zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'releases/v0.11.0'

Browse Source
This commit is contained in:
Todd Gamblin
2018-01-17 14:14:45 -08:00
parent 52a9e5d2a3 ba6c39310b
commit c2a10a2aa2
2961 changed files with 152050 additions and 27661 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.codecov.yml Normal file
View File

@@ -0,0 +1,36 @@
coverage:
precision: 2
round: nearest
range: 60...90
status:
project:
default: true
llnl:
threshold: 0.5
paths:
- lib/spack/llnl
commands:
threshold: 0.5
paths:
- lib/spack/spack/cmd
build_systems:
threshold: 0.5
paths:
- lib/spack/spack/build_systems
modules:
threshold: 0.5
paths:
- lib/spack/spack/modules
core:
threshold: 0.5
paths:
- "!lib/spack/llnl"
- "!lib/spack/spack/cmd"
ignore:
- lib/spack/spack/test/.*
- lib/spack/env/.*
- lib/spack/docs/.*
- lib/spack/external/.*
comment: off

17
.flake8
View File

@@ -1,8 +1,8 @@
# -*- conf -*-
# flake8 settings for Spack.
# flake8 settings for Spack core files.
#
# Below we describe which flake8 checks Spack ignores and what the
# rationale is.
# These exceptions ar for Spack core files. We're slightly more lenient
# with packages. See .flake8_packages for that.
#
# Let people line things up nicely:
# - E129: visually indented line with same indent as next logical line
@@ -11,16 +11,11 @@
# - E272: multiple spaces before keyword
#
# Let people use terse Python features:
# - E731 : lambda expressions
# - E731: lambda expressions
#
# Spack allows wildcard imports:
# - F403: disable wildcard import
#
# These are required to get the package.py files to test clean.
# - F405: `name` may be undefined, or undefined from star imports: `module`
# - F821: undefined name `name` (needed for cmake, configure, etc.)
# These are required to get the package.py files to test clean:
# - F999: syntax error in doctest
#
[flake8]
ignore = E129,E221,E241,E272,E731,F403,F405,F821,F999
ignore = E129,E221,E241,E272,E731,F999
max-line-length = 79

22
.flake8_packages Normal file
View File

@@ -0,0 +1,22 @@
# -*- conf -*-
# flake8 settings for Spack package files.
#
# This should include all the same exceptions that we use for core files.
#
# In Spack packages, we also allow the single `from spack import *`
# wildcard import and dependencies can set globals for their
# dependents. So we add exceptions for checks related to undefined names.
#
# Note that we also add *per-line* exemptions for certain patters in the
# `spack flake8` command. This is where F403 for `from spack import *`
# is added (beause we *only* allow that wildcard).
#
# See .flake8 for regular exceptions.
#
# Redefinition exceptions:
# - F405: `name` may be undefined, or undefined from star imports: `module`
# - F821: undefined name `name` (needed for cmake, configure, etc.)
#
[flake8]
ignore = E129,E221,E241,E272,E731,F999,F405,F821
max-line-length = 79

5
.gitignore vendored
View File

@@ -8,8 +8,9 @@
*~
.DS_Store
.idea
/etc/spack/licenses
/etc/spack/*.yaml
# Ignore everything in /etc/spack except /etc/spack/defaults
/etc/spack/*
!/etc/spack/defaults
/etc/spackconfig
/share/spack/dotkit
/share/spack/modules

129
.travis.yml
View File

@@ -1,8 +1,6 @@
#=============================================================================
# Project settings
#=============================================================================
language: python
# Only build master and develop on push; do not build every branch.
branches:
only:
@@ -13,29 +11,87 @@ branches:
#=============================================================================
# Build matrix
#=============================================================================
python:
- 2.6
- 2.7
env:
- TEST_SUITE=unit
- TEST_SUITE=flake8
- TEST_SUITE=doc
matrix:
# Flake8 and Sphinx no longer support Python 2.6, and one run is enough.
exclude:
- python: 2.6
env: TEST_SUITE=flake8
- python: 2.6
env: TEST_SUITE=doc
# Explicitly include an OS X build with homebrew's python.
# Works around Python issues on Travis for OSX, described here:
# http://blog.fizyk.net.pl/blog/running-python-tests-on-traviss-osx-workers.html
jobs:
fast_finish: true
include:
- os: osx
language: generic
env: TEST_SUITE=unit
- stage: 'flake8'
python: '2.7'
os: linux
language: python
env: TEST_SUITE=flake8
- stage: 'unit tests + documentation'
python: '2.6'
os: linux
language: python
env: TEST_SUITE=unit
- python: '2.7'
os: linux
language: python
env: [ TEST_SUITE=unit, COVERAGE=true ]
- python: '3.3'
os: linux
language: python
env: TEST_SUITE=unit
- python: '3.4'
os: linux
language: python
env: TEST_SUITE=unit
- python: '3.5'
os: linux
language: python
env: TEST_SUITE=unit
- python: '3.6'
os: linux
language: python
env: [ TEST_SUITE=unit, COVERAGE=true ]
- os: osx
language: generic
env: [ TEST_SUITE=unit, PYTHON_VERSION=2.7, COVERAGE=true ]
- python: '2.7'
os: linux
language: python
env: TEST_SUITE=doc
# mpich (AutotoolsPackage)
- stage: 'build tests'
python: '2.7'
os: linux
language: python
env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=mpich' ]
# astyle (MakefilePackage)
- python: '2.7'
os: linux
language: python
env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=astyle' ]
# tut (WafPackage)
- python: '2.7'
os: linux
language: python
env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=tut' ]
# py-setuptools (PythonPackage)
- python: '2.7'
os: linux
language: python
env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=py-setuptools' ]
# perl-dbi (PerlPackage)
# - python: '2.7'
# os: linux
# language: python
# env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=perl-dbi' ]
# openjpeg (CMakePackage + external cmake)
- python: '2.7'
os: linux
language: python
env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=openjpeg' ]
# r-rcpp (RPackage + external R)
- python: '2.7'
os: linux
language: python
env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=r-rcpp' ]
# mpich (AutotoolsPackage)
- python: '3.6'
os: linux
language: python
env: [ TEST_SUITE=build, COVERAGE=true, 'SPEC=mpich' ]
#=============================================================================
# Environment
@@ -48,23 +104,35 @@ addons:
apt:
packages:
- gfortran
- mercurial
- graphviz
- libyaml-dev
- gnupg2
- cmake
- r-base
- r-base-core
- r-base-dev
- perl
- perl-base
cache: pip
# Work around Travis's lack of support for Python on OSX
before_install:
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew ls --versions python > /dev/null || brew install python; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew ls --versions gcc > /dev/null || brew install gcc; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew ls --versions gnupg2 > /dev/null || brew install gnupg2; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then virtualenv venv; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then source venv/bin/activate; fi
# Install various dependencies
install:
- pip install --upgrade coveralls
- pip install --upgrade pip
- pip install --upgrade six
- pip install --upgrade setuptools
- pip install --upgrade codecov
- pip install --upgrade flake8
- pip install --upgrade sphinx
- pip install --upgrade mercurial
- if [[ "$TEST_SUITE" == "doc" ]]; then pip install --upgrade -r lib/spack/docs/requirements.txt; fi
before_script:
# Need this for the git tests to succeed.
@@ -74,13 +142,16 @@ before_script:
# Need this to be able to compute the list of changed files
- git fetch origin develop:develop
# Set up external dependencies for build tests, because the take too long to compile
- if [[ "$TEST_SUITE" == "build" ]]; then cp share/spack/qa/configuration/packages.yaml etc/spack/packages.yaml; fi
#=============================================================================
# Building
#=============================================================================
script: share/spack/qa/run-$TEST_SUITE-tests
after_success:
- if [[ $TEST_SUITE == unit && $TRAVIS_PYTHON_VERSION == 2.7 && $TRAVIS_OS_NAME == "linux" ]]; then coveralls; fi
- codecov --env PY_VERSION
#=============================================================================
# Notifications

46
CODE_OF_CONDUCT.md Normal file
View File

@@ -0,0 +1,46 @@
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
## Scope
This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at maintainers@spack.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/

5
CONTRIBUTING.md Normal file
View File

@@ -0,0 +1,5 @@
# Contributing to Spack
Before contributing to Spack you should read the
[Contribution Guide](https://spack.readthedocs.io/en/latest/contribution_guide.html),
which is maintained as part of Spack's documentation.

574
LICENSE
View File

@@ -1,135 +1,197 @@
########################################################################
GNU LESSER GENERAL PUBLIC LICENSE (Lesser GPL)
Version 2.1, February 1999
########################################################################
Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
Produced at the Lawrence Livermore National Laboratory.
GNU LESSER GENERAL PUBLIC LICENSE
Version 2.1, February 1999
This file is part of Spack.
Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
LLNL-CODE-647188
Copyright (C) 1991, 1999 Free Software Foundation, Inc.
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
For details, see https://github.com/llnl/spack
[This is the first released version of the Lesser GPL. It also counts
as the successor of the GNU Library Public License, version 2, hence
the version number 2.1.]
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License (as
published by the Free Software Foundation) version 2.1, February 1999.
Preamble
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
conditions of the GNU Lesser General Public License for more details.
The licenses for most software are designed to take away your
freedom to share and change it. By contrast, the GNU General Public
Licenses are intended to guarantee your freedom to share and change
free software--to make sure the software is free for all its users.
You should have received a copy of the GNU Lesser General Public
License along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
This license, the Lesser General Public License, applies to some
specially designated software packages--typically libraries--of the
Free Software Foundation and other authors who decide to use it. You
can use it too, but we suggest you first think carefully about whether
this license or the ordinary General Public License is the better
strategy to use in any particular case, based on the explanations below.
########################################################################
LLNL NOTICE AND TERMS AND CONDITIONS OF THE GNU LGPL
When we speak of free software, we are referring to freedom of use,
not price. Our General Public Licenses are designed to make sure that
you have the freedom to distribute copies of free software (and charge
for this service if you wish); that you receive source code or can get
it if you want it; that you can change the software and use pieces of
it in new free programs; and that you are informed that you can do
these things.
LLNL Preamble Notice
To protect your rights, we need to make restrictions that forbid
distributors to deny you these rights or to ask you to surrender these
rights. These restrictions translate to certain responsibilities for
you if you distribute copies of the library or if you modify it.
A. This notice is required to be provided under LLNL's contract with
the U.S. Department of Energy (DOE). This work was produced at the
Lawrence Livermore National Laboratory under Contract
No. DE-AC52-07NA27344 with the DOE.
For example, if you distribute copies of the library, whether gratis
or for a fee, you must give the recipients all the rights that we gave
you. You must make sure that they, too, receive or can get the source
code. If you link other code with the library, you must provide
complete object files to the recipients, so that they can relink them
with the library after making changes to the library and recompiling
it. And you must show them these terms so they know their rights.
B. Neither the United States Government nor Lawrence Livermore
National Security, LLC nor any of their employees, makes any
warranty, express or implied, or assumes any liability or
responsibility for the accuracy, completeness, or usefulness of any
information, apparatus, product, or process disclosed, or
represents that its use would not infringe privately-owned rights.
We protect your rights with a two-step method: (1) we copyright the
library, and (2) we offer you this license, which gives you legal
permission to copy, distribute and/or modify the library.
C. Also, reference herein to any specific commercial products,
process, or services by trade name, trademark, manufacturer or
otherwise does not necessarily constitute or imply its endorsement,
recommendation, or favoring by the United States Government or
Lawrence Livermore National Security, LLC. The views and opinions
of authors expressed herein do not necessarily state or reflect
those of the United States Government or Lawrence Livermore
National Security, LLC, and shall not be used for advertising or
product endorsement purposes.
To protect each distributor, we want to make it very clear that
there is no warranty for the free library. Also, if the library is
modified by someone else and passed on, the recipients should know
that what they have is not the original version, so that the original
author's reputation will not be affected by problems that might be
introduced by others.
The precise terms and conditions for copying, distribution and
modification follows.
Finally, software patents pose a constant threat to the existence of
any free program. We wish to make sure that a company cannot
effectively restrict the users of a free program by obtaining a
restrictive license from a patent holder. Therefore, we insist that
any patent license obtained for a version of the library must be
consistent with the full freedom of use specified in this license.
########################################################################
GNU LESSER GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
Most GNU software, including some libraries, is covered by the
ordinary GNU General Public License. This license, the GNU Lesser
General Public License, applies to certain designated libraries, and
is quite different from the ordinary General Public License. We use
this license for certain libraries in order to permit linking those
libraries into non-free programs.
0. This License Agreement applies to any software library or other
When a program is linked with a library, whether statically or using
a shared library, the combination of the two is legally speaking a
combined work, a derivative of the original library. The ordinary
General Public License therefore permits such linking only if the
entire combination fits its criteria of freedom. The Lesser General
Public License permits more lax criteria for linking other code with
the library.
We call this license the "Lesser" General Public License because it
does Less to protect the user's freedom than the ordinary General
Public License. It also provides other free software developers Less
of an advantage over competing non-free programs. These disadvantages
are the reason we use the ordinary General Public License for many
libraries. However, the Lesser license provides advantages in certain
special circumstances.
For example, on rare occasions, there may be a special need to
encourage the widest possible use of a certain library, so that it becomes
a de-facto standard. To achieve this, non-free programs must be
allowed to use the library. A more frequent case is that a free
library does the same job as widely used non-free libraries. In this
case, there is little to gain by limiting the free library to free
software only, so we use the Lesser General Public License.
In other cases, permission to use a particular library in non-free
programs enables a greater number of people to use a large body of
free software. For example, permission to use the GNU C Library in
non-free programs enables many more people to use the whole GNU
operating system, as well as its variant, the GNU/Linux operating
system.
Although the Lesser General Public License is Less protective of the
users' freedom, it does ensure that the user of a program that is
linked with the Library has the freedom and the wherewithal to run
that program using a modified version of the Library.
The precise terms and conditions for copying, distribution and
modification follow. Pay close attention to the difference between a
"work based on the library" and a "work that uses the library". The
former contains code derived from the library, whereas the latter must
be combined with the library in order to run.
GNU LESSER GENERAL PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. This License Agreement applies to any software library or other
program which contains a notice placed by the copyright holder or
other authorized party saying it may be distributed under the terms of
this Lesser General Public License (also called "this License"). Each
licensee is addressed as "you".
this Lesser General Public License (also called "this License").
Each licensee is addressed as "you".
A "library" means a collection of software functions and/or data
A "library" means a collection of software functions and/or data
prepared so as to be conveniently linked with application programs
(which use some of those functions and data) to form executables.
The "Library", below, refers to any such software library or work
which has been distributed under these terms. A "work based on the
The "Library", below, refers to any such software library or work
which has been distributed under these terms. A "work based on the
Library" means either the Library or any derivative work under
copyright law: that is to say, a work containing the Library or a
portion of it, either verbatim or with modifications and/or translated
straightforwardly into another language. (Hereinafter, translation is
straightforwardly into another language. (Hereinafter, translation is
included without limitation in the term "modification".)
"Source code" for a work means the preferred form of the work for
making modifications to it. For a library, complete source code means
"Source code" for a work means the preferred form of the work for
making modifications to it. For a library, complete source code means
all the source code for all modules it contains, plus any associated
interface definition files, plus the scripts used to control
compilation and installation of the library. Activities other than
copying, distribution and modification are not covered by this
License; they are outside its scope. The act of running a program
using the Library is not restricted, and output from such a program is
covered only if its contents constitute a work based on the Library
(independent of the use of the Library in a tool for writing
it). Whether that is true depends on what the Library does and what
the program that uses the Library does.
interface definition files, plus the scripts used to control compilation
and installation of the library.
1. You may copy and distribute verbatim copies of the Library's
Activities other than copying, distribution and modification are not
covered by this License; they are outside its scope. The act of
running a program using the Library is not restricted, and output from
such a program is covered only if its contents constitute a work based
on the Library (independent of the use of the Library in a tool for
writing it). Whether that is true depends on what the Library does
and what the program that uses the Library does.
1. You may copy and distribute verbatim copies of the Library's
complete source code as you receive it, in any medium, provided that
you conspicuously and appropriately publish on each copy an
appropriate copyright notice and disclaimer of warranty; keep intact
all the notices that refer to this License and to the absence of any
warranty; and distribute a copy of this License along with the
Library. You may charge a fee for the physical act of transferring a
copy, and you may at your option offer warranty protection in exchange
for a fee.
Library.
2. You may modify your copy or copies of the Library or any portion of
it, thus forming a work based on the Library, and copy and distribute
such modifications or work under the terms of Section 1 above,
provided that you also meet all of these conditions:
You may charge a fee for the physical act of transferring a copy,
and you may at your option offer warranty protection in exchange for a
fee.
a) The modified work must itself be a software library.
2. You may modify your copy or copies of the Library or any portion
of it, thus forming a work based on the Library, and copy and
distribute such modifications or work under the terms of Section 1
above, provided that you also meet all of these conditions:
b) You must cause the files modified to carry prominent notices
stating that you changed the files and the date of any change.
a) The modified work must itself be a software library.
c) You must cause the whole of the work to be licensed at no charge to
all third parties under the terms of this License.
b) You must cause the files modified to carry prominent notices
stating that you changed the files and the date of any change.
d) If a facility in the modified Library refers to a function or a
table of data to be supplied by an application program that uses the
facility, other than as an argument passed when the facility is
invoked, then you must make a good faith effort to ensure that, in the
event an application does not supply such function or table, the
facility still operates, and performs whatever part of its purpose
remains meaningful. (For example, a function in a library to compute
square roots has a purpose that is entirely well-defined independent
of the application. Therefore, Subsection 2d requires that any
application-supplied function or table used by this function must be
optional: if the application does not supply it, the square root
function must still compute square roots.)
c) You must cause the whole of the work to be licensed at no
charge to all third parties under the terms of this License.
These requirements apply to the modified work as a whole. If
d) If a facility in the modified Library refers to a function or a
table of data to be supplied by an application program that uses
the facility, other than as an argument passed when the facility
is invoked, then you must make a good faith effort to ensure that,
in the event an application does not supply such function or
table, the facility still operates, and performs whatever part of
its purpose remains meaningful.
(For example, a function in a library to compute square roots has
a purpose that is entirely well-defined independent of the
application. Therefore, Subsection 2d requires that any
application-supplied function or table used by this function must
be optional: if the application does not supply it, the square
root function must still compute square roots.)
These requirements apply to the modified work as a whole. If
identifiable sections of that work are not derived from the Library,
and can be reasonably considered independent and separate works in
themselves, then this License, and its terms, do not apply to those
sections when you distribute them as separate works. But when you
sections when you distribute them as separate works. But when you
distribute the same sections as part of a whole which is a work based
on the Library, the distribution of the whole must be on the terms of
this License, whose permissions for other licensees extend to the
@@ -146,189 +208,191 @@ with the Library (or with a work based on the Library) on a volume of
a storage or distribution medium does not bring the other work under
the scope of this License.
3. You may opt to apply the terms of the ordinary GNU General Public
License instead of this License to a given copy of the Library. To do
3. You may opt to apply the terms of the ordinary GNU General Public
License instead of this License to a given copy of the Library. To do
this, you must alter all the notices that refer to this License, so
that they refer to the ordinary GNU General Public License, version 2,
instead of to this License. (If a newer version than version 2 of the
instead of to this License. (If a newer version than version 2 of the
ordinary GNU General Public License has appeared, then you can specify
that version instead if you wish.) Do not make any other change in
that version instead if you wish.) Do not make any other change in
these notices.
Once this change is made in a given copy, it is irreversible for that
copy, so the ordinary GNU General Public License applies to all
subsequent copies and derivative works made from that copy. This
option is useful when you wish to copy part of the code of the Library
into a program that is not a library.
Once this change is made in a given copy, it is irreversible for
that copy, so the ordinary GNU General Public License applies to all
subsequent copies and derivative works made from that copy.
4. You may copy and distribute the Library (or a portion or derivative
of it, under Section 2) in object code or executable form under the
terms of Sections 1 and 2 above provided that you accompany it with
the complete corresponding machine- readable source code, which must
be distributed under the terms of Sections 1 and 2 above on a medium
customarily used for software interchange.
This option is useful when you wish to copy part of the code of
the Library into a program that is not a library.
If distribution of object code is made by offering access to copy from
a designated place, then offering equivalent access to copy the source
code from the same place satisfies the requirement to distribute the
source code, even though third parties are not compelled to copy the
source along with the object code.
4. You may copy and distribute the Library (or a portion or
derivative of it, under Section 2) in object code or executable form
under the terms of Sections 1 and 2 above provided that you accompany
it with the complete corresponding machine-readable source code, which
must be distributed under the terms of Sections 1 and 2 above on a
medium customarily used for software interchange.
5. A program that contains no derivative of any portion of the
If distribution of object code is made by offering access to copy
from a designated place, then offering equivalent access to copy the
source code from the same place satisfies the requirement to
distribute the source code, even though third parties are not
compelled to copy the source along with the object code.
5. A program that contains no derivative of any portion of the
Library, but is designed to work with the Library by being compiled or
linked with it, is called a "work that uses the Library". Such a work,
in isolation, is not a derivative work of the Library, and therefore
falls outside the scope of this License.
linked with it, is called a "work that uses the Library". Such a
work, in isolation, is not a derivative work of the Library, and
therefore falls outside the scope of this License.
However, linking a "work that uses the Library" with the Library
However, linking a "work that uses the Library" with the Library
creates an executable that is a derivative of the Library (because it
contains portions of the Library), rather than a "work that uses the
library". The executable is therefore covered by this License. Section
6 states terms for distribution of such executables.
library". The executable is therefore covered by this License.
Section 6 states terms for distribution of such executables.
When a "work that uses the Library" uses material from a header file
When a "work that uses the Library" uses material from a header file
that is part of the Library, the object code for the work may be a
derivative work of the Library even though the source code is
not. Whether this is true is especially significant if the work can be
linked without the Library, or if the work is itself a library. The
derivative work of the Library even though the source code is not.
Whether this is true is especially significant if the work can be
linked without the Library, or if the work is itself a library. The
threshold for this to be true is not precisely defined by law.
If such an object file uses only numerical parameters, data structure
layouts and accessors, and small macros and small inline functions
(ten lines or less in length), then the use of the object file is
unrestricted, regardless of whether it is legally a derivative
work. (Executables containing this object code plus portions of the
Library will still fall under section 6.)
If such an object file uses only numerical parameters, data
structure layouts and accessors, and small macros and small inline
functions (ten lines or less in length), then the use of the object
file is unrestricted, regardless of whether it is legally a derivative
work. (Executables containing this object code plus portions of the
Library will still fall under Section 6.)
Otherwise, if the work is a derivative of the Library, you may
distribute the object code for the work under the terms of Section
6. Any executables containing that work also fall under Section 6,
Otherwise, if the work is a derivative of the Library, you may
distribute the object code for the work under the terms of Section 6.
Any executables containing that work also fall under Section 6,
whether or not they are linked directly with the Library itself.
6. As an exception to the Sections above, you may also combine or link
a "work that uses the Library" with the Library to produce a work
containing portions of the Library, and distribute that work under
terms of your choice, provided that the terms permit modification of
the work for the customer's own use and reverse engineering for
debugging such modifications.
6. As an exception to the Sections above, you may also combine or
link a "work that uses the Library" with the Library to produce a
work containing portions of the Library, and distribute that work
under terms of your choice, provided that the terms permit
modification of the work for the customer's own use and reverse
engineering for debugging such modifications.
You must give prominent notice with each copy of the work that the
You must give prominent notice with each copy of the work that the
Library is used in it and that the Library and its use are covered by
this License. You must supply a copy of this License. If the work
this License. You must supply a copy of this License. If the work
during execution displays copyright notices, you must include the
copyright notice for the Library among them, as well as a reference
directing the user to the copy of this License. Also, you must do one
directing the user to the copy of this License. Also, you must do one
of these things:
a) Accompany the work with the complete corresponding machine-readable
source code for the Library including whatever changes were used in
the work (which must be distributed under Sections 1 and 2 above);
and, if the work is an executable liked with the Library, with the
complete machine-readable "work that uses the Library", as object code
and/or source code, so that the user can modify the Library and then
relink to produce a modified executable containing the modified
Library. (It is understood that the user who changes the contents of
definitions files in the Library will not necessarily be able to
recompile the application to use the modified definitions.)
a) Accompany the work with the complete corresponding
machine-readable source code for the Library including whatever
changes were used in the work (which must be distributed under
Sections 1 and 2 above); and, if the work is an executable linked
with the Library, with the complete machine-readable "work that
uses the Library", as object code and/or source code, so that the
user can modify the Library and then relink to produce a modified
executable containing the modified Library. (It is understood
that the user who changes the contents of definitions files in the
Library will not necessarily be able to recompile the application
to use the modified definitions.)
b) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (1) uses at run time a copy
of the library already present on the user's computer system, rather
than copying library functions into the executable, and (2) will
operate properly with a modified version of the library, if the user
installs one, as long as the modified version is interface- compatible
with the version that the work was made with.
b) Use a suitable shared library mechanism for linking with the
Library. A suitable mechanism is one that (1) uses at run time a
copy of the library already present on the user's computer system,
rather than copying library functions into the executable, and (2)
will operate properly with a modified version of the library, if
the user installs one, as long as the modified version is
interface-compatible with the version that the work was made with.
c) Accompany the work with a written offer, valid for at least three
years, to give the same user the materials specified in Subsection 6a,
above, for a charge no more than the cost of performing this
distribution.
c) Accompany the work with a written offer, valid for at
least three years, to give the same user the materials
specified in Subsection 6a, above, for a charge no more
than the cost of performing this distribution.
d) If distribution of the work is made by offering access to copy from
a designated place, offer equivalent access to copy the above
specified materials from the same place.
d) If distribution of the work is made by offering access to copy
from a designated place, offer equivalent access to copy the above
specified materials from the same place.
e) Verify that the user has already received a copy of these materials
or that you have already sent this user a copy.
e) Verify that the user has already received a copy of these
materials or that you have already sent this user a copy.
For an executable, the required form of the "work that uses the
For an executable, the required form of the "work that uses the
Library" must include any data and utility programs needed for
reproducing the executable from it. However, as a special exception,
reproducing the executable from it. However, as a special exception,
the materials to be distributed need not include anything that is
normally distributed (in either source or binary form) with the major
components (compiler, kernel, and so on) of the operating system on
which the executable runs, unless that component itself accompanies
the executable.
It may happen that this requirement contradicts the license
restrictions of other propriety libraries that do not normally
accompany the operating system. Such a contradiction means you cannot
It may happen that this requirement contradicts the license
restrictions of other proprietary libraries that do not normally
accompany the operating system. Such a contradiction means you cannot
use both them and the Library together in an executable that you
distribute.
7. You may place library facilities that are a work based on the
7. You may place library facilities that are a work based on the
Library side-by-side in a single library together with other library
facilities not covered by this License, and distribute such a combined
library, provided that the separate distribution of the work based on
the Library and of the other library facilities is otherwise
permitted, and provided that you do these two things:
a) Accompany the combined library with a copy of the same work based
on the Library, uncombined with any other library facilities. This
must be distributed under the terms of the Sections above.
a) Accompany the combined library with a copy of the same work
based on the Library, uncombined with any other library
facilities. This must be distributed under the terms of the
Sections above.
b) Give prominent notice with the combined library of the fact that
part of it is a work based on the Library, and explaining where to
find the accompanying uncombined form of the same work.
b) Give prominent notice with the combined library of the fact
that part of it is a work based on the Library, and explaining
where to find the accompanying uncombined form of the same work.
1 You may not copy, modify, sublicense, link with, or distribute the
Library except as expressly provided under this License. Any attempt
otherwise to copy, modify, sublicense, link with, or distribute the
Library is void, and will automatically terminate your rights under
this License. However, parties who have received copies, or rights,
from you under this License will not have their licenses terminated so
long as such parties remain in full compliance.
8. You may not copy, modify, sublicense, link with, or distribute
the Library except as expressly provided under this License. Any
attempt otherwise to copy, modify, sublicense, link with, or
distribute the Library is void, and will automatically terminate your
rights under this License. However, parties who have received copies,
or rights, from you under this License will not have their licenses
terminated so long as such parties remain in full compliance.
2 You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Library or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
9. You are not required to accept this License, since you have not
signed it. However, nothing else grants you permission to modify or
distribute the Library or its derivative works. These actions are
prohibited by law if you do not accept this License. Therefore, by
modifying or distributing the Library (or any work based on the
Library), you indicate your acceptance of this License to do so, and
all its terms and conditions for copying, distributing or modifying
the Library or works based on it.
3 Each time you redistribute the Library (or any work based on the
10. Each time you redistribute the Library (or any work based on the
Library), the recipient automatically receives a license from the
original licensor to copy, distribute, link with or modify the Library
subject to these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted
herein. You are not responsible for enforcing compliance by third
parties with this License.
subject to these terms and conditions. You may not impose any further
restrictions on the recipients' exercise of the rights granted herein.
You are not responsible for enforcing compliance by third parties with
this License.
4 If, as a consequence of a court judgment or allegation of patent
11. If, as a consequence of a court judgment or allegation of patent
infringement or for any other reason (not limited to patent issues),
conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot
excuse you from the conditions of this License. If you cannot
distribute so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you
may not distribute the Library at all. For example, if a patent
may not distribute the Library at all. For example, if a patent
license would not permit royalty-free redistribution of the Library by
all those who receive copies directly or indirectly through you, then
the only way you could satisfy both it and this License would be to
refrain entirely from distribution of the Library.
If any portion of this section is held invalid or unenforceable under
any particular circumstance, the balance of the section is intended to
apply, and the section as a whole is intended to apply in other
circumstances.
If any portion of this section is held invalid or unenforceable under any
particular circumstance, the balance of the section is intended to apply,
and the section as a whole is intended to apply in other circumstances.
It is not the purpose of this section to induce you to infringe any
patents or other property right claims or to contest validity of any
such claims; this section has the sole purpose of protecting the
integrity of the free software distribution system which is
implemented by public license practices. Many people have made
implemented by public license practices. Many people have made
generous contributions to the wide range of software distributed
through that system in reliance on consistent application of that
system; it is up to the author/donor to decide if he or she is willing
@@ -338,56 +402,102 @@ impose that choice.
This section is intended to make thoroughly clear what is believed to
be a consequence of the rest of this License.
1 If the distribution and/or use of the Library is restricted in
12. If the distribution and/or use of the Library is restricted in
certain countries either by patents or by copyrighted interfaces, the
original copyright holder who places the Library under this License
may add an explicit geographical distribution limitation excluding
those countries, so that distribution is permitted only in or among
countries not thus excluded. In such case, this License incorporates
the limitation as if written in the body of this License.
original copyright holder who places the Library under this License may add
an explicit geographical distribution limitation excluding those countries,
so that distribution is permitted only in or among countries not thus
excluded. In such case, this License incorporates the limitation as if
written in the body of this License.
13. The Free Software Foundation may publish revised and/or new
versions of the Lesser General Public License from time to time. Such
new versions will be similar in spirit to the present version, but may
differ in detail to address new problems or concerns.
13. The Free Software Foundation may publish revised and/or new
versions of the Lesser General Public License from time to time.
Such new versions will be similar in spirit to the present version,
but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Library
Each version is given a distinguishing version number. If the Library
specifies a version number of this License which applies to it and
"any later version", you have the option of following the terms and
conditions either of that version or of any later version published by
the Free Software Foundation. If the Library does not specify a
the Free Software Foundation. If the Library does not specify a
license version number, you may choose any version ever published by
the Free Software Foundation.
2 If you wish to incorporate parts of the Library into other free
14. If you wish to incorporate parts of the Library into other free
programs whose distribution conditions are incompatible with these,
write to the author to ask for permission. For software which is
write to the author to ask for permission. For software which is
copyrighted by the Free Software Foundation, write to the Free
Software Foundation; we sometimes make exceptions for this. Our
Software Foundation; we sometimes make exceptions for this. Our
decision will be guided by the two goals of preserving the free status
of all derivatives of our free software and of promoting the sharing
and reuse of software generally.
NO WARRANTY
NO WARRANTY
1 BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT
WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER
PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND,
EITHER EXPRESSED OR IMPLIED INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
2 IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE
LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL
OR CONSQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
DAMAGES.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Libraries
If you develop a new library, and you want it to be of the greatest
possible use to the public, we recommend making it free software that
everyone can redistribute and change. You can do so by permitting
redistribution under these terms (or, alternatively, under the terms of the
ordinary General Public License).
To apply these terms, attach the following notices to the library. It is
safest to attach them to the start of each source file to most effectively
convey the exclusion of warranty; and each file should have at least the
"copyright" line and a pointer to where the full notice is found.
<one line to give the library's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Also add information on how to contact you by electronic and paper mail.
You should also get your employer (if you work as a programmer) or your
school, if any, to sign a "copyright disclaimer" for the library, if
necessary. Here is a sample; alter the names:
Yoyodyne, Inc., hereby disclaims all copyright interest in the
library `Frob' (a library for tweaking knobs) written by James Random Hacker.
<signature of Ty Coon>, 1 April 1990
Ty Coon, President of Vice
That's all there is to it!

32
NOTICE Normal file
View File

@@ -0,0 +1,32 @@
########################################################################
LLNL NOTICE AND TERMS AND CONDITIONS OF THE GNU LGPL
########################################################################
LLNL Preamble Notice
A. This notice is required to be provided under LLNL's contract with
the U.S. Department of Energy (DOE). This work was produced at the
Lawrence Livermore National Laboratory under Contract
No. DE-AC52-07NA27344 with the DOE.
B. Neither the United States Government nor Lawrence Livermore
National Security, LLC nor any of their employees, makes any
warranty, express or implied, or assumes any liability or
responsibility for the accuracy, completeness, or usefulness of any
information, apparatus, product, or process disclosed, or
represents that its use would not infringe privately-owned rights.
C. Also, reference herein to any specific commercial products,
process, or services by trade name, trademark, manufacturer or
otherwise does not necessarily constitute or imply its endorsement,
recommendation, or favoring by the United States Government or
Lawrence Livermore National Security, LLC. The views and opinions
of authors expressed herein do not necessarily state or reflect
those of the United States Government or Lawrence Livermore
National Security, LLC, and shall not be used for advertising or
product endorsement purposes.
See the LICENSE file for the precise terms and conditions for copying,
distribution and modification.
########################################################################

83
README.md
View File

@@ -1,31 +1,30 @@
![image](share/spack/logo/spack-logo-text-64.png "Spack")
============
# <img src="https://cdn.rawgit.com/spack/spack/develop/share/spack/logo/spack-logo.svg" width="64" valign="middle" alt="Spack"/> Spack
[![Build Status](https://travis-ci.org/LLNL/spack.svg?branch=develop)](https://travis-ci.org/LLNL/spack)
[![Coverage Status](https://coveralls.io/repos/github/LLNL/spack/badge.svg?branch=develop)](https://coveralls.io/github/LLNL/spack?branch=develop)
[![Build Status](https://travis-ci.org/spack/spack.svg?branch=develop)](https://travis-ci.org/spack/spack)
[![codecov](https://codecov.io/gh/spack/spack/branch/develop/graph/badge.svg)](https://codecov.io/gh/spack/spack)
[![Read the Docs](https://readthedocs.org/projects/spack/badge/?version=latest)](https://spack.readthedocs.io)
[![Slack](https://spackpm.herokuapp.com/badge.svg)](https://spackpm.herokuapp.com)
Spack is a package management tool designed to support multiple
versions and configurations of software on a wide variety of platforms
and environments. It was designed for large supercomputing centers,
where many users and application teams share common installations of
software on clusters with exotic architectures, using libraries that
do not have a standard ABI. Spack is non-destructive: installing a new
version does not break existing installations, so many configurations
can coexist on the same system.
Spack is a multi-platform package manager that builds and installs
multiple versions and configurations of software. It works on Linux,
macOS, and many supercomputers. Spack is non-destructive: installing a
new version of a package does not break existing installations, so many
configurations of the same package can coexist.
Most importantly, Spack is simple. It offers a simple spec syntax so
that users can specify versions and configuration options
concisely. Spack is also simple for package authors: package files are
written in pure Python, and specs allow package authors to write a
single build script for many different builds of the same package.
Spack offers a simple "spec" syntax that allows users to specify versions
and configuration options. Package files are written in pure Python, and
specs allow package authors to write a single script for many different
builds of the same package. With Spack, you can build your software
*all* the ways you want to.
See the
[Feature Overview](http://spack.readthedocs.io/en/latest/features.html)
for examples and highlights.
To install spack and install your first package:
To install spack and your first package, make sure you have Python.
Then:
$ git clone https://github.com/llnl/spack.git
$ git clone https://github.com/spack/spack.git
$ cd spack/bin
$ ./spack install libelf
@@ -35,15 +34,16 @@ Documentation
[**Full documentation**](http://spack.readthedocs.io/) for Spack is
the first place to look.
We've also got a [**Spack 101 Tutorial**](http://spack.readthedocs.io/en/latest/tutorial_sc16.html),
so you can learn Spack yourself, or teach users at your own site.
Try the
[**Spack Tutorial**](http://spack.readthedocs.io/en/latest/tutorial.html),
to learn how to use spack, write packages, or deploy packages for users
at your site.
See also:
* [Technical paper](http://www.computer.org/csdl/proceedings/sc/2015/3723/00/2807623.pdf) and
[slides](https://tgamblin.github.io/files/Gamblin-Spack-SC15-Talk.pdf) on Spack's design and implementation.
* [Short presentation](https://tgamblin.github.io/files/Gamblin-Spack-Lightning-Talk-BOF-SC15.pdf) from the *Getting Scientific Software Installed* BOF session at Supercomputing 2015.
Get Involved!
------------------------
@@ -53,37 +53,44 @@ packages to bugfixes, or even new core features.
### Mailing list
If you are interested in contributing to spack, the first step is to
join the mailing list. We're using a Google Group for this, and you
can join it here:
If you are interested in contributing to spack, join the mailing list.
We're using Google Groups for this:
* [Spack Google Group](https://groups.google.com/d/forum/spack)
### Slack channel
Spack has a Slack channel where you can chat about all things Spack:
* [Spack on Slack](https://spackpm.slack.com)
[Sign up here](https://spackpm.herokuapp.com) to get an invitation mailed
to you.
### Contributions
Contributing to Spack is relatively easy. Just send us a
[pull request](https://help.github.com/articles/using-pull-requests/).
When you send your request, make ``develop`` the destination branch on the
[Spack repository](https://github.com/LLNL/spack).
[Spack repository](https://github.com/spack/spack).
Your PR must pass Spack's unit tests and documentation tests, and must be
[PEP 8](https://www.python.org/dev/peps/pep-0008/) compliant.
We enforce these guidelines with [Travis CI](https://travis-ci.org/LLNL/spack).
To run these tests locally, and for helpful tips on git, see our
[PEP 8](https://www.python.org/dev/peps/pep-0008/) compliant. We enforce
these guidelines with [Travis CI](https://travis-ci.org/spack/spack). To
run these tests locally, and for helpful tips on git, see our
[Contribution Guide](http://spack.readthedocs.io/en/latest/contribution_guide.html).
Spack uses a rough approximation of the [Git
Flow](http://nvie.com/posts/a-successful-git-branching-model/)
Spack uses a rough approximation of the
[Git Flow](http://nvie.com/posts/a-successful-git-branching-model/)
branching model. The ``develop`` branch contains the latest
contributions, and ``master`` is always tagged and points to the
latest stable release.
contributions, and ``master`` is always tagged and points to the latest
stable release.
Authors
----------------
Many thanks go to Spack's [contributors](https://github.com/llnl/spack/graphs/contributors).
Many thanks go to Spack's [contributors](https://github.com/spack/spack/graphs/contributors).
Spack was originally written by Todd Gamblin, tgamblin@llnl.gov.
Spack was created by Todd Gamblin, tgamblin@llnl.gov.
### Citing Spack
@@ -97,6 +104,8 @@ If you are referencing Spack in a publication, please cite the following paper:
Release
----------------
Spack is released under an LGPL license. For more details see the
LICENSE file.
NOTICE and LICENSE files.
``LLNL-CODE-647188``
![Analytics](https://ga-beacon.appspot.com/UA-101208306-3/welcome-page?pixel)

23
bin/sbang
View File

@@ -1,14 +1,14 @@
#!/bin/bash
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
@@ -104,18 +104,27 @@ lines=0
while read line && ((lines < 2)) ; do
if [[ "$line" = '#!'* ]]; then
interpreter="${line#\#!}"
elif [[ "$line" = '//!'*node* ]]; then
interpreter="${line#//!}"
elif [[ "$line" = '--!'*lua* ]]; then
interpreter="${line#--!}"
fi
lines=$((lines+1))
done < "$script"
# this is ineeded for scripts with sbang parameter
# like ones in intltool
# #!/<spack-long-path>/perl -w
# this is the interpreter line with all the parameters as a vector
interpreter_v=(${interpreter})
# this is the single interpreter path
interpreter_f="${interpreter_v[0]}"
# Invoke any interpreter found, or raise an error if none was found.
if [[ -n "$interpreter" ]]; then
if [[ "${interpreter##*/}" = "perl" ]]; then
exec $interpreter -x "$@"
if [[ -n "$interpreter_f" ]]; then
if [[ "${interpreter_f##*/}" = "perl" ]]; then
exec $interpreter_v -x "$@"
else
exec $interpreter "$@"
exec $interpreter_v "$@"
fi
else
echo "error: sbang found no interpreter in $script"

201
bin/spack
View File

@@ -1,15 +1,14 @@
#!/usr/bin/env python
# flake8: noqa
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
@@ -24,27 +23,34 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import sys
if (sys.version_info[0] > 2) or (sys.version_info[:2] < (2, 6)):
v_info = sys.version_info[:3]
sys.exit("Spack requires Python 2.6 or 2.7. "
"This is Python %d.%d.%d." % v_info)
from __future__ import print_function
import os
import inspect
import sys
if sys.version_info[:2] < (2, 6):
v_info = sys.version_info[:3]
sys.exit("Spack requires Python 2.6 or higher."
"This is Python %d.%d.%d." % v_info)
# Find spack's location and its prefix.
SPACK_FILE = os.path.realpath(os.path.expanduser(__file__))
os.environ["SPACK_FILE"] = SPACK_FILE
SPACK_PREFIX = os.path.dirname(os.path.dirname(SPACK_FILE))
spack_file = os.path.realpath(os.path.expanduser(__file__))
spack_prefix = os.path.dirname(os.path.dirname(spack_file))
# Allow spack libs to be imported in our scripts
SPACK_LIB_PATH = os.path.join(SPACK_PREFIX, "lib", "spack")
sys.path.insert(0, SPACK_LIB_PATH)
spack_lib_path = os.path.join(spack_prefix, "lib", "spack")
sys.path.insert(0, spack_lib_path)
# Add external libs
SPACK_EXTERNAL_LIBS = os.path.join(SPACK_LIB_PATH, "external")
sys.path.insert(0, SPACK_EXTERNAL_LIBS)
spack_external_libs = os.path.join(spack_lib_path, "external")
sys.path.insert(0, spack_external_libs)
# Handle vendoring of YAML specially, as it has two versions.
if sys.version_info[0] == 2:
spack_yaml_libs = os.path.join(spack_external_libs, "yaml/lib")
else:
spack_yaml_libs = os.path.join(spack_external_libs, "yaml/lib3")
sys.path.insert(0, spack_yaml_libs)
# Quick and dirty check to clean orphaned .pyc files left over from
# previous revisions. These files were present in earlier versions of
@@ -52,13 +58,13 @@ sys.path.insert(0, SPACK_EXTERNAL_LIBS)
# imports. If we leave them, Spack will fail in mysterious ways.
# TODO: more elegant solution for orphaned pyc files.
orphaned_pyc_files = [
os.path.join(SPACK_EXTERNAL_LIBS, 'functools.pyc'),
os.path.join(SPACK_EXTERNAL_LIBS, 'ordereddict.pyc'),
os.path.join(SPACK_LIB_PATH, 'spack', 'platforms', 'cray_xc.pyc'),
os.path.join(SPACK_LIB_PATH, 'spack', 'cmd', 'package-list.pyc'),
os.path.join(SPACK_LIB_PATH, 'spack', 'cmd', 'test-install.pyc'),
os.path.join(SPACK_LIB_PATH, 'spack', 'cmd', 'url-parse.pyc'),
os.path.join(SPACK_LIB_PATH, 'spack', 'test', 'yaml.pyc')
os.path.join(spack_external_libs, 'functools.pyc'),
os.path.join(spack_external_libs, 'ordereddict.pyc'),
os.path.join(spack_lib_path, 'spack', 'platforms', 'cray_xc.pyc'),
os.path.join(spack_lib_path, 'spack', 'cmd', 'package-list.pyc'),
os.path.join(spack_lib_path, 'spack', 'cmd', 'test-install.pyc'),
os.path.join(spack_lib_path, 'spack', 'cmd', 'url-parse.pyc'),
os.path.join(spack_lib_path, 'spack', 'test', 'yaml.pyc')
]
for pyc_file in orphaned_pyc_files:
@@ -67,146 +73,9 @@ for pyc_file in orphaned_pyc_files:
try:
os.remove(pyc_file)
except OSError as e:
print ("WARNING: Spack may fail mysteriously. "
"Couldn't remove orphaned .pyc file: %s" % pyc_file)
print("WARNING: Spack may fail mysteriously. "
"Couldn't remove orphaned .pyc file: %s" % pyc_file)
# If there is no working directory, use the spack prefix.
try:
working_dir = os.getcwd()
except OSError:
os.chdir(SPACK_PREFIX)
working_dir = SPACK_PREFIX
# clean up the scope and start using spack package instead.
del SPACK_FILE, SPACK_PREFIX, SPACK_LIB_PATH
import llnl.util.tty as tty
from llnl.util.tty.color import *
import spack
from spack.error import SpackError
import argparse
# Command parsing
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
description="Spack: the Supercomputing PACKage Manager." + colorize("""
spec expressions:
PACKAGE [CONSTRAINTS]
CONSTRAINTS:
@c{@version}
@g{%compiler @compiler_version}
@B{+variant}
@r{-variant} or @r{~variant}
@m{=architecture}
[^DEPENDENCY [CONSTRAINTS] ...]"""))
parser.add_argument('-d', '--debug', action='store_true',
help="Write out debug logs during compile")
parser.add_argument('-D', '--pdb', action='store_true',
help="Run spack under the pdb debugger")
parser.add_argument('-k', '--insecure', action='store_true',
help="Do not check ssl certificates when downloading.")
parser.add_argument('-m', '--mock', action='store_true',
help="Use mock packages instead of real ones.")
parser.add_argument('-p', '--profile', action='store_true',
help="Profile execution using cProfile.")
parser.add_argument('-v', '--verbose', action='store_true',
help="Print additional output during builds")
parser.add_argument('-s', '--stacktrace', action='store_true',
help="Add stacktrace information to all printed statements")
parser.add_argument('-V', '--version', action='version',
version="%s" % spack.spack_version)
# each command module implements a parser() function, to which we pass its
# subparser for setup.
subparsers = parser.add_subparsers(metavar='SUBCOMMAND', dest="command")
import spack.cmd
for cmd in spack.cmd.commands:
module = spack.cmd.get_module(cmd)
cmd_name = cmd.replace('_', '-')
subparser = subparsers.add_parser(cmd_name, help=module.description)
module.setup_parser(subparser)
def _main(args, unknown_args):
# Set up environment based on args.
tty.set_verbose(args.verbose)
tty.set_debug(args.debug)
tty.set_stacktrace(args.stacktrace)
spack.debug = args.debug
if spack.debug:
import spack.util.debug as debug
debug.register_interrupt_handler()
# Run any available pre-run hooks
spack.hooks.pre_run()
spack.spack_working_dir = working_dir
if args.mock:
from spack.repository import RepoPath
spack.repo.swap(RepoPath(spack.mock_packages_path))
# If the user asked for it, don't check ssl certs.
if args.insecure:
tty.warn("You asked for --insecure. Will NOT check SSL certificates.")
spack.insecure = True
# Try to load the particular command asked for and run it
command = spack.cmd.get_command(args.command.replace('-', '_'))
# Allow commands to inject an optional argument and get unknown args
# if they want to handle them.
info = dict(inspect.getmembers(command))
varnames = info['__code__'].co_varnames
argcount = info['__code__'].co_argcount
# Actually execute the command
try:
if argcount == 3 and varnames[2] == 'unknown_args':
return_val = command(parser, args, unknown_args)
else:
if unknown_args:
tty.die('unrecognized arguments: %s' % ' '.join(unknown_args))
return_val = command(parser, args)
except SpackError as e:
e.die()
except KeyboardInterrupt:
sys.stderr.write('\n')
tty.die("Keyboard interrupt.")
# Allow commands to return values if they want to exit with some other code.
if return_val is None:
sys.exit(0)
elif isinstance(return_val, int):
sys.exit(return_val)
else:
tty.die("Bad return value from command %s: %s"
% (args.command, return_val))
def main(args):
# Just print help and exit if run with no arguments at all
if len(args) == 1:
parser.print_help()
sys.exit(1)
# actually parse the args.
args, unknown = parser.parse_known_args()
if args.profile:
import cProfile
cProfile.runctx('_main(args, unknown)', globals(), locals(),
sort='time')
elif args.pdb:
import pdb
pdb.runctx('_main(args, unknown)', globals(), locals())
else:
_main(args, unknown)
if __name__ == '__main__':
main(sys.argv)
# Once we've set up the system path, run the spack main method
import spack.main # noqa
sys.exit(spack.main.main())

6
bin/spack-python
View File

@@ -1,14 +1,14 @@
#!/bin/sh
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as

9
etc/spack/defaults/config.yaml
View File

@@ -18,6 +18,9 @@ config:
# You can use $spack here to refer to the root of the spack instance.
install_tree: $spack/opt/spack
# Locations where templates should be found
template_dirs:
- $spack/templates
# Locations where different types of modules should be installed.
module_roots:
@@ -66,3 +69,9 @@ config:
# If set to true, `spack install` and friends will NOT clean
# potentially harmful variables from the build environment. Use wisely.
dirty: false
# The default number of jobs to use when running `make` in parallel.
# If set to 4, for example, `spack install` will run `make -j4`.
# If not set, all available cores are used by default.
# build_jobs: 4

4
etc/spack/defaults/modules.yaml
View File

@@ -40,3 +40,7 @@ modules:
- PKG_CONFIG_PATH
'':
- CMAKE_PREFIX_PATH
lmod:
hierarchy:
- mpi

16
etc/spack/defaults/packages.yaml
View File

@@ -17,7 +17,21 @@ packages:
all:
compiler: [gcc, intel, pgi, clang, xl, nag]
providers:
mpi: [openmpi, mpich]
awk: [gawk]
blas: [openblas]
daal: [intel-daal]
elf: [elfutils]
golang: [gcc]
ipp: [intel-ipp]
java: [jdk]
lapack: [openblas]
mkl: [intel-mkl]
mpe: [mpe2]
mpi: [openmpi, mpich]
opencl: [pocl]
openfoam: [openfoam-com, openfoam-org, foam-extend]
pil: [py-pillow]
scalapack: [netlib-scalapack]
szip: [libszip, libaec]
tbb: [intel-tbb]
jpeg: [libjpeg-turbo, libjpeg]

2
lib/spack/docs/.gitignore vendored
View File

@@ -1,5 +1,5 @@
package_list.rst
command_index.rst
spack*.rst
modules.rst
llnl*.rst
_build

10
lib/spack/docs/Makefile
View File

@@ -3,13 +3,12 @@
# You can set these variables from the command line.
SPHINXOPTS = -E
JOBS ?= $(shell python -c 'import multiprocessing; print multiprocessing.cpu_count()')
SPHINXBUILD = sphinx-build -j $(JOBS)
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
export PYTHONPATH := ../../spack:$(PYTHONPATH)
APIDOC_FILES = spack*.rst
APIDOC_FILES = spack*.rst llnl*.rst
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
@@ -58,7 +57,8 @@ upload:
git push -f github gh-pages
apidoc:
sphinx-apidoc -T -o . $(PYTHONPATH)/spack
sphinx-apidoc -f -T -o . ../spack
sphinx-apidoc -f -T -o . ../llnl
help:
@echo "Please use \`make <target>' where <target> is one of"
@@ -83,7 +83,7 @@ help:
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
clean:
-rm -f package_list.rst command_index.rst modules.rst
-rm -f package_list.rst command_index.rst
-rm -rf $(BUILDDIR)/* $(APIDOC_FILES)
html:

2
lib/spack/docs/_themes/sphinx_rtd_theme/footer.html vendored
View File

@@ -24,7 +24,7 @@
<br/>
Written by Todd Gamblin (<a href="mailto:tgamblin@llnl.gov">tgamblin@llnl.gov</a>) and
many contributors. LLNL-CODE-647188.
many <a href="https://github.com/spack/spack/graphs/contributors">contributors.</a> LLNL-CODE-647188.
{%- if last_updated %}
<br/>

43
lib/spack/docs/basic_usage.rst
View File

@@ -808,17 +808,46 @@ Specifying Specs by Hash
^^^^^^^^^^^^^^^^^^^^^^^^
Complicated specs can become cumbersome to enter on the command line,
especially when many of the qualifications are necessary to
distinguish between similar installs, for example when using the
``uninstall`` command. To avoid this, when referencing an existing spec,
especially when many of the qualifications are necessary to distinguish
between similar installs. To avoid this, when referencing an existing spec,
Spack allows you to reference specs by their hash. We previously
discussed the spec hash that Spack computes. In place of a spec in any
command, substitute ``/<hash>`` where ``<hash>`` is any amount from
the beginning of a spec hash. If the given spec hash is sufficient
to be unique, Spack will replace the reference with the spec to which
it refers. Otherwise, it will prompt for a more qualified hash.
the beginning of a spec hash.
Note that this will not work to reinstall a depencency uninstalled by
For example, lets say that you accidentally installed two different
``mvapich2`` installations. If you want to uninstall one of them but don't
know what the difference is, you can run:
.. code-block:: console
$ spack find --long mvapich2
==> 2 installed packages.
-- linux-centos7-x86_64 / gcc@6.3.0 ----------
qmt35td mvapich2@2.2%gcc
er3die3 mvapich2@2.2%gcc
You can then uninstall the latter installation using:
.. code-block:: console
$ spack uninstall /er3die3
Or, if you want to build with a specific installation as a dependency,
you can use:
.. code-block:: console
$ spack install trilinos ^/er3die3
If the given spec hash is sufficiently long as to be unique, Spack will
replace the reference with the spec to which it refers. Otherwise, it will
prompt for a more qualified hash.
Note that this will not work to reinstall a dependency uninstalled by
``spack uninstall --force``.
.. _cmd-spack-providers:

132
lib/spack/docs/binary_caches.rst Normal file
View File

@@ -0,0 +1,132 @@
.. _binary_caches:
============
Build caches
============
Some sites may encourage users to set up their own test environments
before carrying out central installations, or some users may prefer to set
up these environments on their own motivation. To reduce the load of
recompiling otherwise identical package specs in different installations,
installed packages can be put into build cache tarballs, uploaded to
your Spack mirror and then downloaded and installed by others.
--------------------------
Creating build cache files
--------------------------
A compressed tarball of an installed package is created. Tarballs are created
for all of its link and run dependency packages as well. Compressed tarballs are
signed with gpg and signature and tarball and put in a ``.spack`` file. Optionally,
the rpaths (and ids and deps on macOS) can be changed to paths relative to
the Spack install tree before the tarball is created.
Build caches are created via:
.. code-block:: console
$ spack buildcache create spec
---------------------------------------
Finding or installing build cache files
---------------------------------------
To find build caches or install build caches, a Spack mirror must be configured
with:
.. code-block:: console
$ spack mirror add <name> <url>
Build caches are found via:
.. code-block:: console
$ spack buildcache list
Build caches are installed via:
.. code-block:: console
$ spack buildcache install
----------
Relocation
----------
Initial build and later installation do not necessarily happen at the same
location. Spack provides a relocation capability and corrects for RPATHs and
non-relocatable scripts. However, many packages compile paths into binary
artifacts directly. In such cases, the build instructions of this package would
need to be adjusted for better re-locatability.
.. _cmd-spack-buildcache:
--------------------
``spack buildcache``
--------------------
^^^^^^^^^^^^^^^^^^^^^^^^^^^
``spack buildcache create``
^^^^^^^^^^^^^^^^^^^^^^^^^^^
Create tarball of installed Spack package and all dependencies.
Tarballs are checksummed and signed if gpg2 is available.
Places them in a directory ``build_cache`` that can be copied to a mirror.
Commands like ``spack buildcache install`` will search Spack mirrors for build_cache to get the list of build caches.
============== ========================================================================================================================
Arguments Description
============== ========================================================================================================================
``<specs>`` list of partial specs or hashes with a leading ``/`` to match from installed packages and used for creating build caches
``-d <path>`` directory in which ``build_cache`` directory is created, defaults to ``.``
``-f`` overwrite ``.spack`` file in ``build_cache`` directory if it exists
``-k <key>`` the key to sign package with. In the case where multiple keys exist, the package will be unsigned unless ``-k`` is used.
``-r`` make paths in binaries relative before creating tarball
``-y`` answer yes to all create unsigned ``build_cache`` questions
============== ========================================================================================================================
^^^^^^^^^^^^^^^^^^^^^^^^^
``spack buildcache list``
^^^^^^^^^^^^^^^^^^^^^^^^^
Retrieves all specs for build caches available on a Spack mirror.
============== =====================================================================================
Arguments Description
============== =====================================================================================
``<specs>`` list of partial package specs to be matched against specs downloaded for build caches
============== =====================================================================================
E.g. ``spack buildcache list gcc`` with print only commands to install ``gcc`` package(s)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
``spack buildcache install``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Retrieves all specs for build caches available on a Spack mirror and installs build caches
with specs matching the specs input.
============== ==============================================================================================
Arguments Description
============== ==============================================================================================
``<specs>`` list of partial package specs or hashes with a leading ``/`` to be installed from build caches
``-f`` remove install directory if it exists before unpacking tarball
``-y`` answer yes to all to don't verify package with gpg questions
============== ==============================================================================================
^^^^^^^^^^^^^^^^^^^^^^^^^
``spack buildcache keys``
^^^^^^^^^^^^^^^^^^^^^^^^^
List public keys available on Spack mirror.
========= ==============================================
Arguments Description
========= ==============================================
``-i`` trust the keys downloaded with prompt for each
``-y`` answer yes to all trust all keys downloaded
========= ==============================================

4
lib/spack/docs/build_settings.rst
View File

@@ -71,9 +71,9 @@ package lacks a spec component, such as missing a compiler or
package version, then Spack will guess the missing component based
on its most-favored packages, and it may guess incorrectly.
Each package version and compilers listed in an external should
Each package version and compiler listed in an external should
have entries in Spack's packages and compiler configuration, even
though the package and compiler may not every be built.
though the package and compiler may not ever be built.
The packages configuration can tell Spack to use an external location
for certain package versions, but it does not restrict Spack to using

60
lib/spack/docs/conf.py
View File

@@ -1,14 +1,14 @@
# flake8: noqa
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
@@ -49,8 +49,11 @@
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('exts'))
sys.path.insert(0, os.path.abspath('../external'))
if sys.version_info[0] < 3:
sys.path.insert(0, os.path.abspath('../external/yaml/lib'))
else:
sys.path.insert(0, os.path.abspath('../external/yaml/lib3'))
sys.path.append(os.path.abspath('..'))
# Add the Spack bin directory to the path so that we can use its output in docs.
@@ -58,11 +61,6 @@
os.environ['SPACK_ROOT'] = spack_root
os.environ['PATH'] += '%s%s/bin' % (os.pathsep, spack_root)
# Get the spack version for use in the docs
spack_version = subprocess.Popen(
[spack_root + '/bin/spack', '-V'],
stderr=subprocess.PIPE).communicate()[1].strip().split('.')
# Set an environment variable so that colify will print output like it would to
# a terminal.
os.environ['COLIFY_SIZE'] = '25x120'
@@ -91,10 +89,21 @@
for cmd in sorted(command_names):
index.write(' * :ref:`%s`\n' % cmd)
#
# Run sphinx-apidoc
sphinx_apidoc(['-T', '-o', '.', '../spack'])
os.remove('modules.rst')
#
# Remove any previous API docs
# Read the Docs doesn't clean up after previous builds
# Without this, the API Docs will never actually update
#
apidoc_args = [
'--force', # Older versions of Sphinx ignore the first argument
'--force', # Overwrite existing files
'--no-toc', # Don't create a table of contents file
'--output-dir=.', # Directory to place all output
]
sphinx_apidoc(apidoc_args + ['../spack'])
sphinx_apidoc(apidoc_args + ['../llnl'])
#
# Exclude everything in spack.__all__ from indexing. All of these
@@ -110,17 +119,31 @@
for line in fileinput.input('spack.rst', inplace=1):
if handling_spack:
if not line.startswith(' :noindex:'):
print ' :noindex: %s' % ' '.join(spack.__all__)
print(' :noindex: %s' % ' '.join(spack.__all__))
handling_spack = False
if line.startswith('.. automodule::'):
handling_spack = (line == '.. automodule:: spack\n')
print line,
sys.stdout.write(line)
# Enable todo items
todo_include_todos = True
#
# Disable duplicate cross-reference warnings.
#
from sphinx.domains.python import PythonDomain
class PatchedPythonDomain(PythonDomain):
def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode):
if 'refspecific' in node:
del node['refspecific']
return super(PatchedPythonDomain, self).resolve_xref(
env, fromdocname, builder, typ, target, node, contnode)
def setup(sphinx):
sphinx.override_domain(PatchedPythonDomain)
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
@@ -130,6 +153,7 @@
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc',
'sphinx.ext.graphviz',
'sphinx.ext.napoleon',
'sphinx.ext.todo',
'sphinxcontrib.programoutput']
@@ -156,16 +180,16 @@
# General information about the project.
project = u'Spack'
copyright = u'2013-2015, Lawrence Livermore National Laboratory.'
copyright = u'2013-2017, Lawrence Livermore National Laboratory.'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '.'.join(spack_version[:2])
version = str(spack.spack_version.up_to(2))
# The full version, including alpha/beta/rc tags.
release = '.'.join(spack_version[:2])
release = str(spack.spack_version.up_to(2))
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@@ -225,7 +249,7 @@
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
html_logo = '../../../share/spack/logo/spack-logo-white-text-48.png'
html_logo = '../../../share/spack/logo/spack-logo-white-text.svg'
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32

93
lib/spack/docs/config_yaml.rst
View File

@@ -14,27 +14,6 @@ see the default settings by looking at
These settings can be overridden in ``etc/spack/config.yaml`` or
``~/.spack/config.yaml``. See :ref:`configuration-scopes` for details.
.. _config-file-variables:
------------------------------
Config file variables
------------------------------
You may notice some variables prefixed with ``$`` in the settings above.
Spack understands several variables that can be used in values of
configuration parameters. They are:
* ``$spack``: path to the prefix of this spack installation
* ``$tempdir``: default system temporary directory (as specified in
Python's `tempfile.tempdir
<https://docs.python.org/2/library/tempfile.html#tempfile.tempdir>`_
variable.
* ``$user``: name of the current user
Note that, as with shell variables, you can write these as ``$varname``
or with braces to distinguish the variable from surrounding characters:
``${varname}``.
--------------------
``install_tree``
--------------------
@@ -42,6 +21,43 @@ or with braces to distinguish the variable from surrounding characters:
The location where Spack will install packages and their dependencies.
Default is ``$spack/opt/spack``.
---------------------------------------------------
``install_hash_length`` and ``install_path_scheme``
---------------------------------------------------
The default Spack installation path can be very long and can create
problems for scripts with hardcoded shebangs. There are two parameters
to help with that. Firstly, the ``install_hash_length`` parameter can
set the length of the hash in the installation path from 1 to 32. The
default path uses the full 32 characters.
Secondly, it is
also possible to modify the entire installation scheme. By default
Spack uses
``${ARCHITECTURE}/${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}-${HASH}``
where the tokens that are available for use in this directive are the
same as those understood by the ``Spec.format`` method. Using this parameter it
is possible to use a different package layout or reduce the depth of
the installation paths. For example
.. code-block:: yaml
config:
install_path_scheme: '${PACKAGE}/${VERSION}/${HASH:7}'
would install packages into sub-directories using only the package
name, version and a hash length of 7 characters.
When using either parameter to set the hash length it only affects the
representation of the hash in the installation directory. You
should be aware that the smaller the hash length the more likely
naming conflicts will occur. These parameters are independent of those
used to configure module names.
.. warning:: Modifying the installation hash length or path scheme after
packages have been installed will prevent Spack from being
able to find the old installation directories.
--------------------
``module_roots``
--------------------
@@ -62,8 +78,8 @@ See :ref:`modules` for details.
``build_stage``
--------------------
Spack is designed to run out of a user home directories, and on many
systems the home directory a (slow) network filesystem. On most systems,
Spack is designed to run out of a user home directory, and on many
systems the home directory is a (slow) network filesystem. On most systems,
building in a temporary filesystem results in faster builds than building
in the home directory. Usually, there is also more space available in
the temporary location than in the home directory. So, Spack tries to
@@ -85,12 +101,12 @@ See :ref:`config-file-variables` for more on ``$tempdir`` and ``$spack``.
When Spack builds a package, it creates a temporary directory within the
``build_stage``, and it creates a symbolic link to that directory in
``$spack/var/spack/stage``. This is used totrack the stage.
``$spack/var/spack/stage``. This is used to track the stage.
After a package is successfully installed, Spack deletes the temporary
directory it used to build. Unsuccessful builds are not deleted, but you
can manually purge them with :ref:`spack purge --stage
<cmd-spack-purge>`.
can manually purge them with :ref:`spack clean --stage
<cmd-spack-clean>`.
.. note::
@@ -105,8 +121,8 @@ can manually purge them with :ref:`spack purge --stage
Location to cache downloaded tarballs and repositories. By default these
are stored in ``$spack/var/spack/cache``. These are stored indefinitely
by default. Can be purged with :ref:`spack purge --downloads
<cmd-spack-purge>`.
by default. Can be purged with :ref:`spack clean --downloads
<cmd-spack-clean>`.
--------------------
``misc_cache``
@@ -114,7 +130,7 @@ by default. Can be purged with :ref:`spack purge --downloads
Temporary directory to store long-lived cache files, such as indices of
packages available in repositories. Defaults to ``~/.spack/cache``. Can
be purged with :ref:`spack purge --misc-cache <cmd-spack-purge>`.
be purged with :ref:`spack clean --misc-cache <cmd-spack-clean>`.
--------------------
``verify_ssl``
@@ -143,7 +159,24 @@ the way packages build. This includes ``LD_LIBRARY_PATH``, ``CPATH``,
``LIBRARY_PATH``, ``DYLD_LIBRARY_PATH``, and others.
By default, builds are ``clean``, but on some machines, compilers and
other tools may need custom ``LD_LIBRARY_PATH`` setings to run. You can
other tools may need custom ``LD_LIBRARY_PATH`` settings to run. You can
set ``dirty`` to ``true`` to skip the cleaning step and make all builds
"dirty" by default. Be aware that this will reduce the reproducibility
of builds.
--------------
``build_jobs``
--------------
Unless overridden in a package or on the command line, Spack builds all
packages in parallel. For a build system that uses Makefiles, this means
running ``make -j<build_jobs>``, where ``build_jobs`` is the number of
threads to use.
The default parallelism is equal to the number of cores on your machine.
If you work on a shared login node or have a strict ulimit, it may be
necessary to set the default to a lower value. By setting ``build_jobs``
to 4, for example, commands like ``spack install`` will run ``make -j4``
instead of hogging every core.
To build all software in serial, set ``build_jobs`` to 1.

89
lib/spack/docs/configuration.rst
View File

@@ -45,20 +45,27 @@ Configuration Scopes
-------------------------
Spack pulls configuration data from files in several directories. There
are three configuration scopes. From lowest to highest:
are four configuration scopes. From lowest to highest:
1. **defaults**: Stored in ``$(prefix)/etc/spack/defaults/``. These are
#. **defaults**: Stored in ``$(prefix)/etc/spack/defaults/``. These are
the "factory" settings. Users should generally not modify the settings
here, but should override them in other configuration scopes. The
defaults here will change from version to version of Spack.
2. **site**: Stored in ``$(prefix)/etc/spack/``. Settings here affect
#. **system**: Stored in ``/etc/spack``. These are settings for this
machine, or for all machines on which this file system is
mounted. The site scope can be used for settings idiosyncratic to a
particular machine, such as the locations of compilers or external
packages. These settings are presumably controlled by someone with
root access on the machine.
#. **site**: Stored in ``$(prefix)/etc/spack/``. Settings here affect
only *this instance* of Spack, and they override defaults. The site
scope can can be used for per-project settings (one spack instance per
project) or for site-wide settings on a multi-user machine (e.g., for
a common spack instance).
3. **user**: Stored in the home directory: ``~/.spack/``. These settings
#. **user**: Stored in the home directory: ``~/.spack/``. These settings
affect all instances of Spack and take the highest precedence.
Each configuration directory may contain several configuration files,
@@ -78,22 +85,25 @@ Platform-specific scopes
-------------------------
For each scope above, there can *also* be platform-specific settings.
For example, on Blue Gene/Q machines, Spack needs to know the location of
cross-compilers for the compute nodes. This configuration is in
``etc/spack/defaults/bgq/compilers.yaml``. It will take precedence over
settings in the ``defaults`` scope, but can still be overridden by
settings in ``site``, ``site/bgq``, ``user``, or ``user/bgq``. So, the
full scope precedence is:
For example, on Blue Gene/Q machines, Spack needs to know the location
of cross-compilers for the compute nodes. This configuration is in
``etc/spack/defaults/bgq/compilers.yaml``. It will take precedence
over settings in the ``defaults`` scope, but can still be overridden
by settings in ``system``, ``system/bgq``, ``site``, ``site/bgq``,
``user``, or ``user/bgq``. So, the full scope precedence is:
1. ``defaults``
2. ``defaults/<platform>``
3. ``site``
4. ``site/<platform>``
5. ``user``
6. ``user/<platform>``
3. ``system``
4. ``system/<platform>``
5. ``site``
6. ``site/<platform>``
7. ``user``
8. ``user/<platform>``
You can get the name to use for ``<platform>`` by running ``spack arch
--platform``.
--platform``. The system config scope has a ``<platform>`` section for
sites at which ``/etc`` is mounted on multiple heterogeneous machines.
-------------------------
Scope precedence
@@ -251,3 +261,52 @@ The merged configuration would look like this:
- /lustre-scratch/$user
- ~/mystage
$ _
.. _config-file-variables:
------------------------------
Config file variables
------------------------------
Spack understands several variables which can be used in config file paths
where ever they appear. There are three sets of these variables, Spack specific
variables, environment variables, and user path variables. Spack specific
variables and environment variables both are indicated by prefixing the variable
name with ``$``. User path variables are indicated at the start of the path with
``~`` or ``~user``. Let's discuss each in turn.
^^^^^^^^^^^^^^^^^^^^^^^^
Spack Specific Variables
^^^^^^^^^^^^^^^^^^^^^^^^
Spack understands several special variables. These are:
* ``$spack``: path to the prefix of this spack installation
* ``$tempdir``: default system temporary directory (as specified in
Python's `tempfile.tempdir
<https://docs.python.org/2/library/tempfile.html#tempfile.tempdir>`_
variable.
* ``$user``: name of the current user
Note that, as with shell variables, you can write these as ``$varname``
or with braces to distinguish the variable from surrounding characters:
``${varname}``. Their names are also case insensitive meaning that ``$SPACK``
works just as well as ``$spack``. These special variables are also
substituted first, so any environment variables with the same name will not
be used.
^^^^^^^^^^^^^^^^^^^^^
Environment Variables
^^^^^^^^^^^^^^^^^^^^^
Spack then uses ``os.path.expandvars`` to expand any remaining environment
variables.
^^^^^^^^^^^^^^
User Variables
^^^^^^^^^^^^^^
Spack also uses the ``os.path.expanduser`` function on the path to expand
any user tilde paths such as ``~`` or ``~user``. These tilde paths must appear
at the beginning of the path or ``os.path.expanduser`` will not properly
expand them.

18
lib/spack/docs/contribution_guide.rst
View File

@@ -26,13 +26,13 @@ Spack uses a rough approximation of the `Git Flow <http://nvie.com/posts/a-succe
branching model. The develop branch contains the latest contributions, and
master is always tagged and points to the latest stable release. Therefore, when
you send your request, make ``develop`` the destination branch on the
`Spack repository <https://github.com/LLNL/spack>`_.
`Spack repository <https://github.com/spack/spack>`_.
----------------------
Continuous Integration
----------------------
Spack uses `Travis CI <https://travis-ci.org/LLNL/spack>`_ for Continuous Integration
Spack uses `Travis CI <https://travis-ci.org/spack/spack>`_ for Continuous Integration
testing. This means that every time you submit a pull request, a series of tests will
be run to make sure you didn't accidentally introduce any bugs into Spack. Your PR
will not be accepted until it passes all of these tests. While you can certainly wait
@@ -40,7 +40,7 @@ for the results of these tests after submitting a PR, we recommend that you run
locally to speed up the review process.
If you take a look in ``$SPACK_ROOT/.travis.yml``, you'll notice that we test
against Python 2.6 and 2.7. We currently perform 3 types of tests:
against Python 2.6, 2.7, and 3.3-3.6. We currently perform 3 types of tests:
^^^^^^^^^^
Unit Tests
@@ -75,7 +75,10 @@ This allows you to develop iteratively: make a change, test that change, make
another change, test that change, etc. To get a list of all available unit
tests, run:
.. command-output:: spack test --collect-only
.. command-output:: spack test --list
A more detailed list of available unit tests can be found by running
``spack test --long-list``.
Unit tests are crucial to making sure bugs aren't introduced into Spack. If you
are modifying core Spack libraries or adding new functionality, please consider
@@ -121,7 +124,7 @@ command:
"extends" in ``package.py`` files.
More approved flake8 exemptions can be found
`here <https://github.com/LLNL/spack/blob/develop/.flake8>`_.
`here <https://github.com/spack/spack/blob/develop/.flake8>`_.
If all is well, you'll see something like this:
@@ -186,6 +189,7 @@ Building the documentation requires several dependencies, all of which can be
installed with Spack:
* sphinx
* sphinxcontrib-programoutput
* graphviz
* git
* mercurial
@@ -193,7 +197,7 @@ installed with Spack:
.. warning::
Sphinx has `several required dependencies <https://github.com/LLNL/spack/blob/develop/var/spack/repos/builtin/packages/py-sphinx/package.py>`_.
Sphinx has `several required dependencies <https://github.com/spack/spack/blob/develop/var/spack/repos/builtin/packages/py-sphinx/package.py>`_.
If you installed ``py-sphinx`` with Spack, make sure to add all of these
dependencies to your ``PYTHONPATH``. The easiest way to do this is to run
``spack activate py-sphinx`` so that all of the dependencies are symlinked
@@ -224,7 +228,7 @@ your PR is accepted.
There is also a ``run-doc-tests`` script in the Quality Assurance directory.
The only difference between running this script and running ``make`` by hand
is that the script will exit immediately if it encounters an error or warning.
This is necessary for Travis CI. If you made a lot of documentation tests, it
This is necessary for Travis CI. If you made a lot of documentation changes, it
is much quicker to run ``make`` by hand so that you can see all of the warnings
at once.

147
lib/spack/docs/developer_guide.rst
View File

@@ -300,6 +300,46 @@ Stage objects
Writing commands
----------------
Adding a new command to Spack is easy. Simply add a ``<name>.py`` file to
``lib/spack/spack/cmd/``, where ``<name>`` is the name of the subcommand.
At the bare minimum, two functions are required in this file:
^^^^^^^^^^^^^^^^^^
``setup_parser()``
^^^^^^^^^^^^^^^^^^
Unless your command doesn't accept any arguments, a ``setup_parser()``
function is required to define what arguments and flags your command takes.
See the `Argparse documentation <https://docs.python.org/2.7/library/argparse.html>`_
for more details on how to add arguments.
Some commands have a set of subcommands, like ``spack compiler find`` or
``spack module refresh``. You can add subparsers to your parser to handle
this. Check out ``spack edit --command compiler`` for an example of this.
A lot of commands take the same arguments and flags. These arguments should
be defined in ``lib/spack/spack/cmd/common/arguments.py`` so that they don't
need to be redefined in multiple commands.
^^^^^^^^^^^^
``<name>()``
^^^^^^^^^^^^
In order to run your command, Spack searches for a function with the same
name as your command in ``<name>.py``. This is the main method for your
command, and can call other helper methods to handle common tasks.
Remember, before adding a new command, think to yourself whether or not this
new command is actually necessary. Sometimes, the functionality you desire
can be added to an existing command. Also remember to add unit tests for
your command. If it isn't used very frequently, changes to the rest of
Spack can cause your command to break without sufficient unit tests to
prevent this from happening.
Whenever you add/remove/rename a command or flags for an existing command,
make sure to update Spack's `Bash tab completion script
<https://github.com/adamjstewart/spack/blob/develop/share/spack/spack-completion.bash>`_.
----------
Unit tests
----------
@@ -312,14 +352,121 @@ Unit testing
Developer commands
------------------
.. _cmd-spack-doc:
^^^^^^^^^^^^^
``spack doc``
^^^^^^^^^^^^^
.. _cmd-spack-test:
^^^^^^^^^^^^^^
``spack test``
^^^^^^^^^^^^^^
.. _cmd-spack-python:
^^^^^^^^^^^^^^^^
``spack python``
^^^^^^^^^^^^^^^^
``spack python`` is a command that lets you import and debug things as if
you were in a Spack interactive shell. Without any arguments, it is similar
to a normal interactive Python shell, except you can import spack and any
other Spack modules:
.. code-block:: console
$ spack python
Spack version 0.10.0
Python 2.7.13, Linux x86_64
>>> from spack.version import Version
>>> a = Version('1.2.3')
>>> b = Version('1_2_3')
>>> a == b
True
>>> c = Version('1.2.3b')
>>> c > a
True
>>>
You can also run a single command:
.. code-block:: console
$ spack python -c 'import distro; distro.linux_distribution()'
('Fedora', '25', 'Workstation Edition')
or a file:
.. code-block:: console
$ spack python ~/test_fetching.py
just like you would with the normal ``python`` command.
.. _cmd-spack-url:
^^^^^^^^^^^^^
``spack url``
^^^^^^^^^^^^^
A package containing a single URL can be used to download several different
versions of the package. If you've ever wondered how this works, all of the
magic is in :mod:`spack.url`. This module contains methods for extracting
the name and version of a package from its URL. The name is used by
``spack create`` to guess the name of the package. By determining the version
from the URL, Spack can replace it with other versions to determine where to
download them from.
The regular expressions in ``parse_name_offset`` and ``parse_version_offset``
are used to extract the name and version, but they aren't perfect. In order
to debug Spack's URL parsing support, the ``spack url`` command can be used.
"""""""""""""""""""
``spack url parse``
"""""""""""""""""""
If you need to debug a single URL, you can use the following command:
.. command-output:: spack url parse http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.0.tar.gz
You'll notice that the name and version of this URL are correctly detected,
and you can even see which regular expressions it was matched to. However,
you'll notice that when it substitutes the version number in, it doesn't
replace the ``2.2`` with ``9.9`` where we would expect ``9.9.9b`` to live.
This particular package may require a ``list_url`` or ``url_for_version``
function.
This command also accepts a ``--spider`` flag. If provided, Spack searches
for other versions of the package and prints the matching URLs.
""""""""""""""""""
``spack url list``
""""""""""""""""""
This command lists every URL in every package in Spack. If given the
``--color`` and ``--extrapolation`` flags, it also colors the part of
the string that it detected to be the name and version. The
``--incorrect-name`` and ``--incorrect-version`` flags can be used to
print URLs that were not being parsed correctly.
"""""""""""""""""""""
``spack url summary``
"""""""""""""""""""""
This command attempts to parse every URL for every package in Spack
and prints a summary of how many of them are being correctly parsed.
It also prints a histogram showing which regular expressions are being
matched and how frequently:
.. command-output:: spack url summary
This command is essential for anyone adding or changing the regular
expressions that parse names and versions. By running this command
before and after the change, you can make sure that your regular
expression fixes more packages than it breaks.
---------
Profiling
---------

9
lib/spack/docs/exts/sphinxcontrib/__init__.py
View File

@@ -1,9 +0,0 @@
# -*- coding: utf-8 -*-
"""
sphinxcontrib
~~~~~~~~~~~~~
Contains 3rd party Sphinx extensions.
"""
__import__('pkg_resources').declare_namespace(__name__)

263
lib/spack/docs/exts/sphinxcontrib/programoutput.py
View File

@@ -1,263 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2010, 2011, 2012, Sebastian Wiesner <lunaryorn@gmail.com>
# All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
"""
sphinxcontrib.programoutput
===========================
This extension provides a directive to include the output of commands as
literal block while building the docs.
.. moduleauthor:: Sebastian Wiesner <lunaryorn@gmail.com>
"""
from __future__ import (print_function, division, unicode_literals,
absolute_import)
import sys
import os
import shlex
from subprocess import Popen, PIPE, STDOUT
from collections import defaultdict, namedtuple
from docutils import nodes
from docutils.parsers import rst
from docutils.parsers.rst.directives import flag, unchanged, nonnegative_int
__version__ = '0.9'
class program_output(nodes.Element):
pass
def _slice(value):
parts = [int(v.strip()) for v in value.split(',')]
if len(parts) > 2:
raise ValueError('too many slice parts')
return tuple((parts + [None] * 2)[:2])
class ProgramOutputDirective(rst.Directive):
has_content = False
final_argument_whitespace = True
required_arguments = 1
option_spec = dict(shell=flag, prompt=flag, nostderr=flag,
ellipsis=_slice, extraargs=unchanged,
returncode=nonnegative_int, cwd=unchanged)
def run(self):
env = self.state.document.settings.env
node = program_output()
node.line = self.lineno
node['command'] = self.arguments[0]
if self.name == 'command-output':
node['show_prompt'] = True
else:
node['show_prompt'] = 'prompt' in self.options
node['hide_standard_error'] = 'nostderr' in self.options
node['extraargs'] = self.options.get('extraargs', '')
_, cwd = env.relfn2path(self.options.get('cwd', '/'))
node['working_directory'] = cwd
node['use_shell'] = 'shell' in self.options
node['returncode'] = self.options.get('returncode', 0)
if 'ellipsis' in self.options:
node['strip_lines'] = self.options['ellipsis']
return [node]
_Command = namedtuple(
'Command', 'command shell hide_standard_error working_directory')
class Command(_Command):
"""
A command to be executed.
"""
def __new__(cls, command, shell=False, hide_standard_error=False,
working_directory='/'):
if isinstance(command, list):
command = tuple(command)
# `chdir()` resolves symlinks, so we need to resolve them too for
# caching to make sure that different symlinks to the same directory
# don't result in different cache keys. Also normalize paths to make
# sure that identical paths are also equal as strings.
working_directory = os.path.normpath(os.path.realpath(
working_directory))
return _Command.__new__(cls, command, shell, hide_standard_error,
working_directory)
@classmethod
def from_program_output_node(cls, node):
"""
Create a command from a :class:`program_output` node.
"""
extraargs = node.get('extraargs', '')
command = (node['command'] + ' ' + extraargs).strip()
return cls(command, node['use_shell'],
node['hide_standard_error'], node['working_directory'])
def execute(self):
"""
Execute this command.
Return the :class:`~subprocess.Popen` object representing the running
command.
"""
if self.shell:
if sys.version_info[0] < 3 and isinstance(self.command, unicode):
command = self.command.encode(sys.getfilesystemencoding())
else:
command = self.command
else:
if sys.version_info[0] < 3 and isinstance(self.command, unicode):
command = shlex.split(self.command.encode(
sys.getfilesystemencoding()))
elif isinstance(self.command, str):
command = shlex.split(self.command)
else:
command = self.command
return Popen(command, shell=self.shell, stdout=PIPE,
stderr=PIPE if self.hide_standard_error else STDOUT,
cwd=self.working_directory)
def get_output(self):
"""
Get the output of this command.
Return a tuple ``(returncode, output)``. ``returncode`` is the
integral return code of the process, ``output`` is the output as
unicode string, with final trailing spaces and new lines stripped.
"""
process = self.execute()
output = process.communicate()[0].decode(
sys.getfilesystemencoding(), 'replace').rstrip()
return process.returncode, output
def __str__(self):
if isinstance(self.command, tuple):
return repr(list(self.command))
return repr(self.command)
class ProgramOutputCache(defaultdict):
"""
Execute command and cache their output.
This class is a mapping. Its keys are :class:`Command` objects represeting
command invocations. Its values are tuples of the form ``(returncode,
output)``, where ``returncode`` is the integral return code of the command,
and ``output`` is the output as unicode string.
The first time, a key is retrieved from this object, the command is
invoked, and its result is cached. Subsequent access to the same key
returns the cached value.
"""
def __missing__(self, command):
"""
Called, if a command was not found in the cache.
``command`` is an instance of :class:`Command`.
"""
result = command.get_output()
self[command] = result
return result
def run_programs(app, doctree):
"""
Execute all programs represented by ``program_output`` nodes in
``doctree``. Each ``program_output`` node in ``doctree`` is then
replaced with a node, that represents the output of this program.
The program output is retrieved from the cache in
``app.env.programoutput_cache``.
"""
if app.config.programoutput_use_ansi:
# enable ANSI support, if requested by config
from sphinxcontrib.ansi import ansi_literal_block
node_class = ansi_literal_block
else:
node_class = nodes.literal_block
cache = app.env.programoutput_cache
for node in doctree.traverse(program_output):
command = Command.from_program_output_node(node)
try:
returncode, output = cache[command]
except EnvironmentError as error:
error_message = 'Command {0} failed: {1}'.format(command, error)
error_node = doctree.reporter.error(error_message, base_node=node)
node.replace_self(error_node)
else:
if returncode != node['returncode']:
app.warn('Unexpected return code {0} from command {1}'.format(
returncode, command))
# replace lines with ..., if ellipsis is specified
if 'strip_lines' in node:
lines = output.splitlines()
start, stop = node['strip_lines']
lines[start:stop] = ['...']
output = '\n'.join(lines)
if node['show_prompt']:
tmpl = app.config.programoutput_prompt_template
output = tmpl.format(command=node['command'], output=output,
returncode=returncode)
new_node = node_class(output, output)
new_node['language'] = 'text'
node.replace_self(new_node)
def init_cache(app):
"""
Initialize the cache for program output at
``app.env.programoutput_cache``, if not already present (e.g. being
loaded from a pickled environment).
The cache is of type :class:`ProgramOutputCache`.
"""
if not hasattr(app.env, 'programoutput_cache'):
app.env.programoutput_cache = ProgramOutputCache()
def setup(app):
app.add_config_value('programoutput_use_ansi', False, 'env')
app.add_config_value('programoutput_prompt_template',
'$ {command}\n{output}', 'env')
app.add_directive('program-output', ProgramOutputDirective)
app.add_directive('command-output', ProgramOutputDirective)
app.connect(str('builder-inited'), init_cache)
app.connect(str('doctree-read'), run_programs)

305
lib/spack/docs/getting_started.rst
View File

@@ -11,9 +11,10 @@ Prerequisites
Spack has the following minimum requirements, which must be installed
before Spack is run:
1. Python 2.6 or 2.7
1. Python 2 (2.6 or 2.7) or 3 (3.3 - 3.6)
2. A C/C++ compiler
3. The ``git`` and ``curl`` commands.
4. If using the ``gpg`` subcommand, ``gnupg2`` is required.
These requirements can be easily installed on most modern Linux systems;
on Macintosh, XCode is required. Spack is designed to run on HPC
@@ -26,11 +27,11 @@ Installation
------------
Getting Spack is easy. You can clone it from the `github repository
<https://github.com/llnl/spack>`_ using this command:
<https://github.com/spack/spack>`_ using this command:
.. code-block:: console
$ git clone https://github.com/llnl/spack.git
$ git clone https://github.com/spack/spack.git
This will create a directory called ``spack``.
@@ -51,7 +52,7 @@ For a richer experience, use Spack's shell support:
.. code-block:: console
# For bash users
# For bash/zsh users
$ export SPACK_ROOT=/path/to/spack
$ . $SPACK_ROOT/share/spack/setup-env.sh
@@ -59,10 +60,15 @@ For a richer experience, use Spack's shell support:
$ setenv SPACK_ROOT /path/to/spack
$ source $SPACK_ROOT/share/spack/setup-env.csh
This automatically adds Spack to your ``PATH`` and allows the ``spack``
command to :ref:`load environment modules <shell-support>` and execute
command to be used to execute spack :ref:`commands <shell-support>` and
:ref:`useful packaging commands <packaging-shell-support>`.
If :ref:`environment-modules or dotkit <InstallEnvironmentModules>` is
installed and available, the ``spack`` command can also load and unload
:ref:`modules <modules>`.
^^^^^^^^^^^^^^^^^
Clean Environment
^^^^^^^^^^^^^^^^^
@@ -85,30 +91,20 @@ Check Installation
With Spack installed, you should be able to run some basic Spack
commands. For example:
.. code-block:: console
.. command-output:: spack spec netcdf
$ spack spec netcdf
...
netcdf@4.4.1%gcc@5.3.0~hdf4+mpi arch=linux-SuSE11-x86_64
^curl@7.50.1%gcc@5.3.0 arch=linux-SuSE11-x86_64
^openssl@system%gcc@5.3.0 arch=linux-SuSE11-x86_64
^zlib@1.2.8%gcc@5.3.0 arch=linux-SuSE11-x86_64
^hdf5@1.10.0-patch1%gcc@5.3.0+cxx~debug+fortran+mpi+shared~szip~threadsafe arch=linux-SuSE11-x86_64
^openmpi@1.10.1%gcc@5.3.0~mxm~pmi~psm~psm2~slurm~sqlite3~thread_multiple~tm+verbs+vt arch=linux-SuSE11-x86_64
^m4@1.4.17%gcc@5.3.0+sigsegv arch=linux-SuSE11-x86_64
^libsigsegv@2.10%gcc@5.3.0 arch=linux-SuSE11-x86_64
^^^^^^^^^^^^^^^^^^^^^^^^^^
Optional: Alternate Prefix
^^^^^^^^^^^^^^^^^^^^^^^^^^
You may want to run Spack out of a prefix other than the git repository
you cloned. The ``spack bootstrap`` command provides this
you cloned. The ``spack clone`` command provides this
functionality. To install spack in a new directory, simply type:
.. code-block:: console
$ spack bootstrap /my/favorite/prefix
$ spack clone /my/favorite/prefix
This will install a new spack script in ``/my/favorite/prefix/bin``,
which you can use just like you would the regular spack script. Each
@@ -225,7 +221,7 @@ If you want to see specifics on a particular compiler, you can run
f77 = /usr/local/bin/ifort-15.0.090
fc = /usr/local/bin/ifort-15.0.090
modules = []
operating system = centos6
operating_system = centos6
...
This shows which C, C++, and Fortran compilers were detected by Spack.
@@ -325,19 +321,6 @@ by adding the following to your ``packages.yaml`` file:
compiler: [gcc@4.9.3]
.. tip::
If you are building your own compiler, some users prefer to have a
Spack instance just for that. For example, create a new Spack in
``~/spack-tools`` and then run ``~/spack-tools/bin/spack install
gcc@4.9.3``. Once the compiler is built, don't build anything
more in that Spack instance; instead, create a new "real" Spack
instance, configure Spack to use the compiler you've just built,
and then build your application software in the new Spack
instance. Following this tip makes it easy to delete all your
Spack packages *except* the compiler.
^^^^^^^^^^^^^^^^^^^^^^^^^^^
Compilers Requiring Modules
^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -415,14 +398,62 @@ provides no Fortran compilers. The user is therefore forced to use a
mixed toolchain: XCode-provided Clang for C/C++ and GNU ``gfortran`` for
Fortran.
#. You need to make sure that command-line tools are installed. To that
end run ``$ xcode-select --install``.
#. You need to make sure that Xcode is installed. Run the following command:
#. Run ``$ spack compiler find`` to locate Clang.
.. code-block:: console
$ xcode-select --install
If the Xcode command-line tools are already installed, you will see an
error message:
.. code-block:: none
xcode-select: error: command line tools are already installed, use "Software Update" to install updates
#. For most packages, the Xcode command-line tools are sufficient. However,
some packages like ``qt`` require the full Xcode suite. You can check
to see which you have installed by running:
.. code-block:: console
$ xcode-select -p
If the output is:
.. code-block:: none
/Applications/Xcode.app/Contents/Developer
you already have the full Xcode suite installed. If the output is:
.. code-block:: none
/Library/Developer/CommandLineTools
you only have the command-line tools installed. The full Xcode suite can
be installed through the App Store. Make sure you launch the Xcode
application and accept the license agreement before using Spack.
It may ask you to install additional components. Alternatively, the license
can be accepted through the command line:
.. code-block:: console
$ sudo xcodebuild -license accept
Note: the flag is ``-license``, not ``--license``.
#. Run ``spack compiler find`` to locate Clang.
#. There are different ways to get ``gfortran`` on macOS. For example, you can
install GCC with Spack (``$ spack install gcc``) or with Homebrew
(``$ brew install gcc``).
install GCC with Spack (``spack install gcc``) or with Homebrew
(``brew install gcc``).
#. The only thing left to do is to edit ``~/.spack/compilers.yaml`` to provide
the path to ``gfortran``:
@@ -438,7 +469,7 @@ Fortran.
fc: /path/to/bin/gfortran
If you used Spack to install GCC, you can get the installation prefix by
``$ spack location -i gcc`` (this will only work if you have a single version
``spack location -i gcc`` (this will only work if you have a single version
of GCC installed). Whereas for Homebrew, GCC is installed in
``/usr/local/Cellar/gcc/x.y.z``.
@@ -650,6 +681,7 @@ Or it can be set permanently in your ``compilers.yaml``:
fflags: -mismatch
spec: nag@6.1
---------------
System Packages
---------------
@@ -712,19 +744,22 @@ example:
$ curl -O https://github.com/ImageMagick/ImageMagick/archive/7.0.2-7.tar.gz
The recommended way to tell Spack to use the system-supplied OpenSSL is
to add the following to ``packages.yaml``. Note that the ``@system``
"version" means "I don't care what version it is, just use what is
there." This is reasonable for OpenSSL, which has a stable API.
To tell Spack to use the system-supplied OpenSSL, first determine what
version you have:
.. code-block:: console
$ openssl version
OpenSSL 1.0.2g 1 Mar 2016
Then add the following to ``~/.spack/packages.yaml``:
.. code-block:: yaml
packages:
openssl:
paths:
openssl@system: /usr
version: [system]
openssl@1.0.2g: /usr
buildable: False
@@ -740,8 +775,7 @@ to add the following to ``packages.yaml``:
packages:
netlib-lapack:
paths:
netlib-lapack@system: /usr
version: [system]
netlib-lapack@3.6.1: /usr
buildable: False
all:
providers:
@@ -750,11 +784,9 @@ to add the following to ``packages.yaml``:
.. note::
The ``@system`` "version" means "I don't care what version it is,
just use what is there." Above we pretend that the system-provided
Blas/Lapack is ``netlib-lapack`` only because it is the only BLAS / LAPACK
provider which use standard names for libraries (as opposed to, for example,
`libopenblas.so`).
Above we pretend that the system-provided BLAS / LAPACK is ``netlib-lapack``
only because it is the only BLAS / LAPACK provider which use standard names
for libraries (as opposed to, for example, ``libopenblas.so``).
Although we specify external package in ``/usr``, Spack is smart enough not
to add ``/usr/lib`` to RPATHs, where it could cause unrelated system
@@ -784,7 +816,7 @@ This problem is related to OpenSSL, and in some cases might be solved
by installing a new version of ``git`` and ``openssl``:
#. Run ``spack install git``
#. Add the output of ``spack module loads git`` to your ``.bahsrc``.
#. Add the output of ``spack module loads git`` to your ``.bashrc``.
If this doesn't work, it is also possible to disable checking of SSL
certificates by using:
@@ -857,6 +889,10 @@ well. They can generally be activated as in the ``curl`` example above;
or some systems might already have an appropriate hand-built
environment module that may be loaded. Either way works.
If you find that you are missing some of these programs, ``spack`` can
build some of them for you with ``spack bootstrap``. Currently supported
programs are ``environment-modules``.
A few notes on specific programs in this list:
""""""""""""""""""""""""""
@@ -893,53 +929,72 @@ Environment Modules
In order to use Spack's generated environment modules, you must have
installed one of *Environment Modules* or *Lmod*. On many Linux
distributions, this can be installed from the vendor's repository. For
example: ``yum install environment-modules`` (Fedora/RHEL/CentOS). If
your Linux distribution does not have Environment Modules, you can get it
with Spack:
example: ``yum install environment-modules`` (Fedora/RHEL/CentOS). If
your Linux distribution does not have Environment Modules, Spack can
build it for you!
#. Consider using system tcl (as long as your system has Tcl version 8.0 or later):
What follows are three steps describing how to install and use environment-modules with spack.
#) Identify its location using ``which tclsh``
#) Identify its version using ``echo 'puts $tcl_version;exit 0' | tclsh``
#) Add to ``~/.spack/packages.yaml`` and modify as appropriate:
#. Install ``environment-modules``.
.. code-block:: yaml
* ``spack bootstrap`` will build ``environment-modules`` for you (and may build
other packages that are useful to the operation of Spack)
packages:
tcl:
paths:
tcl@8.5: /usr
version: [8.5]
buildable: False
* Install ``environment-modules`` using ``spack install`` with
``spack install environment-modules~X`` (The ``~X`` variant builds without Xorg
dependencies, but ``environment-modules`` works fine too.)
#. Install with:
#. Add ``modulecmd`` to ``PATH`` and create a ``module`` command.
* If you are using ``bash`` or ``ksh``, Spack can currently do this for you as well.
After installing ``environment-modules`` following the step
above, source Spack's shell integration script. This will automatically
detect the lack of ``modulecmd`` and ``module``, and use the installed
``environment-modules`` from ``spack bootstrap`` or ``spack install``.
.. code-block:: console
# For bash/zsh users
$ export SPACK_ROOT=/path/to/spack
$ . $SPACK_ROOT/share/spack/setup-env.sh
* If you prefer to do it manually, you can activate with the following
script (or apply the updates to your ``.bashrc`` file manually):
.. code-block:: sh
TMP=`tempfile`
echo >$TMP
MODULE_HOME=`spack location --install-dir environment-modules`
MODULE_VERSION=`ls -1 $MODULE_HOME/Modules | head -1`
${MODULE_HOME}/Modules/${MODULE_VERSION}/bin/add.modules <$TMP
cp .bashrc $TMP
echo "MODULE_VERSION=${MODULE_VERSION}" > .bashrc
cat $TMP >>.bashrc
This is added to your ``.bashrc`` (or similar) files, enabling Environment
Modules when you log in.
#. Test that the ``module`` command is found with:
.. code-block:: console
$ spack install environment-modules
$ module avail
#. Activate with the following script (or apply the updates to your
``.bashrc`` file manually):
.. code-block:: sh
If ``tcl`` 8.0 or later is installed on your system, you can prevent
spack from rebuilding ``tcl`` as part of the ``environment-modules`` dependency
stack by adding the following to your ``~/.spack/packages.yaml`` replacing
version 8.5 with whatever version is installed on your system:
TMP=`tempfile`
echo >$TMP
MODULE_HOME=`spack location --install-dir environment-modules`
MODULE_VERSION=`ls -1 $MODULE_HOME/Modules | head -1`
${MODULE_HOME}/Modules/${MODULE_VERSION}/bin/add.modules <$TMP
cp .bashrc $TMP
echo "MODULE_VERSION=${MODULE_VERSION}" > .bashrc
cat $TMP >>.bashrc
This adds to your ``.bashrc`` (or similar) files, enabling Environment
Modules when you log in. Re-load your .bashrc (or log out and in
again), and then test that the ``module`` command is found with:
.. code-block:: console
$ module avail
.. code-block:: yaml
packages:
tcl:
paths:
tcl@8.5: /usr
buildable: False
^^^^^^^^^^^^^^^^^
Package Utilities
@@ -996,6 +1051,73 @@ written in C/C++/Fortran would need it. A potential workaround is to
load a recent ``binutils`` into your environment and use the ``--dirty``
flag.
-----------
GPG Signing
-----------
.. _cmd-spack-gpg:
^^^^^^^^^^^^^
``spack gpg``
^^^^^^^^^^^^^
Spack has support for signing and verifying packages using GPG keys. A
separate keyring is used for Spack, so any keys available in the user's home
directory are not used.
^^^^^^^^^^^^^^^^^^
``spack gpg init``
^^^^^^^^^^^^^^^^^^
When Spack is first installed, its keyring is empty. Keys stored in
:file:`var/spack/gpg` are the default keys for a Spack installation. These
keys may be imported by running ``spack gpg init``. This will import the
default keys into the keyring as trusted keys.
^^^^^^^^^^^^^
Trusting keys
^^^^^^^^^^^^^
Additional keys may be added to the keyring using
``spack gpg trust <keyfile>``. Once a key is trusted, packages signed by the
owner of they key may be installed.
^^^^^^^^^^^^^
Creating keys
^^^^^^^^^^^^^
You may also create your own key so that you may sign your own packages using
``spack gpg create <name> <email>``. By default, the key has no expiration,
but it may be set with the ``--expires <date>`` flag (see the ``gnupg2``
documentation for accepted date formats). It is also recommended to add a
comment as to the use of the key using the ``--comment <comment>`` flag. The
public half of the key can also be exported for sharing with others so that
they may use packages you have signed using the ``--export <keyfile>`` flag.
Secret keys may also be later exported using the
``spack gpg export <location> [<key>...]`` command.
^^^^^^^^^^^^
Listing keys
^^^^^^^^^^^^
In order to list the keys available in the keyring, the
``spack gpg list`` command will list trusted keys with the ``--trusted`` flag
and keys available for signing using ``--signing``. If you would like to
remove keys from your keyring, ``spack gpg untrust <keyid>``. Key IDs can be
email addresses, names, or (best) fingerprints.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Signing and Verifying Packages
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
In order to sign a package, ``spack gpg sign <file>`` should be used. By
default, the signature will be written to ``<file>.asc``, but that may be
changed by using the ``--output <file>`` flag. If there is only one signing
key available, it will be used, but if there is more than one, the key to use
must be specified using the ``--key <keyid>`` flag. The ``--clearsign`` flag
may also be used to create a signed file which contains the contents, but it
is not recommended. Signed packages may be verified by using
``spack gpg verify <file>``.
.. _cray-support:
@@ -1091,10 +1213,13 @@ Here's an example of an external configuration for cray modules:
.. code-block:: yaml
packages:
mpi:
mpich:
modules:
mpich@7.3.1%gcc@5.2.0 arch=cray_xc-haswell-CNL10: cray-mpich
mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-haswell-CNL10: cray-mpich
all:
providers:
mpi: [mpich]
This tells Spack that for whatever package that depends on mpi, load the
cray-mpich module into the environment. You can then be able to use whatever
@@ -1111,7 +1236,7 @@ Here is an example of a full packages.yaml used at NERSC
.. code-block:: yaml
packages:
mpi:
mpich:
modules:
mpich@7.3.1%gcc@5.2.0 arch=cray_xc-CNL10-ivybridge: cray-mpich
mpich@7.3.1%intel@16.0.0.109 arch=cray_xc-SuSE11-ivybridge: cray-mpich
@@ -1128,6 +1253,8 @@ Here is an example of a full packages.yaml used at NERSC
buildable: False
all:
compiler: [gcc@5.2.0, intel@16.0.0.109]
providers:
mpi: [mpich]
Here we tell spack that whenever we want to build with gcc use version 5.2.0 or
if we want to build with intel compilers, use version 16.0.0.109. We add a spec

17
lib/spack/docs/index.rst
View File

@@ -4,9 +4,13 @@
contain the root `toctree` directive.
===================
Spack Documentation
Spack
===================
.. epigraph::
`These are docs for the Spack package manager. For sphere packing, see` `pyspack <https://pyspack.readthedocs.io>`_.
Spack is a package management tool designed to support multiple
versions and configurations of software on a wide variety of platforms
and environments. It was designed for large supercomputing centers,
@@ -25,12 +29,12 @@ maintain a single file for many different builds of the same package.
See the :doc:`features` for examples and highlights.
Get spack from the `github repository
<https://github.com/llnl/spack>`_ and install your first
<https://github.com/spack/spack>`_ and install your first
package:
.. code-block:: console
$ git clone https://github.com/llnl/spack.git
$ git clone https://github.com/spack/spack.git
$ cd spack/bin
$ ./spack install libelf
@@ -46,7 +50,8 @@ or refer to the full manual below.
getting_started
basic_usage
workflows
tutorial_sc16
tutorial
known_issues
.. toctree::
:maxdepth: 2
@@ -58,6 +63,7 @@ or refer to the full manual below.
mirrors
module_file_support
repositories
binary_caches
command_index
package_list
@@ -68,7 +74,8 @@ or refer to the full manual below.
contribution_guide
packaging_guide
developer_guide
API Docs <spack>
Spack API Docs <spack>
LLNL API Docs <llnl>
==================
Indices and tables

110
lib/spack/docs/known_issues.rst Normal file
View File

@@ -0,0 +1,110 @@
============
Known Issues
============
This is a list of known bugs in Spack. It provides ways of getting around these
problems if you encounter them.
-----------------------------------------------------------------
Default variants are not taken into account during concretization
-----------------------------------------------------------------
**Status:** Expected to be fixed in the next release
Current concretization algorithm does not take into account default values
of variants when adding extra constraints to the spec via CLI. For example
you may encounter the following error when trying to specify which MPI provider
to use:
.. code-block:: console
$ spack install hdf5 ^openmpi
==> Error: hdf5 does not depend on openmpi
although the hdf5 package contains:
.. code-block:: python
variant('mpi', default=True, description='Enable MPI support')
depends_on('mpi', when='+mpi')
A workaround is to explicitly activate the variant related to the dependency:
.. code-block:: console
$ spack install hdf5+mpi ^openmpi
See https://github.com/spack/spack/issues/397 for further details.
---------------------------------------------------
Variants are not properly forwarded to dependencies
---------------------------------------------------
**Status:** Expected to be fixed in the next release
Sometimes, a variant of a package can also affect how its dependencies are
built. For example, in order to build MPI support for a package, it may
require that its dependencies are also built with MPI support. In the
``package.py``, this looks like:
.. code-block:: python
depends_on('hdf5~mpi', when='~mpi')
depends_on('hdf5+mpi', when='+mpi')
Spack handles this situation properly for *immediate* dependencies, and
builds ``hdf5`` with the same variant you used for the package that
depends on it. However, for *indirect* dependencies (dependencies of
dependencies), Spack does not backtrack up the DAG far enough to handle
this. Users commonly run into this situation when trying to build R with
X11 support:
.. code-block:: console
$ spack install r+X
...
==> Error: Invalid spec: 'cairo@1.14.8%gcc@6.2.1+X arch=linux-fedora25-x86_64 ^bzip2@1.0.6%gcc@6.2.1+shared arch=linux-fedora25-x86_64 ^font-util@1.3.1%gcc@6.2.1 arch=linux-fedora25-x86_64 ^fontconfig@2.12.1%gcc@6.2.1 arch=linux-fedora25-x86_64 ^freetype@2.7.1%gcc@6.2.1 arch=linux-fedora25-x86_64 ^gettext@0.19.8.1%gcc@6.2.1+bzip2+curses+git~libunistring+libxml2+tar+xz arch=linux-fedora25-x86_64 ^glib@2.53.1%gcc@6.2.1~libmount arch=linux-fedora25-x86_64 ^inputproto@2.3.2%gcc@6.2.1 arch=linux-fedora25-x86_64 ^kbproto@1.0.7%gcc@6.2.1 arch=linux-fedora25-x86_64 ^libffi@3.2.1%gcc@6.2.1 arch=linux-fedora25-x86_64 ^libpng@1.6.29%gcc@6.2.1 arch=linux-fedora25-x86_64 ^libpthread-stubs@0.4%gcc@6.2.1 arch=linux-fedora25-x86_64 ^libx11@1.6.5%gcc@6.2.1 arch=linux-fedora25-x86_64 ^libxau@1.0.8%gcc@6.2.1 arch=linux-fedora25-x86_64 ^libxcb@1.12%gcc@6.2.1 arch=linux-fedora25-x86_64 ^libxdmcp@1.1.2%gcc@6.2.1 arch=linux-fedora25-x86_64 ^libxext@1.3.3%gcc@6.2.1 arch=linux-fedora25-x86_64 ^libxml2@2.9.4%gcc@6.2.1~python arch=linux-fedora25-x86_64 ^libxrender@0.9.10%gcc@6.2.1 arch=linux-fedora25-x86_64 ^ncurses@6.0%gcc@6.2.1~symlinks arch=linux-fedora25-x86_64 ^openssl@1.0.2k%gcc@6.2.1 arch=linux-fedora25-x86_64 ^pcre@8.40%gcc@6.2.1+utf arch=linux-fedora25-x86_64 ^pixman@0.34.0%gcc@6.2.1 arch=linux-fedora25-x86_64 ^pkg-config@0.29.2%gcc@6.2.1+internal_glib arch=linux-fedora25-x86_64 ^python@2.7.13%gcc@6.2.1+shared~tk~ucs4 arch=linux-fedora25-x86_64 ^readline@7.0%gcc@6.2.1 arch=linux-fedora25-x86_64 ^renderproto@0.11.1%gcc@6.2.1 arch=linux-fedora25-x86_64 ^sqlite@3.18.0%gcc@6.2.1 arch=linux-fedora25-x86_64 ^tar^util-macros@1.19.1%gcc@6.2.1 arch=linux-fedora25-x86_64 ^xcb-proto@1.12%gcc@6.2.1 arch=linux-fedora25-x86_64 ^xextproto@7.3.0%gcc@6.2.1 arch=linux-fedora25-x86_64 ^xproto@7.0.31%gcc@6.2.1 arch=linux-fedora25-x86_64 ^xtrans@1.3.5%gcc@6.2.1 arch=linux-fedora25-x86_64 ^xz@5.2.3%gcc@6.2.1 arch=linux-fedora25-x86_64 ^zlib@1.2.11%gcc@6.2.1+pic+shared arch=linux-fedora25-x86_64'.
Package cairo requires variant ~X, but spec asked for +X
A workaround is to explicitly activate the variants of dependencies as well:
.. code-block:: console
$ spack install r+X ^cairo+X ^pango+X
See https://github.com/spack/spack/issues/267 and
https://github.com/spack/spack/issues/2546 for further details.
---------------------------------
``spack extensions`` doesn't work
---------------------------------
**Status:** Up for grabs if you want to try to fix it
Spack provides an ``extensions`` command that lists all available extensions
of a package, the ones that are installed, and the ones that are already
activated. This is very useful in conjunction with ``spack activate``.
Unfortunately, this command no longer works:
.. code-block:: console
$ spack extensions python
==> python@2.7.13%clang@8.0.0-apple~tk~ucs4 arch=darwin-sierra-x86_64 -ckrr4mg has no extensions.
See https://github.com/spack/spack/issues/2895 for further details.
----------------------------
``spack setup`` doesn't work
----------------------------
**Status:** Work in progress
Spack provides a ``setup`` command that is useful for the development of
software outside of Spack. Unfortunately, this command no longer works.
See https://github.com/spack/spack/issues/2597 and
https://github.com/spack/spack/issues/2662 for details. This is expected
to be fixed by https://github.com/spack/spack/pull/2664.

4
lib/spack/docs/mirrors.rst
View File

@@ -162,7 +162,7 @@ your site.
Once you have a mirror, you need to let spack know about it. This is
relatively simple. First, figure out the URL for the mirror. If it's
a file, you can use a file URL like this one:
a directory, you can use a file URL like this one:
.. code-block:: none
@@ -237,7 +237,7 @@ as other Spack mirrors (so it can be copied anywhere and referenced with a URL
like other mirrors). The mirror is maintained locally (within the Spack
installation directory) at :file:`var/spack/cache/`. It is always enabled (and
is always searched first when attempting to retrieve files for an installation)
but can be cleared with :ref:`purge <cmd-spack-purge>`; the cache directory can also
but can be cleared with :ref:`clean <cmd-spack-clean>`; the cache directory can also
be deleted manually without issue.
Caching includes retrieved tarball archives and source control repositories, but

989
lib/spack/docs/module_file_generation.svg Normal file
View File

@@ -0,0 +1,989 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
version="1.1"
width="682"
height="382"
id="svg3341"
inkscape:version="0.48.4 r9939"
sodipodi:docname="module_file_generation.svg">
<metadata
id="metadata3657">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
</cc:Work>
</rdf:RDF>
</metadata>
<sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="1920"
inkscape:window-height="1134"
id="namedview3655"
showgrid="false"
inkscape:zoom="2.0073314"
inkscape:cx="341"
inkscape:cy="191"
inkscape:window-x="1920"
inkscape:window-y="27"
inkscape:window-maximized="1"
inkscape:current-layer="g3345" />
<defs
id="defs3343" />
<g
inkscape:groupmode="layer"
id="layer1"
inkscape:label="Background">
<rect
style="fill:none;stroke:#ff0000;stroke-width:1.40437257;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:8.42623582, 8.42623582;stroke-dashoffset:0"
id="rect3844"
width="442.15341"
height="375.15021"
x="18.745768"
y="3.2206354" />
<rect
style="fill:none;stroke:#0000ff;stroke-width:1.50000000000000000;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-opacity:1;stroke-dasharray:9, 9;stroke-dashoffset:0"
id="rect3844-6"
width="175.74904"
height="179.83459"
x="486.96402"
y="179.3212" />
</g>
<g
id="g3345">
<g
transform="translate(-246,-214) scale(1,1)"
id="g3347">
<rect
fill="#C0C0C0"
stroke="none"
x="297"
y="231"
width="198"
height="104"
opacity="0.2"
id="rect3349" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3351">
<rect
fill="#ffffff"
stroke="none"
x="290"
y="224"
width="198"
height="104"
id="rect3353" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3355">
<path
fill="none"
stroke="#000000"
d="M 290 224 L 488 224 L 488 328 L 290 328 L 290 224 Z Z"
stroke-miterlimit="10"
id="path3357" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3359">
<rect
fill="#ffffff"
stroke="none"
x="280"
y="231"
width="20"
height="10"
id="rect3361" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3363">
<path
fill="none"
stroke="#000000"
d="M 280 231 L 300 231 L 300 241 L 280 241 L 280 231 Z Z"
stroke-miterlimit="10"
id="path3365" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3367">
<rect
fill="#ffffff"
stroke="none"
x="280"
y="251"
width="20"
height="10"
id="rect3369" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3371">
<path
fill="none"
stroke="#000000"
d="M 280 251 L 300 251 L 300 261 L 280 261 L 280 251 Z Z"
stroke-miterlimit="10"
id="path3373" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3375">
<g
id="g3377">
<path
fill="none"
stroke="none"
id="path3379" />
<text
fill="#000000"
stroke="none"
font-family="Arial"
font-size="13px"
font-style="normal"
font-weight="bold"
text-decoration="none"
x="343"
y="243.5"
id="text3381">Configuration files</text>
</g>
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3383">
<rect
fill="#C0C0C0"
stroke="none"
x="327"
y="263"
width="98"
height="24"
opacity="0.2"
id="rect3385" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3387">
<rect
fill="#ffffff"
stroke="none"
x="320"
y="256"
width="98"
height="24"
id="rect3389" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3391">
<path
fill="none"
stroke="#000000"
d="M 320 256 L 418 256 L 418 280 L 320 280 L 320 256 Z Z"
stroke-miterlimit="10"
id="path3393" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3395">
<rect
fill="#ffffff"
stroke="none"
x="320"
y="256"
width="98"
height="24"
id="rect3397" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3399">
<path
fill="none"
stroke="#000000"
d="M 320 256 L 418 256 L 418 280 L 320 280 L 320 256 Z Z"
stroke-miterlimit="10"
id="path3401" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3403">
<g
id="g3405"
transform="translate(-2.8259277,-4.138916)">
<path
id="path3407"
d=""
inkscape:connector-curvature="0"
style="fill:none;stroke:none" />
<text
font-size="13px"
font-style="normal"
font-weight="bold"
text-decoration="none"
x="328.5"
y="275.5"
id="text3409"
style="font-size:13px;font-style:normal;font-weight:bold;text-decoration:none;fill:#000000;stroke:none;font-family:Arial">modules.yaml</text>
</g>
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3411">
<rect
fill="#C0C0C0"
stroke="none"
x="335"
y="295"
width="83"
height="24"
opacity="0.2"
id="rect3413" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3415">
<rect
fill="#ffffff"
stroke="none"
x="328"
y="288"
width="83"
height="24"
id="rect3417" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3419">
<path
fill="none"
stroke="#000000"
d="M 328 288 L 411 288 L 411 312 L 328 312 L 328 288 Z Z"
stroke-miterlimit="10"
id="path3421" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3423">
<rect
fill="#ffffff"
stroke="none"
x="328"
y="288"
width="83"
height="24"
id="rect3425" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3427">
<path
fill="none"
stroke="#000000"
d="M 328 288 L 411 288 L 411 312 L 328 312 L 328 288 Z Z"
stroke-miterlimit="10"
id="path3429" />
</g>
<g
transform="translate(-248.92773,-218.16748)"
id="g3431">
<g
id="g3433">
<path
id="path3435"
d=""
inkscape:connector-curvature="0"
style="fill:none;stroke:none" />
<text
font-size="13px"
font-style="normal"
font-weight="bold"
text-decoration="none"
x="336.5"
y="307.5"
id="text3437"
style="font-size:13px;font-style:normal;font-weight:bold;text-decoration:none;fill:#000000;stroke:none;font-family:Arial">config.yaml</text>
</g>
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3439">
<rect
fill="#C0C0C0"
stroke="none"
x="489"
y="407"
width="198"
height="72"
opacity="0.2"
id="rect3441" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3443">
<rect
fill="#ffffff"
stroke="none"
x="482"
y="400"
width="198"
height="72"
id="rect3445" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3447">
<path
fill="none"
stroke="#000000"
d="M 482 400 L 680 400 L 680 472 L 482 472 L 482 400 Z Z"
stroke-miterlimit="10"
id="path3449" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3451">
<rect
fill="#ffffff"
stroke="none"
x="472"
y="407"
width="20"
height="10"
id="rect3453" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3455">
<path
fill="none"
stroke="#000000"
d="M 472 407 L 492 407 L 492 417 L 472 417 L 472 407 Z Z"
stroke-miterlimit="10"
id="path3457" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3459">
<rect
fill="#ffffff"
stroke="none"
x="472"
y="427"
width="20"
height="10"
id="rect3461" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3463">
<path
fill="none"
stroke="#000000"
d="M 472 427 L 492 427 L 492 437 L 472 437 L 472 427 Z Z"
stroke-miterlimit="10"
id="path3465" />
</g>
<g
transform="translate(-255.37842,-194.16748)"
id="g3467">
<g
id="g3469">
<path
id="path3471"
d=""
inkscape:connector-curvature="0"
style="fill:none;stroke:none" />
<text
font-size="13px"
font-style="normal"
font-weight="bold"
text-decoration="none"
x="528"
y="419.5"
id="text3473"
style="font-size:13px;font-style:normal;font-weight:bold;text-decoration:none;fill:#000000;stroke:none;font-family:Arial">Module subpackage</text>
</g>
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3475">
<path
fill="none"
stroke="#000000"
d="M 369 313 L 368 440 L 471 438"
stroke-miterlimit="10"
stroke-dasharray="3"
id="path3477" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3479">
<path
fill="none"
stroke="#000000"
d="M 460.9209632639215 442.4060207072573 L 471 438 L 460.7575176559406 433.98857189624056"
stroke-miterlimit="10"
id="path3481" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3483">
<path
fill="#ffffff"
stroke="none"
d="M 256 448 L 381 448 L 391 458 L 391 496 L 256 496 L 256 448"
id="path3485" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3487">
<path
fill="none"
stroke="#000000"
d="M 256 448 L 381 448 L 391 458 L 391 496 L 256 496 L 256 448 L 256 448"
stroke-miterlimit="10"
id="path3489" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3491">
<path
fill="none"
stroke="#000000"
d="M 381 448 L 381 458 L 391 458 L 381 448"
stroke-miterlimit="10"
id="path3493" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3495">
<g
id="g3497">
<path
fill="none"
stroke="none"
id="path3499" />
<text
fill="#000000"
stroke="none"
font-family="Arial"
font-size="13px"
font-style="normal"
font-weight="normal"
text-decoration="none"
x="261"
y="466"
id="text3501">template directory</text>
</g>
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3503">
<g
id="g3505">
<path
fill="none"
stroke="none"
id="path3507" />
<text
fill="#000000"
stroke="none"
font-family="Arial"
font-size="13px"
font-style="normal"
font-weight="normal"
text-decoration="none"
x="261"
y="481"
id="text3509">module file directory</text>
</g>
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3511">
<path
fill="none"
stroke="#000000"
d="M 419 439 L 393 448"
stroke-miterlimit="10"
stroke-dasharray="3"
id="path3513" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3515">
<path
d="M 419,267 576.40856,267.26845 576,399"
stroke-miterlimit="10"
id="path3517"
inkscape:connector-curvature="0"
style="fill:none;stroke:#000000;stroke-miterlimit:10;stroke-dasharray:3"
sodipodi:nodetypes="ccc" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3519">
<path
fill="none"
stroke="#000000"
d="M 571.790482243984 388.83732514237585 L 576 399 L 580.209517756016 388.83732514237585"
stroke-miterlimit="10"
id="path3521" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3523">
<path
fill="#ffffff"
stroke="none"
d="M 600 296 L 737 296 L 747 306 L 747 344 L 600 344 L 600 296"
id="path3525" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3527">
<path
fill="none"
stroke="#000000"
d="M 600 296 L 737 296 L 747 306 L 747 344 L 600 344 L 600 296 L 600 296"
stroke-miterlimit="10"
id="path3529" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3531">
<path
fill="none"
stroke="#000000"
d="M 737 296 L 737 306 L 747 306 L 737 296"
stroke-miterlimit="10"
id="path3533" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3535">
<g
id="g3537">
<path
fill="none"
stroke="none"
id="path3539" />
<text
fill="#000000"
stroke="none"
font-family="Arial"
font-size="13px"
font-style="normal"
font-weight="normal"
text-decoration="none"
x="605"
y="314"
id="text3541">content customization</text>
</g>
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3543">
<g
id="g3545">
<path
fill="none"
stroke="none"
id="path3547" />
<text
fill="#000000"
stroke="none"
font-family="Arial"
font-size="13px"
font-style="normal"
font-weight="normal"
text-decoration="none"
x="605"
y="329"
id="text3549">layout customization</text>
</g>
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3551">
<path
fill="none"
stroke="#000000"
d="M 576 331 L 599 328"
stroke-miterlimit="10"
stroke-dasharray="3"
id="path3553" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3555">
<rect
fill="#C0C0C0"
stroke="none"
x="801"
y="415"
width="99"
height="44"
opacity="0.2"
id="rect3557" />
</g>
<g
transform="translate(-246,-210)"
id="g3559">
<rect
x="794"
y="408"
width="99"
height="44"
id="rect3561"
style="fill:#ffffff;stroke:none" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3563">
<path
fill="none"
stroke="#000000"
d="M 794 408 L 893 408 L 893 452 L 794 452 L 794 408 Z Z"
stroke-miterlimit="10"
id="path3565" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3567">
<rect
fill="#ffffff"
stroke="none"
x="784"
y="415"
width="20"
height="10"
id="rect3569" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3571">
<path
fill="none"
stroke="#000000"
d="M 784 415 L 804 415 L 804 425 L 784 425 L 784 415 Z Z"
stroke-miterlimit="10"
id="path3573" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3575">
<rect
fill="#ffffff"
stroke="none"
x="784"
y="435"
width="20"
height="10"
id="rect3577" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3579">
<path
fill="none"
stroke="#000000"
d="M 784 435 L 804 435 L 804 445 L 784 445 L 784 435 Z Z"
stroke-miterlimit="10"
id="path3581" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3583">
<g
id="g3585"
transform="translate(-6.7685547,5.861084)">
<path
id="path3587"
d=""
inkscape:connector-curvature="0"
style="fill:none;stroke:none" />
<text
font-size="13px"
font-style="normal"
font-weight="bold"
text-decoration="none"
x="832"
y="427.5"
id="text3589"
style="font-size:13px;font-style:normal;font-weight:bold;text-decoration:none;fill:#000000;stroke:none;font-family:Arial">Jinja2</text>
</g>
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3591">
<path
fill="#C0C0C0"
stroke="none"
d="M 568.9 519 L 596.34 519 L 608.1 530.76 L 608.1 568 L 568.9 568"
opacity="0.2"
id="path3593" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3595">
<path
fill="#ffffff"
stroke="none"
d="M 561.9 512 L 589.34 512 L 601.1 523.76 L 601.1 561 L 561.9 561"
id="path3597" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3599">
<path
fill="none"
stroke="#000000"
d="M 561.9 512 L 589.34 512 L 601.1 523.76 L 601.1 561 L 561.9 561 L 561.9 512"
stroke-miterlimit="10"
id="path3601" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3603">
<path
fill="none"
stroke="#000000"
d="M 589.34 512 L 589.34 523.76 L 601.1 523.76 L 589.34 512"
stroke-miterlimit="10"
id="path3605" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3607">
<g
id="g3609">
<path
fill="none"
stroke="none"
id="path3611" />
<text
fill="#000000"
stroke="none"
font-family="Arial"
font-size="13px"
font-style="normal"
font-weight="bold"
text-decoration="none"
x="551.5"
y="580.5"
id="text3613">Templates</text>
</g>
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3615">
<path
d="M 620,549 836.73155,549.54866 838,453"
stroke-miterlimit="10"
id="path3617"
inkscape:connector-curvature="0"
style="fill:none;stroke:#000000;stroke-miterlimit:10;stroke-dasharray:3"
sodipodi:nodetypes="ccc" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3619">
<path
fill="none"
stroke="#000000"
d="M 842.4139237018308 463.0755782739462 L 838 453 L 833.9966056593214 463.2456251030878"
stroke-miterlimit="10"
id="path3621" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3623">
<path
d="M 680.18289,431.54866 783,431"
stroke-miterlimit="10"
id="path3625"
inkscape:connector-curvature="0"
style="fill:none;stroke:#000000;stroke-miterlimit:10;stroke-dasharray:3"
sodipodi:nodetypes="cc" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3627">
<path
fill="none"
stroke="#000000"
d="M 772.9654737429415 435.5064712133113 L 783 431 L 772.7179620834851 427.0910747917938"
stroke-miterlimit="10"
id="path3629" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3631">
<path
fill="#C0C0C0"
stroke="none"
d="M 862.4 247 L 889.84 247 L 901.6 258.76 L 901.6 296 L 862.4 296"
opacity="0.2"
id="path3633" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3635">
<path
fill="#ffffff"
stroke="none"
d="M 855.4 240 L 882.84 240 L 894.6 251.76000000000002 L 894.6 289 L 855.4 289"
id="path3637" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3639">
<path
fill="none"
stroke="#000000"
d="M 855.4 240 L 882.84 240 L 894.6 251.76000000000002 L 894.6 289 L 855.4 289 L 855.4 240"
stroke-miterlimit="10"
id="path3641" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3643">
<path
fill="none"
stroke="#000000"
d="M 882.84 240 L 882.84 251.76000000000002 L 894.6 251.76000000000002 L 882.84 240"
stroke-miterlimit="10"
id="path3645" />
</g>
<g
transform="translate(-246,-214) scale(1,1)"
id="g3647">
<g
id="g3649">
<path
fill="none"
stroke="none"
id="path3651" />
<text
fill="#000000"
stroke="none"
font-family="Arial"
font-size="13px"
font-style="normal"
font-weight="bold"
text-decoration="none"
x="840.5"
y="308.5"
id="text3653">Module files</text>
</g>
</g>
<g
transform="translate(-261.34866,-222.82727)"
id="g3635-8">
<path
style="fill:#ffffff;stroke:none"
inkscape:connector-curvature="0"
d="m 855.4,240 27.44,0 11.76,11.76 0,37.24 -39.2,0"
id="path3637-9" />
</g>
<g
transform="translate(-261.34866,-222.82727)"
id="g3639-7">
<path
style="fill:none;stroke:#000000;stroke-miterlimit:10"
inkscape:connector-curvature="0"
d="m 855.4,240 27.44,0 11.76,11.76 0,37.24 -39.2,0 0,-49"
stroke-miterlimit="10"
id="path3641-3" />
</g>
<g
transform="translate(-261.34866,-222.82727)"
id="g3643-6">
<path
style="fill:none;stroke:#000000;stroke-miterlimit:10"
inkscape:connector-curvature="0"
d="m 882.84,240 0,11.76 11.76,0 L 882.84,240"
stroke-miterlimit="10"
id="path3645-1" />
</g>
<g
transform="translate(-278.09946,-233.44973)"
id="g3635-84">
<path
style="fill:#ffffff;stroke:none"
inkscape:connector-curvature="0"
d="m 855.4,240 27.44,0 11.76,11.76 0,37.24 -39.2,0"
id="path3637-5" />
</g>
<g
transform="translate(-278.09946,-233.44973)"
id="g3639-0">
<path
style="fill:none;stroke:#000000;stroke-miterlimit:10"
inkscape:connector-curvature="0"
d="m 855.4,240 27.44,0 11.76,11.76 0,37.24 -39.2,0 0,-49"
stroke-miterlimit="10"
id="path3641-36" />
</g>
<g
transform="translate(-278.09946,-233.44973)"
id="g3643-1">
<path
style="fill:none;stroke:#000000;stroke-miterlimit:10"
inkscape:connector-curvature="0"
d="m 882.84,240 0,11.76 11.76,0 L 882.84,240"
stroke-miterlimit="10"
id="path3645-0" />
</g>
<g
transform="translate(-27.431351,-208.3001)"
id="g3375-6" />
<g
id="g4709"
transform="matrix(1,0,0,0.81117898,54.337968,31.640263)">
<g
id="g3623-5"
transform="matrix(0,-1,1,0,121.9107,875.37876)">
<path
sodipodi:nodetypes="cc"
style="fill:none;stroke:#000000;stroke-miterlimit:10;stroke-dasharray:3"
inkscape:connector-curvature="0"
id="path3625-4"
stroke-miterlimit="10"
d="M 680.18289,431.54866 783,431" />
</g>
<g
id="g3627-7"
transform="matrix(0,-1,1,0,121.98695,871.00978)">
<path
id="path3629-6"
stroke-miterlimit="10"
d="M 772.96547,435.50647 783,431 772.71796,427.09107"
inkscape:connector-curvature="0"
style="fill:none;stroke:#000000;stroke-miterlimit:10" />
</g>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 26 KiB

335
lib/spack/docs/module_file_support.rst
View File

@@ -7,16 +7,10 @@ Modules
The use of module systems to manage user environment in a controlled way
is a common practice at HPC centers that is often embraced also by individual
programmers on their development machines. To support this common practice
Spack provides integration with `Environment Modules
Spack integrates with `Environment Modules
<http://modules.sourceforge.net/>`_ , `LMod
<http://lmod.readthedocs.io/en/latest/>`_ and `Dotkit <https://computing.llnl.gov/?set=jobs&page=dotkit>`_ by:
* generating module files after a successful installation
* providing commands that can leverage the spec syntax to manipulate modules
In the following you will see how to activate shell support for commands in Spack
that requires it, and discover what benefits this may bring with respect to deal
directly with automatically generated module files.
<http://lmod.readthedocs.io/en/latest/>`_ and `Dotkit <https://computing.llnl.gov/?set=jobs&page=dotkit>`_ by
providing post-install hooks that generate module files and commands to manipulate them.
.. note::
@@ -26,13 +20,58 @@ directly with automatically generated module files.
.. _shell-support:
-------------
Shell support
-------------
----------------------------
Using module files via Spack
----------------------------
You can enable shell support by sourcing the appropriate setup file
in the ``$SPACK_ROOT/share/spack`` directory.
For ``bash`` or ``ksh`` users:
If you have installed a supported module system either manually or through
``spack bootstrap``, you should be able to run either ``module avail`` or
``use -l spack`` to see what module files have been installed. Here is
sample output of those programs, showing lots of installed packages:
.. code-block:: console
$ module avail
--------------------------------------------------------------- ~/spack/share/spack/modules/linux-ubuntu14-x86_64 ---------------------------------------------------------------
autoconf-2.69-gcc-4.8-qextxkq hwloc-1.11.6-gcc-6.3.0-akcisez m4-1.4.18-gcc-4.8-ev2znoc openblas-0.2.19-gcc-6.3.0-dhkmed6 py-setuptools-34.2.0-gcc-6.3.0-fadur4s
automake-1.15-gcc-4.8-maqvukj isl-0.18-gcc-4.8-afi6taq m4-1.4.18-gcc-6.3.0-uppywnz openmpi-2.1.0-gcc-6.3.0-go2s4z5 py-six-1.10.0-gcc-6.3.0-p4dhkaw
binutils-2.28-gcc-4.8-5s7c6rs libiconv-1.15-gcc-4.8-at46wg3 mawk-1.3.4-gcc-4.8-acjez57 openssl-1.0.2k-gcc-4.8-dkls5tk python-2.7.13-gcc-6.3.0-tyehea7
bison-3.0.4-gcc-4.8-ek4luo5 libpciaccess-0.13.4-gcc-6.3.0-gmufnvh mawk-1.3.4-gcc-6.3.0-ostdoms openssl-1.0.2k-gcc-6.3.0-gxgr5or readline-7.0-gcc-4.8-xhufqhn
bzip2-1.0.6-gcc-4.8-iffrxzn libsigsegv-2.11-gcc-4.8-pp2cvte mpc-1.0.3-gcc-4.8-g5mztc5 pcre-8.40-gcc-4.8-r5pbrxb readline-7.0-gcc-6.3.0-zzcyicg
bzip2-1.0.6-gcc-6.3.0-bequudr libsigsegv-2.11-gcc-6.3.0-7enifnh mpfr-3.1.5-gcc-4.8-o7xm7az perl-5.24.1-gcc-4.8-dg5j65u sqlite-3.8.5-gcc-6.3.0-6zoruzj
cmake-3.7.2-gcc-6.3.0-fowuuby libtool-2.4.6-gcc-4.8-7a523za mpich-3.2-gcc-6.3.0-dmvd3aw perl-5.24.1-gcc-6.3.0-6uzkpt6 tar-1.29-gcc-4.8-wse2ass
curl-7.53.1-gcc-4.8-3fz46n6 libtool-2.4.6-gcc-6.3.0-n7zmbzt ncurses-6.0-gcc-4.8-dcpe7ia pkg-config-0.29.2-gcc-4.8-ib33t75 tcl-8.6.6-gcc-4.8-tfxzqbr
expat-2.2.0-gcc-4.8-mrv6bd4 libxml2-2.9.4-gcc-4.8-ryzxnsu ncurses-6.0-gcc-6.3.0-ucbhcdy pkg-config-0.29.2-gcc-6.3.0-jpgubk3 util-macros-1.19.1-gcc-6.3.0-xorz2x2
flex-2.6.3-gcc-4.8-yf345oo libxml2-2.9.4-gcc-6.3.0-rltzsdh netlib-lapack-3.6.1-gcc-6.3.0-js33dog py-appdirs-1.4.0-gcc-6.3.0-jxawmw7 xz-5.2.3-gcc-4.8-mew4log
gcc-6.3.0-gcc-4.8-24puqve lmod-7.4.1-gcc-4.8-je4srhr netlib-scalapack-2.0.2-gcc-6.3.0-5aidk4l py-numpy-1.12.0-gcc-6.3.0-oemmoeu xz-5.2.3-gcc-6.3.0-3vqeuvb
gettext-0.19.8.1-gcc-4.8-yymghlh lua-5.3.4-gcc-4.8-im75yaz netlib-scalapack-2.0.2-gcc-6.3.0-hjsemcn py-packaging-16.8-gcc-6.3.0-i2n3dtl zip-3.0-gcc-4.8-rwar22d
gmp-6.1.2-gcc-4.8-5ub2wu5 lua-luafilesystem-1_6_3-gcc-4.8-wkey3nl netlib-scalapack-2.0.2-gcc-6.3.0-jva724b py-pyparsing-2.1.10-gcc-6.3.0-tbo6gmw zlib-1.2.11-gcc-4.8-pgxsxv7
help2man-1.47.4-gcc-4.8-kcnqmau lua-luaposix-33.4.0-gcc-4.8-mdod2ry netlib-scalapack-2.0.2-gcc-6.3.0-rgqfr6d py-scipy-0.19.0-gcc-6.3.0-kr7nat4 zlib-1.2.11-gcc-6.3.0-7cqp6cj
The names should look familiar, as they resemble the output from ``spack find``.
You *can* use the modules here directly. For example, you could type either of these commands
to load the ``cmake`` module:
.. code-block:: console
$ use cmake-3.7.2-gcc-6.3.0-fowuuby
.. code-block:: console
$ module load cmake-3.7.2-gcc-6.3.0-fowuuby
Neither of these is particularly pretty, easy to remember, or
easy to type. Luckily, Spack has its own interface for using modules and dotkits.
^^^^^^^^^^^^^
Shell support
^^^^^^^^^^^^^
To enable additional Spack commands for loading and unloading module files,
and to add the correct path to ``MODULEPATH``, you need to source the appropriate
setup file in the ``$SPACK_ROOT/share/spack`` directory. This will activate shell
support for the commands that need it. For ``bash``, ``ksh`` or ``zsh`` users:
.. code-block:: console
@@ -42,73 +81,20 @@ For ``csh`` and ``tcsh`` instead:
.. code-block:: console
$ set SPACK_ROOT ...
$ source $SPACK_ROOT/share/spack/setup-env.csh
Note that in the latter case it is necessary to explicitly set ``SPACK_ROOT``
before sourcing the setup file (you will get a meaningful error message
if you don't).
.. note::
You can put the source line in your ``.bashrc`` or ``.cshrc`` to
have Spack's shell support available on the command line at any login.
When ``bash`` and ``ksh`` users update their environment with ``setup-env.sh``, it will check for spack-installed environment modules and add the ``module`` command to their environment; This only occurs if the module command is not already available. You can install ``environment-modules`` with ``spack bootstrap`` as described in :ref:`InstallEnvironmentModules`.
Finally, if you want to have Spack's shell support available on the command line at
any login you can put this source line in one of the files that are sourced
at startup (like ``.profile``, ``.bashrc`` or ``.cshrc``). Be aware though
that the startup time may be slightly increased because of that.
----------------------------
Using module files via Spack
----------------------------
If you have shell support enabled you should be able to run either
``module avail`` or ``use -l spack`` to see what module/dotkit files have
been installed. Here is sample output of those programs, showing lots
of installed packages.
.. code-block:: console
$ module avail
------- ~/spack/share/spack/modules/linux-debian7-x86_64 --------
adept-utils@1.0%gcc@4.4.7-5adef8da libelf@0.8.13%gcc@4.4.7
automaded@1.0%gcc@4.4.7-d9691bb0 libelf@0.8.13%intel@15.0.0
boost@1.55.0%gcc@4.4.7 mpc@1.0.2%gcc@4.4.7-559607f5
callpath@1.0.1%gcc@4.4.7-5dce4318 mpfr@3.1.2%gcc@4.4.7
dyninst@8.1.2%gcc@4.4.7-b040c20e mpich@3.0.4%gcc@4.4.7
gcc@4.9.1%gcc@4.4.7-93ab98c5 mpich@3.0.4%gcc@4.9.0
gmp@6.0.0a%gcc@4.4.7 mrnet@4.1.0%gcc@4.4.7-72b7881d
graphlib@2.0.0%gcc@4.4.7 netgauge@2.4.6%gcc@4.9.0-27912b7b
launchmon@1.0.1%gcc@4.4.7 stat@2.1.0%gcc@4.4.7-51101207
libNBC@1.1.1%gcc@4.9.0-27912b7b sundials@2.5.0%gcc@4.9.0-27912b7b
libdwarf@20130729%gcc@4.4.7-b52fac98
.. code-block:: console
$ use -l spack
spack ----------
adept-utils@1.0%gcc@4.4.7-5adef8da - adept-utils @1.0
automaded@1.0%gcc@4.4.7-d9691bb0 - automaded @1.0
boost@1.55.0%gcc@4.4.7 - boost @1.55.0
callpath@1.0.1%gcc@4.4.7-5dce4318 - callpath @1.0.1
dyninst@8.1.2%gcc@4.4.7-b040c20e - dyninst @8.1.2
gmp@6.0.0a%gcc@4.4.7 - gmp @6.0.0a
libNBC@1.1.1%gcc@4.9.0-27912b7b - libNBC @1.1.1
libdwarf@20130729%gcc@4.4.7-b52fac98 - libdwarf @20130729
libelf@0.8.13%gcc@4.4.7 - libelf @0.8.13
libelf@0.8.13%intel@15.0.0 - libelf @0.8.13
mpc@1.0.2%gcc@4.4.7-559607f5 - mpc @1.0.2
mpfr@3.1.2%gcc@4.4.7 - mpfr @3.1.2
mpich@3.0.4%gcc@4.4.7 - mpich @3.0.4
mpich@3.0.4%gcc@4.9.0 - mpich @3.0.4
netgauge@2.4.6%gcc@4.9.0-27912b7b - netgauge @2.4.6
sundials@2.5.0%gcc@4.9.0-27912b7b - sundials @2.5.0
The names here should look familiar, they're the same ones from
``spack find``. You *can* use the names here directly. For example,
you could type either of these commands to load the callpath module:
.. code-block:: console
$ use callpath@1.0.1%gcc@4.4.7-5dce4318
.. code-block:: console
$ module load callpath@1.0.1%gcc@4.4.7-5dce4318
.. _cmd-spack-load:
@@ -116,12 +102,11 @@ you could type either of these commands to load the callpath module:
``spack load / unload``
^^^^^^^^^^^^^^^^^^^^^^^
Neither of these is particularly pretty, easy to remember, or
easy to type. Luckily, Spack has its own interface for using modules
and dotkits. You can use the same spec syntax you're used to:
Once you have shell support enabled you can use the same spec syntax
you're used to:
========================= ==========================
Environment Modules Dotkit
Modules Dotkit
========================= ==========================
``spack load <spec>`` ``spack use <spec>``
``spack unload <spec>`` ``spack unuse <spec>``
@@ -210,7 +195,7 @@ Scripts to load modules recursively may be made with the command:
$ spack module loads --dependencies <spec>
An equivalent alternative is:
An equivalent alternative using `process substitution <http://tldp.org/LDP/abs/html/process-sub.html>`_ is:
.. code-block :: console
@@ -296,43 +281,46 @@ For example, consider the following on one system:
# antlr@2.7.7%gcc@5.3.0~csharp+cxx~java~python arch=linux-SuSE11-x86_64
module load linux-SuSE11-x86_64/antlr-2.7.7-gcc-5.3.0-bdpl46y
----------------------------
Auto-generating Module Files
----------------------------
-------------------------
Module file customization
-------------------------
Module files are generated by post-install hooks after the successful
installation of a package. The following table summarizes the essential
installation of a package. The table below summarizes the essential
information associated with the different file formats
that can be generated by Spack:
+-----------------------------+--------------------+-------------------------------+----------------------+
| | **Hook name** | **Default root directory** | **Compatible tools** |
+=============================+====================+===============================+======================+
| **Dotkit** | ``dotkit`` | share/spack/dotkit | DotKit |
+-----------------------------+--------------------+-------------------------------+----------------------+
| **TCL - Non-Hierarchical** | ``tcl`` | share/spack/modules | Env. Modules/LMod |
+-----------------------------+--------------------+-------------------------------+----------------------+
| **Lua - Hierarchical** | ``lmod`` | share/spack/lmod | LMod |
+-----------------------------+--------------------+-------------------------------+----------------------+
+-----------------------------+--------------------+-------------------------------+----------------------------------+----------------------+
| | **Hook name** | **Default root directory** | **Default template file** | **Compatible tools** |
+=============================+====================+===============================+==================================+======================+
| **Dotkit** | ``dotkit`` | share/spack/dotkit | templates/modules/modulefile.dk | DotKit |
+-----------------------------+--------------------+-------------------------------+----------------------------------+----------------------+
| **TCL - Non-Hierarchical** | ``tcl`` | share/spack/modules | templates/modules/modulefile.tcl | Env. Modules/LMod |
+-----------------------------+--------------------+-------------------------------+----------------------------------+----------------------+
| **Lua - Hierarchical** | ``lmod`` | share/spack/lmod | templates/modules/modulefile.lua | LMod |
+-----------------------------+--------------------+-------------------------------+----------------------------------+----------------------+
Though Spack ships with sensible defaults for the generation of module files,
one can customize many aspects of it to accommodate package or site specific needs.
These customizations are enabled by either:
Spack ships with sensible defaults for the generation of module files, but
you can customize many aspects of it to accommodate package or site specific needs.
In general you can override or extend the default behavior by:
1. overriding certain callback APIs in the Python packages
2. writing specific rules in the ``modules.yaml`` configuration file
3. writing your own templates to override or extend the defaults
The former method fits best cases that are site independent, e.g. injecting variables
from language interpreters into their extensions. The latter instead permits to
fine tune the content, naming and creation of module files to meet site specific conventions.
The former method let you express changes in the run-time environment
that are needed to use the installed software properly, e.g. injecting variables
from language interpreters into their extensions. The latter two instead permit to
fine tune the filesystem layout, content and creation of module files to meet
site specific conventions.
^^^^^^^^^^^^^^^^^^^^
``Package`` file API
^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Override API calls in ``package.py``
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
There are two methods that can be overridden in any ``package.py`` to affect the
content of generated module files. The first one is:
There are two methods that you can override in any ``package.py`` to affect the
content of the module files generated by Spack. The first one:
.. code-block:: python
@@ -340,8 +328,8 @@ content of generated module files. The first one is:
"""Set up the compile and runtime environments for a package."""
pass
and can alter the content of *the same package where it is overridden*
by adding actions to ``run_env``. The second method is:
can alter the content of the module file associated with the same package where it is overridden.
The second method:
.. code-block:: python
@@ -349,12 +337,13 @@ by adding actions to ``run_env``. The second method is:
"""Set up the environment of packages that depend on this one"""
pass
and has similar effects on module file of dependees. Even in this case
``run_env`` must be filled with the desired list of environment modifications.
can instead inject run-time environment modifications in the module files of packages
that depend on it. In both cases you need to fill ``run_env`` with the desired
list of environment modifications.
.. note::
The ``r`` package and callback APIs
A typical example in which overriding both methods prove to be useful
An example in which it is crucial to override both methods
is given by the ``r`` package. This package installs libraries and headers
in non-standard locations and it is possible to prepend the appropriate directory
to the corresponding environment variables:
@@ -375,37 +364,36 @@ and has similar effects on module file of dependees. Even in this case
it appropriately in the override of the second method:
.. literalinclude:: ../../../var/spack/repos/builtin/packages/r/package.py
:lines: 128-129,146-151
:pyobject: R.setup_dependent_environment
.. _modules-yaml:
---------------------------------
Configuration in ``modules.yaml``
---------------------------------
^^^^^^^^^^^^^^^^^^^^^^^^^^
Write a configuration file
^^^^^^^^^^^^^^^^^^^^^^^^^^
The name of the configuration file that controls module generation behavior
is ``modules.yaml``. The default configuration:
The configuration files that control module generation behavior
are named ``modules.yaml``. The default configuration:
.. literalinclude:: ../../../etc/spack/defaults/modules.yaml
:language: yaml
activates generation for ``tcl`` and ``dotkit`` module files and inspects
activates the hooks to generate ``tcl`` and ``dotkit`` module files and inspects
the installation folder of each package for the presence of a set of subdirectories
(``bin``, ``man``, ``share/man``, etc.). If any is found its full path is prepended
to the environment variables listed below the folder name.
^^^^^^^^^^^^^^^^^^^^^^^^^^^
Activation of other systems
^^^^^^^^^^^^^^^^^^^^^^^^^^^
""""""""""""""""""""
Activate other hooks
""""""""""""""""""""
Any other module file generator shipped with Spack can be activated adding it to the
list under the ``enable`` key in the module file. Currently the only generator that
is not activated by default is ``lmod``, which produces hierarchical lua module files.
For each module system that can be enabled a finer configuration is possible.
is not active by default is ``lmod``, which produces hierarchical lua module files.
Directives that are aimed at driving the generation of a particular type of module files
should be listed under a top level key that corresponds to the generator being
customized:
Each module system can then be configured separately. In fact, you should list configuration
options that affect a particular type of module files under a top level key corresponding
to the generator being customized:
.. code-block:: yaml
@@ -421,24 +409,21 @@ customized:
lmod:
# contains lmod specific customizations
All these module sections allow for both:
1. global directives that usually affect the whole layout of modules or the naming scheme
2. directives that affect only a set of packages and modify their content
For the latter point in particular it is possible to use anonymous specs
to select an appropriate set of packages on which the modifications should be applied.
In general, the configuration options that you can use in ``modules.yaml`` will
either change the layout of the module files on the filesystem, or they will affect
their content. For the latter point it is possible to use anonymous specs
to fine tune the set of packages on which the modifications should be applied.
.. _anonymous_specs:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
""""""""""""""""""""""""""""
Selection by anonymous specs
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
""""""""""""""""""""""""""""
The procedure to select packages using anonymous specs is a natural
extension of using them to install packages, the only difference being
that specs in this case **are not required to have a root package**.
Consider for instance this snippet:
In the configuration file you can use *anonymous specs* (i.e. specs
that **are not required to have a root package** and are thus used just
to express constraints) to apply certain modifications on a selected set
of the installed software. For instance, in the snippet below:
.. code-block:: yaml
@@ -467,8 +452,7 @@ Consider for instance this snippet:
unset:
- FOOBAR
During module file generation, the configuration above will instruct
Spack to set the environment variable ``BAR=bar`` for every module,
you are instructing Spack to set the environment variable ``BAR=bar`` for every module,
unless the associated spec satisfies ``^openmpi`` in which case ``BAR=baz``.
In addition in any spec that satisfies ``zlib`` the value ``foo`` will be
prepended to ``LD_LIBRARY_PATH`` and in any spec that satisfies ``zlib%gcc@4.8``
@@ -480,15 +464,15 @@ the variable ``FOOBAR`` will be unset.
first, no matter where they appear in the configuration file. All the other
spec constraints are instead evaluated top to bottom.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Blacklist or whitelist the generation of specific module files
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
""""""""""""""""""""""""""""""""""""""""""""
Blacklist or whitelist specific module files
""""""""""""""""""""""""""""""""""""""""""""
Anonymous specs are also used to prevent module files from being written or
to force them to be written. A common case for that at HPC centers is to hide
from users all of the software that needs to be built with system compilers.
Suppose for instance to have ``gcc@4.4.7`` provided by your system. Then
with a configuration file like this one:
You can use anonymous specs also to prevent module files from being written or
to force them to be written. Consider the case where you want to hide from users
all the boilerplate software that you had to build in order to bootstrap a new
compiler. Suppose for instance that ``gcc@4.4.7`` is the compiler provided by
your system. If you write a configuration file like:
.. code-block:: yaml
@@ -497,13 +481,13 @@ with a configuration file like this one:
whitelist: ['gcc', 'llvm'] # Whitelist will have precedence over blacklist
blacklist: ['%gcc@4.4.7'] # Assuming gcc@4.4.7 is the system compiler
you will skip the generation of module files for any package that
is compiled with ``gcc@4.4.7``, with the exception of any ``gcc``
you will prevent the generation of module files for any package that
is compiled with ``gcc@4.4.7``, with the only exception of any ``gcc``
or any ``llvm`` installation.
^^^^^^^^^^^^^^^^^^^^^^^^^^^
"""""""""""""""""""""""""""
Customize the naming scheme
^^^^^^^^^^^^^^^^^^^^^^^^^^^
"""""""""""""""""""""""""""
The names of environment modules generated by spack are not always easy to
fully comprehend due to the long hash in the name. There are two module
@@ -551,7 +535,9 @@ most likely via the ``+blas`` variant specification.
tcl:
naming_scheme: '${PACKAGE}/${VERSION}-${COMPILERNAME}-${COMPILERVER}'
all:
conflict: ['${PACKAGE}', 'intel/14.0.1']
conflict:
- '${PACKAGE}'
- 'intel/14.0.1'
will create module files that will conflict with ``intel/14.0.1`` and with the
base directory of the same module, effectively preventing the possibility to
@@ -563,9 +549,9 @@ most likely via the ``+blas`` variant specification.
.. note::
LMod hierarchical module files
When ``lmod`` is activated Spack will generate a set of hierarchical lua module
files that are understood by LMod. The generated hierarchy always contains the
three layers ``Core`` / ``Compiler`` / ``MPI`` but can be further extended to
any other virtual dependency present in Spack. A case that could be useful in
files that are understood by LMod. The hierarchy will always contain the
two layers ``Core`` / ``Compiler`` but can be further extended to
any of the virtual dependencies present in Spack. A case that could be useful in
practice is for instance:
.. code-block:: yaml
@@ -574,11 +560,14 @@ most likely via the ``+blas`` variant specification.
enable:
- lmod
lmod:
core_compilers: ['gcc@4.8']
hierarchical_scheme: ['lapack']
core_compilers:
- 'gcc@4.8'
hierarchy:
- 'mpi'
- 'lapack'
that will generate a hierarchy in which the ``lapack`` layer is treated as the ``mpi``
one. This allows a site to build the same libraries or applications against different
that will generate a hierarchy in which the ``lapack`` and ``mpi`` layer can be switched
independently. This allows a site to build the same libraries or applications against different
implementations of ``mpi`` and ``lapack``, and let LMod switch safely from one to the
other.
@@ -587,15 +576,14 @@ most likely via the ``+blas`` variant specification.
For hierarchies that are deeper than three layers ``lmod spider`` may have some issues.
See `this discussion on the LMod project <https://github.com/TACC/Lmod/issues/114>`_.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
""""""""""""""""""""""""""""""""""""
Filter out environment modifications
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
""""""""""""""""""""""""""""""""""""
Modifications to certain environment variables in module files are generated by
default, for instance by prefix inspections in the default configuration file.
There are cases though where some of these modifications are unwanted.
Suppose you need to avoid having ``CPATH`` and ``LIBRARY_PATH``
modified by your ``dotkit`` modules:
Modifications to certain environment variables in module files are there by
default, for instance because they are generated by prefix inspections.
If you want to prevent modifications to some environment variables, you can
do so by using the environment blacklist:
.. code-block:: yaml
@@ -610,11 +598,11 @@ The configuration above will generate dotkit module files that will not contain
modifications to either ``CPATH`` or ``LIBRARY_PATH`` and environment module
files that instead will contain these modifications.
^^^^^^^^^^^^^^^^^^^^^
"""""""""""""""""""""
Autoload dependencies
^^^^^^^^^^^^^^^^^^^^^
"""""""""""""""""""""
In some cases it can be useful to have module files directly autoload
In some cases it can be useful to have module files that automatically load
their dependencies. This may be the case for Python extensions, if not
activated using ``spack activate``:
@@ -626,8 +614,9 @@ activated using ``spack activate``:
autoload: 'direct'
The configuration file above will produce module files that will
automatically load their direct dependencies. The allowed values for the
``autoload`` statement are either ``none``, ``direct`` or ``all``.
load their direct dependencies if the package installed depends on ``python``.
The allowed values for the ``autoload`` statement are either ``none``,
``direct`` or ``all``.
.. note::
TCL prerequisites

1903
lib/spack/docs/packaging_guide.rst
View File

File diff suppressed because it is too large Load Diff

5
lib/spack/docs/requirements.txt Normal file
View File

@@ -0,0 +1,5 @@
# These dependencies should be installed using pip in order
# to build the documentation.
sphinx
sphinxcontrib-programoutput

16
lib/spack/docs/tutorial_sc16.rst → lib/spack/docs/tutorial.rst
View File

@@ -37,12 +37,18 @@ These scripts will take you step-by-step through basic Spack tasks. They
correspond to sections in the slides above.
1. :ref:`basics-tutorial`
2. :ref:`packaging-tutorial`
3. :ref:`modules-tutorial`
2. :ref:`configs-tutorial`
3. :ref:`packaging-tutorial`
4. :ref:`build-systems-tutorial`
5. :ref:`advanced-packaging-tutorial`
6. :ref:`modules-tutorial`
Full contents:
.. toctree::
tutorial_sc16_spack_basics
tutorial_sc16_packaging
tutorial_sc16_modules
tutorial_basics
tutorial_configuration
tutorial_packaging
tutorial_buildsystems
tutorial_advanced_packaging
tutorial_modules

21
lib/spack/docs/tutorial/examples/0.package.py
View File

@@ -1,13 +1,13 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
@@ -40,7 +40,7 @@
from spack import *
class Mpileaks(AutotoolsPackage):
class Mpileaks(Package):
"""FIXME: Put a proper description of your package here."""
# FIXME: Add a proper url for your package's homepage here.
@@ -50,14 +50,9 @@ class Mpileaks(AutotoolsPackage):
version('1.0', '8838c574b39202a57d7c2d68692718aa')
# FIXME: Add dependencies if required.
# depends_on('m4', type='build')
# depends_on('autoconf', type='build')
# depends_on('automake', type='build')
# depends_on('libtool', type='build')
# depends_on('foo')
def configure_args(self):
# FIXME: Add arguments other than --prefix
# FIXME: If not needed delete the function
args = []
return args
def install(self, spec, prefix):
# FIXME: Unknown build system
make()
make('install')

24
lib/spack/docs/tutorial/examples/1.package.py
View File

@@ -1,13 +1,13 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
@@ -25,24 +25,18 @@
from spack import *
class Mpileaks(AutotoolsPackage):
class Mpileaks(Package):
"""Tool to detect and report MPI objects like MPI_Requests and
MPI_Datatypes."""
homepage = "https://github.com/hpc/mpileaks"
url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz" # NOQA
version('1.0', '8838c574b39202a57d7c2d68692718aa')
# FIXME: Add dependencies if required.
# depends_on('m4', type='build')
# depends_on('autoconf', type='build')
# depends_on('automake', type='build')
# depends_on('libtool', type='build')
# depends_on('foo')
def configure_args(self):
# FIXME: Add arguments other than --prefix
# FIXME: If not needed delete the function
args = []
return args
def install(self, spec, prefix):
# FIXME: Unknown build system
make()
make('install')

17
lib/spack/docs/tutorial/examples/2.package.py
View File

@@ -1,13 +1,13 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
@@ -25,7 +25,7 @@
from spack import *
class Mpileaks(AutotoolsPackage):
class Mpileaks(Package):
"""Tool to detect and report MPI objects like MPI_Requests and
MPI_Datatypes."""
@@ -38,8 +38,7 @@ class Mpileaks(AutotoolsPackage):
depends_on('adept-utils')
depends_on('callpath')
def configure_args(self):
# FIXME: Add arguments other than --prefix
# FIXME: If not needed delete the function
args = []
return args
def install(self, spec, prefix):
# FIXME: Unknown build system
make()
make('install')

17
lib/spack/docs/tutorial/examples/3.package.py
View File

@@ -1,13 +1,13 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
@@ -25,9 +25,10 @@
from spack import *
class Mpileaks(AutotoolsPackage):
class Mpileaks(Package):
"""Tool to detect and report MPI objects like MPI_Requests and
MPI_Datatypes."""
homepage = "https://github.com/hpc/mpileaks"
url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
@@ -37,7 +38,7 @@ class Mpileaks(AutotoolsPackage):
depends_on('adept-utils')
depends_on('callpath')
def configure_args(self):
args = ['--with-adept-utils=%s' % self.spec['adept-utils'].prefix,
'--with-callpath=%s' % self.spec['callpath'].prefix]
return args
def install(self, spec, prefix):
configure()
make()
make('install')

24
lib/spack/docs/tutorial/examples/4.package.py
View File

@@ -1,13 +1,13 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
@@ -25,7 +25,7 @@
from spack import *
class Mpileaks(AutotoolsPackage):
class Mpileaks(Package):
"""Tool to detect and report MPI objects like MPI_Requests and
MPI_Datatypes."""
@@ -34,17 +34,13 @@ class Mpileaks(AutotoolsPackage):
version('1.0', '8838c574b39202a57d7c2d68692718aa')
variant('stackstart', default=0, description='Specify the number of stack frames to truncate.')
depends_on('mpi')
depends_on('adept-utils')
depends_on('callpath')
def configure_args(self):
args = ['--with-adept-utils=%s' % self.spec['adept-utils'].prefix,
'--with-callpath=%s' % self.spec['callpath'].prefix]
stackstart = int(self.spec.variants['stackstart'].value)
if stackstart:
args.extend(['--with-stack-start-c=%s' % stackstart,
'--with-stack-start-fortran=%s' % stackstart])
return args
def install(self, spec, prefix):
configure('--with-adept-utils=%s' % self.spec['adept-utils'].prefix,
'--with-callpath=%s' % self.spec['callpath'].prefix,
'--prefix=%s' % self.spec.prefix)
make()
make('install')

53
lib/spack/docs/tutorial/examples/5.package.py Normal file
View File

@@ -0,0 +1,53 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Mpileaks(Package):
"""Tool to detect and report MPI objects like MPI_Requests and
MPI_Datatypes."""
homepage = "https://github.com/hpc/mpileaks"
url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
version('1.0', '8838c574b39202a57d7c2d68692718aa')
variant('stackstart', values=int, default=0, description='Specify the number of stack frames to truncate.')
depends_on('mpi')
depends_on('adept-utils')
depends_on('callpath')
def install(self, spec, prefix):
stackstart = int(self.spec.variants['stackstart'].value)
confargs = ['--with-adept-utils=%s' % self.spec['adept-utils'].prefix,
'--with-callpath=%s' % self.spec['callpath'].prefix,
'--prefix=%s' % self.spec.prefix]
if stackstart:
confargs.extend(['--with-stack-start-c=%s' % stackstart,
'--with-stack-start-fortran=%s' % stackstart])
configure(*confargs)
make()
make('install')

46
lib/spack/docs/tutorial/examples/Autotools/0.package.py Normal file
View File

@@ -0,0 +1,46 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Mpileaks(AutoToolsPackage):
"""Tool to detect and report leaked MPI objects like MPI_Requests and
MPI_Datatypes."""
homepage = "https://github.com/hpc/mpileaks"
url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
version('1.0', '8838c574b39202a57d7c2d68692718aa')
depends_on("mpi")
depends_on("adept-utils")
depends_on("callpath")
def install(self, spec, prefix):
configure("--prefix=" + prefix,
"--with-adept-utils=" + spec['adept-utils'].prefix,
"--with-callpath=" + spec['callpath'].prefix)
make()
make("install")

51
lib/spack/docs/tutorial/examples/Autotools/1.package.py Normal file
View File

@@ -0,0 +1,51 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Mpileaks(AutoToolsPackage):
"""Tool to detect and report leaked MPI objects like MPI_Requests and
MPI_Datatypes."""
homepage = "https://github.com/hpc/mpileaks"
url = "https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz"
version('1.0', '8838c574b39202a57d7c2d68692718aa')
variant("stackstart", values=int, default=0,
description="Specify the number of stack frames to truncate")
depends_on("mpi")
depends_on("adept-utils")
depends_on("callpath")
def configure_args(self):
stackstart = int(self.spec.variants['stackstart'].value)
args = ["--with-adept-utils=" + spec['adept-utils'].prefix,
"--with-callpath=" + spec['callpath'].prefix]
if stackstart:
args.extend(['--with-stack-start-c=%s' % stackstart,
'--with-stack-start-fortran=%s' % stackstart])
return args

460
lib/spack/docs/tutorial/examples/Autotools/autotools_class.py Normal file
View File

@@ -0,0 +1,460 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import inspect
import os
import os.path
import shutil
import stat
from subprocess import PIPE
from subprocess import check_call
import llnl.util.tty as tty
from llnl.util.filesystem import working_dir, join_path, force_remove
from spack.package import PackageBase, run_after, run_before
from spack.util.executable import Executable
class AutotoolsPackage(PackageBase):
"""Specialized class for packages built using GNU Autotools.
This class provides four phases that can be overridden:
1. :py:meth:`~.AutotoolsPackage.autoreconf`
2. :py:meth:`~.AutotoolsPackage.configure`
3. :py:meth:`~.AutotoolsPackage.build`
4. :py:meth:`~.AutotoolsPackage.install`
They all have sensible defaults and for many packages the only thing
necessary will be to override the helper method
:py:meth:`~.AutotoolsPackage.configure_args`.
For a finer tuning you may also override:
+-----------------------------------------------+--------------------+
| **Method** | **Purpose** |
+===============================================+====================+
| :py:attr:`~.AutotoolsPackage.build_targets` | Specify ``make`` |
| | targets for the |
| | build phase |
+-----------------------------------------------+--------------------+
| :py:attr:`~.AutotoolsPackage.install_targets` | Specify ``make`` |
| | targets for the |
| | install phase |
+-----------------------------------------------+--------------------+
| :py:meth:`~.AutotoolsPackage.check` | Run build time |
| | tests if required |
+-----------------------------------------------+--------------------+
"""
#: Phases of a GNU Autotools package
phases = ['autoreconf', 'configure', 'build', 'install']
#: This attribute is used in UI queries that need to know the build
#: system base class
build_system_class = 'AutotoolsPackage'
#: Whether or not to update ``config.guess`` on old architectures
patch_config_guess = True
#: Targets for ``make`` during the :py:meth:`~.AutotoolsPackage.build`
#: phase
build_targets = []
#: Targets for ``make`` during the :py:meth:`~.AutotoolsPackage.install`
#: phase
install_targets = ['install']
#: Callback names for build-time test
build_time_test_callbacks = ['check']
#: Callback names for install-time test
install_time_test_callbacks = ['installcheck']
#: Set to true to force the autoreconf step even if configure is present
force_autoreconf = False
#: Options to be passed to autoreconf when using the default implementation
autoreconf_extra_args = []
@run_after('autoreconf')
def _do_patch_config_guess(self):
"""Some packages ship with an older config.guess and need to have
this updated when installed on a newer architecture. In particular,
config.guess fails for PPC64LE for version prior to a 2013-06-10
build date (automake 1.13.4)."""
if not self.patch_config_guess or not self.spec.satisfies(
'target=ppc64le'
):
return
my_config_guess = None
config_guess = None
if os.path.exists('config.guess'):
# First search the top-level source directory
my_config_guess = 'config.guess'
else:
# Then search in all sub directories.
# We would like to use AC_CONFIG_AUX_DIR, but not all packages
# ship with their configure.in or configure.ac.
d = '.'
dirs = [os.path.join(d, o) for o in os.listdir(d)
if os.path.isdir(os.path.join(d, o))]
for dirname in dirs:
path = os.path.join(dirname, 'config.guess')
if os.path.exists(path):
my_config_guess = path
if my_config_guess is not None:
try:
check_call([my_config_guess], stdout=PIPE, stderr=PIPE)
# The package's config.guess already runs OK, so just use it
return
except Exception:
pass
else:
return
# Look for a spack-installed automake package
if 'automake' in self.spec:
automake_path = os.path.join(self.spec['automake'].prefix, 'share',
'automake-' +
str(self.spec['automake'].version))
path = os.path.join(automake_path, 'config.guess')
if os.path.exists(path):
config_guess = path
# Look for the system's config.guess
if config_guess is None and os.path.exists('/usr/share'):
automake_dir = [s for s in os.listdir('/usr/share') if
"automake" in s]
if automake_dir:
automake_path = os.path.join('/usr/share', automake_dir[0])
path = os.path.join(automake_path, 'config.guess')
if os.path.exists(path):
config_guess = path
if config_guess is not None:
try:
check_call([config_guess], stdout=PIPE, stderr=PIPE)
mod = os.stat(my_config_guess).st_mode & 0o777 | stat.S_IWUSR
os.chmod(my_config_guess, mod)
shutil.copyfile(config_guess, my_config_guess)
return
except Exception:
pass
raise RuntimeError('Failed to find suitable config.guess')
@property
def configure_directory(self):
"""Returns the directory where 'configure' resides.
:return: directory where to find configure
"""
return self.stage.source_path
@property
def configure_abs_path(self):
# Absolute path to configure
configure_abs_path = join_path(
os.path.abspath(self.configure_directory), 'configure'
)
return configure_abs_path
@property
def build_directory(self):
"""Override to provide another place to build the package"""
return self.configure_directory
def default_flag_handler(self, spack_env, flag_val):
# Relies on being the first thing that can affect the spack_env
# EnvironmentModification after it is instantiated or no other
# method trying to affect these variables. Currently both are true
# flag_val is a tuple (flag, value_list).
spack_env.set(flag_val[0].upper(),
' '.join(flag_val[1]))
return []
@run_before('autoreconf')
def delete_configure_to_force_update(self):
if self.force_autoreconf:
force_remove(self.configure_abs_path)
def autoreconf(self, spec, prefix):
"""Not needed usually, configure should be already there"""
# If configure exists nothing needs to be done
if os.path.exists(self.configure_abs_path):
return
# Else try to regenerate it
autotools = ['m4', 'autoconf', 'automake', 'libtool']
missing = [x for x in autotools if x not in spec]
if missing:
msg = 'Cannot generate configure: missing dependencies {0}'
raise RuntimeError(msg.format(missing))
tty.msg('Configure script not found: trying to generate it')
tty.warn('*********************************************************')
tty.warn('* If the default procedure fails, consider implementing *')
tty.warn('* a custom AUTORECONF phase in the package *')
tty.warn('*********************************************************')
with working_dir(self.configure_directory):
m = inspect.getmodule(self)
# This part should be redundant in principle, but
# won't hurt
m.libtoolize()
m.aclocal()
# This line is what is needed most of the time
# --install, --verbose, --force
autoreconf_args = ['-ivf']
if 'pkg-config' in spec:
autoreconf_args += [
'-I',
join_path(spec['pkg-config'].prefix, 'share', 'aclocal'),
]
autoreconf_args += self.autoreconf_extra_args
m.autoreconf(*autoreconf_args)
@run_after('autoreconf')
def set_configure_or_die(self):
"""Checks the presence of a ``configure`` file after the
autoreconf phase. If it is found sets a module attribute
appropriately, otherwise raises an error.
:raises RuntimeError: if a configure script is not found in
:py:meth:`~AutotoolsPackage.configure_directory`
"""
# Check if a configure script is there. If not raise a RuntimeError.
if not os.path.exists(self.configure_abs_path):
msg = 'configure script not found in {0}'
raise RuntimeError(msg.format(self.configure_directory))
# Monkey-patch the configure script in the corresponding module
inspect.getmodule(self).configure = Executable(
self.configure_abs_path
)
def configure_args(self):
"""Produces a list containing all the arguments that must be passed to
configure, except ``--prefix`` which will be pre-pended to the list.
:return: list of arguments for configure
"""
return []
def configure(self, spec, prefix):
"""Runs configure with the arguments specified in
:py:meth:`~.AutotoolsPackage.configure_args`
and an appropriately set prefix.
"""
options = ['--prefix={0}'.format(prefix)] + self.configure_args()
with working_dir(self.build_directory, create=True):
inspect.getmodule(self).configure(*options)
def build(self, spec, prefix):
"""Makes the build targets specified by
:py:attr:``~.AutotoolsPackage.build_targets``
"""
with working_dir(self.build_directory):
inspect.getmodule(self).make(*self.build_targets)
def install(self, spec, prefix):
"""Makes the install targets specified by
:py:attr:``~.AutotoolsPackage.install_targets``
"""
with working_dir(self.build_directory):
inspect.getmodule(self).make(*self.install_targets)
run_after('build')(PackageBase._run_default_build_time_test_callbacks)
def check(self):
"""Searches the Makefile for targets ``test`` and ``check``
and runs them if found.
"""
with working_dir(self.build_directory):
self._if_make_target_execute('test')
self._if_make_target_execute('check')
def _activate_or_not(
self,
name,
activation_word,
deactivation_word,
activation_value=None
):
"""This function contains the current implementation details of
:py:meth:`~.AutotoolsPackage.with_or_without` and
:py:meth:`~.AutotoolsPackage.enable_or_disable`.
Args:
name (str): name of the variant that is being processed
activation_word (str): the default activation word ('with' in the
case of ``with_or_without``)
deactivation_word (str): the default deactivation word ('without'
in the case of ``with_or_without``)
activation_value (callable): callable that accepts a single
value. This value is either one of the allowed values for a
multi-valued variant or the name of a bool-valued variant.
Returns the parameter to be used when the value is activated.
The special value 'prefix' can also be assigned and will return
``spec[name].prefix`` as activation parameter.
Examples:
Given a package with:
.. code-block:: python
variant('foo', values=('x', 'y'), description='')
variant('bar', default=True, description='')
calling this function like:
.. code-block:: python
_activate_or_not(
'foo', 'with', 'without', activation_value='prefix'
)
_activate_or_not('bar', 'with', 'without')
will generate the following configuration options:
.. code-block:: console
--with-x=<prefix-to-x> --without-y --with-bar
for ``<spec-name> foo=x +bar``
Returns:
list of strings that corresponds to the activation/deactivation
of the variant that has been processed
Raises:
KeyError: if name is not among known variants
"""
spec = self.spec
args = []
if activation_value == 'prefix':
activation_value = lambda x: spec[x].prefix
# Defensively look that the name passed as argument is among
# variants
if name not in self.variants:
msg = '"{0}" is not a variant of "{1}"'
raise KeyError(msg.format(name, self.name))
# Create a list of pairs. Each pair includes a configuration
# option and whether or not that option is activated
if set(self.variants[name].values) == set((True, False)):
# BoolValuedVariant carry information about a single option.
# Nonetheless, for uniformity of treatment we'll package them
# in an iterable of one element.
condition = '+{name}'.format(name=name)
options = [(name, condition in spec)]
else:
condition = '{name}={value}'
options = [
(value, condition.format(name=name, value=value) in spec)
for value in self.variants[name].values
]
# For each allowed value in the list of values
for option_value, activated in options:
# Search for an override in the package for this value
override_name = '{0}_or_{1}_{2}'.format(
activation_word, deactivation_word, option_value
)
line_generator = getattr(self, override_name, None)
# If not available use a sensible default
if line_generator is None:
def _default_generator(is_activated):
if is_activated:
line = '--{0}-{1}'.format(
activation_word, option_value
)
if activation_value is not None and activation_value(option_value): # NOQA=ignore=E501
line += '={0}'.format(
activation_value(option_value)
)
return line
return '--{0}-{1}'.format(deactivation_word, option_value)
line_generator = _default_generator
args.append(line_generator(activated))
return args
def with_or_without(self, name, activation_value=None):
"""Inspects a variant and returns the arguments that activate
or deactivate the selected feature(s) for the configure options.
This function works on all type of variants. For bool-valued variants
it will return by default ``--with-{name}`` or ``--without-{name}``.
For other kinds of variants it will cycle over the allowed values and
return either ``--with-{value}`` or ``--without-{value}``.
If activation_value is given, then for each possible value of the
variant, the option ``--with-{value}=activation_value(value)`` or
``--without-{value}`` will be added depending on whether or not
``variant=value`` is in the spec.
Args:
name (str): name of a valid multi-valued variant
activation_value (callable): callable that accepts a single
value and returns the parameter to be used leading to an entry
of the type ``--with-{name}={parameter}``.
The special value 'prefix' can also be assigned and will return
``spec[name].prefix`` as activation parameter.
Returns:
list of arguments to configure
"""
return self._activate_or_not(name, 'with', 'without', activation_value)
def enable_or_disable(self, name, activation_value=None):
"""Same as :py:meth:`~.AutotoolsPackage.with_or_without` but substitute
``with`` with ``enable`` and ``without`` with ``disable``.
Args:
name (str): name of a valid multi-valued variant
activation_value (callable): if present accepts a single value
and returns the parameter to be used leading to an entry of the
type ``--enable-{name}={parameter}``
The special value 'prefix' can also be assigned and will return
``spec[name].prefix`` as activation parameter.
Returns:
list of arguments to configure
"""
return self._activate_or_not(
name, 'enable', 'disable', activation_value
)
run_after('install')(PackageBase._run_default_install_time_test_callbacks)
def installcheck(self):
"""Searches the Makefile for an ``installcheck`` target
and runs it if found.
"""
with working_dir(self.build_directory):
self._if_make_target_execute('installcheck')
# Check that self.prefix is there after installation
run_after('install')(PackageBase.sanity_check_prefix)

60
lib/spack/docs/tutorial/examples/Cmake/0.package.py Normal file
View File

@@ -0,0 +1,60 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
#
# This is a template package file for Spack. We've put "FIXME"
# next to all the things you'll want to change. Once you've handled
# them, you can save this file and test your package like this:
#
# spack install callpath
#
# You can edit this file again by typing:
#
# spack edit callpath
#
# See the Spack documentation for more information on packaging.
# If you submit this package back to Spack as a pull request,
# please first remove this boilerplate and all FIXME comments.
#
from spack import *
class Callpath(CMakePackage):
"""FIXME: Put a proper description of your package here."""
# FIXME: Add a proper url for your package's homepage here.
homepage = "http://www.example.com"
url = "https://github.com/llnl/callpath/archive/v1.0.1.tar.gz"
version('1.0.3', 'c89089b3f1c1ba47b09b8508a574294a')
# FIXME: Add dependencies if required.
# depends_on('foo')
def cmake_args(self):
# FIXME: Add arguments other than
# FIXME: CMAKE_INSTALL_PREFIX and CMAKE_BUILD_TYPE
# FIXME: If not needed delete this function
args = []
return args

42
lib/spack/docs/tutorial/examples/Cmake/1.package.py Normal file
View File

@@ -0,0 +1,42 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Callpath(CMakePackage):
"""Library for representing callpaths consistently in
distributed-memory performance tools."""
homepage = "https://github.com/llnl/callpath"
url = "https://github.com/llnl/callpath/archive/v1.0.3.tar.gz"
version('1.0.3', 'c89089b3f1c1ba47b09b8508a574294a')
depends_on("elf", type="link")
depends_on("libdwarf")
depends_on("dyninst")
depends_on("adept-utils")
depends_on("mpi")
depends_on("cmake@2.8:", type="build")

52
lib/spack/docs/tutorial/examples/Cmake/2.package.py Normal file
View File

@@ -0,0 +1,52 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Callpath(CMakePackage):
"""Library for representing callpaths consistently in
distributed-memory performance tools."""
homepage = "https://github.com/llnl/callpath"
url = "https://github.com/llnl/callpath/archive/v1.0.3.tar.gz"
version('1.0.3', 'c89089b3f1c1ba47b09b8508a574294a')
depends_on("elf", type="link")
depends_on("libdwarf")
depends_on("dyninst")
depends_on("adept-utils")
depends_on("mpi")
depends_on("cmake@2.8:", type="build")
def cmake_args(self):
args = ["-DCALLPATH_WALKER=dyninst"]
if self.spec.satisfies("^dyninst@9.3.0:"):
std.flag = self.compiler.cxx_flag
args.append("-DCMAKE_CXX_FLAGS='{0}' -fpermissive'".format(
std_flag))
return args

224
lib/spack/docs/tutorial/examples/Cmake/cmake_class.py Normal file
View File

@@ -0,0 +1,224 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import inspect
import os
import platform
import spack.build_environment
from llnl.util.filesystem import working_dir, join_path
from spack.util.environment import filter_system_paths
from spack.directives import depends_on, variant
from spack.package import PackageBase, InstallError, run_after
class CMakePackage(PackageBase):
"""Specialized class for packages built using CMake
For more information on the CMake build system, see:
https://cmake.org/cmake/help/latest/
This class provides three phases that can be overridden:
1. :py:meth:`~.CMakePackage.cmake`
2. :py:meth:`~.CMakePackage.build`
3. :py:meth:`~.CMakePackage.install`
They all have sensible defaults and for many packages the only thing
necessary will be to override :py:meth:`~.CMakePackage.cmake_args`.
For a finer tuning you may also override:
+-----------------------------------------------+--------------------+
| **Method** | **Purpose** |
+===============================================+====================+
| :py:meth:`~.CMakePackage.root_cmakelists_dir` | Location of the |
| | root CMakeLists.txt|
+-----------------------------------------------+--------------------+
| :py:meth:`~.CMakePackage.build_directory` | Directory where to |
| | build the package |
+-----------------------------------------------+--------------------+
"""
#: Phases of a CMake package
phases = ['cmake', 'build', 'install']
#: This attribute is used in UI queries that need to know the build
#: system base class
build_system_class = 'CMakePackage'
build_targets = []
install_targets = ['install']
build_time_test_callbacks = ['check']
#: The build system generator to use.
#:
#: See ``cmake --help`` for a list of valid generators.
#: Currently, "Unix Makefiles" and "Ninja" are the only generators
#: that Spack supports. Defaults to "Unix Makefiles".
#:
#: See https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html
#: for more information.
generator = 'Unix Makefiles'
# https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html
variant('build_type', default='RelWithDebInfo',
description='CMake build type',
values=('Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel'))
depends_on('cmake', type='build')
@property
def root_cmakelists_dir(self):
"""The relative path to the directory containing CMakeLists.txt
This path is relative to the root of the extracted tarball,
not to the ``build_directory``. Defaults to the current directory.
:return: directory containing CMakeLists.txt
"""
return self.stage.source_path
@property
def std_cmake_args(self):
"""Standard cmake arguments provided as a property for
convenience of package writers
:return: standard cmake arguments
"""
# standard CMake arguments
return CMakePackage._std_args(self)
@staticmethod
def _std_args(pkg):
"""Computes the standard cmake arguments for a generic package"""
try:
generator = pkg.generator
except AttributeError:
generator = 'Unix Makefiles'
# Make sure a valid generator was chosen
valid_generators = ['Unix Makefiles', 'Ninja']
if generator not in valid_generators:
msg = "Invalid CMake generator: '{0}'\n".format(generator)
msg += "CMakePackage currently supports the following "
msg += "generators: '{0}'".format("', '".join(valid_generators))
raise InstallError(msg)
try:
build_type = pkg.spec.variants['build_type'].value
except KeyError:
build_type = 'RelWithDebInfo'
args = [
'-G', generator,
'-DCMAKE_INSTALL_PREFIX:PATH={0}'.format(pkg.prefix),
'-DCMAKE_BUILD_TYPE:STRING={0}'.format(build_type),
'-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON'
]
if platform.mac_ver()[0]:
args.append('-DCMAKE_FIND_FRAMEWORK:STRING=LAST')
# Set up CMake rpath
args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE')
rpaths = ':'.join(spack.build_environment.get_rpaths(pkg))
args.append('-DCMAKE_INSTALL_RPATH:STRING={0}'.format(rpaths))
# CMake's find_package() looks in CMAKE_PREFIX_PATH first, help CMake
# to find immediate link dependencies in right places:
deps = [d.prefix for d in
pkg.spec.dependencies(deptype=('build', 'link'))]
deps = filter_system_paths(deps)
args.append('-DCMAKE_PREFIX_PATH:STRING={0}'.format(';'.join(deps)))
return args
@property
def build_directory(self):
"""Returns the directory to use when building the package
:return: directory where to build the package
"""
return join_path(self.stage.source_path, 'spack-build')
def default_flag_handler(self, spack_env, flag_val):
# Relies on being the first thing that can affect the spack_env
# EnvironmentModification after it is instantiated or no other
# method trying to affect these variables. Currently both are true
# flag_val is a tuple (flag, value_list)
spack_env.set(flag_val[0].upper(),
' '.join(flag_val[1]))
return []
def cmake_args(self):
"""Produces a list containing all the arguments that must be passed to
cmake, except:
* CMAKE_INSTALL_PREFIX
* CMAKE_BUILD_TYPE
which will be set automatically.
:return: list of arguments for cmake
"""
return []
def cmake(self, spec, prefix):
"""Runs ``cmake`` in the build directory"""
options = [os.path.abspath(self.root_cmakelists_dir)]
options += self.std_cmake_args
options += self.cmake_args()
with working_dir(self.build_directory, create=True):
inspect.getmodule(self).cmake(*options)
def build(self, spec, prefix):
"""Make the build targets"""
with working_dir(self.build_directory):
if self.generator == 'Unix Makefiles':
inspect.getmodule(self).make(*self.build_targets)
elif self.generator == 'Ninja':
inspect.getmodule(self).ninja(*self.build_targets)
def install(self, spec, prefix):
"""Make the install targets"""
with working_dir(self.build_directory):
if self.generator == 'Unix Makefiles':
inspect.getmodule(self).make(*self.install_targets)
elif self.generator == 'Ninja':
inspect.getmodule(self).ninja(*self.install_targets)
run_after('build')(PackageBase._run_default_build_time_test_callbacks)
def check(self):
"""Searches the CMake-generated Makefile for the target ``test``
and runs it if found.
"""
with working_dir(self.build_directory):
if self.generator == 'Unix Makefiles':
self._if_make_target_execute('test')
elif self.generator == 'Ninja':
self._if_ninja_target_execute('test')
# Check that self.prefix is there after installation
run_after('install')(PackageBase.sanity_check_prefix)

45
lib/spack/docs/tutorial/examples/Makefile/0.package.py Normal file
View File

@@ -0,0 +1,45 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Bowtie(MakefilePackage):
"""FIXME: Put a proper description of your package here."""
# FIXME: Add a proper url for your package's homepage here.
homepage = "http://www.example.com"
url = "https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip"
version('1.2.1.1', 'ec06265730c5f587cd58bcfef6697ddf')
# FIXME: Add dependencies if required.
# depends_on('foo')
def edit(self, spec, prefix):
# FIXME: Edit the Makefile if necessary
# FIXME: If not needed delete this function
# makefile = FileFilter('Makefile')
# makefile.filter('CC = .*', 'CC = cc')
return

46
lib/spack/docs/tutorial/examples/Makefile/1.package.py Normal file
View File

@@ -0,0 +1,46 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Bowtie(MakefilePackage):
"""Bowtie is an ultrafast, memory efficient short read aligner
for short DNA sequences (reads) from next-gen sequencers."""
homepage = "https://sourceforge.net/projects/bowtie-bio/"
url = "https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip"
version('1.2.1.1', 'ec06265730c5f587cd58bcfef6697ddf')
variant("tbb", default=False, description="Use Intel thread building block")
depends_on("tbb", when="+tbb")
def edit(self, spec, prefix):
# FIXME: Edit the Makefile if necessary
# FIXME: If not needed delete this function
# makefile = FileFilter('Makefile')
# makefile.filter('CC = .*', 'CC = cc')
return

44
lib/spack/docs/tutorial/examples/Makefile/2.package.py Normal file
View File

@@ -0,0 +1,44 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Bowtie(MakefilePackage):
"""Bowtie is an ultrafast, memory efficient short read aligner
for short DNA sequences (reads) from next-gen sequencers."""
homepage = "https://sourceforge.net/projects/bowtie-bio/"
url = "https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip"
version('1.2.1.1', 'ec06265730c5f587cd58bcfef6697ddf')
variant("tbb", default=False, description="Use Intel thread building block")
depends_on("tbb", when="+tbb")
def edit(self, spec, prefix):
makefile = FileFilter("Makefile")
makefile.filter('CC= .*', 'CC = ' + env['CC'])
makefile.filter('CXX = .*', 'CXX = ' + env['CXX'])

53
lib/spack/docs/tutorial/examples/Makefile/3.package.py Normal file
View File

@@ -0,0 +1,53 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Bowtie(MakefilePackage):
"""Bowtie is an ultrafast, memory efficient short read aligner
for short DNA sequences (reads) from next-gen sequencers."""
homepage = "https://sourceforge.net/projects/bowtie-bio/"
url = "https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip"
version('1.2.1.1', 'ec06265730c5f587cd58bcfef6697ddf')
variant("tbb", default=False, description="Use Intel thread building block")
depends_on("tbb", when="+tbb")
def edit(self, spec, prefix):
makefile = FileFilter("Makefile")
makefile.filter('CC= .*', 'CC = ' + env['CC'])
makefile.filter('CXX = .*', 'CXX = ' + env['CXX'])
def build(self, spec, prefix):
if "+tbb" in spec:
make()
else:
make("NO_TBB=1")
def install(self, spec, prefix):
make('prefix={0}'.format(self.prefix), 'install')

129
lib/spack/docs/tutorial/examples/Makefile/makefile_class.py Normal file
View File

@@ -0,0 +1,129 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import inspect
import llnl.util.tty as tty
from llnl.util.filesystem import working_dir
from spack.package import PackageBase, run_after
class MakefilePackage(PackageBase):
"""Specialized class for packages that are built using editable Makefiles
This class provides three phases that can be overridden:
1. :py:meth:`~.MakefilePackage.edit`
2. :py:meth:`~.MakefilePackage.build`
3. :py:meth:`~.MakefilePackage.install`
It is usually necessary to override the :py:meth:`~.MakefilePackage.edit`
phase, while :py:meth:`~.MakefilePackage.build` and
:py:meth:`~.MakefilePackage.install` have sensible defaults.
For a finer tuning you may override:
+-----------------------------------------------+--------------------+
| **Method** | **Purpose** |
+===============================================+====================+
| :py:attr:`~.MakefilePackage.build_targets` | Specify ``make`` |
| | targets for the |
| | build phase |
+-----------------------------------------------+--------------------+
| :py:attr:`~.MakefilePackage.install_targets` | Specify ``make`` |
| | targets for the |
| | install phase |
+-----------------------------------------------+--------------------+
| :py:meth:`~.MakefilePackage.build_directory` | Directory where the|
| | Makefile is located|
+-----------------------------------------------+--------------------+
"""
#: Phases of a package that is built with an hand-written Makefile
phases = ['edit', 'build', 'install']
#: This attribute is used in UI queries that need to know the build
#: system base class
build_system_class = 'MakefilePackage'
#: Targets for ``make`` during the :py:meth:`~.MakefilePackage.build`
#: phase
build_targets = []
#: Targets for ``make`` during the :py:meth:`~.MakefilePackage.install`
#: phase
install_targets = ['install']
#: Callback names for build-time test
build_time_test_callbacks = ['check']
#: Callback names for install-time test
install_time_test_callbacks = ['installcheck']
@property
def build_directory(self):
"""Returns the directory containing the main Makefile
:return: build directory
"""
return self.stage.source_path
def edit(self, spec, prefix):
"""Edits the Makefile before calling make. This phase cannot
be defaulted.
"""
tty.msg('Using default implementation: skipping edit phase.')
def build(self, spec, prefix):
"""Calls make, passing :py:attr:`~.MakefilePackage.build_targets`
as targets.
"""
with working_dir(self.build_directory):
inspect.getmodule(self).make(*self.build_targets)
def install(self, spec, prefix):
"""Calls make, passing :py:attr:`~.MakefilePackage.install_targets`
as targets.
"""
with working_dir(self.build_directory):
inspect.getmodule(self).make(*self.install_targets)
run_after('build')(PackageBase._run_default_build_time_test_callbacks)
def check(self):
"""Searches the Makefile for targets ``test`` and ``check``
and runs them if found.
"""
with working_dir(self.build_directory):
self._if_make_target_execute('test')
self._if_make_target_execute('check')
run_after('install')(PackageBase._run_default_install_time_test_callbacks)
def installcheck(self):
"""Searches the Makefile for an ``installcheck`` target
and runs it if found.
"""
with working_dir(self.build_directory):
self._if_make_target_execute('installcheck')
# Check that self.prefix is there after installation
run_after('install')(PackageBase.sanity_check_prefix)

60
lib/spack/docs/tutorial/examples/PyPackage/0.package.py Normal file
View File

@@ -0,0 +1,60 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
#
# This is a template package file for Spack. We've put "FIXME"
# next to all the things you'll want to change. Once you've handled
# them, you can save this file and test your package like this:
#
# spack install py-pandas
#
# You can edit this file again by typing:
#
# spack edit py-pandas
#
# See the Spack documentation for more information on packaging.
# If you submit this package back to Spack as a pull request,
# please first remove this boilerplate and all FIXME comments.
#
from spack import *
class PyPandas(PythonPackage):
"""FIXME: Put a proper description of your package here."""
# FIXME: Add a proper url for your package's homepage here.
homepage = "http://www.example.com"
url = "https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz"
version('0.19.0', 'bc9bb7188e510b5d44fbdd249698a2c3')
# FIXME: Add dependencies if required.
# depends_on('py-setuptools', type='build')
# depends_on('py-foo', type=('build', 'run'))
def build_args(self, spec, prefix):
# FIXME: Add arguments other than --prefix
# FIXME: If not needed delete this function
args = []
return args

51
lib/spack/docs/tutorial/examples/PyPackage/1.package.py Normal file
View File

@@ -0,0 +1,51 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class PyPandas(PythonPackage):
"""pandas is a Python package providing fast, flexible, and expressive
data structures designed to make working with relational or
labeled data both easy and intuitive. It aims to be the
fundamental high-level building block for doing practical, real
world data analysis in Python. Additionally, it has the broader
goal of becoming the most powerful and flexible open source data
analysis / manipulation tool available in any language.
"""
homepage = "http://pandas.pydata.org/"
url = "https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz"
version('0.19.0', 'bc9bb7188e510b5d44fbdd249698a2c3')
version('0.18.0', 'f143762cd7a59815e348adf4308d2cf6')
version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8')
version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73')
depends_on('py-dateutil', type=('build', 'run'))
depends_on('py-numpy', type=('build', 'run'))
depends_on('py-setuptools', type='build')
depends_on('py-cython', type='build')
depends_on('py-pytz', type=('build', 'run'))
depends_on('py-numexpr', type=('build', 'run'))
depends_on('py-bottleneck', type=('build', 'run'))

399
lib/spack/docs/tutorial/examples/PyPackage/python_package_class.py Normal file
View File

@@ -0,0 +1,399 @@
##############################################################################
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import inspect
import os
from spack.directives import depends_on, extends
from spack.package import PackageBase, run_after
from llnl.util.filesystem import working_dir
class PythonPackage(PackageBase):
"""Specialized class for packages that are built using Python
setup.py files
This class provides the following phases that can be overridden:
* build
* build_py
* build_ext
* build_clib
* build_scripts
* clean
* install
* install_lib
* install_headers
* install_scripts
* install_data
* sdist
* register
* bdist
* bdist_dumb
* bdist_rpm
* bdist_wininst
* upload
* check
These are all standard setup.py commands and can be found by running:
.. code-block:: console
$ python setup.py --help-commands
By default, only the 'build' and 'install' phases are run, but if you
need to run more phases, simply modify your ``phases`` list like so:
.. code-block:: python
phases = ['build_ext', 'install', 'bdist']
Each phase provides a function <phase> that runs:
.. code-block:: console
$ python setup.py --no-user-cfg <phase>
Each phase also has a <phase_args> function that can pass arguments to
this call. All of these functions are empty except for the ``install_args``
function, which passes ``--prefix=/path/to/installation/directory``.
If you need to run a phase which is not a standard setup.py command,
you'll need to define a function for it like so:
.. code-block:: python
def configure(self, spec, prefix):
self.setup_py('configure')
"""
# Default phases
phases = ['build', 'install']
# Name of modules that the Python package provides
# This is used to test whether or not the installation succeeded
# These names generally come from running:
#
# >>> import setuptools
# >>> setuptools.find_packages()
#
# in the source tarball directory
import_modules = []
# To be used in UI queries that require to know which
# build-system class we are using
build_system_class = 'PythonPackage'
#: Callback names for build-time test
build_time_test_callbacks = ['test']
#: Callback names for install-time test
install_time_test_callbacks = ['import_module_test']
extends('python')
depends_on('python', type=('build', 'run'))
def setup_file(self):
"""Returns the name of the setup file to use."""
return 'setup.py'
@property
def build_directory(self):
"""The directory containing the ``setup.py`` file."""
return self.stage.source_path
def python(self, *args, **kwargs):
inspect.getmodule(self).python(*args, **kwargs)
def setup_py(self, *args, **kwargs):
setup = self.setup_file()
with working_dir(self.build_directory):
self.python(setup, '--no-user-cfg', *args, **kwargs)
def _setup_command_available(self, command):
"""Determines whether or not a setup.py command exists.
Args:
command (str): The command to look for
Returns:
bool: True if the command is found, else False
"""
kwargs = {
'output': os.devnull,
'error': os.devnull,
'fail_on_error': False
}
python = inspect.getmodule(self).python
setup = self.setup_file()
python(setup, '--no-user-cfg', command, '--help', **kwargs)
return python.returncode == 0
# The following phases and their descriptions come from:
# $ python setup.py --help-commands
# Standard commands
def build(self, spec, prefix):
"""Build everything needed to install."""
args = self.build_args(spec, prefix)
self.setup_py('build', *args)
def build_args(self, spec, prefix):
"""Arguments to pass to build."""
return []
def build_py(self, spec, prefix):
'''"Build" pure Python modules (copy to build directory).'''
args = self.build_py_args(spec, prefix)
self.setup_py('build_py', *args)
def build_py_args(self, spec, prefix):
"""Arguments to pass to build_py."""
return []
def build_ext(self, spec, prefix):
"""Build C/C++ extensions (compile/link to build directory)."""
args = self.build_ext_args(spec, prefix)
self.setup_py('build_ext', *args)
def build_ext_args(self, spec, prefix):
"""Arguments to pass to build_ext."""
return []
def build_clib(self, spec, prefix):
"""Build C/C++ libraries used by Python extensions."""
args = self.build_clib_args(spec, prefix)
self.setup_py('build_clib', *args)
def build_clib_args(self, spec, prefix):
"""Arguments to pass to build_clib."""
return []
def build_scripts(self, spec, prefix):
'''"Build" scripts (copy and fixup #! line).'''
args = self.build_scripts_args(spec, prefix)
self.setup_py('build_scripts', *args)
def clean(self, spec, prefix):
"""Clean up temporary files from 'build' command."""
args = self.clean_args(spec, prefix)
self.setup_py('clean', *args)
def clean_args(self, spec, prefix):
"""Arguments to pass to clean."""
return []
def install(self, spec, prefix):
"""Install everything from build directory."""
args = self.install_args(spec, prefix)
self.setup_py('install', *args)
def install_args(self, spec, prefix):
"""Arguments to pass to install."""
args = ['--prefix={0}'.format(prefix)]
# This option causes python packages (including setuptools) NOT
# to create eggs or easy-install.pth files. Instead, they
# install naturally into $prefix/pythonX.Y/site-packages.
#
# Eggs add an extra level of indirection to sys.path, slowing
# down large HPC runs. They are also deprecated in favor of
# wheels, which use a normal layout when installed.
#
# Spack manages the package directory on its own by symlinking
# extensions into the site-packages directory, so we don't really
# need the .pth files or egg directories, anyway.
if ('py-setuptools' == spec.name or # this is setuptools, or
'py-setuptools' in spec._dependencies): # it's an immediate dep
args += ['--single-version-externally-managed', '--root=/']
return args
def install_lib(self, spec, prefix):
"""Install all Python modules (extensions and pure Python)."""
args = self.install_lib_args(spec, prefix)
self.setup_py('install_lib', *args)
def install_lib_args(self, spec, prefix):
"""Arguments to pass to install_lib."""
return []
def install_headers(self, spec, prefix):
"""Install C/C++ header files."""
args = self.install_headers_args(spec, prefix)
self.setup_py('install_headers', *args)
def install_headers_args(self, spec, prefix):
"""Arguments to pass to install_headers."""
return []
def install_scripts(self, spec, prefix):
"""Install scripts (Python or otherwise)."""
args = self.install_scripts_args(spec, prefix)
self.setup_py('install_scripts', *args)
def install_scripts_args(self, spec, prefix):
"""Arguments to pass to install_scripts."""
return []
def install_data(self, spec, prefix):
"""Install data files."""
args = self.install_data_args(spec, prefix)
self.setup_py('install_data', *args)
def install_data_args(self, spec, prefix):
"""Arguments to pass to install_data."""
return []
def sdist(self, spec, prefix):
"""Create a source distribution (tarball, zip file, etc.)."""
args = self.sdist_args(spec, prefix)
self.setup_py('sdist', *args)
def sdist_args(self, spec, prefix):
"""Arguments to pass to sdist."""
return []
def register(self, spec, prefix):
"""Register the distribution with the Python package index."""
args = self.register_args(spec, prefix)
self.setup_py('register', *args)
def register_args(self, spec, prefix):
"""Arguments to pass to register."""
return []
def bdist(self, spec, prefix):
"""Create a built (binary) distribution."""
args = self.bdist_args(spec, prefix)
self.setup_py('bdist', *args)
def bdist_args(self, spec, prefix):
"""Arguments to pass to bdist."""
return []
def bdist_dumb(self, spec, prefix):
'''Create a "dumb" built distribution.'''
args = self.bdist_dumb_args(spec, prefix)
self.setup_py('bdist_dumb', *args)
def bdist_dumb_args(self, spec, prefix):
"""Arguments to pass to bdist_dumb."""
return []
def bdist_rpm(self, spec, prefix):
"""Create an RPM distribution."""
args = self.bdist_rpm(spec, prefix)
self.setup_py('bdist_rpm', *args)
def bdist_rpm_args(self, spec, prefix):
"""Arguments to pass to bdist_rpm."""
return []
def bdist_wininst(self, spec, prefix):
"""Create an executable installer for MS Windows."""
args = self.bdist_wininst_args(spec, prefix)
self.setup_py('bdist_wininst', *args)
def bdist_wininst_args(self, spec, prefix):
"""Arguments to pass to bdist_wininst."""
return []
def upload(self, spec, prefix):
"""Upload binary package to PyPI."""
args = self.upload_args(spec, prefix)
self.setup_py('upload', *args)
def upload_args(self, spec, prefix):
"""Arguments to pass to upload."""
return []
def check(self, spec, prefix):
"""Perform some checks on the package."""
args = self.check_args(spec, prefix)
self.setup_py('check', *args)
def check_args(self, spec, prefix):
"""Arguments to pass to check."""
return []
# Testing
def test(self):
"""Run unit tests after in-place build.
These tests are only run if the package actually has a 'test' command.
"""
if self._setup_command_available('test'):
args = self.test_args(self.spec, self.prefix)
self.setup_py('test', *args)
def test_args(self, spec, prefix):
"""Arguments to pass to test."""
return []
run_after('build')(PackageBase._run_default_build_time_test_callbacks)
def import_module_test(self):
"""Attempts to import the module that was just installed.
This test is only run if the package overrides
:py:attr:`import_modules` with a list of module names."""
# Make sure we are importing the installed modules,
# not the ones in the current directory
with working_dir('..'):
for module in self.import_modules:
self.python('-c', 'import {0}'.format(module))
run_after('install')(PackageBase._run_default_install_time_test_callbacks)
# Check that self.prefix is there after installation
run_after('install')(PackageBase.sanity_check_prefix)

563
lib/spack/docs/tutorial_advanced_packaging.rst Normal file
View File

@@ -0,0 +1,563 @@
.. _advanced-packaging-tutorial:
============================
Advanced Topics in Packaging
============================
While you can quickly accomplish most common tasks with what
was covered in :ref:`packaging-tutorial`, there are times when such
knowledge won't suffice. Usually this happens for libraries that provide
more than one API and need to let dependents decide which one to use
or for packages that provide tools that are invoked at build-time,
or in other similar situations.
In the following we'll dig into some of the details of package
implementation that help us deal with these rare, but important,
occurrences. You can rest assured that in every case Spack remains faithful to
its philosophy: keep simple things simple, but be flexible enough when
complex requests arise!
----------------------
Setup for the tutorial
----------------------
The simplest way to follow along with this tutorial is to use our Docker image,
which comes with Spack and various packages pre-installed:
.. code-block:: console
$ docker pull alalazo/spack:advanced_packaging_tutorial
$ docker run --rm -h advanced-packaging-tutorial -it alalazo/spack:advanced_packaging_tutorial
root@advanced-packaging-tutorial:/#
root@advanced-packaging-tutorial:/# spack find
==> 20 installed packages.
-- linux-ubuntu16.04-x86_64 / gcc@5.4.0 -------------------------
arpack-ng@3.5.0 hdf5@1.10.1 libpciaccess@0.13.5 libtool@2.4.6 m4@1.4.18 ncurses@6.0 openblas@0.2.20 openssl@1.0.2k superlu@5.2.1 xz@5.2.3
cmake@3.9.4 hwloc@1.11.8 libsigsegv@2.11 libxml2@2.9.4 mpich@3.2 netlib-lapack@3.6.1 openmpi@3.0.0 pkg-config@0.29.2 util-macros@1.19.1 zlib@1.2.11
If you already started the image, you can set the ``EDITOR`` environment
variable to your preferred editor (``vi``, ``emacs``, and ``nano`` are included in the image)
and move directly to :ref:`specs_build_interface_tutorial`.
If you choose not to use the Docker image, you can clone the Spack repository
and build the necessary bits yourself:
.. code-block:: console
$ git clone https://github.com/spack/spack.git
Cloning into 'spack'...
remote: Counting objects: 92731, done.
remote: Compressing objects: 100% (1108/1108), done.
remote: Total 92731 (delta 1964), reused 4186 (delta 1637), pack-reused 87932
Receiving objects: 100% (92731/92731), 33.31 MiB | 64.00 KiB/s, done.
Resolving deltas: 100% (43557/43557), done.
Checking connectivity... done.
$ cd spack
$ git checkout tutorials/advanced_packaging
Branch tutorials/advanced_packaging set up to track remote branch tutorials/advanced_packaging from origin.
Switched to a new branch 'tutorials/advanced_packaging'
At this point you can install the software that will be used
during the rest of the tutorial (the output of the commands is omitted
for the sake of brevity):
.. code-block:: console
$ spack install openblas
$ spack install netlib-lapack
$ spack install mpich
$ spack install openmpi
$ spack install --only=dependencies armadillo ^openblas
$ spack install --only=dependencies netcdf
$ spack install --only=dependencies elpa
Now, you are ready to set your preferred ``EDITOR`` and continue with
the rest of the tutorial.
.. _specs_build_interface_tutorial:
----------------------
Spec's build interface
----------------------
Spack is designed with an emphasis on assigning responsibilities
to the appropriate entities, as this results in a clearer and more intuitive interface
for the users.
When it comes to packaging, one of the most fundamental guideline that
emerged from this tenet is that:
*It is a package's responsibility to know
every software it directly depends on and to expose to others how to
use the services it provides*.
Spec's build interface is a protocol-like implementation of this guideline
that allows packages to easily query their dependencies,
and prescribes how they should expose their own build information.
^^^^^^^^^^^^^^^^^^^^
A motivating example
^^^^^^^^^^^^^^^^^^^^
We've started work on a package for ``armadillo``. You should open it,
read through the comment that starts with ``# TUTORIAL:`` and complete
the ``cmake_args`` section:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack edit armadillo
If you followed the instructions in the package, when you are finished your
``cmake_args`` method should look like:
.. code-block:: python
def cmake_args(self):
spec = self.spec
return [
# ARPACK support
'-DARPACK_LIBRARY={0}'.format(spec['arpack-ng'].libs.joined(";")),
# BLAS support
'-DBLAS_LIBRARY={0}'.format(spec['blas'].libs.joined(";")),
# LAPACK support
'-DLAPACK_LIBRARY={0}'.format(spec['lapack'].libs.joined(";")),
# SuperLU support
'-DSuperLU_INCLUDE_DIR={0}'.format(spec['superlu'].prefix.include),
'-DSuperLU_LIBRARY={0}'.format(spec['superlu'].libs.joined(";")),
# HDF5 support
'-DDETECT_HDF5={0}'.format('ON' if '+hdf5' in spec else 'OFF')
]
As you can see, getting the list of libraries that your dependencies provide
is as easy as accessing the their ``libs`` attribute. Furthermore, the interface
remains the same whether you are querying regular or virtual dependencies.
At this point you can complete the installation of ``armadillo`` using ``openblas``
as a LAPACK provider:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack install armadillo ^openblas
==> pkg-config is already installed in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/pkg-config-0.29.2-ae2hwm7q57byfbxtymts55xppqwk7ecj
...
==> superlu is already installed in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/superlu-5.2.1-q2mbtw2wo4kpzis2e2n227ip2fquxrno
==> Installing armadillo
==> Using cached archive: /usr/local/var/spack/cache/armadillo/armadillo-8.100.1.tar.xz
==> Staging archive: /usr/local/var/spack/stage/armadillo-8.100.1-n2eojtazxbku6g4l5izucwwgnpwz77r4/armadillo-8.100.1.tar.xz
==> Created stage in /usr/local/var/spack/stage/armadillo-8.100.1-n2eojtazxbku6g4l5izucwwgnpwz77r4
==> Applied patch undef_linux.patch
==> Building armadillo [CMakePackage]
==> Executing phase: 'cmake'
==> Executing phase: 'build'
==> Executing phase: 'install'
==> Successfully installed armadillo
Fetch: 0.01s. Build: 3.96s. Total: 3.98s.
[+] /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/armadillo-8.100.1-n2eojtazxbku6g4l5izucwwgnpwz77r4
Hopefully the installation went fine and the code we added expanded to the right list
of semicolon separated libraries (you are encouraged to open ``armadillo``'s
build logs to double check).
If we try to build another version tied to ``netlib-lapack`` we'll
notice that this time the installation won't complete:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack install armadillo ^netlib-lapack
==> pkg-config is already installed in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/pkg-config-0.29.2-ae2hwm7q57byfbxtymts55xppqwk7ecj
...
==> openmpi is already installed in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/openmpi-3.0.0-yo5qkfvumpmgmvlbalqcadu46j5bd52f
==> Installing arpack-ng
==> Using cached archive: /usr/local/var/spack/cache/arpack-ng/arpack-ng-3.5.0.tar.gz
==> Already staged arpack-ng-3.5.0-bloz7cqirpdxj33pg7uj32zs5likz2un in /usr/local/var/spack/stage/arpack-ng-3.5.0-bloz7cqirpdxj33pg7uj32zs5likz2un
==> No patches needed for arpack-ng
==> Building arpack-ng [Package]
==> Executing phase: 'install'
==> Error: RuntimeError: Unable to recursively locate netlib-lapack libraries in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/netlib-lapack-3.6.1-jjfe23wgt7nkjnp2adeklhseg3ftpx6z
RuntimeError: RuntimeError: Unable to recursively locate netlib-lapack libraries in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/netlib-lapack-3.6.1-jjfe23wgt7nkjnp2adeklhseg3ftpx6z
/usr/local/var/spack/repos/builtin/packages/arpack-ng/package.py:105, in install:
5 options.append('-DCMAKE_INSTALL_NAME_DIR:PATH=%s/lib' % prefix)
6
7 # Make sure we use Spack's blas/lapack:
>> 8 lapack_libs = spec['lapack'].libs.joined(';')
9 blas_libs = spec['blas'].libs.joined(';')
10
11 options.extend([
See build log for details:
/usr/local/var/spack/stage/arpack-ng-3.5.0-bloz7cqirpdxj33pg7uj32zs5likz2un/arpack-ng-3.5.0/spack-build.out
This is because ``netlib-lapack`` requires extra work, compared to ``openblas``,
to expose its build information to other packages. Let's edit it:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack edit netlib-lapack
and follow the instructions in the ``# TUTORIAL:`` comment as before.
What we need to implement is:
.. code-block:: python
@property
def lapack_libs(self):
shared = True if '+shared' in self.spec else False
return find_libraries(
'liblapack', root=self.prefix, shared=shared, recurse=True
)
i.e. a property that returns the correct list of libraries for the LAPACK interface.
Now we can finally install ``armadillo ^netlib-lapack``:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack install armadillo ^netlib-lapack
...
==> Building armadillo [CMakePackage]
==> Executing phase: 'cmake'
==> Executing phase: 'build'
==> Executing phase: 'install'
==> Successfully installed armadillo
Fetch: 0.01s. Build: 3.75s. Total: 3.76s.
[+] /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/armadillo-8.100.1-sxmpu5an4dshnhickh6ykchyfda7jpyn
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
What happens at subscript time?
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
The example above leaves us with a few questions. How could it be that the
attribute:
.. code-block:: python
spec['lapack'].libs
stems from a property of the ``netlib-lapack`` package that has a different name?
How is it even computed for ``openblas``, given that in its package there's no code
that deals with finding libraries?
The answer is that ``libs`` is one of the few properties of specs that follow the
*build-interface protocol*. The others are currently ``command`` and ``headers``.
These properties exist only on concrete specs that have been retrieved via the
subscript notation.
What happens is that, whenever you retrieve a spec using subscripts:
.. code-block:: python
lapack = spec['lapack']
the key that appears in the query (in this case ``'lapack'``) is attached to the
returned item. When, later on, you access any of the build-interface attributes, this
key is used to compute the result according to the following algorithm:
.. code-block:: none
Given any pair of <query-key> and <build-attribute>:
1. If <query-key> is the name of a virtual spec and the package
providing it has an attribute named '<query-key>_<build-attribute>'
return it
2. Otherwise if the package has an attribute named '<build-attribute>'
return that
3. Otherwise use the default handler for <build-attribute>
Going back to our concrete case this means that, if the spec providing LAPACK
is ``netlib-lapack``, we are returning the value computed in the ``lapack_libs``
property. If it is ``openblas``, we are instead resorting to the default handler
for ``libs`` (which searches for the presence of ``libopenblas`` in the
installation prefix).
.. note::
Types commonly returned by build-interface attributes
Even though there's no enforcement on it, the type of the objects returned most often when
asking for the ``libs`` attributes is :py:class:`LibraryList <llnl.util.filesystem.LibraryList>`.
Similarly the usual type returned for ``headers`` is :py:class:`HeaderList <llnl.util.filesystem.HeaderList>`,
while for ``command`` is :py:class:`Executable <spack.util.executable.Executable>`. You can refer to
these objects' API documentation to discover more about them.
^^^^^^^^^^^^^^^^^^^^^^^
Extra query parameters
^^^^^^^^^^^^^^^^^^^^^^^
An advanced feature of the Spec's build-interface protocol is the support
for extra parameters after the subscript key. In fact, any of the keys used in the query
can be followed by a comma separated list of extra parameters which can be
inspected by the package receiving the request to fine-tune a response.
Let's look at an example and try to install ``netcdf``:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack install netcdf
==> libsigsegv is already installed in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/libsigsegv-2.11-fypapcprssrj3nstp6njprskeyynsgaz
==> m4 is already installed in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/m4-1.4.18-r5envx3kqctwwflhd4qax4ahqtt6x43a
...
==> Error: AttributeError: 'list' object has no attribute 'search_flags'
AttributeError: AttributeError: 'list' object has no attribute 'search_flags'
/usr/local/var/spack/repos/builtin/packages/netcdf/package.py:207, in configure_args:
50 # used instead.
51 hdf5_hl = self.spec['hdf5:hl']
52 CPPFLAGS.append(hdf5_hl.headers.cpp_flags)
>> 53 LDFLAGS.append(hdf5_hl.libs.search_flags)
54
55 if '+parallel-netcdf' in self.spec:
56 config_args.append('--enable-pnetcdf')
See build log for details:
/usr/local/var/spack/stage/netcdf-4.4.1.1-gk2xxhbqijnrdwicawawcll4t3c7dvoj/netcdf-4.4.1.1/spack-build.out
We can see from the error that ``netcdf`` needs to know how to link the *high-level interface*
of ``hdf5``, and thus passes the extra parameter ``hl`` after the request to retrieve it.
Clearly the implementation in the ``hdf5`` package is not complete, and we need to fix it:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack edit hdf5
If you followed the instructions correctly, the code added to the
``lib`` property should be similar to:
.. code-block:: python
:emphasize-lines: 1
query_parameters = self.spec.last_query.extra_parameters
key = tuple(sorted(query_parameters))
libraries = query2libraries[key]
shared = '+shared' in self.spec
return find_libraries(
libraries, root=self.prefix, shared=shared, recurse=True
)
where we highlighted the line retrieving the extra parameters. Now we can successfully
complete the installation of ``netcdf``:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack install netcdf
==> libsigsegv is already installed in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/libsigsegv-2.11-fypapcprssrj3nstp6njprskeyynsgaz
==> m4 is already installed in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/m4-1.4.18-r5envx3kqctwwflhd4qax4ahqtt6x43a
...
==> Installing netcdf
==> Using cached archive: /usr/local/var/spack/cache/netcdf/netcdf-4.4.1.1.tar.gz
==> Already staged netcdf-4.4.1.1-gk2xxhbqijnrdwicawawcll4t3c7dvoj in /usr/local/var/spack/stage/netcdf-4.4.1.1-gk2xxhbqijnrdwicawawcll4t3c7dvoj
==> Already patched netcdf
==> Building netcdf [AutotoolsPackage]
==> Executing phase: 'autoreconf'
==> Executing phase: 'configure'
==> Executing phase: 'build'
==> Executing phase: 'install'
==> Successfully installed netcdf
Fetch: 0.01s. Build: 24.61s. Total: 24.62s.
[+] /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/netcdf-4.4.1.1-gk2xxhbqijnrdwicawawcll4t3c7dvoj
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Single package providing multiple virtual specs
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
At the close of this tutorial's subsection, it may be useful to see where the
build-interface protocol shines the most i.e. when it comes to manage packages
that provide more than one virtual spec. An example of a package of this kind is
``intel-parallel-studio``, and due to its complexity we'll limit our discussion
here to just a few considerations (without any hands-on). You can open
the related ``package.py`` in the usual way:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack edit intel-parallel-studio
As you can see this package provides a lot of virtual specs, and thus it has
more than one function that enters into the build-interface protocol. These
functions will be invoked for *exactly the same spec* according to the key used
by its dependents in the subscript query.
So, for instance, the ``blas_libs`` property will be returned when
``intel-parallel-studio`` is the BLAS provider in the current DAG and
is retrieved by a dependent with:
.. code-block:: python
blas = self.spec['blas']
blas_libs = blas.libs
Within the property we inspect various aspects of the current spec:
.. code-block:: python
@property
def blas_libs(self):
spec = self.spec
prefix = self.prefix
shared = '+shared' in spec
if '+ilp64' in spec:
mkl_integer = ['libmkl_intel_ilp64']
else:
mkl_integer = ['libmkl_intel_lp64']
...
and construct the list of library we need to return accordingly.
What we achieved is that the complexity of dealing with ``intel-parallel-studio``
is now gathered in the package itself, instead of being spread
all over its possible dependents.
Thus, a package that uses MPI or LAPACK doesn't care which implementation it uses,
as each virtual dependency has
*a uniform interface* to ask for libraries or headers and manipulate them.
The packages that provide this virtual spec, on the other hand, have a clear
way to differentiate their answer to the query [#uniforminterface]_.
.. [#uniforminterface] Before this interface was added, each package that
depended on MPI or LAPACK had dozens of lines of code copied from other
packages telling it where to find the libraries and what they are called.
With the addition of this interface, the virtual dependency itself tells
other packages that depend on it where it can find its libraries.
---------------------------
Package's build environment
---------------------------
Besides Spec's build interface, Spack provides means to set environment
variables, either for yourself or for your dependent packages, and to
attach attributes to your dependents. We'll see them next with the help
of a few real use cases.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Set variables at build-time for yourself
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Spack provides a way to manipulate a package's build time and
run time environments using the
:py:func:`setup_environment <spack.package.PackageBase.setup_environment>` function.
Let's try to see how it works by completing the ``elpa`` package:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack edit elpa
In the end your method should look like:
.. code-block:: python
def setup_environment(self, spack_env, run_env):
spec = self.spec
spack_env.set('CC', spec['mpi'].mpicc)
spack_env.set('FC', spec['mpi'].mpifc)
spack_env.set('CXX', spec['mpi'].mpicxx)
spack_env.set('SCALAPACK_LDFLAGS', spec['scalapack'].libs.joined())
spack_env.append_flags('LDFLAGS', spec['lapack'].libs.search_flags)
spack_env.append_flags('LIBS', spec['lapack'].libs.link_flags)
The two arguments, ``spack_env`` and ``run_env``, are both instances of
:py:class:`EnvironmentModifications <spack.environment.EnvironmentModifications>` and
permit you to register modifications to either the build-time or the run-time
environment of the package, respectively.
At this point it's possible to proceed with the installation of ``elpa``:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack install elpa
==> pkg-config is already installed in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/pkg-config-0.29.2-ae2hwm7q57byfbxtymts55xppqwk7ecj
==> ncurses is already installed in /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/ncurses-6.0-ukq4tccptm2rxd56d2bumqthnpcjzlez
...
==> Executing phase: 'build'
==> Executing phase: 'install'
==> Successfully installed elpa
Fetch: 3.94s. Build: 41.93s. Total: 45.87s.
[+] /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/elpa-2016.05.004-sdbfhwcexg7s2zqf52vssb762ocvklbu
If you had modifications to ``run_env``, those would have appeared e.g. in the module files
generated for the package.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Set variables in dependencies at build-time
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Another common occurrence, particularly for packages like ``r`` and ``python``
that support extensions and for packages that provide build tools,
is to require *their dependents* to have some environment variables set.
The mechanism is similar to what we just saw, except that we override the
:py:func:`setup_dependent_environment <spack.package.PackageBase.setup_dependent_environment>`
function, which takes one additional argument, i.e. the dependent spec that needs the modified
environment. Let's practice completing the ``mpich`` package:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack edit mpich
Once you're finished the method should look like this:
.. code-block:: python
def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
spack_env.set('MPICC', join_path(self.prefix.bin, 'mpicc'))
spack_env.set('MPICXX', join_path(self.prefix.bin, 'mpic++'))
spack_env.set('MPIF77', join_path(self.prefix.bin, 'mpif77'))
spack_env.set('MPIF90', join_path(self.prefix.bin, 'mpif90'))
spack_env.set('MPICH_CC', spack_cc)
spack_env.set('MPICH_CXX', spack_cxx)
spack_env.set('MPICH_F77', spack_f77)
spack_env.set('MPICH_F90', spack_fc)
spack_env.set('MPICH_FC', spack_fc)
At this point we can, for instance, install ``netlib-scalapack``:
.. code-block:: console
root@advanced-packaging-tutorial:/# spack install netlib-scalapack ^mpich
...
==> Created stage in /usr/local/var/spack/stage/netlib-scalapack-2.0.2-km7tsbgoyyywonyejkjoojskhc5knz3z
==> No patches needed for netlib-scalapack
==> Building netlib-scalapack [CMakePackage]
==> Executing phase: 'cmake'
==> Executing phase: 'build'
==> Executing phase: 'install'
==> Successfully installed netlib-scalapack
Fetch: 0.01s. Build: 3m 59.86s. Total: 3m 59.87s.
[+] /usr/local/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/netlib-scalapack-2.0.2-km7tsbgoyyywonyejkjoojskhc5knz3z
and double check the environment logs to verify that every variable was
set to the correct value. More complicated examples of the use of this function
may be found in the ``r`` and ``python`` package.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Attach attributes to other packages
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Build tools usually also provide a set of executables that can be used
when another package is being installed. Spack gives the opportunity
to monkey-patch dependent modules and attach attributes to them. This
helps make the packager experience as similar as possible to what would
have been the manual installation of the same package.
An example here is the ``automake`` package, which overrides
:py:func:`setup_dependent_package <spack.package.PackageBase.setup_dependent_package>`:
.. code-block:: python
def setup_dependent_package(self, module, dependent_spec):
# Automake is very likely to be a build dependency,
# so we add the tools it provides to the dependent module
executables = ['aclocal', 'automake']
for name in executables:
setattr(module, name, self._make_executable(name))
so that every other package that depends on it can use directly ``aclocal``
and ``automake`` with the usual function call syntax of :py:class:`Executable <spack.util.executable.Executable>`:
.. code-block:: python
aclocal('--force')

1747
lib/spack/docs/tutorial_basics.rst Normal file
View File

File diff suppressed because it is too large Load Diff

795
lib/spack/docs/tutorial_buildsystems.rst Normal file
View File

@@ -0,0 +1,795 @@
.. _build-systems-tutorial:
==============================
Spack Package Build Systems
==============================
You may begin to notice after writing a couple of package template files a
pattern emerge for some packages. For example, you may find yourself writing
an :code:`install()` method that invokes: :code:`configure`, :code:`cmake`,
:code:`make`, :code:`make install`. You may also find yourself writing
:code:`"prefix=" + prefix` as an argument to :code:`configure` or :code:`cmake`.
Rather than having you repeat these lines for all packages, Spack has
classes that can take care of these patterns. In addition,
these package files allow for finer grained control of these build systems.
In this section, we will describe each build system and give examples on
how these can be manipulated to install a package.
-----------------------
Package Class Hierarchy
-----------------------
.. graphviz::
digraph G {
node [
shape = "record"
]
edge [
arrowhead = "empty"
]
PackageBase -> Package [dir=back]
PackageBase -> MakefilePackage [dir=back]
PackageBase -> AutotoolsPackage [dir=back]
PackageBase -> CMakePackage [dir=back]
PackageBase -> PythonPackage [dir=back]
}
The above diagram gives a high level view of the class hierarchy and how each
package relates. Each subclass inherits from the :code:`PackageBaseClass`
super class. The bulk of the work is done in this super class which includes
fetching, extracting to a staging directory and installing. Each subclass
then adds additional build-system-specific functionality. In the following
sections, we will go over examples of how to utilize each subclass and to see
how powerful these abstractions are when packaging.
-----------------
Package
-----------------
We've already seen examples of a :code:`Package` class in our walkthrough for writing
package files, so we won't be spending much time with them here. Briefly,
the Package class allows for abitrary control over the build process, whereas
subclasses rely on certain patterns (e.g. :code:`configure` :code:`make`
:code:`make install`) to be useful. :code:`Package` classes are particularly useful
for packages that have a non-conventional way of being built since the packager
can utilize some of Spack's helper functions to customize the building and
installing of a package.
-------------------
Autotools
-------------------
As we have seen earlier, packages using :code:`Autotools` use :code:`configure`,
:code:`make` and :code:`make install` commands to execute the build and
install process. In our :code:`Package` class, your typical build incantation will
consist of the following:
.. code-block:: python
def install(self, spec, prefix):
configure("--prefix=" + prefix)
make()
make("install")
You'll see that this looks similar to what we wrote in our packaging tutorial.
The :code:`Autotools` subclass aims to simplify writing package files and provides
convenience methods to manipulate each of the different phases for a :code:`Autotools`
build system.
:code:`Autotools` packages consist of four phases:
1. :code:`autoreconf()`
2. :code:`configure()`
3. :code:`build()`
4. :code:`install()`
Each of these phases have sensible defaults. Let's take a quick look at some
the internals of the :code:`Autotools` class:
.. code-block:: console
$ spack edit --build-system autotools
This will open the :code:`AutotoolsPackage` file in your text editor.
.. note::
The examples showing code for these classes is abridged to avoid having
long examples. We only show what is relevant to the packager.
.. literalinclude:: tutorial/examples/Autotools/autotools_class.py
:language: python
:emphasize-lines: 42,45,62
:lines: 40-95,259-267
:linenos:
Important to note are the highlighted lines. These properties allow the
packager to set what build targets and install targets they want for their
package. If, for example, we wanted to add as our build target :code:`foo`
then we can append to our :code:`build_targets` property:
.. code-block:: python
build_targets = ["foo"]
Which is similiar to invoking make in our Package
.. code-block:: python
make("foo")
This is useful if we have packages that ignore environment variables and need
a command-line argument.
Another thing to take note of is in the :code:`configure()` method.
Here we see that the :code:`prefix` argument is already included since it is a
common pattern amongst packages using :code:`Autotools`. We then only have to
override :code:`configure_args()`, which will then return it's output to
to :code:`configure()`. Then, :code:`configure()` will append the common
arguments
Packagers also have the option to run :code:`autoreconf` in case a package
needs to update the build system and generate a new :code:`configure`. Though,
for the most part this will be unnecessary.
Let's look at the :code:`mpileaks` package.py file that we worked on earlier:
.. code-block:: console
$ spack edit mpileaks
Notice that mpileaks is a :code:`Package` class but uses the :code:`Autotools`
build system. Although this package is acceptable let's make this into an
:code:`AutotoolsPackage` class and simplify it further.
.. literalinclude:: tutorial/examples/Autotools/0.package.py
:language: python
:emphasize-lines: 28
:linenos:
We first inherit from the :code:`AutotoolsPackage` class.
Although we could keep the :code:`install()` method, most of it can be handled
by the :code:`AutotoolsPackage` base class. In fact, the only thing that needs
to be overridden is :code:`configure_args()`.
.. literalinclude:: tutorial/examples/Autotools/1.package.py
:language: python
:emphasize-lines: 42,43
:linenos:
Since Spack takes care of setting the prefix for us we can exclude that as
an argument to :code:`configure`. Our packages look simpler, and the packager
does not need to worry about whether they have properly included :code:`configure`
and :code:`make`.
This version of the :code:`mpileaks` package installs the same as the previous,
but the :code:`AutotoolsPackage` class lets us do it with a cleaner looking
package file.
-----------------
Makefile
-----------------
Packages that utilize :code:`Make` or a :code:`Makefile` usually require you
to edit a :code:`Makefile` to set up platform and compiler specific variables.
These packages are handled by the :code:`Makefile` subclass which provides
convenience methods to help write these types of packages.
A :code:`MakefilePackage` class has three phases that can be overridden. These include:
1. :code:`edit()`
2. :code:`build()`
3. :code:`install()`
Packagers then have the ability to control how a :code:`Makefile` is edited, and
what targets to include for the build phase or install phase.
Let's also take a look inside the :code:`MakefilePackage` class:
.. code-block:: console
$ spack edit --build-system makefile
Take note of the following:
.. literalinclude:: tutorial/examples/Makefile/makefile_class.py
:language: python
:lines: 33-79,89-107
:emphasize-lines: 48,54,61
:linenos:
Similar to :code:`Autotools`, :code:`MakefilePackage` class has properties
that can be set by the packager. We can also override the different
methods highlighted.
Let's try to recreate the Bowtie_ package:
.. _Bowtie: http://bowtie-bio.sourceforge.net/index.shtml
.. code-block:: console
$ spack create -f https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip
==> This looks like a URL for bowtie
==> Found 1 version of bowtie:
1.2.1.1 https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip
==> How many would you like to checksum? (default is 1, q to abort) 1
==> Downloading...
==> Fetching https://downloads.sourceforge.net/project/bowtie-bio/bowtie/1.2.1.1/bowtie-1.2.1.1-src.zip
######################################################################## 100.0%
==> Checksummed 1 version of bowtie
==> This package looks like it uses the makefile build system
==> Created template for bowtie package
==> Created package file: /Users/mamelara/spack/var/spack/repos/builtin/packages/bowtie/package.py
Once the fetching is completed, Spack will open up your text editor in the
usual fashion and create a template of a :code:`MakefilePackage` package.py.
.. literalinclude:: tutorial/examples/Makefile/0.package.py
:language: python
:linenos:
Spack was successfully able to detect that :code:`Bowtie` uses :code:`Make`.
Let's add in the rest of our details for our package:
.. literalinclude:: tutorial/examples/Makefile/1.package.py
:language: python
:emphasize-lines: 29,30,32,33,37,39
:linenos:
As we mentioned earlier, most packages using a :code:`Makefile` have hard-coded
variables that must be edited. These variables are fine if you happen to not
care about setup or types of compilers used but Spack is designed to work with
any compiler. The :code:`MakefilePackage` subclass makes it easy to edit
these :code:`Makefiles` by having an :code:`edit()` method that
can be overridden.
Let's take a look at the default :code:`Makefile` that :code:`Bowtie` provides.
If we look inside, we see that :code:`CC` and :code:`CXX` point to our GNU
compiler:
.. code-block:: console
$ spack stage bowtie
.. note::
As usual make sure you have shell support activated with spack:
:code:`source /path/to/spack_root/spack/share/spack/setup-env.sh`
.. code-block:: console
$ spack cd -s bowtie
$ cd bowtie-1.2
$ vim Makefile
.. code-block:: make
CPP = g++ -w
CXX = $(CPP)
CC = gcc
LIBS = $(LDFLAGS) -lz
HEADERS = $(wildcard *.h)
To fix this, we need to use the :code:`edit()` method to write our custom
:code:`Makefile`.
.. literalinclude:: tutorial/examples/Makefile/2.package.py
:language: python
:emphasize-lines: 42,43,44
:linenos:
Here we use a :code:`FileFilter` object to edit our :code:`Makefile`. It takes
in a regular expression and then replaces :code:`CC` and :code:`CXX` to whatever
Spack sets :code:`CC` and :code:`CXX` environment variables to. This allows us to
build :code:`Bowtie` with whatever compiler we specify through Spack's
:code:`spec` syntax.
Let's change the build and install phases of our package:
.. literalinclude:: tutorial/examples/Makefile/3.package.py
:language: python
:emphasize-lines: 46, 52
:linenos:
Here demonstrate another strategy that we can use to manipulate our package
We can provide command-line arguments to :code:`make()`. Since :code:`Bowtie`
can use :code:`tbb` we can either add :code:`NO_TBB=1` as a argument to prevent
:code:`tbb` support or we can just invoke :code:`make` with no arguments.
:code:`Bowtie` requires our :code:`install_target` to provide a path to
the install directory. We can do this by providing :code:`prefix=` as a command
line argument to :code:`make()`.
Let's look at a couple of other examples and go through them:
.. code-block:: console
$ spack edit cbench
Some packages allow environment variables to be set and will honor them.
Packages that use :code:`?=` for assignment in their :code:`Makefile`
can be set using environment variables. In our :code:`cbench` example we
set two environment variables in our :code:`edit()` method:
.. code-block:: python
def edit(self, spec, prefix):
# The location of the Cbench source tree
env['CBENCHHOME'] = self.stage.source_path
# The location that will contain all your tests and your results
env['CBENCHTEST'] = prefix
# ... more code
As you may have noticed, we didn't really write anything to the :code:`Makefile`
but rather we set environment variables that will override variables set in
the :code:`Makefile`.
Some packages include a configuration file that sets certain compiler variables,
platform specific variables, and the location of dependencies or libraries.
If the file is simple and only requires a couple of changes, we can overwrite
those entries with our :code:`FileFilter` object. If the configuration involves
complex changes, we can write a new configuration file from scratch.
Let's look at an example of this in the :code:`elk` package:
.. code-block:: console
$ spack edit elk
.. code-block:: python
def edit(self, spec, prefix):
# Dictionary of configuration options
config = {
'MAKE': 'make',
'AR': 'ar'
}
# Compiler-specific flags
flags = ''
if self.compiler.name == 'intel':
flags = '-O3 -ip -unroll -no-prec-div'
elif self.compiler.name == 'gcc':
flags = '-O3 -ffast-math -funroll-loops'
elif self.compiler.name == 'pgi':
flags = '-O3 -lpthread'
elif self.compiler.name == 'g95':
flags = '-O3 -fno-second-underscore'
elif self.compiler.name == 'nag':
flags = '-O4 -kind=byte -dusty -dcfuns'
elif self.compiler.name == 'xl':
flags = '-O3'
config['F90_OPTS'] = flags
config['F77_OPTS'] = flags
# BLAS/LAPACK support
# Note: BLAS/LAPACK must be compiled with OpenMP support
# if the +openmp variant is chosen
blas = 'blas.a'
lapack = 'lapack.a'
if '+blas' in spec:
blas = spec['blas'].libs.joined()
if '+lapack' in spec:
lapack = spec['lapack'].libs.joined()
# lapack must come before blas
config['LIB_LPK'] = ' '.join([lapack, blas])
# FFT support
if '+fft' in spec:
config['LIB_FFT'] = join_path(spec['fftw'].prefix.lib,
'libfftw3.so')
config['SRC_FFT'] = 'zfftifc_fftw.f90'
else:
config['LIB_FFT'] = 'fftlib.a'
config['SRC_FFT'] = 'zfftifc.f90'
# MPI support
if '+mpi' in spec:
config['F90'] = spec['mpi'].mpifc
config['F77'] = spec['mpi'].mpif77
else:
config['F90'] = spack_fc
config['F77'] = spack_f77
config['SRC_MPI'] = 'mpi_stub.f90'
# OpenMP support
if '+openmp' in spec:
config['F90_OPTS'] += ' ' + self.compiler.openmp_flag
config['F77_OPTS'] += ' ' + self.compiler.openmp_flag
else:
config['SRC_OMP'] = 'omp_stub.f90'
# Libxc support
if '+libxc' in spec:
config['LIB_libxc'] = ' '.join([
join_path(spec['libxc'].prefix.lib, 'libxcf90.so'),
join_path(spec['libxc'].prefix.lib, 'libxc.so')
])
config['SRC_libxc'] = ' '.join([
'libxc_funcs.f90',
'libxc.f90',
'libxcifc.f90'
])
else:
config['SRC_libxc'] = 'libxcifc_stub.f90'
# Write configuration options to include file
with open('make.inc', 'w') as inc:
for key in config:
inc.write('{0} = {1}\n'.format(key, config[key]))
:code:`config` is just a dictionary that we can add key-value pairs to. By the
end of the :code:`edit()` method we write the contents of our dictionary to
:code:`make.inc`.
---------------
CMake
---------------
CMake_ is another common build system that has been gaining popularity. It works
in a similar manner to :code:`Autotools` but with differences in variable names,
the number of configuration options available, and the handling of shared libraries.
Typical build incantations look like this:
.. _CMake: https://cmake.org
.. code-block:: python
def install(self, spec, prefix):
cmake("-DCMAKE_INSTALL_PREFIX:PATH=/path/to/install_dir ..")
make()
make("install")
As you can see from the example above, it's very similar to invoking
:code:`configure` and :code:`make` in an :code:`Autotools` build system. However,
the variable names and options differ. Most options in CMake are prefixed
with a :code:`'-D'` flag to indicate a configuration setting.
In the :code:`CMakePackage` class we can override the following phases:
1. :code:`cmake()`
2. :code:`build()`
3. :code:`install()`
The :code:`CMakePackage` class also provides sensible defaults so we only need to
override :code:`cmake_args()`.
Let's look at these defaults in the :code:`CMakePackage` class:
.. code-block:: console
$ spack edit --build-system cmake
And go into a bit of detail on the highlighted sections:
.. literalinclude:: tutorial/examples/Cmake/cmake_class.py
:language: python
:lines: 37-92, 94-155, 174-211
:emphasize-lines: 57,68,86,94,96,99,100,101,102,111,117,135,136
:linenos:
Some :code:`CMake` packages use different generators. Spack is able to support
Unix-Makefile_ generators as well as Ninja_ generators.
.. _Unix-Makefile: https://cmake.org/cmake/help/v3.4/generator/Unix%20Makefiles.html
.. _Ninja: https://cmake.org/cmake/help/v3.4/generator/Ninja.html
Default generator is :code:`Unix Makefile`.
Next we setup the build type. In :code:`CMake` you can specify the build type
that you want. Options include:
1. empty
2. Debug
3. Release
4. RelWithDebInfo
5. MinSizeRel
With these options you can specify whether you want your executable to have
the debug version only, release version or the release with debug information.
Release executables tend to be more optimized than Debug. In Spack, we set
the default as RelWithDebInfo unless otherwise specified through a variant.
Spack then automatically sets up the :code:`-DCMAKE_INSTALL_PREFIX` path,
appends the build type (RelDebInfo default), and then specifies a verbose
:code:`Makefile`.
Next we add the :code:`rpaths` to :code:`-DCMAKE_INSTALL_RPATH:STRING`.
Finally we add to :code:`-DCMAKE_PREFIX_PATH:STRING` the locations of all our
dependencies so that :code:`CMake` can find them.
In the end our :code:`cmake` line will look like this (example is :code:`xrootd`):
.. code-block:: console
$ cmake $HOME/spack/var/spack/stage/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/xrootd-4.6.0 -G Unix Makefiles -DCMAKE_INSTALL_PREFIX:PATH=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON -DCMAKE_FIND_FRAMEWORK:STRING=LAST -DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=FALSE -DCMAKE_INSTALL_RPATH:STRING=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/lib:$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/xrootd-4.6.0-4ydm74kbrp4xmcgda5upn33co5pwddyk/lib64 -DCMAKE_PREFIX_PATH:STRING=$HOME/spack/opt/spack/darwin-sierra-x86_64/clang-9.0.0-apple/cmake-3.9.4-hally3vnbzydiwl3skxcxcbzsscaasx5
Saves a lot of typing doesn't it?
Let's try to recreate callpath_:
.. _callpath: https://github.com/LLNL/callpath.git
.. code-block:: console
$ spack create -f https://github.com/llnl/callpath/archive/v1.0.3.tar.gz
==> This looks like a URL for callpath
==> Found 4 versions of callpath:
1.0.3 https://github.com/LLNL/callpath/archive/v1.0.3.tar.gz
1.0.2 https://github.com/LLNL/callpath/archive/v1.0.2.tar.gz
1.0.1 https://github.com/LLNL/callpath/archive/v1.0.1.tar.gz
1.0 https://github.com/LLNL/callpath/archive/v1.0.tar.gz
==> How many would you like to checksum? (default is 1, q to abort) 1
==> Downloading...
==> Fetching https://github.com/LLNL/callpath/archive/v1.0.3.tar.gz
######################################################################## 100.0%
==> Checksummed 1 version of callpath
==> This package looks like it uses the cmake build system
==> Created template for callpath package
==> Created package file: /Users/mamelara/spack/var/spack/repos/builtin/packages/callpath/package.py
which then produces the following template:
.. literalinclude:: tutorial/examples/Cmake/0.package.py
:language: python
:linenos:
Again we fill in the details:
.. literalinclude:: tutorial/examples/Cmake/1.package.py
:language: python
:linenos:
:emphasize-lines: 28,32,33,37,38,39,40,41,42
As mentioned earlier, Spack will use sensible defaults to prevent repeated code
and to make writing :code:`CMake` package files simpler.
In callpath, we want to add options to :code:`CALLPATH_WALKER` as well as add
compiler flags. We add the following options like so:
.. literalinclude:: tutorial/examples/Cmake/2.package.py
:language: python
:linenos:
:emphasize-lines: 45,49,50
Now we can control our build options using :code:`cmake_args()`. If defaults are
sufficient enough for the package, we can leave this method out.
:code:`CMakePackage` classes allow for control of other features in the
build system. For example, you can specify the path to the "out of source"
build directory and also point to the root of the :code:`CMakeLists.txt` file if it
is placed in a non-standard location.
A good example of a package that has its :code:`CMakeLists.txt` file located at a
different location is found in :code:`spades`.
.. code-block:: console
$ spack edit spade
.. code-block:: python
root_cmakelists_dir = "src"
Here :code:`root_cmakelists_dir` will tell Spack where to find the location
of :code:`CMakeLists.txt`. In this example, it is located a directory level below in
the :code:`src` directory.
Some :code:`CMake` packages also require the :code:`install` phase to be
overridden. For example, let's take a look at :code:`sniffles`.
.. code-block:: console
$ spack edit sniffles
In the :code:`install()` method, we have to manually install our targets
so we override the :code:`install()` method to do it for us:
.. code-block:: python
# the build process doesn't actually install anything, do it by hand
def install(self, spec, prefix):
mkdir(prefix.bin)
src = "bin/sniffles-core-{0}".format(spec.version.dotted)
binaries = ['sniffles', 'sniffles-debug']
for b in binaries:
install(join_path(src, b), join_path(prefix.bin, b))
--------------
PythonPackage
--------------
Python extensions and modules are built differently from source than most
applications. Python uses a :code:`setup.py` script to install Python modules.
The script consists of a call to :code:`setup()` which provides the information
required to build a module to Distutils. If you're familiar with pip or
easy_install, setup.py does the same thing.
These modules are usually installed using the following line:
.. code-block:: console
$ python setup.py install
There are also a list of commands and phases that you can call. To see the full
list you can run:
.. code-block:: console
$ python setup.py --help-commands
Standard commands:
build build everything needed to install
build_py "build" pure Python modules (copy to build directory)
build_ext build C/C++ extensions (compile/link to build directory)
build_clib build C/C++ libraries used by Python extensions
build_scripts "build" scripts (copy and fixup #! line)
clean (no description available)
install install everything from build directory
install_lib install all Python modules (extensions and pure Python)
install_headers install C/C++ header files
install_scripts install scripts (Python or otherwise)
install_data install data files
sdist create a source distribution (tarball, zip file, etc.)
register register the distribution with the Python package index
bdist create a built (binary) distribution
bdist_dumb create a "dumb" built distribution
bdist_rpm create an RPM distribution
bdist_wininst create an executable installer for MS Windows
upload upload binary package to PyPI
check perform some checks on the package
To see the defaults that Spack has for each a methods, we will take a look
at the :code:`PythonPackage` class:
.. code-block:: console
$ spack edit --build-system python
We see the following:
.. literalinclude:: tutorial/examples/PyPackage/python_package_class.py
:language: python
:lines: 35, 161-364
:linenos:
Each of these methods have sensible defaults or they can be overridden.
We can write package files for Python packages using the :code:`Package` class,
but the class brings with it a lot of methods that are useless for Python packages.
Instead, Spack has a :code: `PythonPackage` subclass that allows packagers
of Python modules to be able to invoke :code:`setup.py` and use :code:`Distutils`,
which is much more familiar to a typical python user.
We will write a package file for Pandas_:
.. _pandas: https://pandas.pydata.org
.. code-block:: console
$ spack create -f https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz
==> This looks like a URL for pandas
==> Warning: Spack was unable to fetch url list due to a certificate verification problem. You can try running spack -k, which will not check SSL certificates. Use this at your own risk.
==> Found 1 version of pandas:
0.19.0 https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz
==> How many would you like to checksum? (default is 1, q to abort) 1
==> Downloading...
==> Fetching https://pypi.io/packages/source/p/pandas/pandas-0.19.0.tar.gz
######################################################################## 100.0%
==> Checksummed 1 version of pandas
==> This package looks like it uses the python build system
==> Changing package name from pandas to py-pandas
==> Created template for py-pandas package
==> Created package file: /Users/mamelara/spack/var/spack/repos/builtin/packages/py-pandas/package.py
And we are left with the following template:
.. literalinclude:: tutorial/examples/PyPackage/0.package.py
:language: python
:linenos:
As you can see this is not any different than any package template that we have
written. We have the choice of providing build options or using the sensible
defaults
Luckily for us, there is no need to provide build args.
Next we need to find the dependencies of a package. Dependencies are usually
listed in :code:`setup.py`. You can find the dependencies by searching for
:code:`install_requires` keyword in that file. Here it is for :code:`Pandas`:
.. code-block:: python
# ... code
if sys.version_info[0] >= 3:
setuptools_kwargs = {
'zip_safe': False,
'install_requires': ['python-dateutil >= 2',
'pytz >= 2011k',
'numpy >= %s' % min_numpy_ver],
'setup_requires': ['numpy >= %s' % min_numpy_ver],
}
if not _have_setuptools:
sys.exit("need setuptools/distribute for Py3k"
"\n$ pip install distribute")
# ... more code
You can find a more comprehensive list at the Pandas documentation_.
.. _documentation: https://pandas.pydata.org/pandas-docs/stable/install.html
By reading the documentation and :code:`setup.py` we found that :code:`Pandas`
depends on :code:`python-dateutil`, :code:`pytz`, and :code:`numpy`, :code:`numexpr`,
and finally :code:`bottleneck`.
Here is the completed :code:`Pandas` script:
.. literalinclude:: tutorial/examples/PyPackage/1.package.py
:language: python
:linenos:
It is quite important to declare all the dependencies of a Python package.
Spack can "activate" Python packages to prevent the user from having to
load each dependency module explictly. If a dependency is missed, Spack will
be unable to properly activate the package and it will cause an issue. To
learn more about extensions go to :ref:`cmd-spack-extensions`.
From this example, you can see that building Python modules is made easy
through the :code:`PythonPackage` class.
-------------------
Other Build Systems
-------------------
Although we won't get in depth with any of the other build systems that Spack
supports, it is worth mentioning that Spack does provide subclasses
for the following build systems:
1. :code:`IntelPackage`
2. :code:`SconsPackage`
3. :code:`WafPackage`
4. :code:`RPackage`
5. :code:`PerlPackage`
6. :code:`QMake`
Each of these classes have their own abstractions to help assist in writing
package files. For whatever doesn't fit nicely into the other build-systems,
you can use the :code:`Package` class.
Hopefully by now you can see how we aim to make packaging simple and
robust through these classes. If you want to learn more about these build
systems, check out :ref:`installation_procedure` in the Packaging Guide.

843
lib/spack/docs/tutorial_configuration.rst Normal file
View File

@@ -0,0 +1,843 @@
.. _configs-tutorial:
======================
Configuration Tutorial
======================
This tutorial will guide you through various configuration options
that allow you to customize Spack's behavior with respect to
software installation. We will first cover the configuration file
hierarchy. Then, we will cover configuration options for compilers,
focusing on how it can be used to extend Spack's compiler auto-detection.
Next, we will cover the packages configuration file, focusing on
how it can be used to override default build options as well as
specify external package installations to use. Finally, we will
briefly touch on the config configuration file, which manages more
high-level Spack configuration options.
For all of these features we will demonstrate how we build up a full
configuration file. For some we will then demonstrate how the
configuration affects the install command, and for others we will use
the ``spack spec`` command to demonstrate how the configuration
changes have affected Spack's concretization algorithm. The provided
output is all from a server running Ubuntu version 16.04.
.. _configs-tutorial-scopes:
--------------------
Configuration Scopes
--------------------
Depending on your use case, you may want to provide configuration
settings common to everyone on your team, or you may want to set
default behaviors specific to a single user account. Spack provides
4 configuration *scopes* to handle this customization. These scopes,
in order of decreasing priority, are:
====================== ==================================
Scope Directory
====================== ==================================
User configurations ``~/.spack``
Project configurations ``$SPACK_ROOT/etc/spack``
System configurations ``/etc/spack``
Default configurations ``$SPACK_ROOT/etc/spack/defaults``
====================== ==================================
Spack's default configuration settings reside in
``$SPACK_ROOT/etc/spack/defaults``. These are useful for reference,
but should never be directly edited. To override these settings,
create new configuration files in any of the higher-priority
configuration scopes.
A particular cluster may have multiple Spack installations associated
with different projects. To provide settings common to all Spack
installations, put your configuration files in ``/etc/spack``.
To provide settings specific to a particular Spack installation,
you can use the ``$SPACK_ROOT/etc/spack`` directory.
For settings specific to a particular user, you will want to add
configuration files to the ``~/.spack`` directory. When Spack first
checked for compilers on your system, you may have noticed that it
placed your compiler configuration in this directory.
Some facilities manage multiple platforms from a single shared
filesystem. In order to handle this, each of the configuration
scopes listed above has two *sub-scopes*: platform-specific and
platform-independent. For example, compiler settings can be stored
in ``compilers.yaml`` configuration files in the following locations:
- ``~/.spack/<platform>/compilers.yaml``
- ``~/.spack/compilers.yaml``
- ``$SPACK_ROOT/etc/spack/<platform>/compilers.yaml``
- ``$SPACK_ROOT/etc/spack/compilers.yaml``
- ``/etc/spack/<platform>/compilers.yaml``
- ``/etc/spack/compilers.yaml``
- ``$SPACK_ROOT/etc/defaults/<platform>/compilers.yaml``
- ``$SPACK_ROOT/etc/defaults/compilers.yaml``
These files are listed in decreasing order of precedence, so files in
``~/.spack/<platform>`` will override settings in ``~/.spack``.
Spack configurations are YAML dictionaries. Every configuration file
begins with a top-level dictionary that tells Spack which
configuration set it modifies. When Spack checks it's configuration,
the configuration scopes are updated as dictionaries in increasing
order of precedence, allowing higher precedence files to override
lower. YAML dictionaries use a colon ":" to specify key-value
pairs. Spack extends YAML syntax slightly to allow a double-colon
"::" to specify a key-value pair. When a double-colon is used to
specify a key-value pair, instead of adding that section Spack
replaces what was in that section with the new value. For example, a
user compilers configuration file as follows:
.. code-block:: yaml
compilers::
- compiler:
environment: {}
extra_rpaths: []
flags: {}
modules: []
operating_system: ubuntu16.04
paths:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
f77: /usr/bin/gfortran
fc: /usr/bin/gfortran
spec: gcc@5.4.0
target: x86_64
ensures that no other compilers are used, as the user configuration
scope is the last scope searched and the ``compilers::`` line replaces
all previous configuration files information. If the same
configuration file had a single colon instead of the double colon, it
would add the gcc version 5.4.0 compiler to whatever other compilers
were listed in other configuration files.
.. _configs-tutorial-compilers:
----------------------
Compiler Configuration
----------------------
For most tasks, we can use Spack with the compilers auto-detected the
first time Spack runs on a system. As we discussed in the basic
installation section, we can also tell Spack where compilers are
located using the ``spack compiler add`` command. However, in some
circumstances we want even more fine-grained control over the
compilers available. This section will teach you how to exercise that
control using the compilers configuration file.
We will start by opening the compilers configuration file
.. code-block:: console
$ spack config edit compilers
.. code-block:: yaml
compilers:
- compiler:
environment: {}
extra_rpaths: []
flags: {}
modules: []
operating_system: ubuntu16.04
paths:
cc: /usr/bin/clang
cxx: /usr/bin/clang++
f77: null
fc: null
spec: clang@3.8.0-2ubuntu4
target: x86_64
- compiler:
environment: {}
extra_rpaths: []
flags: {}
modules: []
operating_system: ubuntu16.04
paths:
cc: /usr/bin/gcc
cxx: /usr/bin/g++
f77: /usr/bin/gfortran
fc: /usr/bin/gfortran
spec: gcc@5.4.0
target: x86_64
This specifies one version of the gcc compiler and one version of the
clang compiler with no flang compiler. Now suppose we have a code that
we want to compile with the clang compiler for C/C++ code, but with
gfortran for Fortran components. We can do this by adding another entry
to the ``compilers.yaml`` file.
.. code-block:: yaml
- compiler:
environment: {}
extra_rpaths: []
flags: {}
modules: []
operating_system: ubuntu16.04
paths:
cc: /usr/bin/clang
cxx: /usr/bin/clang++
f77: /usr/bin/gfortran
fc: /usr/bin/gfortran
spec: clang@3.8.0-gfortran
target: x86_64
Let's talk about the sections of this compiler entry that we've changed.
The biggest change we've made is to the ``paths`` section. This lists
the paths to the compilers to use for each language/specification.
In this case, we point to the clang compiler for C/C++ and the gfortran
compiler for both specifications of Fortran. We've also changed the
``spec`` entry for this compiler. The ``spec`` entry is effectively the
name of the compiler for Spack. It consists of a name and a version
number, separated by the ``@`` sigil. The name must be one of the supported
compiler names in Spack (gcc, intel, pgi, xl, xl_r, clang, nag, cce).
The version number can be an arbitrary string of alphanumeric characters,
as well as ``-``, ``.``, and ``_``. The ``target`` and ``operating_system``
sections we leave unchanged. These sections specify when Spack can use
different compilers, and are primarily useful for configuration files that
will be used across multiple systems.
We can verify that our new compiler works by invoking it now:
.. code-block:: console
$ spack install zlib %clang@3.8.0-gfortran
...
This new compiler also works on Fortran codes:
.. code-block:: console
$ spack install cfitsio %clang@3.8.0-gfortran
...
^^^^^^^^^^^^^^
Compiler Flags
^^^^^^^^^^^^^^
Some compilers may require specific compiler flags to work properly in
a particular computing environment. Spack provides configuration
options for setting compiler flags every time a specific compiler is
invoked. These flags become part of the package spec and therefore of
the build provenance. As on the command line, the flags are set
through the implicit build variables ``cflags``, ``cxxflags``, ``cppflags``,
``fflags``, ``ldflags``, and ``ldlibs``.
Let's open our compilers configuration file again and add a compiler flag.
.. code-block:: yaml
- compiler:
environment: {}
extra_rpaths: []
flags:
cppflags: -g
modules: []
operating_system: ubuntu16.04
paths:
cc: /usr/bin/clang
cxx: /usr/bin/clang++
f77: /usr/bin/gfortran
fc: /usr/bin/gfortran
spec: clang@3.8.0-gfortran
target: x86_64
We can test this out using the ``spack spec`` command to show how the
spec is concretized.
.. code-block:: console
$ spack spec cfitsio %clang@3.8.0-gfortran
Input spec
--------------------------------
cfitsio%clang@3.8.0-gfortran
Normalized
--------------------------------
cfitsio%clang@3.8.0-gfortran
Concretized
--------------------------------
cfitsio@3.410%clang@3.8.0-gfortran cppflags="-g" +bzip2+shared arch=linux-ubuntu16.04-x86_64
^bzip2@1.0.6%clang@3.8.0-gfortran cppflags="-g" +shared arch=linux-ubuntu16.04-x86_64
We can see that "cppflags=-g" has been added to every node in the DAG.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Advanced Compiler Configuration
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
There are three fields of the compiler configuration entry that we
have not talked about yet.
The ``modules`` field of the compiler is used primarily on Cray systems,
but can be useful on any system that has compilers that are only
useful when a particular module is loaded. Any modules in the
``modules`` field of the compiler configuration will be loaded as part
of the build environment for packages using that compiler.
The ``extra_rpaths`` field of the compiler configuration is used for
compilers that do not rpath all of their dependencies by
default. Since compilers are generally installed externally to Spack,
Spack is unable to manage compiler dependencies and enforce
rpath usage. This can lead to packages not finding link dependencies
imposed by the compiler properly. For compilers that impose link
dependencies on the resulting executables that are not rpath'ed into
the executable automatically, the ``extra_rpath`` field of the compiler
configuration tells Spack which dependencies to rpath into every
executable created by that compiler. The executables will then be able
to find the link dependencies imposed by the compiler.
The ``environment`` field of the compiler configuration is used for
compilers that require environment variables to be set during build
time. For example, if your Intel compiler suite requires the
``INTEL_LICENSE_FILE`` environment variable to point to the proper
license server, you can set this in ``compilers.yaml``.
-------------------------------
Configuring Package Preferences
-------------------------------
Package preferences in Spack are managed through the ``packages.yaml``
configuration file. First, we will look at the default
``packages.yaml`` file.
.. code-block:: console
$ spack config --scope defaults edit packages
.. literalinclude:: ../../../etc/spack/defaults/packages.yaml
:language: yaml
This sets the default preferences for compilers and for providers of
virtual packages. To illustrate how this works, suppose we want to
change the preferences to prefer the clang compiler and to prefer
mpich over openmpi. Currently, we prefer gcc and openmpi
.. code-block:: console
$ spack spec hdf5
Input spec
--------------------------------
hdf5
Normalized
--------------------------------
hdf5
^zlib@1.1.2:
Concretized
--------------------------------
hdf5@1.10.1%gcc@5.4.0+cxx~debug+fortran+mpi+pic+shared~szip~threadsafe arch=linux-ubuntu16.04-x86_64
^openmpi@3.0.0%gcc@5.4.0~cuda fabrics= ~java schedulers= ~sqlite3~thread_multiple+vt arch=linux-ubuntu16.04-x86_64
^hwloc@1.11.7%gcc@5.4.0~cuda+libxml2~pci arch=linux-ubuntu16.04-x86_64
^libxml2@2.9.4%gcc@5.4.0~python arch=linux-ubuntu16.04-x86_64
^pkg-config@0.29.2%gcc@5.4.0+internal_glib arch=linux-ubuntu16.04-x86_64
^xz@5.2.3%gcc@5.4.0 arch=linux-ubuntu16.04-x86_64
^zlib@1.2.11%gcc@5.4.0+pic+shared arch=linux-ubuntu16.04-x86_64
Now we will open the packages configuration file and update our
preferences.
.. code-block:: console
$ spack config edit packages
.. code-block:: yaml
packages:
all:
compiler: [clang, gcc, intel, pgi, xl, nag]
providers:
mpi: [mpich, openmpi]
Because of the configuration scoping we discussed earlier, this
overrides the default settings just for these two items.
.. code-block:: console
$ spack spec hdf5
Input spec
--------------------------------
hdf5
Normalized
--------------------------------
hdf5
^zlib@1.1.2:
Concretized
--------------------------------
hdf5@1.10.1%clang@3.8.0-2ubuntu4+cxx~debug~fortran~hl+mpi+pic+shared~szip~threadsafe arch=linux-ubuntu16.04-x86_64
^mpich@3.2%clang@3.8.0-2ubuntu4 device=ch3 +hydra netmod=tcp +pmi+romio~verbs arch=linux-ubuntu16.04-x86_64
^zlib@1.2.11%clang@3.8.0-2ubuntu4+pic+shared arch=linux-ubuntu16.04-x86_64
^^^^^^^^^^^^^^^^^^^
Variant Preferences
^^^^^^^^^^^^^^^^^^^
The packages configuration file can also set variant preferences for
package variants. For example, let's change our preferences to build all
packages without shared libraries. We will accomplish this by turning
off the ``shared`` variant on all packages that have one.
.. code-block:: yaml
packages:
all:
compiler: [clang, gcc, intel, pgi, xl, nag]
providers:
mpi: [mpich, openmpi]
variants: ~shared
We can check the effect of this command with ``spack spec hdf5`` again.
.. code-block:: console
$ spack spec hdf5
Input spec
--------------------------------
hdf5
Normalized
--------------------------------
hdf5
^zlib@1.1.2:
Concretized
--------------------------------
hdf5@1.10.1%clang@3.8.0-2ubuntu4+cxx~debug~fortran~hl+mpi+pic~shared~szip~threadsafe arch=linux-ubuntu16.04-x86_64
^mpich@3.2%clang@3.8.0-2ubuntu4 device=ch3 +hydra netmod=tcp +pmi+romio~verbs arch=linux-ubuntu16.04-x86_64
^zlib@1.2.11%clang@3.8.0-2ubuntu4+pic~shared arch=linux-ubuntu16.04-x86_64
So far we have only made global changes to the package preferences. As
we've seen throughout this tutorial, hdf5 builds with MPI enabled by
default in Spack. If we were working on a project that would routinely
need serial hdf5, that might get annoying quickly, having to type
``hdf5~mpi`` all the time. Instead, we'll update our preferences for
hdf5.
.. code-block:: yaml
packages:
all:
compiler: [clang, gcc, intel, pgi, xl, nag]
providers:
mpi: [mpich, openmpi]
variants: ~shared
hdf5:
variants: ~mpi
Now hdf5 will concretize without an MPI dependency by default.
.. code-block:: console
$ spack spec hdf5
Input spec
--------------------------------
hdf5
Normalized
--------------------------------
hdf5
^zlib@1.1.2:
Concretized
--------------------------------
hdf5@1.10.1%clang@3.8.0-2ubuntu4+cxx~debug~fortran~hl+mpi+pic~shared~szip~threadsafe arch=linux-ubuntu16.04-x86_64
^zlib@1.2.11%clang@3.8.0-2ubuntu4+pic~shared arch=linux-ubuntu16.04-x86_64
In general, every attribute that we can set for all packages we can
set separately for an individual package.
^^^^^^^^^^^^^^^^^
External Packages
^^^^^^^^^^^^^^^^^
The packages configuration file also controls when Spack will build
against an externally installed package. On these systems we have a
pre-installed zlib.
.. code-block:: yaml
packages:
all:
compiler: [clang, gcc, intel, pgi, xl, nag]
providers:
mpi: [mpich, openmpi]
variants: ~shared
hdf5:
variants: ~mpi
zlib:
paths:
zlib@1.2.8%gcc@5.4.0 arch=linux-ubuntu16.04-x86_64: /usr
Here, we've told Spack that zlib 1.2.8 is installed on our system.
We've also told it the installation prefix where zlib can be found.
We don't know exactly which variants it was built with, but that's
okay.
.. code-block:: console
$ spack spec hdf5
Input spec
--------------------------------
hdf5
Normalized
--------------------------------
hdf5
^zlib@1.1.2:
Concretized
--------------------------------
hdf5@1.10.1%gcc@5.4.0~cxx~debug~fortran~hl~mpi+pic+shared~szip~threadsafe arch=linux-ubuntu16.04-x86_64
^zlib@1.2.8%gcc@5.4.0+optimize+pic~shared arch=linux-ubuntu16.04-x86_64
You'll notice that Spack is now using the external zlib installation,
but the compiler used to build zlib is now overriding our compiler
preference of clang. If we explicitly specify clang:
.. code-block:: console
$ spack spec hdf5 %clang
Input spec
--------------------------------
hdf5%clang
Normalized
--------------------------------
hdf5%clang
^zlib@1.1.2:
Concretized
--------------------------------
hdf5@1.10.1%clang@3.8.0-2ubuntu4~cxx~debug~fortran~hl~mpi+pic+shared~szip~threadsafe arch=linux-ubuntu16.04-x86_64
^zlib@1.2.11%clang@3.8.0-2ubuntu4+optimize+pic~shared arch=linux-ubuntu16.04-x86_64
Spack concretizes to both hdf5 and zlib being built with clang.
This has a side-effect of rebuilding zlib. If we want to force
Spack to use the system zlib, we have two choices. We can either
specify it on the command line, or we can tell Spack that it's
not allowed to build its own zlib. We'll go with the latter.
.. code-block:: yaml
packages:
all:
compiler: [clang, gcc, intel, pgi, xl, nag]
providers:
mpi: [mpich, openmpi]
variants: ~shared
hdf5:
variants: ~mpi
zlib:
paths:
zlib@1.2.8%gcc@5.4.0 arch=linux-ubuntu16.04-x86_64: /usr
buildable: False
Now Spack will be forced to choose the external zlib.
.. code-block:: console
$ spack spec hdf5 %clang
Input spec
--------------------------------
hdf5%clang
Normalized
--------------------------------
hdf5%clang
^zlib@1.1.2:
Concretized
--------------------------------
hdf5@1.10.1%clang@3.8.0-2ubuntu4~cxx~debug~fortran~hl~mpi+pic+shared~szip~threadsafe arch=linux-ubuntu16.04-x86_64
^zlib@1.2.8%gcc@5.4.0+optimize+pic~shared arch=linux-ubuntu16.04-x86_64
This gets slightly more complicated with virtual dependencies. Suppose
we don't want to build our own MPI, but we now want a parallel version
of hdf5? Well, fortunately we have mpich installed on these systems.
.. code-block:: yaml
packages:
all:
compiler: [clang, gcc, intel, pgi, xl, nag]
providers:
mpi: [mpich, openmpi]
variants: ~shared
hdf5:
variants: ~mpi
zlib:
paths:
zlib@1.2.8%gcc@5.4.0 arch=linux-ubuntu16.04-x86_64: /usr
buildable: False
mpich:
paths:
mpich@3.2%gcc@5.4.0 device=ch3 +hydra netmod=tcp +pmi+romio~verbs arch=linux-ubuntu16.04-x86_64: /usr
buildable: False
If we concretize ``hdf5+mpi`` with this configuration file, we will just
build with an alternate MPI implementation.
.. code-block:: console
$ spack spec hdf5 %clang +mpi
Input spec
--------------------------------
hdf5%clang+mpi
Normalized
--------------------------------
hdf5%clang+mpi
^mpi
^zlib@1.1.2:
Concretized
--------------------------------
hdf5@1.10.1%clang@3.8.0-2ubuntu4~cxx~debug~fortran~hl+mpi+pic~shared~szip~threadsafe arch=linux-ubuntu16.04-x86_64
^openmpi@3.0.0%clang@3.8.0-2ubuntu4~cuda fabrics=verbs ~java schedulers= ~sqlite3~thread_multiple+vt arch=linux-ubuntu16.04-x86_64
^hwloc@1.11.8%clang@3.8.0-2ubuntu4~cuda+libxml2+pci arch=linux-ubuntu16.04-x86_64
^libpciaccess@0.13.5%clang@3.8.0-2ubuntu4 arch=linux-ubuntu16.04-x86_64
^libtool@2.4.6%clang@3.8.0-2ubuntu4 arch=linux-ubuntu16.04-x86_64
^m4@1.4.18%clang@3.8.0-2ubuntu4 patches=3877ab548f88597ab2327a2230ee048d2d07ace1062efe81fc92e91b7f39cd00 +sigsegv arch=linux-ubuntu16.04-x86_64
^libsigsegv@2.11%clang@3.8.0-2ubuntu4 arch=linux-ubuntu16.04-x86_64
^pkg-config@0.29.2%clang@3.8.0-2ubuntu4+internal_glib arch=linux-ubuntu16.04-x86_64
^util-macros@1.19.1%clang@3.8.0-2ubuntu4 arch=linux-ubuntu16.04-x86_64
^libxml2@2.9.4%clang@3.8.0-2ubuntu4~python arch=linux-ubuntu16.04-x86_64
^xz@5.2.3%clang@3.8.0-2ubuntu4 arch=linux-ubuntu16.04-x86_64
^zlib@1.2.8%gcc@5.4.0+optimize+pic+shared arch=linux-ubuntu16.04-x86_64
We have only expressed a preference for mpich over other MPI
implementations, and Spack will happily build with one we haven't
forbid it from using. We could resolve this by requesting
``hdf5%clang+mpi^mpich`` explicitly, or we can configure Spack not to
use any other MPI implementation. Since we're focused on
configurations here and the former can get tedious, we'll need to
modify our ``packages.yaml`` file again.
While we're at it, we can configure hdf5 to build with MPI by default
again.
.. code-block:: yaml
packages:
all:
compiler: [clang, gcc, intel, pgi, xl, nag]
providers:
mpi: [mpich, openmpi]
variants: ~shared
zlib:
paths:
zlib@1.2.8%gcc@5.4.0 arch=linux-ubuntu16.04-x86_64: /usr
buildable: False
mpich:
paths:
mpich@3.2%gcc@5.4.0 device=ch3 +hydra netmod=tcp +pmi+romio~verbs arch=linux-ubuntu16.04-x86_64: /usr
buildable: False
openmpi:
buildable: False
mvapich2:
buildable: False
intel-mpi:
buildable: False
spectrum-mpi:
buildable: False
intel-parallel-studio:
buildable: False
Now that we have configured Spack not to build any of the possible
providers for MPI we can try again.
.. code-block:: console
$ spack spec hdf5 %clang
Input spec
--------------------------------
hdf5%clang
Normalized
--------------------------------
hdf5%clang
^mpi
^zlib@1.1.2:
Concretized
--------------------------------
hdf5@1.10.1%clang@3.8.0-2ubuntu4+cxx~debug~fortran~hl+mpi+pic~shared~szip~threadsafe arch=linux-ubuntu16.04-x86_64
^mpich@3.2%gcc@5.4.0 device=ch3 +hydra netmod=tcp +pmi+romio~verbs arch=linux-ubuntu16.04-x86_64
^zlib@1.2.8%gcc@5.4.0+pic+shared arch=linux-ubuntu16.04-x86_64
By configuring most of our package preferences in ``packages.yaml``,
we can cut down on the amount of work we need to do when specifying
a spec on the command line. In addition to compiler and variant
preferences, we can specify version preferences as well. Anything
that you can specify on the command line can be specified in
``packages.yaml`` with the exact same spec syntax.
.. warning::
Make sure to delete or move the ``packages.yaml`` you have been
editing up to this point. Otherwise, it will change the hashes
of your packages, leading to differences in the output of later
tutorial sections.
-----------------
High-level Config
-----------------
In addition to compiler and package settings, Spack allows customization
of several high-level settings. These settings are stored in the generic
``config.yaml`` configuration file. You can see the default settings by
running:
.. code-block:: console
$ spack config --scope defaults edit config
.. literalinclude:: ../../../etc/spack/defaults/config.yaml
:language: yaml
As you can see, many of the directories Spack uses can be customized.
For example, you can tell Spack to install packages to a prefix
outside of the ``$SPACK_ROOT`` hierarchy. Module files can be
written to a central location if you are using multiple Spack
instances. If you have a fast scratch filesystem, you can run builds
from this filesystem with the following ``config.yaml``:
.. code-block:: yaml
config:
build_stage:
- /scratch/$user
On systems with compilers that absolutely *require* environment variables
like ``LD_LIBRARY_PATH``, it is possible to prevent Spack from cleaning
the build environment with the ``dirty`` setting:
.. code-block:: yaml
config:
dirty: true
However, this is strongly discouraged, as it can pull unwanted libraries
into the build.
One last setting that may be of interest to many users is the ability
to customize the parallelism of Spack builds. By default, Spack
installs all packages in parallel with the number of jobs equal to the
number of cores on the node. For example, on a node with 36 cores,
this will look like:
.. code-block:: console
$ spack install --verbose zlib
==> Installing zlib
==> Using cached archive: ~/spack/var/spack/cache/zlib/zlib-1.2.11.tar.gz
==> Staging archive: ~/spack/var/spack/stage/zlib-1.2.11-5nus6knzumx4ik2yl44jxtgtsl7d54xb/zlib-1.2.11.tar.gz
==> Created stage in ~/spack/var/spack/stage/zlib-1.2.11-5nus6knzumx4ik2yl44jxtgtsl7d54xb
==> No patches needed for zlib
==> Building zlib [Package]
==> Executing phase: 'install'
==> './configure' '--prefix=~/spack/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/zlib-1.2.11-5nus6knzumx4ik2yl44jxtgtsl7d54xb'
Checking for shared library support...
Building shared library libz.so.1.2.11 with ~/spack/lib/spack/env/gcc/gcc.
Checking for size_t... Yes.
Checking for off64_t... Yes.
Checking for fseeko... Yes.
Checking for strerror... Yes.
Checking for unistd.h... Yes.
Checking for stdarg.h... Yes.
Checking whether to use vs[n]printf() or s[n]printf()... using vs[n]printf().
Checking for vsnprintf() in stdio.h... Yes.
Checking for return value of vsnprintf()... Yes.
Checking for attribute(visibility) support... Yes.
==> 'make' '-j36'
...
==> 'make' '-j36' 'install'
...
As you can see, we are building with all 36 cores on the node. If you are
on a shared login node, this can slow down the system for other users. If
you have a strict ulimit or restriction on the number of available licenses,
you may not be able to build at all with this many cores. On nodes with 64+
cores, you may not see a significant speedup of the build anyway. To limit
the number of cores our build uses, set ``build_jobs`` like so:
.. code-block:: yaml
config:
build_jobs: 4
If we uninstall and reinstall zlib, we see that it now uses only 4 cores:
.. code-block:: console
$ spack install -v zlib
==> Installing zlib
==> Using cached archive: ~/spack/var/spack/cache/zlib/zlib-1.2.11.tar.gz
==> Staging archive: ~/spack/var/spack/stage/zlib-1.2.11-ezuwp4pa52e75v6iweawzwymmf4ahxxn/zlib-1.2.11.tar.gz
==> Created stage in ~/spack/var/spack/stage/zlib-1.2.11-ezuwp4pa52e75v6iweawzwymmf4ahxxn
==> No patches needed for zlib
==> Building zlib [Package]
==> Executing phase: 'install'
==> './configure' '--prefix=~/spack/opt/spack/linux-ubuntu16.04-x86_64/gcc-7.2.0/zlib-1.2.11-ezuwp4pa52e75v6iweawzwymmf4ahxxn'
Checking for shared library support...
Building shared library libz.so.1.2.11 with ~/spack/lib/spack/env/gcc/gcc.
Checking for size_t... Yes.
Checking for off64_t... Yes.
Checking for fseeko... Yes.
Checking for strerror... Yes.
Checking for unistd.h... Yes.
Checking for stdarg.h... Yes.
Checking whether to use vs[n]printf() or s[n]printf()... using vs[n]printf().
Checking for vsnprintf() in stdio.h... Yes.
Checking for return value of vsnprintf()... Yes.
Checking for attribute(visibility) support... Yes.
==> 'make' '-j4'
...
==> 'make' '-j4' 'install'
...
Obviously, if you want to build everything in serial for whatever reason,
you would set ``build_jobs`` to 1.

1541
lib/spack/docs/tutorial_modules.rst Normal file
View File

File diff suppressed because it is too large Load Diff

230
lib/spack/docs/tutorial_sc16_packaging.rst → lib/spack/docs/tutorial_packaging.rst
View File

@@ -43,7 +43,7 @@ A few things before we get started:
Creating the Package File
-------------------------
Spack comes with a handy command to create a new package: ``spack create``
Spack comes with a handy command to create a new package: ``spack create``.
This command is given the location of a package's source code, downloads
the code, and sets up some basic packaging infrastructure for you. The
@@ -52,12 +52,20 @@ we run ``spack create`` on it:
.. code-block:: console
$ spack create -f https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
==> This looks like a URL for mpileaks version 1.0
==> Creating template for package mpileaks
$ spack create -t generic -f https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
==> This looks like a URL for mpileaks
==> Found 1 version of mpileaks:
1.0 https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
==> How many would you like to checksum? (default is 1, q to abort) 1
==> Downloading...
==> Fetching https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
###################################################################################### 100.0%
==> Fetching https://github.com/LLNL/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
############################################################################# 100.0%
==> Checksummed 1 version of mpileaks
==> Using specified package template: 'generic'
==> Created template for mpileaks package
==> Created package file: $SPACK_ROOT/var/spack/repos/builtin/packages/mpileaks/package.py
And Spack should spawn a text editor with this file:
@@ -192,29 +200,27 @@ Now when we try to install this package a lot more happens:
.. code-block:: console
$ spack install mpileaks
...
==> libdwarf is already installed in SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/libdwarf-20160507-er4jrjynul6uba7wiu5tasuj35roxw6m
==> dyninst is already installed in SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/dyninst-9.3.2-t7mau34jv3e76mpspdzhf2p2a6k7qubg
==> callpath is already installed in SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/callpath-1.0.4-ikbbkvfmsfmqzo624nvvrbooovf7egoc
==> Installing mpileaks
==> openmpi is already installed in /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/openmpi-2.0.1-5ee5j34c2y4kb5c3joygrgahidqnwhnz
==> callpath is already installed in /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/callpath-1.0.2-zm4pf3gasgxeibyu2y262suktvaazube
==> adept-utils is already installed in /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/adept-utils-1.0.1-7p7ezxwtajdglj6cmojy2vybjct4j4jz
==> Using cached archive: /usr/workspace/wsa/legendre/spack/var/spack/cache/mpileaks/mpileaks-1.0.tar.gz
==> Already staged mpileaks-1.0-eum4hmnlt6ovalwjnciaygfb3beja4gk in /usr/workspace/wsa/legendre/spack/var/spack/stage/mpileaks-1.0-eum4hmnlt6ovalwjnciaygfb3beja4gk
==> Already patched mpileaks
==> Building mpileaks [AutotoolsPackage]
==> Executing phase : 'autoreconf'
==> Executing phase : 'configure'
==> Error: ProcessError: Command exited with status 1:
'./configure' '--prefix=/usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/mpileaks-1.0-eum4hmnlt6ovalwjnciaygfb3beja4gk'
/usr/workspace/wsa/legendre/spack/lib/spack/spack/build_systems/autotools.py:150, in configure:
145 def configure(self, spec, prefix):
146 """Runs configure with the arguments specified in `configure_args`
147 and an appropriately set prefix
148 """
149 options = ['--prefix={0}'.format(prefix)] + self.configure_args()
>> 150 inspect.getmodule(self).configure(*options)
==> Using cached archive: SPACK_ROOT/var/spack/cache/mpileaks/mpileaks-1.0.tar.gz
==> Already staged mpileaks-1.0-lfgf53rns5mswq25rxckzgvmjc6ywam7 in SPACK_ROOT/var/spack/stage/mpileaks-1.0-lfgf53rns5mswq25rxckzgvmjc6ywam7
==> No patches needed for mpileaks
==> Building mpileaks [Package]
==> Executing phase: 'install'
==> Error: ProcessError: Command exited with status 2:
'make' '-j36'
1 error found in build log:
1 ==> Executing phase: 'install'
2 ==> 'make' '-j36'
>> 3 make: *** No targets specified and no makefile found. Stop.
See build log for details:
/tmp/legendre/spack-stage/spack-stage-7V5yyk/mpileaks-1.0/spack-build.out
SPACK_ROOT/var/spack/stage/mpileaks-1.0-lfgf53rns5mswq25rxckzgvmjc6ywam7/mpileaks-1.0/spack-build.out
Note that this command may take a while to run and produce more output if
you don't have an MPI already installed or configured in Spack.
@@ -228,44 +234,79 @@ Debugging Package Builds
------------------------
Our ``mpileaks`` package is still not building. It may be obvious to
many of you that we're still missing the configure options. But let's
pretend we're not all intelligent developers and use this opportunity
spend some time debugging. We a few options that can tell us about
many of you that we never ran the configure script. Let's add a
call to ``configure()`` to the top of the install routine. The resulting
package.py is in ``$SPACK_ROOT/lib/spack/docs/tutorial/examples/3.package.py``:
.. literalinclude:: tutorial/examples/3.package.py
:lines: 25-
:language: python
If we re-run we still get errors:
.. code-block:: console
==> Installing mpileaks
==> Using cached archive: SPACK_ROOT/var/spack/cache/mpileaks/mpileaks-1.0.tar.gz
==> Already staged mpileaks-1.0-lfgf53rns5mswq25rxckzgvmjc6ywam7 in SPACK_ROOT/var/spack/stage/mpileaks-1.0-lfgf53rns5mswq25rxckzgvmjc6ywam7
==> No patches needed for mpileaks
==> Building mpileaks [Package]
==> Executing phase: 'install'
==> Error: ProcessError: Command exited with status 1:
'./configure'
1 error found in build log:
[ ... ]
21 checking whether SPACK_ROOT/lib/spack/env/gcc/gcc and cc understand -c and -o together... yes
22 checking whether we are using the GNU C++ compiler... yes
23 checking whether SPACK_ROOT/lib/spack/env/gcc/g++ accepts -g... yes
24 checking dependency style of SPACK_ROOT/lib/spack/env/gcc/g++... gcc3
25 checking for SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/openmpi-3.0.0-yo5qkfvumpmgmvlbalqcadu46j5bd52f/bin/mpicc... SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/openmpi-3.0.0-yo5qkfvumpmgmvlbalqcadu46j5bd52f/bin/mpicc
26 Checking whether SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/openmpi-3.0.0-yo5qkfvumpmgmvlbalqcadu46j5bd52f/bin/mpicc responds to '-showme:compile'... yes
>> 27 configure: error: unable to locate adept-utils installation
See build log for details:
SPACK_ROOT/var/spack/stage/mpileaks-1.0-lfgf53rns5mswq25rxckzgvmjc6ywam7/mpileaks-1.0/spack-build.out
Again, the problem may be obvious. But let's pretend we're not
all intelligent developers and use this opportunity spend some
time debugging. We have a few options that can tell us about
what's going wrong:
As per the error message, Spack has given us a ``spack-build.out`` debug log:
.. code-block:: console
==> './configure' '--prefix=/usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/mpileaks-1.0-eum4hmnlt6ovalwjnciaygfb3beja4gk'
==> Executing phase: 'install'
==> './configure'
checking metadata... no
checking installation directory variables... yes
checking for a BSD-compatible install... /usr/bin/install -c
checking whether build environment is sane... yes
checking for a thread-safe mkdir -p... /usr/bin/mkdir -p
checking for a thread-safe mkdir -p... /bin/mkdir -p
checking for gawk... gawk
checking whether make sets $(MAKE)... yes
checking for gcc... /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/gcc
checking for gcc... SPACK_ROOT/lib/spack/env/gcc/gcc
checking for C compiler default output file name... a.out
checking whether the C compiler works... yes
checking whether we are cross compiling... no
checking for suffix of executables...
checking for suffix of executables...
checking for suffix of object files... o
checking whether we are using the GNU C compiler... yes
checking whether /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/gcc accepts -g... yes
checking for /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/gcc option to accept ISO C89... none needed
checking whether SPACK_ROOT/lib/spack/env/gcc/gcc accepts -g... yes
checking for SPACK_ROOT/lib/spack/env/gcc/gcc option to accept ISO C89... none needed
checking for style of include used by make... GNU
checking dependency style of /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/gcc... gcc3
checking whether /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/gcc and cc understand -c and -o together... yes
checking dependency style of SPACK_ROOT/lib/spack/env/gcc/gcc... gcc3
checking whether SPACK_ROOT/lib/spack/env/gcc/gcc and cc understand -c and -o together... yes
checking whether we are using the GNU C++ compiler... yes
checking whether /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/g++ accepts -g... yes
checking dependency style of /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/g++... gcc3
checking for /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/openmpi-2.0.1-5ee5j34c2y4kb5c3joygrgahidqnwhnz/bin/mpicc... /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/openmpi-2.0.1-5ee5j34c2y4kb5c3joygrgahidqnwhnz/bin/mpicc
Checking whether /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/openmpi-2.0.1-5ee5j34c2y4kb5c3joygrgahidqnwhnz/bin/mpicc responds to '-showme:compile'... yes
configure: error: unable to locate ``adept-utils`` installation
checking whether SPACK_ROOT/lib/spack/env/gcc/g++ accepts -g... yes
checking dependency style of SPACK_ROOT/lib/spack/env/gcc/g++... gcc3
checking for SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/openmpi-3.0.0-yo5qkfvumpmgmvlbalqcadu46j5bd52f/bin/mpicc... SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/openmpi-3.0.0-yo5qkfvumpmgmvlbalqcadu46j5bd52f/bin/mpicc
Checking whether SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/openmpi-3.0.0-yo5qkfvumpmgmvlbalqcadu46j5bd52f/bin/mpicc responds to '-showme:compile'... yes
configure: error: unable to locate adept-utils installation
This gives us the output from the build, and it's fairly obvious that
mpileaks isn't finding its ``adept-utils`` package. Spack has
This gives us the output from the build, and mpileaks isn't
finding its ``adept-utils`` package. Spack has
automatically added the include and library directories of
``adept-utils`` to the compiler's search path, but some packages like
mpileaks can sometimes be picky and still want things spelled out on
@@ -292,26 +333,26 @@ From here we can manually re-run the build:
checking installation directory variables... yes
checking for a BSD-compatible install... /usr/bin/install -c
checking whether build environment is sane... yes
checking for a thread-safe mkdir -p... /usr/bin/mkdir -p
checking for a thread-safe mkdir -p... /bin/mkdir -p
checking for gawk... gawk
checking whether make sets $(MAKE)... yes
checking for gcc... /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/gcc
checking for gcc... SPACK_ROOT/lib/spack/env/gcc/gcc
checking for C compiler default output file name... a.out
checking whether the C compiler works... yes
checking whether we are cross compiling... no
checking for suffix of executables...
checking for suffix of executables...
checking for suffix of object files... o
checking whether we are using the GNU C compiler... yes
checking whether /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/gcc accepts -g... yes
checking for /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/gcc option to accept ISO C89... none needed
checking whether SPACK_ROOT/lib/spack/env/gcc/gcc accepts -g... yes
checking for SPACK_ROOT/lib/spack/env/gcc/gcc option to accept ISO C89... none needed
checking for style of include used by make... GNU
checking dependency style of /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/gcc... gcc3
checking whether /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/gcc and cc understand -c and -o together... yes
checking dependency style of SPACK_ROOT/lib/spack/env/gcc/gcc... gcc3
checking whether SPACK_ROOT/lib/spack/env/gcc/gcc and cc understand -c and -o together... yes
checking whether we are using the GNU C++ compiler... yes
checking whether /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/g++ accepts -g... yes
checking dependency style of /usr/workspace/wsa/legendre/spack/lib/spack/env/gcc/g++... gcc3
checking for /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/openmpi-2.0.1-5ee5j34c2y4kb5c3joygrgahidqnwhnz/bin/mpicc... /usr/workspace/wsa /legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/openmpi-2.0.1-5ee5j34c2y4kb5c3joygrgahidqnwhnz/bin/mpicc
Checking whether /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/openmpi-2.0.1-5ee5j34c2y4kb5c3joygrgahidqnwhnz/bin/mpicc responds to '-showme:compile'... yes
checking whether SPACK_ROOT/lib/spack/env/gcc/g++ accepts -g... yes
checking dependency style of SPACK_ROOT/lib/spack/env/gcc/g++... gcc3
checking for SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/openmpi-3.0.0-yo5qkfvumpmgmvlbalqcadu46j5bd52f/bin/mpicc... SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/openmpi-3.0.0-yo5qkfvumpmgmvlbalqcadu46j5bd52f/bin/mpicc
Checking whether SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/openmpi-3.0.0-yo5qkfvumpmgmvlbalqcadu46j5bd52f/bin/mpicc responds to '-showme:compile'... yes
configure: error: unable to locate adept-utils installation
We're seeing the same error, but now we're in a shell where we can run
@@ -328,9 +369,9 @@ Specifying Configure Arguments
Let's add the configure arguments to the mpileaks' ``package.py``. This
version can be found in
``$SPACK_ROOT/lib/spack/docs/tutorial/examples/3.package.py``:
``$SPACK_ROOT/lib/spack/docs/tutorial/examples/4.package.py``:
.. literalinclude:: tutorial/examples/3.package.py
.. literalinclude:: tutorial/examples/4.package.py
:lines: 25-
:language: python
@@ -339,37 +380,35 @@ This is all we need for working mpileaks! If we install now we'll see:
.. code-block:: console
$ spack install mpileaks
spack install mpileaks
...
==> Installing mpileaks
==> openmpi is already installed in /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/openmpi-2.0.1-5ee5j34c2y4kb5c3joygrgahidqnwhnz
==> callpath is already installed in /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/callpath-1.0.2-zm4pf3gasgxeibyu2y262suktvaazube
==> adept-utils is already installed in /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/adept-utils-1.0.1-7p7ezxwtajdglj6cmojy2vybjct4j4jz
==> Using cached archive: /usr/workspace/wsa/legendre/spack/var/spack/cache/mpileaks/mpileaks-1.0.tar.gz
==> Already staged mpileaks-1.0-eum4hmnlt6ovalwjnciaygfb3beja4gk in /usr/workspace/wsa/legendre/spack/var/spack/stage/mpileaks-1.0-eum4hmnlt6ovalwjnciaygfb3beja4gk
==> Already patched mpileaks
==> Building mpileaks [AutotoolsPackage]
==> Executing phase : 'autoreconf'
==> Executing phase : 'configure'
==> Executing phase : 'build'
==> Executing phase : 'install'
==> Using cached archive: SPACK_ROOT/var/spack/cache/mpileaks/mpileaks-1.0.tar.gz
==> Staging archive: SPACK_ROOT/var/spack/stage/mpileaks-1.0-lfgf53rns5mswq25rxckzgvmjc6ywam7/mpileaks-1.0.tar.gz
==> Created stage in SPACK_ROOT/var/spack/stage/mpileaks-1.0-lfgf53rns5mswq25rxckzgvmjc6ywam7
==> No patches needed for mpileaks
==> Building mpileaks [Package]
==> Executing phase: 'install'
==> Successfully installed mpileaks
Fetch: 0.00s. Build: 14.08s. Total: 14.08s.
[+] /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/mpileaks-1.0-eum4hmnlt6ovalwjnciaygfb3beja4gk
Fetch: 0.00s. Build: 9.01s. Total: 9.01s.
[+] SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/mpileaks-1.0-lfgf53rns5mswq25rxckzgvmjc6ywam7
We took a few shortcuts for this package that are worth highlighting.
Spack automatically detected that mpileaks was an Autotools-based package
when we ran ``spack create``. If this had been a CMake-based package we
would have been filling in a ``cmake_args`` function instead of
``configure_args``. If Spack hadn't been able to detect the build
system, we'd be filling in a generic install method that would manually
be calling build commands, such as is found in the ``zlib`` package:
There are some special circumstances in package that are worth highlighting.
Normally spack would have automatically detected that mpileaks was an
Autotools-based package when we ran ``spack create`` and made it an ``AutoToolsPackage`` class (except we added the ``-t generic`` option to skip this). Instead of
a full install routine we would have just written:
.. code-block:: python
def install(self, spec, prefix):
configure('--prefix={0}'.format(prefix))
make()
make('install')
def configure_args(self):
args = ['--with-adept-utils=%s' % self.spec['adept-utils'].prefix,
'--with-callpath=%s' % self.spec['callpath'].prefix]
return args
Similarly, if this had been a CMake-based package we
would have been filling in a ``cmake_args`` function instead of
``configure_args``. There are similar default package types for
many build environments.
--------
Variants
@@ -381,9 +420,9 @@ that it walks. Let's add a variant to allow users to set this when they
build in Spack.
To do this, we'll add a variant to our package, as per the following (see
``$SPACK_ROOT/lib/spack/docs/tutorial/examples/4.package.py``):
``$SPACK_ROOT/lib/spack/docs/tutorial/examples/5.package.py``):
.. literalinclude:: tutorial/examples/4.package.py
.. literalinclude:: tutorial/examples/5.package.py
:lines: 25-
:language: python
@@ -394,18 +433,15 @@ configure line (output truncated for length):
.. code-block:: console
$ spack install --verbose mpileaks stackstart=4
...
==> Installing mpileaks
==> openmpi is already installed in /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/openmpi-2.0.1-5ee5j34c2y4kb5c3joygrgahidqnwhnz
==> callpath is already installed in /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/callpath-1.0.2-zm4pf3gasgxeibyu2y262suktvaazube
==> adept-utils is already installed in /usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/adept-utils-1.0.1-7p7ezxwtajdglj6cmojy2vybjct4j4jz
==> Using cached archive: /usr/workspace/wsa/legendre/spack/var/spack/cache/mpileaks/mpileaks-1.0.tar.gz
==> Staging archive: /usr/workspace/wsa/legendre/spack/var/spack/stage/mpileaks-1.0-otqo2opkhan5ksujt6tpmdftydrieig7/mpileaks-1.0.tar.gz
==> Created stage in /usr/workspace/wsa/legendre/spack/var/spack/stage/mpileaks-1.0-otqo2opkhan5ksujt6tpmdftydrieig7
==> Ran patch() for mpileaks
==> Building mpileaks [AutotoolsPackage]
==> Executing phase : 'autoreconf'
==> Executing phase : 'configure'
==> './configure' '--prefix=/usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/mpileaks-1.0-otqo2opkhan5ksujt6tpmdftydrieig7' '--with-adept-utils=/usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/adept-utils-1.0.1-7p7ezxwtajdglj6cmojy2vybjct4j4jz' '--with-callpath=/usr/workspace/wsa/legendre/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/callpath-1.0.2-zm4pf3gasgxeibyu2y262suktvaazube' '--with-stack-start-c=4' '--with-stack-start-fortran=4'
==> Using cached archive: SPACK_ROOT/var/spack/cache/mpileaks/mpileaks-1.0.tar.gz
==> Staging archive: SPACK_ROOT/var/spack/stage/mpileaks-1.0-gxxi4fp57b4j6xalra5t65hyx5rj25t7/mpileaks-1.0.tar.gz
==> Created stage in SPACK_ROOT/var/spack/stage/mpileaks-1.0-gxxi4fp57b4j6xalra5t65hyx5rj25t7
==> No patches needed for mpileaks
==> Building mpileaks [Package]
==> Executing phase: 'install'
==> './configure' '--with-adept-utils=SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/adept-utils-1.0.1-pm3gffhrnwsdtqthtvsfvs2tny4r65wb' '--with-callpath=SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/callpath-1.0.4-ikbbkvfmsfmqzo624nvvrbooovf7egoc' '--prefix=SPACK_ROOT/opt/spack/linux-ubuntu16.04-x86_64/gcc-5.4.0/mpileaks-1.0-gxxi4fp57b4j6xalra5t65hyx5rj25t7' '--with-stack-start-c=4' '--with-stack-start-fortran=4'
---------------
The Spec Object

982
lib/spack/docs/tutorial_sc16_modules.rst
View File

@@ -1,982 +0,0 @@
.. _modules-tutorial:
=============================
Module Configuration Tutorial
=============================
This tutorial will guide you through the customization of both
content and naming of module files generated by Spack.
Starting from the default Spack settings you will add an increasing
number of directives to the ``modules.yaml`` configuration file to
satisfy a number of constraints that mimic those that you may encounter
in a typical production environment at HPC sites.
Even though the focus will be for the most part on customizing
TCL non-hierarchical module files, everything
you'll see applies also to other kinds of module files generated by Spack.
The generation of Lua hierarchical
module files will be addressed at the end of the tutorial,
and you'll see that with minor modifications
to an existing ``modules.yaml`` written for TCL
non-hierarchical modules you'll get almost
for free the possibility to try a hierarchical layout.
Let's start!
.. _module_file_tutorial_prerequisites:
-------------
Prerequisites
-------------
Before proceeding further ensure:
- you have LMod or Environment Modules available
- have :ref:`shell support <shell-support>` activated in Spack
If you need to install Lmod or Environment module you can refer
to the documentation :ref:`here <InstallEnvironmentModules>`.
^^^^^^^^^^^^^^^^^^
Add a new compiler
^^^^^^^^^^^^^^^^^^
Spack automatically scans the environment to search for available
compilers on first use. On a Ubuntu 14.04 a fresh clone will show
something like this:
.. code-block:: console
$ uname -a
Linux nuvolari 4.4.0-45-generic #66~14.04.1-Ubuntu SMP Wed Oct 19 15:05:38 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux
$ spack compilers
==> Available compilers
-- gcc ----------------------------------------------------------
gcc@4.8
For the purpose of building a limited set of packages with some features
that will help showcasing the capabilities of
module customization the first thing we need is to build a new compiler:
.. code-block:: console
$ spack install gcc@6.2.0
# ...
# Wait a long time
# ...
Then we can use shell support for modules to add it to the list of known compilers:
.. code-block:: console
# The name of the generated module may vary
$ module load gcc-6.2.0-gcc-4.8-twd5nqg
$ spack compiler add
==> Added 1 new compiler to ~/.spack/linux/compilers.yaml
gcc@6.2.0
$ spack compilers
==> Available compilers
-- gcc ----------------------------------------------------------
gcc@6.2.0 gcc@4.8
Note that the final 7 digits hash at the end of the generated module may vary depending
on architecture or package version.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Build software that will be used in the tutorial
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Next you should install a few modules that will be used in the tutorial:
.. code-block:: console
$ spack install netlib-scalapack ^openmpi ^openblas
# ...
The packages you need to install are:
- ``netlib-scalapack ^openmpi ^openblas``
- ``netlib-scalapack ^mpich ^openblas``
- ``netlib-scalapack ^openmpi ^netlib-lapack``
- ``netlib-scalapack ^mpich ^netlib-lapack``
- ``py-scipy ^openblas``
In the end your environment should look something like:
.. code-block:: console
$ module avail
------------------------------------------------------------------------ ~/spack/share/spack/modules/linux-Ubuntu14-x86_64 ------------------------------------------------------------------------
binutils-2.27-gcc-4.8-dz3xevw libpciaccess-0.13.4-gcc-6.2.0-eo2siet lzo-2.09-gcc-6.2.0-jcngz72 netlib-scalapack-2.0.2-gcc-6.2.0-wnimqhw python-2.7.12-gcc-6.2.0-qu7rc5p
bzip2-1.0.6-gcc-6.2.0-csoc2mq libsigsegv-2.10-gcc-4.8-avb6azw m4-1.4.17-gcc-4.8-iggewke netlib-scalapack-2.0.2-gcc-6.2.0-wojunhq sqlite-3.8.5-gcc-6.2.0-td3zfe7
cmake-3.5.2-gcc-6.2.0-6poypqg libsigsegv-2.10-gcc-6.2.0-g3qpmbi m4-1.4.17-gcc-6.2.0-lhgqa6s nettle-3.2-gcc-6.2.0-djdthlh tcl-8.6.5-gcc-4.8-atddxu7
curl-7.50.3-gcc-6.2.0-2ffacqm libtool-2.4.6-gcc-6.2.0-kiepac6 mpc-1.0.3-gcc-4.8-lylv7lk openblas-0.2.19-gcc-6.2.0-js33umc util-macros-1.19.0-gcc-6.2.0-uoukuqk
expat-2.2.0-gcc-6.2.0-bxqnjar libxml2-2.9.4-gcc-6.2.0-3k4ykbe mpfr-3.1.4-gcc-4.8-bldfx3w openmpi-2.0.1-gcc-6.2.0-s3qbtby xz-5.2.2-gcc-6.2.0-t5lk6in
gcc-6.2.0-gcc-4.8-twd5nqg lmod-6.4.5-gcc-4.8-7v7bh7b mpich-3.2-gcc-6.2.0-5n5xoep openssl-1.0.2j-gcc-6.2.0-hibnfda zlib-1.2.8-gcc-4.8-bds4ies
gmp-6.1.1-gcc-4.8-uq52e2n lua-5.3.2-gcc-4.8-xozf2hx ncurses-6.0-gcc-4.8-u62fit4 pkg-config-0.29.1-gcc-6.2.0-rslsgcs zlib-1.2.8-gcc-6.2.0-asydrba
gmp-6.1.1-gcc-6.2.0-3cfh3hi lua-luafilesystem-1_6_3-gcc-4.8-sbzejlz ncurses-6.0-gcc-6.2.0-7tb426s py-nose-1.3.7-gcc-6.2.0-4gl5c42
hwloc-1.11.4-gcc-6.2.0-3ostwel lua-luaposix-33.4.0-gcc-4.8-xf7y2p5 netlib-lapack-3.6.1-gcc-6.2.0-mirer2l py-numpy-1.11.1-gcc-6.2.0-i3rpk4e
isl-0.14-gcc-4.8-cq73t5m lz4-131-gcc-6.2.0-cagoem4 netlib-scalapack-2.0.2-gcc-6.2.0-6bqlxqy py-scipy-0.18.1-gcc-6.2.0-e6uljfi
libarchive-3.2.1-gcc-6.2.0-2b54aos lzma-4.32.7-gcc-6.2.0-sfmeynw netlib-scalapack-2.0.2-gcc-6.2.0-hpqb3dp py-setuptools-25.2.0-gcc-6.2.0-hkqauaa
------------------------------------------------
Filter unwanted modifications to the environment
------------------------------------------------
The non-hierarchical TCL module files that have been generated so far
follow the default rules for module generation, which are given
:ref:`here <modules-yaml>` in the reference part of the manual. Taking a
look at the ``gcc`` module you'll see something like:
.. code-block:: console
$ module show gcc-6.2.0-gcc-4.8-twd5nqg
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
~/spack/share/spack/modules/linux-Ubuntu14-x86_64/gcc-6.2.0-gcc-4.8-twd5nqg:
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
whatis("gcc @6.2.0 ")
prepend_path("PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/bin")
prepend_path("CMAKE_PREFIX_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/")
prepend_path("MANPATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/share/man")
prepend_path("PKG_CONFIG_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/lib64/pkgconfig")
prepend_path("LIBRARY_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/lib64")
prepend_path("LD_LIBRARY_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/lib64")
prepend_path("CPATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/include")
help([[The GNU Compiler Collection includes front ends for C, C++, Objective-C,
Fortran, and Java.
]])
As expected, a few environment variables representing paths will be modified
by the modules according to the default prefix inspection rules.
Consider now the case that your site has decided that e.g. ``CPATH`` and
``LIBRARY_PATH`` modifications should not be present in module files. What you can
do to abide by the rules is to create a configuration file ``~/.spack/modules.yaml``
with the following content:
.. code-block:: yaml
modules:
tcl:
all:
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
Next you should regenerate all the module files:
.. code-block:: console
$ spack module refresh --module-type tcl
==> You are about to regenerate tcl module files for:
-- linux-Ubuntu14-x86_64 / gcc@4.8 ------------------------------
dz3xevw binutils@2.27 uq52e2n gmp@6.1.1 avb6azw libsigsegv@2.10 xozf2hx lua@5.3.2 xf7y2p5 lua-luaposix@33.4.0 lylv7lk mpc@1.0.3 u62fit4 ncurses@6.0 bds4ies zlib@1.2.8
twd5nqg gcc@6.2.0 cq73t5m isl@0.14 7v7bh7b lmod@6.4.5 sbzejlz lua-luafilesystem@1_6_3 iggewke m4@1.4.17 bldfx3w mpfr@3.1.4 atddxu7 tcl@8.6.5
...
==> Do you want to proceed ? [y/n]
y
==> Regenerating tcl module files
If you take a look now at the module for ``gcc`` you'll see that the unwanted
paths have disappeared:
.. code-block:: console
$ module show gcc-6.2.0-gcc-4.8-twd5nqg
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
~/spack/share/spack/modules/linux-Ubuntu14-x86_64/gcc-6.2.0-gcc-4.8-twd5nqg:
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
whatis("gcc @6.2.0 ")
prepend_path("PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/bin")
prepend_path("CMAKE_PREFIX_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/")
prepend_path("MANPATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/share/man")
prepend_path("PKG_CONFIG_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/lib64/pkgconfig")
prepend_path("LD_LIBRARY_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/lib64")
help([[The GNU Compiler Collection includes front ends for C, C++, Objective-C,
Fortran, and Java.
]])
----------------------------------------------
Prevent some module files from being generated
----------------------------------------------
Another common request at many sites is to avoid exposing software that
is only needed as an intermediate step when building a newer stack.
Let's try to prevent the generation of
module files for anything that is compiled with ``gcc@4.8`` (the OS provided compiler).
To do this you should add a ``blacklist`` keyword to the configuration file:
.. code-block:: yaml
:emphasize-lines: 3,4
modules:
tcl:
blacklist:
- '%gcc@4.8'
all:
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
and regenerate the module files:
.. code-block:: console
$ spack module refresh --module-type tcl --delete-tree
==> You are about to regenerate tcl module files for:
-- linux-Ubuntu14-x86_64 / gcc@4.8 ------------------------------
dz3xevw binutils@2.27 uq52e2n gmp@6.1.1 avb6azw libsigsegv@2.10 xozf2hx lua@5.3.2 xf7y2p5 lua-luaposix@33.4.0 lylv7lk mpc@1.0.3 u62fit4 ncurses@6.0 bds4ies zlib@1.2.8
twd5nqg gcc@6.2.0 cq73t5m isl@0.14 7v7bh7b lmod@6.4.5 sbzejlz lua-luafilesystem@1_6_3 iggewke m4@1.4.17 bldfx3w mpfr@3.1.4 atddxu7 tcl@8.6.5
-- linux-Ubuntu14-x86_64 / gcc@6.2.0 ----------------------------
csoc2mq bzip2@1.0.6 2b54aos libarchive@3.2.1 sfmeynw lzma@4.32.7 wnimqhw netlib-scalapack@2.0.2 s3qbtby openmpi@2.0.1 hkqauaa py-setuptools@25.2.0
6poypqg cmake@3.5.2 eo2siet libpciaccess@0.13.4 jcngz72 lzo@2.09 6bqlxqy netlib-scalapack@2.0.2 hibnfda openssl@1.0.2j qu7rc5p python@2.7.12
2ffacqm curl@7.50.3 g3qpmbi libsigsegv@2.10 lhgqa6s m4@1.4.17 wojunhq netlib-scalapack@2.0.2 rslsgcs pkg-config@0.29.1 td3zfe7 sqlite@3.8.5
bxqnjar expat@2.2.0 kiepac6 libtool@2.4.6 5n5xoep mpich@3.2 hpqb3dp netlib-scalapack@2.0.2 4gl5c42 py-nose@1.3.7 uoukuqk util-macros@1.19.0
3cfh3hi gmp@6.1.1 3k4ykbe libxml2@2.9.4 7tb426s ncurses@6.0 djdthlh nettle@3.2 i3rpk4e py-numpy@1.11.1 t5lk6in xz@5.2.2
3ostwel hwloc@1.11.4 cagoem4 lz4@131 mirer2l netlib-lapack@3.6.1 js33umc openblas@0.2.19 e6uljfi py-scipy@0.18.1 asydrba zlib@1.2.8
==> Do you want to proceed ? [y/n]
y
$ module avail
------------------------------------------------------------------------ ~/spack/share/spack/modules/linux-Ubuntu14-x86_64 ------------------------------------------------------------------------
bzip2-1.0.6-gcc-6.2.0-csoc2mq libsigsegv-2.10-gcc-6.2.0-g3qpmbi ncurses-6.0-gcc-6.2.0-7tb426s openmpi-2.0.1-gcc-6.2.0-s3qbtby sqlite-3.8.5-gcc-6.2.0-td3zfe7
cmake-3.5.2-gcc-6.2.0-6poypqg libtool-2.4.6-gcc-6.2.0-kiepac6 netlib-lapack-3.6.1-gcc-6.2.0-mirer2l openssl-1.0.2j-gcc-6.2.0-hibnfda util-macros-1.19.0-gcc-6.2.0-uoukuqk
curl-7.50.3-gcc-6.2.0-2ffacqm libxml2-2.9.4-gcc-6.2.0-3k4ykbe netlib-scalapack-2.0.2-gcc-6.2.0-6bqlxqy pkg-config-0.29.1-gcc-6.2.0-rslsgcs xz-5.2.2-gcc-6.2.0-t5lk6in
expat-2.2.0-gcc-6.2.0-bxqnjar lz4-131-gcc-6.2.0-cagoem4 netlib-scalapack-2.0.2-gcc-6.2.0-hpqb3dp py-nose-1.3.7-gcc-6.2.0-4gl5c42 zlib-1.2.8-gcc-6.2.0-asydrba
gmp-6.1.1-gcc-6.2.0-3cfh3hi lzma-4.32.7-gcc-6.2.0-sfmeynw netlib-scalapack-2.0.2-gcc-6.2.0-wnimqhw py-numpy-1.11.1-gcc-6.2.0-i3rpk4e
hwloc-1.11.4-gcc-6.2.0-3ostwel lzo-2.09-gcc-6.2.0-jcngz72 netlib-scalapack-2.0.2-gcc-6.2.0-wojunhq py-scipy-0.18.1-gcc-6.2.0-e6uljfi
libarchive-3.2.1-gcc-6.2.0-2b54aos m4-1.4.17-gcc-6.2.0-lhgqa6s nettle-3.2-gcc-6.2.0-djdthlh py-setuptools-25.2.0-gcc-6.2.0-hkqauaa
libpciaccess-0.13.4-gcc-6.2.0-eo2siet mpich-3.2-gcc-6.2.0-5n5xoep openblas-0.2.19-gcc-6.2.0-js33umc python-2.7.12-gcc-6.2.0-qu7rc5p
This time it is convenient to pass the option ``--delete-tree`` to the command that
regenerates the module files to instruct it to delete the existing tree and regenerate
a new one instead of overwriting the files in the existing directory.
If you pay careful attention you'll see though that we went too far in blacklisting modules:
the module for ``gcc@6.2.0`` disappeared as it was bootstrapped with ``gcc@4.8``. To specify
exceptions to the blacklist rules you can use ``whitelist``:
.. code-block:: yaml
:emphasize-lines: 3,4
modules:
tcl:
whitelist:
- gcc
blacklist:
- '%gcc@4.8'
all:
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
``whitelist`` rules always have precedence over ``blacklist`` rules. If you regenerate the modules again:
.. code-block:: console
$ spack module refresh --module-type tcl -y
you'll see that now the module for ``gcc@6.2.0`` has reappeared:
.. code-block:: console
$ module avail gcc-6.2.0-gcc-4.8-twd5nqg
------------------------------------------------------------------------ ~/spack/share/spack/modules/linux-Ubuntu14-x86_64 ------------------------------------------------------------------------
gcc-6.2.0-gcc-4.8-twd5nqg
-------------------------
Change module file naming
-------------------------
The next step in making module files more user-friendly is to
improve their naming scheme.
To reduce the length of the hash or remove it altogether you can
use the ``hash_length`` keyword in the configuration file:
.. TODO: give reasons to remove hashes if they are not evident enough?
.. code-block:: yaml
:emphasize-lines: 3
modules:
tcl:
hash_length: 0
whitelist:
- gcc
blacklist:
- '%gcc@4.8'
all:
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
If you try to regenerate the module files now you will get an error:
.. code-block:: console
$ spack module refresh --module-type tcl --delete-tree -y
==> Error: Name clashes detected in module files:
file : ~/spack/share/spack/modules/linux-Ubuntu14-x86_64/netlib-scalapack-2.0.2-gcc-6.2.0
spec : netlib-scalapack@2.0.2%gcc@6.2.0~fpic+shared arch=linux-Ubuntu14-x86_64
spec : netlib-scalapack@2.0.2%gcc@6.2.0~fpic+shared arch=linux-Ubuntu14-x86_64
spec : netlib-scalapack@2.0.2%gcc@6.2.0~fpic+shared arch=linux-Ubuntu14-x86_64
spec : netlib-scalapack@2.0.2%gcc@6.2.0~fpic+shared arch=linux-Ubuntu14-x86_64
==> Error: Operation aborted
.. note::
We try to check for errors upfront!
In Spack we check for errors upfront whenever possible, so don't worry about your module files:
as a name clash was detected nothing has been changed on disk.
The problem here is that without
the hashes the four different flavors of ``netlib-scalapack`` map to the same module file
name. We have the possibility to add suffixes to differentiate them:
.. code-block:: yaml
:emphasize-lines: 9-11,14-17
modules:
tcl:
hash_length: 0
whitelist:
- gcc
blacklist:
- '%gcc@4.8'
all:
suffixes:
'^openblas': openblas
'^netlib-lapack': netlib
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
netlib-scalapack:
suffixes:
'^openmpi': openmpi
'^mpich': mpich
As you can see it is possible to specify rules that applies only to a
restricted set of packages using :ref:`anonymous specs <anonymous_specs>`.
Regenerating module files now we obtain:
.. code-block:: console
$ spack module refresh --module-type tcl --delete-tree -y
==> Regenerating tcl module files
$ module avail
------------------------------------------------------------------------ ~/spack/share/spack/modules/linux-Ubuntu14-x86_64 ------------------------------------------------------------------------
bzip2-1.0.6-gcc-6.2.0 libpciaccess-0.13.4-gcc-6.2.0 mpich-3.2-gcc-6.2.0 openblas-0.2.19-gcc-6.2.0 python-2.7.12-gcc-6.2.0
cmake-3.5.2-gcc-6.2.0 libsigsegv-2.10-gcc-6.2.0 ncurses-6.0-gcc-6.2.0 openmpi-2.0.1-gcc-6.2.0 sqlite-3.8.5-gcc-6.2.0
curl-7.50.3-gcc-6.2.0 libtool-2.4.6-gcc-6.2.0 netlib-lapack-3.6.1-gcc-6.2.0 openssl-1.0.2j-gcc-6.2.0 util-macros-1.19.0-gcc-6.2.0
expat-2.2.0-gcc-6.2.0 libxml2-2.9.4-gcc-6.2.0 netlib-scalapack-2.0.2-gcc-6.2.0-netlib-mpich pkg-config-0.29.1-gcc-6.2.0 xz-5.2.2-gcc-6.2.0
gcc-6.2.0-gcc-4.8 lz4-131-gcc-6.2.0 netlib-scalapack-2.0.2-gcc-6.2.0-netlib-openmpi py-nose-1.3.7-gcc-6.2.0 zlib-1.2.8-gcc-6.2.0
gmp-6.1.1-gcc-6.2.0 lzma-4.32.7-gcc-6.2.0 netlib-scalapack-2.0.2-gcc-6.2.0-openblas-mpich py-numpy-1.11.1-gcc-6.2.0-openblas
hwloc-1.11.4-gcc-6.2.0 lzo-2.09-gcc-6.2.0 netlib-scalapack-2.0.2-gcc-6.2.0-openblas-openmpi py-scipy-0.18.1-gcc-6.2.0-openblas
libarchive-3.2.1-gcc-6.2.0 m4-1.4.17-gcc-6.2.0 nettle-3.2-gcc-6.2.0 py-setuptools-25.2.0-gcc-6.2.0
Finally we can set a ``naming_scheme`` to prevent users from loading
modules that refer to different flavors of the same library/application:
.. code-block:: yaml
:emphasize-lines: 4,10,11
modules:
tcl:
hash_length: 0
naming_scheme: '${PACKAGE}/${VERSION}-${COMPILERNAME}-${COMPILERVER}'
whitelist:
- gcc
blacklist:
- '%gcc@4.8'
all:
conflict:
- '${PACKAGE}'
suffixes:
'^openblas': openblas
'^netlib-lapack': netlib
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
netlib-scalapack:
suffixes:
'^openmpi': openmpi
'^mpich': mpich
The final result should look like:
.. code-block:: console
$ module avail
------------------------------------------------------------------------ ~/spack/share/spack/modules/linux-Ubuntu14-x86_64 ------------------------------------------------------------------------
bzip2/1.0.6-gcc-6.2.0 libpciaccess/0.13.4-gcc-6.2.0 mpich/3.2-gcc-6.2.0 openblas/0.2.19-gcc-6.2.0 python/2.7.12-gcc-6.2.0
cmake/3.5.2-gcc-6.2.0 libsigsegv/2.10-gcc-6.2.0 ncurses/6.0-gcc-6.2.0 openmpi/2.0.1-gcc-6.2.0 sqlite/3.8.5-gcc-6.2.0
curl/7.50.3-gcc-6.2.0 libtool/2.4.6-gcc-6.2.0 netlib-lapack/3.6.1-gcc-6.2.0 openssl/1.0.2j-gcc-6.2.0 util-macros/1.19.0-gcc-6.2.0
expat/2.2.0-gcc-6.2.0 libxml2/2.9.4-gcc-6.2.0 netlib-scalapack/2.0.2-gcc-6.2.0-netlib-mpich pkg-config/0.29.1-gcc-6.2.0 xz/5.2.2-gcc-6.2.0
gcc/6.2.0-gcc-4.8 lz4/131-gcc-6.2.0 netlib-scalapack/2.0.2-gcc-6.2.0-netlib-openmpi py-nose/1.3.7-gcc-6.2.0 zlib/1.2.8-gcc-6.2.0
gmp/6.1.1-gcc-6.2.0 lzma/4.32.7-gcc-6.2.0 netlib-scalapack/2.0.2-gcc-6.2.0-openblas-mpich py-numpy/1.11.1-gcc-6.2.0-openblas
hwloc/1.11.4-gcc-6.2.0 lzo/2.09-gcc-6.2.0 netlib-scalapack/2.0.2-gcc-6.2.0-openblas-openmpi (D) py-scipy/0.18.1-gcc-6.2.0-openblas
libarchive/3.2.1-gcc-6.2.0 m4/1.4.17-gcc-6.2.0 nettle/3.2-gcc-6.2.0 py-setuptools/25.2.0-gcc-6.2.0
.. note::
TCL specific directive
The directives ``naming_scheme`` and ``conflict`` are TCL specific and do not apply
to the ``dotkit`` or ``lmod`` sections in the configuration file.
------------------------------------
Add custom environment modifications
------------------------------------
At many sites it is customary to set an environment variable in a
package's module file that points to the folder in which the package
is installed. You can achieve this with Spack by adding an
``environment`` directive to the configuration file:
.. code-block:: yaml
:emphasize-lines: 17-19
modules:
tcl:
hash_length: 0
naming_scheme: '${PACKAGE}/${VERSION}-${COMPILERNAME}-${COMPILERVER}'
whitelist:
- gcc
blacklist:
- '%gcc@4.8'
all:
conflict:
- '${PACKAGE}'
suffixes:
'^openblas': openblas
'^netlib-lapack': netlib
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
environment:
set:
'${PACKAGE}_ROOT': '${PREFIX}'
netlib-scalapack:
suffixes:
'^openmpi': openmpi
'^mpich': mpich
There are many variable tokens available to use in the ``environment``
and ``naming_scheme`` directives, such as ``${PACKAGE}``,
``${VERSION}``, etc. (see the :meth:`~spack.spec.Spec.format` API
documentation for the complete list).
Regenerating the module files should result in something like:
.. code-block:: console
:emphasize-lines: 14
$ spack module refresh -y --module-type tcl
==> Regenerating tcl module files
$ module show gcc
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
~/spack/share/spack/modules/linux-Ubuntu14-x86_64/gcc/6.2.0-gcc-4.8:
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
whatis("gcc @6.2.0 ")
prepend_path("PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/bin")
prepend_path("CMAKE_PREFIX_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/")
prepend_path("MANPATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/share/man")
prepend_path("PKG_CONFIG_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/lib64/pkgconfig")
prepend_path("LD_LIBRARY_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/lib64")
setenv("GCC_ROOT","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u")
conflict("gcc")
help([[The GNU Compiler Collection includes front ends for C, C++, Objective-C,
Fortran, and Java.
]])
As you see the ``gcc`` module has the environment variable ``GCC_ROOT`` set.
Sometimes it's also useful to apply environment modifications selectively and target
only certain packages. You can, for instance set the common variables ``CC``, ``CXX``,
etc. in the ``gcc`` module file and apply other custom modifications to the
``openmpi`` modules as follows:
.. code-block:: yaml
:emphasize-lines: 20-32
modules:
tcl:
hash_length: 0
naming_scheme: '${PACKAGE}/${VERSION}-${COMPILERNAME}-${COMPILERVER}'
whitelist:
- gcc
blacklist:
- '%gcc@4.8'
all:
conflict:
- '${PACKAGE}'
suffixes:
'^openblas': openblas
'^netlib-lapack': netlib
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
environment:
set:
'${PACKAGE}_ROOT': '${PREFIX}'
gcc:
environment:
set:
CC: gcc
CXX: g++
FC: gfortran
F90: gfortran
F77: gfortran
openmpi:
environment:
set:
SLURM_MPI_TYPE: pmi2
OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
netlib-scalapack:
suffixes:
'^openmpi': openmpi
'^mpich': mpich
This time we will be more selective and regenerate only the ``gcc`` and
``openmpi`` module files:
.. code-block:: console
$ spack module refresh -y --module-type tcl gcc
==> Regenerating tcl module files
$ spack module refresh -y --module-type tcl openmpi
==> Regenerating tcl module files
$ module show gcc
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
~/spack/share/spack/modules/linux-Ubuntu14-x86_64/gcc/6.2.0-gcc-4.8:
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
whatis("gcc @6.2.0 ")
prepend_path("PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/bin")
prepend_path("CMAKE_PREFIX_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/")
prepend_path("MANPATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/share/man")
prepend_path("PKG_CONFIG_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/lib64/pkgconfig")
prepend_path("LD_LIBRARY_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u/lib64")
setenv("GCC_ROOT","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-4.8/gcc-6.2.0-twd5nqg33hrrssqclcfi5k42eccwxz5u")
setenv("CC","gcc")
setenv("CXX","g++")
setenv("F90","gfortran")
setenv("FC","gfortran")
setenv("F77","gfortran")
conflict("gcc")
help([[The GNU Compiler Collection includes front ends for C, C++, Objective-C,
Fortran, and Java.
]])
$ module show openmpi
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
~/spack/share/spack/modules/linux-Ubuntu14-x86_64/openmpi/2.0.1-gcc-6.2.0:
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
whatis("openmpi @2.0.1 ")
prepend_path("PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-6.2.0/openmpi-2.0.1-s3qbtbyh3y5y4gkchmhcuak7th44l53w/bin")
prepend_path("CMAKE_PREFIX_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-6.2.0/openmpi-2.0.1-s3qbtbyh3y5y4gkchmhcuak7th44l53w/")
prepend_path("LD_LIBRARY_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-6.2.0/openmpi-2.0.1-s3qbtbyh3y5y4gkchmhcuak7th44l53w/lib")
prepend_path("PKG_CONFIG_PATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-6.2.0/openmpi-2.0.1-s3qbtbyh3y5y4gkchmhcuak7th44l53w/lib/pkgconfig")
prepend_path("MANPATH","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-6.2.0/openmpi-2.0.1-s3qbtbyh3y5y4gkchmhcuak7th44l53w/share/man")
setenv("SLURM_MPI_TYPE","pmi2")
setenv("OMPI_MCA_BTL_OPENIB_WARN_DEFAULT_GID_PREFIX","0")
setenv("OPENMPI_ROOT","~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-6.2.0/openmpi-2.0.1-s3qbtbyh3y5y4gkchmhcuak7th44l53w")
conflict("openmpi")
help([[The Open MPI Project is an open source Message Passing Interface
implementation that is developed and maintained by a consortium of
academic, research, and industry partners. Open MPI is therefore able to
combine the expertise, technologies, and resources from all across the
High Performance Computing community in order to build the best MPI
library available. Open MPI offers advantages for system and software
vendors, application developers and computer science researchers.
]])
---------------------
Autoload dependencies
---------------------
Spack can also generate module files that contain code to load the
dependencies automatically. You can, for instance generate python
modules that load their dependencies by adding the ``autoload``
directive and assigning it the value ``direct``:
.. code-block:: yaml
:emphasize-lines: 37,38
modules:
tcl:
hash_length: 0
naming_scheme: '${PACKAGE}/${VERSION}-${COMPILERNAME}-${COMPILERVER}'
whitelist:
- gcc
blacklist:
- '%gcc@4.8'
all:
conflict:
- '${PACKAGE}'
suffixes:
'^openblas': openblas
'^netlib-lapack': netlib
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
environment:
set:
'${PACKAGE}_ROOT': '${PREFIX}'
gcc:
environment:
set:
CC: gcc
CXX: g++
FC: gfortran
F90: gfortran
F77: gfortran
openmpi:
environment:
set:
SLURM_MPI_TYPE: pmi2
OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
netlib-scalapack:
suffixes:
'^openmpi': openmpi
'^mpich': mpich
^python:
autoload: 'direct'
and regenerating the module files for every package that depends on ``python``:
.. code-block:: console
$ spack module refresh -y --module-type tcl ^python
==> Regenerating tcl module files
Now the ``py-scipy`` module will be:
.. code-block:: tcl
#%Module1.0
## Module file created by spack (https://github.com/LLNL/spack) on 2016-11-02 20:53:21.283547
##
## py-scipy@0.18.1%gcc@6.2.0 arch=linux-Ubuntu14-x86_64-e6uljfi
##
module-whatis "py-scipy @0.18.1"
proc ModulesHelp { } {
puts stderr "SciPy (pronounced "Sigh Pie") is a Scientific Library for Python. It"
puts stderr "provides many user-friendly and efficient numerical routines such as"
puts stderr "routines for numerical integration and optimization."
}
if ![ is-loaded python/2.7.12-gcc-6.2.0 ] {
puts stderr "Autoloading python/2.7.12-gcc-6.2.0"
module load python/2.7.12-gcc-6.2.0
}
if ![ is-loaded openblas/0.2.19-gcc-6.2.0 ] {
puts stderr "Autoloading openblas/0.2.19-gcc-6.2.0"
module load openblas/0.2.19-gcc-6.2.0
}
if ![ is-loaded py-numpy/1.11.1-gcc-6.2.0-openblas ] {
puts stderr "Autoloading py-numpy/1.11.1-gcc-6.2.0-openblas"
module load py-numpy/1.11.1-gcc-6.2.0-openblas
}
prepend-path CMAKE_PREFIX_PATH "~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-6.2.0/py-scipy-0.18.1-e6uljfiffgym4xvj6wveevqxfqnfb3gh/"
prepend-path LD_LIBRARY_PATH "~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-6.2.0/py-scipy-0.18.1-e6uljfiffgym4xvj6wveevqxfqnfb3gh/lib"
prepend-path PYTHONPATH "~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-6.2.0/py-scipy-0.18.1-e6uljfiffgym4xvj6wveevqxfqnfb3gh/lib/python2.7/site-packages"
setenv PY_SCIPY_ROOT "~/spack/opt/spack/linux-Ubuntu14-x86_64/gcc-6.2.0/py-scipy-0.18.1-e6uljfiffgym4xvj6wveevqxfqnfb3gh"
conflict py-scipy
and will contain code to autoload all the dependencies:
.. code-block:: console
$ module load py-scipy
Autoloading python/2.7.12-gcc-6.2.0
Autoloading openblas/0.2.19-gcc-6.2.0
Autoloading py-numpy/1.11.1-gcc-6.2.0-openblas
-----------------------------
Lua hierarchical module files
-----------------------------
In the final part of this tutorial you will modify ``modules.yaml`` to generate
Lua hierarchical module files. You will see that most of the directives used before
are also valid in the ``lmod`` context.
^^^^^^^^^^^^^^^^^
Core/Compiler/MPI
^^^^^^^^^^^^^^^^^
.. warning::
Only LMod supports Lua hierarchical module files
For this part of the tutorial you need to be using LMod to
manage your environment.
The most common hierarchy is the so called ``Core/Compiler/MPI``. To have an idea
how a hierarchy is organized you may refer to the
`Lmod guide <https://www.tacc.utexas.edu/research-development/tacc-projects/lmod/user-guide/module-hierarchy>`_.
Since ``lmod`` is not enabled by default, you need to add it to the list of
enabled module file generators. The other things you need to do are:
- change the ``tcl`` tag to ``lmod``
- remove ``tcl`` specific directives (``naming_scheme`` and ``conflict``)
- set which compilers are considered ``core``
- remove the ``mpi`` related suffixes (as they will be substituted by hierarchies)
After modifications the configuration file will be:
.. code-block:: yaml
:emphasize-lines: 2-6
modules:
enable::
- lmod
lmod:
core_compilers:
- 'gcc@4.8'
hash_length: 0
whitelist:
- gcc
blacklist:
- '%gcc@4.8'
all:
suffixes:
'^openblas': openblas
'^netlib-lapack': netlib
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
environment:
set:
'${PACKAGE}_ROOT': '${PREFIX}'
gcc:
environment:
set:
CC: gcc
CXX: g++
FC: gfortran
F90: gfortran
F77: gfortran
openmpi:
environment:
set:
SLURM_MPI_TYPE: pmi2
OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
.. note::
The double colon
The double colon after ``enable`` is intentional and it serves the
purpose of overriding the default list of enabled generators so
that only ``lmod`` will be active (see :ref:`the reference
manual <config-overrides>` for a more detailed explanation of
config scopes).
The directive ``core_compilers`` accepts a list of compilers : everything built
using these compilers will create a module in the ``Core`` part of the hierarchy. It is
common practice to put the OS provided compilers in the list and only build common utilities
and other compilers in ``Core``.
If you regenerate the module files
.. code-block:: console
$ spack module refresh --module-type lmod --delete-tree -y
and update ``MODULEPATH`` to point to the ``Core`` folder, and
list the available modules, you'll see:
.. code-block:: console
$ module unuse ~/spack/share/spack/modules/linux-Ubuntu14-x86_64
$ module use ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/Core
$ module avail
----------------------------------------------------------------------- ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/Core -----------------------------------------------------------------------
gcc/6.2.0
The only module visible now is ``gcc``. Loading that you will make
visible the ``Compiler`` part of the software stack that was built with ``gcc/6.2.0``:
.. code-block:: console
$ module load gcc
$ module avail
-------------------------------------------------------------------- ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/gcc/6.2.0 ---------------------------------------------------------------------
binutils/2.27 curl/7.50.3 hwloc/1.11.4 libtool/2.4.6 lzo/2.09 netlib-lapack/3.6.1 openssl/1.0.2j py-scipy/0.18.1-openblas util-macros/1.19.0
bison/3.0.4 expat/2.2.0 libarchive/3.2.1 libxml2/2.9.4 m4/1.4.17 nettle/3.2 pkg-config/0.29.1 py-setuptools/25.2.0 xz/5.2.2
bzip2/1.0.6 flex/2.6.0 libpciaccess/0.13.4 lz4/131 mpich/3.2 openblas/0.2.19 py-nose/1.3.7 python/2.7.12 zlib/1.2.8
cmake/3.6.1 gmp/6.1.1 libsigsegv/2.10 lzma/4.32.7 ncurses/6.0 openmpi/2.0.1 py-numpy/1.11.1-openblas sqlite/3.8.5
----------------------------------------------------------------------- ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/Core -----------------------------------------------------------------------
gcc/6.2.0 (L)
The same holds true for the ``MPI`` part of the stack, that you can enable by loading
either ``mpich`` or ``openmpi``. The nice features of LMod will become evident
once you'll try switching among different stacks:
.. code-block:: console
$ module load mpich
$ module avail
----------------------------------------------------------- ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/mpich/3.2-5n5xoep/gcc/6.2.0 ------------------------------------------------------------
netlib-scalapack/2.0.2-netlib netlib-scalapack/2.0.2-openblas (D)
-------------------------------------------------------------------- ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/gcc/6.2.0 ---------------------------------------------------------------------
binutils/2.27 curl/7.50.3 hwloc/1.11.4 libtool/2.4.6 lzo/2.09 netlib-lapack/3.6.1 openssl/1.0.2j py-scipy/0.18.1-openblas util-macros/1.19.0
bison/3.0.4 expat/2.2.0 libarchive/3.2.1 libxml2/2.9.4 m4/1.4.17 nettle/3.2 pkg-config/0.29.1 py-setuptools/25.2.0 xz/5.2.2
bzip2/1.0.6 flex/2.6.0 libpciaccess/0.13.4 lz4/131 mpich/3.2 (L) openblas/0.2.19 py-nose/1.3.7 python/2.7.12 zlib/1.2.8
cmake/3.6.1 gmp/6.1.1 libsigsegv/2.10 lzma/4.32.7 ncurses/6.0 openmpi/2.0.1 py-numpy/1.11.1-openblas sqlite/3.8.5
----------------------------------------------------------------------- ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/Core -----------------------------------------------------------------------
gcc/6.2.0 (L)
$ module load openblas netlib-scalapack/2.0.2-openblas
$ module list
Currently Loaded Modules:
1) gcc/6.2.0 2) mpich/3.2 3) openblas/0.2.19 4) netlib-scalapack/2.0.2-openblas
$ module load openmpi
Lmod is automatically replacing "mpich/3.2" with "openmpi/2.0.1"
Due to MODULEPATH changes the following have been reloaded:
1) netlib-scalapack/2.0.2-openblas
This layout is already a great improvement over the usual non-hierarchical layout,
but it still has an asymmetry: ``LAPACK`` providers are semantically the same as ``MPI``
providers, but they are still not part of the hierarchy. We'll see a possible solution
next.
.. Activate lmod and turn the previous modifications into lmod:
Add core compilers
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Extend the hierarchy to other virtual providers
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. warning::
This is an experimental feature
Having a hierarchy deeper than ``Core``/``Compiler``/``MPI`` is an experimental
feature, still not fully supported by ``module spider``,
see `here <https://github.com/TACC/Lmod/issues/114>`_. Furthermore its use
with hierarchies more complex than ``Core``/``Compiler``/``MPI``/``LAPACK``
has not been thoroughly tested in production environments.
Spack permits you to generate Lua hierarchical module files where users
can add an arbitrary list of virtual providers to the triplet
``Core``/``Compiler``/``MPI``. A configuration file like:
.. code-block:: yaml
:emphasize-lines: 7,8
modules:
enable::
- lmod
lmod:
core_compilers:
- 'gcc@4.8'
hierarchical_scheme:
- lapack
hash_length: 0
whitelist:
- gcc
blacklist:
- '%gcc@4.8'
- readline
all:
filter:
environment_blacklist: ['CPATH', 'LIBRARY_PATH']
environment:
set:
'${PACKAGE}_ROOT': '${PREFIX}'
gcc:
environment:
set:
CC: gcc
CXX: g++
FC: gfortran
F90: gfortran
F77: gfortran
openmpi:
environment:
set:
SLURM_MPI_TYPE: pmi2
OMPI_MCA_btl_openib_warn_default_gid_prefix: '0'
will add ``lapack`` providers to the mix. After the usual regeneration of module files:
.. code-block:: console
$ module purge
$ spack module refresh --module-type lmod --delete-tree -y
==> Regenerating lmod module files
you will have something like:
.. code-block:: console
$ module load gcc
$ module load openblas
$ module load openmpi
$ module avail
--------------------------------------------- ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/openblas/0.2.19-js33umc/openmpi/2.0.1-s3qbtby/gcc/6.2.0 ----------------------------------------------
netlib-scalapack/2.0.2
-------------------------------------------------------- ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/openblas/0.2.19-js33umc/gcc/6.2.0 ---------------------------------------------------------
py-numpy/1.11.1 py-scipy/0.18.1
-------------------------------------------------------------------- ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/gcc/6.2.0 ---------------------------------------------------------------------
binutils/2.27 curl/7.50.3 hwloc/1.11.4 libtool/2.4.6 lzo/2.09 netlib-lapack/3.6.1 openssl/1.0.2j python/2.7.12 zlib/1.2.8
bison/3.0.4 expat/2.2.0 libarchive/3.2.1 libxml2/2.9.4 m4/1.4.17 nettle/3.2 pkg-config/0.29.1 sqlite/3.8.5
bzip2/1.0.6 flex/2.6.0 libpciaccess/0.13.4 lz4/131 mpich/3.2 openblas/0.2.19 (L) py-nose/1.3.7 util-macros/1.19.0
cmake/3.6.1 gmp/6.1.1 libsigsegv/2.10 lzma/4.32.7 ncurses/6.0 openmpi/2.0.1 (L) py-setuptools/25.2.0 xz/5.2.2
----------------------------------------------------------------------- ~/spack/share/spack/lmod/linux-Ubuntu14-x86_64/Core -----------------------------------------------------------------------
gcc/6.2.0 (L)
Now both the ``MPI`` and the ``LAPACK`` providers are handled by LMod as hierarchies:
.. code-block:: console
$ module load py-numpy netlib-scalapack
$ module load mpich
Lmod is automatically replacing "openmpi/2.0.1" with "mpich/3.2"
Due to MODULEPATH changes the following have been reloaded:
1) netlib-scalapack/2.0.2
$ module load netlib-lapack
Lmod is automatically replacing "openblas/0.2.19" with "netlib-lapack/3.6.1"
Inactive Modules:
1) py-numpy
Due to MODULEPATH changes the following have been reloaded:
1) netlib-scalapack/2.0.2
making the use of tags to differentiate them unnecessary.
Note that because we only compiled ``py-numpy`` with ``openblas`` the module
is made inactive when we switch the ``LAPACK`` provider. The user
environment will now be consistent by design!

1255
lib/spack/docs/tutorial_sc16_spack_basics.rst
View File

File diff suppressed because it is too large Load Diff

263
lib/spack/docs/workflows.rst
View File

@@ -33,24 +33,12 @@ possible realization of a particular package, out of combinatorially
many other realizations. For example, here is a concrete spec
instantiated from ``curl``:
.. code-block:: console
curl@7.50.1%gcc@5.3.0 arch=linux-SuSE11-x86_64
^openssl@system%gcc@5.3.0 arch=linux-SuSE11-x86_64
^zlib@1.2.8%gcc@5.3.0 arch=linux-SuSE11-x86_64
.. command-output:: spack spec curl
Spack's core concretization algorithm generates concrete specs by
instantiating packages from its repo, based on a set of "hints",
including user input and the ``packages.yaml`` file. This algorithm
may be accessed at any time with the ``spack spec`` command. For
example:
.. code-block:: console
$ spack spec curl
curl@7.50.1%gcc@5.3.0 arch=linux-SuSE11-x86_64
^openssl@system%gcc@5.3.0 arch=linux-SuSE11-x86_64
^zlib@1.2.8%gcc@5.3.0 arch=linux-SuSE11-x86_64
may be accessed at any time with the ``spack spec`` command.
Every time Spack installs a package, that installation corresponds to
a concrete spec. Only a vanishingly small fraction of possible
@@ -68,7 +56,7 @@ variant, compiler, etc. For example, the following set is consistent:
.. code-block:: console
curl@7.50.1%gcc@5.3.0 arch=linux-SuSE11-x86_64
^openssl@system%gcc@5.3.0 arch=linux-SuSE11-x86_64
^openssl@1.0.2k%gcc@5.3.0 arch=linux-SuSE11-x86_64
^zlib@1.2.8%gcc@5.3.0 arch=linux-SuSE11-x86_64
zlib@1.2.8%gcc@5.3.0 arch=linux-SuSE11-x86_64
@@ -77,7 +65,7 @@ The following set is not consistent:
.. code-block:: console
curl@7.50.1%gcc@5.3.0 arch=linux-SuSE11-x86_64
^openssl@system%gcc@5.3.0 arch=linux-SuSE11-x86_64
^openssl@1.0.2k%gcc@5.3.0 arch=linux-SuSE11-x86_64
^zlib@1.2.8%gcc@5.3.0 arch=linux-SuSE11-x86_64
zlib@1.2.7%gcc@5.3.0 arch=linux-SuSE11-x86_64
@@ -182,7 +170,7 @@ of usage:
.. code-block:: sh
#!/bin/sh
#!/bin/bash
compilers=(
%gcc
@@ -367,7 +355,7 @@ Transitive Dependencies
In the script above, each ``spack module loads`` command generates a
*single* ``module load`` line. Transitive dependencies do not usually
need to be loaded, only modules the user needs in in ``$PATH``. This is
need to be loaded, only modules the user needs in ``$PATH``. This is
because Spack builds binaries with RPATH. Spack's RPATH policy has
some nice features:
@@ -488,10 +476,11 @@ if the view is built with hardlinks.
.. FIXME: reference the relocation work of Hegner and Gartung (PR #1013)
.. _cmd-spack-view:
""""""""""""""""""""""
Using Filesystem Views
""""""""""""""""""""""
""""""""""""""
``spack view``
""""""""""""""
A filesystem view is created, and packages are linked in, by the ``spack
view`` command's ``symlink`` and ``hardlink`` sub-commands. The
@@ -1039,6 +1028,232 @@ or filesystem views. However, it has some drawbacks:
integrate Spack explicitly in their workflow. Not all users are
willing to do this.
------------------------
Using Spack on Travis-CI
------------------------
Spack can be deployed as a provider for userland software in
`Travis-CI <https://http://travis-ci.org>`_.
A starting-point for a ``.travis.yml`` file can look as follows.
It uses `caching <https://docs.travis-ci.com/user/caching/>`_ for
already built environments, so make sure to clean the Travis cache if
you run into problems.
The main points that are implemented below:
#. Travis is detected as having up to 34 cores available, but only 2
are actually allocated for the user. We limit the parallelism of
the spack builds in the config.
(The Travis yaml parser is a bit buggy on the echo command.)
#. Builds over 10 minutes need to be prefixed with ``travis_wait``.
Alternatively, generate output once with ``spack install -v``.
#. Travis builds are non-interactive. This prevents using bash
aliases and functions for modules. We fix that by sourcing
``/etc/profile`` first (or running everything in a subshell with
``bash -l -c '...'``).
.. code-block:: yaml
language: cpp
sudo: false
dist: trusty
cache:
apt: true
directories:
- $HOME/.cache
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- g++-4.9
- environment-modules
env:
global:
- SPACK_ROOT: $HOME/.cache/spack
- PATH: $PATH:$HOME/.cache/spack/bin
before_install:
- export CXX=g++-4.9
- export CC=gcc-4.9
- export FC=gfortran-4.9
- export CXXFLAGS="-std=c++11"
install:
- if ! which spack >/dev/null; then
mkdir -p $SPACK_ROOT &&
git clone --depth 50 https://github.com/spack/spack.git $SPACK_ROOT &&
echo -e "config:""\n build_jobs:"" 2" > $SPACK_ROOT/etc/spack/config.yaml;
fi
- travis_wait spack install cmake@3.7.2~openssl~ncurses
- travis_wait spack install boost@1.62.0~graph~iostream~locale~log~wave
- spack clean -a
- source /etc/profile &&
source $SPACK_ROOT/share/spack/setup-env.sh
- spack load cmake
- spack load boost
script:
- mkdir -p $HOME/build
- cd $HOME/build
- cmake $TRAVIS_BUILD_DIR
- make -j 2
- make test
.. _workflow_create_docker_image:
-----------------------------------
Using Spack to Create Docker Images
-----------------------------------
Spack can be the ideal tool to set up images for Docker (and Singularity).
An example ``Dockerfile`` is given below, downloading the latest spack
version.
The following functionality is prepared:
#. Base image: the example starts from a minimal ubuntu.
#. Installing as root: docker images are usually set up as root.
Since some autotools scripts might complain about this being unsafe, we set
``FORCE_UNSAFE_CONFIGURE=1`` to avoid configure errors.
#. Pre-install the spack dependencies, including modules from the packages.
This avoids needing to build those from scratch via ``spack bootstrap``.
Package installs are followed by a clean-up of the system package index,
to avoid outdated information and it saves space.
#. Install spack in ``/usr/local``.
Add ``setup-env.sh`` to profile scripts, so commands in *login* shells
can use the whole spack functionality, including modules.
#. Install an example package (``tar``).
As with system package managers above, ``spack install`` commands should be
concatenated with a ``&& spack clean -a`` in order to keep image sizes small.
#. Add a startup hook to an *interactive login shell* so spack modules will be
usable.
In order to build and run the image, execute:
.. code-block:: bash
docker build -t spack .
docker run -it spack
.. code-block:: docker
FROM ubuntu:16.04
MAINTAINER Your Name <someone@example.com>
# general environment for docker
ENV DEBIAN_FRONTEND=noninteractive \
SPACK_ROOT=/usr/local \
FORCE_UNSAFE_CONFIGURE=1
# install minimal spack depedencies
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
autoconf \
build-essential \
ca-certificates \
coreutils \
curl \
environment-modules \
git \
python \
unzip \
vim \
&& rm -rf /var/lib/apt/lists/*
# load spack environment on login
RUN echo "source $SPACK_ROOT/share/spack/setup-env.sh" \
> /etc/profile.d/spack.sh
# spack settings
# note: if you wish to change default settings, add files alongside
# the Dockerfile with your desired settings. Then uncomment this line
#COPY packages.yaml modules.yaml $SPACK_ROOT/etc/spack/
# install spack
RUN curl -s -L https://api.github.com/repos/spack/spack/tarball \
| tar xzC $SPACK_ROOT --strip 1
# note: at this point one could also run ``spack bootstrap`` to avoid
# parts of the long apt-get install list above
# install software
RUN spack install tar \
&& spack clean -a
# need the modules already during image build?
#RUN /bin/bash -l -c ' \
# spack load tar \
# && which tar'
# image run hook: the -l will make sure /etc/profile environments are loaded
CMD /bin/bash -l
^^^^^^^^^^^^^^
Best Practices
^^^^^^^^^^^^^^
"""
MPI
"""
Due to the dependency on Fortran for OpenMPI, which is the spack default
implementation, consider adding ``gfortran`` to the ``apt-get install`` list.
Recent versions of OpenMPI will require you to pass ``--allow-run-as-root``
to your ``mpirun`` calls if started as root user inside Docker.
For execution on HPC clusters, it can be helpful to import the docker
image into Singularity in order to start a program with an *external*
MPI. Otherwise, also add ``openssh-server`` to the ``apt-get install`` list.
""""
CUDA
""""
Starting from CUDA 9.0, Nvidia provides minimal CUDA images based on
Ubuntu.
Please see `their instructions <https://hub.docker.com/r/nvidia/cuda/>`_.
Avoid double-installing CUDA by adding, e.g.
.. code-block:: yaml
packages:
cuda:
paths:
cuda@9.0.176%gcc@5.4.0 arch=linux-ubuntu16-x86_64: /usr/local/cuda
buildable: False
to your ``packages.yaml``.
Then ``COPY`` in that file into the image as in the example above.
Users will either need ``nvidia-docker`` or e.g. Singularity to *execute*
device kernels.
"""""""""""
Singularity
"""""""""""
Importing and running the image created above into
`Singularity <http://singularity.lbl.gov/>`_ works like a charm.
Just use the `docker bootstraping mechanism <http://singularity.lbl.gov/quickstart#bootstrap-recipes>`_:
.. code-block:: none
Bootstrap: docker
From: registry/user/image:tag
%runscript
exec /bin/bash -l
------------------
Upstream Bug Fixes
------------------
@@ -1055,7 +1270,7 @@ Buggy New Version
Sometimes, the old version of a package works fine, but a new version
is buggy. For example, it was once found that `Adios did not build
with hdf5@1.10 <https://github.com/LLNL/spack/issues/1683>`_. If the
with hdf5@1.10 <https://github.com/spack/spack/issues/1683>`_. If the
old version of ``hdf5`` will work with ``adios``, the suggested
procedure is:
@@ -1065,7 +1280,7 @@ procedure is:
.. code-block:: python
# Adios does not build with HDF5 1.10
# See: https://github.com/LLNL/spack/issues/1683
# See: https://github.com/spack/spack/issues/1683
depends_on('hdf5@:1.9')
#. Determine whether the problem is with ``hdf5`` or ``adios``, and
@@ -1078,7 +1293,7 @@ procedure is:
.. code-block:: python
# Adios up to v1.10.0 does not build with HDF5 1.10
# See: https://github.com/LLNL/spack/issues/1683
# See: https://github.com/spack/spack/issues/1683
depends_on('hdf5@:1.9', when='@:1.10.0')
depends_on('hdf5', when='@1.10.1:')

27
lib/spack/env/cc vendored
View File

@@ -1,14 +1,14 @@
#!/bin/bash
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
@@ -45,6 +45,7 @@ parameters=(
SPACK_PREFIX
SPACK_ENV_PATH
SPACK_DEBUG_LOG_DIR
SPACK_DEBUG_LOG_ID
SPACK_COMPILER_SPEC
SPACK_CC_RPATH_ARG
SPACK_CXX_RPATH_ARG
@@ -58,7 +59,7 @@ parameters=(
# The default compiler flags are passed from these variables:
# SPACK_CFLAGS, SPACK_CXXFLAGS, SPACK_FCFLAGS, SPACK_FFLAGS,
# SPACK_LDFLAGS, SPACK_LDLIBS
# Debug env var is optional; set to true for debug logging:
# Debug env var is optional; set to "TRUE" for debug logging:
# SPACK_DEBUG
# Test command is used to unit test the compiler script.
# SPACK_TEST_COMMAND
@@ -98,25 +99,25 @@ case "$command" in
cpp)
mode=cpp
;;
cc|c89|c99|gcc|clang|icc|pgcc|xlc)
cc|c89|c99|gcc|clang|icc|pgcc|xlc|xlc_r)
command="$SPACK_CC"
language="C"
comp="CC"
lang_flags=C
;;
c++|CC|g++|clang++|icpc|pgc++|xlc++)
c++|CC|g++|clang++|icpc|pgc++|xlc++|xlc++_r)
command="$SPACK_CXX"
language="C++"
comp="CXX"
lang_flags=CXX
;;
ftn|f90|fc|f95|gfortran|ifort|pgfortran|xlf90|nagfor)
ftn|f90|fc|f95|gfortran|flang|ifort|pgfortran|xlf90|xlf90_r|nagfor)
command="$SPACK_FC"
language="Fortran 90"
comp="FC"
lang_flags=F
;;
f77|gfortran|ifort|pgfortran|xlf|nagfor|ftn)
f77|gfortran|flang|ifort|pgfortran|xlf|xlf_r|nagfor|ftn)
command="$SPACK_F77"
language="Fortran 77"
comp="F77"
@@ -133,7 +134,7 @@ esac
# If any of the arguments below are present, then the mode is vcheck.
# In vcheck mode, nothing is added in terms of extra search paths or
# libraries.
if [[ -z $mode ]]; then
if [[ -z $mode ]] || [[ $mode == ld ]]; then
for arg in "$@"; do
if [[ $arg == -v || $arg == -V || $arg == --version || $arg == -dumpversion ]]; then
mode=vcheck
@@ -218,7 +219,7 @@ fi
add_rpaths=true
if [[ ($mode == ld || $mode == ccld) && "$SPACK_SHORT_SPEC" =~ "darwin" ]]; then
for arg in "$@"; do
if [[ ($arg == -r && $mode == ld) || ($arg == -Wl,-r && $mode == ccld) ]]; then
if [[ ($arg == -r && $mode == ld) || ($arg == -r && $mode == ccld) || ($arg == -Wl,-r && $mode == ccld) ]]; then
add_rpaths=false
break
fi
@@ -328,8 +329,10 @@ IFS=':' read -ra extra_rpaths <<< "$SPACK_COMPILER_EXTRA_RPATHS"
for extra_rpath in "${extra_rpaths[@]}"; do
if [[ $mode == ccld ]]; then
$add_rpaths && args=("$rpath$extra_rpath" "${args[@]}")
args=("-L$extra_rpath" "${args[@]}")
elif [[ $mode == ld ]]; then
$add_rpaths && args=("-rpath" "$extra_rpath" "${args[@]}")
args=("-L$extra_rpath" "${args[@]}")
fi
done
@@ -353,8 +356,8 @@ fi
# Write the input and output commands to debug logs if it's asked for.
#
if [[ $SPACK_DEBUG == TRUE ]]; then
input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.in.log"
output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_SHORT_SPEC.out.log"
input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.in.log"
output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.out.log"
echo "[$mode] $command $input_command" >> "$input_log"
echo "[$mode] ${full_command[@]}" >> "$output_log"
fi

1
lib/spack/env/clang/flang vendored Symbolic link
View File

@@ -0,0 +1 @@
../cc

1
lib/spack/env/xl_r/xlc++_r vendored Symbolic link
View File

@@ -0,0 +1 @@
../cc

1
lib/spack/env/xl_r/xlc_r vendored Symbolic link
View File

@@ -0,0 +1 @@
../cc

1
lib/spack/env/xl_r/xlf90_r vendored Symbolic link
View File

@@ -0,0 +1 @@
../cc

1
lib/spack/env/xl_r/xlf_r vendored Symbolic link
View File

@@ -0,0 +1 @@
../cc

8
lib/spack/external/__init__.py vendored
View File

@@ -1,13 +1,13 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Copyright (c) 2013-2017, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
@@ -33,6 +33,8 @@
functools: Used for implementation of total_ordering.
jinja2: A modern and designer-friendly templating language for Python
jsonschema: An implementation of JSON Schema for Python.
ordereddict: We include our own version to be Python 2.6 compatible.

2
lib/spack/external/_pytest/pytester.py vendored
View File

@@ -551,7 +551,7 @@ def syspathinsert(self, path=None):
def _possibly_invalidate_import_caches(self):
# invalidate caches if we can (py33 and above)
try:
import importlib
import importlib # nopyqver
except ImportError:
pass
else:

313
lib/spack/external/ctest_log_parser.py vendored Normal file
View File

@@ -0,0 +1,313 @@
# -----------------------------------------------------------------------------
# CMake - Cross Platform Makefile Generator
# Copyright 2000-2017 Kitware, Inc. and Contributors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# * Neither the name of Kitware, Inc. nor the names of Contributors
# may be used to endorse or promote products derived from this
# software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# -----------------------------------------------------------------------------
#
# The above copyright and license notice applies to distributions of
# CMake in source and binary form. Third-party software packages supplied
# with CMake under compatible licenses provide their own copyright notices
# documented in corresponding subdirectories or source files.
#
# -----------------------------------------------------------------------------
#
# CMake was initially developed by Kitware with the following sponsorship:
#
# * National Library of Medicine at the National Institutes of Health
# as part of the Insight Segmentation and Registration Toolkit (ITK).
#
# * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel
# Visualization Initiative.
#
# * National Alliance for Medical Image Computing (NAMIC) is funded by the
# National Institutes of Health through the NIH Roadmap for Medical
# Research, Grant U54 EB005149.
#
# * Kitware, Inc.
# -----------------------------------------------------------------------------
"""Functions to parse build logs and extract error messages.
This is a python port of the regular expressions CTest uses to parse log
files here:
https://github.com/Kitware/CMake/blob/master/Source/CTest/cmCTestBuildHandler.cxx
This file takes the regexes verbatim from there and adds some parsing
algorithms that duplicate the way CTest scrapes log files. To keep this
up to date with CTest, just make sure the ``*_matches`` and
``*_exceptions`` lists are kept up to date with CTest's build handler.
"""
import re
from six import StringIO
from six import string_types
error_matches = [
"^[Bb]us [Ee]rror",
"^[Ss]egmentation [Vv]iolation",
"^[Ss]egmentation [Ff]ault",
":.*[Pp]ermission [Dd]enied",
"([^ :]+):([0-9]+): ([^ \\t])",
"([^:]+): error[ \\t]*[0-9]+[ \\t]*:",
"^Error ([0-9]+):",
"^Fatal",
"^Error: ",
"^Error ",
"[0-9] ERROR: ",
"^\"[^\"]+\", line [0-9]+: [^Ww]",
"^cc[^C]*CC: ERROR File = ([^,]+), Line = ([0-9]+)",
"^ld([^:])*:([ \\t])*ERROR([^:])*:",
"^ild:([ \\t])*\\(undefined symbol\\)",
"([^ :]+) : (error|fatal error|catastrophic error)",
"([^:]+): (Error:|error|undefined reference|multiply defined)",
"([^:]+)\\(([^\\)]+)\\) ?: (error|fatal error|catastrophic error)",
"^fatal error C[0-9]+:",
": syntax error ",
"^collect2: ld returned 1 exit status",
"ld terminated with signal",
"Unsatisfied symbol",
"^Unresolved:",
"Undefined symbol",
"^Undefined[ \\t]+first referenced",
"^CMake Error.*:",
":[ \\t]cannot find",
":[ \\t]can't find",
": \\*\\*\\* No rule to make target [`'].*\\'. Stop",
": \\*\\*\\* No targets specified and no makefile found",
": Invalid loader fixup for symbol",
": Invalid fixups exist",
": Can't find library for",
": internal link edit command failed",
": Unrecognized option [`'].*\\'",
"\", line [0-9]+\\.[0-9]+: [0-9]+-[0-9]+ \\([^WI]\\)",
"ld: 0706-006 Cannot find or open library file: -l ",
"ild: \\(argument error\\) can't find library argument ::",
"^could not be found and will not be loaded.",
"s:616 string too big",
"make: Fatal error: ",
"ld: 0711-993 Error occurred while writing to the output file:",
"ld: fatal: ",
"final link failed:",
"make: \\*\\*\\*.*Error",
"make\\[.*\\]: \\*\\*\\*.*Error",
"\\*\\*\\* Error code",
"nternal error:",
"Makefile:[0-9]+: \\*\\*\\* .* Stop\\.",
": No such file or directory",
": Invalid argument",
"^The project cannot be built\\.",
"^\\[ERROR\\]",
"^Command .* failed with exit code",
]
error_exceptions = [
"instantiated from ",
"candidates are:",
": warning",
": \\(Warning\\)",
": note",
"Note:",
"makefile:",
"Makefile:",
":[ \\t]+Where:",
"([^ :]+):([0-9]+): Warning",
"------ Build started: .* ------",
]
#: Regexes to match file/line numbers in error/warning messages
warning_matches = [
"([^ :]+):([0-9]+): warning:",
"([^ :]+):([0-9]+): note:",
"^cc[^C]*CC: WARNING File = ([^,]+), Line = ([0-9]+)",
"^ld([^:])*:([ \\t])*WARNING([^:])*:",
"([^:]+): warning ([0-9]+):",
"^\"[^\"]+\", line [0-9]+: [Ww](arning|arnung)",
"([^:]+): warning[ \\t]*[0-9]+[ \\t]*:",
"^(Warning|Warnung) ([0-9]+):",
"^(Warning|Warnung)[ :]",
"WARNING: ",
"([^ :]+) : warning",
"([^:]+): warning",
"\", line [0-9]+\\.[0-9]+: [0-9]+-[0-9]+ \\([WI]\\)",
"^cxx: Warning:",
".*file: .* has no symbols",
"([^ :]+):([0-9]+): (Warning|Warnung)",
"\\([0-9]*\\): remark #[0-9]*",
"\".*\", line [0-9]+: remark\\([0-9]*\\):",
"cc-[0-9]* CC: REMARK File = .*, Line = [0-9]*",
"^CMake Warning.*:",
"^\\[WARNING\\]",
]
#: Regexes to match file/line numbers in error/warning messages
warning_exceptions = [
"/usr/.*/X11/Xlib\\.h:[0-9]+: war.*: ANSI C\\+\\+ forbids declaration",
"/usr/.*/X11/Xutil\\.h:[0-9]+: war.*: ANSI C\\+\\+ forbids declaration",
"/usr/.*/X11/XResource\\.h:[0-9]+: war.*: ANSI C\\+\\+ forbids declaration",
"WARNING 84 :",
"WARNING 47 :",
"makefile:",
"Makefile:",
"warning: Clock skew detected. Your build may be incomplete.",
"/usr/openwin/include/GL/[^:]+:",
"bind_at_load",
"XrmQGetResource",
"IceFlush",
"warning LNK4089: all references to [^ \\t]+ discarded by .OPT:REF",
"ld32: WARNING 85: definition of dataKey in",
"cc: warning 422: Unknown option \"\\+b",
"_with_warning_C",
]
#: Regexes to match file/line numbers in error/warning messages
file_line_matches = [
"^Warning W[0-9]+ ([a-zA-Z.\\:/0-9_+ ~-]+) ([0-9]+):",
"^([a-zA-Z./0-9_+ ~-]+):([0-9]+):",
"^([a-zA-Z.\\:/0-9_+ ~-]+)\\(([0-9]+)\\)",
"^[0-9]+>([a-zA-Z.\\:/0-9_+ ~-]+)\\(([0-9]+)\\)",
"^([a-zA-Z./0-9_+ ~-]+)\\(([0-9]+)\\)",
"\"([a-zA-Z./0-9_+ ~-]+)\", line ([0-9]+)",
"File = ([a-zA-Z./0-9_+ ~-]+), Line = ([0-9]+)",
]
class LogEvent(object):
"""Class representing interesting events (e.g., errors) in a build log."""
def __init__(self, text, line_no,
source_file=None, source_line_no=None,
pre_context=None, post_context=None):
self.text = text
self.line_no = line_no
self.source_file = source_file,
self.source_line_no = source_line_no,
self.pre_context = pre_context if pre_context is not None else []
self.post_context = post_context if post_context is not None else []
self.repeat_count = 0
@property
def start(self):
"""First line in the log with text for the event or its context."""
return self.line_no - len(self.pre_context)
@property
def end(self):
"""Last line in the log with text for event or its context."""
return self.line_no + len(self.post_context) + 1
def __getitem__(self, line_no):
"""Index event text and context by actual line number in file."""
if line_no == self.line_no:
return self.text
elif line_no < self.line_no:
return self.pre_context[line_no - self.line_no]
elif line_no > self.line_no:
return self.post_context[line_no - self.line_no - 1]
def __str__(self):
"""Returns event lines and context."""
out = StringIO()
for i in range(self.start, self.end):
if i == self.line_no:
out.write(' >> %-6d%s' % (i, self[i]))
else:
out.write(' %-6d%s' % (i, self[i]))
return out.getvalue()
class BuildError(LogEvent):
"""LogEvent subclass for build errors."""
class BuildWarning(LogEvent):
"""LogEvent subclass for build warnings."""
def _match(matches, exceptions, line):
"""True if line matches a regex in matches and none in exceptions."""
return (any(m.search(line) for m in matches) and
not any(e.search(line) for e in exceptions))
class CTestLogParser(object):
"""Log file parser that extracts errors and warnings."""
def __init__(self):
def compile(regex_array):
return [re.compile(regex) for regex in regex_array]
self.error_matches = compile(error_matches)
self.error_exceptions = compile(error_exceptions)
self.warning_matches = compile(warning_matches)
self.warning_exceptions = compile(warning_exceptions)
self.file_line_matches = compile(file_line_matches)
def parse(self, stream, context=6):
"""Parse a log file by searching each line for errors and warnings.
Args:
stream (str or file-like): filename or stream to read from
context (int): lines of context to extract around each log event
Returns:
(tuple): two lists containig ``BuildError`` and
``BuildWarning`` objects.
"""
if isinstance(stream, string_types):
with open(stream) as f:
return self.parse(f)
lines = [line for line in stream]
errors = []
warnings = []
for i, line in enumerate(lines):
# use CTest's regular expressions to scrape the log for events
if _match(self.error_matches, self.error_exceptions, line):
event = BuildError(line.strip(), i + 1)
errors.append(event)
elif _match(self.warning_matches, self.warning_exceptions, line):
event = BuildWarning(line.strip(), i + 1)
warnings.append(event)
else:
continue
# get file/line number for each event, if possible
for flm in self.file_line_matches:
match = flm.search(line)
if match:
event.source_file, source_line_no = match.groups()
# add log context, as well
event.pre_context = [
l.rstrip() for l in lines[i - context:i]]
event.post_context = [
l.rstrip() for l in lines[i + 1:i + context + 1]]
return errors, warnings

19
lib/spack/external/distro.py vendored
View File

@@ -993,13 +993,18 @@ def _get_distro_release_info(self):
continue
match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename)
if match:
filepath = os.path.join(_UNIXCONFDIR, basename)
distro_info = self._parse_distro_release_file(filepath)
if 'name' in distro_info:
# The name is always present if the pattern matches
self.distro_release_file = filepath
distro_info['id'] = match.group(1)
return distro_info
try:
filepath = os.path.join(_UNIXCONFDIR, basename)
distro_info = self._parse_distro_release_file(filepath)
if 'name' in distro_info:
# The name is always present if the pattern matches
self.distro_release_file = filepath
distro_info['id'] = match.group(1)
return distro_info
except IOError:
# We found a file we do not have permission to read
# Continue checking candidate files for distro file.
continue
return {}
def _parse_distro_release_file(self, filepath):

17
lib/spack/external/functools_backport.py vendored
View File

@@ -28,3 +28,20 @@ def total_ordering(cls):
opfunc.__doc__ = getattr(int, opname).__doc__
setattr(cls, opname, opfunc)
return cls
@total_ordering
class reverse_order(object):
"""Helper for creating key functions.
This is a wrapper that inverts the sense of the natural
comparisons on the object.
"""
def __init__(self, value):
self.value = value
def __eq__(self, other):
return other.value == self.value
def __lt__(self, other):
return other.value < self.value

33
lib/spack/external/jinja2/AUTHORS vendored Normal file
View File

@@ -0,0 +1,33 @@
Jinja is written and maintained by the Jinja Team and various
contributors:
Lead Developer:
- Armin Ronacher <armin.ronacher@active-4.com>
Developers:
- Christoph Hack
- Georg Brandl
Contributors:
- Bryan McLemore
- Mickaël Guérin <kael@crocobox.org>
- Cameron Knight
- Lawrence Journal-World.
- David Cramer
Patches and suggestions:
- Ronny Pfannschmidt
- Axel Böhm
- Alexey Melchakov
- Bryan McLemore
- Clovis Fabricio (nosklo)
- Cameron Knight
- Peter van Dijk (Habbie)
- Stefan Ebner
- Rene Leonhardt
- Thomas Waldmann
- Cory Benfield (Lukasa)

56
lib/spack/docs/exts/sphinxcontrib/LICENSE → lib/spack/external/jinja2/LICENSE vendored
View File

@@ -1,25 +1,31 @@
Copyright (c) 2010, 2011, 2012 Sebastian Wiesner <lunaryorn@googlemail.com>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details.
Some rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following
disclaimer in the documentation and/or other materials provided
with the distribution.
* The names of the contributors may not be used to endorse or
promote products derived from this software without specific
prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

51
lib/spack/external/jinja2/README.rst vendored Normal file
View File

@@ -0,0 +1,51 @@
Jinja2
~~~~~~
Jinja2 is a template engine written in pure Python. It provides a
`Django`_ inspired non-XML syntax but supports inline expressions and
an optional `sandboxed`_ environment.
Nutshell
--------
Here a small example of a Jinja template:
.. code-block:: jinja
{% extends 'base.html' %}
{% block title %}Memberlist{% endblock %}
{% block content %}
<ul>
{% for user in users %}
<li><a href="{{ user.url }}">{{ user.username }}</a></li>
{% endfor %}
</ul>
{% endblock %}
Philosophy
----------
Application logic is for the controller, but don't make the template designer's
life difficult by restricting functionality too much.
For more information visit the new `Jinja2 webpage`_ and `documentation`_.
The `Jinja2 tip`_ is installable via ``pip`` with ``pip install
https://github.com/pallets/jinja/zipball/master``.
.. _sandboxed: http://en.wikipedia.org/wiki/Sandbox_(computer_security)
.. _Django: http://www.djangoproject.com/
.. _Jinja2 webpage: http://jinja.pocoo.org/
.. _documentation: http://jinja.pocoo.org/docs/
.. _Jinja2 tip: http://jinja.pocoo.org/docs/intro/#as-a-python-egg-via-easy-install
Builds
------
+---------------------+------------------------------------------------------------------------------+
| ``master`` | .. image:: https://travis-ci.org/pallets/jinja.svg?branch=master |
| | :target: https://travis-ci.org/pallets/jinja |
+---------------------+------------------------------------------------------------------------------+
| ``2.9-maintenance`` | .. image:: https://travis-ci.org/pallets/jinja.svg?branch=2.9-maintenance |
| | :target: https://travis-ci.org/pallets/jinja |
+---------------------+------------------------------------------------------------------------------+

82
lib/spack/external/jinja2/__init__.py vendored Normal file
View File

@@ -0,0 +1,82 @@
# -*- coding: utf-8 -*-
"""
jinja2
~~~~~~
Jinja2 is a template engine written in pure Python. It provides a
Django inspired non-XML syntax but supports inline expressions and
an optional sandboxed environment.
Nutshell
--------
Here a small example of a Jinja2 template::
{% extends 'base.html' %}
{% block title %}Memberlist{% endblock %}
{% block content %}
<ul>
{% for user in users %}
<li><a href="{{ user.url }}">{{ user.username }}</a></li>
{% endfor %}
</ul>
{% endblock %}
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
__docformat__ = 'restructuredtext en'
__version__ = '2.9.6'
# high level interface
from jinja2.environment import Environment, Template
# loaders
from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \
DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \
ModuleLoader
# bytecode caches
from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \
MemcachedBytecodeCache
# undefined types
from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined, \
make_logging_undefined
# exceptions
from jinja2.exceptions import TemplateError, UndefinedError, \
TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \
TemplateAssertionError
# decorators and public utilities
from jinja2.filters import environmentfilter, contextfilter, \
evalcontextfilter
from jinja2.utils import Markup, escape, clear_caches, \
environmentfunction, evalcontextfunction, contextfunction, \
is_undefined, select_autoescape
__all__ = [
'Environment', 'Template', 'BaseLoader', 'FileSystemLoader',
'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader',
'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache',
'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined',
'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound',
'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError',
'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape',
'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined',
'evalcontextfilter', 'evalcontextfunction', 'make_logging_undefined',
'select_autoescape',
]
def _patch_async():
from jinja2.utils import have_async_gen
if have_async_gen:
from jinja2.asyncsupport import patch_all
patch_all()
_patch_async()
del _patch_async

99
lib/spack/external/jinja2/_compat.py vendored Normal file
View File

@@ -0,0 +1,99 @@
# -*- coding: utf-8 -*-
"""
jinja2._compat
~~~~~~~~~~~~~~
Some py2/py3 compatibility support based on a stripped down
version of six so we don't have to depend on a specific version
of it.
:copyright: Copyright 2013 by the Jinja team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import sys
PY2 = sys.version_info[0] == 2
PYPY = hasattr(sys, 'pypy_translation_info')
_identity = lambda x: x
if not PY2:
unichr = chr
range_type = range
text_type = str
string_types = (str,)
integer_types = (int,)
iterkeys = lambda d: iter(d.keys())
itervalues = lambda d: iter(d.values())
iteritems = lambda d: iter(d.items())
import pickle
from io import BytesIO, StringIO
NativeStringIO = StringIO
def reraise(tp, value, tb=None):
if value.__traceback__ is not tb:
raise value.with_traceback(tb)
raise value
ifilter = filter
imap = map
izip = zip
intern = sys.intern
implements_iterator = _identity
implements_to_string = _identity
encode_filename = _identity
else:
unichr = unichr
text_type = unicode
range_type = xrange
string_types = (str, unicode)
integer_types = (int, long)
iterkeys = lambda d: d.iterkeys()
itervalues = lambda d: d.itervalues()
iteritems = lambda d: d.iteritems()
import cPickle as pickle
from cStringIO import StringIO as BytesIO, StringIO
NativeStringIO = BytesIO
exec('def reraise(tp, value, tb=None):\n raise tp, value, tb')
from itertools import imap, izip, ifilter
intern = intern
def implements_iterator(cls):
cls.next = cls.__next__
del cls.__next__
return cls
def implements_to_string(cls):
cls.__unicode__ = cls.__str__
cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
return cls
def encode_filename(filename):
if isinstance(filename, unicode):
return filename.encode('utf-8')
return filename
def with_metaclass(meta, *bases):
"""Create a base class with a metaclass."""
# This requires a bit of explanation: the basic idea is to make a
# dummy metaclass for one level of class instantiation that replaces
# itself with the actual metaclass.
class metaclass(type):
def __new__(cls, name, this_bases, d):
return meta(name, bases, d)
return type.__new__(metaclass, 'temporary_class', (), {})
try:
from urllib.parse import quote_from_bytes as url_quote
except ImportError:
from urllib import quote as url_quote

71
lib/spack/external/jinja2/_stringdefs.py vendored Normal file
View File

File diff suppressed because one or more lines are too long

146
lib/spack/external/jinja2/asyncfilters.py vendored Normal file
View File

@@ -0,0 +1,146 @@
from functools import wraps
from jinja2.asyncsupport import auto_aiter
from jinja2 import filters
async def auto_to_seq(value):
seq = []
if hasattr(value, '__aiter__'):
async for item in value:
seq.append(item)
else:
for item in value:
seq.append(item)
return seq
async def async_select_or_reject(args, kwargs, modfunc, lookup_attr):
seq, func = filters.prepare_select_or_reject(
args, kwargs, modfunc, lookup_attr)
if seq:
async for item in auto_aiter(seq):
if func(item):
yield item
def dualfilter(normal_filter, async_filter):
wrap_evalctx = False
if getattr(normal_filter, 'environmentfilter', False):
is_async = lambda args: args[0].is_async
wrap_evalctx = False
else:
if not getattr(normal_filter, 'evalcontextfilter', False) and \
not getattr(normal_filter, 'contextfilter', False):
wrap_evalctx = True
is_async = lambda args: args[0].environment.is_async
@wraps(normal_filter)
def wrapper(*args, **kwargs):
b = is_async(args)
if wrap_evalctx:
args = args[1:]
if b:
return async_filter(*args, **kwargs)
return normal_filter(*args, **kwargs)
if wrap_evalctx:
wrapper.evalcontextfilter = True
wrapper.asyncfiltervariant = True
return wrapper
def asyncfiltervariant(original):
def decorator(f):
return dualfilter(original, f)
return decorator
@asyncfiltervariant(filters.do_first)
async def do_first(environment, seq):
try:
return await auto_aiter(seq).__anext__()
except StopAsyncIteration:
return environment.undefined('No first item, sequence was empty.')
@asyncfiltervariant(filters.do_groupby)
async def do_groupby(environment, value, attribute):
expr = filters.make_attrgetter(environment, attribute)
return [filters._GroupTuple(key, await auto_to_seq(values))
for key, values in filters.groupby(sorted(
await auto_to_seq(value), key=expr), expr)]
@asyncfiltervariant(filters.do_join)
async def do_join(eval_ctx, value, d=u'', attribute=None):
return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute)
@asyncfiltervariant(filters.do_list)
async def do_list(value):
return await auto_to_seq(value)
@asyncfiltervariant(filters.do_reject)
async def do_reject(*args, **kwargs):
return async_select_or_reject(args, kwargs, lambda x: not x, False)
@asyncfiltervariant(filters.do_rejectattr)
async def do_rejectattr(*args, **kwargs):
return async_select_or_reject(args, kwargs, lambda x: not x, True)
@asyncfiltervariant(filters.do_select)
async def do_select(*args, **kwargs):
return async_select_or_reject(args, kwargs, lambda x: x, False)
@asyncfiltervariant(filters.do_selectattr)
async def do_selectattr(*args, **kwargs):
return async_select_or_reject(args, kwargs, lambda x: x, True)
@asyncfiltervariant(filters.do_map)
async def do_map(*args, **kwargs):
seq, func = filters.prepare_map(args, kwargs)
if seq:
async for item in auto_aiter(seq):
yield func(item)
@asyncfiltervariant(filters.do_sum)
async def do_sum(environment, iterable, attribute=None, start=0):
rv = start
if attribute is not None:
func = filters.make_attrgetter(environment, attribute)
else:
func = lambda x: x
async for item in auto_aiter(iterable):
rv += func(item)
return rv
@asyncfiltervariant(filters.do_slice)
async def do_slice(value, slices, fill_with=None):
return filters.do_slice(await auto_to_seq(value), slices, fill_with)
ASYNC_FILTERS = {
'first': do_first,
'groupby': do_groupby,
'join': do_join,
'list': do_list,
# we intentionally do not support do_last because that would be
# ridiculous
'reject': do_reject,
'rejectattr': do_rejectattr,
'map': do_map,
'select': do_select,
'selectattr': do_selectattr,
'sum': do_sum,
'slice': do_slice,
}

254
lib/spack/external/jinja2/asyncsupport.py vendored Normal file
View File

@@ -0,0 +1,254 @@
# -*- coding: utf-8 -*-
"""
jinja2.asyncsupport
~~~~~~~~~~~~~~~~~~~
Has all the code for async support which is implemented as a patch
for supported Python versions.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
import sys
import asyncio
import inspect
from functools import update_wrapper
from jinja2.utils import concat, internalcode, Markup
from jinja2.environment import TemplateModule
from jinja2.runtime import LoopContextBase, _last_iteration
async def concat_async(async_gen):
rv = []
async def collect():
async for event in async_gen:
rv.append(event)
await collect()
return concat(rv)
async def generate_async(self, *args, **kwargs):
vars = dict(*args, **kwargs)
try:
async for event in self.root_render_func(self.new_context(vars)):
yield event
except Exception:
exc_info = sys.exc_info()
else:
return
yield self.environment.handle_exception(exc_info, True)
def wrap_generate_func(original_generate):
def _convert_generator(self, loop, args, kwargs):
async_gen = self.generate_async(*args, **kwargs)
try:
while 1:
yield loop.run_until_complete(async_gen.__anext__())
except StopAsyncIteration:
pass
def generate(self, *args, **kwargs):
if not self.environment.is_async:
return original_generate(self, *args, **kwargs)
return _convert_generator(self, asyncio.get_event_loop(), args, kwargs)
return update_wrapper(generate, original_generate)
async def render_async(self, *args, **kwargs):
if not self.environment.is_async:
raise RuntimeError('The environment was not created with async mode '
'enabled.')
vars = dict(*args, **kwargs)
ctx = self.new_context(vars)
try:
return await concat_async(self.root_render_func(ctx))
except Exception:
exc_info = sys.exc_info()
return self.environment.handle_exception(exc_info, True)
def wrap_render_func(original_render):
def render(self, *args, **kwargs):
if not self.environment.is_async:
return original_render(self, *args, **kwargs)
loop = asyncio.get_event_loop()
return loop.run_until_complete(self.render_async(*args, **kwargs))
return update_wrapper(render, original_render)
def wrap_block_reference_call(original_call):
@internalcode
async def async_call(self):
rv = await concat_async(self._stack[self._depth](self._context))
if self._context.eval_ctx.autoescape:
rv = Markup(rv)
return rv
@internalcode
def __call__(self):
if not self._context.environment.is_async:
return original_call(self)
return async_call(self)
return update_wrapper(__call__, original_call)
def wrap_macro_invoke(original_invoke):
@internalcode
async def async_invoke(self, arguments, autoescape):
rv = await self._func(*arguments)
if autoescape:
rv = Markup(rv)
return rv
@internalcode
def _invoke(self, arguments, autoescape):
if not self._environment.is_async:
return original_invoke(self, arguments, autoescape)
return async_invoke(self, arguments, autoescape)
return update_wrapper(_invoke, original_invoke)
@internalcode
async def get_default_module_async(self):
if self._module is not None:
return self._module
self._module = rv = await self.make_module_async()
return rv
def wrap_default_module(original_default_module):
@internalcode
def _get_default_module(self):
if self.environment.is_async:
raise RuntimeError('Template module attribute is unavailable '
'in async mode')
return original_default_module(self)
return _get_default_module
async def make_module_async(self, vars=None, shared=False, locals=None):
context = self.new_context(vars, shared, locals)
body_stream = []
async for item in self.root_render_func(context):
body_stream.append(item)
return TemplateModule(self, context, body_stream)
def patch_template():
from jinja2 import Template
Template.generate = wrap_generate_func(Template.generate)
Template.generate_async = update_wrapper(
generate_async, Template.generate_async)
Template.render_async = update_wrapper(
render_async, Template.render_async)
Template.render = wrap_render_func(Template.render)
Template._get_default_module = wrap_default_module(
Template._get_default_module)
Template._get_default_module_async = get_default_module_async
Template.make_module_async = update_wrapper(
make_module_async, Template.make_module_async)
def patch_runtime():
from jinja2.runtime import BlockReference, Macro
BlockReference.__call__ = wrap_block_reference_call(
BlockReference.__call__)
Macro._invoke = wrap_macro_invoke(Macro._invoke)
def patch_filters():
from jinja2.filters import FILTERS
from jinja2.asyncfilters import ASYNC_FILTERS
FILTERS.update(ASYNC_FILTERS)
def patch_all():
patch_template()
patch_runtime()
patch_filters()
async def auto_await(value):
if inspect.isawaitable(value):
return await value
return value
async def auto_aiter(iterable):
if hasattr(iterable, '__aiter__'):
async for item in iterable:
yield item
return
for item in iterable:
yield item
class AsyncLoopContext(LoopContextBase):
def __init__(self, async_iterator, after, length, recurse=None,
depth0=0):
LoopContextBase.__init__(self, recurse, depth0)
self._async_iterator = async_iterator
self._after = after
self._length = length
@property
def length(self):
if self._length is None:
raise TypeError('Loop length for some iterators cannot be '
'lazily calculated in async mode')
return self._length
def __aiter__(self):
return AsyncLoopContextIterator(self)
class AsyncLoopContextIterator(object):
__slots__ = ('context',)
def __init__(self, context):
self.context = context
def __aiter__(self):
return self
async def __anext__(self):
ctx = self.context
ctx.index0 += 1
if ctx._after is _last_iteration:
raise StopAsyncIteration()
next_elem = ctx._after
try:
ctx._after = await ctx._async_iterator.__anext__()
except StopAsyncIteration:
ctx._after = _last_iteration
return next_elem, ctx
async def make_async_loop_context(iterable, recurse=None, depth0=0):
# Length is more complicated and less efficient in async mode. The
# reason for this is that we cannot know if length will be used
# upfront but because length is a property we cannot lazily execute it
# later. This means that we need to buffer it up and measure :(
#
# We however only do this for actual iterators, not for async
# iterators as blocking here does not seem like the best idea in the
# world.
try:
length = len(iterable)
except (TypeError, AttributeError):
if not hasattr(iterable, '__aiter__'):
iterable = tuple(iterable)
length = len(iterable)
else:
length = None
async_iterator = auto_aiter(iterable)
try:
after = await async_iterator.__anext__()
except StopAsyncIteration:
after = _last_iteration
return AsyncLoopContext(async_iterator, after, length, recurse, depth0)

362
lib/spack/external/jinja2/bccache.py vendored Normal file
View File

@@ -0,0 +1,362 @@
# -*- coding: utf-8 -*-
"""
jinja2.bccache
~~~~~~~~~~~~~~
This module implements the bytecode cache system Jinja is optionally
using. This is useful if you have very complex template situations and
the compiliation of all those templates slow down your application too
much.
Situations where this is useful are often forking web applications that
are initialized on the first request.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD.
"""
from os import path, listdir
import os
import sys
import stat
import errno
import marshal
import tempfile
import fnmatch
from hashlib import sha1
from jinja2.utils import open_if_exists
from jinja2._compat import BytesIO, pickle, PY2, text_type
# marshal works better on 3.x, one hack less required
if not PY2:
marshal_dump = marshal.dump
marshal_load = marshal.load
else:
def marshal_dump(code, f):
if isinstance(f, file):
marshal.dump(code, f)
else:
f.write(marshal.dumps(code))
def marshal_load(f):
if isinstance(f, file):
return marshal.load(f)
return marshal.loads(f.read())
bc_version = 3
# magic version used to only change with new jinja versions. With 2.6
# we change this to also take Python version changes into account. The
# reason for this is that Python tends to segfault if fed earlier bytecode
# versions because someone thought it would be a good idea to reuse opcodes
# or make Python incompatible with earlier versions.
bc_magic = 'j2'.encode('ascii') + \
pickle.dumps(bc_version, 2) + \
pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1])
class Bucket(object):
"""Buckets are used to store the bytecode for one template. It's created
and initialized by the bytecode cache and passed to the loading functions.
The buckets get an internal checksum from the cache assigned and use this
to automatically reject outdated cache material. Individual bytecode
cache subclasses don't have to care about cache invalidation.
"""
def __init__(self, environment, key, checksum):
self.environment = environment
self.key = key
self.checksum = checksum
self.reset()
def reset(self):
"""Resets the bucket (unloads the bytecode)."""
self.code = None
def load_bytecode(self, f):
"""Loads bytecode from a file or file like object."""
# make sure the magic header is correct
magic = f.read(len(bc_magic))
if magic != bc_magic:
self.reset()
return
# the source code of the file changed, we need to reload
checksum = pickle.load(f)
if self.checksum != checksum:
self.reset()
return
# if marshal_load fails then we need to reload
try:
self.code = marshal_load(f)
except (EOFError, ValueError, TypeError):
self.reset()
return
def write_bytecode(self, f):
"""Dump the bytecode into the file or file like object passed."""
if self.code is None:
raise TypeError('can\'t write empty bucket')
f.write(bc_magic)
pickle.dump(self.checksum, f, 2)
marshal_dump(self.code, f)
def bytecode_from_string(self, string):
"""Load bytecode from a string."""
self.load_bytecode(BytesIO(string))
def bytecode_to_string(self):
"""Return the bytecode as string."""
out = BytesIO()
self.write_bytecode(out)
return out.getvalue()
class BytecodeCache(object):
"""To implement your own bytecode cache you have to subclass this class
and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of
these methods are passed a :class:`~jinja2.bccache.Bucket`.
A very basic bytecode cache that saves the bytecode on the file system::
from os import path
class MyCache(BytecodeCache):
def __init__(self, directory):
self.directory = directory
def load_bytecode(self, bucket):
filename = path.join(self.directory, bucket.key)
if path.exists(filename):
with open(filename, 'rb') as f:
bucket.load_bytecode(f)
def dump_bytecode(self, bucket):
filename = path.join(self.directory, bucket.key)
with open(filename, 'wb') as f:
bucket.write_bytecode(f)
A more advanced version of a filesystem based bytecode cache is part of
Jinja2.
"""
def load_bytecode(self, bucket):
"""Subclasses have to override this method to load bytecode into a
bucket. If they are not able to find code in the cache for the
bucket, it must not do anything.
"""
raise NotImplementedError()
def dump_bytecode(self, bucket):
"""Subclasses have to override this method to write the bytecode
from a bucket back to the cache. If it unable to do so it must not
fail silently but raise an exception.
"""
raise NotImplementedError()
def clear(self):
"""Clears the cache. This method is not used by Jinja2 but should be
implemented to allow applications to clear the bytecode cache used
by a particular environment.
"""
def get_cache_key(self, name, filename=None):
"""Returns the unique hash key for this template name."""
hash = sha1(name.encode('utf-8'))
if filename is not None:
filename = '|' + filename
if isinstance(filename, text_type):
filename = filename.encode('utf-8')
hash.update(filename)
return hash.hexdigest()
def get_source_checksum(self, source):
"""Returns a checksum for the source."""
return sha1(source.encode('utf-8')).hexdigest()
def get_bucket(self, environment, name, filename, source):
"""Return a cache bucket for the given template. All arguments are
mandatory but filename may be `None`.
"""
key = self.get_cache_key(name, filename)
checksum = self.get_source_checksum(source)
bucket = Bucket(environment, key, checksum)
self.load_bytecode(bucket)
return bucket
def set_bucket(self, bucket):
"""Put the bucket into the cache."""
self.dump_bytecode(bucket)
class FileSystemBytecodeCache(BytecodeCache):
"""A bytecode cache that stores bytecode on the filesystem. It accepts
two arguments: The directory where the cache items are stored and a
pattern string that is used to build the filename.
If no directory is specified a default cache directory is selected. On
Windows the user's temp directory is used, on UNIX systems a directory
is created for the user in the system temp directory.
The pattern can be used to have multiple separate caches operate on the
same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s``
is replaced with the cache key.
>>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache')
This bytecode cache supports clearing of the cache using the clear method.
"""
def __init__(self, directory=None, pattern='__jinja2_%s.cache'):
if directory is None:
directory = self._get_default_cache_dir()
self.directory = directory
self.pattern = pattern
def _get_default_cache_dir(self):
def _unsafe_dir():
raise RuntimeError('Cannot determine safe temp directory. You '
'need to explicitly provide one.')
tmpdir = tempfile.gettempdir()
# On windows the temporary directory is used specific unless
# explicitly forced otherwise. We can just use that.
if os.name == 'nt':
return tmpdir
if not hasattr(os, 'getuid'):
_unsafe_dir()
dirname = '_jinja2-cache-%d' % os.getuid()
actual_dir = os.path.join(tmpdir, dirname)
try:
os.mkdir(actual_dir, stat.S_IRWXU)
except OSError as e:
if e.errno != errno.EEXIST:
raise
try:
os.chmod(actual_dir, stat.S_IRWXU)
actual_dir_stat = os.lstat(actual_dir)
if actual_dir_stat.st_uid != os.getuid() \
or not stat.S_ISDIR(actual_dir_stat.st_mode) \
or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU:
_unsafe_dir()
except OSError as e:
if e.errno != errno.EEXIST:
raise
actual_dir_stat = os.lstat(actual_dir)
if actual_dir_stat.st_uid != os.getuid() \
or not stat.S_ISDIR(actual_dir_stat.st_mode) \
or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU:
_unsafe_dir()
return actual_dir
def _get_cache_filename(self, bucket):
return path.join(self.directory, self.pattern % bucket.key)
def load_bytecode(self, bucket):
f = open_if_exists(self._get_cache_filename(bucket), 'rb')
if f is not None:
try:
bucket.load_bytecode(f)
finally:
f.close()
def dump_bytecode(self, bucket):
f = open(self._get_cache_filename(bucket), 'wb')
try:
bucket.write_bytecode(f)
finally:
f.close()
def clear(self):
# imported lazily here because google app-engine doesn't support
# write access on the file system and the function does not exist
# normally.
from os import remove
files = fnmatch.filter(listdir(self.directory), self.pattern % '*')
for filename in files:
try:
remove(path.join(self.directory, filename))
except OSError:
pass
class MemcachedBytecodeCache(BytecodeCache):
"""This class implements a bytecode cache that uses a memcache cache for
storing the information. It does not enforce a specific memcache library
(tummy's memcache or cmemcache) but will accept any class that provides
the minimal interface required.
Libraries compatible with this class:
- `werkzeug <http://werkzeug.pocoo.org/>`_.contrib.cache
- `python-memcached <http://www.tummy.com/Community/software/python-memcached/>`_
- `cmemcache <http://gijsbert.org/cmemcache/>`_
(Unfortunately the django cache interface is not compatible because it
does not support storing binary data, only unicode. You can however pass
the underlying cache client to the bytecode cache which is available
as `django.core.cache.cache._client`.)
The minimal interface for the client passed to the constructor is this:
.. class:: MinimalClientInterface
.. method:: set(key, value[, timeout])
Stores the bytecode in the cache. `value` is a string and
`timeout` the timeout of the key. If timeout is not provided
a default timeout or no timeout should be assumed, if it's
provided it's an integer with the number of seconds the cache
item should exist.
.. method:: get(key)
Returns the value for the cache key. If the item does not
exist in the cache the return value must be `None`.
The other arguments to the constructor are the prefix for all keys that
is added before the actual cache key and the timeout for the bytecode in
the cache system. We recommend a high (or no) timeout.
This bytecode cache does not support clearing of used items in the cache.
The clear method is a no-operation function.
.. versionadded:: 2.7
Added support for ignoring memcache errors through the
`ignore_memcache_errors` parameter.
"""
def __init__(self, client, prefix='jinja2/bytecode/', timeout=None,
ignore_memcache_errors=True):
self.client = client
self.prefix = prefix
self.timeout = timeout
self.ignore_memcache_errors = ignore_memcache_errors
def load_bytecode(self, bucket):
try:
code = self.client.get(self.prefix + bucket.key)
except Exception:
if not self.ignore_memcache_errors:
raise
code = None
if code is not None:
bucket.bytecode_from_string(code)
def dump_bytecode(self, bucket):
args = (self.prefix + bucket.key, bucket.bytecode_to_string())
if self.timeout is not None:
args += (self.timeout,)
try:
self.client.set(*args)
except Exception:
if not self.ignore_memcache_errors:
raise

1653
lib/spack/external/jinja2/compiler.py vendored Normal file
View File

File diff suppressed because it is too large Load Diff

32
lib/spack/external/jinja2/constants.py vendored Normal file
View File

@@ -0,0 +1,32 @@
# -*- coding: utf-8 -*-
"""
jinja.constants
~~~~~~~~~~~~~~~
Various constants.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
#: list of lorem ipsum words used by the lipsum() helper function
LOREM_IPSUM_WORDS = u'''\
a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at
auctor augue bibendum blandit class commodo condimentum congue consectetuer
consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus
diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend
elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames
faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac
hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum
justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem
luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie
mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non
nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque
penatibus per pharetra phasellus placerat platea porta porttitor posuere
potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus
ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit
sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor
tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices
ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus
viverra volutpat vulputate'''

372
lib/spack/external/jinja2/debug.py vendored Normal file
View File

@@ -0,0 +1,372 @@
# -*- coding: utf-8 -*-
"""
jinja2.debug
~~~~~~~~~~~~
Implements the debug interface for Jinja. This module does some pretty
ugly stuff with the Python traceback system in order to achieve tracebacks
with correct line numbers, locals and contents.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
import sys
import traceback
from types import TracebackType, CodeType
from jinja2.utils import missing, internal_code
from jinja2.exceptions import TemplateSyntaxError
from jinja2._compat import iteritems, reraise, PY2
# on pypy we can take advantage of transparent proxies
try:
from __pypy__ import tproxy
except ImportError:
tproxy = None
# how does the raise helper look like?
try:
exec("raise TypeError, 'foo'")
except SyntaxError:
raise_helper = 'raise __jinja_exception__[1]'
except TypeError:
raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]'
class TracebackFrameProxy(object):
"""Proxies a traceback frame."""
def __init__(self, tb):
self.tb = tb
self._tb_next = None
@property
def tb_next(self):
return self._tb_next
def set_next(self, next):
if tb_set_next is not None:
try:
tb_set_next(self.tb, next and next.tb or None)
except Exception:
# this function can fail due to all the hackery it does
# on various python implementations. We just catch errors
# down and ignore them if necessary.
pass
self._tb_next = next
@property
def is_jinja_frame(self):
return '__jinja_template__' in self.tb.tb_frame.f_globals
def __getattr__(self, name):
return getattr(self.tb, name)
def make_frame_proxy(frame):
proxy = TracebackFrameProxy(frame)
if tproxy is None:
return proxy
def operation_handler(operation, *args, **kwargs):
if operation in ('__getattribute__', '__getattr__'):
return getattr(proxy, args[0])
elif operation == '__setattr__':
proxy.__setattr__(*args, **kwargs)
else:
return getattr(proxy, operation)(*args, **kwargs)
return tproxy(TracebackType, operation_handler)
class ProcessedTraceback(object):
"""Holds a Jinja preprocessed traceback for printing or reraising."""
def __init__(self, exc_type, exc_value, frames):
assert frames, 'no frames for this traceback?'
self.exc_type = exc_type
self.exc_value = exc_value
self.frames = frames
# newly concatenate the frames (which are proxies)
prev_tb = None
for tb in self.frames:
if prev_tb is not None:
prev_tb.set_next(tb)
prev_tb = tb
prev_tb.set_next(None)
def render_as_text(self, limit=None):
"""Return a string with the traceback."""
lines = traceback.format_exception(self.exc_type, self.exc_value,
self.frames[0], limit=limit)
return ''.join(lines).rstrip()
def render_as_html(self, full=False):
"""Return a unicode string with the traceback as rendered HTML."""
from jinja2.debugrenderer import render_traceback
return u'%s\n\n<!--\n%s\n-->' % (
render_traceback(self, full=full),
self.render_as_text().decode('utf-8', 'replace')
)
@property
def is_template_syntax_error(self):
"""`True` if this is a template syntax error."""
return isinstance(self.exc_value, TemplateSyntaxError)
@property
def exc_info(self):
"""Exception info tuple with a proxy around the frame objects."""
return self.exc_type, self.exc_value, self.frames[0]
@property
def standard_exc_info(self):
"""Standard python exc_info for re-raising"""
tb = self.frames[0]
# the frame will be an actual traceback (or transparent proxy) if
# we are on pypy or a python implementation with support for tproxy
if type(tb) is not TracebackType:
tb = tb.tb
return self.exc_type, self.exc_value, tb
def make_traceback(exc_info, source_hint=None):
"""Creates a processed traceback object from the exc_info."""
exc_type, exc_value, tb = exc_info
if isinstance(exc_value, TemplateSyntaxError):
exc_info = translate_syntax_error(exc_value, source_hint)
initial_skip = 0
else:
initial_skip = 1
return translate_exception(exc_info, initial_skip)
def translate_syntax_error(error, source=None):
"""Rewrites a syntax error to please traceback systems."""
error.source = source
error.translated = True
exc_info = (error.__class__, error, None)
filename = error.filename
if filename is None:
filename = '<unknown>'
return fake_exc_info(exc_info, filename, error.lineno)
def translate_exception(exc_info, initial_skip=0):
"""If passed an exc_info it will automatically rewrite the exceptions
all the way down to the correct line numbers and frames.
"""
tb = exc_info[2]
frames = []
# skip some internal frames if wanted
for x in range(initial_skip):
if tb is not None:
tb = tb.tb_next
initial_tb = tb
while tb is not None:
# skip frames decorated with @internalcode. These are internal
# calls we can't avoid and that are useless in template debugging
# output.
if tb.tb_frame.f_code in internal_code:
tb = tb.tb_next
continue
# save a reference to the next frame if we override the current
# one with a faked one.
next = tb.tb_next
# fake template exceptions
template = tb.tb_frame.f_globals.get('__jinja_template__')
if template is not None:
lineno = template.get_corresponding_lineno(tb.tb_lineno)
tb = fake_exc_info(exc_info[:2] + (tb,), template.filename,
lineno)[2]
frames.append(make_frame_proxy(tb))
tb = next
# if we don't have any exceptions in the frames left, we have to
# reraise it unchanged.
# XXX: can we backup here? when could this happen?
if not frames:
reraise(exc_info[0], exc_info[1], exc_info[2])
return ProcessedTraceback(exc_info[0], exc_info[1], frames)
def get_jinja_locals(real_locals):
ctx = real_locals.get('context')
if ctx:
locals = ctx.get_all()
else:
locals = {}
local_overrides = {}
for name, value in iteritems(real_locals):
if not name.startswith('l_') or value is missing:
continue
try:
_, depth, name = name.split('_', 2)
depth = int(depth)
except ValueError:
continue
cur_depth = local_overrides.get(name, (-1,))[0]
if cur_depth < depth:
local_overrides[name] = (depth, value)
for name, (_, value) in iteritems(local_overrides):
if value is missing:
locals.pop(name, None)
else:
locals[name] = value
return locals
def fake_exc_info(exc_info, filename, lineno):
"""Helper for `translate_exception`."""
exc_type, exc_value, tb = exc_info
# figure the real context out
if tb is not None:
locals = get_jinja_locals(tb.tb_frame.f_locals)
# if there is a local called __jinja_exception__, we get
# rid of it to not break the debug functionality.
locals.pop('__jinja_exception__', None)
else:
locals = {}
# assamble fake globals we need
globals = {
'__name__': filename,
'__file__': filename,
'__jinja_exception__': exc_info[:2],
# we don't want to keep the reference to the template around
# to not cause circular dependencies, but we mark it as Jinja
# frame for the ProcessedTraceback
'__jinja_template__': None
}
# and fake the exception
code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec')
# if it's possible, change the name of the code. This won't work
# on some python environments such as google appengine
try:
if tb is None:
location = 'template'
else:
function = tb.tb_frame.f_code.co_name
if function == 'root':
location = 'top-level template code'
elif function.startswith('block_'):
location = 'block "%s"' % function[6:]
else:
location = 'template'
if PY2:
code = CodeType(0, code.co_nlocals, code.co_stacksize,
code.co_flags, code.co_code, code.co_consts,
code.co_names, code.co_varnames, filename,
location, code.co_firstlineno,
code.co_lnotab, (), ())
else:
code = CodeType(0, code.co_kwonlyargcount,
code.co_nlocals, code.co_stacksize,
code.co_flags, code.co_code, code.co_consts,
code.co_names, code.co_varnames, filename,
location, code.co_firstlineno,
code.co_lnotab, (), ())
except Exception as e:
pass
# execute the code and catch the new traceback
try:
exec(code, globals, locals)
except:
exc_info = sys.exc_info()
new_tb = exc_info[2].tb_next
# return without this frame
return exc_info[:2] + (new_tb,)
def _init_ugly_crap():
"""This function implements a few ugly things so that we can patch the
traceback objects. The function returned allows resetting `tb_next` on
any python traceback object. Do not attempt to use this on non cpython
interpreters
"""
import ctypes
from types import TracebackType
if PY2:
# figure out size of _Py_ssize_t for Python 2:
if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'):
_Py_ssize_t = ctypes.c_int64
else:
_Py_ssize_t = ctypes.c_int
else:
# platform ssize_t on Python 3
_Py_ssize_t = ctypes.c_ssize_t
# regular python
class _PyObject(ctypes.Structure):
pass
_PyObject._fields_ = [
('ob_refcnt', _Py_ssize_t),
('ob_type', ctypes.POINTER(_PyObject))
]
# python with trace
if hasattr(sys, 'getobjects'):
class _PyObject(ctypes.Structure):
pass
_PyObject._fields_ = [
('_ob_next', ctypes.POINTER(_PyObject)),
('_ob_prev', ctypes.POINTER(_PyObject)),
('ob_refcnt', _Py_ssize_t),
('ob_type', ctypes.POINTER(_PyObject))
]
class _Traceback(_PyObject):
pass
_Traceback._fields_ = [
('tb_next', ctypes.POINTER(_Traceback)),
('tb_frame', ctypes.POINTER(_PyObject)),
('tb_lasti', ctypes.c_int),
('tb_lineno', ctypes.c_int)
]
def tb_set_next(tb, next):
"""Set the tb_next attribute of a traceback object."""
if not (isinstance(tb, TracebackType) and
(next is None or isinstance(next, TracebackType))):
raise TypeError('tb_set_next arguments must be traceback objects')
obj = _Traceback.from_address(id(tb))
if tb.tb_next is not None:
old = _Traceback.from_address(id(tb.tb_next))
old.ob_refcnt -= 1
if next is None:
obj.tb_next = ctypes.POINTER(_Traceback)()
else:
next = _Traceback.from_address(id(next))
next.ob_refcnt += 1
obj.tb_next = ctypes.pointer(next)
return tb_set_next
# try to get a tb_set_next implementation if we don't have transparent
# proxies.
tb_set_next = None
if tproxy is None:
try:
tb_set_next = _init_ugly_crap()
except:
pass
del _init_ugly_crap

54
lib/spack/external/jinja2/defaults.py vendored Normal file
View File

@@ -0,0 +1,54 @@
# -*- coding: utf-8 -*-
"""
jinja2.defaults
~~~~~~~~~~~~~~~
Jinja default filters and tags.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
from jinja2._compat import range_type
from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner
# defaults for the parser / lexer
BLOCK_START_STRING = '{%'
BLOCK_END_STRING = '%}'
VARIABLE_START_STRING = '{{'
VARIABLE_END_STRING = '}}'
COMMENT_START_STRING = '{#'
COMMENT_END_STRING = '#}'
LINE_STATEMENT_PREFIX = None
LINE_COMMENT_PREFIX = None
TRIM_BLOCKS = False
LSTRIP_BLOCKS = False
NEWLINE_SEQUENCE = '\n'
KEEP_TRAILING_NEWLINE = False
# default filters, tests and namespace
from jinja2.filters import FILTERS as DEFAULT_FILTERS
from jinja2.tests import TESTS as DEFAULT_TESTS
DEFAULT_NAMESPACE = {
'range': range_type,
'dict': dict,
'lipsum': generate_lorem_ipsum,
'cycler': Cycler,
'joiner': Joiner
}
# default policies
DEFAULT_POLICIES = {
'compiler.ascii_str': True,
'urlize.rel': 'noopener',
'urlize.target': None,
'truncate.leeway': 5,
'json.dumps_function': None,
'json.dumps_kwargs': {'sort_keys': True},
}
# export all constants
__all__ = tuple(x for x in locals().keys() if x.isupper())

1276
lib/spack/external/jinja2/environment.py vendored Normal file
View File

File diff suppressed because it is too large Load Diff

146
lib/spack/external/jinja2/exceptions.py vendored Normal file
View File

@@ -0,0 +1,146 @@
# -*- coding: utf-8 -*-
"""
jinja2.exceptions
~~~~~~~~~~~~~~~~~
Jinja exceptions.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
from jinja2._compat import imap, text_type, PY2, implements_to_string
class TemplateError(Exception):
"""Baseclass for all template errors."""
if PY2:
def __init__(self, message=None):
if message is not None:
message = text_type(message).encode('utf-8')
Exception.__init__(self, message)
@property
def message(self):
if self.args:
message = self.args[0]
if message is not None:
return message.decode('utf-8', 'replace')
def __unicode__(self):
return self.message or u''
else:
def __init__(self, message=None):
Exception.__init__(self, message)
@property
def message(self):
if self.args:
message = self.args[0]
if message is not None:
return message
@implements_to_string
class TemplateNotFound(IOError, LookupError, TemplateError):
"""Raised if a template does not exist."""
# looks weird, but removes the warning descriptor that just
# bogusly warns us about message being deprecated
message = None
def __init__(self, name, message=None):
IOError.__init__(self)
if message is None:
message = name
self.message = message
self.name = name
self.templates = [name]
def __str__(self):
return self.message
class TemplatesNotFound(TemplateNotFound):
"""Like :class:`TemplateNotFound` but raised if multiple templates
are selected. This is a subclass of :class:`TemplateNotFound`
exception, so just catching the base exception will catch both.
.. versionadded:: 2.2
"""
def __init__(self, names=(), message=None):
if message is None:
message = u'none of the templates given were found: ' + \
u', '.join(imap(text_type, names))
TemplateNotFound.__init__(self, names and names[-1] or None, message)
self.templates = list(names)
@implements_to_string
class TemplateSyntaxError(TemplateError):
"""Raised to tell the user that there is a problem with the template."""
def __init__(self, message, lineno, name=None, filename=None):
TemplateError.__init__(self, message)
self.lineno = lineno
self.name = name
self.filename = filename
self.source = None
# this is set to True if the debug.translate_syntax_error
# function translated the syntax error into a new traceback
self.translated = False
def __str__(self):
# for translated errors we only return the message
if self.translated:
return self.message
# otherwise attach some stuff
location = 'line %d' % self.lineno
name = self.filename or self.name
if name:
location = 'File "%s", %s' % (name, location)
lines = [self.message, ' ' + location]
# if the source is set, add the line to the output
if self.source is not None:
try:
line = self.source.splitlines()[self.lineno - 1]
except IndexError:
line = None
if line:
lines.append(' ' + line.strip())
return u'\n'.join(lines)
class TemplateAssertionError(TemplateSyntaxError):
"""Like a template syntax error, but covers cases where something in the
template caused an error at compile time that wasn't necessarily caused
by a syntax error. However it's a direct subclass of
:exc:`TemplateSyntaxError` and has the same attributes.
"""
class TemplateRuntimeError(TemplateError):
"""A generic runtime error in the template engine. Under some situations
Jinja may raise this exception.
"""
class UndefinedError(TemplateRuntimeError):
"""Raised if a template tries to operate on :class:`Undefined`."""
class SecurityError(TemplateRuntimeError):
"""Raised if a template tries to do something insecure if the
sandbox is enabled.
"""
class FilterArgumentError(TemplateRuntimeError):
"""This error is raised if a filter was called with inappropriate
arguments
"""

609
lib/spack/external/jinja2/ext.py vendored Normal file
View File

@@ -0,0 +1,609 @@
# -*- coding: utf-8 -*-
"""
jinja2.ext
~~~~~~~~~~
Jinja extensions allow to add custom tags similar to the way django custom
tags work. By default two example extensions exist: an i18n and a cache
extension.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD.
"""
from jinja2 import nodes
from jinja2.defaults import BLOCK_START_STRING, \
BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \
COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \
LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \
KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS
from jinja2.environment import Environment
from jinja2.runtime import concat
from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError
from jinja2.utils import contextfunction, import_string, Markup
from jinja2._compat import with_metaclass, string_types, iteritems
# the only real useful gettext functions for a Jinja template. Note
# that ugettext must be assigned to gettext as Jinja doesn't support
# non unicode strings.
GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext')
class ExtensionRegistry(type):
"""Gives the extension an unique identifier."""
def __new__(cls, name, bases, d):
rv = type.__new__(cls, name, bases, d)
rv.identifier = rv.__module__ + '.' + rv.__name__
return rv
class Extension(with_metaclass(ExtensionRegistry, object)):
"""Extensions can be used to add extra functionality to the Jinja template
system at the parser level. Custom extensions are bound to an environment
but may not store environment specific data on `self`. The reason for
this is that an extension can be bound to another environment (for
overlays) by creating a copy and reassigning the `environment` attribute.
As extensions are created by the environment they cannot accept any
arguments for configuration. One may want to work around that by using
a factory function, but that is not possible as extensions are identified
by their import name. The correct way to configure the extension is
storing the configuration values on the environment. Because this way the
environment ends up acting as central configuration storage the
attributes may clash which is why extensions have to ensure that the names
they choose for configuration are not too generic. ``prefix`` for example
is a terrible name, ``fragment_cache_prefix`` on the other hand is a good
name as includes the name of the extension (fragment cache).
"""
#: if this extension parses this is the list of tags it's listening to.
tags = set()
#: the priority of that extension. This is especially useful for
#: extensions that preprocess values. A lower value means higher
#: priority.
#:
#: .. versionadded:: 2.4
priority = 100
def __init__(self, environment):
self.environment = environment
def bind(self, environment):
"""Create a copy of this extension bound to another environment."""
rv = object.__new__(self.__class__)
rv.__dict__.update(self.__dict__)
rv.environment = environment
return rv
def preprocess(self, source, name, filename=None):
"""This method is called before the actual lexing and can be used to
preprocess the source. The `filename` is optional. The return value
must be the preprocessed source.
"""
return source
def filter_stream(self, stream):
"""It's passed a :class:`~jinja2.lexer.TokenStream` that can be used
to filter tokens returned. This method has to return an iterable of
:class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a
:class:`~jinja2.lexer.TokenStream`.
In the `ext` folder of the Jinja2 source distribution there is a file
called `inlinegettext.py` which implements a filter that utilizes this
method.
"""
return stream
def parse(self, parser):
"""If any of the :attr:`tags` matched this method is called with the
parser as first argument. The token the parser stream is pointing at
is the name token that matched. This method has to return one or a
list of multiple nodes.
"""
raise NotImplementedError()
def attr(self, name, lineno=None):
"""Return an attribute node for the current extension. This is useful
to pass constants on extensions to generated template code.
::
self.attr('_my_attribute', lineno=lineno)
"""
return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno)
def call_method(self, name, args=None, kwargs=None, dyn_args=None,
dyn_kwargs=None, lineno=None):
"""Call a method of the extension. This is a shortcut for
:meth:`attr` + :class:`jinja2.nodes.Call`.
"""
if args is None:
args = []
if kwargs is None:
kwargs = []
return nodes.Call(self.attr(name, lineno=lineno), args, kwargs,
dyn_args, dyn_kwargs, lineno=lineno)
@contextfunction
def _gettext_alias(__context, *args, **kwargs):
return __context.call(__context.resolve('gettext'), *args, **kwargs)
def _make_new_gettext(func):
@contextfunction
def gettext(__context, __string, **variables):
rv = __context.call(func, __string)
if __context.eval_ctx.autoescape:
rv = Markup(rv)
return rv % variables
return gettext
def _make_new_ngettext(func):
@contextfunction
def ngettext(__context, __singular, __plural, __num, **variables):
variables.setdefault('num', __num)
rv = __context.call(func, __singular, __plural, __num)
if __context.eval_ctx.autoescape:
rv = Markup(rv)
return rv % variables
return ngettext
class InternationalizationExtension(Extension):
"""This extension adds gettext support to Jinja2."""
tags = set(['trans'])
# TODO: the i18n extension is currently reevaluating values in a few
# situations. Take this example:
# {% trans count=something() %}{{ count }} foo{% pluralize
# %}{{ count }} fooss{% endtrans %}
# something is called twice here. One time for the gettext value and
# the other time for the n-parameter of the ngettext function.
def __init__(self, environment):
Extension.__init__(self, environment)
environment.globals['_'] = _gettext_alias
environment.extend(
install_gettext_translations=self._install,
install_null_translations=self._install_null,
install_gettext_callables=self._install_callables,
uninstall_gettext_translations=self._uninstall,
extract_translations=self._extract,
newstyle_gettext=False
)
def _install(self, translations, newstyle=None):
gettext = getattr(translations, 'ugettext', None)
if gettext is None:
gettext = translations.gettext
ngettext = getattr(translations, 'ungettext', None)
if ngettext is None:
ngettext = translations.ngettext
self._install_callables(gettext, ngettext, newstyle)
def _install_null(self, newstyle=None):
self._install_callables(
lambda x: x,
lambda s, p, n: (n != 1 and (p,) or (s,))[0],
newstyle
)
def _install_callables(self, gettext, ngettext, newstyle=None):
if newstyle is not None:
self.environment.newstyle_gettext = newstyle
if self.environment.newstyle_gettext:
gettext = _make_new_gettext(gettext)
ngettext = _make_new_ngettext(ngettext)
self.environment.globals.update(
gettext=gettext,
ngettext=ngettext
)
def _uninstall(self, translations):
for key in 'gettext', 'ngettext':
self.environment.globals.pop(key, None)
def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS):
if isinstance(source, string_types):
source = self.environment.parse(source)
return extract_from_ast(source, gettext_functions)
def parse(self, parser):
"""Parse a translatable tag."""
lineno = next(parser.stream).lineno
num_called_num = False
# find all the variables referenced. Additionally a variable can be
# defined in the body of the trans block too, but this is checked at
# a later state.
plural_expr = None
plural_expr_assignment = None
variables = {}
while parser.stream.current.type != 'block_end':
if variables:
parser.stream.expect('comma')
# skip colon for python compatibility
if parser.stream.skip_if('colon'):
break
name = parser.stream.expect('name')
if name.value in variables:
parser.fail('translatable variable %r defined twice.' %
name.value, name.lineno,
exc=TemplateAssertionError)
# expressions
if parser.stream.current.type == 'assign':
next(parser.stream)
variables[name.value] = var = parser.parse_expression()
else:
variables[name.value] = var = nodes.Name(name.value, 'load')
if plural_expr is None:
if isinstance(var, nodes.Call):
plural_expr = nodes.Name('_trans', 'load')
variables[name.value] = plural_expr
plural_expr_assignment = nodes.Assign(
nodes.Name('_trans', 'store'), var)
else:
plural_expr = var
num_called_num = name.value == 'num'
parser.stream.expect('block_end')
plural = plural_names = None
have_plural = False
referenced = set()
# now parse until endtrans or pluralize
singular_names, singular = self._parse_block(parser, True)
if singular_names:
referenced.update(singular_names)
if plural_expr is None:
plural_expr = nodes.Name(singular_names[0], 'load')
num_called_num = singular_names[0] == 'num'
# if we have a pluralize block, we parse that too
if parser.stream.current.test('name:pluralize'):
have_plural = True
next(parser.stream)
if parser.stream.current.type != 'block_end':
name = parser.stream.expect('name')
if name.value not in variables:
parser.fail('unknown variable %r for pluralization' %
name.value, name.lineno,
exc=TemplateAssertionError)
plural_expr = variables[name.value]
num_called_num = name.value == 'num'
parser.stream.expect('block_end')
plural_names, plural = self._parse_block(parser, False)
next(parser.stream)
referenced.update(plural_names)
else:
next(parser.stream)
# register free names as simple name expressions
for var in referenced:
if var not in variables:
variables[var] = nodes.Name(var, 'load')
if not have_plural:
plural_expr = None
elif plural_expr is None:
parser.fail('pluralize without variables', lineno)
node = self._make_node(singular, plural, variables, plural_expr,
bool(referenced),
num_called_num and have_plural)
node.set_lineno(lineno)
if plural_expr_assignment is not None:
return [plural_expr_assignment, node]
else:
return node
def _parse_block(self, parser, allow_pluralize):
"""Parse until the next block tag with a given name."""
referenced = []
buf = []
while 1:
if parser.stream.current.type == 'data':
buf.append(parser.stream.current.value.replace('%', '%%'))
next(parser.stream)
elif parser.stream.current.type == 'variable_begin':
next(parser.stream)
name = parser.stream.expect('name').value
referenced.append(name)
buf.append('%%(%s)s' % name)
parser.stream.expect('variable_end')
elif parser.stream.current.type == 'block_begin':
next(parser.stream)
if parser.stream.current.test('name:endtrans'):
break
elif parser.stream.current.test('name:pluralize'):
if allow_pluralize:
break
parser.fail('a translatable section can have only one '
'pluralize section')
parser.fail('control structures in translatable sections are '
'not allowed')
elif parser.stream.eos:
parser.fail('unclosed translation block')
else:
assert False, 'internal parser error'
return referenced, concat(buf)
def _make_node(self, singular, plural, variables, plural_expr,
vars_referenced, num_called_num):
"""Generates a useful node from the data provided."""
# no variables referenced? no need to escape for old style
# gettext invocations only if there are vars.
if not vars_referenced and not self.environment.newstyle_gettext:
singular = singular.replace('%%', '%')
if plural:
plural = plural.replace('%%', '%')
# singular only:
if plural_expr is None:
gettext = nodes.Name('gettext', 'load')
node = nodes.Call(gettext, [nodes.Const(singular)],
[], None, None)
# singular and plural
else:
ngettext = nodes.Name('ngettext', 'load')
node = nodes.Call(ngettext, [
nodes.Const(singular),
nodes.Const(plural),
plural_expr
], [], None, None)
# in case newstyle gettext is used, the method is powerful
# enough to handle the variable expansion and autoescape
# handling itself
if self.environment.newstyle_gettext:
for key, value in iteritems(variables):
# the function adds that later anyways in case num was
# called num, so just skip it.
if num_called_num and key == 'num':
continue
node.kwargs.append(nodes.Keyword(key, value))
# otherwise do that here
else:
# mark the return value as safe if we are in an
# environment with autoescaping turned on
node = nodes.MarkSafeIfAutoescape(node)
if variables:
node = nodes.Mod(node, nodes.Dict([
nodes.Pair(nodes.Const(key), value)
for key, value in variables.items()
]))
return nodes.Output([node])
class ExprStmtExtension(Extension):
"""Adds a `do` tag to Jinja2 that works like the print statement just
that it doesn't print the return value.
"""
tags = set(['do'])
def parse(self, parser):
node = nodes.ExprStmt(lineno=next(parser.stream).lineno)
node.node = parser.parse_tuple()
return node
class LoopControlExtension(Extension):
"""Adds break and continue to the template engine."""
tags = set(['break', 'continue'])
def parse(self, parser):
token = next(parser.stream)
if token.value == 'break':
return nodes.Break(lineno=token.lineno)
return nodes.Continue(lineno=token.lineno)
class WithExtension(Extension):
pass
class AutoEscapeExtension(Extension):
pass
def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS,
babel_style=True):
"""Extract localizable strings from the given template node. Per
default this function returns matches in babel style that means non string
parameters as well as keyword arguments are returned as `None`. This
allows Babel to figure out what you really meant if you are using
gettext functions that allow keyword arguments for placeholder expansion.
If you don't want that behavior set the `babel_style` parameter to `False`
which causes only strings to be returned and parameters are always stored
in tuples. As a consequence invalid gettext calls (calls without a single
string parameter or string parameters after non-string parameters) are
skipped.
This example explains the behavior:
>>> from jinja2 import Environment
>>> env = Environment()
>>> node = env.parse('{{ (_("foo"), _(), ngettext("foo", "bar", 42)) }}')
>>> list(extract_from_ast(node))
[(1, '_', 'foo'), (1, '_', ()), (1, 'ngettext', ('foo', 'bar', None))]
>>> list(extract_from_ast(node, babel_style=False))
[(1, '_', ('foo',)), (1, 'ngettext', ('foo', 'bar'))]
For every string found this function yields a ``(lineno, function,
message)`` tuple, where:
* ``lineno`` is the number of the line on which the string was found,
* ``function`` is the name of the ``gettext`` function used (if the
string was extracted from embedded Python code), and
* ``message`` is the string itself (a ``unicode`` object, or a tuple
of ``unicode`` objects for functions with multiple string arguments).
This extraction function operates on the AST and is because of that unable
to extract any comments. For comment support you have to use the babel
extraction interface or extract comments yourself.
"""
for node in node.find_all(nodes.Call):
if not isinstance(node.node, nodes.Name) or \
node.node.name not in gettext_functions:
continue
strings = []
for arg in node.args:
if isinstance(arg, nodes.Const) and \
isinstance(arg.value, string_types):
strings.append(arg.value)
else:
strings.append(None)
for arg in node.kwargs:
strings.append(None)
if node.dyn_args is not None:
strings.append(None)
if node.dyn_kwargs is not None:
strings.append(None)
if not babel_style:
strings = tuple(x for x in strings if x is not None)
if not strings:
continue
else:
if len(strings) == 1:
strings = strings[0]
else:
strings = tuple(strings)
yield node.lineno, node.node.name, strings
class _CommentFinder(object):
"""Helper class to find comments in a token stream. Can only
find comments for gettext calls forwards. Once the comment
from line 4 is found, a comment for line 1 will not return a
usable value.
"""
def __init__(self, tokens, comment_tags):
self.tokens = tokens
self.comment_tags = comment_tags
self.offset = 0
self.last_lineno = 0
def find_backwards(self, offset):
try:
for _, token_type, token_value in \
reversed(self.tokens[self.offset:offset]):
if token_type in ('comment', 'linecomment'):
try:
prefix, comment = token_value.split(None, 1)
except ValueError:
continue
if prefix in self.comment_tags:
return [comment.rstrip()]
return []
finally:
self.offset = offset
def find_comments(self, lineno):
if not self.comment_tags or self.last_lineno > lineno:
return []
for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]):
if token_lineno > lineno:
return self.find_backwards(self.offset + idx)
return self.find_backwards(len(self.tokens))
def babel_extract(fileobj, keywords, comment_tags, options):
"""Babel extraction method for Jinja templates.
.. versionchanged:: 2.3
Basic support for translation comments was added. If `comment_tags`
is now set to a list of keywords for extraction, the extractor will
try to find the best preceeding comment that begins with one of the
keywords. For best results, make sure to not have more than one
gettext call in one line of code and the matching comment in the
same line or the line before.
.. versionchanged:: 2.5.1
The `newstyle_gettext` flag can be set to `True` to enable newstyle
gettext calls.
.. versionchanged:: 2.7
A `silent` option can now be provided. If set to `False` template
syntax errors are propagated instead of being ignored.
:param fileobj: the file-like object the messages should be extracted from
:param keywords: a list of keywords (i.e. function names) that should be
recognized as translation functions
:param comment_tags: a list of translator tags to search for and include
in the results.
:param options: a dictionary of additional options (optional)
:return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
(comments will be empty currently)
"""
extensions = set()
for extension in options.get('extensions', '').split(','):
extension = extension.strip()
if not extension:
continue
extensions.add(import_string(extension))
if InternationalizationExtension not in extensions:
extensions.add(InternationalizationExtension)
def getbool(options, key, default=False):
return options.get(key, str(default)).lower() in \
('1', 'on', 'yes', 'true')
silent = getbool(options, 'silent', True)
environment = Environment(
options.get('block_start_string', BLOCK_START_STRING),
options.get('block_end_string', BLOCK_END_STRING),
options.get('variable_start_string', VARIABLE_START_STRING),
options.get('variable_end_string', VARIABLE_END_STRING),
options.get('comment_start_string', COMMENT_START_STRING),
options.get('comment_end_string', COMMENT_END_STRING),
options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX,
options.get('line_comment_prefix') or LINE_COMMENT_PREFIX,
getbool(options, 'trim_blocks', TRIM_BLOCKS),
getbool(options, 'lstrip_blocks', LSTRIP_BLOCKS),
NEWLINE_SEQUENCE,
getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE),
frozenset(extensions),
cache_size=0,
auto_reload=False
)
if getbool(options, 'newstyle_gettext'):
environment.newstyle_gettext = True
source = fileobj.read().decode(options.get('encoding', 'utf-8'))
try:
node = environment.parse(source)
tokens = list(environment.lex(environment.preprocess(source)))
except TemplateSyntaxError as e:
if not silent:
raise
# skip templates with syntax errors
return
finder = _CommentFinder(tokens, comment_tags)
for lineno, func, message in extract_from_ast(node, keywords):
yield lineno, func, message, finder.find_comments(lineno)
#: nicer import names
i18n = InternationalizationExtension
do = ExprStmtExtension
loopcontrols = LoopControlExtension
with_ = WithExtension
autoescape = AutoEscapeExtension

1073
lib/spack/external/jinja2/filters.py vendored Normal file
View File

File diff suppressed because it is too large Load Diff

273
lib/spack/external/jinja2/idtracking.py vendored Normal file
View File

@@ -0,0 +1,273 @@
from jinja2.visitor import NodeVisitor
from jinja2._compat import iteritems
VAR_LOAD_PARAMETER = 'param'
VAR_LOAD_RESOLVE = 'resolve'
VAR_LOAD_ALIAS = 'alias'
VAR_LOAD_UNDEFINED = 'undefined'
def find_symbols(nodes, parent_symbols=None):
sym = Symbols(parent=parent_symbols)
visitor = FrameSymbolVisitor(sym)
for node in nodes:
visitor.visit(node)
return sym
def symbols_for_node(node, parent_symbols=None):
sym = Symbols(parent=parent_symbols)
sym.analyze_node(node)
return sym
class Symbols(object):
def __init__(self, parent=None):
if parent is None:
self.level = 0
else:
self.level = parent.level + 1
self.parent = parent
self.refs = {}
self.loads = {}
self.stores = set()
def analyze_node(self, node, **kwargs):
visitor = RootVisitor(self)
visitor.visit(node, **kwargs)
def _define_ref(self, name, load=None):
ident = 'l_%d_%s' % (self.level, name)
self.refs[name] = ident
if load is not None:
self.loads[ident] = load
return ident
def find_load(self, target):
if target in self.loads:
return self.loads[target]
if self.parent is not None:
return self.parent.find_load(target)
def find_ref(self, name):
if name in self.refs:
return self.refs[name]
if self.parent is not None:
return self.parent.find_ref(name)
def ref(self, name):
rv = self.find_ref(name)
if rv is None:
raise AssertionError('Tried to resolve a name to a reference that '
'was unknown to the frame (%r)' % name)
return rv
def copy(self):
rv = object.__new__(self.__class__)
rv.__dict__.update(self.__dict__)
rv.refs = self.refs.copy()
rv.loads = self.loads.copy()
rv.stores = self.stores.copy()
return rv
def store(self, name):
self.stores.add(name)
# If we have not see the name referenced yet, we need to figure
# out what to set it to.
if name not in self.refs:
# If there is a parent scope we check if the name has a
# reference there. If it does it means we might have to alias
# to a variable there.
if self.parent is not None:
outer_ref = self.parent.find_ref(name)
if outer_ref is not None:
self._define_ref(name, load=(VAR_LOAD_ALIAS, outer_ref))
return
# Otherwise we can just set it to undefined.
self._define_ref(name, load=(VAR_LOAD_UNDEFINED, None))
def declare_parameter(self, name):
self.stores.add(name)
return self._define_ref(name, load=(VAR_LOAD_PARAMETER, None))
def load(self, name):
target = self.find_ref(name)
if target is None:
self._define_ref(name, load=(VAR_LOAD_RESOLVE, name))
def branch_update(self, branch_symbols):
stores = {}
for branch in branch_symbols:
for target in branch.stores:
if target in self.stores:
continue
stores[target] = stores.get(target, 0) + 1
for sym in branch_symbols:
self.refs.update(sym.refs)
self.loads.update(sym.loads)
self.stores.update(sym.stores)
for name, branch_count in iteritems(stores):
if branch_count == len(branch_symbols):
continue
target = self.find_ref(name)
assert target is not None, 'should not happen'
if self.parent is not None:
outer_target = self.parent.find_ref(name)
if outer_target is not None:
self.loads[target] = (VAR_LOAD_ALIAS, outer_target)
continue
self.loads[target] = (VAR_LOAD_RESOLVE, name)
def dump_stores(self):
rv = {}
node = self
while node is not None:
for name in node.stores:
if name not in rv:
rv[name] = self.find_ref(name)
node = node.parent
return rv
def dump_param_targets(self):
rv = set()
node = self
while node is not None:
for target, (instr, _) in iteritems(self.loads):
if instr == VAR_LOAD_PARAMETER:
rv.add(target)
node = node.parent
return rv
class RootVisitor(NodeVisitor):
def __init__(self, symbols):
self.sym_visitor = FrameSymbolVisitor(symbols)
def _simple_visit(self, node, **kwargs):
for child in node.iter_child_nodes():
self.sym_visitor.visit(child)
visit_Template = visit_Block = visit_Macro = visit_FilterBlock = \
visit_Scope = visit_If = visit_ScopedEvalContextModifier = \
_simple_visit
def visit_AssignBlock(self, node, **kwargs):
for child in node.body:
self.sym_visitor.visit(child)
def visit_CallBlock(self, node, **kwargs):
for child in node.iter_child_nodes(exclude=('call',)):
self.sym_visitor.visit(child)
def visit_For(self, node, for_branch='body', **kwargs):
if for_branch == 'body':
self.sym_visitor.visit(node.target, store_as_param=True)
branch = node.body
elif for_branch == 'else':
branch = node.else_
elif for_branch == 'test':
self.sym_visitor.visit(node.target, store_as_param=True)
if node.test is not None:
self.sym_visitor.visit(node.test)
return
else:
raise RuntimeError('Unknown for branch')
for item in branch or ():
self.sym_visitor.visit(item)
def visit_With(self, node, **kwargs):
for target in node.targets:
self.sym_visitor.visit(target)
for child in node.body:
self.sym_visitor.visit(child)
def generic_visit(self, node, *args, **kwargs):
raise NotImplementedError('Cannot find symbols for %r' %
node.__class__.__name__)
class FrameSymbolVisitor(NodeVisitor):
"""A visitor for `Frame.inspect`."""
def __init__(self, symbols):
self.symbols = symbols
def visit_Name(self, node, store_as_param=False, **kwargs):
"""All assignments to names go through this function."""
if store_as_param or node.ctx == 'param':
self.symbols.declare_parameter(node.name)
elif node.ctx == 'store':
self.symbols.store(node.name)
elif node.ctx == 'load':
self.symbols.load(node.name)
def visit_If(self, node, **kwargs):
self.visit(node.test, **kwargs)
original_symbols = self.symbols
def inner_visit(nodes):
self.symbols = rv = original_symbols.copy()
for subnode in nodes:
self.visit(subnode, **kwargs)
self.symbols = original_symbols
return rv
body_symbols = inner_visit(node.body)
else_symbols = inner_visit(node.else_ or ())
self.symbols.branch_update([body_symbols, else_symbols])
def visit_Macro(self, node, **kwargs):
self.symbols.store(node.name)
def visit_Import(self, node, **kwargs):
self.generic_visit(node, **kwargs)
self.symbols.store(node.target)
def visit_FromImport(self, node, **kwargs):
self.generic_visit(node, **kwargs)
for name in node.names:
if isinstance(name, tuple):
self.symbols.store(name[1])
else:
self.symbols.store(name)
def visit_Assign(self, node, **kwargs):
"""Visit assignments in the correct order."""
self.visit(node.node, **kwargs)
self.visit(node.target, **kwargs)
def visit_For(self, node, **kwargs):
"""Visiting stops at for blocks. However the block sequence
is visited as part of the outer scope.
"""
self.visit(node.iter, **kwargs)
def visit_CallBlock(self, node, **kwargs):
self.visit(node.call, **kwargs)
def visit_FilterBlock(self, node, **kwargs):
self.visit(node.filter, **kwargs)
def visit_With(self, node, **kwargs):
for target in node.values:
self.visit(target)
def visit_AssignBlock(self, node, **kwargs):
"""Stop visiting at block assigns."""
self.visit(node.target, **kwargs)
def visit_Scope(self, node, **kwargs):
"""Stop visiting at scopes."""
def visit_Block(self, node, **kwargs):
"""Stop visiting at blocks."""

737
lib/spack/external/jinja2/lexer.py vendored Normal file
View File

@@ -0,0 +1,737 @@
# -*- coding: utf-8 -*-
"""
jinja2.lexer
~~~~~~~~~~~~
This module implements a Jinja / Python combination lexer. The
`Lexer` class provided by this module is used to do some preprocessing
for Jinja.
On the one hand it filters out invalid operators like the bitshift
operators we don't allow in templates. On the other hand it separates
template code and python code in expressions.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
import re
import sys
from operator import itemgetter
from collections import deque
from jinja2.exceptions import TemplateSyntaxError
from jinja2.utils import LRUCache
from jinja2._compat import iteritems, implements_iterator, text_type, intern
# cache for the lexers. Exists in order to be able to have multiple
# environments with the same lexer
_lexer_cache = LRUCache(50)
# static regular expressions
whitespace_re = re.compile(r'\s+', re.U)
string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)
integer_re = re.compile(r'\d+')
def _make_name_re():
try:
compile('föö', '<unknown>', 'eval')
except SyntaxError:
return re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b')
import jinja2
from jinja2 import _stringdefs
name_re = re.compile(r'[%s][%s]*' % (_stringdefs.xid_start,
_stringdefs.xid_continue))
# Save some memory here
sys.modules.pop('jinja2._stringdefs')
del _stringdefs
del jinja2._stringdefs
return name_re
# we use the unicode identifier rule if this python version is able
# to handle unicode identifiers, otherwise the standard ASCII one.
name_re = _make_name_re()
del _make_name_re
float_re = re.compile(r'(?<!\.)\d+\.\d+')
newline_re = re.compile(r'(\r\n|\r|\n)')
# internal the tokens and keep references to them
TOKEN_ADD = intern('add')
TOKEN_ASSIGN = intern('assign')
TOKEN_COLON = intern('colon')
TOKEN_COMMA = intern('comma')
TOKEN_DIV = intern('div')
TOKEN_DOT = intern('dot')
TOKEN_EQ = intern('eq')
TOKEN_FLOORDIV = intern('floordiv')
TOKEN_GT = intern('gt')
TOKEN_GTEQ = intern('gteq')
TOKEN_LBRACE = intern('lbrace')
TOKEN_LBRACKET = intern('lbracket')
TOKEN_LPAREN = intern('lparen')
TOKEN_LT = intern('lt')
TOKEN_LTEQ = intern('lteq')
TOKEN_MOD = intern('mod')
TOKEN_MUL = intern('mul')
TOKEN_NE = intern('ne')
TOKEN_PIPE = intern('pipe')
TOKEN_POW = intern('pow')
TOKEN_RBRACE = intern('rbrace')
TOKEN_RBRACKET = intern('rbracket')
TOKEN_RPAREN = intern('rparen')
TOKEN_SEMICOLON = intern('semicolon')
TOKEN_SUB = intern('sub')
TOKEN_TILDE = intern('tilde')
TOKEN_WHITESPACE = intern('whitespace')
TOKEN_FLOAT = intern('float')
TOKEN_INTEGER = intern('integer')
TOKEN_NAME = intern('name')
TOKEN_STRING = intern('string')
TOKEN_OPERATOR = intern('operator')
TOKEN_BLOCK_BEGIN = intern('block_begin')
TOKEN_BLOCK_END = intern('block_end')
TOKEN_VARIABLE_BEGIN = intern('variable_begin')
TOKEN_VARIABLE_END = intern('variable_end')
TOKEN_RAW_BEGIN = intern('raw_begin')
TOKEN_RAW_END = intern('raw_end')
TOKEN_COMMENT_BEGIN = intern('comment_begin')
TOKEN_COMMENT_END = intern('comment_end')
TOKEN_COMMENT = intern('comment')
TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin')
TOKEN_LINESTATEMENT_END = intern('linestatement_end')
TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin')
TOKEN_LINECOMMENT_END = intern('linecomment_end')
TOKEN_LINECOMMENT = intern('linecomment')
TOKEN_DATA = intern('data')
TOKEN_INITIAL = intern('initial')
TOKEN_EOF = intern('eof')
# bind operators to token types
operators = {
'+': TOKEN_ADD,
'-': TOKEN_SUB,
'/': TOKEN_DIV,
'//': TOKEN_FLOORDIV,
'*': TOKEN_MUL,
'%': TOKEN_MOD,
'**': TOKEN_POW,
'~': TOKEN_TILDE,
'[': TOKEN_LBRACKET,
']': TOKEN_RBRACKET,
'(': TOKEN_LPAREN,
')': TOKEN_RPAREN,
'{': TOKEN_LBRACE,
'}': TOKEN_RBRACE,
'==': TOKEN_EQ,
'!=': TOKEN_NE,
'>': TOKEN_GT,
'>=': TOKEN_GTEQ,
'<': TOKEN_LT,
'<=': TOKEN_LTEQ,
'=': TOKEN_ASSIGN,
'.': TOKEN_DOT,
':': TOKEN_COLON,
'|': TOKEN_PIPE,
',': TOKEN_COMMA,
';': TOKEN_SEMICOLON
}
reverse_operators = dict([(v, k) for k, v in iteritems(operators)])
assert len(operators) == len(reverse_operators), 'operators dropped'
operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
sorted(operators, key=lambda x: -len(x))))
ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT,
TOKEN_COMMENT_END, TOKEN_WHITESPACE,
TOKEN_LINECOMMENT_BEGIN, TOKEN_LINECOMMENT_END,
TOKEN_LINECOMMENT])
ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA,
TOKEN_COMMENT, TOKEN_LINECOMMENT])
def _describe_token_type(token_type):
if token_type in reverse_operators:
return reverse_operators[token_type]
return {
TOKEN_COMMENT_BEGIN: 'begin of comment',
TOKEN_COMMENT_END: 'end of comment',
TOKEN_COMMENT: 'comment',
TOKEN_LINECOMMENT: 'comment',
TOKEN_BLOCK_BEGIN: 'begin of statement block',
TOKEN_BLOCK_END: 'end of statement block',
TOKEN_VARIABLE_BEGIN: 'begin of print statement',
TOKEN_VARIABLE_END: 'end of print statement',
TOKEN_LINESTATEMENT_BEGIN: 'begin of line statement',
TOKEN_LINESTATEMENT_END: 'end of line statement',
TOKEN_DATA: 'template data / text',
TOKEN_EOF: 'end of template'
}.get(token_type, token_type)
def describe_token(token):
"""Returns a description of the token."""
if token.type == 'name':
return token.value
return _describe_token_type(token.type)
def describe_token_expr(expr):
"""Like `describe_token` but for token expressions."""
if ':' in expr:
type, value = expr.split(':', 1)
if type == 'name':
return value
else:
type = expr
return _describe_token_type(type)
def count_newlines(value):
"""Count the number of newline characters in the string. This is
useful for extensions that filter a stream.
"""
return len(newline_re.findall(value))
def compile_rules(environment):
"""Compiles all the rules from the environment into a list of rules."""
e = re.escape
rules = [
(len(environment.comment_start_string), 'comment',
e(environment.comment_start_string)),
(len(environment.block_start_string), 'block',
e(environment.block_start_string)),
(len(environment.variable_start_string), 'variable',
e(environment.variable_start_string))
]
if environment.line_statement_prefix is not None:
rules.append((len(environment.line_statement_prefix), 'linestatement',
r'^[ \t\v]*' + e(environment.line_statement_prefix)))
if environment.line_comment_prefix is not None:
rules.append((len(environment.line_comment_prefix), 'linecomment',
r'(?:^|(?<=\S))[^\S\r\n]*' +
e(environment.line_comment_prefix)))
return [x[1:] for x in sorted(rules, reverse=True)]
class Failure(object):
"""Class that raises a `TemplateSyntaxError` if called.
Used by the `Lexer` to specify known errors.
"""
def __init__(self, message, cls=TemplateSyntaxError):
self.message = message
self.error_class = cls
def __call__(self, lineno, filename):
raise self.error_class(self.message, lineno, filename)
class Token(tuple):
"""Token class."""
__slots__ = ()
lineno, type, value = (property(itemgetter(x)) for x in range(3))
def __new__(cls, lineno, type, value):
return tuple.__new__(cls, (lineno, intern(str(type)), value))
def __str__(self):
if self.type in reverse_operators:
return reverse_operators[self.type]
elif self.type == 'name':
return self.value
return self.type
def test(self, expr):
"""Test a token against a token expression. This can either be a
token type or ``'token_type:token_value'``. This can only test
against string values and types.
"""
# here we do a regular string equality check as test_any is usually
# passed an iterable of not interned strings.
if self.type == expr:
return True
elif ':' in expr:
return expr.split(':', 1) == [self.type, self.value]
return False
def test_any(self, *iterable):
"""Test against multiple token expressions."""
for expr in iterable:
if self.test(expr):
return True
return False
def __repr__(self):
return 'Token(%r, %r, %r)' % (
self.lineno,
self.type,
self.value
)
@implements_iterator
class TokenStreamIterator(object):
"""The iterator for tokenstreams. Iterate over the stream
until the eof token is reached.
"""
def __init__(self, stream):
self.stream = stream
def __iter__(self):
return self
def __next__(self):
token = self.stream.current
if token.type is TOKEN_EOF:
self.stream.close()
raise StopIteration()
next(self.stream)
return token
@implements_iterator
class TokenStream(object):
"""A token stream is an iterable that yields :class:`Token`\\s. The
parser however does not iterate over it but calls :meth:`next` to go
one token ahead. The current active token is stored as :attr:`current`.
"""
def __init__(self, generator, name, filename):
self._iter = iter(generator)
self._pushed = deque()
self.name = name
self.filename = filename
self.closed = False
self.current = Token(1, TOKEN_INITIAL, '')
next(self)
def __iter__(self):
return TokenStreamIterator(self)
def __bool__(self):
return bool(self._pushed) or self.current.type is not TOKEN_EOF
__nonzero__ = __bool__ # py2
eos = property(lambda x: not x, doc="Are we at the end of the stream?")
def push(self, token):
"""Push a token back to the stream."""
self._pushed.append(token)
def look(self):
"""Look at the next token."""
old_token = next(self)
result = self.current
self.push(result)
self.current = old_token
return result
def skip(self, n=1):
"""Got n tokens ahead."""
for x in range(n):
next(self)
def next_if(self, expr):
"""Perform the token test and return the token if it matched.
Otherwise the return value is `None`.
"""
if self.current.test(expr):
return next(self)
def skip_if(self, expr):
"""Like :meth:`next_if` but only returns `True` or `False`."""
return self.next_if(expr) is not None
def __next__(self):
"""Go one token ahead and return the old one"""
rv = self.current
if self._pushed:
self.current = self._pushed.popleft()
elif self.current.type is not TOKEN_EOF:
try:
self.current = next(self._iter)
except StopIteration:
self.close()
return rv
def close(self):
"""Close the stream."""
self.current = Token(self.current.lineno, TOKEN_EOF, '')
self._iter = None
self.closed = True
def expect(self, expr):
"""Expect a given token type and return it. This accepts the same
argument as :meth:`jinja2.lexer.Token.test`.
"""
if not self.current.test(expr):
expr = describe_token_expr(expr)
if self.current.type is TOKEN_EOF:
raise TemplateSyntaxError('unexpected end of template, '
'expected %r.' % expr,
self.current.lineno,
self.name, self.filename)
raise TemplateSyntaxError("expected token %r, got %r" %
(expr, describe_token(self.current)),
self.current.lineno,
self.name, self.filename)
try:
return self.current
finally:
next(self)
def get_lexer(environment):
"""Return a lexer which is probably cached."""
key = (environment.block_start_string,
environment.block_end_string,
environment.variable_start_string,
environment.variable_end_string,
environment.comment_start_string,
environment.comment_end_string,
environment.line_statement_prefix,
environment.line_comment_prefix,
environment.trim_blocks,
environment.lstrip_blocks,
environment.newline_sequence,
environment.keep_trailing_newline)
lexer = _lexer_cache.get(key)
if lexer is None:
lexer = Lexer(environment)
_lexer_cache[key] = lexer
return lexer
class Lexer(object):
"""Class that implements a lexer for a given environment. Automatically
created by the environment class, usually you don't have to do that.
Note that the lexer is not automatically bound to an environment.
Multiple environments can share the same lexer.
"""
def __init__(self, environment):
# shortcuts
c = lambda x: re.compile(x, re.M | re.S)
e = re.escape
# lexing rules for tags
tag_rules = [
(whitespace_re, TOKEN_WHITESPACE, None),
(float_re, TOKEN_FLOAT, None),
(integer_re, TOKEN_INTEGER, None),
(name_re, TOKEN_NAME, None),
(string_re, TOKEN_STRING, None),
(operator_re, TOKEN_OPERATOR, None)
]
# assemble the root lexing rule. because "|" is ungreedy
# we have to sort by length so that the lexer continues working
# as expected when we have parsing rules like <% for block and
# <%= for variables. (if someone wants asp like syntax)
# variables are just part of the rules if variable processing
# is required.
root_tag_rules = compile_rules(environment)
# block suffix if trimming is enabled
block_suffix_re = environment.trim_blocks and '\\n?' or ''
# strip leading spaces if lstrip_blocks is enabled
prefix_re = {}
if environment.lstrip_blocks:
# use '{%+' to manually disable lstrip_blocks behavior
no_lstrip_re = e('+')
# detect overlap between block and variable or comment strings
block_diff = c(r'^%s(.*)' % e(environment.block_start_string))
# make sure we don't mistake a block for a variable or a comment
m = block_diff.match(environment.comment_start_string)
no_lstrip_re += m and r'|%s' % e(m.group(1)) or ''
m = block_diff.match(environment.variable_start_string)
no_lstrip_re += m and r'|%s' % e(m.group(1)) or ''
# detect overlap between comment and variable strings
comment_diff = c(r'^%s(.*)' % e(environment.comment_start_string))
m = comment_diff.match(environment.variable_start_string)
no_variable_re = m and r'(?!%s)' % e(m.group(1)) or ''
lstrip_re = r'^[ \t]*'
block_prefix_re = r'%s%s(?!%s)|%s\+?' % (
lstrip_re,
e(environment.block_start_string),
no_lstrip_re,
e(environment.block_start_string),
)
comment_prefix_re = r'%s%s%s|%s\+?' % (
lstrip_re,
e(environment.comment_start_string),
no_variable_re,
e(environment.comment_start_string),
)
prefix_re['block'] = block_prefix_re
prefix_re['comment'] = comment_prefix_re
else:
block_prefix_re = '%s' % e(environment.block_start_string)
self.newline_sequence = environment.newline_sequence
self.keep_trailing_newline = environment.keep_trailing_newline
# global lexing rules
self.rules = {
'root': [
# directives
(c('(.*?)(?:%s)' % '|'.join(
[r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % (
e(environment.block_start_string),
block_prefix_re,
e(environment.block_end_string),
e(environment.block_end_string)
)] + [
r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, prefix_re.get(n,r))
for n, r in root_tag_rules
])), (TOKEN_DATA, '#bygroup'), '#bygroup'),
# data
(c('.+'), TOKEN_DATA, None)
],
# comments
TOKEN_COMMENT_BEGIN: [
(c(r'(.*?)((?:\-%s\s*|%s)%s)' % (
e(environment.comment_end_string),
e(environment.comment_end_string),
block_suffix_re
)), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'),
(c('(.)'), (Failure('Missing end of comment tag'),), None)
],
# blocks
TOKEN_BLOCK_BEGIN: [
(c(r'(?:\-%s\s*|%s)%s' % (
e(environment.block_end_string),
e(environment.block_end_string),
block_suffix_re
)), TOKEN_BLOCK_END, '#pop'),
] + tag_rules,
# variables
TOKEN_VARIABLE_BEGIN: [
(c(r'\-%s\s*|%s' % (
e(environment.variable_end_string),
e(environment.variable_end_string)
)), TOKEN_VARIABLE_END, '#pop')
] + tag_rules,
# raw block
TOKEN_RAW_BEGIN: [
(c(r'(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % (
e(environment.block_start_string),
block_prefix_re,
e(environment.block_end_string),
e(environment.block_end_string),
block_suffix_re
)), (TOKEN_DATA, TOKEN_RAW_END), '#pop'),
(c('(.)'), (Failure('Missing end of raw directive'),), None)
],
# line statements
TOKEN_LINESTATEMENT_BEGIN: [
(c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop')
] + tag_rules,
# line comments
TOKEN_LINECOMMENT_BEGIN: [
(c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT,
TOKEN_LINECOMMENT_END), '#pop')
]
}
def _normalize_newlines(self, value):
"""Called for strings and template data to normalize it to unicode."""
return newline_re.sub(self.newline_sequence, value)
def tokenize(self, source, name=None, filename=None, state=None):
"""Calls tokeniter + tokenize and wraps it in a token stream.
"""
stream = self.tokeniter(source, name, filename, state)
return TokenStream(self.wrap(stream, name, filename), name, filename)
def wrap(self, stream, name=None, filename=None):
"""This is called with the stream as returned by `tokenize` and wraps
every token in a :class:`Token` and converts the value.
"""
for lineno, token, value in stream:
if token in ignored_tokens:
continue
elif token == 'linestatement_begin':
token = 'block_begin'
elif token == 'linestatement_end':
token = 'block_end'
# we are not interested in those tokens in the parser
elif token in ('raw_begin', 'raw_end'):
continue
elif token == 'data':
value = self._normalize_newlines(value)
elif token == 'keyword':
token = value
elif token == 'name':
value = str(value)
elif token == 'string':
# try to unescape string
try:
value = self._normalize_newlines(value[1:-1]) \
.encode('ascii', 'backslashreplace') \
.decode('unicode-escape')
except Exception as e:
msg = str(e).split(':')[-1].strip()
raise TemplateSyntaxError(msg, lineno, name, filename)
elif token == 'integer':
value = int(value)
elif token == 'float':
value = float(value)
elif token == 'operator':
token = operators[value]
yield Token(lineno, token, value)
def tokeniter(self, source, name, filename=None, state=None):
"""This method tokenizes the text and returns the tokens in a
generator. Use this method if you just want to tokenize a template.
"""
source = text_type(source)
lines = source.splitlines()
if self.keep_trailing_newline and source:
for newline in ('\r\n', '\r', '\n'):
if source.endswith(newline):
lines.append('')
break
source = '\n'.join(lines)
pos = 0
lineno = 1
stack = ['root']
if state is not None and state != 'root':
assert state in ('variable', 'block'), 'invalid state'
stack.append(state + '_begin')
else:
state = 'root'
statetokens = self.rules[stack[-1]]
source_length = len(source)
balancing_stack = []
while 1:
# tokenizer loop
for regex, tokens, new_state in statetokens:
m = regex.match(source, pos)
# if no match we try again with the next rule
if m is None:
continue
# we only match blocks and variables if braces / parentheses
# are balanced. continue parsing with the lower rule which
# is the operator rule. do this only if the end tags look
# like operators
if balancing_stack and \
tokens in ('variable_end', 'block_end',
'linestatement_end'):
continue
# tuples support more options
if isinstance(tokens, tuple):
for idx, token in enumerate(tokens):
# failure group
if token.__class__ is Failure:
raise token(lineno, filename)
# bygroup is a bit more complex, in that case we
# yield for the current token the first named
# group that matched
elif token == '#bygroup':
for key, value in iteritems(m.groupdict()):
if value is not None:
yield lineno, key, value
lineno += value.count('\n')
break
else:
raise RuntimeError('%r wanted to resolve '
'the token dynamically'
' but no group matched'
% regex)
# normal group
else:
data = m.group(idx + 1)
if data or token not in ignore_if_empty:
yield lineno, token, data
lineno += data.count('\n')
# strings as token just are yielded as it.
else:
data = m.group()
# update brace/parentheses balance
if tokens == 'operator':
if data == '{':
balancing_stack.append('}')
elif data == '(':
balancing_stack.append(')')
elif data == '[':
balancing_stack.append(']')
elif data in ('}', ')', ']'):
if not balancing_stack:
raise TemplateSyntaxError('unexpected \'%s\'' %
data, lineno, name,
filename)
expected_op = balancing_stack.pop()
if expected_op != data:
raise TemplateSyntaxError('unexpected \'%s\', '
'expected \'%s\'' %
(data, expected_op),
lineno, name,
filename)
# yield items
if data or tokens not in ignore_if_empty:
yield lineno, tokens, data
lineno += data.count('\n')
# fetch new position into new variable so that we can check
# if there is a internal parsing error which would result
# in an infinite loop
pos2 = m.end()
# handle state changes
if new_state is not None:
# remove the uppermost state
if new_state == '#pop':
stack.pop()
# resolve the new state by group checking
elif new_state == '#bygroup':
for key, value in iteritems(m.groupdict()):
if value is not None:
stack.append(key)
break
else:
raise RuntimeError('%r wanted to resolve the '
'new state dynamically but'
' no group matched' %
regex)
# direct state name given
else:
stack.append(new_state)
statetokens = self.rules[stack[-1]]
# we are still at the same position and no stack change.
# this means a loop without break condition, avoid that and
# raise error
elif pos2 == pos:
raise RuntimeError('%r yielded empty string without '
'stack change' % regex)
# publish new function and start again
pos = pos2
break
# if loop terminated without break we haven't found a single match
# either we are at the end of the file or we have a problem
else:
# end of text
if pos >= source_length:
return
# something went wrong
raise TemplateSyntaxError('unexpected char %r at %d' %
(source[pos], pos), lineno,
name, filename)

481
lib/spack/external/jinja2/loaders.py vendored Normal file
View File

@@ -0,0 +1,481 @@
# -*- coding: utf-8 -*-
"""
jinja2.loaders
~~~~~~~~~~~~~~
Jinja loader classes.
:copyright: (c) 2017 by the Jinja Team.
:license: BSD, see LICENSE for more details.
"""
import os
import sys
import weakref
from types import ModuleType
from os import path
from hashlib import sha1
from jinja2.exceptions import TemplateNotFound
from jinja2.utils import open_if_exists, internalcode
from jinja2._compat import string_types, iteritems
def split_template_path(template):
"""Split a path into segments and perform a sanity check. If it detects
'..' in the path it will raise a `TemplateNotFound` error.
"""
pieces = []
for piece in template.split('/'):
if path.sep in piece \
or (path.altsep and path.altsep in piece) or \
piece == path.pardir:
raise TemplateNotFound(template)
elif piece and piece != '.':
pieces.append(piece)
return pieces
class BaseLoader(object):
"""Baseclass for all loaders. Subclass this and override `get_source` to
implement a custom loading mechanism. The environment provides a
`get_template` method that calls the loader's `load` method to get the
:class:`Template` object.
A very basic example for a loader that looks up templates on the file
system could look like this::
from jinja2 import BaseLoader, TemplateNotFound
from os.path import join, exists, getmtime
class MyLoader(BaseLoader):
def __init__(self, path):
self.path = path
def get_source(self, environment, template):
path = join(self.path, template)
if not exists(path):
raise TemplateNotFound(template)
mtime = getmtime(path)
with file(path) as f:
source = f.read().decode('utf-8')
return source, path, lambda: mtime == getmtime(path)
"""
#: if set to `False` it indicates that the loader cannot provide access
#: to the source of templates.
#:
#: .. versionadded:: 2.4
has_source_access = True
def get_source(self, environment, template):
"""Get the template source, filename and reload helper for a template.
It's passed the environment and template name and has to return a
tuple in the form ``(source, filename, uptodate)`` or raise a
`TemplateNotFound` error if it can't locate the template.
The source part of the returned tuple must be the source of the
template as unicode string or a ASCII bytestring. The filename should
be the name of the file on the filesystem if it was loaded from there,
otherwise `None`. The filename is used by python for the tracebacks
if no loader extension is used.
The last item in the tuple is the `uptodate` function. If auto
reloading is enabled it's always called to check if the template
changed. No arguments are passed so the function must store the
old state somewhere (for example in a closure). If it returns `False`
the template will be reloaded.
"""
if not self.has_source_access:
raise RuntimeError('%s cannot provide access to the source' %
self.__class__.__name__)
raise TemplateNotFound(template)
def list_templates(self):
"""Iterates over all templates. If the loader does not support that
it should raise a :exc:`TypeError` which is the default behavior.
"""
raise TypeError('this loader cannot iterate over all templates')
@internalcode
def load(self, environment, name, globals=None):
"""Loads a template. This method looks up the template in the cache
or loads one by calling :meth:`get_source`. Subclasses should not
override this method as loaders working on collections of other
loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`)
will not call this method but `get_source` directly.
"""
code = None
if globals is None:
globals = {}
# first we try to get the source for this template together
# with the filename and the uptodate function.
source, filename, uptodate = self.get_source(environment, name)
# try to load the code from the bytecode cache if there is a
# bytecode cache configured.
bcc = environment.bytecode_cache
if bcc is not None:
bucket = bcc.get_bucket(environment, name, filename, source)
code = bucket.code
# if we don't have code so far (not cached, no longer up to
# date) etc. we compile the template
if code is None:
code = environment.compile(source, name, filename)
# if the bytecode cache is available and the bucket doesn't
# have a code so far, we give the bucket the new code and put
# it back to the bytecode cache.
if bcc is not None and bucket.code is None:
bucket.code = code
bcc.set_bucket(bucket)
return environment.template_class.from_code(environment, code,
globals, uptodate)
class FileSystemLoader(BaseLoader):
"""Loads templates from the file system. This loader can find templates
in folders on the file system and is the preferred way to load them.
The loader takes the path to the templates as string, or if multiple
locations are wanted a list of them which is then looked up in the
given order::
>>> loader = FileSystemLoader('/path/to/templates')
>>> loader = FileSystemLoader(['/path/to/templates', '/other/path'])
Per default the template encoding is ``'utf-8'`` which can be changed
by setting the `encoding` parameter to something else.
To follow symbolic links, set the *followlinks* parameter to ``True``::
>>> loader = FileSystemLoader('/path/to/templates', followlinks=True)
.. versionchanged:: 2.8+
The *followlinks* parameter was added.
"""
def __init__(self, searchpath, encoding='utf-8', followlinks=False):
if isinstance(searchpath, string_types):
searchpath = [searchpath]
self.searchpath = list(searchpath)
self.encoding = encoding
self.followlinks = followlinks
def get_source(self, environment, template):
pieces = split_template_path(template)
for searchpath in self.searchpath:
filename = path.join(searchpath, *pieces)
f = open_if_exists(filename)
if f is None:
continue
try:
contents = f.read().decode(self.encoding)
finally:
f.close()
mtime = path.getmtime(filename)
def uptodate():
try:
return path.getmtime(filename) == mtime
except OSError:
return False
return contents, filename, uptodate
raise TemplateNotFound(template)
def list_templates(self):
found = set()
for searchpath in self.searchpath:
walk_dir = os.walk(searchpath, followlinks=self.followlinks)
for dirpath, dirnames, filenames in walk_dir:
for filename in filenames:
template = os.path.join(dirpath, filename) \
[len(searchpath):].strip(os.path.sep) \
.replace(os.path.sep, '/')
if template[:2] == './':
template = template[2:]
if template not in found:
found.add(template)
return sorted(found)
class PackageLoader(BaseLoader):
"""Load templates from python eggs or packages. It is constructed with
the name of the python package and the path to the templates in that
package::
loader = PackageLoader('mypackage', 'views')
If the package path is not given, ``'templates'`` is assumed.
Per default the template encoding is ``'utf-8'`` which can be changed
by setting the `encoding` parameter to something else. Due to the nature
of eggs it's only possible to reload templates if the package was loaded
from the file system and not a zip file.
"""
def __init__(self, package_name, package_path='templates',
encoding='utf-8'):
from pkg_resources import DefaultProvider, ResourceManager, \
get_provider
provider = get_provider(package_name)
self.encoding = encoding
self.manager = ResourceManager()
self.filesystem_bound = isinstance(provider, DefaultProvider)
self.provider = provider
self.package_path = package_path
def get_source(self, environment, template):
pieces = split_template_path(template)
p = '/'.join((self.package_path,) + tuple(pieces))
if not self.provider.has_resource(p):
raise TemplateNotFound(template)
filename = uptodate = None
if self.filesystem_bound:
filename = self.provider.get_resource_filename(self.manager, p)
mtime = path.getmtime(filename)
def uptodate():
try:
return path.getmtime(filename) == mtime
except OSError:
return False
source = self.provider.get_resource_string(self.manager, p)
return source.decode(self.encoding), filename, uptodate
def list_templates(self):
path = self.package_path
if path[:2] == './':
path = path[2:]
elif path == '.':
path = ''
offset = len(path)
results = []
def _walk(path):
for filename in self.provider.resource_listdir(path):
fullname = path + '/' + filename
if self.provider.resource_isdir(fullname):
_walk(fullname)
else:
results.append(fullname[offset:].lstrip('/'))
_walk(path)
results.sort()
return results
class DictLoader(BaseLoader):
"""Loads a template from a python dict. It's passed a dict of unicode
strings bound to template names. This loader is useful for unittesting:
>>> loader = DictLoader({'index.html': 'source here'})
Because auto reloading is rarely useful this is disabled per default.
"""
def __init__(self, mapping):
self.mapping = mapping
def get_source(self, environment, template):
if template in self.mapping:
source = self.mapping[template]
return source, None, lambda: source == self.mapping.get(template)
raise TemplateNotFound(template)
def list_templates(self):
return sorted(self.mapping)
class FunctionLoader(BaseLoader):
"""A loader that is passed a function which does the loading. The
function receives the name of the template and has to return either
an unicode string with the template source, a tuple in the form ``(source,
filename, uptodatefunc)`` or `None` if the template does not exist.
>>> def load_template(name):
... if name == 'index.html':
... return '...'
...
>>> loader = FunctionLoader(load_template)
The `uptodatefunc` is a function that is called if autoreload is enabled
and has to return `True` if the template is still up to date. For more
details have a look at :meth:`BaseLoader.get_source` which has the same
return value.
"""
def __init__(self, load_func):
self.load_func = load_func
def get_source(self, environment, template):
rv = self.load_func(template)
if rv is None:
raise TemplateNotFound(template)
elif isinstance(rv, string_types):
return rv, None, None
return rv
class PrefixLoader(BaseLoader):
"""A loader that is passed a dict of loaders where each loader is bound
to a prefix. The prefix is delimited from the template by a slash per
default, which can be changed by setting the `delimiter` argument to
something else::
loader = PrefixLoader({
'app1': PackageLoader('mypackage.app1'),
'app2': PackageLoader('mypackage.app2')
})
By loading ``'app1/index.html'`` the file from the app1 package is loaded,
by loading ``'app2/index.html'`` the file from the second.
"""
def __init__(self, mapping, delimiter='/'):
self.mapping = mapping
self.delimiter = delimiter
def get_loader(self, template):
try:
prefix, name = template.split(self.delimiter, 1)
loader = self.mapping[prefix]
except (ValueError, KeyError):
raise TemplateNotFound(template)
return loader, name
def get_source(self, environment, template):
loader, name = self.get_loader(template)
try:
return loader.get_source(environment, name)
except TemplateNotFound:
# re-raise the exception with the correct filename here.
# (the one that includes the prefix)
raise TemplateNotFound(template)
@internalcode
def load(self, environment, name, globals=None):
loader, local_name = self.get_loader(name)
try:
return loader.load(environment, local_name, globals)
except TemplateNotFound:
# re-raise the exception with the correct filename here.
# (the one that includes the prefix)
raise TemplateNotFound(name)
def list_templates(self):
result = []
for prefix, loader in iteritems(self.mapping):
for template in loader.list_templates():
result.append(prefix + self.delimiter + template)
return result
class ChoiceLoader(BaseLoader):
"""This loader works like the `PrefixLoader` just that no prefix is
specified. If a template could not be found by one loader the next one
is tried.
>>> loader = ChoiceLoader([
... FileSystemLoader('/path/to/user/templates'),
... FileSystemLoader('/path/to/system/templates')
... ])
This is useful if you want to allow users to override builtin templates
from a different location.
"""
def __init__(self, loaders):
self.loaders = loaders
def get_source(self, environment, template):
for loader in self.loaders:
try:
return loader.get_source(environment, template)
except TemplateNotFound:
pass
raise TemplateNotFound(template)
@internalcode
def load(self, environment, name, globals=None):
for loader in self.loaders:
try:
return loader.load(environment, name, globals)
except TemplateNotFound:
pass
raise TemplateNotFound(name)
def list_templates(self):
found = set()
for loader in self.loaders:
found.update(loader.list_templates())
return sorted(found)
class _TemplateModule(ModuleType):
"""Like a normal module but with support for weak references"""
class ModuleLoader(BaseLoader):
"""This loader loads templates from precompiled templates.
Example usage:
>>> loader = ChoiceLoader([
... ModuleLoader('/path/to/compiled/templates'),
... FileSystemLoader('/path/to/templates')
... ])
Templates can be precompiled with :meth:`Environment.compile_templates`.
"""
has_source_access = False
def __init__(self, path):
package_name = '_jinja2_module_templates_%x' % id(self)
# create a fake module that looks for the templates in the
# path given.
mod = _TemplateModule(package_name)
if isinstance(path, string_types):
path = [path]
else:
path = list(path)
mod.__path__ = path
sys.modules[package_name] = weakref.proxy(mod,
lambda x: sys.modules.pop(package_name, None))
# the only strong reference, the sys.modules entry is weak
# so that the garbage collector can remove it once the
# loader that created it goes out of business.
self.module = mod
self.package_name = package_name
@staticmethod
def get_template_key(name):
return 'tmpl_' + sha1(name.encode('utf-8')).hexdigest()
@staticmethod
def get_module_filename(name):
return ModuleLoader.get_template_key(name) + '.py'
@internalcode
def load(self, environment, name, globals=None):
key = self.get_template_key(name)
module = '%s.%s' % (self.package_name, key)
mod = getattr(self.module, module, None)
if mod is None:
try:
mod = __import__(module, None, None, ['root'])
except ImportError:
raise TemplateNotFound(name)
# remove the entry from sys.modules, we only want the attribute
# on the module object we have stored on the loader.
sys.modules.pop(module, None)
return environment.template_class.from_module_dict(
environment, mod.__dict__, globals)

106
lib/spack/external/jinja2/meta.py vendored Normal file
View File

@@ -0,0 +1,106 @@
# -*- coding: utf-8 -*-
"""
jinja2.meta
~~~~~~~~~~~
This module implements various functions that exposes information about
templates that might be interesting for various kinds of applications.
:copyright: (c) 2017 by the Jinja Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""
from jinja2 import nodes
from jinja2.compiler import CodeGenerator
from jinja2._compat import string_types, iteritems
class TrackingCodeGenerator(CodeGenerator):
"""We abuse the code generator for introspection."""
def __init__(self, environment):
CodeGenerator.__init__(self, environment, '<introspection>',
'<introspection>')
self.undeclared_identifiers = set()
def write(self, x):
"""Don't write."""
def enter_frame(self, frame):
"""Remember all undeclared identifiers."""
CodeGenerator.enter_frame(self, frame)
for _, (action, param) in iteritems(frame.symbols.loads):
if action == 'resolve':
self.undeclared_identifiers.add(param)
def find_undeclared_variables(ast):
"""Returns a set of all variables in the AST that will be looked up from
the context at runtime. Because at compile time it's not known which
variables will be used depending on the path the execution takes at
runtime, all variables are returned.
>>> from jinja2 import Environment, meta
>>> env = Environment()
>>> ast = env.parse('{% set foo = 42 %}{{ bar + foo }}')
>>> meta.find_undeclared_variables(ast) == set(['bar'])
True
.. admonition:: Implementation
Internally the code generator is used for finding undeclared variables.
This is good to know because the code generator might raise a
:exc:`TemplateAssertionError` during compilation and as a matter of
fact this function can currently raise that exception as well.
"""
codegen = TrackingCodeGenerator(ast.environment)
codegen.visit(ast)
return codegen.undeclared_identifiers
def find_referenced_templates(ast):
"""Finds all the referenced templates from the AST. This will return an
iterator over all the hardcoded template extensions, inclusions and
imports. If dynamic inheritance or inclusion is used, `None` will be
yielded.
>>> from jinja2 import Environment, meta
>>> env = Environment()
>>> ast = env.parse('{% extends "layout.html" %}{% include helper %}')
>>> list(meta.find_referenced_templates(ast))
['layout.html', None]
This function is useful for dependency tracking. For example if you want
to rebuild parts of the website after a layout template has changed.
"""
for node in ast.find_all((nodes.Extends, nodes.FromImport, nodes.Import,
nodes.Include)):
if not isinstance(node.template, nodes.Const):
# a tuple with some non consts in there
if isinstance(node.template, (nodes.Tuple, nodes.List)):
for template_name in node.template.items:
# something const, only yield the strings and ignore
# non-string consts that really just make no sense
if isinstance(template_name, nodes.Const):
if isinstance(template_name.value, string_types):
yield template_name.value
# something dynamic in there
else:
yield None
# something dynamic we don't know about here
else:
yield None
continue
# constant is a basestring, direct template name
if isinstance(node.template.value, string_types):
yield node.template.value
# a tuple or list (latter *should* not happen) made of consts,
# yield the consts that are strings. We could warn here for
# non string values
elif isinstance(node, nodes.Include) and \
isinstance(node.template.value, (tuple, list)):
for template_name in node.template.value:
if isinstance(template_name, string_types):
yield template_name
# something else we don't care about, we could warn here
else:
yield None

Some files were not shown because too many files have changed in this diff Show More