zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor IntelInstaller into IntelPackage base class (#4300)

Browse Source 
* Refactor IntelInstaller into IntelPackage base class

* Move license attributes from __init__ to class-level

* Flake8 fixes: remove unused imports

* Fix logic that writes the silent.cfg file

* More specific version numbers for Intel MPI

* Rework logic that selects components to install

* Final changes necessary to get intel package working

* Various updates to intel-parallel-studio

* Add latest version of every Intel package

* Add environment variables for Intel packages

* Update env vars for intel package

* Finalize components for intel-parallel-studio package

Adds a +tbb variant to intel-parallel-studio.
The tbb package was renamed to intel-tbb.
Now both intel-tbb and intel-parallel-studio+tbb
provide tbb.

* Overhaul environment variables set by intel-parallel-studio

* Point dependent packages to the correct MPI wrappers

* Never default to intel-parallel-studio

* Gather env vars by sourcing setup scripts

* Use mpiicc instead of mpicc when using Intel compiler

* Undo change to ARCH

* Add changes from intel-mpi to intel-parallel-studio

* Add comment explaining mpicc vs mpiicc

* Prepend env vars containing 'PATH' or separators

* Flake8 fix

* Fix bugs in from_sourcing_file

* Indentation fix

* Prepend, not set if contains separator

* Fix license symlinking broken by changes to intel-parallel-studio

* Use comments instead of docstrings to document attributes

* Flake8 fixes

* Use a set instead of a list to prevent duplicate components

* Fix MKL and MPI library linking directories

* Remove +all variant from intel-parallel-studio

* It is not possible to build with MKL, GCC, and OpenMP at this time

* Found a workaround for locating GCC libraries

* Typos and variable names

* Fix initialization of empty LibraryList
This commit is contained in:
Adam J. Stewart
2017-08-16 12:21:07 -05:00
committed by GitHub
parent ad8c60239f
commit db657d938d
17 changed files with 937 additions and 597 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/spack/docs/packaging_guide.rst
View File

@@ -2136,6 +2136,9 @@ The classes that are currently provided by Spack are:
| :py:class:`.PerlPackage` | Specialized class for |
| | :py:class:`.Perl` extensions |
+-------------------------------+----------------------------------+
| :py:class:`.IntelPackage` | Specialized class for licensed |
| | Intel software |
+-------------------------------+----------------------------------+
.. note::

2
lib/spack/spack/__init__.py
View File

@@ -178,6 +178,7 @@
from spack.build_systems.python import PythonPackage
from spack.build_systems.r import RPackage
from spack.build_systems.perl import PerlPackage
from spack.build_systems.intel import IntelPackage
__all__ += [
'run_before',
@@ -193,6 +194,7 @@
'PythonPackage',
'RPackage',
'PerlPackage',
'IntelPackage',
]
from spack.version import Version, ver

4
lib/spack/spack/build_environment.py
View File

@@ -229,7 +229,7 @@ def set_build_environment_variables(pkg, env, dirty=False):
# Install root prefix
env.set(SPACK_INSTALL, spack.store.root)
# Stuff in here sanitizes the build environemnt to eliminate
# Stuff in here sanitizes the build environment to eliminate
# anything the user has set that may interfere.
if not dirty:
# Remove these vars from the environment during build because they
@@ -518,7 +518,7 @@ def fork(pkg, function, dirty=False):
Args:
pkg (PackageBase): package whose environemnt we should set up the
pkg (PackageBase): package whose environment we should set up the
forked process for.
function (callable): argless function to run in the child
process.

192
lib/spack/spack/build_systems/intel.py Normal file
View File

@@ -0,0 +1,192 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import os
import xml.etree.ElementTree as ET
from llnl.util.filesystem import install, join_path
from spack.package import PackageBase, run_after
from spack.util.executable import Executable
def _valid_components():
"""A generator that yields valid components."""
tree = ET.parse('pset/mediaconfig.xml')
root = tree.getroot()
components = root.findall('.//Abbr')
for component in components:
yield component.text
class IntelPackage(PackageBase):
"""Specialized class for licensed Intel software.
This class provides two phases that can be overridden:
1. :py:meth:`~.IntelPackage.configure`
2. :py:meth:`~.IntelPackage.install`
They both have sensible defaults and for many packages the
only thing necessary will be to override ``setup_environment``
to set the appropriate environment variables.
"""
#: Phases of an Intel package
phases = ['configure', 'install']
#: This attribute is used in UI queries that need to know the build
#: system base class
build_system_class = 'IntelPackage'
#: By default, we assume that all Intel software requires a license.
#: This can be overridden for packages that do not require a license.
license_required = True
#: Comment symbol used in the ``license.lic`` file
license_comment = '#'
#: Location where Intel searches for a license file
license_files = ['Licenses/license.lic']
#: Environment variables that Intel searches for a license file
license_vars = ['INTEL_LICENSE_FILE']
#: URL providing information on how to acquire a license key
license_url = 'https://software.intel.com/en-us/articles/intel-license-manager-faq'
#: Components of the package to install.
#: By default, install 'ALL' components.
components = ['ALL']
@property
def _filtered_components(self):
"""Returns a list or set of valid components that match
the requested components from ``components``."""
# Don't filter 'ALL'
if self.components == ['ALL']:
return self.components
# mediaconfig.xml is known to contain duplicate components.
# If more than one copy of the same component is used, you
# will get an error message about invalid components.
# Use a set to store components to prevent duplicates.
matches = set()
for valid in _valid_components():
for requested in self.components:
if valid.startswith(requested):
matches.add(valid)
return matches
@property
def global_license_file(self):
"""Returns the path where a global license file should be stored.
All Intel software shares the same license, so we store it in a
common 'intel' directory."""
return join_path(self.global_license_dir, 'intel',
os.path.basename(self.license_files[0]))
def configure(self, spec, prefix):
"""Writes the ``silent.cfg`` file used to configure the installation.
See https://software.intel.com/en-us/articles/configuration-file-format
"""
# Patterns used to check silent configuration file
#
# anythingpat - any string
# filepat - the file location pattern (/path/to/license.lic)
# lspat - the license server address pattern (0123@hostname)
# snpat - the serial number pattern (ABCD-01234567)
config = {
# Accept EULA, valid values are: {accept, decline}
'ACCEPT_EULA': 'accept',
# Optional error behavior, valid values are: {yes, no}
'CONTINUE_WITH_OPTIONAL_ERROR': 'yes',
# Install location, valid values are: {/opt/intel, filepat}
'PSET_INSTALL_DIR': prefix,
# Continue with overwrite of existing installation directory,
# valid values are: {yes, no}
'CONTINUE_WITH_INSTALLDIR_OVERWRITE': 'yes',
# List of components to install,
# valid values are: {ALL, DEFAULTS, anythingpat}
'COMPONENTS': ';'.join(self._filtered_components),
# Installation mode, valid values are: {install, repair, uninstall}
'PSET_MODE': 'install',
# Directory for non-RPM database, valid values are: {filepat}
'NONRPM_DB_DIR': prefix,
# Perform validation of digital signatures of RPM files,
# valid values are: {yes, no}
'SIGNING_ENABLED': 'no',
# Select target architecture of your applications,
# valid values are: {IA32, INTEL64, ALL}
'ARCH_SELECTED': 'ALL',
}
# Not all Intel software requires a license. Trying to specify
# one anyway will cause the installation to fail.
if self.license_required:
config.update({
# License file or license server,
# valid values are: {lspat, filepat}
'ACTIVATION_LICENSE_FILE': self.global_license_file,
# Activation type, valid values are: {exist_lic,
# license_server, license_file, trial_lic, serial_number}
'ACTIVATION_TYPE': 'license_file',
# Intel(R) Software Improvement Program opt-in,
# valid values are: {yes, no}
'PHONEHOME_SEND_USAGE_DATA': 'no',
})
with open('silent.cfg', 'w') as cfg:
for key in config:
cfg.write('{0}={1}\n'.format(key, config[key]))
def install(self, spec, prefix):
"""Runs the ``install.sh`` installation script."""
install_script = Executable('./install.sh')
install_script('--silent', 'silent.cfg')
@run_after('install')
def save_silent_cfg(self):
"""Copies the silent.cfg configuration file to ``<prefix>/.spack``."""
install('silent.cfg', join_path(self.prefix, '.spack'))
# Check that self.prefix is there after installation
run_after('install')(PackageBase.sanity_check_prefix)

1
lib/spack/spack/cmd/configure.py
View File

@@ -41,6 +41,7 @@
QMakePackage: 'qmake',
WafPackage: 'configure',
PerlPackage: 'configure',
IntelPackage: 'configure',
}

10
lib/spack/spack/cmd/create.py
View File

@@ -370,6 +370,15 @@ def edit(self, spec, prefix):
# makefile.filter('CC = .*', 'CC = cc')"""
class IntelPackageTemplate(PackageTemplate):
"""Provides appropriate overrides for licensed Intel software"""
base_class_name = 'IntelPackage'
body = """\
# FIXME: Override `setup_environment` if necessary."""
templates = {
'autotools': AutotoolsPackageTemplate,
'autoreconf': AutoreconfPackageTemplate,
@@ -384,6 +393,7 @@ def edit(self, spec, prefix):
'perlbuild': PerlbuildPackageTemplate,
'octave': OctavePackageTemplate,
'makefile': MakefilePackageTemplate,
'intel': IntelPackageTemplate,
'generic': PackageTemplate,
}

174
lib/spack/spack/environment.py
View File

@@ -284,132 +284,157 @@ def apply_modifications(self):
x.execute()
@staticmethod
def from_sourcing_files(*args, **kwargs):
"""Returns modifications that would be made by sourcing files.
def from_sourcing_file(filename, *args, **kwargs):
"""Returns modifications that would be made by sourcing a file.
Args:
*args (list of str): list of files to be sourced
Parameters:
filename (str): The file to source
*args (list of str): Arguments to pass on the command line
Keyword Arguments:
shell (str): The shell to use (default: ``bash``)
shell_options (str): Options passed to the shell (default: ``-c``)
source_command (str): The command to run (default: ``source``)
suppress_output (str): Redirect used to suppress output of command
(default: ``&> /dev/null``)
concatenate_on_success (str): Operator used to execute a command
only when the previous command succeeds (default: ``&&``)
Returns:
EnvironmentModifications: an object that, if executed, has
the same effect on the environment as sourcing the files
passed as parameters
the same effect on the environment as sourcing the file
"""
env = EnvironmentModifications()
# Check if the file actually exists
if not os.path.isfile(filename):
msg = 'Trying to source non-existing file: {0}'.format(filename)
raise RuntimeError(msg)
# Check if the files are actually there
files = [line.split(' ')[0] for line in args]
non_existing = [file for file in files if not os.path.isfile(file)]
if non_existing:
message = 'trying to source non-existing files\n'
message += '\n'.join(non_existing)
raise RuntimeError(message)
# Kwargs parsing and default values
shell = kwargs.get('shell', '/bin/bash')
shell_options = kwargs.get('shell_options', '-c')
source_command = kwargs.get('source_command', 'source')
suppress_output = kwargs.get('suppress_output', '&> /dev/null')
concatenate_on_success = kwargs.get('concatenate_on_success', '&&')
# Relevant kwd parameters and formats
info = dict(kwargs)
info.setdefault('shell', '/bin/bash')
info.setdefault('shell_options', '-c')
info.setdefault('source_command', 'source')
info.setdefault('suppress_output', '&> /dev/null')
info.setdefault('concatenate_on_success', '&&')
source_file = [source_command, filename]
source_file.extend(args)
source_file = ' '.join(source_file)
shell = '{shell}'.format(**info)
shell_options = '{shell_options}'.format(**info)
source_file = '{source_command} {file} {concatenate_on_success}'
dump_cmd = "import os, json; print(json.dumps(dict(os.environ)))"
dump_environment = 'python -c "%s"' % dump_cmd
dump_cmd = 'import os, json; print(json.dumps(dict(os.environ)))'
dump_environment = 'python -c "{0}"'.format(dump_cmd)
# Construct the command that will be executed
command = [source_file.format(file=file, **info) for file in args]
command.append(dump_environment)
command = ' '.join(command)
command = [
shell,
shell_options,
command
' '.join([
source_file, suppress_output,
concatenate_on_success, dump_environment,
]),
]
# Try to source all the files,
# Try to source the file
proc = subprocess.Popen(
command, stdout=subprocess.PIPE, env=os.environ)
proc.wait()
if proc.returncode != 0:
raise RuntimeError('sourcing files returned a non-zero exit code')
msg = 'Sourcing file {0} returned a non-zero exit code'.format(
filename)
raise RuntimeError(msg)
output = ''.join([line.decode('utf-8') for line in proc.stdout])
# Construct a dictionaries of the environment before and after
# sourcing the files, so that we can diff them.
this_environment = dict(os.environ)
after_source_env = json.loads(output)
# Construct dictionaries of the environment before and after
# sourcing the file, so that we can diff them.
env_before = dict(os.environ)
env_after = json.loads(output)
# If we're in python2, convert to str objects instead of unicode
# like json gives us. We can't put unicode in os.environ anyway.
if sys.version_info[0] < 3:
after_source_env = dict((k.encode('utf-8'), v.encode('utf-8'))
for k, v in after_source_env.items())
env_after = dict((k.encode('utf-8'), v.encode('utf-8'))
for k, v in env_after.items())
# Filter variables that are not related to sourcing a file
to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD'
for d in after_source_env, this_environment:
to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD', 'PS2'
for d in env_after, env_before:
for name in to_be_filtered:
d.pop(name, None)
# Fill the EnvironmentModifications instance
env = EnvironmentModifications()
# New variables
new_variables = set(after_source_env) - set(this_environment)
for x in new_variables:
env.set(x, after_source_env[x])
new_variables = set(env_after) - set(env_before)
# Variables that have been unset
unset_variables = set(this_environment) - set(after_source_env)
for x in unset_variables:
env.unset(x)
unset_variables = set(env_before) - set(env_after)
# Variables that have been modified
common_variables = set(
this_environment).intersection(set(after_source_env))
env_before).intersection(set(env_after))
modified_variables = [x for x in common_variables
if this_environment[x] != after_source_env[x]]
if env_before[x] != env_after[x]]
def return_separator_if_any(first_value, second_value):
def return_separator_if_any(*args):
separators = ':', ';'
for separator in separators:
if separator in first_value and separator in second_value:
return separator
for arg in args:
if separator in arg:
return separator
return None
for x in modified_variables:
current = this_environment[x]
modified = after_source_env[x]
sep = return_separator_if_any(current, modified)
if sep is None:
# We just need to set the variable to the new value
env.set(x, after_source_env[x])
# Add variables to env.
# Assume that variables with 'PATH' in the name or that contain
# separators like ':' or ';' are more likely to be paths
for x in new_variables:
sep = return_separator_if_any(env_after[x])
if sep:
env.prepend_path(x, env_after[x], separator=sep)
elif 'PATH' in x:
env.prepend_path(x, env_after[x])
else:
current_list = current.split(sep)
modified_list = modified.split(sep)
# We just need to set the variable to the new value
env.set(x, env_after[x])
for x in unset_variables:
env.unset(x)
for x in modified_variables:
before = env_before[x]
after = env_after[x]
sep = return_separator_if_any(before, after)
if sep:
before_list = before.split(sep)
after_list = after.split(sep)
# Filter out empty strings
before_list = list(filter(None, before_list))
after_list = list(filter(None, after_list))
# Paths that have been removed
remove_list = [
ii for ii in current_list if ii not in modified_list]
# Check that nothing has been added in the middle of vurrent
# list
ii for ii in before_list if ii not in after_list]
# Check that nothing has been added in the middle of
# before_list
remaining_list = [
ii for ii in current_list if ii in modified_list]
start = modified_list.index(remaining_list[0])
end = modified_list.index(remaining_list[-1])
search = sep.join(modified_list[start:end + 1])
ii for ii in before_list if ii in after_list]
try:
start = after_list.index(remaining_list[0])
end = after_list.index(remaining_list[-1])
search = sep.join(after_list[start:end + 1])
except IndexError:
env.prepend_path(x, env_after[x])
if search not in current:
if search not in before:
# We just need to set the variable to the new value
env.set(x, after_source_env[x])
break
env.prepend_path(x, env_after[x])
else:
try:
prepend_list = modified_list[:start]
prepend_list = after_list[:start]
except KeyError:
prepend_list = []
try:
append_list = modified_list[end + 1:]
append_list = after_list[end + 1:]
except KeyError:
append_list = []
@@ -419,6 +444,9 @@ def return_separator_if_any(first_value, second_value):
env.append_path(x, item)
for item in prepend_list:
env.prepend_path(x, item)
else:
# We just need to set the variable to the new value
env.set(x, env_after[x])
return env

73
lib/spack/spack/package.py
View File

@@ -484,38 +484,65 @@ class SomePackage(Package):
#
# These are default values for instance variables.
#
"""By default we build in parallel. Subclasses can override this."""
#: By default we build in parallel. Subclasses can override this.
parallel = True
"""# jobs to use for parallel make. If set, overrides default of ncpus."""
#: # jobs to use for parallel make. If set, overrides default of ncpus.
make_jobs = spack.build_jobs
"""By default do not run tests within package's install()"""
#: By default do not run tests within package's install()
run_tests = False
# FIXME: this is a bad object-oriented design, should be moved to Clang.
"""By default do not setup mockup XCode on macOS with Clang"""
#: By default do not setup mockup XCode on macOS with Clang
use_xcode = False
"""Most packages are NOT extendable. Set to True if you want extensions."""
#: Most packages are NOT extendable. Set to True if you want extensions.
extendable = False
"""When True, add RPATHs for the entire DAG. When False, add RPATHs only
for immediate dependencies."""
#: When True, add RPATHs for the entire DAG. When False, add RPATHs only
#: for immediate dependencies.
transitive_rpaths = True
"""List of prefix-relative file paths (or a single path). If these do
not exist after install, or if they exist but are not files,
sanity checks fail.
"""
#: List of prefix-relative file paths (or a single path). If these do
#: not exist after install, or if they exist but are not files,
#: sanity checks fail.
sanity_check_is_file = []
"""List of prefix-relative directory paths (or a single path). If
these do not exist after install, or if they exist but are not
directories, sanity checks will fail.
"""
#: List of prefix-relative directory paths (or a single path). If
#: these do not exist after install, or if they exist but are not
#: directories, sanity checks will fail.
sanity_check_is_dir = []
#
# Set default licensing information
#
#: Boolean. If set to ``True``, this software requires a license.
#: If set to ``False``, all of the ``license_*`` attributes will
#: be ignored. Defaults to ``False``.
license_required = False
#: String. Contains the symbol used by the license manager to denote
#: a comment. Defaults to ``#``.
license_comment = '#'
#: List of strings. These are files that the software searches for when
#: looking for a license. All file paths must be relative to the
#: installation directory. More complex packages like Intel may require
#: multiple licenses for individual components. Defaults to the empty list.
license_files = []
#: List of strings. Environment variables that can be set to tell the
#: software where to look for a license if it is not in the usual location.
#: Defaults to the empty list.
license_vars = []
#: String. A URL pointing to license setup instructions for the software.
#: Defaults to the empty string.
license_url = ''
def __init__(self, spec):
# this determines how the package should be built.
self.spec = spec
@@ -569,22 +596,6 @@ def __init__(self, spec):
if not hasattr(self, 'list_depth'):
self.list_depth = 0
# Set default licensing information
if not hasattr(self, 'license_required'):
self.license_required = False
if not hasattr(self, 'license_comment'):
self.license_comment = '#'
if not hasattr(self, 'license_files'):
self.license_files = []
if not hasattr(self, 'license_vars'):
self.license_vars = []
if not hasattr(self, 'license_url'):
self.license_url = None
# Set up some internal variables for timing.
self._fetch_time = 0.0
self._total_time = 0.0

11
lib/spack/spack/test/environment.py
View File

@@ -89,7 +89,7 @@ def files_to_be_sourced():
files = [
os.path.join(datadir, 'sourceme_first.sh'),
os.path.join(datadir, 'sourceme_second.sh'),
os.path.join(datadir, 'sourceme_parameters.sh intel64'),
os.path.join(datadir, 'sourceme_parameters.sh'),
os.path.join(datadir, 'sourceme_unicode.sh')
]
@@ -224,7 +224,14 @@ def test_source_files(files_to_be_sourced):
"""Tests the construction of a list of environment modifications that are
the result of sourcing a file.
"""
env = EnvironmentModifications.from_sourcing_files(*files_to_be_sourced)
env = EnvironmentModifications()
for filename in files_to_be_sourced:
if filename.endswith('sourceme_parameters.sh'):
env.extend(EnvironmentModifications.from_sourcing_file(
filename, 'intel64'))
else:
env.extend(EnvironmentModifications.from_sourcing_file(filename))
modifications = env.group_by_name()
# This is sensitive to the user's environment; can include