From a4c19eee14103e9944e04917a44edec2b3209f94 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 19 Jan 2015 14:00:54 -0800 Subject: [PATCH 01/97] Qt5 webkit requires gperf --- var/spack/packages/gperf/package.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 var/spack/packages/gperf/package.py diff --git a/var/spack/packages/gperf/package.py b/var/spack/packages/gperf/package.py new file mode 100644 index 00000000000..32551b67b40 --- /dev/null +++ b/var/spack/packages/gperf/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Gperf(Package): + """GNU gperf is a perfect hash function generator. For a given + list of strings, it produces a hash function and hash table, in + form of C or C++ code, for looking up a value depending on the + input string. The hash function is perfect, which means that the + hash table has no collisions, and the hash table lookup needs a + single string comparison only.""" + + homepage = "https://www.gnu.org/software/gperf/" + url = "http://ftp.gnu.org/pub/gnu/gperf/gperf-3.0.4.tar.gz" + + version('3.0.4', 'c1f1db32fb6598d6a93e6e88796a8632') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") From f35b8b8db42332a93168e3f8a707442186359a6c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 19 Jan 2015 14:05:48 -0800 Subject: [PATCH 02/97] Better location error output. --- lib/spack/spack/cmd/location.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 509c336b69d..e422eaf9668 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import sys from external import argparse import llnl.util.tty as tty @@ -86,11 +87,12 @@ def location(parser, args): tty.die("Spec '%s' matches no installed packages." % spec) elif len(matching_specs) > 1: - args = ["%s matches multiple packages." % spec, - "Matching packages:"] - args += [" " + str(s) for s in matching_specs] - args += ["Use a more specific spec."] - tty.die(*args) + tty.error("%s matches multiple packages:" % spec) + for s in matching_specs: + sys.stderr.write(s.tree(color=True)) + sys.stderr.write("\n") + sys.stderr.write("Use a more specific spec.\n") + sys.exit(1) print matching_specs[0].prefix From 53c8b4249ad5b875678ebe6d784fbc5493aef160 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 19 Jan 2015 14:06:09 -0800 Subject: [PATCH 03/97] Make dbus put a machine id file in the right place. --- var/spack/packages/dbus/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/packages/dbus/package.py b/var/spack/packages/dbus/package.py index 5fee103f038..f7f394498ce 100644 --- a/var/spack/packages/dbus/package.py +++ b/var/spack/packages/dbus/package.py @@ -23,3 +23,7 @@ def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() make("install") + + # dbus needs a machine id generated after install + dbus_uuidgen = Executable(join_path(prefix.bin, 'dbus-uuidgen')) + dbus_uuidgen('--ensure') From 0211adbdb6010b403495644aab619ba8a9bed31e Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 19 Jan 2015 14:06:25 -0800 Subject: [PATCH 04/97] version bump libpng --- var/spack/packages/libpng/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/libpng/package.py b/var/spack/packages/libpng/package.py index a6d9bf0b46c..affc14ea926 100644 --- a/var/spack/packages/libpng/package.py +++ b/var/spack/packages/libpng/package.py @@ -3,9 +3,9 @@ class Libpng(Package): """libpng graphics file format""" homepage = "http://www.libpng.org/pub/png/libpng.html" - url = "http://sourceforge.net/projects/libpng/files/libpng16/1.6.14/libpng-1.6.14.tar.gz/download" + url = "http://prdownloads.sourceforge.net/libpng/libpng-1.6.16.tar.gz?download" - version('1.6.14', '2101b3de1d5f348925990f9aa8405660') + version('1.6.16', '1a4ad377919ab15b54f6cb6a3ae2622d') def install(self, spec, prefix): configure("--prefix=%s" % prefix) From b7dacb427d2d9e80778b6c0eec05e0cc64d60859 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 19 Jan 2015 14:06:45 -0800 Subject: [PATCH 05/97] Qt5 & VTK builds. VTK works with Qt 4 and 5. --- var/spack/packages/qt/package.py | 76 +++++++++++++++++++++++++------ var/spack/packages/vtk/package.py | 40 ++++++++++++++++ 2 files changed, 101 insertions(+), 15 deletions(-) create mode 100644 var/spack/packages/vtk/package.py diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py index 01f9de7f3c0..4f82a9d9d69 100644 --- a/var/spack/packages/qt/package.py +++ b/var/spack/packages/qt/package.py @@ -3,7 +3,16 @@ class Qt(Package): """Qt is a comprehensive cross-platform C++ application framework.""" homepage = "http://qt.io" + list_url = 'http://download.qt-project.org/official_releases/qt/' + list_depth = 2 + version('5.4.0', 'e8654e4b37dd98039ba20da7a53877e6', + url='http://download.qt-project.org/official_releases/qt/5.4/5.4.0/single/qt-everywhere-opensource-src-5.4.0.tar.gz') + version('5.3.2', 'febb001129927a70174467ecb508a682', + url='http://download.qt.io/archive/qt/5.3/5.3.2/single/qt-everywhere-opensource-src-5.3.2.tar.gz') + + version('5.2.1', 'a78408c887c04c34ce615da690e0b4c8', + url='http://download.qt.io/archive/qt/5.2/5.2.1/single/qt-everywhere-opensource-src-5.2.1.tar.gz') version('4.8.6', '2edbe4d6c2eff33ef91732602f3518eb', url="http://download.qt-project.org/official_releases/qt/4.8/4.8.6/qt-everywhere-opensource-src-4.8.6.tar.gz") @@ -20,25 +29,62 @@ class Qt(Package): depends_on("libmng") depends_on("jpeg") + depends_on("gperf") # Needed to build Qt with webkit. + def patch(self): + if self.spec.satisfies('@4'): + qmake_conf = 'mkspecs/common/g++-base.conf' + elif self.spec.satisfies('@5'): + qmake_conf = 'qtbase/mkspecs/common/g++-base.conf' + else: + return + # Fix qmake compilers in the default mkspec - qmake_conf = 'mkspecs/common/g++-base.conf' - filter_file(r'^QMAKE_CC *=.*$', 'QMAKE_CC = cc', qmake_conf) - filter_file(r'^QMAKE_CXX *=.*$', 'QMAKE_CXX = c++', qmake_conf) + filter_file(r'^QMAKE_COMPILER *=.*$', 'QMAKE_COMPILER = cc', qmake_conf) + filter_file(r'^QMAKE_CC *=.*$', 'QMAKE_CC = cc', qmake_conf) + filter_file(r'^QMAKE_CXX *=.*$', 'QMAKE_CXX = c++', qmake_conf) + + + @property + def common_config_args(self): + return [ + '-prefix', self.prefix, + '-v', + '-opensource', + "-release", + '-shared', + '-confirm-license', + '-openssl-linked', + '-dbus-linked', + '-optimized-qmake', + '-no-openvg', + '-no-pch', + # For now, disable all the database drivers + "-no-sql-db2", "-no-sql-ibase", "-no-sql-mysql", "-no-sql-oci", "-no-sql-odbc", + "-no-sql-psql", "-no-sql-sqlite", "-no-sql-sqlite2", "-no-sql-tds", + # NIS is deprecated in more recent glibc + "-no-nis"] + + + @when('@4') + def configure(self): + configure('-no-phonon', + '-no-phonon-backend', + '-fast', + *self.common_config_args) + + + @when('@5') + def configure(self): + configure('-no-eglfs', + '-no-directfb', + '-qt-xcb', + # If someone wants to get a webkit build working, be my guest! + '-skip', 'qtwebkit', + *self.common_config_args) def install(self, spec, prefix): - configure('-v', - '-confirm-license', - '-opensource', - '-prefix', prefix, - '-openssl-linked', - '-dbus-linked', - '-fast', - '-optimized-qmake', - '-no-pch', - '-no-phonon', - '-no-phonon-backend', - '-no-openvg') + self.configure() make() make("install") diff --git a/var/spack/packages/vtk/package.py b/var/spack/packages/vtk/package.py new file mode 100644 index 00000000000..4a27a8fedba --- /dev/null +++ b/var/spack/packages/vtk/package.py @@ -0,0 +1,40 @@ +from spack import * + +class Vtk(Package): + """The Visualization Toolkit (VTK) is an open-source, freely + available software system for 3D computer graphics, image + processing and visualization. """ + homepage = "http://www.vtk.org" + url = "http://www.vtk.org/files/release/6.1/VTK-6.1.0.tar.gz" + + version('6.1.0', '25e4dfb3bad778722dcaec80cd5dab7d') + + depends_on("qt") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake_args = [ + "..", + "-DBUILD_SHARED_LIBS=ON", + # Disable wrappers for other languages. + "-DVTK_WRAP_PYTHON=OFF", + "-DVTK_WRAP_JAVA=OFF", + "-DVTK_WRAP_TCL=OFF"] + cmake_args.extend(std_cmake_args) + + # Enable Qt support here. + cmake_args.extend([ + "-DQT_QMAKE_EXECUTABLE:PATH=%s/qmake" % spec['qt'].prefix.bin, + "-DVTK_Group_Qt:BOOL=ON", + # Ignore webkit because it's hard to build w/Qt + "-DVTK_Group_Qt=OFF", + "-DModule_vtkGUISupportQt:BOOL=ON", + "-DModule_vtkGUISupportQtOpenGL:BOOL=ON" + ]) + + if spec['qt'].satisfies('@5'): + cmake_args.append("-DVTK_QT_VERSION:STRING=5") + + cmake(*cmake_args) + make() + make("install") From d08c0703a06825311a4fabf2c49f7f04cceadbf9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 19 Jan 2015 14:07:41 -0800 Subject: [PATCH 06/97] Initial build of MemAxes GUI. --- var/spack/packages/memaxes/package.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 var/spack/packages/memaxes/package.py diff --git a/var/spack/packages/memaxes/package.py b/var/spack/packages/memaxes/package.py new file mode 100644 index 00000000000..afa62009a62 --- /dev/null +++ b/var/spack/packages/memaxes/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Memaxes(Package): + """MemAxes is a visualizer for sampled memory trace data.""" + + homepage = "https://github.com/scalability-llnl/MemAxes" + + version('0.5', 'b0f561d48aa7301e028d074bc4b5751b', + url='https://github.com/scalability-llnl/MemAxes/archive/v0.5.tar.gz') + + depends_on("cmake@2.8.9:") + depends_on("qt@5:") + depends_on("vtk") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") + From 2a0e33876ec9f6e33a0408228463a021a1809668 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 19 Jan 2015 20:41:45 -0800 Subject: [PATCH 07/97] Add PSAPI --- var/spack/packages/psapi/package.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 var/spack/packages/psapi/package.py diff --git a/var/spack/packages/psapi/package.py b/var/spack/packages/psapi/package.py new file mode 100644 index 00000000000..b73c75c221c --- /dev/null +++ b/var/spack/packages/psapi/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Psapi(Package): + """PSAPI is a library and a tool for collecting sampled memory + performance data to view with MemAxes""" + + homepage = "https://github.com/scalability-llnl/PSAPI" + url = "http://www.example.com/memaxes-psapi-1.0.tar.gz" + + version('0.5', git='https://github.com/scalability-llnl/PSAPI.git', tag='v0.5') + + depends_on('dyninst') + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") From 51ed0d3f6f3ba8cf3c7a17db967aead0b691e3a4 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 19 Jan 2015 20:59:23 -0800 Subject: [PATCH 08/97] Properly set install RPATHS for cmake builds. --- lib/spack/spack/build_environment.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 45353ec6402..70b5c89411b 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -183,6 +183,10 @@ def set_module_variables_for_package(pkg): if platform.mac_ver()[0]: m.std_cmake_args.append('-DCMAKE_FIND_FRAMEWORK=LAST') + # Set up CMake rpath + m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE') + m.std_cmake_args.append('-DCMAKE_INSTALL_RPATH=%s' % ":".join(get_rpaths(pkg))) + # Emulate some shell commands for convenience m.pwd = os.getcwd m.cd = os.chdir @@ -202,6 +206,16 @@ def set_module_variables_for_package(pkg): m.prefix = pkg.prefix +def get_rpaths(pkg): + """Get a list of all the rpaths for a package.""" + rpaths = [pkg.prefix.lib, pkg.prefix.lib64] + rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) + if os.path.isdir(d.prefix.lib)) + rpaths.extend(d.prefix.lib64 for d in pkg.spec.traverse(root=False) + if os.path.isdir(d.prefix.lib64)) + return rpaths + + def setup_package(pkg): """Execute all environment setup routines.""" set_compiler_environment_variables(pkg) From e97db785d6d2f3a5b54d3a9e6ff0782901aa1b11 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Wed, 21 Jan 2015 20:42:44 -0800 Subject: [PATCH 09/97] psapi v0.6 --- var/spack/packages/psapi/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/psapi/package.py b/var/spack/packages/psapi/package.py index b73c75c221c..489c4f530e5 100644 --- a/var/spack/packages/psapi/package.py +++ b/var/spack/packages/psapi/package.py @@ -7,7 +7,7 @@ class Psapi(Package): homepage = "https://github.com/scalability-llnl/PSAPI" url = "http://www.example.com/memaxes-psapi-1.0.tar.gz" - version('0.5', git='https://github.com/scalability-llnl/PSAPI.git', tag='v0.5') + version('0.6', git='https://github.com/scalability-llnl/PSAPI.git', tag='v0.6') depends_on('dyninst') From 6a496ef62038f40ad1f4c22abd816f990fdfd4db Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Fri, 23 Jan 2015 16:58:15 -0800 Subject: [PATCH 10/97] PSAPI v0.6 -> Mitos v0.7 --- var/spack/packages/Mitos/package.py | 18 ++++++++++++++++++ var/spack/packages/psapi/package.py | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 var/spack/packages/Mitos/package.py delete mode 100644 var/spack/packages/psapi/package.py diff --git a/var/spack/packages/Mitos/package.py b/var/spack/packages/Mitos/package.py new file mode 100644 index 00000000000..272360a55c4 --- /dev/null +++ b/var/spack/packages/Mitos/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Mitos(Package): + """Mitos is a library and a tool for collecting sampled memory + performance data to view with MemAxes""" + + homepage = "https://github.com/scalability-llnl/Mitos" + url = "https://github.com/scalability-llnl/Mitos" + + version('0.7', git='https://github.com/scalability-llnl/Mitos.git', tag='v0.7') + + depends_on('dyninst') + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") diff --git a/var/spack/packages/psapi/package.py b/var/spack/packages/psapi/package.py deleted file mode 100644 index 489c4f530e5..00000000000 --- a/var/spack/packages/psapi/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Psapi(Package): - """PSAPI is a library and a tool for collecting sampled memory - performance data to view with MemAxes""" - - homepage = "https://github.com/scalability-llnl/PSAPI" - url = "http://www.example.com/memaxes-psapi-1.0.tar.gz" - - version('0.6', git='https://github.com/scalability-llnl/PSAPI.git', tag='v0.6') - - depends_on('dyninst') - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', *std_cmake_args) - make() - make("install") From 88afad3e46326c56aaeaed6adc2058033ad7ad33 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 6 Jan 2015 14:47:51 -0500 Subject: [PATCH 11/97] Directory layout can now track installed extensions per package. --- lib/spack/spack/directory_layout.py | 97 ++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 42cac0c9d21..4ab9a515cf9 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -71,6 +71,21 @@ def make_path_for_spec(self, spec): raise NotImplementedError() + def get_extensions(self, spec): + """Get a set of currently installed extension packages for a spec.""" + raise NotImplementedError() + + + def add_extension(self, spec, extension_spec): + """Add to the list of currently installed extensions.""" + raise NotImplementedError() + + + def remove_extension(self, spec, extension_spec): + """Remove from the list of currently installed extensions.""" + raise NotImplementedError() + + def path_for_spec(self, spec): """Return an absolute path from the root to a directory for the spec.""" _check_concrete(spec) @@ -134,9 +149,11 @@ def __init__(self, root, **kwargs): """Prefix size is number of characters in the SHA-1 prefix to use to make each hash unique. """ - spec_file_name = kwargs.get('spec_file_name', '.spec') + spec_file_name = kwargs.get('spec_file_name', '.spec') + extension_file_name = kwargs.get('extension_file_name', '.extensions') super(SpecHashDirectoryLayout, self).__init__(root) self.spec_file_name = spec_file_name + self.extension_file_name = extension_file_name def relative_path_for_spec(self, spec): @@ -225,6 +242,55 @@ def all_specs(self): yield spec + def extension_file_path(self, spec): + """Gets full path to an installed package's extension file""" + _check_concrete(spec) + return join_path(self.path_for_spec(spec), self.extension_file_name) + + + def get_extensions(self, spec): + path = self.extension_file_path(spec) + + extensions = set() + if os.path.exists(path): + with closing(open(path)) as spec_file: + for line in spec_file: + try: + extensions.add(Spec(line)) + except SpecError, e: + raise InvalidExtensionSpecError(str(e)) + return extensions + + + def write_extensions(self, extensions): + path = self.extension_file_path(spec) + with closing(open(path, 'w')) as spec_file: + for extension in sorted(extensions): + spec_file.write("%s\n" % extensions) + + + def add_extension(self, spec, extension_spec): + exts = get_extensions(spec) + if extension_spec in exts: + raise ExtensionAlreadyInstalledError(spec, extension_spec) + else: + for already_installed in exts: + if spec.name == extension_spec.name: + raise ExtensionConflictError(spec, extension_spec, already_installed) + + exts.add(extension_spec) + self.write_extensions(exts) + + + def remove_extension(self, spec, extension_spec): + exts = get_extensions(spec) + if not extension_spec in exts: + raise NoSuchExtensionError(spec, extension_spec) + + exts.remove(extension_spec) + self.write_extensions(exts) + + class DirectoryLayoutError(SpackError): """Superclass for directory layout errors.""" def __init__(self, message): @@ -250,3 +316,32 @@ class InstallDirectoryAlreadyExistsError(DirectoryLayoutError): def __init__(self, path): super(InstallDirectoryAlreadyExistsError, self).__init__( "Install path %s already exists!") + + +class InvalidExtensionSpecError(DirectoryLayoutError): + """Raised when an extension file has a bad spec in it.""" + def __init__(self, message): + super(InvalidExtensionSpecError, self).__init__(message) + + +class ExtensionAlreadyInstalledError(DirectoryLayoutError): + """Raised when an extension is added to a package that already has it.""" + def __init__(self, spec, extension_spec): + super(ExtensionAlreadyInstalledError, self).__init__( + "%s is already installed in %s" % (extension_spec, spec)) + + +class ExtensionConflictError(DirectoryLayoutError): + """Raised when an extension is added to a package that already has it.""" + def __init__(self, spec, extension_spec, conflict): + super(ExtensionConflictError, self).__init__( + "%s cannot be installed in %s because it conflicts with %s."% ( + extension_spec, spec, conflict)) + + +class NoSuchExtensionError(DirectoryLayoutError): + """Raised when an extension isn't there on remove.""" + def __init__(self, spec, extension_spec): + super(NoSuchExtensionError, self).__init__( + "%s cannot be removed from %s beacuse it's not installed."% ( + extension_spec, spec, conflict)) From ebe0c1d83ac1380a6320a8dadcfa2ad4fc07c279 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 6 Jan 2015 14:49:13 -0500 Subject: [PATCH 12/97] New "extends" relation adds another special list to the package class. --- lib/spack/spack/package.py | 3 +++ lib/spack/spack/relations.py | 26 +++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1dfd3d1c834..c256ea479f0 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -314,6 +314,9 @@ class SomePackage(Package): """Specs of virtual packages provided by this package, keyed by name.""" provided = {} + """Specs of packages this one extends, keyed by name.""" + extendees = {} + """Specs of conflicting packages, keyed by name. """ conflicted = {} diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py index b1f43489454..aaca9c199e4 100644 --- a/lib/spack/spack/relations.py +++ b/lib/spack/spack/relations.py @@ -107,8 +107,9 @@ def depends_on(*specs): """Adds a dependencies local variable in the locals of the calling class, based on args. """ pkg = get_calling_package_name() + clocals = caller_locals() + dependencies = clocals.setdefault('dependencies', {}) - dependencies = caller_locals().setdefault('dependencies', {}) for string in specs: for spec in spack.spec.parse(string): if pkg == spec.name: @@ -116,6 +117,29 @@ def depends_on(*specs): dependencies[spec.name] = spec +def extends(*specs): + """Same as depends_on, but dependency is symlinked into parent prefix. + + This is for Python and other language modules where the module + needs to be installed into the prefix of the Python installation. + Spack handles this by installing modules into their own prefix, + but allowing ONE module version to be symlinked into a parent + Python install at a time. + + """ + pkg = get_calling_package_name() + clocals = caller_locals() + dependencies = clocals.setdefault('dependencies', {}) + extendees = clocals.setdefault('extendees', {}) + + for string in specs: + for spec in spack.spec.parse(string): + if pkg == spec.name: + raise CircularReferenceError('depends_on', pkg) + dependencies[spec.name] = spec + extendees[spec.name] = spec + + def provides(*specs, **kwargs): """Allows packages to provide a virtual dependency. If a package provides 'mpi', other packages can declare that they depend on "mpi", and spack From adb7d614e69a0c176c86b3b4aaa1e81d403d0a71 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 6 Jan 2015 14:50:14 -0500 Subject: [PATCH 13/97] Add pre-install and pre-uninstall hooks. --- lib/spack/spack/hooks/__init__.py | 7 ++++++- lib/spack/spack/package.py | 8 ++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/hooks/__init__.py b/lib/spack/spack/hooks/__init__.py index 98b7f2323f2..1c44e8abaaa 100644 --- a/lib/spack/spack/hooks/__init__.py +++ b/lib/spack/spack/hooks/__init__.py @@ -31,7 +31,9 @@ Currently the following hooks are supported: + * pre_install() * post_install() + * pre_uninstall() * post_uninstall() This can be used to implement support for things like module @@ -70,5 +72,8 @@ def __call__(self, pkg): # # Define some functions that can be called to fire off hooks. # -post_install = HookRunner('post_install') +pre_install = HookRunner('pre_install') +post_install = HookRunner('post_install') + +pre_uninstall = HookRunner('pre_uninstall') post_uninstall = HookRunner('post_uninstall') diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c256ea479f0..aa79721266c 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -768,6 +768,10 @@ def do_install(self, **kwargs): # package naming scheme it likes. spack.install_layout.make_path_for_spec(self.spec) + # Run the pre-install hook in the child process after + # the directory is created. + spack.hooks.pre_install(self) + # Set up process's build environment before running install. self.stage.chdir_to_source() build_env.setup_package(self) @@ -862,6 +866,10 @@ def do_uninstall(self, **kwargs): "The following installed packages depend on it: %s" % ' '.join(formatted_deps)) + # Pre-uninstall hook runs first. + spack.hooks.pre_uninstall(self) + + # Uninstalling in Spack only requires removing the prefix. self.remove_prefix() tty.msg("Successfully uninstalled %s." % self.spec.short_spec) From 2c1eda66c4d7a0df8f0a05ad16be38942f54dcee Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 6 Jan 2015 14:50:40 -0500 Subject: [PATCH 14/97] First python extension package: setuptools --- var/spack/packages/py-setuptools/package.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 var/spack/packages/py-setuptools/package.py diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py new file mode 100644 index 00000000000..e2c4e1a0be6 --- /dev/null +++ b/var/spack/packages/py-setuptools/package.py @@ -0,0 +1,19 @@ +from spack import * + +class PySetuptools(Package): + """Easily download, build, install, upgrade, and uninstall Python packages.""" + homepage = "https://pypi.python.org/pypi/setuptools" + url = "https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz" + + version('11.3.1', '01f69212e019a2420c1693fb43593930') + + extends('python') + + def install(self, spec, prefix): + site_packages_dir = "%s/lib/python2.7/site-packages" % prefix + mkdirp(site_packages_dir) + + env['PYTHONPATH'] = site_packages_dir + + python = which('python') + python('setup.py', 'install', '--prefix=%s' % prefix) From 7215aee224150d954e8a5bd6b632b6d8f66948d2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 7 Jan 2015 17:45:14 -0500 Subject: [PATCH 15/97] do_install() passes kwargs to dependencies. --- lib/spack/spack/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index aa79721266c..04f0d842da9 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -744,7 +744,7 @@ def do_install(self, **kwargs): tty.msg("Installing %s" % self.name) if not ignore_deps: - self.do_install_dependencies() + self.do_install_dependencies(**kwargs) start_time = time.time() if not fake_install: @@ -832,10 +832,10 @@ def do_install(self, **kwargs): spack.hooks.post_install(self) - def do_install_dependencies(self): + def do_install_dependencies(self, **kwargs): # Pass along paths of dependencies here for dep in self.spec.dependencies.values(): - dep.package.do_install() + dep.package.do_install(**kwargs) @property From 99775434785779d223997a9e41972da470214e5d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 7 Jan 2015 11:48:21 -0800 Subject: [PATCH 16/97] Added feature: package extensions - packages can be "extended" by others - allows extension to be symlinked into extendee's prefix. - used for python modules. - first module: py-setuptools --- lib/spack/llnl/util/filesystem.py | 82 ++++++++++++++++++++++++++- lib/spack/spack/__init__.py | 2 +- lib/spack/spack/directory_layout.py | 55 +++++++++++++----- lib/spack/spack/hooks/extensions.py | 49 ++++++++++++++++ lib/spack/spack/package.py | 85 ++++++++++++++++++++++++++++ lib/spack/spack/relations.py | 4 +- var/spack/packages/python/package.py | 3 + 7 files changed, 261 insertions(+), 19 deletions(-) create mode 100644 lib/spack/spack/hooks/extensions.py diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 05784156530..9fb76d3a35d 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -24,7 +24,8 @@ ############################################################################## __all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir', 'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor', - 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe'] + 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe', + 'check_link_tree', 'merge_link_tree', 'unmerge_link_tree'] import os import sys @@ -222,3 +223,82 @@ def ancestor(dir, n=1): def can_access(file_name): """True if we have read/write access to the file.""" return os.access(file_name, os.R_OK|os.W_OK) + + +def traverse_link_tree(src_root, dest_root, follow_nonexisting=True, **kwargs): + # Yield directories before or after their contents. + order = kwargs.get('order', 'pre') + if order not in ('pre', 'post'): + raise ValueError("Order must be 'pre' or 'post'.") + + # List of relative paths to ignore under the src root. + ignore = kwargs.get('ignore', None) + if isinstance(ignore, basestring): + ignore = (ignore,) + + for dirpath, dirnames, filenames in os.walk(src_root): + rel_path = dirpath[len(src_root):] + rel_path = rel_path.lstrip(os.path.sep) + dest_dirpath = os.path.join(dest_root, rel_path) + + # Don't descend into ignored directories + if ignore and dest_dirpath in ignore: + return + + # Don't descend into dirs in dest that do not exist in src. + if not follow_nonexisting: + dirnames[:] = [ + d for d in dirnames + if os.path.exists(os.path.join(dest_dirpath, d))] + + # preorder yields directories before children + if order == 'pre': + yield (dirpath, dest_dirpath) + + for name in filenames: + src_file = os.path.join(dirpath, name) + dest_file = os.path.join(dest_dirpath, name) + + # Ignore particular paths inside the install root. + src_relpath = src_file[len(src_root):] + src_relpath = src_relpath.lstrip(os.path.sep) + if ignore and src_relpath in ignore: + continue + + yield (src_file, dest_file) + + # postorder yields directories after children + if order == 'post': + yield (dirpath, dest_dirpath) + + + +def check_link_tree(src_root, dest_root, **kwargs): + for src, dest in traverse_link_tree(src_root, dest_root, False, **kwargs): + if os.path.exists(dest) and not os.path.isdir(dest): + return dest + return None + + +def merge_link_tree(src_root, dest_root, **kwargs): + kwargs['order'] = 'pre' + for src, dest in traverse_link_tree(src_root, dest_root, **kwargs): + if os.path.isdir(src): + mkdirp(dest) + else: + assert(not os.path.exists(dest)) + os.symlink(src, dest) + + +def unmerge_link_tree(src_root, dest_root, **kwargs): + kwargs['order'] = 'post' + for src, dest in traverse_link_tree(src_root, dest_root, **kwargs): + if os.path.isdir(dest): + if not os.listdir(dest): + # TODO: what if empty directories were present pre-merge? + shutil.rmtree(dest, ignore_errors=True) + + elif os.path.exists(dest): + if not os.path.islink(dest): + raise ValueError("%s is not a link tree!" % dest) + os.remove(dest) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6697e00e40e..6763411f7d3 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -138,7 +138,7 @@ # should live. This file is overloaded for spack core vs. for packages. # __all__ = ['Package', 'Version', 'when', 'ver'] -from spack.package import Package +from spack.package import Package, ExtensionConflictError from spack.version import Version, ver from spack.multimethod import when diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 4ab9a515cf9..ff327ed5047 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -53,6 +53,19 @@ def __init__(self, root): self.root = root + @property + def hidden_file_paths(self): + """Return a list of hidden files used by the directory layout. + + Paths are relative to the root of an install directory. + + If the directory layout uses no hidden files to maintain + state, this should return an empty container, e.g. [] or (,). + + """ + raise NotImplementedError() + + def all_specs(self): """To be implemented by subclasses to traverse all specs for which there is a directory within the root. @@ -156,6 +169,11 @@ def __init__(self, root, **kwargs): self.extension_file_name = extension_file_name + @property + def hidden_file_paths(self): + return ('.spec', '.extensions') + + def relative_path_for_spec(self, spec): _check_concrete(spec) dir_name = spec.format('$_$@$+$#') @@ -249,28 +267,32 @@ def extension_file_path(self, spec): def get_extensions(self, spec): - path = self.extension_file_path(spec) + _check_concrete(spec) + path = self.extension_file_path(spec) extensions = set() if os.path.exists(path): - with closing(open(path)) as spec_file: - for line in spec_file: + with closing(open(path)) as ext_file: + for line in ext_file: try: - extensions.add(Spec(line)) - except SpecError, e: + extensions.add(Spec(line.strip())) + except spack.error.SpackError, e: raise InvalidExtensionSpecError(str(e)) return extensions - def write_extensions(self, extensions): + def write_extensions(self, spec, extensions): path = self.extension_file_path(spec) with closing(open(path, 'w')) as spec_file: for extension in sorted(extensions): - spec_file.write("%s\n" % extensions) + spec_file.write("%s\n" % extension) def add_extension(self, spec, extension_spec): - exts = get_extensions(spec) + _check_concrete(spec) + _check_concrete(extension_spec) + + exts = self.get_extensions(spec) if extension_spec in exts: raise ExtensionAlreadyInstalledError(spec, extension_spec) else: @@ -279,16 +301,19 @@ def add_extension(self, spec, extension_spec): raise ExtensionConflictError(spec, extension_spec, already_installed) exts.add(extension_spec) - self.write_extensions(exts) + self.write_extensions(spec, exts) def remove_extension(self, spec, extension_spec): - exts = get_extensions(spec) + _check_concrete(spec) + _check_concrete(extension_spec) + + exts = self.get_extensions(spec) if not extension_spec in exts: raise NoSuchExtensionError(spec, extension_spec) exts.remove(extension_spec) - self.write_extensions(exts) + self.write_extensions(spec, exts) class DirectoryLayoutError(SpackError): @@ -328,7 +353,7 @@ class ExtensionAlreadyInstalledError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" def __init__(self, spec, extension_spec): super(ExtensionAlreadyInstalledError, self).__init__( - "%s is already installed in %s" % (extension_spec, spec)) + "%s is already installed in %s" % (extension_spec.short_spec, spec.short_spec)) class ExtensionConflictError(DirectoryLayoutError): @@ -336,12 +361,12 @@ class ExtensionConflictError(DirectoryLayoutError): def __init__(self, spec, extension_spec, conflict): super(ExtensionConflictError, self).__init__( "%s cannot be installed in %s because it conflicts with %s."% ( - extension_spec, spec, conflict)) + extension_spec.short_spec, spec.short_spec, conflict.short_spec)) class NoSuchExtensionError(DirectoryLayoutError): """Raised when an extension isn't there on remove.""" def __init__(self, spec, extension_spec): super(NoSuchExtensionError, self).__init__( - "%s cannot be removed from %s beacuse it's not installed."% ( - extension_spec, spec, conflict)) + "%s cannot be removed from %s because it's not installed."% ( + extension_spec.short_spec, spec.short_spec)) diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py new file mode 100644 index 00000000000..444472bffa1 --- /dev/null +++ b/lib/spack/spack/hooks/extensions.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import spack + + +def post_install(pkg): + assert(pkg.spec.concrete) + for name, spec in pkg.extendees.items(): + ext = pkg.spec[name] + epkg = ext.package + if epkg.installed: + epkg.do_activate(pkg) + + +def pre_uninstall(pkg): + assert(pkg.spec.concrete) + + # Need to do this b/c uninstall does not automatically do it. + # TODO: store full graph info in stored .spec file. + pkg.spec.normalize() + + for name, spec in pkg.extendees.items(): + ext = pkg.spec[name] + epkg = ext.package + if epkg.installed: + epkg.do_deactivate(pkg) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 04f0d842da9..b7dae552e47 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -329,6 +329,9 @@ class SomePackage(Package): """By default we build in parallel. Subclasses can override this.""" parallel = True + """Most packages are NOT extendable. Set to True if you want extensions.""" + extendable = False + def __init__(self, spec): # this determines how the package should be built. @@ -398,6 +401,9 @@ def ensure_has_dict(attr_name): self._fetch_time = 0.0 self._total_time = 0.0 + for name, spec in self.extendees.items(): + spack.db.get(spec)._check_extendable() + @property def version(self): @@ -877,6 +883,79 @@ def do_uninstall(self, **kwargs): spack.hooks.post_uninstall(self) + def _check_extendable(self): + if not self.extendable: + raise ValueError("Package %s is not extendable!" % self.name) + + + def _sanity_check_extension(self, extension): + self._check_extendable() + if not self.installed: + raise ValueError("Can only (de)activate extensions for installed packages.") + if not extension.installed: + raise ValueError("Extensions must first be installed.") + if not self.name in extension.extendees: + raise ValueError("%s does not extend %s!" % (extension.name, self.name)) + if not self.spec.satisfies(extension.extendees[self.name]): + raise ValueError("%s does not satisfy %s!" % (self.spec, extension.spec)) + + + def do_activate(self, extension): + self._sanity_check_extension(extension) + + self.activate(extension) + spack.install_layout.add_extension(self.spec, extension.spec) + tty.msg("Activated extension %s for %s." + % (extension.spec.short_spec, self.spec.short_spec)) + + + def activate(self, extension): + """Symlinks all files from the extension into extendee's install dir. + + Package authors can override this method to support other + extension mechanisms. Spack internals (commands, hooks, etc.) + should call do_activate() method so that proper checks are + always executed. + + """ + conflict = check_link_tree( + extension.prefix, self.prefix, + ignore=spack.install_layout.hidden_file_paths) + + if conflict: + raise ExtensionConflictError(conflict) + + merge_link_tree(extension.prefix, self.prefix, + ignore=spack.install_layout.hidden_file_paths) + + + def do_deactivate(self, extension): + self._sanity_check_extension(extension) + self.deactivate(extension) + + ext = extension.spec + if ext in spack.install_layout.get_extensions(self.spec): + spack.install_layout.remove_extension(self.spec, ext) + + tty.msg("Deactivated extension %s for %s." + % (extension.spec.short_spec, self.spec.short_spec)) + + + def deactivate(self, extension): + """Unlinks all files from extension out of extendee's install dir. + + Package authors can override this method to support other + extension mechanisms. Spack internals (commands, hooks, etc.) + should call do_deactivate() method so that proper checks are + always executed. + + """ + unmerge_link_tree(extension.prefix, self.prefix, + ignore=spack.install_layout.hidden_file_paths) + tty.msg("Deactivated %s as extension of %s." + % (extension.spec.short_spec, self.spec.short_spec)) + + def do_clean(self): if self.stage.expanded_archive_path: self.stage.chdir_to_source() @@ -1068,3 +1147,9 @@ class NoURLError(PackageError): def __init__(self, cls): super(NoURLError, self).__init__( "Package %s has no version with a URL." % cls.__name__) + + +class ExtensionConflictError(PackageError): + def __init__(self, path): + super(ExtensionConflictError, self).__init__( + "Extension blocked by file: %s" % path) diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py index aaca9c199e4..17bec1664f6 100644 --- a/lib/spack/spack/relations.py +++ b/lib/spack/spack/relations.py @@ -68,7 +68,7 @@ class Mpileaks(Package): spack install mpileaks ^mvapich spack install mpileaks ^mpich """ -__all__ = [ 'depends_on', 'provides', 'patch', 'version' ] +__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version' ] import re import inspect @@ -135,7 +135,7 @@ def extends(*specs): for string in specs: for spec in spack.spec.parse(string): if pkg == spec.name: - raise CircularReferenceError('depends_on', pkg) + raise CircularReferenceError('extends', pkg) dependencies[spec.name] = spec extendees[spec.name] = spec diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index e6c3e28820a..953be69cc23 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,10 +1,13 @@ from spack import * + class Python(Package): """The Python programming language.""" homepage = "http://www.python.org" url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tar.xz" + extendable = True + version('2.7.8', 'd235bdfa75b8396942e360a70487ee00') depends_on("openssl") From 82946d29147bbe63855f94b9c2ebd4a21cd0a3d6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 8 Jan 2015 22:46:31 -0800 Subject: [PATCH 17/97] Move symlink tree routines to LinkTree class. --- lib/spack/llnl/util/filesystem.py | 82 +-------------- lib/spack/llnl/util/link_tree.py | 168 ++++++++++++++++++++++++++++++ lib/spack/spack/package.py | 16 ++- 3 files changed, 176 insertions(+), 90 deletions(-) create mode 100644 lib/spack/llnl/util/link_tree.py diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 9fb76d3a35d..05784156530 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -24,8 +24,7 @@ ############################################################################## __all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir', 'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor', - 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe', - 'check_link_tree', 'merge_link_tree', 'unmerge_link_tree'] + 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe'] import os import sys @@ -223,82 +222,3 @@ def ancestor(dir, n=1): def can_access(file_name): """True if we have read/write access to the file.""" return os.access(file_name, os.R_OK|os.W_OK) - - -def traverse_link_tree(src_root, dest_root, follow_nonexisting=True, **kwargs): - # Yield directories before or after their contents. - order = kwargs.get('order', 'pre') - if order not in ('pre', 'post'): - raise ValueError("Order must be 'pre' or 'post'.") - - # List of relative paths to ignore under the src root. - ignore = kwargs.get('ignore', None) - if isinstance(ignore, basestring): - ignore = (ignore,) - - for dirpath, dirnames, filenames in os.walk(src_root): - rel_path = dirpath[len(src_root):] - rel_path = rel_path.lstrip(os.path.sep) - dest_dirpath = os.path.join(dest_root, rel_path) - - # Don't descend into ignored directories - if ignore and dest_dirpath in ignore: - return - - # Don't descend into dirs in dest that do not exist in src. - if not follow_nonexisting: - dirnames[:] = [ - d for d in dirnames - if os.path.exists(os.path.join(dest_dirpath, d))] - - # preorder yields directories before children - if order == 'pre': - yield (dirpath, dest_dirpath) - - for name in filenames: - src_file = os.path.join(dirpath, name) - dest_file = os.path.join(dest_dirpath, name) - - # Ignore particular paths inside the install root. - src_relpath = src_file[len(src_root):] - src_relpath = src_relpath.lstrip(os.path.sep) - if ignore and src_relpath in ignore: - continue - - yield (src_file, dest_file) - - # postorder yields directories after children - if order == 'post': - yield (dirpath, dest_dirpath) - - - -def check_link_tree(src_root, dest_root, **kwargs): - for src, dest in traverse_link_tree(src_root, dest_root, False, **kwargs): - if os.path.exists(dest) and not os.path.isdir(dest): - return dest - return None - - -def merge_link_tree(src_root, dest_root, **kwargs): - kwargs['order'] = 'pre' - for src, dest in traverse_link_tree(src_root, dest_root, **kwargs): - if os.path.isdir(src): - mkdirp(dest) - else: - assert(not os.path.exists(dest)) - os.symlink(src, dest) - - -def unmerge_link_tree(src_root, dest_root, **kwargs): - kwargs['order'] = 'post' - for src, dest in traverse_link_tree(src_root, dest_root, **kwargs): - if os.path.isdir(dest): - if not os.listdir(dest): - # TODO: what if empty directories were present pre-merge? - shutil.rmtree(dest, ignore_errors=True) - - elif os.path.exists(dest): - if not os.path.islink(dest): - raise ValueError("%s is not a link tree!" % dest) - os.remove(dest) diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py new file mode 100644 index 00000000000..19c2d46938f --- /dev/null +++ b/lib/spack/llnl/util/link_tree.py @@ -0,0 +1,168 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""LinkTree class for setting up trees of symbolic links.""" +__all__ = ['LinkTree'] + +import os +import shutil +from llnl.util.filesystem import mkdirp + + +class LinkTree(object): + """Class to create trees of symbolic links from a source directory. + + LinkTree objects are constructed with a source root. Their + methods allow you to create and delete trees of symbolic links + back to the source tree in specific destination directories. + Trees comprise symlinks only to files; directries are never + symlinked to, to prevent the source directory from ever being + modified. + + """ + def __init__(self, source_root): + self._root = source_root + + + def traverse(self, dest_root, **kwargs): + """Traverse LinkTree root and dest simultaneously. + + Walks the LinkTree directory in pre or post order. Yields + each file in the source directory with a matching path from + the dest directory. e.g., for this tree:: + + root/ + a/ + file1 + file2 + b/ + file3 + + When called on dest, this yields:: + + ('root', 'dest') + ('root/a', 'dest/a') + ('root/a/file1', 'dest/a/file1') + ('root/a/file2', 'dest/a/file2') + ('root/b', 'dest/b') + ('root/b/file3', 'dest/b/file3') + + Optional args: + + order=[pre|post] -- Whether to do pre- or post-order traveral. + + ignore= -- Optional container of root-relative + paths to ignore. + + follow_nonexisting -- Whether to descend into directories in + src that do not exit in dest. + + """ + # Yield directories before or after their contents. + order = kwargs.get('order', 'pre') + if order not in ('pre', 'post'): + raise ValueError("Order must be 'pre' or 'post'.") + + # List of relative paths to ignore under the src root. + ignore = kwargs.get('ignore', None) + if isinstance(ignore, basestring): + ignore = (ignore,) + + # Whether to descend when dirs dont' exist in dest. + follow_nonexisting = kwargs.get('follow_nonexisting', True) + + for dirpath, dirnames, filenames in os.walk(self._root): + rel_path = dirpath[len(self._root):] + rel_path = rel_path.lstrip(os.path.sep) + dest_dirpath = os.path.join(dest_root, rel_path) + + # Don't descend into ignored directories + if ignore and dest_dirpath in ignore: + return + + # Don't descend into dirs in dest that do not exist in src. + if not follow_nonexisting: + dirnames[:] = [ + d for d in dirnames + if os.path.exists(os.path.join(dest_dirpath, d))] + + # preorder yields directories before children + if order == 'pre': + yield (dirpath, dest_dirpath) + + for name in filenames: + src_file = os.path.join(dirpath, name) + dest_file = os.path.join(dest_dirpath, name) + + # Ignore particular paths inside the install root. + src_relpath = src_file[len(self._root):] + src_relpath = src_relpath.lstrip(os.path.sep) + if ignore and src_relpath in ignore: + continue + + yield (src_file, dest_file) + + # postorder yields directories after children + if order == 'post': + yield (dirpath, dest_dirpath) + + + + def find_conflict(self, dest_root, **kwargs): + """Returns the first file in dest that also exists in src.""" + kwargs['follow_nonexisting'] = False + for src, dest in self.traverse(dest_root, **kwargs): + if os.path.exists(dest) and not os.path.isdir(dest): + return dest + return None + + + def merge(self, dest_root, **kwargs): + """Link all files in src into dest, creating directories if necessary.""" + kwargs['order'] = 'pre' + for src, dest in self.traverse(dest_root, **kwargs): + if os.path.isdir(src): + mkdirp(dest) + else: + assert(not os.path.exists(dest)) + os.symlink(src, dest) + + + def unmerge(self, dest_root, **kwargs): + """Unlink all files in dest that exist in src. + + Unlinks directories in dest if they are empty. + + """ + kwargs['order'] = 'post' + for src, dest in self.traverse(dest_root, **kwargs): + if os.path.isdir(dest): + if not os.listdir(dest): + # TODO: what if empty directories were present pre-merge? + shutil.rmtree(dest, ignore_errors=True) + + elif os.path.exists(dest): + if not os.path.islink(dest): + raise ValueError("%s is not a link tree!" % dest) + os.remove(dest) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b7dae552e47..da251dc4e83 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -45,6 +45,7 @@ from StringIO import StringIO import llnl.util.tty as tty +from llnl.util.link_tree import LinkTree from llnl.util.filesystem import * from llnl.util.lang import * @@ -918,15 +919,12 @@ def activate(self, extension): always executed. """ - conflict = check_link_tree( - extension.prefix, self.prefix, - ignore=spack.install_layout.hidden_file_paths) - + tree = LinkTree(extension.prefix) + conflict = tree.find_conflict( + self.prefix, ignore=spack.install_layout.hidden_file_paths) if conflict: raise ExtensionConflictError(conflict) - - merge_link_tree(extension.prefix, self.prefix, - ignore=spack.install_layout.hidden_file_paths) + tree.merge(self.prefix, ignore=spack.install_layout.hidden_file_paths) def do_deactivate(self, extension): @@ -950,8 +948,8 @@ def deactivate(self, extension): always executed. """ - unmerge_link_tree(extension.prefix, self.prefix, - ignore=spack.install_layout.hidden_file_paths) + tree = LinkTree(extension.prefix) + tree.unmerge(self.prefix, ignore=spack.install_layout.hidden_file_paths) tty.msg("Deactivated %s as extension of %s." % (extension.spec.short_spec, self.spec.short_spec)) From bcccf020204a556e382c0af2897ad9126bb24984 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 10 Jan 2015 19:37:01 -0800 Subject: [PATCH 18/97] Add setup_extension_environment() method. - lets packages do some setup before their extensions run install() --- lib/spack/spack/package.py | 30 +++++++++++++++++++++ var/spack/packages/py-setuptools/package.py | 6 ----- var/spack/packages/python/package.py | 23 +++++++++++++++- 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index da251dc4e83..8504b96fcf5 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -783,6 +783,12 @@ def do_install(self, **kwargs): self.stage.chdir_to_source() build_env.setup_package(self) + # Allow extendees to further set up the environment. + for ext_name in self.extendees: + ext_spec = self.spec[ext_name] + ext_spec.package.setup_extension_environment( + self.module, ext_spec, self.spec) + if fake_install: self.do_fake_install() else: @@ -854,6 +860,30 @@ def module(self): fromlist=[self.__class__.__name__]) + def setup_extension_environment(self, module, spec, ext_spec): + """Called before the install() method of extensions. + + Default implementation does nothing, but this can be + overridden by an extendable package to set up the install + environment for its extensions. This is useful if there are + some common steps to installing all extensions for a + certain package. + + Some examples: + + 1. Installing python modules generally requires PYTHONPATH to + point to the lib/pythonX.Y/site-packages directory in the + module's install prefix. This could set that variable. + + 2. Extensions often need to invoke the 'python' interpreter + from the Python installation being extended. This routine can + put a 'python' Execuable object in the module scope for the + extension package to simplify extension installs. + + """ + pass + + def install(self, spec, prefix): """Package implementations override this with their own build configuration.""" raise InstallError("Package %s provides no install method!" % self.name) diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py index e2c4e1a0be6..755288d55c3 100644 --- a/var/spack/packages/py-setuptools/package.py +++ b/var/spack/packages/py-setuptools/package.py @@ -10,10 +10,4 @@ class PySetuptools(Package): extends('python') def install(self, spec, prefix): - site_packages_dir = "%s/lib/python2.7/site-packages" % prefix - mkdirp(site_packages_dir) - - env['PYTHONPATH'] = site_packages_dir - - python = which('python') python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 953be69cc23..9700179ab8e 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,5 +1,5 @@ from spack import * - +import os class Python(Package): """The Python programming language.""" @@ -26,3 +26,24 @@ def install(self, spec, prefix): "--enable-shared") make() make("install") + + + def setup_extension_environment(self, module, spec, ext_spec): + """Called before python modules' install() methods. + + In most cases, extensions will only need to have one line:: + + python('setup.py', 'install', '--prefix=%s' % prefix) + """ + # Python extension builds can have a global python executable function + module.python = Executable(join_path(spec.prefix.bin, 'python')) + + # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. + module.python_lib_dir = join_path(ext_spec.prefix.lib, 'python%d.%d' % self.version[:2]) + module.site_packages_dir = join_path(module.python_lib_dir, 'site-packages') + + # Add site packages directory to the PYTHONPATH + os.environ['PYTHONPATH'] = module.site_packages_dir + + # Make the site packages directory if it does not exist already. + mkdirp(module.site_packages_dir) From d13bbeb605f56214db919f6f122a8fa6ba67ddbc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 10 Jan 2015 19:52:07 -0800 Subject: [PATCH 19/97] Add PYTOHNPATH to modules for python extensions. --- lib/spack/spack/modules.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 755e9ea9004..7d2ca97a62b 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -49,6 +49,7 @@ import re import textwrap import shutil +from glob import glob from contextlib import closing import llnl.util.tty as tty @@ -123,6 +124,13 @@ def add_path(path_name, directory): if os.path.isdir(directory): add_path(var, directory) + # Add python path unless it's an actual python installation + # TODO: is there a better way to do this? + if self.spec.name != 'python': + site_packages = glob(join_path(self.spec.prefix.lib, "python*/site-packages")) + if site_packages: + add_path('PYTHONPATH', site_packages[0]) + # short description is just the package + version # TODO: maybe packages can optionally provide it. self.short_description = self.spec.format("$_ $@") From acc62abbd08f6d44c930b0ceed74fb9e47f365cf Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 12 Jan 2015 22:39:18 -0800 Subject: [PATCH 20/97] Rework do_activate/activate and do_deactivate/deactivate semantics. - packages can now extend only one other package. - do_activate() and do_deactivate() are now called on the extension, and they automatically find the extendee - activate() and deactivate() are still called on the extendee and are passed the extension. --- lib/spack/spack/cmd/__init__.py | 15 +++++ lib/spack/spack/cmd/location.py | 47 ++++++-------- lib/spack/spack/cmd/uninstall.py | 1 - lib/spack/spack/hooks/extensions.py | 15 +---- lib/spack/spack/package.py | 97 +++++++++++++++++++---------- lib/spack/spack/packages.py | 2 + lib/spack/spack/relations.py | 2 + 7 files changed, 105 insertions(+), 74 deletions(-) diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py index 537db536ddd..b96ac5af510 100644 --- a/lib/spack/spack/cmd/__init__.py +++ b/lib/spack/spack/cmd/__init__.py @@ -121,3 +121,18 @@ def elide_list(line_list, max_num=10): return line_list[:max_num-1] + ['...'] + line_list[-1:] else: return line_list + + +def disambiguate_spec(spec): + matching_specs = spack.db.get_installed(spec) + if not matching_specs: + tty.die("Spec '%s' matches no installed packages." % spec) + + elif len(matching_specs) > 1: + args = ["%s matches multiple packages." % spec, + "Matching packages:"] + args += [" " + str(s) for s in matching_specs] + args += ["Use a more specific spec."] + tty.die(*args) + + return matching_specs[0] diff --git a/lib/spack/spack/cmd/location.py b/lib/spack/spack/cmd/location.py index 509c336b69d..810c34d0a61 100644 --- a/lib/spack/spack/cmd/location.py +++ b/lib/spack/spack/cmd/location.py @@ -77,37 +77,30 @@ def location(parser, args): tty.die("You must supply a spec.") if len(specs) != 1: tty.die("Too many specs. Supply only one.") - spec = specs[0] if args.install_dir: # install_dir command matches against installed specs. - matching_specs = spack.db.get_installed(spec) - if not matching_specs: - tty.die("Spec '%s' matches no installed packages." % spec) - - elif len(matching_specs) > 1: - args = ["%s matches multiple packages." % spec, - "Matching packages:"] - args += [" " + str(s) for s in matching_specs] - args += ["Use a more specific spec."] - tty.die(*args) - - print matching_specs[0].prefix - - elif args.package_dir: - # This one just needs the spec name. - print join_path(spack.db.root, spec.name) + spec = spack.cmd.disambiguate_spec(specs[0]) + print spec.prefix else: - # These versions need concretized specs. - spec.concretize() - pkg = spack.db.get(spec) + spec = specs[0] - if args.stage_dir: - print pkg.stage.path + if args.package_dir: + # This one just needs the spec name. + print join_path(spack.db.root, spec.name) + + else: + # These versions need concretized specs. + spec.concretize() + pkg = spack.db.get(spec) + + if args.stage_dir: + print pkg.stage.path + + else: # args.build_dir is the default. + if not pkg.stage.source_path: + tty.die("Build directory does not exist yet. Run this to create it:", + "spack stage " + " ".join(args.spec)) + print pkg.stage.source_path - else: # args.build_dir is the default. - if not pkg.stage.source_path: - tty.die("Build directory does not exist yet. Run this to create it:", - "spack stage " + " ".join(args.spec)) - print pkg.stage.source_path diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index e787c460ad3..0962942f437 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -65,7 +65,6 @@ def uninstall(parser, args): " b) use a more specific spec."] tty.die(*args) - if len(matching_specs) == 0: tty.die("%s does not match any installed packages." % spec) diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index 444472bffa1..2cf506beed6 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -27,23 +27,12 @@ def post_install(pkg): - assert(pkg.spec.concrete) - for name, spec in pkg.extendees.items(): - ext = pkg.spec[name] - epkg = ext.package - if epkg.installed: - epkg.do_activate(pkg) + pkg.do_activate() def pre_uninstall(pkg): - assert(pkg.spec.concrete) - # Need to do this b/c uninstall does not automatically do it. # TODO: store full graph info in stored .spec file. pkg.spec.normalize() - for name, spec in pkg.extendees.items(): - ext = pkg.spec[name] - epkg = ext.package - if epkg.installed: - epkg.do_deactivate(pkg) + pkg.do_deactivate() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 8504b96fcf5..ae34f8ae455 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -315,15 +315,18 @@ class SomePackage(Package): """Specs of virtual packages provided by this package, keyed by name.""" provided = {} - """Specs of packages this one extends, keyed by name.""" - extendees = {} - """Specs of conflicting packages, keyed by name. """ conflicted = {} """Patches to apply to newly expanded source, if any.""" patches = {} + """Specs of package this one extends, or None. + + Currently, ppackages can extend at most one other package. + """ + extendees = {} + # # These are default values for instance variables. # @@ -402,8 +405,8 @@ def ensure_has_dict(attr_name): self._fetch_time = 0.0 self._total_time = 0.0 - for name, spec in self.extendees.items(): - spack.db.get(spec)._check_extendable() + if self.is_extension: + spack.db.get(self.extendee_spec)._check_extendable() @property @@ -491,6 +494,34 @@ def fetcher(self, f): self._fetcher = f + @property + def extendee_spec(self): + """Spec of the extendee of this package, or None if it is not an extension.""" + if not self.extendees: return None + + name = next(iter(self.extendees)) + if not name in self.spec: + return self.extendees[name] + + # Need to do this to get the concrete version of the spec + return self.spec[name] + + + @property + def is_extension(self): + return len(self.extendees) > 0 + + + @property + def activated(self): + if not self.spec.concrete: + raise ValueError("Only concrete package extensions can be activated.") + if not self.is_extension: + raise ValueError("is_extension called on package that is not an extension.") + + return self.spec in spack.install_layout.get_extensions(self.extendee_spec) + + def preorder_traversal(self, visited=None, **kwargs): """This does a preorder traversal of the package's dependence DAG.""" virtual = kwargs.get("virtual", False) @@ -784,10 +815,9 @@ def do_install(self, **kwargs): build_env.setup_package(self) # Allow extendees to further set up the environment. - for ext_name in self.extendees: - ext_spec = self.spec[ext_name] - ext_spec.package.setup_extension_environment( - self.module, ext_spec, self.spec) + if self.is_extension: + self.extendee_spec.package.setup_extension_environment( + self.module, self.extendee_spec, self.spec) if fake_install: self.do_fake_install() @@ -840,7 +870,6 @@ def do_install(self, **kwargs): if returncode != 0: sys.exit(1) - # Once everything else is done, run post install hooks spack.hooks.post_install(self) @@ -919,25 +948,30 @@ def _check_extendable(self): raise ValueError("Package %s is not extendable!" % self.name) - def _sanity_check_extension(self, extension): - self._check_extendable() - if not self.installed: + def _sanity_check_extension(self): + extendee_package = self.extendee_spec.package + extendee_package._check_extendable() + + if not extendee_package.installed: raise ValueError("Can only (de)activate extensions for installed packages.") - if not extension.installed: + if not self.installed: raise ValueError("Extensions must first be installed.") - if not self.name in extension.extendees: - raise ValueError("%s does not extend %s!" % (extension.name, self.name)) - if not self.spec.satisfies(extension.extendees[self.name]): - raise ValueError("%s does not satisfy %s!" % (self.spec, extension.spec)) + if not self.extendee_spec.name in self.extendees: + raise ValueError("%s does not extend %s!" % (self.name, self.extendee.name)) - def do_activate(self, extension): - self._sanity_check_extension(extension) + def do_activate(self): + """Called on an etension to invoke the extendee's activate method. - self.activate(extension) - spack.install_layout.add_extension(self.spec, extension.spec) + Commands should call this routine, and should not call + activate() directly. + """ + self._sanity_check_extension() + self.extendee_spec.package.activate(self) + + spack.install_layout.add_extension(self.extendee_spec, self.spec) tty.msg("Activated extension %s for %s." - % (extension.spec.short_spec, self.spec.short_spec)) + % (self.spec.short_spec, self.extendee_spec.short_spec)) def activate(self, extension): @@ -957,20 +991,19 @@ def activate(self, extension): tree.merge(self.prefix, ignore=spack.install_layout.hidden_file_paths) - def do_deactivate(self, extension): - self._sanity_check_extension(extension) - self.deactivate(extension) + def do_deactivate(self): + self._sanity_check_extension() + self.extendee_spec.package.deactivate(self) - ext = extension.spec - if ext in spack.install_layout.get_extensions(self.spec): - spack.install_layout.remove_extension(self.spec, ext) + if self.spec in spack.install_layout.get_extensions(self.extendee_spec): + spack.install_layout.remove_extension(self.extendee_spec, self.spec) tty.msg("Deactivated extension %s for %s." - % (extension.spec.short_spec, self.spec.short_spec)) + % (self.spec.short_spec, self.extendee_spec.short_spec)) def deactivate(self, extension): - """Unlinks all files from extension out of extendee's install dir. + """Unlinks all files from extension out of this package's install dir. Package authors can override this method to support other extension mechanisms. Spack internals (commands, hooks, etc.) @@ -980,8 +1013,6 @@ def deactivate(self, extension): """ tree = LinkTree(extension.prefix) tree.unmerge(self.prefix, ignore=spack.install_layout.hidden_file_paths) - tty.msg("Deactivated %s as extension of %s." - % (extension.spec.short_spec, self.spec.short_spec)) def do_clean(self): diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index db43d3909ad..bb5a94bcab9 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -77,6 +77,8 @@ def get(self, spec, **kwargs): copy = spec.copy() self.instances[copy] = package_class(copy) except Exception, e: + if spack.debug: + sys.excepthook(*sys.exc_info()) raise FailedConstructorError(spec.name, e) return self.instances[spec] diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py index 17bec1664f6..60ff5bef34e 100644 --- a/lib/spack/spack/relations.py +++ b/lib/spack/spack/relations.py @@ -131,6 +131,8 @@ def extends(*specs): clocals = caller_locals() dependencies = clocals.setdefault('dependencies', {}) extendees = clocals.setdefault('extendees', {}) + if extendees: + raise RelationError("Packages can extend at most one other package.") for string in specs: for spec in spack.spec.parse(string): From 89ccdf92cd42a52fa42d39e2be580eac110e264a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 12 Jan 2015 22:41:27 -0800 Subject: [PATCH 21/97] Add activate and deactivate commands for extensions. --- lib/spack/spack/cmd/activate.py | 50 +++++++++++++++++++++++++++++++ lib/spack/spack/cmd/deactivate.py | 50 +++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 lib/spack/spack/cmd/activate.py create mode 100644 lib/spack/spack/cmd/deactivate.py diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py new file mode 100644 index 00000000000..c1e23852d64 --- /dev/null +++ b/lib/spack/spack/cmd/activate.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse +import llnl.util.tty as tty +import spack +import spack.cmd + +description = "Activate a package extension." + +def setup_parser(subparser): + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, help="spec of package extension to activate.") + + +def activate(parser, args): + specs = spack.cmd.parse_specs(args.spec, concretize=True) + if len(specs) != 1: + tty.die("activate requires one spec. %d given." % len(specs)) + + # TODO: remove this hack when DAG info is stored in dir layout. + # This ensures the ext spec is always normalized properly. + spack.db.get(specs[0]) + + spec = spack.cmd.disambiguate_spec(specs[0]) + if spec.package.activated: + tty.die("Package %s is already activated." % specs[0].short_spec) + + spec.package.do_activate() diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py new file mode 100644 index 00000000000..fd13f051dfb --- /dev/null +++ b/lib/spack/spack/cmd/deactivate.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse +import llnl.util.tty as tty +import spack +import spack.cmd + +description = "Deactivate a package extension." + +def setup_parser(subparser): + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, help="spec of package extension to deactivate.") + + +def deactivate(parser, args): + specs = spack.cmd.parse_specs(args.spec, concretize=True) + if len(specs) != 1: + tty.die("deactivate requires one spec. %d given." % len(specs)) + + # TODO: remove this hack when DAG info is stored in dir layout. + # This ensures the ext spec is always normalized properly. + spack.db.get(specs[0]) + + spec = spack.cmd.disambiguate_spec(specs[0]) + if not spec.package.activated: + tty.die("Package %s is not activated." % specs[0].short_spec) + + spec.package.do_deactivate() From 2ae7f53b8359841fc5d1d7b6c70ff72f38bf2d88 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 15 Jan 2015 09:33:51 -0800 Subject: [PATCH 22/97] Bugfix: Extension hooks shoudl only run for extensions. --- lib/spack/spack/hooks/extensions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index 2cf506beed6..718b24b9655 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -27,7 +27,8 @@ def post_install(pkg): - pkg.do_activate() + if pkg.is_extension: + pkg.do_activate() def pre_uninstall(pkg): @@ -35,4 +36,5 @@ def pre_uninstall(pkg): # TODO: store full graph info in stored .spec file. pkg.spec.normalize() - pkg.do_deactivate() + if pkg.is_extension: + pkg.do_deactivate() From 7992f415fe0e50c5e15964f348ed23c1fc1795b5 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 15 Jan 2015 10:11:40 -0800 Subject: [PATCH 23/97] added py-nose --- var/spack/packages/py-nose/package.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 var/spack/packages/py-nose/package.py diff --git a/var/spack/packages/py-nose/package.py b/var/spack/packages/py-nose/package.py new file mode 100644 index 00000000000..7bd7106b8c3 --- /dev/null +++ b/var/spack/packages/py-nose/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyNose(Package): + """nose extends the test loading and running features of unittest, making it easier to write, find and run tests.""" + homepage = "https://pypi.python.org/pypi/nose" + url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" + + version('1.3.4', '6ed7169887580ddc9a8e16048d38274d') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 9fa489b7f2dab452929729e300f189f8c50b817c Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Tue, 20 Jan 2015 09:39:21 -0800 Subject: [PATCH 24/97] added several modules --- var/spack/packages/py-cython/package.py | 13 +++++++++++++ var/spack/packages/py-dateutil/package.py | 13 +++++++++++++ var/spack/packages/py-epydoc/package.py | 13 +++++++++++++ var/spack/packages/py-ipython/package.py | 14 ++++++++++++++ var/spack/packages/py-matplotlib/package.py | 20 ++++++++++++++++++++ var/spack/packages/py-numpy/package.py | 14 ++++++++++++++ var/spack/packages/py-pexpect/package.py | 13 +++++++++++++ var/spack/packages/py-pygments/package.py | 14 ++++++++++++++ var/spack/packages/py-pyparsing/package.py | 13 +++++++++++++ var/spack/packages/py-pyside/package.py | 18 ++++++++++++++++++ var/spack/packages/py-pytz/package.py | 13 +++++++++++++ var/spack/packages/py-scipy/package.py | 15 +++++++++++++++ var/spack/packages/py-six/package.py | 13 +++++++++++++ var/spack/packages/py-virtualenv/package.py | 18 ++++++++++++++++++ 14 files changed, 204 insertions(+) create mode 100644 var/spack/packages/py-cython/package.py create mode 100644 var/spack/packages/py-dateutil/package.py create mode 100644 var/spack/packages/py-epydoc/package.py create mode 100644 var/spack/packages/py-ipython/package.py create mode 100644 var/spack/packages/py-matplotlib/package.py create mode 100644 var/spack/packages/py-numpy/package.py create mode 100644 var/spack/packages/py-pexpect/package.py create mode 100644 var/spack/packages/py-pygments/package.py create mode 100644 var/spack/packages/py-pyparsing/package.py create mode 100644 var/spack/packages/py-pyside/package.py create mode 100644 var/spack/packages/py-pytz/package.py create mode 100644 var/spack/packages/py-scipy/package.py create mode 100644 var/spack/packages/py-six/package.py create mode 100644 var/spack/packages/py-virtualenv/package.py diff --git a/var/spack/packages/py-cython/package.py b/var/spack/packages/py-cython/package.py new file mode 100644 index 00000000000..af67a155264 --- /dev/null +++ b/var/spack/packages/py-cython/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyCython(Package): + """The Cython compiler for writing C extensions for the Python language.""" + homepage = "https://pypi.python.org/pypi/cython" + url = "https://pypi.python.org/packages/source/C/Cython/Cython-0.21.2.tar.gz" + + version('0.21.2', 'd21adb870c75680dc857cd05d41046a4') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-dateutil/package.py b/var/spack/packages/py-dateutil/package.py new file mode 100644 index 00000000000..96e3ecab075 --- /dev/null +++ b/var/spack/packages/py-dateutil/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyDateutil(Package): + """Extensions to the standard Python datetime module.""" + homepage = "https://pypi.python.org/pypi/dateutil" + url = "https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.4.0.tar.gz" + + version('2.4.0', '75714163bb96bedd07685cdb2071b8bc') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-epydoc/package.py b/var/spack/packages/py-epydoc/package.py new file mode 100644 index 00000000000..af055105043 --- /dev/null +++ b/var/spack/packages/py-epydoc/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyEpydoc(Package): + """Epydoc is a tool for generating API documentation documentation for Python modules, based on their docstrings.""" + homepage = "https://pypi.python.org/pypi/epydoc" + url = "https://pypi.python.org/packages/source/e/epydoc/epydoc-3.0.1.tar.gz" + + version('3.0.1', '36407974bd5da2af00bf90ca27feeb44') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py new file mode 100644 index 00000000000..731e661dfda --- /dev/null +++ b/var/spack/packages/py-ipython/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyIpython(Package): + """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" + homepage = "https://pypi.python.org/pypi/ipython" + url = "https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz" + + version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf') + + extends('python') + depends_on('py-pygments') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py new file mode 100644 index 00000000000..836273c9237 --- /dev/null +++ b/var/spack/packages/py-matplotlib/package.py @@ -0,0 +1,20 @@ +from spack import * + +class PyMatplotlib(Package): + """Python plotting package.""" + homepage = "https://pypi.python.org/pypi/matplotlib" + url = "https://pypi.python.org/packages/source/m/matplotlib/matplotlib-1.4.2.tar.gz" + + version('1.4.2', '7d22efb6cce475025733c50487bd8898') + + extends('python') + depends_on('py-pyside') + depends_on('py-ipython') + depends_on('py-pyparsing') + depends_on('py-six') + depends_on('py-dateutil') + depends_on('py-pytz') + depends_on('py-nose') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/packages/py-numpy/package.py new file mode 100644 index 00000000000..e6cb6a464f4 --- /dev/null +++ b/var/spack/packages/py-numpy/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyNumpy(Package): + """array processing for numbers, strings, records, and objects.""" + homepage = "https://pypi.python.org/pypi/numpy" + url = "https://pypi.python.org/packages/source/n/numpy/numpy-1.9.1.tar.gz" + + version('1.9.1', '78842b73560ec378142665e712ae4ad9') + + extends('python') + depends_on('py-nose') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pexpect/package.py b/var/spack/packages/py-pexpect/package.py new file mode 100644 index 00000000000..ff5fac84e06 --- /dev/null +++ b/var/spack/packages/py-pexpect/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPexpect(Package): + """Pexpect allows easy control of interactive console applications.""" + homepage = "https://pypi.python.org/pypi/pexpect" + url = "https://pypi.python.org/packages/source/p/pexpect/pexpect-3.3.tar.gz" + + version('3.3', '0de72541d3f1374b795472fed841dce8') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pygments/package.py b/var/spack/packages/py-pygments/package.py new file mode 100644 index 00000000000..990eebde650 --- /dev/null +++ b/var/spack/packages/py-pygments/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyPygments(Package): + """Pygments is a syntax highlighting package written in Python.""" + homepage = "https://pypi.python.org/pypi/pygments" + url = "https://pypi.python.org/packages/source/P/Pygments/Pygments-2.0.1.tar.gz" + + version('2.0.1', 'e0daf4c14a4fe5b630da765904de4d6c') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pyparsing/package.py b/var/spack/packages/py-pyparsing/package.py new file mode 100644 index 00000000000..a6e50ad1398 --- /dev/null +++ b/var/spack/packages/py-pyparsing/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPyparsing(Package): + """A Python Parsing Module.""" + homepage = "https://pypi.python.org/pypi/pyparsing" + url = "https://pypi.python.org/packages/source/p/pyparsing/pyparsing-2.0.3.tar.gz" + + version('2.0.3', '0fe479be09fc2cf005f753d3acc35939') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py new file mode 100644 index 00000000000..b01e16d7e6c --- /dev/null +++ b/var/spack/packages/py-pyside/package.py @@ -0,0 +1,18 @@ +from spack import * +import spack.package +import os + +class PyPyside(Package): + """array processing for numbers, strings, records, and objects.""" + homepage = "https://pypi.python.org/pypi/pyside" + url = "https://pypi.python.org/packages/source/P/PySide/PySide-1.2.2.tar.gz" + + version('1.2.2', 'c45bc400c8a86d6b35f34c29e379e44d') + + extends('python') + + def install(self, spec, prefix): + qmake_path = '/usr/lib64/qt4/bin/qmake' + if not os.path.exists(qmake_path): + raise spack.package.InstallError("Failed to find qmake in %s" % qmake_path) + python('setup.py', 'install', '--prefix=%s' % prefix, '--qmake=%s' % qmake_path) diff --git a/var/spack/packages/py-pytz/package.py b/var/spack/packages/py-pytz/package.py new file mode 100644 index 00000000000..80bcfe82ca9 --- /dev/null +++ b/var/spack/packages/py-pytz/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPytz(Package): + """World timezone definitions, modern and historical.""" + homepage = "https://pypi.python.org/pypi/pytz" + url = "https://pypi.python.org/packages/source/p/pytz/pytz-2014.10.tar.gz" + + version('2014.10', 'eb1cb941a20c5b751352c52486aa1dd7') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scipy/package.py b/var/spack/packages/py-scipy/package.py new file mode 100644 index 00000000000..b5325b919f3 --- /dev/null +++ b/var/spack/packages/py-scipy/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyScipy(Package): + """Scientific Library for Python.""" + homepage = "https://pypi.python.org/pypi/scipy" + url = "https://pypi.python.org/packages/source/s/scipy/scipy-0.15.0.tar.gz" + + version('0.15.0', '639112f077f0aeb6d80718dc5019dc7a') + + extends('python') + depends_on('py-nose') + depends_on('py-numpy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-six/package.py b/var/spack/packages/py-six/package.py new file mode 100644 index 00000000000..04d29adcedf --- /dev/null +++ b/var/spack/packages/py-six/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PySix(Package): + """Python 2 and 3 compatibility utilities.""" + homepage = "https://pypi.python.org/pypi/six" + url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz" + + version('1.9.0', '476881ef4012262dfc8adc645ee786c4') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-virtualenv/package.py b/var/spack/packages/py-virtualenv/package.py new file mode 100644 index 00000000000..c1b359e1649 --- /dev/null +++ b/var/spack/packages/py-virtualenv/package.py @@ -0,0 +1,18 @@ +from spack import * +import shutil + +class PyVirtualenv(Package): + """virtualenv is a tool to create isolated Python environments.""" + homepage = "http://virtualenv.readthedocs.org/projects/virtualenv/" + url = "https://pypi.python.org/packages/source/v/virtualenv/virtualenv-1.11.6.tar.gz" + + version('1.11.6', 'f61cdd983d2c4e6aeabb70b1060d6f49') + + extends('python') + + def clean(self): + if os.path.exists('build'): + shutil.rmtree('build') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From ff9cb94f4f92112739f53881bcb0a9a19811684d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 20 Jan 2015 00:23:16 -0800 Subject: [PATCH 25/97] Add arguements to extends() and activate/deactivate. --- lib/spack/spack/package.py | 40 ++++++++++++++++++++-------- lib/spack/spack/relations.py | 17 +++++++----- var/spack/packages/python/package.py | 32 ++++++++++++++++++++-- 3 files changed, 69 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index ae34f8ae455..bd63c2e0c0b 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -497,16 +497,26 @@ def fetcher(self, f): @property def extendee_spec(self): """Spec of the extendee of this package, or None if it is not an extension.""" - if not self.extendees: return None - + if not self.extendees: + return None name = next(iter(self.extendees)) if not name in self.spec: - return self.extendees[name] + spec, kwargs = self.extendees[name] + return spec # Need to do this to get the concrete version of the spec return self.spec[name] + @property + def extendee_args(self): + """Spec of the extendee of this package, or None if it is not an extension.""" + if not self.extendees: + return None + name = next(iter(self.extendees)) + return self.extendees[name][1] + + @property def is_extension(self): return len(self.extendees) > 0 @@ -949,6 +959,8 @@ def _check_extendable(self): def _sanity_check_extension(self): + if not self.is_extension: + raise ValueError("This package is not an extension.") extendee_package = self.extendee_spec.package extendee_package._check_extendable() @@ -967,14 +979,14 @@ def do_activate(self): activate() directly. """ self._sanity_check_extension() - self.extendee_spec.package.activate(self) + self.extendee_spec.package.activate(self, **self.extendee_args) spack.install_layout.add_extension(self.extendee_spec, self.spec) tty.msg("Activated extension %s for %s." % (self.spec.short_spec, self.extendee_spec.short_spec)) - def activate(self, extension): + def activate(self, extension, **kwargs): """Symlinks all files from the extension into extendee's install dir. Package authors can override this method to support other @@ -983,17 +995,20 @@ def activate(self, extension): always executed. """ + ignore_files = set(spack.install_layout.hidden_file_paths) + ignore_files.update(kwargs.get('ignore', ())) + tree = LinkTree(extension.prefix) - conflict = tree.find_conflict( - self.prefix, ignore=spack.install_layout.hidden_file_paths) + conflict = tree.find_conflict(self.prefix, ignore=ignore_files) if conflict: raise ExtensionConflictError(conflict) - tree.merge(self.prefix, ignore=spack.install_layout.hidden_file_paths) + tree.merge(self.prefix, ignore=ignore_files) def do_deactivate(self): + """Called on the extension to invoke extendee's deactivate() method.""" self._sanity_check_extension() - self.extendee_spec.package.deactivate(self) + self.extendee_spec.package.deactivate(self, **self.extendee_args) if self.spec in spack.install_layout.get_extensions(self.extendee_spec): spack.install_layout.remove_extension(self.extendee_spec, self.spec) @@ -1002,7 +1017,7 @@ def do_deactivate(self): % (self.spec.short_spec, self.extendee_spec.short_spec)) - def deactivate(self, extension): + def deactivate(self, extension, **kwargs): """Unlinks all files from extension out of this package's install dir. Package authors can override this method to support other @@ -1011,8 +1026,11 @@ def deactivate(self, extension): always executed. """ + ignore_files = set(spack.install_layout.hidden_file_paths) + ignore_files.update(kwargs.get('ignore', ())) + tree = LinkTree(extension.prefix) - tree.unmerge(self.prefix, ignore=spack.install_layout.hidden_file_paths) + tree.unmerge(self.prefix, ignore=ignore_files) def do_clean(self): diff --git a/lib/spack/spack/relations.py b/lib/spack/spack/relations.py index 60ff5bef34e..a0c7723473c 100644 --- a/lib/spack/spack/relations.py +++ b/lib/spack/spack/relations.py @@ -117,7 +117,7 @@ def depends_on(*specs): dependencies[spec.name] = spec -def extends(*specs): +def extends(spec, **kwargs): """Same as depends_on, but dependency is symlinked into parent prefix. This is for Python and other language modules where the module @@ -126,6 +126,10 @@ def extends(*specs): but allowing ONE module version to be symlinked into a parent Python install at a time. + keyword arguments can be passed to extends() so that extension + packages can pass parameters to the extendee's extension + mechanism. + """ pkg = get_calling_package_name() clocals = caller_locals() @@ -134,12 +138,11 @@ def extends(*specs): if extendees: raise RelationError("Packages can extend at most one other package.") - for string in specs: - for spec in spack.spec.parse(string): - if pkg == spec.name: - raise CircularReferenceError('extends', pkg) - dependencies[spec.name] = spec - extendees[spec.name] = spec + spec = Spec(spec) + if pkg == spec.name: + raise CircularReferenceError('extends', pkg) + dependencies[spec.name] = spec + extendees[spec.name] = (spec, kwargs) def provides(*specs, **kwargs): diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 9700179ab8e..86b903bc23b 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -28,6 +28,16 @@ def install(self, spec, prefix): make("install") + @property + def python_lib_dir(self): + return os.path.join('lib', 'python%d.%d' % self.version[:2]) + + + @property + def site_packages_dir(self): + return os.path.join(self.python_lib_dir, 'site-packages') + + def setup_extension_environment(self, module, spec, ext_spec): """Called before python modules' install() methods. @@ -39,11 +49,29 @@ def setup_extension_environment(self, module, spec, ext_spec): module.python = Executable(join_path(spec.prefix.bin, 'python')) # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. - module.python_lib_dir = join_path(ext_spec.prefix.lib, 'python%d.%d' % self.version[:2]) - module.site_packages_dir = join_path(module.python_lib_dir, 'site-packages') + module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) + module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) # Add site packages directory to the PYTHONPATH os.environ['PYTHONPATH'] = module.site_packages_dir # Make the site packages directory if it does not exist already. mkdirp(module.site_packages_dir) + + + def add_ignore_files(self, args): + """Add some ignore files to activate/deactivate args.""" + ignore = set(args.get('ignore', ())) + ignore.add(os.path.join(self.site_packages_dir, 'site.py')) + ignore.add(os.path.join(self.site_packages_dir, 'site.pyc')) + args.update(ignore=ignore) + + + def activate(self, ext_pkg, **args): + self.add_ignore_files(args) + super(Python, self).activate(ext_pkg, **args) + + + def deactivate(self, ext_pkg, **args): + self.add_ignore_files(args) + super(Python, self).deactivate(ext_pkg, **args) From de91c95e8e45b8ab066ba3dfc8f89c92da761b5a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 20 Jan 2015 15:07:53 -0800 Subject: [PATCH 26/97] Ability to ignore files in activate/deactivate for extensions. --- lib/spack/llnl/util/link_tree.py | 11 ++++------- lib/spack/spack/package.py | 16 +++++++++------- var/spack/packages/py-nose/package.py | 4 +++- var/spack/packages/python/package.py | 17 ++++++++++------- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 19c2d46938f..2d7126be2c4 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -72,8 +72,7 @@ def traverse(self, dest_root, **kwargs): order=[pre|post] -- Whether to do pre- or post-order traveral. - ignore= -- Optional container of root-relative - paths to ignore. + ignore= -- Predicate indicating which files to ignore. follow_nonexisting -- Whether to descend into directories in src that do not exit in dest. @@ -85,9 +84,7 @@ def traverse(self, dest_root, **kwargs): raise ValueError("Order must be 'pre' or 'post'.") # List of relative paths to ignore under the src root. - ignore = kwargs.get('ignore', None) - if isinstance(ignore, basestring): - ignore = (ignore,) + ignore = kwargs.get('ignore', lambda filename: False) # Whether to descend when dirs dont' exist in dest. follow_nonexisting = kwargs.get('follow_nonexisting', True) @@ -98,7 +95,7 @@ def traverse(self, dest_root, **kwargs): dest_dirpath = os.path.join(dest_root, rel_path) # Don't descend into ignored directories - if ignore and dest_dirpath in ignore: + if ignore(dest_dirpath): return # Don't descend into dirs in dest that do not exist in src. @@ -118,7 +115,7 @@ def traverse(self, dest_root, **kwargs): # Ignore particular paths inside the install root. src_relpath = src_file[len(self._root):] src_relpath = src_relpath.lstrip(os.path.sep) - if ignore and src_relpath in ignore: + if ignore(src_relpath): continue yield (src_file, dest_file) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index bd63c2e0c0b..43b1fcd9c8a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -995,14 +995,15 @@ def activate(self, extension, **kwargs): always executed. """ - ignore_files = set(spack.install_layout.hidden_file_paths) - ignore_files.update(kwargs.get('ignore', ())) + def ignore(filename): + return (filename in spack.install_layout.hidden_file_paths or + kwargs.get('ignore', lambda f: False)(filename)) tree = LinkTree(extension.prefix) - conflict = tree.find_conflict(self.prefix, ignore=ignore_files) + conflict = tree.find_conflict(self.prefix, ignore=ignore) if conflict: raise ExtensionConflictError(conflict) - tree.merge(self.prefix, ignore=ignore_files) + tree.merge(self.prefix, ignore=ignore) def do_deactivate(self): @@ -1026,11 +1027,12 @@ def deactivate(self, extension, **kwargs): always executed. """ - ignore_files = set(spack.install_layout.hidden_file_paths) - ignore_files.update(kwargs.get('ignore', ())) + def ignore(filename): + return (filename in spack.install_layout.hidden_file_paths or + kwargs.get('ignore', lambda f: False)(filename)) tree = LinkTree(extension.prefix) - tree.unmerge(self.prefix, ignore=ignore_files) + tree.unmerge(self.prefix, ignore=ignore) def do_clean(self): diff --git a/var/spack/packages/py-nose/package.py b/var/spack/packages/py-nose/package.py index 7bd7106b8c3..6df84e831d1 100644 --- a/var/spack/packages/py-nose/package.py +++ b/var/spack/packages/py-nose/package.py @@ -1,7 +1,9 @@ from spack import * class PyNose(Package): - """nose extends the test loading and running features of unittest, making it easier to write, find and run tests.""" + """nose extends the test loading and running features of unittest, + making it easier to write, find and run tests.""" + homepage = "https://pypi.python.org/pypi/nose" url = "https://pypi.python.org/packages/source/n/nose/nose-1.3.4.tar.gz" diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 86b903bc23b..a22bd54c82b 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,5 +1,6 @@ from spack import * import os +import re class Python(Package): """The Python programming language.""" @@ -59,19 +60,21 @@ def setup_extension_environment(self, module, spec, ext_spec): mkdirp(module.site_packages_dir) - def add_ignore_files(self, args): + def make_ignore(self, args): """Add some ignore files to activate/deactivate args.""" - ignore = set(args.get('ignore', ())) - ignore.add(os.path.join(self.site_packages_dir, 'site.py')) - ignore.add(os.path.join(self.site_packages_dir, 'site.pyc')) - args.update(ignore=ignore) + orig_ignore = args.get('ignore', lambda f: False) + def ignore(filename): + return (re.search(r'/site\.pyc?$', filename) or + re.search(r'\.pth$', filename) or + orig_ignore(filename)) + return ignore def activate(self, ext_pkg, **args): - self.add_ignore_files(args) + args.update(ignore=self.make_ignore(args)) super(Python, self).activate(ext_pkg, **args) def deactivate(self, ext_pkg, **args): - self.add_ignore_files(args) + args.update(ignore=self.make_ignore(args)) super(Python, self).deactivate(ext_pkg, **args) From 2bc3f74df263eb92a99e5477f4ce04972eb76994 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 22 Jan 2015 11:50:01 -0800 Subject: [PATCH 27/97] added more Python modules --- var/spack/packages/hdf5/package.py | 6 +++-- var/spack/packages/py-basemap/package.py | 24 +++++++++++++++++++ var/spack/packages/py-biopython/package.py | 14 +++++++++++ var/spack/packages/py-gnuplot/package.py | 13 ++++++++++ var/spack/packages/py-h5py/package.py | 18 ++++++++++++++ var/spack/packages/py-matplotlib/package.py | 17 +++++++++++++ var/spack/packages/py-mpi4py/package.py | 13 ++++++++++ var/spack/packages/py-mx/package.py | 13 ++++++++++ var/spack/packages/py-pil/package.py | 14 +++++++++++ var/spack/packages/py-pmw/package.py | 13 ++++++++++ var/spack/packages/py-pylint/package.py | 16 +++++++++++++ var/spack/packages/py-rpy2/package.py | 14 +++++++++++ .../packages/py-scientificpython/package.py | 13 ++++++++++ var/spack/packages/py-scikit-learn/package.py | 13 ++++++++++ var/spack/packages/py-sympy/package.py | 13 ++++++++++ 15 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 var/spack/packages/py-basemap/package.py create mode 100644 var/spack/packages/py-biopython/package.py create mode 100644 var/spack/packages/py-gnuplot/package.py create mode 100644 var/spack/packages/py-h5py/package.py create mode 100644 var/spack/packages/py-mpi4py/package.py create mode 100644 var/spack/packages/py-mx/package.py create mode 100644 var/spack/packages/py-pil/package.py create mode 100644 var/spack/packages/py-pmw/package.py create mode 100644 var/spack/packages/py-pylint/package.py create mode 100644 var/spack/packages/py-rpy2/package.py create mode 100644 var/spack/packages/py-scientificpython/package.py create mode 100644 var/spack/packages/py-scikit-learn/package.py create mode 100644 var/spack/packages/py-sympy/package.py diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py index 615c2a7fe4b..992dd8ec700 100644 --- a/var/spack/packages/hdf5/package.py +++ b/var/spack/packages/hdf5/package.py @@ -18,12 +18,14 @@ class Hdf5(Package): # TODO: currently hard-coded to use OpenMPI def install(self, spec, prefix): + configure( "--prefix=%s" % prefix, "--with-zlib=%s" % spec['zlib'].prefix, "--enable-parallel", - "CC=%s" % spec['openmpi'].prefix.bin + "/mpicc", - "CXX=%s" % spec['openmpi'].prefix.bin + "/mpic++") + "--enable-shared", + "CC=%s" % spec['mpich'].prefix.bin + "/mpicc", + "CXX=%s" % spec['mpich'].prefix.bin + "/mpic++") make() make("install") diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py new file mode 100644 index 00000000000..8955bf8827f --- /dev/null +++ b/var/spack/packages/py-basemap/package.py @@ -0,0 +1,24 @@ +from spack import * +import os + +class PyBasemap(Package): + """The matplotlib basemap toolkit is a library for plotting 2D data on maps in Python.""" + homepage = "http://matplotlib.org/basemap/" + url = "https://downloads.sourceforge.net/project/matplotlib/matplotlib-toolkits/basemap-1.0.7/basemap-1.0.7.tar.gz" + + version('1.0.7', '48c0557ced9e2c6e440b28b3caff2de8') + + geos_version = {'1.0.7' : '3.3.3'} + + extends('python') + depends_on('py-numpy') + depends_on('py-matplotlib') + depends_on('py-pil') + + def install(self, spec, prefix): + with working_dir('geos-%s' % self.geos_version[str(self.version)]): + configure("--prefix=" + prefix) + make() + make("install") + os.environ['GEOS_DIR'] = prefix + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-biopython/package.py b/var/spack/packages/py-biopython/package.py new file mode 100644 index 00000000000..2ed04c389e7 --- /dev/null +++ b/var/spack/packages/py-biopython/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyBiopython(Package): + """It is a distributed collaborative effort to develop Python libraries and applications which address the needs of current and future work in bioinformatics.""" + homepage = "http://biopython.org/wiki/Main_Page" + url = "http://biopython.org/DIST/biopython-1.65.tar.gz" + + version('1.65', '143e7861ade85c0a8b5e2bbdd1da1f67') + + extends('python') + depends_on('py-mx') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-gnuplot/package.py b/var/spack/packages/py-gnuplot/package.py new file mode 100644 index 00000000000..0a2c073a491 --- /dev/null +++ b/var/spack/packages/py-gnuplot/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyGnuplot(Package): + """Gnuplot.py is a Python package that allows you to create graphs from within Python using the gnuplot plotting program.""" + homepage = "http://gnuplot-py.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/gnuplot-py/Gnuplot-py/1.8/gnuplot-py-1.8.tar.gz" + + version('1.8', 'abd6f571e7aec68ae7db90a5217cd5b1') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/packages/py-h5py/package.py new file mode 100644 index 00000000000..f72b3ac06ea --- /dev/null +++ b/var/spack/packages/py-h5py/package.py @@ -0,0 +1,18 @@ +from spack import * +import re + +class PyH5py(Package): + """The h5py package provides both a high- and low-level interface to the HDF5 library from Python.""" + homepage = "https://pypi.python.org/pypi/h5py" + url = "https://pypi.python.org/packages/source/h/h5py/h5py-2.4.0.tar.gz" + + version('2.4.0', '80c9a94ae31f84885cc2ebe1323d6758') + + extends('python', ignore=lambda f: re.match(r'cy*', f)) + depends_on('hdf5') + depends_on('py-numpy') + depends_on('py-cython') + + def install(self, spec, prefix): + python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index 836273c9237..270a1ebfe4c 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -1,4 +1,5 @@ from spack import * +import os class PyMatplotlib(Package): """Python plotting package.""" @@ -18,3 +19,19 @@ class PyMatplotlib(Package): def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) + if str(self.version) == '1.4.2': + # hack to fix configuration file + config_file = None + for p,d,f in os.walk(prefix.lib): + for file in f: + if file.find('matplotlibrc') != -1: + config_file = join_path(p, 'matplotlibrc') + print config_file + if config_file == None: + raise InstallError('could not find config file') + filter_file(r'backend : pyside', + 'backend : Qt4Agg', + config_file) + filter_file(r'#backend.qt4 : PyQt4', + 'backend.qt4 : PySide', + config_file) diff --git a/var/spack/packages/py-mpi4py/package.py b/var/spack/packages/py-mpi4py/package.py new file mode 100644 index 00000000000..fdea340dc21 --- /dev/null +++ b/var/spack/packages/py-mpi4py/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyMpi4py(Package): + """This package provides Python bindings for the Message Passing Interface (MPI) standard. It is implemented on top of the MPI-1/MPI-2 specification and exposes an API which grounds on the standard MPI-2 C++ bindings.""" + homepage = "https://pypi.python.org/pypi/mpi4py" + url = "https://pypi.python.org/packages/source/m/mpi4py/mpi4py-1.3.1.tar.gz" + + version('1.3.1', 'dbe9d22bdc8ed965c23a7ceb6f32fc3c') + extends('python') + depends_on('mpi') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-mx/package.py b/var/spack/packages/py-mx/package.py new file mode 100644 index 00000000000..717ee0562b6 --- /dev/null +++ b/var/spack/packages/py-mx/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyMx(Package): + """The eGenix.com mx Base Distribution for Python is a collection of professional quality software tools which enhance Python's usability in many important areas such as fast text searching, date/time processing and high speed data types.""" + homepage = "http://www.egenix.com/products/python/mxBase/" + url = "https://downloads.egenix.com/python/egenix-mx-base-3.2.8.tar.gz" + + version('3.2.8', '9d9d3a25f9dc051a15e97f452413423b') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pil/package.py b/var/spack/packages/py-pil/package.py new file mode 100644 index 00000000000..743b7619817 --- /dev/null +++ b/var/spack/packages/py-pil/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyPil(Package): + """The Python Imaging Library (PIL) adds image processing capabilities to your Python interpreter. This library supports many file formats, and provides powerful image processing and graphics capabilities.""" + + homepage = "http://www.pythonware.com/products/pil/" + url = "http://effbot.org/media/downloads/Imaging-1.1.7.tar.gz" + + version('1.1.7', 'fc14a54e1ce02a0225be8854bfba478e') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pmw/package.py b/var/spack/packages/py-pmw/package.py new file mode 100644 index 00000000000..56131811e9b --- /dev/null +++ b/var/spack/packages/py-pmw/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPmw(Package): + """Pmw is a toolkit for building high-level compound widgets, or megawidgets, constructed using other widgets as component parts.""" + homepage = "https://pypi.python.org/pypi/Pmw" + url = "https://pypi.python.org/packages/source/P/Pmw/Pmw-2.0.0.tar.gz" + + version('2.0.0', 'c7c3f26c4f5abaa99807edefee578fc0') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pylint/package.py b/var/spack/packages/py-pylint/package.py new file mode 100644 index 00000000000..ebde861f947 --- /dev/null +++ b/var/spack/packages/py-pylint/package.py @@ -0,0 +1,16 @@ +from spack import * +import re + +class PyPylint(Package): + """array processing for numbers, strings, records, and objects.""" + homepage = "https://pypi.python.org/pypi/pylint" + url = "https://pypi.python.org/packages/source/p/pylint/pylint-1.4.1.tar.gz" + + version('1.4.1', 'df7c679bdcce5019389038847e4de622') + +# extends('python') + extends('python', ignore=lambda f:re.match(r"site.py*", f)) + depends_on('py-nose') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-rpy2/package.py b/var/spack/packages/py-rpy2/package.py new file mode 100644 index 00000000000..3817059911d --- /dev/null +++ b/var/spack/packages/py-rpy2/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyRpy2(Package): + """rpy2 is a redesign and rewrite of rpy. It is providing a low-level interface to R from Python, a proposed high-level interface, including wrappers to graphical libraries, as well as R-like structures and functions.""" + homepage = "https://pypi.python.org/pypi/rpy2" + url = "https://pypi.python.org/packages/source/r/rpy2/rpy2-2.5.4.tar.gz" + + version('2.5.4', '115a20ac30883f096da2bdfcab55196d') + + extends('python') + depends_on('py-setuptools') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scientificpython/package.py b/var/spack/packages/py-scientificpython/package.py new file mode 100644 index 00000000000..73600e6cb97 --- /dev/null +++ b/var/spack/packages/py-scientificpython/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyScientificpython(Package): + """ScientificPython is a collection of Python modules for scientific computing. It contains support for geometry, mathematical functions, statistics, physical units, IO, visualization, and parallelization.""" + homepage = "https://sourcesup.renater.fr/projects/scientific-py/" + url = "https://sourcesup.renater.fr/frs/download.php/4411/ScientificPython-2.8.1.tar.gz" + + version('2.8.1', '73ee0df19c7b58cdf2954261f0763c77') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scikit-learn/package.py b/var/spack/packages/py-scikit-learn/package.py new file mode 100644 index 00000000000..c59c05a6190 --- /dev/null +++ b/var/spack/packages/py-scikit-learn/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyScikitLearn(Package): + """""" + homepage = "https://pypi.python.org/pypi/scikit-learn" + url = "https://pypi.python.org/packages/source/s/scikit-learn/scikit-learn-0.15.2.tar.gz" + + version('0.15.2', 'd9822ad0238e17b382a3c756ea94fe0d') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-sympy/package.py b/var/spack/packages/py-sympy/package.py new file mode 100644 index 00000000000..c17e35b95f1 --- /dev/null +++ b/var/spack/packages/py-sympy/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PySympy(Package): + """SymPy is a Python library for symbolic mathematics.""" + homepage = "https://pypi.python.org/pypi/sympy" + url = "https://pypi.python.org/packages/source/s/sympy/sympy-0.7.6.tar.gz" + + version('0.7.6', '3d04753974306d8a13830008e17babca') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 48f1ff87f836f214e72e1d02f47eac4678f0292a Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Fri, 23 Jan 2015 13:53:36 -0800 Subject: [PATCH 28/97] added more Python modules --- var/spack/packages/py-dateutil/package.py | 1 + var/spack/packages/py-libxml2/package.py | 13 +++++++++++++ var/spack/packages/py-matplotlib/package.py | 1 + var/spack/packages/py-pychecker/package.py | 13 +++++++++++++ var/spack/packages/py-pyqt4/package.py | 18 ++++++++++++++++++ var/spack/packages/py-sip/package.py | 15 +++++++++++++++ var/spack/packages/qt/package.py | 5 +++-- 7 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 var/spack/packages/py-libxml2/package.py create mode 100644 var/spack/packages/py-pychecker/package.py create mode 100644 var/spack/packages/py-pyqt4/package.py create mode 100644 var/spack/packages/py-sip/package.py diff --git a/var/spack/packages/py-dateutil/package.py b/var/spack/packages/py-dateutil/package.py index 96e3ecab075..11699e07ee5 100644 --- a/var/spack/packages/py-dateutil/package.py +++ b/var/spack/packages/py-dateutil/package.py @@ -8,6 +8,7 @@ class PyDateutil(Package): version('2.4.0', '75714163bb96bedd07685cdb2071b8bc') extends('python') + depends_on('py-setuptools') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-libxml2/package.py b/var/spack/packages/py-libxml2/package.py new file mode 100644 index 00000000000..0dcefbd9cf8 --- /dev/null +++ b/var/spack/packages/py-libxml2/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyLibxml2(Package): + """A Python wrapper around libxml2.""" + homepage = "https://xmlsoft.org/python.html" + url = "ftp://xmlsoft.org/libxml2/python/libxml2-python-2.6.21.tar.gz" + + version('2.6.21', '229dd2b3d110a77defeeaa73af83f7f3') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index 270a1ebfe4c..5979ceeab0d 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -16,6 +16,7 @@ class PyMatplotlib(Package): depends_on('py-dateutil') depends_on('py-pytz') depends_on('py-nose') + depends_on('py-numpy') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pychecker/package.py b/var/spack/packages/py-pychecker/package.py new file mode 100644 index 00000000000..bda5a746aaa --- /dev/null +++ b/var/spack/packages/py-pychecker/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyPychecker(Package): + """""" + homepage = "http://pychecker.sourceforge.net/" + url = "http://sourceforge.net/projects/pychecker/files/pychecker/0.8.19/pychecker-0.8.19.tar.gz" + + version('0.8.19', 'c37182863dfb09209d6ba4f38fce9d2b') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pyqt4/package.py b/var/spack/packages/py-pyqt4/package.py new file mode 100644 index 00000000000..eeb13825605 --- /dev/null +++ b/var/spack/packages/py-pyqt4/package.py @@ -0,0 +1,18 @@ +from spack import * + +class PyPyqt4(Package): + """PyQt is a set of Python v2 and v3 bindings for Digia's Qt application framework and runs on all platforms supported by Qt including Windows, MacOS/X and Linux.""" + homepage = "http://www.riverbankcomputing.com/software/pyqt/intro" + url = "http://sourceforge.net/projects/pyqt/files/PyQt4/PyQt-4.11.3/PyQt-x11-gpl-4.11.3.tar.gz" + + version('4.11.3', '997c3e443165a89a559e0d96b061bf70') + + extends('python') + depends_on('qt') + depends_on('py-sip') + + def install(self, spec, prefix): + version_array = str(spec['python'].version).split('.') + python('configure.py', '--confirm-license', '--destdir=%s/python%s.%s/site-packages' %(self.prefix.lib, version_array[0], version_array[1])) + make() + make('install') diff --git a/var/spack/packages/py-sip/package.py b/var/spack/packages/py-sip/package.py new file mode 100644 index 00000000000..06aea35a745 --- /dev/null +++ b/var/spack/packages/py-sip/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PySip(Package): + """SIP is a tool that makes it very easy to create Python bindings for C and C++ libraries.""" + homepage = "http://www.riverbankcomputing.com/software/sip/intro" + url = "http://sourceforge.net/projects/pyqt/files/sip/sip-4.16.5/sip-4.16.5.tar.gz" + + version('4.16.5', '6d01ea966a53e4c7ae5c5e48c40e49e5') + + extends('python') + + def install(self, spec, prefix): + python('configure.py') + make() + make('install') diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py index 01f9de7f3c0..6a55c897015 100644 --- a/var/spack/packages/qt/package.py +++ b/var/spack/packages/qt/package.py @@ -37,8 +37,9 @@ def install(self, spec, prefix): '-fast', '-optimized-qmake', '-no-pch', - '-no-phonon', - '-no-phonon-backend', +# phonon required for py-pyqt4 +# '-no-phonon', +# '-no-phonon-backend', '-no-openvg') make() make("install") From 70c8bf44b8d2613c54423562f81e049fbb956780 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 26 Jan 2015 14:47:33 -0800 Subject: [PATCH 29/97] Fix for install sanity check -- don't count hidden dir layout files. --- lib/spack/spack/package.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 43b1fcd9c8a..0b6bc4ce6cd 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -836,10 +836,7 @@ def do_install(self, **kwargs): self.install(self.spec, self.prefix) # Ensure that something was actually installed. - if not os.listdir(self.prefix): - raise InstallError( - "Install failed for %s. Nothing was installed!" - % self.name) + self._sanity_check_install() # On successful install, remove the stage. if not keep_stage: @@ -884,6 +881,15 @@ def do_install(self, **kwargs): spack.hooks.post_install(self) + + def _sanity_check_install(self): + installed = set(os.listdir(self.prefix)) + installed.difference_update(spack.install_layout.hidden_file_paths) + if not installed: + raise InstallError( + "Install failed for %s. Nothing was installed!" % self.name) + + def do_install_dependencies(self, **kwargs): # Pass along paths of dependencies here for dep in self.spec.dependencies.values(): From 6400ace90152a08a32684f97490369467ae1e37d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 26 Jan 2015 15:45:19 -0800 Subject: [PATCH 30/97] Add "spack extensions" command to list activated extensions. --- lib/spack/spack/cmd/extensions.py | 75 +++++++++++++++++++++++++++ lib/spack/spack/cmd/find.py | 85 ++++++++++++++++++------------- 2 files changed, 125 insertions(+), 35 deletions(-) create mode 100644 lib/spack/spack/cmd/extensions.py diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py new file mode 100644 index 00000000000..961d7e3f248 --- /dev/null +++ b/lib/spack/spack/cmd/extensions.py @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import sys +from external import argparse + +import llnl.util.tty as tty + +import spack +import spack.cmd +import spack.cmd.find + +description = "List extensions for package." + +def setup_parser(subparser): + format_group = subparser.add_mutually_exclusive_group() + format_group.add_argument( + '-l', '--long', action='store_const', dest='mode', const='long', + help='Show dependency hashes as well as versions.') + format_group.add_argument( + '-p', '--paths', action='store_const', dest='mode', const='paths', + help='Show paths to extension install directories') + format_group.add_argument( + '-d', '--deps', action='store_const', dest='mode', const='deps', + help='Show full dependency DAG of extensions') + + subparser.add_argument( + 'spec', nargs=argparse.REMAINDER, help='Spec of package to list extensions for') + + +def extensions(parser, args): + if not args.spec: + tty.die("extensions requires a package spec.") + + spec = spack.cmd.parse_specs(args.spec) + if len(spec) > 1: + tty.die("Can only list extensions for one package.") + spec = spack.cmd.disambiguate_spec(spec[0]) + + if not spec.package.extendable: + tty.die("%s does not have extensions." % spec.short_spec) + + if not args.mode: + args.mode = 'short' + + exts = spack.install_layout.get_extensions(spec) + if not exts: + tty.msg("%s has no activated extensions." % spec.short_spec) + else: + tty.msg("Showing %d activated extension%s for package:" + % (len(exts), 's' if len(exts) > 1 else ''), + spec.short_spec) + print + spack.cmd.find.display_specs(exts, mode=args.mode) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 1de3413d42d..f6f503afe5d 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -41,13 +41,13 @@ def setup_parser(subparser): format_group = subparser.add_mutually_exclusive_group() format_group.add_argument( - '-l', '--long', action='store_true', dest='long', + '-l', '--long', action='store_const', dest='mode', const='long', help='Show dependency hashes as well as versions.') format_group.add_argument( - '-p', '--paths', action='store_true', dest='paths', + '-p', '--paths', action='store_const', dest='mode', const='paths', help='Show paths to package install directories') format_group.add_argument( - '-d', '--deps', action='store_true', dest='full_deps', + '-d', '--deps', action='store_const', dest='mode', const='deps', help='Show full dependency DAG of installed packages') subparser.add_argument( @@ -55,6 +55,50 @@ def setup_parser(subparser): help='optional specs to filter results') +def display_specs(specs, **kwargs): + mode = kwargs.get('mode', 'short') + + # Make a dict with specs keyed by architecture and compiler. + index = index_by(specs, ('architecture', 'compiler')) + + # Traverse the index and print out each package + for i, (architecture, compiler) in enumerate(sorted(index)): + if i > 0: print + + header = "%s{%s} / %s{%s}" % ( + spack.spec.architecture_color, architecture, + spack.spec.compiler_color, compiler) + tty.hline(colorize(header), char='-') + + specs = index[(architecture,compiler)] + specs.sort() + + abbreviated = [s.format('$_$@$+', color=True) for s in specs] + if mode == 'paths': + # Print one spec per line along with prefix path + width = max(len(s) for s in abbreviated) + width += 2 + format = " %-{}s%s".format(width) + + for abbrv, spec in zip(abbreviated, specs): + print format % (abbrv, spec.prefix) + + elif mode == 'deps': + for spec in specs: + print spec.tree(indent=4, format='$_$@$+', color=True), + + elif mode in ('short', 'long'): + fmt = '$-_$@$+' + if mode == 'long': + fmt += '$#' + colify(s.format(fmt, color=True) for s in specs) + + else: + raise ValueError( + "Invalid mode for display_specs: %s. Must be one of (paths, deps, short)." % mode) + + + def find(parser, args): # Filter out specs that don't exist. query_specs = spack.cmd.parse_specs(args.query_specs) @@ -76,36 +120,7 @@ def find(parser, args): results = [set(spack.db.get_installed(qs)) for qs in query_specs] specs = set.union(*results) - # Make a dict with specs keyed by architecture and compiler. - index = index_by(specs, ('architecture', 'compiler')) + if not args.mode: + args.mode = 'short' + display_specs(specs, mode=args.mode) - # Traverse the index and print out each package - for i, (architecture, compiler) in enumerate(sorted(index)): - if i > 0: print - - header = "%s{%s} / %s{%s}" % ( - spack.spec.architecture_color, architecture, - spack.spec.compiler_color, compiler) - tty.hline(colorize(header), char='-') - - specs = index[(architecture,compiler)] - specs.sort() - - abbreviated = [s.format('$_$@$+', color=True) for s in specs] - if args.paths: - # Print one spec per line along with prefix path - width = max(len(s) for s in abbreviated) - width += 2 - format = " %-{}s%s".format(width) - - for abbrv, spec in zip(abbreviated, specs): - print format % (abbrv, spec.prefix) - - elif args.full_deps: - for spec in specs: - print spec.tree(indent=4, format='$_$@$+', color=True), - else: - fmt = '$-_$@$+' - if args.long: - fmt += '$#' - colify(s.format(fmt, color=True) for s in specs) From 6b90017efa1f3157fe4be7d0c7b199b6e51b9fa8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 28 Jan 2015 22:05:57 -0800 Subject: [PATCH 31/97] Fixed dumb link_tree bug, added test for link tree. --- lib/spack/llnl/util/filesystem.py | 8 +- lib/spack/llnl/util/link_tree.py | 205 +++++++++++++++++------------- lib/spack/spack/test/__init__.py | 3 +- lib/spack/spack/test/link_tree.py | 153 ++++++++++++++++++++++ 4 files changed, 278 insertions(+), 91 deletions(-) create mode 100644 lib/spack/spack/test/link_tree.py diff --git a/lib/spack/llnl/util/filesystem.py b/lib/spack/llnl/util/filesystem.py index 05784156530..576aeb16bd3 100644 --- a/lib/spack/llnl/util/filesystem.py +++ b/lib/spack/llnl/util/filesystem.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## __all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir', - 'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor', + 'touch', 'touchp', 'mkdirp', 'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe'] import os @@ -204,6 +204,12 @@ def touch(path): os.utime(path, None) +def touchp(path): + """Like touch, but creates any parent directories needed for the file.""" + mkdirp(os.path.dirname(path)) + touch(path) + + def join_path(prefix, *args): path = str(prefix) for elt in args: diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 2d7126be2c4..887f6f4d26a 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -29,6 +29,89 @@ import shutil from llnl.util.filesystem import mkdirp +empty_file_name = '.spack-empty' + + +def traverse_tree(source_root, dest_root, rel_path='', **kwargs): + """Traverse two filesystem trees simultaneously. + + Walks the LinkTree directory in pre or post order. Yields each + file in the source directory with a matching path from the dest + directory, along with whether the file is a directory. + e.g., for this tree:: + + root/ + a/ + file1 + file2 + b/ + file3 + + When called on dest, this yields:: + + ('root', 'dest') + ('root/a', 'dest/a') + ('root/a/file1', 'dest/a/file1') + ('root/a/file2', 'dest/a/file2') + ('root/b', 'dest/b') + ('root/b/file3', 'dest/b/file3') + + Optional args: + + order=[pre|post] -- Whether to do pre- or post-order traveral. + + ignore= -- Predicate indicating which files to ignore. + + follow_nonexisting -- Whether to descend into directories in + src that do not exit in dest. Default True. + + follow_links -- Whether to descend into symlinks in src. + + """ + follow_nonexisting = kwargs.get('follow_nonexisting', True) + follow_links = kwargs.get('follow_link', False) + + # Yield in pre or post order? + order = kwargs.get('order', 'pre') + if order not in ('pre', 'post'): + raise ValueError("Order must be 'pre' or 'post'.") + + # List of relative paths to ignore under the src root. + ignore = kwargs.get('ignore', lambda filename: False) + + # Don't descend into ignored directories + if ignore(rel_path): + return + + source_path = os.path.join(source_root, rel_path) + dest_path = os.path.join(dest_root, rel_path) + + # preorder yields directories before children + if order == 'pre': + yield (source_path, dest_path) + + for f in os.listdir(source_path): + source_child = os.path.join(source_path, f) + dest_child = os.path.join(dest_path, f) + + # Treat as a directory + if os.path.isdir(source_child) and ( + follow_links or not os.path.islink(source_child)): + + # When follow_nonexisting isn't set, don't descend into dirs + # in source that do not exist in dest + if follow_nonexisting or os.path.exists(dest_child): + tuples = traverse_tree(source_child, dest_child, rel_path, **kwargs) + for t in tuples: yield t + + # Treat as a file. + elif not ignore(os.path.join(rel_path, f)): + yield (source_child, dest_child) + + if order == 'post': + yield (source_path, dest_path) + + class LinkTree(object): """Class to create trees of symbolic links from a source directory. @@ -42,95 +125,20 @@ class LinkTree(object): """ def __init__(self, source_root): + if not os.path.exists(source_root): + raise IOError("No such file or directory: '%s'", source_root) + self._root = source_root - def traverse(self, dest_root, **kwargs): - """Traverse LinkTree root and dest simultaneously. - - Walks the LinkTree directory in pre or post order. Yields - each file in the source directory with a matching path from - the dest directory. e.g., for this tree:: - - root/ - a/ - file1 - file2 - b/ - file3 - - When called on dest, this yields:: - - ('root', 'dest') - ('root/a', 'dest/a') - ('root/a/file1', 'dest/a/file1') - ('root/a/file2', 'dest/a/file2') - ('root/b', 'dest/b') - ('root/b/file3', 'dest/b/file3') - - Optional args: - - order=[pre|post] -- Whether to do pre- or post-order traveral. - - ignore= -- Predicate indicating which files to ignore. - - follow_nonexisting -- Whether to descend into directories in - src that do not exit in dest. - - """ - # Yield directories before or after their contents. - order = kwargs.get('order', 'pre') - if order not in ('pre', 'post'): - raise ValueError("Order must be 'pre' or 'post'.") - - # List of relative paths to ignore under the src root. - ignore = kwargs.get('ignore', lambda filename: False) - - # Whether to descend when dirs dont' exist in dest. - follow_nonexisting = kwargs.get('follow_nonexisting', True) - - for dirpath, dirnames, filenames in os.walk(self._root): - rel_path = dirpath[len(self._root):] - rel_path = rel_path.lstrip(os.path.sep) - dest_dirpath = os.path.join(dest_root, rel_path) - - # Don't descend into ignored directories - if ignore(dest_dirpath): - return - - # Don't descend into dirs in dest that do not exist in src. - if not follow_nonexisting: - dirnames[:] = [ - d for d in dirnames - if os.path.exists(os.path.join(dest_dirpath, d))] - - # preorder yields directories before children - if order == 'pre': - yield (dirpath, dest_dirpath) - - for name in filenames: - src_file = os.path.join(dirpath, name) - dest_file = os.path.join(dest_dirpath, name) - - # Ignore particular paths inside the install root. - src_relpath = src_file[len(self._root):] - src_relpath = src_relpath.lstrip(os.path.sep) - if ignore(src_relpath): - continue - - yield (src_file, dest_file) - - # postorder yields directories after children - if order == 'post': - yield (dirpath, dest_dirpath) - - - def find_conflict(self, dest_root, **kwargs): - """Returns the first file in dest that also exists in src.""" + """Returns the first file in dest that conflicts with src""" kwargs['follow_nonexisting'] = False - for src, dest in self.traverse(dest_root, **kwargs): - if os.path.exists(dest) and not os.path.isdir(dest): + for src, dest in traverse_tree(self._root, dest_root, **kwargs): + if os.path.isdir(src): + if os.path.exists(dest) and not os.path.isdir(dest): + return dest + elif os.path.exists(dest): return dest return None @@ -138,9 +146,20 @@ def find_conflict(self, dest_root, **kwargs): def merge(self, dest_root, **kwargs): """Link all files in src into dest, creating directories if necessary.""" kwargs['order'] = 'pre' - for src, dest in self.traverse(dest_root, **kwargs): + for src, dest in traverse_tree(self._root, dest_root, **kwargs): if os.path.isdir(src): - mkdirp(dest) + if not os.path.exists(dest): + mkdirp(dest) + continue + + if not os.path.isdir(dest): + raise ValueError("File blocks directory: %s" % dest) + + # mark empty directories so they aren't removed on unmerge. + if not os.listdir(dest): + marker = os.path.join(dest, empty_file_name) + touch(marker) + else: assert(not os.path.exists(dest)) os.symlink(src, dest) @@ -153,12 +172,20 @@ def unmerge(self, dest_root, **kwargs): """ kwargs['order'] = 'post' - for src, dest in self.traverse(dest_root, **kwargs): - if os.path.isdir(dest): + for src, dest in traverse_tree(self._root, dest_root, **kwargs): + if os.path.isdir(src): + if not os.path.isdir(dest): + raise ValueError("File blocks directory: %s" % dest) + + # remove directory if it is empty. if not os.listdir(dest): - # TODO: what if empty directories were present pre-merge? shutil.rmtree(dest, ignore_errors=True) + # remove empty dir marker if present. + marker = os.path.join(dest, empty_file_name) + if os.path.exists(marker): + os.remove(marker) + elif os.path.exists(dest): if not os.path.islink(dest): raise ValueError("%s is not a link tree!" % dest) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 0eda667abc0..c53e6774fc8 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -51,7 +51,8 @@ 'hg_fetch', 'mirror', 'url_extrapolate', - 'cc'] + 'cc', + 'link_tree'] def list_tests(): diff --git a/lib/spack/spack/test/link_tree.py b/lib/spack/spack/test/link_tree.py new file mode 100644 index 00000000000..bc7c2c6b5e2 --- /dev/null +++ b/lib/spack/spack/test/link_tree.py @@ -0,0 +1,153 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import unittest +import shutil +import tempfile +from contextlib import closing + +from llnl.util.filesystem import * +from llnl.util.link_tree import LinkTree + +from spack.stage import Stage + + +class LinkTreeTest(unittest.TestCase): + """Tests Spack's LinkTree class.""" + + def setUp(self): + self.stage = Stage('link-tree-test') + + with working_dir(self.stage.path): + touchp('source/1') + touchp('source/a/b/2') + touchp('source/a/b/3') + touchp('source/c/4') + touchp('source/c/d/5') + touchp('source/c/d/6') + touchp('source/c/d/e/7') + + source_path = os.path.join(self.stage.path, 'source') + self.link_tree = LinkTree(source_path) + + + def tearDown(self): + if self.stage: + self.stage.destroy() + + + def check_file_link(self, filename): + self.assertTrue(os.path.isfile(filename)) + self.assertTrue(os.path.islink(filename)) + + + def check_dir(self, filename): + self.assertTrue(os.path.isdir(filename)) + + + def test_merge_to_new_directory(self): + with working_dir(self.stage.path): + self.link_tree.merge('dest') + + self.check_file_link('dest/1') + self.check_file_link('dest/a/b/2') + self.check_file_link('dest/a/b/3') + self.check_file_link('dest/c/4') + self.check_file_link('dest/c/d/5') + self.check_file_link('dest/c/d/6') + self.check_file_link('dest/c/d/e/7') + + self.link_tree.unmerge('dest') + + self.assertFalse(os.path.exists('dest')) + + + def test_merge_to_existing_directory(self): + with working_dir(self.stage.path): + + touchp('dest/x') + touchp('dest/a/b/y') + + self.link_tree.merge('dest') + + self.check_file_link('dest/1') + self.check_file_link('dest/a/b/2') + self.check_file_link('dest/a/b/3') + self.check_file_link('dest/c/4') + self.check_file_link('dest/c/d/5') + self.check_file_link('dest/c/d/6') + self.check_file_link('dest/c/d/e/7') + + self.assertTrue(os.path.isfile('dest/x')) + self.assertTrue(os.path.isfile('dest/a/b/y')) + + self.link_tree.unmerge('dest') + + self.assertTrue(os.path.isfile('dest/x')) + self.assertTrue(os.path.isfile('dest/a/b/y')) + + self.assertFalse(os.path.isfile('dest/1')) + self.assertFalse(os.path.isfile('dest/a/b/2')) + self.assertFalse(os.path.isfile('dest/a/b/3')) + self.assertFalse(os.path.isfile('dest/c/4')) + self.assertFalse(os.path.isfile('dest/c/d/5')) + self.assertFalse(os.path.isfile('dest/c/d/6')) + self.assertFalse(os.path.isfile('dest/c/d/e/7')) + + + def test_merge_with_empty_directories(self): + with working_dir(self.stage.path): + mkdirp('dest/f/g') + mkdirp('dest/a/b/h') + + self.link_tree.merge('dest') + self.link_tree.unmerge('dest') + + self.assertFalse(os.path.exists('dest/1')) + self.assertFalse(os.path.exists('dest/a/b/2')) + self.assertFalse(os.path.exists('dest/a/b/3')) + self.assertFalse(os.path.exists('dest/c/4')) + self.assertFalse(os.path.exists('dest/c/d/5')) + self.assertFalse(os.path.exists('dest/c/d/6')) + self.assertFalse(os.path.exists('dest/c/d/e/7')) + + self.assertTrue(os.path.isdir('dest/a/b/h')) + self.assertTrue(os.path.isdir('dest/f/g')) + + + def test_ignore(self): + with working_dir(self.stage.path): + touchp('source/.spec') + touchp('dest/.spec') + + self.link_tree.merge('dest', ignore=lambda x: x == '.spec') + self.link_tree.unmerge('dest', ignore=lambda x: x == '.spec') + + self.assertFalse(os.path.exists('dest/1')) + self.assertFalse(os.path.exists('dest/a')) + self.assertFalse(os.path.exists('dest/c')) + + self.assertTrue(os.path.isfile('source/.spec')) + self.assertTrue(os.path.isfile('dest/.spec')) From 2d9190d264dd276853aca41998fffbab1baecdb0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 2 Feb 2015 06:09:35 -0800 Subject: [PATCH 32/97] Add extensions command. --- lib/spack/llnl/util/link_tree.py | 5 +- lib/spack/spack/cmd/extensions.py | 9 +-- lib/spack/spack/directory_layout.py | 2 +- lib/spack/spack/package.py | 5 ++ lib/spack/spack/packages.py | 5 ++ lib/spack/spack/spec.py | 7 +++ var/spack/packages/py-basemap/package.py | 1 + var/spack/packages/python/package.py | 77 ++++++++++++++++++++++-- 8 files changed, 99 insertions(+), 12 deletions(-) diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 887f6f4d26a..4e4e48316e8 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -27,7 +27,7 @@ import os import shutil -from llnl.util.filesystem import mkdirp +from llnl.util.filesystem import * empty_file_name = '.spack-empty' @@ -93,6 +93,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): for f in os.listdir(source_path): source_child = os.path.join(source_path, f) dest_child = os.path.join(dest_path, f) + rel_child = os.path.join(rel_path, f) # Treat as a directory if os.path.isdir(source_child) and ( @@ -101,7 +102,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs): # When follow_nonexisting isn't set, don't descend into dirs # in source that do not exist in dest if follow_nonexisting or os.path.exists(dest_child): - tuples = traverse_tree(source_child, dest_child, rel_path, **kwargs) + tuples = traverse_tree(source_root, dest_root, rel_child, **kwargs) for t in tuples: yield t # Treat as a file. diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 961d7e3f248..f28a388bf2a 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -26,6 +26,7 @@ from external import argparse import llnl.util.tty as tty +from llnl.util.tty.colify import colify import spack import spack.cmd @@ -66,10 +67,10 @@ def extensions(parser, args): exts = spack.install_layout.get_extensions(spec) if not exts: - tty.msg("%s has no activated extensions." % spec.short_spec) + tty.msg("%s has no activated extensions." % spec.cshort_spec) else: - tty.msg("Showing %d activated extension%s for package:" - % (len(exts), 's' if len(exts) > 1 else ''), - spec.short_spec) + tty.msg("Extensions for package %s:" % spec.cshort_spec) + colify(pkg.name for pkg in spack.db.extensions_for(spec)) print + tty.msg("%d currently activated:" % len(exts)) spack.cmd.find.display_specs(exts, mode=args.mode) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index ff327ed5047..efc40a17a4f 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -269,8 +269,8 @@ def extension_file_path(self, spec): def get_extensions(self, spec): _check_concrete(spec) - path = self.extension_file_path(spec) extensions = set() + path = self.extension_file_path(spec) if os.path.exists(path): with closing(open(path)) as ext_file: for line in ext_file: diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 0b6bc4ce6cd..b9059685405 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -522,6 +522,11 @@ def is_extension(self): return len(self.extendees) > 0 + def extends(self, spec): + return (spec.name in self.extendees and + spec.satisfies(self.extendees[spec.name][0])) + + @property def activated(self): if not self.spec.concrete: diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index bb5a94bcab9..b3049e812f0 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -112,6 +112,11 @@ def providers_for(self, vpkg_spec): return providers + @_autospec + def extensions_for(self, extendee_spec): + return [p for p in self.all_packages() if p.extends(extendee_spec)] + + def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 2f4fe9ca24d..dffdccaddb9 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -552,6 +552,13 @@ def short_spec(self): return self.format('$_$@$%@$+$=$#') + @property + def cshort_spec(self): + """Returns a version of the spec with the dependencies hashed + instead of completely enumerated.""" + return self.format('$_$@$%@$+$=$#', color=True) + + @property def prefix(self): return Prefix(spack.install_layout.path_for_spec(self)) diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py index 8955bf8827f..7b6d8e7e657 100644 --- a/var/spack/packages/py-basemap/package.py +++ b/var/spack/packages/py-basemap/package.py @@ -11,6 +11,7 @@ class PyBasemap(Package): geos_version = {'1.0.7' : '3.3.3'} extends('python') + depends_on('py-setuptools') depends_on('py-numpy') depends_on('py-matplotlib') depends_on('py-pil') diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index a22bd54c82b..8a6d574d9b2 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,6 +1,9 @@ from spack import * +import spack import os import re +from contextlib import closing + class Python(Package): """The Python programming language.""" @@ -29,6 +32,10 @@ def install(self, spec, prefix): make("install") + # ======================================================================== + # Set up environment to make install easy for python extensions. + # ======================================================================== + @property def python_lib_dir(self): return os.path.join('lib', 'python%d.%d' % self.version[:2]) @@ -60,21 +67,81 @@ def setup_extension_environment(self, module, spec, ext_spec): mkdirp(module.site_packages_dir) - def make_ignore(self, args): + # ======================================================================== + # Handle specifics of activating and deactivating python modules. + # ======================================================================== + + def python_ignore(self, ext_pkg, args): """Add some ignore files to activate/deactivate args.""" orig_ignore = args.get('ignore', lambda f: False) + def ignore(filename): - return (re.search(r'/site\.pyc?$', filename) or - re.search(r'\.pth$', filename) or + # Always ignore easy-install.pth, as it needs to be merged. + patterns = [r'easy-install\.pth$'] + + # Ignore pieces of setuptools installed by other packages. + if ext_pkg.name != 'py-setuptools': + patterns.append(r'/site\.pyc?$') + patterns.append(r'setuptools\.pth') + patterns.append(r'bin/easy_install[^/]*$') + patterns.append(r'setuptools.*egg$') + + return (any(re.search(p, filename) for p in patterns) or orig_ignore(filename)) + return ignore + def write_easy_install_pth(self, extensions): + paths = [] + for ext in extensions: + ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir) + easy_pth = "%s/easy-install.pth" % ext_site_packages + + if not os.path.isfile(easy_pth): + continue + + with closing(open(easy_pth)) as f: + for line in f: + line = line.rstrip() + + # Skip lines matching these criteria + if not line: continue + if re.search(r'^(import|#)', line): continue + if (ext.name != 'py-setuptools' and + re.search(r'setuptools.*egg$', line)): continue + + paths.append(line) + + site_packages = os.path.join(self.prefix, self.site_packages_dir) + main_pth = "%s/easy-install.pth" % site_packages + + if not paths: + if os.path.isfile(main_pth): + os.remove(main_pth) + + else: + with closing(open(main_pth, 'w')) as f: + f.write("import sys; sys.__plen = len(sys.path)\n") + for path in paths: + f.write("%s\n" % path) + f.write("import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; " + "p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)\n") + + def activate(self, ext_pkg, **args): - args.update(ignore=self.make_ignore(args)) + args.update(ignore=self.python_ignore(ext_pkg, args)) super(Python, self).activate(ext_pkg, **args) + extensions = set(spack.install_layout.get_extensions(self.spec)) + extensions.add(ext_pkg.spec) + self.write_easy_install_pth(extensions) + def deactivate(self, ext_pkg, **args): - args.update(ignore=self.make_ignore(args)) + args.update(ignore=self.python_ignore(ext_pkg, args)) super(Python, self).deactivate(ext_pkg, **args) + + extensions = set(spack.install_layout.get_extensions(self.spec)) + extensions.remove(ext_pkg.spec) + self.write_easy_install_pth(extensions) From 5bde8359e8378bf8595a4bef343d1f50258f663d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 2 Feb 2015 07:58:52 -0800 Subject: [PATCH 33/97] More information in extensions command. --- lib/spack/spack/cmd/extensions.py | 39 ++++++++++++++++++++++++------- lib/spack/spack/packages.py | 6 +++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index f28a388bf2a..f6ccd7b5151 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -54,9 +54,14 @@ def extensions(parser, args): if not args.spec: tty.die("extensions requires a package spec.") + # Checks spec = spack.cmd.parse_specs(args.spec) if len(spec) > 1: tty.die("Can only list extensions for one package.") + + if not spec[0].package.extendable: + tty.die("%s is not an extendable package." % spec[0].name) + spec = spack.cmd.disambiguate_spec(spec[0]) if not spec.package.extendable: @@ -65,12 +70,28 @@ def extensions(parser, args): if not args.mode: args.mode = 'short' - exts = spack.install_layout.get_extensions(spec) - if not exts: - tty.msg("%s has no activated extensions." % spec.cshort_spec) - else: - tty.msg("Extensions for package %s:" % spec.cshort_spec) - colify(pkg.name for pkg in spack.db.extensions_for(spec)) - print - tty.msg("%d currently activated:" % len(exts)) - spack.cmd.find.display_specs(exts, mode=args.mode) + # List package names of extensions + extensions = spack.db.extensions_for(spec) + if not extensions: + tty.msg("%s has no extensions." % spec.cshort_spec) + return + tty.msg("%s extensions:" % spec.cshort_spec) + colify(ext.name for ext in extensions) + + # List specs of installed extensions. + installed = [s.spec for s in spack.db.installed_extensions_for(spec)] + print + if not installed: + tty.msg("None activated.") + return + tty.msg("%d installed:" % len(installed)) + spack.cmd.find.display_specs(installed, mode=args.mode) + + # List specs of activated extensions. + activated = spack.install_layout.get_extensions(spec) + print + if not activated: + tty.msg("None activated.") + return + tty.msg("%d currently activated:" % len(exts)) + spack.cmd.find.display_specs(installed, mode=args.mode) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index b3049e812f0..7ef8135c1a9 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -117,6 +117,12 @@ def extensions_for(self, extendee_spec): return [p for p in self.all_packages() if p.extends(extendee_spec)] + @_autospec + def installed_extensions_for(self, extendee_spec): + return [s.package for s in self.installed_package_specs() + if s.package.extends(extendee_spec)] + + def dirname_for_package_name(self, pkg_name): """Get the directory name for a particular package. This is the directory that contains its package.py file.""" From a9e189972a547fb5f7996f0f5acafc1200702b72 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 4 Feb 2015 15:42:41 -0800 Subject: [PATCH 34/97] Bugfix in spak extensions --- lib/spack/spack/cmd/extensions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index f6ccd7b5151..3b189895b0b 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -93,5 +93,5 @@ def extensions(parser, args): if not activated: tty.msg("None activated.") return - tty.msg("%d currently activated:" % len(exts)) - spack.cmd.find.display_specs(installed, mode=args.mode) + tty.msg("%d currently activated:" % len(activated)) + spack.cmd.find.display_specs(activated, mode=args.mode) From 3a3e4d4391977327bca18c416205971dd6a85509 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 4 Feb 2015 15:47:03 -0800 Subject: [PATCH 35/97] Do not automatically activate extensions on install. --- lib/spack/spack/hooks/extensions.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index 718b24b9655..9d6fa23d034 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -26,15 +26,11 @@ import spack -def post_install(pkg): - if pkg.is_extension: - pkg.do_activate() - - def pre_uninstall(pkg): # Need to do this b/c uninstall does not automatically do it. # TODO: store full graph info in stored .spec file. pkg.spec.normalize() if pkg.is_extension: - pkg.do_deactivate() + if pkg.activated: + pkg.do_deactivate() From 5cc369c2b831446f1afaaba41cbf0dbdba75b4ed Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Fri, 6 Feb 2015 16:35:35 -0800 Subject: [PATCH 36/97] add dependent packages to PYTHONPATH for build --- var/spack/packages/python/package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 8a6d574d9b2..23b528b0897 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -66,6 +66,11 @@ def setup_extension_environment(self, module, spec, ext_spec): # Make the site packages directory if it does not exist already. mkdirp(module.site_packages_dir) + # Add dependent packages' site-packages directory to PYTHONPATH + for d in ext_spec.traverse(): + if d.package.extends(self.spec): + os.environ['PYTHONPATH'] += ':' + os.path.join(d.prefix, self.site_packages_dir) + # ======================================================================== # Handle specifics of activating and deactivating python modules. From d95d48bbe69eed2772bea8d155ccc4f8d708e4d2 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Fri, 6 Feb 2015 16:41:43 -0800 Subject: [PATCH 37/97] py-mako and fix for setup-env.sh --- share/spack/setup-env.sh | 4 ++-- var/spack/packages/py-mako/package.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 var/spack/packages/py-mako/package.py diff --git a/share/spack/setup-env.sh b/share/spack/setup-env.sh index 91b1dc46302..b2bcbaf6c7c 100755 --- a/share/spack/setup-env.sh +++ b/share/spack/setup-env.sh @@ -165,8 +165,8 @@ fi # # Set up modules and dotkit search paths in the user environment # -_sp_share_dir="$(dirname $_sp_source_file)" -_sp_prefix="$(dirname $(dirname $_sp_share_dir))" +_sp_share_dir=$(cd "$(dirname $_sp_source_file)" && pwd) +_sp_prefix=$(cd "$(dirname $(dirname $_sp_share_dir))" && pwd) # TODO: fix SYS_TYPE to something non-LLNL-specific _spack_pathadd DK_NODE "$_sp_share_dir/dotkit/$SYS_TYPE" diff --git a/var/spack/packages/py-mako/package.py b/var/spack/packages/py-mako/package.py new file mode 100644 index 00000000000..3e91ffd8e57 --- /dev/null +++ b/var/spack/packages/py-mako/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyMako(Package): + """A super-fast templating language that borrows the best + ideas from the existing templating languages.""" + + homepage = "https://pypi.python.org/pypi/mako" + url = "https://pypi.python.org/packages/source/M/Mako/Mako-1.0.1.tar.gz" + + version('1.0.1', '9f0aafd177b039ef67b90ea350497a54') + + depends_on('py-setuptools') + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 5fdf5438ea4375dd5d9ceee7c5af1a4cf6896d53 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Fri, 6 Feb 2015 16:55:48 -0800 Subject: [PATCH 38/97] flex and bison --- var/spack/packages/bison/package.py | 17 +++++++++++++++++ var/spack/packages/flex/package.py | 15 +++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 var/spack/packages/bison/package.py create mode 100644 var/spack/packages/flex/package.py diff --git a/var/spack/packages/bison/package.py b/var/spack/packages/bison/package.py new file mode 100644 index 00000000000..7c526fb9589 --- /dev/null +++ b/var/spack/packages/bison/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Bison(Package): + """Bison is a general-purpose parser generator that converts + an annotated context-free grammar into a deterministic LR or + generalized LR (GLR) parser employing LALR(1) parser tables.""" + + homepage = "http://www.gnu.org/software/bison/" + url = "http://ftp.gnu.org/gnu/bison/bison-3.0.tar.gz" + + version('3.0.4', 'a586e11cd4aff49c3ff6d3b6a4c9ccf8') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/packages/flex/package.py b/var/spack/packages/flex/package.py new file mode 100644 index 00000000000..b065904912c --- /dev/null +++ b/var/spack/packages/flex/package.py @@ -0,0 +1,15 @@ +from spack import * + +class Flex(Package): + """Flex is a tool for generating scanners.""" + + homepage = "http://flex.sourceforge.net/" + url = "http://download.sourceforge.net/flex/flex-2.5.39.tar.gz" + + version('2.5.39', 'e133e9ead8ec0a58d81166b461244fde') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") From 676cc84c9e0a845fab117fde4ff0b1dca0b2d792 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Fri, 6 Feb 2015 17:24:55 -0800 Subject: [PATCH 39/97] more mesa dependencies --- var/spack/packages/dri2proto/package.py | 14 ++++++++++++++ var/spack/packages/libdrm/package.py | 22 ++++++++++++++++++++++ var/spack/packages/libxcb/package.py | 21 +++++++++++++++++++++ var/spack/packages/libxshmfence/package.py | 16 ++++++++++++++++ var/spack/packages/xcb-proto/package.py | 15 +++++++++++++++ 5 files changed, 88 insertions(+) create mode 100644 var/spack/packages/dri2proto/package.py create mode 100644 var/spack/packages/libdrm/package.py create mode 100644 var/spack/packages/libxcb/package.py create mode 100644 var/spack/packages/libxshmfence/package.py create mode 100644 var/spack/packages/xcb-proto/package.py diff --git a/var/spack/packages/dri2proto/package.py b/var/spack/packages/dri2proto/package.py new file mode 100644 index 00000000000..11dfa568e27 --- /dev/null +++ b/var/spack/packages/dri2proto/package.py @@ -0,0 +1,14 @@ +from spack import * + +class Dri2proto(Package): + """DRI2 Protocol Headers.""" + homepage = "http://http://cgit.freedesktop.org/xorg/proto/dri2proto/" + url = "http://xorg.freedesktop.org/releases/individual/proto/dri2proto-2.8.tar.gz" + + version('2.8', '19ea18f63d8ae8053c9fa84b60365b77') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/packages/libdrm/package.py b/var/spack/packages/libdrm/package.py new file mode 100644 index 00000000000..9164db81416 --- /dev/null +++ b/var/spack/packages/libdrm/package.py @@ -0,0 +1,22 @@ +from spack import * + +class Libdrm(Package): + """A userspace library for accessing the DRM, direct + rendering manager, on Linux, BSD and other operating + systems that support the ioctl interface.""" + + homepage = "http://dri.freedesktop.org/libdrm/" # no real website... + url = "http://dri.freedesktop.org/libdrm/libdrm-2.4.59.tar.gz" + + version('2.4.59', '105ac7af1afcd742d402ca7b4eb168b6') + + # FIXME: Add dependencies if this package requires them. + # depends_on("foo") + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure("--prefix=%s" % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/packages/libxcb/package.py b/var/spack/packages/libxcb/package.py new file mode 100644 index 00000000000..521cd0d4757 --- /dev/null +++ b/var/spack/packages/libxcb/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Libxcb(Package): + """The X protocol C-language Binding (XCB) is a replacement + for Xlib featuring a small footprint, latency hiding, direct + access to the protocol, improved threading support, and + extensibility.""" + + homepage = "http://xcb.freedesktop.org/" + url = "http://xcb.freedesktop.org/dist/libxcb-1.11.tar.gz" + + version('1.11', '1698dd837d7e6e94d029dbe8b3a82deb') + + depends_on("python") + depends_on("xcb-proto") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/packages/libxshmfence/package.py b/var/spack/packages/libxshmfence/package.py new file mode 100644 index 00000000000..3aa2448b46b --- /dev/null +++ b/var/spack/packages/libxshmfence/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Libxshmfence(Package): + """This is a tiny library that exposes a event API on top of Linux + futexes.""" + + homepage = "http://keithp.com/blogs/dri3_extension/" # not really... + url = "http://xorg.freedesktop.org/archive/individual/lib/libxshmfence-1.2.tar.gz" + + version('1.2', 'f0b30c0fc568b22ec524859ee28556f1') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/packages/xcb-proto/package.py b/var/spack/packages/xcb-proto/package.py new file mode 100644 index 00000000000..17a94bd8924 --- /dev/null +++ b/var/spack/packages/xcb-proto/package.py @@ -0,0 +1,15 @@ +from spack import * + +class XcbProto(Package): + """Protocol for libxcb""" + + homepage = "http://xcb.freedesktop.org/" + url = "http://xcb.freedesktop.org/dist/xcb-proto-1.11.tar.gz" + + version('1.11', 'c8c6cb72c84f58270f4db1f39607f66a') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") From 932f3930f43e10f22c06ede6bb963a23989443c1 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Sat, 7 Feb 2015 09:18:34 -0800 Subject: [PATCH 40/97] util-linux added --- var/spack/packages/libdrm/package.py | 5 ----- var/spack/packages/util-linux/package.py | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 var/spack/packages/util-linux/package.py diff --git a/var/spack/packages/libdrm/package.py b/var/spack/packages/libdrm/package.py index 9164db81416..3a657e8ecc2 100644 --- a/var/spack/packages/libdrm/package.py +++ b/var/spack/packages/libdrm/package.py @@ -10,13 +10,8 @@ class Libdrm(Package): version('2.4.59', '105ac7af1afcd742d402ca7b4eb168b6') - # FIXME: Add dependencies if this package requires them. - # depends_on("foo") - def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. configure("--prefix=%s" % prefix) - # FIXME: Add logic to build and install here make() make("install") diff --git a/var/spack/packages/util-linux/package.py b/var/spack/packages/util-linux/package.py new file mode 100644 index 00000000000..cb7ceabf571 --- /dev/null +++ b/var/spack/packages/util-linux/package.py @@ -0,0 +1,20 @@ +from spack import * +import os + +class UtilLinux(Package): + """Util-linux is a suite of essential utilities for any Linux system.""" + + homepage = "http://freecode.com/projects/util-linux" + url = "https://www.kernel.org/pub/linux/utils/util-linux/v2.25/util-linux-2.25.tar.gz" + + version('2.25', 'f6d7fc6952ec69c4dc62c8d7c59c1d57') + + depends_on("python@2.7:") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "PKG_CONFIG_PATH=%s/pkgconfig" % spec['python'].prefix.lib, + "--disable-use-tty-group") + + make() + make("install") From 1605e04d444583284a19462206755ecacb219d3b Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Sat, 7 Feb 2015 22:08:50 -0800 Subject: [PATCH 41/97] mesa and systemd (systemd not working yet) --- var/spack/packages/mesa/package.py | 26 ++++++++++++++++++ var/spack/packages/systemd/package.py | 38 +++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 var/spack/packages/mesa/package.py create mode 100644 var/spack/packages/systemd/package.py diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py new file mode 100644 index 00000000000..df28d877006 --- /dev/null +++ b/var/spack/packages/mesa/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Mesa(Package): + """Mesa is an open-source implementation of the OpenGL + specification - a system for rendering interactive 3D graphics.""" + + homepage = "http://www.mesa3d.org" + url = "ftp://ftp.freedesktop.org/pub/mesa/10.4.4/MesaLib-10.4.4.tar.gz" + + version('10.4.4', '8d863a3c209bf5116b2babfccccc68ce') + + depends_on("py-mako") + depends_on("flex") + depends_on("bison") + depends_on("libdrm") + depends_on("dri2proto") + depends_on("libxcb") + depends_on("libxshmfence") + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure("--prefix=%s" % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/packages/systemd/package.py b/var/spack/packages/systemd/package.py new file mode 100644 index 00000000000..647b3bda7b9 --- /dev/null +++ b/var/spack/packages/systemd/package.py @@ -0,0 +1,38 @@ +# FIXME: +# This is a template package file for Spack. We've conveniently +# put "FIXME" labels next to all the things you'll want to change. +# +# Once you've edited all the FIXME's, delete this whole message, +# save this file, and test out your package like this: +# +# spack install systemd +# +# You can always get back here to change things with: +# +# spack edit systemd +# +# See the spack documentation for more information on building +# packages. +# +from spack import * + +class Systemd(Package): + """Systemd is a suite of basic building blocks for a Linux system.""" + + homepage = "http://www.freedesktop.org/wiki/Software/systemd/" + url = "http://www.freedesktop.org/software/systemd/systemd-218.tar.xz" + + version('218', '4e2c511b0a7932d7fc9d79822273aac6') + + depends_on("gperf") + depends_on("coreutils@8.16:") # ln --relative + depends_on("util-linux") # libmount + depends_on("python@2.7:") + depends_on("gcc@4.5:") # pragma gcc diagnostic not allowed inside functions + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "CC=%s/gcc -std=gnu99" % spec['gcc'].prefix.bin) + + make() + make("install") From cc684a3ebeaa939b9ae87474991e210aab7c3185 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Sun, 8 Feb 2015 13:34:45 -0800 Subject: [PATCH 42/97] older mesa for 2.6 kernel (not workin yet) --- var/spack/packages/mesa/package.py | 8 +++--- var/spack/packages/systemd/package.py | 38 --------------------------- 2 files changed, 4 insertions(+), 42 deletions(-) delete mode 100644 var/spack/packages/systemd/package.py diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py index df28d877006..6da13e52b20 100644 --- a/var/spack/packages/mesa/package.py +++ b/var/spack/packages/mesa/package.py @@ -5,9 +5,11 @@ class Mesa(Package): specification - a system for rendering interactive 3D graphics.""" homepage = "http://www.mesa3d.org" - url = "ftp://ftp.freedesktop.org/pub/mesa/10.4.4/MesaLib-10.4.4.tar.gz" + url = "ftp://ftp.freedesktop.org/pub/mesa/older-versions/7.x/7.11.2/MesaLib-7.11.2.tar.gz" + # url = "ftp://ftp.freedesktop.org/pub/mesa/10.4.4/MesaLib-10.4.4.tar.gz" - version('10.4.4', '8d863a3c209bf5116b2babfccccc68ce') + # version('10.4.4', '8d863a3c209bf5116b2babfccccc68ce') + version('7.11.2', 'b9e84efee3931c0acbccd1bb5a860554') depends_on("py-mako") depends_on("flex") @@ -18,9 +20,7 @@ class Mesa(Package): depends_on("libxshmfence") def install(self, spec, prefix): - # FIXME: Modify the configure line to suit your build system here. configure("--prefix=%s" % prefix) - # FIXME: Add logic to build and install here make() make("install") diff --git a/var/spack/packages/systemd/package.py b/var/spack/packages/systemd/package.py deleted file mode 100644 index 647b3bda7b9..00000000000 --- a/var/spack/packages/systemd/package.py +++ /dev/null @@ -1,38 +0,0 @@ -# FIXME: -# This is a template package file for Spack. We've conveniently -# put "FIXME" labels next to all the things you'll want to change. -# -# Once you've edited all the FIXME's, delete this whole message, -# save this file, and test out your package like this: -# -# spack install systemd -# -# You can always get back here to change things with: -# -# spack edit systemd -# -# See the spack documentation for more information on building -# packages. -# -from spack import * - -class Systemd(Package): - """Systemd is a suite of basic building blocks for a Linux system.""" - - homepage = "http://www.freedesktop.org/wiki/Software/systemd/" - url = "http://www.freedesktop.org/software/systemd/systemd-218.tar.xz" - - version('218', '4e2c511b0a7932d7fc9d79822273aac6') - - depends_on("gperf") - depends_on("coreutils@8.16:") # ln --relative - depends_on("util-linux") # libmount - depends_on("python@2.7:") - depends_on("gcc@4.5:") # pragma gcc diagnostic not allowed inside functions - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "CC=%s/gcc -std=gnu99" % spec['gcc'].prefix.bin) - - make() - make("install") From 9e878075aceace1439e85b1d75770a7a6f248566 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Sun, 8 Feb 2015 16:09:13 -0800 Subject: [PATCH 43/97] mesa 8.0.5 working --- var/spack/packages/libdrm/package.py | 1 + var/spack/packages/llvm/package.py | 6 ++++++ var/spack/packages/mesa/package.py | 25 ++++++++++++++++--------- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/var/spack/packages/libdrm/package.py b/var/spack/packages/libdrm/package.py index 3a657e8ecc2..00736b7811a 100644 --- a/var/spack/packages/libdrm/package.py +++ b/var/spack/packages/libdrm/package.py @@ -9,6 +9,7 @@ class Libdrm(Package): url = "http://dri.freedesktop.org/libdrm/libdrm-2.4.59.tar.gz" version('2.4.59', '105ac7af1afcd742d402ca7b4eb168b6') + version('2.4.33', '86e4e3debe7087d5404461e0032231c8') def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index c7a10df55ae..69354a5c909 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -35,6 +35,12 @@ class Llvm(Package): url = "http://llvm.org/releases/3.4.2/llvm-3.4.2.src.tar.gz" version('3.4.2', 'a20669f75967440de949ac3b1bad439c') + version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', + url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') + version('2.9', '793138412d2af2c7c7f54615f8943771', + url='http://llvm.org/releases/2.9/llvm-2.9.tgz') + version('2.8', '220d361b4d17051ff4bb21c64abe05ba', + url='http://llvm.org/releases/2.8/llvm-2.8.tgz') def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py index 6da13e52b20..2dba878a77c 100644 --- a/var/spack/packages/mesa/package.py +++ b/var/spack/packages/mesa/package.py @@ -5,19 +5,26 @@ class Mesa(Package): specification - a system for rendering interactive 3D graphics.""" homepage = "http://www.mesa3d.org" - url = "ftp://ftp.freedesktop.org/pub/mesa/older-versions/7.x/7.11.2/MesaLib-7.11.2.tar.gz" + url = "ftp://ftp.freedesktop.org/pub/mesa/older-versions/8.x/8.0.5/MesaLib-8.0.5.tar.gz" # url = "ftp://ftp.freedesktop.org/pub/mesa/10.4.4/MesaLib-10.4.4.tar.gz" # version('10.4.4', '8d863a3c209bf5116b2babfccccc68ce') - version('7.11.2', 'b9e84efee3931c0acbccd1bb5a860554') + version('8.0.5', 'cda5d101f43b8784fa60bdeaca4056f2') + + # mesa 7.x, 8.x, 9.x + depends_on("libdrm@2.4.33") + depends_on("llvm@3.0") + + # patch("llvm-fixes.patch") # using newer llvm + + # mesa 10.x + # depends_on("py-mako") + # depends_on("flex") + # depends_on("bison") + # depends_on("dri2proto") + # depends_on("libxcb") + # depends_on("libxshmfence") - depends_on("py-mako") - depends_on("flex") - depends_on("bison") - depends_on("libdrm") - depends_on("dri2proto") - depends_on("libxcb") - depends_on("libxshmfence") def install(self, spec, prefix): configure("--prefix=%s" % prefix) From befe72b9b9d1f4107eef8f24e6f194f27a15bca4 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 8 Feb 2015 19:36:30 -0800 Subject: [PATCH 44/97] directory_layout now raises an error when an install fails. --- lib/spack/spack/directory_layout.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index efc40a17a4f..37740720a20 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -109,12 +109,17 @@ def path_for_spec(self, spec): def remove_path_for_spec(self, spec): - """Removes a prefix and any empty parent directories from the root.""" + """Removes a prefix and any empty parent directories from the root. + Raised RemoveFailedError if something goes wrong. + """ path = self.path_for_spec(spec) assert(path.startswith(self.root)) if os.path.exists(path): - shutil.rmtree(path, True) + try: + shutil.rmtree(path) + except exceptions.OSError, e: + raise RemoveFailedError(spec, path, e) path = os.path.dirname(path) while path != self.root: @@ -330,6 +335,15 @@ def __init__(self, installed_spec, new_spec): % installed_spec, new_spec) +class RemoveFailedError(DirectoryLayoutError): + """Raised when a DirectoryLayout cannot remove an install prefix.""" + def __init__(self, installed_spec, prefix, error): + super(RemoveFailedError, self).__init__( + 'Could not remove prefix %s for %s : %s' + % prefix, installed_spec.short_spec, error) + self.cause = error + + class InconsistentInstallDirectoryError(DirectoryLayoutError): """Raised when a package seems to be installed to the wrong place.""" def __init__(self, message): @@ -370,3 +384,5 @@ def __init__(self, spec, extension_spec): super(NoSuchExtensionError, self).__init__( "%s cannot be removed from %s because it's not installed."% ( extension_spec.short_spec, spec.short_spec)) + + From e51e01f4f066f9b0f412354cb8abab3642845d43 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 8 Feb 2015 19:39:36 -0800 Subject: [PATCH 45/97] Cleaned up python to remove redundant line. --- var/spack/packages/python/package.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 23b528b0897..8bffbf393c3 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -60,16 +60,16 @@ def setup_extension_environment(self, module, spec, ext_spec): module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) - # Add site packages directory to the PYTHONPATH - os.environ['PYTHONPATH'] = module.site_packages_dir - # Make the site packages directory if it does not exist already. mkdirp(module.site_packages_dir) - # Add dependent packages' site-packages directory to PYTHONPATH + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. + python_paths = [] for d in ext_spec.traverse(): if d.package.extends(self.spec): - os.environ['PYTHONPATH'] += ':' + os.path.join(d.prefix, self.site_packages_dir) + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + os.environ['PYTHONPATH'] = ':'.join(python_paths) # ======================================================================== From 60a385d4a44b16bce77725dbb4f71a26aa610ffc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 8 Feb 2015 19:40:28 -0800 Subject: [PATCH 46/97] Minor textual error in extensions command. --- lib/spack/spack/cmd/extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index 3b189895b0b..c6b61453218 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -82,7 +82,7 @@ def extensions(parser, args): installed = [s.spec for s in spack.db.installed_extensions_for(spec)] print if not installed: - tty.msg("None activated.") + tty.msg("None installed.") return tty.msg("%d installed:" % len(installed)) spack.cmd.find.display_specs(installed, mode=args.mode) From 20ec80295dbf2a36b633860c139d113f9c0e6388 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 8 Feb 2015 19:41:17 -0800 Subject: [PATCH 47/97] setup_extension_environment is now setup_dependent_environment. - other packages, like Qt, can now use this to set up relevant build variables and env vars for their dependencies. - not just extensions anymore. --- lib/spack/spack/package.py | 14 ++++++++------ var/spack/packages/python/package.py | 2 +- var/spack/packages/qt/package.py | 6 ++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b9059685405..6e319a1f87a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -829,10 +829,10 @@ def do_install(self, **kwargs): self.stage.chdir_to_source() build_env.setup_package(self) - # Allow extendees to further set up the environment. - if self.is_extension: - self.extendee_spec.package.setup_extension_environment( - self.module, self.extendee_spec, self.spec) + # Allow dependencies to further set up the environment. + for dep_spec in self.spec.traverse(root=False): + dep_spec.package.setup_dependent_environment( + self.module, dep_spec, self.spec) if fake_install: self.do_fake_install() @@ -910,8 +910,8 @@ def module(self): fromlist=[self.__class__.__name__]) - def setup_extension_environment(self, module, spec, ext_spec): - """Called before the install() method of extensions. + def setup_dependent_environment(self, module, spec, dependent_spec): + """Called before the install() method of dependents. Default implementation does nothing, but this can be overridden by an extendable package to set up the install @@ -930,6 +930,8 @@ def setup_extension_environment(self, module, spec, ext_spec): put a 'python' Execuable object in the module scope for the extension package to simplify extension installs. + 3. A lot of Qt extensions need QTDIR set. This can be used to do that. + """ pass diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 8bffbf393c3..eed81d095c9 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -46,7 +46,7 @@ def site_packages_dir(self): return os.path.join(self.python_lib_dir, 'site-packages') - def setup_extension_environment(self, module, spec, ext_spec): + def setup_dependent_environment(self, module, spec, ext_spec): """Called before python modules' install() methods. In most cases, extensions will only need to have one line:: diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py index 6a55c897015..fcbcd2491a0 100644 --- a/var/spack/packages/qt/package.py +++ b/var/spack/packages/qt/package.py @@ -1,3 +1,4 @@ +import os from spack import * class Qt(Package): @@ -20,6 +21,11 @@ class Qt(Package): depends_on("libmng") depends_on("jpeg") + def setup_dependent_environment(self, module, spec, dep_spec): + """Dependencies of Qt find it using the QTDIR environment variable.""" + os.environ['QTDIR'] = self.prefix + + def patch(self): # Fix qmake compilers in the default mkspec qmake_conf = 'mkspecs/common/g++-base.conf' From f81b136547347997d043bd99adf9d7013f097c69 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 8 Feb 2015 19:43:10 -0800 Subject: [PATCH 48/97] import fix in cmd/clean --- lib/spack/spack/cmd/clean.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index 79dd91c5bf3..ec3b221988a 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from external import argparse +import subprocess import llnl.util.tty as tty From c077f0570531aac76b9182a96a2a37ee0e633989 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 8 Feb 2015 22:01:00 -0800 Subject: [PATCH 49/97] Move dpeendency environment setup to build_environemnt. --- lib/spack/spack/build_environment.py | 5 +++++ lib/spack/spack/package.py | 5 ----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 45353ec6402..cabde7dc86e 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -207,3 +207,8 @@ def setup_package(pkg): set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) set_module_variables_for_package(pkg) + + # Allow dependencies to set up environment as well. + for dep_spec in pkg.spec.traverse(root=False): + dep_spec.package.setup_dependent_environment( + pkg.module, dep_spec, pkg.spec) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 6e319a1f87a..b18d0549901 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -829,11 +829,6 @@ def do_install(self, **kwargs): self.stage.chdir_to_source() build_env.setup_package(self) - # Allow dependencies to further set up the environment. - for dep_spec in self.spec.traverse(root=False): - dep_spec.package.setup_dependent_environment( - self.module, dep_spec, self.spec) - if fake_install: self.do_fake_install() else: From aae364b4c9ffdf8949a8c7ce5a7e63c9772703c0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 8 Feb 2015 23:26:15 -0800 Subject: [PATCH 50/97] "spack extensions" shows total extension count. --- lib/spack/spack/cmd/extensions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index c6b61453218..ae73d8ac553 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -75,7 +75,8 @@ def extensions(parser, args): if not extensions: tty.msg("%s has no extensions." % spec.cshort_spec) return - tty.msg("%s extensions:" % spec.cshort_spec) + tty.msg(spec.cshort_spec) + tty.msg("%d extensions:" % len(extensions)) colify(ext.name for ext in extensions) # List specs of installed extensions. From 27617670f013bd1dbde24079bd7ae38331530695 Mon Sep 17 00:00:00 2001 From: Alfredo Gimenez Date: Mon, 9 Feb 2015 00:01:07 -0800 Subject: [PATCH 51/97] qt with hardware accelerated opengl working --- var/spack/packages/icu4c/package.py | 17 +++++++++++++++++ var/spack/packages/qt/package.py | 20 ++++++++++++++++++-- var/spack/packages/ruby/package.py | 16 ++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 var/spack/packages/icu4c/package.py create mode 100644 var/spack/packages/ruby/package.py diff --git a/var/spack/packages/icu4c/package.py b/var/spack/packages/icu4c/package.py new file mode 100644 index 00000000000..55b44463b2c --- /dev/null +++ b/var/spack/packages/icu4c/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Icu4c(Package): + """ICU is a mature, widely used set of C/C++ and Java libraries + providing Unicode and Globalization support for software applications.""" + + homepage = "http://site.icu-project.org/" + url = "http://downloads.sourceforge.net/project/icu/ICU4C/54.1/icu4c-54_1-src.tgz" + + version('54_1', 'e844caed8f2ca24c088505b0d6271bc0') + + def install(self, spec, prefix): + cd("source") + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py index 96e484b9959..c8b19d07f5c 100644 --- a/var/spack/packages/qt/package.py +++ b/var/spack/packages/qt/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Qt(Package): """Qt is a comprehensive cross-platform C++ application framework.""" @@ -29,7 +30,16 @@ class Qt(Package): depends_on("libmng") depends_on("jpeg") - depends_on("gperf") # Needed to build Qt with webkit. + # Webkit + # depends_on("gperf") + # depends_on("flex") + # depends_on("bison") + # depends_on("ruby") + # depends_on("icu4c") + + # OpenGL hardware acceleration + depends_on("mesa") + depends_on("libxcb") def patch(self): if self.spec.satisfies('@4'): @@ -46,13 +56,19 @@ def patch(self): def install(self, spec, prefix): + # Apparently this is the only way to + # "truly" get rid of webkit compiles now... + os.rename("qtwebkit","no-qtwebkit") + os.rename("qtwebkit-examples","no-qtwebkit-examples") configure('-v', '-confirm-license', '-opensource', '-prefix', prefix, '-openssl-linked', '-dbus-linked', - '-fast', + #'-fast', + '-opengl', + '-qt-xcb', '-optimized-qmake', '-no-pch', # phonon required for py-pyqt4 diff --git a/var/spack/packages/ruby/package.py b/var/spack/packages/ruby/package.py new file mode 100644 index 00000000000..718fd0a3be1 --- /dev/null +++ b/var/spack/packages/ruby/package.py @@ -0,0 +1,16 @@ +from spack import * + +class Ruby(Package): + """A dynamic, open source programming language with a focus on + simplicity and productivity.""" + + homepage = "https://www.ruby-lang.org/" + url = "http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.0.tar.gz" + + version('2.2.0', 'cd03b28fd0b555970f5c4fd481700852') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") From d1d0b85d80a1cf46e6417c0d467da20bbea1eba0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 9 Feb 2015 01:13:56 -0800 Subject: [PATCH 52/97] Add Alfredo to contributors. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7a13dba2a8b..74a327b517a 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ people: * David Beckingsale * David Boehme + * Alfredo Gimenez * Luc Jaulmes * Matt Legendre * Greg Lee From 25af341954fa7c9a88943fe24cad88275ee00795 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 9 Feb 2015 02:54:49 -0800 Subject: [PATCH 53/97] Python package improvements. --- var/spack/packages/geos/package.py | 31 +++++++++++++++++++++ var/spack/packages/py-basemap/package.py | 9 ++---- var/spack/packages/py-biopython/package.py | 1 + var/spack/packages/py-gnuplot/package.py | 1 + var/spack/packages/py-libxml2/package.py | 1 + var/spack/packages/py-matplotlib/package.py | 3 ++ var/spack/packages/py-pyside/package.py | 31 ++++++++++++++++++--- var/spack/packages/py-shiboken/package.py | 21 ++++++++++++++ var/spack/packages/python/package.py | 5 ++-- 9 files changed, 90 insertions(+), 13 deletions(-) create mode 100644 var/spack/packages/geos/package.py create mode 100644 var/spack/packages/py-shiboken/package.py diff --git a/var/spack/packages/geos/package.py b/var/spack/packages/geos/package.py new file mode 100644 index 00000000000..4a2657e32f2 --- /dev/null +++ b/var/spack/packages/geos/package.py @@ -0,0 +1,31 @@ +from spack import * + +class Geos(Package): + """GEOS (Geometry Engine - Open Source) is a C++ port of the Java + Topology Suite (JTS). As such, it aims to contain the complete + functionality of JTS in C++. This includes all the OpenGIS + Simple Features for SQL spatial predicate functions and spatial + operators, as well as specific JTS enhanced topology functions.""" + + homepage = "http://trac.osgeo.org/geos/" + url = "http://download.osgeo.org/geos/geos-3.4.2.tar.bz2" + + version('3.4.2', 'fc5df2d926eb7e67f988a43a92683bae') + version('3.4.1', '4c930dec44c45c49cd71f3e0931ded7e') + version('3.4.0', 'e41318fc76b5dc764a69d43ac6b18488') + version('3.3.9', '4794c20f07721d5011c93efc6ccb8e4e') + version('3.3.8', '75be476d0831a2d14958fed76ca266de') + version('3.3.7', '95ab996d22672b067d92c7dee2170460') + version('3.3.6', '6fadfb941541875f4976f75fb0bbc800') + version('3.3.5', '2ba61afb7fe2c5ddf642d82d7b16e75b') + version('3.3.4', '1bb9f14d57ef06ffa41cb1d67acb55a1') + version('3.3.3', '8454e653d7ecca475153cc88fd1daa26') + + extends('python') + depends_on('swig') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--enable-python") + make() + make("install") diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py index 7b6d8e7e657..45f1085ba1a 100644 --- a/var/spack/packages/py-basemap/package.py +++ b/var/spack/packages/py-basemap/package.py @@ -8,18 +8,13 @@ class PyBasemap(Package): version('1.0.7', '48c0557ced9e2c6e440b28b3caff2de8') - geos_version = {'1.0.7' : '3.3.3'} - extends('python') depends_on('py-setuptools') depends_on('py-numpy') depends_on('py-matplotlib') depends_on('py-pil') + depends_on("geos") def install(self, spec, prefix): - with working_dir('geos-%s' % self.geos_version[str(self.version)]): - configure("--prefix=" + prefix) - make() - make("install") - os.environ['GEOS_DIR'] = prefix + env['GEOS_DIR'] = spec['geos'].prefix python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-biopython/package.py b/var/spack/packages/py-biopython/package.py index 2ed04c389e7..8ecaf486262 100644 --- a/var/spack/packages/py-biopython/package.py +++ b/var/spack/packages/py-biopython/package.py @@ -9,6 +9,7 @@ class PyBiopython(Package): extends('python') depends_on('py-mx') + depends_on('py-numpy') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-gnuplot/package.py b/var/spack/packages/py-gnuplot/package.py index 0a2c073a491..ede4472c034 100644 --- a/var/spack/packages/py-gnuplot/package.py +++ b/var/spack/packages/py-gnuplot/package.py @@ -8,6 +8,7 @@ class PyGnuplot(Package): version('1.8', 'abd6f571e7aec68ae7db90a5217cd5b1') extends('python') + depends_on('py-numpy') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-libxml2/package.py b/var/spack/packages/py-libxml2/package.py index 0dcefbd9cf8..e645acb5dd8 100644 --- a/var/spack/packages/py-libxml2/package.py +++ b/var/spack/packages/py-libxml2/package.py @@ -8,6 +8,7 @@ class PyLibxml2(Package): version('2.6.21', '229dd2b3d110a77defeeaa73af83f7f3') extends('python') + depends_on('libxml2') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index 5979ceeab0d..8b8684c5632 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -17,9 +17,12 @@ class PyMatplotlib(Package): depends_on('py-pytz') depends_on('py-nose') depends_on('py-numpy') + depends_on('qt') + def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) + if str(self.version) == '1.4.2': # hack to fix configuration file config_file = None diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py index b01e16d7e6c..7528a0bf72b 100644 --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -10,9 +10,32 @@ class PyPyside(Package): version('1.2.2', 'c45bc400c8a86d6b35f34c29e379e44d') extends('python') + depends_on('py-setuptools') + depends_on('qt@:4') + + + def patch(Self): + """Undo PySide RPATH handling and add Spack RPATH.""" + # Add Spack's standard CMake args to the sub-builds. + # They're called BY setup.py so we have to patch it. + filter_file( + r'OPTION_CMAKE,', + r'OPTION_CMAKE, ' + ( + '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' + '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(self.rpath)), + 'setup.py') + + # PySide tries to patch ELF files to remove RPATHs + # Disable this and go with the one we set. + filter_file( + r'rpath_cmd\(pyside_path, srcpath\)', + r'#rpath_cmd(pyside_path, srcpath)', + 'pyside_postinstall.py') + def install(self, spec, prefix): - qmake_path = '/usr/lib64/qt4/bin/qmake' - if not os.path.exists(qmake_path): - raise spack.package.InstallError("Failed to find qmake in %s" % qmake_path) - python('setup.py', 'install', '--prefix=%s' % prefix, '--qmake=%s' % qmake_path) + python('setup.py', 'install', + '--prefix=%s' % prefix, + '--jobs=%s' % make_jobs) + + diff --git a/var/spack/packages/py-shiboken/package.py b/var/spack/packages/py-shiboken/package.py new file mode 100644 index 00000000000..47abe64e651 --- /dev/null +++ b/var/spack/packages/py-shiboken/package.py @@ -0,0 +1,21 @@ +from spack import * + +class PyShiboken(Package): + """Shiboken generates bindings for C++ libraries using CPython source code.""" + homepage = "https://shiboken.readthedocs.org/" + url = "https://pypi.python.org/packages/source/S/Shiboken/Shiboken-1.2.2.tar.gz" + + version('1.2.2', '345cfebda221f525842e079a6141e555') + + # TODO: make build dependency + # depends_on("cmake") + + extends('python') + depends_on("py-setuptools") + depends_on("libxml2") + depends_on("qt@:4.8") + + def install(self, spec, prefix): + python('setup.py', 'install', + '--prefix=%s' % prefix, + '--jobs=%s' % make_jobs) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index eed81d095c9..4b3b31eb6bb 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,9 +1,10 @@ -from spack import * -import spack import os import re from contextlib import closing +from spack import * +import spack + class Python(Package): """The Python programming language.""" From 5c2608b032f04c063c437eca7d8360baaf6a2a16 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Mon, 9 Feb 2015 15:55:18 -0800 Subject: [PATCH 54/97] typo: Self -> self --- var/spack/packages/py-pyside/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py index 7528a0bf72b..1fd037d75f5 100644 --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -14,7 +14,7 @@ class PyPyside(Package): depends_on('qt@:4') - def patch(Self): + def patch(self): """Undo PySide RPATH handling and add Spack RPATH.""" # Add Spack's standard CMake args to the sub-builds. # They're called BY setup.py so we have to patch it. From 93067d0d63570425e473862de7ce72dc6091c34c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 01:44:38 -0800 Subject: [PATCH 55/97] Add profile option to spack script. --- bin/spack | 83 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/bin/spack b/bin/spack index 626d9d9d11f..c49caf37f9a 100755 --- a/bin/spack +++ b/bin/spack @@ -58,14 +58,16 @@ parser = argparse.ArgumentParser( description='Spack: the Supercomputing PACKage Manager.') parser.add_argument('-V', '--version', action='version', version="%s" % spack.spack_version) -parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', +parser.add_argument('-v', '--verbose', action='store_true', help="Print additional output during builds") -parser.add_argument('-d', '--debug', action='store_true', dest='debug', +parser.add_argument('-d', '--debug', action='store_true', help="Write out debug logs during compile") -parser.add_argument('-k', '--insecure', action='store_true', dest='insecure', +parser.add_argument('-k', '--insecure', action='store_true', help="Do not check ssl certificates when downloading archives.") -parser.add_argument('-m', '--mock', action='store_true', dest='mock', +parser.add_argument('-m', '--mock', action='store_true', help="Use mock packages instead of real ones.") +parser.add_argument('-p', '--profile', action='store_true', + help="Profile execution using cProfile.") # each command module implements a parser() function, to which we pass its # subparser for setup. @@ -85,42 +87,49 @@ if len(sys.argv) == 1: # actually parse the args. args = parser.parse_args() -# Set up environment based on args. -tty.set_verbose(args.verbose) -tty.set_debug(args.debug) -spack.debug = args.debug +def main(): + # Set up environment based on args. + tty.set_verbose(args.verbose) + tty.set_debug(args.debug) + spack.debug = args.debug -spack.spack_working_dir = working_dir -if args.mock: - from spack.packages import PackageDB - spack.db = PackageDB(spack.mock_packages_path) + spack.spack_working_dir = working_dir + if args.mock: + from spack.packages import PackageDB + spack.db = PackageDB(spack.mock_packages_path) -# If the user asked for it, don't check ssl certs. -if args.insecure: - tty.warn("You asked for --insecure, which does not check SSL certificates or checksums.") - spack.curl.add_default_arg('-k') + # If the user asked for it, don't check ssl certs. + if args.insecure: + tty.warn("You asked for --insecure, which does not check SSL certificates or checksums.") + spack.curl.add_default_arg('-k') -# Try to load the particular command asked for and run it -command = spack.cmd.get_command(args.command) -try: - return_val = command(parser, args) -except SpackError, e: - if spack.debug: - # In debug mode, raise with a full stack trace. - raise - elif e.long_message: - tty.die(e.message, e.long_message) + # Try to load the particular command asked for and run it + command = spack.cmd.get_command(args.command) + try: + return_val = command(parser, args) + except SpackError, e: + if spack.debug: + # In debug mode, raise with a full stack trace. + raise + elif e.long_message: + tty.die(e.message, e.long_message) + else: + tty.die(e.message) + + except KeyboardInterrupt: + sys.stderr.write('\n') + tty.die("Keyboard interrupt.") + + # Allow commands to return values if they want to exit with some ohter code. + if return_val is None: + sys.exit(0) + elif isinstance(return_val, int): + sys.exit(return_val) else: - tty.die(e.message) + tty.die("Bad return value from command %s: %s" % (args.command, return_val)) -except KeyboardInterrupt: - sys.stderr.write('\n') - tty.die("Keyboard interrupt.") - -# Allow commands to return values if they want to exit with some ohter code. -if return_val is None: - sys.exit(0) -elif isinstance(return_val, int): - sys.exit(return_val) +if args.profile: + import cProfile + cProfile.run('main()', sort='tottime') else: - tty.die("Bad return value from command %s: %s" % (args.command, return_val)) + main() From 82dc935a50874e899380f32a9a35b7cc4f76df87 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 01:49:50 -0800 Subject: [PATCH 56/97] installed_extensions_for no longer fails when nothing known about pkg --- lib/spack/spack/packages.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 7ef8135c1a9..3c81863c113 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -119,8 +119,15 @@ def extensions_for(self, extendee_spec): @_autospec def installed_extensions_for(self, extendee_spec): - return [s.package for s in self.installed_package_specs() - if s.package.extends(extendee_spec)] + for s in self.installed_package_specs(): + try: + if s.package.extends(extendee_spec): + yield s.package + except UnknownPackageError, e: + # Skip packages we know nothing about + continue + # TODO: add some conditional way to do this instead of + # catching exceptions. def dirname_for_package_name(self, pkg_name): From c0c08799249fb56c281f62b3659e7cf7d7080188 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 01:58:35 -0800 Subject: [PATCH 57/97] Better extension activation/deactivation --- lib/spack/spack/cmd/extensions.py | 4 +- lib/spack/spack/directory_layout.py | 154 +++++++++++++++++++-------- lib/spack/spack/package.py | 19 +++- var/spack/packages/python/package.py | 16 +-- 4 files changed, 135 insertions(+), 58 deletions(-) diff --git a/lib/spack/spack/cmd/extensions.py b/lib/spack/spack/cmd/extensions.py index ae73d8ac553..fc8e6842c3b 100644 --- a/lib/spack/spack/cmd/extensions.py +++ b/lib/spack/spack/cmd/extensions.py @@ -89,10 +89,10 @@ def extensions(parser, args): spack.cmd.find.display_specs(installed, mode=args.mode) # List specs of activated extensions. - activated = spack.install_layout.get_extensions(spec) + activated = spack.install_layout.extension_map(spec) print if not activated: tty.msg("None activated.") return tty.msg("%d currently activated:" % len(activated)) - spack.cmd.find.display_specs(activated, mode=args.mode) + spack.cmd.find.display_specs(activated.values(), mode=args.mode) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 37740720a20..562c0bd3eda 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -27,6 +27,7 @@ import exceptions import hashlib import shutil +import tempfile from contextlib import closing import llnl.util.tty as tty @@ -84,17 +85,38 @@ def make_path_for_spec(self, spec): raise NotImplementedError() - def get_extensions(self, spec): - """Get a set of currently installed extension packages for a spec.""" + def extension_map(self, spec): + """Get a dict of currently installed extension packages for a spec. + + Dict maps { name : extension_spec } + Modifying dict does not affect internals of this layout. + """ raise NotImplementedError() - def add_extension(self, spec, extension_spec): + def check_extension_conflict(self, spec, ext_spec): + """Ensure that ext_spec can be activated in spec. + + If not, raise ExtensionAlreadyInstalledError or + ExtensionConflictError. + """ + raise NotImplementedError() + + + def check_activated(self, spec, ext_spec): + """Ensure that ext_spec can be removed from spec. + + If not, raise NoSuchExtensionError. + """ + raise NotImplementedError() + + + def add_extension(self, spec, ext_spec): """Add to the list of currently installed extensions.""" raise NotImplementedError() - def remove_extension(self, spec, extension_spec): + def remove_extension(self, spec, ext_spec): """Remove from the list of currently installed extensions.""" raise NotImplementedError() @@ -173,6 +195,8 @@ def __init__(self, root, **kwargs): self.spec_file_name = spec_file_name self.extension_file_name = extension_file_name + # Cache of already written/read extension maps. + self._extension_maps = {} @property def hidden_file_paths(self): @@ -271,54 +295,94 @@ def extension_file_path(self, spec): return join_path(self.path_for_spec(spec), self.extension_file_name) - def get_extensions(self, spec): + def _extension_map(self, spec): + """Get a dict spec> for all extensions currnetly + installed for this package.""" _check_concrete(spec) - extensions = set() + if not spec in self._extension_maps: + path = self.extension_file_path(spec) + if not os.path.exists(path): + self._extension_maps[spec] = {} + + else: + exts = {} + with closing(open(path)) as ext_file: + for line in ext_file: + try: + spec = Spec(line.strip()) + exts[spec.name] = spec + except spack.error.SpackError, e: + # TODO: do something better here -- should be + # resilient to corrupt files. + raise InvalidExtensionSpecError(str(e)) + self._extension_maps[spec] = exts + + return self._extension_maps[spec] + + + def extension_map(self, spec): + """Defensive copying version of _extension_map() for external API.""" + return self._extension_map(spec).copy() + + + def check_extension_conflict(self, spec, ext_spec): + exts = self._extension_map(spec) + if ext_spec.name in exts: + installed_spec = exts[ext_spec.name] + if ext_spec == installed_spec: + raise ExtensionAlreadyInstalledError(spec, ext_spec) + else: + raise ExtensionConflictError(spec, ext_spec, installed_spec) + + + def check_activated(self, spec, ext_spec): + exts = self._extension_map(spec) + if (not ext_spec.name in exts) or (ext_spec != exts[ext_spec.name]): + raise NoSuchExtensionError(spec, ext_spec) + + + def _write_extensions(self, spec, extensions): path = self.extension_file_path(spec) - if os.path.exists(path): - with closing(open(path)) as ext_file: - for line in ext_file: - try: - extensions.add(Spec(line.strip())) - except spack.error.SpackError, e: - raise InvalidExtensionSpecError(str(e)) - return extensions + + # Create a temp file in the same directory as the actual file. + dirname, basename = os.path.split(path) + tmp = tempfile.NamedTemporaryFile( + prefix=basename, dir=dirname, delete=False) + + # Write temp file. + with closing(tmp): + for extension in sorted(extensions.values()): + tmp.write("%s\n" % extension) + + # Atomic update by moving tmpfile on top of old one. + os.rename(tmp.name, path) - def write_extensions(self, spec, extensions): - path = self.extension_file_path(spec) - with closing(open(path, 'w')) as spec_file: - for extension in sorted(extensions): - spec_file.write("%s\n" % extension) - - - def add_extension(self, spec, extension_spec): + def add_extension(self, spec, ext_spec): _check_concrete(spec) - _check_concrete(extension_spec) + _check_concrete(ext_spec) - exts = self.get_extensions(spec) - if extension_spec in exts: - raise ExtensionAlreadyInstalledError(spec, extension_spec) - else: - for already_installed in exts: - if spec.name == extension_spec.name: - raise ExtensionConflictError(spec, extension_spec, already_installed) + # Check whether it's already installed or if it's a conflict. + exts = self.extension_map(spec) + self.check_extension_conflict(spec, ext_spec) - exts.add(extension_spec) - self.write_extensions(spec, exts) + # do the actual adding. + exts[ext_spec.name] = ext_spec + self._write_extensions(spec, exts) - def remove_extension(self, spec, extension_spec): + def remove_extension(self, spec, ext_spec): _check_concrete(spec) - _check_concrete(extension_spec) + _check_concrete(ext_spec) - exts = self.get_extensions(spec) - if not extension_spec in exts: - raise NoSuchExtensionError(spec, extension_spec) + # Make sure it's installed before removing. + exts = self.extension_map(spec) + self.check_activated(spec, ext_spec) - exts.remove(extension_spec) - self.write_extensions(spec, exts) + # do the actual removing. + del exts[ext_spec.name] + self._write_extensions(spec, exts) class DirectoryLayoutError(SpackError): @@ -365,24 +429,24 @@ def __init__(self, message): class ExtensionAlreadyInstalledError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" - def __init__(self, spec, extension_spec): + def __init__(self, spec, ext_spec): super(ExtensionAlreadyInstalledError, self).__init__( - "%s is already installed in %s" % (extension_spec.short_spec, spec.short_spec)) + "%s is already installed in %s" % (ext_spec.short_spec, spec.short_spec)) class ExtensionConflictError(DirectoryLayoutError): """Raised when an extension is added to a package that already has it.""" - def __init__(self, spec, extension_spec, conflict): + def __init__(self, spec, ext_spec, conflict): super(ExtensionConflictError, self).__init__( "%s cannot be installed in %s because it conflicts with %s."% ( - extension_spec.short_spec, spec.short_spec, conflict.short_spec)) + ext_spec.short_spec, spec.short_spec, conflict.short_spec)) class NoSuchExtensionError(DirectoryLayoutError): """Raised when an extension isn't there on remove.""" - def __init__(self, spec, extension_spec): + def __init__(self, spec, ext_spec): super(NoSuchExtensionError, self).__init__( "%s cannot be removed from %s because it's not installed."% ( - extension_spec.short_spec, spec.short_spec)) + ext_spec.short_spec, spec.short_spec)) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b18d0549901..a624c1ebf50 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -534,7 +534,8 @@ def activated(self): if not self.is_extension: raise ValueError("is_extension called on package that is not an extension.") - return self.spec in spack.install_layout.get_extensions(self.extendee_spec) + exts = spack.install_layout.extension_map(self.extendee_spec) + return (self.name in exts) and (exts[self.name] == self.spec) def preorder_traversal(self, visited=None, **kwargs): @@ -987,6 +988,8 @@ def do_activate(self): activate() directly. """ self._sanity_check_extension() + spack.install_layout.check_extension_conflict(self.extendee_spec, self.spec) + self.extendee_spec.package.activate(self, **self.extendee_args) spack.install_layout.add_extension(self.extendee_spec, self.spec) @@ -1014,12 +1017,22 @@ def ignore(filename): tree.merge(self.prefix, ignore=ignore) - def do_deactivate(self): + def do_deactivate(self, **kwargs): """Called on the extension to invoke extendee's deactivate() method.""" + force = kwargs.get('force', False) + self._sanity_check_extension() + + # Allow a force deactivate to happen. This can unlink + # spurious files if something was corrupted. + if not force: + spack.install_layout.check_activated(self.extendee_spec, self.spec) + self.extendee_spec.package.deactivate(self, **self.extendee_args) - if self.spec in spack.install_layout.get_extensions(self.extendee_spec): + # redundant activation check -- makes SURE the spec is not + # still activated even if something was wrong above. + if self.activated: spack.install_layout.remove_extension(self.extendee_spec, self.spec) tty.msg("Deactivated extension %s for %s." diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 4b3b31eb6bb..de7f412b529 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -98,9 +98,9 @@ def ignore(filename): return ignore - def write_easy_install_pth(self, extensions): + def write_easy_install_pth(self, exts): paths = [] - for ext in extensions: + for ext in sorted(exts.values()): ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir) easy_pth = "%s/easy-install.pth" % ext_site_packages @@ -139,15 +139,15 @@ def activate(self, ext_pkg, **args): args.update(ignore=self.python_ignore(ext_pkg, args)) super(Python, self).activate(ext_pkg, **args) - extensions = set(spack.install_layout.get_extensions(self.spec)) - extensions.add(ext_pkg.spec) - self.write_easy_install_pth(extensions) + exts = spack.install_layout.extension_map(self.spec) + exts[ext_pkg.name] = ext_pkg.spec + self.write_easy_install_pth(exts) def deactivate(self, ext_pkg, **args): args.update(ignore=self.python_ignore(ext_pkg, args)) super(Python, self).deactivate(ext_pkg, **args) - extensions = set(spack.install_layout.get_extensions(self.spec)) - extensions.remove(ext_pkg.spec) - self.write_easy_install_pth(extensions) + exts = spack.install_layout.extension_map(self.spec) + del exts[ext_pkg.name] + self.write_easy_install_pth(exts) From 3c0048dd89e4d18ac95afd19b65d7ce54a48d862 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 01:59:36 -0800 Subject: [PATCH 58/97] py-sip installs properly into a prefix --- var/spack/packages/py-sip/package.py | 7 ++++++- var/spack/packages/python/package.py | 10 ++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/py-sip/package.py b/var/spack/packages/py-sip/package.py index 06aea35a745..6753bdd2a52 100644 --- a/var/spack/packages/py-sip/package.py +++ b/var/spack/packages/py-sip/package.py @@ -1,4 +1,5 @@ from spack import * +import os class PySip(Package): """SIP is a tool that makes it very easy to create Python bindings for C and C++ libraries.""" @@ -10,6 +11,10 @@ class PySip(Package): extends('python') def install(self, spec, prefix): - python('configure.py') + python('configure.py', + '--destdir=%s' % site_packages_dir, + '--bindir=%s' % spec.prefix.bin, + '--incdir=%s' % python_include_dir, + '--sipdir=%s' % os.path.join(spec.prefix.share, 'sip')) make() make('install') diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index de7f412b529..fb875a7eeb2 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -42,6 +42,11 @@ def python_lib_dir(self): return os.path.join('lib', 'python%d.%d' % self.version[:2]) + @property + def python_include_dir(self): + return os.path.join('include', 'python%d.%d' % self.version[:2]) + + @property def site_packages_dir(self): return os.path.join(self.python_lib_dir, 'site-packages') @@ -58,8 +63,9 @@ def setup_dependent_environment(self, module, spec, ext_spec): module.python = Executable(join_path(spec.prefix.bin, 'python')) # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. - module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) - module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) + module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) + module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) + module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) # Make the site packages directory if it does not exist already. mkdirp(module.site_packages_dir) From d1e03329c5c16ba38b082c3473e7c6970f168990 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 11:50:13 -0800 Subject: [PATCH 59/97] Memoize all_specs() and exists() for better performance. - Real bottleneck is calling normalize() for every spec when we read it. - Need to store graph information in spec files to avoid the need for this. - Also, normalizing old specs isn't always possible, so we need to do this anyway. --- lib/spack/spack/directory_layout.py | 11 +++++++++-- lib/spack/spack/packages.py | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 562c0bd3eda..5b80e93d6b8 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -31,6 +31,7 @@ from contextlib import closing import llnl.util.tty as tty +from llnl.util.lang import memoized from llnl.util.filesystem import join_path, mkdirp import spack @@ -223,6 +224,9 @@ def read_spec(self, path): if all(spack.db.exists(s.name) for s in spec.traverse()): copy = spec.copy() + + # TODO: It takes a lot of time to normalize every spec on read. + # TODO: Storing graph info with spec files would fix this. copy.normalize() if copy.concrete: return copy # These are specs spack still understands. @@ -276,17 +280,20 @@ def make_path_for_spec(self, spec): self.write_spec(spec, spec_file_path) + @memoized def all_specs(self): if not os.path.isdir(self.root): - return + return [] + specs = [] for path in traverse_dirs_at_depth(self.root, 3): arch, compiler, last_dir = path spec_file_path = join_path( self.root, arch, compiler, last_dir, self.spec_file_name) if os.path.exists(spec_file_path): spec = self.read_spec(spec_file_path) - yield spec + specs.append(spec) + return specs def extension_file_path(self, spec): diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 3c81863c113..43c4c191c1f 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -192,6 +192,7 @@ def all_packages(self): yield self.get(name) + @memoized def exists(self, pkg_name): """Whether a package with the supplied name exists .""" return os.path.exists(self.filename_for_package_name(pkg_name)) From 2f67cdaf10a3e64474f7ac242518fdfe93e9c87a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 12:39:10 -0800 Subject: [PATCH 60/97] Better time output on build completion. --- lib/spack/spack/package.py | 16 ++++++++++++++-- .../packages/{py-pyqt4 => py-pyqt}/package.py | 13 ++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) rename var/spack/packages/{py-pyqt4 => py-pyqt}/package.py (55%) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index a624c1ebf50..c48816cb5b6 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -848,8 +848,8 @@ def do_install(self, **kwargs): build_time = self._total_time - self._fetch_time tty.msg("Successfully installed %s." % self.name, - "Fetch: %.2f sec. Build: %.2f sec. Total: %.2f sec." - % (self._fetch_time, build_time, self._total_time)) + "Fetch: %s. Build: %s. Total: %s." + % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) print_pkg(self.prefix) # Use os._exit here to avoid raising a SystemExit exception, @@ -1201,6 +1201,18 @@ def print_pkg(message): print message +def _hms(seconds): + """Convert time in seconds to hours, minutes, seconds.""" + m, s = divmod(seconds, 60) + h, m = divmod(m, 60) + + parts = [] + if h: parts.append("%dh" % h) + if m: parts.append("%dm" % m) + if s: parts.append("%.2fs" % s) + return ' '.join(parts) + + class FetchError(spack.error.SpackError): """Raised when something goes wrong during fetch.""" def __init__(self, message, long_msg=None): diff --git a/var/spack/packages/py-pyqt4/package.py b/var/spack/packages/py-pyqt/package.py similarity index 55% rename from var/spack/packages/py-pyqt4/package.py rename to var/spack/packages/py-pyqt/package.py index eeb13825605..cb40af351aa 100644 --- a/var/spack/packages/py-pyqt4/package.py +++ b/var/spack/packages/py-pyqt/package.py @@ -1,18 +1,21 @@ from spack import * -class PyPyqt4(Package): - """PyQt is a set of Python v2 and v3 bindings for Digia's Qt application framework and runs on all platforms supported by Qt including Windows, MacOS/X and Linux.""" +class PyPyqt(Package): + """PyQt is a set of Python v2 and v3 bindings for Digia's Qt + application framework and runs on all platforms supported by Qt + including Windows, MacOS/X and Linux.""" homepage = "http://www.riverbankcomputing.com/software/pyqt/intro" url = "http://sourceforge.net/projects/pyqt/files/PyQt4/PyQt-4.11.3/PyQt-x11-gpl-4.11.3.tar.gz" version('4.11.3', '997c3e443165a89a559e0d96b061bf70') extends('python') - depends_on('qt') + depends_on('qt@4') # TODO: allow qt5 when conditional deps are supported. depends_on('py-sip') def install(self, spec, prefix): - version_array = str(spec['python'].version).split('.') - python('configure.py', '--confirm-license', '--destdir=%s/python%s.%s/site-packages' %(self.prefix.lib, version_array[0], version_array[1])) + python('configure.py', + '--confirm-license', + '--destdir=%s' % site_packages_dir) make() make('install') From b11061f99d9e501f69d6d39bae56f0b69d17eaa1 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 12:40:02 -0800 Subject: [PATCH 61/97] Rename py-pyqt4 to py-pyqt. --- var/spack/packages/py-pyqt/package.py | 5 ++++- var/spack/packages/qt/package.py | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/py-pyqt/package.py b/var/spack/packages/py-pyqt/package.py index cb40af351aa..8edca105bb6 100644 --- a/var/spack/packages/py-pyqt/package.py +++ b/var/spack/packages/py-pyqt/package.py @@ -10,9 +10,12 @@ class PyPyqt(Package): version('4.11.3', '997c3e443165a89a559e0d96b061bf70') extends('python') - depends_on('qt@4') # TODO: allow qt5 when conditional deps are supported. depends_on('py-sip') + # TODO: allow qt5 when conditional deps are supported. + # TODO: Fix version matching so that @4 works like @:4 + depends_on('qt@:4') + def install(self, spec, prefix): python('configure.py', '--confirm-license', diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py index fcbcd2491a0..3b5096c4f0c 100644 --- a/var/spack/packages/qt/package.py +++ b/var/spack/packages/qt/package.py @@ -43,9 +43,9 @@ def install(self, spec, prefix): '-fast', '-optimized-qmake', '-no-pch', -# phonon required for py-pyqt4 -# '-no-phonon', -# '-no-phonon-backend', + # phonon required for py-pyqt + # '-no-phonon', + # '-no-phonon-backend', '-no-openvg') make() make("install") From ce011501f9b0184888ba2b6648fcf6d360f7c404 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 23:02:21 -0800 Subject: [PATCH 62/97] Add R package. --- var/spack/packages/R/package.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 var/spack/packages/R/package.py diff --git a/var/spack/packages/R/package.py b/var/spack/packages/R/package.py new file mode 100644 index 00000000000..2e6f65a7429 --- /dev/null +++ b/var/spack/packages/R/package.py @@ -0,0 +1,33 @@ +from spack import * + +class R(Package): + """R is 'GNU S', a freely available language and environment for + statistical computing and graphics which provides a wide va + riety of statistical and graphical techniques: linear and + nonlinear modelling, statistical tests, time series analysis, + classification, clustering, etc. Please consult the R project + homepage for further information.""" + homepage = "http://www.example.com" + url = "http://cran.cnr.berkeley.edu/src/base/R-3/R-3.1.2.tar.gz" + + version('3.1.2', '3af29ec06704cbd08d4ba8d69250ae74') + + depends_on("readline") + depends_on("ncurses") + depends_on("icu") + depends_on("glib") + depends_on("zlib") + depends_on("libtiff") + depends_on("jpeg") + depends_on("cairo") + depends_on("pango") + depends_on("freetype") + depends_on("tcl") + depends_on("tk") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix, + "--enable-R-shlib", + "--enable-BLAS-shlib") + make() + make("install") From 36579844d9f105b5c9182beed83a65bf0bb556a9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 23:02:36 -0800 Subject: [PATCH 63/97] Add Tcl/Tk packages. --- var/spack/packages/tcl/package.py | 22 ++++++++++++++++++++++ var/spack/packages/tk/package.py | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 var/spack/packages/tcl/package.py create mode 100644 var/spack/packages/tk/package.py diff --git a/var/spack/packages/tcl/package.py b/var/spack/packages/tcl/package.py new file mode 100644 index 00000000000..529adf77883 --- /dev/null +++ b/var/spack/packages/tcl/package.py @@ -0,0 +1,22 @@ +from spack import * + +class Tcl(Package): + """Tcl (Tool Command Language) is a very powerful but easy to + learn dynamic programming language, suitable for a very wide + range of uses, including web and desktop applications, + networking, administration, testing and many more. Open source + and business-friendly, Tcl is a mature yet evolving language + that is truly cross platform, easily deployed and highly + extensible.""" + homepage = "http://www.tcl.tk" + + version('8.6.3', 'db382feca91754b7f93da16dc4cdad1f', + url="http://prdownloads.sourceforge.net/tcl/tcl8.6.3-src.tar.gz") + + depends_on('zlib') + + def install(self, spec, prefix): + with working_dir('unix'): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/packages/tk/package.py b/var/spack/packages/tk/package.py new file mode 100644 index 00000000000..96736f6f95d --- /dev/null +++ b/var/spack/packages/tk/package.py @@ -0,0 +1,22 @@ +from spack import * + +class Tk(Package): + """Tk is a graphical user interface toolkit that takes developing + desktop applications to a higher level than conventional + approaches. Tk is the standard GUI not only for Tcl, but for + many other dynamic languages, and can produce rich, native + applications that run unchanged across Windows, Mac OS X, Linux + and more.""" + homepage = "http://www.tcl.tk" + url = "http://prdownloads.sourceforge.net/tcl/tk8.6.3-src.tar.gz" + + version('src', '85ca4dbf4dcc19777fd456f6ee5d0221') + + depends_on("tcl") + + def install(self, spec, prefix): + with working_dir('unix'): + configure("--prefix=%s" % prefix, + "--with-tcl=%s" % spec['tcl'].prefix.lib) + make() + make("install") From 65d60f92f5c479bdcf8b4cbfa244135a131d08f9 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 23:02:51 -0800 Subject: [PATCH 64/97] qhull package. --- var/spack/packages/qhull/package.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 var/spack/packages/qhull/package.py diff --git a/var/spack/packages/qhull/package.py b/var/spack/packages/qhull/package.py new file mode 100644 index 00000000000..9da4078a702 --- /dev/null +++ b/var/spack/packages/qhull/package.py @@ -0,0 +1,27 @@ +from spack import * + +class Qhull(Package): + """Qhull computes the convex hull, Delaunay triangulation, Voronoi + diagram, halfspace intersection about a point, furt hest-site + Delaunay triangulation, and furthest-site Voronoi diagram. The + source code runs in 2-d, 3-d, 4-d, and higher dimensions. Qhull + implements the Quickhull algorithm for computing the convex + hull. It handles roundoff errors from floating point + arithmetic. It computes volumes, surface areas, and + approximations to the convex hull. + + Qhull does not support triangulation of non-convex surfaces, + mesh generation of non-convex objects, medium-sized inputs in + 9-D and higher, alpha shapes, weighted Voronoi diagrams, + Voronoi volumes, or constrained Delaunay triangulations.""" + + homepage = "http://www.qhull.org" + + version('1.0', 'd0f978c0d8dfb2e919caefa56ea2953c', + url="http://www.qhull.org/download/qhull-2012.1-src.tgz") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") From b86eb695523c235371169d0139486bcae15a7948 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 23:03:33 -0800 Subject: [PATCH 65/97] libgcrypt and libgpg-error packages. --- var/spack/packages/libgcrypt/package.py | 19 +++++++++++++++++++ var/spack/packages/libgpg-error/package.py | 17 +++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 var/spack/packages/libgcrypt/package.py create mode 100644 var/spack/packages/libgpg-error/package.py diff --git a/var/spack/packages/libgcrypt/package.py b/var/spack/packages/libgcrypt/package.py new file mode 100644 index 00000000000..1d0a57f3176 --- /dev/null +++ b/var/spack/packages/libgcrypt/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Libgcrypt(Package): + """Libgcrypt is a general purpose cryptographic library based on + the code from GnuPG. It provides functions for all cryptographic + building blocks: symmetric ciphers, hash algorithms, MACs, public + key algorithms, large integer functions, random numbers and a lot + of supporting functions. """ + homepage = "http://www.gnu.org/software/libgcrypt/" + url = "ftp://ftp.gnupg.org/gcrypt/libgcrypt/libgcrypt-1.6.2.tar.bz2" + + version('1.6.2', 'b54395a93cb1e57619943c082da09d5f') + + depends_on("libgpg-error") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/packages/libgpg-error/package.py b/var/spack/packages/libgpg-error/package.py new file mode 100644 index 00000000000..6c1d1a10a7e --- /dev/null +++ b/var/spack/packages/libgpg-error/package.py @@ -0,0 +1,17 @@ +from spack import * + +class LibgpgError(Package): + """Libgpg-error is a small library that defines common error + values for all GnuPG components. Among these are GPG, GPGSM, + GPGME, GPG-Agent, libgcrypt, Libksba, DirMngr, Pinentry, + SmartCard Daemon and possibly more in the future. """ + + homepage = "https://www.gnupg.org/related_software/libgpg-error" + url = "ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.18.tar.bz2" + + version('1.18', '12312802d2065774b787cbfc22cc04e9') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") From 847ed8ad399973477a7889b6367911f10c56f6bf Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 23:04:04 -0800 Subject: [PATCH 66/97] Add libxslt, cleanup libxml2. --- var/spack/packages/libxml2/package.py | 3 +++ var/spack/packages/libxslt/package.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 var/spack/packages/libxslt/package.py diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py index 5eaed36d948..72199d8def8 100644 --- a/var/spack/packages/libxml2/package.py +++ b/var/spack/packages/libxml2/package.py @@ -9,6 +9,9 @@ class Libxml2(Package): version('2.9.2', '9e6a9aca9d155737868b3dc5fd82f788') + depends_on('zlib') + depends_on('xz') + def install(self, spec, prefix): configure("--prefix=%s" % prefix, "--without-python") diff --git a/var/spack/packages/libxslt/package.py b/var/spack/packages/libxslt/package.py new file mode 100644 index 00000000000..f97332d0203 --- /dev/null +++ b/var/spack/packages/libxslt/package.py @@ -0,0 +1,24 @@ +from spack import * + +class Libxslt(Package): + """Libxslt is the XSLT C library developed for the GNOME + project. XSLT itself is a an XML language to define + transformation for XML. Libxslt is based on libxml2 the XML C + library developed for the GNOME project. It also implements + most of the EXSLT set of processor-portable extensions + functions and some of Saxon's evaluate and expressions + extensions.""" + homepage = "http://www.xmlsoft.org/XSLT/index.html" + url = "http://xmlsoft.org/sources/libxslt-1.1.28.tar.gz" + + version('1.1.28', '9667bf6f9310b957254fdcf6596600b7') + + depends_on("libxml2") + depends_on("xz") + depends_on("zlib") + depends_on("libgcrypt") + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") From 8aa3afcfde9cebd34fdb534141c258c214ae4132 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 23:04:20 -0800 Subject: [PATCH 67/97] Python package cleanup. - Added a number of dependencies to python packages. - Python packages may still not build without some OS support. - Example: Numpy needs ATLAS, and will use a system ATLAS install. - Atlas requires turning off CPU throttling to build. - can't do this as a regular user -- how to build ATLAS with Spack - currnetly relying on a system ATLAS install. --- var/spack/packages/py-libxml2/package.py | 1 + var/spack/packages/py-matplotlib/package.py | 6 +++++- var/spack/packages/py-pyside/package.py | 1 - var/spack/packages/py-rpy2/package.py | 2 ++ .../packages/py-scientificpython/package.py | 6 +++++- var/spack/packages/py-shiboken/package.py | 19 +++++++++++++++++++ 6 files changed, 32 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/py-libxml2/package.py b/var/spack/packages/py-libxml2/package.py index e645acb5dd8..59005428e4d 100644 --- a/var/spack/packages/py-libxml2/package.py +++ b/var/spack/packages/py-libxml2/package.py @@ -9,6 +9,7 @@ class PyLibxml2(Package): extends('python') depends_on('libxml2') + depends_on('libxslt') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index 8b8684c5632..f6b9c587fd3 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -17,8 +17,12 @@ class PyMatplotlib(Package): depends_on('py-pytz') depends_on('py-nose') depends_on('py-numpy') - depends_on('qt') + depends_on('qt') + depends_on('bzip2') + depends_on('tcl') + depends_on('tk') + depends_on('qhull') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py index 1fd037d75f5..c165d9b3bf2 100644 --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -13,7 +13,6 @@ class PyPyside(Package): depends_on('py-setuptools') depends_on('qt@:4') - def patch(self): """Undo PySide RPATH handling and add Spack RPATH.""" # Add Spack's standard CMake args to the sub-builds. diff --git a/var/spack/packages/py-rpy2/package.py b/var/spack/packages/py-rpy2/package.py index 3817059911d..dd0c0672afe 100644 --- a/var/spack/packages/py-rpy2/package.py +++ b/var/spack/packages/py-rpy2/package.py @@ -10,5 +10,7 @@ class PyRpy2(Package): extends('python') depends_on('py-setuptools') + depends_on('R') + def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scientificpython/package.py b/var/spack/packages/py-scientificpython/package.py index 73600e6cb97..020d8307032 100644 --- a/var/spack/packages/py-scientificpython/package.py +++ b/var/spack/packages/py-scientificpython/package.py @@ -1,7 +1,11 @@ from spack import * class PyScientificpython(Package): - """ScientificPython is a collection of Python modules for scientific computing. It contains support for geometry, mathematical functions, statistics, physical units, IO, visualization, and parallelization.""" + """ScientificPython is a collection of Python modules for + scientific computing. It contains support for geometry, + mathematical functions, statistics, physical units, IO, + visualization, and parallelization.""" + homepage = "https://sourcesup.renater.fr/projects/scientific-py/" url = "https://sourcesup.renater.fr/frs/download.php/4411/ScientificPython-2.8.1.tar.gz" diff --git a/var/spack/packages/py-shiboken/package.py b/var/spack/packages/py-shiboken/package.py index 47abe64e651..e9009479395 100644 --- a/var/spack/packages/py-shiboken/package.py +++ b/var/spack/packages/py-shiboken/package.py @@ -15,6 +15,25 @@ class PyShiboken(Package): depends_on("libxml2") depends_on("qt@:4.8") + def patch(self): + """Undo Shiboken RPATH handling and add Spack RPATH.""" + # Add Spack's standard CMake args to the sub-builds. + # They're called BY setup.py so we have to patch it. + filter_file( + r'OPTION_CMAKE,', + r'OPTION_CMAKE, ' + ( + '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' + '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(self.rpath)), + 'setup.py') + + # Shiboken tries to patch ELF files to remove RPATHs + # Disable this and go with the one we set. + filter_file( + r'^\s*rpath_cmd\(shiboken_path, srcpath\)', + r'#rpath_cmd(shiboken_path, srcpath)', + 'shiboken_postinstall.py') + + def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix, From 614c22fc1b1ac10c85ed9e27a1e59eeb88de0898 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 16 Feb 2015 12:41:22 -0800 Subject: [PATCH 68/97] Allow forced deactivation -- best effort unlinking spack deactivate -f will unlink even if Spack thinks the package isn't enabled. Made deactivate routines idempotent. --- lib/spack/llnl/util/link_tree.py | 4 ++++ lib/spack/spack/cmd/deactivate.py | 7 +++++-- lib/spack/spack/cmd/find.py | 3 +++ var/spack/packages/python/package.py | 5 +++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/spack/llnl/util/link_tree.py b/lib/spack/llnl/util/link_tree.py index 4e4e48316e8..4d778eca1e3 100644 --- a/lib/spack/llnl/util/link_tree.py +++ b/lib/spack/llnl/util/link_tree.py @@ -175,6 +175,10 @@ def unmerge(self, dest_root, **kwargs): kwargs['order'] = 'post' for src, dest in traverse_tree(self._root, dest_root, **kwargs): if os.path.isdir(src): + # Skip non-existing links. + if not os.path.exists(dest): + continue + if not os.path.isdir(dest): raise ValueError("File blocks directory: %s" % dest) diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index fd13f051dfb..f37dfd79ed8 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -30,6 +30,9 @@ description = "Deactivate a package extension." def setup_parser(subparser): + subparser.add_argument( + '-f', '--force', action='store_true', + help="Run deactivation even if spec is NOT currently activated.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="spec of package extension to deactivate.") @@ -44,7 +47,7 @@ def deactivate(parser, args): spack.db.get(specs[0]) spec = spack.cmd.disambiguate_spec(specs[0]) - if not spec.package.activated: + if not args.force and not spec.package.activated: tty.die("Package %s is not activated." % specs[0].short_spec) - spec.package.do_deactivate() + spec.package.do_deactivate(force=args.force) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index f6f503afe5d..dee1dfece71 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -122,5 +122,8 @@ def find(parser, args): if not args.mode: args.mode = 'short' + + if sys.stdout.isatty(): + tty.msg("%d installed packages." % len(specs)) display_specs(specs, mode=args.mode) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index fb875a7eeb2..705d002e80f 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -155,5 +155,6 @@ def deactivate(self, ext_pkg, **args): super(Python, self).deactivate(ext_pkg, **args) exts = spack.install_layout.extension_map(self.spec) - del exts[ext_pkg.name] - self.write_easy_install_pth(exts) + if ext_pkg.name in exts: # Make deactivate idempotent. + del exts[ext_pkg.name] + self.write_easy_install_pth(exts) From e6b2c2701184dd9128b729e11fc1f142272caa1d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 16 Feb 2015 20:38:22 -0800 Subject: [PATCH 69/97] Factor out forking logic to build_environment.py. --- lib/spack/spack/build_environment.py | 56 ++++++++++++++++++++++++++++ lib/spack/spack/package.py | 39 ++++--------------- 2 files changed, 63 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index cabde7dc86e..84d2bd77ef3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -28,6 +28,7 @@ calls you can make from within the install() function. """ import os +import sys import shutil import multiprocessing import platform @@ -212,3 +213,58 @@ def setup_package(pkg): for dep_spec in pkg.spec.traverse(root=False): dep_spec.package.setup_dependent_environment( pkg.module, dep_spec, pkg.spec) + + +def fork(pkg, function): + """Fork a child process to do part of a spack build. + + Arguments: + + pkg -- pkg whose environemnt we should set up the + forked process for. + function -- arg-less function to run in the child process. + + Usage: + def child_fun(): + # do stuff + build_env.fork(pkg, child_fun) + + Forked processes are run with the build environemnt set up by + spack.build_environment. This allows package authors to have + full control over the environment, etc. without offecting + other builds that might be executed in the same spack call. + + If something goes wrong, the child process is expected toprint + the error and the parent process will exit with error as + well. If things go well, the child exits and the parent + carries on. + """ + try: + pid = os.fork() + except OSError, e: + raise InstallError("Unable to fork build process: %s" % e) + + if pid == 0: + # Give the child process the package's build environemnt. + setup_package(pkg) + + try: + # call the forked function. + function() + + # Use os._exit here to avoid raising a SystemExit exception, + # which interferes with unit tests. + os._exit(0) + except: + # Child doesn't raise or return to main spack code. + # Just runs default exception handler and exits. + sys.excepthook(*sys.exc_info()) + os._exit(1) + + else: + # Parent process just waits for the child to complete. If the + # child exited badly, assume it already printed an appropriate + # message. Just make the parent exit with an error code. + pid, returncode = os.waitpid(pid, 0) + if returncode != 0: + sys.exit(1) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index c48816cb5b6..5d04fed8ffd 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -804,32 +804,21 @@ def do_install(self, **kwargs): if not fake_install: self.do_patch() - # Fork a child process to do the build. This allows each - # package authors to have full control over their environment, - # etc. without offecting other builds that might be executed - # in the same spack call. - try: - pid = os.fork() - except OSError, e: - raise InstallError("Unable to fork build process: %s" % e) + # create the install directory. The install layout + # handles this in case so that it can use whatever + # package naming scheme it likes. + spack.install_layout.make_path_for_spec(self.spec) - if pid == 0: + def real_work(): try: tty.msg("Building %s." % self.name) - # create the install directory. The install layout - # handles this in case so that it can use whatever - # package naming scheme it likes. - spack.install_layout.make_path_for_spec(self.spec) - # Run the pre-install hook in the child process after # the directory is created. spack.hooks.pre_install(self) # Set up process's build environment before running install. self.stage.chdir_to_source() - build_env.setup_package(self) - if fake_install: self.do_fake_install() else: @@ -852,10 +841,6 @@ def do_install(self, **kwargs): % (_hms(self._fetch_time), _hms(build_time), _hms(self._total_time))) print_pkg(self.prefix) - # Use os._exit here to avoid raising a SystemExit exception, - # which interferes with unit tests. - os._exit(0) - except: if not keep_prefix: # If anything goes wrong, remove the install prefix @@ -865,24 +850,14 @@ def do_install(self, **kwargs): "Spack will think this package is installed." + "Manually remove this directory to fix:", self.prefix) + raise - # Child doesn't raise or return to main spack code. - # Just runs default exception handler and exits. - sys.excepthook(*sys.exc_info()) - os._exit(1) - - # Parent process just waits for the child to complete. If the - # child exited badly, assume it already printed an appropriate - # message. Just make the parent exit with an error code. - pid, returncode = os.waitpid(pid, 0) - if returncode != 0: - sys.exit(1) + build_env.fork(self, real_work) # Once everything else is done, run post install hooks spack.hooks.post_install(self) - def _sanity_check_install(self): installed = set(os.listdir(self.prefix)) installed.difference_update(spack.install_layout.hidden_file_paths) From 13376efafc42c6eeb1bf0ad3b35f509924f1a6df Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 16 Feb 2015 21:53:34 -0800 Subject: [PATCH 70/97] Add package-specific rpath back to shiboken and pyside. --- var/spack/packages/py-pyside/package.py | 13 ++++++++++--- var/spack/packages/py-shiboken/package.py | 7 ++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py index c165d9b3bf2..6583431124c 100644 --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -1,5 +1,4 @@ from spack import * -import spack.package import os class PyPyside(Package): @@ -9,25 +8,33 @@ class PyPyside(Package): version('1.2.2', 'c45bc400c8a86d6b35f34c29e379e44d') + # TODO: make build dependency + # depends_on("cmake") + extends('python') depends_on('py-setuptools') depends_on('qt@:4') def patch(self): """Undo PySide RPATH handling and add Spack RPATH.""" + # Figure out the special RPATH + pypkg = self.spec['python'].package + rpath = self.rpath + rpath.append(os.path.join(self.prefix, pypkg.site_packages_dir, 'PySide')) + # Add Spack's standard CMake args to the sub-builds. # They're called BY setup.py so we have to patch it. filter_file( r'OPTION_CMAKE,', r'OPTION_CMAKE, ' + ( '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' - '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(self.rpath)), + '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(rpath)), 'setup.py') # PySide tries to patch ELF files to remove RPATHs # Disable this and go with the one we set. filter_file( - r'rpath_cmd\(pyside_path, srcpath\)', + r'^\s*rpath_cmd\(pyside_path, srcpath\)', r'#rpath_cmd(pyside_path, srcpath)', 'pyside_postinstall.py') diff --git a/var/spack/packages/py-shiboken/package.py b/var/spack/packages/py-shiboken/package.py index e9009479395..e4bf4ce07e6 100644 --- a/var/spack/packages/py-shiboken/package.py +++ b/var/spack/packages/py-shiboken/package.py @@ -1,4 +1,5 @@ from spack import * +import os class PyShiboken(Package): """Shiboken generates bindings for C++ libraries using CPython source code.""" @@ -19,11 +20,15 @@ def patch(self): """Undo Shiboken RPATH handling and add Spack RPATH.""" # Add Spack's standard CMake args to the sub-builds. # They're called BY setup.py so we have to patch it. + pypkg = self.spec['python'].package + rpath = self.rpath + rpath.append(os.path.join(self.prefix, pypkg.site_packages_dir, 'Shiboken')) + filter_file( r'OPTION_CMAKE,', r'OPTION_CMAKE, ' + ( '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' - '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(self.rpath)), + '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(rpath)), 'setup.py') # Shiboken tries to patch ELF files to remove RPATHs From 06d6b0b205095f849ace216aa51393b907cf821b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 16 Feb 2015 21:53:55 -0800 Subject: [PATCH 71/97] More py-setuptools dependencies added. --- var/spack/packages/py-ipython/package.py | 1 + var/spack/packages/py-mpi4py/package.py | 1 + var/spack/packages/py-nose/package.py | 1 + var/spack/packages/py-pylint/package.py | 4 ++-- var/spack/packages/py-virtualenv/package.py | 1 + 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py index 731e661dfda..907ea9edcdb 100644 --- a/var/spack/packages/py-ipython/package.py +++ b/var/spack/packages/py-ipython/package.py @@ -9,6 +9,7 @@ class PyIpython(Package): extends('python') depends_on('py-pygments') + depends_on('py-setuptools') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-mpi4py/package.py b/var/spack/packages/py-mpi4py/package.py index fdea340dc21..8001689a18d 100644 --- a/var/spack/packages/py-mpi4py/package.py +++ b/var/spack/packages/py-mpi4py/package.py @@ -7,6 +7,7 @@ class PyMpi4py(Package): version('1.3.1', 'dbe9d22bdc8ed965c23a7ceb6f32fc3c') extends('python') + depends_on('py-setuptools') depends_on('mpi') def install(self, spec, prefix): diff --git a/var/spack/packages/py-nose/package.py b/var/spack/packages/py-nose/package.py index 6df84e831d1..b902a35fbb0 100644 --- a/var/spack/packages/py-nose/package.py +++ b/var/spack/packages/py-nose/package.py @@ -10,6 +10,7 @@ class PyNose(Package): version('1.3.4', '6ed7169887580ddc9a8e16048d38274d') extends('python') + depends_on('py-setuptools') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pylint/package.py b/var/spack/packages/py-pylint/package.py index ebde861f947..7a6ee7dbbce 100644 --- a/var/spack/packages/py-pylint/package.py +++ b/var/spack/packages/py-pylint/package.py @@ -8,9 +8,9 @@ class PyPylint(Package): version('1.4.1', 'df7c679bdcce5019389038847e4de622') -# extends('python') - extends('python', ignore=lambda f:re.match(r"site.py*", f)) + extends('python') depends_on('py-nose') + depends_on('py-setuptools') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-virtualenv/package.py b/var/spack/packages/py-virtualenv/package.py index c1b359e1649..9d94c2dcda0 100644 --- a/var/spack/packages/py-virtualenv/package.py +++ b/var/spack/packages/py-virtualenv/package.py @@ -9,6 +9,7 @@ class PyVirtualenv(Package): version('1.11.6', 'f61cdd983d2c4e6aeabb70b1060d6f49') extends('python') + depends_on('py-setuptools') def clean(self): if os.path.exists('build'): From 67db8ddca8ac7ab9adeb827a7dadd34a385b2b6b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 17 Feb 2015 00:21:15 -0800 Subject: [PATCH 72/97] Factor ignore logic into a predicate builder. --- lib/spack/llnl/util/lang.py | 31 ++++++++++++++++++++++++++++ var/spack/packages/python/package.py | 25 ++++++++++------------ 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/lib/spack/llnl/util/lang.py b/lib/spack/llnl/util/lang.py index db15da05064..332367f5373 100644 --- a/lib/spack/llnl/util/lang.py +++ b/lib/spack/llnl/util/lang.py @@ -291,6 +291,37 @@ def foo(self, **kwargs): % (next(kwargs.iterkeys()), fun.__name__)) +def match_predicate(*args): + """Utility function for making string matching predicates. + + Each arg can be a: + - regex + - list or tuple of regexes + - predicate that takes a string. + + This returns a predicate that is true if: + - any arg regex matches + - any regex in a list or tuple of regexes matches. + - any predicate in args matches. + """ + def match(string): + for arg in args: + if isinstance(arg, basestring): + if re.search(arg, string): + return True + elif isinstance(arg, list) or isinstance(arg, tuple): + if any(re.search(i, string) for i in arg): + return True + elif callable(arg): + if arg(string): + return True + else: + raise ValueError("args to match_predicate must be regex, " + "list of regexes, or callable.") + return False + return match + + class RequiredAttributeError(ValueError): def __init__(self, message): super(RequiredAttributeError, self).__init__(message) diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py index 705d002e80f..31a12ea653d 100644 --- a/var/spack/packages/python/package.py +++ b/var/spack/packages/python/package.py @@ -1,6 +1,7 @@ import os import re from contextlib import closing +from llnl.util.lang import match_predicate from spack import * import spack @@ -85,23 +86,19 @@ def setup_dependent_environment(self, module, spec, ext_spec): def python_ignore(self, ext_pkg, args): """Add some ignore files to activate/deactivate args.""" - orig_ignore = args.get('ignore', lambda f: False) + ignore_arg = args.get('ignore', lambda f: False) - def ignore(filename): - # Always ignore easy-install.pth, as it needs to be merged. - patterns = [r'easy-install\.pth$'] + # Always ignore easy-install.pth, as it needs to be merged. + patterns = [r'easy-install\.pth$'] - # Ignore pieces of setuptools installed by other packages. - if ext_pkg.name != 'py-setuptools': - patterns.append(r'/site\.pyc?$') - patterns.append(r'setuptools\.pth') - patterns.append(r'bin/easy_install[^/]*$') - patterns.append(r'setuptools.*egg$') + # Ignore pieces of setuptools installed by other packages. + if ext_pkg.name != 'py-setuptools': + patterns.append(r'/site\.pyc?$') + patterns.append(r'setuptools\.pth') + patterns.append(r'bin/easy_install[^/]*$') + patterns.append(r'setuptools.*egg$') - return (any(re.search(p, filename) for p in patterns) or - orig_ignore(filename)) - - return ignore + return match_predicate(ignore_arg, patterns) def write_easy_install_pth(self, exts): From 57f331e2acf75c5bf4c464d0df888fd882295a68 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 17 Feb 2015 00:22:18 -0800 Subject: [PATCH 73/97] Ignore conflicting nose tests in py-nose and py-matplotlib. --- var/spack/packages/py-matplotlib/package.py | 3 ++- var/spack/packages/py-nose/package.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index f6b9c587fd3..04037f004e7 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -8,7 +8,8 @@ class PyMatplotlib(Package): version('1.4.2', '7d22efb6cce475025733c50487bd8898') - extends('python') + extends('python', ignore=r'bin/nosetests.*$') + depends_on('py-pyside') depends_on('py-ipython') depends_on('py-pyparsing') diff --git a/var/spack/packages/py-nose/package.py b/var/spack/packages/py-nose/package.py index b902a35fbb0..155019289d5 100644 --- a/var/spack/packages/py-nose/package.py +++ b/var/spack/packages/py-nose/package.py @@ -9,7 +9,7 @@ class PyNose(Package): version('1.3.4', '6ed7169887580ddc9a8e16048d38274d') - extends('python') + extends('python', ignore=r'bin/nosetests.*$') depends_on('py-setuptools') def install(self, spec, prefix): From d800c23cecd9c87b17991411512e3aa42855815d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 17 Feb 2015 00:24:58 -0800 Subject: [PATCH 74/97] Better activate/deactivate logic. spack activate - now activates dependency extensions - ensures dependencies are activated in the python installation. - -f/--force option still allows the old activate behavior. spack deactivate - checks for dependents before deactivating (like uninstall) - deactivate -a/--all will deactviate a package and ALL of its dependency extensions. - deactivate -a/--all activates all extensions of e.g.: spack deactivate -a python - deactivate -f/--force option allows removing regardless of dependents. - deactivate -f can be run EVEN if a package is not activated. - allows for clenup of activations gone wrong. --- lib/spack/spack/cmd/activate.py | 6 ++++ lib/spack/spack/cmd/deactivate.py | 54 ++++++++++++++++++++++++++--- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/directory_layout.py | 8 ++--- lib/spack/spack/hooks/extensions.py | 2 +- lib/spack/spack/package.py | 42 +++++++++++++++++----- 6 files changed, 95 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py index c1e23852d64..71eca4f4537 100644 --- a/lib/spack/spack/cmd/activate.py +++ b/lib/spack/spack/cmd/activate.py @@ -30,6 +30,9 @@ description = "Activate a package extension." def setup_parser(subparser): + subparser.add_argument( + '-f', '--force', action='store_true', + help="Activate without first activating dependencies.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="spec of package extension to activate.") @@ -44,6 +47,9 @@ def activate(parser, args): spack.db.get(specs[0]) spec = spack.cmd.disambiguate_spec(specs[0]) + if not spec.package.is_extension: + tty.die("%s is not an extension." % spec.name) + if spec.package.activated: tty.die("Package %s is already activated." % specs[0].short_spec) diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index f37dfd79ed8..bfec618c8e2 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -24,8 +24,10 @@ ############################################################################## from external import argparse import llnl.util.tty as tty + import spack import spack.cmd +from spack.graph import topological_sort description = "Deactivate a package extension." @@ -33,6 +35,10 @@ def setup_parser(subparser): subparser.add_argument( '-f', '--force', action='store_true', help="Run deactivation even if spec is NOT currently activated.") + subparser.add_argument( + '-a', '--all', action='store_true', + help="Deactivate all extensions of an extendable pacakge, or " + "deactivate an extension AND its dependencies.") subparser.add_argument( 'spec', nargs=argparse.REMAINDER, help="spec of package extension to deactivate.") @@ -42,12 +48,52 @@ def deactivate(parser, args): if len(specs) != 1: tty.die("deactivate requires one spec. %d given." % len(specs)) - # TODO: remove this hack when DAG info is stored in dir layout. + # TODO: remove this hack when DAG info is stored properly. # This ensures the ext spec is always normalized properly. spack.db.get(specs[0]) spec = spack.cmd.disambiguate_spec(specs[0]) - if not args.force and not spec.package.activated: - tty.die("Package %s is not activated." % specs[0].short_spec) + pkg = spec.package - spec.package.do_deactivate(force=args.force) + if args.all: + if pkg.extendable: + tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) + ext_pkgs = spack.db.installed_extensions_for(spec) + for ext_pkg in ext_pkgs: + ext_pkg.spec.normalize() + if ext_pkg.activated: + ext_pkg.do_deactivate(force=True) + + elif pkg.is_extension: + # TODO: store DAG info properly (see above) + spec.normalize() + + tty.msg("Deactivating %s and all dependencies." % pkg.spec.short_spec) + + topo_order = topological_sort(spec) + index = spec.index() + + for name in topo_order: + espec = index[name] + epkg = espec.package + + # TODO: store DAG info properly (see above) + epkg.spec.normalize() + + if epkg.extends(pkg.extendee_spec): + if epkg.activated or args.force: + + epkg.do_deactivate(force=args.force) + + else: + tty.die("spack deactivate --all requires an extendable package or an extension.") + + else: + if not pkg.is_extension: + tty.die("spack deactivate requires an extension.", + "Did you mean 'spack deactivate --all'?") + + if not args.force and not spec.package.activated: + tty.die("Package %s is not activated." % specs[0].short_spec) + + spec.package.do_deactivate(force=args.force) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index dee1dfece71..70b10edb4eb 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -85,7 +85,7 @@ def display_specs(specs, **kwargs): elif mode == 'deps': for spec in specs: - print spec.tree(indent=4, format='$_$@$+', color=True), + print spec.tree(indent=4, format='$_$@$+$#', color=True), elif mode in ('short', 'long'): fmt = '$-_$@$+' diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 5b80e93d6b8..b2cf5dc8017 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -371,7 +371,7 @@ def add_extension(self, spec, ext_spec): _check_concrete(ext_spec) # Check whether it's already installed or if it's a conflict. - exts = self.extension_map(spec) + exts = self._extension_map(spec) self.check_extension_conflict(spec, ext_spec) # do the actual adding. @@ -384,7 +384,7 @@ def remove_extension(self, spec, ext_spec): _check_concrete(ext_spec) # Make sure it's installed before removing. - exts = self.extension_map(spec) + exts = self._extension_map(spec) self.check_activated(spec, ext_spec) # do the actual removing. @@ -450,10 +450,10 @@ def __init__(self, spec, ext_spec, conflict): class NoSuchExtensionError(DirectoryLayoutError): - """Raised when an extension isn't there on remove.""" + """Raised when an extension isn't there on deactivate.""" def __init__(self, spec, ext_spec): super(NoSuchExtensionError, self).__init__( - "%s cannot be removed from %s because it's not installed."% ( + "%s cannot be removed from %s because it's not activated."% ( ext_spec.short_spec, spec.short_spec)) diff --git a/lib/spack/spack/hooks/extensions.py b/lib/spack/spack/hooks/extensions.py index 9d6fa23d034..cf87a78c8c8 100644 --- a/lib/spack/spack/hooks/extensions.py +++ b/lib/spack/spack/hooks/extensions.py @@ -33,4 +33,4 @@ def pre_uninstall(pkg): if pkg.is_extension: if pkg.activated: - pkg.do_deactivate() + pkg.do_deactivate(force=True) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 5d04fed8ffd..bc8541a184f 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -529,11 +529,8 @@ def extends(self, spec): @property def activated(self): - if not self.spec.concrete: - raise ValueError("Only concrete package extensions can be activated.") if not self.is_extension: raise ValueError("is_extension called on package that is not an extension.") - exts = spack.install_layout.extension_map(self.extendee_spec) return (self.name in exts) and (exts[self.name] == self.spec) @@ -956,20 +953,33 @@ def _sanity_check_extension(self): raise ValueError("%s does not extend %s!" % (self.name, self.extendee.name)) - def do_activate(self): + def do_activate(self, **kwargs): """Called on an etension to invoke the extendee's activate method. Commands should call this routine, and should not call activate() directly. """ self._sanity_check_extension() + force = kwargs.get('force', False) + + # TODO: get rid of this normalize - DAG handling. + self.spec.normalize() + spack.install_layout.check_extension_conflict(self.extendee_spec, self.spec) + if not force: + for spec in self.spec.traverse(root=False): + if spec.package.extends(self.extendee_spec): + # TODO: fix this normalize() requirement -- revisit DAG handling. + spec.package.spec.normalize() + if not spec.package.activated: + spec.package.do_activate(**kwargs) + self.extendee_spec.package.activate(self, **self.extendee_args) spack.install_layout.add_extension(self.extendee_spec, self.spec) tty.msg("Activated extension %s for %s." - % (self.spec.short_spec, self.extendee_spec.short_spec)) + % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) def activate(self, extension, **kwargs): @@ -994,15 +1004,21 @@ def ignore(filename): def do_deactivate(self, **kwargs): """Called on the extension to invoke extendee's deactivate() method.""" - force = kwargs.get('force', False) - self._sanity_check_extension() + force = kwargs.get('force', False) # Allow a force deactivate to happen. This can unlink # spurious files if something was corrupted. if not force: spack.install_layout.check_activated(self.extendee_spec, self.spec) + activated = spack.install_layout.extension_map(self.extendee_spec) + for name, aspec in activated.items(): + if aspec != self.spec and self.spec in aspec: + raise ActivationError( + "Cannot deactivate %s beacuse %s is activated and depends on it." + % (self.spec.short_spec, aspec.short_spec)) + self.extendee_spec.package.deactivate(self, **self.extendee_args) # redundant activation check -- makes SURE the spec is not @@ -1011,7 +1027,7 @@ def do_deactivate(self, **kwargs): spack.install_layout.remove_extension(self.extendee_spec, self.spec) tty.msg("Deactivated extension %s for %s." - % (self.spec.short_spec, self.extendee_spec.short_spec)) + % (self.spec.short_spec, self.extendee_spec.format("$_$@$+$%@"))) def deactivate(self, extension, **kwargs): @@ -1236,7 +1252,15 @@ def __init__(self, cls): "Package %s has no version with a URL." % cls.__name__) -class ExtensionConflictError(PackageError): +class ExtensionError(PackageError): pass + + +class ExtensionConflictError(ExtensionError): def __init__(self, path): super(ExtensionConflictError, self).__init__( "Extension blocked by file: %s" % path) + + +class ActivationError(ExtensionError): + def __init__(self, msg, long_msg=None): + super(ActivationError, self).__init__(msg, long_msg) From 724b72bdaf5c7c569e502e383e2aadcb8105983c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 17 Feb 2015 00:47:35 -0800 Subject: [PATCH 75/97] take out dyninst 8.2 for now. - doesn't build correctly with boost 1.55 --- var/spack/packages/dyninst/package.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py index 2e6f3e010a3..f3d661f9a97 100644 --- a/var/spack/packages/dyninst/package.py +++ b/var/spack/packages/dyninst/package.py @@ -31,8 +31,9 @@ class Dyninst(Package): url = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1.2/DyninstAPI-8.1.2.tgz" list_url = "http://www.dyninst.org/downloads/dyninst-8.x" - version('8.2.1', 'abf60b7faabe7a2e4b54395757be39c7', - url="http://www.paradyn.org/release8.2/DyninstAPI-8.2.1.tgz") +# Doesn't build right with boost@1.55.0 +# version('8.2.1', 'abf60b7faabe7a2e4b54395757be39c7', +# url="http://www.paradyn.org/release8.2/DyninstAPI-8.2.1.tgz") version('8.1.2', 'bf03b33375afa66fe0efa46ce3f4b17a', url="http://www.paradyn.org/release8.1.2/DyninstAPI-8.1.2.tgz") version('8.1.1', 'd1a04e995b7aa70960cd1d1fac8bd6ac', @@ -40,7 +41,7 @@ class Dyninst(Package): depends_on("libelf") depends_on("libdwarf") - depends_on("boost@1.42:1.43") + depends_on("boost@1.42:") # new version uses cmake def install(self, spec, prefix): From 44003449d594e9ba8ea1adaed84c7c938b439972 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Tue, 17 Feb 2015 16:26:00 -0800 Subject: [PATCH 76/97] fixed install steps for version 4 --- var/spack/packages/qt/package.py | 55 +++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py index 30f46c08dc3..1535bd59480 100644 --- a/var/spack/packages/qt/package.py +++ b/var/spack/packages/qt/package.py @@ -63,24 +63,41 @@ def patch(self): def install(self, spec, prefix): - # Apparently this is the only way to - # "truly" get rid of webkit compiles now... - os.rename("qtwebkit","no-qtwebkit") - os.rename("qtwebkit-examples","no-qtwebkit-examples") - configure('-v', - '-confirm-license', - '-opensource', - '-prefix', prefix, - '-openssl-linked', - '-dbus-linked', - #'-fast', - '-opengl', - '-qt-xcb', - '-optimized-qmake', - '-no-pch', - # phonon required for py-pyqt - # '-no-phonon', - # '-no-phonon-backend', - '-no-openvg') + if self.spec.satisfies('@4'): + configure('-v', + '-confirm-license', + '-opensource', + '-prefix', prefix, + '-openssl-linked', + '-dbus-linked', + #'-fast', + '-opengl', + '-optimized-qmake', + '-no-pch', + # phonon required for py-pyqt + # '-no-phonon', + # '-no-phonon-backend', + '-no-openvg') + elif self.spec.satisfies('@5'): + # Apparently this is the only way to + # "truly" get rid of webkit compiles now... + os.rename("qtwebkit","no-qtwebkit") + os.rename("qtwebkit-examples","no-qtwebkit-examples") + + configure('-v', + '-confirm-license', + '-opensource', + '-prefix', prefix, + '-openssl-linked', + '-dbus-linked', + #'-fast', + '-opengl', + '-qt-xcb', + '-optimized-qmake', + '-no-pch', + # phonon required for py-pyqt + # '-no-phonon', + # '-no-phonon-backend', + '-no-openvg') make() make("install") From 14097e39ccc77e03574119eac59524ad2bb60c6f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 17 Feb 2015 01:28:47 -0800 Subject: [PATCH 77/97] Suppress download status meter when routing I/O to a file. --- lib/spack/spack/fetch_strategy.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index a71f3a1531f..48313e2b376 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -41,6 +41,7 @@ Archive a source directory, e.g. for creating a mirror. """ import os +import sys import re import shutil from functools import wraps @@ -141,13 +142,19 @@ def fetch(self): tty.msg("Trying to fetch from %s" % self.url) + curl_args = ['-O', # save file to disk + '-f', # fail on >400 errors + '-D', '-', # print out HTML headers + '-L', self.url,] + + if sys.stdout.isatty(): + curl_args.append('-#') # status bar when using a tty + else: + curl_args.append('-sS') # just errors when not. + # Run curl but grab the mime type from the http headers - headers = spack.curl('-#', # status bar - '-O', # save file to disk - '-f', # fail on >400 errors - '-D', '-', # print out HTML headers - '-L', self.url, - return_output=True, fail_on_error=False) + headers = spack.curl( + *curl_args, return_output=True, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. From 959ce4f98577c872c69a29a30a2e0659e12cff75 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Feb 2015 10:43:29 -0800 Subject: [PATCH 78/97] Downgrade standard version of ImageMagick to a non-changing URL. - bleeding edge still available but commented by default. --- var/spack/packages/ImageMagick/package.py | 24 ++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/ImageMagick/package.py b/var/spack/packages/ImageMagick/package.py index 657b9255a32..753ea80ca6e 100644 --- a/var/spack/packages/ImageMagick/package.py +++ b/var/spack/packages/ImageMagick/package.py @@ -3,18 +3,32 @@ class Imagemagick(Package): """ImageMagick is a image processing library""" homepage = "http://www.imagemagic.org" - url = "http://www.imagemagick.org/download/ImageMagick-6.8.9-10.tar.gz" - version('6.9.0-0', '2cf094cb86ec518fa5bc669ce2d21613') - version('6.8.9-10', 'aa050bf9785e571c956c111377bbf57c') - version('6.8.9-9', 'e63fed3e3550851328352c708f800676') + #------------------------------------------------------------------------- + # ImageMagick does not keep around anything but *-10 versions, so + # this URL may change. If you want the bleeding edge, you can + # uncomment it and see if it works but you may need to try to + # fetch a newer version (-6, -7, -8, -9, etc.) or you can stick + # wtih the older, stable, archived -10 versions below. + # + # TODO: would be nice if spack had a way to recommend avoiding a + # TODO: bleeding edge version, but not comment it out. + # ------------------------------------------------------------------------- + # version('6.9.0-6', 'c1bce7396c22995b8bdb56b7797b4a1b', + # url="http://www.imagemagick.org/download/ImageMagick-6.9.0-6.tar.bz2") + + #------------------------------------------------------------------------- + # *-10 versions are archived, so these versions should fetch reliably. + # ------------------------------------------------------------------------- + version('6.8.9-10', 'aa050bf9785e571c956c111377bbf57c', + url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download") depends_on('libtool') depends_on('jpeg') depends_on('libpng') depends_on('freetype') depends_on('fontconfig') -# depends_on('libtiff') + depends_on('libtiff') def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 3e5aa4b0f5c2b1fba3a81abb465298de93009532 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Feb 2015 10:58:10 -0800 Subject: [PATCH 79/97] llvm/clang version bump --- var/spack/packages/clang/package.py | 5 ++--- var/spack/packages/llvm/package.py | 14 ++++++-------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py index b0097bd1261..4f10385dbda 100644 --- a/var/spack/packages/clang/package.py +++ b/var/spack/packages/clang/package.py @@ -29,11 +29,10 @@ class Clang(Package): Objective C and Objective C++ front-end for the LLVM compiler. """ homepage = "http://clang.llvm.org" - url = "http://llvm.org/releases/3.4.2/cfe-3.4.2.src.tar.gz" + list_url = "http://llvm.org/releases/download.html" depends_on("llvm") - - version('3.4.2', '87945973b7c73038871c5f849a818588') + version('3.4.2', '87945973b7c73038871c5f849a818588', url='http://llvm.org/releases/3.4.2/cfe-3.4.2.src.tar.xz') def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py index 69354a5c909..9d2be690bb1 100644 --- a/var/spack/packages/llvm/package.py +++ b/var/spack/packages/llvm/package.py @@ -32,15 +32,13 @@ class Llvm(Package): it is the full name of the project. """ homepage = "http://llvm.org/" - url = "http://llvm.org/releases/3.4.2/llvm-3.4.2.src.tar.gz" + list_url = "http://llvm.org/releases/download.html" - version('3.4.2', 'a20669f75967440de949ac3b1bad439c') - version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', - url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') - version('2.9', '793138412d2af2c7c7f54615f8943771', - url='http://llvm.org/releases/2.9/llvm-2.9.tgz') - version('2.8', '220d361b4d17051ff4bb21c64abe05ba', - url='http://llvm.org/releases/2.8/llvm-2.8.tgz') + version('3.5.1', '2d3d8004f38852aa679e5945b8ce0b14', url='http://llvm.org/releases/3.5.1/llvm-3.5.1.src.tar.xz') + version('3.4.2', 'a20669f75967440de949ac3b1bad439c', url='http://llvm.org/releases/3.4.2/llvm-3.4.2.src.tar.gz') + version('3.0', 'a8e5f5f1c1adebae7b4a654c376a6005', url='http://llvm.org/releases/3.0/llvm-3.0.tar.gz') + version('2.9', '793138412d2af2c7c7f54615f8943771', url='http://llvm.org/releases/2.9/llvm-2.9.tgz') + version('2.8', '220d361b4d17051ff4bb21c64abe05ba', url='http://llvm.org/releases/2.8/llvm-2.8.tgz') def install(self, spec, prefix): env['CXXFLAGS'] = self.compiler.cxx11_flag From 6e13d0985cf457b3d2b64908c6652ec4fc2349f0 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Wed, 18 Feb 2015 13:13:19 -0800 Subject: [PATCH 80/97] fixed deps for python packages --- var/spack/packages/py-dateutil/package.py | 1 + var/spack/packages/py-six/package.py | 1 + 2 files changed, 2 insertions(+) diff --git a/var/spack/packages/py-dateutil/package.py b/var/spack/packages/py-dateutil/package.py index 11699e07ee5..3bd2f2ca139 100644 --- a/var/spack/packages/py-dateutil/package.py +++ b/var/spack/packages/py-dateutil/package.py @@ -9,6 +9,7 @@ class PyDateutil(Package): extends('python') depends_on('py-setuptools') + depends_on('py-six') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-six/package.py b/var/spack/packages/py-six/package.py index 04d29adcedf..05c5bd00a98 100644 --- a/var/spack/packages/py-six/package.py +++ b/var/spack/packages/py-six/package.py @@ -8,6 +8,7 @@ class PySix(Package): version('1.9.0', '476881ef4012262dfc8adc645ee786c4') extends('python') + depends_on('py-setuptools') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) From db113733513345059b0a6d36aae8cddc56e6560a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Feb 2015 11:08:25 -0800 Subject: [PATCH 81/97] Resurrect combined qt4/5 package from b7dacb --- var/spack/packages/qt/package.py | 77 +++++++++++++++++--------------- 1 file changed, 41 insertions(+), 36 deletions(-) diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py index 1535bd59480..1dc3e1e51de 100644 --- a/var/spack/packages/qt/package.py +++ b/var/spack/packages/qt/package.py @@ -62,42 +62,47 @@ def patch(self): filter_file(r'^QMAKE_CXX *=.*$', 'QMAKE_CXX = c++', qmake_conf) - def install(self, spec, prefix): - if self.spec.satisfies('@4'): - configure('-v', - '-confirm-license', - '-opensource', - '-prefix', prefix, - '-openssl-linked', - '-dbus-linked', - #'-fast', - '-opengl', - '-optimized-qmake', - '-no-pch', - # phonon required for py-pyqt - # '-no-phonon', - # '-no-phonon-backend', - '-no-openvg') - elif self.spec.satisfies('@5'): - # Apparently this is the only way to - # "truly" get rid of webkit compiles now... - os.rename("qtwebkit","no-qtwebkit") - os.rename("qtwebkit-examples","no-qtwebkit-examples") - configure('-v', - '-confirm-license', - '-opensource', - '-prefix', prefix, - '-openssl-linked', - '-dbus-linked', - #'-fast', - '-opengl', - '-qt-xcb', - '-optimized-qmake', - '-no-pch', - # phonon required for py-pyqt - # '-no-phonon', - # '-no-phonon-backend', - '-no-openvg') + @property + def common_config_args(self): + return [ + '-prefix', self.prefix, + '-v', + '-opensource', + '-opengl', + "-release", + '-shared', + '-confirm-license', + '-openssl-linked', + '-dbus-linked', + '-optimized-qmake', + '-no-openvg', + '-no-pch', + # NIS is deprecated in more recent glibc + "-no-nis", + # For now, disable all the database drivers + "-no-sql-db2", "-no-sql-ibase", "-no-sql-mysql", "-no-sql-oci", "-no-sql-odbc", + "-no-sql-psql", "-no-sql-sqlite", "-no-sql-sqlite2", "-no-sql-tds"] + + + @when('@4') + def configure(self): + configure('-fast', + '-no-webkit', + *self.common_config_args) + + + @when('@5') + def configure(self): + configure('-no-eglfs', + '-no-directfb', + '-qt-xcb', + # If someone wants to get a webkit build working, be my guest! + '-skip', 'qtwebkit', + *self.common_config_args) + + + def install(self, spec, prefix): + self.configure() make() make("install") From c7b8a4e25cde8c2f32e514fdfe5280feca42e4ea Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Feb 2015 14:00:37 -0800 Subject: [PATCH 82/97] Fix for SPACK-46: cleanup spack clean, spack restage. --- lib/spack/spack/cmd/clean.py | 29 ++----------- lib/spack/spack/cmd/restage.py | 46 +++++++++++++++++++++ lib/spack/spack/package.py | 21 ++-------- var/spack/packages/libdwarf/package.py | 7 ---- var/spack/packages/py-virtualenv/package.py | 4 -- 5 files changed, 54 insertions(+), 53 deletions(-) create mode 100644 lib/spack/spack/cmd/restage.py diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index ec3b221988a..c20136ebe5d 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013-2014, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. @@ -23,45 +23,24 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from external import argparse -import subprocess import llnl.util.tty as tty import spack import spack.cmd -import spack.stage as stage -description = "Remove staged files for packages" +description = "Remove build stage and source tarball for packages." def setup_parser(subparser): - subparser.add_argument('-c', "--clean", action="store_true", dest='clean', - help="run make clean in the build directory (default)") - subparser.add_argument('-w', "--work", action="store_true", dest='work', - help="delete the build directory and re-expand it from its archive.") - subparser.add_argument('-d', "--dist", action="store_true", dest='dist', - help="delete the downloaded archive.") subparser.add_argument('packages', nargs=argparse.REMAINDER, help="specs of packages to clean") def clean(parser, args): if not args.packages: - tty.die("spack clean requires at least one package argument") + tty.die("spack clean requires at least one package spec.") specs = spack.cmd.parse_specs(args.packages, concretize=True) for spec in specs: package = spack.db.get(spec) - if args.dist: - package.do_clean_dist() - tty.msg("Cleaned %s" % package.name) - - elif args.work: - package.do_clean_work() - tty.msg("Restaged %s" % package.name) - - else: - try: - package.do_clean() - except subprocess.CalledProcessError, e: - tty.warn("Warning: 'make clean' didn't work. Consider 'spack clean --work'.") - tty.msg("Made clean for %s" % package.name) + package.do_clean() diff --git a/lib/spack/spack/cmd/restage.py b/lib/spack/spack/cmd/restage.py new file mode 100644 index 00000000000..e735a12c327 --- /dev/null +++ b/lib/spack/spack/cmd/restage.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013-2014, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from external import argparse + +import llnl.util.tty as tty + +import spack +import spack.cmd + +description = "Revert checked out package source code." + +def setup_parser(subparser): + subparser.add_argument('packages', nargs=argparse.REMAINDER, + help="specs of packages to restage") + + +def restage(parser, args): + if not args.packages: + tty.die("spack restage requires at least one package spec.") + + specs = spack.cmd.parse_specs(args.packages, concretize=True) + for spec in specs: + package = spack.db.get(spec) + package.do_restage() diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index bc8541a184f..492af12053e 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1047,26 +1047,13 @@ def ignore(filename): tree.unmerge(self.prefix, ignore=ignore) - def do_clean(self): - if self.stage.expanded_archive_path: - self.stage.chdir_to_source() - self.clean() - - - def clean(self): - """By default just runs make clean. Override if this isn't good.""" - # TODO: should we really call make clean, ro just blow away the directory? - make = build_env.MakeExecutable('make', self.parallel) - make('clean') - - - def do_clean_work(self): - """By default just blows away the stage directory and re-stages.""" + def do_restage(self): + """Reverts expanded/checked out source to a pristine state.""" self.stage.restage() - def do_clean_dist(self): - """Removes the stage directory where this package was built.""" + def do_clean(self): + """Removes the package's build stage and source tarball.""" if os.path.exists(self.stage.path): self.stage.destroy() diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/packages/libdwarf/package.py index c4d71ebc010..099a974e934 100644 --- a/var/spack/packages/libdwarf/package.py +++ b/var/spack/packages/libdwarf/package.py @@ -53,13 +53,6 @@ class Libdwarf(Package): parallel = False - def clean(self): - for dir in dwarf_dirs: - with working_dir(dir): - if os.path.exists('Makefile'): - make('clean') - - def install(self, spec, prefix): # dwarf build does not set arguments for ar properly make.add_default_arg('ARFLAGS=rcs') diff --git a/var/spack/packages/py-virtualenv/package.py b/var/spack/packages/py-virtualenv/package.py index 9d94c2dcda0..2d10d440a6c 100644 --- a/var/spack/packages/py-virtualenv/package.py +++ b/var/spack/packages/py-virtualenv/package.py @@ -11,9 +11,5 @@ class PyVirtualenv(Package): extends('python') depends_on('py-setuptools') - def clean(self): - if os.path.exists('build'): - shutil.rmtree('build') - def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) From e67655c31a9d98a65b3c9cd43ca329d8b97ba95b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Feb 2015 14:29:55 -0800 Subject: [PATCH 83/97] docs autodetect version. --- lib/spack/docs/conf.py | 14 ++++++++++---- lib/spack/spack/__init__.py | 2 +- lib/spack/spack/cmd/package-list.py | 9 +++++---- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index b4d49c594db..b01f33d4b82 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -35,7 +35,9 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import sys +import os +import subprocess # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -43,9 +45,13 @@ sys.path.insert(0, os.path.abspath('exts')) # Add the Spack bin directory to the path so that we can use its output in docs. -os.environ['SPACK_ROOT'] = '../../..' +spack_root = '../../..' +os.environ['SPACK_ROOT'] = spack_root os.environ['PATH'] += os.pathsep + '$SPACK_ROOT/bin' +spack_version = subprocess.Popen( + ['spack', '-V'], stderr=subprocess.PIPE).communicate()[1].strip().split('.') + # Set an environment variable so that colify will print output like it would to # a terminal. os.environ['COLIFY_TTY'] = 'true' @@ -97,9 +103,9 @@ # built documents. # # The short X.Y version. -version = '1.0' +version = '.'.join(spack_version[:2]) # The full version, including alpha/beta/rc tags. -release = '1.0' +release = '.'.join(spack_version[:2]) # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6763411f7d3..eb891e3d57a 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -78,7 +78,7 @@ # Version information from spack.version import Version -spack_version = Version("0.8") +spack_version = Version("0.8.15") # # Executables used by Spack diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index 87c528881e5..809c64a5b94 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -23,6 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import re +import cgi from StringIO import StringIO import llnl.util.tty as tty from llnl.util.tty.colify import * @@ -70,9 +71,9 @@ def print_rst_package_list(): print print pkg.name print "-" * len(pkg.name) - print "Links" - print " * `Homepage <%s>`__" % pkg.homepage - print " * `%s/package.py <%s>`__" % (pkg.name, github_url(pkg)) + print "Links:" + print " * `%s <%s>`__" % (cgi.escape(pkg.homepage), pkg.homepage) + print " * `%s/package.py <%s>`__" % (pkg.name, github_url(pkg)) print if pkg.versions: print "Versions:" @@ -82,7 +83,7 @@ def print_rst_package_list(): print " " + ", ".join("`%s`_" % d if d != "mpi" else d for d in pkg.dependencies) print - print "Description" + print "Description:" print pkg.format_doc(indent=2) print print "-----" From 2755171e08777a2b2e8449166c792956b7e8304c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Feb 2015 14:46:00 -0800 Subject: [PATCH 84/97] Update documentation to reflect new restage/clean behavior. --- lib/spack/docs/packaging_guide.rst | 40 ++++++++++++++---------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index ec2ca4d0998..076d3ca0e60 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1964,35 +1964,33 @@ apply cleanly on some previous run, then it will restage the entire package before patching. -``spack clean`` +``spack restage`` ~~~~~~~~~~~~~~~~~ +Restores the source code to pristine state, as it was before building. -There are several variations of ``spack clean``. With no arguments, -``spack clean`` runs ``make clean`` in the expanded archive directory. -This is useful if an attempted build failed, and something needs to be -changed to get a package to build. If a particular package does not -have a ``make clean`` target, this will do nothing. +Does this in one of two ways: -``spack clean -w / --work`` + 1. If the source was fetched as a tarball, deletes the entire build + directory and re-expands the tarball. + + 2. If the source was checked out from a repository, this deletes the + build directory and checks it out again. + + +``spack clean`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Deletes the entire build directory and re-expands it from the downloaded -archive. This is useful if a package does not support a proper ``make clean`` -target. +Cleans up temporary files for a particular package, by deleting the +expanded/checked out source code *and* any downloaded archive. If +``fetch``, ``stage``, or ``install`` are run again after this, Spack's +build process will start from scratch. -``spack clean -d / --dist`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Deletes the build directory *and* the downloaded archive. If -``fetch``, ``stage``, or ``install`` are run again after this, the -process will start from scratch, and the archive archive will be -downloaded again. Useful if somehow a bad archive is downloaded -accidentally and needs to be cleaned out of the staging area. ``spack purge`` ~~~~~~~~~~~~~~~~~ - -Cleans up *everything* in the build directory. You can use this to -recover disk space if temporary files from interrupted or failed -installs accumulate in the staging area. +Cleans up all of Spack's temporary files. Use this to recover disk +space if temporary files from interrupted or failed installs +accumulate in the staging area. This is equivalent to running ``spack +clean`` for every package you have fetched or staged. Keeping the stage directory on success From 2eda01c703cbd1692130ffc8ed48a9163fda87b6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Feb 2015 16:21:15 -0800 Subject: [PATCH 85/97] uninstall -f ignores nonexisting packages. --- lib/spack/spack/cmd/uninstall.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 0962942f437..6ded4553905 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -66,6 +66,7 @@ def uninstall(parser, args): tty.die(*args) if len(matching_specs) == 0: + if args.force: continue tty.die("%s does not match any installed packages." % spec) for s in matching_specs: From 2374eb4dca2a3bb4db5586251964d5ca9166b45a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Feb 2015 16:45:12 -0800 Subject: [PATCH 86/97] Fix for SPACK-62 - deactivate -a errors if arg is not activated - deactivate -af does not. --- lib/spack/spack/cmd/activate.py | 2 ++ lib/spack/spack/cmd/deactivate.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/lib/spack/spack/cmd/activate.py b/lib/spack/spack/cmd/activate.py index 71eca4f4537..4070baaa702 100644 --- a/lib/spack/spack/cmd/activate.py +++ b/lib/spack/spack/cmd/activate.py @@ -38,6 +38,7 @@ def setup_parser(subparser): def activate(parser, args): + # TODO: shouldn't have to concretize here. Fix DAG issues. specs = spack.cmd.parse_specs(args.spec, concretize=True) if len(specs) != 1: tty.die("activate requires one spec. %d given." % len(specs)) @@ -47,6 +48,7 @@ def activate(parser, args): spack.db.get(specs[0]) spec = spack.cmd.disambiguate_spec(specs[0]) + if not spec.package.is_extension: tty.die("%s is not an extension." % spec.name) diff --git a/lib/spack/spack/cmd/deactivate.py b/lib/spack/spack/cmd/deactivate.py index bfec618c8e2..c9a4d4b2f66 100644 --- a/lib/spack/spack/cmd/deactivate.py +++ b/lib/spack/spack/cmd/deactivate.py @@ -44,6 +44,7 @@ def setup_parser(subparser): def deactivate(parser, args): + # TODO: shouldn't have to concretize here. Fix DAG issues. specs = spack.cmd.parse_specs(args.spec, concretize=True) if len(specs) != 1: tty.die("deactivate requires one spec. %d given." % len(specs)) @@ -59,6 +60,7 @@ def deactivate(parser, args): if pkg.extendable: tty.msg("Deactivating all extensions of %s" % pkg.spec.short_spec) ext_pkgs = spack.db.installed_extensions_for(spec) + for ext_pkg in ext_pkgs: ext_pkg.spec.normalize() if ext_pkg.activated: @@ -68,6 +70,9 @@ def deactivate(parser, args): # TODO: store DAG info properly (see above) spec.normalize() + if not args.force and not spec.package.activated: + tty.die("%s is not activated." % pkg.spec.short_spec) + tty.msg("Deactivating %s and all dependencies." % pkg.spec.short_spec) topo_order = topological_sort(spec) From 02e316e7724e8f39e69306f03bcbff1eaa12827c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Feb 2015 16:45:54 -0800 Subject: [PATCH 87/97] Convert ValueErrors to SpackError subclass. --- lib/spack/spack/package.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 492af12053e..fed62f6cb76 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -941,16 +941,17 @@ def _check_extendable(self): def _sanity_check_extension(self): if not self.is_extension: - raise ValueError("This package is not an extension.") + raise ActivationError("This package is not an extension.") + extendee_package = self.extendee_spec.package extendee_package._check_extendable() if not extendee_package.installed: - raise ValueError("Can only (de)activate extensions for installed packages.") + raise ActivationError("Can only (de)activate extensions for installed packages.") if not self.installed: - raise ValueError("Extensions must first be installed.") + raise ActivationError("Extensions must first be installed.") if not self.extendee_spec.name in self.extendees: - raise ValueError("%s does not extend %s!" % (self.name, self.extendee.name)) + raise ActivationError("%s does not extend %s!" % (self.name, self.extendee.name)) def do_activate(self, **kwargs): From 14e70ad68959097bcfa4bda38cbafe8d1e71127b Mon Sep 17 00:00:00 2001 From: Saravan Pantham Date: Wed, 18 Feb 2015 18:05:57 -0800 Subject: [PATCH 88/97] Added netcdf package support. --- var/spack/packages/netcdf/package.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 var/spack/packages/netcdf/package.py diff --git a/var/spack/packages/netcdf/package.py b/var/spack/packages/netcdf/package.py new file mode 100644 index 00000000000..34284ea725a --- /dev/null +++ b/var/spack/packages/netcdf/package.py @@ -0,0 +1,28 @@ +from spack import * + +class Netcdf(Package): + """NetCDF is a set of software libraries and self-describing, machine-independent + data formats that support the creation, access, and sharing of array-oriented + scientific data.""" + + homepage = "http://www.unidata.ucar.edu/software/netcdf/" + url = "ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.3.3.tar.gz" + + version('4.3.3', '5fbd0e108a54bd82cb5702a73f56d2ae') + + # Dependencies: + # >HDF5 + depends_on("hdf5") + + def install(self, spec, prefix): + configure( + "--prefix=%s" % prefix, + "--disable-dap", # Disable DAP. + "--disable-shared", # Don't build shared libraries (use static libs). + "CPPFLAGS=-I%s/include" % spec['hdf5'].prefix, # Link HDF5's include dir. + "LDFLAGS=-L%s/lib" % spec['hdf5'].prefix) # Link HDF5's lib dir. + + make("install") + + # Check the newly installed netcdf package. Currently disabled. + # make("check") From 065e5ccd1a09cdb24db78c871cd512b4d305e0e6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 18 Feb 2015 20:51:50 -0800 Subject: [PATCH 89/97] Update contributors list. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 74a327b517a..69632b8933a 100644 --- a/README.md +++ b/README.md @@ -36,8 +36,7 @@ Authors ---------------- Spack was written by Todd Gamblin, tgamblin@llnl.gov. -Significant contributions were also made by the following awesome -people: +Significant contributions were also made by: * David Beckingsale * David Boehme @@ -46,7 +45,10 @@ people: * Matt Legendre * Greg Lee * Adam Moody + * Saravan Pantham + * Joachim Protze * Bob Robey + * Justin Too Release ---------------- From d49c98188a68d4aad0410ac1d0df759117773937 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 22 Feb 2015 21:17:18 -0800 Subject: [PATCH 90/97] Add an override to colify so we can set terminal dimensions. --- lib/spack/docs/conf.py | 4 +--- lib/spack/llnl/util/tty/colify.py | 9 +++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/spack/docs/conf.py b/lib/spack/docs/conf.py index b4d49c594db..7b350f73b5a 100644 --- a/lib/spack/docs/conf.py +++ b/lib/spack/docs/conf.py @@ -48,9 +48,7 @@ # Set an environment variable so that colify will print output like it would to # a terminal. -os.environ['COLIFY_TTY'] = 'true' -os.environ['COLUMNS'] = '80' -os.environ['LINES'] = '25' +os.environ['COLIFY_SIZE'] = '25x80' # Enable todo items todo_include_todos = True diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index 6b2909990ca..66c52c39687 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -169,6 +169,15 @@ def colify(elts, **options): if not elts: return (0, ()) + # environment size is of the form "x" + env_size = os.environ.get('COLIFY_SIZE') + if env_size: + try: + r, c = env_size.split('x') + console_rows, console_cols = int(r), int(c) + tty = True + except: pass + # Use only one column if not a tty. if not tty: if tty is False or not output.isatty(): From 6dab133d9f001d5e03fde25bdd3d412adf2a7b56 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 22 Feb 2015 23:00:14 -0800 Subject: [PATCH 91/97] Same package add icon on mac and linux. --- lib/spack/spack/package.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index fed62f6cb76..137e8f88373 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1171,12 +1171,8 @@ def validate_package_url(url_string): def print_pkg(message): """Outputs a message with a package icon.""" - mac_ver = py_platform.mac_ver()[0] - if mac_ver and Version(mac_ver) >= Version('10.7'): - print u"\U0001F4E6" + tty.indent, - else: - from llnl.util.tty.color import cwrite - cwrite('@*g{[+]} ') + from llnl.util.tty.color import cwrite + cwrite('@*g{[+]} ') print message From 5699cbb597468911c0b34988512ddea8b4c62ecc Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 23 Feb 2015 01:22:49 -0800 Subject: [PATCH 92/97] Fix SPACK-60: 0.8.15 basic docs. --- lib/spack/docs/basic_usage.rst | 716 ++++++++++++++++++++-------- lib/spack/docs/packaging_guide.rst | 74 +++ lib/spack/spack/cmd/package-list.py | 2 + 3 files changed, 583 insertions(+), 209 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 196b7077f9c..bd25d739ea8 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -4,18 +4,17 @@ Basic usage ===================== Spack is implemented as a single command (``spack``) with many -*subcommands*, much like ``git``, ``svn``, ``yum``, or ``apt-get``. -Only a small subset of commands are needed for typical usage. - -This section covers a small set of subcommands that should cover most -general use cases for Spack. +*subcommands*. Only a small subset of commands is needed for typical +usage. Listing available packages ------------------------------ -The first thing you will likely want to do with spack is find out what -software is available to install. There are a few relevant commands. +The first thing you likely want to do with spack is to install some +software. Before that, you need to know what's available. You can +see avaialble package names either using the :ref:`package-list`, or +using the commands below. ``spack list`` ~~~~~~~~~~~~~~~~ @@ -36,33 +35,273 @@ do wildcard searches using ``*``: ``spack info`` ~~~~~~~~~~~~~~~~ -To get information on a particular package from the full list, run -``spack info ``. For example, for ``mpich`` the output -looks like this: +To get more information on a particular package from `spack list`, use +`spack info`. Just supply the name of a package: .. command-output:: spack info mpich -This includes basic information about the package: where to download -it, its dependencies, virtual packages it provides (e.g. an MPI -implementation will provide the MPI interface), and a text -description, if one is available. :ref:`Dependencies -` and :ref:`virtual dependencies -` are described in more detail later. +Most of the information is self-explanatory. *Safe versions* are +versions that Spack has a checksum for, and Spack will use the +checksum to ensure they downloaded without any errors or malicious +attacks. :ref:`Dependencies ` and :ref:`virtual +dependencies `, are described in more detail +later. ``spack versions`` ~~~~~~~~~~~~~~~~~~~~~~~~ -To see available versions of a package, run ``spack versions``, for -example: +To see *more* available versions of a package, run ``spack versions``, +for example: .. command-output:: spack versions libelf -Since it has to manage many different software packages, Spack doesn't -place many restrictions on what a package version has to look like. -Packages like ``mpich`` use traditional version numbers like -``3.0.4``. Other packages, like ``libdwarf`` use date-stamp versions -like ``20130729``. Versions can contain numbers, letters, dashes, -underscores, and periods. +There are two sections in the output. *Safe versions* are ones that +have already been checksummed. Spack goes a step further, though, and +also shows you what versions are available out on the web---these are +*remote versions*. Spack gets this information by scraping it +directly from webpages. Depending on the package, Spack may or may +not be able to find any remote versions. + + +Installing and uninstalling +------------------------------ + +Now that you know how to list avaiable packages and versions, you're +ready to start installing things. + +``spack install`` +~~~~~~~~~~~~~~~~~~~~~ + +``spack install`` will install any package shown by ``spack list``. +To install the latest version of a pacakge, along with all of its +dependencies, simply give it a package name: + +.. code-block:: sh + + $ spack install mpileaks + +If `mpileaks` depends on other packages, Spack will install those +first. It then fetches the tarball for ``mpileaks``, expands it, +verifies that it was downloaded without errors, builds it, and +installs it in its own directory under ``$SPACK_HOME/opt``. You'll see +a number of messages from spack, a lot of build output, and a message +that the packages is installed: + +.. code-block:: sh + + $ spack install mpileaks + ==> Installing mpileaks + ==> mpich is already installed in /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/mpich@3.0.4. + ==> callpath is already installed in /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/callpath@1.0.2-5dce4318. + ==> adept-utils is already installed in /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/adept-utils@1.0-5adef8da. + ==> Trying to fetch from https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz + ######################################################################## 100.0% + ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7=chaos_5_x86_64_ib-59f6ad23/mpileaks-1.0.tar.gz + ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7=chaos_5_x86_64_ib-59f6ad23. + ==> No patches needed for mpileaks. + ==> Building mpileaks. + + ... build output ... + + ==> Successfully installed mpileaks. + Fetch: 2.16s. Build: 9.82s. Total: 11.98s. + [+] /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/mpileaks@1.0-59f6ad23 + +The last line, with the ``[+]``, indicates where the package is +installed. + +Building a specific version +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spack can also build *specific versions* of a package. To do this, +just add ``@`` after the package name, followed by a version: + +.. code-block:: sh + + $ spack install mpich@3.0.4 + +Any number of versions of the same package can be installed at once +without interfering with each other. This is good for multi-user +sites, as installing a version that one user needs will not disrupt +existing installations for other users. + +In addition to different versions, Spack can customize the compiler, +compile-time options (variants), and platform (for cross compiles) of +an installation. Spack is unique in that it can also configure the +*dependencies* a package is built with. For example, two +configurations of the same version of a package, one built with boost +1.39.0, and the other version built with version 1.43.0, can coexist. + +This can all be done on the command line using special syntax. Spack +calls the descriptor used to refer to a particular package +configuration a **spec**. In the command lines above, both +``mpileaks`` and ``mpileaks@3.0.4`` are specs. Specs are described in +detail in :ref:`sec-specs`. + + +``spack uninstall`` +~~~~~~~~~~~~~~~~~~~~~ + +To uninstall a package, type ``spack uninstall ``. This will +completely remove the directory in which the package was installed. + +.. code-block:: sh + + spack uninstall mpich + +If there are still installed packages that depend on the package to be +uninstalled, spack will refuse to uninstall it. You can override this +behavior with ``spack uninstall -f ``, but you risk breaking +other installed packages. In general, it is safer to remove dependent +packages *before* removing their dependencies. + +A line like ``spack uninstall mpich`` may be ambiguous, if multiple +``mpich`` configurations are installed. For example, if both +``mpich@3.0.2`` and ``mpich@3.1`` are installed, ``mpich`` could refer +to either one. Because it cannot determine which one to uninstall, +Spack will ask you to provide a version number to remove the +ambiguity. As an example, ``spack uninstall mpich@3.1`` is +unambiguous in this scenario. + + +Seeing installed packages +----------------------------------- + +We know that ``spack list`` shows you the names of available packages, +but how do you figure out which are installed? + + +``spack find`` +~~~~~~~~~~~~~~~~~~~~~~ + +``spack find`` shows the *specs* of installed packages. A spec is +like a name, but it has a version, compiler, architecture, and build +options associated with it. In spack, you can have many installations +of the same package with different specs. + +Running ``spack find`` with no arguments lists installed packages: + +.. code-block:: sh + + $ spack find + ==> 74 installed packages. + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + ImageMagick@6.8.9-10 libdwarf@20130729 py-dateutil@2.4.0 + adept-utils@1.0 libdwarf@20130729 py-ipython@2.3.1 + atk@2.14.0 libelf@0.8.12 py-matplotlib@1.4.2 + boost@1.55.0 libelf@0.8.13 py-nose@1.3.4 + bzip2@1.0.6 libffi@3.1 py-numpy@1.9.1 + cairo@1.14.0 libmng@2.0.2 py-pygments@2.0.1 + callpath@1.0.2 libpng@1.6.16 py-pyparsing@2.0.3 + cmake@3.0.2 libtiff@4.0.3 py-pyside@1.2.2 + dbus@1.8.6 libtool@2.4.2 py-pytz@2014.10 + dbus@1.9.0 libxcb@1.11 py-setuptools@11.3.1 + dyninst@8.1.2 libxml2@2.9.2 py-six@1.9.0 + fontconfig@2.11.1 libxml2@2.9.2 python@2.7.8 + freetype@2.5.3 llvm@3.0 qhull@1.0 + gdk-pixbuf@2.31.2 memaxes@0.5 qt@4.8.6 + glib@2.42.1 mesa@8.0.5 qt@5.4.0 + graphlib@2.0.0 mpich@3.0.4 readline@6.3 + gtkplus@2.24.25 mpileaks@1.0 sqlite@3.8.5 + harfbuzz@0.9.37 mrnet@4.1.0 stat@2.1.0 + hdf5@1.8.13 ncurses@5.9 tcl@8.6.3 + icu@54.1 netcdf@4.3.3 tk@src + jpeg@9a openssl@1.0.1h vtk@6.1.0 + launchmon@1.0.1 pango@1.36.8 xcb-proto@1.11 + lcms@2.6 pixman@0.32.6 xz@5.2.0 + libdrm@2.4.33 py-dateutil@2.4.0 zlib@1.2.8 + + -- chaos_5_x86_64_ib / gcc@4.9.2 -------------------------------- + libelf@0.8.10 mpich@3.0.4 + +Packages are divided into groups according to their architecture and +compiler. Within each group, Spack tries to keep the view simple, and +only shows the version of installed packages. + +In some cases, there may be differnt configurations of the *same* +version of a package installed. For example, there are two +installations of of ``libdwarf@20130729`` above. We can look at them +in more detail using ``spack find -d``, and by asking only to show +``libdwarf`` packages: + +.. code-block:: sh + + $ spack find --deps libdwarf + ==> 2 installed packages. + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + libdwarf@20130729-d9b90962 + ^libelf@0.8.12 + libdwarf@20130729-b52fac98 + ^libelf@0.8.13 + +Now we see that the two instances of ``libdwarf`` depend on +*different* versions of ``libelf``: 0.8.12 and 0.8.13. This view can +become complicated for packages with many dependencies. If you just +want to know whether two packages' dependencies differ, you can use +``spack find -l``: + +.. code-block:: sh + + $ spack find -l libdwarf + ==> 2 installed packages. + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + libdwarf@20130729-d9b90962 libdwarf@20130729-b52fac98 + +Now the ``libwarf`` installs have hashes after their names. These are +hashes over all of the dependencies of each package. If the hashes +are the same, then the packages have the same dependency configuration. + +If you want to know the path where each package is installed, you can +use ``spack find -p``: + +.. code-block:: sh + + $ spack find -p + ==> 74 installed packages. + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + ImageMagick@6.8.9-10 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/ImageMagick@6.8.9-10-4df950dd + adept-utils@1.0 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/adept-utils@1.0-5adef8da + atk@2.14.0 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/atk@2.14.0-3d09ac09 + boost@1.55.0 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/boost@1.55.0 + bzip2@1.0.6 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/bzip2@1.0.6 + cairo@1.14.0 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/cairo@1.14.0-fcc2ab44 + callpath@1.0.2 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/callpath@1.0.2-5dce4318 + ... + +And, finally, you can restrict your search to a particular package +by supplying its name: + +.. code-block:: sh + + $ spack find -p libelf + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + libelf@0.8.11 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.11 + libelf@0.8.12 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.12 + libelf@0.8.13 /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.13 + +``spack find`` actually does a lot more than this. You can use +*specs* to query for specific configurations and builds of each +package. If you want to find only libelf versions greater than version +0.8.12, you could say: + +.. code-block:: sh + + $ spack find libelf@0.8.12: + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + libelf@0.8.12 libelf@0.8.13 + +Finding just the versions of libdwarf built with a particular version +of libelf would look like this: + +.. code-block:: sh + + $ spack find -l libdwarf ^libelf@0.8.12 + ==> 1 installed packages. + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + libdwarf@20130729-d9b90962 + +The full spec syntax is discussed in detail in :ref:`sec-specs`. + Compiler Configuration ----------------------------------- @@ -110,15 +349,18 @@ where the compiler is installed. For example:: intel@13.0.079 Or you can run ``spack compiler add`` with no arguments to force -autodetection. This is useful if you do not know where compilers -live, but new compilers have been added to your ``PATH``. For -example, using dotkit, you might do this:: +autodetection. This is useful if you do not know where compilers are +installed, but you know that new compilers have been added to your +``PATH``. For example, using dotkit, you might do this:: - $ use gcc-4.9.0 + $ module load gcc-4.9.0 $ spack compiler add ==> Added 1 new compiler to /Users/gamblin2/.spackconfig gcc@4.9.0 +This loads the environment module for gcc-4.9.0 to get it into the +``PATH``, and then it adds the compiler to Spack. + ``spack compiler info`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -126,17 +368,20 @@ example, using dotkit, you might do this:: If you want to see specifics on a particular compiler, you can run ``spack compiler info`` on it:: - $ spack compiler info intel@12.1.3 - intel@12.1.3: - cc = /usr/local/bin/icc-12.1.293 - cxx = /usr/local/bin/icpc-12.1.293 - f77 = /usr/local/bin/ifort-12.1.293 - fc = /usr/local/bin/ifort-12.1.293 + $ spack compiler info intel@15 + intel@15.0.0: + cc = /usr/local/bin/icc-15.0.090 + cxx = /usr/local/bin/icpc-15.0.090 + f77 = /usr/local/bin/ifort-15.0.090 + fc = /usr/local/bin/ifort-15.0.090 This shows which C, C++, and Fortran compilers were detected by Spack. +Notice also that we didn't have to be too specific about the +version. We just said ``intel@15``, and information about the only +matching Intel compiler was displayed. -Manual configuration +Manual compiler configuration ~~~~~~~~~~~~~~~~~~~~~~~ If autodetection fails, you can manually conigure a compiler by @@ -153,8 +398,8 @@ Each compiler configuration in the file looks like this:: fc = /usr/local/bin/ifort-15.0.024-beta ... -For compilers, like ``clang``, that do not support Fortran, you can simply -put ``None`` for ``f77`` and ``fc``:: +For compilers, like ``clang``, that do not support Fortran, put +``None`` for ``f77`` and ``fc``:: [compiler "clang@3.3svn"] cc = /usr/bin/clang @@ -163,169 +408,7 @@ put ``None`` for ``f77`` and ``fc``:: fc = None Once you save the file, the configured compilers will show up in the -list displayed when you run ``spack compilers``. - - -Seeing installed packages ----------------------------------- - -``spack find`` -~~~~~~~~~~~~~~~~~~~~~~ - -The second thing you're likely to want to do with Spack, and the first -thing users of your system will likely want to do, is to find what -software is already installed and ready to use. You can do that with -``spack find``. - -Running ``spack find`` with no arguments will list all the installed -packages: - -.. code-block:: sh - - $ spack find - == chaos_5_x86_64_ib =========================================== - -- gcc@4.4.7 --------------------------------------------------- - libdwarf@20130207-d9b909 - libdwarf@20130729-d9b909 - libdwarf@20130729-b52fac - libelf@0.8.11 - libelf@0.8.12 - libelf@0.8.13 - -Packages are grouped by architecture, then by the compiler used to -build them, and then by their versions and options. If a package has -dependencies, there will also be a hash at the end of the name -indicating the dependency configuration. Packages with the same hash -have the same dependency configuration. If you want ALL information -about dependencies, as well, then you can supply ``-l`` or ``--long``: - -.. code-block:: sh - - $ spack find -l - == chaos_5_x86_64_ib =========================================== - -- gcc@4.4.7 --------------------------------------------------- - libdwarf@20130207 - ^libelf@0.8.12 - libdwarf@20130729 - ^libelf@0.8.12 - libdwarf@20130729 - ^libelf@0.8.13 - libelf@0.8.11 - libelf@0.8.12 - libelf@0.8.13 - -Now you can see which versions of ``libelf`` each version of -``libdwarf`` was built with. - -If you want to know the path where each of these packages is -installed, do ``spack find -p`` or ``--path``: - -.. code-block:: sh - - $ spack find -p - == chaos_5_x86_64_ib =========================================== - -- gcc@4.4.7 --------------------------------------------------- - libdwarf@20130207-d9b909 /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libdwarf@20130207-d9b909 - libdwarf@20130729-d9b909 /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libdwarf@20130729-d9b909 - libdwarf@20130729-b52fac /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libdwarf@20130729-b52fac - libelf@0.8.11 /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.11 - libelf@0.8.12 /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.12 - libelf@0.8.13 /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.13 - - -And, finally, you can restrict your search to a particular package -by supplying its name: - -.. code-block:: sh - - $ spack find -p libelf - == chaos_5_x86_64_ib =========================================== - -- gcc@4.4.7 --------------------------------------------------- - libelf@0.8.11 /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.11 - libelf@0.8.12 /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.12 - libelf@0.8.13 /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/libelf@0.8.13 - - -``spack find`` actually does a lot more than this. You can use -*specs* to query for specific configurations and builds of each -package. The full spec syntax is discussed in detail in -:ref:`sec-specs`. - - - -Installing and uninstalling ------------------------------- - -``spack install`` -~~~~~~~~~~~~~~~~~~~~~ - -``spack install`` will install any package that appears in the output -of ``spack list``. To install the latest version of a pacakge and all -of its dependencies, simply run ``spack install ``: - -.. code-block:: sh - - spack install mpileaks - -Spack will fetch the tarball for ``mpileaks``, expand it, verify that -it was downloaded without errors, build it, and install it in its own -directory under ``$SPACK_HOME/opt``. If the requested package depends -on other packages in order to build, Spack fetches them as well, and -installs them before it installs the requested package. Like the main -package, each dependency is also installed in its own directory. - -Spack can also build *specific* configurations of a package. For -example, to install something with a specific version, add ``@`` after -the package name, followed by a version string: - -.. code-block:: sh - - spack install mpich@3.0.4 - -Any number of configurations of the same package can be installed at -once without interfering with each other. This is good for multi-user -sites, as installing a version that one user needs will not disrupt -existing installations for other users. - -In addition to version configuraitons, Spack can customize the -compiler, compile-time options (variants), and platform (for cross -compiles) of an installation. Spack is unique in that it can also -configure the *dependencies* a package is built with. For example, -two configurations of the same version of a package, one built with -boost 1.39.0, and the other version built with version 1.43.0, can -coexist. - -This can all be done on the command line using special syntax. Spack -calls the descriptor used to refer to a particular package -configuration a **spec**. In the command lines above, both -``mpileaks`` and ``mpileaks@3.0.4`` are specs. To customize -additional properties, simply add more attributes to the spec. Specs -and their syntax are covered in more detail in :ref:`sec-specs`. - - -``spack uninstall`` -~~~~~~~~~~~~~~~~~~~~~ - -To uninstall a package, type ``spack uninstall ``. This will -completely remove the directory in which the package was installed. - -.. code-block:: sh - - spack uninstall mpich - -If there are still installed packages that depend on the package to be -uninstalled, spack will refuse to uninstall. If you know what you're -doing, you can override this with ``spack uninstall -f ``. -However, running this risks breaking other installed packages. In -general, it is safer to remove dependent packages *before* removing -their dependencies. - -A line like ``spack uninstall mpich`` may be ambiguous, if multiple -``mpich`` configurations are installed. For example, if both -``mpich@3.0.2`` and ``mpich@3.1`` are installed, it could refer to -either one, and Spack cannot determine which one to uninstall. Spack -will ask you to provide a version number to remove the ambiguity. For -example, ``spack uninstall mpich@3.1`` is unambiguous in the above -scenario. +list displayed by ``spack compilers``. .. _sec-specs: @@ -333,10 +416,10 @@ scenario. Specs & Dependencies ------------------------- -We now know that ``spack install`` and ``spack uninstall`` both take a -package name with an optional version specifier. In Spack, that -descriptor is called a *spec*. Spack uses specs to refer to a -particular build configuration (or configurations) of a package. +We know that ``spack install``, ``spack uninstall``, and other +commands take a package name with an optional version specifier. In +Spack, that descriptor is called a *spec*. Spack uses specs to refer +to a particular build configuration (or configurations) of a package. Specs are more than a package name and a version; you can use them to specify the compiler, compiler version, architecture, compile options, and dependency options for a build. In this section, we'll go over @@ -499,6 +582,11 @@ based on site policies. Variants ~~~~~~~~~~~~~~~~~~~~~~~ +.. Note:: + + Variants are not yet supported, but will be in the next Spack + release (0.9), due in Q2 2015. + Variants are named options associated with a particular package, and they can be turned on or off. For example, above, supplying ``+debug`` causes ``mpileaks`` to be built with debug flags. The @@ -544,6 +632,11 @@ the command line is provided for convenience and legibility. Architecture specifier ~~~~~~~~~~~~~~~~~~~~~~~ +.. Note:: + + Architecture specifiers are part of specs but are not yet + functional. They will be in Spack version 1.0, due in Q3 2015. + The architecture specifier starts with a ``=`` and also comes after some package name within a spec. It allows a user to specify a particular architecture for the package to be built. This is mostly @@ -678,10 +771,6 @@ For ``csh`` and ``tcsh`` run: You can put the above code in your ``.bashrc`` or ``.cshrc``, and Spack's shell support will be available on the command line. - -------------------------------- - - When you install a package with Spack, it automatically generates an environment module that lets you add the package to your environment. @@ -710,7 +799,7 @@ of installed packages. $ module avail - ------- /g/g21/gamblin2/src/spack/share/spack/modules/chaos_5_x86_64_ib -------- + ------- /home/gamblin2/spack/share/spack/modules/chaos_5_x86_64_ib -------- adept-utils@1.0%gcc@4.4.7-5adef8da libelf@0.8.13%gcc@4.4.7 automaded@1.0%gcc@4.4.7-d9691bb0 libelf@0.8.13%intel@15.0.0 boost@1.55.0%gcc@4.4.7 mpc@1.0.2%gcc@4.4.7-559607f5 @@ -858,6 +947,215 @@ regenerate all module and dotkit files from scratch: ==> Regenerating tcl module files. ==> Regenerating dotkit module files. + +.. _extensions: + +Extensions & Python Support +------------------------------------ + +Spack's installation model assumes that each package will live in its +own install prefix. However, certain packages are typically installed +*within* the directory hierarchy of other packages. For example, +modules in interpreted languages like `Python +`_ are typically installed in the +``$prefix/lib/python-2.7/site-packages`` directory. + +Spack has support for this type of installation as well. In Spack, +a package that can live inside the prefix of another package is called +an *extension*. Suppose you have Python installed like so: + +.. code-block:: sh + + $ spack find python + ==> 1 installed packages. + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + python@2.7.8 + +``spack extensions`` +~~~~~~~~~~~~~~~~~~~~~~~ + +You can find extensions for your Python installation like this: + +.. code-block:: sh + + $ spack extensions python + ==> python@2.7.8%gcc@4.4.7=chaos_5_x86_64_ib-703c7a96 + ==> 36 extensions: + geos py-ipython py-pexpect py-pyside py-sip + py-basemap py-libxml2 py-pil py-pytz py-six + py-biopython py-mako py-pmw py-rpy2 py-sympy + py-cython py-matplotlib py-pychecker py-scientificpython py-virtualenv + py-dateutil py-mpi4py py-pygments py-scikit-learn + py-epydoc py-mx py-pylint py-scipy + py-gnuplot py-nose py-pyparsing py-setuptools + py-h5py py-numpy py-pyqt py-shiboken + + ==> 12 installed: + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + py-dateutil@2.4.0 py-nose@1.3.4 py-pyside@1.2.2 + py-dateutil@2.4.0 py-numpy@1.9.1 py-pytz@2014.10 + py-ipython@2.3.1 py-pygments@2.0.1 py-setuptools@11.3.1 + py-matplotlib@1.4.2 py-pyparsing@2.0.3 py-six@1.9.0 + + ==> None activated. + +The extensions are a subset of what's returned by ``spack list``, and +they are packages like any ohter. They are installed into their own +prefixes, and you can see this with ``spack find -p``: + +.. code-block:: sh + $ spack find -p py-numpy + ==> 1 installed packages. + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + py-numpy@1.9.1 /g/g21/gamblin2/src/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/py-numpy@1.9.1-66733244 + +However, even though this package is installed, you cannot use it +directly when you run ``python``: + +.. code-block:: sh + + $ spack load python + $ python + Python 2.7.8 (default, Feb 17 2015, 01:35:25) + [GCC 4.4.7 20120313 (Red Hat 4.4.7-11)] on linux2 + Type "help", "copyright", "credits" or "license" for more information. + >>> import numpy + Traceback (most recent call last): + File "", line 1, in + ImportError: No module named numpy + >>> + +Extensions & Environment Modules +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are two ways to get ``numpy`` working in Python. The first is +to use :ref:`shell-support`. You can simply ``use`` or ``load`` the +module for the extension, and it will be added to the ``PYTHONPATH`` +in your current shell. + +For tcl modules: + +.. code-block:: sh + + $ spack load python + $ spack load py-numpy + +or, for dotkit: + +.. code-block:: sh + + $ spack use python + $ spack use py-numpy + +Now ``import numpy`` will succeed for as long as you keep your current +session open. + + +Activating Extensions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is often desirable to have certain packages *always* available as +part of a Python installation. Spack offers a more permanent solution +for this case. Instead of requiring users to load particular +environment modules, you can *activate* the package within the Python +installation: + +``spack activate`` +^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: sh + + $ spack activate py-numpy + ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7=chaos_5_x86_64_ib-3c74eb69 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-nose@1.3.4%gcc@4.4.7=chaos_5_x86_64_ib-5f70f816 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-numpy@1.9.1%gcc@4.4.7=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. + +Several things have happened here. The user requested that +``py-numpy`` be activated in the ``python`` installation it was built +with. Spack knows that ``py-numpy`` depends on ``py-nose`` and +``py-setuptools``, so it activated those packages first. Finally, +once all dpeendencies were activated in the ``python`` installation, +``py-numpy`` was activated as well. + +If we run ``spack extensions`` again, we now see the three new +packages listed as activated: + +.. code-block:: sh + + $ spack extensions python + ==> python@2.7.8%gcc@4.4.7=chaos_5_x86_64_ib-703c7a96 + ==> 36 extensions: + geos py-ipython py-pexpect py-pyside py-sip + py-basemap py-libxml2 py-pil py-pytz py-six + py-biopython py-mako py-pmw py-rpy2 py-sympy + py-cython py-matplotlib py-pychecker py-scientificpython py-virtualenv + py-dateutil py-mpi4py py-pygments py-scikit-learn + py-epydoc py-mx py-pylint py-scipy + py-gnuplot py-nose py-pyparsing py-setuptools + py-h5py py-numpy py-pyqt py-shiboken + + ==> 12 installed: + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + py-dateutil@2.4.0 py-nose@1.3.4 py-pyside@1.2.2 + py-dateutil@2.4.0 py-numpy@1.9.1 py-pytz@2014.10 + py-ipython@2.3.1 py-pygments@2.0.1 py-setuptools@11.3.1 + py-matplotlib@1.4.2 py-pyparsing@2.0.3 py-six@1.9.0 + + ==> 3 currently activated: + -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- + py-nose@1.3.4 py-numpy@1.9.1 py-setuptools@11.3.1 + + +Now, when a user runs python, ``numpy`` will be avaiable for import +*without* the user having to explicitly loaded. ``python@2.7.8`` now +acts like a system Python installation with ``numpy`` installed inside +of it. + +Spack accomplishes this by symbolically linking the *entire* prefix of +the ``py-numpy`` into the prefix of the ``python`` package. To the +python interpreter, it looks like ``numpy`` is installed in the +``site-packages`` directory. + +The only limitation of activation is that you can only have a *single* +version of an extension activated at a time. This is because multiple +versions of the same extension would conflict if symbolically linked +into the same prefix. Users who want a different version of a package +can still get it by using environment modules, but they will have to +explicitly load their preferred version. + +``spack activate -f`` +^^^^^^^^^^^^^^^^^^^^^^^^^ +If, for some reason, you want to activate a package *without* its +dependencies, you can use ``spack activate -f``: + +.. code-block:: sh + + $ spack activate -f py-numpy + ==> Activated extension py-numpy@1.9.1%gcc@4.4.7=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. + + +``spack deactivate`` +^^^^^^^^^^^^^^^^^^^^^^^^^ + +We've seen how activating an extension can be used to set up a default +version of a Python module. Obviously, you may want to change that at +some point. ``spack deactivate`` is the command for this. There are +several variants: + + * ``spack deactivate `` will deactivate a single + extension. If another activated extension depends on this one, + Spack will warn you and exit with an error. + * ``spack deactivate -f `` deactivates an extension + regardless of packages that depend on it. + * ``spack deactivate -a `` deactivates an extension and + all of its dependencies. Use ``-f`` to disregard dependents. + * ``spack deactivate -a `` deactivates *all* activated + extensions of a package. For example, to deactivate *all* python + extensions, use:: + + spack deactivate -a python + + Getting Help ----------------------- diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index ec2ca4d0998..e5ebdb4cd2a 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -2032,6 +2032,80 @@ to get rid of the install prefix before you build again: spack uninstall -f +Graphing Dependencies +-------------------------- + +Spack provides the ``spack graph`` command for graphing dependencies. +The command by default generates an ASCII rendering of a spec's +dependency graph. For example:: + + $ spack graph mpileaks + o mpileaks + |\ + | |\ + | o | callpath + |/| | + | |\| + | |\ \ + | | |\ \ + | | | | o adept-utils + | |_|_|/| + |/| | | | + o | | | | mpi + / / / / + | | o | dyninst + | |/| | + |/|/| | + | | |/ + | o | libdwarf + |/ / + o | libelf + / + o boost + +At the top is the root package in the DAG, with dependency edges +emerging from it. On a color terminal, the edges are colored by which +dependency they lead to. + +You can also use ``spack graph`` to generate graphs in the widely used +`Dot `_ format. For +example:: + + $ spack graph --dot mpileaks + digraph G { + label = "Spack Dependencies" + labelloc = "b" + rankdir = "LR" + ranksep = "5" + + "boost" [label="boost"] + "callpath" [label="callpath"] + "libdwarf" [label="libdwarf"] + "mpileaks" [label="mpileaks"] + "mpi" [label="mpi"] + "adept-utils" [label="adept-utils"] + "dyninst" [label="dyninst"] + "libelf" [label="libelf"] + + "callpath" -> "dyninst" + "callpath" -> "adept-utils" + "callpath" -> "mpi" + "callpath" -> "libelf" + "callpath" -> "libdwarf" + "libdwarf" -> "libelf" + "mpileaks" -> "adept-utils" + "mpileaks" -> "callpath" + "mpileaks" -> "mpi" + "adept-utils" -> "boost" + "adept-utils" -> "mpi" + "dyninst" -> "boost" + "dyninst" -> "libelf" + "dyninst" -> "libdwarf" + } + +This graph can be provided as input to other graphing tools, such as +those in `Graphviz `_. + Interactive Shell Support -------------------------- diff --git a/lib/spack/spack/cmd/package-list.py b/lib/spack/spack/cmd/package-list.py index 87c528881e5..073363db0fe 100644 --- a/lib/spack/spack/cmd/package-list.py +++ b/lib/spack/spack/cmd/package-list.py @@ -49,6 +49,8 @@ def print_rst_package_list(): """Print out information on all packages in restructured text.""" pkgs = sorted(spack.db.all_packages(), key=lambda s:s.name.lower()) + print ".. _package-list:" + print print "Package List" print "==================" From daef78f53853f30d342c6f3f650f04b25d7457f2 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 23 Feb 2015 10:31:22 -0800 Subject: [PATCH 93/97] Update packaging documentataion. --- lib/spack/docs/packaging_guide.rst | 235 ++++++++++++++++-------- var/spack/packages/py-pyside/package.py | 2 - 2 files changed, 156 insertions(+), 81 deletions(-) diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index e5ebdb4cd2a..48986f80657 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -16,16 +16,17 @@ There are two key parts of Spack: software according to a spec. Specs allow a user to describe a *particular* build in a way that a -package author can understand. Packages allow a developer to -encapsulate the logic build logic for different versions, compilers, +package author can understand. Packages allow a the packager to +encapsulate the build logic for different versions, compilers, options, platforms, and dependency combinations in one place. +Essentially, a package translates a spec into build logic. Packages in Spack are written in pure Python, so you can do anything in Spack that you can do in Python. Python was chosen as the implementation language for two reasons. First, Python is becoming -ubiquitous in the HPC community due to its use in numerical codes. -Second, it's a modern language and has many powerful features to help -make package writing easy. +ubiquitous in the scientific software community. Second, it's a modern +language and has many powerful features to help make package writing +easy. Creating & Editing Packages ---------------------------------- @@ -35,24 +36,23 @@ Creating & Editing Packages ``spack create`` ~~~~~~~~~~~~~~~~~~~~~ -The ``spack create`` command generates boilerplate package template -from a URL pointing to a tarball or other software archive. In most -cases, you'll only need to run this once, then slightly modify the -boilerplate to get your package working. +The ``spack create`` command generates a boilerplate package template +from a URL. The URL should point to a tarball or other software +archive. In most cases, ``spack create`` plus a few modifications is +all you need to get a package working. -All you need is the URL to a tarball (other archive formats are ok -too) you want to package: +Here's an example: .. code-block:: sh $ spack create http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz -When you run this, Spack looks at the tarball URL and tries to figure -out the name of the package to be created. It also tries to determine -out what version strings look like for this package. Using this -information, it tries to find *additional* versions by spidering the -package's webpage. If it finds multiple versions, Spack prompts you -to tell it how many versions you want to download and checksum. +Spack examines the tarball URL and tries to figure out the name of the +package to be created. It also tries to determine what version strings +look like for this package. Using this information, it will try to +find *additional* versions by spidering the package's webpage. If it +finds multiple versions, Spack prompts you to tell it how many +versions you want to download and checksum: .. code-block:: sh @@ -63,12 +63,6 @@ to tell it how many versions you want to download and checksum. 2.8.12.1 http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz 2.8.12 http://www.cmake.org/files/v2.8/cmake-2.8.12.tar.gz 2.8.11.2 http://www.cmake.org/files/v2.8/cmake-2.8.11.2.tar.gz - 2.8.11.1 http://www.cmake.org/files/v2.8/cmake-2.8.11.1.tar.gz - 2.8.11 http://www.cmake.org/files/v2.8/cmake-2.8.11.tar.gz - 2.8.10.2 http://www.cmake.org/files/v2.8/cmake-2.8.10.2.tar.gz - 2.8.10.1 http://www.cmake.org/files/v2.8/cmake-2.8.10.1.tar.gz - 2.8.10 http://www.cmake.org/files/v2.8/cmake-2.8.10.tar.gz - 2.8.9 http://www.cmake.org/files/v2.8/cmake-2.8.9.tar.gz ... 2.8.0 http://www.cmake.org/files/v2.8/cmake-2.8.0.tar.gz @@ -77,10 +71,30 @@ to tell it how many versions you want to download and checksum. Spack will automatically download the number of tarballs you specify (starting with the most recent) and checksum each of them. -Note that you don't need to do everything up front. If your package -is large, you can always choose to download just one tarball for now, -then run :ref:`spack checksum ` later if you end up -wanting more. Let's say you choose to download 3 tarballs: +You do not *have* to download all of the versions up front. You can +always choose to download just one tarball initially, and run +:ref:`spack checksum ` later if you need more. + +.. note:: + + If ``spack create`` fails to detect the package name correctly, + you can try supplying it yourself, e.g.:: + + $ spack create --name cmake http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz + + If it fails entirely, you can get minimal boilerplate by using + :ref:`spack-edit-f`, or you can manually create a directory and + ``package.py`` file for the package in ``var/spack/packages``. + +.. note:: + + Spack can fetch packages from source code repositories, but, + ``spack create`` will *not* currently create a boilerplate package + from a repository URL. You will need to use :ref:`spack-edit-f` + and manually edit the ``version()`` directives to fetch from a + repo. See :ref:`vcs-fetch` for details. + +Let's say you download 3 tarballs: .. code-block:: sh @@ -93,8 +107,8 @@ wanting more. Let's say you choose to download 3 tarballs: ==> Fetching http://www.cmake.org/files/v2.8/cmake-2.8.11.2.tar.gz #################################################################### 95.2% -Now Spack generates boilerplate code and opens the new -``package.py`` file in your favorite ``$EDITOR``: +Now Spack generates boilerplate code and opens a new ``package.py`` +file in your favorite ``$EDITOR``: .. code-block:: python :linenos: @@ -141,12 +155,6 @@ Now Spack generates boilerplate code and opens the new The tedious stuff (creating the class, checksumming archives) has been done for you. -.. note:: - - If ``spack create`` fails to download or to detect the package - version, you can use ``spack edit -f`` to generate simpler - boilerplate. See the next section for more on this. - In the generated package, the download ``url`` attribute is already set. All the things you still need to change are marked with ``FIXME`` labels. The first ``FIXME`` refers to the commented @@ -199,27 +207,30 @@ The ``cmake`` package actually lives in a much simpler shortcut and saves you the trouble of typing the full path. - -``spack edit -f`` -~~~~~~~~~~~~~~~~~~~~ If you try to edit a package that doesn't exist, Spack will recommend -using ``spack create``: +using ``spack create`` or ``spack edit -f``: .. code-block:: sh $ spack edit foo ==> Error: No package 'foo'. Use spack create, or supply -f/--force to edit a new file. -As the output advises, You can use ``spack edit -f/--force`` to force -the creation of a new, *very* simple boilerplate package: +.. _spack-edit-f: + +``spack edit -f`` +~~~~~~~~~~~~~~~~~~~~ + +``spack edit -f`` can be used to create a new, minimal boilerplate +package: .. code-block:: sh $ spack edit -f foo -Unlike ``spack create``, which tries to infer names and versions, and -which actually downloads the tarball and checksums it for you, ``spack -edit -f`` will substitute dummy values for you to fill in yourself: +Unlike ``spack create``, which infers names and versions, and which +actually downloads the tarball and checksums it for you, ``spack edit +-f`` has no such fanciness. It will substitute dummy values for you +to fill in yourself: .. code-block:: python :linenos: @@ -246,6 +257,13 @@ version of your package from the archive URL. Naming & Directory Structure -------------------------------------- +.. note:: + + Spack's default naming and directory structure will change in + version 0.9. Specifically, 0.9 will stop using directory names + with special characters like ``@``, to avoid interfering with + certain packages that do not handle this well. + This section describes how packages need to be named, and where they live in Spack's directory structure. In general, `spack-create`_ and `spack-edit`_ handle creating package files for you, so you can skip @@ -264,6 +282,7 @@ package: .. command-output:: cd $SPACK_ROOT/var/spack/packages; ls -CF :shell: + :ellipsis: 10 Each directory contains a file called ``package.py``, which is where all the python code for the package goes. For example, the ``libelf`` @@ -280,11 +299,9 @@ Package Names Packages are named after the directory containing ``package.py``. So, ``libelf``'s ``package.py`` lives in a directory called ``libelf``. -The ``package.py`` file contains a class called ``Libelf``, which -extends Spack's ``Package`` class. This is what makes it a Spack -package: - -``var/spack/packages/libelf/package.py`` +The ``package.py`` file defines a class called ``Libelf``, which +extends Spack's ``Package`` class. for example, here is +``$SPACK_ROOT/var/spack/packages/libelf/package.py``: .. code-block:: python :linenos: @@ -301,8 +318,9 @@ package: def install(): ... -The **directory name** (``libelf``) is what users need to provide on -the command line. e.g., if you type any of these: +The **directory name** (``libelf``) determines the package name that +users should provide on the command line. e.g., if you type any of +these: .. code-block:: sh @@ -311,8 +329,8 @@ the command line. e.g., if you type any of these: Spack sees the package name in the spec and looks for ``libelf/package.py`` in ``var/spack/packages``. Likewise, if you say -``spack install docbook-xml``, then Spack looks for -``docbook-xml/package.py``. +``spack install py-numpy``, then Spack looks for +``py-numpy/package.py``. Spack uses the directory name as the package name in order to give packagers more freedom in naming their packages. Package names can @@ -342,8 +360,7 @@ some examples: ================= ================= In general, you won't have to remember this naming convention because -`spack-create`_ and `spack-edit`_ will generate boilerplate for you, -and you can just fill in the blanks. +`spack-create`_ and `spack-edit`_ handle the details for you. Adding new versions @@ -381,9 +398,8 @@ For the URL above, you might have to add an explicit URL because the version can't simply be substituted in the original ``url`` to construct the new one for ``8.2.1``. -Wehn you supply a custom URL for a version, Spack uses that URL -*verbatim* when fetching the version, and will *not* perform -extrapolation. +When you supply a custom URL for a version, Spack uses that URL +*verbatim* and does not perform extrapolation. Checksums ~~~~~~~~~~~~~~~~~ @@ -392,10 +408,11 @@ Spack uses a checksum to ensure that the downloaded package version is not corrupted or compromised. This is especially important when fetching from insecure sources, like unencrypted http. By default, a package will *not* be installed if it doesn't pass a checksum test -(though users can overried this with ``spack install --no-checksum``). +(though you can override this with ``spack install --no-checksum``). Spack can currently support checksums using the MD5, SHA-1, SHA-224, -SHA-256, SHA-384, and SHA-512 algorithms. +SHA-256, SHA-384, and SHA-512 algorithms. It determines the algorithm +to use based on the hash length. ``spack md5`` ^^^^^^^^^^^^^^^^^^^^^^ @@ -459,16 +476,18 @@ By default, Spack will search for new tarball downloads by scraping the parent directory of the tarball you gave it. So, if your tarball is at ``http://example.com/downloads/foo-1.0.tar.gz``, Spack will look in ``http://example.com/downloads/`` for links to additional versions. -If you need to search another path for download links, see the -reference documentation on `attribute_list_url`_ and +If you need to search another path for download links, you can supply +some extra attributes that control how your package finds new +versions. See the documentation on `attribute_list_url`_ and `attributee_list_depth`_. .. note:: * This command assumes that Spack can extrapolate new URLs from an existing URL in the package, and that Spack can find similar URLs - on a webpage. If that's not possible, you'll need to manually add - ``version`` calls yourself. + on a webpage. If that's not possible, e.g. if the package's + developers don't name their tarballs consistently, you'll need to + manually add ``version`` calls yourself. * For ``spack checksum`` to work, Spack needs to be able to ``import`` your pacakge in Python. That means it can't have any @@ -481,32 +500,33 @@ reference documentation on `attribute_list_url`_ and Fetching from VCS Repositories -------------------------------------- -For some packages, source code is hosted in a Version Control System -(VCS) repository rather than as a tarball. Packages can be set up to -fetch from a repository instead of a tarball. Currently, Spack -supports fetching with `Git `_, `Mercurial (hg) -`_, and `Subversion (SVN) `_. +For some packages, source code is provided in a Version Control System +(VCS) repository rather than in a tarball. Spack can fetch packages +from VCS repositories. Currently, Spack supports fetching with `Git +`_, `Mercurial (hg) `_, and `Subversion (SVN) +`_. To fetch a package from a source repository, you add a ``version()`` call to your package with parameters indicating the repository URL and -any branch, tag, or revision to fetch. See below for the paramters +any branch, tag, or revision to fetch. See below for the parameters you'll need for each VCS system. Repositories and versions ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The package author is responsible for coming up with a sensible name -for each version. For example, if you're fetching from a tag like -``v1.0``, you might call that ``1.0``. If you're fetching a nameless -git commit or an older subversion revision, you might give the commit -an intuitive name, like ``dev`` for a development version, or -``some-fancy-new-feature`` if you want to be more specific. +for each version to be fetched from a repository. For example, if +you're fetching from a tag like ``v1.0``, you might call that ``1.0``. +If you're fetching a nameless git commit or an older subversion +revision, you might give the commit an intuitive name, like ``dev`` +for a development version, or ``some-fancy-new-feature`` if you want +to be more specific. In general, it's recommended to fetch tags or particular commits/revisions, NOT branches or the repository mainline, as branches move forward over time and you aren't guaranteed to get the same thing every time you fetch a particular version. Life isn't -simple, though, so this is not strictly enforced. +always simple, though, so this is not strictly enforced. In some future release, Spack may support extrapolating repository versions as it does for tarball URLs, but currently this is not @@ -633,7 +653,7 @@ Subversion To fetch with subversion, use the ``svn`` and ``revision`` parameters: -Head +Fetching the head Simply add an ``svn`` parameter to ``version``: .. code-block:: python @@ -642,7 +662,7 @@ Head This is not recommended, as the head will move forward over time. -Revisions +Fetching a revision To fetch a particular revision, add a ``revision`` to the version call: @@ -746,6 +766,53 @@ from the URL and then applied to your source code. applies cleanly with ``-p1``, but if you're using a patch you didn't create yourself, ``level`` can be handy. +``patch()`` functions +~~~~~~~~~~~~~~~~~~~~~~~~ + +In addition to supplying patch files, you can write a custom function +to patch a package's source. For example, the ``py-pyside`` package +contains some custom code for tweaking the way the PySide build +handles ``RPATH``: + +.. code-block:: python + :linenos: + + class PyPyside(Package): + ... + + def patch(self): + """Undo PySide RPATH handling and add Spack RPATH.""" + # Figure out the special RPATH + pypkg = self.spec['python'].package + rpath = self.rpath + rpath.append(os.path.join(self.prefix, pypkg.site_packages_dir, 'PySide')) + + # Add Spack's standard CMake args to the sub-builds. + # They're called BY setup.py so we have to patch it. + filter_file( + r'OPTION_CMAKE,', + r'OPTION_CMAKE, ' + ( + '"-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=FALSE", ' + '"-DCMAKE_INSTALL_RPATH=%s",' % ':'.join(rpath)), + 'setup.py') + + # PySide tries to patch ELF files to remove RPATHs + # Disable this and go with the one we set. + filter_file( + r'^\s*rpath_cmd\(pyside_path, srcpath\)', + r'#rpath_cmd(pyside_path, srcpath)', + 'pyside_postinstall.py') + +A ``patch`` function, if present, will be run after patch files are +applied and before ``install()`` is run. + +You could put this logic in ``install()``, but putting it in a patch +function gives you some benefits. First, spack ensures that the +``patch()`` function is run once per code checkout. That means that +if you run install, hit ctrl-C, and run install again, the code in the +patch function is only run once. Also, you can tell Spack to run only the patching part of the build using the .. + + Finding Package Downloads ---------------------------- @@ -1932,6 +1999,8 @@ A typical package workflow might look like this: Below are some commands that will allow you some finer-grained controll over the install process. +.. _spack-fetch: + ``spack fetch`` ~~~~~~~~~~~~~~~~~ @@ -1944,6 +2013,8 @@ directory will be located under ``$SPACK_HOME/var/spack``. When run after the archive has already been downloaded, ``spack fetch`` is idempotent and will not download the archive again. +.. _spack-stage: + ``spack stage`` ~~~~~~~~~~~~~~~~~ @@ -1952,6 +2023,8 @@ the downloaded archive in its temporary directory, where it will be built by ``spack install``. Similar to ``fetch``, if the archive has already been expanded, ``stage`` is idempotent. +.. _spack-patch: + ``spack patch`` ~~~~~~~~~~~~~~~~~ @@ -1963,7 +2036,6 @@ this step if they have been. If Spack discovers that patches didn't apply cleanly on some previous run, then it will restage the entire package before patching. - ``spack clean`` ~~~~~~~~~~~~~~~~~ @@ -2035,6 +2107,11 @@ to get rid of the install prefix before you build again: Graphing Dependencies -------------------------- +.. _spack-graph: + +``spack graph`` +~~~~~~~~~~~~~~~~~~~ + Spack provides the ``spack graph`` command for graphing dependencies. The command by default generates an ASCII rendering of a spec's dependency graph. For example:: diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py index 6583431124c..bb5da44d027 100644 --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -43,5 +43,3 @@ def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix, '--jobs=%s' % make_jobs) - - From daa38d2ff4acd75b3b592bac5abd9ef642ee801f Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 24 Feb 2015 02:33:29 -0800 Subject: [PATCH 94/97] SPACK-59: Documentation updates, bugfix in fetching. --- lib/spack/docs/.gitignore | 1 + lib/spack/docs/Makefile | 16 +- lib/spack/docs/basic_usage.rst | 37 ++- lib/spack/docs/command_index.in | 10 + lib/spack/docs/features.rst | 2 +- lib/spack/docs/index.rst | 2 + lib/spack/docs/mirrors.rst | 217 ++++++++++++++++++ lib/spack/docs/packaging_guide.rst | 316 ++++++++++++++++++++++++-- lib/spack/docs/site_configuration.rst | 193 ---------------- lib/spack/spack/mirror.py | 2 +- lib/spack/spack/package.py | 5 +- lib/spack/spack/test/git_fetch.py | 4 +- lib/spack/spack/test/hg_fetch.py | 4 +- lib/spack/spack/test/svn_fetch.py | 4 +- 14 files changed, 587 insertions(+), 226 deletions(-) create mode 100644 lib/spack/docs/command_index.in create mode 100644 lib/spack/docs/mirrors.rst diff --git a/lib/spack/docs/.gitignore b/lib/spack/docs/.gitignore index 7701dd9f125..26c343d3eb1 100644 --- a/lib/spack/docs/.gitignore +++ b/lib/spack/docs/.gitignore @@ -1,3 +1,4 @@ package_list.rst +command_index.rst spack*.rst _build diff --git a/lib/spack/docs/Makefile b/lib/spack/docs/Makefile index a660e1255d0..00203b5b61c 100644 --- a/lib/spack/docs/Makefile +++ b/lib/spack/docs/Makefile @@ -27,6 +27,18 @@ all: html package_list: spack package-list > package_list.rst +# +# Generate a command index +# +command_index: + cp command_index.in command_index.rst + echo >> command_index.rst + grep -ho '.. _spack-.*:' *rst \ + | perl -pe 's/.. _([^:]*):/ * :ref:`\1`/' \ + | sort >> command_index.rst + +custom_targets: package_list command_index + # # This creates a git repository and commits generated html docs. # It them pushes the new branch into THIS repository as gh-pages. @@ -77,10 +89,10 @@ help: @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: - -rm -f package_list.rst + -rm -f package_list.rst command_index.rst -rm -rf $(BUILDDIR)/* $(APIDOC_FILES) -html: apidoc package_list +html: apidoc custom_targets $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index bd25d739ea8..3d808708e1f 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -16,6 +16,8 @@ software. Before that, you need to know what's available. You can see avaialble package names either using the :ref:`package-list`, or using the commands below. +.. _spack-list: + ``spack list`` ~~~~~~~~~~~~~~~~ @@ -31,6 +33,7 @@ do wildcard searches using ``*``: .. command-output:: spack list *util* +.. _spack-info: ``spack info`` ~~~~~~~~~~~~~~~~ @@ -47,6 +50,8 @@ attacks. :ref:`Dependencies ` and :ref:`virtual dependencies `, are described in more detail later. +.. _spack-versions: + ``spack versions`` ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -69,6 +74,8 @@ Installing and uninstalling Now that you know how to list avaiable packages and versions, you're ready to start installing things. +.. _spack-install: + ``spack install`` ~~~~~~~~~~~~~~~~~~~~~ @@ -138,6 +145,7 @@ configuration a **spec**. In the command lines above, both ``mpileaks`` and ``mpileaks@3.0.4`` are specs. Specs are described in detail in :ref:`sec-specs`. +.. _spack-uninstall: ``spack uninstall`` ~~~~~~~~~~~~~~~~~~~~~ @@ -170,6 +178,7 @@ Seeing installed packages We know that ``spack list`` shows you the names of available packages, but how do you figure out which are installed? +.. _spack-find: ``spack find`` ~~~~~~~~~~~~~~~~~~~~~~ @@ -303,7 +312,7 @@ of libelf would look like this: The full spec syntax is discussed in detail in :ref:`sec-specs`. -Compiler Configuration +Compiler configuration ----------------------------------- Spack has the ability to build packages with multiple compilers and @@ -311,6 +320,8 @@ compiler versions. Spack searches for compilers on your machine automatically the first time it is run. It does this by inspecting your path. +.. _spack-compilers: + ``spack compilers`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -337,6 +348,8 @@ compilers`` or ``spack compiler list``:: Any of these compilers can be used to build Spack packages. More on how this is done is in :ref:`sec-specs`. +.. _spack-compiler-add: + ``spack compiler add`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -361,6 +374,7 @@ installed, but you know that new compilers have been added to your This loads the environment module for gcc-4.9.0 to get it into the ``PATH``, and then it adds the compiler to Spack. +.. _spack-compiler-info: ``spack compiler info`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -382,7 +396,7 @@ matching Intel compiler was displayed. Manual compiler configuration -~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If autodetection fails, you can manually conigure a compiler by editing your ``~/.spackconfig`` file. You can do this by running @@ -413,7 +427,7 @@ list displayed by ``spack compilers``. .. _sec-specs: -Specs & Dependencies +Specs & dependencies ------------------------- We know that ``spack install``, ``spack uninstall``, and other @@ -720,6 +734,8 @@ any MPI implementation will do. If another package depends on error. Likewise, if you try to plug in some package that doesn't provide MPI, Spack will raise an error. +.. _spack-providers: + ``spack providers`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -739,7 +755,7 @@ versions are now filtered out. .. _shell-support: -Environment Modules +Environment modules ------------------------------- .. note:: @@ -787,6 +803,7 @@ The directories are automatically added to your ``MODULEPATH`` and ``DK_NODE`` environment variables when you enable Spack's `shell support `_. + Using Modules & Dotkits ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -934,6 +951,8 @@ if newer, fancier module support is added to Spack at some later date, you may want to regenerate all the modules to take advantage of these new features. +.. _spack-module: + ``spack module refresh`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -950,7 +969,7 @@ regenerate all module and dotkit files from scratch: .. _extensions: -Extensions & Python Support +Extensions & Python support ------------------------------------ Spack's installation model assumes that each package will live in its @@ -971,6 +990,8 @@ an *extension*. Suppose you have Python installed like so: -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- python@2.7.8 +.. _spack-extensions: + ``spack extensions`` ~~~~~~~~~~~~~~~~~~~~~~~ @@ -1004,6 +1025,7 @@ they are packages like any ohter. They are installed into their own prefixes, and you can see this with ``spack find -p``: .. code-block:: sh + $ spack find -p py-numpy ==> 1 installed packages. -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- @@ -1060,6 +1082,8 @@ for this case. Instead of requiring users to load particular environment modules, you can *activate* the package within the Python installation: +.. _spack-activate: + ``spack activate`` ^^^^^^^^^^^^^^^^^^^^^^^ @@ -1133,6 +1157,7 @@ dependencies, you can use ``spack activate -f``: $ spack activate -f py-numpy ==> Activated extension py-numpy@1.9.1%gcc@4.4.7=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. +.. _spack-deactivate: ``spack deactivate`` ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1159,6 +1184,8 @@ several variants: Getting Help ----------------------- +.. _spack-help: + ``spack help`` ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/spack/docs/command_index.in b/lib/spack/docs/command_index.in new file mode 100644 index 00000000000..94cdf381091 --- /dev/null +++ b/lib/spack/docs/command_index.in @@ -0,0 +1,10 @@ +.. _command_index: + +Command index +================= + +This is an alphabetical list of commands with links to the places they +appear in the documentation. + +.. hlist:: + :columns: 3 diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index b39dcd33906..fcb810086da 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -1,4 +1,4 @@ -Feature Overview +Feature overview ================== This is a high-level overview of features that make Spack different diff --git a/lib/spack/docs/index.rst b/lib/spack/docs/index.rst index 73eff43ab7e..2382678cc39 100644 --- a/lib/spack/docs/index.rst +++ b/lib/spack/docs/index.rst @@ -46,8 +46,10 @@ Table of Contents getting_started basic_usage packaging_guide + mirrors site_configuration developer_guide + command_index package_list API Docs diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst new file mode 100644 index 00000000000..57ca1af0681 --- /dev/null +++ b/lib/spack/docs/mirrors.rst @@ -0,0 +1,217 @@ +.. _mirrors: + +Mirrors +============================ + +Some sites may not have access to the internet for fetching packages. +These sites will need a local repository of tarballs from which they +can get their files. Spack has support for this with *mirrors*. A +mirror is a URL that points to a directory, either on the local +filesystem or on some server, containing tarballs for all of Spack's +packages. + +Here's an example of a mirror's directory structure:: + + mirror/ + cmake/ + cmake-2.8.10.2.tar.gz + dyninst/ + dyninst-8.1.1.tgz + dyninst-8.1.2.tgz + libdwarf/ + libdwarf-20130126.tar.gz + libdwarf-20130207.tar.gz + libdwarf-20130729.tar.gz + libelf/ + libelf-0.8.12.tar.gz + libelf-0.8.13.tar.gz + libunwind/ + libunwind-1.1.tar.gz + mpich/ + mpich-3.0.4.tar.gz + mvapich2/ + mvapich2-1.9.tgz + +The structure is very simple. There is a top-level directory. The +second level directories are named after packages, and the third level +contains tarballs for each package, named after each package. + +.. note:: + + Archives are **not** named exactly they were in the package's fetch + URL. They have the form ``-.``, where + ```` is Spack's name for the package, ```` is the + version of the tarball, and ```` is whatever format the + package's fetch URL contains. + + In order to make mirror creation reasonably fast, we copy the + tarball in its original format to the mirror directory, but we do + not standardize on a particular compression algorithm, because this + would potentially require expanding and recompressing each archive. + +.. _spack-mirror: + +``spack mirror`` +---------------------------- + +Mirrors are managed with the ``spack mirror`` command. The help for +``spack mirror`` looks like this:: + + $ spack mirror -h + usage: spack mirror [-h] SUBCOMMAND ... + + positional arguments: + SUBCOMMAND + create Create a directory to be used as a spack mirror, and fill + it with package archives. + add Add a mirror to Spack. + remove Remove a mirror by name. + list Print out available mirrors to the console. + + optional arguments: + -h, --help show this help message and exit + +The ``create`` command actually builds a mirror by fetching all of its +packages from the internet and checksumming them. + +The other three commands are for managing mirror configuration. They +control the URL(s) from which Spack downloads its packages. + +.. _spack-mirror-create: + +``spack mirror create`` +---------------------------- + +You can create a mirror using the ``spack mirror create`` command, assuming +you're on a machine where you can access the internet. + +The command will iterate through all of Spack's packages and download +the safe ones into a directory structure like the one above. Here is +what it looks like: + + +.. code-block:: bash + + $ spack mirror create libelf libdwarf + ==> Created new mirror in spack-mirror-2014-06-24 + ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz + ########################################################## 81.6% + ==> Checksum passed for libelf@0.8.13 + ==> Added libelf@0.8.13 + ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.12.tar.gz + ###################################################################### 98.6% + ==> Checksum passed for libelf@0.8.12 + ==> Added libelf@0.8.12 + ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130207.tar.gz + ###################################################################### 97.3% + ==> Checksum passed for libdwarf@20130207 + ==> Added libdwarf@20130207 + ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130126.tar.gz + ######################################################## 78.9% + ==> Checksum passed for libdwarf@20130126 + ==> Added libdwarf@20130126 + ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130729.tar.gz + ############################################################# 84.7% + ==> Added libdwarf@20130729 + ==> Added spack-mirror-2014-06-24/libdwarf/libdwarf-20130729.tar.gz to mirror + ==> Added python@2.7.8. + ==> Successfully updated mirror in spack-mirror-2015-02-24. + Archive stats: + 0 already present + 5 added + 0 failed to fetch. + +Once this is done, you can tar up the ``spack-mirror-2014-06-24`` directory and +copy it over to the machine you want it hosted on. + +Custom package sets +~~~~~~~~~~~~~~~~~~~~~~~ + +Normally, ``spack mirror create`` downloads all the archives it has +checksums for. If you want to only create a mirror for a subset of +packages, you can do that by supplying a list of package specs on the +command line after ``spack mirror create``. For example, this +command:: + + $ spack mirror create libelf@0.8.12: boost@1.44: + +Will create a mirror for libelf versions greater than or equal to +0.8.12 and boost versions greater than or equal to 1.44. + +Mirror files +~~~~~~~~~~~~~~~~~~~~~~~ + +If you have a *very* large number of packages you want to mirror, you +can supply a file with specs in it, one per line:: + + $ cat specs.txt + libdwarf + libelf@0.8.12: + boost@1.44: + boost@1.39.0 + ... + $ spack mirror create -f specs.txt + ... + +This is useful if there is a specific suite of software managed by +your site. + +.. _spack-mirror-add: + +``spack mirror add`` +---------------------------- + +Once you have a mirrror, you need to let spack know about it. This is +relatively simple. First, figure out the URL for the mirror. If it's +a file, you can use a file URL like this one:: + + file:///Users/gamblin2/spack-mirror-2014-06-24 + +That points to the directory on the local filesystem. If it were on a +web server, you could use a URL like this one: + + https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 + +Spack will use the URL as the root for all of the packages it fetches. +You can tell your Spack installation to use that mirror like this: + +.. code-block:: bash + + $ spack mirror add local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 + +Each mirror has a name so that you can refer to it again later. + +.. _spack-mirror-list: + +``spack mirror list`` +---------------------------- + +If you want to see all the mirrors Spack knows about you can run ``spack mirror list``:: + + $ spack mirror list + local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 + +.. _spack-mirror-remove: + +``spack mirror remove`` +---------------------------- + +And, if you want to remove a mirror, just remove it by name:: + + $ spack mirror remove local_filesystem + $ spack mirror list + ==> No mirrors configured. + +Mirror precedence +---------------------------- + +Adding a mirror really just adds a section in ``~/.spackconfig``:: + + [mirror "local_filesystem"] + url = file:///Users/gamblin2/spack-mirror-2014-06-24 + [mirror "remote_server"] + url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 + +If you want to change the order in which mirrors are searched for +packages, you can edit this file and reorder the sections. Spack will +search the topmost mirror first and the bottom-most mirror last. diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index bc3bacf8f21..8b4c0a4ce17 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -28,7 +28,7 @@ ubiquitous in the scientific software community. Second, it's a modern language and has many powerful features to help make package writing easy. -Creating & Editing Packages +Creating & editing packages ---------------------------------- .. _spack-create: @@ -254,7 +254,7 @@ This is useful when ``spack create`` cannot figure out the name and version of your package from the archive URL. -Naming & Directory Structure +Naming & directory structure -------------------------------------- .. note:: @@ -497,7 +497,7 @@ versions. See the documentation on `attribute_list_url`_ and .. _vcs-fetch: -Fetching from VCS Repositories +Fetching from VCS repositories -------------------------------------- For some packages, source code is provided in a Version Control System @@ -774,6 +774,8 @@ to patch a package's source. For example, the ``py-pyside`` package contains some custom code for tweaking the way the PySide build handles ``RPATH``: +.. _pyside-patch: + .. code-block:: python :linenos: @@ -810,14 +812,59 @@ You could put this logic in ``install()``, but putting it in a patch function gives you some benefits. First, spack ensures that the ``patch()`` function is run once per code checkout. That means that if you run install, hit ctrl-C, and run install again, the code in the -patch function is only run once. Also, you can tell Spack to run only the patching part of the build using the .. +patch function is only run once. Also, you can tell Spack to run only +the patching part of the build using the :ref:`spack-patch` command. - - -Finding Package Downloads +Handling RPATHs ---------------------------- -We've already seen the ``homepage`` and ``url`` package attributes: +Spack installs each package in a way that ensures that all of its +dependencies are found when it runs. It does this using `RPATHs +`_. An RPATH is a search +path, stored in a binary (an executable or library), that tells the +dynamic loader where to find its dependencies at runtime. You may be +familiar with ```LD_LIBRARY_PATH`` +`_ +on Linux or ```DYLD_LIBRARY_PATH`` +` +on Mac OS X. RPATH is similar to these paths, in that it tells +the loader where to find libraries. Unlike them, it is embedded in +the binary and not set in each user's environment. + +RPATHs in Spack are handled in one of three ways: + + 1. For most packages, RPATHs are handled automatically using Spack's + :ref:`compiler wrappers `. These wrappers are + set in standard variables like ``CC``, ``CXX``, and ``FC``, so + most build systems (autotools and many gmake systems) pick them + up and use them. + 2. CMake also respects Spack's compiler wrappers, but many CMake + builds have logic to overwrite RPATHs when binaries are + installed. Spack provides the ``std_cmake_args`` variable, which + includes parameters necessary for CMake build use the right + installation RPATH. It can be used like this when ``cmake`` is + invoked: + + .. code-block:: python + + class MyPackage(Package): + ... + def install(self, spec, prefix): + cmake('..', *std_cmake_args) + make() + make('install') + + 3. If you need to modify the build to add your own RPATHs, you can + use the ``self.rpath`` property of your package, which will + return a list of all the RPATHs that Spack will use when it + links. You can see this how this is used in the :ref:`PySide + example ` above. + + +Finding new versions +---------------------------- + +You've already seen the ``homepage`` and ``url`` package attributes: .. code-block:: python :linenos: @@ -853,7 +900,7 @@ url is: url = "http://www.mr511.de/software/libelf-0.8.13.tar.gz" -Spack spiders ``http://www.mr511.de/software/`` to find similar +Here, Spack spiders ``http://www.mr511.de/software/`` to find similar tarball links and ultimately to make a list of available versions of ``libelf``. @@ -907,7 +954,7 @@ when spidering the page. .. _attribute_parallel: -Parallel Builds +Parallel builds ------------------ By default, Spack will invoke ``make()`` with a ``-j `` @@ -1036,6 +1083,203 @@ command line to find installed packages or to install packages with particular constraints, and package authors can use specs to describe relationships between packages. +.. _setup-dependent-environment: + +``setup_dependent_environment()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spack provides a mechanism for dependencies to provide variables that +can be used in their dependents' build. Any package can declare a +``setup_dependent_environment()`` function, and this function will be +called before the ``install()`` method of any dependent packages. +This allows dependencies to set up environment variables and other +properties to be used by dependents. + +The funciton declaration should look like this: + +.. code-block:: python + + class Qt(Package): + ... + def setup_dependent_environment(self, module, spec, dep_spec): + """Dependencies of Qt find it using the QTDIR environment variable.""" + os.environ['QTDIR'] = self.prefix + +Here, the Qt package sets the ``QTDIR`` environment variable so that +packages that depend on a particular Qt installation will find it. + +The arguments to this function are: + + * **module**: the module of the dependent package, where global + properties can be assigned. + * **spec**: the spec of the *dependency package* (the one the function is called on). + * **dep_spec**: the spec of the dependent package (i.e. dep_spec depends on spec). + +A goo example of using these is in the Python packge: + +.. code-block:: python + + def setup_dependent_environment(self, module, spec, dep_spec): + # Python extension builds can have a global python executable function + module.python = Executable(join_path(spec.prefix.bin, 'python')) + + # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. + module.python_lib_dir = os.path.join(dep_spec.prefix, self.python_lib_dir) + module.python_include_dir = os.path.join(dep_spec.prefix, self.python_include_dir) + module.site_packages_dir = os.path.join(dep_spec.prefix, self.site_packages_dir) + + # Make the site packages directory if it does not exist already. + mkdirp(module.site_packages_dir) + + # Set PYTHONPATH to include site-packages dir for the + # extension and any other python extensions it depends on. + python_paths = [] + for d in dep_spec.traverse(): + if d.package.extends(self.spec): + python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + os.environ['PYTHONPATH'] = ':'.join(python_paths) + +The first thing that happens here is that the ``python`` command is +inserted into module scope of the dependent. This allows most python +packages to have a very simple install method, like this: + +.. code-block:: python + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + +Python's ``setup_dependent_environment`` method also sets up smoe +other variables, creates a directory, and sets up the ``PYTHONPATH`` +so that dependent packages can find their dependencies at build time. + + +.. _packaging_extensions: + +Extensions +------------------------- + +Spack's support for package extensions is documented extensively in +:ref:`extensions`. This section documents how to make your own +extendable packages and extensions. + +To support extensions, a package needs to set its ``extendable`` +property to ``True``, e.g.: + +.. code-block:: python + + class Python(Package): + ... + extendable = True + ... + +To make a package into an extension, simply add simply add an +``extends`` call in the package definition, and pass it the name of an +extendable package: + +.. code-block:: python + + class PyNumpy(Package): + ... + extends('python') + ... + +Now, the ``py-numpy`` package can be used as an argument to ``spack +activate``. When it is activated, all the files in its prefix will be +symbolically linked into the prefix of the python package. + +Sometimes, certain files in one package will conflict with those in +another, which means they cannot both be activated (symlinked) at the +same time. In this case, you can tell Spack to ignore those files +when it does the activation: + +.. code-block:: python + + class PyNose(Package): + ... + extends('python', ignore=r'bin/nosetests.*$') + ... + +The code above will prevent ``$prefix/bin/nosetests`` from being +linked in at activation time. + +.. note:: + + You can call *either* ``depends_on`` or ``extends`` on any one + package, but not both. For example you cannot both + ``depends_on('python')`` and ``extends(python)`` in the same + package. ``extends`` implies ``depends_on``. + + + +Activation & deactivation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Spack's ``Package`` class has default ``activate`` and ``deactivate`` +implementations that handle symbolically linking extensions' prefixes +into the directory of the parent package. However, extendable +packages can override these methdos to add custom activate/deactivate +logic of their own. For example, the ``activate`` and ``deactivate`` +methods in the Python class use the symbolic linking, but they also +handle details surrounding Python's ``.pth`` files, and other aspects +of Python packaging. + +Spack's extensions mechanism is designed to be extensible, so that +other packages (like Ruby, R, Perl, etc.) can provide their own +custom extension management logic, as they may not handle modules the +same way that Python does. + +Let's look at Python's activate function: + +.. code-block:: python + + def activate(self, ext_pkg, **kwargs): + kwargs.update(ignore=self.python_ignore(ext_pkg, kwargs)) + super(Python, self).activate(ext_pkg, **kwargs) + + exts = spack.install_layout.extension_map(self.spec) + exts[ext_pkg.name] = ext_pkg.spec + self.write_easy_install_pth(exts) + +This function is called on the *extendee* (Python). It first calls +``activate`` in the superclass, which handles symlinking the +extension package's prefix into this package's prefix. It then does +some special handling of the ``easy-install.pth`` file, part of +Python's setuptools. + +Deactivate behaves similarly to activate, but it unlinks files: + +.. code-block:: python + + def deactivate(self, ext_pkg, **kwargs): + kwargs.update(ignore=self.python_ignore(ext_pkg, kwargs)) + super(Python, self).deactivate(ext_pkg, **kwargs) + + exts = spack.install_layout.extension_map(self.spec) + if ext_pkg.name in exts: # Make deactivate idempotent. + del exts[ext_pkg.name] + self.write_easy_install_pth(exts) + +Both of these methods call some custom functions in the Python +package. See the source for Spack's Python package for details. + + +Activation arguments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You may have noticed that the ``activate`` function defined above +takes keyword arguments. These are the keyword arguments from +``extends()``, and they are passed to both activate and deactivate. + +This capability allows an extension to customize its own activation by +passing arguments to the extendee. Extendees can likewise implement +custom ``activate()`` and ``deactivate()`` functions to suit their +needs. + +The only keyword argument supported by default is the ``ignore`` +argument, which can take a regex, list of regexes, or a predicate to +determine which files *not* to symlink during activation. + + .. _virtual-dependencies: Virtual dependencies @@ -1257,6 +1501,7 @@ explicitly. Concretization policies are discussed in more detail in :ref:`site-configuration`. Sites using Spack can customize them to match the preferences of their own users. +.. _spack-spec: ``spack spec`` ~~~~~~~~~~~~~~~~~~~~ @@ -1354,7 +1599,7 @@ information. .. _install-environment: -The Install environment +The install environment -------------------------- In general, you should not have to do much differently in your install @@ -1414,6 +1659,7 @@ easily: ``PATH`` Set to point to ``/bin`` directories of dpeendencies ``CMAKE_PREFIX_PATH`` Path to dependency prefixes for CMake ``PKG_CONFIG_PATH`` Path to any pkgconfig directories for dependencies + ``PYTHONPATH`` Path to site-packages dir of any python dependencies ======================= ============================= ``PATH`` is set up to point to dependencies ``/bin`` directories so @@ -1433,6 +1679,12 @@ dependencies using the GNU ``pkg-config`` tool. It is similar to ``CMAKE_PREFIX_PATH`` in that it allows a build to automatically discover its dependencies. +If you want to see the environment that a package will build with, or +if you want to run commands in that environment to test them out, you +can use the :ref:```spack env`` ` command, documented +below. + +.. _compiler-wrappers: Compiler interceptors ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1498,7 +1750,6 @@ process runs. Packages are free to change the environment or to modify Spack internals, because each ``install()`` call has its own dedicated process. - .. _prefix-objects: Prefix objects @@ -1970,7 +2221,7 @@ File functions .. _pacakge-lifecycle: -Package Workflow Commands +Packaging workflow commands --------------------------------- When you are building packages, you will likely not get things @@ -2036,6 +2287,8 @@ this step if they have been. If Spack discovers that patches didn't apply cleanly on some previous run, then it will restage the entire package before patching. +.. _spack-restage: + ``spack restage`` ~~~~~~~~~~~~~~~~~ Restores the source code to pristine state, as it was before building. @@ -2048,6 +2301,7 @@ Does this in one of two ways: 2. If the source was checked out from a repository, this deletes the build directory and checks it out again. +.. _spack-clean: ``spack clean`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2057,6 +2311,8 @@ expanded/checked out source code *and* any downloaded archive. If build process will start from scratch. +.. _spack-purge: + ``spack purge`` ~~~~~~~~~~~~~~~~~ Cleans up all of Spack's temporary files. Use this to recover disk @@ -2102,7 +2358,7 @@ to get rid of the install prefix before you build again: spack uninstall -f -Graphing Dependencies +Graphing dependencies -------------------------- .. _spack-graph: @@ -2181,7 +2437,7 @@ example:: This graph can be provided as input to other graphing tools, such as those in `Graphviz `_. -Interactive Shell Support +Interactive shell support -------------------------- Spack provides some limited shell support to make life easier for @@ -2197,6 +2453,7 @@ For ``csh`` and ``tcsh`` run: ``spack cd`` will then be available. +.. _spack-cd: ``spack cd`` ~~~~~~~~~~~~~~~~~ @@ -2227,6 +2484,35 @@ directory, install directory, package directory) and others change to core spack locations. For example, ``spack cd -m`` will take you to the main python source directory of your spack install. +.. _spack-env: + +``spack env`` +~~~~~~~~~~~~~~~~~~~~~~ + +``spack env`` functions much like the standard unix ``env`` command, +but it takes a spec as an argument. You can use it to see the +environment variables that will be set when a particular build runs, +for example: + +.. code-block:: sh + + $ spack env mpileaks@1.1%intel + +This will display the entire environment that will be set when the +``mpileaks@1.1%intel`` build runs. + +To run commands in a package's build environment, you can simply provided them after the spec argument to ``spack env``: + +.. code-block:: sh + + $ spack cd mpileaks@1.1%intel + $ spack env mpileaks@1.1%intel ./configure + +This will cd to the build directory and then run ``configure`` in the +package's build environment. + + +.. _spack-location: ``spack location`` ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index 4936e3052c1..a3e19cc8006 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -3,199 +3,6 @@ Site-specific configuration =================================== -.. _mirrors: - -Mirrors ----------------------------- - -Some sites may not have access to the internet for fetching packages. -These sites will need a local repository of tarballs from which they -can get their files. Spack has support for this with *mirrors*. A -mirror is a URL that points to a directory, either on the local -filesystem or on some server, containing tarballs for all of Spack's -packages. - -Here's an example of a mirror's directory structure:: - - mirror/ - cmake/ - cmake-2.8.10.2.tar.gz - dyninst/ - DyninstAPI-8.1.1.tgz - DyninstAPI-8.1.2.tgz - libdwarf/ - libdwarf-20130126.tar.gz - libdwarf-20130207.tar.gz - libdwarf-20130729.tar.gz - libelf/ - libelf-0.8.12.tar.gz - libelf-0.8.13.tar.gz - libunwind/ - libunwind-1.1.tar.gz - mpich/ - mpich-3.0.4.tar.gz - mvapich2/ - mvapich2-1.9.tgz - -The structure is very simple. There is a top-level directory. The -second level directories are named after packages, and the third level -contains tarballs for each package, named as they were in the -package's fetch URL. - -``spack mirror`` -~~~~~~~~~~~~~~~~~~~~~~~ - -Mirrors are managed with the ``spack mirror`` command. The help for -``spack mirror`` looks like this:: - - $ spack mirror -h - usage: spack mirror [-h] SUBCOMMAND ... - - positional arguments: - SUBCOMMAND - create Create a directory to be used as a spack mirror, and fill - it with package archives. - add Add a mirror to Spack. - remove Remove a mirror by name. - list Print out available mirrors to the console. - - optional arguments: - -h, --help show this help message and exit - -The ``create`` command actually builds a mirror by fetching all of its -packages from the internet and checksumming them. - -The other three commands are for managing mirror configuration. They -control the URL(s) from which Spack downloads its packages. - - -``spack mirror create`` -~~~~~~~~~~~~~~~~~~~~~~~ - -You can create a mirror using the ``spack mirror create`` command, assuming -you're on a machine where you can access the internet. - -The command will iterate through all of Spack's packages and download -the safe ones into a directory structure like the one above. Here is -what it looks like: - - -.. code-block:: bash - - $ spack mirror create libelf libdwarf - ==> Created new mirror in spack-mirror-2014-06-24 - ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz - ########################################################## 81.6% - ==> Checksum passed for libelf@0.8.13 - ==> Added spack-mirror-2014-06-24/libelf/libelf-0.8.13.tar.gz to mirror - ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.12.tar.gz - ###################################################################### 98.6% - ==> Checksum passed for libelf@0.8.12 - ==> Added spack-mirror-2014-06-24/libelf/libelf-0.8.12.tar.gz to mirror - ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130207.tar.gz - ###################################################################### 97.3% - ==> Checksum passed for libdwarf@20130207 - ==> Added spack-mirror-2014-06-24/libdwarf/libdwarf-20130207.tar.gz to mirror - ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130126.tar.gz - ######################################################## 78.9% - ==> Checksum passed for libdwarf@20130126 - ==> Added spack-mirror-2014-06-24/libdwarf/libdwarf-20130126.tar.gz to mirror - ==> Trying to fetch from http://www.prevanders.net/libdwarf-20130729.tar.gz - ############################################################# 84.7% - ==> Checksum passed for libdwarf@20130729 - ==> Added spack-mirror-2014-06-24/libdwarf/libdwarf-20130729.tar.gz to mirror - -Once this is done, you can tar up the ``spack-mirror-2014-06-24`` directory and -copy it over to the machine you want it hosted on. - -Custom package sets -^^^^^^^^^^^^^^^^^^^^^^^^ - -Normally, ``spack mirror create`` downloads all the archives it has -checksums for. If you want to only create a mirror for a subset of -packages, you can do that by supplying a list of package specs on the -command line after ``spack mirror create``. For example, this -command:: - - $ spack mirror create libelf@0.8.12: boost@1.44: - -Will create a mirror for libelf versions greater than or equal to -0.8.12 and boost versions greater than or equal to 1.44. - -Mirror files -^^^^^^^^^^^^^^^^^^^^^^^^ - -If you have a *very* large number of packages you want to mirror, you -can supply a file with specs in it, one per line:: - - $ cat specs.txt - libdwarf - libelf@0.8.12: - boost@1.44: - boost@1.39.0 - ... - $ spack mirror create -f specs.txt - ... - -This is useful if there is a specific suite of software managed by -your site. - - -``spack mirror add`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Once you have a mirrror, you need to let spack know about it. This is -relatively simple. First, figure out the URL for the mirror. If it's -a file, you can use a file URL like this one:: - - file:///Users/gamblin2/spack-mirror-2014-06-24 - -That points to the directory on the local filesystem. If it were on a -web server, you could use a URL like this one: - - https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 - -Spack will use the URL as the root for all of the packages it fetches. -You can tell your Spack installation to use that mirror like this: - -.. code-block:: bash - - $ spack mirror add local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 - -Each mirror has a name so that you can refer to it again later. - -``spack mirror list`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want to see all the mirrors Spack knows about you can run ``spack mirror list``:: - - $ spack mirror list - local_filesystem file:///Users/gamblin2/spack-mirror-2014-06-24 - -``spack mirror remove`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -And, if you want to remove a mirror, just remove it by name:: - - $ spack mirror remove local_filesystem - $ spack mirror list - ==> No mirrors configured. - -Mirror precedence -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Adding a mirror really just adds a section in ``~/.spackconfig``:: - - [mirror "local_filesystem"] - url = file:///Users/gamblin2/spack-mirror-2014-06-24 - [mirror "remote_server"] - url = https://example.com/some/web-hosted/directory/spack-mirror-2014-06-24 - -If you want to change the order in which mirrors are searched for -packages, you can edit this file and reorder the sections. Spack will -search the topmost mirror first and the bottom-most mirror last. - - .. _temp-space: Temporary space diff --git a/lib/spack/spack/mirror.py b/lib/spack/spack/mirror.py index 114c7b6a353..306c8085aa5 100644 --- a/lib/spack/spack/mirror.py +++ b/lib/spack/spack/mirror.py @@ -146,7 +146,7 @@ def create(path, specs, **kwargs): stage = None try: # create a subdirectory for the current package@version - archive_path = join_path(path, mirror_archive_path(spec)) + archive_path = os.path.abspath(join_path(path, mirror_archive_path(spec))) subdir = os.path.dirname(archive_path) mkdirp(subdir) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 137e8f88373..75e6142a9d2 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -287,10 +287,9 @@ class SomePackage(Package): .. code-block:: python - p.do_clean() # runs make clean - p.do_clean_work() # removes the build directory and + p.do_clean() # removes the stage directory entirely + p.do_restage() # removes the build directory and # re-expands the archive. - p.do_clean_dist() # removes the stage directory entirely The convention used here is that a do_* function is intended to be called internally by Spack commands (in spack.cmd). These aren't for package diff --git a/lib/spack/spack/test/git_fetch.py b/lib/spack/spack/test/git_fetch.py index f6d9bfcf058..04422adb57e 100644 --- a/lib/spack/spack/test/git_fetch.py +++ b/lib/spack/spack/test/git_fetch.py @@ -61,7 +61,7 @@ def tearDown(self): if self.repo.stage is not None: self.repo.stage.destroy() - self.pkg.do_clean_dist() + self.pkg.do_clean() def assert_rev(self, rev): @@ -93,7 +93,7 @@ def try_fetch(self, rev, test_file, args): untracked_file = 'foobarbaz' touch(untracked_file) self.assertTrue(os.path.isfile(untracked_file)) - self.pkg.do_clean_work() + self.pkg.do_restage() self.assertFalse(os.path.isfile(untracked_file)) self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) diff --git a/lib/spack/spack/test/hg_fetch.py b/lib/spack/spack/test/hg_fetch.py index 97c5b665e74..e1ab2cffe6b 100644 --- a/lib/spack/spack/test/hg_fetch.py +++ b/lib/spack/spack/test/hg_fetch.py @@ -60,7 +60,7 @@ def tearDown(self): if self.repo.stage is not None: self.repo.stage.destroy() - self.pkg.do_clean_dist() + self.pkg.do_clean() def try_fetch(self, rev, test_file, args): @@ -87,7 +87,7 @@ def try_fetch(self, rev, test_file, args): untracked = 'foobarbaz' touch(untracked) self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_clean_work() + self.pkg.do_restage() self.assertFalse(os.path.isfile(untracked)) self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) diff --git a/lib/spack/spack/test/svn_fetch.py b/lib/spack/spack/test/svn_fetch.py index a48a86dcc3a..0159fb087fa 100644 --- a/lib/spack/spack/test/svn_fetch.py +++ b/lib/spack/spack/test/svn_fetch.py @@ -60,7 +60,7 @@ def tearDown(self): if self.repo.stage is not None: self.repo.stage.destroy() - self.pkg.do_clean_dist() + self.pkg.do_clean() def assert_rev(self, rev): @@ -99,7 +99,7 @@ def try_fetch(self, rev, test_file, args): untracked = 'foobarbaz' touch(untracked) self.assertTrue(os.path.isfile(untracked)) - self.pkg.do_clean_work() + self.pkg.do_restage() self.assertFalse(os.path.isfile(untracked)) self.assertTrue(os.path.isdir(self.pkg.stage.source_path)) From 447e2959476b84b317dbc5acba623a66968aa7c4 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 24 Feb 2015 02:45:21 -0800 Subject: [PATCH 95/97] SPACK-62: fix for site docs. --- lib/spack/docs/site_configuration.rst | 52 ++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index a3e19cc8006..e076aec7638 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -1,6 +1,6 @@ .. _site-configuration: -Site-specific configuration +Site configuration =================================== .. _temp-space: @@ -134,3 +134,53 @@ Set concretizer to *your own* class instead of the default: concretizer = MyConcretizer() The next time you run Spack, your changes should take effect. + + +Profiling +~~~~~~~~~~~~~~~~~~~~~ + +Spack has some limited builtin support for profiling, and can report +statistics using standard Python timing tools. To use this feature, +supply ``-p`` to Spack on the command line, before any subcommands. + +.. _spack-p: + +``spack -p`` +^^^^^^^^^^^^^^^^^^ + +``spack -p`` output looks like this: + +.. code-block:: sh + + $ spack -p graph dyninst + o dyninst + |\ + | |\ + | o | libdwarf + |/ / + o | libelf + / + o boost + + 307670 function calls (305943 primitive calls) in 0.127 seconds + + Ordered by: internal time + + ncalls tottime percall cumtime percall filename:lineno(function) + 853 0.021 0.000 0.066 0.000 inspect.py:472(getmodule) + 51197 0.011 0.000 0.018 0.000 inspect.py:51(ismodule) + 73961 0.010 0.000 0.010 0.000 {isinstance} + 1762 0.006 0.000 0.053 0.000 inspect.py:440(getsourcefile) + 32075 0.006 0.000 0.006 0.000 {hasattr} + 1760 0.004 0.000 0.004 0.000 {posix.stat} + 2240 0.004 0.000 0.004 0.000 {posix.lstat} + 2602 0.004 0.000 0.011 0.000 inspect.py:398(getfile) + 771 0.004 0.000 0.077 0.000 inspect.py:518(findsource) + 2656 0.004 0.000 0.004 0.000 {method 'match' of '_sre.SRE_Pattern' objects} + 30772 0.003 0.000 0.003 0.000 {method 'get' of 'dict' objects} + ... + +The bottom of the output shows the top most time consuming functions, +slowest on top. The profiling support is from Python's builtin tool, +`cProfile +`_. From 5eb7e466545ffc2e954a283074ec661bca87afa3 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 24 Feb 2015 10:26:26 -0800 Subject: [PATCH 96/97] Spell check docs --- lib/spack/docs/basic_usage.rst | 22 ++++++------ lib/spack/docs/developer_guide.rst | 10 +++--- lib/spack/docs/mirrors.rst | 4 +-- lib/spack/docs/packaging_guide.rst | 52 +++++++++++++-------------- lib/spack/docs/site_configuration.rst | 8 ++--- 5 files changed, 48 insertions(+), 48 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 3d808708e1f..ed79790bb93 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -13,7 +13,7 @@ Listing available packages The first thing you likely want to do with spack is to install some software. Before that, you need to know what's available. You can -see avaialble package names either using the :ref:`package-list`, or +see available package names either using the :ref:`package-list`, or using the commands below. .. _spack-list: @@ -27,7 +27,7 @@ Spack can install: .. command-output:: spack list The packages are listed by name in alphabetical order. You can also -do wildcard searches using ``*``: +do wildcats searches using ``*``: .. command-output:: spack list m* @@ -64,14 +64,14 @@ There are two sections in the output. *Safe versions* are ones that have already been checksummed. Spack goes a step further, though, and also shows you what versions are available out on the web---these are *remote versions*. Spack gets this information by scraping it -directly from webpages. Depending on the package, Spack may or may +directly from web pages. Depending on the package, Spack may or may not be able to find any remote versions. Installing and uninstalling ------------------------------ -Now that you know how to list avaiable packages and versions, you're +Now that you know how to list available packages and versions, you're ready to start installing things. .. _spack-install: @@ -80,7 +80,7 @@ ready to start installing things. ~~~~~~~~~~~~~~~~~~~~~ ``spack install`` will install any package shown by ``spack list``. -To install the latest version of a pacakge, along with all of its +To install the latest version of a package, along with all of its dependencies, simply give it a package name: .. code-block:: sh @@ -227,7 +227,7 @@ Packages are divided into groups according to their architecture and compiler. Within each group, Spack tries to keep the view simple, and only shows the version of installed packages. -In some cases, there may be differnt configurations of the *same* +In some cases, there may be different configurations of the *same* version of a package installed. For example, there are two installations of of ``libdwarf@20130729`` above. We can look at them in more detail using ``spack find -d``, and by asking only to show @@ -362,7 +362,7 @@ where the compiler is installed. For example:: intel@13.0.079 Or you can run ``spack compiler add`` with no arguments to force -autodetection. This is useful if you do not know where compilers are +auto-detection. This is useful if you do not know where compilers are installed, but you know that new compilers have been added to your ``PATH``. For example, using dotkit, you might do this:: @@ -398,7 +398,7 @@ matching Intel compiler was displayed. Manual compiler configuration ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If autodetection fails, you can manually conigure a compiler by +If auto-detection fails, you can manually configure a compiler by editing your ``~/.spackconfig`` file. You can do this by running ``spack config edit``, which will open the file in your ``$EDITOR``. @@ -1021,7 +1021,7 @@ You can find extensions for your Python installation like this: ==> None activated. The extensions are a subset of what's returned by ``spack list``, and -they are packages like any ohter. They are installed into their own +they are packages like any other. They are installed into their own prefixes, and you can see this with ``spack find -p``: .. code-block:: sh @@ -1098,7 +1098,7 @@ Several things have happened here. The user requested that ``py-numpy`` be activated in the ``python`` installation it was built with. Spack knows that ``py-numpy`` depends on ``py-nose`` and ``py-setuptools``, so it activated those packages first. Finally, -once all dpeendencies were activated in the ``python`` installation, +once all dependencies were activated in the ``python`` installation, ``py-numpy`` was activated as well. If we run ``spack extensions`` again, we now see the three new @@ -1130,7 +1130,7 @@ packages listed as activated: py-nose@1.3.4 py-numpy@1.9.1 py-setuptools@11.3.1 -Now, when a user runs python, ``numpy`` will be avaiable for import +Now, when a user runs python, ``numpy`` will be available for import *without* the user having to explicitly loaded. ``python@2.7.8`` now acts like a system Python installation with ``numpy`` installed inside of it. diff --git a/lib/spack/docs/developer_guide.rst b/lib/spack/docs/developer_guide.rst index 969ed60b151..db47de80f59 100644 --- a/lib/spack/docs/developer_guide.rst +++ b/lib/spack/docs/developer_guide.rst @@ -50,11 +50,11 @@ as a descriptor for one or more instances of that template. Users express the configuration they want using a spec, and a package turns the spec into a complete build. -The obvious difficulty with this design is that users underspecify +The obvious difficulty with this design is that users under-specify what they want. To build a software package, the package object needs a *complete* specification. In Spack, if a spec describes only one instance of a package, then we say it is **concrete**. If a spec -could describes many instances, (i.e. it is underspecified in one way +could describes many instances, (i.e. it is under-specified in one way or another), then we say it is **abstract**. Spack's job is to take an *abstract* spec from the user, find a @@ -92,7 +92,7 @@ with a high level view of Spack's directory structure:: Spack is designed so that it could live within a `standard UNIX directory hierarchy `_, so ``lib``, ``var``, and ``opt`` all contain a ``spack`` subdirectory in case -Spack is installed alongside other software. Most of the insteresting +Spack is installed alongside other software. Most of the interesting parts of Spack live in ``lib/spack``. Files under ``var`` are created as needed, so there is no ``var`` directory when you initially clone Spack from the repository. @@ -123,13 +123,13 @@ Package-related modules Contains the :class:`Package ` class, which is the superclass for all packages in Spack. Methods on ``Package`` implement all phases of the :ref:`package lifecycle - ` and manage the build process. + ` and manage the build process. :mod:`spack.packages` Contains all of the packages in Spack and methods for managing them. Functions like :func:`packages.get ` and :func:`class_name_for_package_name - ` handle mapping packge module + ` handle mapping package module names to class names and dynamically instantiating packages by name from module files. diff --git a/lib/spack/docs/mirrors.rst b/lib/spack/docs/mirrors.rst index 57ca1af0681..d732a3dd54f 100644 --- a/lib/spack/docs/mirrors.rst +++ b/lib/spack/docs/mirrors.rst @@ -47,7 +47,7 @@ contains tarballs for each package, named after each package. In order to make mirror creation reasonably fast, we copy the tarball in its original format to the mirror directory, but we do not standardize on a particular compression algorithm, because this - would potentially require expanding and recompressing each archive. + would potentially require expanding and re-compressing each archive. .. _spack-mirror: @@ -161,7 +161,7 @@ your site. ``spack mirror add`` ---------------------------- -Once you have a mirrror, you need to let spack know about it. This is +Once you have a mirror, you need to let spack know about it. This is relatively simple. First, figure out the URL for the mirror. If it's a file, you can use a file URL like this one:: diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 8b4c0a4ce17..59ba63fa355 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -5,7 +5,7 @@ Packaging Guide This guide is intended for developers or administrators who want to package software so that Spack can install it. It assumes that you -have at least some familiarty with Python, and that you've read the +have at least some familiarity with Python, and that you've read the :ref:`basic usage guide `, especially the part about :ref:`specs `. @@ -459,7 +459,7 @@ example for ``libelf``: How many would you like to checksum? (default is 5, q to abort) This does the same thing that ``spack create`` does, but it allows you -to go back and add new vesrions easily as you need them (e.g., as +to go back and add new versions easily as you need them (e.g., as they're released). It fetches the tarballs you ask for and prints out a list of ``version`` commands ready to copy/paste into your package file: @@ -479,7 +479,7 @@ in ``http://example.com/downloads/`` for links to additional versions. If you need to search another path for download links, you can supply some extra attributes that control how your package finds new versions. See the documentation on `attribute_list_url`_ and -`attributee_list_depth`_. +`attribute_list_depth`_. .. note:: @@ -490,7 +490,7 @@ versions. See the documentation on `attribute_list_url`_ and manually add ``version`` calls yourself. * For ``spack checksum`` to work, Spack needs to be able to - ``import`` your pacakge in Python. That means it can't have any + ``import`` your package in Python. That means it can't have any syntax errors, or the ``import`` will fail. Use this once you've got your package in working order. @@ -643,7 +643,7 @@ Revisions revisions, you can use ``revision`` for branches, tags, and commits when you fetch with Mercurial. -As wtih git, you can fetch these versions using the ``spack install +As with git, you can fetch these versions using the ``spack install example@`` command-line syntax. .. _svn-fetch: @@ -927,7 +927,7 @@ the ``list_url``, because that is where links to old versions are: ~~~~~~~~~~~~~~~~~~~~~ ``libdwarf`` and many other packages have a listing of available -verisons on a single webpage, but not all do. For example, ``mpich`` +versions on a single webpage, but not all do. For example, ``mpich`` has a tarball URL that looks like this: url = "http://www.mpich.org/static/downloads/3.0.4/mpich-3.0.4.tar.gz" @@ -1095,7 +1095,7 @@ called before the ``install()`` method of any dependent packages. This allows dependencies to set up environment variables and other properties to be used by dependents. -The funciton declaration should look like this: +The function declaration should look like this: .. code-block:: python @@ -1115,7 +1115,7 @@ The arguments to this function are: * **spec**: the spec of the *dependency package* (the one the function is called on). * **dep_spec**: the spec of the dependent package (i.e. dep_spec depends on spec). -A goo example of using these is in the Python packge: +A good example of using these is in the Python package: .. code-block:: python @@ -1148,7 +1148,7 @@ packages to have a very simple install method, like this: def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) -Python's ``setup_dependent_environment`` method also sets up smoe +Python's ``setup_dependent_environment`` method also sets up some other variables, creates a directory, and sets up the ``PYTHONPATH`` so that dependent packages can find their dependencies at build time. @@ -1217,7 +1217,7 @@ Activation & deactivation Spack's ``Package`` class has default ``activate`` and ``deactivate`` implementations that handle symbolically linking extensions' prefixes into the directory of the parent package. However, extendable -packages can override these methdos to add custom activate/deactivate +packages can override these methods to add custom activate/deactivate logic of their own. For example, the ``activate`` and ``deactivate`` methods in the Python class use the symbolic linking, but they also handle details surrounding Python's ``.pth`` files, and other aspects @@ -1286,7 +1286,7 @@ Virtual dependencies ----------------------------- In some cases, more than one package can satisfy another package's -dependency. One way this can happen is if a pacakge depends on a +dependency. One way this can happen is if a package depends on a particular *interface*, but there are multiple *implementations* of the interface, and the package could be built with any of them. A *very* common interface in HPC is the `Message Passing Interface (MPI) @@ -1299,7 +1299,7 @@ MPI has several different implementations (e.g., `MPICH applications can be built with any one of them. Complicating matters, MPI does not have a standardized ABI, so a package built with one implementation cannot simply be relinked with another implementation. -Many pacakage managers handle interfaces like this by requiring many +Many package managers handle interfaces like this by requiring many similar package files, e.g., ``foo``, ``foo-mvapich``, ``foo-mpich``, but Spack avoids this explosion of package files by providing support for *virtual dependencies*. @@ -1325,7 +1325,7 @@ supplying a ``depends_on`` call in the package definition. For example: depends_on("adept-utils") depends_on("callpath") -Here, ``callpath`` and ``adept-utils`` are concrete pacakges, but +Here, ``callpath`` and ``adept-utils`` are concrete packages, but there is no actual package file for ``mpi``, so we say it is a *virtual* package. The syntax of ``depends_on``, is the same for both. If we look inside the package file of an MPI implementation, @@ -1349,7 +1349,7 @@ to ``provides`` to add constraints. This allows Spack to support the notion of *versioned interfaces*. The MPI standard has gone through many revisions, each with new functions added, and each revision of the standard has a version number. Some packages may require a recent -implementation that supports MPI-3 fuctions, but some MPI versions may +implementation that supports MPI-3 functions, but some MPI versions may only provide up to MPI-2. Others may need MPI 2.1 or higher. You can indicate this by adding a version constraint to the spec passed to ``provides``: @@ -1381,7 +1381,7 @@ constraints on the *providing* package, or the *provider*. The provider only provides the declared virtual spec when *it* matches the constraints in the when clause. Here, when ``mpich`` is at version 3 or higher, it provides MPI up to version 3. When ``mpich`` -is at version 1 or higher, it provides the MPI virtual pacakge at +is at version 1 or higher, it provides the MPI virtual package at version 1. The ``when`` qualifier ensures that Spack selects a suitably high @@ -1544,7 +1544,7 @@ software should be installed. Spack provides wrapper functions for ``configure`` and ``make`` so that you can call them in a similar way to how you'd call a shell -comamnd. In reality, these are Python functions. Spack provides +command. In reality, these are Python functions. Spack provides these functions to make writing packages more natural. See the section on :ref:`shell wrappers `. @@ -1603,7 +1603,7 @@ The install environment -------------------------- In general, you should not have to do much differently in your install -method than you would when installing a pacakge on the command line. +method than you would when installing a package on the command line. In fact, you may need to do *less* than you would on the command line. Spack tries to set environment variables and modify compiler calls so @@ -1626,7 +1626,7 @@ purposes: #. Make build systems use Spack's compiler wrappers for their builds. #. Allow build systems to find dependencies more easily -The Compiler enviroment variables that Spack sets are: +The Compiler environment variables that Spack sets are: ============ =============================== Variable Purpose @@ -1656,7 +1656,7 @@ entering ``install()`` so that packages can locate dependencies easily: ======================= ============================= - ``PATH`` Set to point to ``/bin`` directories of dpeendencies + ``PATH`` Set to point to ``/bin`` directories of dependencies ``CMAKE_PREFIX_PATH`` Path to dependency prefixes for CMake ``PKG_CONFIG_PATH`` Path to any pkgconfig directories for dependencies ``PYTHONPATH`` Path to site-packages dir of any python dependencies @@ -1742,7 +1742,7 @@ the command line. Forking ``install()`` ~~~~~~~~~~~~~~~~~~~~~ -To give packagers free reign over their install environemnt, Spack +To give packagers free reign over their install environment, Spack forks a new process each time it invokes a package's ``install()`` method. This allows packages to have their own completely sandboxed build environment, without impacting other jobs that the main Spack @@ -1870,7 +1870,7 @@ dependency version. You can use ``satisfies()`` to test for particular dependencies, e.g. ``foo.satisfies('^openmpi@1.2')`` or ``foo.satisfies('^mpich')``, -or you can use Python's builtin ``in`` operator: +or you can use Python's built-in ``in`` operator: .. code-block:: python @@ -1899,7 +1899,7 @@ Accessing Dependencies ~~~~~~~~~~~~~~~~~~~~~~~~~~ You may need to get at some file or binary that's in the prefix of one -of your dependencies. You can do that by subscripting the spec: +of your dependencies. You can do that by sub-scripting the spec: .. code-block:: python @@ -2219,7 +2219,7 @@ File functions Create an empty file at ``path``. -.. _pacakge-lifecycle: +.. _package-lifecycle: Packaging workflow commands --------------------------------- @@ -2248,7 +2248,7 @@ A typical package workflow might look like this: ... repeat clean/install until install works ... Below are some commands that will allow you some finer-grained -controll over the install process. +control over the install process. .. _spack-fetch: @@ -2325,7 +2325,7 @@ Keeping the stage directory on success ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ By default, ``spack install`` will delete the staging area once a -pacakge has been successfully built and installed. Use +package has been successfully built and installed. Use ``--keep-stage`` to leave the build directory intact: .. code-block:: sh @@ -2474,7 +2474,7 @@ build it: /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3=linux-ppc64/libelf-0.8.13 ``spack cd`` here changed he current working directory to the -directory containing theexpanded ``libelf`` source code. There are a +directory containing the expanded ``libelf`` source code. There are a number of other places you can cd to in the spack directory hierarchy: .. command-output:: spack cd -h diff --git a/lib/spack/docs/site_configuration.rst b/lib/spack/docs/site_configuration.rst index e076aec7638..44071bbfc64 100644 --- a/lib/spack/docs/site_configuration.rst +++ b/lib/spack/docs/site_configuration.rst @@ -9,7 +9,7 @@ Temporary space ---------------------------- .. warning:: Temporary space configuration will be moved to configuration files. - The intructions here are old and refer to ``__init__.py`` + The instructions here are old and refer to ``__init__.py`` By default, Spack will try to do all of its building in temporary space. There are two main reasons for this. First, Spack is designed @@ -93,7 +93,7 @@ the virtual spec to specs for possible implementations, and later, so there is no need to fully concretize the spec when returning it. -The ``DefaultConcretizer`` is intendend to provide sensible defaults +The ``DefaultConcretizer`` is intended to provide sensible defaults for each policy, but there are certain choices that it can't know about. For example, one site might prefer ``OpenMPI`` over ``MPICH``, or another might prefer an old version of some packages. These types @@ -139,7 +139,7 @@ The next time you run Spack, your changes should take effect. Profiling ~~~~~~~~~~~~~~~~~~~~~ -Spack has some limited builtin support for profiling, and can report +Spack has some limited built-in support for profiling, and can report statistics using standard Python timing tools. To use this feature, supply ``-p`` to Spack on the command line, before any subcommands. @@ -181,6 +181,6 @@ supply ``-p`` to Spack on the command line, before any subcommands. ... The bottom of the output shows the top most time consuming functions, -slowest on top. The profiling support is from Python's builtin tool, +slowest on top. The profiling support is from Python's built-in tool, `cProfile `_. From ffdb90f39a56e5f0dacefd19ccf85ddd0e7a01c0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 24 Feb 2015 10:42:35 -0800 Subject: [PATCH 97/97] Last minute Qt bugfix. --- var/spack/packages/qt/package.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py index 1dc3e1e51de..4a9a867511f 100644 --- a/var/spack/packages/qt/package.py +++ b/var/spack/packages/qt/package.py @@ -79,10 +79,9 @@ def common_config_args(self): '-no-openvg', '-no-pch', # NIS is deprecated in more recent glibc - "-no-nis", - # For now, disable all the database drivers - "-no-sql-db2", "-no-sql-ibase", "-no-sql-mysql", "-no-sql-oci", "-no-sql-odbc", - "-no-sql-psql", "-no-sql-sqlite", "-no-sql-sqlite2", "-no-sql-tds"] + "-no-nis"] + # Don't disable all the database drivers, but should + # really get them into spack at some point. @when('@4')