From c6cd8ae243c07fa80d54207c334883585eca0d97 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 1 Oct 2015 21:35:25 -0700 Subject: [PATCH 001/375] Use more recent version of libffi. 3.1 has a known bug that is fixed in current version --- var/spack/packages/libffi/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/var/spack/packages/libffi/package.py b/var/spack/packages/libffi/package.py index 2c1c4eed4d3..18acabb00cf 100644 --- a/var/spack/packages/libffi/package.py +++ b/var/spack/packages/libffi/package.py @@ -6,9 +6,9 @@ class Libffi(Package): to call any function specified by a call interface description at run time.""" homepage = "https://sourceware.org/libffi/" - url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz" - - version('3.1', 'f5898b29bbfd70502831a212d9249d10') + + version('3.2.1','83b89587607e3eb65c70d361f13bab43',url = "ftp://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz") + #version('3.1', 'f5898b29bbfd70502831a212d9249d10',url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz") # Has a bug $(lib64) instead of ${lib64} in libffi.pc def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 1501786fd9be0a01abe7f6bb917e416fb3256aa6 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 13:04:37 -0700 Subject: [PATCH 002/375] adding python_recipe_parser as a submodule under the folder utils --- .gitmodules | 3 +++ utils/python_recipe_parser | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 utils/python_recipe_parser diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..a097fcd8cf3 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "utils/python_recipe_parser"] + path = utils/python_recipe_parser + url = https://github.com/karenyyng/spack_python_package_parser diff --git a/utils/python_recipe_parser b/utils/python_recipe_parser new file mode 160000 index 00000000000..3158074a2d7 --- /dev/null +++ b/utils/python_recipe_parser @@ -0,0 +1 @@ +Subproject commit 3158074a2d74f99aa72289dfcf093b1b0ba4474e From 79808da760eabcb3e692c88e29ea3c7c558b131d Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 18:47:36 -0700 Subject: [PATCH 003/375] added a bunch of packages and added submodule --- .gitignore | 1 + .gitmodules | 4 +- utils/python_recipe_parser | 1 - var/spack/packages/ImageMagick/package.py | 0 var/spack/packages/Mitos/package.py | 0 var/spack/packages/R/package.py | 0 var/spack/packages/SAMRAI/no-tool-build.patch | 0 var/spack/packages/SAMRAI/package.py | 0 var/spack/packages/adept-utils/package.py | 0 var/spack/packages/arpack/package.py | 0 var/spack/packages/asciidoc/package.py | 18 --------- var/spack/packages/atk/package.py | 0 var/spack/packages/atlas/package.py | 0 var/spack/packages/autoconf/package.py | 0 var/spack/packages/automaded/package.py | 0 var/spack/packages/automake/package.py | 0 var/spack/packages/bear/package.py | 0 var/spack/packages/bib2xhtml/package.py | 0 var/spack/packages/binutils/package.py | 0 var/spack/packages/bison/package.py | 0 var/spack/packages/boost/package.py | 4 -- var/spack/packages/bowtie2/bowtie2-2.5.patch | 16 -------- var/spack/packages/bowtie2/package.py | 24 ------------ var/spack/packages/boxlib/package.py | 0 var/spack/packages/bzip2/package.py | 0 var/spack/packages/cairo/package.py | 0 var/spack/packages/callpath/package.py | 0 var/spack/packages/cblas/package.py | 0 var/spack/packages/cgm/package.py | 0 var/spack/packages/clang/package.py | 0 var/spack/packages/cloog/package.py | 0 var/spack/packages/cmake/package.py | 0 var/spack/packages/coreutils/package.py | 0 var/spack/packages/cppcheck/package.py | 0 var/spack/packages/cram/package.py | 0 var/spack/packages/cscope/package.py | 17 -------- var/spack/packages/cube/package.py | 0 var/spack/packages/czmq/package.py | 19 --------- var/spack/packages/dbus/package.py | 0 var/spack/packages/docbook-xml/package.py | 19 --------- var/spack/packages/doxygen/package.py | 25 ------------ var/spack/packages/dri2proto/package.py | 0 var/spack/packages/dtcmp/package.py | 0 var/spack/packages/dyninst/package.py | 0 var/spack/packages/eigen/package.py | 17 ++++++++ var/spack/packages/elfutils/package.py | 26 ------------- var/spack/packages/extrae/package.py | 0 var/spack/packages/exuberant-ctags/package.py | 0 var/spack/packages/fish/package.py | 18 --------- var/spack/packages/flex/package.py | 0 var/spack/packages/flux/package.py | 36 ----------------- var/spack/packages/fontconfig/package.py | 0 var/spack/packages/freetype/package.py | 0 var/spack/packages/gasnet/package.py | 0 var/spack/packages/gcc/package.py | 0 var/spack/packages/gdk-pixbuf/package.py | 0 var/spack/packages/geos/package.py | 0 var/spack/packages/gflags/package.py | 21 ---------- var/spack/packages/ghostscript/package.py | 17 -------- var/spack/packages/git/package.py | 0 var/spack/packages/glib/package.py | 0 var/spack/packages/global/package.py | 0 var/spack/packages/glog/package.py | 15 ------- var/spack/packages/gmp/package.py | 0 var/spack/packages/gnutls/package.py | 0 var/spack/packages/gperf/package.py | 0 var/spack/packages/gperftools/package.py | 0 var/spack/packages/graphlib/package.py | 0 var/spack/packages/graphviz/package.py | 19 +++++---- var/spack/packages/gtkplus/package.py | 0 var/spack/packages/harfbuzz/package.py | 0 var/spack/packages/hdf5/package.py | 0 var/spack/packages/hwloc/package.py | 0 var/spack/packages/hypre/package.py | 0 var/spack/packages/icu/package.py | 0 var/spack/packages/icu4c/package.py | 0 var/spack/packages/isl/package.py | 0 var/spack/packages/jdk/package.py | 0 var/spack/packages/jpeg/package.py | 0 .../{netlib-lapack => lapack}/package.py | 18 +-------- var/spack/packages/launchmon/package.py | 0 .../packages/launchmon/patch.lmon_install_dir | 0 var/spack/packages/lcms/package.py | 0 var/spack/packages/leveldb/package.py | 29 -------------- var/spack/packages/libNBC/package.py | 0 var/spack/packages/libarchive/package.py | 0 var/spack/packages/libcircle/package.py | 0 var/spack/packages/libdrm/package.py | 5 ++- var/spack/packages/libdwarf/package.py | 0 var/spack/packages/libelf/package.py | 2 - var/spack/packages/libevent/package.py | 0 var/spack/packages/libffi/package.py | 6 +-- var/spack/packages/libgcrypt/package.py | 0 var/spack/packages/libgpg-error/package.py | 0 var/spack/packages/libjpeg-turbo/package.py | 0 var/spack/packages/libjson-c/package.py | 14 ------- var/spack/packages/libmng/package.py | 0 var/spack/packages/libmonitor/package.py | 0 var/spack/packages/libpciaccess/package.py | 17 +++----- var/spack/packages/libpng/package.py | 0 var/spack/packages/libsodium/package.py | 19 --------- var/spack/packages/libtiff/package.py | 0 var/spack/packages/libtool/package.py | 0 var/spack/packages/libunwind/package.py | 0 var/spack/packages/libuuid/package.py | 0 var/spack/packages/libxcb/package.py | 0 var/spack/packages/libxml2/package.py | 5 +-- var/spack/packages/libxshmfence/package.py | 0 var/spack/packages/libxslt/package.py | 0 var/spack/packages/llvm-lld/package.py | 0 var/spack/packages/llvm/package.py | 0 var/spack/packages/lmdb/package.py | 39 ------------------- var/spack/packages/lua/package.py | 26 ------------- var/spack/packages/lwgrp/package.py | 0 var/spack/packages/lwm2/package.py | 0 var/spack/packages/memaxes/package.py | 0 var/spack/packages/mesa/package.py | 5 +-- var/spack/packages/metis/package.py | 0 var/spack/packages/mpc/package.py | 0 var/spack/packages/mpe2/mpe2.patch | 0 var/spack/packages/mpe2/package.py | 0 var/spack/packages/mpfr/package.py | 0 var/spack/packages/mpibash/mpibash-4.3.patch | 0 var/spack/packages/mpibash/package.py | 0 var/spack/packages/mpich/package.py | 1 + var/spack/packages/mpileaks/package.py | 0 var/spack/packages/mrnet/package.py | 0 var/spack/packages/munge/package.py | 20 ---------- var/spack/packages/muster/package.py | 0 .../ad_lustre_rwcontig_open_source.patch | 0 var/spack/packages/mvapich2/package.py | 0 var/spack/packages/nasm/package.py | 0 var/spack/packages/ncurses/package.py | 8 ---- var/spack/packages/netcdf/package.py | 0 var/spack/packages/netgauge/package.py | 0 var/spack/packages/netlib-blas/package.py | 6 --- var/spack/packages/nettle/package.py | 0 var/spack/packages/ompss/package.py | 0 var/spack/packages/opari2/package.py | 0 .../ad_lustre_rwcontig_open_source.patch | 0 .../packages/openmpi/llnl-platforms.patch | 0 var/spack/packages/openmpi/package.py | 0 var/spack/packages/openssl/package.py | 0 var/spack/packages/otf/package.py | 0 var/spack/packages/otf2/package.py | 0 var/spack/packages/pango/package.py | 0 var/spack/packages/papi/package.py | 0 var/spack/packages/paraver/package.py | 0 var/spack/packages/parmetis/package.py | 0 var/spack/packages/parpack/package.py | 0 var/spack/packages/pcre/package.py | 0 var/spack/packages/petsc/package.py | 0 var/spack/packages/pidx/package.py | 21 ---------- var/spack/packages/pixman/package.py | 0 var/spack/packages/pkg-config/package.py | 17 -------- var/spack/packages/pmgr_collective/package.py | 0 var/spack/packages/postgresql/package.py | 0 var/spack/packages/ppl/package.py | 0 var/spack/packages/protobuf/package.py | 16 -------- var/spack/packages/py-Cython/package.py | 12 ++++++ var/spack/packages/py-Distutils2/package.py | 15 +++++++ var/spack/packages/py-astroML/package.py | 12 ++++++ var/spack/packages/py-astropy/package.py | 12 ++++++ var/spack/packages/py-autopep8/package.py | 12 ++++++ var/spack/packages/py-basemap/package.py | 0 var/spack/packages/py-biopython/package.py | 0 var/spack/packages/py-certifi/package.py | 12 ++++++ var/spack/packages/py-cffi/package.py | 0 var/spack/packages/py-cython/package.py | 0 var/spack/packages/py-dateutil/package.py | 0 var/spack/packages/py-decorator/package.py | 12 ++++++ var/spack/packages/py-emcee/package.py | 12 ++++++ var/spack/packages/py-epydoc/package.py | 0 var/spack/packages/py-fitsio/package.py | 13 +++++++ var/spack/packages/py-genders/package.py | 0 var/spack/packages/py-gnuplot/package.py | 0 var/spack/packages/py-gnureadline/package.py | 12 ++++++ var/spack/packages/py-h5py/package.py | 0 var/spack/packages/py-ipython/package.py | 21 ++++++++-- .../packages/py-ipython/package.py.backup | 27 +++++++++++++ .../packages/py-ipython_genutils/package.py | 12 ++++++ var/spack/packages/py-joblib/package.py | 12 ++++++ var/spack/packages/py-jsonschema/package.py | 16 ++++++++ var/spack/packages/py-libxml2/package.py | 0 var/spack/packages/py-lockfile/package.py | 0 var/spack/packages/py-mako/package.py | 0 var/spack/packages/py-matplotlib/package.py | 1 - var/spack/packages/py-mock/package.py | 17 +++----- var/spack/packages/py-mpi4py/package.py | 0 var/spack/packages/py-mx/package.py | 0 var/spack/packages/py-nose/package.py | 0 var/spack/packages/py-numexpr/package.py | 12 ++++++ var/spack/packages/py-numpy/package.py | 15 +------ var/spack/packages/py-pandas/package.py | 2 + var/spack/packages/py-pep8/package.py | 12 ++++++ var/spack/packages/py-pexpect/package.py | 0 var/spack/packages/py-pickleshare/package.py | 12 ++++++ var/spack/packages/py-pil/package.py | 0 var/spack/packages/py-pip/package.py | 12 ++++++ var/spack/packages/py-pmw/package.py | 0 var/spack/packages/py-py/package.py | 12 ++++++ var/spack/packages/py-py4j/package.py | 15 +++++++ var/spack/packages/py-pychecker/package.py | 0 var/spack/packages/py-pycparser/package.py | 0 var/spack/packages/py-pyelftools/package.py | 0 var/spack/packages/py-pygments/package.py | 0 var/spack/packages/py-pylint/package.py | 0 var/spack/packages/py-pypar/package.py | 0 var/spack/packages/py-pyparsing/package.py | 0 var/spack/packages/py-pyqt/package.py | 0 var/spack/packages/py-pyside/package.py | 5 +-- var/spack/packages/py-pytest/package.py | 12 ++++++ .../packages/py-python-daemon/package.py | 0 .../packages/py-python-dateutil/package.py | 12 ++++++ var/spack/packages/py-pytz/package.py | 2 + var/spack/packages/py-pyzmq/package.py | 16 ++++++++ var/spack/packages/py-rpy2/package.py | 0 .../packages/py-scientificpython/package.py | 0 var/spack/packages/py-scikit-learn/package.py | 3 ++ var/spack/packages/py-scipy/package.py | 4 +- var/spack/packages/py-seaborn/package.py | 12 ++++++ var/spack/packages/py-setuptools/package.py | 9 +++-- var/spack/packages/py-shiboken/package.py | 0 var/spack/packages/py-sip/package.py | 0 var/spack/packages/py-six/package.py | 2 + var/spack/packages/py-statsmodels/package.py | 12 ++++++ var/spack/packages/py-sympy/package.py | 0 var/spack/packages/py-tables/package.py | 12 ++++++ var/spack/packages/py-tornado/package.py | 17 ++++++++ var/spack/packages/py-traitlets/package.py | 14 +++++++ .../packages/py-triangle-plot/package.py | 12 ++++++ var/spack/packages/py-virtualenv/package.py | 0 var/spack/packages/py-yapf/package.py | 0 var/spack/packages/python/package.py | 0 var/spack/packages/qhull/package.py | 0 var/spack/packages/qt/package.py | 0 var/spack/packages/qthreads/package.py | 0 var/spack/packages/ravel/package.py | 0 var/spack/packages/readline/package.py | 0 .../rose/add_spack_compiler_recognition.patch | 0 var/spack/packages/rose/package.py | 0 var/spack/packages/ruby/package.py | 0 var/spack/packages/samtools/package.py | 18 --------- var/spack/packages/samtools/samtools1.2.patch | 20 ---------- var/spack/packages/scalasca/package.py | 0 var/spack/packages/scorep/package.py | 15 +++---- var/spack/packages/scotch/package.py | 0 var/spack/packages/scr/package.py | 0 var/spack/packages/silo/package.py | 0 var/spack/packages/snappy/package.py | 15 ------- var/spack/packages/spindle/package.py | 0 var/spack/packages/sqlite/package.py | 0 .../packages/stat/configure_mpicxx.patch | 0 var/spack/packages/stat/package.py | 11 +----- var/spack/packages/sundials/package.py | 0 var/spack/packages/swig/package.py | 0 var/spack/packages/task/package.py | 0 var/spack/packages/taskd/package.py | 0 var/spack/packages/tau/package.py | 0 var/spack/packages/tcl/package.py | 0 .../packages/the_silver_searcher/package.py | 0 var/spack/packages/thrift/package.py | 0 var/spack/packages/tk/package.py | 0 var/spack/packages/tmux/package.py | 0 var/spack/packages/tmuxinator/package.py | 0 var/spack/packages/uncrustify/package.py | 0 var/spack/packages/util-linux/package.py | 0 var/spack/packages/vim/package.py | 23 ++--------- var/spack/packages/vtk/package.py | 0 var/spack/packages/wget/package.py | 0 var/spack/packages/wx/package.py | 0 var/spack/packages/wxpropgrid/package.py | 0 var/spack/packages/xcb-proto/package.py | 0 var/spack/packages/xz/package.py | 0 var/spack/packages/yasm/package.py | 0 var/spack/packages/zeromq/package.py | 20 ---------- var/spack/packages/zlib/package.py | 0 var/spack/packages/zsh/package.py | 0 278 files changed, 491 insertions(+), 688 deletions(-) delete mode 160000 utils/python_recipe_parser mode change 100644 => 100755 var/spack/packages/ImageMagick/package.py mode change 100644 => 100755 var/spack/packages/Mitos/package.py mode change 100644 => 100755 var/spack/packages/R/package.py mode change 100644 => 100755 var/spack/packages/SAMRAI/no-tool-build.patch mode change 100644 => 100755 var/spack/packages/SAMRAI/package.py mode change 100644 => 100755 var/spack/packages/adept-utils/package.py mode change 100644 => 100755 var/spack/packages/arpack/package.py delete mode 100644 var/spack/packages/asciidoc/package.py mode change 100644 => 100755 var/spack/packages/atk/package.py mode change 100644 => 100755 var/spack/packages/atlas/package.py mode change 100644 => 100755 var/spack/packages/autoconf/package.py mode change 100644 => 100755 var/spack/packages/automaded/package.py mode change 100644 => 100755 var/spack/packages/automake/package.py mode change 100644 => 100755 var/spack/packages/bear/package.py mode change 100644 => 100755 var/spack/packages/bib2xhtml/package.py mode change 100644 => 100755 var/spack/packages/binutils/package.py mode change 100644 => 100755 var/spack/packages/bison/package.py mode change 100644 => 100755 var/spack/packages/boost/package.py delete mode 100644 var/spack/packages/bowtie2/bowtie2-2.5.patch delete mode 100644 var/spack/packages/bowtie2/package.py mode change 100644 => 100755 var/spack/packages/boxlib/package.py mode change 100644 => 100755 var/spack/packages/bzip2/package.py mode change 100644 => 100755 var/spack/packages/cairo/package.py mode change 100644 => 100755 var/spack/packages/callpath/package.py mode change 100644 => 100755 var/spack/packages/cblas/package.py mode change 100644 => 100755 var/spack/packages/cgm/package.py mode change 100644 => 100755 var/spack/packages/clang/package.py mode change 100644 => 100755 var/spack/packages/cloog/package.py mode change 100644 => 100755 var/spack/packages/cmake/package.py mode change 100644 => 100755 var/spack/packages/coreutils/package.py mode change 100644 => 100755 var/spack/packages/cppcheck/package.py mode change 100644 => 100755 var/spack/packages/cram/package.py delete mode 100644 var/spack/packages/cscope/package.py mode change 100644 => 100755 var/spack/packages/cube/package.py delete mode 100644 var/spack/packages/czmq/package.py mode change 100644 => 100755 var/spack/packages/dbus/package.py delete mode 100644 var/spack/packages/docbook-xml/package.py delete mode 100644 var/spack/packages/doxygen/package.py mode change 100644 => 100755 var/spack/packages/dri2proto/package.py mode change 100644 => 100755 var/spack/packages/dtcmp/package.py mode change 100644 => 100755 var/spack/packages/dyninst/package.py create mode 100644 var/spack/packages/eigen/package.py delete mode 100644 var/spack/packages/elfutils/package.py mode change 100644 => 100755 var/spack/packages/extrae/package.py mode change 100644 => 100755 var/spack/packages/exuberant-ctags/package.py delete mode 100644 var/spack/packages/fish/package.py mode change 100644 => 100755 var/spack/packages/flex/package.py delete mode 100644 var/spack/packages/flux/package.py mode change 100644 => 100755 var/spack/packages/fontconfig/package.py mode change 100644 => 100755 var/spack/packages/freetype/package.py mode change 100644 => 100755 var/spack/packages/gasnet/package.py mode change 100644 => 100755 var/spack/packages/gcc/package.py mode change 100644 => 100755 var/spack/packages/gdk-pixbuf/package.py mode change 100644 => 100755 var/spack/packages/geos/package.py delete mode 100644 var/spack/packages/gflags/package.py delete mode 100644 var/spack/packages/ghostscript/package.py mode change 100644 => 100755 var/spack/packages/git/package.py mode change 100644 => 100755 var/spack/packages/glib/package.py mode change 100644 => 100755 var/spack/packages/global/package.py delete mode 100644 var/spack/packages/glog/package.py mode change 100644 => 100755 var/spack/packages/gmp/package.py mode change 100644 => 100755 var/spack/packages/gnutls/package.py mode change 100644 => 100755 var/spack/packages/gperf/package.py mode change 100644 => 100755 var/spack/packages/gperftools/package.py mode change 100644 => 100755 var/spack/packages/graphlib/package.py mode change 100644 => 100755 var/spack/packages/gtkplus/package.py mode change 100644 => 100755 var/spack/packages/harfbuzz/package.py mode change 100644 => 100755 var/spack/packages/hdf5/package.py mode change 100644 => 100755 var/spack/packages/hwloc/package.py mode change 100644 => 100755 var/spack/packages/hypre/package.py mode change 100644 => 100755 var/spack/packages/icu/package.py mode change 100644 => 100755 var/spack/packages/icu4c/package.py mode change 100644 => 100755 var/spack/packages/isl/package.py mode change 100644 => 100755 var/spack/packages/jdk/package.py mode change 100644 => 100755 var/spack/packages/jpeg/package.py rename var/spack/packages/{netlib-lapack => lapack}/package.py (77%) mode change 100644 => 100755 mode change 100644 => 100755 var/spack/packages/launchmon/package.py mode change 100644 => 100755 var/spack/packages/launchmon/patch.lmon_install_dir mode change 100644 => 100755 var/spack/packages/lcms/package.py delete mode 100644 var/spack/packages/leveldb/package.py mode change 100644 => 100755 var/spack/packages/libNBC/package.py mode change 100644 => 100755 var/spack/packages/libarchive/package.py mode change 100644 => 100755 var/spack/packages/libcircle/package.py mode change 100644 => 100755 var/spack/packages/libdrm/package.py mode change 100644 => 100755 var/spack/packages/libdwarf/package.py mode change 100644 => 100755 var/spack/packages/libelf/package.py mode change 100644 => 100755 var/spack/packages/libevent/package.py mode change 100644 => 100755 var/spack/packages/libffi/package.py mode change 100644 => 100755 var/spack/packages/libgcrypt/package.py mode change 100644 => 100755 var/spack/packages/libgpg-error/package.py mode change 100644 => 100755 var/spack/packages/libjpeg-turbo/package.py delete mode 100644 var/spack/packages/libjson-c/package.py mode change 100644 => 100755 var/spack/packages/libmng/package.py mode change 100644 => 100755 var/spack/packages/libmonitor/package.py mode change 100644 => 100755 var/spack/packages/libpciaccess/package.py mode change 100644 => 100755 var/spack/packages/libpng/package.py delete mode 100644 var/spack/packages/libsodium/package.py mode change 100644 => 100755 var/spack/packages/libtiff/package.py mode change 100644 => 100755 var/spack/packages/libtool/package.py mode change 100644 => 100755 var/spack/packages/libunwind/package.py mode change 100644 => 100755 var/spack/packages/libuuid/package.py mode change 100644 => 100755 var/spack/packages/libxcb/package.py mode change 100644 => 100755 var/spack/packages/libxml2/package.py mode change 100644 => 100755 var/spack/packages/libxshmfence/package.py mode change 100644 => 100755 var/spack/packages/libxslt/package.py mode change 100644 => 100755 var/spack/packages/llvm-lld/package.py mode change 100644 => 100755 var/spack/packages/llvm/package.py delete mode 100644 var/spack/packages/lmdb/package.py delete mode 100644 var/spack/packages/lua/package.py mode change 100644 => 100755 var/spack/packages/lwgrp/package.py mode change 100644 => 100755 var/spack/packages/lwm2/package.py mode change 100644 => 100755 var/spack/packages/memaxes/package.py mode change 100644 => 100755 var/spack/packages/mesa/package.py mode change 100644 => 100755 var/spack/packages/metis/package.py mode change 100644 => 100755 var/spack/packages/mpc/package.py mode change 100644 => 100755 var/spack/packages/mpe2/mpe2.patch mode change 100644 => 100755 var/spack/packages/mpe2/package.py mode change 100644 => 100755 var/spack/packages/mpfr/package.py mode change 100644 => 100755 var/spack/packages/mpibash/mpibash-4.3.patch mode change 100644 => 100755 var/spack/packages/mpibash/package.py mode change 100644 => 100755 var/spack/packages/mpich/package.py mode change 100644 => 100755 var/spack/packages/mpileaks/package.py mode change 100644 => 100755 var/spack/packages/mrnet/package.py delete mode 100644 var/spack/packages/munge/package.py mode change 100644 => 100755 var/spack/packages/muster/package.py mode change 100644 => 100755 var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch mode change 100644 => 100755 var/spack/packages/mvapich2/package.py mode change 100644 => 100755 var/spack/packages/nasm/package.py mode change 100644 => 100755 var/spack/packages/ncurses/package.py mode change 100644 => 100755 var/spack/packages/netcdf/package.py mode change 100644 => 100755 var/spack/packages/netgauge/package.py mode change 100644 => 100755 var/spack/packages/netlib-blas/package.py mode change 100644 => 100755 var/spack/packages/nettle/package.py mode change 100644 => 100755 var/spack/packages/ompss/package.py mode change 100644 => 100755 var/spack/packages/opari2/package.py mode change 100644 => 100755 var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch mode change 100644 => 100755 var/spack/packages/openmpi/llnl-platforms.patch mode change 100644 => 100755 var/spack/packages/openmpi/package.py mode change 100644 => 100755 var/spack/packages/openssl/package.py mode change 100644 => 100755 var/spack/packages/otf/package.py mode change 100644 => 100755 var/spack/packages/otf2/package.py mode change 100644 => 100755 var/spack/packages/pango/package.py mode change 100644 => 100755 var/spack/packages/papi/package.py mode change 100644 => 100755 var/spack/packages/paraver/package.py mode change 100644 => 100755 var/spack/packages/parmetis/package.py mode change 100644 => 100755 var/spack/packages/parpack/package.py mode change 100644 => 100755 var/spack/packages/pcre/package.py mode change 100644 => 100755 var/spack/packages/petsc/package.py delete mode 100644 var/spack/packages/pidx/package.py mode change 100644 => 100755 var/spack/packages/pixman/package.py delete mode 100644 var/spack/packages/pkg-config/package.py mode change 100644 => 100755 var/spack/packages/pmgr_collective/package.py mode change 100644 => 100755 var/spack/packages/postgresql/package.py mode change 100644 => 100755 var/spack/packages/ppl/package.py delete mode 100644 var/spack/packages/protobuf/package.py create mode 100644 var/spack/packages/py-Cython/package.py create mode 100644 var/spack/packages/py-Distutils2/package.py create mode 100644 var/spack/packages/py-astroML/package.py create mode 100644 var/spack/packages/py-astropy/package.py create mode 100644 var/spack/packages/py-autopep8/package.py mode change 100644 => 100755 var/spack/packages/py-basemap/package.py mode change 100644 => 100755 var/spack/packages/py-biopython/package.py create mode 100644 var/spack/packages/py-certifi/package.py mode change 100644 => 100755 var/spack/packages/py-cffi/package.py mode change 100644 => 100755 var/spack/packages/py-cython/package.py mode change 100644 => 100755 var/spack/packages/py-dateutil/package.py create mode 100644 var/spack/packages/py-decorator/package.py create mode 100644 var/spack/packages/py-emcee/package.py mode change 100644 => 100755 var/spack/packages/py-epydoc/package.py create mode 100644 var/spack/packages/py-fitsio/package.py mode change 100644 => 100755 var/spack/packages/py-genders/package.py mode change 100644 => 100755 var/spack/packages/py-gnuplot/package.py create mode 100644 var/spack/packages/py-gnureadline/package.py mode change 100644 => 100755 var/spack/packages/py-h5py/package.py mode change 100644 => 100755 var/spack/packages/py-ipython/package.py create mode 100755 var/spack/packages/py-ipython/package.py.backup create mode 100644 var/spack/packages/py-ipython_genutils/package.py create mode 100644 var/spack/packages/py-joblib/package.py create mode 100755 var/spack/packages/py-jsonschema/package.py mode change 100644 => 100755 var/spack/packages/py-libxml2/package.py mode change 100644 => 100755 var/spack/packages/py-lockfile/package.py mode change 100644 => 100755 var/spack/packages/py-mako/package.py mode change 100644 => 100755 var/spack/packages/py-matplotlib/package.py mode change 100644 => 100755 var/spack/packages/py-mpi4py/package.py mode change 100644 => 100755 var/spack/packages/py-mx/package.py mode change 100644 => 100755 var/spack/packages/py-nose/package.py create mode 100644 var/spack/packages/py-numexpr/package.py mode change 100644 => 100755 var/spack/packages/py-numpy/package.py mode change 100644 => 100755 var/spack/packages/py-pandas/package.py create mode 100644 var/spack/packages/py-pep8/package.py mode change 100644 => 100755 var/spack/packages/py-pexpect/package.py create mode 100644 var/spack/packages/py-pickleshare/package.py mode change 100644 => 100755 var/spack/packages/py-pil/package.py create mode 100644 var/spack/packages/py-pip/package.py mode change 100644 => 100755 var/spack/packages/py-pmw/package.py create mode 100644 var/spack/packages/py-py/package.py create mode 100755 var/spack/packages/py-py4j/package.py mode change 100644 => 100755 var/spack/packages/py-pychecker/package.py mode change 100644 => 100755 var/spack/packages/py-pycparser/package.py mode change 100644 => 100755 var/spack/packages/py-pyelftools/package.py mode change 100644 => 100755 var/spack/packages/py-pygments/package.py mode change 100644 => 100755 var/spack/packages/py-pylint/package.py mode change 100644 => 100755 var/spack/packages/py-pypar/package.py mode change 100644 => 100755 var/spack/packages/py-pyparsing/package.py mode change 100644 => 100755 var/spack/packages/py-pyqt/package.py mode change 100644 => 100755 var/spack/packages/py-pyside/package.py create mode 100644 var/spack/packages/py-pytest/package.py mode change 100644 => 100755 var/spack/packages/py-python-daemon/package.py create mode 100644 var/spack/packages/py-python-dateutil/package.py mode change 100644 => 100755 var/spack/packages/py-pytz/package.py create mode 100755 var/spack/packages/py-pyzmq/package.py mode change 100644 => 100755 var/spack/packages/py-rpy2/package.py mode change 100644 => 100755 var/spack/packages/py-scientificpython/package.py mode change 100644 => 100755 var/spack/packages/py-scikit-learn/package.py mode change 100644 => 100755 var/spack/packages/py-scipy/package.py create mode 100644 var/spack/packages/py-seaborn/package.py mode change 100644 => 100755 var/spack/packages/py-setuptools/package.py mode change 100644 => 100755 var/spack/packages/py-shiboken/package.py mode change 100644 => 100755 var/spack/packages/py-sip/package.py mode change 100644 => 100755 var/spack/packages/py-six/package.py create mode 100644 var/spack/packages/py-statsmodels/package.py mode change 100644 => 100755 var/spack/packages/py-sympy/package.py create mode 100644 var/spack/packages/py-tables/package.py create mode 100755 var/spack/packages/py-tornado/package.py create mode 100755 var/spack/packages/py-traitlets/package.py create mode 100644 var/spack/packages/py-triangle-plot/package.py mode change 100644 => 100755 var/spack/packages/py-virtualenv/package.py mode change 100644 => 100755 var/spack/packages/py-yapf/package.py mode change 100644 => 100755 var/spack/packages/python/package.py mode change 100644 => 100755 var/spack/packages/qhull/package.py mode change 100644 => 100755 var/spack/packages/qt/package.py mode change 100644 => 100755 var/spack/packages/qthreads/package.py mode change 100644 => 100755 var/spack/packages/ravel/package.py mode change 100644 => 100755 var/spack/packages/readline/package.py mode change 100644 => 100755 var/spack/packages/rose/add_spack_compiler_recognition.patch mode change 100644 => 100755 var/spack/packages/rose/package.py mode change 100644 => 100755 var/spack/packages/ruby/package.py delete mode 100644 var/spack/packages/samtools/package.py delete mode 100644 var/spack/packages/samtools/samtools1.2.patch mode change 100644 => 100755 var/spack/packages/scalasca/package.py mode change 100644 => 100755 var/spack/packages/scorep/package.py mode change 100644 => 100755 var/spack/packages/scotch/package.py mode change 100644 => 100755 var/spack/packages/scr/package.py mode change 100644 => 100755 var/spack/packages/silo/package.py delete mode 100644 var/spack/packages/snappy/package.py mode change 100644 => 100755 var/spack/packages/spindle/package.py mode change 100644 => 100755 var/spack/packages/sqlite/package.py mode change 100644 => 100755 var/spack/packages/stat/configure_mpicxx.patch mode change 100644 => 100755 var/spack/packages/stat/package.py mode change 100644 => 100755 var/spack/packages/sundials/package.py mode change 100644 => 100755 var/spack/packages/swig/package.py mode change 100644 => 100755 var/spack/packages/task/package.py mode change 100644 => 100755 var/spack/packages/taskd/package.py mode change 100644 => 100755 var/spack/packages/tau/package.py mode change 100644 => 100755 var/spack/packages/tcl/package.py mode change 100644 => 100755 var/spack/packages/the_silver_searcher/package.py mode change 100644 => 100755 var/spack/packages/thrift/package.py mode change 100644 => 100755 var/spack/packages/tk/package.py mode change 100644 => 100755 var/spack/packages/tmux/package.py mode change 100644 => 100755 var/spack/packages/tmuxinator/package.py mode change 100644 => 100755 var/spack/packages/uncrustify/package.py mode change 100644 => 100755 var/spack/packages/util-linux/package.py mode change 100644 => 100755 var/spack/packages/vim/package.py mode change 100644 => 100755 var/spack/packages/vtk/package.py mode change 100644 => 100755 var/spack/packages/wget/package.py mode change 100644 => 100755 var/spack/packages/wx/package.py mode change 100644 => 100755 var/spack/packages/wxpropgrid/package.py mode change 100644 => 100755 var/spack/packages/xcb-proto/package.py mode change 100644 => 100755 var/spack/packages/xz/package.py mode change 100644 => 100755 var/spack/packages/yasm/package.py delete mode 100644 var/spack/packages/zeromq/package.py mode change 100644 => 100755 var/spack/packages/zlib/package.py mode change 100644 => 100755 var/spack/packages/zsh/package.py diff --git a/.gitignore b/.gitignore index 1c6ca4c99e1..8146921fd85 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /etc/spackconfig /share/spack/dotkit /share/spack/modules +*.ropeproject diff --git a/.gitmodules b/.gitmodules index a097fcd8cf3..1d770f415bd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "utils/python_recipe_parser"] - path = utils/python_recipe_parser +[submodule "/lib/spack/spack/util/python_recipe_parser"] + path = /lib/spack/spack/util/python_recipe_parser url = https://github.com/karenyyng/spack_python_package_parser diff --git a/utils/python_recipe_parser b/utils/python_recipe_parser deleted file mode 160000 index 3158074a2d7..00000000000 --- a/utils/python_recipe_parser +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3158074a2d74f99aa72289dfcf093b1b0ba4474e diff --git a/var/spack/packages/ImageMagick/package.py b/var/spack/packages/ImageMagick/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/Mitos/package.py b/var/spack/packages/Mitos/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/R/package.py b/var/spack/packages/R/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/SAMRAI/no-tool-build.patch b/var/spack/packages/SAMRAI/no-tool-build.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/packages/SAMRAI/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/adept-utils/package.py b/var/spack/packages/adept-utils/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/arpack/package.py b/var/spack/packages/arpack/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/asciidoc/package.py b/var/spack/packages/asciidoc/package.py deleted file mode 100644 index 828f3b3f4f7..00000000000 --- a/var/spack/packages/asciidoc/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Asciidoc(Package): - """ A presentable text document format for writing articles, UNIX man - pages and other small to medium sized documents.""" - homepage = "http://asciidoc.org" - url = "http://downloads.sourceforge.net/project/asciidoc/asciidoc/8.6.9/asciidoc-8.6.9.tar.gz" - - version('8.6.9', 'c59018f105be8d022714b826b0be130a') - - depends_on('libxml2') - depends_on('libxslt') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/atk/package.py b/var/spack/packages/atk/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/atlas/package.py b/var/spack/packages/atlas/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/autoconf/package.py b/var/spack/packages/autoconf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/automaded/package.py b/var/spack/packages/automaded/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/automake/package.py b/var/spack/packages/automake/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/bear/package.py b/var/spack/packages/bear/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/bib2xhtml/package.py b/var/spack/packages/bib2xhtml/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/bison/package.py b/var/spack/packages/bison/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py old mode 100644 new mode 100755 index 35824d53a26..254d1afda1d --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -14,10 +14,6 @@ class Boost(Package): list_url = "http://sourceforge.net/projects/boost/files/boost/" list_depth = 2 - version('1.59.0', '6aa9a5c6a4ca1016edd0ed1178e3cb87') - version('1.58.0', 'b8839650e61e9c1c0a89f371dd475546') - version('1.57.0', '1be49befbdd9a5ce9def2983ba3e7b76') - version('1.56.0', 'a744cf167b05d72335f27c88115f211d') version('1.55.0', 'd6eef4b4cacb2183f2bf265a5a03a354') version('1.54.0', '15cb8c0803064faef0c4ddf5bc5ca279') version('1.53.0', 'a00d22605d5dbcfb4c9936a9b35bc4c2') diff --git a/var/spack/packages/bowtie2/bowtie2-2.5.patch b/var/spack/packages/bowtie2/bowtie2-2.5.patch deleted file mode 100644 index 290be39c732..00000000000 --- a/var/spack/packages/bowtie2/bowtie2-2.5.patch +++ /dev/null @@ -1,16 +0,0 @@ ---- Makefile 2015-02-26 10:50:00.000000000 -0800 -+++ Makefile.new 2015-07-29 18:03:59.891357399 -0700 -@@ -22,10 +22,10 @@ - # - - INC = --GCC_PREFIX = $(shell dirname `which gcc`) -+GCC_PREFIX = - GCC_SUFFIX = --CC = $(GCC_PREFIX)/gcc$(GCC_SUFFIX) --CPP = $(GCC_PREFIX)/g++$(GCC_SUFFIX) -+CC = cc -+CPP = c++ - CXX = $(CPP) - HEADERS = $(wildcard *.h) - BOWTIE_MM = 1 diff --git a/var/spack/packages/bowtie2/package.py b/var/spack/packages/bowtie2/package.py deleted file mode 100644 index 339aab6598d..00000000000 --- a/var/spack/packages/bowtie2/package.py +++ /dev/null @@ -1,24 +0,0 @@ -from spack import * -from glob import glob -class Bowtie2(Package): - """Description""" - homepage = "bowtie-bio.sourceforge.net/bowtie2/index.shtml" - version('2.2.5','51fa97a862d248d7ee660efc1147c75f', url = "http://downloads.sourceforge.net/project/bowtie-bio/bowtie2/2.2.5/bowtie2-2.2.5-source.zip") - - patch('bowtie2-2.5.patch',when='@2.2.5', level=0) - - def install(self, spec, prefix): - make() - mkdirp(prefix.bin) - for bow in glob("bowtie2*"): - install(bow, prefix.bin) - # install('bowtie2',prefix.bin) - # install('bowtie2-align-l',prefix.bin) - # install('bowtie2-align-s',prefix.bin) - # install('bowtie2-build',prefix.bin) - # install('bowtie2-build-l',prefix.bin) - # install('bowtie2-build-s',prefix.bin) - # install('bowtie2-inspect',prefix.bin) - # install('bowtie2-inspect-l',prefix.bin) - # install('bowtie2-inspect-s',prefix.bin) - diff --git a/var/spack/packages/boxlib/package.py b/var/spack/packages/boxlib/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/bzip2/package.py b/var/spack/packages/bzip2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cairo/package.py b/var/spack/packages/cairo/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/callpath/package.py b/var/spack/packages/callpath/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cblas/package.py b/var/spack/packages/cblas/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cgm/package.py b/var/spack/packages/cgm/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cloog/package.py b/var/spack/packages/cloog/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/coreutils/package.py b/var/spack/packages/coreutils/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cppcheck/package.py b/var/spack/packages/cppcheck/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cram/package.py b/var/spack/packages/cram/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/cscope/package.py b/var/spack/packages/cscope/package.py deleted file mode 100644 index 9aac0f73042..00000000000 --- a/var/spack/packages/cscope/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Cscope(Package): - """Cscope is a developer's tool for browsing source code.""" - homepage = "http://http://cscope.sourceforge.net/" - url = "http://downloads.sourceforge.net/project/cscope/cscope/15.8b/cscope-15.8b.tar.gz" - - version('15.8b', '8f9409a238ee313a96f9f87fe0f3b176') - - # Can be configured to use flex (not necessary) - # ./configure --with-flex - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/cube/package.py b/var/spack/packages/cube/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/czmq/package.py b/var/spack/packages/czmq/package.py deleted file mode 100644 index a2f19475541..00000000000 --- a/var/spack/packages/czmq/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Czmq(Package): - """ A C interface to the ZMQ library """ - homepage = "http://czmq.zeromq.org" - url = "https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz" - - version('3.0.2', '23e9885f7ee3ce88d99d0425f52e9be1', url='https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz') - - depends_on('zeromq') - - def install(self, spec, prefix): - bash = which("bash") - bash("./autogen.sh") - configure("--prefix=%s" % prefix) - - make() - make("install") - diff --git a/var/spack/packages/dbus/package.py b/var/spack/packages/dbus/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/docbook-xml/package.py b/var/spack/packages/docbook-xml/package.py deleted file mode 100644 index fce1de7debf..00000000000 --- a/var/spack/packages/docbook-xml/package.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import glob -from spack import * - - -class DocbookXml(Package): - """Docbook DTD XML files.""" - homepage = "http://www.oasis-open.org/docbook" - url = "http://www.oasis-open.org/docbook/xml/4.5/docbook-xml-4.5.zip" - - version('4.5', '03083e288e87a7e829e437358da7ef9e') - - def install(self, spec, prefix): - cp = which('cp') - - install_args = ['-a', '-t', prefix] - install_args.extend(glob.glob('*')) - - cp(*install_args) diff --git a/var/spack/packages/doxygen/package.py b/var/spack/packages/doxygen/package.py deleted file mode 100644 index 3d4a4e47a78..00000000000 --- a/var/spack/packages/doxygen/package.py +++ /dev/null @@ -1,25 +0,0 @@ -#------------------------------------------------------------------------------ -# Author: Justin Too -# Date: September 11, 2015 -#------------------------------------------------------------------------------ - -from spack import * - -class Doxygen(Package): - """Doxygen is the de facto standard tool for generating documentation - from annotated C++ sources, but it also supports other popular programming - languages such as C, Objective-C, C#, PHP, Java, Python, IDL (Corba, - Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, Tcl, and to some extent D.. - """ - homepage = "http://www.stack.nl/~dimitri/doxygen/" - url = "http://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.10.src.tar.gz" - - version('1.8.10', '79767ccd986f12a0f949015efb5f058f') - - depends_on("cmake@2.8.12:") - - def install(self, spec, prefix): - cmake('.', *std_cmake_args) - - make() - make("install") diff --git a/var/spack/packages/dri2proto/package.py b/var/spack/packages/dri2proto/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/dtcmp/package.py b/var/spack/packages/dtcmp/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/eigen/package.py b/var/spack/packages/eigen/package.py new file mode 100644 index 00000000000..b4d89ea3a0e --- /dev/null +++ b/var/spack/packages/eigen/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Eigen(Package): + """Eigen is a C++ template library for linear algebra: matrices, vectors, + numerical solvers, and related algorithms..""" + homepage = "http://eigen.tuxfamily.org/" + url = "http://bitbucket.org/eigen/eigen/get/3.2.5.tar.gz" + + version('3.2.5', '8cc513ac6ec687117acadddfcacf551b') + + def install(self, spec, prefix): + import os + os.system("mkdir ./build_dir && cd ./build_dir") + cmake('../', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/packages/elfutils/package.py b/var/spack/packages/elfutils/package.py deleted file mode 100644 index 926d2345846..00000000000 --- a/var/spack/packages/elfutils/package.py +++ /dev/null @@ -1,26 +0,0 @@ -from spack import * - -class Elfutils(Package): - """elfutils is a collection of various binary tools such as - eu-objdump, eu-readelf, and other utilities that allow you to - inspect and manipulate ELF files. Refer to Table 5.Tools Included - in elfutils for Red Hat Developer for a complete list of binary - tools that are distributed with the Red Hat Developer Toolset - version of elfutils.""" - - homepage = "https://fedorahosted.org/elfutils/" - - version('0.163', - git='git://git.fedorahosted.org/git/elfutils.git', - tag='elfutils-0.163') - - provides('elf') - - def install(self, spec, prefix): - autoreconf = which('autoreconf') - autoreconf('-if') - - configure('--prefix=%s' % prefix, '--enable-maintainer-mode') - make() - make("install") - diff --git a/var/spack/packages/extrae/package.py b/var/spack/packages/extrae/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/exuberant-ctags/package.py b/var/spack/packages/exuberant-ctags/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/fish/package.py b/var/spack/packages/fish/package.py deleted file mode 100644 index 12255587057..00000000000 --- a/var/spack/packages/fish/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Fish(Package): - """fish is a smart and user-friendly command line shell for OS X, Linux, and - the rest of the family. - """ - - homepage = "http://fishshell.com/" - url = "http://fishshell.com/files/2.2.0/fish-2.2.0.tar.gz" - list_url = homepage - - version('2.2.0', 'a76339fd14ce2ec229283c53e805faac48c3e99d9e3ede9d82c0554acfc7b77a') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/flex/package.py b/var/spack/packages/flex/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/flux/package.py b/var/spack/packages/flux/package.py deleted file mode 100644 index c128f46be8b..00000000000 --- a/var/spack/packages/flux/package.py +++ /dev/null @@ -1,36 +0,0 @@ -from spack import * -import os - -class Flux(Package): - """ A next-generation resource manager (pre-alpha) """ - - homepage = "https://github.com/flux-framework/flux-core" - url = "https://github.com/flux-framework/flux-core" - - version('master', branch='master', git='https://github.com/flux-framework/flux-core') - - # Also needs autotools, but should use the system version if available - depends_on("zeromq@4.0.4:") - depends_on("czmq@2.2:") - depends_on("lua@5.1:5.1.99") - depends_on("munge") - depends_on("libjson-c") - depends_on("libxslt") - # TODO: This provides a catalog, hacked with environment below for now - depends_on("docbook-xml") - depends_on("asciidoc") - depends_on("python") - depends_on("py-cffi") - - def install(self, spec, prefix): - # Bootstrap with autotools - bash = which('bash') - bash('./autogen.sh') - - # Fix asciidoc dependency on xml style sheets and whatnot - os.environ['XML_CATALOG_FILES'] = os.path.join(spec['docbook-xml'].prefix, - 'catalog.xml') - # Configure, compile & install - configure("--prefix=" + prefix) - make("install", "V=1") - diff --git a/var/spack/packages/fontconfig/package.py b/var/spack/packages/fontconfig/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/freetype/package.py b/var/spack/packages/freetype/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gasnet/package.py b/var/spack/packages/gasnet/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gdk-pixbuf/package.py b/var/spack/packages/gdk-pixbuf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/geos/package.py b/var/spack/packages/geos/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gflags/package.py b/var/spack/packages/gflags/package.py deleted file mode 100644 index 62dd80a0944..00000000000 --- a/var/spack/packages/gflags/package.py +++ /dev/null @@ -1,21 +0,0 @@ -import os -from spack import * - -class Gflags(Package): - """The gflags package contains a C++ library that implements - commandline flags processing. It includes built-in support for - standard types such as string and the ability to define flags - in the source file in which they are used. Online documentation - available at: https://gflags.github.io/gflags/""" - - homepage = "https://gflags.github.io/gflags" - url = "https://github.com/gflags/gflags/archive/v2.1.2.tar.gz" - - version('2.1.2', 'ac432de923f9de1e9780b5254884599f') - - def install(self, spec, prefix): - cmake("-DCMAKE_INSTALL_PREFIX=" + prefix, - "-DBUILD_SHARED_LIBS=ON") - make() - make("test") - make("install") diff --git a/var/spack/packages/ghostscript/package.py b/var/spack/packages/ghostscript/package.py deleted file mode 100644 index 0ab49d425f7..00000000000 --- a/var/spack/packages/ghostscript/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Ghostscript(Package): - """an interpreter for the PostScript language and for PDF. """ - homepage = "http://ghostscript.com/" - url = "http://downloads.ghostscript.com/public/ghostscript-9.16.tar.gz" - - version('9.16', '829319325bbdb83f5c81379a8f86f38f') - - parallel = False - - def install(self, spec, prefix): - configure("--prefix=%s" %prefix, "--enable-shared") - - make() - make("install") - diff --git a/var/spack/packages/git/package.py b/var/spack/packages/git/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/glib/package.py b/var/spack/packages/glib/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/global/package.py b/var/spack/packages/global/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/glog/package.py b/var/spack/packages/glog/package.py deleted file mode 100644 index d73386b394b..00000000000 --- a/var/spack/packages/glog/package.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -from spack import * - -class Glog(Package): - """C++ implementation of the Google logging module.""" - - homepage = "https://github.com/google/glog" - url = "https://github.com/google/glog/archive/v0.3.3.tar.gz" - - version('0.3.3', 'c1f86af27bd9c73186730aa957607ed0') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/gmp/package.py b/var/spack/packages/gmp/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gnutls/package.py b/var/spack/packages/gnutls/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gperf/package.py b/var/spack/packages/gperf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/gperftools/package.py b/var/spack/packages/gperftools/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/graphlib/package.py b/var/spack/packages/graphlib/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/graphviz/package.py b/var/spack/packages/graphviz/package.py index 7af7da1881a..8244feccfdb 100644 --- a/var/spack/packages/graphviz/package.py +++ b/var/spack/packages/graphviz/package.py @@ -1,21 +1,24 @@ from spack import * class Graphviz(Package): - """Graph Visualization Software""" + """graph visualization software.""" homepage = "http://www.graphviz.org" url = "http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.38.0.tar.gz" version('2.38.0', '5b6a829b2ac94efcd5fa3c223ed6d3ae') + version('2.36.0', '1f41664dba0c93109ac8b71216bf2b57') - parallel = False - - depends_on("swig") - depends_on("python") - depends_on("ghostscript") + depends_on("cairo@1.1.10") + depends_on("freetype@2.1.10") + depends_on("fontconfig") + depends_on("zlib@1.2.3") + # depends_on("libpng@1.2.10") + # depends_on("expat@2.0.0") + # depends_on("gd@2.0.34") def install(self, spec, prefix): - configure("--prefix=%s" %prefix) + configure('--prefix=%s' % prefix) + # FIXME: Add logic to build and install here make() make("install") - diff --git a/var/spack/packages/gtkplus/package.py b/var/spack/packages/gtkplus/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/harfbuzz/package.py b/var/spack/packages/harfbuzz/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/hypre/package.py b/var/spack/packages/hypre/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/icu/package.py b/var/spack/packages/icu/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/icu4c/package.py b/var/spack/packages/icu4c/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/isl/package.py b/var/spack/packages/isl/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/jdk/package.py b/var/spack/packages/jdk/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/jpeg/package.py b/var/spack/packages/jpeg/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/netlib-lapack/package.py b/var/spack/packages/lapack/package.py old mode 100644 new mode 100755 similarity index 77% rename from var/spack/packages/netlib-lapack/package.py rename to var/spack/packages/lapack/package.py index fb6b99e27c3..d9d37e3e4aa --- a/var/spack/packages/netlib-lapack/package.py +++ b/var/spack/packages/lapack/package.py @@ -1,6 +1,6 @@ from spack import * -class NetlibLapack(Package): +class Lapack(Package): """ LAPACK version 3.X is a comprehensive FORTRAN library that does linear algebra operations including matrix inversions, least @@ -18,16 +18,9 @@ class NetlibLapack(Package): version('3.4.0', '02d5706ec03ba885fc246e5fa10d8c70') version('3.3.1', 'd0d533ec9a5b74933c2a1e84eedc58b4') - variant('shared', default=False, description="Build shared library version") - - # virtual dependency - provides('lapack') - # blas is a virtual dependency. depends_on('blas') - depends_on('cmake') - # Doesn't always build correctly in parallel parallel = False @@ -46,14 +39,7 @@ def get_blas_libs(self): def install(self, spec, prefix): blas_libs = ";".join(self.get_blas_libs()) - cmake_args = [".", '-DBLAS_LIBRARIES=' + blas_libs] - - if '+shared' in spec: - cmake_args.append('-DBUILD_SHARED_LIBS=ON') - - cmake_args += std_cmake_args - - cmake(*cmake_args) + cmake(".", '-DBLAS_LIBRARIES=' + blas_libs, *std_cmake_args) make() make("install") diff --git a/var/spack/packages/launchmon/package.py b/var/spack/packages/launchmon/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/launchmon/patch.lmon_install_dir b/var/spack/packages/launchmon/patch.lmon_install_dir old mode 100644 new mode 100755 diff --git a/var/spack/packages/lcms/package.py b/var/spack/packages/lcms/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/leveldb/package.py b/var/spack/packages/leveldb/package.py deleted file mode 100644 index da68a9cbcb2..00000000000 --- a/var/spack/packages/leveldb/package.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import glob -from spack import * - -class Leveldb(Package): - """LevelDB is a fast key-value storage library written at Google - that provides an ordered mapping from string keys to string values.""" - - homepage = "https://github.com/google/leveldb" - url = "https://github.com/google/leveldb/archive/v1.18.tar.gz" - - version('1.18', '73770de34a2a5ab34498d2e05b2b7fa0') - - depends_on("snappy") - - def install(self, spec, prefix): - make() - - mkdirp(prefix.include) - mkdirp(prefix.lib) - - cp = which('cp') - - # cp --preserve=links libleveldb.* prefix/lib - args = glob.glob('libleveldb.*') - args.append(prefix + '/lib') - cp('--preserve=links', *args) - - cp('-r', 'include/leveldb', prefix + '/include') diff --git a/var/spack/packages/libNBC/package.py b/var/spack/packages/libNBC/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libarchive/package.py b/var/spack/packages/libarchive/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libcircle/package.py b/var/spack/packages/libcircle/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libdrm/package.py b/var/spack/packages/libdrm/package.py old mode 100644 new mode 100755 index 00736b7811a..1cab5372bec --- a/var/spack/packages/libdrm/package.py +++ b/var/spack/packages/libdrm/package.py @@ -2,7 +2,7 @@ class Libdrm(Package): """A userspace library for accessing the DRM, direct - rendering manager, on Linux, BSD and other operating + rendering manager, on Linux, BSD and other operating systems that support the ioctl interface.""" homepage = "http://dri.freedesktop.org/libdrm/" # no real website... @@ -11,8 +11,11 @@ class Libdrm(Package): version('2.4.59', '105ac7af1afcd742d402ca7b4eb168b6') version('2.4.33', '86e4e3debe7087d5404461e0032231c8') + depends_on("libpciaccess") + def install(self, spec, prefix): configure("--prefix=%s" % prefix) + make("clean") make() make("install") diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/packages/libdwarf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libelf/package.py b/var/spack/packages/libelf/package.py old mode 100644 new mode 100755 index 9338b8f393e..bf2fefabd50 --- a/var/spack/packages/libelf/package.py +++ b/var/spack/packages/libelf/package.py @@ -36,8 +36,6 @@ class Libelf(Package): version('0.8.13', '4136d7b4c04df68b686570afa26988ac') version('0.8.12', 'e21f8273d9f5f6d43a59878dc274fec7') - provides('elf') - def install(self, spec, prefix): configure("--prefix=" + prefix, "--enable-shared", diff --git a/var/spack/packages/libevent/package.py b/var/spack/packages/libevent/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libffi/package.py b/var/spack/packages/libffi/package.py old mode 100644 new mode 100755 index 18acabb00cf..2c1c4eed4d3 --- a/var/spack/packages/libffi/package.py +++ b/var/spack/packages/libffi/package.py @@ -6,9 +6,9 @@ class Libffi(Package): to call any function specified by a call interface description at run time.""" homepage = "https://sourceware.org/libffi/" - - version('3.2.1','83b89587607e3eb65c70d361f13bab43',url = "ftp://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz") - #version('3.1', 'f5898b29bbfd70502831a212d9249d10',url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz") # Has a bug $(lib64) instead of ${lib64} in libffi.pc + url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz" + + version('3.1', 'f5898b29bbfd70502831a212d9249d10') def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/packages/libgcrypt/package.py b/var/spack/packages/libgcrypt/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libgpg-error/package.py b/var/spack/packages/libgpg-error/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libjpeg-turbo/package.py b/var/spack/packages/libjpeg-turbo/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libjson-c/package.py b/var/spack/packages/libjson-c/package.py deleted file mode 100644 index c0801cce9cd..00000000000 --- a/var/spack/packages/libjson-c/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class LibjsonC(Package): - """ A JSON implementation in C """ - homepage = "https://github.com/json-c/json-c/wiki" - url = "https://s3.amazonaws.com/json-c_releases/releases/json-c-0.11.tar.gz" - - version('0.11', 'aa02367d2f7a830bf1e3376f77881e98') - - def install(self, spec, prefix): - configure('--prefix=%s' % prefix) - - make() - make("install") diff --git a/var/spack/packages/libmng/package.py b/var/spack/packages/libmng/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libmonitor/package.py b/var/spack/packages/libmonitor/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/packages/libpciaccess/package.py old mode 100644 new mode 100755 index 6022fc34a30..371fbb66854 --- a/var/spack/packages/libpciaccess/package.py +++ b/var/spack/packages/libpciaccess/package.py @@ -1,21 +1,16 @@ from spack import * class Libpciaccess(Package): - """Generic PCI access library.""" - + """Generic PCI access library""" homepage = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" - url = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" + url = "http://pkgs.fedoraproject.org/repo/pkgs/libpciaccess/libpciaccess-0.13.2.tar.bz2/b7c0d3afce14eedca57312a3141ec13a/libpciaccess-0.13.2.tar.bz2" - version('0.13.4', git='http://anongit.freedesktop.org/git/xorg/lib/libpciaccess.git', - tag='libpciaccess-0.13.4') - - depends_on('autoconf') - depends_on('libtool') + version('0.13.2', 'b7c0d3afce14eedca57312a3141ec13a') def install(self, spec, prefix): - from subprocess import call - call(["./autogen.sh"]) - configure("--prefix=%s" % prefix) + import os + os.system("autoconf") + configure('--prefix=%s' % prefix) make() make("install") diff --git a/var/spack/packages/libpng/package.py b/var/spack/packages/libpng/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libsodium/package.py b/var/spack/packages/libsodium/package.py deleted file mode 100644 index 1c8a16d9984..00000000000 --- a/var/spack/packages/libsodium/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class Libsodium(Package): - """Sodium is a modern, easy-to-use software library for encryption, - decryption, signatures, password hashing and more.""" - homepage = "https://download.libsodium.org/doc/" - url = "https://download.libsodium.org/libsodium/releases/libsodium-1.0.3.tar.gz" - - version('1.0.3', 'b3bcc98e34d3250f55ae196822307fab') - version('1.0.2', 'dc40eb23e293448c6fc908757738003f') - version('1.0.1', '9a221b49fba7281ceaaf5e278d0f4430') - version('1.0.0', '3093dabe4e038d09f0d150cef064b2f7') - version('0.7.1', 'c224fe3923d1dcfe418c65c8a7246316') - - def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/libtiff/package.py b/var/spack/packages/libtiff/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libtool/package.py b/var/spack/packages/libtool/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libunwind/package.py b/var/spack/packages/libunwind/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libuuid/package.py b/var/spack/packages/libuuid/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libxcb/package.py b/var/spack/packages/libxcb/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py old mode 100644 new mode 100755 index 3a0af6b3684..72199d8def8 --- a/var/spack/packages/libxml2/package.py +++ b/var/spack/packages/libxml2/package.py @@ -9,12 +9,11 @@ class Libxml2(Package): version('2.9.2', '9e6a9aca9d155737868b3dc5fd82f788') - extends('python') depends_on('zlib') depends_on('xz') def install(self, spec, prefix): - configure("--prefix=%s" % prefix) - + configure("--prefix=%s" % prefix, + "--without-python") make() make("install") diff --git a/var/spack/packages/libxshmfence/package.py b/var/spack/packages/libxshmfence/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/libxslt/package.py b/var/spack/packages/libxslt/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/llvm-lld/package.py b/var/spack/packages/llvm-lld/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/lmdb/package.py b/var/spack/packages/lmdb/package.py deleted file mode 100644 index 875b8100c5a..00000000000 --- a/var/spack/packages/lmdb/package.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -from spack import * - -class Lmdb(Package): - """Read-only mirror of official repo on openldap.org. Issues and - pull requests here are ignored. Use OpenLDAP ITS for issues. - http://www.openldap.org/software/repo.html""" - - - homepage = "http://www.openldap.org/software/repo.html" - url = "https://github.com/LMDB/lmdb/archive/LMDB_0.9.16.tar.gz" - - version('0.9.16', '0de89730b8f3f5711c2b3a4ba517b648') - - def install(self, spec, prefix): - os.chdir('libraries/liblmdb') - - make() - - mkdirp(prefix.bin) - mkdirp(prefix + '/man/man1') - mkdirp(prefix.lib) - mkdirp(prefix.include) - - bins = ['mdb_stat', 'mdb_copy', 'mdb_dump', 'mdb_load'] - for f in bins: - install(f, prefix.bin) - - mans = ['mdb_stat.1', 'mdb_copy.1', 'mdb_dump.1', 'mdb_load.1'] - for f in mans: - install(f, prefix + '/man/man1') - - libs = ['liblmdb.a', 'liblmdb.so'] - for f in libs: - install(f, prefix.lib) - - includes = ['lmdb.h'] - for f in includes: - install(f, prefix.include) diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py deleted file mode 100644 index 57c443cc2d6..00000000000 --- a/var/spack/packages/lua/package.py +++ /dev/null @@ -1,26 +0,0 @@ -from spack import * -import os - -class Lua(Package): - """ The Lua programming language interpreter and library """ - homepage = "http://www.lua.org" - url = "http://www.lua.org/ftp/lua-5.1.5.tar.gz" - - version('5.3.1', '797adacada8d85761c079390ff1d9961') - version('5.3.0', 'a1b0a7e92d0c85bbff7a8d27bf29f8af') - version('5.2.4', '913fdb32207046b273fdb17aad70be13') - version('5.2.3', 'dc7f94ec6ff15c985d2d6ad0f1b35654') - version('5.2.2', 'efbb645e897eae37cad4344ce8b0a614') - version('5.2.1', 'ae08f641b45d737d12d30291a5e5f6e3') - version('5.2.0', 'f1ea831f397214bae8a265995ab1a93e') - version('5.1.5', '2e115fe26e435e33b0d5c022e4490567') - version('5.1.4', 'd0870f2de55d59c1c8419f36e8fac150') - version('5.1.3', 'a70a8dfaa150e047866dc01a46272599') - - depends_on('ncurses') - - def install(self, spec, prefix): - make('INSTALL_TOP=%s' % prefix, - 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, - 'linux', - 'install') diff --git a/var/spack/packages/lwgrp/package.py b/var/spack/packages/lwgrp/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/lwm2/package.py b/var/spack/packages/lwm2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/memaxes/package.py b/var/spack/packages/memaxes/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py old mode 100644 new mode 100755 index 2a04a8fd511..2dba878a77c --- a/var/spack/packages/mesa/package.py +++ b/var/spack/packages/mesa/package.py @@ -1,7 +1,7 @@ from spack import * class Mesa(Package): - """Mesa is an open-source implementation of the OpenGL + """Mesa is an open-source implementation of the OpenGL specification - a system for rendering interactive 3D graphics.""" homepage = "http://www.mesa3d.org" @@ -11,10 +11,9 @@ class Mesa(Package): # version('10.4.4', '8d863a3c209bf5116b2babfccccc68ce') version('8.0.5', 'cda5d101f43b8784fa60bdeaca4056f2') - # mesa 7.x, 8.x, 9.x + # mesa 7.x, 8.x, 9.x depends_on("libdrm@2.4.33") depends_on("llvm@3.0") - depends_on("libxml2") # patch("llvm-fixes.patch") # using newer llvm diff --git a/var/spack/packages/metis/package.py b/var/spack/packages/metis/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpe2/mpe2.patch b/var/spack/packages/mpe2/mpe2.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpe2/package.py b/var/spack/packages/mpe2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpfr/package.py b/var/spack/packages/mpfr/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpibash/mpibash-4.3.patch b/var/spack/packages/mpibash/mpibash-4.3.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpibash/package.py b/var/spack/packages/mpibash/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py old mode 100644 new mode 100755 index b6b2dfde214..6aa8f2eca4b --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -38,6 +38,7 @@ class Mpich(Package): provides('mpi@:3', when='@3:') provides('mpi@:1', when='@1:') + def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" os.environ['MPICH_CC'] = 'cc' diff --git a/var/spack/packages/mpileaks/package.py b/var/spack/packages/mpileaks/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mrnet/package.py b/var/spack/packages/mrnet/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/munge/package.py b/var/spack/packages/munge/package.py deleted file mode 100644 index c737ca03546..00000000000 --- a/var/spack/packages/munge/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * -import os - -class Munge(Package): - """ MUNGE Uid 'N' Gid Emporium """ - homepage = "https://code.google.com/p/munge/" - url = "https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2" - - version('0.5.11', 'bd8fca8d5f4c1fcbef1816482d49ee01', url='https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2') - - depends_on('openssl') - depends_on('libgcrypt') - - def install(self, spec, prefix): - os.makedirs(os.path.join(prefix, "lib/systemd/system")) - configure("--prefix=%s" % prefix) - - make() - make("install") - diff --git a/var/spack/packages/muster/package.py b/var/spack/packages/muster/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch b/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/nasm/package.py b/var/spack/packages/nasm/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/ncurses/package.py b/var/spack/packages/ncurses/package.py old mode 100644 new mode 100755 index 8f5763bfdd2..a22e83503f9 --- a/var/spack/packages/ncurses/package.py +++ b/var/spack/packages/ncurses/package.py @@ -21,11 +21,3 @@ def install(self, spec, prefix): make() make("install") - configure("--prefix=%s" % prefix, - "--with-shared", - "--disable-widec", - "--disable-pc-files", - "--without-ada") - make() - make("install") - diff --git a/var/spack/packages/netcdf/package.py b/var/spack/packages/netcdf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/netgauge/package.py b/var/spack/packages/netgauge/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/netlib-blas/package.py b/var/spack/packages/netlib-blas/package.py old mode 100644 new mode 100755 index 85e97323d35..0a6cdb04423 --- a/var/spack/packages/netlib-blas/package.py +++ b/var/spack/packages/netlib-blas/package.py @@ -9,8 +9,6 @@ class NetlibBlas(Package): version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') - variant('fpic', default=False, description="Build with -fpic compiler option") - # virtual dependency provides('blas') @@ -25,10 +23,6 @@ def patch(self): mf.filter('^LOADER.*', 'LOADER = f90') mf.filter('^CC =.*', 'CC = cc') - if '+fpic' in self.spec: - mf.filter('^OPTS.*=.*', 'OPTS = -O2 -frecursive -fpic') - mf.filter('^CFLAGS =.*', 'CFLAGS = -O3 -fpic') - def install(self, spec, prefix): make('blaslib') diff --git a/var/spack/packages/nettle/package.py b/var/spack/packages/nettle/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/ompss/package.py b/var/spack/packages/ompss/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/opari2/package.py b/var/spack/packages/opari2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch b/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/openmpi/llnl-platforms.patch b/var/spack/packages/openmpi/llnl-platforms.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/openmpi/package.py b/var/spack/packages/openmpi/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/otf/package.py b/var/spack/packages/otf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/otf2/package.py b/var/spack/packages/otf2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/pango/package.py b/var/spack/packages/pango/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/papi/package.py b/var/spack/packages/papi/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/paraver/package.py b/var/spack/packages/paraver/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/parmetis/package.py b/var/spack/packages/parmetis/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/parpack/package.py b/var/spack/packages/parpack/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/pcre/package.py b/var/spack/packages/pcre/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/petsc/package.py b/var/spack/packages/petsc/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/pidx/package.py b/var/spack/packages/pidx/package.py deleted file mode 100644 index 81aed62fb10..00000000000 --- a/var/spack/packages/pidx/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack import * - -class Pidx(Package): - """PIDX Parallel I/O Library. - - PIDX is an efficient parallel I/O library that reads and writes - multiresolution IDX data files. - """ - - homepage = "http://www.cedmav.com/pidx" - - version('1.0', git='https://github.com/sci-visus/PIDX.git', - commit='6afa1cf71d1c41263296dc049c8fabaf73c296da') - - depends_on("mpi") - - def install(self, spec, prefix): - with working_dir('spack-build', create=True): - cmake('..', *std_cmake_args) - make() - make("install") diff --git a/var/spack/packages/pixman/package.py b/var/spack/packages/pixman/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/pkg-config/package.py b/var/spack/packages/pkg-config/package.py deleted file mode 100644 index 9964c6ce346..00000000000 --- a/var/spack/packages/pkg-config/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class PkgConfig(Package): - """pkg-config is a helper tool used when compiling applications and libraries""" - homepage = "http://www.freedesktop.org/wiki/Software/pkg-config/" - url = "http://pkgconfig.freedesktop.org/releases/pkg-config-0.28.tar.gz" - - version('0.28', 'aa3c86e67551adc3ac865160e34a2a0d') - - parallel = False - - def install(self, spec, prefix): - configure("--prefix=%s" %prefix, "--enable-shared") - - make() - make("install") - diff --git a/var/spack/packages/pmgr_collective/package.py b/var/spack/packages/pmgr_collective/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/postgresql/package.py b/var/spack/packages/postgresql/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/ppl/package.py b/var/spack/packages/ppl/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/protobuf/package.py b/var/spack/packages/protobuf/package.py deleted file mode 100644 index 34085c7ce95..00000000000 --- a/var/spack/packages/protobuf/package.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -from spack import * - -class Protobuf(Package): - """Google's data interchange format.""" - - homepage = "https://developers.google.com/protocol-buffers" - url = "https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.bz2" - - version('2.5.0', 'a72001a9067a4c2c4e0e836d0f92ece4') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("check") - make("install") diff --git a/var/spack/packages/py-Cython/package.py b/var/spack/packages/py-Cython/package.py new file mode 100644 index 00000000000..c2501391143 --- /dev/null +++ b/var/spack/packages/py-Cython/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyCython(Package): + """The Cython compiler for writing C extensions for the Python language.""" + homepage = "http://www.cython.org" + version("0.21.2", "d21adb870c75680dc857cd05d41046a4", + url="https://pypi.python.org/packages/source/C/Cython/Cython-0.21.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-Distutils2/package.py b/var/spack/packages/py-Distutils2/package.py new file mode 100644 index 00000000000..8690f569fee --- /dev/null +++ b/var/spack/packages/py-Distutils2/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyDistutils2(Package): + """Python Packaging Library.""" + + homepage = "https://hg.python.org/distutils2" + url = "https://pypi.python.org/packages/source/D/Distutils2/Distutils2-1.0a4.tar.gz" + + version('1.0a4', '52bc9dffb394970c27e02853ae3a3241') + + depends_on("python") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + diff --git a/var/spack/packages/py-astroML/package.py b/var/spack/packages/py-astroML/package.py new file mode 100644 index 00000000000..1583e249d21 --- /dev/null +++ b/var/spack/packages/py-astroML/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyAstroml(Package): + """tools for machine learning and data mining in Astronomy""" + homepage = "http://astroML.github.com" + version("0.2", "85f558368546660564f20b30efafb024", + url="https://pypi.python.org/packages/source/a/astroML/astroML-0.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-astropy/package.py b/var/spack/packages/py-astropy/package.py new file mode 100644 index 00000000000..5d2d0619ccf --- /dev/null +++ b/var/spack/packages/py-astropy/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyAstropy(Package): + """Community-developed python astronomy tools""" + homepage = "http://astropy.org" + version("0.4.2", "6dc4f643cde37ba0a8b4967dc8becee8", + url="https://pypi.python.org/packages/source/a/astropy/astropy-0.4.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-autopep8/package.py b/var/spack/packages/py-autopep8/package.py new file mode 100644 index 00000000000..39a2288ea04 --- /dev/null +++ b/var/spack/packages/py-autopep8/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyAutopep8(Package): + """A tool that automatically formats Python code to conform to the PEP 8 style guide""" + homepage = "https://github.com/hhatto/autopep8" + version("1.1", "7998358d8f0efd77dcb2cc8e34e3cb5c", + url="https://pypi.python.org/packages/source/a/autopep8/autopep8-1.1.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-biopython/package.py b/var/spack/packages/py-biopython/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-certifi/package.py b/var/spack/packages/py-certifi/package.py new file mode 100644 index 00000000000..442903e6a0f --- /dev/null +++ b/var/spack/packages/py-certifi/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyCertifi(Package): + """Python package for providing Mozilla's CA Bundle.""" + homepage = "http://python-requests.org" + version("14.05.14", "315ea4e50673a16ab047099f816fd32a", + url="https://pypi.python.org/packages/source/c/certifi/certifi-14.05.14.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-cffi/package.py b/var/spack/packages/py-cffi/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-cython/package.py b/var/spack/packages/py-cython/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-dateutil/package.py b/var/spack/packages/py-dateutil/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-decorator/package.py b/var/spack/packages/py-decorator/package.py new file mode 100644 index 00000000000..00c1c541d44 --- /dev/null +++ b/var/spack/packages/py-decorator/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyDecorator(Package): + """Better living through Python with decorators""" + homepage = "https://github.com/micheles/decorator" + version("4.0.4", "dd3a0669e1e6f09699eefa2c7fbd9756", + url="https://pypi.python.org/packages/source/d/decorator/decorator-4.0.4.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-emcee/package.py b/var/spack/packages/py-emcee/package.py new file mode 100644 index 00000000000..544066b92d1 --- /dev/null +++ b/var/spack/packages/py-emcee/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyEmcee(Package): + """Kick ass affine-invariant ensemble MCMC sampling""" + homepage = "http://dan.iel.fm/emcee/" + version("2.1.0", "c6b6fad05c824d40671d4a4fc58dfff7", + url="https://pypi.python.org/packages/source/e/emcee/emcee-2.1.0.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-epydoc/package.py b/var/spack/packages/py-epydoc/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-fitsio/package.py b/var/spack/packages/py-fitsio/package.py new file mode 100644 index 00000000000..046bc386d57 --- /dev/null +++ b/var/spack/packages/py-fitsio/package.py @@ -0,0 +1,13 @@ +from spack import * + +class PyFitsio(Package): + """A full featured python library to read from and write to FITS files.""" + homepage = "https://github.com/esheldon/fitsio" + version("0.9.7", "75fa05f999ae8f55c8290bd78ada49e7", + url="https://pypi.python.org/packages/source/f/fitsio/fitsio-0.9.7.tar.gz") + + extends("python") + depends_on("py-numpy") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-genders/package.py b/var/spack/packages/py-genders/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-gnuplot/package.py b/var/spack/packages/py-gnuplot/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-gnureadline/package.py b/var/spack/packages/py-gnureadline/package.py new file mode 100644 index 00000000000..51d1c6227eb --- /dev/null +++ b/var/spack/packages/py-gnureadline/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyGnureadline(Package): + """The standard Python readline extension statically linked against the GNU readline library.""" + homepage = "http://github.com/ludwigschwardt/python-gnureadline" + version("6.3.3", "c4af83c9a3fbeac8f2da9b5a7c60e51c", + url="https://pypi.python.org/packages/source/g/gnureadline/gnureadline-6.3.3.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/packages/py-h5py/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py old mode 100644 new mode 100755 index 8d0e64a07f1..3b94b71f295 --- a/var/spack/packages/py-ipython/package.py +++ b/var/spack/packages/py-ipython/package.py @@ -3,14 +3,29 @@ class PyIpython(Package): """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" homepage = "https://pypi.python.org/pypi/ipython" - url = "https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz" - version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf') + version('3.0.0','b3f00f3c0be036fafef3b0b9d663f27e', + url='https://pypi.python.org/packages/source/i/ipython/ipython-3.0.0.tar.gz') + version('4.0.0','c2fecbcf1c0fbdc82625c77a50733dd6', + url='https://pypi.python.org/packages/source/i/ipython/ipython-4.0.0.tar.gz') + version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', + url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') + version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', + url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') extends('python') depends_on('py-pygments') - depends_on('py-setuptools') + depends_on('py-setuptools@18.1') + depends_on('py-traitlets') + depends_on('py-pyzmq') + depends_on('py-pexpect') + depends_on('py-tornado') + depends_on('py-jinja2') def install(self, spec, prefix): + import os + # os.system("spack activate py-setuptools@18.1") + # os.system("easy_install pip") + # os.system("pip install ipython[notebook]") python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython/package.py.backup b/var/spack/packages/py-ipython/package.py.backup new file mode 100755 index 00000000000..1ed8b57e605 --- /dev/null +++ b/var/spack/packages/py-ipython/package.py.backup @@ -0,0 +1,27 @@ +from spack import * + +class PyIpython(Package): + """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" + homepage = "https://pypi.python.org/pypi/ipython" + + version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', + url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") + version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') + version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', + url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') + + extends('python') + depends_on('py-pygments') + depends_on('py-setuptools@18.1') + depends_on('py-traitlets') + depends_on('py-pyzmq') + depends_on('py-pexpect') + depends_on('py-tornado') + depends_on('py-Jinja2') + + def install(self, spec, prefix): + import os + # os.system("spack activate py-setuptools@18.1") + # os.system("easy_install pip") + # os.system("pip install ipython[notebook]") + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython_genutils/package.py b/var/spack/packages/py-ipython_genutils/package.py new file mode 100644 index 00000000000..39b15eec730 --- /dev/null +++ b/var/spack/packages/py-ipython_genutils/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyIpythonGenutils(Package): + """Vestigial utilities from IPython""" + homepage = "http://ipython.org" + version("0.1.0", "9a8afbe0978adbcbfcb3b35b2d015a56", + url="https://pypi.python.org/packages/source/i/ipython_genutils/ipython_genutils-0.1.0.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-joblib/package.py b/var/spack/packages/py-joblib/package.py new file mode 100644 index 00000000000..10672314def --- /dev/null +++ b/var/spack/packages/py-joblib/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyJoblib(Package): + """Lightweight pipelining: using Python functions as pipeline jobs.""" + homepage = "http://packages.python.org/joblib/" + version("0.8.4", "90a1c25cc4dc4a8e3536093dbc35cff3", + url="https://pypi.python.org/packages/source/j/joblib/joblib-0.8.4.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-jsonschema/package.py b/var/spack/packages/py-jsonschema/package.py new file mode 100755 index 00000000000..9a7fc218622 --- /dev/null +++ b/var/spack/packages/py-jsonschema/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyJsonschema(Package): + """implementation of JSON Schema validation for Python""" + homepage = "https://github.com/Julian/jsonschema" + url = "https://pypi.python.org/packages/source/j/jsonschema/jsonschema-2.5.1.tar.gz" + + version('2.4.0','661f85c3d23094afbb9ac3c0673840bf', + url='https://pypi.python.org/packages/source/j/jsonschema/jsonschema-2.4.0.tar.gz') + version('2.5.1', '374e848fdb69a3ce8b7e778b47c30640') + + extends("python") + depends_on("py-setuptools@18.1") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-libxml2/package.py b/var/spack/packages/py-libxml2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-lockfile/package.py b/var/spack/packages/py-lockfile/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-mako/package.py b/var/spack/packages/py-mako/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py old mode 100644 new mode 100755 index e7ce3dfd24a..b9244daf562 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -12,7 +12,6 @@ class PyMatplotlib(Package): extends('python', ignore=r'bin/nosetests.*$') depends_on('py-pyside') - depends_on('py-ipython') depends_on('py-pyparsing') depends_on('py-six') depends_on('py-dateutil') diff --git a/var/spack/packages/py-mock/package.py b/var/spack/packages/py-mock/package.py index 3b08428ba0c..c43842c1878 100644 --- a/var/spack/packages/py-mock/package.py +++ b/var/spack/packages/py-mock/package.py @@ -1,17 +1,12 @@ from spack import * class PyMock(Package): - """mock is a library for testing in Python. It allows you to replace parts - of your system under test with mock objects and make assertions about how - they have been used.""" + """A Python Mocking and Patching Library for Testing""" + homepage = "http://www.voidspace.org.uk/python/mock/" + version("1.0.1", "c3971991738caa55ec7c356bbc154ee2", + url="https://pypi.python.org/packages/source/m/mock/mock-1.0.1.tar.gz") - homepage = "https://github.com/testing-cabal/mock" - url = "https://pypi.python.org/packages/source/m/mock/mock-1.3.0.tar.gz" - - version('1.3.0', '73ee8a4afb3ff4da1b4afa287f39fdeb') - - extends('python') - depends_on('py-setuptools@17.1:') + extends("python") def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-mpi4py/package.py b/var/spack/packages/py-mpi4py/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-mx/package.py b/var/spack/packages/py-mx/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-nose/package.py b/var/spack/packages/py-nose/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-numexpr/package.py b/var/spack/packages/py-numexpr/package.py new file mode 100644 index 00000000000..283ea18772e --- /dev/null +++ b/var/spack/packages/py-numexpr/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyNumexpr(Package): + """Fast numerical expression evaluator for NumPy""" + homepage = "https://github.com/pydata/numexpr" + version("2.4", "df7e8d9e9dbb145b56d43c465c2bf854", + url="https://pypi.python.org/packages/source/n/numexpr/numexpr-2.4.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/packages/py-numpy/package.py old mode 100644 new mode 100755 index efa109a3e9c..6534e063d23 --- a/var/spack/packages/py-numpy/package.py +++ b/var/spack/packages/py-numpy/package.py @@ -7,22 +7,9 @@ class PyNumpy(Package): version('1.9.1', '78842b73560ec378142665e712ae4ad9') version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645') - + extends('python') depends_on('py-nose') - depends_on('netlib-blas+fpic') - depends_on('netlib-lapack+shared') - - def patch(self): - filter_file( - "possible_executables = \['(gfortran|g77|ifort|efl)", - "possible_executables = ['fc", - "numpy/distutils/fcompiler/gnu.py", - "numpy/distutils/fcompiler/intel.py") def install(self, spec, prefix): - with open('site.cfg', 'w') as f: - f.write('[DEFAULT]\n') - f.write('libraries=lapack,blas\n') - f.write('library_dirs=%s/lib:%s/lib\n' % (spec['blas'].prefix, spec['lapack'].prefix)) python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pandas/package.py b/var/spack/packages/py-pandas/package.py old mode 100644 new mode 100755 index 5b9997faa94..07ee12782f3 --- a/var/spack/packages/py-pandas/package.py +++ b/var/spack/packages/py-pandas/package.py @@ -6,6 +6,8 @@ class PyPandas(Package): homepage = "http://pandas.pydata.org/" url = "https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73" + version('0.15.1','f6fd8d49fc801f6f3fa48f9117a6290b', + url='https://pypi.python.org/packages/source/p/pandas/pandas-0.15.1.tar.gz') version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73') version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8') diff --git a/var/spack/packages/py-pep8/package.py b/var/spack/packages/py-pep8/package.py new file mode 100644 index 00000000000..0bcf4169228 --- /dev/null +++ b/var/spack/packages/py-pep8/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPep8(Package): + """Python style guide checker""" + homepage = "http://pep8.readthedocs.org/" + version("1.6.1", "76cf60b245f8549cb458ffcd85710738", + url="https://pypi.python.org/packages/source/p/pep8/pep8-1.6.1.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pexpect/package.py b/var/spack/packages/py-pexpect/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pickleshare/package.py b/var/spack/packages/py-pickleshare/package.py new file mode 100644 index 00000000000..cf9ddb7db88 --- /dev/null +++ b/var/spack/packages/py-pickleshare/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPickleshare(Package): + """Tiny 'shelve'-like database with concurrency support""" + homepage = "https://github.com/vivainio/pickleshare" + version("0.5", "25337740507cb855ad58bfcf60f7710e", + url="https://pypi.python.org/packages/source/p/pickleshare/pickleshare-0.5.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pil/package.py b/var/spack/packages/py-pil/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pip/package.py b/var/spack/packages/py-pip/package.py new file mode 100644 index 00000000000..89b8570fe46 --- /dev/null +++ b/var/spack/packages/py-pip/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPip(Package): + """The PyPA recommended tool for installing Python packages.""" + homepage = "https://pip.pypa.io/" + version("7.1.2", "3823d2343d9f3aaab21cf9c917710196", + url="https://pypi.python.org/packages/source/p/pip/pip-7.1.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pmw/package.py b/var/spack/packages/py-pmw/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-py/package.py b/var/spack/packages/py-py/package.py new file mode 100644 index 00000000000..0ef04031243 --- /dev/null +++ b/var/spack/packages/py-py/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPy(Package): + """library with cross-python path, ini-parsing, io, code, log facilities""" + homepage = "http://pylib.readthedocs.org/" + version("1.4.26", "30c3fd92a53f1a5ed6f3591c1fe75c0e", + url="https://pypi.python.org/packages/source/p/py/py-1.4.26.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-py4j/package.py b/var/spack/packages/py-py4j/package.py new file mode 100755 index 00000000000..4c31f572982 --- /dev/null +++ b/var/spack/packages/py-py4j/package.py @@ -0,0 +1,15 @@ +from spack import * + + +class PyPy4j(Package): + """Enables Python programs to dynamically access arbitrary Java objects""" + homepage = "https://www.py4j.org/" + url = "https://pypi.python.org/packages/source/p/py4j/py4j-0.9.tar.gz" + + version('0.9', 'b6fed5faef81a5368e3d50a91a5c9a60') + + extends("python") + depends_on("py-setuptools@18.1") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pychecker/package.py b/var/spack/packages/py-pychecker/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pycparser/package.py b/var/spack/packages/py-pycparser/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pyelftools/package.py b/var/spack/packages/py-pyelftools/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pygments/package.py b/var/spack/packages/py-pygments/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pylint/package.py b/var/spack/packages/py-pylint/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pypar/package.py b/var/spack/packages/py-pypar/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pyparsing/package.py b/var/spack/packages/py-pyparsing/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pyqt/package.py b/var/spack/packages/py-pyqt/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py old mode 100644 new mode 100755 index bb5da44d027..8d093efd2c7 --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -8,12 +8,9 @@ class PyPyside(Package): version('1.2.2', 'c45bc400c8a86d6b35f34c29e379e44d') - # TODO: make build dependency - # depends_on("cmake") - extends('python') depends_on('py-setuptools') - depends_on('qt@:4') + depends_on('qt@5.4.0') def patch(self): """Undo PySide RPATH handling and add Spack RPATH.""" diff --git a/var/spack/packages/py-pytest/package.py b/var/spack/packages/py-pytest/package.py new file mode 100644 index 00000000000..3611b68862f --- /dev/null +++ b/var/spack/packages/py-pytest/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPytest(Package): + """pytest: simple powerful testing with Python""" + homepage = "http://pytest.org" + version("2.6.4", "14341e122f7e9031a0948eb6b01a2640", + url="https://pypi.python.org/packages/source/p/pytest/pytest-2.6.4.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-python-daemon/package.py b/var/spack/packages/py-python-daemon/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-python-dateutil/package.py b/var/spack/packages/py-python-dateutil/package.py new file mode 100644 index 00000000000..8efa114db1b --- /dev/null +++ b/var/spack/packages/py-python-dateutil/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyPythonDateutil(Package): + """Extensions to the standard Python datetime module""" + homepage = "http://labix.org/python-dateutil" + version("2.2", "c1f654d0ff7e33999380a8ba9783fd5c", + url="https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pytz/package.py b/var/spack/packages/py-pytz/package.py old mode 100644 new mode 100755 index da6311a784a..f67bb428549 --- a/var/spack/packages/py-pytz/package.py +++ b/var/spack/packages/py-pytz/package.py @@ -5,6 +5,8 @@ class PyPytz(Package): homepage = "https://pypi.python.org/pypi/pytz" url = "https://pypi.python.org/packages/source/p/pytz/pytz-2014.10.tar.gz" + version('2014.9','d42bda2f4c1e873e02fbd1e4acfd1b8c', + url='https://pypi.python.org/packages/source/p/pytz/pytz-2014.9.tar.gz') version('2014.10', 'eb1cb941a20c5b751352c52486aa1dd7') version('2015.4', '417a47b1c432d90333e42084a605d3d8') diff --git a/var/spack/packages/py-pyzmq/package.py b/var/spack/packages/py-pyzmq/package.py new file mode 100755 index 00000000000..f5551182b32 --- /dev/null +++ b/var/spack/packages/py-pyzmq/package.py @@ -0,0 +1,16 @@ +from spack import * + +class PyPyzmq(Package): + """official Python binding for the ZeroMQ Messaging Library""" + homepage = "https://github.com/zeromq/pyzmq" + url = "https://pypi.python.org/packages/source/p/pyzmq/pyzmq-14.7.0.tar.gz" + + version('14.4.1','a6a7fb6950b1bf5333c6795f3edf196c', + url='https://pypi.python.org/packages/source/p/pyzmq/pyzmq-14.4.1.tar.gz') + version('14.7.0', '87e3abb33af5794db5ae85c667bbf324') + + extends('python') + depends_on('py-setuptools@18.1') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-rpy2/package.py b/var/spack/packages/py-rpy2/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-scientificpython/package.py b/var/spack/packages/py-scientificpython/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-scikit-learn/package.py b/var/spack/packages/py-scikit-learn/package.py old mode 100644 new mode 100755 index 5b078ce9011..744b37e7e0e --- a/var/spack/packages/py-scikit-learn/package.py +++ b/var/spack/packages/py-scikit-learn/package.py @@ -9,6 +9,9 @@ class PyScikitLearn(Package): version('0.16.1', '363ddda501e3b6b61726aa40b8dbdb7e') extends('python') + depends_on('python@2.7.10') + depends_on('py-numpy@1.9.2') + depends_on('py-scipy@0.15.1') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scipy/package.py b/var/spack/packages/py-scipy/package.py old mode 100644 new mode 100755 index 3a1124cc15a..bcbdb2100f0 --- a/var/spack/packages/py-scipy/package.py +++ b/var/spack/packages/py-scipy/package.py @@ -5,14 +5,14 @@ class PyScipy(Package): homepage = "https://pypi.python.org/pypi/scipy" url = "https://pypi.python.org/packages/source/s/scipy/scipy-0.15.0.tar.gz" + version('0.14.0','d7c7f4ccf8b07b08d6fe49d5cd51f85d', + url='https://pypi.python.org/packages/source/s/scipy/scipy-0.14.0.tar.gz') version('0.15.0', '639112f077f0aeb6d80718dc5019dc7a') version('0.15.1', 'be56cd8e60591d6332aac792a5880110') extends('python') depends_on('py-nose') depends_on('py-numpy') - depends_on('blas') - depends_on('lapack') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-seaborn/package.py b/var/spack/packages/py-seaborn/package.py new file mode 100644 index 00000000000..1e228b06227 --- /dev/null +++ b/var/spack/packages/py-seaborn/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PySeaborn(Package): + """Seaborn: statistical data visualization""" + homepage = "http://stanford.edu/~mwaskom/software/seaborn/" + version("0.5.1", "2ce6ea7d3c67858c0b1f5793fa2043b8", + url="https://pypi.python.org/packages/source/s/seaborn/seaborn-0.5.1.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py old mode 100644 new mode 100755 index 760ad4d6dbf..053fe1ba80d --- a/var/spack/packages/py-setuptools/package.py +++ b/var/spack/packages/py-setuptools/package.py @@ -3,11 +3,12 @@ class PySetuptools(Package): """Easily download, build, install, upgrade, and uninstall Python packages.""" homepage = "https://pypi.python.org/pypi/setuptools" - url = "https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz" - version('11.3.1', '01f69212e019a2420c1693fb43593930') - version('16.0', '0ace0b96233516fc5f7c857d086aa3ad') - version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') + version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06', + url='https://pypi.python.org/packages/source/s/setuptools/setuptools-18.1.tar.gz') + version('11.3.1', '01f69212e019a2420c1693fb43593930', + url="https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz") + version('16.0', '0ace0b96233516fc5f7c857d086aa3ad',) extends('python') diff --git a/var/spack/packages/py-shiboken/package.py b/var/spack/packages/py-shiboken/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-sip/package.py b/var/spack/packages/py-sip/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-six/package.py b/var/spack/packages/py-six/package.py old mode 100644 new mode 100755 index 05c5bd00a98..e9946a1e566 --- a/var/spack/packages/py-six/package.py +++ b/var/spack/packages/py-six/package.py @@ -5,6 +5,8 @@ class PySix(Package): homepage = "https://pypi.python.org/pypi/six" url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz" + version('1.8.0','1626eb24cc889110c38f7e786ec69885', + url='https://pypi.python.org/packages/source/s/six/six-1.8.0.tar.gz') version('1.9.0', '476881ef4012262dfc8adc645ee786c4') extends('python') diff --git a/var/spack/packages/py-statsmodels/package.py b/var/spack/packages/py-statsmodels/package.py new file mode 100644 index 00000000000..393c772738c --- /dev/null +++ b/var/spack/packages/py-statsmodels/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyStatsmodels(Package): + """Statistical computations and models for use with SciPy""" + homepage = "http://statsmodels.sourceforge.net/" + version("0.6.1", "f7580ebf7d2a2c9b87abfad190dcb9a3", + url="https://pypi.python.org/packages/source/s/statsmodels/statsmodels-0.6.1.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-sympy/package.py b/var/spack/packages/py-sympy/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-tables/package.py b/var/spack/packages/py-tables/package.py new file mode 100644 index 00000000000..637ccf34cd6 --- /dev/null +++ b/var/spack/packages/py-tables/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyTables(Package): + """Hierarchical datasets for Python""" + homepage = "http://www.pytables.org/" + version("3.1.1", "38d917f0c6dfb0bc28ce9ea0c3492524", + url="https://pypi.python.org/packages/source/t/tables/tables-3.1.1.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-tornado/package.py b/var/spack/packages/py-tornado/package.py new file mode 100755 index 00000000000..4fc2d42c7bb --- /dev/null +++ b/var/spack/packages/py-tornado/package.py @@ -0,0 +1,17 @@ +from spack import * + +class PyTornado(Package): + """Python web framework and asynchronous networking library""" + homepage = "http://www.tornadoweb.org/" + url = "https://pypi.python.org/packages/source/t/tornado/tornado-4.2.1.tar.gz" + + version('4.0.2','985c0e704b765c33a6193d49d1935588', + url='https://pypi.python.org/packages/source/t/tornado/tornado-4.0.2.tar.gz') + version('4.2.1', 'd523204389cfb70121bb69709f551b20') + + extends('python') + depends_on('py-certifi') + depends_on('py-setuptools@18.1') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-traitlets/package.py b/var/spack/packages/py-traitlets/package.py new file mode 100755 index 00000000000..7b7f5c64cc4 --- /dev/null +++ b/var/spack/packages/py-traitlets/package.py @@ -0,0 +1,14 @@ +from spack import * + +class PyTraitlets(Package): + """A lightweight Traits like module""" + homepage = "http://traitlets.readthedocs.org/" + url = "https://github.com/ipython/traitlets/archive/4.0.0.tar.gz" + + version('4.0.0', 'b5b95ea5941fd9619b4704dfd8201568') + + extends('python') + depends_on("py-setuptools@18.1") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-triangle-plot/package.py b/var/spack/packages/py-triangle-plot/package.py new file mode 100644 index 00000000000..29c5f92857e --- /dev/null +++ b/var/spack/packages/py-triangle-plot/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyTrianglePlot(Package): + """Make some beautiful corner plots of samples.""" + homepage = "https://github.com/dfm/triangle.py" + version("0.0.6", "970a35a9bde6002d673e3188fe39f0ed", + url="https://pypi.python.org/packages/source/t/triangle_plot/triangle_plot-0.0.6.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-virtualenv/package.py b/var/spack/packages/py-virtualenv/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/py-yapf/package.py b/var/spack/packages/py-yapf/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/qhull/package.py b/var/spack/packages/qhull/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/qthreads/package.py b/var/spack/packages/qthreads/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/ravel/package.py b/var/spack/packages/ravel/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/readline/package.py b/var/spack/packages/readline/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/rose/add_spack_compiler_recognition.patch b/var/spack/packages/rose/add_spack_compiler_recognition.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/rose/package.py b/var/spack/packages/rose/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/ruby/package.py b/var/spack/packages/ruby/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/samtools/package.py b/var/spack/packages/samtools/package.py deleted file mode 100644 index 72900398d86..00000000000 --- a/var/spack/packages/samtools/package.py +++ /dev/null @@ -1,18 +0,0 @@ -from spack import * - -class Samtools(Package): - """SAM Tools provide various utilities for manipulating alignments in the SAM format, - including sorting, merging, indexing and generating - alignments in a per-position format""" - - homepage = "www.htslib.org" - version('1.2','988ec4c3058a6ceda36503eebecd4122',url = "https://github.com/samtools/samtools/releases/download/1.2/samtools-1.2.tar.bz2") - - depends_on("zlib") - depends_on("mpc") - parallel=False - patch("samtools1.2.patch",level=0) - - def install(self, spec, prefix): - make("prefix=%s" % prefix, "install") - diff --git a/var/spack/packages/samtools/samtools1.2.patch b/var/spack/packages/samtools/samtools1.2.patch deleted file mode 100644 index ead3ab4e2cb..00000000000 --- a/var/spack/packages/samtools/samtools1.2.patch +++ /dev/null @@ -1,20 +0,0 @@ ---- Makefile 2015-02-03 08:27:34.000000000 -0800 -+++ Makefile.new 2015-07-21 10:38:27.881406892 -0700 -@@ -26,7 +26,7 @@ - CFLAGS = -g -Wall -O2 - LDFLAGS = - LDLIBS = --DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=1 -+DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=0 - LOBJS= bam_aux.o bam.o bam_import.o sam.o \ - sam_header.o bam_plbuf.o - AOBJS= bam_index.o bam_plcmd.o sam_view.o \ -@@ -37,7 +37,7 @@ - faidx.o stats.o stats_isize.o bam_flags.o bam_split.o \ - bam_tview.o bam_tview_curses.o bam_tview_html.o bam_lpileup.o - INCLUDES= -I. -I$(HTSDIR) --LIBCURSES= -lcurses # -lXCurses -+#LIBCURSES= -lcurses # -lXCurses - - prefix = /usr/local - exec_prefix = $(prefix) diff --git a/var/spack/packages/scalasca/package.py b/var/spack/packages/scalasca/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py old mode 100644 new mode 100755 index f013bd1cbb7..32a772e3dbe --- a/var/spack/packages/scorep/package.py +++ b/var/spack/packages/scorep/package.py @@ -1,10 +1,11 @@ # FIXME: Add copyright statement from spack import * +from contextlib import closing class Scorep(Package): - """The Score-P measurement infrastructure is a highly scalable and - easy-to-use tool suite for profiling, event tracing, and online + """The Score-P measurement infrastructure is a highly scalable and + easy-to-use tool suite for profiling, event tracing, and online analysis of HPC applications.""" # FIXME: add a proper url for your package's homepage here. @@ -19,7 +20,7 @@ class Scorep(Package): depends_on("mpi") depends_on("papi") # depends_on("otf2@1.2:1.2.1") # only Score-P 1.2.x - depends_on("otf2") + depends_on("otf2") depends_on("opari2") depends_on("cube@4.2:4.2.3") @@ -52,12 +53,12 @@ def install(self, spec, prefix): # Use a custom compiler configuration, otherwise the score-p # build system messes with spack's compiler settings. # Create these three files in the build directory - with open("platform-backend-user-provided", "w") as backend_file: + with closing(open("platform-backend-user-provided", "w")) as backend_file: backend_file.write(self.backend_user_provided) - with open("platform-frontend-user-provided", "w") as frontend_file: + with closing(open("platform-frontend-user-provided", "w")) as frontend_file: frontend_file.write(self.frontend_user_provided) - with open("platform-mpi-user-provided", "w") as mpi_file: - mpi_file.write(self.mpi_user_provided) + with closing(open("platform-mpi-user-provided", "w")) as mpi_file: + mpi_file.write(self.mpi_user_provided) configure_args = ["--prefix=%s" % prefix, "--with-custom-compilers", diff --git a/var/spack/packages/scotch/package.py b/var/spack/packages/scotch/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/scr/package.py b/var/spack/packages/scr/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/silo/package.py b/var/spack/packages/silo/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/snappy/package.py b/var/spack/packages/snappy/package.py deleted file mode 100644 index c8f9ceef7da..00000000000 --- a/var/spack/packages/snappy/package.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -from spack import * - -class Snappy(Package): - """A fast compressor/decompressor: https://code.google.com/p/snappy""" - - homepage = "https://code.google.com/p/snappy" - url = "https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz" - - version('1.1.3', '7358c82f133dc77798e4c2062a749b73') - - def install(self, spec, prefix): - configure("--prefix=" + prefix) - make() - make("install") diff --git a/var/spack/packages/spindle/package.py b/var/spack/packages/spindle/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/sqlite/package.py b/var/spack/packages/sqlite/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/stat/configure_mpicxx.patch b/var/spack/packages/stat/configure_mpicxx.patch old mode 100644 new mode 100755 diff --git a/var/spack/packages/stat/package.py b/var/spack/packages/stat/package.py old mode 100644 new mode 100755 index 5d81e627317..c40d65ae783 --- a/var/spack/packages/stat/package.py +++ b/var/spack/packages/stat/package.py @@ -9,20 +9,17 @@ class Stat(Package): version('2.1.0', 'ece26beaf057aa9134d62adcdda1ba91') version('2.0.0', 'c7494210b0ba26b577171b92838e1a9b') - variant('dysect', default=False, description="enable DySectAPI") - depends_on('libelf') depends_on('libdwarf') depends_on('dyninst') depends_on('graphlib') - depends_on('graphviz') depends_on('launchmon') depends_on('mrnet') patch('configure_mpicxx.patch', when='@2.1.0') def install(self, spec, prefix): - configure_args = [ + configure( "--enable-gui", "--prefix=%s" % prefix, "--disable-examples", # Examples require MPI: avoid this dependency. @@ -30,11 +27,7 @@ def install(self, spec, prefix): "--with-mrnet=%s" % spec['mrnet'].prefix, "--with-graphlib=%s" % spec['graphlib'].prefix, "--with-stackwalker=%s" % spec['dyninst'].prefix, - "--with-libdwarf=%s" % spec['libdwarf'].prefix - ] - if '+dysect' in spec: - configure_args.append('--enable-dysectapi') - configure(*configure_args) + "--with-libdwarf=%s" % spec['libdwarf'].prefix) make(parallel=False) make("install") diff --git a/var/spack/packages/sundials/package.py b/var/spack/packages/sundials/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/swig/package.py b/var/spack/packages/swig/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/task/package.py b/var/spack/packages/task/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/taskd/package.py b/var/spack/packages/taskd/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/tcl/package.py b/var/spack/packages/tcl/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/the_silver_searcher/package.py b/var/spack/packages/the_silver_searcher/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/thrift/package.py b/var/spack/packages/thrift/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/tk/package.py b/var/spack/packages/tk/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/tmux/package.py b/var/spack/packages/tmux/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/tmuxinator/package.py b/var/spack/packages/tmuxinator/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/uncrustify/package.py b/var/spack/packages/uncrustify/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/util-linux/package.py b/var/spack/packages/util-linux/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/vim/package.py b/var/spack/packages/vim/package.py old mode 100644 new mode 100755 index 4099b3257f7..d0cf80b898d --- a/var/spack/packages/vim/package.py +++ b/var/spack/packages/vim/package.py @@ -35,26 +35,15 @@ class Vim(Package): variant('ruby', default=False, description="build with Ruby") depends_on('ruby', when='+ruby') - variant('cscope', default=False, description="build with cscope support") - depends_on('cscope', when='+cscope') - - variant('gui', default=False, description="build with gui (gvim)") - # virtual dependency? - def install(self, spec, prefix): feature_set = None for fs in self.feature_sets: if "+" + fs in spec: if feature_set is not None: - tty.error("Only one feature set allowed, both %s and %s specified" - % (feature_set, fs)) + tty.error("Only one feature set allowed, both {} and {} specified".format( + feature_set, + fs)) feature_set = fs - if '+gui' in spec: - if feature_set is not None: - if feature_set is not 'huge': - tty.error("+gui variant requires 'huge' feature set, %s was specified" - % feature_set) - feature_set = 'huge' if feature_set is None: feature_set = 'normal' @@ -71,12 +60,6 @@ def install(self, spec, prefix): else: configure_args.append("--enable-rubyinterp=dynamic") - if '+gui' in spec: - configure_args.append("--enable-gui=auto") - - if '+cscope' in spec: - configure_args.append("--enable-cscope") - configure("--prefix=%s" % prefix, *configure_args) make() diff --git a/var/spack/packages/vtk/package.py b/var/spack/packages/vtk/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/wget/package.py b/var/spack/packages/wget/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/wx/package.py b/var/spack/packages/wx/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/wxpropgrid/package.py b/var/spack/packages/wxpropgrid/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/xcb-proto/package.py b/var/spack/packages/xcb-proto/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/xz/package.py b/var/spack/packages/xz/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/yasm/package.py b/var/spack/packages/yasm/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/zeromq/package.py b/var/spack/packages/zeromq/package.py deleted file mode 100644 index b5a1e3d4cde..00000000000 --- a/var/spack/packages/zeromq/package.py +++ /dev/null @@ -1,20 +0,0 @@ -from spack import * - -class Zeromq(Package): - """ The ZMQ networking/concurrency library and core API """ - homepage = "http://zguide.zeromq.org/" - url = "http://download.zeromq.org/zeromq-4.1.2.tar.gz" - - version('4.1.2', '159c0c56a895472f02668e692d122685') - version('4.1.1', '0a4b44aa085644f25c177f79dc13f253') - version('4.0.7', '9b46f7e7b0704b83638ef0d461fd59ab') - version('4.0.6', 'd47dd09ed7ae6e7fd6f9a816d7f5fdf6') - version('4.0.5', '73c39f5eb01b9d7eaf74a5d899f1d03d') - - depends_on("libsodium") - - def install(self, spec, prefix): - configure("--with-libsodium","--prefix=%s" % prefix) - - make() - make("install") diff --git a/var/spack/packages/zlib/package.py b/var/spack/packages/zlib/package.py old mode 100644 new mode 100755 diff --git a/var/spack/packages/zsh/package.py b/var/spack/packages/zsh/package.py old mode 100644 new mode 100755 From e9f7d033ff04138905c99cb427e2071cc55cdc16 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 19:13:08 -0700 Subject: [PATCH 004/375] make sure submodule is added in the correct location --- .gitmodules | 6 +++--- lib/spack/spack/util/python_recipe_parser | 1 + var/spack/packages/py-astropy/package.py | 1 + var/spack/packages/py-jsonschema/package.py | 2 ++ 4 files changed, 7 insertions(+), 3 deletions(-) create mode 160000 lib/spack/spack/util/python_recipe_parser diff --git a/.gitmodules b/.gitmodules index 1d770f415bd..d5995e8e4b0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "/lib/spack/spack/util/python_recipe_parser"] - path = /lib/spack/spack/util/python_recipe_parser - url = https://github.com/karenyyng/spack_python_package_parser +[submodule "lib/spack/spack/util/python_recipe_parser"] + path = lib/spack/spack/util/python_recipe_parser + url = git@github.com:karenyyng/spack_python_package_parser.git diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser new file mode 160000 index 00000000000..6777dd98e75 --- /dev/null +++ b/lib/spack/spack/util/python_recipe_parser @@ -0,0 +1 @@ +Subproject commit 6777dd98e75229856d94da6671676b2a66e75986 diff --git a/var/spack/packages/py-astropy/package.py b/var/spack/packages/py-astropy/package.py index 5d2d0619ccf..9fda37519e2 100644 --- a/var/spack/packages/py-astropy/package.py +++ b/var/spack/packages/py-astropy/package.py @@ -7,6 +7,7 @@ class PyAstropy(Package): url="https://pypi.python.org/packages/source/a/astropy/astropy-0.4.2.tar.gz") extends("python") + depends_on("py-numpy") def install(self, spec, prefix): python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-jsonschema/package.py b/var/spack/packages/py-jsonschema/package.py index 9a7fc218622..fa233635ce8 100755 --- a/var/spack/packages/py-jsonschema/package.py +++ b/var/spack/packages/py-jsonschema/package.py @@ -11,6 +11,8 @@ class PyJsonschema(Package): extends("python") depends_on("py-setuptools@18.1") + depends_on("py-vcversioner") + depends_on("py-functools32") def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) From f3254ff02d589c9e177aab58276f5c1971866018 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 19:22:53 -0700 Subject: [PATCH 005/375] removed package for python packages with problematic / duplicate names --- var/spack/packages/py-Cython/package.py | 12 ------------ var/spack/packages/py-Distutils2/package.py | 15 --------------- 2 files changed, 27 deletions(-) delete mode 100644 var/spack/packages/py-Cython/package.py delete mode 100644 var/spack/packages/py-Distutils2/package.py diff --git a/var/spack/packages/py-Cython/package.py b/var/spack/packages/py-Cython/package.py deleted file mode 100644 index c2501391143..00000000000 --- a/var/spack/packages/py-Cython/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyCython(Package): - """The Cython compiler for writing C extensions for the Python language.""" - homepage = "http://www.cython.org" - version("0.21.2", "d21adb870c75680dc857cd05d41046a4", - url="https://pypi.python.org/packages/source/C/Cython/Cython-0.21.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-Distutils2/package.py b/var/spack/packages/py-Distutils2/package.py deleted file mode 100644 index 8690f569fee..00000000000 --- a/var/spack/packages/py-Distutils2/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyDistutils2(Package): - """Python Packaging Library.""" - - homepage = "https://hg.python.org/distutils2" - url = "https://pypi.python.org/packages/source/D/Distutils2/Distutils2-1.0a4.tar.gz" - - version('1.0a4', '52bc9dffb394970c27e02853ae3a3241') - - depends_on("python") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) - From c809cc92735c7ff57dab2aca02bc7fa40c901a43 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Sat, 3 Oct 2015 19:24:15 -0700 Subject: [PATCH 006/375] clean up package.py file and remove duplicate file --- var/spack/packages/py-ipython/package.py | 4 --- .../packages/py-ipython/package.py.backup | 27 ------------------- 2 files changed, 31 deletions(-) delete mode 100755 var/spack/packages/py-ipython/package.py.backup diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py index 3b94b71f295..225366552a1 100755 --- a/var/spack/packages/py-ipython/package.py +++ b/var/spack/packages/py-ipython/package.py @@ -24,8 +24,4 @@ class PyIpython(Package): depends_on('py-jinja2') def install(self, spec, prefix): - import os - # os.system("spack activate py-setuptools@18.1") - # os.system("easy_install pip") - # os.system("pip install ipython[notebook]") python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython/package.py.backup b/var/spack/packages/py-ipython/package.py.backup deleted file mode 100755 index 1ed8b57e605..00000000000 --- a/var/spack/packages/py-ipython/package.py.backup +++ /dev/null @@ -1,27 +0,0 @@ -from spack import * - -class PyIpython(Package): - """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" - homepage = "https://pypi.python.org/pypi/ipython" - - version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', - url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") - version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') - version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', - url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') - - extends('python') - depends_on('py-pygments') - depends_on('py-setuptools@18.1') - depends_on('py-traitlets') - depends_on('py-pyzmq') - depends_on('py-pexpect') - depends_on('py-tornado') - depends_on('py-Jinja2') - - def install(self, spec, prefix): - import os - # os.system("spack activate py-setuptools@18.1") - # os.system("easy_install pip") - # os.system("pip install ipython[notebook]") - python('setup.py', 'install', '--prefix=%s' % prefix) From d00314c621bd427b142fecafc11c25006f3b5279 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 5 Oct 2015 01:30:25 -0700 Subject: [PATCH 007/375] Protptype cray compiler detection and support --- lib/spack/env/cc | 17 +++++++ lib/spack/spack/build_environment.py | 9 ++++ lib/spack/spack/compiler.py | 67 ++++++++++++++++++++++++++- lib/spack/spack/compilers/__init__.py | 9 ++++ lib/spack/spack/compilers/clang.py | 2 +- lib/spack/spack/compilers/gcc.py | 7 ++- lib/spack/spack/compilers/intel.py | 3 ++ lib/spack/spack/compilers/pgi.py | 3 ++ lib/spack/spack/compilers/xl.py | 4 +- 9 files changed, 116 insertions(+), 5 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index fa85bb595ee..b6c6e03e428 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -70,6 +70,23 @@ for param in $parameters; do fi done + +# +# Cray module environment-related stuff. +# +if [ ! -z "$SPACK_CRAYPE" ]; then + cur_pe=$(module list 2>&1 | grep PrgEnv | grep -o 'PrgEnv-[^/]*') + if [ ! -z "$cur_pe" ]; then + module swap $cur_pe $SPACK_CRAYPE + else + module load $SPACK_CRAYPE + fi +fi + +if [ ! -z "$SPACK_COMP_MODULE" ]; then + module load $SPACK_COMP_MODULE +fi + # # Figure out the type of compiler, the language, and the mode so that # the compiler script knows what to do. diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a133faa6295..68388958f52 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -57,6 +57,9 @@ SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' +SPACK_CRAYPE = 'SPACK_CRAYPE' +SPACK_COMP_MODULE = 'SPACK_COMP_MODULE' + class MakeExecutable(Executable): """Special callable executable object for make so the user can @@ -105,6 +108,12 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + if compiler.PrgEnv: + os.environ['SPACK_CRAYPE'] = compiler.PrgEnv + os.environ['SPACK_COMP_MODULE'] = compiler.module + + + def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1e800a89795..e7d450ee8bf 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -25,6 +25,7 @@ import os import re import itertools +import subprocess from datetime import datetime import llnl.util.tty as tty @@ -98,7 +99,14 @@ class Compiler(object): cxx11_flag = "-std=c++11" - def __init__(self, cspec, cc, cxx, f77, fc): + # Cray PrgEnv name that can be used to load this compiler + PrgEnv = None + + # Name of module used to switch versions of this compiler + PrgEnv_compiler = None + + + def __init__(self, cspec, cc, cxx, f77, fc, module=None): def check(exe): if exe is None: return None @@ -111,6 +119,8 @@ def check(exe): self.fc = check(fc) self.spec = cspec + self.module = module + @property @@ -255,6 +265,61 @@ def find(cls, *path): return list(compilers.values()) + @classmethod + def find_in_modules(cls): + compilers = [] + + if cls.PrgEnv: + if not cls.PrgEnv_compiler: + tty.die('Must supply PrgEnv_compiler with PrgEnv') + + output = _shell('module avail %s' % cls.PrgEnv_compiler) + matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) + + for name, version in matches: + v = version + '-craype' + comp = cls(spack.spec.CompilerSpec(name + '@' + v), + 'cc', 'CC', 'ftn', 'ftn', name +'/' + v) + + compilers.append(comp) + + return compilers + + +def _cur_prgenv(): + out, err = subprocess.Popen( + ['module list'], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + matches = re.findall(r'(PrgEnv-[^/]*)/', err) + return matches[0] + + +def _module_shell(module, *args): + cmd = 'module swap %s %s;' % (_cur_prgenv(), module) + cmd += 'module load %s;' % compiler + cmd += 'module unload cray-libsci;' + +# + +# 'module load craype-network-gemini;' + +# 'module load %s;' % module + +# 'module swap gcc/4.6.1;' + +# 'module load eswrap; ' + +# 'module load craype-mc12; ' + +# 'module load cray-shmem; ' + +# 'module load cray-mpich; ') + cmd += ' '.join(args) + out, err = subprocess.Popen([cmd + ' '.join(args)], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() + return out + + +def _shell(*args): + return subprocess.Popen([' '.join(args)], shell=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1] + + + + def __repr__(self): """Return a string represntation of the compiler toolchain.""" return self.__str__() diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b7b021a1ac8..8c4cb38926a 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -44,6 +44,7 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +_optional_instance_vars = ['module'] _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] @@ -127,11 +128,18 @@ def add_compilers_to_config(scope, *compilers): compiler_config_tree = {} for compiler in compilers: compiler_entry = {} + for c in _required_instance_vars: val = getattr(compiler, c) if not val: val = "None" compiler_entry[c] = val + + for c in _optional_instance_vars: + val = getattr(compiler, c) + if val: + compiler_entry[c] = val + compiler_config_tree[str(compiler.spec)] = compiler_entry spack.config.add_to_compiler_config(compiler_config_tree, scope) @@ -220,6 +228,7 @@ def class_for_compiler_name(compiler_name): def all_compiler_types(): +# return [class_for_compiler_name(c) for c in ['gcc']] return [class_for_compiler_name(c) for c in supported_compilers()] diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index 790901c86e4..8d32608ff4e 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -39,7 +39,7 @@ class Clang(Compiler): @classmethod - def default_version(self, comp): + def default_version(cls, comp): """The '--version' option works for clang compilers. Output looks like this:: diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index f0d27d590e8..ff0b8889a87 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -42,6 +42,9 @@ class Gcc(Compiler): # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. suffixes = [r'-mp-\d\.\d'] + PrgEnv = 'gnu' + PrgEnv_compiler = 'gcc' + @property def cxx11_flag(self): if self.version < ver('4.3'): @@ -56,9 +59,9 @@ def fc_version(cls, fc): return get_compiler_version( fc, '-dumpversion', # older gfortran versions don't have simple dumpversion output. - r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)') + r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)', module) @classmethod def f77_version(cls, f77): - return cls.fc_version(f77) + return cls.fc_version(f77, module) diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 2a72c4eaea0..7c485fe69d5 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -37,6 +37,9 @@ class Intel(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['ifort'] + PrgEnv = 'intel' + PrgEnv_compiler = 'intel' + @property def cxx11_flag(self): if self.version < ver('11.1'): diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index d97f24c12e8..8f1ed288258 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -37,6 +37,9 @@ class Pgi(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['pgf95', 'pgf90'] + PrgEnv = 'pgi' + PrgEnv_compiler = 'pgi' + @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. diff --git a/lib/spack/spack/compilers/xl.py b/lib/spack/spack/compilers/xl.py index 562186b865f..179b7209182 100644 --- a/lib/spack/spack/compilers/xl.py +++ b/lib/spack/spack/compilers/xl.py @@ -45,8 +45,9 @@ def cxx11_flag(self): else: return "-qlanglvl=extended0x" + @classmethod - def default_version(self, comp): + def default_version(cls, comp): """The '-qversion' is the standard option fo XL compilers. Output looks like this:: @@ -72,6 +73,7 @@ def default_version(self, comp): return get_compiler_version( comp, '-qversion',r'([0-9]?[0-9]\.[0-9])') + @classmethod def fc_version(cls, fc): """The fortran and C/C++ versions of the XL compiler are always two units apart. From f4e72f33c8ba988507aba667e318d1861c7b2b20 Mon Sep 17 00:00:00 2001 From: karenyyng Date: Mon, 5 Oct 2015 08:24:33 -0700 Subject: [PATCH 008/375] added dependencies for py-h5py --- lib/spack/spack/util/python_recipe_parser | 2 +- var/spack/packages/py-h5py/package.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser index 6777dd98e75..437a62abb3d 160000 --- a/lib/spack/spack/util/python_recipe_parser +++ b/lib/spack/spack/util/python_recipe_parser @@ -1 +1 @@ -Subproject commit 6777dd98e75229856d94da6671676b2a66e75986 +Subproject commit 437a62abb3df7212e3ee20269c0089a0a9766fe0 diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/packages/py-h5py/package.py index 6293da54073..04072ca76b3 100755 --- a/var/spack/packages/py-h5py/package.py +++ b/var/spack/packages/py-h5py/package.py @@ -13,6 +13,8 @@ class PyH5py(Package): depends_on('hdf5') depends_on('py-numpy') depends_on('py-cython') + depends_on('py-six') + depends_on('py-pkgconfig') def install(self, spec, prefix): python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) From b6d2a12ceb090692aa2be363b46d7f25486c0245 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 7 Oct 2015 15:57:29 -0700 Subject: [PATCH 009/375] Started changing the find in modules method written by Todd --- lib/spack/spack/compiler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e7d450ee8bf..cbe6d8cd7da 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -272,9 +272,11 @@ def find_in_modules(cls): if cls.PrgEnv: if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') - - output = _shell('module avail %s' % cls.PrgEnv_compiler) - matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) + + loaded_modules = os.environ["LOADEDMODULES"].split(":") + #output = _shell('module avail %s' % cls.PrgEnv_compiler) + for module in loaded_modules: + match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) for name, version in matches: v = version + '-craype' From 4f21344e87da327b4166fff1fd2ce32afaa07dbc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 14 Oct 2015 19:41:07 -0700 Subject: [PATCH 010/375] Started created the Architecture class for Spack to use --- lib/spack/spack/architecture.py | 71 ++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 0c4b605e910..829aaa4c1cb 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -30,7 +30,7 @@ import spack import spack.error as serr from spack.version import Version - +from external import yaml class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): @@ -43,41 +43,81 @@ def __init__(self): super(NoSysTypeError, self).__init__( "Could not determine sys_type for this machine.") +class Architecture(object): + def __init__(self, *arch_name): + + """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for + searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. + """ + self.arch_dict = {} + self.arch_name = arch_name + + def add_arch_strategy(self): + """ Create a dictionary using the tuples of arch_names""" + for n in self.arch_name: + if 'cray' in n.lower(): + self.arch_dict[n] = "MODULES" + if 'linux' in n.lower() or 'x86_64' in n.lower(): + self.arch_dict[n] = "PATH" + else: + self.arch_dict[n] = None def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" if not hasattr(spack, "sys_type"): - return None + return None elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() + return Architecture(spack.sys_type()) else: - return spack.sys_type - - -def get_sys_type_from_environment(): - """Return $SYS_TYPE or None if it's not defined.""" - return os.environ.get('SYS_TYPE') + return Architecture(spack.sys_type) +# This is livermore dependent. Hard coded for livermore +#def get_sys_type_from_environment(): +# """Return $SYS_TYPE or None if it's not defined.""" +# return os.environ.get('SYS_TYPE') def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac.""" + """Return a Mac OS SYS_TYPE or None if this isn't a mac. + Front-end config + """ + mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return "macosx_%s_%s" % ( - Version(mac_ver).up_to(2), py_platform.machine()) + return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) +def get_sys_type_from_uname(): + """ Returns a sys_type from the uname argument + Front-end config + """ + return Architecture(os.uname()[0] + " " + os.uname()[-1]) + +def get_sys_type_from_config_file(): + """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since + The user can specify that the architecture is a cray-xc40 + """ + + home_dir = os.environ["HOME"] + yaml_file = os.path.join(home_dir, ".spack/architecture.yaml") + if os.path.isfile(yaml_file): + with open(yaml_file) as config: + config_dict = config['architecture'] + front_end = config_dict['front'] + back_end = config_dict['back'] + return Architecture(front_end) @memoized def sys_type(): - """Returns a SysType for the current machine.""" + """Returns a SysType for the current machine. Should return output to an + Architecture class + """ methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, get_mac_sys_type] # search for a method that doesn't return None - sys_type = None + sys_type = (None,None) for method in methods: sys_type = method() if sys_type: break @@ -90,3 +130,4 @@ def sys_type(): raise InvalidSysTypeError(sys_type) return sys_type + From 29e03ac851f807e9fb5aefb687902e703d208615 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:46:44 -0700 Subject: [PATCH 011/375] Added __eq__ testing method. Created tests for it in test/ folder --- lib/spack/spack/architecture.py | 56 +++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 829aaa4c1cb..97c2cdbd6d9 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -57,10 +57,20 @@ def add_arch_strategy(self): for n in self.arch_name: if 'cray' in n.lower(): self.arch_dict[n] = "MODULES" - if 'linux' in n.lower() or 'x86_64' in n.lower(): + elif 'linux' in n.lower() or 'x86_64' in n.lower(): self.arch_dict[n] = "PATH" else: self.arch_dict[n] = None + + def get_arch_dict(self): + """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ + return self.arch_dict + + def __eq__(self, other): + if self.arch_dict != {} and other.arch_dict != {}: + return self.arch_dict == other.arch_dict + else: + return self.arch_name == self.arch_name def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" @@ -95,29 +105,43 @@ def get_sys_type_from_uname(): def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since - The user can specify that the architecture is a cray-xc40 + The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack + is installed. Similar to .spackconfig """ - home_dir = os.environ["HOME"] - yaml_file = os.path.join(home_dir, ".spack/architecture.yaml") - if os.path.isfile(yaml_file): - with open(yaml_file) as config: - config_dict = config['architecture'] - front_end = config_dict['front'] - back_end = config_dict['back'] - return Architecture(front_end) + spack_home_dir = os.environ["HOME"] + "/.spack" + yaml_file = os.path.join(spack_home_dir, "architecture.yaml") + + try: + config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() + arch = config_dict['architecture'] + front = arch['front'] + back = arch['back'] + + except: + print "No architecture.yaml config file found" + + return Architecture(front,back) @memoized def sys_type(): - """Returns a SysType for the current machine. Should return output to an - Architecture class + """Priority of gathering sys-type. + 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 + 2. UNAME + 3. GLOBALS + 4. MAC OSX + Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. + If there is no yaml present then it should move on to the next function and stop immediately once it gets a + arch name + """ - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, - get_mac_sys_type] + methods = [get_sys_type_from_config_file, + get_sys_type_from_uname, + get_sys_type_from_spack_globals, + get_mac_sys_type] # search for a method that doesn't return None - sys_type = (None,None) + sys_type = None for method in methods: sys_type = method() if sys_type: break From d328f4c3b631e59ed14f52cf38f3d05774100c15 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:47:26 -0700 Subject: [PATCH 012/375] Test suite for architecture class and functions --- lib/spack/spack/test/architecture.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lib/spack/spack/test/architecture.py diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py new file mode 100644 index 00000000000..6ff22aaa59a --- /dev/null +++ b/lib/spack/spack/test/architecture.py @@ -0,0 +1,19 @@ +""" Test checks if the architecture class is created correctly and also that + the functions are looking for the correct architecture name +""" +import unittest +import spack +from spack.architecture import * + +class ArchitectureTest(unittest.TestCase): + + def test_Architecture_class(self): + a = Architecture('Cray-XC40') + a.add_arch_strategy() + self.assertEquals(a.get_arch_dict(), {'Cray-XC40': 'MODULES'}) + + def test_get_sys_type_from_config_file(self): + output_arch_class = get_sys_type_from_config_file() + my_arch_class = Architecture('Linux x86_64','Cray-xc40') + + self.assertEqual(output_arch_class, my_arch_class) From ccdf1057592b48e13ba3b98df9972324e91d0be8 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 12:48:12 -0700 Subject: [PATCH 013/375] Commented out a long list of tests to just include my arch test --- lib/spack/spack/test/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 6b3715be6f9..ed51fac33a7 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,7 +31,8 @@ import spack """Names of tests to be included in Spack's test suite""" -test_names = ['versions', +"""test_names = ['architecture', + 'versions', 'url_parse', 'url_substitution', 'packages', @@ -57,7 +58,8 @@ 'optional_deps', 'make_executable', 'configure_guess'] - +""" +test_names = ['architecture'] def list_tests(): """Return names of all tests that can be run for Spack.""" From fec197ccac94af485745b6b40d8150b4ae030e99 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 15 Oct 2015 15:25:13 -0700 Subject: [PATCH 014/375] Fixed the output of sys_type(), might need to add back the error handling part --- lib/spack/spack/architecture.py | 47 ++++++++++++++++----------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 97c2cdbd6d9..8f74fbc2e1f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -60,7 +60,7 @@ def add_arch_strategy(self): elif 'linux' in n.lower() or 'x86_64' in n.lower(): self.arch_dict[n] = "PATH" else: - self.arch_dict[n] = None + self.arch_dict[n] = "" def get_arch_dict(self): """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ @@ -72,6 +72,7 @@ def __eq__(self, other): else: return self.arch_name == self.arch_name + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" if not hasattr(spack, "sys_type"): @@ -86,6 +87,7 @@ def get_sys_type_from_spack_globals(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') + def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config @@ -97,12 +99,14 @@ def get_mac_sys_type(): return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) + def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ return Architecture(os.uname()[0] + " " + os.uname()[-1]) + def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack @@ -110,21 +114,22 @@ def get_sys_type_from_config_file(): """ spack_home_dir = os.environ["HOME"] + "/.spack" - yaml_file = os.path.join(spack_home_dir, "architecture.yaml") + yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() arch = config_dict['architecture'] front = arch['front'] back = arch['back'] + return Architecture(front,back) except: print "No architecture.yaml config file found" - - return Architecture(front,back) + return None + @memoized -def sys_type(): +def sys_type(): # This function is going to give me issues isn't it?? """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME @@ -133,25 +138,19 @@ def sys_type(): Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name - """ - methods = [get_sys_type_from_config_file, - get_sys_type_from_uname, - get_sys_type_from_spack_globals, - get_mac_sys_type] - - # search for a method that doesn't return None - sys_type = None - for method in methods: - sys_type = method() - if sys_type: break - - # Couldn't determine the sys_type for this machine. - if sys_type is None: - return "unknown_arch" - - if not isinstance(sys_type, basestring): - raise InvalidSysTypeError(sys_type) - + # Try to create an architecture object using the config file FIRST + functions = [get_sys_type_from_config_file, + get_sys_type_from_uname, + get_sys_type_from_spack_globals, + get_mac_sys_type] + + # TODO: Test for mac OSX system type but I'm sure it will be okay + for func in functions: + sys_type = None + sys_type = func() + if sys_type: + break + return sys_type From 3ba2842b53752e9a3ec8766f3a1350c6cc7577a2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 13:33:21 -0700 Subject: [PATCH 015/375] Got Architecture class working the way i wanted to. Next to write tests --- lib/spack/spack/architecture.py | 68 +++++++++++++++++---------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8f74fbc2e1f..1a9f9de2cb3 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -34,47 +34,50 @@ class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") + class Architecture(object): - def __init__(self, *arch_name): + """ Architecture class that contains a dictionary of architecture name and compiler search strategy methods. + The idea is to create an object that Spack can interact with and know how to search for the compiler + If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. + """ + + def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for - searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. + """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) + and a strategy for searching for that architecture's compiler. + The target passed to it should be a dictionary of names and strategies. """ - self.arch_dict = {} - self.arch_name = arch_name + names = [] + names.append(front) + names.append(back) - def add_arch_strategy(self): - """ Create a dictionary using the tuples of arch_names""" - for n in self.arch_name: - if 'cray' in n.lower(): - self.arch_dict[n] = "MODULES" - elif 'linux' in n.lower() or 'x86_64' in n.lower(): - self.arch_dict[n] = "PATH" - else: - self.arch_dict[n] = "" - - def get_arch_dict(self): - """ Grab the dictionary from the Architecture class, rather than access the internal Architecture attributes """ - return self.arch_dict - - def __eq__(self, other): - if self.arch_dict != {} and other.arch_dict != {}: - return self.arch_dict == other.arch_dict - else: - return self.arch_name == self.arch_name + def add_compiler_strategy(names): + """ Create a dictionary of {'arch-name': 'strategy'} + This will tell Spack whether to look in the $PATH + or $MODULES location for compilers + """ + d = {} + for n in names: + if n: + if 'cray' in n.lower(): + d[n] = "MODULES" + elif 'linux' in n.lower(): + d[n] = "PATH" + else: + d[n] = 'No Strategy' + return d + + self.arch_dict = add_compiler_strategy(names) - -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set. Front-end""" +def get_sys_type_from_spack_globals(): #TODO: Figure out how this function works + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): @@ -104,7 +107,7 @@ def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ - return Architecture(os.uname()[0] + " " + os.uname()[-1]) + return Architecture(os.uname()[0]) def get_sys_type_from_config_file(): @@ -149,6 +152,7 @@ def sys_type(): # This function is going to give me issues isn't it?? for func in functions: sys_type = None sys_type = func() + if sys_type: break From ca3cc5b23e94b8168ba768ad343822b39aac4a25 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 13:37:06 -0700 Subject: [PATCH 016/375] Refactored architecture class. Now it will automagically create a dict upon instantiation. --- lib/spack/spack/architecture.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1a9f9de2cb3..760c9cddd17 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,7 +49,6 @@ class Architecture(object): """ def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) and a strategy for searching for that architecture's compiler. The target passed to it should be a dictionary of names and strategies. @@ -63,6 +62,7 @@ def add_compiler_strategy(names): This will tell Spack whether to look in the $PATH or $MODULES location for compilers """ + #TODO: Look for other strategies d = {} for n in names: if n: @@ -76,21 +76,20 @@ def add_compiler_strategy(names): self.arch_dict = add_compiler_strategy(names) -def get_sys_type_from_spack_globals(): #TODO: Figure out how this function works +def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): - return Architecture(spack.sys_type()) + return Architecture(spack.sys_type()) #If in __init__.py there is a sys_type() then call that else: - return Architecture(spack.sys_type) + return Architecture(spack.sys_type) # Else use the attributed which defaults to None # This is livermore dependent. Hard coded for livermore #def get_sys_type_from_environment(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') - def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config From c31da9bc8f10c3a74b3bf26c2dbac9a13c073eba Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 20 Oct 2015 14:10:54 -0700 Subject: [PATCH 017/375] Made sure architecture works with yaml file --- lib/spack/spack/architecture.py | 48 +++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 760c9cddd17..8460b0de1ee 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,18 +49,24 @@ class Architecture(object): """ def __init__(self, front=None, back=None): - """ Constructor for the architecture class. Should return a dictionary of name (grabbed from uname) - and a strategy for searching for that architecture's compiler. - The target passed to it should be a dictionary of names and strategies. - """ - names = [] - names.append(front) - names.append(back) + """ Constructor for the architecture class. It will create a list from the given arguments and iterate + through that list. It will then create a dictionary of {arch_name : strategy} + Takes in two parameters: - def add_compiler_strategy(names): + front = None defaults to None. Should be the front-end architecture of the machine + back = None defaults to None. Should be the back-end architecture of the machine + + If no arguments are given it will return an empty dictionary + """ + _names = [] + _names.append(front) + _names.append(back) + + def _add_compiler_strategy(names): """ Create a dictionary of {'arch-name': 'strategy'} This will tell Spack whether to look in the $PATH or $MODULES location for compilers + Else it will return No Strategy """ #TODO: Look for other strategies d = {} @@ -74,7 +80,7 @@ def add_compiler_strategy(names): d[n] = 'No Strategy' return d - self.arch_dict = add_compiler_strategy(names) + self.arch_dict = _add_compiler_strategy(_names) def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" @@ -85,20 +91,20 @@ def get_sys_type_from_spack_globals(): else: return Architecture(spack.sys_type) # Else use the attributed which defaults to None + # This is livermore dependent. Hard coded for livermore #def get_sys_type_from_environment(): # """Return $SYS_TYPE or None if it's not defined.""" # return os.environ.get('SYS_TYPE') + def get_mac_sys_type(): """Return a Mac OS SYS_TYPE or None if this isn't a mac. Front-end config """ - mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) @@ -113,11 +119,9 @@ def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack is installed. Similar to .spackconfig - """ - + """ spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') - try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() arch = config_dict['architecture'] @@ -131,7 +135,7 @@ def get_sys_type_from_config_file(): @memoized -def sys_type(): # This function is going to give me issues isn't it?? +def sys_type(): """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME @@ -146,14 +150,18 @@ def sys_type(): # This function is going to give me issues isn't it?? get_sys_type_from_uname, get_sys_type_from_spack_globals, get_mac_sys_type] - - # TODO: Test for mac OSX system type but I'm sure it will be okay + + sys_type = None for func in functions: - sys_type = None sys_type = func() - if sys_type: - break + break + if sys_type is None: + return Architecture("unknown_arch") + + if not isinstance(sys_type, Architecture): + raise InvalidSysTypeError(sys_type) + return sys_type From 51b69ef00b947f4cf1801358b1a5da413d0b21bc Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 21 Oct 2015 09:27:57 -0700 Subject: [PATCH 018/375] initial fetch on crayport --- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/directives.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e7d450ee8bf..4ce6e3f11d7 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -119,7 +119,7 @@ def check(exe): self.fc = check(fc) self.spec = cspec - self.module = module + self.modules = modules.split() diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 9297d6dac36..4bdc8772b81 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -252,7 +252,7 @@ def variant(pkg, name, default=False, description=""): """Define a variant for the package. Packager can specify a default value (on or off) as well as a text description.""" - default = bool(default) + default = default description = str(description).strip() if not re.match(spack.spec.identifier_re, name): From 9b387e7682c22b74661e3363442adcfb569d8680 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 21 Oct 2015 11:09:05 -0700 Subject: [PATCH 019/375] Added strategy method in init --- lib/spack/spack/architecture.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8460b0de1ee..3968b82124c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -57,18 +57,13 @@ def __init__(self, front=None, back=None): back = None defaults to None. Should be the back-end architecture of the machine If no arguments are given it will return an empty dictionary + Uses the _add_compiler_strategy(front, back) to create the dictionary """ - _names = [] - _names.append(front) - _names.append(back) - - def _add_compiler_strategy(names): - """ Create a dictionary of {'arch-name': 'strategy'} - This will tell Spack whether to look in the $PATH - or $MODULES location for compilers - Else it will return No Strategy - """ - #TODO: Look for other strategies + + def _add_compiler_strategy(front,back): + names = [] + names.append(front) + names.append(back) d = {} for n in names: if n: @@ -80,7 +75,8 @@ def _add_compiler_strategy(names): d[n] = 'No Strategy' return d - self.arch_dict = _add_compiler_strategy(_names) + self.arch_dict = _add_compiler_strategy(front, back) + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" From a89abb435f288c2864d9a21089b20f6eaac5d98b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 21 Oct 2015 11:32:59 -0700 Subject: [PATCH 020/375] Changed structure of class, add compiler strategy is a method and can create a dict --- lib/spack/spack/architecture.py | 35 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3968b82124c..7c1bdfb20f9 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -48,6 +48,21 @@ class Architecture(object): If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. """ + def add_compiler_strategy(self, front,back): + names = [] + names.append(front) + names.append(back) + d = {} + for n in names: + if n: + if 'cray' in n.lower(): + d[n] = "MODULES" + elif 'linux' in n.lower(): + d[n] = "PATH" + else: + d[n] = 'No Strategy' + return d + def __init__(self, front=None, back=None): """ Constructor for the architecture class. It will create a list from the given arguments and iterate through that list. It will then create a dictionary of {arch_name : strategy} @@ -59,23 +74,9 @@ def __init__(self, front=None, back=None): If no arguments are given it will return an empty dictionary Uses the _add_compiler_strategy(front, back) to create the dictionary """ - - def _add_compiler_strategy(front,back): - names = [] - names.append(front) - names.append(back) - d = {} - for n in names: - if n: - if 'cray' in n.lower(): - d[n] = "MODULES" - elif 'linux' in n.lower(): - d[n] = "PATH" - else: - d[n] = 'No Strategy' - return d - - self.arch_dict = _add_compiler_strategy(front, back) + self.front = front + self.back = back + self.arch_dict = self.add_compiler_strategy(front, back) def get_sys_type_from_spack_globals(): From 38508c5a3f294f730b31e209dfa90545e1920588 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 22 Oct 2015 12:02:26 -0700 Subject: [PATCH 021/375] Created a control flow logic that will loop through the strategies and find compilers using that. TODO: Need to find a way to locate their executables --- lib/spack/spack/cmd/compiler.py | 47 ++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index 2a64dc914ed..cbe3aba864f 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -29,6 +29,8 @@ from llnl.util.tty.colify import colify from llnl.util.lang import index_by +import spack.architecture +import spack.compiler import spack.compilers import spack.spec import spack.config @@ -36,13 +38,12 @@ from spack.spec import CompilerSpec description = "Manage compilers" +ARCHITECTURE = spack.architecture.sys_type() def setup_parser(subparser): - sp = subparser.add_subparsers( - metavar='SUBCOMMAND', dest='compiler_command') + sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command') - update_parser = sp.add_parser( - 'add', help='Add compilers to the Spack configuration.') + update_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.') update_parser.add_argument('add_paths', nargs=argparse.REMAINDER) remove_parser = sp.add_parser('remove', help='remove compiler') @@ -55,23 +56,33 @@ def setup_parser(subparser): def compiler_add(args): - """Search either $PATH or a list of paths for compilers and add them + """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" - paths = args.add_paths - if not paths: - paths = get_path('PATH') - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers()] + strategies = ARCHITECTURE.strategy() + + for strategy in strategies: + if strategy == 'PATH': + paths = args.add_paths # This might be a parser method. Parsing method to add_paths + if not paths: + paths = get_path('PATH') + + compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + if c.spec not in spack.compilers.all_compilers()] + + elif strategy == "MODULES": + from spack.compilers.cray import Cray + compilers = Cray.find_in_modules() + #TODO: Find a way to locate the executables - if compilers: - spack.compilers.add_compilers_to_config('user', *compilers) - n = len(compilers) - tty.msg("Added %d new compiler%s to %s" % ( - n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) - colify(reversed(sorted(c.spec for c in compilers)), indent=4) - else: - tty.msg("Found no new compilers") + if compilers: + spack.compilers.add_compilers_to_config('user', *compilers) + n = len(compilers) + tty.msg("Added %d new compiler%s to %s" % ( + n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) + colify(reversed(sorted(c.spec for c in compilers)), indent=4) + else: + tty.msg("Found no new compilers") def compiler_remove(args): From 09597fe8dccbd7d49acf1b3198f24ab928874cdb Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 09:44:28 -0700 Subject: [PATCH 022/375] updated the executible to return stderr when specified. Added load_module to build_environment.py, loads target --- lib/spack/spack/build_environment.py | 36 ++++++++++++++++++++++------ lib/spack/spack/util/executable.py | 21 ++++++++++++---- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 68388958f52..191e8587357 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -86,6 +86,28 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) +def load_module(mod): + """Takes a module name and removes modules until it is possible to + load that module. It then loads the provided module. Depends on the + modulecmd implementation of modules used in cray and lmod. + """ + #Create an executable of the module command that will output python code + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module and remove any conflicting modules + # We do this without checking that they are already installed + # for ease of programming because unloading a module that is not + # loaded does nothing. + text = modulecmd('show', mod, return_oe=True).split() + for i, word in enumerate(text): + if word == 'conflict': + exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) + # Load the module now that there are no conflicts + load = modulecmd('load', mod, return_oe=True) + exec(compile(load, '', 'exec')) + + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -108,11 +130,9 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.PrgEnv: - os.environ['SPACK_CRAYPE'] = compiler.PrgEnv - os.environ['SPACK_COMP_MODULE'] = compiler.module - - + if compiler.modules: + for mod in compiler.modules: + load_module(mod) def set_build_environment_variables(pkg): @@ -163,8 +183,10 @@ def set_build_environment_variables(pkg): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - path_set("PKG_CONFIG_PATH", pkg_config_dirs) + path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) + if pkg.spec.architecture.compiler_strategy.lower() == 'module': + load_module(pkg.spec.architecture.module_name) def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. @@ -239,8 +261,8 @@ def get_rpaths(pkg): def setup_package(pkg): """Execute all environment setup routines.""" - set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) + set_compiler_environment_variables(pkg) set_module_variables_for_package(pkg) # Allow dependencies to set up environment as well. diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb6..15354089ac8 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -56,7 +56,11 @@ def command(self): def __call__(self, *args, **kwargs): """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) + # Return oe returns a combined stream, setting both output and error + # without setting return oe returns them concatenated by a double line break + return_oe = kwargs.get("return_oe", False) + return_output = True if return_oe else kwargs.get("return_output", False) + return_error = True if return_oe else kwargs.get("return_error", False) fail_on_error = kwargs.get("fail_on_error", True) ignore_errors = kwargs.get("ignore_errors", ()) @@ -95,8 +99,8 @@ def streamify(arg, mode): proc = subprocess.Popen( cmd, stdin=input, - stderr=error, - stdout=subprocess.PIPE if return_output else output) + stdout=subprocess.PIPE if return_output else output, + stderr=subprocess.STDOUT if return_oe else (subprocess.PIPE if return_error else error)) out, err = proc.communicate() self.returncode = proc.returncode @@ -104,8 +108,15 @@ def streamify(arg, mode): if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) - if return_output: - return out + # Return out or error if specified. Return combined stream if requested, + # otherwise return them concatenated by double line break if both requested. + if return_output or return_error: + if return_oe or not return_error: + return out + elif return_output: + return out+'\n\n'+err + else: + return err except OSError, e: raise ProcessError( From 382d8478bf325099582b4814e0d71ff8522696aa Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 10:23:33 -0700 Subject: [PATCH 023/375] prototype of new architecture concretization saved as new_concretize_architecture in concretize.py --- lib/spack/spack/concretize.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 66002492cb8..c5041d67be3 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -122,6 +122,34 @@ def concretize_architecture(self, spec): return True # changed + def new_concretize_architecture(self, spec): + """If the spec already has an architecture and it is a an architecture type, + return. Otherwise, if it has an architecture that is a string type, generate an + architecture based on that type. If it has no architecture and the root of the + DAG has an architecture, then use that. Otherwise, take the system's default + architecture. + """ + if spec.architecture is not None: + if isinstance(spec.architecture,spack.architecture.Target): + return False + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target(spec.architecture) + return True #changed + + if spec.root.architecture: + if isinstance(spec.root.architecture,spack.architecture.Target): + spec.architecture = spec.root.architecture + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target(spec.root.architecture) + else: + arch = spack.architecture.sys_type() + spec.architecture = arch.target('default') + + return True #changed + + def concretize_variants(self, spec): """If the spec already has variants filled in, return. Otherwise, add the default variants from the package specification. From 7ab921ff02a7d0a05287e4a49738ec9c51361ef0 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 14:46:26 -0700 Subject: [PATCH 024/375] Changed architecture class and added class Target --- lib/spack/spack/architecture.py | 141 +++++++++++++++++++------------- 1 file changed, 85 insertions(+), 56 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 7c1bdfb20f9..ea1f98e06e2 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -23,11 +23,16 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import os +import imp import platform as py_platform +import inspect -from llnl.util.lang import memoized +from llnl.util.lang import memoized, list_modules +from llnl.util.filesystem import join_path +import llnl.util.tty as tty import spack +from spack.util.naming import mod_to_class import spack.error as serr from spack.version import Version from external import yaml @@ -42,41 +47,55 @@ def __init__(self): super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") -class Architecture(object): - """ Architecture class that contains a dictionary of architecture name and compiler search strategy methods. - The idea is to create an object that Spack can interact with and know how to search for the compiler - If it is on a Cray architecture it should look in modules. If it is anything else search $PATH. - """ - - def add_compiler_strategy(self, front,back): - names = [] - names.append(front) - names.append(back) - d = {} - for n in names: - if n: - if 'cray' in n.lower(): - d[n] = "MODULES" - elif 'linux' in n.lower(): - d[n] = "PATH" - else: - d[n] = 'No Strategy' - return d - - def __init__(self, front=None, back=None): - """ Constructor for the architecture class. It will create a list from the given arguments and iterate - through that list. It will then create a dictionary of {arch_name : strategy} - Takes in two parameters: +class Target(object): + """ This is the processor type e.g. cray-ivybridge """ + # Front end or back end target. Target needs to know which one this is + # Should autodetect from the machine + # features of a target + # - a module name + # -a compiler finding strategy + # -a name + # architecture classes handling the aliasing for front-end, back-end and default - front = None defaults to None. Should be the front-end architecture of the machine - back = None defaults to None. Should be the back-end architecture of the machine + def __init__(self,name, module_name=None): + self.name = name # case of cray "ivybridge but if it's x86_64 + self.module_name = module_name # craype-ivybridge + + def compiler_strategy(self): + if self.module_name: # If there is a module_name given then use MODULES + return "MODULES" + else: + return "PATH" + +class Architecture(object): + """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it + is returned + """ + + priority = None # Subclass needs to set this number. This controls order in which arch is detected. + front = None + back = None + default_front = None # The default front end target. On cray sandybridge + default_back = None # The default back end target. On cray ivybridge + + def __init__(self, name): + self.targets = {} + self.name = name + + def add_target(self, name, target): + self.targets[name] = target - If no arguments are given it will return an empty dictionary - Uses the _add_compiler_strategy(front, back) to create the dictionary + + @classmethod + def detect(self): + """ Subclass is responsible for implementing this method. + Returns True if the architecture detects if it is the current architecture + and False if it's not. """ - self.front = front - self.back = back - self.arch_dict = self.add_compiler_strategy(front, back) + raise NotImplementedError() + + def __str__(self): + return self.name def get_sys_type_from_spack_globals(): @@ -84,9 +103,9 @@ def get_sys_type_from_spack_globals(): if not hasattr(spack, "sys_type"): return None elif hasattr(spack.sys_type, "__call__"): - return Architecture(spack.sys_type()) #If in __init__.py there is a sys_type() then call that + return spack.sys_type() #If in __init__.py there is a sys_type() then call that else: - return Architecture(spack.sys_type) # Else use the attributed which defaults to None + return spack.sys_type # Else use the attributed which defaults to None # This is livermore dependent. Hard coded for livermore @@ -102,15 +121,19 @@ def get_mac_sys_type(): mac_ver = py_platform.mac_ver()[0] if not mac_ver: return None - return Architecture("macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())) + return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) def get_sys_type_from_uname(): """ Returns a sys_type from the uname argument Front-end config """ - return Architecture(os.uname()[0]) - + try: + arch_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch, _ = arch_proc.communicate() + return arch.strip() + except: + return None def get_sys_type_from_config_file(): """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since @@ -131,34 +154,40 @@ def get_sys_type_from_config_file(): return None +@memoized +def all_architectures(): + modules = [] + for name in list_modules(spack.arch_path): + mod_name = 'spack.architectures.' + name + path = join_path(spack.arch_path, name) + ".py" + mod = imp.load_source(mod_name, path) + class_name = mod_to_class(name) + if not hasattr(mod, class_name): + tty.die('No class %s defined in %s' % (class_name, mod_name)) + cls = getattr(mod, class_name) + if not inspect.isclass(cls): + tty.die('%s.%s is not a class' % (mod_name, class_name)) + + modules.append(cls) + + return modules + @memoized def sys_type(): """Priority of gathering sys-type. 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME 3. GLOBALS - 4. MAC OSX + 4. MAC OSX Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name """ # Try to create an architecture object using the config file FIRST - functions = [get_sys_type_from_config_file, - get_sys_type_from_uname, - get_sys_type_from_spack_globals, - get_mac_sys_type] - - sys_type = None - for func in functions: - sys_type = func() - if sys_type: - break + architecture_list = all_architectures() + architecture_list.sort(key = lambda a: a.priority) - if sys_type is None: - return Architecture("unknown_arch") - - if not isinstance(sys_type, Architecture): - raise InvalidSysTypeError(sys_type) - - return sys_type + for arch in architecture_list: + if arch.detect(): + return arch() From d177184777591d21cc6a59771ed6f817960394f5 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 15:12:29 -0700 Subject: [PATCH 025/375] added target method to architecture.py and minor cleanup --- lib/spack/spack/architecture.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index ea1f98e06e2..f04fda2ee9c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -58,9 +58,10 @@ class Target(object): # architecture classes handling the aliasing for front-end, back-end and default def __init__(self,name, module_name=None): - self.name = name # case of cray "ivybridge but if it's x86_64 + self.name = name # case of cray "ivybridge" but if it's x86_64 self.module_name = module_name # craype-ivybridge + @property def compiler_strategy(self): if self.module_name: # If there is a module_name given then use MODULES return "MODULES" @@ -73,10 +74,9 @@ class Architecture(object): """ priority = None # Subclass needs to set this number. This controls order in which arch is detected. - front = None - back = None - default_front = None # The default front end target. On cray sandybridge - default_back = None # The default back end target. On cray ivybridge + front-end = None + back-end = None + default = None # The default back end target. On cray ivybridge def __init__(self, name): self.targets = {} @@ -85,7 +85,19 @@ def __init__(self, name): def add_target(self, name, target): self.targets[name] = target - + def target(self, name): + """This is a getter method for the target dictionary that handles defaulting based + on the values provided by default, front-end, and back-end. This can be overwritten + by a subclass for which we want to provide further aliasing options. + """ + if name == 'default': + name = default + elif name == 'front_end': + name = front-end + elif name == 'back_end': + name = back-end + return self.targets[name] + @classmethod def detect(self): """ Subclass is responsible for implementing this method. From 6e560703862794632176733a1d063550215dc573 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 30 Oct 2015 15:15:36 -0700 Subject: [PATCH 026/375] improved aliasing in target method in architecture.py --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f04fda2ee9c..3f38540c0eb 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -92,9 +92,9 @@ def target(self, name): """ if name == 'default': name = default - elif name == 'front_end': + elif name == 'front_end' or name == 'fe': name = front-end - elif name == 'back_end': + elif name == 'back_end' or name == 'be': name = back-end return self.targets[name] From db0695e46a6c6fd4d2ac10dba04f7feb18c25a39 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:57:00 -0700 Subject: [PATCH 027/375] architectures folder where the Architecture subclasses reside --- lib/spack/spack/architectures/__init__.py | 0 lib/spack/spack/architectures/cray.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 lib/spack/spack/architectures/__init__.py create mode 100644 lib/spack/spack/architectures/cray.py diff --git a/lib/spack/spack/architectures/__init__.py b/lib/spack/spack/architectures/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py new file mode 100644 index 00000000000..e6d6d849227 --- /dev/null +++ b/lib/spack/spack/architectures/cray.py @@ -0,0 +1,17 @@ +import os + +from spack.architecture import Architecture + +class Cray(Architecture): + priority = 20 + front_end = None + back_end = None + default = None + + def __init__(self): + super(Cray, self).__init__('cray') + + @classmethod + def detect(self): + return os.path.exists('/opt/cray/craype') + From 2d87bb92edb95cf1c9ec7d58f5d8a34e44bd11a4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:58:20 -0700 Subject: [PATCH 028/375] Changed front-end and back-end to front_end and back_end to avoid error --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3f38540c0eb..f488b65cc66 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -74,8 +74,8 @@ class Architecture(object): """ priority = None # Subclass needs to set this number. This controls order in which arch is detected. - front-end = None - back-end = None + front_end = None + back_end = None default = None # The default back end target. On cray ivybridge def __init__(self, name): From e39586c81da152e8d02fd26bf48c66e00e26519a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:58:49 -0700 Subject: [PATCH 029/375] Reverted back all architecture way of finding modules --- lib/spack/spack/cmd/compiler.py | 38 +++++++++++++-------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index cbe3aba864f..3e869289775 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -38,7 +38,6 @@ from spack.spec import CompilerSpec description = "Manage compilers" -ARCHITECTURE = spack.architecture.sys_type() def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command') @@ -59,30 +58,23 @@ def compiler_add(args): """Search either $PATH or a list of paths OR MODULES for compilers and add them to Spack's configuration.""" - strategies = ARCHITECTURE.strategy() - for strategy in strategies: - if strategy == 'PATH': - paths = args.add_paths # This might be a parser method. Parsing method to add_paths - if not paths: - paths = get_path('PATH') - - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) - if c.spec not in spack.compilers.all_compilers()] - - elif strategy == "MODULES": - from spack.compilers.cray import Cray - compilers = Cray.find_in_modules() - #TODO: Find a way to locate the executables + paths = args.add_paths # This might be a parser method. Parsing method to add_paths + if not paths: + paths = get_path('PATH') + + compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + if c.spec not in spack.compilers.all_compilers()] - if compilers: - spack.compilers.add_compilers_to_config('user', *compilers) - n = len(compilers) - tty.msg("Added %d new compiler%s to %s" % ( - n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) - colify(reversed(sorted(c.spec for c in compilers)), indent=4) - else: - tty.msg("Found no new compilers") + + if compilers: + spack.compilers.add_compilers_to_config('user', *compilers) + n = len(compilers) + tty.msg("Added %d new compiler%s to %s" % ( + n, 's' if n > 1 else '', spack.config.get_config_scope_filename('user', 'compilers'))) + colify(reversed(sorted(c.spec for c in compilers)), indent=4) + else: + tty.msg("Found no new compilers") def compiler_remove(args): From 9a91da9ccd8044e147a416fcfa9b16405ef22b3d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 30 Oct 2015 15:59:28 -0700 Subject: [PATCH 030/375] Added arch_path to list of modules path --- lib/spack/spack/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index caa09eb6e00..0f23f616147 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -37,6 +37,7 @@ lib_path = join_path(prefix, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") +arch_path = join_path(module_path, 'architectures') compilers_path = join_path(module_path, "compilers") test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") From eb2cf1698fa8cd606a8b80bcb39de2ee0b724c69 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 2 Nov 2015 11:08:08 -0800 Subject: [PATCH 031/375] Fixed some issues with naming --- lib/spack/spack/architecture.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f488b65cc66..442180242b9 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -49,13 +49,6 @@ def __init__(self): class Target(object): """ This is the processor type e.g. cray-ivybridge """ - # Front end or back end target. Target needs to know which one this is - # Should autodetect from the machine - # features of a target - # - a module name - # -a compiler finding strategy - # -a name - # architecture classes handling the aliasing for front-end, back-end and default def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 @@ -67,7 +60,8 @@ def compiler_strategy(self): return "MODULES" else: return "PATH" - + + class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -85,17 +79,19 @@ def __init__(self, name): def add_target(self, name, target): self.targets[name] = target + def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': - name = default + name = self.default elif name == 'front_end' or name == 'fe': - name = front-end + name = self.front_end elif name == 'back_end' or name == 'be': - name = back-end + name = self.back_end + return self.targets[name] @classmethod @@ -148,10 +144,7 @@ def get_sys_type_from_uname(): return None def get_sys_type_from_config_file(): - """ Should read in a sys_type from the config yaml file. This should be the first thing looked at since - The user can specify that the architecture is a cray-xc40. A template yaml should be created when spack - is installed. Similar to .spackconfig - """ + spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: From 058e72d29c3cd934f91ba626392b25ebaa50e2cc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 2 Nov 2015 11:08:55 -0800 Subject: [PATCH 032/375] Added default target and also front end and back end targets --- lib/spack/spack/architectures/cray.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index e6d6d849227..a79c9166842 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -4,9 +4,9 @@ class Cray(Architecture): priority = 20 - front_end = None - back_end = None - default = None + front_end = 'sandybridge' + back_end = 'ivybridge' + default = os.environ["CRAY_CPU_TARGET"] def __init__(self): super(Cray, self).__init__('cray') From 5ac974c9b2072631eab490cce8f2922420eef9e4 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 12:50:22 -0800 Subject: [PATCH 033/375] Enforced that the architecture subclass cannot add a target that shares a name with a target alias --- lib/spack/spack/architecture.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 442180242b9..890df9b1e59 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -77,9 +77,14 @@ def __init__(self, name): self.name = name def add_target(self, name, target): - self.targets[name] = target - - + """Used by the architecture specific subclass to list available targets. Raises an error + if the architecture specifies a name that is reserved by spack as an alias. + """ + if name in ['front_end', 'fe', 'back_end', 'be', 'default']: + raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) + self.targets[name] = target + + def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten From 37260962e545366d0d882b205be807562c36f3b7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 12:57:29 -0800 Subject: [PATCH 034/375] changed some potential syntax errors and added a way for target to recognize class --- lib/spack/spack/architecture.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 442180242b9..b251b82dcc7 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -48,20 +48,29 @@ def __init__(self): class Target(object): - """ This is the processor type e.g. cray-ivybridge """ + """ Target is the processor of the host machine. The host machine may have different front-end + and back-end targets, especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also recognize which architecture + they came from using the set_architecture method. Targets will have compiler finding strategies + """ + default_strategy = None # Can probably add a compiler path here def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 self.module_name = module_name # craype-ivybridge + def set_architecture(self, architecture): # Target should get the architecture class. + self.architecture = architecture + @property def compiler_strategy(self): - if self.module_name: # If there is a module_name given then use MODULES + if default_strategy: + return default_strategy + elif self.module_name: # If there is a module_name given then use MODULES return "MODULES" else: return "PATH" - class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -77,8 +86,8 @@ def __init__(self, name): self.name = name def add_target(self, name, target): - self.targets[name] = target - + target.set_architecture(self) + self.targets[name] = target def target(self, name): """This is a getter method for the target dictionary that handles defaulting based @@ -163,7 +172,7 @@ def get_sys_type_from_config_file(): def all_architectures(): modules = [] for name in list_modules(spack.arch_path): - mod_name = 'spack.architectures.' + name + mod_name = 'spack.architectures' + name path = join_path(spack.arch_path, name) + ".py" mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) From 35532d6b0aaa428e1d8234513e9e887d08c3914c Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 13:00:35 -0800 Subject: [PATCH 035/375] Changed cray architecture subclass to add proper targets for front at back end nodes --- lib/spack/spack/architectures/cray.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index a79c9166842..420b7c589e2 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -10,6 +10,11 @@ class Cray(Architecture): def __init__(self): super(Cray, self).__init__('cray') + # Back End compiler needs the proper target module loaded. + self.add_target('ivybridge','craype-ivybridge') + # Could switch to use modules and fe targets for front end + # Currently using compilers by path for front end. + self.add_target('sandybridge') @classmethod def detect(self): From 9bf8e8573cff42581074ed9e492ab0e0bc6c7253 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 13:08:48 -0800 Subject: [PATCH 036/375] generic linux architecture subclass --- lib/spack/spack/architectures/linux.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 lib/spack/spack/architectures/linux.py diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py new file mode 100644 index 00000000000..72385756604 --- /dev/null +++ b/lib/spack/spack/architectures/linux.py @@ -0,0 +1,17 @@ +import subprocess +from spack.architecture import Architecture + +class Linux(Architecture): + priority = 60 + front_end = "x86_64" + back_end = "x86_64" + default = "x86_64" + + def __init__(self): + super(Linux, self).__init__('linux') + + @classmethod + def detect(self): + arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch, _ = arch.communicate() + return 'x86_64' in arch.strip() From 3a73ae1683c4e83791f273d2034a99824e81aeca Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 Nov 2015 13:12:11 -0800 Subject: [PATCH 037/375] Fixed the previous commit --- lib/spack/spack/architectures/cray.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index 420b7c589e2..640a3933e06 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -1,20 +1,20 @@ import os -from spack.architecture import Architecture +from spack.architecture import Architecture, Target class Cray(Architecture): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' - default = os.environ["CRAY_CPU_TARGET"] + default = 'ivybridge' def __init__(self): super(Cray, self).__init__('cray') # Back End compiler needs the proper target module loaded. - self.add_target('ivybridge','craype-ivybridge') + self.add_target('ivybridge', Target('ivybridge','craype-ivybridge')) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target('sandybridge') + self.add_target('sandybridge', Target('sandybridge')) @classmethod def detect(self): From b61d554dc818c2514384b017c74dfa61c04a8c3a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 4 Nov 2015 13:28:12 -0800 Subject: [PATCH 038/375] Fixed architecture.py file so it doesn't have the weird merge changes i.e <<>>>>>> 8b3f2ec1d117e1a8b206927f51db8684396c231b def target(self, name): """This is a getter method for the target dictionary that handles defaulting based on the values provided by default, front-end, and back-end. This can be overwritten From 08729315c617b42e6fd1eaa1af4919aa52f96a0f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 5 Nov 2015 09:21:56 -0800 Subject: [PATCH 039/375] Added bgq architecture subclass: may require additional functionality to support --- lib/spack/spack/architectures/bgq.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lib/spack/spack/architectures/bgq.py diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py new file mode 100644 index 00000000000..d3d4446e09c --- /dev/null +++ b/lib/spack/spack/architectures/bgq.py @@ -0,0 +1,19 @@ +import os + +from spack.architecture import Architecture, Target + +class Bgq(Architecture): + priority = 30 + front_end = 'power7' + back_end = 'powerpc' + default = 'powerpc' + + def __init__(self): + super(Bgq, self).__init__('cray') + self.add_target('power7', Target('power7')) + self.add_target('powerpc', Target('powerpc')) + + @classmethod + def detect(self): + return os.path.exists('/bgsys') + From 0fcb368f478621b3eb8faf2b812018d43c46ffef Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 5 Nov 2015 13:42:50 -0800 Subject: [PATCH 040/375] Adding adios package to spack's package list --- var/spack/packages/adios/package.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 var/spack/packages/adios/package.py diff --git a/var/spack/packages/adios/package.py b/var/spack/packages/adios/package.py new file mode 100644 index 00000000000..5f298fc6b5f --- /dev/null +++ b/var/spack/packages/adios/package.py @@ -0,0 +1,28 @@ +from spack import * + +class Adios(Package): + """The Adaptable IO System (ADIOS) provides a simple, + flexible way for scientists to describe the data in their code that may need to be written, + read, or processed outside of the running simulation + """ + + homepage = "http://www.olcf.ornl.gov/center-projects/adios/" + url = "http://users.nccs.gov/~pnorbert/adios-1.9.0.tar.gz" + + version('1.9.0', 'dbf5cb10e32add2f04c9b4052b7ffa76') + + # Lots of setting up here for this package + # module swap PrgEnv-intel PrgEnv-$COMP + # module load cray-netcdf/4.3.3.1 + # module load cray-hdf5/1.8.14 + # module load python/2.7.10 + depends_on('mxml') + + def install(self, spec, prefix): + configure_args = ["--prefix=%s" % prefix, "--with-mxml=%s" % spec['mxml'].prefix, + "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-netcdf=%s" % os.environ['NETCDF_DIR'], + "--with-infiniband=no"] + + configure(*configure_args) + make() + make("install") From e0498ce54ad15787db3c4f187902b424c33013d7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 5 Nov 2015 13:43:11 -0800 Subject: [PATCH 041/375] mxml is adios's package dependency --- var/spack/packages/mxml/package.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 var/spack/packages/mxml/package.py diff --git a/var/spack/packages/mxml/package.py b/var/spack/packages/mxml/package.py new file mode 100644 index 00000000000..fc32b56323c --- /dev/null +++ b/var/spack/packages/mxml/package.py @@ -0,0 +1,24 @@ +from spack import * + +class Mxml(Package): + """Mini-XML is a small XML library that you can use to read and write XML + and XML-like data files in your application without requiring large + non-standard libraries""" + + homepage = "http://www.msweet.org" + url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz" + + version('2.9', 'e21cad0f7aacd18f942aa0568a8dee19') + version('2.8', 'd85ee6d30de053581242c4a86e79a5d2') + version('2.7', '76f2ae49bf0f5745d5cb5d9507774dc9') + version('2.6', '68977789ae64985dddbd1a1a1652642e') + version('2.5', 'f706377fba630b39fa02fd63642b17e5') + + # module swap PrgEnv-intel PrgEnv-$COMP (Can use whatever compiler you want to use) + # Case statement to change CC and CXX flags + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix, "--disable-shared") + make() + make("install") + From 5347f460b4f2ce8ad64f6ec225779c2a2f944ad4 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 10:31:37 -0800 Subject: [PATCH 042/375] adding module find to find command --- lib/spack/spack/compiler.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 4ce6e3f11d7..e4bfeea608c 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,7 +106,7 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, cc, cxx, f77, fc, module=None): + def __init__(self, cspec, cc, cxx, f77, fc, modules=None): def check(exe): if exe is None: return None @@ -119,7 +119,7 @@ def check(exe): self.fc = check(fc) self.spec = cspec - self.modules = modules.split() + self.modules = modules @@ -216,6 +216,10 @@ def check(key): @classmethod def find(cls, *path): + return cls.find_in_path(*path) + cls.find_in_modules() + + @classmethod + def find_in_path(cls, *path): """Try to find this type of compiler in the user's environment. For each set of compilers found, this returns compiler objects with the cc, cxx, f77, fc paths and the @@ -273,7 +277,9 @@ def find_in_modules(cls): if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') - output = _shell('module avail %s' % cls.PrgEnv_compiler) +# output = _shell('module avail %s' % cls.PrgEnv_compiler) + modulecmd = which('modulecmd') + modulecmd matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) for name, version in matches: From 95a34628a3bebe3d263afbd2707cb5d47a2b7a33 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 11:32:18 -0800 Subject: [PATCH 043/375] Add modules to compilers. Changed compiler to take paths as a list. Changed compiler_for_spec to be aware of different compiler stratigies --- lib/spack/spack/compiler.py | 61 +++++++-------------------- lib/spack/spack/compilers/__init__.py | 13 +++++- lib/spack/spack/package.py | 2 +- 3 files changed, 27 insertions(+), 49 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 67765dfb7be..4be573fb126 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,17 +106,20 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, cc, cxx, f77, fc, modules=None): + def __init__(self, cspec, paths, modules=None): def check(exe): if exe is None: return None _verify_executables(exe) return exe - self.cc = check(cc) - self.cxx = check(cxx) - self.f77 = check(f77) - self.fc = check(fc) + self.cc = check(paths[0]) + self.cxx = check(paths[1]) + self.f77 = check(paths[2]) + if len(paths) == 3: + self.fc = self.f77 + else: + self.fc = check(paths[3]) self.spec = cspec self.modules = modules @@ -264,7 +267,7 @@ def find_in_path(cls, *path): if newcount <= prevcount: continue - compilers[ver] = cls(spec, *paths) + compilers[ver] = cls(spec, paths) return list(compilers.values()) @@ -277,60 +280,26 @@ def find_in_modules(cls): if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') -# output = _shell('module avail %s' % cls.PrgEnv_compiler) modulecmd = which('modulecmd') - modulecmd + modulecmd.add_default_arg('python') + output = modulecmd('avail', return_oe=True) matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) - loaded_modules = os.environ["LOADEDMODULES"].split(":") +# loaded_modules = os.environ["LOADEDMODULES"].split(":") #output = _shell('module avail %s' % cls.PrgEnv_compiler) - for module in loaded_modules: - match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) +# for module in loaded_modules: +# match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) for name, version in matches: v = version + '-craype' comp = cls(spack.spec.CompilerSpec(name + '@' + v), - 'cc', 'CC', 'ftn', 'ftn', name +'/' + v) + ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) compilers.append(comp) return compilers -def _cur_prgenv(): - out, err = subprocess.Popen( - ['module list'], shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - matches = re.findall(r'(PrgEnv-[^/]*)/', err) - return matches[0] - - -def _module_shell(module, *args): - cmd = 'module swap %s %s;' % (_cur_prgenv(), module) - cmd += 'module load %s;' % compiler - cmd += 'module unload cray-libsci;' - -# + -# 'module load craype-network-gemini;' + -# 'module load %s;' % module + -# 'module swap gcc/4.6.1;' + -# 'module load eswrap; ' + -# 'module load craype-mc12; ' + -# 'module load cray-shmem; ' + -# 'module load cray-mpich; ') - cmd += ' '.join(args) - out, err = subprocess.Popen([cmd + ' '.join(args)], shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() - return out - - -def _shell(*args): - return subprocess.Popen([' '.join(args)], shell=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[1] - - - - def __repr__(self): """Return a string represntation of the compiler toolchain.""" return self.__str__() diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 8c4cb38926a..6fc54daa61c 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -197,18 +197,27 @@ def get_compiler(cspec): else: compiler_paths.append(None) - return cls(cspec, *compiler_paths) + for m in _optional_instance_vars: + if m not in items: + items[m] = None + mods = items[m] + + return cls(cspec, compiler_paths, mods) matches = find(compiler_spec) return [get_compiler(cspec) for cspec in matches] @_auto_compiler_spec -def compiler_for_spec(compiler_spec): +def compiler_for_spec(compiler_spec, target): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" assert(compiler_spec.concrete) compilers = compilers_for_spec(compiler_spec) + if target.compiler_strategy == "PATH": + filter(lambda c: c.modules is None, compilers) + elif target.compiler_strategy == "MODULES": + filter(lambda c: c.modules is not None, compilers) assert(len(compilers) == 1) return compilers[0] diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 61606d05905..b6a98f3ee62 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -592,7 +592,7 @@ def compiler(self): """Get the spack.compiler.Compiler object used to build this package.""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler) + return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.architecture) def url_version(self, version): From 271a839957ace370ce332f8edd3321e60127a9e6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 14:22:07 -0800 Subject: [PATCH 044/375] First possibly working version of the crayport. Not sufficiently tested at all. --- lib/spack/spack/architecture.py | 41 +++++++++++++++----------- lib/spack/spack/architectures/linux.py | 13 ++++---- lib/spack/spack/build_environment.py | 6 ++-- lib/spack/spack/compiler.py | 8 +++-- lib/spack/spack/compilers/__init__.py | 6 ++-- lib/spack/spack/concretize.py | 28 +----------------- lib/spack/spack/config.py | 5 ++-- 7 files changed, 46 insertions(+), 61 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 64416e8b4c3..b1da8ae4a29 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -61,18 +61,20 @@ def __init__(self,name, module_name=None): def set_architecture(self, architecture): # Target should get the architecture class. self.architecture = architecture - + @property def compiler_strategy(self): - if default_strategy: - return default_strategy - elif self.module_name: # If there is a module_name given then use MODULES + if self.module_name: # If there is a module_name given then use MODULES return "MODULES" else: return "PATH" + def __str__(self): + return self.name + + class Architecture(object): - """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it + """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned """ @@ -110,12 +112,15 @@ def target(self, name): @classmethod def detect(self): - """ Subclass is responsible for implementing this method. + """ Subclass is responsible for implementing this method. Returns True if the architecture detects if it is the current architecture and False if it's not. """ raise NotImplementedError() - + + def __repr__(self): + return self.__str__ + def __str__(self): return self.name @@ -123,7 +128,7 @@ def __str__(self): def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): - return None + return None elif hasattr(spack.sys_type, "__call__"): return spack.sys_type() #If in __init__.py there is a sys_type() then call that else: @@ -147,7 +152,7 @@ def get_mac_sys_type(): def get_sys_type_from_uname(): - """ Returns a sys_type from the uname argument + """ Returns a sys_type from the uname argument Front-end config """ try: @@ -158,8 +163,8 @@ def get_sys_type_from_uname(): return None def get_sys_type_from_config_file(): - - spack_home_dir = os.environ["HOME"] + "/.spack" + + spack_home_dir = os.environ["HOME"] + "/.spack" yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') try: config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() @@ -167,7 +172,7 @@ def get_sys_type_from_config_file(): front = arch['front'] back = arch['back'] return Architecture(front,back) - + except: print "No architecture.yaml config file found" return None @@ -182,7 +187,7 @@ def all_architectures(): mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) if not hasattr(mod, class_name): - tty.die('No class %s defined in %s' % (class_name, mod_name)) + tty.die('No class %s defined in %s' % (class_name, mod_name)) cls = getattr(mod, class_name) if not inspect.isclass(cls): tty.die('%s.%s is not a class' % (mod_name, class_name)) @@ -197,15 +202,15 @@ def sys_type(): 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 2. UNAME 3. GLOBALS - 4. MAC OSX + 4. MAC OSX Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. - If there is no yaml present then it should move on to the next function and stop immediately once it gets a + If there is no yaml present then it should move on to the next function and stop immediately once it gets a arch name """ # Try to create an architecture object using the config file FIRST - architecture_list = all_architectures() - architecture_list.sort(key = lambda a: a.priority) - + architecture_list = all_architectures() + architecture_list.sort(key = lambda a: a.priority) + for arch in architecture_list: if arch.detect(): return arch() diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index 72385756604..d7d9a994faa 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -1,14 +1,15 @@ -import subprocess -from spack.architecture import Architecture +import subprocess +from spack.architecture import Architecture, Target class Linux(Architecture): - priority = 60 - front_end = "x86_64" - back_end = "x86_64" - default = "x86_64" + priority = 60 + front_end = 'linux' + back_end = 'linux' + default = 'linux' def __init__(self): super(Linux, self).__init__('linux') + self.add_target('linux', Target('linux')) @classmethod def detect(self): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 191e8587357..9e3be433fb0 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -88,13 +88,13 @@ def __call__(self, *args, **kwargs): def load_module(mod): """Takes a module name and removes modules until it is possible to - load that module. It then loads the provided module. Depends on the + load that module. It then loads the provided module. Depends on the modulecmd implementation of modules used in cray and lmod. """ #Create an executable of the module command that will output python code modulecmd = which('modulecmd') modulecmd.add_default_arg('python') - + # Read the module and remove any conflicting modules # We do this without checking that they are already installed # for ease of programming because unloading a module that is not @@ -102,7 +102,7 @@ def load_module(mod): text = modulecmd('show', mod, return_oe=True).split() for i, word in enumerate(text): if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) + exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) # Load the module now that there are no conflicts load = modulecmd('load', mod, return_oe=True) exec(compile(load, '', 'exec')) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 4be573fb126..41ff89a1515 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -307,8 +307,12 @@ def __repr__(self): def __str__(self): """Return a string represntation of the compiler toolchain.""" - return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) + if self.modules: + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules)))) + else: + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6fc54daa61c..23ff6cced44 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -44,15 +44,15 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_optional_instance_vars = ['module'] +_optional_instance_vars = ['modules'] _default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] def _auto_compiler_spec(function): - def converter(cspec_like): + def converter(cspec_like, *args): if not isinstance(cspec_like, spack.spec.CompilerSpec): cspec_like = spack.spec.CompilerSpec(cspec_like) - return function(cspec_like) + return function(cspec_like, *args) return converter diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index c5041d67be3..4b4cb690330 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -97,32 +97,6 @@ def concretize_version(self, spec): def concretize_architecture(self, spec): - """If the spec already had an architecture, return. Otherwise if - the root of the DAG has an architecture, then use that. - Otherwise take the system's default architecture. - - Intuition: Architectures won't be set a lot, and generally you - want the host system's architecture. When architectures are - mised in a spec, it is likely because the tool requries a - cross-compiled component, e.g. for tools that run on BlueGene - or Cray machines. These constraints will likely come directly - from packages, so require the user to be explicit if they want - to mess with the architecture, and revert to the default when - they're not explicit. - """ - if spec.architecture is not None: - return False - - if spec.root.architecture: - spec.architecture = spec.root.architecture - else: - spec.architecture = spack.architecture.sys_type() - - assert(spec.architecture is not None) - return True # changed - - - def new_concretize_architecture(self, spec): """If the spec already has an architecture and it is a an architecture type, return. Otherwise, if it has an architecture that is a string type, generate an architecture based on that type. If it has no architecture and the root of the @@ -146,7 +120,7 @@ def new_concretize_architecture(self, spec): else: arch = spack.architecture.sys_type() spec.architecture = arch.target('default') - + return True #changed diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3e91958c2c2..bc655d60512 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -199,6 +199,7 @@ def get_config(category_name): category.result_dict = _merge_dicts(category.result_dict, result) else: category.result_dict = result + return category.result_dict @@ -208,7 +209,7 @@ def get_compilers_config(arch=None): configuration""" global _compiler_by_arch if not arch: - arch = spack.architecture.sys_type() + arch = str(spack.architecture.sys_type()) if arch in _compiler_by_arch: return _compiler_by_arch[arch] @@ -305,7 +306,7 @@ def add_to_compiler_config(addition_dict, scope=None, arch=None): """Add compilerss to the configuration files""" if not arch: arch = spack.architecture.sys_type() - add_to_config('compilers', { arch : addition_dict }, scope) + add_to_config('compilers', { str(arch) : addition_dict }, scope) clear_config_caches() From 8a13d344a898195c6eb33d8bea151487d0d209bd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 15:07:57 -0800 Subject: [PATCH 045/375] bug fix in spec.py --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e1fbb844234..3ba9c139a69 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1998,4 +1998,4 @@ def __init__(self, provided, required): class SpackYAMLError(spack.error.SpackError): def __init__(self, msg, yaml_error): - super(SpackError, self).__init__(msg, str(yaml_error)) + super(SpackYAMLError, self).__init__(msg, str(yaml_error)) From 3067705c109aef062823e62f0bb8a9d977a10a35 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 16:16:47 -0800 Subject: [PATCH 046/375] made a to_dict for targets. Also a from_dict for sanity --- lib/spack/spack/architecture.py | 19 +++++++++++++++++++ lib/spack/spack/spec.py | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index b1da8ae4a29..642a589f357 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -69,6 +69,25 @@ def compiler_strategy(self): else: return "PATH" + def to_dict(self): + print "to_dict" + d = {} + d['name'] = self.name + d['module_name'] = self.module_name + if self.architecture: + d['architecture'] = self.architecture + return d + + @staticmethod + def from_dict(d): + print "from_dict" + target = Target.__new__(Target) + target.name = d['name'] + target.module_name = d['module_name'] + if 'architecture' in d: + target.architecture = d['architecture'] + return target + def __str__(self): return self.name diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 3ba9c139a69..395faecde65 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -651,7 +651,7 @@ def to_node_dict(self): d = { 'variants' : dict( (name,v.enabled) for name, v in self.variants.items()), - 'arch' : self.architecture, + 'arch' : self.architecture.to_dict(), 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } @@ -680,7 +680,7 @@ def from_node_dict(node): spec = Spec(name) spec.versions = VersionList.from_dict(node) - spec.architecture = node['arch'] + spec.architecture = spack.architecture.Target.from_dict(node['arch']) if node['compiler'] is None: spec.compiler = None From 185f40eb8bbc19ecd7e2634844c97067388d0422 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 16:33:22 -0800 Subject: [PATCH 047/375] fixed my git problems --- lib/spack/spack/compiler.py | 2 +- lib/spack/spack/compilers/gcc.py | 2 +- lib/spack/spack/compilers/intel.py | 2 +- lib/spack/spack/compilers/pgi.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 41ff89a1515..abf72231176 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -291,7 +291,7 @@ def find_in_modules(cls): # match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) for name, version in matches: - v = version + '-craype' + v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) diff --git a/lib/spack/spack/compilers/gcc.py b/lib/spack/spack/compilers/gcc.py index ff0b8889a87..a9a4f729ebd 100644 --- a/lib/spack/spack/compilers/gcc.py +++ b/lib/spack/spack/compilers/gcc.py @@ -42,7 +42,7 @@ class Gcc(Compiler): # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. suffixes = [r'-mp-\d\.\d'] - PrgEnv = 'gnu' + PrgEnv = 'PrgEnv-gnu' PrgEnv_compiler = 'gcc' @property diff --git a/lib/spack/spack/compilers/intel.py b/lib/spack/spack/compilers/intel.py index 7c485fe69d5..4096e32a08f 100644 --- a/lib/spack/spack/compilers/intel.py +++ b/lib/spack/spack/compilers/intel.py @@ -37,7 +37,7 @@ class Intel(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['ifort'] - PrgEnv = 'intel' + PrgEnv = 'PrgEnv-intel' PrgEnv_compiler = 'intel' @property diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index 8f1ed288258..8010c1d04d6 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -37,7 +37,7 @@ class Pgi(Compiler): # Subclasses use possible names of Fortran 90 compiler fc_names = ['pgf95', 'pgf90'] - PrgEnv = 'pgi' + PrgEnv = 'PrgEnv-pgi' PrgEnv_compiler = 'pgi' @classmethod From eb2c08315905e0484dbd4d866396563487d7d249 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 11 Nov 2015 17:29:47 -0800 Subject: [PATCH 048/375] Fixed the to_dict from_dict for targets on cray. First version to build/find properly on Cray --- lib/spack/spack/architecture.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 642a589f357..349a7b33dbe 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -70,12 +70,11 @@ def compiler_strategy(self): return "PATH" def to_dict(self): - print "to_dict" d = {} d['name'] = self.name d['module_name'] = self.module_name - if self.architecture: - d['architecture'] = self.architecture +# if self.architecture: +# d['architecture'] = self.architecture return d @staticmethod @@ -84,10 +83,13 @@ def from_dict(d): target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] - if 'architecture' in d: - target.architecture = d['architecture'] +# if 'architecture' in d: +# target.architecture = d['architecture'] return target + def __repr__(self): + return self.__str__() + def __str__(self): return self.name @@ -138,7 +140,7 @@ def detect(self): raise NotImplementedError() def __repr__(self): - return self.__str__ + return self.__str__() def __str__(self): return self.name From e78b8c16be5334aa4aea29313a77da8923319a8f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 12 Nov 2015 13:27:10 -0800 Subject: [PATCH 049/375] used key_comparator decorator from util.lang to compare targets. Fixes find bug --- lib/spack/spack/architecture.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 349a7b33dbe..f4b8585c2f4 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -27,7 +27,7 @@ import platform as py_platform import inspect -from llnl.util.lang import memoized, list_modules +from llnl.util.lang import memoized, list_modules, key_ordering from llnl.util.filesystem import join_path import llnl.util.tty as tty @@ -47,6 +47,7 @@ def __init__(self): super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") +@key_ordering class Target(object): """ Target is the processor of the host machine. The host machine may have different front-end and back-end targets, especially if it is a Cray machine. The target will have a name and @@ -79,7 +80,6 @@ def to_dict(self): @staticmethod def from_dict(d): - print "from_dict" target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] @@ -87,6 +87,10 @@ def from_dict(d): # target.architecture = d['architecture'] return target + + def _cmp_key(self): + return (self.name, self.module_name) + def __repr__(self): return self.__str__() From 297cebd83328040aef71ed3aaf486a336011116d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 12 Nov 2015 13:34:40 -0800 Subject: [PATCH 050/375] Got rid of print statement --- lib/spack/spack/architecture.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 349a7b33dbe..d65acd92ba4 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -79,7 +79,6 @@ def to_dict(self): @staticmethod def from_dict(d): - print "from_dict" target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] From 7c89f9d18c57e3e0abe40cf608f63015ea9b68f3 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 12 Nov 2015 14:31:41 -0800 Subject: [PATCH 051/375] Added the basic ouline of a cray compiler class in lib/spack/spack/compilers/cray.py --- lib/spack/spack/compilers/cray.py | 50 +++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 lib/spack/spack/compilers/cray.py diff --git a/lib/spack/spack/compilers/cray.py b/lib/spack/spack/compilers/cray.py new file mode 100644 index 00000000000..c2f3ecd6f4e --- /dev/null +++ b/lib/spack/spack/compilers/cray.py @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import llnl.util.tty as tty +from spack.compiler import * +from spack.version import ver + +class Cray(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['cc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['CC'] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['ftn'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['ftn'] + + # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. + suffixes = [r'-mp-\d\.\d'] + + PrgEnv = 'PrgEnv-cray' + PrgEnv_compiler = 'craype' + + @property + def cxx11_flag(self): + return "-hstd=c++11" From a5ba69d68d1472f22454a93d045632b1340e2b04 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:46:13 -0800 Subject: [PATCH 052/375] Added a _cmp_key for the architecture class --- lib/spack/spack/architecture.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f4b8585c2f4..786ef340715 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -97,7 +97,7 @@ def __repr__(self): def __str__(self): return self.name - +@key_ordering class Architecture(object): """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned @@ -148,7 +148,9 @@ def __repr__(self): def __str__(self): return self.name - + + def _cmp_key(self): + return (self.name, (_cmp_key(t) for t in self.targets.values())) def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" From 9458f7c7d0ffebfbe33345c1a6b4b07c39e41e5f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:47:36 -0800 Subject: [PATCH 053/375] Got rid of my old method of finding in LOADEDMODULES --- lib/spack/spack/compiler.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index abf72231176..5d875882452 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -283,13 +283,9 @@ def find_in_modules(cls): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') output = modulecmd('avail', return_oe=True) - matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output) - -# loaded_modules = os.environ["LOADEDMODULES"].split(":") - #output = _shell('module avail %s' % cls.PrgEnv_compiler) -# for module in loaded_modules: -# match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module) - + matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) +# It's finding a weird third attribute + print matches for name, version in matches: v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), From 72fa3c04922b3630c15c69ff51022d88a8bf1de9 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:48:15 -0800 Subject: [PATCH 054/375] Added craype support --- lib/spack/spack/compilers/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 23ff6cced44..6731318f43a 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -46,7 +46,7 @@ _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] _optional_instance_vars = ['modules'] -_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc'] +_default_order = ['gcc', 'intel', 'pgi', 'clang', 'xlc','craype'] def _auto_compiler_spec(function): def converter(cspec_like, *args): @@ -180,6 +180,7 @@ def compilers_for_spec(compiler_spec): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ + # cray issue might be located here config = _get_config() def get_compiler(cspec): From bfd05d3d2702bb2a75fe310fb04eb78ac2526824 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:48:49 -0800 Subject: [PATCH 055/375] Changed name of file and class to craype to avoid spack freaking out about versions --- lib/spack/spack/compilers/craype.py | 57 +++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 lib/spack/spack/compilers/craype.py diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py new file mode 100644 index 00000000000..56d6d8d36c9 --- /dev/null +++ b/lib/spack/spack/compilers/craype.py @@ -0,0 +1,57 @@ +##############################################################################} +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import llnl.util.tty as tty + +from spack.build_environment import load_module +from spack.compiler import * +from spack.version import ver + +class Craype(Compiler): + # Subclasses use possible names of C compiler + cc_names = ['cc'] + + # Subclasses use possible names of C++ compiler + cxx_names = ['CC'] + + # Subclasses use possible names of Fortran 77 compiler + f77_names = ['ftn'] + + # Subclasses use possible names of Fortran 90 compiler + fc_names = ['ftn'] + + # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. + suffixes = [r'-mp-\d\.\d'] + + PrgEnv = 'PrgEnv-cray' + PrgEnv_compiler = 'craype' + + @property + def cxx11_flag(self): + return "-hstd=c++11" + + @classmethod + def default_version(cls, comp): + return get_compiler_version(comp, r'([Vv]ersion).*(\d+(\.\d+)+)') + From fceb5a75b06b639a6f6d7edf33c3b727e71a491b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 Nov 2015 11:53:38 -0800 Subject: [PATCH 056/375] Adding new files to compilers --- lib/spack/spack/compilers/cray.py | 50 ------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 lib/spack/spack/compilers/cray.py diff --git a/lib/spack/spack/compilers/cray.py b/lib/spack/spack/compilers/cray.py deleted file mode 100644 index c2f3ecd6f4e..00000000000 --- a/lib/spack/spack/compilers/cray.py +++ /dev/null @@ -1,50 +0,0 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://scalability-llnl.github.io/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## -import llnl.util.tty as tty -from spack.compiler import * -from spack.version import ver - -class Cray(Compiler): - # Subclasses use possible names of C compiler - cc_names = ['cc'] - - # Subclasses use possible names of C++ compiler - cxx_names = ['CC'] - - # Subclasses use possible names of Fortran 77 compiler - f77_names = ['ftn'] - - # Subclasses use possible names of Fortran 90 compiler - fc_names = ['ftn'] - - # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes. - suffixes = [r'-mp-\d\.\d'] - - PrgEnv = 'PrgEnv-cray' - PrgEnv_compiler = 'craype' - - @property - def cxx11_flag(self): - return "-hstd=c++11" From 8f56f5cfccb681286ab5d31dbdbed29cdf49a930 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 23 Nov 2015 14:02:59 -0800 Subject: [PATCH 057/375] Added path to netcdf and hdf5 using the env vars NETCDF_DIR and HDF5_DIR. --- var/spack/packages/adios/package.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/adios/package.py b/var/spack/packages/adios/package.py index 5f298fc6b5f..05cbb5f9638 100644 --- a/var/spack/packages/adios/package.py +++ b/var/spack/packages/adios/package.py @@ -1,8 +1,10 @@ -from spack import * +import os +from spack import * class Adios(Package): """The Adaptable IO System (ADIOS) provides a simple, - flexible way for scientists to describe the data in their code that may need to be written, + flexible way for scientists to describe the + data in their code that may need to be written, read, or processed outside of the running simulation """ @@ -19,10 +21,12 @@ class Adios(Package): depends_on('mxml') def install(self, spec, prefix): - configure_args = ["--prefix=%s" % prefix, "--with-mxml=%s" % spec['mxml'].prefix, - "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-netcdf=%s" % os.environ['NETCDF_DIR'], + configure_args = ["--prefix=%s" % prefix, + "--with-mxml=%s" % spec['mxml'].prefix, + "--with-hdf5="+os.environ["HDF5_DIR"], + "--with-netcdf="+os.environ["NETCDF_DIR"], "--with-infiniband=no"] - + configure(*configure_args) make() make("install") From 89fbe4fdfa01b2461a5355a8922e6083f528e409 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:55:38 -0800 Subject: [PATCH 058/375] Changed add_target to include variables rather than hard-coded strings --- lib/spack/spack/architectures/bgq.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py index d3d4446e09c..85487626f1b 100644 --- a/lib/spack/spack/architectures/bgq.py +++ b/lib/spack/spack/architectures/bgq.py @@ -10,8 +10,8 @@ class Bgq(Architecture): def __init__(self): super(Bgq, self).__init__('cray') - self.add_target('power7', Target('power7')) - self.add_target('powerpc', Target('powerpc')) + self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.back_end, Target(self.back_end)) @classmethod def detect(self): From 58f2b39bc844d1c8beffd6449a9047c5ec5522e5 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:57:17 -0800 Subject: [PATCH 059/375] Changed default to CPU_TARGET env var. Helps deal with target differences between cori and edison --- lib/spack/spack/architectures/cray.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index 640a3933e06..e4a3617dc93 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -6,15 +6,23 @@ class Cray(Architecture): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' - default = 'ivybridge' - + default = os.environ['CRAY_CPU_TARGET'] + #default = 'ivybridge' + def __init__(self): + ''' Since cori doesn't have ivybridge as a front end it's better + if we use CRAY_CPU_TARGET as the default. This will ensure + that if we're on a XC-40 or XC-30 then we can detect the target + ''' super(Cray, self).__init__('cray') + # Back End compiler needs the proper target module loaded. - self.add_target('ivybridge', Target('ivybridge','craype-ivybridge')) + self.add_target(self.front_end, Target(self.front_end,'craype-' + self.front_end)) + self.add_target(self.default, Target(self.default,'craype-' + self.default)) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target('sandybridge', Target('sandybridge')) + self.add_target(self.back_end, Target('craype-' + self.back_end)) + @classmethod def detect(self): From d7fdb8e015d5e6c5b0e06608166307f50f2f046f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:57:54 -0800 Subject: [PATCH 060/375] Got rid of hard coded strings for adding default --- lib/spack/spack/architectures/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index d7d9a994faa..6c454f7a40e 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -9,7 +9,7 @@ class Linux(Architecture): def __init__(self): super(Linux, self).__init__('linux') - self.add_target('linux', Target('linux')) + self.add_target(self.default, Target(self.default)) @classmethod def detect(self): From 4e0d47f1dd1d0eb31ef29e1e25ab68870f087604 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:59:19 -0800 Subject: [PATCH 061/375] building with GCC needs special variables added. Intel compilers build just fine --- var/spack/packages/adios/package.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/var/spack/packages/adios/package.py b/var/spack/packages/adios/package.py index 05cbb5f9638..224c3b37d51 100644 --- a/var/spack/packages/adios/package.py +++ b/var/spack/packages/adios/package.py @@ -19,14 +19,19 @@ class Adios(Package): # module load cray-hdf5/1.8.14 # module load python/2.7.10 depends_on('mxml') - + def install(self, spec, prefix): configure_args = ["--prefix=%s" % prefix, "--with-mxml=%s" % spec['mxml'].prefix, - "--with-hdf5="+os.environ["HDF5_DIR"], - "--with-netcdf="+os.environ["NETCDF_DIR"], - "--with-infiniband=no"] - + "--with-hdf5=%s" % os.environ["HDF5_DIR"], + "--with-netcdf=%s" % os.environ["NETCDF_DIR"], + "--with-infiniband=no", + "MPICC=cc","MPICXX=CC","MPIFC=ftn", + "CPPFLAGS=-DMPICH_IGNORE_CXX_SEEK"] + + if spec.satisfies('%gcc'): + configure_args.extend(["CC=gcc", "CXX=g++", "FC=gfortran"]) + configure(*configure_args) make() make("install") From 0544b164fc717f5221d7545139c18752129499d4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 1 Dec 2015 11:59:49 -0800 Subject: [PATCH 062/375] Updated script of mini-xml for adios package --- var/spack/packages/mxml/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/var/spack/packages/mxml/package.py b/var/spack/packages/mxml/package.py index fc32b56323c..f79251d312f 100644 --- a/var/spack/packages/mxml/package.py +++ b/var/spack/packages/mxml/package.py @@ -1,9 +1,11 @@ +import os from spack import * class Mxml(Package): """Mini-XML is a small XML library that you can use to read and write XML and XML-like data files in your application without requiring large - non-standard libraries""" + non-standard libraries + """ homepage = "http://www.msweet.org" url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz" @@ -18,7 +20,7 @@ class Mxml(Package): # Case statement to change CC and CXX flags def install(self, spec, prefix): - configure('--prefix=%s' % prefix, "--disable-shared") + configure('--prefix=%s' % prefix, "--disable-shared", 'CFLAGS=-static') make() make("install") From 028cca16e66c4a7dd867ef6be4cf38db9daa6e51 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Dec 2015 10:15:33 -0800 Subject: [PATCH 063/375] Got rid of the unusued default strategy=None --- lib/spack/spack/architecture.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 786ef340715..a53222d36f2 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -54,7 +54,6 @@ class Target(object): also the module_name (e.g craype-compiler). Targets will also recognize which architecture they came from using the set_architecture method. Targets will have compiler finding strategies """ - default_strategy = None # Can probably add a compiler path here def __init__(self,name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 @@ -99,7 +98,8 @@ def __str__(self): @key_ordering class Architecture(object): - """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it + """ Abstract class that each type of Architecture will subclass. + Will return a instance of it once it is returned """ From fee88d289d00f6886d0cd336aab4cabbb5d45acc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 11 Dec 2015 12:03:13 -0800 Subject: [PATCH 064/375] Rewrote docstrings for sys_type() and got rid of unused functions --- lib/spack/spack/architecture.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a53222d36f2..beebeba4bf7 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -189,22 +189,6 @@ def get_sys_type_from_uname(): except: return None -def get_sys_type_from_config_file(): - - spack_home_dir = os.environ["HOME"] + "/.spack" - yaml_file = os.path.join(spack_home_dir, 'architecture.yaml') - try: - config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load() - arch = config_dict['architecture'] - front = arch['front'] - back = arch['back'] - return Architecture(front,back) - - except: - print "No architecture.yaml config file found" - return None - - @memoized def all_architectures(): modules = [] @@ -225,14 +209,10 @@ def all_architectures(): @memoized def sys_type(): - """Priority of gathering sys-type. - 1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30 - 2. UNAME - 3. GLOBALS - 4. MAC OSX - Yaml should be a priority here because we want the user to be able to specify the type of architecture to use. - If there is no yaml present then it should move on to the next function and stop immediately once it gets a - arch name + """ Gather a list of all available subclasses of architectures. + Sorts the list according to their priority looking. Priority is + an arbitrarily set number. Detects arch either using uname or + a file path (/opt/cray...) """ # Try to create an architecture object using the config file FIRST architecture_list = all_architectures() From 31df2dd9dd713e75d161eb5a0187536886363469 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 11 Dec 2015 12:04:47 -0800 Subject: [PATCH 065/375] Fixed targets ti be x86_64, important note: have not tested this on linux machine --- lib/spack/spack/architectures/linux.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index 6c454f7a40e..052b10c0196 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -3,9 +3,9 @@ class Linux(Architecture): priority = 60 - front_end = 'linux' - back_end = 'linux' - default = 'linux' + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' def __init__(self): super(Linux, self).__init__('linux') @@ -15,4 +15,4 @@ def __init__(self): def detect(self): arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) arch, _ = arch.communicate() - return 'x86_64' in arch.strip() + return 'linux' in arch.strip().lower() From b5216f6ec8fc5b1cd7061d8610b51ceec5f162e8 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 11 Dec 2015 12:07:32 -0800 Subject: [PATCH 066/375] Commented out cxxflag. Not 100% sure it's needed --- lib/spack/spack/compilers/craype.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py index 56d6d8d36c9..e8ae284f5bb 100644 --- a/lib/spack/spack/compilers/craype.py +++ b/lib/spack/spack/compilers/craype.py @@ -24,9 +24,9 @@ ############################################################################## import llnl.util.tty as tty -from spack.build_environment import load_module +#from spack.build_environment import load_module from spack.compiler import * -from spack.version import ver +#from spack.version import ver class Craype(Compiler): # Subclasses use possible names of C compiler @@ -47,9 +47,9 @@ class Craype(Compiler): PrgEnv = 'PrgEnv-cray' PrgEnv_compiler = 'craype' - @property - def cxx11_flag(self): - return "-hstd=c++11" +# @property +# def cxx11_flag(self): +# return "-hstd=c++11" @classmethod def default_version(cls, comp): From 217a2d9ea5bd6184d36ea0c32429c979aa088fbc Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 17 Dec 2015 15:26:27 -0800 Subject: [PATCH 067/375] fixed yaml error --- lib/spack/spack/architecture.py | 2 ++ lib/spack/spack/spec.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index beebeba4bf7..3d1de393544 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -79,6 +79,8 @@ def to_dict(self): @staticmethod def from_dict(d): + if d is None: + return None target = Target.__new__(Target) target.name = d['name'] target.module_name = d['module_name'] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 395faecde65..ff580916564 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -651,10 +651,13 @@ def to_node_dict(self): d = { 'variants' : dict( (name,v.enabled) for name, v in self.variants.items()), - 'arch' : self.architecture.to_dict(), 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } + if self.architecture: + d['arch'] = self.architecture.to_dict() + else: + d['arch'] = None if self.compiler: d.update(self.compiler.to_dict()) else: From 53808f254efc6919ec66043c709039c78c34e11a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 4 Jan 2016 12:36:48 -0800 Subject: [PATCH 068/375] Support for cray external dependencies implemented in modules --- lib/spack/spack/build_environment.py | 54 +++++++++++++++++++++++++++- lib/spack/spack/concretize.py | 6 +++- lib/spack/spack/config.py | 13 +++++-- lib/spack/spack/spec.py | 9 +++-- 4 files changed, 75 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 9e3be433fb0..781039e073d 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -32,10 +32,11 @@ import shutil import multiprocessing import platform +import re + from llnl.util.filesystem import * import spack -import spack.compilers as compilers from spack.util.executable import Executable, which from spack.util.environment import * @@ -108,6 +109,46 @@ def load_module(mod): exec(compile(load, '', 'exec')) +def get_path_from_module(mod): + """Inspects a TCL module for entries that indicate the absolute path + at which the library supported by said module can be found. + """ + # Create a modulecmd executable + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module + text = modulecmd('show', mod, return_oe=True).split('\n') + + # If it lists its package directory, return that + for line in text: + if line.find(mod.upper()+'_DIR') >= 0: + words = line.split() + return words[2] + + # If it lists a -rpath instruction, use that + for line in text: + rpath = line.find('-rpath/') + if rpath >= 0: + return line[rpath+6:line.find('/lib')] + + # If it lists a -L instruction, use that + for line in text: + L = line.find('-L/') + if L >= 0: + return line[L+2:line.find('/lib')] + + # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that + for line in text: + if line.find('LD_LIBRARY_PATH') >= 0: + words = line.split() + path = words[2] + return path[:path.find('/lib')] + + # Unable to find module path + return None + + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -251,6 +292,17 @@ def set_module_variables_for_package(pkg): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" + + # First load all modules for external packages and update the external + # packages' paths to reflect what is found in the modules so that we can + # rpath through the modules when possible, but if not possible they are + # already loaded. + for spec in pkg.spec.traverse(root=False): + if spec.external_module: + load_module(spec.external_module) + spec.external = get_path_from_module(spec.external_module) + + # Construct rpaths from the paths of each dep rpaths = [pkg.prefix.lib, pkg.prefix.lib64] rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) if os.path.isdir(d.prefix.lib)) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 13e9b477dca..7c7453353a5 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -97,7 +97,7 @@ def _valid_virtuals_and_externals(self, spec): externals = spec_externals(pkg) buildable = not is_spec_nobuild(pkg) if buildable: - result.append((pkg, None)) + result.append((pkg, None, None)) if externals: sorted_externals = sorted(externals, cmp=lambda a,b: a[0].__cmp__(b[0])) for external in sorted_externals: @@ -131,6 +131,7 @@ def concretize_virtual_and_external(self, spec): if not candidate: #No ABI matches. Pick the top choice based on the orignal preferences. candidate = candidates[0] + external_module = candidate[2] external = candidate[1] candidate_spec = candidate[0] @@ -144,6 +145,9 @@ def concretize_virtual_and_external(self, spec): if not spec.external and external: spec.external = external changed = True + if not spec.external_module and external_module: + spec.external_module = external_module + changed = True #If we're external then trim the dependencies if external and spec.dependencies: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9919dcc0457..d66ffb338e1 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -103,6 +103,8 @@ from llnl.util.filesystem import mkdirp import copy +from spack.build_environment import get_path_from_module + _config_sections = {} class _ConfigCategory: name = None @@ -255,7 +257,8 @@ def get_packages_config(): package_name = spack.spec.Spec(p.keys()[0]).name if package_name not in indexed_packages: indexed_packages[package_name] = [] - indexed_packages[package_name].append({ spack.spec.Spec(key) : val for key, val in p.iteritems() }) + pkg_dict = dict([ (spack.spec.Spec(key), val) for key, val in p.iteritems()]) + indexed_packages[package_name].append( pkg_dict ) return indexed_packages @@ -286,9 +289,13 @@ def spec_externals(spec): if not pkg.satisfies(spec): continue path = conf.get('path', None) + module = conf.get('module', None) if not path: - continue - spec_locations.append( (pkg, path) ) + if not module: + continue + else: + path = get_path_from_module(module) + spec_locations.append( (pkg, path, module) ) return spec_locations diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 4289fe19cb2..f496011d62c 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -420,6 +420,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) self.external = None + self.external_module = None # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -1352,8 +1353,9 @@ def _dup(self, other, **kwargs): changed = (self.name != other.name and self.versions != other.versions and \ self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ - self.concrete != other.concrete and self.external != other.external) - + self.concrete != other.concrete and self.external != other.external and \ + self.external_module != other.external_module) + # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() @@ -1365,6 +1367,7 @@ def _dup(self, other, **kwargs): self.variants = other.variants.copy() self.variants.spec = self self.external = other.external + self.external_module = other.external_module # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1383,6 +1386,7 @@ def _dup(self, other, **kwargs): self._normal = other._normal self._concrete = other._concrete self.external = other.external + self.external_module = other.external_module return changed @@ -1809,6 +1813,7 @@ def spec(self): spec.architecture = None spec.compiler = None spec.external = None + spec.external_module = None spec.dependents = DependencyMap() spec.dependencies = DependencyMap() From 751208cedf3506ebcd5119fd0d4d904bc28392e8 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 4 Jan 2016 16:35:06 -0800 Subject: [PATCH 069/375] pick last compiler instead of first, will be overwritten by more advanced preferences as mentioned in the email group --- lib/spack/spack/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 7c7453353a5..48d05f7f8d5 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -287,7 +287,7 @@ def concretize_compiler(self, spec): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[0].copy() + spec.compiler = matches[len(matches)-1].copy() assert(spec.compiler.concrete) return True # things changed. From d2c2c46541d54e4dcde34e6199ba5dafa949bef1 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 11:12:14 -0800 Subject: [PATCH 070/375] bug hunting --- lib/spack/spack/compilers/__init__.py | 4 ++-- lib/spack/spack/config.py | 2 +- lib/spack/spack/spec.py | 5 +++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 6731318f43a..87106282cf0 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -216,9 +216,9 @@ def compiler_for_spec(compiler_spec, target): assert(compiler_spec.concrete) compilers = compilers_for_spec(compiler_spec) if target.compiler_strategy == "PATH": - filter(lambda c: c.modules is None, compilers) + compilers = [c for c in compilers if c.modules is None] elif target.compiler_strategy == "MODULES": - filter(lambda c: c.modules is not None, compilers) + compilers = [c for c in compilers if c.modules is not None] assert(len(compilers) == 1) return compilers[0] diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index d66ffb338e1..6e11cfa4753 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -277,7 +277,7 @@ def is_spec_nobuild(spec): def spec_externals(spec): - """Return a list of spec, directory pairs for each external location for spec""" + """Return a list of spec, directory, module triples for each external location for spec""" allpkgs = get_packages_config() name = spec.name spec_locations = [] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f496011d62c..126ffc4f95d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -771,8 +771,9 @@ def _replace_with(self, concrete): """Replace this virtual spec with a concrete spec.""" assert(self.virtual) for name, dependent in self.dependents.items(): - del dependent.dependencies[self.name] - dependent._add_dependency(concrete) + if not dependent.external: + del dependent.dependencies[self.name] + dependent._add_dependency(concrete) def _expand_virtual_packages(self): From 9848ad32fdadaf7decfb19aba03829708d8b6c85 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 13:39:53 -0800 Subject: [PATCH 071/375] fixed errors caused by crayport on linux and added a darwin architecture --- lib/spack/spack/architecture.py | 6 +++-- lib/spack/spack/architectures/cray.py | 21 +++++++++------- lib/spack/spack/architectures/darwin.py | 18 ++++++++++++++ lib/spack/spack/architectures/linux.py | 4 ++-- lib/spack/spack/test/__init__.py | 5 ++-- lib/spack/spack/test/architecture.py | 32 ++++++++++++++++++------- 6 files changed, 62 insertions(+), 24 deletions(-) create mode 100644 lib/spack/spack/architectures/darwin.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3d1de393544..1f608bd1875 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -96,11 +96,13 @@ def __repr__(self): return self.__str__() def __str__(self): + if self.module_name: + return self.name + ' module: ' + self.module_name return self.name @key_ordering class Architecture(object): - """ Abstract class that each type of Architecture will subclass. + """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it is returned """ @@ -150,7 +152,7 @@ def __repr__(self): def __str__(self): return self.name - + def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index e4a3617dc93..47ede301454 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -4,27 +4,30 @@ class Cray(Architecture): priority = 20 - front_end = 'sandybridge' + front_end = 'sandybridge' back_end = 'ivybridge' - default = os.environ['CRAY_CPU_TARGET'] - #default = 'ivybridge' - + default = 'ivybridge' + def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' super(Cray, self).__init__('cray') - + + # Handle the default here so we can check for a key error + if 'CRAY_CPU_TARGET' in os.environ: + default = os.environ['CRAY_CPU_TARGET'] + # Back End compiler needs the proper target module loaded. - self.add_target(self.front_end, Target(self.front_end,'craype-' + self.front_end)) + self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) self.add_target(self.default, Target(self.default,'craype-' + self.default)) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target(self.back_end, Target('craype-' + self.back_end)) - + self.add_target(self.front_end, Target(self.front_end)) + @classmethod def detect(self): return os.path.exists('/opt/cray/craype') - + diff --git a/lib/spack/spack/architectures/darwin.py b/lib/spack/spack/architectures/darwin.py new file mode 100644 index 00000000000..2f0d34c38d3 --- /dev/null +++ b/lib/spack/spack/architectures/darwin.py @@ -0,0 +1,18 @@ +import subprocess +from spack.architecture import Architecture, Target + +class Darwin(Architecture): + priority = 89 + front_end = 'x86_64' + back_end = 'x86_64' + default = 'x86_64' + + def __init__(self): + super(Darwin, self).__init__('darwin') + self.add_target(self.default, Target(self.default)) + + @classmethod + def detect(self): + arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + arch, _ = arch.communicate() + return 'darwin' in arch.strip().lower() diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index 052b10c0196..d63cf9179a9 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -2,7 +2,7 @@ from spack.architecture import Architecture, Target class Linux(Architecture): - priority = 60 + priority = 90 front_end = 'x86_64' back_end = 'x86_64' default = 'x86_64' @@ -13,6 +13,6 @@ def __init__(self): @classmethod def detect(self): - arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) arch, _ = arch.communicate() return 'linux' in arch.strip().lower() diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index ed51fac33a7..96dc1f4331a 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,7 +31,7 @@ import spack """Names of tests to be included in Spack's test suite""" -"""test_names = ['architecture', +test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', @@ -58,8 +58,7 @@ 'optional_deps', 'make_executable', 'configure_guess'] -""" -test_names = ['architecture'] + def list_tests(): """Return names of all tests that can be run for Spack.""" diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 6ff22aaa59a..3a7474799aa 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -2,18 +2,34 @@ the functions are looking for the correct architecture name """ import unittest +import os +import platform import spack from spack.architecture import * +from spack.architectures.cray import Cray +from spack.architectures.linux import Linux +from spack.architectures.bgq import Bgq class ArchitectureTest(unittest.TestCase): - def test_Architecture_class(self): - a = Architecture('Cray-XC40') - a.add_arch_strategy() - self.assertEquals(a.get_arch_dict(), {'Cray-XC40': 'MODULES'}) + def test_Architecture_class_and_compiler_strategies(self): + a = Cray() + t = a.target('default') + self.assertEquals(t.compiler_strategy, 'MODULES') + b = Linux() + s = b.target('default') + self.assertEquals(s.compiler_strategy, 'PATH') - def test_get_sys_type_from_config_file(self): - output_arch_class = get_sys_type_from_config_file() - my_arch_class = Architecture('Linux x86_64','Cray-xc40') + def test_sys_type(self): + output_arch_class = sys_type() + my_arch_class = None + if os.path.exists('/opt/cray/craype'): + my_arch_class = Cray() + elif os.path.exists('/bgsys'): + my_arch_class = Bgq() + elif 'Linux' in platform.system(): + my_arch_class = Linux() +# elif 'Darwin' in platform.system(): +# my_arch_class = Darwin() - self.assertEqual(output_arch_class, my_arch_class) + self.assertEqual(str(output_arch_class), str(my_arch_class)) From 1edbaa4e62d2eb59b53cf6b10f7854d4edcf8497 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 14:13:09 -0800 Subject: [PATCH 072/375] removed debug printing --- lib/spack/spack/architecture.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1f608bd1875..1b4b7730b2d 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -96,8 +96,6 @@ def __repr__(self): return self.__str__() def __str__(self): - if self.module_name: - return self.name + ' module: ' + self.module_name return self.name @key_ordering From 751503c434bd249ed9f76b6ad353d2adcdcc3997 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 14:47:14 -0800 Subject: [PATCH 073/375] fixed haswell targeting bug --- lib/spack/spack/architectures/cray.py | 17 ++++++++++++----- lib/spack/spack/compilers/__init__.py | 1 + 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index 47ede301454..dac3943fb73 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -17,15 +17,22 @@ def __init__(self): # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: - default = os.environ['CRAY_CPU_TARGET'] + self.default = os.environ['CRAY_CPU_TARGET'] + + # Change the defaults to haswell if we're on an XC40 + if self.default == 'haswell': + self.front_end = self.default + self.back_end = self.default - # Back End compiler needs the proper target module loaded. - self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) - self.add_target(self.default, Target(self.default,'craype-' + self.default)) # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. self.add_target(self.front_end, Target(self.front_end)) - + # Back End compiler needs the proper target module loaded. + self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) + self.add_target(self.default, Target(self.default,'craype-' + self.default)) + # This is kludgy and the order matters when the targets are all haswell + # This is because the last one overwrites the others when they have the + # same name. @classmethod def detect(self): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 87106282cf0..21a41f38ea9 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -206,6 +206,7 @@ def get_compiler(cspec): return cls(cspec, compiler_paths, mods) matches = find(compiler_spec) + print matches, 'matches' return [get_compiler(cspec) for cspec in matches] From 48b9023de48ba01f789705db421c9bb326202c0e Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 14:51:18 -0800 Subject: [PATCH 074/375] removed debug printing --- lib/spack/spack/compilers/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 21a41f38ea9..87106282cf0 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -206,7 +206,6 @@ def get_compiler(cspec): return cls(cspec, compiler_paths, mods) matches = find(compiler_spec) - print matches, 'matches' return [get_compiler(cspec) for cspec in matches] From d6768cf9210288ec562330fac0978cfa0b8b8683 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 16:23:32 -0800 Subject: [PATCH 075/375] minor bug fix --- lib/spack/spack/compiler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 5d875882452..b40f68b2c59 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -284,8 +284,7 @@ def find_in_modules(cls): modulecmd.add_default_arg('python') output = modulecmd('avail', return_oe=True) matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) -# It's finding a weird third attribute - print matches + for name, version in matches: v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), From c649610473a722388a09bf1000a8e3e9d6a4f9ff Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 17:00:28 -0800 Subject: [PATCH 076/375] fixed concretization to only consider compilers found by the proper strategy --- lib/spack/spack/concretize.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 48d05f7f8d5..f32e956a131 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -260,8 +260,27 @@ def concretize_compiler(self, spec): build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ + # Pass on concretizing the compiler if the architecture is not yet determined + if not spec.architecture: + #Although this usually means changed, this means awaiting other changes + return True + + # Examine only those compilers found by the proper compiler strategy for this architecture + # Takes advantage of the proper logic already existing in compiler_for_spec + # Should be redone more efficiently if this works all_compilers = spack.compilers.all_compilers() + def _proper_compiler_style(cspec, target): + compilers = spack.compilers.compilers_for_spec(cspec) + if target.compiler_strategy == 'PATH': + filter(lambda c: not c.modules, compilers) + if target.compiler_strategy == 'MODULES': + filter(lambda c: c.modules, compilers) + return compilers + + filter(lambda c: _proper_compiler_style(c, spec.architecture), all_compilers) + + if (spec.compiler and spec.compiler.concrete and spec.compiler in all_compilers): @@ -287,7 +306,7 @@ def concretize_compiler(self, spec): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[len(matches)-1].copy() + spec.compiler = matches[0].copy() assert(spec.compiler.concrete) return True # things changed. From 29e0ff61d7dc10b20b4b5d0d627cc066690ea84b Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 17:14:35 -0800 Subject: [PATCH 077/375] fixed compiler finding so as not to identify non-existent versions of the intel compiler based on the version numbers of the PrgEnv-intel module --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index b40f68b2c59..225c65917a5 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -282,7 +282,7 @@ def find_in_modules(cls): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') - output = modulecmd('avail', return_oe=True) + output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) for name, version in matches: From 93c9c45580d89ccbab441c5808192e149731c85f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 17:19:04 -0800 Subject: [PATCH 078/375] minor tweak of compiler priority while waiting for customizable compiler priorities --- lib/spack/spack/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f32e956a131..2013ae2e84a 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -306,7 +306,7 @@ def _proper_compiler_style(cspec, target): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[0].copy() + spec.compiler = matches[-1].copy() assert(spec.compiler.concrete) return True # things changed. From 61b03b72b0fb1aa5c4ddbaff8079d9c0258efb76 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 5 Jan 2016 19:03:25 -0800 Subject: [PATCH 079/375] improved concretize efficiency for determining whether compilers come from the proper strategy --- lib/spack/spack/concretize.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 2013ae2e84a..854db949e31 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -265,11 +265,9 @@ def concretize_compiler(self, spec): #Although this usually means changed, this means awaiting other changes return True - # Examine only those compilers found by the proper compiler strategy for this architecture + # Only use a matching compiler if it is of the proper style # Takes advantage of the proper logic already existing in compiler_for_spec - # Should be redone more efficiently if this works - all_compilers = spack.compilers.all_compilers() - + # Should think whether this can be more efficient def _proper_compiler_style(cspec, target): compilers = spack.compilers.compilers_for_spec(cspec) if target.compiler_strategy == 'PATH': @@ -278,8 +276,8 @@ def _proper_compiler_style(cspec, target): filter(lambda c: c.modules, compilers) return compilers - filter(lambda c: _proper_compiler_style(c, spec.architecture), all_compilers) - + + all_compilers = spack.compilers.all_compilers() if (spec.compiler and spec.compiler.concrete and @@ -306,7 +304,12 @@ def _proper_compiler_style(cspec, target): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - spec.compiler = matches[-1].copy() + index = len(matches)-1 + while not _proper_compiler_style(matches[index], spec.architecture): + index -= 1 + if index == 0: + raise NoValidVersionError(spec) + spec.compiler = matches[index].copy() assert(spec.compiler.concrete) return True # things changed. From 51bd91edc0679d21c26df8af17eaafb2a0ca6d56 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 6 Jan 2016 13:00:22 -0800 Subject: [PATCH 080/375] made the compiler strategy more easily controllable --- lib/spack/spack/architecture.py | 15 ++++++++------- lib/spack/spack/architectures/bgq.py | 4 ++-- lib/spack/spack/architectures/cray.py | 6 +++--- lib/spack/spack/architectures/darwin.py | 2 +- lib/spack/spack/architectures/linux.py | 2 +- lib/spack/spack/build_environment.py | 2 +- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1b4b7730b2d..81aac6cc339 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -55,19 +55,20 @@ class Target(object): they came from using the set_architecture method. Targets will have compiler finding strategies """ - def __init__(self,name, module_name=None): + def __init__(self, name, compiler_strategy, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 + self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge def set_architecture(self, architecture): # Target should get the architecture class. self.architecture = architecture - @property - def compiler_strategy(self): - if self.module_name: # If there is a module_name given then use MODULES - return "MODULES" - else: - return "PATH" +# @property +# def compiler_strategy(self): +# if self.module_name: # If there is a module_name given then use MODULES +# return "MODULES" +# else: +# return "PATH" def to_dict(self): d = {} diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/architectures/bgq.py index 85487626f1b..3ac5a59546a 100644 --- a/lib/spack/spack/architectures/bgq.py +++ b/lib/spack/spack/architectures/bgq.py @@ -10,8 +10,8 @@ class Bgq(Architecture): def __init__(self): super(Bgq, self).__init__('cray') - self.add_target(self.front_end, Target(self.front_end)) - self.add_target(self.back_end, Target(self.back_end)) + self.add_target(self.front_end, Target(self.front_end, 'PATH')) + self.add_target(self.back_end, Target(self.back_end, 'PATH')) @classmethod def detect(self): diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/architectures/cray.py index dac3943fb73..f109b488673 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/architectures/cray.py @@ -26,10 +26,10 @@ def __init__(self): # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.front_end, Target(self.front_end, 'PATH')) # Back End compiler needs the proper target module loaded. - self.add_target(self.back_end, Target(self.front_end,'craype-'+ self.back_end)) - self.add_target(self.default, Target(self.default,'craype-' + self.default)) +# self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) + self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) # This is kludgy and the order matters when the targets are all haswell # This is because the last one overwrites the others when they have the # same name. diff --git a/lib/spack/spack/architectures/darwin.py b/lib/spack/spack/architectures/darwin.py index 2f0d34c38d3..30fbde39bb0 100644 --- a/lib/spack/spack/architectures/darwin.py +++ b/lib/spack/spack/architectures/darwin.py @@ -9,7 +9,7 @@ class Darwin(Architecture): def __init__(self): super(Darwin, self).__init__('darwin') - self.add_target(self.default, Target(self.default)) + self.add_target(self.default, Target(self.default, 'PATH')) @classmethod def detect(self): diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/architectures/linux.py index d63cf9179a9..bb3089ebf66 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/architectures/linux.py @@ -9,7 +9,7 @@ class Linux(Architecture): def __init__(self): super(Linux, self).__init__('linux') - self.add_target(self.default, Target(self.default)) + self.add_target(self.default, Target(self.default, 'PATH')) @classmethod def detect(self): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 781039e073d..fc71ae03314 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -226,7 +226,7 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.architecture.compiler_strategy.lower() == 'module': + if pkg.spec.architecture.module_name: load_module(pkg.spec.architecture.module_name) def set_module_variables_for_package(pkg): From b4a0004f44af89c034423adef5444a234c49d464 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 6 Jan 2016 14:50:31 -0800 Subject: [PATCH 081/375] Improved target cmp_key and to/from yaml functions --- lib/spack/spack/architecture.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 81aac6cc339..aaea7c81ac0 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -60,22 +60,17 @@ def __init__(self, name, compiler_strategy, module_name=None): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - def set_architecture(self, architecture): # Target should get the architecture class. - self.architecture = architecture - -# @property -# def compiler_strategy(self): -# if self.module_name: # If there is a module_name given then use MODULES -# return "MODULES" -# else: -# return "PATH" + # Sets only the architecture name to avoid recursiveness + def set_architecture(self, architecture): + self.architecture_name = architecture.name def to_dict(self): d = {} d['name'] = self.name + d['compiler_strategy'] = self.compiler_strategy d['module_name'] = self.module_name -# if self.architecture: -# d['architecture'] = self.architecture + if self.architecture_name: + d['architecture'] = self.architecture_name return d @staticmethod @@ -84,14 +79,15 @@ def from_dict(d): return None target = Target.__new__(Target) target.name = d['name'] + target.compiler_strategy = d['compiler_strategy'] target.module_name = d['module_name'] -# if 'architecture' in d: -# target.architecture = d['architecture'] + if 'architecture' in d: + target.architecture_name = d['architecture'] return target def _cmp_key(self): - return (self.name, self.module_name) + return (self.name, self.compiler_strategy, self.module_name) def __repr__(self): return self.__str__() From 6ccd9d6fa4baa9b14e0cea19e55b24ad19d053f0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 11:49:01 -0800 Subject: [PATCH 082/375] Cleaned up naming conventions for architecture, split into platform and target --- lib/spack/spack/__init__.py | 2 +- lib/spack/spack/abi.py | 8 +- lib/spack/spack/architecture.py | 62 +++++----- lib/spack/spack/build_environment.py | 4 +- lib/spack/spack/cmd/find.py | 10 +- lib/spack/spack/concretize.py | 40 +++---- lib/spack/spack/config.py | 42 +++---- lib/spack/spack/directory_layout.py | 4 +- lib/spack/spack/modules.py | 4 +- lib/spack/spack/package.py | 2 +- .../{architectures => platforms}/__init__.py | 0 .../spack/{architectures => platforms}/bgq.py | 4 +- .../{architectures => platforms}/cray.py | 4 +- .../{architectures => platforms}/darwin.py | 10 +- .../{architectures => platforms}/linux.py | 10 +- lib/spack/spack/preferred_packages.py | 10 +- lib/spack/spack/spec.py | 108 +++++++++--------- lib/spack/spack/test/architecture.py | 23 ++-- lib/spack/spack/test/concretize.py | 4 +- lib/spack/spack/test/multimethod.py | 12 +- lib/spack/spack/test/spec_dag.py | 4 +- lib/spack/spack/test/spec_semantics.py | 4 +- .../mock_packages/multimethod/package.py | 10 +- 23 files changed, 191 insertions(+), 190 deletions(-) rename lib/spack/spack/{architectures => platforms}/__init__.py (100%) rename lib/spack/spack/{architectures => platforms}/bgq.py (84%) rename lib/spack/spack/{architectures => platforms}/cray.py (95%) rename lib/spack/spack/{architectures => platforms}/darwin.py (54%) rename lib/spack/spack/{architectures => platforms}/linux.py (54%) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index b2ff2c692bb..e1ef094f17c 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -37,7 +37,7 @@ lib_path = join_path(prefix, "lib", "spack") build_env_path = join_path(lib_path, "env") module_path = join_path(lib_path, "spack") -arch_path = join_path(module_path, 'architectures') +platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index f0a997703cd..1dd49d6b2c4 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -34,9 +34,9 @@ class ABI(object): """This class provides methods to test ABI compatibility between specs. The current implementation is rather rough and could be improved.""" - def architecture_compatible(self, parent, child): - """Returns true iff the parent and child specs have ABI compatible architectures.""" - return not parent.architecture or not child.architecture or parent.architecture == child.architecture + def target_compatible(self, parent, child): + """Returns true iff the parent and child specs have ABI compatible targets.""" + return not parent.target or not child.target or parent.target == child.target @memoized @@ -123,6 +123,6 @@ def compiler_compatible(self, parent, child, **kwargs): def compatible(self, parent, child, **kwargs): """Returns true iff a parent and child spec are ABI compatible""" loosematch = kwargs.get('loose', False) - return self.architecture_compatible(parent, child) and \ + return self.target_compatible(parent, child) and \ self.compiler_compatible(parent, child, loose=loosematch) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index aaea7c81ac0..a269767fab5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -51,8 +51,8 @@ def __init__(self): class Target(object): """ Target is the processor of the host machine. The host machine may have different front-end and back-end targets, especially if it is a Cray machine. The target will have a name and - also the module_name (e.g craype-compiler). Targets will also recognize which architecture - they came from using the set_architecture method. Targets will have compiler finding strategies + also the module_name (e.g craype-compiler). Targets will also recognize which platform + they came from using the set_platform method. Targets will have compiler finding strategies """ def __init__(self, name, compiler_strategy, module_name=None): @@ -60,17 +60,17 @@ def __init__(self, name, compiler_strategy, module_name=None): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - # Sets only the architecture name to avoid recursiveness - def set_architecture(self, architecture): - self.architecture_name = architecture.name + # Sets only the platform name to avoid recursiveness + def set_platform(self, platform): + self.platform_name = platform.name def to_dict(self): d = {} d['name'] = self.name d['compiler_strategy'] = self.compiler_strategy d['module_name'] = self.module_name - if self.architecture_name: - d['architecture'] = self.architecture_name + if self.platform_name: + d['platform'] = self.platform_name return d @staticmethod @@ -81,8 +81,8 @@ def from_dict(d): target.name = d['name'] target.compiler_strategy = d['compiler_strategy'] target.module_name = d['module_name'] - if 'architecture' in d: - target.architecture_name = d['architecture'] + if 'platform' in d: + target.platform_name = d['platform'] return target @@ -96,13 +96,13 @@ def __str__(self): return self.name @key_ordering -class Architecture(object): - """ Abstract class that each type of Architecture will subclass. +class Platform(object): + """ Abstract class that each type of Platform will subclass. Will return a instance of it once it is returned """ - priority = None # Subclass needs to set this number. This controls order in which arch is detected. + priority = None # Subclass needs to set this number. This controls order in which platform is detected. front_end = None back_end = None default = None # The default back end target. On cray ivybridge @@ -112,12 +112,12 @@ def __init__(self, name): self.name = name def add_target(self, name, target): - """Used by the architecture specific subclass to list available targets. Raises an error - if the architecture specifies a name that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. Raises an error + if the platform specifies a name that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) - target.set_architecture(self) + target.set_platform(self) self.targets[name] = target def target(self, name): @@ -137,7 +137,7 @@ def target(self, name): @classmethod def detect(self): """ Subclass is responsible for implementing this method. - Returns True if the architecture detects if it is the current architecture + Returns True if the Platform class detects that it is the current platform and False if it's not. """ raise NotImplementedError() @@ -182,18 +182,18 @@ def get_sys_type_from_uname(): Front-end config """ try: - arch_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) - arch, _ = arch_proc.communicate() - return arch.strip() + platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + platform, _ = platform_proc.communicate() + return platform.strip() except: return None @memoized -def all_architectures(): +def all_platforms(): modules = [] - for name in list_modules(spack.arch_path): - mod_name = 'spack.architectures' + name - path = join_path(spack.arch_path, name) + ".py" + for name in list_modules(spack.platform_path): + mod_name = 'spack.platformss' + name + path = join_path(spack.platform_path, name) + ".py" mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) if not hasattr(mod, class_name): @@ -208,16 +208,16 @@ def all_architectures(): @memoized def sys_type(): - """ Gather a list of all available subclasses of architectures. + """ Gather a list of all available subclasses of platforms. Sorts the list according to their priority looking. Priority is - an arbitrarily set number. Detects arch either using uname or + an arbitrarily set number. Detects platform either using uname or a file path (/opt/cray...) """ - # Try to create an architecture object using the config file FIRST - architecture_list = all_architectures() - architecture_list.sort(key = lambda a: a.priority) + # Try to create a Platform object using the config file FIRST + platform_list = all_platforms() + platform_list.sort(key = lambda a: a.priority) - for arch in architecture_list: - if arch.detect(): - return arch() + for platform in platform_list: + if platform.detect(): + return platform() diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index fc71ae03314..d849b4b56c3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -226,8 +226,8 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.architecture.module_name: - load_module(pkg.spec.architecture.module_name) + if pkg.spec.target.module_name: + load_module(pkg.spec.target.module_name) def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 3c993990b14..7ea16bf09f3 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -72,19 +72,19 @@ def display_specs(specs, **kwargs): hashes = True hlen = None - # Make a dict with specs keyed by architecture and compiler. - index = index_by(specs, ('architecture', 'compiler')) + # Make a dict with specs keyed by target and compiler. + index = index_by(specs, ('target', 'compiler')) # Traverse the index and print out each package - for i, (architecture, compiler) in enumerate(sorted(index)): + for i, (target, compiler) in enumerate(sorted(index)): if i > 0: print header = "%s{%s} / %s{%s}" % ( - spack.spec.architecture_color, architecture, + spack.spec.target_color, target, spack.spec.compiler_color, compiler) tty.hline(colorize(header), char='-') - specs = index[(architecture,compiler)] + specs = index[(target,compiler)] specs.sort() abbreviated = [s.format('$_$@$+', color=True) for s in specs] diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 854db949e31..fcd23a60550 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -208,30 +208,30 @@ def concretize_version(self, spec): return True # Things changed - def concretize_architecture(self, spec): - """If the spec already has an architecture and it is a an architecture type, - return. Otherwise, if it has an architecture that is a string type, generate an - architecture based on that type. If it has no architecture and the root of the - DAG has an architecture, then use that. Otherwise, take the system's default - architecture. + def concretize_target(self, spec): + """If the spec already has an target and it is a an target type, + return. Otherwise, if it has a target that is a string type, generate a + target based on that type. If it has no target and the root of the + DAG has an target, then use that. Otherwise, take the system's default + target. """ - if spec.architecture is not None: - if isinstance(spec.architecture,spack.architecture.Target): + if spec.target is not None: + if isinstance(spec.target,spack.architecture.Target): return False else: - arch = spack.architecture.sys_type() - spec.architecture = arch.target(spec.architecture) + platform = spack.architecture.sys_type() + spec.target = platform.target(spec.target) return True #changed - if spec.root.architecture: - if isinstance(spec.root.architecture,spack.architecture.Target): - spec.architecture = spec.root.architecture + if spec.root.target: + if isinstance(spec.root.target,spack.architecture.Target): + spec.target = spec.root.target else: - arch = spack.architecture.sys_type() - spec.architecture = arch.target(spec.root.architecture) + platform = spack.architecture.sys_type() + spec.target = platform.target(spec.root.target) else: - arch = spack.architecture.sys_type() - spec.architecture = arch.target('default') + platform = spack.architecture.sys_type() + spec.target = platform.target('default') return True #changed @@ -260,8 +260,8 @@ def concretize_compiler(self, spec): build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ - # Pass on concretizing the compiler if the architecture is not yet determined - if not spec.architecture: + # Pass on concretizing the compiler if the target is not yet determined + if not spec.target: #Although this usually means changed, this means awaiting other changes return True @@ -305,7 +305,7 @@ def _proper_compiler_style(cspec, target): # copy concrete version into other_compiler index = len(matches)-1 - while not _proper_compiler_style(matches[index], spec.architecture): + while not _proper_compiler_style(matches[index], spec.target): index -= 1 if index == 0: raise NoValidVersionError(spec) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 6e11cfa4753..0622606a4f5 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -129,7 +129,7 @@ def __init__(self, n, f, m): config_scopes = [('site', os.path.join(spack.etc_path, 'spack')), ('user', os.path.expanduser('~/.spack'))] -_compiler_by_arch = {} +_compiler_by_platform = {} _read_config_file_result = {} def _read_config_file(filename): """Read a given YAML configuration file""" @@ -156,7 +156,7 @@ def clear_config_caches(): s.files_read_from = [] s.result_dict = {} spack.config._read_config_file_result = {} - spack.config._compiler_by_arch = {} + spack.config._compiler_by_platform = {} spack.compilers._cached_default_compiler = None @@ -213,27 +213,27 @@ def get_config(category_name): return category.result_dict -def get_compilers_config(arch=None): +def get_compilers_config(platform=None): """Get the compiler configuration from config files for the given - architecture. Strips off the architecture component of the + platform. Strips off the platform component of the configuration""" - global _compiler_by_arch - if not arch: - arch = str(spack.architecture.sys_type()) - if arch in _compiler_by_arch: - return _compiler_by_arch[arch] + global _compiler_by_platform + if not platform: + platform = str(spack.architecture.sys_type()) + if platform in _compiler_by_platform: + return _compiler_by_platform[platform] cc_config = get_config('compilers') - if arch in cc_config and 'all' in cc_config: - arch_compiler = dict(cc_config[arch]) - _compiler_by_arch[arch] = _merge_dict(arch_compiler, cc_config['all']) - elif arch in cc_config: - _compiler_by_arch[arch] = cc_config[arch] + if platform in cc_config and 'all' in cc_config: + platform_compiler = dict(cc_config[platform]) + _compiler_by_platform[platform] = _merge_dict(platform_compiler, cc_config['all']) + elif platform in cc_config: + _compiler_by_platform[platform] = cc_config[platform] elif 'all' in cc_config: - _compiler_by_arch[arch] = cc_config['all'] + _compiler_by_platform[platform] = cc_config['all'] else: - _compiler_by_arch[arch] = {} - return _compiler_by_arch[arch] + _compiler_by_platform[platform] = {} + return _compiler_by_platform[platform] def get_mirror_config(): @@ -371,11 +371,11 @@ def add_to_mirror_config(addition_dict, scope=None): add_to_config('mirrors', addition_dict, scope) -def add_to_compiler_config(addition_dict, scope=None, arch=None): +def add_to_compiler_config(addition_dict, scope=None, platform=None): """Add compilers to the configuration files""" - if not arch: - arch = spack.architecture.sys_type() - add_to_config('compilers', { str(arch) : addition_dict }, scope) + if not platform: + platform = spack.architecture.sys_type() + add_to_config('compilers', { str(platform) : addition_dict }, scope) clear_config_caches() diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 83e6eb566a7..da3441fa139 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -156,7 +156,7 @@ def remove_install_directory(self, spec): class YamlDirectoryLayout(DirectoryLayout): """Lays out installation directories like this:: / - / + / -/ --- @@ -201,7 +201,7 @@ def relative_path_for_spec(self, spec): spec.dag_hash(self.hash_len)) path = join_path( - spec.architecture, + spec.target, "%s-%s" % (spec.compiler.name, spec.compiler.version), dir_name) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index 56a61adefb9..f84ac75c773 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -193,7 +193,7 @@ class Dotkit(EnvModule): @property def file_name(self): - return join_path(Dotkit.path, self.spec.architecture, + return join_path(Dotkit.path, self.spec.target, self.spec.format('$_$@$%@$+$#.dk')) @property @@ -230,7 +230,7 @@ class TclModule(EnvModule): @property def file_name(self): - return join_path(TclModule.path, self.spec.architecture, self.use_name) + return join_path(TclModule.path, self.spec.target, self.use_name) @property diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index e0097fa88d6..1148e4ac3df 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -592,7 +592,7 @@ def compiler(self): """Get the spack.compiler.Compiler object used to build this package.""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.architecture) + return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.target) def url_version(self, version): diff --git a/lib/spack/spack/architectures/__init__.py b/lib/spack/spack/platforms/__init__.py similarity index 100% rename from lib/spack/spack/architectures/__init__.py rename to lib/spack/spack/platforms/__init__.py diff --git a/lib/spack/spack/architectures/bgq.py b/lib/spack/spack/platforms/bgq.py similarity index 84% rename from lib/spack/spack/architectures/bgq.py rename to lib/spack/spack/platforms/bgq.py index 3ac5a59546a..988e67023c3 100644 --- a/lib/spack/spack/architectures/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -1,8 +1,8 @@ import os -from spack.architecture import Architecture, Target +from spack.architecture import Platform, Target -class Bgq(Architecture): +class Bgq(Platform): priority = 30 front_end = 'power7' back_end = 'powerpc' diff --git a/lib/spack/spack/architectures/cray.py b/lib/spack/spack/platforms/cray.py similarity index 95% rename from lib/spack/spack/architectures/cray.py rename to lib/spack/spack/platforms/cray.py index f109b488673..815a9aea98e 100644 --- a/lib/spack/spack/architectures/cray.py +++ b/lib/spack/spack/platforms/cray.py @@ -1,8 +1,8 @@ import os -from spack.architecture import Architecture, Target +from spack.architecture import Platform, Target -class Cray(Architecture): +class Cray(Platform): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' diff --git a/lib/spack/spack/architectures/darwin.py b/lib/spack/spack/platforms/darwin.py similarity index 54% rename from lib/spack/spack/architectures/darwin.py rename to lib/spack/spack/platforms/darwin.py index 30fbde39bb0..52284826b12 100644 --- a/lib/spack/spack/architectures/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,7 +1,7 @@ import subprocess -from spack.architecture import Architecture, Target +from spack.architecture import Platform, Target -class Darwin(Architecture): +class Darwin(Platform): priority = 89 front_end = 'x86_64' back_end = 'x86_64' @@ -13,6 +13,6 @@ def __init__(self): @classmethod def detect(self): - arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) - arch, _ = arch.communicate() - return 'darwin' in arch.strip().lower() + platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + platform, _ = platform.communicate() + return 'darwin' in platform.strip().lower() diff --git a/lib/spack/spack/architectures/linux.py b/lib/spack/spack/platforms/linux.py similarity index 54% rename from lib/spack/spack/architectures/linux.py rename to lib/spack/spack/platforms/linux.py index bb3089ebf66..7f94d80c34e 100644 --- a/lib/spack/spack/architectures/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,7 +1,7 @@ import subprocess -from spack.architecture import Architecture, Target +from spack.architecture import Platform, Target -class Linux(Architecture): +class Linux(Platform): priority = 90 front_end = 'x86_64' back_end = 'x86_64' @@ -13,6 +13,6 @@ def __init__(self): @classmethod def detect(self): - arch = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) - arch, _ = arch.communicate() - return 'linux' in arch.strip().lower() + platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE) + platform, _ = platform.communicate() + return 'linux' in platform.strip().lower() diff --git a/lib/spack/spack/preferred_packages.py b/lib/spack/spack/preferred_packages.py index bc2a4ac234d..993683a62fe 100644 --- a/lib/spack/spack/preferred_packages.py +++ b/lib/spack/spack/preferred_packages.py @@ -163,11 +163,11 @@ def variant_compare(self, pkgname, a, b): return self._component_compare(pkgname, 'variant', a, b, False, None) - def architecture_compare(self, pkgname, a, b): - """Return less-than-0, 0, or greater than 0 if architecture a of pkgname is - respecively less-than, equal-to, or greater-than architecture b of pkgname. - One architecture is less-than another if it is preferred over the other.""" - return self._component_compare(pkgname, 'architecture', a, b, False, None) + def target_compare(self, pkgname, a, b): + """Return less-than-0, 0, or greater than 0 if target a of pkgname is + respecively less-than, equal-to, or greater-than target b of pkgname. + One target is less-than another if it is preferred over the other.""" + return self._component_compare(pkgname, 'target', a, b, False, None) def compiler_compare(self, pkgname, a, b): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 126ffc4f95d..9e011bfb9ff 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -63,7 +63,7 @@ if it comes immediately after the compiler name. Otherwise it will be associated with the current package spec. -6. The architecture to build with. This is needed on machines where +6. The target to build with. This is needed on machines where cross-compilation is required Here is the EBNF grammar for a spec:: @@ -72,9 +72,9 @@ dep_list = { ^ spec } spec = id [ options ] options = { @version-list | +variant | -variant | ~variant | - %compiler | =architecture } + %compiler | =target } variant = id - architecture = id + target = id compiler = id [ version-list ] version-list = version [ { , version } ] version = id | id: | :id | id:id @@ -119,7 +119,7 @@ # Convenient names for color formats so that other things can use them compiler_color = '@g' version_color = '@c' -architecture_color = '@m' +target_color = '@m' enabled_variant_color = '@B' disabled_variant_color = '@r' dependency_color = '@.' @@ -130,7 +130,7 @@ See spack.color for descriptions of the color codes. """ color_formats = {'%' : compiler_color, '@' : version_color, - '=' : architecture_color, + '=' : target_color, '+' : enabled_variant_color, '~' : disabled_variant_color, '^' : dependency_color, @@ -407,7 +407,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): self.name = other.name self.dependents = other.dependents self.versions = other.versions - self.architecture = other.architecture + self.target = other.target self.compiler = other.compiler self.dependencies = other.dependencies self.variants = other.variants @@ -452,11 +452,11 @@ def _set_compiler(self, compiler): self.compiler = compiler - def _set_architecture(self, architecture): - """Called by the parser to set the architecture.""" - if self.architecture: raise DuplicateArchitectureError( - "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture + def _set_target(self, target): + """Called by the parser to set the target.""" + if self.target: raise DuplicateTargetError( + "Spec for '%s' cannot have two targets." % self.name) + self.target = target def _add_dependency(self, spec): @@ -512,7 +512,7 @@ def is_virtual(name): @property def concrete(self): """A spec is concrete if it can describe only ONE build of a package. - If any of the name, version, architecture, compiler, + If any of the name, version, target, compiler, variants, or depdenencies are ambiguous,then it is not concrete. """ if self._concrete: @@ -521,7 +521,7 @@ def concrete(self): self._concrete = bool(not self.virtual and self.versions.concrete and self.variants.concrete - and self.architecture + and self.target and self.compiler and self.compiler.concrete and self.dependencies.concrete) return self._concrete @@ -656,10 +656,10 @@ def to_node_dict(self): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } - if self.architecture: - d['arch'] = self.architecture.to_dict() + if self.target: + d['target'] = self.target.to_dict() else: - d['arch'] = None + d['target'] = None if self.compiler: d.update(self.compiler.to_dict()) else: @@ -685,7 +685,7 @@ def from_node_dict(node): spec = Spec(name) spec.versions = VersionList.from_dict(node) - spec.architecture = spack.architecture.Target.from_dict(node['arch']) + spec.target = spack.architecture.Target.from_dict(node['target']) if node['compiler'] is None: spec.compiler = None @@ -757,7 +757,7 @@ def _concretize_helper(self, presets=None, visited=None): # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. changed |= any( - (spack.concretizer.concretize_architecture(self), + (spack.concretizer.concretize_target(self), spack.concretizer.concretize_compiler(self), spack.concretizer.concretize_version(self), spack.concretizer.concretize_variants(self))) @@ -1145,10 +1145,10 @@ def constrain(self, other, deps=True): raise UnsatisfiableVariantSpecError(self.variants[v], other.variants[v]) - if self.architecture is not None and other.architecture is not None: - if self.architecture != other.architecture: - raise UnsatisfiableArchitectureSpecError(self.architecture, - other.architecture) + if self.target is not None and other.target is not None: + if self.target != other.target: + raise UnsatisfiableTargetSpecError(self.target, + other.target) changed = False if self.compiler is not None and other.compiler is not None: @@ -1160,9 +1160,9 @@ def constrain(self, other, deps=True): changed |= self.versions.intersect(other.versions) changed |= self.variants.constrain(other.variants) - old = self.architecture - self.architecture = self.architecture or other.architecture - changed |= (self.architecture != old) + old = self.target + self.target = self.target or other.target + changed |= (self.target != old) if deps: changed |= self._constrain_dependencies(other) @@ -1273,12 +1273,12 @@ def satisfies(self, other, deps=True, strict=False): if not self.variants.satisfies(other.variants, strict=strict): return False - # Architecture satisfaction is currently just string equality. + # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. - if self.architecture and other.architecture: - if self.architecture != other.architecture: + if self.target and other.target: + if self.target != other.target: return False - elif strict and (other.architecture and not self.architecture): + elif strict and (other.target and not self.target): return False # If we need to descend into dependencies, do it, otherwise we're done. @@ -1352,7 +1352,7 @@ def _dup(self, other, **kwargs): changed = True if hasattr(self, 'name'): changed = (self.name != other.name and self.versions != other.versions and \ - self.architecture != other.architecture and self.compiler != other.compiler and \ + self.target != other.target and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external and \ self.external_module != other.external_module) @@ -1360,7 +1360,7 @@ def _dup(self, other, **kwargs): # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() - self.architecture = other.architecture + self.target = other.target self.compiler = other.compiler.copy() if other.compiler else None if kwargs.get('cleardeps', True): self.dependents = DependencyMap() @@ -1490,7 +1490,7 @@ def ne_dag(self, other): def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" return (self.name, self.versions, self.variants, - self.architecture, self.compiler) + self.target, self.compiler) def eq_node(self, other): @@ -1524,7 +1524,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): $% Compiler with '%' prefix $%@ Compiler with '%' prefix & compiler version with '@' prefix $+ Options - $= Architecture with '=' prefix + $= Target with '=' prefix $# 7-char prefix of DAG hash with '-' prefix $$ $ @@ -1536,7 +1536,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): ${COMPILERNAME} Compiler name ${COMPILERVER} Compiler version ${OPTIONS} Options - ${ARCHITECTURE} Architecture + ${TARGET} Target ${SHA1} Dependencies 8-char sha1 prefix ${SPACK_ROOT} The spack root directory @@ -1549,7 +1549,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): Anything else is copied verbatim into the output stream. *Example:* ``$_$@$+`` translates to the name, version, and options - of the package, but no dependencies, arch, or compiler. + of the package, but no dependencies, target, or compiler. TODO: allow, e.g., $6# to customize short hash length TODO: allow, e.g., $## for full hash. @@ -1593,8 +1593,8 @@ def write(s, c): if self.variants: write(fmt % str(self.variants), c) elif c == '=': - if self.architecture: - write(fmt % (c + str(self.architecture)), c) + if self.target: + write(fmt % (c + str(self.target)), c) elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) elif c == '$': @@ -1641,9 +1641,9 @@ def write(s, c): elif named_str == 'OPTIONS': if self.variants: write(fmt % str(self.variants), '+') - elif named_str == 'ARCHITECTURE': - if self.architecture: - write(fmt % str(self.architecture), '=') + elif named_str == 'TARGET': + if self.target: + write(fmt % str(self.target), '=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dep_hash(8))) @@ -1691,10 +1691,10 @@ def __cmp__(self, other): return spack.pkgsort.variant_compare(pkgname, self.variants, other.variants) - #Architecture - if self.architecture != other.architecture: - return spack.pkgsort.architecture_compare(pkgname, - self.architecture, other.architecture) + #Target + if self.target != other.target: + return spack.pkgsort.target_compare(pkgname, + self.target, other.target) #Dependency is not configurable if self.dep_hash() != other.dep_hash(): @@ -1811,7 +1811,7 @@ def spec(self): spec.name = self.token.value spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.architecture = None + spec.target = None spec.compiler = None spec.external = None spec.external_module = None @@ -1842,7 +1842,7 @@ def spec(self): spec._set_compiler(self.compiler()) elif self.accept(EQ): - spec._set_architecture(self.architecture()) + spec._set_target(self.target()) else: break @@ -1860,7 +1860,7 @@ def variant(self): return self.token.value - def architecture(self): + def target(self): self.expect(ID) return self.token.value @@ -2000,10 +2000,10 @@ def __init__(self, pkg, variant): "Package %s has no variant %s!" % (pkg, variant)) -class DuplicateArchitectureError(SpecError): - """Raised when the same architecture occurs in a spec twice.""" +class DuplicateTargetError(SpecError): + """Raised when the same target occurs in a spec twice.""" def __init__(self, message): - super(DuplicateArchitectureError, self).__init__(message) + super(DuplicateTargetError, self).__init__(message) class InconsistentSpecError(SpecError): @@ -2082,11 +2082,11 @@ def __init__(self, provided, required): provided, required, "variant") -class UnsatisfiableArchitectureSpecError(UnsatisfiableSpecError): - """Raised when a spec architecture conflicts with package constraints.""" +class UnsatisfiableTargetSpecError(UnsatisfiableSpecError): + """Raised when a spec target conflicts with package constraints.""" def __init__(self, provided, required): - super(UnsatisfiableArchitectureSpecError, self).__init__( - provided, required, "architecture") + super(UnsatisfiableTargetSpecError, self).__init__( + provided, required, "target") class UnsatisfiableProviderSpecError(UnsatisfiableSpecError): diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 3a7474799aa..32b16feeb14 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,13 +6,14 @@ import platform import spack from spack.architecture import * -from spack.architectures.cray import Cray -from spack.architectures.linux import Linux -from spack.architectures.bgq import Bgq +from spack.platforms.cray import Cray +from spack.platforms.linux import Linux +from spack.platforms.bgq import Bgq +from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): - def test_Architecture_class_and_compiler_strategies(self): + def test_platform_class_and_compiler_strategies(self): a = Cray() t = a.target('default') self.assertEquals(t.compiler_strategy, 'MODULES') @@ -21,15 +22,15 @@ def test_Architecture_class_and_compiler_strategies(self): self.assertEquals(s.compiler_strategy, 'PATH') def test_sys_type(self): - output_arch_class = sys_type() + output_platform_class = sys_type() my_arch_class = None if os.path.exists('/opt/cray/craype'): - my_arch_class = Cray() + my_platform_class = Cray() elif os.path.exists('/bgsys'): - my_arch_class = Bgq() + my_platform_class = Bgq() elif 'Linux' in platform.system(): - my_arch_class = Linux() -# elif 'Darwin' in platform.system(): -# my_arch_class = Darwin() + my_platform_class = Linux() + elif 'Darwin' in platform.system(): + my_platform_class = Darwin() - self.assertEqual(str(output_arch_class), str(my_arch_class)) + self.assertEqual(str(output_platform_class), str(my_platform_class)) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index f81a2f5af81..24037191347 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -46,8 +46,8 @@ def check_spec(self, abstract, concrete): if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) - if abstract.architecture and abstract.architecture.concrete: - self.assertEqual(abstract.architecture, concrete.architecture) + if abstract.target and abstract.target.concrete: + self.assertEqual(abstract.target, concrete.target) def check_concretize(self, abstract_spec): diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index cd5d9e625e3..741dc960721 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -91,21 +91,21 @@ def test_default_works(self): self.assertEqual(pkg.has_a_default(), 'default') - def test_architecture_match(self): + def test_target_match(self): pkg = spack.db.get('multimethod=x86_64') - self.assertEqual(pkg.different_by_architecture(), 'x86_64') + self.assertEqual(pkg.different_by_target(), 'x86_64') pkg = spack.db.get('multimethod=ppc64') - self.assertEqual(pkg.different_by_architecture(), 'ppc64') + self.assertEqual(pkg.different_by_target(), 'ppc64') pkg = spack.db.get('multimethod=ppc32') - self.assertEqual(pkg.different_by_architecture(), 'ppc32') + self.assertEqual(pkg.different_by_target(), 'ppc32') pkg = spack.db.get('multimethod=arm64') - self.assertEqual(pkg.different_by_architecture(), 'arm64') + self.assertEqual(pkg.different_by_target(), 'arm64') pkg = spack.db.get('multimethod=macos') - self.assertRaises(NoSuchMethodError, pkg.different_by_architecture) + self.assertRaises(NoSuchMethodError, pkg.different_by_target) def test_dependency_match(self): diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 549f829d3eb..6a2dd6140fa 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -240,10 +240,10 @@ def test_unsatisfiable_compiler_version(self): self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) - def test_unsatisfiable_architecture(self): + def test_unsatisfiable_target(self): set_pkg_dep('mpileaks', 'mpich=bgqos_0') spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) + self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) def test_invalid_dep(self): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 6666dbbb52c..b8b4fb951cc 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -110,7 +110,7 @@ def test_satisfies_compiler_version(self): self.check_unsatisfiable('foo %gcc@4.7', '%gcc@4.7.3') - def test_satisfies_architecture(self): + def test_satisfies_target(self): self.check_satisfies('foo=chaos_5_x86_64_ib', '=chaos_5_x86_64_ib') self.check_satisfies('foo=bgqos_0', '=bgqos_0') @@ -266,7 +266,7 @@ def test_constrain_variants(self): self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo') - def test_constrain_arch(self): + def test_constrain_target(self): self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 75b1606ffc6..f78ef3bb3dd 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -101,22 +101,22 @@ def has_a_default(self): # - # Make sure we can switch methods on different architectures + # Make sure we can switch methods on different target # @when('=x86_64') - def different_by_architecture(self): + def different_by_target(self): return 'x86_64' @when('=ppc64') - def different_by_architecture(self): + def different_by_target(self): return 'ppc64' @when('=ppc32') - def different_by_architecture(self): + def different_by_target(self): return 'ppc32' @when('=arm64') - def different_by_architecture(self): + def different_by_target(self): return 'arm64' From 83917c4c302851a0d4ff91ef652fdd1b26fb1e08 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 12:43:39 -0800 Subject: [PATCH 083/375] Improved target specification --- lib/spack/spack/architecture.py | 2 ++ lib/spack/spack/concretize.py | 38 ++++++++++++++++++++++++++++---- lib/spack/spack/platforms/bgq.py | 2 +- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a269767fab5..6b9ae33b3e3 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -93,6 +93,8 @@ def __repr__(self): return self.__str__() def __str__(self): + if self.platform_name: + return self.platform_name + '-' + self.name return self.name @key_ordering diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index fcd23a60550..ce867860041 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,11 +33,13 @@ TODO: make this customizable and allow users to configure concretization policies. """ +from llnl.util.filesystem import join_path import spack import spack.spec import spack.compilers import spack.architecture import spack.error +from spack.util.naming import mod_to_class from spack.version import * from functools import partial from spec import DependencyMap @@ -207,6 +209,36 @@ def concretize_version(self, spec): return True # Things changed + def class_from_platform_name(self, platform_name): + file_path = join_path(spack.platform_path, platform_name) + platform_mod = imp.load_source('spack.platforms', file_path + '.py') + cls = getattr(platform_mod, mod_to_class(platform_name)) + + return cls + + def spec_add_target_from_string(self, spec, target): + """If only a target is provided, spack will assume the default architecture. + A platform-target pair can be input delimited by a '-'. If either portion of + a platform-target pair is empty, spack will supply a default, in the case of + a blank target the default will be dependent on the platform. + E.g. x86_64 -> 64 bit x86 + bgq- -> default bgq target (back end/powerpc) + cray-hawswell -> haswell target on cray platform + """ + if '-' in target: + platform, target = target.split('-') + else: + platform = '' + + if platform != '': + cls = self.class_from_platform_name(platform) + platform = cls() + else: + platform = spack.architecture.sys_type() + if target != '': + spec.target = platform.target(target) + else: + spec.target = platform.target('default') def concretize_target(self, spec): """If the spec already has an target and it is a an target type, @@ -219,16 +251,14 @@ def concretize_target(self, spec): if isinstance(spec.target,spack.architecture.Target): return False else: - platform = spack.architecture.sys_type() - spec.target = platform.target(spec.target) + self.spec_add_target_from_string(spec, spec.target) return True #changed if spec.root.target: if isinstance(spec.root.target,spack.architecture.Target): spec.target = spec.root.target else: - platform = spack.architecture.sys_type() - spec.target = platform.target(spec.root.target) + self.spec_add_target_from_string(spec, spec.root.target) else: platform = spack.architecture.sys_type() spec.target = platform.target('default') diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py index 988e67023c3..6e872d2e729 100644 --- a/lib/spack/spack/platforms/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -9,7 +9,7 @@ class Bgq(Platform): default = 'powerpc' def __init__(self): - super(Bgq, self).__init__('cray') + super(Bgq, self).__init__('bgq') self.add_target(self.front_end, Target(self.front_end, 'PATH')) self.add_target(self.back_end, Target(self.back_end, 'PATH')) From 53d4f82ce1863d0871f53a7e20b75d73b8d4fad6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 13:01:05 -0800 Subject: [PATCH 084/375] Improved cray architecture class --- lib/spack/spack/platforms/{cray.py => cray_xc.py} | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) rename lib/spack/spack/platforms/{cray.py => cray_xc.py} (73%) diff --git a/lib/spack/spack/platforms/cray.py b/lib/spack/spack/platforms/cray_xc.py similarity index 73% rename from lib/spack/spack/platforms/cray.py rename to lib/spack/spack/platforms/cray_xc.py index 815a9aea98e..e3adb182ead 100644 --- a/lib/spack/spack/platforms/cray.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -2,7 +2,7 @@ from spack.architecture import Platform, Target -class Cray(Platform): +class CrayXc(Platform): priority = 20 front_end = 'sandybridge' back_end = 'ivybridge' @@ -13,7 +13,7 @@ def __init__(self): if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' - super(Cray, self).__init__('cray') + super(CrayXc, self).__init__('cray_xc') # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: @@ -24,12 +24,17 @@ def __init__(self): self.front_end = self.default self.back_end = self.default + # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target(self.front_end, Target(self.front_end, 'PATH')) + self.add_target('sandybridge', Target('sandybridge', 'PATH')) + self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) + self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) + +# self.add_target(self.front_end, Target(self.front_end, 'PATH')) # Back End compiler needs the proper target module loaded. # self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) - self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) +# self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) # This is kludgy and the order matters when the targets are all haswell # This is because the last one overwrites the others when they have the # same name. From ba63111f453e258a0231a285dcbb11a876cccc19 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 7 Jan 2016 13:05:58 -0800 Subject: [PATCH 085/375] brought the architecture test up to date with new changes --- lib/spack/spack/test/architecture.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 32b16feeb14..cf7938f5d69 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,7 +6,7 @@ import platform import spack from spack.architecture import * -from spack.platforms.cray import Cray +from spack.platforms.cray_xc import CrayXc from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin @@ -14,7 +14,7 @@ class ArchitectureTest(unittest.TestCase): def test_platform_class_and_compiler_strategies(self): - a = Cray() + a = CrayXc() t = a.target('default') self.assertEquals(t.compiler_strategy, 'MODULES') b = Linux() @@ -25,7 +25,7 @@ def test_sys_type(self): output_platform_class = sys_type() my_arch_class = None if os.path.exists('/opt/cray/craype'): - my_platform_class = Cray() + my_platform_class = CrayXc() elif os.path.exists('/bgsys'): my_platform_class = Bgq() elif 'Linux' in platform.system(): From 49908386ef5d14be5cb34adb2040a7f2d535019c Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 11:48:59 -0800 Subject: [PATCH 086/375] changing problematic package.py file permissions back to 0644 --- var/spack/packages/ImageMagick/package.py | 0 var/spack/packages/Mitos/package.py | 0 var/spack/packages/R/package.py | 0 var/spack/packages/SAMRAI/package.py | 0 var/spack/packages/adept-utils/package.py | 0 var/spack/packages/adios/package.py | 5 +++-- var/spack/packages/arpack/package.py | 0 var/spack/packages/atk/package.py | 0 var/spack/packages/atlas/package.py | 0 var/spack/packages/autoconf/package.py | 0 var/spack/packages/automaded/package.py | 0 var/spack/packages/automake/package.py | 0 var/spack/packages/bear/package.py | 0 var/spack/packages/bib2xhtml/package.py | 0 var/spack/packages/binutils/package.py | 0 var/spack/packages/bison/package.py | 0 var/spack/packages/boost/package.py | 0 var/spack/packages/boxlib/package.py | 0 var/spack/packages/bzip2/package.py | 0 var/spack/packages/cairo/package.py | 0 var/spack/packages/callpath/package.py | 0 var/spack/packages/cblas/package.py | 0 var/spack/packages/cgm/package.py | 0 var/spack/packages/clang/package.py | 0 var/spack/packages/cloog/package.py | 0 var/spack/packages/cmake/package.py | 0 var/spack/packages/coreutils/package.py | 0 var/spack/packages/cppcheck/package.py | 0 var/spack/packages/cram/package.py | 0 var/spack/packages/cube/package.py | 0 var/spack/packages/dbus/package.py | 0 var/spack/packages/dri2proto/package.py | 0 var/spack/packages/dtcmp/package.py | 0 var/spack/packages/dyninst/package.py | 0 var/spack/packages/extrae/package.py | 0 var/spack/packages/exuberant-ctags/package.py | 0 var/spack/packages/flex/package.py | 0 var/spack/packages/fontconfig/package.py | 0 var/spack/packages/freetype/package.py | 0 var/spack/packages/gasnet/package.py | 0 var/spack/packages/gcc/package.py | 0 var/spack/packages/gdk-pixbuf/package.py | 0 var/spack/packages/geos/package.py | 0 var/spack/packages/git/package.py | 0 var/spack/packages/glib/package.py | 0 var/spack/packages/global/package.py | 0 var/spack/packages/gmp/package.py | 0 var/spack/packages/gnutls/package.py | 0 var/spack/packages/gperf/package.py | 0 var/spack/packages/gperftools/package.py | 0 var/spack/packages/graphlib/package.py | 0 var/spack/packages/gtkplus/package.py | 0 var/spack/packages/harfbuzz/package.py | 0 var/spack/packages/hdf5/package.py | 0 var/spack/packages/hwloc/package.py | 0 var/spack/packages/hypre/package.py | 0 var/spack/packages/icu/package.py | 0 var/spack/packages/icu4c/package.py | 0 var/spack/packages/isl/package.py | 0 var/spack/packages/jdk/package.py | 0 var/spack/packages/jpeg/package.py | 0 var/spack/packages/lapack/package.py | 0 var/spack/packages/launchmon/package.py | 0 var/spack/packages/lcms/package.py | 0 var/spack/packages/libNBC/package.py | 0 var/spack/packages/libarchive/package.py | 0 var/spack/packages/libcircle/package.py | 0 var/spack/packages/libdrm/package.py | 0 var/spack/packages/libdwarf/package.py | 0 var/spack/packages/libelf/package.py | 0 var/spack/packages/libevent/package.py | 0 var/spack/packages/libffi/package.py | 0 var/spack/packages/libgcrypt/package.py | 0 var/spack/packages/libgpg-error/package.py | 0 var/spack/packages/libjpeg-turbo/package.py | 0 var/spack/packages/libmng/package.py | 0 var/spack/packages/libmonitor/package.py | 0 var/spack/packages/libpciaccess/package.py | 0 var/spack/packages/libpng/package.py | 0 var/spack/packages/libtiff/package.py | 0 var/spack/packages/libtool/package.py | 0 var/spack/packages/libunwind/package.py | 0 var/spack/packages/libuuid/package.py | 0 var/spack/packages/libxcb/package.py | 0 var/spack/packages/libxml2/package.py | 0 var/spack/packages/libxshmfence/package.py | 0 var/spack/packages/libxslt/package.py | 0 var/spack/packages/llvm-lld/package.py | 0 var/spack/packages/llvm/package.py | 0 var/spack/packages/lwgrp/package.py | 0 var/spack/packages/lwm2/package.py | 0 var/spack/packages/memaxes/package.py | 0 var/spack/packages/mesa/package.py | 0 var/spack/packages/metis/package.py | 0 var/spack/packages/mpc/package.py | 0 var/spack/packages/mpe2/package.py | 0 var/spack/packages/mpfr/package.py | 0 var/spack/packages/mpibash/package.py | 0 var/spack/packages/mpich/package.py | 0 var/spack/packages/mpileaks/package.py | 0 var/spack/packages/mrnet/package.py | 0 var/spack/packages/muster/package.py | 0 var/spack/packages/mvapich2/package.py | 0 var/spack/packages/nasm/package.py | 0 var/spack/packages/ncurses/package.py | 0 var/spack/packages/netcdf/package.py | 0 var/spack/packages/netgauge/package.py | 0 var/spack/packages/netlib-blas/package.py | 0 var/spack/packages/nettle/package.py | 0 var/spack/packages/ompss/package.py | 0 var/spack/packages/opari2/package.py | 0 var/spack/packages/openmpi/package.py | 0 var/spack/packages/openssl/package.py | 0 var/spack/packages/otf/package.py | 0 var/spack/packages/otf2/package.py | 0 var/spack/packages/pango/package.py | 0 var/spack/packages/papi/package.py | 0 var/spack/packages/paraver/package.py | 0 var/spack/packages/parmetis/package.py | 0 var/spack/packages/parpack/package.py | 0 var/spack/packages/pcre/package.py | 0 var/spack/packages/petsc/package.py | 0 var/spack/packages/pixman/package.py | 0 var/spack/packages/pmgr_collective/package.py | 0 var/spack/packages/postgresql/package.py | 0 var/spack/packages/ppl/package.py | 0 var/spack/packages/py-basemap/package.py | 0 var/spack/packages/py-biopython/package.py | 0 var/spack/packages/py-cffi/package.py | 0 var/spack/packages/py-cython/package.py | 0 var/spack/packages/py-dateutil/package.py | 0 var/spack/packages/py-epydoc/package.py | 0 var/spack/packages/py-genders/package.py | 0 var/spack/packages/py-gnuplot/package.py | 0 var/spack/packages/py-h5py/package.py | 0 var/spack/packages/py-ipython/package.py | 0 var/spack/packages/py-jsonschema/package.py | 0 var/spack/packages/py-libxml2/package.py | 0 var/spack/packages/py-lockfile/package.py | 0 var/spack/packages/py-mako/package.py | 0 var/spack/packages/py-matplotlib/package.py | 0 var/spack/packages/py-mpi4py/package.py | 0 var/spack/packages/py-mx/package.py | 0 var/spack/packages/py-nose/package.py | 0 var/spack/packages/py-numpy/package.py | 0 var/spack/packages/py-pandas/package.py | 0 var/spack/packages/py-pexpect/package.py | 0 var/spack/packages/py-pil/package.py | 0 var/spack/packages/py-pmw/package.py | 0 var/spack/packages/py-py4j/package.py | 0 var/spack/packages/py-pychecker/package.py | 0 var/spack/packages/py-pycparser/package.py | 0 var/spack/packages/py-pyelftools/package.py | 0 var/spack/packages/py-pygments/package.py | 0 var/spack/packages/py-pylint/package.py | 0 var/spack/packages/py-pypar/package.py | 0 var/spack/packages/py-pyparsing/package.py | 0 var/spack/packages/py-pyqt/package.py | 0 var/spack/packages/py-pyside/package.py | 0 var/spack/packages/py-python-daemon/package.py | 0 var/spack/packages/py-pytz/package.py | 0 var/spack/packages/py-pyzmq/package.py | 0 var/spack/packages/py-rpy2/package.py | 0 var/spack/packages/py-scientificpython/package.py | 0 var/spack/packages/py-scikit-learn/package.py | 0 var/spack/packages/py-scipy/package.py | 0 var/spack/packages/py-setuptools/package.py | 0 var/spack/packages/py-shiboken/package.py | 0 var/spack/packages/py-sip/package.py | 0 var/spack/packages/py-six/package.py | 0 var/spack/packages/py-sympy/package.py | 0 var/spack/packages/py-tornado/package.py | 0 var/spack/packages/py-traitlets/package.py | 0 var/spack/packages/py-virtualenv/package.py | 0 var/spack/packages/py-yapf/package.py | 0 var/spack/packages/python/package.py | 0 var/spack/packages/qhull/package.py | 0 var/spack/packages/qt/package.py | 0 var/spack/packages/qthreads/package.py | 0 var/spack/packages/ravel/package.py | 0 var/spack/packages/readline/package.py | 0 var/spack/packages/rose/package.py | 0 var/spack/packages/ruby/package.py | 0 var/spack/packages/scalasca/package.py | 0 var/spack/packages/scorep/package.py | 0 var/spack/packages/scotch/package.py | 0 var/spack/packages/scr/package.py | 0 var/spack/packages/silo/package.py | 0 var/spack/packages/spindle/package.py | 0 var/spack/packages/sqlite/package.py | 0 var/spack/packages/stat/package.py | 0 var/spack/packages/sundials/package.py | 0 var/spack/packages/swig/package.py | 0 var/spack/packages/task/package.py | 0 var/spack/packages/taskd/package.py | 0 var/spack/packages/tau/package.py | 0 var/spack/packages/tcl/package.py | 0 var/spack/packages/the_silver_searcher/package.py | 0 var/spack/packages/thrift/package.py | 0 var/spack/packages/tk/package.py | 0 var/spack/packages/tmux/package.py | 0 var/spack/packages/tmuxinator/package.py | 0 var/spack/packages/uncrustify/package.py | 0 var/spack/packages/util-linux/package.py | 0 var/spack/packages/vim/package.py | 0 var/spack/packages/vtk/package.py | 0 var/spack/packages/wget/package.py | 0 var/spack/packages/wx/package.py | 0 var/spack/packages/wxpropgrid/package.py | 0 var/spack/packages/xcb-proto/package.py | 0 var/spack/packages/xz/package.py | 0 var/spack/packages/yasm/package.py | 0 var/spack/packages/zlib/package.py | 0 var/spack/packages/zsh/package.py | 0 214 files changed, 3 insertions(+), 2 deletions(-) mode change 100755 => 100644 var/spack/packages/ImageMagick/package.py mode change 100755 => 100644 var/spack/packages/Mitos/package.py mode change 100755 => 100644 var/spack/packages/R/package.py mode change 100755 => 100644 var/spack/packages/SAMRAI/package.py mode change 100755 => 100644 var/spack/packages/adept-utils/package.py mode change 100755 => 100644 var/spack/packages/arpack/package.py mode change 100755 => 100644 var/spack/packages/atk/package.py mode change 100755 => 100644 var/spack/packages/atlas/package.py mode change 100755 => 100644 var/spack/packages/autoconf/package.py mode change 100755 => 100644 var/spack/packages/automaded/package.py mode change 100755 => 100644 var/spack/packages/automake/package.py mode change 100755 => 100644 var/spack/packages/bear/package.py mode change 100755 => 100644 var/spack/packages/bib2xhtml/package.py mode change 100755 => 100644 var/spack/packages/binutils/package.py mode change 100755 => 100644 var/spack/packages/bison/package.py mode change 100755 => 100644 var/spack/packages/boost/package.py mode change 100755 => 100644 var/spack/packages/boxlib/package.py mode change 100755 => 100644 var/spack/packages/bzip2/package.py mode change 100755 => 100644 var/spack/packages/cairo/package.py mode change 100755 => 100644 var/spack/packages/callpath/package.py mode change 100755 => 100644 var/spack/packages/cblas/package.py mode change 100755 => 100644 var/spack/packages/cgm/package.py mode change 100755 => 100644 var/spack/packages/clang/package.py mode change 100755 => 100644 var/spack/packages/cloog/package.py mode change 100755 => 100644 var/spack/packages/cmake/package.py mode change 100755 => 100644 var/spack/packages/coreutils/package.py mode change 100755 => 100644 var/spack/packages/cppcheck/package.py mode change 100755 => 100644 var/spack/packages/cram/package.py mode change 100755 => 100644 var/spack/packages/cube/package.py mode change 100755 => 100644 var/spack/packages/dbus/package.py mode change 100755 => 100644 var/spack/packages/dri2proto/package.py mode change 100755 => 100644 var/spack/packages/dtcmp/package.py mode change 100755 => 100644 var/spack/packages/dyninst/package.py mode change 100755 => 100644 var/spack/packages/extrae/package.py mode change 100755 => 100644 var/spack/packages/exuberant-ctags/package.py mode change 100755 => 100644 var/spack/packages/flex/package.py mode change 100755 => 100644 var/spack/packages/fontconfig/package.py mode change 100755 => 100644 var/spack/packages/freetype/package.py mode change 100755 => 100644 var/spack/packages/gasnet/package.py mode change 100755 => 100644 var/spack/packages/gcc/package.py mode change 100755 => 100644 var/spack/packages/gdk-pixbuf/package.py mode change 100755 => 100644 var/spack/packages/geos/package.py mode change 100755 => 100644 var/spack/packages/git/package.py mode change 100755 => 100644 var/spack/packages/glib/package.py mode change 100755 => 100644 var/spack/packages/global/package.py mode change 100755 => 100644 var/spack/packages/gmp/package.py mode change 100755 => 100644 var/spack/packages/gnutls/package.py mode change 100755 => 100644 var/spack/packages/gperf/package.py mode change 100755 => 100644 var/spack/packages/gperftools/package.py mode change 100755 => 100644 var/spack/packages/graphlib/package.py mode change 100755 => 100644 var/spack/packages/gtkplus/package.py mode change 100755 => 100644 var/spack/packages/harfbuzz/package.py mode change 100755 => 100644 var/spack/packages/hdf5/package.py mode change 100755 => 100644 var/spack/packages/hwloc/package.py mode change 100755 => 100644 var/spack/packages/hypre/package.py mode change 100755 => 100644 var/spack/packages/icu/package.py mode change 100755 => 100644 var/spack/packages/icu4c/package.py mode change 100755 => 100644 var/spack/packages/isl/package.py mode change 100755 => 100644 var/spack/packages/jdk/package.py mode change 100755 => 100644 var/spack/packages/jpeg/package.py mode change 100755 => 100644 var/spack/packages/lapack/package.py mode change 100755 => 100644 var/spack/packages/launchmon/package.py mode change 100755 => 100644 var/spack/packages/lcms/package.py mode change 100755 => 100644 var/spack/packages/libNBC/package.py mode change 100755 => 100644 var/spack/packages/libarchive/package.py mode change 100755 => 100644 var/spack/packages/libcircle/package.py mode change 100755 => 100644 var/spack/packages/libdrm/package.py mode change 100755 => 100644 var/spack/packages/libdwarf/package.py mode change 100755 => 100644 var/spack/packages/libelf/package.py mode change 100755 => 100644 var/spack/packages/libevent/package.py mode change 100755 => 100644 var/spack/packages/libffi/package.py mode change 100755 => 100644 var/spack/packages/libgcrypt/package.py mode change 100755 => 100644 var/spack/packages/libgpg-error/package.py mode change 100755 => 100644 var/spack/packages/libjpeg-turbo/package.py mode change 100755 => 100644 var/spack/packages/libmng/package.py mode change 100755 => 100644 var/spack/packages/libmonitor/package.py mode change 100755 => 100644 var/spack/packages/libpciaccess/package.py mode change 100755 => 100644 var/spack/packages/libpng/package.py mode change 100755 => 100644 var/spack/packages/libtiff/package.py mode change 100755 => 100644 var/spack/packages/libtool/package.py mode change 100755 => 100644 var/spack/packages/libunwind/package.py mode change 100755 => 100644 var/spack/packages/libuuid/package.py mode change 100755 => 100644 var/spack/packages/libxcb/package.py mode change 100755 => 100644 var/spack/packages/libxml2/package.py mode change 100755 => 100644 var/spack/packages/libxshmfence/package.py mode change 100755 => 100644 var/spack/packages/libxslt/package.py mode change 100755 => 100644 var/spack/packages/llvm-lld/package.py mode change 100755 => 100644 var/spack/packages/llvm/package.py mode change 100755 => 100644 var/spack/packages/lwgrp/package.py mode change 100755 => 100644 var/spack/packages/lwm2/package.py mode change 100755 => 100644 var/spack/packages/memaxes/package.py mode change 100755 => 100644 var/spack/packages/mesa/package.py mode change 100755 => 100644 var/spack/packages/metis/package.py mode change 100755 => 100644 var/spack/packages/mpc/package.py mode change 100755 => 100644 var/spack/packages/mpe2/package.py mode change 100755 => 100644 var/spack/packages/mpfr/package.py mode change 100755 => 100644 var/spack/packages/mpibash/package.py mode change 100755 => 100644 var/spack/packages/mpich/package.py mode change 100755 => 100644 var/spack/packages/mpileaks/package.py mode change 100755 => 100644 var/spack/packages/mrnet/package.py mode change 100755 => 100644 var/spack/packages/muster/package.py mode change 100755 => 100644 var/spack/packages/mvapich2/package.py mode change 100755 => 100644 var/spack/packages/nasm/package.py mode change 100755 => 100644 var/spack/packages/ncurses/package.py mode change 100755 => 100644 var/spack/packages/netcdf/package.py mode change 100755 => 100644 var/spack/packages/netgauge/package.py mode change 100755 => 100644 var/spack/packages/netlib-blas/package.py mode change 100755 => 100644 var/spack/packages/nettle/package.py mode change 100755 => 100644 var/spack/packages/ompss/package.py mode change 100755 => 100644 var/spack/packages/opari2/package.py mode change 100755 => 100644 var/spack/packages/openmpi/package.py mode change 100755 => 100644 var/spack/packages/openssl/package.py mode change 100755 => 100644 var/spack/packages/otf/package.py mode change 100755 => 100644 var/spack/packages/otf2/package.py mode change 100755 => 100644 var/spack/packages/pango/package.py mode change 100755 => 100644 var/spack/packages/papi/package.py mode change 100755 => 100644 var/spack/packages/paraver/package.py mode change 100755 => 100644 var/spack/packages/parmetis/package.py mode change 100755 => 100644 var/spack/packages/parpack/package.py mode change 100755 => 100644 var/spack/packages/pcre/package.py mode change 100755 => 100644 var/spack/packages/petsc/package.py mode change 100755 => 100644 var/spack/packages/pixman/package.py mode change 100755 => 100644 var/spack/packages/pmgr_collective/package.py mode change 100755 => 100644 var/spack/packages/postgresql/package.py mode change 100755 => 100644 var/spack/packages/ppl/package.py mode change 100755 => 100644 var/spack/packages/py-basemap/package.py mode change 100755 => 100644 var/spack/packages/py-biopython/package.py mode change 100755 => 100644 var/spack/packages/py-cffi/package.py mode change 100755 => 100644 var/spack/packages/py-cython/package.py mode change 100755 => 100644 var/spack/packages/py-dateutil/package.py mode change 100755 => 100644 var/spack/packages/py-epydoc/package.py mode change 100755 => 100644 var/spack/packages/py-genders/package.py mode change 100755 => 100644 var/spack/packages/py-gnuplot/package.py mode change 100755 => 100644 var/spack/packages/py-h5py/package.py mode change 100755 => 100644 var/spack/packages/py-ipython/package.py mode change 100755 => 100644 var/spack/packages/py-jsonschema/package.py mode change 100755 => 100644 var/spack/packages/py-libxml2/package.py mode change 100755 => 100644 var/spack/packages/py-lockfile/package.py mode change 100755 => 100644 var/spack/packages/py-mako/package.py mode change 100755 => 100644 var/spack/packages/py-matplotlib/package.py mode change 100755 => 100644 var/spack/packages/py-mpi4py/package.py mode change 100755 => 100644 var/spack/packages/py-mx/package.py mode change 100755 => 100644 var/spack/packages/py-nose/package.py mode change 100755 => 100644 var/spack/packages/py-numpy/package.py mode change 100755 => 100644 var/spack/packages/py-pandas/package.py mode change 100755 => 100644 var/spack/packages/py-pexpect/package.py mode change 100755 => 100644 var/spack/packages/py-pil/package.py mode change 100755 => 100644 var/spack/packages/py-pmw/package.py mode change 100755 => 100644 var/spack/packages/py-py4j/package.py mode change 100755 => 100644 var/spack/packages/py-pychecker/package.py mode change 100755 => 100644 var/spack/packages/py-pycparser/package.py mode change 100755 => 100644 var/spack/packages/py-pyelftools/package.py mode change 100755 => 100644 var/spack/packages/py-pygments/package.py mode change 100755 => 100644 var/spack/packages/py-pylint/package.py mode change 100755 => 100644 var/spack/packages/py-pypar/package.py mode change 100755 => 100644 var/spack/packages/py-pyparsing/package.py mode change 100755 => 100644 var/spack/packages/py-pyqt/package.py mode change 100755 => 100644 var/spack/packages/py-pyside/package.py mode change 100755 => 100644 var/spack/packages/py-python-daemon/package.py mode change 100755 => 100644 var/spack/packages/py-pytz/package.py mode change 100755 => 100644 var/spack/packages/py-pyzmq/package.py mode change 100755 => 100644 var/spack/packages/py-rpy2/package.py mode change 100755 => 100644 var/spack/packages/py-scientificpython/package.py mode change 100755 => 100644 var/spack/packages/py-scikit-learn/package.py mode change 100755 => 100644 var/spack/packages/py-scipy/package.py mode change 100755 => 100644 var/spack/packages/py-setuptools/package.py mode change 100755 => 100644 var/spack/packages/py-shiboken/package.py mode change 100755 => 100644 var/spack/packages/py-sip/package.py mode change 100755 => 100644 var/spack/packages/py-six/package.py mode change 100755 => 100644 var/spack/packages/py-sympy/package.py mode change 100755 => 100644 var/spack/packages/py-tornado/package.py mode change 100755 => 100644 var/spack/packages/py-traitlets/package.py mode change 100755 => 100644 var/spack/packages/py-virtualenv/package.py mode change 100755 => 100644 var/spack/packages/py-yapf/package.py mode change 100755 => 100644 var/spack/packages/python/package.py mode change 100755 => 100644 var/spack/packages/qhull/package.py mode change 100755 => 100644 var/spack/packages/qt/package.py mode change 100755 => 100644 var/spack/packages/qthreads/package.py mode change 100755 => 100644 var/spack/packages/ravel/package.py mode change 100755 => 100644 var/spack/packages/readline/package.py mode change 100755 => 100644 var/spack/packages/rose/package.py mode change 100755 => 100644 var/spack/packages/ruby/package.py mode change 100755 => 100644 var/spack/packages/scalasca/package.py mode change 100755 => 100644 var/spack/packages/scorep/package.py mode change 100755 => 100644 var/spack/packages/scotch/package.py mode change 100755 => 100644 var/spack/packages/scr/package.py mode change 100755 => 100644 var/spack/packages/silo/package.py mode change 100755 => 100644 var/spack/packages/spindle/package.py mode change 100755 => 100644 var/spack/packages/sqlite/package.py mode change 100755 => 100644 var/spack/packages/stat/package.py mode change 100755 => 100644 var/spack/packages/sundials/package.py mode change 100755 => 100644 var/spack/packages/swig/package.py mode change 100755 => 100644 var/spack/packages/task/package.py mode change 100755 => 100644 var/spack/packages/taskd/package.py mode change 100755 => 100644 var/spack/packages/tau/package.py mode change 100755 => 100644 var/spack/packages/tcl/package.py mode change 100755 => 100644 var/spack/packages/the_silver_searcher/package.py mode change 100755 => 100644 var/spack/packages/thrift/package.py mode change 100755 => 100644 var/spack/packages/tk/package.py mode change 100755 => 100644 var/spack/packages/tmux/package.py mode change 100755 => 100644 var/spack/packages/tmuxinator/package.py mode change 100755 => 100644 var/spack/packages/uncrustify/package.py mode change 100755 => 100644 var/spack/packages/util-linux/package.py mode change 100755 => 100644 var/spack/packages/vim/package.py mode change 100755 => 100644 var/spack/packages/vtk/package.py mode change 100755 => 100644 var/spack/packages/wget/package.py mode change 100755 => 100644 var/spack/packages/wx/package.py mode change 100755 => 100644 var/spack/packages/wxpropgrid/package.py mode change 100755 => 100644 var/spack/packages/xcb-proto/package.py mode change 100755 => 100644 var/spack/packages/xz/package.py mode change 100755 => 100644 var/spack/packages/yasm/package.py mode change 100755 => 100644 var/spack/packages/zlib/package.py mode change 100755 => 100644 var/spack/packages/zsh/package.py diff --git a/var/spack/packages/ImageMagick/package.py b/var/spack/packages/ImageMagick/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/Mitos/package.py b/var/spack/packages/Mitos/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/R/package.py b/var/spack/packages/R/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/SAMRAI/package.py b/var/spack/packages/SAMRAI/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/adept-utils/package.py b/var/spack/packages/adept-utils/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/adios/package.py b/var/spack/packages/adios/package.py index 224c3b37d51..260dcbe851d 100644 --- a/var/spack/packages/adios/package.py +++ b/var/spack/packages/adios/package.py @@ -18,12 +18,13 @@ class Adios(Package): # module load cray-netcdf/4.3.3.1 # module load cray-hdf5/1.8.14 # module load python/2.7.10 + depends_on('hdf5') depends_on('mxml') - + def install(self, spec, prefix): configure_args = ["--prefix=%s" % prefix, "--with-mxml=%s" % spec['mxml'].prefix, - "--with-hdf5=%s" % os.environ["HDF5_DIR"], + "--with-hdf5=%s" % spec['hdf5'].prefix, "--with-netcdf=%s" % os.environ["NETCDF_DIR"], "--with-infiniband=no", "MPICC=cc","MPICXX=CC","MPIFC=ftn", diff --git a/var/spack/packages/arpack/package.py b/var/spack/packages/arpack/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/atk/package.py b/var/spack/packages/atk/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/atlas/package.py b/var/spack/packages/atlas/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/autoconf/package.py b/var/spack/packages/autoconf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/automaded/package.py b/var/spack/packages/automaded/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/automake/package.py b/var/spack/packages/automake/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/bear/package.py b/var/spack/packages/bear/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/bib2xhtml/package.py b/var/spack/packages/bib2xhtml/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/binutils/package.py b/var/spack/packages/binutils/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/bison/package.py b/var/spack/packages/bison/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/boxlib/package.py b/var/spack/packages/boxlib/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/bzip2/package.py b/var/spack/packages/bzip2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cairo/package.py b/var/spack/packages/cairo/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/callpath/package.py b/var/spack/packages/callpath/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cblas/package.py b/var/spack/packages/cblas/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cgm/package.py b/var/spack/packages/cgm/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/clang/package.py b/var/spack/packages/clang/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cloog/package.py b/var/spack/packages/cloog/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cmake/package.py b/var/spack/packages/cmake/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/coreutils/package.py b/var/spack/packages/coreutils/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cppcheck/package.py b/var/spack/packages/cppcheck/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cram/package.py b/var/spack/packages/cram/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/cube/package.py b/var/spack/packages/cube/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/dbus/package.py b/var/spack/packages/dbus/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/dri2proto/package.py b/var/spack/packages/dri2proto/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/dtcmp/package.py b/var/spack/packages/dtcmp/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/dyninst/package.py b/var/spack/packages/dyninst/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/extrae/package.py b/var/spack/packages/extrae/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/exuberant-ctags/package.py b/var/spack/packages/exuberant-ctags/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/flex/package.py b/var/spack/packages/flex/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/fontconfig/package.py b/var/spack/packages/fontconfig/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/freetype/package.py b/var/spack/packages/freetype/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gasnet/package.py b/var/spack/packages/gasnet/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gcc/package.py b/var/spack/packages/gcc/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gdk-pixbuf/package.py b/var/spack/packages/gdk-pixbuf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/geos/package.py b/var/spack/packages/geos/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/git/package.py b/var/spack/packages/git/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/glib/package.py b/var/spack/packages/glib/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/global/package.py b/var/spack/packages/global/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gmp/package.py b/var/spack/packages/gmp/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gnutls/package.py b/var/spack/packages/gnutls/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gperf/package.py b/var/spack/packages/gperf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gperftools/package.py b/var/spack/packages/gperftools/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/graphlib/package.py b/var/spack/packages/graphlib/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/gtkplus/package.py b/var/spack/packages/gtkplus/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/harfbuzz/package.py b/var/spack/packages/harfbuzz/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/hdf5/package.py b/var/spack/packages/hdf5/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/hwloc/package.py b/var/spack/packages/hwloc/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/hypre/package.py b/var/spack/packages/hypre/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/icu/package.py b/var/spack/packages/icu/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/icu4c/package.py b/var/spack/packages/icu4c/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/isl/package.py b/var/spack/packages/isl/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/jdk/package.py b/var/spack/packages/jdk/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/jpeg/package.py b/var/spack/packages/jpeg/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/lapack/package.py b/var/spack/packages/lapack/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/launchmon/package.py b/var/spack/packages/launchmon/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/lcms/package.py b/var/spack/packages/lcms/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libNBC/package.py b/var/spack/packages/libNBC/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libarchive/package.py b/var/spack/packages/libarchive/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libcircle/package.py b/var/spack/packages/libcircle/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libdrm/package.py b/var/spack/packages/libdrm/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libdwarf/package.py b/var/spack/packages/libdwarf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libelf/package.py b/var/spack/packages/libelf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libevent/package.py b/var/spack/packages/libevent/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libffi/package.py b/var/spack/packages/libffi/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libgcrypt/package.py b/var/spack/packages/libgcrypt/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libgpg-error/package.py b/var/spack/packages/libgpg-error/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libjpeg-turbo/package.py b/var/spack/packages/libjpeg-turbo/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libmng/package.py b/var/spack/packages/libmng/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libmonitor/package.py b/var/spack/packages/libmonitor/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/packages/libpciaccess/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libpng/package.py b/var/spack/packages/libpng/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libtiff/package.py b/var/spack/packages/libtiff/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libtool/package.py b/var/spack/packages/libtool/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libunwind/package.py b/var/spack/packages/libunwind/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libuuid/package.py b/var/spack/packages/libuuid/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libxcb/package.py b/var/spack/packages/libxcb/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libxshmfence/package.py b/var/spack/packages/libxshmfence/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/libxslt/package.py b/var/spack/packages/libxslt/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/llvm-lld/package.py b/var/spack/packages/llvm-lld/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/llvm/package.py b/var/spack/packages/llvm/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/lwgrp/package.py b/var/spack/packages/lwgrp/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/lwm2/package.py b/var/spack/packages/lwm2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/memaxes/package.py b/var/spack/packages/memaxes/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/metis/package.py b/var/spack/packages/metis/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpc/package.py b/var/spack/packages/mpc/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpe2/package.py b/var/spack/packages/mpe2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpfr/package.py b/var/spack/packages/mpfr/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpibash/package.py b/var/spack/packages/mpibash/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpileaks/package.py b/var/spack/packages/mpileaks/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mrnet/package.py b/var/spack/packages/mrnet/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/muster/package.py b/var/spack/packages/muster/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/mvapich2/package.py b/var/spack/packages/mvapich2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/nasm/package.py b/var/spack/packages/nasm/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/ncurses/package.py b/var/spack/packages/ncurses/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/netcdf/package.py b/var/spack/packages/netcdf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/netgauge/package.py b/var/spack/packages/netgauge/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/netlib-blas/package.py b/var/spack/packages/netlib-blas/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/nettle/package.py b/var/spack/packages/nettle/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/ompss/package.py b/var/spack/packages/ompss/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/opari2/package.py b/var/spack/packages/opari2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/openmpi/package.py b/var/spack/packages/openmpi/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/openssl/package.py b/var/spack/packages/openssl/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/otf/package.py b/var/spack/packages/otf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/otf2/package.py b/var/spack/packages/otf2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/pango/package.py b/var/spack/packages/pango/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/papi/package.py b/var/spack/packages/papi/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/paraver/package.py b/var/spack/packages/paraver/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/parmetis/package.py b/var/spack/packages/parmetis/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/parpack/package.py b/var/spack/packages/parpack/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/pcre/package.py b/var/spack/packages/pcre/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/petsc/package.py b/var/spack/packages/petsc/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/pixman/package.py b/var/spack/packages/pixman/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/pmgr_collective/package.py b/var/spack/packages/pmgr_collective/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/postgresql/package.py b/var/spack/packages/postgresql/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/ppl/package.py b/var/spack/packages/ppl/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-basemap/package.py b/var/spack/packages/py-basemap/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-biopython/package.py b/var/spack/packages/py-biopython/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-cffi/package.py b/var/spack/packages/py-cffi/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-cython/package.py b/var/spack/packages/py-cython/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-dateutil/package.py b/var/spack/packages/py-dateutil/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-epydoc/package.py b/var/spack/packages/py-epydoc/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-genders/package.py b/var/spack/packages/py-genders/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-gnuplot/package.py b/var/spack/packages/py-gnuplot/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-h5py/package.py b/var/spack/packages/py-h5py/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-jsonschema/package.py b/var/spack/packages/py-jsonschema/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-libxml2/package.py b/var/spack/packages/py-libxml2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-lockfile/package.py b/var/spack/packages/py-lockfile/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-mako/package.py b/var/spack/packages/py-mako/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-mpi4py/package.py b/var/spack/packages/py-mpi4py/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-mx/package.py b/var/spack/packages/py-mx/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-nose/package.py b/var/spack/packages/py-nose/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/packages/py-numpy/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pandas/package.py b/var/spack/packages/py-pandas/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pexpect/package.py b/var/spack/packages/py-pexpect/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pil/package.py b/var/spack/packages/py-pil/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pmw/package.py b/var/spack/packages/py-pmw/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-py4j/package.py b/var/spack/packages/py-py4j/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pychecker/package.py b/var/spack/packages/py-pychecker/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pycparser/package.py b/var/spack/packages/py-pycparser/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pyelftools/package.py b/var/spack/packages/py-pyelftools/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pygments/package.py b/var/spack/packages/py-pygments/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pylint/package.py b/var/spack/packages/py-pylint/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pypar/package.py b/var/spack/packages/py-pypar/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pyparsing/package.py b/var/spack/packages/py-pyparsing/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pyqt/package.py b/var/spack/packages/py-pyqt/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-python-daemon/package.py b/var/spack/packages/py-python-daemon/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pytz/package.py b/var/spack/packages/py-pytz/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-pyzmq/package.py b/var/spack/packages/py-pyzmq/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-rpy2/package.py b/var/spack/packages/py-rpy2/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-scientificpython/package.py b/var/spack/packages/py-scientificpython/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-scikit-learn/package.py b/var/spack/packages/py-scikit-learn/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-scipy/package.py b/var/spack/packages/py-scipy/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-shiboken/package.py b/var/spack/packages/py-shiboken/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-sip/package.py b/var/spack/packages/py-sip/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-six/package.py b/var/spack/packages/py-six/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-sympy/package.py b/var/spack/packages/py-sympy/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-tornado/package.py b/var/spack/packages/py-tornado/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-traitlets/package.py b/var/spack/packages/py-traitlets/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-virtualenv/package.py b/var/spack/packages/py-virtualenv/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/py-yapf/package.py b/var/spack/packages/py-yapf/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/python/package.py b/var/spack/packages/python/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/qhull/package.py b/var/spack/packages/qhull/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/qt/package.py b/var/spack/packages/qt/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/qthreads/package.py b/var/spack/packages/qthreads/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/ravel/package.py b/var/spack/packages/ravel/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/readline/package.py b/var/spack/packages/readline/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/rose/package.py b/var/spack/packages/rose/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/ruby/package.py b/var/spack/packages/ruby/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/scalasca/package.py b/var/spack/packages/scalasca/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/scotch/package.py b/var/spack/packages/scotch/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/scr/package.py b/var/spack/packages/scr/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/silo/package.py b/var/spack/packages/silo/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/spindle/package.py b/var/spack/packages/spindle/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/sqlite/package.py b/var/spack/packages/sqlite/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/stat/package.py b/var/spack/packages/stat/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/sundials/package.py b/var/spack/packages/sundials/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/swig/package.py b/var/spack/packages/swig/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/task/package.py b/var/spack/packages/task/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/taskd/package.py b/var/spack/packages/taskd/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/tau/package.py b/var/spack/packages/tau/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/tcl/package.py b/var/spack/packages/tcl/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/the_silver_searcher/package.py b/var/spack/packages/the_silver_searcher/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/thrift/package.py b/var/spack/packages/thrift/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/tk/package.py b/var/spack/packages/tk/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/tmux/package.py b/var/spack/packages/tmux/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/tmuxinator/package.py b/var/spack/packages/tmuxinator/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/uncrustify/package.py b/var/spack/packages/uncrustify/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/util-linux/package.py b/var/spack/packages/util-linux/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/vim/package.py b/var/spack/packages/vim/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/vtk/package.py b/var/spack/packages/vtk/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/wget/package.py b/var/spack/packages/wget/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/wx/package.py b/var/spack/packages/wx/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/wxpropgrid/package.py b/var/spack/packages/wxpropgrid/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/xcb-proto/package.py b/var/spack/packages/xcb-proto/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/xz/package.py b/var/spack/packages/xz/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/yasm/package.py b/var/spack/packages/yasm/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/zlib/package.py b/var/spack/packages/zlib/package.py old mode 100755 new mode 100644 diff --git a/var/spack/packages/zsh/package.py b/var/spack/packages/zsh/package.py old mode 100755 new mode 100644 From d2da41b0bd21d4150ec1e161576efb1e356db1cb Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 12:12:03 -0800 Subject: [PATCH 087/375] More file permissions chasing --- var/spack/packages/SAMRAI/no-tool-build.patch | 0 var/spack/packages/launchmon/patch.lmon_install_dir | 0 var/spack/packages/mpe2/mpe2.patch | 0 var/spack/packages/mpibash/mpibash-4.3.patch | 0 var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch | 0 var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch | 0 var/spack/packages/openmpi/llnl-platforms.patch | 0 var/spack/packages/rose/add_spack_compiler_recognition.patch | 0 var/spack/packages/stat/configure_mpicxx.patch | 0 9 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 var/spack/packages/SAMRAI/no-tool-build.patch mode change 100755 => 100644 var/spack/packages/launchmon/patch.lmon_install_dir mode change 100755 => 100644 var/spack/packages/mpe2/mpe2.patch mode change 100755 => 100644 var/spack/packages/mpibash/mpibash-4.3.patch mode change 100755 => 100644 var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch mode change 100755 => 100644 var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch mode change 100755 => 100644 var/spack/packages/openmpi/llnl-platforms.patch mode change 100755 => 100644 var/spack/packages/rose/add_spack_compiler_recognition.patch mode change 100755 => 100644 var/spack/packages/stat/configure_mpicxx.patch diff --git a/var/spack/packages/SAMRAI/no-tool-build.patch b/var/spack/packages/SAMRAI/no-tool-build.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/launchmon/patch.lmon_install_dir b/var/spack/packages/launchmon/patch.lmon_install_dir old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpe2/mpe2.patch b/var/spack/packages/mpe2/mpe2.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/mpibash/mpibash-4.3.patch b/var/spack/packages/mpibash/mpibash-4.3.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch b/var/spack/packages/mvapich2/ad_lustre_rwcontig_open_source.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch b/var/spack/packages/openmpi/ad_lustre_rwcontig_open_source.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/openmpi/llnl-platforms.patch b/var/spack/packages/openmpi/llnl-platforms.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/rose/add_spack_compiler_recognition.patch b/var/spack/packages/rose/add_spack_compiler_recognition.patch old mode 100755 new mode 100644 diff --git a/var/spack/packages/stat/configure_mpicxx.patch b/var/spack/packages/stat/configure_mpicxx.patch old mode 100755 new mode 100644 From 1b0d06474359797d4f9e0c59aa813e0e607f0e5a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 14:13:33 -0800 Subject: [PATCH 088/375] Revert "clean up package.py file and remove duplicate file" This reverts commit c809cc92735c7ff57dab2aca02bc7fa40c901a43. --- var/spack/packages/py-ipython/package.py | 4 +++ .../packages/py-ipython/package.py.backup | 27 +++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100755 var/spack/packages/py-ipython/package.py.backup diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py index 225366552a1..3b94b71f295 100644 --- a/var/spack/packages/py-ipython/package.py +++ b/var/spack/packages/py-ipython/package.py @@ -24,4 +24,8 @@ class PyIpython(Package): depends_on('py-jinja2') def install(self, spec, prefix): + import os + # os.system("spack activate py-setuptools@18.1") + # os.system("easy_install pip") + # os.system("pip install ipython[notebook]") python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython/package.py.backup b/var/spack/packages/py-ipython/package.py.backup new file mode 100755 index 00000000000..1ed8b57e605 --- /dev/null +++ b/var/spack/packages/py-ipython/package.py.backup @@ -0,0 +1,27 @@ +from spack import * + +class PyIpython(Package): + """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" + homepage = "https://pypi.python.org/pypi/ipython" + + version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', + url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") + version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') + version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', + url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') + + extends('python') + depends_on('py-pygments') + depends_on('py-setuptools@18.1') + depends_on('py-traitlets') + depends_on('py-pyzmq') + depends_on('py-pexpect') + depends_on('py-tornado') + depends_on('py-Jinja2') + + def install(self, spec, prefix): + import os + # os.system("spack activate py-setuptools@18.1") + # os.system("easy_install pip") + # os.system("pip install ipython[notebook]") + python('setup.py', 'install', '--prefix=%s' % prefix) From a9e9f9afb04394ddb85ad197fa7b19999ecc3b32 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 14:14:26 -0800 Subject: [PATCH 089/375] Revert "removed package for python packages with problematic / duplicate names" This reverts commit f3254ff02d589c9e177aab58276f5c1971866018. --- var/spack/packages/py-Cython/package.py | 12 ++++++++++++ var/spack/packages/py-Distutils2/package.py | 15 +++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 var/spack/packages/py-Cython/package.py create mode 100644 var/spack/packages/py-Distutils2/package.py diff --git a/var/spack/packages/py-Cython/package.py b/var/spack/packages/py-Cython/package.py new file mode 100644 index 00000000000..c2501391143 --- /dev/null +++ b/var/spack/packages/py-Cython/package.py @@ -0,0 +1,12 @@ +from spack import * + +class PyCython(Package): + """The Cython compiler for writing C extensions for the Python language.""" + homepage = "http://www.cython.org" + version("0.21.2", "d21adb870c75680dc857cd05d41046a4", + url="https://pypi.python.org/packages/source/C/Cython/Cython-0.21.2.tar.gz") + + extends("python") + + def install(self, spec, prefix): + python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-Distutils2/package.py b/var/spack/packages/py-Distutils2/package.py new file mode 100644 index 00000000000..8690f569fee --- /dev/null +++ b/var/spack/packages/py-Distutils2/package.py @@ -0,0 +1,15 @@ +from spack import * + +class PyDistutils2(Package): + """Python Packaging Library.""" + + homepage = "https://hg.python.org/distutils2" + url = "https://pypi.python.org/packages/source/D/Distutils2/Distutils2-1.0a4.tar.gz" + + version('1.0a4', '52bc9dffb394970c27e02853ae3a3241') + + depends_on("python") + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) + From d3d37ad0cebe4d42d48f5f5f5a53077ae6126631 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 14:29:01 -0800 Subject: [PATCH 090/375] removed the submodule as part of reversion of extraneous commits --- lib/spack/spack/util/python_recipe_parser | 1 - 1 file changed, 1 deletion(-) delete mode 160000 lib/spack/spack/util/python_recipe_parser diff --git a/lib/spack/spack/util/python_recipe_parser b/lib/spack/spack/util/python_recipe_parser deleted file mode 160000 index 437a62abb3d..00000000000 --- a/lib/spack/spack/util/python_recipe_parser +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 437a62abb3df7212e3ee20269c0089a0a9766fe0 From 86230b9ac8ed42f3f3b309a0a8a77483978a306c Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 15:01:46 -0800 Subject: [PATCH 091/375] Revert "make sure submodule is added in the correct location" This reverts commit e9f7d033ff04138905c99cb427e2071cc55cdc16. --- .gitmodules | 6 +++--- var/spack/packages/py-astropy/package.py | 1 - var/spack/packages/py-jsonschema/package.py | 2 -- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitmodules b/.gitmodules index d5995e8e4b0..1d770f415bd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "lib/spack/spack/util/python_recipe_parser"] - path = lib/spack/spack/util/python_recipe_parser - url = git@github.com:karenyyng/spack_python_package_parser.git +[submodule "/lib/spack/spack/util/python_recipe_parser"] + path = /lib/spack/spack/util/python_recipe_parser + url = https://github.com/karenyyng/spack_python_package_parser diff --git a/var/spack/packages/py-astropy/package.py b/var/spack/packages/py-astropy/package.py index 9fda37519e2..5d2d0619ccf 100644 --- a/var/spack/packages/py-astropy/package.py +++ b/var/spack/packages/py-astropy/package.py @@ -7,7 +7,6 @@ class PyAstropy(Package): url="https://pypi.python.org/packages/source/a/astropy/astropy-0.4.2.tar.gz") extends("python") - depends_on("py-numpy") def install(self, spec, prefix): python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-jsonschema/package.py b/var/spack/packages/py-jsonschema/package.py index fa233635ce8..9a7fc218622 100644 --- a/var/spack/packages/py-jsonschema/package.py +++ b/var/spack/packages/py-jsonschema/package.py @@ -11,8 +11,6 @@ class PyJsonschema(Package): extends("python") depends_on("py-setuptools@18.1") - depends_on("py-vcversioner") - depends_on("py-functools32") def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) From 6ca1e5c4b07e1677363cf0f3787144f67af688c8 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 15:02:29 -0800 Subject: [PATCH 092/375] Revert "added a bunch of packages and added submodule" This reverts commit 79808da760eabcb3e692c88e29ea3c7c558b131d. --- .gitignore | 1 - .gitmodules | 4 +- utils/python_recipe_parser | 1 + var/spack/packages/asciidoc/package.py | 18 +++++++++ var/spack/packages/boost/package.py | 4 ++ var/spack/packages/bowtie2/bowtie2-2.5.patch | 16 ++++++++ var/spack/packages/bowtie2/package.py | 24 ++++++++++++ var/spack/packages/cscope/package.py | 17 ++++++++ var/spack/packages/czmq/package.py | 19 +++++++++ var/spack/packages/docbook-xml/package.py | 19 +++++++++ var/spack/packages/doxygen/package.py | 25 ++++++++++++ var/spack/packages/eigen/package.py | 17 -------- var/spack/packages/elfutils/package.py | 26 +++++++++++++ var/spack/packages/fish/package.py | 18 +++++++++ var/spack/packages/flux/package.py | 36 +++++++++++++++++ var/spack/packages/gflags/package.py | 21 ++++++++++ var/spack/packages/ghostscript/package.py | 17 ++++++++ var/spack/packages/glog/package.py | 15 +++++++ var/spack/packages/graphviz/package.py | 19 ++++----- var/spack/packages/leveldb/package.py | 29 ++++++++++++++ var/spack/packages/libdrm/package.py | 5 +-- var/spack/packages/libelf/package.py | 2 + var/spack/packages/libffi/package.py | 6 +-- var/spack/packages/libjson-c/package.py | 14 +++++++ var/spack/packages/libpciaccess/package.py | 19 +++++---- var/spack/packages/libsodium/package.py | 19 +++++++++ var/spack/packages/libxml2/package.py | 5 ++- var/spack/packages/lmdb/package.py | 39 +++++++++++++++++++ var/spack/packages/lua/package.py | 26 +++++++++++++ var/spack/packages/mesa/package.py | 5 ++- var/spack/packages/mpich/package.py | 1 - var/spack/packages/munge/package.py | 20 ++++++++++ var/spack/packages/ncurses/package.py | 8 ++++ var/spack/packages/netlib-blas/package.py | 6 +++ .../{lapack => netlib-lapack}/package.py | 18 ++++++++- var/spack/packages/pidx/package.py | 21 ++++++++++ var/spack/packages/pkg-config/package.py | 17 ++++++++ var/spack/packages/protobuf/package.py | 16 ++++++++ var/spack/packages/py-Cython/package.py | 12 ------ var/spack/packages/py-Distutils2/package.py | 15 ------- var/spack/packages/py-astroML/package.py | 12 ------ var/spack/packages/py-astropy/package.py | 12 ------ var/spack/packages/py-autopep8/package.py | 12 ------ var/spack/packages/py-certifi/package.py | 12 ------ var/spack/packages/py-decorator/package.py | 12 ------ var/spack/packages/py-emcee/package.py | 12 ------ var/spack/packages/py-fitsio/package.py | 13 ------- var/spack/packages/py-gnureadline/package.py | 12 ------ var/spack/packages/py-ipython/package.py | 21 ++-------- .../packages/py-ipython/package.py.backup | 27 ------------- .../packages/py-ipython_genutils/package.py | 12 ------ var/spack/packages/py-joblib/package.py | 12 ------ var/spack/packages/py-jsonschema/package.py | 16 -------- var/spack/packages/py-matplotlib/package.py | 1 + var/spack/packages/py-mock/package.py | 17 +++++--- var/spack/packages/py-numexpr/package.py | 12 ------ var/spack/packages/py-numpy/package.py | 15 ++++++- var/spack/packages/py-pandas/package.py | 2 - var/spack/packages/py-pep8/package.py | 12 ------ var/spack/packages/py-pickleshare/package.py | 12 ------ var/spack/packages/py-pip/package.py | 12 ------ var/spack/packages/py-py/package.py | 12 ------ var/spack/packages/py-py4j/package.py | 15 ------- var/spack/packages/py-pyside/package.py | 5 ++- var/spack/packages/py-pytest/package.py | 12 ------ .../packages/py-python-dateutil/package.py | 12 ------ var/spack/packages/py-pytz/package.py | 2 - var/spack/packages/py-pyzmq/package.py | 16 -------- var/spack/packages/py-scikit-learn/package.py | 3 -- var/spack/packages/py-scipy/package.py | 4 +- var/spack/packages/py-seaborn/package.py | 12 ------ var/spack/packages/py-setuptools/package.py | 9 ++--- var/spack/packages/py-six/package.py | 2 - var/spack/packages/py-statsmodels/package.py | 12 ------ var/spack/packages/py-tables/package.py | 12 ------ var/spack/packages/py-tornado/package.py | 17 -------- var/spack/packages/py-traitlets/package.py | 14 ------- .../packages/py-triangle-plot/package.py | 12 ------ var/spack/packages/samtools/package.py | 18 +++++++++ var/spack/packages/samtools/samtools1.2.patch | 20 ++++++++++ var/spack/packages/scorep/package.py | 15 ++++--- var/spack/packages/snappy/package.py | 15 +++++++ var/spack/packages/stat/package.py | 11 +++++- var/spack/packages/vim/package.py | 23 +++++++++-- var/spack/packages/zeromq/package.py | 20 ++++++++++ 85 files changed, 689 insertions(+), 492 deletions(-) create mode 160000 utils/python_recipe_parser create mode 100644 var/spack/packages/asciidoc/package.py create mode 100644 var/spack/packages/bowtie2/bowtie2-2.5.patch create mode 100644 var/spack/packages/bowtie2/package.py create mode 100644 var/spack/packages/cscope/package.py create mode 100644 var/spack/packages/czmq/package.py create mode 100644 var/spack/packages/docbook-xml/package.py create mode 100644 var/spack/packages/doxygen/package.py delete mode 100644 var/spack/packages/eigen/package.py create mode 100644 var/spack/packages/elfutils/package.py create mode 100644 var/spack/packages/fish/package.py create mode 100644 var/spack/packages/flux/package.py create mode 100644 var/spack/packages/gflags/package.py create mode 100644 var/spack/packages/ghostscript/package.py create mode 100644 var/spack/packages/glog/package.py create mode 100644 var/spack/packages/leveldb/package.py create mode 100644 var/spack/packages/libjson-c/package.py create mode 100644 var/spack/packages/libsodium/package.py create mode 100644 var/spack/packages/lmdb/package.py create mode 100644 var/spack/packages/lua/package.py create mode 100644 var/spack/packages/munge/package.py rename var/spack/packages/{lapack => netlib-lapack}/package.py (77%) create mode 100644 var/spack/packages/pidx/package.py create mode 100644 var/spack/packages/pkg-config/package.py create mode 100644 var/spack/packages/protobuf/package.py delete mode 100644 var/spack/packages/py-Cython/package.py delete mode 100644 var/spack/packages/py-Distutils2/package.py delete mode 100644 var/spack/packages/py-astroML/package.py delete mode 100644 var/spack/packages/py-astropy/package.py delete mode 100644 var/spack/packages/py-autopep8/package.py delete mode 100644 var/spack/packages/py-certifi/package.py delete mode 100644 var/spack/packages/py-decorator/package.py delete mode 100644 var/spack/packages/py-emcee/package.py delete mode 100644 var/spack/packages/py-fitsio/package.py delete mode 100644 var/spack/packages/py-gnureadline/package.py delete mode 100755 var/spack/packages/py-ipython/package.py.backup delete mode 100644 var/spack/packages/py-ipython_genutils/package.py delete mode 100644 var/spack/packages/py-joblib/package.py delete mode 100644 var/spack/packages/py-jsonschema/package.py delete mode 100644 var/spack/packages/py-numexpr/package.py delete mode 100644 var/spack/packages/py-pep8/package.py delete mode 100644 var/spack/packages/py-pickleshare/package.py delete mode 100644 var/spack/packages/py-pip/package.py delete mode 100644 var/spack/packages/py-py/package.py delete mode 100644 var/spack/packages/py-py4j/package.py delete mode 100644 var/spack/packages/py-pytest/package.py delete mode 100644 var/spack/packages/py-python-dateutil/package.py delete mode 100644 var/spack/packages/py-pyzmq/package.py delete mode 100644 var/spack/packages/py-seaborn/package.py delete mode 100644 var/spack/packages/py-statsmodels/package.py delete mode 100644 var/spack/packages/py-tables/package.py delete mode 100644 var/spack/packages/py-tornado/package.py delete mode 100644 var/spack/packages/py-traitlets/package.py delete mode 100644 var/spack/packages/py-triangle-plot/package.py create mode 100644 var/spack/packages/samtools/package.py create mode 100644 var/spack/packages/samtools/samtools1.2.patch create mode 100644 var/spack/packages/snappy/package.py create mode 100644 var/spack/packages/zeromq/package.py diff --git a/.gitignore b/.gitignore index 8146921fd85..1c6ca4c99e1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,3 @@ /etc/spackconfig /share/spack/dotkit /share/spack/modules -*.ropeproject diff --git a/.gitmodules b/.gitmodules index 1d770f415bd..a097fcd8cf3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "/lib/spack/spack/util/python_recipe_parser"] - path = /lib/spack/spack/util/python_recipe_parser +[submodule "utils/python_recipe_parser"] + path = utils/python_recipe_parser url = https://github.com/karenyyng/spack_python_package_parser diff --git a/utils/python_recipe_parser b/utils/python_recipe_parser new file mode 160000 index 00000000000..3158074a2d7 --- /dev/null +++ b/utils/python_recipe_parser @@ -0,0 +1 @@ +Subproject commit 3158074a2d74f99aa72289dfcf093b1b0ba4474e diff --git a/var/spack/packages/asciidoc/package.py b/var/spack/packages/asciidoc/package.py new file mode 100644 index 00000000000..828f3b3f4f7 --- /dev/null +++ b/var/spack/packages/asciidoc/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Asciidoc(Package): + """ A presentable text document format for writing articles, UNIX man + pages and other small to medium sized documents.""" + homepage = "http://asciidoc.org" + url = "http://downloads.sourceforge.net/project/asciidoc/asciidoc/8.6.9/asciidoc-8.6.9.tar.gz" + + version('8.6.9', 'c59018f105be8d022714b826b0be130a') + + depends_on('libxml2') + depends_on('libxslt') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/boost/package.py b/var/spack/packages/boost/package.py index 254d1afda1d..35824d53a26 100644 --- a/var/spack/packages/boost/package.py +++ b/var/spack/packages/boost/package.py @@ -14,6 +14,10 @@ class Boost(Package): list_url = "http://sourceforge.net/projects/boost/files/boost/" list_depth = 2 + version('1.59.0', '6aa9a5c6a4ca1016edd0ed1178e3cb87') + version('1.58.0', 'b8839650e61e9c1c0a89f371dd475546') + version('1.57.0', '1be49befbdd9a5ce9def2983ba3e7b76') + version('1.56.0', 'a744cf167b05d72335f27c88115f211d') version('1.55.0', 'd6eef4b4cacb2183f2bf265a5a03a354') version('1.54.0', '15cb8c0803064faef0c4ddf5bc5ca279') version('1.53.0', 'a00d22605d5dbcfb4c9936a9b35bc4c2') diff --git a/var/spack/packages/bowtie2/bowtie2-2.5.patch b/var/spack/packages/bowtie2/bowtie2-2.5.patch new file mode 100644 index 00000000000..290be39c732 --- /dev/null +++ b/var/spack/packages/bowtie2/bowtie2-2.5.patch @@ -0,0 +1,16 @@ +--- Makefile 2015-02-26 10:50:00.000000000 -0800 ++++ Makefile.new 2015-07-29 18:03:59.891357399 -0700 +@@ -22,10 +22,10 @@ + # + + INC = +-GCC_PREFIX = $(shell dirname `which gcc`) ++GCC_PREFIX = + GCC_SUFFIX = +-CC = $(GCC_PREFIX)/gcc$(GCC_SUFFIX) +-CPP = $(GCC_PREFIX)/g++$(GCC_SUFFIX) ++CC = cc ++CPP = c++ + CXX = $(CPP) + HEADERS = $(wildcard *.h) + BOWTIE_MM = 1 diff --git a/var/spack/packages/bowtie2/package.py b/var/spack/packages/bowtie2/package.py new file mode 100644 index 00000000000..339aab6598d --- /dev/null +++ b/var/spack/packages/bowtie2/package.py @@ -0,0 +1,24 @@ +from spack import * +from glob import glob +class Bowtie2(Package): + """Description""" + homepage = "bowtie-bio.sourceforge.net/bowtie2/index.shtml" + version('2.2.5','51fa97a862d248d7ee660efc1147c75f', url = "http://downloads.sourceforge.net/project/bowtie-bio/bowtie2/2.2.5/bowtie2-2.2.5-source.zip") + + patch('bowtie2-2.5.patch',when='@2.2.5', level=0) + + def install(self, spec, prefix): + make() + mkdirp(prefix.bin) + for bow in glob("bowtie2*"): + install(bow, prefix.bin) + # install('bowtie2',prefix.bin) + # install('bowtie2-align-l',prefix.bin) + # install('bowtie2-align-s',prefix.bin) + # install('bowtie2-build',prefix.bin) + # install('bowtie2-build-l',prefix.bin) + # install('bowtie2-build-s',prefix.bin) + # install('bowtie2-inspect',prefix.bin) + # install('bowtie2-inspect-l',prefix.bin) + # install('bowtie2-inspect-s',prefix.bin) + diff --git a/var/spack/packages/cscope/package.py b/var/spack/packages/cscope/package.py new file mode 100644 index 00000000000..9aac0f73042 --- /dev/null +++ b/var/spack/packages/cscope/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Cscope(Package): + """Cscope is a developer's tool for browsing source code.""" + homepage = "http://http://cscope.sourceforge.net/" + url = "http://downloads.sourceforge.net/project/cscope/cscope/15.8b/cscope-15.8b.tar.gz" + + version('15.8b', '8f9409a238ee313a96f9f87fe0f3b176') + + # Can be configured to use flex (not necessary) + # ./configure --with-flex + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/czmq/package.py b/var/spack/packages/czmq/package.py new file mode 100644 index 00000000000..a2f19475541 --- /dev/null +++ b/var/spack/packages/czmq/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Czmq(Package): + """ A C interface to the ZMQ library """ + homepage = "http://czmq.zeromq.org" + url = "https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz" + + version('3.0.2', '23e9885f7ee3ce88d99d0425f52e9be1', url='https://github.com/zeromq/czmq/archive/v3.0.2.tar.gz') + + depends_on('zeromq') + + def install(self, spec, prefix): + bash = which("bash") + bash("./autogen.sh") + configure("--prefix=%s" % prefix) + + make() + make("install") + diff --git a/var/spack/packages/docbook-xml/package.py b/var/spack/packages/docbook-xml/package.py new file mode 100644 index 00000000000..fce1de7debf --- /dev/null +++ b/var/spack/packages/docbook-xml/package.py @@ -0,0 +1,19 @@ +import os +import glob +from spack import * + + +class DocbookXml(Package): + """Docbook DTD XML files.""" + homepage = "http://www.oasis-open.org/docbook" + url = "http://www.oasis-open.org/docbook/xml/4.5/docbook-xml-4.5.zip" + + version('4.5', '03083e288e87a7e829e437358da7ef9e') + + def install(self, spec, prefix): + cp = which('cp') + + install_args = ['-a', '-t', prefix] + install_args.extend(glob.glob('*')) + + cp(*install_args) diff --git a/var/spack/packages/doxygen/package.py b/var/spack/packages/doxygen/package.py new file mode 100644 index 00000000000..3d4a4e47a78 --- /dev/null +++ b/var/spack/packages/doxygen/package.py @@ -0,0 +1,25 @@ +#------------------------------------------------------------------------------ +# Author: Justin Too +# Date: September 11, 2015 +#------------------------------------------------------------------------------ + +from spack import * + +class Doxygen(Package): + """Doxygen is the de facto standard tool for generating documentation + from annotated C++ sources, but it also supports other popular programming + languages such as C, Objective-C, C#, PHP, Java, Python, IDL (Corba, + Microsoft, and UNO/OpenOffice flavors), Fortran, VHDL, Tcl, and to some extent D.. + """ + homepage = "http://www.stack.nl/~dimitri/doxygen/" + url = "http://ftp.stack.nl/pub/users/dimitri/doxygen-1.8.10.src.tar.gz" + + version('1.8.10', '79767ccd986f12a0f949015efb5f058f') + + depends_on("cmake@2.8.12:") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/packages/eigen/package.py b/var/spack/packages/eigen/package.py deleted file mode 100644 index b4d89ea3a0e..00000000000 --- a/var/spack/packages/eigen/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class Eigen(Package): - """Eigen is a C++ template library for linear algebra: matrices, vectors, - numerical solvers, and related algorithms..""" - homepage = "http://eigen.tuxfamily.org/" - url = "http://bitbucket.org/eigen/eigen/get/3.2.5.tar.gz" - - version('3.2.5', '8cc513ac6ec687117acadddfcacf551b') - - def install(self, spec, prefix): - import os - os.system("mkdir ./build_dir && cd ./build_dir") - cmake('../', *std_cmake_args) - - make() - make("install") diff --git a/var/spack/packages/elfutils/package.py b/var/spack/packages/elfutils/package.py new file mode 100644 index 00000000000..926d2345846 --- /dev/null +++ b/var/spack/packages/elfutils/package.py @@ -0,0 +1,26 @@ +from spack import * + +class Elfutils(Package): + """elfutils is a collection of various binary tools such as + eu-objdump, eu-readelf, and other utilities that allow you to + inspect and manipulate ELF files. Refer to Table 5.Tools Included + in elfutils for Red Hat Developer for a complete list of binary + tools that are distributed with the Red Hat Developer Toolset + version of elfutils.""" + + homepage = "https://fedorahosted.org/elfutils/" + + version('0.163', + git='git://git.fedorahosted.org/git/elfutils.git', + tag='elfutils-0.163') + + provides('elf') + + def install(self, spec, prefix): + autoreconf = which('autoreconf') + autoreconf('-if') + + configure('--prefix=%s' % prefix, '--enable-maintainer-mode') + make() + make("install") + diff --git a/var/spack/packages/fish/package.py b/var/spack/packages/fish/package.py new file mode 100644 index 00000000000..12255587057 --- /dev/null +++ b/var/spack/packages/fish/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Fish(Package): + """fish is a smart and user-friendly command line shell for OS X, Linux, and + the rest of the family. + """ + + homepage = "http://fishshell.com/" + url = "http://fishshell.com/files/2.2.0/fish-2.2.0.tar.gz" + list_url = homepage + + version('2.2.0', 'a76339fd14ce2ec229283c53e805faac48c3e99d9e3ede9d82c0554acfc7b77a') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/flux/package.py b/var/spack/packages/flux/package.py new file mode 100644 index 00000000000..c128f46be8b --- /dev/null +++ b/var/spack/packages/flux/package.py @@ -0,0 +1,36 @@ +from spack import * +import os + +class Flux(Package): + """ A next-generation resource manager (pre-alpha) """ + + homepage = "https://github.com/flux-framework/flux-core" + url = "https://github.com/flux-framework/flux-core" + + version('master', branch='master', git='https://github.com/flux-framework/flux-core') + + # Also needs autotools, but should use the system version if available + depends_on("zeromq@4.0.4:") + depends_on("czmq@2.2:") + depends_on("lua@5.1:5.1.99") + depends_on("munge") + depends_on("libjson-c") + depends_on("libxslt") + # TODO: This provides a catalog, hacked with environment below for now + depends_on("docbook-xml") + depends_on("asciidoc") + depends_on("python") + depends_on("py-cffi") + + def install(self, spec, prefix): + # Bootstrap with autotools + bash = which('bash') + bash('./autogen.sh') + + # Fix asciidoc dependency on xml style sheets and whatnot + os.environ['XML_CATALOG_FILES'] = os.path.join(spec['docbook-xml'].prefix, + 'catalog.xml') + # Configure, compile & install + configure("--prefix=" + prefix) + make("install", "V=1") + diff --git a/var/spack/packages/gflags/package.py b/var/spack/packages/gflags/package.py new file mode 100644 index 00000000000..62dd80a0944 --- /dev/null +++ b/var/spack/packages/gflags/package.py @@ -0,0 +1,21 @@ +import os +from spack import * + +class Gflags(Package): + """The gflags package contains a C++ library that implements + commandline flags processing. It includes built-in support for + standard types such as string and the ability to define flags + in the source file in which they are used. Online documentation + available at: https://gflags.github.io/gflags/""" + + homepage = "https://gflags.github.io/gflags" + url = "https://github.com/gflags/gflags/archive/v2.1.2.tar.gz" + + version('2.1.2', 'ac432de923f9de1e9780b5254884599f') + + def install(self, spec, prefix): + cmake("-DCMAKE_INSTALL_PREFIX=" + prefix, + "-DBUILD_SHARED_LIBS=ON") + make() + make("test") + make("install") diff --git a/var/spack/packages/ghostscript/package.py b/var/spack/packages/ghostscript/package.py new file mode 100644 index 00000000000..0ab49d425f7 --- /dev/null +++ b/var/spack/packages/ghostscript/package.py @@ -0,0 +1,17 @@ +from spack import * + +class Ghostscript(Package): + """an interpreter for the PostScript language and for PDF. """ + homepage = "http://ghostscript.com/" + url = "http://downloads.ghostscript.com/public/ghostscript-9.16.tar.gz" + + version('9.16', '829319325bbdb83f5c81379a8f86f38f') + + parallel = False + + def install(self, spec, prefix): + configure("--prefix=%s" %prefix, "--enable-shared") + + make() + make("install") + diff --git a/var/spack/packages/glog/package.py b/var/spack/packages/glog/package.py new file mode 100644 index 00000000000..d73386b394b --- /dev/null +++ b/var/spack/packages/glog/package.py @@ -0,0 +1,15 @@ +import os +from spack import * + +class Glog(Package): + """C++ implementation of the Google logging module.""" + + homepage = "https://github.com/google/glog" + url = "https://github.com/google/glog/archive/v0.3.3.tar.gz" + + version('0.3.3', 'c1f86af27bd9c73186730aa957607ed0') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/packages/graphviz/package.py b/var/spack/packages/graphviz/package.py index 8244feccfdb..7af7da1881a 100644 --- a/var/spack/packages/graphviz/package.py +++ b/var/spack/packages/graphviz/package.py @@ -1,24 +1,21 @@ from spack import * class Graphviz(Package): - """graph visualization software.""" + """Graph Visualization Software""" homepage = "http://www.graphviz.org" url = "http://www.graphviz.org/pub/graphviz/stable/SOURCES/graphviz-2.38.0.tar.gz" version('2.38.0', '5b6a829b2ac94efcd5fa3c223ed6d3ae') - version('2.36.0', '1f41664dba0c93109ac8b71216bf2b57') - depends_on("cairo@1.1.10") - depends_on("freetype@2.1.10") - depends_on("fontconfig") - depends_on("zlib@1.2.3") - # depends_on("libpng@1.2.10") - # depends_on("expat@2.0.0") - # depends_on("gd@2.0.34") + parallel = False + + depends_on("swig") + depends_on("python") + depends_on("ghostscript") def install(self, spec, prefix): - configure('--prefix=%s' % prefix) + configure("--prefix=%s" %prefix) - # FIXME: Add logic to build and install here make() make("install") + diff --git a/var/spack/packages/leveldb/package.py b/var/spack/packages/leveldb/package.py new file mode 100644 index 00000000000..da68a9cbcb2 --- /dev/null +++ b/var/spack/packages/leveldb/package.py @@ -0,0 +1,29 @@ +import os +import glob +from spack import * + +class Leveldb(Package): + """LevelDB is a fast key-value storage library written at Google + that provides an ordered mapping from string keys to string values.""" + + homepage = "https://github.com/google/leveldb" + url = "https://github.com/google/leveldb/archive/v1.18.tar.gz" + + version('1.18', '73770de34a2a5ab34498d2e05b2b7fa0') + + depends_on("snappy") + + def install(self, spec, prefix): + make() + + mkdirp(prefix.include) + mkdirp(prefix.lib) + + cp = which('cp') + + # cp --preserve=links libleveldb.* prefix/lib + args = glob.glob('libleveldb.*') + args.append(prefix + '/lib') + cp('--preserve=links', *args) + + cp('-r', 'include/leveldb', prefix + '/include') diff --git a/var/spack/packages/libdrm/package.py b/var/spack/packages/libdrm/package.py index 1cab5372bec..00736b7811a 100644 --- a/var/spack/packages/libdrm/package.py +++ b/var/spack/packages/libdrm/package.py @@ -2,7 +2,7 @@ class Libdrm(Package): """A userspace library for accessing the DRM, direct - rendering manager, on Linux, BSD and other operating + rendering manager, on Linux, BSD and other operating systems that support the ioctl interface.""" homepage = "http://dri.freedesktop.org/libdrm/" # no real website... @@ -11,11 +11,8 @@ class Libdrm(Package): version('2.4.59', '105ac7af1afcd742d402ca7b4eb168b6') version('2.4.33', '86e4e3debe7087d5404461e0032231c8') - depends_on("libpciaccess") - def install(self, spec, prefix): configure("--prefix=%s" % prefix) - make("clean") make() make("install") diff --git a/var/spack/packages/libelf/package.py b/var/spack/packages/libelf/package.py index bf2fefabd50..9338b8f393e 100644 --- a/var/spack/packages/libelf/package.py +++ b/var/spack/packages/libelf/package.py @@ -36,6 +36,8 @@ class Libelf(Package): version('0.8.13', '4136d7b4c04df68b686570afa26988ac') version('0.8.12', 'e21f8273d9f5f6d43a59878dc274fec7') + provides('elf') + def install(self, spec, prefix): configure("--prefix=" + prefix, "--enable-shared", diff --git a/var/spack/packages/libffi/package.py b/var/spack/packages/libffi/package.py index 2c1c4eed4d3..18acabb00cf 100644 --- a/var/spack/packages/libffi/package.py +++ b/var/spack/packages/libffi/package.py @@ -6,9 +6,9 @@ class Libffi(Package): to call any function specified by a call interface description at run time.""" homepage = "https://sourceware.org/libffi/" - url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz" - - version('3.1', 'f5898b29bbfd70502831a212d9249d10') + + version('3.2.1','83b89587607e3eb65c70d361f13bab43',url = "ftp://sourceware.org/pub/libffi/libffi-3.2.1.tar.gz") + #version('3.1', 'f5898b29bbfd70502831a212d9249d10',url = "ftp://sourceware.org/pub/libffi/libffi-3.1.tar.gz") # Has a bug $(lib64) instead of ${lib64} in libffi.pc def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/packages/libjson-c/package.py b/var/spack/packages/libjson-c/package.py new file mode 100644 index 00000000000..c0801cce9cd --- /dev/null +++ b/var/spack/packages/libjson-c/package.py @@ -0,0 +1,14 @@ +from spack import * + +class LibjsonC(Package): + """ A JSON implementation in C """ + homepage = "https://github.com/json-c/json-c/wiki" + url = "https://s3.amazonaws.com/json-c_releases/releases/json-c-0.11.tar.gz" + + version('0.11', 'aa02367d2f7a830bf1e3376f77881e98') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/packages/libpciaccess/package.py b/var/spack/packages/libpciaccess/package.py index 371fbb66854..6022fc34a30 100644 --- a/var/spack/packages/libpciaccess/package.py +++ b/var/spack/packages/libpciaccess/package.py @@ -1,16 +1,21 @@ from spack import * class Libpciaccess(Package): - """Generic PCI access library""" - homepage = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" - url = "http://pkgs.fedoraproject.org/repo/pkgs/libpciaccess/libpciaccess-0.13.2.tar.bz2/b7c0d3afce14eedca57312a3141ec13a/libpciaccess-0.13.2.tar.bz2" + """Generic PCI access library.""" - version('0.13.2', 'b7c0d3afce14eedca57312a3141ec13a') + homepage = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" + url = "http://cgit.freedesktop.org/xorg/lib/libpciaccess/" + + version('0.13.4', git='http://anongit.freedesktop.org/git/xorg/lib/libpciaccess.git', + tag='libpciaccess-0.13.4') + + depends_on('autoconf') + depends_on('libtool') def install(self, spec, prefix): - import os - os.system("autoconf") - configure('--prefix=%s' % prefix) + from subprocess import call + call(["./autogen.sh"]) + configure("--prefix=%s" % prefix) make() make("install") diff --git a/var/spack/packages/libsodium/package.py b/var/spack/packages/libsodium/package.py new file mode 100644 index 00000000000..1c8a16d9984 --- /dev/null +++ b/var/spack/packages/libsodium/package.py @@ -0,0 +1,19 @@ +from spack import * + +class Libsodium(Package): + """Sodium is a modern, easy-to-use software library for encryption, + decryption, signatures, password hashing and more.""" + homepage = "https://download.libsodium.org/doc/" + url = "https://download.libsodium.org/libsodium/releases/libsodium-1.0.3.tar.gz" + + version('1.0.3', 'b3bcc98e34d3250f55ae196822307fab') + version('1.0.2', 'dc40eb23e293448c6fc908757738003f') + version('1.0.1', '9a221b49fba7281ceaaf5e278d0f4430') + version('1.0.0', '3093dabe4e038d09f0d150cef064b2f7') + version('0.7.1', 'c224fe3923d1dcfe418c65c8a7246316') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + + make() + make("install") diff --git a/var/spack/packages/libxml2/package.py b/var/spack/packages/libxml2/package.py index 72199d8def8..3a0af6b3684 100644 --- a/var/spack/packages/libxml2/package.py +++ b/var/spack/packages/libxml2/package.py @@ -9,11 +9,12 @@ class Libxml2(Package): version('2.9.2', '9e6a9aca9d155737868b3dc5fd82f788') + extends('python') depends_on('zlib') depends_on('xz') def install(self, spec, prefix): - configure("--prefix=%s" % prefix, - "--without-python") + configure("--prefix=%s" % prefix) + make() make("install") diff --git a/var/spack/packages/lmdb/package.py b/var/spack/packages/lmdb/package.py new file mode 100644 index 00000000000..875b8100c5a --- /dev/null +++ b/var/spack/packages/lmdb/package.py @@ -0,0 +1,39 @@ +import os +from spack import * + +class Lmdb(Package): + """Read-only mirror of official repo on openldap.org. Issues and + pull requests here are ignored. Use OpenLDAP ITS for issues. + http://www.openldap.org/software/repo.html""" + + + homepage = "http://www.openldap.org/software/repo.html" + url = "https://github.com/LMDB/lmdb/archive/LMDB_0.9.16.tar.gz" + + version('0.9.16', '0de89730b8f3f5711c2b3a4ba517b648') + + def install(self, spec, prefix): + os.chdir('libraries/liblmdb') + + make() + + mkdirp(prefix.bin) + mkdirp(prefix + '/man/man1') + mkdirp(prefix.lib) + mkdirp(prefix.include) + + bins = ['mdb_stat', 'mdb_copy', 'mdb_dump', 'mdb_load'] + for f in bins: + install(f, prefix.bin) + + mans = ['mdb_stat.1', 'mdb_copy.1', 'mdb_dump.1', 'mdb_load.1'] + for f in mans: + install(f, prefix + '/man/man1') + + libs = ['liblmdb.a', 'liblmdb.so'] + for f in libs: + install(f, prefix.lib) + + includes = ['lmdb.h'] + for f in includes: + install(f, prefix.include) diff --git a/var/spack/packages/lua/package.py b/var/spack/packages/lua/package.py new file mode 100644 index 00000000000..57c443cc2d6 --- /dev/null +++ b/var/spack/packages/lua/package.py @@ -0,0 +1,26 @@ +from spack import * +import os + +class Lua(Package): + """ The Lua programming language interpreter and library """ + homepage = "http://www.lua.org" + url = "http://www.lua.org/ftp/lua-5.1.5.tar.gz" + + version('5.3.1', '797adacada8d85761c079390ff1d9961') + version('5.3.0', 'a1b0a7e92d0c85bbff7a8d27bf29f8af') + version('5.2.4', '913fdb32207046b273fdb17aad70be13') + version('5.2.3', 'dc7f94ec6ff15c985d2d6ad0f1b35654') + version('5.2.2', 'efbb645e897eae37cad4344ce8b0a614') + version('5.2.1', 'ae08f641b45d737d12d30291a5e5f6e3') + version('5.2.0', 'f1ea831f397214bae8a265995ab1a93e') + version('5.1.5', '2e115fe26e435e33b0d5c022e4490567') + version('5.1.4', 'd0870f2de55d59c1c8419f36e8fac150') + version('5.1.3', 'a70a8dfaa150e047866dc01a46272599') + + depends_on('ncurses') + + def install(self, spec, prefix): + make('INSTALL_TOP=%s' % prefix, + 'MYLDFLAGS=-L%s/lib' % spec['ncurses'].prefix, + 'linux', + 'install') diff --git a/var/spack/packages/mesa/package.py b/var/spack/packages/mesa/package.py index 2dba878a77c..2a04a8fd511 100644 --- a/var/spack/packages/mesa/package.py +++ b/var/spack/packages/mesa/package.py @@ -1,7 +1,7 @@ from spack import * class Mesa(Package): - """Mesa is an open-source implementation of the OpenGL + """Mesa is an open-source implementation of the OpenGL specification - a system for rendering interactive 3D graphics.""" homepage = "http://www.mesa3d.org" @@ -11,9 +11,10 @@ class Mesa(Package): # version('10.4.4', '8d863a3c209bf5116b2babfccccc68ce') version('8.0.5', 'cda5d101f43b8784fa60bdeaca4056f2') - # mesa 7.x, 8.x, 9.x + # mesa 7.x, 8.x, 9.x depends_on("libdrm@2.4.33") depends_on("llvm@3.0") + depends_on("libxml2") # patch("llvm-fixes.patch") # using newer llvm diff --git a/var/spack/packages/mpich/package.py b/var/spack/packages/mpich/package.py index ce07335bba0..dfff22152da 100644 --- a/var/spack/packages/mpich/package.py +++ b/var/spack/packages/mpich/package.py @@ -38,7 +38,6 @@ class Mpich(Package): provides('mpi@:3', when='@3:') provides('mpi@:1', when='@1:') - def setup_dependent_environment(self, module, spec, dep_spec): """For dependencies, make mpicc's use spack wrapper.""" os.environ['MPICH_CC'] = 'cc' diff --git a/var/spack/packages/munge/package.py b/var/spack/packages/munge/package.py new file mode 100644 index 00000000000..c737ca03546 --- /dev/null +++ b/var/spack/packages/munge/package.py @@ -0,0 +1,20 @@ +from spack import * +import os + +class Munge(Package): + """ MUNGE Uid 'N' Gid Emporium """ + homepage = "https://code.google.com/p/munge/" + url = "https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2" + + version('0.5.11', 'bd8fca8d5f4c1fcbef1816482d49ee01', url='https://github.com/dun/munge/releases/download/munge-0.5.11/munge-0.5.11.tar.bz2') + + depends_on('openssl') + depends_on('libgcrypt') + + def install(self, spec, prefix): + os.makedirs(os.path.join(prefix, "lib/systemd/system")) + configure("--prefix=%s" % prefix) + + make() + make("install") + diff --git a/var/spack/packages/ncurses/package.py b/var/spack/packages/ncurses/package.py index a22e83503f9..8f5763bfdd2 100644 --- a/var/spack/packages/ncurses/package.py +++ b/var/spack/packages/ncurses/package.py @@ -21,3 +21,11 @@ def install(self, spec, prefix): make() make("install") + configure("--prefix=%s" % prefix, + "--with-shared", + "--disable-widec", + "--disable-pc-files", + "--without-ada") + make() + make("install") + diff --git a/var/spack/packages/netlib-blas/package.py b/var/spack/packages/netlib-blas/package.py index 0a6cdb04423..85e97323d35 100644 --- a/var/spack/packages/netlib-blas/package.py +++ b/var/spack/packages/netlib-blas/package.py @@ -9,6 +9,8 @@ class NetlibBlas(Package): version('3.5.0', 'b1d3e3e425b2e44a06760ff173104bdf') + variant('fpic', default=False, description="Build with -fpic compiler option") + # virtual dependency provides('blas') @@ -23,6 +25,10 @@ def patch(self): mf.filter('^LOADER.*', 'LOADER = f90') mf.filter('^CC =.*', 'CC = cc') + if '+fpic' in self.spec: + mf.filter('^OPTS.*=.*', 'OPTS = -O2 -frecursive -fpic') + mf.filter('^CFLAGS =.*', 'CFLAGS = -O3 -fpic') + def install(self, spec, prefix): make('blaslib') diff --git a/var/spack/packages/lapack/package.py b/var/spack/packages/netlib-lapack/package.py similarity index 77% rename from var/spack/packages/lapack/package.py rename to var/spack/packages/netlib-lapack/package.py index d9d37e3e4aa..fb6b99e27c3 100644 --- a/var/spack/packages/lapack/package.py +++ b/var/spack/packages/netlib-lapack/package.py @@ -1,6 +1,6 @@ from spack import * -class Lapack(Package): +class NetlibLapack(Package): """ LAPACK version 3.X is a comprehensive FORTRAN library that does linear algebra operations including matrix inversions, least @@ -18,9 +18,16 @@ class Lapack(Package): version('3.4.0', '02d5706ec03ba885fc246e5fa10d8c70') version('3.3.1', 'd0d533ec9a5b74933c2a1e84eedc58b4') + variant('shared', default=False, description="Build shared library version") + + # virtual dependency + provides('lapack') + # blas is a virtual dependency. depends_on('blas') + depends_on('cmake') + # Doesn't always build correctly in parallel parallel = False @@ -39,7 +46,14 @@ def get_blas_libs(self): def install(self, spec, prefix): blas_libs = ";".join(self.get_blas_libs()) - cmake(".", '-DBLAS_LIBRARIES=' + blas_libs, *std_cmake_args) + cmake_args = [".", '-DBLAS_LIBRARIES=' + blas_libs] + + if '+shared' in spec: + cmake_args.append('-DBUILD_SHARED_LIBS=ON') + + cmake_args += std_cmake_args + + cmake(*cmake_args) make() make("install") diff --git a/var/spack/packages/pidx/package.py b/var/spack/packages/pidx/package.py new file mode 100644 index 00000000000..81aed62fb10 --- /dev/null +++ b/var/spack/packages/pidx/package.py @@ -0,0 +1,21 @@ +from spack import * + +class Pidx(Package): + """PIDX Parallel I/O Library. + + PIDX is an efficient parallel I/O library that reads and writes + multiresolution IDX data files. + """ + + homepage = "http://www.cedmav.com/pidx" + + version('1.0', git='https://github.com/sci-visus/PIDX.git', + commit='6afa1cf71d1c41263296dc049c8fabaf73c296da') + + depends_on("mpi") + + def install(self, spec, prefix): + with working_dir('spack-build', create=True): + cmake('..', *std_cmake_args) + make() + make("install") diff --git a/var/spack/packages/pkg-config/package.py b/var/spack/packages/pkg-config/package.py new file mode 100644 index 00000000000..9964c6ce346 --- /dev/null +++ b/var/spack/packages/pkg-config/package.py @@ -0,0 +1,17 @@ +from spack import * + +class PkgConfig(Package): + """pkg-config is a helper tool used when compiling applications and libraries""" + homepage = "http://www.freedesktop.org/wiki/Software/pkg-config/" + url = "http://pkgconfig.freedesktop.org/releases/pkg-config-0.28.tar.gz" + + version('0.28', 'aa3c86e67551adc3ac865160e34a2a0d') + + parallel = False + + def install(self, spec, prefix): + configure("--prefix=%s" %prefix, "--enable-shared") + + make() + make("install") + diff --git a/var/spack/packages/protobuf/package.py b/var/spack/packages/protobuf/package.py new file mode 100644 index 00000000000..34085c7ce95 --- /dev/null +++ b/var/spack/packages/protobuf/package.py @@ -0,0 +1,16 @@ +import os +from spack import * + +class Protobuf(Package): + """Google's data interchange format.""" + + homepage = "https://developers.google.com/protocol-buffers" + url = "https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.bz2" + + version('2.5.0', 'a72001a9067a4c2c4e0e836d0f92ece4') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("check") + make("install") diff --git a/var/spack/packages/py-Cython/package.py b/var/spack/packages/py-Cython/package.py deleted file mode 100644 index c2501391143..00000000000 --- a/var/spack/packages/py-Cython/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyCython(Package): - """The Cython compiler for writing C extensions for the Python language.""" - homepage = "http://www.cython.org" - version("0.21.2", "d21adb870c75680dc857cd05d41046a4", - url="https://pypi.python.org/packages/source/C/Cython/Cython-0.21.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-Distutils2/package.py b/var/spack/packages/py-Distutils2/package.py deleted file mode 100644 index 8690f569fee..00000000000 --- a/var/spack/packages/py-Distutils2/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - -class PyDistutils2(Package): - """Python Packaging Library.""" - - homepage = "https://hg.python.org/distutils2" - url = "https://pypi.python.org/packages/source/D/Distutils2/Distutils2-1.0a4.tar.gz" - - version('1.0a4', '52bc9dffb394970c27e02853ae3a3241') - - depends_on("python") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) - diff --git a/var/spack/packages/py-astroML/package.py b/var/spack/packages/py-astroML/package.py deleted file mode 100644 index 1583e249d21..00000000000 --- a/var/spack/packages/py-astroML/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyAstroml(Package): - """tools for machine learning and data mining in Astronomy""" - homepage = "http://astroML.github.com" - version("0.2", "85f558368546660564f20b30efafb024", - url="https://pypi.python.org/packages/source/a/astroML/astroML-0.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-astropy/package.py b/var/spack/packages/py-astropy/package.py deleted file mode 100644 index 5d2d0619ccf..00000000000 --- a/var/spack/packages/py-astropy/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyAstropy(Package): - """Community-developed python astronomy tools""" - homepage = "http://astropy.org" - version("0.4.2", "6dc4f643cde37ba0a8b4967dc8becee8", - url="https://pypi.python.org/packages/source/a/astropy/astropy-0.4.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-autopep8/package.py b/var/spack/packages/py-autopep8/package.py deleted file mode 100644 index 39a2288ea04..00000000000 --- a/var/spack/packages/py-autopep8/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyAutopep8(Package): - """A tool that automatically formats Python code to conform to the PEP 8 style guide""" - homepage = "https://github.com/hhatto/autopep8" - version("1.1", "7998358d8f0efd77dcb2cc8e34e3cb5c", - url="https://pypi.python.org/packages/source/a/autopep8/autopep8-1.1.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-certifi/package.py b/var/spack/packages/py-certifi/package.py deleted file mode 100644 index 442903e6a0f..00000000000 --- a/var/spack/packages/py-certifi/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyCertifi(Package): - """Python package for providing Mozilla's CA Bundle.""" - homepage = "http://python-requests.org" - version("14.05.14", "315ea4e50673a16ab047099f816fd32a", - url="https://pypi.python.org/packages/source/c/certifi/certifi-14.05.14.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-decorator/package.py b/var/spack/packages/py-decorator/package.py deleted file mode 100644 index 00c1c541d44..00000000000 --- a/var/spack/packages/py-decorator/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyDecorator(Package): - """Better living through Python with decorators""" - homepage = "https://github.com/micheles/decorator" - version("4.0.4", "dd3a0669e1e6f09699eefa2c7fbd9756", - url="https://pypi.python.org/packages/source/d/decorator/decorator-4.0.4.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-emcee/package.py b/var/spack/packages/py-emcee/package.py deleted file mode 100644 index 544066b92d1..00000000000 --- a/var/spack/packages/py-emcee/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyEmcee(Package): - """Kick ass affine-invariant ensemble MCMC sampling""" - homepage = "http://dan.iel.fm/emcee/" - version("2.1.0", "c6b6fad05c824d40671d4a4fc58dfff7", - url="https://pypi.python.org/packages/source/e/emcee/emcee-2.1.0.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-fitsio/package.py b/var/spack/packages/py-fitsio/package.py deleted file mode 100644 index 046bc386d57..00000000000 --- a/var/spack/packages/py-fitsio/package.py +++ /dev/null @@ -1,13 +0,0 @@ -from spack import * - -class PyFitsio(Package): - """A full featured python library to read from and write to FITS files.""" - homepage = "https://github.com/esheldon/fitsio" - version("0.9.7", "75fa05f999ae8f55c8290bd78ada49e7", - url="https://pypi.python.org/packages/source/f/fitsio/fitsio-0.9.7.tar.gz") - - extends("python") - depends_on("py-numpy") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-gnureadline/package.py b/var/spack/packages/py-gnureadline/package.py deleted file mode 100644 index 51d1c6227eb..00000000000 --- a/var/spack/packages/py-gnureadline/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyGnureadline(Package): - """The standard Python readline extension statically linked against the GNU readline library.""" - homepage = "http://github.com/ludwigschwardt/python-gnureadline" - version("6.3.3", "c4af83c9a3fbeac8f2da9b5a7c60e51c", - url="https://pypi.python.org/packages/source/g/gnureadline/gnureadline-6.3.3.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-ipython/package.py b/var/spack/packages/py-ipython/package.py index 3b94b71f295..8d0e64a07f1 100644 --- a/var/spack/packages/py-ipython/package.py +++ b/var/spack/packages/py-ipython/package.py @@ -3,29 +3,14 @@ class PyIpython(Package): """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" homepage = "https://pypi.python.org/pypi/ipython" + url = "https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz" - version('3.0.0','b3f00f3c0be036fafef3b0b9d663f27e', - url='https://pypi.python.org/packages/source/i/ipython/ipython-3.0.0.tar.gz') - version('4.0.0','c2fecbcf1c0fbdc82625c77a50733dd6', - url='https://pypi.python.org/packages/source/i/ipython/ipython-4.0.0.tar.gz') - version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', - url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") + version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf') version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') - version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', - url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') extends('python') depends_on('py-pygments') - depends_on('py-setuptools@18.1') - depends_on('py-traitlets') - depends_on('py-pyzmq') - depends_on('py-pexpect') - depends_on('py-tornado') - depends_on('py-jinja2') + depends_on('py-setuptools') def install(self, spec, prefix): - import os - # os.system("spack activate py-setuptools@18.1") - # os.system("easy_install pip") - # os.system("pip install ipython[notebook]") python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython/package.py.backup b/var/spack/packages/py-ipython/package.py.backup deleted file mode 100755 index 1ed8b57e605..00000000000 --- a/var/spack/packages/py-ipython/package.py.backup +++ /dev/null @@ -1,27 +0,0 @@ -from spack import * - -class PyIpython(Package): - """IPython provides a rich toolkit to help you make the most out of using Python interactively.""" - homepage = "https://pypi.python.org/pypi/ipython" - - version('2.3.1', '2b7085525dac11190bfb45bb8ec8dcbf', - url="https://pypi.python.org/packages/source/i/ipython/ipython-2.3.1.tar.gz") - version('3.1.0', 'a749d90c16068687b0ec45a27e72ef8f') - version('3.2.1', 'f4c93d67ac4b2d4fc69df693b6f3c9e0', - url='https://github.com/ipython/ipython/archive/rel-3.2.1.tar.gz') - - extends('python') - depends_on('py-pygments') - depends_on('py-setuptools@18.1') - depends_on('py-traitlets') - depends_on('py-pyzmq') - depends_on('py-pexpect') - depends_on('py-tornado') - depends_on('py-Jinja2') - - def install(self, spec, prefix): - import os - # os.system("spack activate py-setuptools@18.1") - # os.system("easy_install pip") - # os.system("pip install ipython[notebook]") - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-ipython_genutils/package.py b/var/spack/packages/py-ipython_genutils/package.py deleted file mode 100644 index 39b15eec730..00000000000 --- a/var/spack/packages/py-ipython_genutils/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyIpythonGenutils(Package): - """Vestigial utilities from IPython""" - homepage = "http://ipython.org" - version("0.1.0", "9a8afbe0978adbcbfcb3b35b2d015a56", - url="https://pypi.python.org/packages/source/i/ipython_genutils/ipython_genutils-0.1.0.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-joblib/package.py b/var/spack/packages/py-joblib/package.py deleted file mode 100644 index 10672314def..00000000000 --- a/var/spack/packages/py-joblib/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyJoblib(Package): - """Lightweight pipelining: using Python functions as pipeline jobs.""" - homepage = "http://packages.python.org/joblib/" - version("0.8.4", "90a1c25cc4dc4a8e3536093dbc35cff3", - url="https://pypi.python.org/packages/source/j/joblib/joblib-0.8.4.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-jsonschema/package.py b/var/spack/packages/py-jsonschema/package.py deleted file mode 100644 index 9a7fc218622..00000000000 --- a/var/spack/packages/py-jsonschema/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class PyJsonschema(Package): - """implementation of JSON Schema validation for Python""" - homepage = "https://github.com/Julian/jsonschema" - url = "https://pypi.python.org/packages/source/j/jsonschema/jsonschema-2.5.1.tar.gz" - - version('2.4.0','661f85c3d23094afbb9ac3c0673840bf', - url='https://pypi.python.org/packages/source/j/jsonschema/jsonschema-2.4.0.tar.gz') - version('2.5.1', '374e848fdb69a3ce8b7e778b47c30640') - - extends("python") - depends_on("py-setuptools@18.1") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-matplotlib/package.py b/var/spack/packages/py-matplotlib/package.py index b9244daf562..e7ce3dfd24a 100644 --- a/var/spack/packages/py-matplotlib/package.py +++ b/var/spack/packages/py-matplotlib/package.py @@ -12,6 +12,7 @@ class PyMatplotlib(Package): extends('python', ignore=r'bin/nosetests.*$') depends_on('py-pyside') + depends_on('py-ipython') depends_on('py-pyparsing') depends_on('py-six') depends_on('py-dateutil') diff --git a/var/spack/packages/py-mock/package.py b/var/spack/packages/py-mock/package.py index c43842c1878..3b08428ba0c 100644 --- a/var/spack/packages/py-mock/package.py +++ b/var/spack/packages/py-mock/package.py @@ -1,12 +1,17 @@ from spack import * class PyMock(Package): - """A Python Mocking and Patching Library for Testing""" - homepage = "http://www.voidspace.org.uk/python/mock/" - version("1.0.1", "c3971991738caa55ec7c356bbc154ee2", - url="https://pypi.python.org/packages/source/m/mock/mock-1.0.1.tar.gz") + """mock is a library for testing in Python. It allows you to replace parts + of your system under test with mock objects and make assertions about how + they have been used.""" - extends("python") + homepage = "https://github.com/testing-cabal/mock" + url = "https://pypi.python.org/packages/source/m/mock/mock-1.3.0.tar.gz" + + version('1.3.0', '73ee8a4afb3ff4da1b4afa287f39fdeb') + + extends('python') + depends_on('py-setuptools@17.1:') def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) + python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-numexpr/package.py b/var/spack/packages/py-numexpr/package.py deleted file mode 100644 index 283ea18772e..00000000000 --- a/var/spack/packages/py-numexpr/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyNumexpr(Package): - """Fast numerical expression evaluator for NumPy""" - homepage = "https://github.com/pydata/numexpr" - version("2.4", "df7e8d9e9dbb145b56d43c465c2bf854", - url="https://pypi.python.org/packages/source/n/numexpr/numexpr-2.4.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-numpy/package.py b/var/spack/packages/py-numpy/package.py index 6534e063d23..efa109a3e9c 100644 --- a/var/spack/packages/py-numpy/package.py +++ b/var/spack/packages/py-numpy/package.py @@ -7,9 +7,22 @@ class PyNumpy(Package): version('1.9.1', '78842b73560ec378142665e712ae4ad9') version('1.9.2', 'a1ed53432dbcd256398898d35bc8e645') - + extends('python') depends_on('py-nose') + depends_on('netlib-blas+fpic') + depends_on('netlib-lapack+shared') + + def patch(self): + filter_file( + "possible_executables = \['(gfortran|g77|ifort|efl)", + "possible_executables = ['fc", + "numpy/distutils/fcompiler/gnu.py", + "numpy/distutils/fcompiler/intel.py") def install(self, spec, prefix): + with open('site.cfg', 'w') as f: + f.write('[DEFAULT]\n') + f.write('libraries=lapack,blas\n') + f.write('library_dirs=%s/lib:%s/lib\n' % (spec['blas'].prefix, spec['lapack'].prefix)) python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pandas/package.py b/var/spack/packages/py-pandas/package.py index 07ee12782f3..5b9997faa94 100644 --- a/var/spack/packages/py-pandas/package.py +++ b/var/spack/packages/py-pandas/package.py @@ -6,8 +6,6 @@ class PyPandas(Package): homepage = "http://pandas.pydata.org/" url = "https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73" - version('0.15.1','f6fd8d49fc801f6f3fa48f9117a6290b', - url='https://pypi.python.org/packages/source/p/pandas/pandas-0.15.1.tar.gz') version('0.16.0', 'bfe311f05dc0c351f8955fbd1e296e73') version('0.16.1', 'fac4f25748f9610a3e00e765474bdea8') diff --git a/var/spack/packages/py-pep8/package.py b/var/spack/packages/py-pep8/package.py deleted file mode 100644 index 0bcf4169228..00000000000 --- a/var/spack/packages/py-pep8/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPep8(Package): - """Python style guide checker""" - homepage = "http://pep8.readthedocs.org/" - version("1.6.1", "76cf60b245f8549cb458ffcd85710738", - url="https://pypi.python.org/packages/source/p/pep8/pep8-1.6.1.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pickleshare/package.py b/var/spack/packages/py-pickleshare/package.py deleted file mode 100644 index cf9ddb7db88..00000000000 --- a/var/spack/packages/py-pickleshare/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPickleshare(Package): - """Tiny 'shelve'-like database with concurrency support""" - homepage = "https://github.com/vivainio/pickleshare" - version("0.5", "25337740507cb855ad58bfcf60f7710e", - url="https://pypi.python.org/packages/source/p/pickleshare/pickleshare-0.5.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pip/package.py b/var/spack/packages/py-pip/package.py deleted file mode 100644 index 89b8570fe46..00000000000 --- a/var/spack/packages/py-pip/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPip(Package): - """The PyPA recommended tool for installing Python packages.""" - homepage = "https://pip.pypa.io/" - version("7.1.2", "3823d2343d9f3aaab21cf9c917710196", - url="https://pypi.python.org/packages/source/p/pip/pip-7.1.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-py/package.py b/var/spack/packages/py-py/package.py deleted file mode 100644 index 0ef04031243..00000000000 --- a/var/spack/packages/py-py/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPy(Package): - """library with cross-python path, ini-parsing, io, code, log facilities""" - homepage = "http://pylib.readthedocs.org/" - version("1.4.26", "30c3fd92a53f1a5ed6f3591c1fe75c0e", - url="https://pypi.python.org/packages/source/p/py/py-1.4.26.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-py4j/package.py b/var/spack/packages/py-py4j/package.py deleted file mode 100644 index 4c31f572982..00000000000 --- a/var/spack/packages/py-py4j/package.py +++ /dev/null @@ -1,15 +0,0 @@ -from spack import * - - -class PyPy4j(Package): - """Enables Python programs to dynamically access arbitrary Java objects""" - homepage = "https://www.py4j.org/" - url = "https://pypi.python.org/packages/source/p/py4j/py4j-0.9.tar.gz" - - version('0.9', 'b6fed5faef81a5368e3d50a91a5c9a60') - - extends("python") - depends_on("py-setuptools@18.1") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-pyside/package.py b/var/spack/packages/py-pyside/package.py index 8d093efd2c7..bb5da44d027 100644 --- a/var/spack/packages/py-pyside/package.py +++ b/var/spack/packages/py-pyside/package.py @@ -8,9 +8,12 @@ class PyPyside(Package): version('1.2.2', 'c45bc400c8a86d6b35f34c29e379e44d') + # TODO: make build dependency + # depends_on("cmake") + extends('python') depends_on('py-setuptools') - depends_on('qt@5.4.0') + depends_on('qt@:4') def patch(self): """Undo PySide RPATH handling and add Spack RPATH.""" diff --git a/var/spack/packages/py-pytest/package.py b/var/spack/packages/py-pytest/package.py deleted file mode 100644 index 3611b68862f..00000000000 --- a/var/spack/packages/py-pytest/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPytest(Package): - """pytest: simple powerful testing with Python""" - homepage = "http://pytest.org" - version("2.6.4", "14341e122f7e9031a0948eb6b01a2640", - url="https://pypi.python.org/packages/source/p/pytest/pytest-2.6.4.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-python-dateutil/package.py b/var/spack/packages/py-python-dateutil/package.py deleted file mode 100644 index 8efa114db1b..00000000000 --- a/var/spack/packages/py-python-dateutil/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyPythonDateutil(Package): - """Extensions to the standard Python datetime module""" - homepage = "http://labix.org/python-dateutil" - version("2.2", "c1f654d0ff7e33999380a8ba9783fd5c", - url="https://pypi.python.org/packages/source/p/python-dateutil/python-dateutil-2.2.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-pytz/package.py b/var/spack/packages/py-pytz/package.py index f67bb428549..da6311a784a 100644 --- a/var/spack/packages/py-pytz/package.py +++ b/var/spack/packages/py-pytz/package.py @@ -5,8 +5,6 @@ class PyPytz(Package): homepage = "https://pypi.python.org/pypi/pytz" url = "https://pypi.python.org/packages/source/p/pytz/pytz-2014.10.tar.gz" - version('2014.9','d42bda2f4c1e873e02fbd1e4acfd1b8c', - url='https://pypi.python.org/packages/source/p/pytz/pytz-2014.9.tar.gz') version('2014.10', 'eb1cb941a20c5b751352c52486aa1dd7') version('2015.4', '417a47b1c432d90333e42084a605d3d8') diff --git a/var/spack/packages/py-pyzmq/package.py b/var/spack/packages/py-pyzmq/package.py deleted file mode 100644 index f5551182b32..00000000000 --- a/var/spack/packages/py-pyzmq/package.py +++ /dev/null @@ -1,16 +0,0 @@ -from spack import * - -class PyPyzmq(Package): - """official Python binding for the ZeroMQ Messaging Library""" - homepage = "https://github.com/zeromq/pyzmq" - url = "https://pypi.python.org/packages/source/p/pyzmq/pyzmq-14.7.0.tar.gz" - - version('14.4.1','a6a7fb6950b1bf5333c6795f3edf196c', - url='https://pypi.python.org/packages/source/p/pyzmq/pyzmq-14.4.1.tar.gz') - version('14.7.0', '87e3abb33af5794db5ae85c667bbf324') - - extends('python') - depends_on('py-setuptools@18.1') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scikit-learn/package.py b/var/spack/packages/py-scikit-learn/package.py index 744b37e7e0e..5b078ce9011 100644 --- a/var/spack/packages/py-scikit-learn/package.py +++ b/var/spack/packages/py-scikit-learn/package.py @@ -9,9 +9,6 @@ class PyScikitLearn(Package): version('0.16.1', '363ddda501e3b6b61726aa40b8dbdb7e') extends('python') - depends_on('python@2.7.10') - depends_on('py-numpy@1.9.2') - depends_on('py-scipy@0.15.1') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-scipy/package.py b/var/spack/packages/py-scipy/package.py index bcbdb2100f0..3a1124cc15a 100644 --- a/var/spack/packages/py-scipy/package.py +++ b/var/spack/packages/py-scipy/package.py @@ -5,14 +5,14 @@ class PyScipy(Package): homepage = "https://pypi.python.org/pypi/scipy" url = "https://pypi.python.org/packages/source/s/scipy/scipy-0.15.0.tar.gz" - version('0.14.0','d7c7f4ccf8b07b08d6fe49d5cd51f85d', - url='https://pypi.python.org/packages/source/s/scipy/scipy-0.14.0.tar.gz') version('0.15.0', '639112f077f0aeb6d80718dc5019dc7a') version('0.15.1', 'be56cd8e60591d6332aac792a5880110') extends('python') depends_on('py-nose') depends_on('py-numpy') + depends_on('blas') + depends_on('lapack') def install(self, spec, prefix): python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-seaborn/package.py b/var/spack/packages/py-seaborn/package.py deleted file mode 100644 index 1e228b06227..00000000000 --- a/var/spack/packages/py-seaborn/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PySeaborn(Package): - """Seaborn: statistical data visualization""" - homepage = "http://stanford.edu/~mwaskom/software/seaborn/" - version("0.5.1", "2ce6ea7d3c67858c0b1f5793fa2043b8", - url="https://pypi.python.org/packages/source/s/seaborn/seaborn-0.5.1.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-setuptools/package.py b/var/spack/packages/py-setuptools/package.py index 053fe1ba80d..760ad4d6dbf 100644 --- a/var/spack/packages/py-setuptools/package.py +++ b/var/spack/packages/py-setuptools/package.py @@ -3,12 +3,11 @@ class PySetuptools(Package): """Easily download, build, install, upgrade, and uninstall Python packages.""" homepage = "https://pypi.python.org/pypi/setuptools" + url = "https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz" - version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06', - url='https://pypi.python.org/packages/source/s/setuptools/setuptools-18.1.tar.gz') - version('11.3.1', '01f69212e019a2420c1693fb43593930', - url="https://pypi.python.org/packages/source/s/setuptools/setuptools-11.3.tar.gz") - version('16.0', '0ace0b96233516fc5f7c857d086aa3ad',) + version('11.3.1', '01f69212e019a2420c1693fb43593930') + version('16.0', '0ace0b96233516fc5f7c857d086aa3ad') + version('18.1', 'f72e87f34fbf07f299f6cb46256a0b06') extends('python') diff --git a/var/spack/packages/py-six/package.py b/var/spack/packages/py-six/package.py index e9946a1e566..05c5bd00a98 100644 --- a/var/spack/packages/py-six/package.py +++ b/var/spack/packages/py-six/package.py @@ -5,8 +5,6 @@ class PySix(Package): homepage = "https://pypi.python.org/pypi/six" url = "https://pypi.python.org/packages/source/s/six/six-1.9.0.tar.gz" - version('1.8.0','1626eb24cc889110c38f7e786ec69885', - url='https://pypi.python.org/packages/source/s/six/six-1.8.0.tar.gz') version('1.9.0', '476881ef4012262dfc8adc645ee786c4') extends('python') diff --git a/var/spack/packages/py-statsmodels/package.py b/var/spack/packages/py-statsmodels/package.py deleted file mode 100644 index 393c772738c..00000000000 --- a/var/spack/packages/py-statsmodels/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyStatsmodels(Package): - """Statistical computations and models for use with SciPy""" - homepage = "http://statsmodels.sourceforge.net/" - version("0.6.1", "f7580ebf7d2a2c9b87abfad190dcb9a3", - url="https://pypi.python.org/packages/source/s/statsmodels/statsmodels-0.6.1.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-tables/package.py b/var/spack/packages/py-tables/package.py deleted file mode 100644 index 637ccf34cd6..00000000000 --- a/var/spack/packages/py-tables/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyTables(Package): - """Hierarchical datasets for Python""" - homepage = "http://www.pytables.org/" - version("3.1.1", "38d917f0c6dfb0bc28ce9ea0c3492524", - url="https://pypi.python.org/packages/source/t/tables/tables-3.1.1.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/py-tornado/package.py b/var/spack/packages/py-tornado/package.py deleted file mode 100644 index 4fc2d42c7bb..00000000000 --- a/var/spack/packages/py-tornado/package.py +++ /dev/null @@ -1,17 +0,0 @@ -from spack import * - -class PyTornado(Package): - """Python web framework and asynchronous networking library""" - homepage = "http://www.tornadoweb.org/" - url = "https://pypi.python.org/packages/source/t/tornado/tornado-4.2.1.tar.gz" - - version('4.0.2','985c0e704b765c33a6193d49d1935588', - url='https://pypi.python.org/packages/source/t/tornado/tornado-4.0.2.tar.gz') - version('4.2.1', 'd523204389cfb70121bb69709f551b20') - - extends('python') - depends_on('py-certifi') - depends_on('py-setuptools@18.1') - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-traitlets/package.py b/var/spack/packages/py-traitlets/package.py deleted file mode 100644 index 7b7f5c64cc4..00000000000 --- a/var/spack/packages/py-traitlets/package.py +++ /dev/null @@ -1,14 +0,0 @@ -from spack import * - -class PyTraitlets(Package): - """A lightweight Traits like module""" - homepage = "http://traitlets.readthedocs.org/" - url = "https://github.com/ipython/traitlets/archive/4.0.0.tar.gz" - - version('4.0.0', 'b5b95ea5941fd9619b4704dfd8201568') - - extends('python') - depends_on("py-setuptools@18.1") - - def install(self, spec, prefix): - python('setup.py', 'install', '--prefix=%s' % prefix) diff --git a/var/spack/packages/py-triangle-plot/package.py b/var/spack/packages/py-triangle-plot/package.py deleted file mode 100644 index 29c5f92857e..00000000000 --- a/var/spack/packages/py-triangle-plot/package.py +++ /dev/null @@ -1,12 +0,0 @@ -from spack import * - -class PyTrianglePlot(Package): - """Make some beautiful corner plots of samples.""" - homepage = "https://github.com/dfm/triangle.py" - version("0.0.6", "970a35a9bde6002d673e3188fe39f0ed", - url="https://pypi.python.org/packages/source/t/triangle_plot/triangle_plot-0.0.6.tar.gz") - - extends("python") - - def install(self, spec, prefix): - python("setup.py", "install", "--prefix=%s" % prefix) diff --git a/var/spack/packages/samtools/package.py b/var/spack/packages/samtools/package.py new file mode 100644 index 00000000000..72900398d86 --- /dev/null +++ b/var/spack/packages/samtools/package.py @@ -0,0 +1,18 @@ +from spack import * + +class Samtools(Package): + """SAM Tools provide various utilities for manipulating alignments in the SAM format, + including sorting, merging, indexing and generating + alignments in a per-position format""" + + homepage = "www.htslib.org" + version('1.2','988ec4c3058a6ceda36503eebecd4122',url = "https://github.com/samtools/samtools/releases/download/1.2/samtools-1.2.tar.bz2") + + depends_on("zlib") + depends_on("mpc") + parallel=False + patch("samtools1.2.patch",level=0) + + def install(self, spec, prefix): + make("prefix=%s" % prefix, "install") + diff --git a/var/spack/packages/samtools/samtools1.2.patch b/var/spack/packages/samtools/samtools1.2.patch new file mode 100644 index 00000000000..ead3ab4e2cb --- /dev/null +++ b/var/spack/packages/samtools/samtools1.2.patch @@ -0,0 +1,20 @@ +--- Makefile 2015-02-03 08:27:34.000000000 -0800 ++++ Makefile.new 2015-07-21 10:38:27.881406892 -0700 +@@ -26,7 +26,7 @@ + CFLAGS = -g -Wall -O2 + LDFLAGS = + LDLIBS = +-DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=1 ++DFLAGS= -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE -D_CURSES_LIB=0 + LOBJS= bam_aux.o bam.o bam_import.o sam.o \ + sam_header.o bam_plbuf.o + AOBJS= bam_index.o bam_plcmd.o sam_view.o \ +@@ -37,7 +37,7 @@ + faidx.o stats.o stats_isize.o bam_flags.o bam_split.o \ + bam_tview.o bam_tview_curses.o bam_tview_html.o bam_lpileup.o + INCLUDES= -I. -I$(HTSDIR) +-LIBCURSES= -lcurses # -lXCurses ++#LIBCURSES= -lcurses # -lXCurses + + prefix = /usr/local + exec_prefix = $(prefix) diff --git a/var/spack/packages/scorep/package.py b/var/spack/packages/scorep/package.py index 32a772e3dbe..f013bd1cbb7 100644 --- a/var/spack/packages/scorep/package.py +++ b/var/spack/packages/scorep/package.py @@ -1,11 +1,10 @@ # FIXME: Add copyright statement from spack import * -from contextlib import closing class Scorep(Package): - """The Score-P measurement infrastructure is a highly scalable and - easy-to-use tool suite for profiling, event tracing, and online + """The Score-P measurement infrastructure is a highly scalable and + easy-to-use tool suite for profiling, event tracing, and online analysis of HPC applications.""" # FIXME: add a proper url for your package's homepage here. @@ -20,7 +19,7 @@ class Scorep(Package): depends_on("mpi") depends_on("papi") # depends_on("otf2@1.2:1.2.1") # only Score-P 1.2.x - depends_on("otf2") + depends_on("otf2") depends_on("opari2") depends_on("cube@4.2:4.2.3") @@ -53,12 +52,12 @@ def install(self, spec, prefix): # Use a custom compiler configuration, otherwise the score-p # build system messes with spack's compiler settings. # Create these three files in the build directory - with closing(open("platform-backend-user-provided", "w")) as backend_file: + with open("platform-backend-user-provided", "w") as backend_file: backend_file.write(self.backend_user_provided) - with closing(open("platform-frontend-user-provided", "w")) as frontend_file: + with open("platform-frontend-user-provided", "w") as frontend_file: frontend_file.write(self.frontend_user_provided) - with closing(open("platform-mpi-user-provided", "w")) as mpi_file: - mpi_file.write(self.mpi_user_provided) + with open("platform-mpi-user-provided", "w") as mpi_file: + mpi_file.write(self.mpi_user_provided) configure_args = ["--prefix=%s" % prefix, "--with-custom-compilers", diff --git a/var/spack/packages/snappy/package.py b/var/spack/packages/snappy/package.py new file mode 100644 index 00000000000..c8f9ceef7da --- /dev/null +++ b/var/spack/packages/snappy/package.py @@ -0,0 +1,15 @@ +import os +from spack import * + +class Snappy(Package): + """A fast compressor/decompressor: https://code.google.com/p/snappy""" + + homepage = "https://code.google.com/p/snappy" + url = "https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz" + + version('1.1.3', '7358c82f133dc77798e4c2062a749b73') + + def install(self, spec, prefix): + configure("--prefix=" + prefix) + make() + make("install") diff --git a/var/spack/packages/stat/package.py b/var/spack/packages/stat/package.py index c40d65ae783..5d81e627317 100644 --- a/var/spack/packages/stat/package.py +++ b/var/spack/packages/stat/package.py @@ -9,17 +9,20 @@ class Stat(Package): version('2.1.0', 'ece26beaf057aa9134d62adcdda1ba91') version('2.0.0', 'c7494210b0ba26b577171b92838e1a9b') + variant('dysect', default=False, description="enable DySectAPI") + depends_on('libelf') depends_on('libdwarf') depends_on('dyninst') depends_on('graphlib') + depends_on('graphviz') depends_on('launchmon') depends_on('mrnet') patch('configure_mpicxx.patch', when='@2.1.0') def install(self, spec, prefix): - configure( + configure_args = [ "--enable-gui", "--prefix=%s" % prefix, "--disable-examples", # Examples require MPI: avoid this dependency. @@ -27,7 +30,11 @@ def install(self, spec, prefix): "--with-mrnet=%s" % spec['mrnet'].prefix, "--with-graphlib=%s" % spec['graphlib'].prefix, "--with-stackwalker=%s" % spec['dyninst'].prefix, - "--with-libdwarf=%s" % spec['libdwarf'].prefix) + "--with-libdwarf=%s" % spec['libdwarf'].prefix + ] + if '+dysect' in spec: + configure_args.append('--enable-dysectapi') + configure(*configure_args) make(parallel=False) make("install") diff --git a/var/spack/packages/vim/package.py b/var/spack/packages/vim/package.py index d0cf80b898d..4099b3257f7 100644 --- a/var/spack/packages/vim/package.py +++ b/var/spack/packages/vim/package.py @@ -35,15 +35,26 @@ class Vim(Package): variant('ruby', default=False, description="build with Ruby") depends_on('ruby', when='+ruby') + variant('cscope', default=False, description="build with cscope support") + depends_on('cscope', when='+cscope') + + variant('gui', default=False, description="build with gui (gvim)") + # virtual dependency? + def install(self, spec, prefix): feature_set = None for fs in self.feature_sets: if "+" + fs in spec: if feature_set is not None: - tty.error("Only one feature set allowed, both {} and {} specified".format( - feature_set, - fs)) + tty.error("Only one feature set allowed, both %s and %s specified" + % (feature_set, fs)) feature_set = fs + if '+gui' in spec: + if feature_set is not None: + if feature_set is not 'huge': + tty.error("+gui variant requires 'huge' feature set, %s was specified" + % feature_set) + feature_set = 'huge' if feature_set is None: feature_set = 'normal' @@ -60,6 +71,12 @@ def install(self, spec, prefix): else: configure_args.append("--enable-rubyinterp=dynamic") + if '+gui' in spec: + configure_args.append("--enable-gui=auto") + + if '+cscope' in spec: + configure_args.append("--enable-cscope") + configure("--prefix=%s" % prefix, *configure_args) make() diff --git a/var/spack/packages/zeromq/package.py b/var/spack/packages/zeromq/package.py new file mode 100644 index 00000000000..b5a1e3d4cde --- /dev/null +++ b/var/spack/packages/zeromq/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Zeromq(Package): + """ The ZMQ networking/concurrency library and core API """ + homepage = "http://zguide.zeromq.org/" + url = "http://download.zeromq.org/zeromq-4.1.2.tar.gz" + + version('4.1.2', '159c0c56a895472f02668e692d122685') + version('4.1.1', '0a4b44aa085644f25c177f79dc13f253') + version('4.0.7', '9b46f7e7b0704b83638ef0d461fd59ab') + version('4.0.6', 'd47dd09ed7ae6e7fd6f9a816d7f5fdf6') + version('4.0.5', '73c39f5eb01b9d7eaf74a5d899f1d03d') + + depends_on("libsodium") + + def install(self, spec, prefix): + configure("--with-libsodium","--prefix=%s" % prefix) + + make() + make("install") From 6da1a105cb448ec3c4f8ca4910b5bc9f12044001 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 15:03:34 -0800 Subject: [PATCH 093/375] Revert "adding python_recipe_parser as a submodule under the folder utils" This reverts commit 1501786fd9be0a01abe7f6bb917e416fb3256aa6. --- .gitmodules | 3 --- utils/python_recipe_parser | 1 - 2 files changed, 4 deletions(-) delete mode 100644 .gitmodules delete mode 160000 utils/python_recipe_parser diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index a097fcd8cf3..00000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "utils/python_recipe_parser"] - path = utils/python_recipe_parser - url = https://github.com/karenyyng/spack_python_package_parser diff --git a/utils/python_recipe_parser b/utils/python_recipe_parser deleted file mode 160000 index 3158074a2d7..00000000000 --- a/utils/python_recipe_parser +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3158074a2d74f99aa72289dfcf093b1b0ba4474e From aa28e4e81f80a1a388aabe589ca23955ebd9721b Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 15:13:48 -0800 Subject: [PATCH 094/375] Improved error messages for compiler_for_spec when either zero or multiple compilers returned. --- lib/spack/spack/compilers/__init__.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 87106282cf0..3a0a88b5e3f 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -219,7 +219,10 @@ def compiler_for_spec(compiler_spec, target): compilers = [c for c in compilers if c.modules is None] elif target.compiler_strategy == "MODULES": compilers = [c for c in compilers if c.modules is not None] - assert(len(compilers) == 1) + if len(compilers) < 1: + raise NoCompilerForSpecError(compiler_spec, target) + if len(compilers) > 1: + raise CompilerSpecInsufficientlySpecificError(compiler_spec) return compilers[0] @@ -253,3 +256,13 @@ def __init__(self, compiler_spec): class NoCompilersError(spack.error.SpackError): def __init__(self): super(NoCompilersError, self).__init__("Spack could not find any compilers!") + +class NoCompilerForSpecError(spack.error.SpackError): + def __init__(self, compiler_spec, target): + super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s", + compiler_spec, target) + +class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): + def __init__(self, compiler_spec): + super(CompilerSpecInsufficientlySpecificError, self).__init__("Multiple compilers satisfy spec %s", + compiler_spec) From 2b4dd8b9af427e2fc3c4a743f4f058a26c7d583f Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 16:30:27 -0800 Subject: [PATCH 095/375] Fixed target satisfaction and updated tests accordingly --- lib/spack/spack/concretize.py | 35 +--------------- lib/spack/spack/spec.py | 35 ++++++++++++++++ lib/spack/spack/test/multimethod.py | 20 ++++----- lib/spack/spack/test/spec_dag.py | 10 +++-- lib/spack/spack/test/spec_semantics.py | 41 ++++++++++++------- .../mock_packages/multimethod/package.py | 23 ++++------- 6 files changed, 85 insertions(+), 79 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index ce867860041..43637ed468d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -209,37 +209,6 @@ def concretize_version(self, spec): return True # Things changed - def class_from_platform_name(self, platform_name): - file_path = join_path(spack.platform_path, platform_name) - platform_mod = imp.load_source('spack.platforms', file_path + '.py') - cls = getattr(platform_mod, mod_to_class(platform_name)) - - return cls - - def spec_add_target_from_string(self, spec, target): - """If only a target is provided, spack will assume the default architecture. - A platform-target pair can be input delimited by a '-'. If either portion of - a platform-target pair is empty, spack will supply a default, in the case of - a blank target the default will be dependent on the platform. - E.g. x86_64 -> 64 bit x86 - bgq- -> default bgq target (back end/powerpc) - cray-hawswell -> haswell target on cray platform - """ - if '-' in target: - platform, target = target.split('-') - else: - platform = '' - - if platform != '': - cls = self.class_from_platform_name(platform) - platform = cls() - else: - platform = spack.architecture.sys_type() - if target != '': - spec.target = platform.target(target) - else: - spec.target = platform.target('default') - def concretize_target(self, spec): """If the spec already has an target and it is a an target type, return. Otherwise, if it has a target that is a string type, generate a @@ -251,14 +220,14 @@ def concretize_target(self, spec): if isinstance(spec.target,spack.architecture.Target): return False else: - self.spec_add_target_from_string(spec, spec.target) + spec.add_target_from_string(spec, spec.target) return True #changed if spec.root.target: if isinstance(spec.root.target,spack.architecture.Target): spec.target = spec.root.target else: - self.spec_add_target_from_string(spec, spec.root.target) + spec.add_target_from_string(spec, spec.root.target) else: platform = spack.architecture.sys_type() spec.target = platform.target('default') diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 9e011bfb9ff..8f90cc0d7fd 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1228,6 +1228,36 @@ def _autospec(self, spec_like): return parse_anonymous_spec(spec_like, self.name) + def add_target_from_string(self, target): + """If only a target is provided, spack will assume the default architecture. + A platform-target pair can be input delimited by a '-'. If either portion of + a platform-target pair is empty, spack will supply a default, in the case of + a blank target the default will be dependent on the platform. + E.g. x86_64 -> 64 bit x86 + bgq- -> default bgq target (back end/powerpc) + cray-hawswell -> haswell target on cray platform + """ + if target is None: + return + if '-' in target: + platform, target = target.split('-') + else: + platform = '' + + if platform != '': + # Find the class for the platform name given + file_path = join_path(spack.platform_path, platform_name) + platform_mod = imp.load_source('spack.platforms', file_path + '.py') + cls = getattr(platform_mod, mod_to_class(platform_name)) + platform = cls() + else: + platform = spack.architecture.sys_type() + if target != '': + self.target = platform.target(target) + else: + self.target = platform.target('default') + + def satisfies(self, other, deps=True, strict=False): """Determine if this spec satisfies all constraints of another. @@ -1275,6 +1305,11 @@ def satisfies(self, other, deps=True, strict=False): # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. + if not isinstance(self.target, spack.architecture.Target): + self.add_target_from_string(self.target) + if not isinstance(other.target, spack.architecture.Target): + other.add_target_from_string(other.target) + if self.target and other.target: if self.target != other.target: return False diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 741dc960721..c651b7ad4a6 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -92,22 +92,16 @@ def test_default_works(self): def test_target_match(self): - pkg = spack.db.get('multimethod=x86_64') - self.assertEqual(pkg.different_by_target(), 'x86_64') + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets[:-1]: + print target + pkg = spack.db.get('multimethod='+target.name) + self.assertEqual(pkg.different_by_target(), target.name) - pkg = spack.db.get('multimethod=ppc64') - self.assertEqual(pkg.different_by_target(), 'ppc64') - - pkg = spack.db.get('multimethod=ppc32') - self.assertEqual(pkg.different_by_target(), 'ppc32') - - pkg = spack.db.get('multimethod=arm64') - self.assertEqual(pkg.different_by_target(), 'arm64') - - pkg = spack.db.get('multimethod=macos') + pkg = spack.db.get('multimethod='+targets[-1].name) self.assertRaises(NoSuchMethodError, pkg.different_by_target) - def test_dependency_match(self): pkg = spack.db.get('multimethod^zmpi') self.assertEqual(pkg.different_by_dep(), 'zmpi') diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 6a2dd6140fa..d116454d5c3 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -241,9 +241,13 @@ def test_unsatisfiable_compiler_version(self): def test_unsatisfiable_target(self): - set_pkg_dep('mpileaks', 'mpich=bgqos_0') - spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) + platform = spack.architecture.sys_type() + if len(platform.targets) > 1: + first = platform.targets.values()[0].name + second = platform.targets.values()[1].name + set_pkg_dep('mpileaks', 'mpich='+first) + spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') + self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) def test_invalid_dep(self): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index b8b4fb951cc..ef4db3fe651 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -111,12 +111,13 @@ def test_satisfies_compiler_version(self): def test_satisfies_target(self): - self.check_satisfies('foo=chaos_5_x86_64_ib', '=chaos_5_x86_64_ib') - self.check_satisfies('foo=bgqos_0', '=bgqos_0') - - self.check_unsatisfiable('foo=bgqos_0', '=chaos_5_x86_64_ib') - self.check_unsatisfiable('foo=chaos_5_x86_64_ib', '=bgqos_0') + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets: + self.check_satisfies('foo='+target.name, '='+target.name) + for i in range(1,len(targets)): + self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) def test_satisfies_dependencies(self): self.check_satisfies('mpileaks^mpich', '^mpich') @@ -267,13 +268,15 @@ def test_constrain_variants(self): def test_constrain_target(self): - self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') - self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') + platform = spack.architecture.sys_type() + target = platform.target('default').name + self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) + self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) def test_constrain_compiler(self): - self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0') - self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0') + self.check_constrain('libelf%intel', 'libelf%intel', 'libelf%intel') + self.check_constrain('libelf%intel', 'libelf', 'libelf%intel') def test_invalid_constraint(self): @@ -283,7 +286,10 @@ def test_invalid_constraint(self): self.check_invalid_constraint('libelf+debug', 'libelf~debug') self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') - self.check_invalid_constraint('libelf=bgqos_0', 'libelf=x86_54') + platform = spack.architecture.sys_type() + targets = platform.targets.values() + if len(targets) > 1: + self.check_invalid_constraint('libelf='+targets[0].name, 'libelf='+targets[1].name) def test_constrain_changed(self): @@ -293,7 +299,8 @@ def test_constrain_changed(self): self.check_constrain_changed('libelf%gcc', '%gcc@4.5') self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') - self.check_constrain_changed('libelf', '=bgqos_0') + platform = spack.architecture.sys_type() + self.check_constrain_changed('libelf', '='+platform.target('default').name) def test_constrain_not_changed(self): @@ -304,7 +311,9 @@ def test_constrain_not_changed(self): self.check_constrain_not_changed('libelf%gcc@4.5', '%gcc@4.5') self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') - self.check_constrain_not_changed('libelf=bgqos_0', '=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_not_changed('libelf='+default, '='+default) self.check_constrain_not_changed('libelf^foo', 'libelf^foo') self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') @@ -316,7 +325,9 @@ def test_constrain_dependency_changed(self): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') - self.check_constrain_changed('libelf^foo', 'libelf^foo=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_changed('libelf^foo', 'libelf^foo='+default) def test_constrain_dependency_not_changed(self): @@ -326,5 +337,7 @@ def test_constrain_dependency_not_changed(self): self.check_constrain_not_changed('libelf^foo%gcc@4.5', 'libelf^foo%gcc@4.5') self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') - self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0') + platform = spack.architecture.sys_type() + default = platform.target('default').name + self.check_constrain_not_changed('libelf^foo='+default, 'libelf^foo='+default) diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index f78ef3bb3dd..0a1e991d37e 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * - +import spack.architecture class Multimethod(Package): """This package is designed for use with Spack's multimethod test. @@ -103,21 +103,12 @@ def has_a_default(self): # # Make sure we can switch methods on different target # - @when('=x86_64') - def different_by_target(self): - return 'x86_64' - - @when('=ppc64') - def different_by_target(self): - return 'ppc64' - - @when('=ppc32') - def different_by_target(self): - return 'ppc32' - - @when('=arm64') - def different_by_target(self): - return 'arm64' + platform = spack.architecture.sys_type() + targets = platform.targets.values() + for target in targets[:-1]: + @when('='+target.name) + def different_by_target(self): + return self.spec.target.name # From 2b2d4bae4e48151ae17c0f2f0514873b1247ff15 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 17:38:37 -0800 Subject: [PATCH 096/375] Fixing multimethod test for new platforms --- lib/spack/spack/test/multimethod.py | 6 +++++- var/spack/mock_packages/multimethod/package.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index c651b7ad4a6..116cb2d2ffe 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -100,7 +100,11 @@ def test_target_match(self): self.assertEqual(pkg.different_by_target(), target.name) pkg = spack.db.get('multimethod='+targets[-1].name) - self.assertRaises(NoSuchMethodError, pkg.different_by_target) + if len(targets) == 1: + self.assertEqual(pkg.different_by_target(), targets[-1].name) + else: + self.assertRaises(NoSuchMethodError, pkg.different_by_target) + def test_dependency_match(self): pkg = spack.db.get('multimethod^zmpi') diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 0a1e991d37e..4e9a2438a7a 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -22,6 +22,9 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import imp +from llnl.util.filesystem import join_path +from spack.util.naming import mod_to_class from spack import * import spack.architecture @@ -103,8 +106,15 @@ def has_a_default(self): # # Make sure we can switch methods on different target # +# for platform_name in ['cray_xc', 'darwin', 'linux']: +# file_path = join_path(spack.platform_path, platform_name) +# platform_mod = imp.load_source('spack.platforms', file_path + '.py') +# cls = getattr(platform_mod, mod_to_class(platform_name)) + +# platform = cls() platform = spack.architecture.sys_type() targets = platform.targets.values() + for target in targets[:-1]: @when('='+target.name) def different_by_target(self): From 6e5238d037aebe4e610a16f200d860cee1a1355d Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 17:44:16 -0800 Subject: [PATCH 097/375] Fixing the fix and removing debug printing --- lib/spack/spack/test/multimethod.py | 1 - var/spack/mock_packages/multimethod/package.py | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 116cb2d2ffe..9d9f39e87d9 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -95,7 +95,6 @@ def test_target_match(self): platform = spack.architecture.sys_type() targets = platform.targets.values() for target in targets[:-1]: - print target pkg = spack.db.get('multimethod='+target.name) self.assertEqual(pkg.different_by_target(), target.name) diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index 4e9a2438a7a..b37db9f1eb4 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -114,8 +114,10 @@ def has_a_default(self): # platform = cls() platform = spack.architecture.sys_type() targets = platform.targets.values() + if len(targets) > 1: + targets = targets[:-1] - for target in targets[:-1]: + for target in targets: @when('='+target.name) def different_by_target(self): return self.spec.target.name From 41046499c69150a7a81518f2ffba3405de0d6577 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 18:51:38 -0800 Subject: [PATCH 098/375] minor bug chasing --- lib/spack/env/cc | 16 ---------------- lib/spack/spack/compilers/__init__.py | 4 ++-- lib/spack/spack/concretize.py | 4 ++-- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index b6c6e03e428..1704ae3280e 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -71,22 +71,6 @@ for param in $parameters; do done -# -# Cray module environment-related stuff. -# -if [ ! -z "$SPACK_CRAYPE" ]; then - cur_pe=$(module list 2>&1 | grep PrgEnv | grep -o 'PrgEnv-[^/]*') - if [ ! -z "$cur_pe" ]; then - module swap $cur_pe $SPACK_CRAYPE - else - module load $SPACK_CRAYPE - fi -fi - -if [ ! -z "$SPACK_COMP_MODULE" ]; then - module load $SPACK_COMP_MODULE -fi - # # Figure out the type of compiler, the language, and the mode so that # the compiler script knows what to do. diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 3a0a88b5e3f..e38ef949d7d 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -259,8 +259,8 @@ def __init__(self): class NoCompilerForSpecError(spack.error.SpackError): def __init__(self, compiler_spec, target): - super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s", - compiler_spec, target) + super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s" % ( + target, compiler_spec)) class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): def __init__(self, compiler_spec): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 43637ed468d..b8057cd46c9 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -220,14 +220,14 @@ def concretize_target(self, spec): if isinstance(spec.target,spack.architecture.Target): return False else: - spec.add_target_from_string(spec, spec.target) + spec.add_target_from_string(spec.target) return True #changed if spec.root.target: if isinstance(spec.root.target,spack.architecture.Target): spec.target = spec.root.target else: - spec.add_target_from_string(spec, spec.root.target) + spec.add_target_from_string(spec.root.target) else: platform = spack.architecture.sys_type() spec.target = platform.target('default') From 9615efd940c425b049a142f8c22545c9a04124df Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 8 Jan 2016 18:53:19 -0800 Subject: [PATCH 099/375] cleanup: Removing an unnecessary line in an otherwise unchanged file --- lib/spack/env/cc | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 1704ae3280e..fa85bb595ee 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -70,7 +70,6 @@ for param in $parameters; do fi done - # # Figure out the type of compiler, the language, and the mode so that # the compiler script knows what to do. From aab1a67d056bf9f0979f2cebfff607d05ac8b7e9 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 9 Jan 2016 09:42:36 -0800 Subject: [PATCH 100/375] Adding needed import modules. Changed platform_name to platform to prevent syntax errors --- lib/spack/spack/spec.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 8f90cc0d7fd..264e0c05067 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -91,6 +91,7 @@ expansion when it is the first character in an id typed on the command line. """ import sys +import imp import itertools import hashlib import base64 @@ -100,6 +101,7 @@ from external.yaml.error import MarkedYAMLError import llnl.util.tty as tty +from llnl.util.filesystem import join_path from llnl.util.lang import * from llnl.util.tty.color import * @@ -109,6 +111,7 @@ import spack.compilers as compilers from spack.version import * +from spack.util.naming import mod_to_class from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex @@ -1246,9 +1249,9 @@ def add_target_from_string(self, target): if platform != '': # Find the class for the platform name given - file_path = join_path(spack.platform_path, platform_name) + file_path = join_path(spack.platform_path, platform) platform_mod = imp.load_source('spack.platforms', file_path + '.py') - cls = getattr(platform_mod, mod_to_class(platform_name)) + cls = getattr(platform_mod, mod_to_class(platform)) platform = cls() else: platform = spack.architecture.sys_type() From 15713219e57a97048f4e3b085f9c75e188a67787 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 20 Jan 2016 10:32:56 -0800 Subject: [PATCH 101/375] Better regular expression searching. Tested on edison was finding test compilers and then spack was giving multiple compiler match errors --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 225c65917a5..1a5be3c3feb 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -283,7 +283,7 @@ def find_in_modules(cls): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) - matches = re.findall(r'(%s)/(\d+[\.\d]+)' % cls.PrgEnv_compiler, output) + matches = re.findall(r'(%s)/([\d\.]+[\d$])' % cls.PrgEnv_compiler, output) for name, version in matches: v = version From 840b41c450267764d021456875dbe87af52c2176 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 20 Jan 2016 10:36:15 -0800 Subject: [PATCH 102/375] Removed unneccessary $. Still learning regexp --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1a5be3c3feb..46f2dfaec72 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -283,7 +283,7 @@ def find_in_modules(cls): modulecmd = which('modulecmd') modulecmd.add_default_arg('python') output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) - matches = re.findall(r'(%s)/([\d\.]+[\d$])' % cls.PrgEnv_compiler, output) + matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output) for name, version in matches: v = version From 03585225338e2ccd61174800b42242c7f256f533 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 6 Feb 2016 15:41:22 -0800 Subject: [PATCH 103/375] new branch and also pushing some architecture changes where os is detected by linux and darwin and manually set by cray and bgq --- lib/spack/spack/architecture.py | 136 ++++++++++++++++++++------------ 1 file changed, 84 insertions(+), 52 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 6b9ae33b3e3..4f36a3fa92b 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -39,20 +39,25 @@ class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") @key_ordering class Target(object): - """ Target is the processor of the host machine. The host machine may have different front-end - and back-end targets, especially if it is a Cray machine. The target will have a name and - also the module_name (e.g craype-compiler). Targets will also recognize which platform - they came from using the set_platform method. Targets will have compiler finding strategies + """ Target is the processor of the host machine. + The host machine may have different front-end + and back-end targets, especially if it is a Cray machine. + The target will have a name and module_name (e.g craype-compiler). + Targets will also recognize which platform + they came from using the set_platform method. + Targets will have compiler finding strategies """ def __init__(self, name, compiler_strategy, module_name=None): @@ -60,10 +65,13 @@ def __init__(self, name, compiler_strategy, module_name=None): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - # Sets only the platform name to avoid recursiveness + # Sets only the platform name to avoid recursiveness def set_platform(self, platform): self.platform_name = platform.name + def set_operating_system(self, operating_sys): + self.platform_os = operating_sys + def to_dict(self): d = {} d['name'] = self.name @@ -71,6 +79,7 @@ def to_dict(self): d['module_name'] = self.module_name if self.platform_name: d['platform'] = self.platform_name + return d @staticmethod @@ -87,14 +96,16 @@ def from_dict(d): def _cmp_key(self): - return (self.name, self.compiler_strategy, self.module_name) + return (self.name, self.compiler_strategy, + self.module_name, self.platform_os) def __repr__(self): return self.__str__() def __str__(self): - if self.platform_name: - return self.platform_name + '-' + self.name + if self.platform_name and self.platform_os: + return (self.platform_name + '-' + + self.platform_os + '-' + self.name) return self.name @key_ordering @@ -105,26 +116,38 @@ class Platform(object): """ priority = None # Subclass needs to set this number. This controls order in which platform is detected. + front_end = None back_end = None default = None # The default back end target. On cray ivybridge + + front_os = None + back_os = None + default_os = None def __init__(self, name): self.targets = {} self.name = name def add_target(self, name, target): - """Used by the platform specific subclass to list available targets. Raises an error - if the platform specifies a name that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. + Raises an error if the platform specifies a name + that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: - raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) + raise ValueError( + "%s is a spack reserved" \ + "alias and cannot be the name of a target" + % name) + + target.set_operating_system(self.platform_os()) target.set_platform(self) self.targets[name] = target def target(self, name): - """This is a getter method for the target dictionary that handles defaulting based - on the values provided by default, front-end, and back-end. This can be overwritten + """This is a getter method for the target dictionary that + handles defaulting based on the values provided by default, + front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': @@ -135,6 +158,51 @@ def target(self, name): name = self.back_end return self.targets[name] + + def _detect_linux_os(self): + """ If it is one a linux machine use the python method platform.dist() + """ + os_name = py_platform.dist()[0] + version = py_platform.dist()[1] + a return os_name + version + + def _detect_mac_os(self): + """If it is on a mac machine then use the python method platform.mac_ver + """ + mac_releases = {'10.6' : 'snowleopard', '10.7' : 'lion', + '10.8' : 'mountainlion', '10.9' : 'mavericks', + '10.10' : 'yosemite', '10.11' : 'elcapitan'} + + mac_ver = py_platform.mac_ver() + try: + os_name = mac_releases[mac_ver] + mac_ver = Version(mac_ver) + + except KeyError: + os_name = 'mac_os' + + return os_name + + def set_os(self): + """ Set the OS according to the platform it is on. Darwin and Linux + will simply be an auto-detected linux distro or mac release. The + special cases will be for Cray and BGQ machines which have two + different OS for login and compute nodes. The client should provide + the name and major version of the operating system + """ + if self.name == 'darwin': + self.default_os = self._detect_mac_os() + else: + self.default_os = self._detect_linux_os() + + def platform_os(self, name=None): + """ Get the platform operating system from the platform """ + if name == 'front_os': + return self.front_os + elif name == 'back_os': + return self.back_os + else: + return self.default_os @classmethod def detect(self): @@ -144,6 +212,7 @@ def detect(self): """ raise NotImplementedError() + def __repr__(self): return self.__str__() @@ -153,43 +222,6 @@ def __str__(self): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() #If in __init__.py there is a sys_type() then call that - else: - return spack.sys_type # Else use the attributed which defaults to None - - -# This is livermore dependent. Hard coded for livermore -#def get_sys_type_from_environment(): -# """Return $SYS_TYPE or None if it's not defined.""" -# return os.environ.get('SYS_TYPE') - - -def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac. - Front-end config - """ - mac_ver = py_platform.mac_ver()[0] - if not mac_ver: - return None - return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) - - -def get_sys_type_from_uname(): - """ Returns a sys_type from the uname argument - Front-end config - """ - try: - platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) - platform, _ = platform_proc.communicate() - return platform.strip() - except: - return None - @memoized def all_platforms(): modules = [] From 21a5a3404134c1676a7aa498ec201ed911e80029 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 10:59:16 -0800 Subject: [PATCH 104/375] Revert "new branch and also pushing some architecture changes where os is detected by linux and darwin and manually set by cray and bgq" This reverts commit 70088be24b2ed34076f7f5292a2a465a8655a886. --- lib/spack/spack/architecture.py | 136 ++++++++++++-------------------- 1 file changed, 52 insertions(+), 84 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 4f36a3fa92b..6b9ae33b3e3 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -39,25 +39,20 @@ class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") @key_ordering class Target(object): - """ Target is the processor of the host machine. - The host machine may have different front-end - and back-end targets, especially if it is a Cray machine. - The target will have a name and module_name (e.g craype-compiler). - Targets will also recognize which platform - they came from using the set_platform method. - Targets will have compiler finding strategies + """ Target is the processor of the host machine. The host machine may have different front-end + and back-end targets, especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also recognize which platform + they came from using the set_platform method. Targets will have compiler finding strategies """ def __init__(self, name, compiler_strategy, module_name=None): @@ -65,13 +60,10 @@ def __init__(self, name, compiler_strategy, module_name=None): self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge - # Sets only the platform name to avoid recursiveness + # Sets only the platform name to avoid recursiveness def set_platform(self, platform): self.platform_name = platform.name - def set_operating_system(self, operating_sys): - self.platform_os = operating_sys - def to_dict(self): d = {} d['name'] = self.name @@ -79,7 +71,6 @@ def to_dict(self): d['module_name'] = self.module_name if self.platform_name: d['platform'] = self.platform_name - return d @staticmethod @@ -96,16 +87,14 @@ def from_dict(d): def _cmp_key(self): - return (self.name, self.compiler_strategy, - self.module_name, self.platform_os) + return (self.name, self.compiler_strategy, self.module_name) def __repr__(self): return self.__str__() def __str__(self): - if self.platform_name and self.platform_os: - return (self.platform_name + '-' + - self.platform_os + '-' + self.name) + if self.platform_name: + return self.platform_name + '-' + self.name return self.name @key_ordering @@ -116,38 +105,26 @@ class Platform(object): """ priority = None # Subclass needs to set this number. This controls order in which platform is detected. - front_end = None back_end = None default = None # The default back end target. On cray ivybridge - - front_os = None - back_os = None - default_os = None def __init__(self, name): self.targets = {} self.name = name def add_target(self, name, target): - """Used by the platform specific subclass to list available targets. - Raises an error if the platform specifies a name - that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. Raises an error + if the platform specifies a name that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: - raise ValueError( - "%s is a spack reserved" \ - "alias and cannot be the name of a target" - % name) - - target.set_operating_system(self.platform_os()) + raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) target.set_platform(self) self.targets[name] = target def target(self, name): - """This is a getter method for the target dictionary that - handles defaulting based on the values provided by default, - front-end, and back-end. This can be overwritten + """This is a getter method for the target dictionary that handles defaulting based + on the values provided by default, front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': @@ -158,51 +135,6 @@ def target(self, name): name = self.back_end return self.targets[name] - - def _detect_linux_os(self): - """ If it is one a linux machine use the python method platform.dist() - """ - os_name = py_platform.dist()[0] - version = py_platform.dist()[1] - a return os_name + version - - def _detect_mac_os(self): - """If it is on a mac machine then use the python method platform.mac_ver - """ - mac_releases = {'10.6' : 'snowleopard', '10.7' : 'lion', - '10.8' : 'mountainlion', '10.9' : 'mavericks', - '10.10' : 'yosemite', '10.11' : 'elcapitan'} - - mac_ver = py_platform.mac_ver() - try: - os_name = mac_releases[mac_ver] - mac_ver = Version(mac_ver) - - except KeyError: - os_name = 'mac_os' - - return os_name - - def set_os(self): - """ Set the OS according to the platform it is on. Darwin and Linux - will simply be an auto-detected linux distro or mac release. The - special cases will be for Cray and BGQ machines which have two - different OS for login and compute nodes. The client should provide - the name and major version of the operating system - """ - if self.name == 'darwin': - self.default_os = self._detect_mac_os() - else: - self.default_os = self._detect_linux_os() - - def platform_os(self, name=None): - """ Get the platform operating system from the platform """ - if name == 'front_os': - return self.front_os - elif name == 'back_os': - return self.back_os - else: - return self.default_os @classmethod def detect(self): @@ -212,7 +144,6 @@ def detect(self): """ raise NotImplementedError() - def __repr__(self): return self.__str__() @@ -222,6 +153,43 @@ def __str__(self): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) +def get_sys_type_from_spack_globals(): + """Return the SYS_TYPE from spack globals, or None if it isn't set.""" + if not hasattr(spack, "sys_type"): + return None + elif hasattr(spack.sys_type, "__call__"): + return spack.sys_type() #If in __init__.py there is a sys_type() then call that + else: + return spack.sys_type # Else use the attributed which defaults to None + + +# This is livermore dependent. Hard coded for livermore +#def get_sys_type_from_environment(): +# """Return $SYS_TYPE or None if it's not defined.""" +# return os.environ.get('SYS_TYPE') + + +def get_mac_sys_type(): + """Return a Mac OS SYS_TYPE or None if this isn't a mac. + Front-end config + """ + mac_ver = py_platform.mac_ver()[0] + if not mac_ver: + return None + return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) + + +def get_sys_type_from_uname(): + """ Returns a sys_type from the uname argument + Front-end config + """ + try: + platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) + platform, _ = platform_proc.communicate() + return platform.strip() + except: + return None + @memoized def all_platforms(): modules = [] From 2650c60374a0857679d52f98fc5f94b23c7d6660 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:14:50 -0800 Subject: [PATCH 105/375] Added operating system class prototype and some autodetect features for operating system --- lib/spack/spack/architecture.py | 61 ++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 6b9ae33b3e3..2914d36ba21 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -109,8 +109,13 @@ class Platform(object): back_end = None default = None # The default back end target. On cray ivybridge + front_os = None + back_os = None + default_os = None + def __init__(self, name): self.targets = {} + self.operating_sys = {} self.name = name def add_target(self, name, target): @@ -136,10 +141,49 @@ def target(self, name): return self.targets[name] + def _detect_linux_os(self): + return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) + + def _detect_mac_os(self): + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + mac_ver = py_platform.mac_ver()[:-2] + try: + os_name = mac_releases[mac_ver] + return OperatingSystem(os_name, mac_ver) + except KeyError: + os_name = "mac_os" + return OperatingSystem(os_name, mac_ver) + + def add_operating_system(self, name=None, operating_system=None): + if self.name == 'linux': + linux_os = self._detect_linux_os() + self.operating_sys[linux_os.name] = linux_os + elif self.name == 'darwin': + mac_os = self._detect_mac_os() + self.operating_sys[mac_os.name] = mac_os + else: + self.operating_sys[name] = operating_system + + def operating_system(self, name): + if name == 'default': + name = self.default_os + if name == 'front_os': + name = self.front_os + if name == 'back_os': + name = self.back_os + + return self.operating_sys[name] + @classmethod def detect(self): """ Subclass is responsible for implementing this method. - Returns True if the Platform class detects that it is the current platform + Returns True if the Platform class detects that + it is the current platform and False if it's not. """ raise NotImplementedError() @@ -153,6 +197,21 @@ def __str__(self): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values())) + +class OperatingSystem(object): + """ The operating system will contain a name and version. It will + also represent itself simple with it's name + """ + def __init__(self, name, version): + self.name = name + self.version = version + + def __str__(self): + return self.name + + def __repr__(self): + return self.__str__() + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): From d0ae6dd40159c1e3b043db254f56c8b7ca22294c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:15:25 -0800 Subject: [PATCH 106/375] Changed add_target_from_string to include methods to concretize operating_system --- lib/spack/spack/spec.py | 64 ++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 264e0c05067..503f80e1e6e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -90,6 +90,7 @@ specs to avoid ambiguity. Both are provided because ~ can cause shell expansion when it is the first character in an id typed on the command line. """ +from collections import namedtuple import sys import imp import itertools @@ -459,7 +460,7 @@ def _set_target(self, target): """Called by the parser to set the target.""" if self.target: raise DuplicateTargetError( "Spec for '%s' cannot have two targets." % self.name) - self.target = target + self.target = target # a string can be set def _add_dependency(self, spec): @@ -1231,7 +1232,7 @@ def _autospec(self, spec_like): return parse_anonymous_spec(spec_like, self.name) - def add_target_from_string(self, target): + def add_target_from_string(self, arch): """If only a target is provided, spack will assume the default architecture. A platform-target pair can be input delimited by a '-'. If either portion of a platform-target pair is empty, spack will supply a default, in the case of @@ -1240,31 +1241,28 @@ def add_target_from_string(self, target): bgq- -> default bgq target (back end/powerpc) cray-hawswell -> haswell target on cray platform """ - if target is None: + Arch = namedtuple("Arch", "arch_os target") + platform = spack.architecture.sys_type() + + if arch is None: return - if '-' in target: - platform, target = target.split('-') - else: - platform = '' - - if platform != '': - # Find the class for the platform name given - file_path = join_path(spack.platform_path, platform) - platform_mod = imp.load_source('spack.platforms', file_path + '.py') - cls = getattr(platform_mod, mod_to_class(platform)) - platform = cls() - else: - platform = spack.architecture.sys_type() - if target != '': - self.target = platform.target(target) - else: - self.target = platform.target('default') + if '-' in arch: + os_name, target = arch.split('-') + self.target = Arch(arch_os=platform.operating_system(os_name), + target = platform.target(target)) + elif arch in platform.targets: + self.target = Arch(arch_os=platform.operating_system('default'), + target=platform.target(target)) + else: + os_name = arch # Odd naming here, definitely need to change + self.target = Arch(arch_os=platform.operating_system(os_name), + target=platform.target('default')) def satisfies(self, other, deps=True, strict=False): - """Determine if this spec satisfies all constraints of another. + """determine if this spec satisfies all constraints of another. - There are two senses for satisfies: + there are two senses for satisfies: * `loose` (default): the absence of a constraint in self implies that it *could* be satisfied by other, so we only @@ -1276,32 +1274,32 @@ def satisfies(self, other, deps=True, strict=False): """ other = self._autospec(other) - # A concrete provider can satisfy a virtual dependency. + # a concrete provider can satisfy a virtual dependency. if not self.virtual and other.virtual: pkg = spack.db.get(self.name) if pkg.provides(other.name): for provided, when_spec in pkg.provided.items(): - if self.satisfies(when_spec, deps=False, strict=strict): + if self.satisfies(when_spec, deps=false, strict=strict): if provided.satisfies(other): - return True - return False + return True + return False - # Otherwise, first thing we care about is whether the name matches + # otherwise, first thing we care about is whether the name matches if self.name != other.name: - return False + return False if self.versions and other.versions: if not self.versions.satisfies(other.versions, strict=strict): - return False + return False elif strict and (self.versions or other.versions): - return False + return False - # None indicates no constraints when not strict. + # none indicates no constraints when not strict. if self.compiler and other.compiler: if not self.compiler.satisfies(other.compiler, strict=strict): - return False + return False elif strict and (other.compiler and not self.compiler): - return False + return True if not self.variants.satisfies(other.variants, strict=strict): return False From fb234205c210930112f7fcd2c7c360e0c950d042 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:16:11 -0800 Subject: [PATCH 107/375] Added method to concretize target and os pair --- lib/spack/spack/concretize.py | 75 ++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index b8057cd46c9..d78c102d93c 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -33,6 +33,7 @@ TODO: make this customizable and allow users to configure concretization policies. """ +import collections from llnl.util.filesystem import join_path import spack import spack.spec @@ -209,6 +210,22 @@ def concretize_version(self, spec): return True # Things changed + def _concretize_operating_system(self, arch, platform): + """ Future method for concretizing operating system """ + if isinstance(arch.arch_os, OperatingSystem): + return False + else: + arch.arch_os = platform.operating_system('default') + return True + + + def _concretize_arch_target(self, arch, platform): + if isinstance(arch.target, spack.architecture.Target): + return False + else: + arch.target = platform.target('default') + return True + def concretize_target(self, spec): """If the spec already has an target and it is a an target type, return. Otherwise, if it has a target that is a string type, generate a @@ -216,23 +233,53 @@ def concretize_target(self, spec): DAG has an target, then use that. Otherwise, take the system's default target. """ - if spec.target is not None: - if isinstance(spec.target,spack.architecture.Target): - return False - else: - spec.add_target_from_string(spec.target) - return True #changed + platform = spack.architecture.sys_type() - if spec.root.target: - if isinstance(spec.root.target,spack.architecture.Target): - spec.target = spec.root.target - else: - spec.add_target_from_string(spec.root.target) + if spec.target is None: + # Create an empty tuple + Arch = collections.namedtuple("Arch", "arch_os target") + spec.target = Arch(arch_os=platform.operating_system('default'), + target=platform.target('default')) + + return True + # If there is a target and it is a tuple and has both filled return + # False + if not isinstance(spec.target, basestring): + return any((self._concretize_operating_system(spec.target, platform), + self._concretize_arch_target(spec.target, plarform))) else: - platform = spack.architecture.sys_type() - spec.target = platform.target('default') + spec.add_target_from_string(spec.target) - return True #changed + if spec.root.target and \ + not isinstance(spec.root.target, basestring): + bool_flag = any( + (self._concretize_operating_system(spec.root.target, platform), + self._concretize_arch_target(spec.root.target, platform))) + spec.target =spec.root.target + return bool_flag + else: + spec.add_target_from_string(spec.root.target) + + + # if there is no target specified used the defaults + + #if spec.target is not None: + # if isinstance(spec.target,spack.architecture.Target): + # return False + # else: + # spec.add_target_from_string(spec.target) + # return True #changed + + #if spec.root.target: + # if isinstance(spec.root.target,spack.architecture.Target): + # spec.target = spec.root.target + # else: + # spec.add_target_from_string(spec.root.target) + #else: + # platform = spack.architecture.sys_type() + # spec.target = platform.target('default') + + #return True #changed def concretize_variants(self, spec): From cb8d5ab1837cfddefa31bd09592ca4ac999692dc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:16:35 -0800 Subject: [PATCH 108/375] Added new add_os method --- lib/spack/spack/platforms/cray_xc.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index e3adb182ead..b37d36a62c8 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,6 +1,6 @@ import os -from spack.architecture import Platform, Target +from spack.architecture import Platform, Target, OperatingSystem class CrayXc(Platform): priority = 20 @@ -8,6 +8,10 @@ class CrayXc(Platform): back_end = 'ivybridge' default = 'ivybridge' + front_os = "SuSE11" + back_os = "GNU/Linux" + default_os = "GNU/Linux" + def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better if we use CRAY_CPU_TARGET as the default. This will ensure @@ -30,6 +34,9 @@ def __init__(self): self.add_target('sandybridge', Target('sandybridge', 'PATH')) self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) + + self.add_operating_system('SuSE11', OperatingSystem('SuSE11', '11')) + self.add_operating_system('GNU/Linux', OperatingSystem('GNU/Linux', '10')) # self.add_target(self.front_end, Target(self.front_end, 'PATH')) # Back End compiler needs the proper target module loaded. From 77e93e1b791f1c264a999de3b59f2e6396e13391 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 10 Feb 2016 16:16:58 -0800 Subject: [PATCH 109/375] Added add_os method to platform subclasses --- lib/spack/spack/platforms/darwin.py | 1 + lib/spack/spack/platforms/linux.py | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 52284826b12..cfa1d957a17 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -10,6 +10,7 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default, 'PATH')) + self.add_operating_system() @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 7f94d80c34e..dcb08569a8a 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -10,6 +10,7 @@ class Linux(Platform): def __init__(self): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default, 'PATH')) + self.add_operating_system() @classmethod def detect(self): From b9d09202c9856bf173c4bbabd4b73573d355d408 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 11 Feb 2016 11:15:19 -0800 Subject: [PATCH 110/375] changed some variables to account for the fact that target is now a tuple --- lib/spack/spack/concretize.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index d78c102d93c..8886b3cd931 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -212,7 +212,7 @@ def concretize_version(self, spec): def _concretize_operating_system(self, arch, platform): """ Future method for concretizing operating system """ - if isinstance(arch.arch_os, OperatingSystem): + if isinstance(arch.arch_os, spack.architecture.OperatingSystem): return False else: arch.arch_os = platform.operating_system('default') @@ -240,13 +240,12 @@ def concretize_target(self, spec): Arch = collections.namedtuple("Arch", "arch_os target") spec.target = Arch(arch_os=platform.operating_system('default'), target=platform.target('default')) - - return True + return True # If there is a target and it is a tuple and has both filled return # False if not isinstance(spec.target, basestring): return any((self._concretize_operating_system(spec.target, platform), - self._concretize_arch_target(spec.target, plarform))) + self._concretize_arch_target(spec.target, platform))) else: spec.add_target_from_string(spec.target) @@ -316,9 +315,9 @@ def concretize_compiler(self, spec): # Should think whether this can be more efficient def _proper_compiler_style(cspec, target): compilers = spack.compilers.compilers_for_spec(cspec) - if target.compiler_strategy == 'PATH': + if target.target.compiler_strategy == 'PATH': filter(lambda c: not c.modules, compilers) - if target.compiler_strategy == 'MODULES': + if target.target.compiler_strategy == 'MODULES': filter(lambda c: c.modules, compilers) return compilers From 3e1be63b0f8e745a04e041288a7583333d31673c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 11 Feb 2016 11:15:50 -0800 Subject: [PATCH 111/375] Changed some variables to account for the fact that target is now a tuple --- lib/spack/spack/spec.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 503f80e1e6e..3b2d54d0473 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -566,7 +566,7 @@ def traverse(self, visited=None, d=0, **kwargs): in the traversal. root [=True] - If false, this won't yield the root node, just its descendents. + If False, this won't yield the root node, just its descendents. direction [=children|parents] If 'children', does a traversal of this spec's children. If @@ -661,7 +661,7 @@ def to_node_dict(self): for d in sorted(self.dependencies)) } if self.target: - d['target'] = self.target.to_dict() + d['target'] = self.target.target.to_dict() else: d['target'] = None if self.compiler: @@ -755,7 +755,6 @@ def _concretize_helper(self, presets=None, visited=None): if self.name in presets: changed |= self.constrain(presets[self.name]) - else: # Concretize virtual dependencies last. Because they're added # to presets below, their constraints will all be merged, but we'll @@ -830,6 +829,7 @@ def concretize(self): changed = True force = False + # Loops forever here in my implementation while changed: changes = (self.normalize(force=force), self._expand_virtual_packages(), @@ -1279,7 +1279,7 @@ def satisfies(self, other, deps=True, strict=False): pkg = spack.db.get(self.name) if pkg.provides(other.name): for provided, when_spec in pkg.provided.items(): - if self.satisfies(when_spec, deps=false, strict=strict): + if self.satisfies(when_spec, deps=False, strict=strict): if provided.satisfies(other): return True return False @@ -1306,9 +1306,9 @@ def satisfies(self, other, deps=True, strict=False): # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. - if not isinstance(self.target, spack.architecture.Target): + if isinstance(self.target, basestring): self.add_target_from_string(self.target) - if not isinstance(other.target, spack.architecture.Target): + if isinstance(other.target, basestring): other.add_target_from_string(other.target) if self.target and other.target: @@ -1380,7 +1380,7 @@ def _dup(self, other, **kwargs): Options: dependencies[=True] - Whether deps should be copied too. Set to false to copy a + Whether deps should be copied too. Set to False to copy a spec but not its dependencies. """ From 8e8c63bd679b57a969274287d402ea9843ecd1ae Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 11 Feb 2016 11:47:39 -0800 Subject: [PATCH 112/375] Using pylint, fixed some of the indentation and spacing errors --- lib/spack/spack/architecture.py | 94 ++++++++++++--------------------- 1 file changed, 33 insertions(+), 61 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 2914d36ba21..c4ef0805ac2 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,7 +22,6 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os import imp import platform as py_platform import inspect @@ -34,31 +33,34 @@ import spack from spack.util.naming import mod_to_class import spack.error as serr -from spack.version import Version -from external import yaml + class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, self).__init__( + "Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.") + super(NoSysTypeError, self).__init__( + "Could not determine sys_type for this machine.") @key_ordering class Target(object): - """ Target is the processor of the host machine. The host machine may have different front-end - and back-end targets, especially if it is a Cray machine. The target will have a name and - also the module_name (e.g craype-compiler). Targets will also recognize which platform - they came from using the set_platform method. Targets will have compiler finding strategies - """ + """ Target is the processor of the host machine. + The host machine may have different front-end and back-end targets, + especially if it is a Cray machine. The target will have a name and + also the module_name (e.g craype-compiler). Targets will also + recognize which platform they came from using the set_platform method. + Targets will have compiler finding strategies + """ def __init__(self, name, compiler_strategy, module_name=None): - self.name = name # case of cray "ivybridge" but if it's x86_64 + self.name = name # case of cray "ivybridge" but if it's x86_64 self.compiler_strategy = compiler_strategy - self.module_name = module_name # craype-ivybridge + self.module_name = module_name # craype-ivybridge # Sets only the platform name to avoid recursiveness def set_platform(self, platform): @@ -85,7 +87,6 @@ def from_dict(d): target.platform_name = d['platform'] return target - def _cmp_key(self): return (self.name, self.compiler_strategy, self.module_name) @@ -97,6 +98,7 @@ def __str__(self): return self.platform_name + '-' + self.name return self.name + @key_ordering class Platform(object): """ Abstract class that each type of Platform will subclass. @@ -104,10 +106,10 @@ class Platform(object): is returned """ - priority = None # Subclass needs to set this number. This controls order in which platform is detected. + priority = None # Subclass needs to set this number. This controls order in which platform is detected. front_end = None back_end = None - default = None # The default back end target. On cray ivybridge + default = None # The default back end target. On cray ivybridge front_os = None back_os = None @@ -119,17 +121,21 @@ def __init__(self, name): self.name = name def add_target(self, name, target): - """Used by the platform specific subclass to list available targets. Raises an error - if the platform specifies a name that is reserved by spack as an alias. + """Used by the platform specific subclass to list available targets. + Raises an error if the platform specifies a name + that is reserved by spack as an alias. """ if name in ['front_end', 'fe', 'back_end', 'be', 'default']: - raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name) + raise ValueError( + "%s is a spack reserved alias " + "and cannot be the name of a target" % name) target.set_platform(self) self.targets[name] = target def target(self, name): - """This is a getter method for the target dictionary that handles defaulting based - on the values provided by default, front-end, and back-end. This can be overwritten + """This is a getter method for the target dictionary + that handles defaulting based on the values provided by default, + front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ if name == 'default': @@ -150,7 +156,7 @@ def _detect_mac_os(self): "10.8": "mountainlion", "10.9": "mavericks", "10.10": "yosemite", - "10.11": "elcapitan"} + "10.11": "elcapitan"} mac_ver = py_platform.mac_ver()[:-2] try: os_name = mac_releases[mac_ver] @@ -199,8 +205,10 @@ def _cmp_key(self): class OperatingSystem(object): - """ The operating system will contain a name and version. It will - also represent itself simple with it's name + """ Operating System class. It will include the name and version. + The biggest attribute to this class will be the compiler finding + strategy. At the moment the compiler finding strategy is going to + be a string. """ def __init__(self, name, version): self.name = name @@ -212,42 +220,6 @@ def __str__(self): def __repr__(self): return self.__str__() -def get_sys_type_from_spack_globals(): - """Return the SYS_TYPE from spack globals, or None if it isn't set.""" - if not hasattr(spack, "sys_type"): - return None - elif hasattr(spack.sys_type, "__call__"): - return spack.sys_type() #If in __init__.py there is a sys_type() then call that - else: - return spack.sys_type # Else use the attributed which defaults to None - - -# This is livermore dependent. Hard coded for livermore -#def get_sys_type_from_environment(): -# """Return $SYS_TYPE or None if it's not defined.""" -# return os.environ.get('SYS_TYPE') - - -def get_mac_sys_type(): - """Return a Mac OS SYS_TYPE or None if this isn't a mac. - Front-end config - """ - mac_ver = py_platform.mac_ver()[0] - if not mac_ver: - return None - return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine()) - - -def get_sys_type_from_uname(): - """ Returns a sys_type from the uname argument - Front-end config - """ - try: - platform_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE) - platform, _ = platform_proc.communicate() - return platform.strip() - except: - return None @memoized def all_platforms(): @@ -267,6 +239,7 @@ def all_platforms(): return modules + @memoized def sys_type(): """ Gather a list of all available subclasses of platforms. @@ -276,9 +249,8 @@ def sys_type(): """ # Try to create a Platform object using the config file FIRST platform_list = all_platforms() - platform_list.sort(key = lambda a: a.priority) + platform_list.sort(key=lambda a: a.priority) for platform in platform_list: if platform.detect(): return platform() - From 7732f375d24c7089d571cbb1659cbd1d60250888 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 13 Feb 2016 14:36:01 -0800 Subject: [PATCH 113/375] Changed operating system getter so that linux and darwin autodetected operating systems are set as class default_os --- lib/spack/spack/architecture.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index c4ef0805ac2..0e006bdf40c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -168,9 +168,11 @@ def _detect_mac_os(self): def add_operating_system(self, name=None, operating_system=None): if self.name == 'linux': linux_os = self._detect_linux_os() + self.default_os = linux_os.name self.operating_sys[linux_os.name] = linux_os elif self.name == 'darwin': mac_os = self._detect_mac_os() + self.default_os = mac_os.name self.operating_sys[mac_os.name] = mac_os else: self.operating_sys[name] = operating_system From c7993010116e02426c73dc32d8cfb74b8d542868 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 13 Feb 2016 14:37:07 -0800 Subject: [PATCH 114/375] Changed the method in which architecture is converted from string to namedtuple. Also added some TODO comments to later consider changing how architecture is handled --- lib/spack/spack/spec.py | 166 +++++++++++++++++++++++++--------------- 1 file changed, 104 insertions(+), 62 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 3b2d54d0473..9b5df9a8252 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -63,7 +63,7 @@ if it comes immediately after the compiler name. Otherwise it will be associated with the current package spec. -6. The target to build with. This is needed on machines where +6. The architecture to build with. This is needed on machines where cross-compilation is required Here is the EBNF grammar for a spec:: @@ -72,9 +72,9 @@ dep_list = { ^ spec } spec = id [ options ] options = { @version-list | +variant | -variant | ~variant | - %compiler | =target } + %compiler | =architecture } variant = id - target = id + architecture = id compiler = id [ version-list ] version-list = version [ { , version } ] version = id | id: | :id | id:id @@ -107,6 +107,7 @@ from llnl.util.tty.color import * import spack +import spack.architecture import spack.parse import spack.error import spack.compilers as compilers @@ -123,7 +124,7 @@ # Convenient names for color formats so that other things can use them compiler_color = '@g' version_color = '@c' -target_color = '@m' +architecture_color = '@m' enabled_variant_color = '@B' disabled_variant_color = '@r' dependency_color = '@.' @@ -134,7 +135,7 @@ See spack.color for descriptions of the color codes. """ color_formats = {'%' : compiler_color, '@' : version_color, - '=' : target_color, + '=' : architecture_color, '+' : enabled_variant_color, '~' : disabled_variant_color, '^' : dependency_color, @@ -411,7 +412,7 @@ def __init__(self, spec_like, *dep_like, **kwargs): self.name = other.name self.dependents = other.dependents self.versions = other.versions - self.target = other.target + self.architecture = other.architecture self.compiler = other.compiler self.dependencies = other.dependencies self.variants = other.variants @@ -456,11 +457,11 @@ def _set_compiler(self, compiler): self.compiler = compiler - def _set_target(self, target): - """Called by the parser to set the target.""" - if self.target: raise DuplicateTargetError( - "Spec for '%s' cannot have two targets." % self.name) - self.target = target # a string can be set + def _set_architecture(self, architecture): + """Called by the parser to set the architecture.""" + if self.architecture: raise DuplicateArchitectureError( + "Spec for '%s' cannot have two architectures." % self.name) + self.architecture = architecture # a string can be set def _add_dependency(self, spec): @@ -516,7 +517,7 @@ def is_virtual(name): @property def concrete(self): """A spec is concrete if it can describe only ONE build of a package. - If any of the name, version, target, compiler, + If any of the name, version, architecture, compiler, variants, or depdenencies are ambiguous,then it is not concrete. """ if self._concrete: @@ -525,7 +526,7 @@ def concrete(self): self._concrete = bool(not self.virtual and self.versions.concrete and self.variants.concrete - and self.target + and self.architecture and self.compiler and self.compiler.concrete and self.dependencies.concrete) return self._concrete @@ -660,10 +661,12 @@ def to_node_dict(self): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } - if self.target: - d['target'] = self.target.target.to_dict() + if self.architecture: + # TODO: Fix the target.to_dict to account for the tuple + # Want it to be a dict of dicts + d['architecture'] = self.architecture.target.to_dict() else: - d['target'] = None + d['architecture'] = None if self.compiler: d.update(self.compiler.to_dict()) else: @@ -689,7 +692,9 @@ def from_node_dict(node): spec = Spec(name) spec.versions = VersionList.from_dict(node) - spec.target = spack.architecture.Target.from_dict(node['target']) + # TODO: Need to fix the architecture.Target.from_dict + spec.architecture = spack.architecture.Target.from_dict( + node['architecture']) if node['compiler'] is None: spec.compiler = None @@ -760,7 +765,7 @@ def _concretize_helper(self, presets=None, visited=None): # to presets below, their constraints will all be merged, but we'll # still need to select a concrete package later. changed |= any( - (spack.concretizer.concretize_target(self), + (spack.concretizer.concretize_architecture(self), spack.concretizer.concretize_compiler(self), spack.concretizer.concretize_version(self), spack.concretizer.concretize_variants(self))) @@ -1149,10 +1154,11 @@ def constrain(self, other, deps=True): raise UnsatisfiableVariantSpecError(self.variants[v], other.variants[v]) - if self.target is not None and other.target is not None: - if self.target != other.target: - raise UnsatisfiableTargetSpecError(self.target, - other.target) + # TODO: Check out the logic here + if self.architecture is not None and other.architecture is not None: + if self.architecture != other.architecture: + raise UnsatisfiableTargetSpecError(self.architecture, + other.architecture) changed = False if self.compiler is not None and other.compiler is not None: @@ -1164,9 +1170,9 @@ def constrain(self, other, deps=True): changed |= self.versions.intersect(other.versions) changed |= self.variants.constrain(other.variants) - old = self.target - self.target = self.target or other.target - changed |= (self.target != old) + old = self.architecture + self.architecture = self.architecture or other.architecture + changed |= (self.architecture != old) if deps: changed |= self._constrain_dependencies(other) @@ -1231,34 +1237,67 @@ def _autospec(self, spec_like): except SpecError: return parse_anonymous_spec(spec_like, self.name) + def _is_valid_platform(self, platform, platform_list): + if platform in platform_names: + return True + return False - def add_target_from_string(self, arch): - """If only a target is provided, spack will assume the default architecture. - A platform-target pair can be input delimited by a '-'. If either portion of - a platform-target pair is empty, spack will supply a default, in the case of - a blank target the default will be dependent on the platform. - E.g. x86_64 -> 64 bit x86 - bgq- -> default bgq target (back end/powerpc) - cray-hawswell -> haswell target on cray platform + def _is_valid_target(self, target, platform): + if target in platform.targets: + return True + return False + + def add_architecture_from_string(self, arch): + """ The user is able to provide a architecture string of the form + platform-os-target. This string will be parsed by this function + and turn the architecture string into an architecture tuple of + platform, operating system and target processor classes. + The platform-os-target triplet can be delimited by a '-'. If any + portion of the architecture triplet is empty, spack will supply + the default. If the architecture is blank then defaults will + be provided based off of the platform + + e.g + =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) + + =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) + + =bgq -> (bgq, + default_os, + default_target) + + =elcapitan -> (darwin, elcapitan, x86_64) + + =x86_64 -> (autodetected platform, + default_os, + x86_64) """ - Arch = namedtuple("Arch", "arch_os target") - platform = spack.architecture.sys_type() - + + platform_list = spack.architecture.all_platforms() + platform_names = [plat.__name__.lower() for plat in platform_list] if arch is None: return + # Get all the platforms to check whether string is valid + Arch = namedtuple("Arch", "platform platform_os target") + arch_list = arch.split("-") + platform = spack.architecture.sys_type() + for entry in arch_list: + if _is_valid_platform(entry, platform_names): + platform = entry() + elif _is_valid_target(entry, platform): + target = entry + else: + platform_os = entry - if '-' in arch: - os_name, target = arch.split('-') - self.target = Arch(arch_os=platform.operating_system(os_name), - target = platform.target(target)) - elif arch in platform.targets: - self.target = Arch(arch_os=platform.operating_system('default'), - target=platform.target(target)) - else: - os_name = arch # Odd naming here, definitely need to change - self.target = Arch(arch_os=platform.operating_system(os_name), - target=platform.target('default')) + if target is None: + target = platform.target('default') + if platform_os is None: + platform_os = platform.operating_system('default') + self.architecture = Arch(platform=platform, + platform_os=platform_os, + target=target) + def satisfies(self, other, deps=True, strict=False): """determine if this spec satisfies all constraints of another. @@ -1306,15 +1345,16 @@ def satisfies(self, other, deps=True, strict=False): # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. - if isinstance(self.target, basestring): - self.add_target_from_string(self.target) - if isinstance(other.target, basestring): - other.add_target_from_string(other.target) + if isinstance(self.architecture, basestring): + self.add_architecture_from_string(self.architecture) + if isinstance(other.architecture, basestring): + other.add_architecture_from_string(other.architecture) - if self.target and other.target: - if self.target != other.target: + # TODO: Need to make sure that comparisons can be made via classes + if self.architecture and other.architecture: + if self.architecture != other.architecture: return False - elif strict and (other.target and not self.target): + elif strict and (other.architecture and not self.architecture): return False # If we need to descend into dependencies, do it, otherwise we're done. @@ -1384,11 +1424,12 @@ def _dup(self, other, **kwargs): spec but not its dependencies. """ + # TODO: Check if comparisons for tuple are valid # We don't count dependencies as changes here changed = True if hasattr(self, 'name'): changed = (self.name != other.name and self.versions != other.versions and \ - self.target != other.target and self.compiler != other.compiler and \ + self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external and \ self.external_module != other.external_module) @@ -1396,7 +1437,7 @@ def _dup(self, other, **kwargs): # Local node attributes get copied first. self.name = other.name self.versions = other.versions.copy() - self.target = other.target + self.architecture = other.architecture self.compiler = other.compiler.copy() if other.compiler else None if kwargs.get('cleardeps', True): self.dependents = DependencyMap() @@ -1526,7 +1567,7 @@ def ne_dag(self, other): def _cmp_node(self): """Comparison key for just *this node* and not its deps.""" return (self.name, self.versions, self.variants, - self.target, self.compiler) + self.architecture, self.compiler) def eq_node(self, other): @@ -1585,7 +1626,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs): Anything else is copied verbatim into the output stream. *Example:* ``$_$@$+`` translates to the name, version, and options - of the package, but no dependencies, target, or compiler. + of the package, but no dependencies, architecture, or compiler. TODO: allow, e.g., $6# to customize short hash length TODO: allow, e.g., $## for full hash. @@ -1628,9 +1669,10 @@ def write(s, c): elif c == '+': if self.variants: write(fmt % str(self.variants), c) + # TODO: Check string methods here elif c == '=': - if self.target: - write(fmt % (c + str(self.target)), c) + if self.architecture: + write(fmt % (c + str(self.architecture)), c) elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) elif c == '$': @@ -2036,10 +2078,10 @@ def __init__(self, pkg, variant): "Package %s has no variant %s!" % (pkg, variant)) -class DuplicateTargetError(SpecError): +class DuplicateArchitectureError(SpecError): """Raised when the same target occurs in a spec twice.""" def __init__(self, message): - super(DuplicateTargetError, self).__init__(message) + super(DuplicateArchitectureError, self).__init__(message) class InconsistentSpecError(SpecError): From 35a602baaf39fb5056f3dbb7eccbd638e126159f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:17:57 -0800 Subject: [PATCH 115/375] Changed target checking to architecture checking for abi compatible --- lib/spack/spack/abi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index 1dd49d6b2c4..0f07feda5f1 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -34,9 +34,10 @@ class ABI(object): """This class provides methods to test ABI compatibility between specs. The current implementation is rather rough and could be improved.""" - def target_compatible(self, parent, child): + def architecture_compatible(self, parent, child): """Returns true iff the parent and child specs have ABI compatible targets.""" - return not parent.target or not child.target or parent.target == child.target + return not parent.architecture or not \ + child.achitecture or parent.architecture == child.architecture @memoized @@ -123,6 +124,6 @@ def compiler_compatible(self, parent, child, **kwargs): def compatible(self, parent, child, **kwargs): """Returns true iff a parent and child spec are ABI compatible""" loosematch = kwargs.get('loose', False) - return self.target_compatible(parent, child) and \ + return self.architecture_compatible(parent, child) and \ self.compiler_compatible(parent, child, loose=loosematch) From 19dca4bcc9dcdd902fb5490b4ee49b9dc92aa5ba Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:21:36 -0800 Subject: [PATCH 116/375] Added some _cmp_key methods to OperatingSystem class, and also changed the way Platforms are compared. Created a mini inherited class named Arch that inherits from namedtuple. The reason for this is to override the namedtuple __str__ method to make arch tuples look prettier when printed out. Rather than Arch(platform= ... ) it prints to platform-os-target which makes directory paths to packages look cleaner. --- lib/spack/spack/architecture.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 0e006bdf40c..3cbce5dc526 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +from collections import namedtuple import imp import platform as py_platform import inspect @@ -178,7 +179,7 @@ def add_operating_system(self, name=None, operating_system=None): self.operating_sys[name] = operating_system def operating_system(self, name): - if name == 'default': + if name == 'default_os': name = self.default_os if name == 'front_os': name = self.front_os @@ -203,9 +204,10 @@ def __str__(self): return self.name def _cmp_key(self): - return (self.name, (_cmp_key(t) for t in self.targets.values())) - + return (self.name, (_cmp_key(t) for t in self.targets.values()), + (_cmp_key(o) for o in self.operating_sys.values())) +@key_ordering class OperatingSystem(object): """ Operating System class. It will include the name and version. The biggest attribute to this class will be the compiler finding @@ -222,6 +224,19 @@ def __str__(self): def __repr__(self): return self.__str__() + def _cmp_key(self): + return (self.name, self.version) + + +class Arch(namedtuple("Arch", "platform platform_os target")): + """ namedtuple for Architecture. Will have it's own __str__ method + to make printing out the tuple easier and also won't make directory + paths look odd """ + __slots__ = () + + def __str__(self): + return (self.platform.name +"-"+ + self.platform_os.name + "-" + self.target.name) @memoized def all_platforms(): From 31ab2383067830e47d54788a2cac01f949e77798 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:22:23 -0800 Subject: [PATCH 117/375] Beginning attemps to fix concretization method to account for the new tuple architecture --- lib/spack/spack/concretize.py | 73 ++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 8886b3cd931..1c373679a2d 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,55 +210,74 @@ def concretize_version(self, spec): return True # Things changed + def _concretize_platform(self, arch, platform): + if issubclass(arch.platform.__class__, spack.architecture.Platform): + return True + else: + arch.platform = platform + def _concretize_operating_system(self, arch, platform): """ Future method for concretizing operating system """ - if isinstance(arch.arch_os, spack.architecture.OperatingSystem): + if isinstance(arch.platform_os, spack.architecture.OperatingSystem): return False else: - arch.arch_os = platform.operating_system('default') + arch.arch_os = platform.operating_system('default_os') return True - def _concretize_arch_target(self, arch, platform): + def _concretize_target(self, arch, platform): if isinstance(arch.target, spack.architecture.Target): return False else: arch.target = platform.target('default') return True - def concretize_target(self, spec): - """If the spec already has an target and it is a an target type, - return. Otherwise, if it has a target that is a string type, generate a - target based on that type. If it has no target and the root of the - DAG has an target, then use that. Otherwise, take the system's default - target. + def concretize_architecture(self, spec): + """If the spec is empty provide the defaults of the platform. If the + architecture is not a basestring, then check if either the platform, + target or operating system are concretized. If any of the fields are + changed then return True. If everything is concretized (i.e the + architecture attribute is a namedtuple of classes) then return False. + If the target is a string type, then convert the string into a + concretized architecture. If it has no architecture and the root of the + DAG has an architecture, then use the root otherwise use the defaults + on the platform. """ - platform = spack.architecture.sys_type() - if spec.target is None: + platform = spack.architecture.sys_type() + import ipdb;ipdb.set_trace() + if spec.architecture is None: # Create an empty tuple - Arch = collections.namedtuple("Arch", "arch_os target") - spec.target = Arch(arch_os=platform.operating_system('default'), - target=platform.target('default')) - return True - # If there is a target and it is a tuple and has both filled return - # False - if not isinstance(spec.target, basestring): - return any((self._concretize_operating_system(spec.target, platform), - self._concretize_arch_target(spec.target, platform))) + Arch = spack.architecture.Arch + # Set the architecture to all defaults + spec.architecture = Arch(platform=platform, + platform_os=platform.operating_system('default_os'), + target=platform.target('default')) + return True + #If there is a target and it is a tuple and has both filled return + #False + if not isinstance(spec.architecture, basestring): + return any(( + self._concretize_platform(spec.architecture, platform), + self._concretize_operating_system(spec.architecture, platform), + self._concretize_arch_target(spec.architecture, platform))) else: spec.add_target_from_string(spec.target) - if spec.root.target and \ - not isinstance(spec.root.target, basestring): - bool_flag = any( - (self._concretize_operating_system(spec.root.target, platform), - self._concretize_arch_target(spec.root.target, platform))) - spec.target =spec.root.target + # Does not look pretty at all!!! + if spec.root.architecture and \ + not isinstance(spec.root.architecture, basestring): + bool_flag = any(( + self._concretize_platform(spec.root.architecture, platform), + self._concretize_operating_system(spec.root.architecture, + platform), + self._concretize_target(spec.root.target, platform))) + spec.architecture =spec.root.architecture return bool_flag else: - spec.add_target_from_string(spec.root.target) + spec.add_architecture_from_string(spec.root.architecture) + return True # if there is no target specified used the defaults From 62b0293963bd21ff734993e5ac577650af0faa3f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:23:38 -0800 Subject: [PATCH 118/375] Cleaned up the file. Got rid of commented out sections of code since they weren't going to be used anyway --- lib/spack/spack/platforms/cray_xc.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index b37d36a62c8..d48b6ffcfa2 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -17,7 +17,7 @@ def __init__(self): if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' - super(CrayXc, self).__init__('cray_xc') + super(CrayXc, self).__init__('crayxc') # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: @@ -28,23 +28,21 @@ def __init__(self): self.front_end = self.default self.back_end = self.default - # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. self.add_target('sandybridge', Target('sandybridge', 'PATH')) - self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) - self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) - - self.add_operating_system('SuSE11', OperatingSystem('SuSE11', '11')) - self.add_operating_system('GNU/Linux', OperatingSystem('GNU/Linux', '10')) -# self.add_target(self.front_end, Target(self.front_end, 'PATH')) - # Back End compiler needs the proper target module loaded. -# self.add_target(self.back_end, Target(self.front_end, 'MODULES', 'craype-'+ self.back_end)) -# self.add_target(self.default, Target(self.default, 'MODULES', 'craype-' + self.default)) - # This is kludgy and the order matters when the targets are all haswell - # This is because the last one overwrites the others when they have the - # same name. + self.add_target('ivybridge', + Target('ivybridge', 'MODULES', 'craype-ivybridge')) + + self.add_target('haswell', + Target('haswell', 'MODULES', 'craype-haswell')) + + self.add_operating_system('SuSE11', + OperatingSystem('SuSE11', '11')) + + self.add_operating_system('GNU/Linux', + OperatingSystem('GNU/Linux', '10')) @classmethod def detect(self): From a3039c4c67b376d2e8577c2227cffac8873bdc74 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 16 Feb 2016 15:26:07 -0800 Subject: [PATCH 119/375] Changed add_architecture_from_string, it now loops through the string and checks whether each piece of string is a valid platform, operating system and target. If the operating system or target are none it will use the defaults. Updated the documentation for that method. One thing that bothers me is how to account for the multitude of operating systems when cross compiling. If someone wants to compile with another operating system not found on current platform. How can spack check to see if it is valid? --- lib/spack/spack/spec.py | 64 +++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 9b5df9a8252..91e9e432eba 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1238,14 +1238,18 @@ def _autospec(self, spec_like): return parse_anonymous_spec(spec_like, self.name) def _is_valid_platform(self, platform, platform_list): - if platform in platform_names: + if platform in platform_list: return True return False def _is_valid_target(self, target, platform): if target in platform.targets: return True - return False + return False + def _is_valid_os(self, os_string, platform): + if os_string in platform.operating_sys: + return True + return False def add_architecture_from_string(self, arch): """ The user is able to provide a architecture string of the form @@ -1254,8 +1258,9 @@ def add_architecture_from_string(self, arch): platform, operating system and target processor classes. The platform-os-target triplet can be delimited by a '-'. If any portion of the architecture triplet is empty, spack will supply - the default. If the architecture is blank then defaults will - be provided based off of the platform + the default. If the entire architecture field is blank then + defaults will be provided based off of the platform. + This happens in the concretize_architecture method in concretize.py e.g =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) @@ -1272,27 +1277,38 @@ def add_architecture_from_string(self, arch): default_os, x86_64) """ - + if arch is None: return + platform_list = spack.architecture.all_platforms() - platform_names = [plat.__name__.lower() for plat in platform_list] - if arch is None: - return - # Get all the platforms to check whether string is valid - Arch = namedtuple("Arch", "platform platform_os target") + platform_names = {plat.__name__.lower():plat for plat in platform_list} + Arch = spack.architecture.Arch arch_list = arch.split("-") - platform = spack.architecture.sys_type() + + # Create instance of current platform, gets overwritten if user + # provided a platform spec. + platform = spack.architecture.sys_type() + target = None + platform_os = None + for entry in arch_list: - if _is_valid_platform(entry, platform_names): - platform = entry() - elif _is_valid_target(entry, platform): - target = entry + if self._is_valid_platform(entry, platform_names): + if entry != platform.name: + platform = platform_dict[entry]() # Create instance of platform + elif self._is_valid_target(entry, platform): + target = platform.target(entry) + # Need to figure out if we're supporting arbitrary os's and how + # to account for them + # Not really a good implementation since the user can add + # gibberish and spack will see it as an os + elif self._is_valid_os(entry, platform): + platform_os = platform.operating_system(entry) else: - platform_os = entry + raise UnknownArchitectureSpecError(entry) if target is None: target = platform.target('default') if platform_os is None: - platform_os = platform.operating_system('default') + platform_os = platform.operating_system('default_os') self.architecture = Arch(platform=platform, platform_os=platform_os, @@ -1854,7 +1870,6 @@ def __init__(self): def do_parse(self): specs = [] - try: while self.next: if self.accept(ID): @@ -1889,7 +1904,7 @@ def spec(self): spec.name = self.token.value spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.target = None + spec.architecture = None spec.compiler = None spec.external = None spec.external_module = None @@ -1920,7 +1935,7 @@ def spec(self): spec._set_compiler(self.compiler()) elif self.accept(EQ): - spec._set_target(self.target()) + spec._set_architecture(self.architecture()) else: break @@ -1938,7 +1953,7 @@ def variant(self): return self.token.value - def target(self): + def architecture(self): self.expect(ID) return self.token.value @@ -2077,6 +2092,13 @@ def __init__(self, pkg, variant): super(UnknownVariantError, self).__init__( "Package %s has no variant %s!" % (pkg, variant)) +class UnknownArchitectureSpecError(SpecError): + """ Raised when an entry in a string field is neither a platform, + operating system or a target. """ + def __init__(self, architecture_spec_entry): + super(UnknownArchitectureSpecError, self).__init__( + "Architecture spec %s is not a valid spec entry" % ( + architecture_spec_entry)) class DuplicateArchitectureError(SpecError): """Raised when the same target occurs in a spec twice.""" From 2a79537ba1120620f79d39bd4081a6a11b7862c4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:44:38 -0800 Subject: [PATCH 120/375] Fixed silly typo that gave errors in tests --- lib/spack/spack/abi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/abi.py b/lib/spack/spack/abi.py index 0f07feda5f1..bd9bc30fd1b 100644 --- a/lib/spack/spack/abi.py +++ b/lib/spack/spack/abi.py @@ -36,8 +36,8 @@ class ABI(object): def architecture_compatible(self, parent, child): """Returns true iff the parent and child specs have ABI compatible targets.""" - return not parent.architecture or not \ - child.achitecture or parent.architecture == child.architecture + return not parent.architecture or not child.architecture \ + or parent.architecture == child.architecture @memoized From 2c20fc2ebf8e9b360f3935a1867d9b95c69513ac Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:45:53 -0800 Subject: [PATCH 121/375] Changed module loading to use arch tuple instead of regular target.module --- lib/spack/spack/build_environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index d849b4b56c3..6e03394777a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -226,8 +226,8 @@ def set_build_environment_variables(pkg): pkg_config_dirs.append(pcdir) path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.target.module_name: - load_module(pkg.spec.target.module_name) + if pkg.spec.architecture.target.module_name: + load_module(pkg.spec.architecture.target.module_name) def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. From 524e9b372ee85508abad1163368c7a9fdc09fcc3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:47:29 -0800 Subject: [PATCH 122/375] From now on all targets are now architecture. Architecture is a named tuple so any time we need access to target we do architecture.target. --- lib/spack/spack/concretize.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 1c373679a2d..6c2aaeda011 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -180,7 +180,7 @@ def concretize_version(self, spec): # If there are known available versions, return the most recent # version that satisfies the spec - pkg = spec.package + pkg = spec.package # Gives error here with dynist cmp_versions = partial(spack.pkgsort.version_compare, spec.name) valid_versions = sorted( [v for v in pkg.versions @@ -212,9 +212,10 @@ def concretize_version(self, spec): def _concretize_platform(self, arch, platform): if issubclass(arch.platform.__class__, spack.architecture.Platform): - return True + return False else: arch.platform = platform + return True def _concretize_operating_system(self, arch, platform): """ Future method for concretizing operating system """ @@ -245,7 +246,6 @@ def concretize_architecture(self, spec): """ platform = spack.architecture.sys_type() - import ipdb;ipdb.set_trace() if spec.architecture is None: # Create an empty tuple Arch = spack.architecture.Arch @@ -260,9 +260,9 @@ def concretize_architecture(self, spec): return any(( self._concretize_platform(spec.architecture, platform), self._concretize_operating_system(spec.architecture, platform), - self._concretize_arch_target(spec.architecture, platform))) + self._concretize_target(spec.architecture, platform))) else: - spec.add_target_from_string(spec.target) + spec.add_architecture_from_string(spec.target) # Does not look pretty at all!!! if spec.root.architecture and \ @@ -325,18 +325,18 @@ def concretize_compiler(self, spec): link to this one, to maximize compatibility. """ # Pass on concretizing the compiler if the target is not yet determined - if not spec.target: + if not spec.architecture.target: #Although this usually means changed, this means awaiting other changes return True # Only use a matching compiler if it is of the proper style # Takes advantage of the proper logic already existing in compiler_for_spec # Should think whether this can be more efficient - def _proper_compiler_style(cspec, target): + def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - if target.target.compiler_strategy == 'PATH': + if architecture.target.compiler_strategy == 'PATH': filter(lambda c: not c.modules, compilers) - if target.target.compiler_strategy == 'MODULES': + if architecture.target.compiler_strategy == 'MODULES': filter(lambda c: c.modules, compilers) return compilers @@ -369,7 +369,7 @@ def _proper_compiler_style(cspec, target): # copy concrete version into other_compiler index = len(matches)-1 - while not _proper_compiler_style(matches[index], spec.target): + while not _proper_compiler_style(matches[index], spec.architecture): index -= 1 if index == 0: raise NoValidVersionError(spec) From ab4006ee2c4e7ecbf256a9de56e54e8573b37e01 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:48:53 -0800 Subject: [PATCH 123/375] Changed so that directory layout uses the platform-os-target string version of the arch tuple --- lib/spack/spack/directory_layout.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index da3441fa139..435ecfdf4dd 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -200,8 +200,7 @@ def relative_path_for_spec(self, spec): spec.version, spec.dag_hash(self.hash_len)) - path = join_path( - spec.target, + path = join_path(spec.architecture, "%s-%s" % (spec.compiler.name, spec.compiler.version), dir_name) From 1d484dbe5fb70885875dcbb2e795c4d517646e77 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:49:36 -0800 Subject: [PATCH 124/375] Changed so that modules use correct file path with spec.architecture instead of spec.target --- lib/spack/spack/modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/modules.py b/lib/spack/spack/modules.py index f84ac75c773..56a61adefb9 100644 --- a/lib/spack/spack/modules.py +++ b/lib/spack/spack/modules.py @@ -193,7 +193,7 @@ class Dotkit(EnvModule): @property def file_name(self): - return join_path(Dotkit.path, self.spec.target, + return join_path(Dotkit.path, self.spec.architecture, self.spec.format('$_$@$%@$+$#.dk')) @property @@ -230,7 +230,7 @@ class TclModule(EnvModule): @property def file_name(self): - return join_path(TclModule.path, self.spec.target, self.use_name) + return join_path(TclModule.path, self.spec.architecture, self.use_name) @property From 54042e399b5cfb4344643d8447c4274d34842643 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:50:10 -0800 Subject: [PATCH 125/375] Changed target to architecture.target --- lib/spack/spack/package.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 1148e4ac3df..b0d84863de0 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -588,11 +588,13 @@ def prefix(self): @property + #TODO: Change this to architecture def compiler(self): """Get the spack.compiler.Compiler object used to build this package.""" if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") - return spack.compilers.compiler_for_spec(self.spec.compiler, self.spec.target) + return spack.compilers.compiler_for_spec(self.spec.compiler, + self.spec.architecture.target) def url_version(self, version): From d9e8bf18078bc4df381b05820b14365f6e0ae01f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:51:14 -0800 Subject: [PATCH 126/375] Added ARCHITECTURE field when showing specs --- lib/spack/spack/spec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 91e9e432eba..0b424989109 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1735,9 +1735,9 @@ def write(s, c): elif named_str == 'OPTIONS': if self.variants: write(fmt % str(self.variants), '+') - elif named_str == 'TARGET': - if self.target: - write(fmt % str(self.target), '=') + elif named_str == 'ARCHITECTURE': + if self.architecture: + write(fmt % self.architecture, '=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dep_hash(8))) From 93a92a00eeaf6e3e721b2150fbc44bd1b388ee66 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:53:09 -0800 Subject: [PATCH 127/375] Changed abstract.target to abstract.architecture.target and abstract.architecture.target.concrete --- lib/spack/spack/test/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 24037191347..c12aada0764 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -46,7 +46,7 @@ def check_spec(self, abstract, concrete): if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) - if abstract.target and abstract.target.concrete: + if abstract.architecture and abstract.architecture.target.concrete: self.assertEqual(abstract.target, concrete.target) From 0a6b54a8c40045dac7d763880de496cd01984c0c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:53:56 -0800 Subject: [PATCH 128/375] Added module compilers so that no CompilerSpecError is thrown --- var/spack/mock_configs/site_spackconfig/compilers.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml index 0a2dc893e29..fcbf7a53f16 100644 --- a/var/spack/mock_configs/site_spackconfig/compilers.yaml +++ b/var/spack/mock_configs/site_spackconfig/compilers.yaml @@ -5,8 +5,18 @@ compilers: cxx: /path/to/clang++ f77: None fc: None + modules: None gcc@4.5.0: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran + modules: None + gcc@5.2.0: + cc: cc + cxx: CC + f77: ftn + fc: ftn + modules: + - PrgEnv-gnu + - gcc/5.2.0 From 3558404c037cdd0730cd4672c8522b9bb3bd6bd2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 17 Feb 2016 14:54:39 -0800 Subject: [PATCH 129/375] Check difference by using arch tuple rather than .target --- var/spack/mock_packages/multimethod/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index b37db9f1eb4..b926d42d7be 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -120,7 +120,7 @@ def has_a_default(self): for target in targets: @when('='+target.name) def different_by_target(self): - return self.spec.target.name + return self.spec.architecture.target.name # From 28218755a5b11dd02c3c7e4c4a503a110ade94e2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 18 Feb 2016 13:06:55 -0800 Subject: [PATCH 130/375] Changed spec.target to spec.architecture --- lib/spack/spack/concretize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 6c2aaeda011..0e7bf53b445 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -262,7 +262,7 @@ def concretize_architecture(self, spec): self._concretize_operating_system(spec.architecture, platform), self._concretize_target(spec.architecture, platform))) else: - spec.add_architecture_from_string(spec.target) + spec.add_architecture_from_string(spec.architecture) # Does not look pretty at all!!! if spec.root.architecture and \ From 18ddbae60e0fa91cbbc73a1b501c489c3f7fc8d2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:49:53 -0800 Subject: [PATCH 131/375] Added new module path to operating system file --- lib/spack/spack/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index e1ef094f17c..6ad9b87ca22 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -39,6 +39,7 @@ module_path = join_path(lib_path, "spack") platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") +operating_system_path = join_path(module_path, 'operating_system') test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") var_path = join_path(prefix, "var", "spack") From a8de45ce28b8277c0ca1740aa53e1e4e334596d1 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:50:51 -0800 Subject: [PATCH 132/375] Got rid of methods used to detect mac osx and linux osx. Now those methods are used my operating system subclasses --- lib/spack/spack/architecture.py | 88 ++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 34 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3cbce5dc526..3ba45f3d824 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -148,35 +148,29 @@ def target(self, name): return self.targets[name] - def _detect_linux_os(self): - return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) + #def _detect_linux_os(self): + # return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) - def _detect_mac_os(self): - mac_releases = {'10.6': "snowleopard", - "10.7": "lion", - "10.8": "mountainlion", - "10.9": "mavericks", - "10.10": "yosemite", - "10.11": "elcapitan"} - mac_ver = py_platform.mac_ver()[:-2] - try: - os_name = mac_releases[mac_ver] - return OperatingSystem(os_name, mac_ver) - except KeyError: - os_name = "mac_os" - return OperatingSystem(os_name, mac_ver) - - def add_operating_system(self, name=None, operating_system=None): - if self.name == 'linux': - linux_os = self._detect_linux_os() - self.default_os = linux_os.name - self.operating_sys[linux_os.name] = linux_os - elif self.name == 'darwin': - mac_os = self._detect_mac_os() - self.default_os = mac_os.name - self.operating_sys[mac_os.name] = mac_os - else: - self.operating_sys[name] = operating_system + #def _detect_mac_os(self): + # mac_releases = {'10.6': "snowleopard", + # "10.7": "lion", + # "10.8": "mountainlion", + # "10.9": "mavericks", + # "10.10": "yosemite", + # "10.11": "elcapitan"} + # mac_ver = py_platform.mac_ver()[:-2] + # try: + # os_name = mac_releases[mac_ver] + # return OperatingSystem(os_name, mac_ver) + # except KeyError: + # os_name = "mac_os" + # return OperatingSystem(os_name, mac_ver) + + def add_operating_system(self, name, os_class): + """ Add the operating_system class object into the + platform.operating_sys dictionary + """ + self.operating_sys[name] = os_class def operating_system(self, name): if name == 'default_os': @@ -207,27 +201,37 @@ def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values()), (_cmp_key(o) for o in self.operating_sys.values())) + @key_ordering class OperatingSystem(object): - """ Operating System class. It will include the name and version. - The biggest attribute to this class will be the compiler finding - strategy. At the moment the compiler finding strategy is going to - be a string. + """ Operating System will be like a class similar to platform extended + by subclasses for the specifics. Operating System will contain the + compiler finding logic. Instead of calling two separate methods to + find compilers we call find_compilers method for each operating system """ def __init__(self, name, version): self.name = name self.version = version def __str__(self): - return self.name + return self.name + self.version def __repr__(self): return self.__str__() + + def compiler_strategy(self): + """ The compiler strategy will be overwritten by the subclass. + Depending on where it comes from it will either use compilers + based off of MODULES search method or the PATH search method + """ + raise NotImplementedError() def _cmp_key(self): return (self.name, self.version) + - +#NOTE: Key error caused because Architecture has no comparison method +@key_ordering class Arch(namedtuple("Arch", "platform platform_os target")): """ namedtuple for Architecture. Will have it's own __str__ method to make printing out the tuple easier and also won't make directory @@ -238,6 +242,22 @@ def __str__(self): return (self.platform.name +"-"+ self.platform_os.name + "-" + self.target.name) + def _cmp_key(self): + return (self.platform.name, self.platform_os.name, self.target.name) + + @staticmethod + def to_dict(platform, os_name, target): + """ This function will handle all the converting of dictionaries + for each of the architecture classes. This is better than having + each class have a to_dict method when creating the dict of dicts + arch class + """ + d = {} + d['platform'] = platform + d['operating_system'] = os_name + d['target'] = target.to_dict() + return d + @memoized def all_platforms(): modules = [] From 725d6d5fce56aa7d0ae401ec71d1565835248250 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:52:09 -0800 Subject: [PATCH 133/375] import linux distro operating system subclass, set front-end os to LinuxDistro() --- lib/spack/spack/platforms/cray_xc.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index d48b6ffcfa2..b58c84d3eb1 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,6 +1,7 @@ import os -from spack.architecture import Platform, Target, OperatingSystem +from spack.architecture import Platform, Target +from spack.operating_system.linux_distro import LinuxDistro class CrayXc(Platform): priority = 20 @@ -9,8 +10,8 @@ class CrayXc(Platform): default = 'ivybridge' front_os = "SuSE11" - back_os = "GNU/Linux" - default_os = "GNU/Linux" + back_os = "CNL" + default_os = "CNL" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better @@ -31,18 +32,12 @@ def __init__(self): # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. self.add_target('sandybridge', Target('sandybridge', 'PATH')) - self.add_target('ivybridge', Target('ivybridge', 'MODULES', 'craype-ivybridge')) - self.add_target('haswell', - Target('haswell', 'MODULES', 'craype-haswell')) + Target('haswell', 'MODULES', 'craype-haswell')) - self.add_operating_system('SuSE11', - OperatingSystem('SuSE11', '11')) - - self.add_operating_system('GNU/Linux', - OperatingSystem('GNU/Linux', '10')) + self.add_operating_system('SuSE11', LinuxDistro()) @classmethod def detect(self): From 1a7d6ed49a8a3fd18fd3deb8ddc035ce0063766d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:52:52 -0800 Subject: [PATCH 134/375] import MacOSX subclass and added add_operating_system method to use subclass --- lib/spack/spack/platforms/darwin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index cfa1d957a17..5b797a78dbe 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,5 +1,6 @@ import subprocess from spack.architecture import Platform, Target +from spack.operating_system.mac_osx import MacOSX class Darwin(Platform): priority = 89 @@ -10,7 +11,9 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default, 'PATH')) - self.add_operating_system() + mac_os = MacOSX() + self.default_os = mac_os.name + self.add_operating_system(mac_os.name, mac_os) @classmethod def detect(self): From 22bf4bc08020898106ef2c8043f102ceeb83ec5c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:53:16 -0800 Subject: [PATCH 135/375] import linux distro subclass. Added method to add operating system to platform dictionary --- lib/spack/spack/platforms/linux.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index dcb08569a8a..220f1bbaf40 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,5 +1,6 @@ import subprocess from spack.architecture import Platform, Target +from spack.operating_system.linux_distro import LinuxDistro class Linux(Platform): priority = 90 @@ -10,7 +11,9 @@ class Linux(Platform): def __init__(self): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default, 'PATH')) - self.add_operating_system() + linux_dist = LinuxDistro() + self.default_os = str(linux_dist) + self.add_operating_system(str(linux_dist), linux_dist) @classmethod def detect(self): From 1367ccab93e6263e11477abe7ea05f1ad74d88a2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:53:55 -0800 Subject: [PATCH 136/375] New folder that will hold operating system subclasses --- lib/spack/spack/operating_system/__init__.py | 0 .../spack/operating_system/linux_distro.py | 25 +++++++++++ lib/spack/spack/operating_system/mac_osx.py | 45 +++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100644 lib/spack/spack/operating_system/__init__.py create mode 100644 lib/spack/spack/operating_system/linux_distro.py create mode 100644 lib/spack/spack/operating_system/mac_osx.py diff --git a/lib/spack/spack/operating_system/__init__.py b/lib/spack/spack/operating_system/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py new file mode 100644 index 00000000000..30a85fe61cb --- /dev/null +++ b/lib/spack/spack/operating_system/linux_distro.py @@ -0,0 +1,25 @@ +import platform as py_platform +import spack +from spack.architecture import Platform, OperatingSystem + +class LinuxDistro(OperatingSystem): + """ This class will represent the autodetected operating system + for a Linux System. Since there are many different flavors of + Linux, this class will attempt to encompass them all through + autodetection using the python module platform and the method + platform.dist() + """ + def __init__(self): + def detect_operating_system(): + name = py_platform.dist()[0] + version = py_platform.dist()[1] + return name, version + + name, version = detect_operating_system() + + super(LinuxDistro, self).__init__(name, version) + + @property + def compiler_strategy(self): + return "PATH" + diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py new file mode 100644 index 00000000000..0b939a5546f --- /dev/null +++ b/lib/spack/spack/operating_system/mac_osx.py @@ -0,0 +1,45 @@ +""" This class represents the MAC_OSX operating system. This will be auto + detected using the python platform.mac_ver. The MAC_OSX platform + will be represented using the major version operating system name, i.e + el capitan, yosemite...etc. +""" + +import spack +import os +import platform as py_platform +from spack.architecture import Platform, OperatingSystem + +class MacOSX(OperatingSystem): + def __init__(self): + """ Autodetects the mac version from a dictionary. Goes back as + far as 10.6 snowleopard. If the user has an older mac then + the version will just be a generic mac_os. + """ + + def get_mac_release(): + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + + mac_ver = py_platform.mac_ver()[0][:-2] + try: + name = mac_releases[mac_ver] + return name, mac_ver + except KeyError: + name = "mac_os" + return name, mac_ver + + name, version = get_mac_release() + + super(MacOSX, self).__init__(name, version) + + @property + def compiler_strategy(self): + return "PATH" + + + + From a385dae1aee6b7624707f42af8a6c52b9d9889bd Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 14:54:30 -0800 Subject: [PATCH 137/375] Unit tests to test operating system subclass creation and whether compiler strategy is set correctly --- lib/spack/spack/test/operating_system.py | 41 ++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 lib/spack/spack/test/operating_system.py diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py new file mode 100644 index 00000000000..9d6850bfa69 --- /dev/null +++ b/lib/spack/spack/test/operating_system.py @@ -0,0 +1,41 @@ +""" Test checks if the operating_system class is created correctly and that +the functions are using the correct operating_system. Also checks whether +the operating_system correctly uses the compiler_strategy +""" + +import unittest +import os +import platform +from spack.platforms.cray_xc import CrayXc +from spack.platforms.linux import Linux +from spack.platforms.darwin import Darwin +from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_system.mac_osx import MacOSX + +class TestOperatingSystem(unittest.TestCase): + + def setUp(self): + cray_xc = CrayXc() + linux = Linux() + darwin = Darwin() + self.cray_operating_sys = cray_xc.operating_system('front_os') + self.darwin_operating_sys = darwin.operating_system('default_os') + self.linux_operating_sys = linux.operating_system('default_os') + + def test_cray_front_end_operating_system(self): + self.assertIsInstance(self.cray_operating_sys, LinuxDistro) + + def test_cray_front_end_compiler_strategy(self): + self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") + + def test_linux_operating_system(self): + print self.linux_operating_sys + self.assertIsInstance(self.linux_operating_sys, LinuxDistro) + + def test_linux_compiler_strategy(self): + self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") + + + + + From ebc5e26c2d95971f414378631089fee270beeb51 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:16:20 -0800 Subject: [PATCH 138/375] Added compiler strategy field to Operating System class --- lib/spack/spack/architecture.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 3ba45f3d824..574076bae4b 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -209,26 +209,28 @@ class OperatingSystem(object): compiler finding logic. Instead of calling two separate methods to find compilers we call find_compilers method for each operating system """ - def __init__(self, name, version): + + def __init__(self, name, version, compiler_strategy): self.name = name self.version = version + self.compiler_strategy = compiler_strategy def __str__(self): return self.name + self.version def __repr__(self): return self.__str__() - - def compiler_strategy(self): - """ The compiler strategy will be overwritten by the subclass. - Depending on where it comes from it will either use compilers - based off of MODULES search method or the PATH search method - """ - raise NotImplementedError() def _cmp_key(self): return (self.name, self.version) + def compiler_strategy(self): + """ Operating Systems will now be in charge of the compiler finding + strategy. They will each have their own find_compilers method + which depending on their compiler strategy will find the compilers + using a specific method (i.e PATH vs MODULES) + """ + raise NotImplementedError() #NOTE: Key error caused because Architecture has no comparison method @key_ordering From 3c87d137a3f0823289c52ffff87e58b8b1d7e271 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:17:33 -0800 Subject: [PATCH 139/375] Added compiler strategy entry and also added new operating system subclass called CNL for compute node linux (name will probably change because I can't find docs on the EXACT name) --- lib/spack/spack/operating_system/cnl.py | 20 +++++++++++++++++++ .../spack/operating_system/linux_distro.py | 10 +++++++--- lib/spack/spack/operating_system/mac_osx.py | 9 ++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 lib/spack/spack/operating_system/cnl.py diff --git a/lib/spack/spack/operating_system/cnl.py b/lib/spack/spack/operating_system/cnl.py new file mode 100644 index 00000000000..e52052dfa8e --- /dev/null +++ b/lib/spack/spack/operating_system/cnl.py @@ -0,0 +1,20 @@ +import spack +from spack.architecture import OperatingSystem + +class ComputeNodeLinux(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'CNL' + version = '10' + super(ComputeNodeLinux, self).__init__(name, version, "MODULES") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py index 30a85fe61cb..b11c7a88fa5 100644 --- a/lib/spack/spack/operating_system/linux_distro.py +++ b/lib/spack/spack/operating_system/linux_distro.py @@ -17,9 +17,13 @@ def detect_operating_system(): name, version = detect_operating_system() - super(LinuxDistro, self).__init__(name, version) + super(LinuxDistro, self).__init__(name, version, "PATH") - @property def compiler_strategy(self): - return "PATH" + return self.compiler_strategy + + def find_compilers(self): + pass + + diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py index 0b939a5546f..f52cdd6bc74 100644 --- a/lib/spack/spack/operating_system/mac_osx.py +++ b/lib/spack/spack/operating_system/mac_osx.py @@ -34,11 +34,14 @@ def get_mac_release(): name, version = get_mac_release() - super(MacOSX, self).__init__(name, version) + super(MacOSX, self).__init__(name, version, "PATH") - @property def compiler_strategy(self): - return "PATH" + return self.compiler_strategy + + def find_compilers(self): + pass + From 083b7b46d92605269df05a9267657acf5e4b4677 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:17:54 -0800 Subject: [PATCH 140/375] Added more tests --- lib/spack/spack/test/operating_system.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index 9d6850bfa69..a4327dde837 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -11,6 +11,7 @@ from spack.platforms.darwin import Darwin from spack.operating_system.linux_distro import LinuxDistro from spack.operating_system.mac_osx import MacOSX +from spack.operating_system.cnl import ComputeNodeLinux class TestOperatingSystem(unittest.TestCase): @@ -19,6 +20,8 @@ def setUp(self): linux = Linux() darwin = Darwin() self.cray_operating_sys = cray_xc.operating_system('front_os') + self.cray_default_os = cray_xc.operating_system('default_os') + self.cray_back_os = cray_xc.operating_system('back_os') self.darwin_operating_sys = darwin.operating_system('default_os') self.linux_operating_sys = linux.operating_system('default_os') @@ -28,6 +31,12 @@ def test_cray_front_end_operating_system(self): def test_cray_front_end_compiler_strategy(self): self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") + def test_cray_back_end_operating_system(self): + self.assertIsInstance(self.cray_back_os,ComputeNodeLinux) + + def test_cray_back_end_compiler_strategy(self): + self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") + def test_linux_operating_system(self): print self.linux_operating_sys self.assertIsInstance(self.linux_operating_sys, LinuxDistro) @@ -35,7 +44,16 @@ def test_linux_operating_system(self): def test_linux_compiler_strategy(self): self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") - + + def test_cray_front_end_compiler_list(self): + """ Operating systems will now be in charge of finding compilers. + So, depending on which operating system you want to build for + or which operating system you are on, then you could detect + compilers in a certain way. Cray linux environment on the front + end is just a regular linux distro whereas the Cray linux compute + node is a stripped down version which modules are important + """ + self.assertEquals(True, False) From cfa7c4feb8419f82a864f9d1afcab69965e9c996 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 22 Feb 2016 16:18:25 -0800 Subject: [PATCH 141/375] Added CNL10 as back_os and default_os --- lib/spack/spack/platforms/cray_xc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index b58c84d3eb1..dffa2e40312 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -2,6 +2,7 @@ from spack.architecture import Platform, Target from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_system.cnl import ComputeNodeLinux class CrayXc(Platform): priority = 20 @@ -10,8 +11,8 @@ class CrayXc(Platform): default = 'ivybridge' front_os = "SuSE11" - back_os = "CNL" - default_os = "CNL" + back_os = "CNL10" + default_os = "CNL10" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better @@ -38,6 +39,7 @@ def __init__(self): Target('haswell', 'MODULES', 'craype-haswell')) self.add_operating_system('SuSE11', LinuxDistro()) + self.add_operating_system('CNL10', ComputeNodeLinux()) @classmethod def detect(self): From 5d5d3c5858a8882b615fb97fb07e335dc6c17f14 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 12:15:52 -0800 Subject: [PATCH 142/375] Added to dictionary method --- lib/spack/spack/architecture.py | 95 +++++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 574076bae4b..109e599470e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -89,7 +89,7 @@ def from_dict(d): return target def _cmp_key(self): - return (self.name, self.compiler_strategy, self.module_name) + return (self.name, self.module_name) def __repr__(self): return self.__str__() @@ -200,7 +200,6 @@ def __str__(self): def _cmp_key(self): return (self.name, (_cmp_key(t) for t in self.targets.values()), (_cmp_key(o) for o in self.operating_sys.values())) - @key_ordering class OperatingSystem(object): @@ -221,17 +220,19 @@ def __str__(self): def __repr__(self): return self.__str__() - def _cmp_key(self): - return (self.name, self.version) def compiler_strategy(self): """ Operating Systems will now be in charge of the compiler finding strategy. They will each have their own find_compilers method which depending on their compiler strategy will find the compilers - using a specific method (i.e PATH vs MODULES) + using a specific method (i.e PATH vs MODULES). """ raise NotImplementedError() + def _cmp_key(self): + return (self.name, self.version, self.compiler_strategy) + + #NOTE: Key error caused because Architecture has no comparison method @key_ordering class Arch(namedtuple("Arch", "platform platform_os target")): @@ -241,24 +242,77 @@ class Arch(namedtuple("Arch", "platform platform_os target")): __slots__ = () def __str__(self): - return (self.platform.name +"-"+ - self.platform_os.name + "-" + self.target.name) - + return (self.platform.name +"-"+ self.platform_os.name + "-" + self.target.name) + def _cmp_key(self): return (self.platform.name, self.platform_os.name, self.target.name) - @staticmethod - def to_dict(platform, os_name, target): - """ This function will handle all the converting of dictionaries - for each of the architecture classes. This is better than having - each class have a to_dict method when creating the dict of dicts - arch class - """ - d = {} - d['platform'] = platform - d['operating_system'] = os_name - d['target'] = target.to_dict() - return d + +def _helper_to_dict(arch_field_dict, arch_field_name, *args): + """ General method to turn each class in architecture into a + dictionary. Takes as argument the class dictionary, + """ + d = {} + d[arch_field_name] = {} + for items in args: + d[arch_field_name][items] = arch_field_dict[items] + return d + +def to_dict(arch): + d = {} + d['architecture'] = {} + + platform = arch.platform.__dict__ + platform_os = arch.platform_os.__dict__ + target = arch.target.__dict__ + + platform_dict = _helper_to_dict(platform,'platform','name') + os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', + 'compiler_strategy') + target_dict = _helper_to_dict(target,'target', 'name', + 'module_name','platform_name') + + d['architecture'].update(platform_dict) + d['architecture'].update(os_dict) + d['architecture'].update(target_dict) + + return d + +def _platform_from_dict(platform): + platform_list = all_platforms() + platform_names = {plat.__name__.lower():plat for plat in platform_list} + return platform_names[platform['name']]() + + +def _target_from_dict(target): + target = Target.__new__(Target) + target.name = d['name'] + target.compiler_strategy = d['compiler_strategy'] + target.module_name = d['module_name'] + if 'platform' in d: + target.platform_name = d['platform'] + return target + +def _operating_system_from_dict(os_dict): + pass + +def arch_from_dict(d): + if d is None: + return None + arch = Arch + platform_dict = d['platform'] + platform_os_dict = d['platform_os'] + target_dict = d['target'] + + platform = _platform_from_dict(platform_dict) + platform_os = _operating_system_from_dict(platform_os_dict) + target = _target_from_dict(target_dict) + + arch.platform = platform + arch.platform_os = platform_os + arch.target = target + + return arch @memoized def all_platforms(): @@ -278,7 +332,6 @@ def all_platforms(): return modules - @memoized def sys_type(): """ Gather a list of all available subclasses of platforms. From 339f4bfd23b097d009009b6dabd524f8bc831449 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 12:16:09 -0800 Subject: [PATCH 143/375] Added unit testing for to_dict method --- lib/spack/spack/test/architecture.py | 43 ++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index cf7938f5d69..5bc78d1b658 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,19 +6,58 @@ import platform import spack from spack.architecture import * +import spack.spec from spack.platforms.cray_xc import CrayXc from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): + + def setUp(self): + zlib = spack.spec.Spec("zlib") + zlib.concretize() + self.architecture = zlib.architecture + self.platform = sys_type() + self.platform_os = self.platform.operating_system('default_os') + self.target = self.platform.target('default') + + #def test_to_dict_function_with_target(self): + # d = spack.architecture.to_dict(self.architecture) + # print d['target'] + # self.assertEquals(d['target'], {'name': self.target.name, + # 'module_name' : self.target.module_name, + # 'platform_name' : self.target.platform_name, + # 'compiler_strategy': 'MODULES' + # }) + + def test_to_dict_function_with_architecture(self): + d = spack.architecture.to_dict(self.architecture) + self.assertEquals(d, {'architecture': + {'platform' : {'name': 'crayxc'}, + 'platform_os': { + 'compiler_strategy': 'MODULES', + 'name':'CNL', + 'version':'10'}, + 'target' : {'platform_name' :'crayxc', + 'module_name': 'craype-haswell', + 'name':'haswell'}}}) + + #def test_to_dict_function_with_operating_system(self): + # d = spack.architecture.to_dict(self.architecture) + # self.assertEquals(d['platform_os'], {'name': self.platform_os.name, + # 'version': self.platform_os.version, + # 'compiler_strategy': self.platform_os.compiler_strategy}) + + def test_architecture_from_dict(self): + pass def test_platform_class_and_compiler_strategies(self): a = CrayXc() - t = a.target('default') + t = a.operating_system('default_os') self.assertEquals(t.compiler_strategy, 'MODULES') b = Linux() - s = b.target('default') + s = b.operating_system('default_os') self.assertEquals(s.compiler_strategy, 'PATH') def test_sys_type(self): From 77799741430796974809fce0f08d099303ec6da6 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:02:40 -0800 Subject: [PATCH 144/375] arch_from_dict worked on --- lib/spack/spack/architecture.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 109e599470e..f0fc00242c5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -294,7 +294,11 @@ def _target_from_dict(target): return target def _operating_system_from_dict(os_dict): - pass + name = os_dict['name'] + os_list = all_platforms(True) + os_classes = {o.__name__.lower():o for o in os_list} + return os_classes[name]() + def arch_from_dict(d): if d is None: @@ -315,11 +319,19 @@ def arch_from_dict(d): return arch @memoized -def all_platforms(): +def all_platforms(operating_system=False): modules = [] - for name in list_modules(spack.platform_path): - mod_name = 'spack.platformss' + name - path = join_path(spack.platform_path, name) + ".py" + + if operating_system: + mod_path = spack.operating_system_path + mod_string = "spack.operating_system." + else: + mod_path = spack.platform_path + mod_string = "spack.platformss" + + for name in list_modules(mod_path): + mod_name = mod_string + name + path = join_path(mod_path, name) + ".py" mod = imp.load_source(mod_name, path) class_name = mod_to_class(name) if not hasattr(mod, class_name): From 9e844d974c2bce42b3e6e647c16b10aa122e54ce Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:30:20 -0800 Subject: [PATCH 145/375] Added comments. Need to figure out how to make arch_tuple from_dict --- lib/spack/spack/architecture.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index f0fc00242c5..4b24680501c 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -250,7 +250,8 @@ def _cmp_key(self): def _helper_to_dict(arch_field_dict, arch_field_name, *args): """ General method to turn each class in architecture into a - dictionary. Takes as argument the class dictionary, + dictionary. Takes as argument the class dictionary, the field name + (platform, platform_os, target) and then any attribute args """ d = {} d[arch_field_name] = {} @@ -294,10 +295,11 @@ def _target_from_dict(target): return target def _operating_system_from_dict(os_dict): + #TODO: Have a way to recognize the OS subclasses name = os_dict['name'] os_list = all_platforms(True) - os_classes = {o.__name__.lower():o for o in os_list} - return os_classes[name]() + os_classes = {o.__name__:o for o in os_list} + def arch_from_dict(d): @@ -318,6 +320,7 @@ def arch_from_dict(d): return arch +#TODO: Haven't changed name here but all_platforms should now pull os class list @memoized def all_platforms(operating_system=False): modules = [] From 2de81cfc62ca456b1f303d64198590eb003fdcd3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:31:21 -0800 Subject: [PATCH 146/375] Changed name to appropriate camelcase --- lib/spack/spack/operating_system/cnl.py | 4 +- .../spack/operating_system/linux_distro.py | 10 ++--- lib/spack/spack/operating_system/mac_osx.py | 38 ++++++++----------- 3 files changed, 20 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/operating_system/cnl.py b/lib/spack/spack/operating_system/cnl.py index e52052dfa8e..07221630f2e 100644 --- a/lib/spack/spack/operating_system/cnl.py +++ b/lib/spack/spack/operating_system/cnl.py @@ -1,7 +1,7 @@ import spack from spack.architecture import OperatingSystem -class ComputeNodeLinux(OperatingSystem): +class Cnl(OperatingSystem): """ Compute Node Linux (CNL) is the operating system used for the Cray XC series super computers. It is a very stripped down version of GNU/Linux. Any compilers found through this operating system will be used with @@ -11,7 +11,7 @@ class ComputeNodeLinux(OperatingSystem): def __init__(self): name = 'CNL' version = '10' - super(ComputeNodeLinux, self).__init__(name, version, "MODULES") + super(Cnl, self).__init__(name, version, "MODULES") def compiler_strategy(self): return self.compiler_strategy diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py index b11c7a88fa5..1345b7d6187 100644 --- a/lib/spack/spack/operating_system/linux_distro.py +++ b/lib/spack/spack/operating_system/linux_distro.py @@ -1,6 +1,6 @@ import platform as py_platform import spack -from spack.architecture import Platform, OperatingSystem +from spack.architecture import OperatingSystem class LinuxDistro(OperatingSystem): """ This class will represent the autodetected operating system @@ -10,13 +10,9 @@ class LinuxDistro(OperatingSystem): platform.dist() """ def __init__(self): - def detect_operating_system(): - name = py_platform.dist()[0] - version = py_platform.dist()[1] - return name, version + name = py_platform.dist()[0] + version = py_platform.dist()[1] - name, version = detect_operating_system() - super(LinuxDistro, self).__init__(name, version, "PATH") def compiler_strategy(self): diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py index f52cdd6bc74..6fb6fef1381 100644 --- a/lib/spack/spack/operating_system/mac_osx.py +++ b/lib/spack/spack/operating_system/mac_osx.py @@ -3,38 +3,30 @@ will be represented using the major version operating system name, i.e el capitan, yosemite...etc. """ - -import spack -import os import platform as py_platform -from spack.architecture import Platform, OperatingSystem +import spack +from spack.architecture import OperatingSystem -class MacOSX(OperatingSystem): +class MacOsx(OperatingSystem): def __init__(self): """ Autodetects the mac version from a dictionary. Goes back as far as 10.6 snowleopard. If the user has an older mac then the version will just be a generic mac_os. """ - - def get_mac_release(): - mac_releases = {'10.6': "snowleopard", - "10.7": "lion", - "10.8": "mountainlion", - "10.9": "mavericks", - "10.10": "yosemite", - "10.11": "elcapitan"} + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} - mac_ver = py_platform.mac_ver()[0][:-2] - try: - name = mac_releases[mac_ver] - return name, mac_ver - except KeyError: - name = "mac_os" - return name, mac_ver + mac_ver = py_platform.mac_ver()[0][:-2] + try: + name = mac_releases[mac_ver] + except KeyError: + name = "mac_os" - name, version = get_mac_release() - - super(MacOSX, self).__init__(name, version, "PATH") + super(MacOsx, self).__init__(name, mac_ver, "PATH") def compiler_strategy(self): return self.compiler_strategy From 234681306cdfb9aea9772781fcb68c4045bdc315 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:32:23 -0800 Subject: [PATCH 147/375] Changed to appropriate class names for operating system --- lib/spack/spack/platforms/cray_xc.py | 5 ++--- lib/spack/spack/platforms/darwin.py | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index dffa2e40312..da90726fefc 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,8 +1,7 @@ import os - from spack.architecture import Platform, Target from spack.operating_system.linux_distro import LinuxDistro -from spack.operating_system.cnl import ComputeNodeLinux +from spack.operating_system.cnl import Cnl class CrayXc(Platform): priority = 20 @@ -39,7 +38,7 @@ def __init__(self): Target('haswell', 'MODULES', 'craype-haswell')) self.add_operating_system('SuSE11', LinuxDistro()) - self.add_operating_system('CNL10', ComputeNodeLinux()) + self.add_operating_system('CNL10', Cnl()) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 5b797a78dbe..45128573714 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_system.mac_osx import MacOSX +from spack.operating_system.mac_osx import MacOsx class Darwin(Platform): priority = 89 @@ -11,7 +11,7 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default, 'PATH')) - mac_os = MacOSX() + mac_os = MacOsx() self.default_os = mac_os.name self.add_operating_system(mac_os.name, mac_os) From 5c3c6e7f066a3f95f9113c46640546516d7a8257 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:34:32 -0800 Subject: [PATCH 148/375] Got rid of print statements --- lib/spack/spack/test/operating_system.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index a4327dde837..205408db3ff 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -38,7 +38,6 @@ def test_cray_back_end_compiler_strategy(self): self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") def test_linux_operating_system(self): - print self.linux_operating_sys self.assertIsInstance(self.linux_operating_sys, LinuxDistro) def test_linux_compiler_strategy(self): From bd7c189c123154e88e092eadbbd93b00ce43cc6a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:37:50 -0800 Subject: [PATCH 149/375] Added more comments --- lib/spack/spack/spec.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 0b424989109..c8c114a8b1f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -661,12 +661,14 @@ def to_node_dict(self): 'dependencies' : dict((d, self.dependencies[d].dag_hash()) for d in sorted(self.dependencies)) } + if self.architecture: # TODO: Fix the target.to_dict to account for the tuple # Want it to be a dict of dicts - d['architecture'] = self.architecture.target.to_dict() + d['architecture'] = spack.architecture.to_dict(self.architecture) else: d['architecture'] = None + if self.compiler: d.update(self.compiler.to_dict()) else: @@ -693,8 +695,7 @@ def from_node_dict(node): spec = Spec(name) spec.versions = VersionList.from_dict(node) # TODO: Need to fix the architecture.Target.from_dict - spec.architecture = spack.architecture.Target.from_dict( - node['architecture']) + spec.architecture = spack.architecture.from_dict(node['architecture']) if node['compiler'] is None: spec.compiler = None @@ -1292,17 +1293,17 @@ def add_architecture_from_string(self, arch): for entry in arch_list: if self._is_valid_platform(entry, platform_names): + # If entry is different from platform name then create it. + # else just keep the already instantiated platform class if entry != platform.name: platform = platform_dict[entry]() # Create instance of platform elif self._is_valid_target(entry, platform): target = platform.target(entry) - # Need to figure out if we're supporting arbitrary os's and how - # to account for them - # Not really a good implementation since the user can add - # gibberish and spack will see it as an os + # check if os is valid by checking platform operating sys dict elif self._is_valid_os(entry, platform): platform_os = platform.operating_system(entry) else: + # throw error since entry is unknown raise UnknownArchitectureSpecError(entry) if target is None: From 8d1c06d14125d86c24584c7e6f0a9ab0d300d10f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 24 Feb 2016 15:38:33 -0800 Subject: [PATCH 150/375] Changed to dict method. Didn't realize this was being assigned to a dictionary already so got rid of d['architecture'] --- lib/spack/spack/architecture.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 4b24680501c..e76e411ae07 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -261,8 +261,7 @@ def _helper_to_dict(arch_field_dict, arch_field_name, *args): def to_dict(arch): d = {} - d['architecture'] = {} - + platform = arch.platform.__dict__ platform_os = arch.platform_os.__dict__ target = arch.target.__dict__ @@ -273,9 +272,9 @@ def to_dict(arch): target_dict = _helper_to_dict(target,'target', 'name', 'module_name','platform_name') - d['architecture'].update(platform_dict) - d['architecture'].update(os_dict) - d['architecture'].update(target_dict) + d.update(platform_dict) + d.update(os_dict) + d.update(target_dict) return d From 863a5bc4fa7d6b880769192e4fcb7a461a6a0063 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:34:55 -0800 Subject: [PATCH 151/375] Fix for 'Parent module spack.operating_systems not found while handling absolute import' --- lib/spack/spack/architecture.py | 40 ++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e76e411ae07..58274270033 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -67,26 +67,26 @@ def __init__(self, name, compiler_strategy, module_name=None): def set_platform(self, platform): self.platform_name = platform.name - def to_dict(self): - d = {} - d['name'] = self.name - d['compiler_strategy'] = self.compiler_strategy - d['module_name'] = self.module_name - if self.platform_name: - d['platform'] = self.platform_name - return d + #def to_dict(self): + # d = {} + # d['name'] = self.name + # d['compiler_strategy'] = self.compiler_strategy + # d['module_name'] = self.module_name + # if self.platform_name: + # d['platform'] = self.platform_name + # return d - @staticmethod - def from_dict(d): - if d is None: - return None - target = Target.__new__(Target) - target.name = d['name'] - target.compiler_strategy = d['compiler_strategy'] - target.module_name = d['module_name'] - if 'platform' in d: - target.platform_name = d['platform'] - return target + #@staticmethod + #def from_dict(d): + # if d is None: + # return None + # target = Target.__new__(Target) + # target.name = d['name'] + # target.compiler_strategy = d['compiler_strategy'] + # target.module_name = d['module_name'] + # if 'platform' in d: + # target.platform_name = d['platform'] + # return target def _cmp_key(self): return (self.name, self.module_name) @@ -326,7 +326,7 @@ def all_platforms(operating_system=False): if operating_system: mod_path = spack.operating_system_path - mod_string = "spack.operating_system." + mod_string = "spack.operating_systems" else: mod_path = spack.platform_path mod_string = "spack.platformss" From 34be473b7cfbb23d4deb5c3f9774bc1acea22574 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:35:58 -0800 Subject: [PATCH 152/375] Deleted files --- lib/spack/spack/operating_systems/cnl.py | 19 +++++++++ .../spack/operating_systems/linux_distro.py | 24 +++++++++++ lib/spack/spack/operating_systems/mac_osx.py | 40 +++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 lib/spack/spack/operating_systems/cnl.py create mode 100644 lib/spack/spack/operating_systems/linux_distro.py create mode 100644 lib/spack/spack/operating_systems/mac_osx.py diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py new file mode 100644 index 00000000000..20cb4c215b4 --- /dev/null +++ b/lib/spack/spack/operating_systems/cnl.py @@ -0,0 +1,19 @@ +from spack.architecture import OperatingSystem + +class Cnl(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'CNL' + version = '10' + super(Cnl, self).__init__(name, version, "MODULES") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py new file mode 100644 index 00000000000..c5547749d8d --- /dev/null +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -0,0 +1,24 @@ +import platform as py_platform +from spack.architecture import OperatingSystem + +class LinuxDistro(OperatingSystem): + """ This class will represent the autodetected operating system + for a Linux System. Since there are many different flavors of + Linux, this class will attempt to encompass them all through + autodetection using the python module platform and the method + platform.dist() + """ + def __init__(self): + name = py_platform.dist()[0] + version = py_platform.dist()[1] + + super(LinuxDistro, self).__init__(name, version, "PATH") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass + + + diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_osx.py new file mode 100644 index 00000000000..555c5cfb14e --- /dev/null +++ b/lib/spack/spack/operating_systems/mac_osx.py @@ -0,0 +1,40 @@ +import platform as py_platform +from spack.architecture import OperatingSystem + +class MacOsx(OperatingSystem): + """ This class represents the MAC_OSX operating system. This will be auto + detected using the python platform.mac_ver. The MAC_OSX platform + will be represented using the major version operating system name, i.e + el capitan, yosemite...etc. + """ + + def __init__(self): + """ Autodetects the mac version from a dictionary. Goes back as + far as 10.6 snowleopard. If the user has an older mac then + the version will just be a generic mac_os. + """ + mac_releases = {'10.6': "snowleopard", + "10.7": "lion", + "10.8": "mountainlion", + "10.9": "mavericks", + "10.10": "yosemite", + "10.11": "elcapitan"} + + mac_ver = py_platform.mac_ver()[0][:-2] + try: + name = mac_releases[mac_ver] + except KeyError: + name = "mac_os" + + super(MacOsx, self).__init__(name, mac_ver, "PATH") + + def compiler_strategy(self): + return self.compiler_strategy + + def find_compilers(self): + pass + + + + + From f1616099292251f208ec0cfdec3d1b092d4441b9 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:36:12 -0800 Subject: [PATCH 153/375] Renamed operating_system to operating_systems --- lib/spack/spack/operating_systems/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/spack/spack/operating_systems/__init__.py diff --git a/lib/spack/spack/operating_systems/__init__.py b/lib/spack/spack/operating_systems/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From b43a5498a159c634ca88341d3010bc1c98197e24 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:36:47 -0800 Subject: [PATCH 154/375] Changed import path to operating_systems --- lib/spack/spack/platforms/bgq.py | 1 - lib/spack/spack/platforms/cray_xc.py | 4 ++-- lib/spack/spack/platforms/darwin.py | 2 +- lib/spack/spack/platforms/linux.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py index 6e872d2e729..b550f1d5cb8 100644 --- a/lib/spack/spack/platforms/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -1,5 +1,4 @@ import os - from spack.architecture import Platform, Target class Bgq(Platform): diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index da90726fefc..ea3ae0e477d 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -1,7 +1,7 @@ import os from spack.architecture import Platform, Target -from spack.operating_system.linux_distro import LinuxDistro -from spack.operating_system.cnl import Cnl +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl class CrayXc(Platform): priority = 20 diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 45128573714..c48f7d17d2b 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_system.mac_osx import MacOsx +from spack.operating_systems.mac_osx import MacOsx class Darwin(Platform): priority = 89 diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 220f1bbaf40..1b22b69048f 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_system.linux_distro import LinuxDistro +from spack.operating_systems.linux_distro import LinuxDistro class Linux(Platform): priority = 90 From 9ac2556285ae38dd6d661d9bba811a5e007cb551 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 10:41:28 -0800 Subject: [PATCH 155/375] Deleted old operating_system (without the s) file --- lib/spack/spack/operating_system/__init__.py | 0 lib/spack/spack/operating_system/cnl.py | 20 ---------- .../spack/operating_system/linux_distro.py | 25 ------------ lib/spack/spack/operating_system/mac_osx.py | 40 ------------------- 4 files changed, 85 deletions(-) delete mode 100644 lib/spack/spack/operating_system/__init__.py delete mode 100644 lib/spack/spack/operating_system/cnl.py delete mode 100644 lib/spack/spack/operating_system/linux_distro.py delete mode 100644 lib/spack/spack/operating_system/mac_osx.py diff --git a/lib/spack/spack/operating_system/__init__.py b/lib/spack/spack/operating_system/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/lib/spack/spack/operating_system/cnl.py b/lib/spack/spack/operating_system/cnl.py deleted file mode 100644 index 07221630f2e..00000000000 --- a/lib/spack/spack/operating_system/cnl.py +++ /dev/null @@ -1,20 +0,0 @@ -import spack -from spack.architecture import OperatingSystem - -class Cnl(OperatingSystem): - """ Compute Node Linux (CNL) is the operating system used for the Cray XC - series super computers. It is a very stripped down version of GNU/Linux. - Any compilers found through this operating system will be used with - modules. If updated, user must make sure that version and name are - updated to indicate that OS has been upgraded (or downgraded) - """ - def __init__(self): - name = 'CNL' - version = '10' - super(Cnl, self).__init__(name, version, "MODULES") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass diff --git a/lib/spack/spack/operating_system/linux_distro.py b/lib/spack/spack/operating_system/linux_distro.py deleted file mode 100644 index 1345b7d6187..00000000000 --- a/lib/spack/spack/operating_system/linux_distro.py +++ /dev/null @@ -1,25 +0,0 @@ -import platform as py_platform -import spack -from spack.architecture import OperatingSystem - -class LinuxDistro(OperatingSystem): - """ This class will represent the autodetected operating system - for a Linux System. Since there are many different flavors of - Linux, this class will attempt to encompass them all through - autodetection using the python module platform and the method - platform.dist() - """ - def __init__(self): - name = py_platform.dist()[0] - version = py_platform.dist()[1] - - super(LinuxDistro, self).__init__(name, version, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - diff --git a/lib/spack/spack/operating_system/mac_osx.py b/lib/spack/spack/operating_system/mac_osx.py deleted file mode 100644 index 6fb6fef1381..00000000000 --- a/lib/spack/spack/operating_system/mac_osx.py +++ /dev/null @@ -1,40 +0,0 @@ -""" This class represents the MAC_OSX operating system. This will be auto - detected using the python platform.mac_ver. The MAC_OSX platform - will be represented using the major version operating system name, i.e - el capitan, yosemite...etc. -""" -import platform as py_platform -import spack -from spack.architecture import OperatingSystem - -class MacOsx(OperatingSystem): - def __init__(self): - """ Autodetects the mac version from a dictionary. Goes back as - far as 10.6 snowleopard. If the user has an older mac then - the version will just be a generic mac_os. - """ - mac_releases = {'10.6': "snowleopard", - "10.7": "lion", - "10.8": "mountainlion", - "10.9": "mavericks", - "10.10": "yosemite", - "10.11": "elcapitan"} - - mac_ver = py_platform.mac_ver()[0][:-2] - try: - name = mac_releases[mac_ver] - except KeyError: - name = "mac_os" - - super(MacOsx, self).__init__(name, mac_ver, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - - - From 550df4cdd6ce1760370e18a858da624167a3139a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 11:47:27 -0800 Subject: [PATCH 156/375] Some cleaning up. Finally got arch_from_dict working successfully. --- lib/spack/spack/architecture.py | 81 ++++++--------------------------- 1 file changed, 15 insertions(+), 66 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 58274270033..7bf5afcf63f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -67,27 +67,6 @@ def __init__(self, name, compiler_strategy, module_name=None): def set_platform(self, platform): self.platform_name = platform.name - #def to_dict(self): - # d = {} - # d['name'] = self.name - # d['compiler_strategy'] = self.compiler_strategy - # d['module_name'] = self.module_name - # if self.platform_name: - # d['platform'] = self.platform_name - # return d - - #@staticmethod - #def from_dict(d): - # if d is None: - # return None - # target = Target.__new__(Target) - # target.name = d['name'] - # target.compiler_strategy = d['compiler_strategy'] - # target.module_name = d['module_name'] - # if 'platform' in d: - # target.platform_name = d['platform'] - # return target - def _cmp_key(self): return (self.name, self.module_name) @@ -147,24 +126,6 @@ def target(self, name): name = self.back_end return self.targets[name] - - #def _detect_linux_os(self): - # return OperatingSystem(py_platform.dist()[0], py_platform.dist()[1]) - - #def _detect_mac_os(self): - # mac_releases = {'10.6': "snowleopard", - # "10.7": "lion", - # "10.8": "mountainlion", - # "10.9": "mavericks", - # "10.10": "yosemite", - # "10.11": "elcapitan"} - # mac_ver = py_platform.mac_ver()[:-2] - # try: - # os_name = mac_releases[mac_ver] - # return OperatingSystem(os_name, mac_ver) - # except KeyError: - # os_name = "mac_os" - # return OperatingSystem(os_name, mac_ver) def add_operating_system(self, name, os_class): """ Add the operating_system class object into the @@ -242,7 +203,8 @@ class Arch(namedtuple("Arch", "platform platform_os target")): __slots__ = () def __str__(self): - return (self.platform.name +"-"+ self.platform_os.name + "-" + self.target.name) + return (self.platform.name +"-"+ + self.platform_os.name + "-" + self.target.name) def _cmp_key(self): return (self.platform.name, self.platform_os.name, self.target.name) @@ -284,52 +246,39 @@ def _platform_from_dict(platform): return platform_names[platform['name']]() -def _target_from_dict(target): +def _target_from_dict(target_dict): target = Target.__new__(Target) - target.name = d['name'] - target.compiler_strategy = d['compiler_strategy'] - target.module_name = d['module_name'] - if 'platform' in d: - target.platform_name = d['platform'] + target.name = target_dict['name'] + #target.compiler_strategy = target_dict['compiler_strategy'] + target.module_name = target_dict['module_name'] + if 'platform_name' in target_dict: + target.platform_name = target_dict['platform_name'] return target -def _operating_system_from_dict(os_dict): - #TODO: Have a way to recognize the OS subclasses +def _operating_system_from_dict(os_dict, platform_class): name = os_dict['name'] - os_list = all_platforms(True) - os_classes = {o.__name__:o for o in os_list} - + return platform_class.operating_system(name) def arch_from_dict(d): if d is None: return None - arch = Arch platform_dict = d['platform'] platform_os_dict = d['platform_os'] target_dict = d['target'] platform = _platform_from_dict(platform_dict) - platform_os = _operating_system_from_dict(platform_os_dict) + platform_os = _operating_system_from_dict(platform_os_dict, platform) target = _target_from_dict(target_dict) - arch.platform = platform - arch.platform_os = platform_os - arch.target = target + return Arch(platform, platform_os, target) - return arch - -#TODO: Haven't changed name here but all_platforms should now pull os class list @memoized -def all_platforms(operating_system=False): +def all_platforms(): modules = [] - if operating_system: - mod_path = spack.operating_system_path - mod_string = "spack.operating_systems" - else: - mod_path = spack.platform_path - mod_string = "spack.platformss" + mod_path = spack.platform_path + mod_string = "spack.platformss" for name in list_modules(mod_path): mod_name = mod_string + name From 5e4d3e7b8280e633bb9b9dd28510dc394feb2000 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 11:48:43 -0800 Subject: [PATCH 157/375] Changed naming convention of operating_system, makes it easier to create instance from dictionary just using name without the version attached. Object will stil display as name+version --- lib/spack/spack/platforms/cray_xc.py | 10 +++++----- lib/spack/spack/platforms/linux.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index ea3ae0e477d..b71481e46e8 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -9,9 +9,9 @@ class CrayXc(Platform): back_end = 'ivybridge' default = 'ivybridge' - front_os = "SuSE11" - back_os = "CNL10" - default_os = "CNL10" + front_os = "SuSE" + back_os = "CNL" + default_os = "CNL" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better @@ -37,8 +37,8 @@ def __init__(self): self.add_target('haswell', Target('haswell', 'MODULES', 'craype-haswell')) - self.add_operating_system('SuSE11', LinuxDistro()) - self.add_operating_system('CNL10', Cnl()) + self.add_operating_system('SuSE', LinuxDistro()) + self.add_operating_system('CNL', Cnl()) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 1b22b69048f..d9e33511a13 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -12,8 +12,8 @@ def __init__(self): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default, 'PATH')) linux_dist = LinuxDistro() - self.default_os = str(linux_dist) - self.add_operating_system(str(linux_dist), linux_dist) + self.default_os = linux_dist.name + self.add_operating_system(linux_dist.name, linux_dist) @classmethod def detect(self): From 5989e3f65d83afdd497059bba9442576e2df0509 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 11:53:02 -0800 Subject: [PATCH 158/375] Changed from_dict to arch_from_dict method --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index c8c114a8b1f..abe17a8f2bc 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -695,7 +695,7 @@ def from_node_dict(node): spec = Spec(name) spec.versions = VersionList.from_dict(node) # TODO: Need to fix the architecture.Target.from_dict - spec.architecture = spack.architecture.from_dict(node['architecture']) + spec.architecture = spack.architecture.arch_from_dict(node['architecture']) if node['compiler'] is None: spec.compiler = None From ac55ce989d0b5a6e7a1160dc32ac7fb12f805f76 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:47:41 -0800 Subject: [PATCH 159/375] Changed compiler finding strategy to come from operating system instead of target --- lib/spack/spack/concretize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 0e7bf53b445..3434ca66f79 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -334,9 +334,9 @@ def concretize_compiler(self, spec): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - if architecture.target.compiler_strategy == 'PATH': + if architecture.platform_os.compiler_strategy == 'PATH': filter(lambda c: not c.modules, compilers) - if architecture.target.compiler_strategy == 'MODULES': + if architecture.platform_os.compiler_strategy == 'MODULES': filter(lambda c: c.modules, compilers) return compilers From f0149faf88b97b63635ba451efe008846478d115 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:48:00 -0800 Subject: [PATCH 160/375] Added doc strings to functions in architecture.py --- lib/spack/spack/architecture.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 7bf5afcf63f..14a8c567938 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -222,6 +222,12 @@ def _helper_to_dict(arch_field_dict, arch_field_name, *args): return d def to_dict(arch): + """ Convert the Arch tuple into a dictionary for yaml dumping. This + uses the _helper_to_dict method to create the dictionary from the + provided architecture field. Can assign the architecture + field name (either platform, platform_os or target) and any + attributes that make up that architecture field, + """ d = {} platform = arch.platform.__dict__ @@ -241,12 +247,19 @@ def to_dict(arch): return d def _platform_from_dict(platform): + """Creates all the platform class module names into a dictionary of + name : key-value pairs. From there we can construct the + platform subclass + """ platform_list = all_platforms() platform_names = {plat.__name__.lower():plat for plat in platform_list} return platform_names[platform['name']]() def _target_from_dict(target_dict): + """ Creates new instance of target and assigns all the attributes of + that target from the dictionary + """ target = Target.__new__(Target) target.name = target_dict['name'] #target.compiler_strategy = target_dict['compiler_strategy'] @@ -256,11 +269,19 @@ def _target_from_dict(target_dict): return target def _operating_system_from_dict(os_dict, platform_class): + """ uses platform's operating system method to grab the constructed + operating systems that are valid on the platform. + """ +# NOTE: Might need a better way to create operating system objects name = os_dict['name'] return platform_class.operating_system(name) def arch_from_dict(d): + """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict + helper methods to recreate the arch tuple from the dictionary read from + a yaml file + """ if d is None: return None platform_dict = d['platform'] From eb96f3829865492c3c484de0d89e14cba578a37f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:48:25 -0800 Subject: [PATCH 161/375] Got rid of debug comment --- lib/spack/spack/spec.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index abe17a8f2bc..0a55ec5d760 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -835,7 +835,6 @@ def concretize(self): changed = True force = False - # Loops forever here in my implementation while changed: changes = (self.normalize(force=force), self._expand_virtual_packages(), From ccd4a79b39b56a8408980ed27b0c5de5a1995fbc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:49:27 -0800 Subject: [PATCH 162/375] changed architecture.target to architecture.platform_os, so that compiler_for_spec uses operating system strategy --- lib/spack/spack/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index b0d84863de0..5d922f6e582 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -594,7 +594,7 @@ def compiler(self): if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") return spack.compilers.compiler_for_spec(self.spec.compiler, - self.spec.architecture.target) + self.spec.architecture.platform_os) def url_version(self, version): From b591d2b5015fdde5d551dd81a8906ac0b2a51260 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 25 Feb 2016 18:49:52 -0800 Subject: [PATCH 163/375] Changed operating_system path --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 6ad9b87ca22..48fa9edb67c 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -39,7 +39,7 @@ module_path = join_path(lib_path, "spack") platform_path = join_path(module_path, 'platforms') compilers_path = join_path(module_path, "compilers") -operating_system_path = join_path(module_path, 'operating_system') +operating_system_path = join_path(module_path, 'operating_systems') test_path = join_path(module_path, "test") hooks_path = join_path(module_path, "hooks") var_path = join_path(prefix, "var", "spack") From 4601c36f572ed7cf66f539c11ebe0835976655dc Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 09:55:44 -0800 Subject: [PATCH 164/375] cleaned up concretize architecture --- lib/spack/spack/concretize.py | 91 ++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 3434ca66f79..04f932aa5de 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -218,20 +218,55 @@ def _concretize_platform(self, arch, platform): return True def _concretize_operating_system(self, arch, platform): - """ Future method for concretizing operating system """ - if isinstance(arch.platform_os, spack.architecture.OperatingSystem): - return False + if spec.architecture.platform_os is not None: + if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): + return False + else: + spec.add_operating_system_from_string(spec.architecture.platform_os) + return True #changed + if spec.root.architecture.platform_os: + if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): + spec.architecture.platform_os = spec.root.architecture.platform_os + else: + spec.add_operating_system_from_string(spec.root.architecture.platform_os) else: - arch.arch_os = platform.operating_system('default_os') - return True + platform = spack.architecture.sys_type() + spec.architecture.platform_os = platform.operating_system('default_os') + + return True #changed + +# """ Future method for concretizing operating system """ +# if isinstance(arch.platform_os, spack.architecture.OperatingSystem): +# return False +# else: +# arch.arch_os = platform.operating_system('default_os') +# return True def _concretize_target(self, arch, platform): - if isinstance(arch.target, spack.architecture.Target): - return False + if spec.target is not None: + if isinstance(spec.target,spack.architecture.Target): + return False + else: + spec.add_target_from_string(spec.target) + return True #changed + + if spec.root.target: + if isinstance(spec.root.target,spack.architecture.Target): + spec.target = spec.root.target + else: + spec.add_target_from_string(spec.root.target) else: - arch.target = platform.target('default') - return True + platform = spack.architecture.sys_type() + spec.target = platform.target('default') + + return True #changed + +# if isinstance(arch.target, spack.architecture.Target): +# return False +# else: +# arch.target = platform.target('default') +# return True def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the @@ -251,33 +286,35 @@ def concretize_architecture(self, spec): Arch = spack.architecture.Arch # Set the architecture to all defaults spec.architecture = Arch(platform=platform, - platform_os=platform.operating_system('default_os'), - target=platform.target('default')) + None, + None) return True #If there is a target and it is a tuple and has both filled return #False - if not isinstance(spec.architecture, basestring): - return any(( +# if isinstance(spec.architecture, basestring): +# spec.split_architecture_string(spec.architecture) + + + ret = any(( self._concretize_platform(spec.architecture, platform), self._concretize_operating_system(spec.architecture, platform), self._concretize_target(spec.architecture, platform))) - else: - spec.add_architecture_from_string(spec.architecture) + # Does not look pretty at all!!! - if spec.root.architecture and \ - not isinstance(spec.root.architecture, basestring): - bool_flag = any(( - self._concretize_platform(spec.root.architecture, platform), - self._concretize_operating_system(spec.root.architecture, - platform), - self._concretize_target(spec.root.target, platform))) - spec.architecture =spec.root.architecture - return bool_flag - else: - spec.add_architecture_from_string(spec.root.architecture) +# if spec.root.architecture and \ +# not isinstance(spec.root.architecture, basestring): +# bool_flag = any(( +# self._concretize_platform(spec.root.architecture, platform), +# self._concretize_operating_system(spec.root.architecture, +# platform), +# self._concretize_target(spec.root.target, platform))) +# spec.architecture =spec.root.architecture +# return bool_flag +# else: +# spec.add_architecture_from_string(spec.root.architecture) - return True + return ret # if there is no target specified used the defaults From 615ea969f8a998417dfbbd07935848eb37f71c47 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 10:14:50 -0800 Subject: [PATCH 165/375] made arch instantiate as a tuple --- lib/spack/spack/spec.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 0a55ec5d760..57dd0c21811 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -461,7 +461,13 @@ def _set_architecture(self, architecture): """Called by the parser to set the architecture.""" if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture # a string can be set + platform = spack.architecture.sys_type() + if '-' in architecture: + os, target = architecture.split('-') + else: + os = architecture + target = None + self.architecture = spack.architecture.Arch(platform, os, target) def _add_dependency(self, spec): From 527bb7abfeeb982d0bf22dadf4b8400502b606dd Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 10:26:09 -0800 Subject: [PATCH 166/375] Changed _cmp_key --- lib/spack/spack/architecture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 14a8c567938..dac3a45b798 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -207,7 +207,7 @@ def __str__(self): self.platform_os.name + "-" + self.target.name) def _cmp_key(self): - return (self.platform.name, self.platform_os.name, self.target.name) + return (self.platform, self.platform_os, self.target) def _helper_to_dict(arch_field_dict, arch_field_name, *args): From 4d7478420968182ab9535de9deffbdd671937972 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 10:27:00 -0800 Subject: [PATCH 167/375] Changed add_architecture_from_string and split the work done into two methods add_target_from_string and add_operating_system_from_string --- lib/spack/spack/spec.py | 122 ++++++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 55 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 57dd0c21811..06accc514b8 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1256,70 +1256,82 @@ def _is_valid_os(self, os_string, platform): if os_string in platform.operating_sys: return True return False + + def add_target_from_string(self, arch): + if arch.target is None: + return arch.platform.target('default_target') + else: + return arch.platform.target(arch.target) - def add_architecture_from_string(self, arch): - """ The user is able to provide a architecture string of the form - platform-os-target. This string will be parsed by this function - and turn the architecture string into an architecture tuple of - platform, operating system and target processor classes. - The platform-os-target triplet can be delimited by a '-'. If any - portion of the architecture triplet is empty, spack will supply - the default. If the entire architecture field is blank then - defaults will be provided based off of the platform. - This happens in the concretize_architecture method in concretize.py + def add_operating_system_from_string(self, arch): + if arch.platform_os is None: + return arch.platform.operating_system('default_os') + else: + return arch.platform.operating_system(arch.platform_os) - e.g - =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) + #def add_architecture_from_string(self, arch): + # """ The user is able to provide a architecture string of the form + # platform-os-target. This string will be parsed by this function + # and turn the architecture string into an architecture tuple of + # platform, operating system and target processor classes. + # The platform-os-target triplet can be delimited by a '-'. If any + # portion of the architecture triplet is empty, spack will supply + # the default. If the entire architecture field is blank then + # defaults will be provided based off of the platform. + # This happens in the concretize_architecture method in concretize.py - =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) + # e.g + # =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) - =bgq -> (bgq, - default_os, - default_target) + # =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) - =elcapitan -> (darwin, elcapitan, x86_64) + # =bgq -> (bgq, + # default_os, + # default_target) - =x86_64 -> (autodetected platform, - default_os, - x86_64) - """ - if arch is None: return - - platform_list = spack.architecture.all_platforms() - platform_names = {plat.__name__.lower():plat for plat in platform_list} - Arch = spack.architecture.Arch - arch_list = arch.split("-") - - # Create instance of current platform, gets overwritten if user - # provided a platform spec. - platform = spack.architecture.sys_type() - target = None - platform_os = None + # =elcapitan -> (darwin, elcapitan, x86_64) - for entry in arch_list: - if self._is_valid_platform(entry, platform_names): - # If entry is different from platform name then create it. - # else just keep the already instantiated platform class - if entry != platform.name: - platform = platform_dict[entry]() # Create instance of platform - elif self._is_valid_target(entry, platform): - target = platform.target(entry) - # check if os is valid by checking platform operating sys dict - elif self._is_valid_os(entry, platform): - platform_os = platform.operating_system(entry) - else: - # throw error since entry is unknown - raise UnknownArchitectureSpecError(entry) + # =x86_64 -> (autodetected platform, + # default_os, + # x86_64) + # """ + # if arch is None: return + # + # platform_list = spack.architecture.all_platforms() + # platform_names = {plat.__name__.lower():plat for plat in platform_list} + # Arch = spack.architecture.Arch + # arch_list = arch.split("-") + # + # # Create instance of current platform, gets overwritten if user + # # provided a platform spec. + # platform = spack.architecture.sys_type() + # target = None + # platform_os = None - if target is None: - target = platform.target('default') - if platform_os is None: - platform_os = platform.operating_system('default_os') + # for entry in arch_list: + # if self._is_valid_platform(entry, platform_names): + # # If entry is different from platform name then create it. + # # else just keep the already instantiated platform class + # if entry != platform.name: + # platform = platform_dict[entry]() # Create instance of platform + # elif self._is_valid_target(entry, platform): + # target = platform.target(entry) + # # check if os is valid by checking platform operating sys dict + # elif self._is_valid_os(entry, platform): + # platform_os = platform.operating_system(entry) + # else: + # # throw error since entry is unknown + # raise UnknownArchitectureSpecError(entry) - self.architecture = Arch(platform=platform, - platform_os=platform_os, - target=target) - + # if target is None: + # target = platform.target('default') + # if platform_os is None: + # platform_os = platform.operating_system('default_os') + + # self.architecture = Arch(platform=platform, + # platform_os=platform_os, + # target=target) + # def satisfies(self, other, deps=True, strict=False): """determine if this spec satisfies all constraints of another. From f27f2f8e491c7e3a433e070497d074f8376233a0 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 10:27:12 -0800 Subject: [PATCH 168/375] Fixed some indentation errors --- lib/spack/spack/concretize.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 04f932aa5de..60efba59c2b 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -225,7 +225,7 @@ def _concretize_operating_system(self, arch, platform): spec.add_operating_system_from_string(spec.architecture.platform_os) return True #changed if spec.root.architecture.platform_os: - if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): + if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): spec.architecture.platform_os = spec.root.architecture.platform_os else: spec.add_operating_system_from_string(spec.root.architecture.platform_os) @@ -281,13 +281,13 @@ def concretize_architecture(self, spec): """ platform = spack.architecture.sys_type() + if spec.architecture is None: # Create an empty tuple Arch = spack.architecture.Arch # Set the architecture to all defaults - spec.architecture = Arch(platform=platform, - None, - None) + spec.architecture = Arch(platform=platform, platform_os=None, + target=None) return True #If there is a target and it is a tuple and has both filled return #False From 45887dec8e703232a4d38bd241a20e36606a00cf Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 10:56:46 -0800 Subject: [PATCH 169/375] partial work on bug hunting --- lib/spack/spack/architecture.py | 22 +++++++++++----------- lib/spack/spack/concretize.py | 32 +++++++++++++++----------------- lib/spack/spack/spec.py | 21 +++++++++++---------- 3 files changed, 37 insertions(+), 38 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index dac3a45b798..45a9dac8ba7 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -203,8 +203,8 @@ class Arch(namedtuple("Arch", "platform platform_os target")): __slots__ = () def __str__(self): - return (self.platform.name +"-"+ - self.platform_os.name + "-" + self.target.name) + return (str(self.platform) +"-"+ + str(self.platform_os) + "-" + str(self.target) ) def _cmp_key(self): return (self.platform, self.platform_os, self.target) @@ -246,14 +246,14 @@ def to_dict(arch): return d -def _platform_from_dict(platform): - """Creates all the platform class module names into a dictionary of - name : key-value pairs. From there we can construct the - platform subclass - """ - platform_list = all_platforms() - platform_names = {plat.__name__.lower():plat for plat in platform_list} - return platform_names[platform['name']]() +#def _platform_from_dict(platform): +# """Creates all the platform class module names into a dictionary of +# name : key-value pairs. From there we can construct the +# platform subclass +# """ +# platform_list = all_platforms() +# platform_names = {plat.__name__.lower():plat for plat in platform_list} +# return platform_names[platform['name']]() def _target_from_dict(target_dict): @@ -288,7 +288,7 @@ def arch_from_dict(d): platform_os_dict = d['platform_os'] target_dict = d['target'] - platform = _platform_from_dict(platform_dict) + platform = d['platform'] platform_os = _operating_system_from_dict(platform_os_dict, platform) target = _target_from_dict(target_dict) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 60efba59c2b..afb12838e8f 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -217,14 +217,14 @@ def _concretize_platform(self, arch, platform): arch.platform = platform return True - def _concretize_operating_system(self, arch, platform): + def _concretize_operating_system(self, spec, platform): if spec.architecture.platform_os is not None: if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): return False else: spec.add_operating_system_from_string(spec.architecture.platform_os) return True #changed - if spec.root.architecture.platform_os: + if spec.root.architecture and spec.root.architecture.platform_os: if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): spec.architecture.platform_os = spec.root.architecture.platform_os else: @@ -243,22 +243,22 @@ def _concretize_operating_system(self, arch, platform): # return True - def _concretize_target(self, arch, platform): - if spec.target is not None: - if isinstance(spec.target,spack.architecture.Target): + def _concretize_target(self, spec, platform): + if spec.architecture.target is not None: + if isinstance(spec.architecture.target,spack.architecture.Target): return False else: - spec.add_target_from_string(spec.target) + spec.add_target_from_string(spec.architecture.target) return True #changed - if spec.root.target: - if isinstance(spec.root.target,spack.architecture.Target): - spec.target = spec.root.target + if spec.root.architecture and spec.root.architecture.target: + if isinstance(spec.root.architecture.target,spack.architecture.Target): + spec.architecture.target = spec.root.architecture.target else: - spec.add_target_from_string(spec.root.target) + spec.add_target_from_string(spec.root.architecture.target) else: platform = spack.architecture.sys_type() - spec.target = platform.target('default') + spec.architecture.target = platform.target('default') return True #changed @@ -283,22 +283,20 @@ def concretize_architecture(self, spec): platform = spack.architecture.sys_type() if spec.architecture is None: - # Create an empty tuple - Arch = spack.architecture.Arch # Set the architecture to all defaults - spec.architecture = Arch(platform=platform, platform_os=None, + spec.architecture = spack.architecture.Arch(platform=platform, platform_os=None, target=None) return True #If there is a target and it is a tuple and has both filled return #False # if isinstance(spec.architecture, basestring): # spec.split_architecture_string(spec.architecture) - + print spec.architecture ret = any(( self._concretize_platform(spec.architecture, platform), - self._concretize_operating_system(spec.architecture, platform), - self._concretize_target(spec.architecture, platform))) + self._concretize_operating_system(spec, platform), + self._concretize_target(spec, platform))) # Does not look pretty at all!!! diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 06accc514b8..5cb94907d3f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -462,11 +462,12 @@ def _set_architecture(self, architecture): if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) platform = spack.architecture.sys_type() + print architecture if '-' in architecture: os, target = architecture.split('-') else: - os = architecture - target = None + os = None + target = architecture self.architecture = spack.architecture.Arch(platform, os, target) @@ -1257,17 +1258,17 @@ def _is_valid_os(self, os_string, platform): return True return False - def add_target_from_string(self, arch): - if arch.target is None: - return arch.platform.target('default_target') + def add_target_from_string(self, target): + if target is None: + self.architecture.target = self.architecture.platform.target('default_target') else: - return arch.platform.target(arch.target) + self.architecture.target = self.architecture.platform.target(target) - def add_operating_system_from_string(self, arch): - if arch.platform_os is None: - return arch.platform.operating_system('default_os') + def add_operating_system_from_string(self, os): + if os is None: + self.architecture.platform_os = self.architecture.platform.operating_system('default_os') else: - return arch.platform.operating_system(arch.platform_os) + self.architecture.platform_os = self.architecture.platform.operating_system(os) #def add_architecture_from_string(self, arch): # """ The user is able to provide a architecture string of the form From 81e236b2de157405a2bc65e71a9a7fc7d1a13d7d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 11:04:55 -0800 Subject: [PATCH 170/375] Changed Arch to class instead of namedtuple. Now platform is automatically set upon creation of class --- lib/spack/spack/architecture.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 45a9dac8ba7..94bd59a9d5e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -196,11 +196,13 @@ def _cmp_key(self): #NOTE: Key error caused because Architecture has no comparison method @key_ordering -class Arch(namedtuple("Arch", "platform platform_os target")): - """ namedtuple for Architecture. Will have it's own __str__ method - to make printing out the tuple easier and also won't make directory - paths look odd """ - __slots__ = () +class Arch(object): + "Architecture is now a class to help with setting attributes" + + def __init__(self, platform_os=None, target=None): + self.platform = sys_type() + self.platform_os = platform_os + self.target = target def __str__(self): return (str(self.platform) +"-"+ From 86e90bba8741eec38eae3898576da272ed41d9c2 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 11:05:05 -0800 Subject: [PATCH 171/375] another partial, narrowing in on proper arch --- lib/spack/spack/concretize.py | 14 +------ lib/spack/spack/spec.py | 74 +++-------------------------------- 2 files changed, 6 insertions(+), 82 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index afb12838e8f..add3b8435d0 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,13 +210,6 @@ def concretize_version(self, spec): return True # Things changed - def _concretize_platform(self, arch, platform): - if issubclass(arch.platform.__class__, spack.architecture.Platform): - return False - else: - arch.platform = platform - return True - def _concretize_operating_system(self, spec, platform): if spec.architecture.platform_os is not None: if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): @@ -279,13 +272,9 @@ def concretize_architecture(self, spec): DAG has an architecture, then use the root otherwise use the defaults on the platform. """ - - platform = spack.architecture.sys_type() - if spec.architecture is None: # Set the architecture to all defaults - spec.architecture = spack.architecture.Arch(platform=platform, platform_os=None, - target=None) + spec.architecture = spack.architecture.Arch() return True #If there is a target and it is a tuple and has both filled return #False @@ -294,7 +283,6 @@ def concretize_architecture(self, spec): print spec.architecture ret = any(( - self._concretize_platform(spec.architecture, platform), self._concretize_operating_system(spec, platform), self._concretize_target(spec, platform))) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 5cb94907d3f..169c06242e6 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -468,7 +468,7 @@ def _set_architecture(self, architecture): else: os = None target = architecture - self.architecture = spack.architecture.Arch(platform, os, target) + self.architecture = spack.architecture.Arch(os, target) def _add_dependency(self, spec): @@ -1250,13 +1250,10 @@ def _is_valid_platform(self, platform, platform_list): return False def _is_valid_target(self, target, platform): - if target in platform.targets: - return True - return False + return target in platform.targets + def _is_valid_os(self, os_string, platform): - if os_string in platform.operating_sys: - return True - return False + return os_string in platform.operating_sys def add_target_from_string(self, target): if target is None: @@ -1270,69 +1267,7 @@ def add_operating_system_from_string(self, os): else: self.architecture.platform_os = self.architecture.platform.operating_system(os) - #def add_architecture_from_string(self, arch): - # """ The user is able to provide a architecture string of the form - # platform-os-target. This string will be parsed by this function - # and turn the architecture string into an architecture tuple of - # platform, operating system and target processor classes. - # The platform-os-target triplet can be delimited by a '-'. If any - # portion of the architecture triplet is empty, spack will supply - # the default. If the entire architecture field is blank then - # defaults will be provided based off of the platform. - # This happens in the concretize_architecture method in concretize.py - # e.g - # =linux-ubuntu10-x84_64 -> (linux, ubuntu10, x86_64) - - # =cray_xc-SuSE11-haswell -> (cray_xc, SuSE11, haswell) - - # =bgq -> (bgq, - # default_os, - # default_target) - - # =elcapitan -> (darwin, elcapitan, x86_64) - - # =x86_64 -> (autodetected platform, - # default_os, - # x86_64) - # """ - # if arch is None: return - # - # platform_list = spack.architecture.all_platforms() - # platform_names = {plat.__name__.lower():plat for plat in platform_list} - # Arch = spack.architecture.Arch - # arch_list = arch.split("-") - # - # # Create instance of current platform, gets overwritten if user - # # provided a platform spec. - # platform = spack.architecture.sys_type() - # target = None - # platform_os = None - - # for entry in arch_list: - # if self._is_valid_platform(entry, platform_names): - # # If entry is different from platform name then create it. - # # else just keep the already instantiated platform class - # if entry != platform.name: - # platform = platform_dict[entry]() # Create instance of platform - # elif self._is_valid_target(entry, platform): - # target = platform.target(entry) - # # check if os is valid by checking platform operating sys dict - # elif self._is_valid_os(entry, platform): - # platform_os = platform.operating_system(entry) - # else: - # # throw error since entry is unknown - # raise UnknownArchitectureSpecError(entry) - - # if target is None: - # target = platform.target('default') - # if platform_os is None: - # platform_os = platform.operating_system('default_os') - - # self.architecture = Arch(platform=platform, - # platform_os=platform_os, - # target=target) - # def satisfies(self, other, deps=True, strict=False): """determine if this spec satisfies all constraints of another. @@ -1378,6 +1313,7 @@ def satisfies(self, other, deps=True, strict=False): if not self.variants.satisfies(other.variants, strict=strict): return False + # Target satisfaction is currently just class equality. # If not strict, None means unconstrained. if isinstance(self.architecture, basestring): From 25f20b19673d71fb11ada83567c9e89de5555839 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 11:40:21 -0800 Subject: [PATCH 172/375] bug hunting --- lib/spack/spack/architecture.py | 5 ++++- lib/spack/spack/concretize.py | 15 ++++++--------- lib/spack/spack/spec.py | 1 - var/spack/mock_packages/multimethod/package.py | 7 ++++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 94bd59a9d5e..c9e4eb132db 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -209,7 +209,10 @@ def __str__(self): str(self.platform_os) + "-" + str(self.target) ) def _cmp_key(self): - return (self.platform, self.platform_os, self.target) + platform = self.platform.name + os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os + target = self.target.name if isinstance(self.target, Target) else self.target + return (platform, os, target) def _helper_to_dict(arch_field_dict, arch_field_name, *args): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index add3b8435d0..37204e32a4e 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,7 +210,7 @@ def concretize_version(self, spec): return True # Things changed - def _concretize_operating_system(self, spec, platform): + def _concretize_operating_system(self, spec): if spec.architecture.platform_os is not None: if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): return False @@ -223,8 +223,7 @@ def _concretize_operating_system(self, spec, platform): else: spec.add_operating_system_from_string(spec.root.architecture.platform_os) else: - platform = spack.architecture.sys_type() - spec.architecture.platform_os = platform.operating_system('default_os') + spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') return True #changed @@ -236,7 +235,7 @@ def _concretize_operating_system(self, spec, platform): # return True - def _concretize_target(self, spec, platform): + def _concretize_target(self, spec): if spec.architecture.target is not None: if isinstance(spec.architecture.target,spack.architecture.Target): return False @@ -250,8 +249,7 @@ def _concretize_target(self, spec, platform): else: spec.add_target_from_string(spec.root.architecture.target) else: - platform = spack.architecture.sys_type() - spec.architecture.target = platform.target('default') + spec.architecture.target = spec.architecture.platform.target('default') return True #changed @@ -280,11 +278,10 @@ def concretize_architecture(self, spec): #False # if isinstance(spec.architecture, basestring): # spec.split_architecture_string(spec.architecture) - print spec.architecture ret = any(( - self._concretize_operating_system(spec, platform), - self._concretize_target(spec, platform))) + self._concretize_operating_system(spec), + self._concretize_target(spec))) # Does not look pretty at all!!! diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 169c06242e6..fdc5e3bc963 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -462,7 +462,6 @@ def _set_architecture(self, architecture): if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) platform = spack.architecture.sys_type() - print architecture if '-' in architecture: os, target = architecture.split('-') else: diff --git a/var/spack/mock_packages/multimethod/package.py b/var/spack/mock_packages/multimethod/package.py index b926d42d7be..03da9ef69d3 100644 --- a/var/spack/mock_packages/multimethod/package.py +++ b/var/spack/mock_packages/multimethod/package.py @@ -120,9 +120,10 @@ def has_a_default(self): for target in targets: @when('='+target.name) def different_by_target(self): - return self.spec.architecture.target.name - - + if isinstance(self.spec.architecture.target,basestring): + return self.spec.architecture.target + else: + return self.spec.architecture.target.name # # Make sure we can switch methods on different dependencies # From 47c8e1366f9659850704b03fd39dbfa3dd54fbbb Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 12:08:17 -0800 Subject: [PATCH 173/375] Changed to_dictionary and from_dictionary methods --- lib/spack/spack/architecture.py | 112 +++++++++++++++++++------------- 1 file changed, 67 insertions(+), 45 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index c9e4eb132db..e432374781f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -58,14 +58,11 @@ class Target(object): Targets will have compiler finding strategies """ - def __init__(self, name, compiler_strategy, module_name=None): + def __init__(self, name, module_name=None): self.name = name # case of cray "ivybridge" but if it's x86_64 - self.compiler_strategy = compiler_strategy self.module_name = module_name # craype-ivybridge # Sets only the platform name to avoid recursiveness - def set_platform(self, platform): - self.platform_name = platform.name def _cmp_key(self): return (self.name, self.module_name) @@ -74,10 +71,14 @@ def __repr__(self): return self.__str__() def __str__(self): - if self.platform_name: - return self.platform_name + '-' + self.name return self.name + def to_dict(self): + d = {} + d['name'] = self.name + d['module_name'] = self.module_name + + return d @key_ordering class Platform(object): @@ -109,7 +110,6 @@ def add_target(self, name, target): raise ValueError( "%s is a spack reserved alias " "and cannot be the name of a target" % name) - target.set_platform(self) self.targets[name] = target def target(self, name): @@ -193,6 +193,14 @@ def compiler_strategy(self): def _cmp_key(self): return (self.name, self.version, self.compiler_strategy) + + def to_dict(self): + d = {} + d['name'] = self.name + d['version'] = self.version + d['compiler_strategy'] = self.compiler_strategy + + return d #NOTE: Key error caused because Architecture has no comparison method @key_ordering @@ -213,43 +221,55 @@ def _cmp_key(self): os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os target = self.target.name if isinstance(self.target, Target) else self.target return (platform, os, target) - - -def _helper_to_dict(arch_field_dict, arch_field_name, *args): - """ General method to turn each class in architecture into a - dictionary. Takes as argument the class dictionary, the field name - (platform, platform_os, target) and then any attribute args - """ - d = {} - d[arch_field_name] = {} - for items in args: - d[arch_field_name][items] = arch_field_dict[items] - return d - -def to_dict(arch): - """ Convert the Arch tuple into a dictionary for yaml dumping. This - uses the _helper_to_dict method to create the dictionary from the - provided architecture field. Can assign the architecture - field name (either platform, platform_os or target) and any - attributes that make up that architecture field, - """ - d = {} - platform = arch.platform.__dict__ - platform_os = arch.platform_os.__dict__ - target = arch.target.__dict__ + def to_dict(self): + d = {} + platform = self.platform + platform_os = self.platform_os + target = self.target - platform_dict = _helper_to_dict(platform,'platform','name') - os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', - 'compiler_strategy') - target_dict = _helper_to_dict(target,'target', 'name', - 'module_name','platform_name') + d['platform'] = self.platform.name + d['platform_os'] = self.platform_os.to_dict() + d['target'] = self.target.to_dict() + + return d - d.update(platform_dict) - d.update(os_dict) - d.update(target_dict) +#def _helper_to_dict(arch_field_dict, arch_field_name, *args): +# """ General method to turn each class in architecture into a +# dictionary. Takes as argument the class dictionary, the field name +# (platform, platform_os, target) and then any attribute args +# """ +# d = {} +# d[arch_field_name] = {} +# for items in args: +# d[arch_field_name][items] = arch_field_dict[items] +# return d +# - return d +#def to_dict(arch): +# """ Convert the Arch tuple into a dictionary for yaml dumping. This +# uses the _helper_to_dict method to create the dictionary from the +# provided architecture field. Can assign the architecture +# field name (either platform, platform_os or target) and any +# attributes that make up that architecture field, +# """ +# d = {} +# +# platform = arch.platform.__dict__ +# platform_os = arch.platform_os.__dict__ +# target = arch.target.__dict__ +# +# platform_dict = _helper_to_dict(platform,'platform','name') +# os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', +# 'compiler_strategy') +# target_dict = _helper_to_dict(target,'target', 'name', +# 'module_name','platform_name') +# +# d.update(platform_dict) +# d.update(os_dict) +# d.update(target_dict) +# +# return d #def _platform_from_dict(platform): # """Creates all the platform class module names into a dictionary of @@ -287,17 +307,19 @@ def arch_from_dict(d): helper methods to recreate the arch tuple from the dictionary read from a yaml file """ + arch = Arch() + if d is None: return None - platform_dict = d['platform'] - platform_os_dict = d['platform_os'] + os_dict = d['platform_os'] target_dict = d['target'] - platform = d['platform'] - platform_os = _operating_system_from_dict(platform_os_dict, platform) target = _target_from_dict(target_dict) + platform_os = _operating_system_from_dict(os_dict, arch.platform) + arch.target =target + arch.platform_os = platform_os - return Arch(platform, platform_os, target) + return arch @memoized def all_platforms(): From 90e90f61c161facfb87dd2dc6389f80d56eba2d7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 12:08:43 -0800 Subject: [PATCH 174/375] Got rid of the compiler strategy from target --- lib/spack/spack/platforms/bgq.py | 4 ++-- lib/spack/spack/platforms/cray_xc.py | 6 +++--- lib/spack/spack/platforms/darwin.py | 2 +- lib/spack/spack/platforms/linux.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/platforms/bgq.py b/lib/spack/spack/platforms/bgq.py index b550f1d5cb8..e0eb76f3363 100644 --- a/lib/spack/spack/platforms/bgq.py +++ b/lib/spack/spack/platforms/bgq.py @@ -9,8 +9,8 @@ class Bgq(Platform): def __init__(self): super(Bgq, self).__init__('bgq') - self.add_target(self.front_end, Target(self.front_end, 'PATH')) - self.add_target(self.back_end, Target(self.back_end, 'PATH')) + self.add_target(self.front_end, Target(self.front_end)) + self.add_target(self.back_end, Target(self.back_end,)) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index b71481e46e8..4843a47c622 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -31,11 +31,11 @@ def __init__(self): # Could switch to use modules and fe targets for front end # Currently using compilers by path for front end. - self.add_target('sandybridge', Target('sandybridge', 'PATH')) + self.add_target('sandybridge', Target('sandybridge')) self.add_target('ivybridge', - Target('ivybridge', 'MODULES', 'craype-ivybridge')) + Target('ivybridge', 'craype-ivybridge')) self.add_target('haswell', - Target('haswell', 'MODULES', 'craype-haswell')) + Target('haswell','craype-haswell')) self.add_operating_system('SuSE', LinuxDistro()) self.add_operating_system('CNL', Cnl()) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index c48f7d17d2b..4c3d38851f8 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -10,7 +10,7 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') - self.add_target(self.default, Target(self.default, 'PATH')) + self.add_target(self.default, Target(self.default)) mac_os = MacOsx() self.default_os = mac_os.name self.add_operating_system(mac_os.name, mac_os) diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index d9e33511a13..3243a1dcdf0 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -10,7 +10,7 @@ class Linux(Platform): def __init__(self): super(Linux, self).__init__('linux') - self.add_target(self.default, Target(self.default, 'PATH')) + self.add_target(self.default, Target(self.default)) linux_dist = LinuxDistro() self.default_os = linux_dist.name self.add_operating_system(linux_dist.name, linux_dist) From e46bac19fa9baafe1e4b32ff940a883218f8b019 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 12:09:02 -0800 Subject: [PATCH 175/375] Changed architecture to dict --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fdc5e3bc963..6cca61ca4e7 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -671,7 +671,7 @@ def to_node_dict(self): if self.architecture: # TODO: Fix the target.to_dict to account for the tuple # Want it to be a dict of dicts - d['architecture'] = spack.architecture.to_dict(self.architecture) + d['architecture'] = self.architecture.to_dict() else: d['architecture'] = None From 975cba295bc3141a9888292623702f1ee45f3d3a Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 14:16:57 -0800 Subject: [PATCH 176/375] Added 'better' tests --- lib/spack/spack/test/architecture.py | 58 +++++++++++----------------- 1 file changed, 23 insertions(+), 35 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 5bc78d1b658..a390a3fbaa6 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -14,43 +14,31 @@ class ArchitectureTest(unittest.TestCase): - def setUp(self): - zlib = spack.spec.Spec("zlib") - zlib.concretize() - self.architecture = zlib.architecture - self.platform = sys_type() - self.platform_os = self.platform.operating_system('default_os') - self.target = self.platform.target('default') - - #def test_to_dict_function_with_target(self): - # d = spack.architecture.to_dict(self.architecture) - # print d['target'] - # self.assertEquals(d['target'], {'name': self.target.name, - # 'module_name' : self.target.module_name, - # 'platform_name' : self.target.platform_name, - # 'compiler_strategy': 'MODULES' - # }) - def test_to_dict_function_with_architecture(self): - d = spack.architecture.to_dict(self.architecture) - self.assertEquals(d, {'architecture': - {'platform' : {'name': 'crayxc'}, - 'platform_os': { - 'compiler_strategy': 'MODULES', - 'name':'CNL', - 'version':'10'}, - 'target' : {'platform_name' :'crayxc', - 'module_name': 'craype-haswell', - 'name':'haswell'}}}) + arch = Arch() + arch.platform_os = arch.platform.operating_system('default_os') + arch.target = arch.platform.target('default') - #def test_to_dict_function_with_operating_system(self): - # d = spack.architecture.to_dict(self.architecture) - # self.assertEquals(d['platform_os'], {'name': self.platform_os.name, - # 'version': self.platform_os.version, - # 'compiler_strategy': self.platform_os.compiler_strategy}) - - def test_architecture_from_dict(self): - pass + d = arch.to_dict() + self.assertEqual(d, {'platform' : 'crayxc', + 'platform_os' : {'name': 'CNL', + 'compiler_strategy' : 'MODULES', + 'version':'10'}, + 'target' : {'name': 'haswell', + 'module_name': 'craype-haswell'}}) + + def test_from_dict_function_with_architecture(self): + d = {'platform':'crayxc', + 'platform_os' : {'name' : 'CNL', 'compiler_strategy': 'MODULES', + 'version': '10'}, + 'target' : {'name':'haswell', 'module_name': 'craype-haswell'}} + + arch = spack.architecture.arch_from_dict(d) + self.assertIsInstance(arch, Arch) + self.assertIsInstance(arch.platform, Platform) + self.assertIsInstance(arch.platform_os, OperatingSystem) + self.assertIsInstance(arch.target, Target) + def test_platform_class_and_compiler_strategies(self): a = CrayXc() From 676591ffc0d62a4d3b20653a2faa788c06d96d77 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 14:17:46 -0800 Subject: [PATCH 177/375] fixed a bug and a test bug --- lib/spack/spack/spec.py | 2 +- .../site_spackconfig/compilers.yaml | 23 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 6cca61ca4e7..6d2138d4c8e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1307,7 +1307,7 @@ def satisfies(self, other, deps=True, strict=False): if not self.compiler.satisfies(other.compiler, strict=strict): return False elif strict and (other.compiler and not self.compiler): - return True + return False if not self.variants.satisfies(other.variants, strict=strict): return False diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml index fcbf7a53f16..72fa252881e 100644 --- a/var/spack/mock_configs/site_spackconfig/compilers.yaml +++ b/var/spack/mock_configs/site_spackconfig/compilers.yaml @@ -13,10 +13,23 @@ compilers: fc: /path/to/gfortran modules: None gcc@5.2.0: - cc: cc - cxx: CC - f77: ftn - fc: ftn - modules: + cc: cc + cxx: CC + f77: ftn + fc: ftn + modules: - PrgEnv-gnu - gcc/5.2.0 + intel@15.0.1: + cc: cc + ccx: CC + f77: ftn + fc: ftn + modules: + - PrgEnv-intel + - intel/15.0.1 + intel@15.1.2: + cc: /path/to/icc + cxx: /path/to/ic++ + f77: /path/to/ifort + fc: /path/to/ifort From 6e7b00a0f6229a2c1df7a5b2f7244bdf5a793d65 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 15:54:23 -0800 Subject: [PATCH 178/375] Made module handling much saner and safer --- lib/spack/spack/build_environment.py | 2 +- lib/spack/spack/compiler.py | 26 ++++++++++--------- lib/spack/spack/compilers/__init__.py | 21 ++++++++++----- lib/spack/spack/concretize.py | 9 ++++--- lib/spack/spack/test/architecture.py | 8 +++--- .../site_spackconfig/compilers.yaml | 5 ++++ 6 files changed, 43 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 6e03394777a..0cb4d774997 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -171,7 +171,7 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.modules: + if compiler.strategy == 'MODULES': for mod in compiler.modules: load_module(mod) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 46f2dfaec72..5d851f5a3da 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,26 +106,28 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, paths, modules=None): + def __init__(self, cspec, strategy, paths, modules=None): def check(exe): if exe is None: return None _verify_executables(exe) return exe + self.strategy = strategy + self.cc = check(paths[0]) self.cxx = check(paths[1]) - self.f77 = check(paths[2]) - if len(paths) == 3: - self.fc = self.f77 - else: - self.fc = check(paths[3]) + if len(paths) > 2: + self.f77 = check(paths[2]) + if len(paths) == 3: + self.fc = self.f77 + else: + self.fc = check(paths[3]) self.spec = cspec self.modules = modules - @property def version(self): return self.spec.version @@ -267,7 +269,7 @@ def find_in_path(cls, *path): if newcount <= prevcount: continue - compilers[ver] = cls(spec, paths) + compilers[ver] = cls(spec, 'PATH', paths) return list(compilers.values()) @@ -287,7 +289,7 @@ def find_in_modules(cls): for name, version in matches: v = version - comp = cls(spack.spec.CompilerSpec(name + '@' + v), + comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES', ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) compilers.append(comp) @@ -302,12 +304,12 @@ def __repr__(self): def __str__(self): """Return a string represntation of the compiler toolchain.""" - if self.modules: + if self.strategy is 'MODULES': return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules)))) + self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc, self.modules)))) else: return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc)))) + self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc)))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index e38ef949d7d..32885062fd3 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -129,6 +129,11 @@ def add_compilers_to_config(scope, *compilers): for compiler in compilers: compiler_entry = {} + val = getattr(compiler, 'strategy') + if not val: + val = 'None' + compiler_entry[c] = val + for c in _required_instance_vars: val = getattr(compiler, c) if not val: @@ -190,6 +195,11 @@ def get_compiler(cspec): raise InvalidCompilerConfigurationError(cspec) cls = class_for_compiler_name(cspec.name) + + strategy = items['strategy'] + if not strategy: + raise InvalidCompilerConfigurationError(cspec) + compiler_paths = [] for c in _required_instance_vars: compiler_path = items[c] @@ -203,22 +213,19 @@ def get_compiler(cspec): items[m] = None mods = items[m] - return cls(cspec, compiler_paths, mods) + return cls(cspec, strategy, compiler_paths, mods) matches = find(compiler_spec) return [get_compiler(cspec) for cspec in matches] @_auto_compiler_spec -def compiler_for_spec(compiler_spec, target): +def compiler_for_spec(compiler_spec, operating_system): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" assert(compiler_spec.concrete) - compilers = compilers_for_spec(compiler_spec) - if target.compiler_strategy == "PATH": - compilers = [c for c in compilers if c.modules is None] - elif target.compiler_strategy == "MODULES": - compilers = [c for c in compilers if c.modules is not None] + compilers = [c for c in compilers_for_spec(compiler_spec) + if c.strategy == operating_system.strategy] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, target) if len(compilers) > 1: diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 37204e32a4e..7a87a824d51 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -354,10 +354,11 @@ def concretize_compiler(self, spec): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - if architecture.platform_os.compiler_strategy == 'PATH': - filter(lambda c: not c.modules, compilers) - if architecture.platform_os.compiler_strategy == 'MODULES': - filter(lambda c: c.modules, compilers) + filter(lambda c: c.strategy == architecture.platform_os.compiler_strategy, compilers) +#if architecture.platform_os.compiler_strategy == 'PATH': + # filter(lambda c: not c.modules, compilers) + #if architecture.platform_os.compiler_strategy == 'MODULES': + # filter(lambda c: c.modules, compilers) return compilers diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index a390a3fbaa6..75a67bf02f9 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -34,10 +34,10 @@ def test_from_dict_function_with_architecture(self): 'target' : {'name':'haswell', 'module_name': 'craype-haswell'}} arch = spack.architecture.arch_from_dict(d) - self.assertIsInstance(arch, Arch) - self.assertIsInstance(arch.platform, Platform) - self.assertIsInstance(arch.platform_os, OperatingSystem) - self.assertIsInstance(arch.target, Target) + self.assertTrue( isinstance(arch, Arch) ) + self.assertTrue( isinstance(arch.platform, Platform) ) + self.assertTrue( isinstance(arch.platform_os, OperatingSystem) ) + self.assertTrue( isinstance(arch.target, Target) ) def test_platform_class_and_compiler_strategies(self): diff --git a/var/spack/mock_configs/site_spackconfig/compilers.yaml b/var/spack/mock_configs/site_spackconfig/compilers.yaml index 72fa252881e..5f8b38007bb 100644 --- a/var/spack/mock_configs/site_spackconfig/compilers.yaml +++ b/var/spack/mock_configs/site_spackconfig/compilers.yaml @@ -6,12 +6,14 @@ compilers: f77: None fc: None modules: None + strategy: PATH gcc@4.5.0: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran modules: None + strategy: PATH gcc@5.2.0: cc: cc cxx: CC @@ -20,6 +22,7 @@ compilers: modules: - PrgEnv-gnu - gcc/5.2.0 + strategy: MODULES intel@15.0.1: cc: cc ccx: CC @@ -28,8 +31,10 @@ compilers: modules: - PrgEnv-intel - intel/15.0.1 + strategy: MODULES intel@15.1.2: cc: /path/to/icc cxx: /path/to/ic++ f77: /path/to/ifort fc: /path/to/ifort + strategy: PATH \ No newline at end of file From ffb9574312e8b6368c7696c2eadd17e46ec9a400 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 15:59:23 -0800 Subject: [PATCH 179/375] fixed type/bug --- lib/spack/spack/compilers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 32885062fd3..b50ad02b071 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -225,7 +225,7 @@ def compiler_for_spec(compiler_spec, operating_system): be concrete.""" assert(compiler_spec.concrete) compilers = [c for c in compilers_for_spec(compiler_spec) - if c.strategy == operating_system.strategy] + if c.strategy == operating_system.compiler_strategy] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, target) if len(compilers) > 1: From 65d4169f003ce73c59f12e426e1e7fa4bf2062f3 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:03:11 -0800 Subject: [PATCH 180/375] fixed type/bug --- lib/spack/spack/compilers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index b50ad02b071..518eea55c45 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -132,7 +132,7 @@ def add_compilers_to_config(scope, *compilers): val = getattr(compiler, 'strategy') if not val: val = 'None' - compiler_entry[c] = val + compiler_entry['strategy'] = val for c in _required_instance_vars: val = getattr(compiler, c) From 16d8c25b231e795f8c4f3d11f937a4658596b238 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:12:32 -0800 Subject: [PATCH 181/375] fixed floating 'target' --- lib/spack/spack/compilers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 518eea55c45..d271b602614 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -227,7 +227,7 @@ def compiler_for_spec(compiler_spec, operating_system): compilers = [c for c in compilers_for_spec(compiler_spec) if c.strategy == operating_system.compiler_strategy] if len(compilers) < 1: - raise NoCompilerForSpecError(compiler_spec, target) + raise NoCompilerForSpecError(compiler_spec, operating_system) if len(compilers) > 1: raise CompilerSpecInsufficientlySpecificError(compiler_spec) return compilers[0] From 58efe1550d913e0bf31f2d5b6e279066cf5363d5 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 16:14:42 -0800 Subject: [PATCH 182/375] Default PATH arg --- lib/spack/spack/architecture.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e432374781f..85ced6e942b 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -170,7 +170,7 @@ class OperatingSystem(object): find compilers we call find_compilers method for each operating system """ - def __init__(self, name, version, compiler_strategy): + def __init__(self, name, version, compiler_strategy="PATH"): self.name = name self.version = version self.compiler_strategy = compiler_strategy @@ -180,19 +180,9 @@ def __str__(self): def __repr__(self): return self.__str__() - - - def compiler_strategy(self): - """ Operating Systems will now be in charge of the compiler finding - strategy. They will each have their own find_compilers method - which depending on their compiler strategy will find the compilers - using a specific method (i.e PATH vs MODULES). - """ - raise NotImplementedError() def _cmp_key(self): return (self.name, self.version, self.compiler_strategy) - def to_dict(self): d = {} From 2b487287419528142b16d9e4201a44a034c11d20 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 2 Mar 2016 16:15:34 -0800 Subject: [PATCH 183/375] Changed os subclasses CNL to change attribute to MODULES --- lib/spack/spack/operating_systems/cnl.py | 6 ------ lib/spack/spack/operating_systems/linux_distro.py | 11 +---------- lib/spack/spack/operating_systems/mac_osx.py | 13 +------------ 3 files changed, 2 insertions(+), 28 deletions(-) diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 20cb4c215b4..9a0bf6c1942 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -11,9 +11,3 @@ def __init__(self): name = 'CNL' version = '10' super(Cnl, self).__init__(name, version, "MODULES") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index c5547749d8d..81c7a864304 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -12,13 +12,4 @@ def __init__(self): name = py_platform.dist()[0] version = py_platform.dist()[1] - super(LinuxDistro, self).__init__(name, version, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - + super(LinuxDistro, self).__init__(name, version) diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_osx.py index 555c5cfb14e..a9de03d2cc7 100644 --- a/lib/spack/spack/operating_systems/mac_osx.py +++ b/lib/spack/spack/operating_systems/mac_osx.py @@ -26,15 +26,4 @@ def __init__(self): except KeyError: name = "mac_os" - super(MacOsx, self).__init__(name, mac_ver, "PATH") - - def compiler_strategy(self): - return self.compiler_strategy - - def find_compilers(self): - pass - - - - - + super(MacOsx, self).__init__(name, mac_ver) From 487b2495a1dbda730a6525f17f175f3eff1674c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:33:10 -0800 Subject: [PATCH 184/375] made spack more conservative in compiler strategies attempted --- lib/spack/spack/compiler.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 5d851f5a3da..999984740f7 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -34,6 +34,7 @@ import spack.error import spack.spec +import spack.architecture from spack.util.multiproc import parmap from spack.util.executable import * from spack.util.environment import get_path @@ -221,7 +222,15 @@ def check(key): @classmethod def find(cls, *path): - return cls.find_in_path(*path) + cls.find_in_modules() + compilers = [] + platform = spack.architecture.sys_type() + strategies = [o.compiler_strategy for o in platform.operating_systems.values()] + if 'PATH' in strategies: + compilers.extend(cls.find_in_path(*path)) + if 'MODULES' in strategies: + compilers.extend(cls.find_in_modules()) + return compilers + @classmethod def find_in_path(cls, *path): From 4f9a309de850f339ed43cd788be45d208bebf6c6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 2 Mar 2016 16:35:02 -0800 Subject: [PATCH 185/375] made spack more conservative in compiler strategies attempted --- lib/spack/spack/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 999984740f7..c8a025f83cb 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -224,7 +224,7 @@ def check(key): def find(cls, *path): compilers = [] platform = spack.architecture.sys_type() - strategies = [o.compiler_strategy for o in platform.operating_systems.values()] + strategies = [o.compiler_strategy for o in platform.operating_sys.values()] if 'PATH' in strategies: compilers.extend(cls.find_in_path(*path)) if 'MODULES' in strategies: From 688eca23ff4488a4cd90125b69d46cb69271a5cd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 23 Mar 2016 09:54:57 -0700 Subject: [PATCH 186/375] cleanup --- lib/spack/spack/spec.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 4d2c61c0c4e..d800a1667dc 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -110,7 +110,7 @@ import spack.architecture import spack.parse import spack.error -import spack.compilers as compilers +#import spack.compilers as compilers from spack.version import * from spack.util.naming import mod_to_class @@ -1174,9 +1174,9 @@ def validate_names(self): spack.repo.get(spec.fullname) # validate compiler in addition to the package name. - if spec.compiler: - if not compilers.supported(spec.compiler): - raise UnsupportedCompilerError(spec.compiler.name) +# if spec.compiler: +# if not compilers.supported(spec.compiler): +# raise UnsupportedCompilerError(spec.compiler.name) # Ensure that variants all exist. for vname, variant in spec.variants.items(): From 315623d3612d64fd058324d4dbbe5264def759c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 24 Mar 2016 16:55:46 -0700 Subject: [PATCH 187/375] Fixed things from merge. --- lib/spack/spack/architecture.py | 7 ++++++ lib/spack/spack/build_environment.py | 10 ++++---- lib/spack/spack/cmd/compiler.py | 2 +- lib/spack/spack/compilers/__init__.py | 34 ++++++++++++++------------- lib/spack/spack/concretize.py | 6 ++--- lib/spack/spack/config.py | 8 +++++++ lib/spack/spack/spec.py | 1 + lib/spack/spack/test/multimethod.py | 4 ++-- 8 files changed, 45 insertions(+), 27 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index aef38b8a557..8d9686cf998 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -202,6 +202,13 @@ def __init__(self, platform_os=None, target=None): self.platform_os = platform_os self.target = target + @property + def concrete(self): + return all( (self.platform is not None, isinstance(self.platform, Platform), + self.platform_os is not None, isinstance(self.platform_os, OperatingSystem), + self.target is not None, isinstance(self.target, Target) ) ) + + def __str__(self): return (str(self.platform) +"-"+ str(self.platform_os) + "-" + str(self.target) ) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index a64ce8aeb44..8ed3c4e1a11 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -101,12 +101,12 @@ def load_module(mod): # We do this without checking that they are already installed # for ease of programming because unloading a module that is not # loaded does nothing. - text = modulecmd('show', mod, return_oe=True).split() + text = modulecmd('show', mod, output=str, error=str).split() for i, word in enumerate(text): if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], return_oe=True), '', 'exec')) + exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) # Load the module now that there are no conflicts - load = modulecmd('load', mod, return_oe=True) + load = modulecmd('load', mod, output=str, error=str) exec(compile(load, '', 'exec')) @@ -119,7 +119,7 @@ def get_path_from_module(mod): modulecmd.add_default_arg('python') # Read the module - text = modulecmd('show', mod, return_oe=True).split('\n') + text = modulecmd('show', mod, output=str, error=str).split('\n') # If it lists its package directory, return that for line in text: @@ -348,8 +348,8 @@ def parent_class_modules(cls): def setup_package(pkg): """Execute all environment setup routines.""" - set_compiler_environment_variables(pkg) set_build_environment_variables(pkg) + set_compiler_environment_variables(pkg) # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index bed5db3d47c..a7de61e1e53 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -23,7 +23,7 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import sys -import argparse +from external import argparse import llnl.util.tty as tty from llnl.util.tty.color import colorize diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 93c5172ddee..55e2db700fe 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -46,7 +46,7 @@ _imported_compilers_module = 'spack.compilers' _required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_optional_instance_vars = ['modules'] +_optional_instance_vars = ['modules', 'strategy'] _default_order = [] # TODO: customize order in config file @@ -69,7 +69,7 @@ def _to_dict(compiler): return { str(compiler.spec) : dict( (attr, getattr(compiler, attr, None)) - for attr in _required_instance_vars) + for attr in _required_instance_vars + _optional_instance_vars) } @@ -77,20 +77,24 @@ def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. """ # Check whether we're on a front-end (native) architecture. - my_arch = spack.architecture.sys_type() + + my_arch = spack.architecture.Arch() + if isinstance(arch, basestring): + if arch == 'all': + my_arch.platform.name = 'all' + arch = my_arch if arch is None: arch = my_arch def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config[arch] = {} + config[arch.platform.name] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: - config[arch].update(_to_dict(compiler)) + config[arch.platform.name].update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) config = spack.config.get_config('compilers', scope=scope) - # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. @@ -105,7 +109,7 @@ def init_compiler_config(): if not site_config: init_compiler_config() - return config[arch] if arch in config else {} + return config[arch.platform.name] if arch.platform.name in config else {} def add_compilers_to_config(compilers, arch=None, scope=None): @@ -117,15 +121,15 @@ def add_compilers_to_config(compilers, arch=None, scope=None): - scope: configuration scope to modify. """ if arch is None: - arch = spack.architecture.sys_type() + arch = spack.architecture.Arch() compiler_config = get_compiler_config(arch, scope) for compiler in compilers: compiler_config[str(compiler.spec)] = dict( (c, getattr(compiler, c, "None")) - for c in _required_instance_vars + ['strategy'] + _optional_instance_vars) + for c in _required_instance_vars + _optional_instance_vars) - update = { arch : compiler_config } + update = { arch.platform.name : compiler_config } spack.config.update_config('compilers', update, scope) @@ -139,7 +143,7 @@ def remove_compiler_from_config(compiler_spec, arch=None, scope=None): - scope: configuration scope to modify. """ if arch is None: - arch = spack.architecture.sys_type() + arch = spack.architecture.Arch() compiler_config = get_compiler_config(arch, scope) del compiler_config[str(compiler_spec)] @@ -154,7 +158,6 @@ def all_compilers_config(arch=None, scope=None): """ # Get compilers for this architecture. arch_config = get_compiler_config(arch, scope) - # Merge 'all' compilers with arch-specific ones. # Arch-specific compilers have higher precedence. merged_config = get_compiler_config('all', scope=scope) @@ -262,10 +265,9 @@ def get_compiler(cspec): else: compiler_paths.append(None) - for m in _optional_instance_vars: - if m not in items: - items[m] = None - mods = items[m] + if 'modules' not in items: + items['modules'] = None + mods = items['modules'] return cls(cspec, strategy, compiler_paths, mods) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 330ecdf5096..c5dad93aefd 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -79,7 +79,7 @@ def _valid_virtuals_and_externals(self, spec): externals = spec_externals(pkg) buildable = not is_spec_nobuild(pkg) if buildable: - result.append((pkg, None)) + result.append((pkg, None, None)) for ext in externals: if ext[0].satisfies(spec): result.append(ext) @@ -354,7 +354,7 @@ def concretize_compiler(self, spec): link to this one, to maximize compatibility. """ # Pass on concretizing the compiler if the target is not yet determined - if not spec.architecture.target: + if not spec.architecture.platform_os: #Although this usually means changed, this means awaiting other changes return True @@ -371,7 +371,7 @@ def _proper_compiler_style(cspec, architecture): return compilers - all_compilers = spack.compilers.all_compilers() + all_compilers = spack.compilers.all_compilers(spec.architecture) if (spec.compiler and spec.compiler.concrete and diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 182c174baa7..c9a770a53fd 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -168,6 +168,10 @@ {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, + 'strategy': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'modules': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]} },},},},},},},}, 'mirrors': { @@ -224,6 +228,10 @@ 'type': 'boolean', 'default': False, }, + 'module': { + 'anyOf' : [{'type': 'string'}, + {'type': 'null'}] + }, 'providers': { 'type': 'object', 'default': {}, diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index d800a1667dc..fa2e9fd4c82 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -542,6 +542,7 @@ def concrete(self): and self.versions.concrete and self.variants.concrete and self.architecture + and self.architecture.concrete and self.compiler and self.compiler.concrete and self.dependencies.concrete) diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 020d25015d7..6a723d123e3 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -92,10 +92,10 @@ def test_target_match(self): platform = spack.architecture.sys_type() targets = platform.targets.values() for target in targets[:-1]: - pkg = spack.db.get('multimethod='+target.name) + pkg = spack.repo.get('multimethod='+target.name) self.assertEqual(pkg.different_by_target(), target.name) - pkg = spack.db.get('multimethod='+targets[-1].name) + pkg = spack.repo.get('multimethod='+targets[-1].name) if len(targets) == 1: self.assertEqual(pkg.different_by_target(), targets[-1].name) else: From 618a9b2d7b1fbe297a2165181016fc45153cecf1 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 25 Mar 2016 15:41:22 -0700 Subject: [PATCH 188/375] Fixed bug in platform names that I introduced yesterday --- lib/spack/spack/compilers/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 55e2db700fe..9601ae3e3ac 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -28,6 +28,7 @@ import imp import os import platform +import copy from llnl.util.lang import memoized, list_modules from llnl.util.filesystem import join_path @@ -77,28 +78,27 @@ def get_compiler_config(arch=None, scope=None): """Return the compiler configuration for the specified architecture. """ # Check whether we're on a front-end (native) architecture. - my_arch = spack.architecture.Arch() - if isinstance(arch, basestring): - if arch == 'all': - my_arch.platform.name = 'all' - arch = my_arch if arch is None: arch = my_arch + if isinstance(arch, basestring) and arch == 'all': + name = 'all' + else: + name = arch.platform.name def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config[arch.platform.name] = {} + config[name] = {} compilers = find_compilers(*get_path('PATH')) for compiler in compilers: - config[arch.platform.name].update(_to_dict(compiler)) + config[name].update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. - if arch == my_arch and arch not in config: + if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: if scope is None: # We know no compilers were configured in any scope. init_compiler_config() @@ -109,7 +109,7 @@ def init_compiler_config(): if not site_config: init_compiler_config() - return config[arch.platform.name] if arch.platform.name in config else {} + return config[name] if name in config else {} def add_compilers_to_config(compilers, arch=None, scope=None): From 57ced372291bfa537e37d325d8e4333623eb306d Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 25 Mar 2016 17:03:49 -0700 Subject: [PATCH 189/375] Fixed a bug in the testing --- lib/spack/spack/test/mock_packages_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 079cbcc1365..8f04076f194 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -42,11 +42,15 @@ cxx: /path/to/clang++ f77: None fc: None + strategy: PATH + modules: None gcc@4.5.0: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran + strategy: PATH + modules: None """ mock_packages_config = """\ From 7e9baf9e2533eb040aa142bd3bf6ffbd142516ac Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 29 Mar 2016 11:23:36 -0700 Subject: [PATCH 190/375] Made architecture test robust for more architectures --- lib/spack/spack/test/architecture.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 75a67bf02f9..2274a9901dc 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -13,32 +13,26 @@ from spack.platforms.darwin import Darwin class ArchitectureTest(unittest.TestCase): - - def test_to_dict_function_with_architecture(self): + + def test_dict_functions_for_architecture(self): arch = Arch() arch.platform_os = arch.platform.operating_system('default_os') arch.target = arch.platform.target('default') d = arch.to_dict() - self.assertEqual(d, {'platform' : 'crayxc', - 'platform_os' : {'name': 'CNL', - 'compiler_strategy' : 'MODULES', - 'version':'10'}, - 'target' : {'name': 'haswell', - 'module_name': 'craype-haswell'}}) - - def test_from_dict_function_with_architecture(self): - d = {'platform':'crayxc', - 'platform_os' : {'name' : 'CNL', 'compiler_strategy': 'MODULES', - 'version': '10'}, - 'target' : {'name':'haswell', 'module_name': 'craype-haswell'}} - - arch = spack.architecture.arch_from_dict(d) + + new_arch = spack.architecture.arch_from_dict(d) + self.assertEqual(arch, new_arch) + self.assertTrue( isinstance(arch, Arch) ) self.assertTrue( isinstance(arch.platform, Platform) ) self.assertTrue( isinstance(arch.platform_os, OperatingSystem) ) self.assertTrue( isinstance(arch.target, Target) ) - + self.assertTrue( isinstance(new_arch, Arch) ) + self.assertTrue( isinstance(new_arch.platform, Platform) ) + self.assertTrue( isinstance(new_arch.platform_os, OperatingSystem) ) + self.assertTrue( isinstance(new_arch.target, Target) ) + def test_platform_class_and_compiler_strategies(self): a = CrayXc() From 9c071ea40d6bacef2056184f7b256c2f1fb7cbd0 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 4 Apr 2016 13:44:24 -0700 Subject: [PATCH 191/375] Changed modulecmd parameters. return_oe is not one of the parameters in the mainline spack repo. Changed the args to the appropriate ones in the new spack --- lib/spack/spack/compiler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e3ea918acad..c285c8a29d5 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -285,16 +285,15 @@ def find_in_path(cls, *path): @classmethod def find_in_modules(cls): compilers = [] - if cls.PrgEnv: if not cls.PrgEnv_compiler: tty.die('Must supply PrgEnv_compiler with PrgEnv') modulecmd = which('modulecmd') modulecmd.add_default_arg('python') - output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True) + + output = modulecmd('avail', cls.PrgEnv_compiler, output=str, error=str) matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output) - for name, version in matches: v = version comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES', From 6d73c862096ecd6426d42dfe666c0908c2069a21 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 4 Apr 2016 13:45:15 -0700 Subject: [PATCH 192/375] Gives error on cori that [PrgEnv-xx, gcc/x.x.x] is not part of the schema. Changed it to oneOf : type : array to get rid of the error. --- lib/spack/spack/config.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index c9a770a53fd..f978ed12b09 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -159,7 +159,9 @@ 'type': 'object', 'additionalProperties': False, 'required': ['cc', 'cxx', 'f77', 'fc'], - 'properties': { + 'properties': { + 'strategy': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, 'cc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'cxx': { 'anyOf': [ {'type' : 'string' }, @@ -168,10 +170,10 @@ {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, - 'strategy': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, - 'modules': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]} + 'modules': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }, + {'type': 'array'}, + ]} },},},},},},},}, 'mirrors': { @@ -221,7 +223,7 @@ 'items' : { 'anyOf' : [ { 'type' : 'string' }, { 'type' : 'number'}]}}, #version strings 'compiler': { - 'type' : 'array', + 'typee' : 'array', 'default' : [], 'items' : { 'type' : 'string' } }, #compiler specs 'nobuild': { From 9bfc83ac5c3ae09cbcb281296d210d17984cd1db Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 4 Apr 2016 15:36:10 -0700 Subject: [PATCH 193/375] Changed so that py_platform.dist() grabs the major version number. --- lib/spack/spack/operating_systems/linux_distro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index 81c7a864304..d0f24d842f9 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -10,6 +10,6 @@ class LinuxDistro(OperatingSystem): """ def __init__(self): name = py_platform.dist()[0] - version = py_platform.dist()[1] + version = py_platform.dist()[1].split(".")[0] # Grabs major version from tuple super(LinuxDistro, self).__init__(name, version) From 3bf75fab3f37bc2edb70bd49060c3831777c21dc Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 12:03:26 -0700 Subject: [PATCH 194/375] Cleaned up architecture. Changed operating_system from dictionary method --- lib/spack/spack/architecture.py | 68 +++++++-------------------------- 1 file changed, 14 insertions(+), 54 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 8d9686cf998..555fb14408e 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -210,8 +210,13 @@ def concrete(self): def __str__(self): + if self.platform.name == 'darwin': + os_name = self.platform_os.name + else: + os_name = str(self.platform_os) + return (str(self.platform) +"-"+ - str(self.platform_os) + "-" + str(self.target) ) + os_name + "-" + str(self.target)) def _cmp_key(self): platform = self.platform.name @@ -231,52 +236,6 @@ def to_dict(self): return d -#def _helper_to_dict(arch_field_dict, arch_field_name, *args): -# """ General method to turn each class in architecture into a -# dictionary. Takes as argument the class dictionary, the field name -# (platform, platform_os, target) and then any attribute args -# """ -# d = {} -# d[arch_field_name] = {} -# for items in args: -# d[arch_field_name][items] = arch_field_dict[items] -# return d -# - -#def to_dict(arch): -# """ Convert the Arch tuple into a dictionary for yaml dumping. This -# uses the _helper_to_dict method to create the dictionary from the -# provided architecture field. Can assign the architecture -# field name (either platform, platform_os or target) and any -# attributes that make up that architecture field, -# """ -# d = {} -# -# platform = arch.platform.__dict__ -# platform_os = arch.platform_os.__dict__ -# target = arch.target.__dict__ -# -# platform_dict = _helper_to_dict(platform,'platform','name') -# os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version', -# 'compiler_strategy') -# target_dict = _helper_to_dict(target,'target', 'name', -# 'module_name','platform_name') -# -# d.update(platform_dict) -# d.update(os_dict) -# d.update(target_dict) -# -# return d - -#def _platform_from_dict(platform): -# """Creates all the platform class module names into a dictionary of -# name : key-value pairs. From there we can construct the -# platform subclass -# """ -# platform_list = all_platforms() -# platform_names = {plat.__name__.lower():plat for plat in platform_list} -# return platform_names[platform['name']]() - def _target_from_dict(target_dict): """ Creates new instance of target and assigns all the attributes of @@ -284,20 +243,21 @@ def _target_from_dict(target_dict): """ target = Target.__new__(Target) target.name = target_dict['name'] - #target.compiler_strategy = target_dict['compiler_strategy'] target.module_name = target_dict['module_name'] if 'platform_name' in target_dict: target.platform_name = target_dict['platform_name'] return target -def _operating_system_from_dict(os_dict, platform_class): +def _operating_system_from_dict(os_dict): """ uses platform's operating system method to grab the constructed operating systems that are valid on the platform. """ # NOTE: Might need a better way to create operating system objects - name = os_dict['name'] - return platform_class.operating_system(name) - + operating_system = OperatingSystem.__new__(OperatingSystem) + operating_system.name = os_dict['name'] + operating_system.version = os_dict['version'] + operating_system.compiler_strategy = os_dict['compiler_strategy'] + return operating_system def arch_from_dict(d): """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict @@ -312,8 +272,8 @@ def arch_from_dict(d): target_dict = d['target'] target = _target_from_dict(target_dict) - platform_os = _operating_system_from_dict(os_dict, arch.platform) - arch.target =target + platform_os = _operating_system_from_dict(os_dict) + arch.target = target arch.platform_os = platform_os return arch From 749508af010ee90009a61bb42a8f8214a6fabd21 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 12:05:29 -0700 Subject: [PATCH 195/375] Found loops calling all_compilers that were always reading in the config file which lead to a slowdown, the fix is to cache the config file so we don't waste time re-reading the config file in again. --- lib/spack/spack/compilers/__init__.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 9601ae3e3ac..caf438e232f 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -151,20 +151,25 @@ def remove_compiler_from_config(compiler_spec, arch=None, scope=None): spack.config.update_config('compilers', update, scope) +_cache_config_file = {} def all_compilers_config(arch=None, scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ # Get compilers for this architecture. - arch_config = get_compiler_config(arch, scope) - # Merge 'all' compilers with arch-specific ones. - # Arch-specific compilers have higher precedence. - merged_config = get_compiler_config('all', scope=scope) - merged_config = spack.config._merge_yaml(merged_config, arch_config) + global _cache_config_file #Create a cache of the config file so we don't load all the time. - return merged_config + if not _cache_config_file: + arch_config = get_compiler_config(arch, scope) + # Merge 'all' compilers with arch-specific ones. + # Arch-specific compilers have higher precedence. + _cache_config_file = get_compiler_config('all', scope=scope) + _cache_config_file = spack.config._merge_yaml(_cache_config_file, arch_config) + return _cache_config_file + else: + return _cache_config_file def all_compilers(arch=None, scope=None): # Return compiler specs from the merged config. From 918ad28a936abf6112ba91ca7eb2e121f84e60cf Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 13:58:29 -0700 Subject: [PATCH 196/375] Changed all target values to architecture. Also added a sort_key method so that older architecture specs can be sorted with the newer ones --- lib/spack/spack/build_environment.py | 90 +--------------------------- lib/spack/spack/cmd/find.py | 18 ++++-- 2 files changed, 16 insertions(+), 92 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 8ed3c4e1a11..87fc310b5a3 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -32,8 +32,6 @@ import shutil import multiprocessing import platform -import re - from llnl.util.filesystem import * import spack @@ -59,9 +57,6 @@ SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC' SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR' -SPACK_CRAYPE = 'SPACK_CRAYPE' -SPACK_COMP_MODULE = 'SPACK_COMP_MODULE' - class MakeExecutable(Executable): """Special callable executable object for make so the user can @@ -88,68 +83,6 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) -def load_module(mod): - """Takes a module name and removes modules until it is possible to - load that module. It then loads the provided module. Depends on the - modulecmd implementation of modules used in cray and lmod. - """ - #Create an executable of the module command that will output python code - modulecmd = which('modulecmd') - modulecmd.add_default_arg('python') - - # Read the module and remove any conflicting modules - # We do this without checking that they are already installed - # for ease of programming because unloading a module that is not - # loaded does nothing. - text = modulecmd('show', mod, output=str, error=str).split() - for i, word in enumerate(text): - if word == 'conflict': - exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) - # Load the module now that there are no conflicts - load = modulecmd('load', mod, output=str, error=str) - exec(compile(load, '', 'exec')) - - -def get_path_from_module(mod): - """Inspects a TCL module for entries that indicate the absolute path - at which the library supported by said module can be found. - """ - # Create a modulecmd executable - modulecmd = which('modulecmd') - modulecmd.add_default_arg('python') - - # Read the module - text = modulecmd('show', mod, output=str, error=str).split('\n') - - # If it lists its package directory, return that - for line in text: - if line.find(mod.upper()+'_DIR') >= 0: - words = line.split() - return words[2] - - # If it lists a -rpath instruction, use that - for line in text: - rpath = line.find('-rpath/') - if rpath >= 0: - return line[rpath+6:line.find('/lib')] - - # If it lists a -L instruction, use that - for line in text: - L = line.find('-L/') - if L >= 0: - return line[L+2:line.find('/lib')] - - # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that - for line in text: - if line.find('LD_LIBRARY_PATH') >= 0: - words = line.split() - path = words[2] - return path[:path.find('/lib')] - - # Unable to find module path - return None - - def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -176,10 +109,6 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.strategy == 'MODULES': - for mod in compiler.modules: - load_module(mod) - def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. @@ -243,8 +172,6 @@ def add_env_path(path): pkg_config_dirs.append(pcdir) path_set("PKG_CONFIG_PATH", pkg_config_dirs) - if pkg.spec.architecture.target.module_name: - load_module(pkg.spec.architecture.target.module_name) def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. @@ -314,21 +241,10 @@ def set_module_variables_for_package(pkg, m): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" - - # First load all modules for external packages and update the external - # packages' paths to reflect what is found in the modules so that we can - # rpath through the modules when possible, but if not possible they are - # already loaded. - for spec in pkg.spec.traverse(root=False): - if spec.external_module: - load_module(spec.external_module) - spec.external = get_path_from_module(spec.external_module) - - # Construct rpaths from the paths of each dep rpaths = [pkg.prefix.lib, pkg.prefix.lib64] - rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) - rpaths.extend(d.prefix.lib64 for d in pkg.spec.traverse(root=False) + rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) return rpaths @@ -348,8 +264,8 @@ def parent_class_modules(cls): def setup_package(pkg): """Execute all environment setup routines.""" - set_build_environment_variables(pkg) set_compiler_environment_variables(pkg) + set_build_environment_variables(pkg) # If a user makes their own package repo, e.g. # spack.repos.mystuff.libelf.Libelf, and they inherit from diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 4223c53b064..95f035c7bf9 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -90,18 +90,26 @@ def display_specs(specs, **kwargs): hlen = None # Make a dict with specs keyed by target and compiler. - index = index_by(specs, ('target', 'compiler')) + index = index_by(specs, ('architecture', 'compiler')) - # Traverse the index and print out each package - for i, (target, compiler) in enumerate(sorted(index)): + # Create sort key based off of whether read architecture is string + def sort_key(index_dict): + if isinstance(index_dict[0],basestring): + return(str, index_dict[1]._cmp_key()) + elif isinstance(index_dict[0], spack.architecture.Arch): + return (index_dict[0]._cmp_key(), index_dict[1]._cmp_key()) + + sorted_index = sorted(index, key=sort_key) + + for i, (architecture, compiler) in enumerate(sorted_index): if i > 0: print header = "%s{%s} / %s{%s}" % ( - spack.spec.target_color, target, + spack.spec.architecture_color, architecture, spack.spec.compiler_color, compiler) tty.hline(colorize(header), char='-') - specs = index[(target,compiler)] + specs = index[(architecture, compiler)] specs.sort() nfmt = '.' if namespace else '_' From 99117cbc9067df814c99f8427f658ad9a811f69b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 6 Apr 2016 14:07:22 -0700 Subject: [PATCH 197/375] pulled from current repo the build_environment.py. Added module loading methods from our cray port --- lib/spack/spack/build_environment.py | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 87fc310b5a3..12a529d070a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -83,6 +83,66 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) +def load_module(mod): + """Takes a module name and removes modules until it is possible to + load that module. It then loads the provided module. Depends on the + modulecmd implementation of modules used in cray and lmod. + """ + #Create an executable of the module command that will output python code + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module and remove any conflicting modules + # We do this without checking that they are already installed + # for ease of programming because unloading a module that is not + # loaded does nothing. + text = modulecmd('show', mod, output=str, error=str).split() + for i, word in enumerate(text): + if word == 'conflict': + exec(compile(modulecmd('unload', text[i+1], output=str, error=str), '', 'exec')) + # Load the module now that there are no conflicts + load = modulecmd('load', mod, output=str, error=str) + exec(compile(load, '', 'exec')) + +def get_path_from_module(mod): + """Inspects a TCL module for entries that indicate the absolute path + at which the library supported by said module can be found. + """ + # Create a modulecmd executable + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Read the module + text = modulecmd('show', mod, output=str, error=str).split('\n') + + # If it lists its package directory, return that + for line in text: + if line.find(mod.upper()+'_DIR') >= 0: + words = line.split() + return words[2] + + # If it lists a -rpath instruction, use that + for line in text: + rpath = line.find('-rpath/') + if rpath >= 0: + return line[rpath+6:line.find('/lib')] + + # If it lists a -L instruction, use that + for line in text: + L = line.find('-L/') + if L >= 0: + return line[L+2:line.find('/lib')] + + # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that + for line in text: + if line.find('LD_LIBRARY_PATH') >= 0: + words = line.split() + path = words[2] + return path[:path.find('/lib')] + + # Unable to find module path + return None + def set_compiler_environment_variables(pkg): assert(pkg.spec.concrete) compiler = pkg.compiler @@ -109,6 +169,9 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) + if compiler.strategy == 'MODULES': + for mod in compiler.modules: + load_module(mod) def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. @@ -172,6 +235,9 @@ def add_env_path(path): pkg_config_dirs.append(pcdir) path_set("PKG_CONFIG_PATH", pkg_config_dirs) + if pkg.spec.architecture.target.module_name: + load_module(pkg.spec.architecture.target.module_name) + def set_module_variables_for_package(pkg, m): """Populate the module scope of install() with some useful functions. @@ -241,6 +307,11 @@ def set_module_variables_for_package(pkg, m): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" + for spec in pkg.spec.traverse(root=False): + if spec.external_module: + load_module(spec.external_module) + spec.external = get_path_from_module(spec.external_module) + rpaths = [pkg.prefix.lib, pkg.prefix.lib64] rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) From 143a4d7f7688b506811a564768ed49a2a12d1992 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:36:26 -0700 Subject: [PATCH 198/375] Changed default to default_target, also added frontend and backend to operating system --- lib/spack/spack/architecture.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 555fb14408e..ca306b04847 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -118,14 +118,14 @@ def target(self, name): front-end, and back-end. This can be overwritten by a subclass for which we want to provide further aliasing options. """ - if name == 'default': + if name == 'default_target': name = self.default - elif name == 'front_end' or name == 'fe': + elif name == 'frontend' or name == 'fe': name = self.front_end - elif name == 'back_end' or name == 'be': + elif name == 'backend' or name == 'be': name = self.back_end - return self.targets[name] + return self.targets.get(name, None) def add_operating_system(self, name, os_class): """ Add the operating_system class object into the @@ -136,12 +136,12 @@ def add_operating_system(self, name, os_class): def operating_system(self, name): if name == 'default_os': name = self.default_os - if name == 'front_os': + if name == 'frontend' or name == "fe": name = self.front_os - if name == 'back_os': + if name == 'backend' or name == 'be': name = self.back_os - return self.operating_sys[name] + return self.operating_sys.get(name, None) @classmethod def detect(self): From 07df40320331c118306c40c36deae8d57ca6073c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:37:00 -0700 Subject: [PATCH 199/375] Cleaned up the concretize_architecture method by removing commented out code --- lib/spack/spack/concretize.py | 62 +++-------------------------------- 1 file changed, 5 insertions(+), 57 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index c5dad93aefd..14edcf1f3db 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -236,14 +236,6 @@ def _concretize_operating_system(self, spec): return True #changed -# """ Future method for concretizing operating system """ -# if isinstance(arch.platform_os, spack.architecture.OperatingSystem): -# return False -# else: -# arch.arch_os = platform.operating_system('default_os') -# return True - - def _concretize_target(self, spec): if spec.architecture.target is not None: if isinstance(spec.architecture.target,spack.architecture.Target): @@ -258,16 +250,10 @@ def _concretize_target(self, spec): else: spec.add_target_from_string(spec.root.architecture.target) else: - spec.architecture.target = spec.architecture.platform.target('default') + spec.architecture.target = spec.architecture.platform.target('default_target') return True #changed -# if isinstance(arch.target, spack.architecture.Target): -# return False -# else: -# arch.target = platform.target('default') -# return True - def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the architecture is not a basestring, then check if either the platform, @@ -283,50 +269,12 @@ def concretize_architecture(self, spec): # Set the architecture to all defaults spec.architecture = spack.architecture.Arch() return True - #If there is a target and it is a tuple and has both filled return - #False -# if isinstance(spec.architecture, basestring): -# spec.split_architecture_string(spec.architecture) - - ret = any(( - self._concretize_operating_system(spec), - self._concretize_target(spec))) - - - # Does not look pretty at all!!! -# if spec.root.architecture and \ -# not isinstance(spec.root.architecture, basestring): -# bool_flag = any(( -# self._concretize_platform(spec.root.architecture, platform), -# self._concretize_operating_system(spec.root.architecture, -# platform), -# self._concretize_target(spec.root.target, platform))) -# spec.architecture =spec.root.architecture -# return bool_flag -# else: -# spec.add_architecture_from_string(spec.root.architecture) - + + # Concretize the operating_system and target based of the spec + ret = any((self._concretize_operating_system(spec), + self._concretize_target(spec))) return ret - # if there is no target specified used the defaults - - #if spec.target is not None: - # if isinstance(spec.target,spack.architecture.Target): - # return False - # else: - # spec.add_target_from_string(spec.target) - # return True #changed - - #if spec.root.target: - # if isinstance(spec.root.target,spack.architecture.Target): - # spec.target = spec.root.target - # else: - # spec.add_target_from_string(spec.root.target) - #else: - # platform = spack.architecture.sys_type() - # spec.target = platform.target('default') - - #return True #changed def concretize_variants(self, spec): From 0ad317213c0f32c125c7043b5e57aa678ba63f8f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:38:04 -0700 Subject: [PATCH 200/375] Changed architecture parser so that if user just enters in frontend or backend, then both the os and target will take those names. In the concretize method the frontend target/os and backend target/os will be picked to match each other --- lib/spack/spack/spec.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fa2e9fd4c82..94d881da28b 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -462,11 +462,10 @@ def _set_architecture(self, architecture): """Called by the parser to set the architecture.""" if self.architecture: raise DuplicateArchitectureError( "Spec for '%s' cannot have two architectures." % self.name) - platform = spack.architecture.sys_type() if '-' in architecture: os, target = architecture.split('-') - else: - os = None + elif architecture == 'frontend' or architecture == 'backend': + os = architecture target = architecture self.architecture = spack.architecture.Arch(os, target) From 7b777568ed27f2dc0f15283d0209541b65b47b24 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 7 Apr 2016 14:38:43 -0700 Subject: [PATCH 201/375] Added new concretization of architecture tests, test each combination of user input to make sure the correct os and target are chosen --- lib/spack/spack/test/architecture.py | 45 ++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 2274a9901dc..1df1e3ffa29 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -6,7 +6,7 @@ import platform import spack from spack.architecture import * -import spack.spec +from spack.spec import * from spack.platforms.cray_xc import CrayXc from spack.platforms.linux import Linux from spack.platforms.bgq import Bgq @@ -17,7 +17,7 @@ class ArchitectureTest(unittest.TestCase): def test_dict_functions_for_architecture(self): arch = Arch() arch.platform_os = arch.platform.operating_system('default_os') - arch.target = arch.platform.target('default') + arch.target = arch.platform.target('default_target') d = arch.to_dict() @@ -55,3 +55,44 @@ def test_sys_type(self): my_platform_class = Darwin() self.assertEqual(str(output_platform_class), str(my_platform_class)) + + def setUp(self): + self.platform = sys_type() + + def test_user_front_end_input(self): + """Test when user inputs just frontend that both the frontend target + and frontend operating system match + """ + frontend_os = self.platform.operating_system("frontend") + frontend_target = self.platform.target("frontend") + print frontend_os + print frontend_target + frontend_spec = Spec("zlib=frontend") + frontend_spec.concretize() + self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) + self.assertEqual(frontend_target, frontend_spec.architecture.target) + + def test_user_back_end_input(self): + """Test when user inputs backend that both the backend target and + backend operating system match + """ + backend_os = self.platform.operating_system("backend") + backend_target = self.platform.target("backend") + print backend_os + print backend_target + backend_spec = Spec("zlib=backend") + backend_spec.concretize() + self.assertEqual(backend_os, backend_spec.architecture.platform_os) + self.assertEqual(backend_target, backend_spec.architecture.target) + + def test_user_defaults(self): + default_os = self.platform.operating_system("default_os") + default_target = self.platform.target("default_target") + + default_spec = Spec("zlib") # default is no args + default_spec.concretize() + self.assertEqual(default_os, default_spec.architecture.platform_os) + self.assertEqual(default_target, default_spec.architecture.target) + + def test_user_combination(self): + From 6ff6c805af2a4626e0611f3798eedd3a6f1b3e06 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 10:28:30 -0700 Subject: [PATCH 202/375] Assigned self.front_os, and back_os to self.default. Helps with testing. If we're on a darwin or linux machine, chances are there is no back end or front end operating system, but rather than have those as None, I just assign them the default which is detected via python platform. --- lib/spack/spack/platforms/darwin.py | 4 ++++ lib/spack/spack/platforms/linux.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 4c3d38851f8..7901f82d06a 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -12,7 +12,11 @@ def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default)) mac_os = MacOsx() + self.default_os = mac_os.name + self.front_os = mac_os.name + self.back_os = mac_os.name + self.add_operating_system(mac_os.name, mac_os) @classmethod diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 3243a1dcdf0..18050ac79e1 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -13,6 +13,8 @@ def __init__(self): self.add_target(self.default, Target(self.default)) linux_dist = LinuxDistro() self.default_os = linux_dist.name + self.front_os = linux_dist.name + self.back_os = linux_dist.name self.add_operating_system(linux_dist.name, linux_dist) @classmethod From 0d1a1b7526ce1c129bed352b175f33f2875b8b41 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 10:29:37 -0700 Subject: [PATCH 203/375] Changed _set_architecture so if user inputs only the target, then os is None. This prevents the os being referenced before assignment error --- lib/spack/spack/spec.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 94d881da28b..28cd6a02747 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -461,12 +461,17 @@ def _set_compiler(self, compiler): def _set_architecture(self, architecture): """Called by the parser to set the architecture.""" if self.architecture: raise DuplicateArchitectureError( - "Spec for '%s' cannot have two architectures." % self.name) + "Spec for '%s' cannot have two architectures." % self.name) if '-' in architecture: os, target = architecture.split('-') + elif architecture == 'frontend' or architecture == 'backend': os = architecture target = architecture + else: + os = None + target = architecture + self.architecture = spack.architecture.Arch(os, target) From 277efc1dfbb00117db65918ff22928d42e1194e2 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 11:17:56 -0700 Subject: [PATCH 204/375] Added test that works on every type of combination the user could possibly enter for arch spec --- lib/spack/spack/test/architecture.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 1df1e3ffa29..1ca3e58831d 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -65,8 +65,6 @@ def test_user_front_end_input(self): """ frontend_os = self.platform.operating_system("frontend") frontend_target = self.platform.target("frontend") - print frontend_os - print frontend_target frontend_spec = Spec("zlib=frontend") frontend_spec.concretize() self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) @@ -78,8 +76,6 @@ def test_user_back_end_input(self): """ backend_os = self.platform.operating_system("backend") backend_target = self.platform.target("backend") - print backend_os - print backend_target backend_spec = Spec("zlib=backend") backend_spec.concretize() self.assertEqual(backend_os, backend_spec.architecture.platform_os) @@ -94,5 +90,24 @@ def test_user_defaults(self): self.assertEqual(default_os, default_spec.architecture.platform_os) self.assertEqual(default_target, default_spec.architecture.target) - def test_user_combination(self): + def test_user_input_combination(self): + os_list = self.platform.operating_sys.keys() + target_list = self.platform.targets.keys() + additional = ["fe", "be", "frontend", "backend"] + + os_list.extend(additional) + target_list.extend(additional) + combinations = itertools.product(os_list, target_list) + results = [] + for arch in combinations: + o,t = arch + arch_spec = "-".join(arch) + spec = Spec("zlib=%s" % arch_spec) + spec.concretize() + results.append(spec.architecture.platform_os == self.platform.operating_system(o)) + results.append(spec.architecture.target == self.platform.target(t)) + res = all(results) + print res + self.assertTrue(res) + From 618b3f5f2d149fe3c20f90c0890748c8f3589afb Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 11:18:30 -0700 Subject: [PATCH 205/375] Changed default to default_target so the tests can pass --- lib/spack/spack/test/spec_semantics.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 0b310488a90..84d8c0edd8c 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -308,7 +308,7 @@ def test_constrain_variants(self): def test_constrain_target(self): platform = spack.architecture.sys_type() - target = platform.target('default').name + target = platform.target('default_target').name self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) @@ -339,7 +339,7 @@ def test_constrain_changed(self): self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') platform = spack.architecture.sys_type() - self.check_constrain_changed('libelf', '='+platform.target('default').name) + self.check_constrain_changed('libelf', '='+platform.target('default_target').name) def test_constrain_not_changed(self): @@ -351,8 +351,8 @@ def test_constrain_not_changed(self): self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') platform = spack.architecture.sys_type() - default = platform.target('default').name - self.check_constrain_not_changed('libelf='+default, '='+default) + default_target = platform.target('default_target').name + self.check_constrain_not_changed('libelf='+default_target, '='+default_target) self.check_constrain_not_changed('libelf^foo', 'libelf^foo') self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') @@ -365,8 +365,8 @@ def test_constrain_dependency_changed(self): self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') platform = spack.architecture.sys_type() - default = platform.target('default').name - self.check_constrain_changed('libelf^foo', 'libelf^foo='+default) + default_target = platform.target('default_target').name + self.check_constrain_changed('libelf^foo', 'libelf^foo='+default_target) def test_constrain_dependency_not_changed(self): @@ -377,6 +377,6 @@ def test_constrain_dependency_not_changed(self): self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') platform = spack.architecture.sys_type() - default = platform.target('default').name - self.check_constrain_not_changed('libelf^foo='+default, 'libelf^foo='+default) + default_target = platform.target('default_target').name + self.check_constrain_not_changed('libelf^foo='+default_target, 'libelf^foo='+default_target) From 5bcd1e7ccd9ff6568c06a13998f2d552d1b37b60 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 12:43:08 -0700 Subject: [PATCH 206/375] If we already have the name and the platform, then we should probably go ahead and concretize it in the constructor. Else leave it as None and concretize it later. --- lib/spack/spack/architecture.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index ca306b04847..e2c189980a5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -199,8 +199,13 @@ class Arch(object): def __init__(self, platform_os=None, target=None): self.platform = sys_type() + if platform_os: + platform_os = self.platform.operating_system(platform_os) self.platform_os = platform_os - self.target = target + if target: + target = self.platform.target(target) + self.target = target + @property def concrete(self): From 01a36ab33341e2af91c6e6f5ca58efbeecfb6ec4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 12:44:27 -0700 Subject: [PATCH 207/375] Changed the logic of concretize. We don't have to worry about whether the part of the spec is a string since any string parsed is made into an object via the Arch constructor. Any dependencies will take the root arch spec as well --- lib/spack/spack/concretize.py | 32 +++++++++++--------------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 14edcf1f3db..3422e86ae83 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -220,39 +220,29 @@ def concretize_version(self, spec): return True # Things changed def _concretize_operating_system(self, spec): - if spec.architecture.platform_os is not None: - if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem): - return False - else: - spec.add_operating_system_from_string(spec.architecture.platform_os) - return True #changed + platform = spec.architecture.platform + if spec.architecture.platform_os is not None and isinstance( + spec.architecture.platform_os,spack.architecture.OperatingSystem): + return False + if spec.root.architecture and spec.root.architecture.platform_os: if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): spec.architecture.platform_os = spec.root.architecture.platform_os - else: - spec.add_operating_system_from_string(spec.root.architecture.platform_os) else: spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') - - return True #changed + return True #changed def _concretize_target(self, spec): - if spec.architecture.target is not None: - if isinstance(spec.architecture.target,spack.architecture.Target): - return False - else: - spec.add_target_from_string(spec.architecture.target) - return True #changed - + platform = spec.architecture.platform + if spec.architecture.target is not None and isinstance( + spec.architecture.target, spack.architecture.Target): + return False if spec.root.architecture and spec.root.architecture.target: if isinstance(spec.root.architecture.target,spack.architecture.Target): spec.architecture.target = spec.root.architecture.target - else: - spec.add_target_from_string(spec.root.architecture.target) else: spec.architecture.target = spec.architecture.platform.target('default_target') - - return True #changed + return True #changed def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the From 3a281b239998133c7ed3fe03a54e44d643d0d0bd Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 8 Apr 2016 12:46:44 -0700 Subject: [PATCH 208/375] Commented out the add_x_from_string methods since we don't need to worry about it. Also, changed the way architecture is set. Todd wanted to have users be able to input just frontend and/or backend in the architecture spec. This will be able to parse those inputs. --- lib/spack/spack/spec.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 28cd6a02747..eee93185503 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -465,7 +465,7 @@ def _set_architecture(self, architecture): if '-' in architecture: os, target = architecture.split('-') - elif architecture == 'frontend' or architecture == 'backend': + elif architecture in ['frontend','backend','fe','be']: os = architecture target = architecture else: @@ -1302,28 +1302,17 @@ def _autospec(self, spec_like): except SpecError: return parse_anonymous_spec(spec_like, self.name) - def _is_valid_platform(self, platform, platform_list): - if platform in platform_list: - return True - return False + #def add_target_from_string(self, target): + # if target is None: + # self.architecture.target = self.architecture.platform.target('default_target') + # else: + # self.architecture.target = self.architecture.platform.target(target) - def _is_valid_target(self, target, platform): - return target in platform.targets - - def _is_valid_os(self, os_string, platform): - return os_string in platform.operating_sys - - def add_target_from_string(self, target): - if target is None: - self.architecture.target = self.architecture.platform.target('default_target') - else: - self.architecture.target = self.architecture.platform.target(target) - - def add_operating_system_from_string(self, os): - if os is None: - self.architecture.platform_os = self.architecture.platform.operating_system('default_os') - else: - self.architecture.platform_os = self.architecture.platform.operating_system(os) + #def add_operating_system_from_string(self, os): + # if os is None: + # self.architecture.platform_os = self.architecture.platform.operating_system('default_os') + # else: + # self.architecture.platform_os = self.architecture.platform.operating_system(os) def satisfies(self, other, deps=True, strict=False): From 49956faab97e2dea52371755d0fcb9b013567ac7 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Wed, 24 Feb 2016 12:19:06 +0100 Subject: [PATCH 209/375] First try to make file system views of specs. --- lib/spack/spack/cmd/view.py | 108 ++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 lib/spack/spack/cmd/view.py diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py new file mode 100644 index 00000000000..c7f1c21f67e --- /dev/null +++ b/lib/spack/spack/cmd/view.py @@ -0,0 +1,108 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import os +import argparse + +import spack +import spack.cmd +import llnl.util.tty as tty + +description = "Produce a single-rooted directory view of a spec." + +def setup_parser(subparser): + setup_parser.parser = subparser + + subparser.add_argument('prefix', nargs=1, + help="Path to top-level directory for the view.") + subparser.add_argument('specs', nargs=argparse.REMAINDER, + help="specs of packages to expose in the view.") + +def assuredir(path): + 'Assure path exists as a directory' + if not os.path.exists(path): + os.makedirs(path) + +def relative_to(prefix, path): + assert 0 == path.find(prefix) + reldir = path[len(prefix):] + if reldir.startswith('/'): + reldir = reldir[1:] + return reldir + +def view_one(prefix, spec): + print spec.name,spec.prefix + + dotspack = os.path.join(prefix, '.spack', spec.name) + + if os.path.exists(os.path.join(dotspack)): + tty.warn("Skipping previously added package %s"%spec.name) + return + + + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: + continue # avoid explicitly making empty dirs + + reldir = relative_to(spec.prefix, dirpath) + + # fixme: assumes .spack has no subdirs + if dirpath.endswith('.spack'): + targdir = dotspack + else: + targdir = os.path.join(prefix, reldir) + + assuredir(targdir) + + print '\t%s...'%reldir, + nlinks = 0 + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if os.path.exists(dst): + tty.warn("Skipping existing file for view: %s" % dst) + continue + os.symlink(src,dst) + nlinks += 1 + print nlinks + + +def view(parser, args): + # if not args.specs: + # tty.die("view creation requires at least one spec argument") + + specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) + if not specs: + setup_parser.parser.print_help() + return 1 + + prefix = args.prefix[0] + assuredir(prefix) + + flat = set() + for spec in specs: + flat.update(spec.normalized().traverse()) + + for spec in flat: + view_one(prefix, spec) From 8cd13d4b3552e6737ed9fc768bbe544024e05b10 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 28 Apr 2016 12:49:21 -0700 Subject: [PATCH 210/375] Make arch command print out the platform. --- lib/spack/spack/cmd/arch.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/spack/spack/cmd/arch.py b/lib/spack/spack/cmd/arch.py index db27544ffd1..cd0ce2bb2e6 100644 --- a/lib/spack/spack/cmd/arch.py +++ b/lib/spack/spack/cmd/arch.py @@ -28,8 +28,4 @@ description = "Print the architecture for this machine" def arch(parser, args): - configured_sys_type = architecture.get_sys_type_from_spack_globals() - if not configured_sys_type: - configured_sys_type = "autodetect" - print "Configured sys_type: %s" % configured_sys_type - print "Autodetected default sys_type: %s" % architecture.sys_type() + print architecture.sys_type() From a2b9a000dcb18868ea9f1c110632a251fbea92f3 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Fri, 29 Apr 2016 15:08:54 -0400 Subject: [PATCH 211/375] Add add removal and status actions in addition to link and add various ways to filter what is done. --- lib/spack/spack/cmd/view.py | 130 +++++++++++++++++++++++++++++------- 1 file changed, 107 insertions(+), 23 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index c7f1c21f67e..dd698350b21 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -1,3 +1,8 @@ +''' +Produce a file-system "view" of a Spack DAG. + +Concept from Nix, implemented by brett.viren@gmail.com ca 2016. +''' ############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -22,7 +27,9 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## + import os +import re import argparse import spack @@ -34,8 +41,15 @@ def setup_parser(subparser): setup_parser.parser = subparser - subparser.add_argument('prefix', nargs=1, - help="Path to top-level directory for the view.") + subparser.add_argument('-e','--exclude', action='append', default=[], + help="exclude any packages which the given re pattern") + subparser.add_argument('-a','--action', default='link', + choices=['link','remove','status'], # names match action_*() functions below + help="what action to perform on the view") + subparser.add_argument('--no-dependencies', action='store_true', default=False, + help="just operate on named packages and do not follow dependencies") + subparser.add_argument('-p','--prefix', required=True, + help="Path to a top-level directory to receive the view.") subparser.add_argument('specs', nargs=argparse.REMAINDER, help="specs of packages to expose in the view.") @@ -45,64 +59,134 @@ def assuredir(path): os.makedirs(path) def relative_to(prefix, path): + 'Return end of `path` relative to `prefix`' assert 0 == path.find(prefix) reldir = path[len(prefix):] if reldir.startswith('/'): reldir = reldir[1:] return reldir -def view_one(prefix, spec): - print spec.name,spec.prefix + +def transform_path(spec, path, prefix=None): + 'Return the a relative path corresponding to given path spec.prefix' + if os.path.isabs(path): + path = relative_to(spec.prefix, path) + subdirs = path.split(os.path.sep) + if subdirs[0] == '.spack': + lst = ['.spack',spec.name]+subdirs[1:] + path = os.path.join(*lst) + if prefix: + path = os.path.join(prefix, path) + return path +def action_status(spec, prefix): + 'Check status of view in prefix against spec' dotspack = os.path.join(prefix, '.spack', spec.name) - if os.path.exists(os.path.join(dotspack)): - tty.warn("Skipping previously added package %s"%spec.name) + tty.info("Package added: %s"%spec.name) + return + tty.info("Package missing: %s"%spec.name) + return + +def action_remove(spec, prefix): + 'Remove any files found in spec from prefix and purge empty directories.' + + if not os.path.exists(prefix): return + dotspack = transform_path(spec, '.spack', prefix) + if not os.path.exists(dotspack): + tty.info("Skipping nonexistent package %s"%spec.name) + return + + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: + continue + + targdir = transform_path(spec, dirpath, prefix) + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if not os.path.exists(dst): + #tty.warn("Skipping nonexistent file for view: %s" % dst) + continue + os.unlink(dst) + + +def action_link(spec, prefix): + 'Symlink all files in `spec` into directory `prefix`.' + + dotspack = transform_path(spec, '.spack', prefix) + if os.path.exists(dotspack): + tty.warn("Skipping previously added package %s"%spec.name) + return for dirpath,dirnames,filenames in os.walk(spec.prefix): if not filenames: continue # avoid explicitly making empty dirs - reldir = relative_to(spec.prefix, dirpath) - - # fixme: assumes .spack has no subdirs - if dirpath.endswith('.spack'): - targdir = dotspack - else: - targdir = os.path.join(prefix, reldir) - + targdir = transform_path(spec, dirpath, prefix) assuredir(targdir) - print '\t%s...'%reldir, - nlinks = 0 for fname in filenames: src = os.path.join(dirpath, fname) dst = os.path.join(targdir, fname) if os.path.exists(dst): + if '.spack' in dst.split(os.path.sep): + continue # silence these tty.warn("Skipping existing file for view: %s" % dst) continue os.symlink(src,dst) - nlinks += 1 - print nlinks - + + + +def purge_empty_directories(path): + 'Ascend up from the leaves accessible from `path` and remove empty directories.' + for dirpath, subdirs, files in os.walk(path, topdown=False): + for sd in subdirs: + sdp = os.path.join(dirpath,sd) + try: + os.rmdir(sdp) + except OSError: + tty.warn("Not removing directory with contents: %s" % sdp) + def view(parser, args): - # if not args.specs: - # tty.die("view creation requires at least one spec argument") + 'The view command.' + to_exclude = [re.compile(e) for e in args.exclude] + def exclude(spec): + for e in to_exclude: + if e.match(spec.name): + return True + return False specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) if not specs: setup_parser.parser.print_help() return 1 - prefix = args.prefix[0] + prefix = args.prefix assuredir(prefix) flat = set() for spec in specs: + if args.no_dependencies: + flat.add(spec) + continue flat.update(spec.normalized().traverse()) + action = args.action.lower() + method = eval('action_' + action) + for spec in flat: - view_one(prefix, spec) + if exclude(spec): + tty.info('Skipping excluded package: "%s"' % spec.name) + continue + if not os.path.exists(spec.prefix): + tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) + continue + tty.info("%s %s" % (action, spec.name)) + method(spec, prefix) + + if action in ['remove']: + purge_empty_directories(prefix) From 8c140f4725608510d0d7ecc6195736ca513b02ab Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 30 Apr 2016 08:46:47 -0400 Subject: [PATCH 212/375] Change the action from parameter to subcommand to fit Spack convention. --- lib/spack/spack/cmd/view.py | 58 ++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index dd698350b21..a82c0343989 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -41,17 +41,30 @@ def setup_parser(subparser): setup_parser.parser = subparser - subparser.add_argument('-e','--exclude', action='append', default=[], - help="exclude any packages which the given re pattern") - subparser.add_argument('-a','--action', default='link', - choices=['link','remove','status'], # names match action_*() functions below - help="what action to perform on the view") - subparser.add_argument('--no-dependencies', action='store_true', default=False, - help="just operate on named packages and do not follow dependencies") - subparser.add_argument('-p','--prefix', required=True, - help="Path to a top-level directory to receive the view.") - subparser.add_argument('specs', nargs=argparse.REMAINDER, - help="specs of packages to expose in the view.") + sp = subparser.add_subparsers(metavar='ACTION', dest='action') + + # The action parameterizes the command but in keeping with Spack + # patterns we make it a subcommand. + sps = [ + sp.add_parser('link', aliases=['add'], + help='Add packages to the view, create view if needed.'), + sp.add_parser('remove', aliases=['rm'], + help='Remove packages from the view, and view if empty.'), + sp.add_parser('status', aliases=['check'], + help='Check status of packages in the view.') + ] + + # All these options and arguments are common to every action. + for p in sps: + p.add_argument('-e','--exclude', action='append', default=[], + help="exclude any packages which the given re pattern") + p.add_argument('--no-dependencies', action='store_true', default=False, + help="just operate on named packages and do not follow dependencies") + p.add_argument('prefix', nargs=1, + help="Path to a top-level directory to receive the view.") + p.add_argument('specs', nargs=argparse.REMAINDER, + help="specs of packages to expose in the view.") + def assuredir(path): 'Assure path exists as a directory' @@ -151,7 +164,9 @@ def purge_empty_directories(path): tty.warn("Not removing directory with contents: %s" % sdp) -def view(parser, args): + + +def view_action(action, parser, args): 'The view command.' to_exclude = [re.compile(e) for e in args.exclude] def exclude(spec): @@ -162,7 +177,7 @@ def exclude(spec): specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) if not specs: - setup_parser.parser.print_help() + parser.print_help() return 1 prefix = args.prefix @@ -175,9 +190,6 @@ def exclude(spec): continue flat.update(spec.normalized().traverse()) - action = args.action.lower() - method = eval('action_' + action) - for spec in flat: if exclude(spec): tty.info('Skipping excluded package: "%s"' % spec.name) @@ -186,7 +198,19 @@ def exclude(spec): tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) continue tty.info("%s %s" % (action, spec.name)) - method(spec, prefix) + action(spec, prefix) if action in ['remove']: purge_empty_directories(prefix) + + +def view(parser, args): + action = { + 'add': action_link, + 'link': action_link, + 'remove': action_remove, + 'rm': action_remove, + 'status': action_status, + 'check': action_status + }[args.action] + view_action(action, parser, args) From 346f1022343d608ffabd565636ec8358c90464c8 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sun, 1 May 2016 13:54:07 -0400 Subject: [PATCH 213/375] Bug fixes, tty tweaks. --- lib/spack/spack/cmd/view.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index a82c0343989..1bdaaecadc3 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -46,7 +46,7 @@ def setup_parser(subparser): # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. sps = [ - sp.add_parser('link', aliases=['add'], + sp.add_parser('add', aliases=['link'], help='Add packages to the view, create view if needed.'), sp.add_parser('remove', aliases=['rm'], help='Remove packages from the view, and view if empty.'), @@ -112,6 +112,7 @@ def action_remove(spec, prefix): tty.info("Skipping nonexistent package %s"%spec.name) return + tty.info("remove %s"%spec.name) for dirpath,dirnames,filenames in os.walk(spec.prefix): if not filenames: continue @@ -125,7 +126,6 @@ def action_remove(spec, prefix): continue os.unlink(dst) - def action_link(spec, prefix): 'Symlink all files in `spec` into directory `prefix`.' @@ -134,6 +134,7 @@ def action_link(spec, prefix): tty.warn("Skipping previously added package %s"%spec.name) return + tty.info("link %s" % spec.name) for dirpath,dirnames,filenames in os.walk(spec.prefix): if not filenames: continue # avoid explicitly making empty dirs @@ -151,8 +152,6 @@ def action_link(spec, prefix): continue os.symlink(src,dst) - - def purge_empty_directories(path): 'Ascend up from the leaves accessible from `path` and remove empty directories.' for dirpath, subdirs, files in os.walk(path, topdown=False): @@ -161,13 +160,14 @@ def purge_empty_directories(path): try: os.rmdir(sdp) except OSError: - tty.warn("Not removing directory with contents: %s" % sdp) + #tty.warn("Not removing directory with contents: %s" % sdp) + pass def view_action(action, parser, args): - 'The view command.' + 'The view command parameterized by the action.' to_exclude = [re.compile(e) for e in args.exclude] def exclude(spec): for e in to_exclude: @@ -180,7 +180,7 @@ def exclude(spec): parser.print_help() return 1 - prefix = args.prefix + prefix = args.prefix[0] assuredir(prefix) flat = set() @@ -197,14 +197,15 @@ def exclude(spec): if not os.path.exists(spec.prefix): tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) continue - tty.info("%s %s" % (action, spec.name)) action(spec, prefix) - if action in ['remove']: + if args.action in ['remove','rm']: purge_empty_directories(prefix) + def view(parser, args): + 'The view command.' action = { 'add': action_link, 'link': action_link, From 26c5bc9d979b95df560bdb7d7a6bd747ef93f83d Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sun, 1 May 2016 17:10:32 -0400 Subject: [PATCH 214/375] A few minimal test packages, fodder for testing 'spack view' command. --- .../builtin.mock/packages/test-a/package.py | 29 ++++++++++++++++ .../packages/test-a/test-a-0.0.tar.gz | Bin 0 -> 112 bytes .../builtin.mock/packages/test-b/package.py | 31 +++++++++++++++++ .../packages/test-b/test-b-0.0.tar.gz | Bin 0 -> 112 bytes .../builtin.mock/packages/test-c/package.py | 31 +++++++++++++++++ .../packages/test-c/test-c-0.0.tar.gz | Bin 0 -> 112 bytes .../builtin.mock/packages/test-d/package.py | 32 ++++++++++++++++++ .../packages/test-d/test-d-0.0.tar.gz | Bin 0 -> 112 bytes 8 files changed, 123 insertions(+) create mode 100644 var/spack/repos/builtin.mock/packages/test-a/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz create mode 100644 var/spack/repos/builtin.mock/packages/test-b/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz create mode 100644 var/spack/repos/builtin.mock/packages/test-c/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz create mode 100644 var/spack/repos/builtin.mock/packages/test-d/package.py create mode 100644 var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz diff --git a/var/spack/repos/builtin.mock/packages/test-a/package.py b/var/spack/repos/builtin.mock/packages/test-a/package.py new file mode 100644 index 00000000000..2f72370580a --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-a/package.py @@ -0,0 +1,29 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-a-0.0.tar.gz') + +class TestA(Package): + """The test-a package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-a') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 diff --git a/var/spack/repos/builtin.mock/packages/test-b/package.py b/var/spack/repos/builtin.mock/packages/test-b/package.py new file mode 100644 index 00000000000..db64b0a5565 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-b/package.py @@ -0,0 +1,31 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-b-0.0.tar.gz') + +class TestB(Package): + """The test-b package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + depends_on('test-a') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-b') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 diff --git a/var/spack/repos/builtin.mock/packages/test-c/package.py b/var/spack/repos/builtin.mock/packages/test-c/package.py new file mode 100644 index 00000000000..0b030374669 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-c/package.py @@ -0,0 +1,31 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-c-0.0.tar.gz') + +class TestC(Package): + """The test-c package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + depends_on('test-a+var',when='+var') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-c') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 diff --git a/var/spack/repos/builtin.mock/packages/test-d/package.py b/var/spack/repos/builtin.mock/packages/test-d/package.py new file mode 100644 index 00000000000..5cb7dcb2cb2 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/test-d/package.py @@ -0,0 +1,32 @@ +from spack import * + +import os + +mydir = os.path.dirname(__file__) +source = os.path.join(mydir,'test-d-0.0.tar.gz') + +class TestD(Package): + """The test-d package""" + + url = 'file://'+source + + version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') + + variant('nom', default=True, description='Nominal variant') + variant('var', default=False, description='Variant variant') + + depends_on('test-b') + depends_on('test-c') + + def install(self, spec, prefix): + bindir = os.path.join(prefix,'bin') + os.makedirs(bindir) + script = os.path.join(bindir, 'test-d') + with open(script,'w') as fp: + fp.write("""#!/bin/bash +echo 'name: %s' +echo 'prefix: %s' +echo 'spec: %s' + """ % (spec.name, prefix, spec)) + + os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..3588e05765cd799dfe19e1f77e057bf7508ec47e GIT binary patch literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX literal 0 HcmV?d00001 From 90b7b7ba5c483054a3b6687f84306601d0e62d15 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 4 May 2016 21:42:59 -0700 Subject: [PATCH 215/375] Reworked compiler finding/storing/format to allow for multiple compilers with the same spec for different operating systems. TODO: fix config tests. All others up to date --- lib/spack/spack/architecture.py | 96 ++++++++- lib/spack/spack/build_environment.py | 5 +- lib/spack/spack/cmd/compiler.py | 4 +- lib/spack/spack/compiler.py | 166 ++++----------- lib/spack/spack/compilers/__init__.py | 224 ++++++++++----------- lib/spack/spack/concretize.py | 8 +- lib/spack/spack/config.py | 35 ++-- lib/spack/spack/operating_systems/cnl.py | 51 ++++- lib/spack/spack/test/architecture.py | 39 ++-- lib/spack/spack/test/config.py | 95 ++++++--- lib/spack/spack/test/mock_packages_test.py | 43 +++- lib/spack/spack/test/spec_dag.py | 2 +- 12 files changed, 451 insertions(+), 317 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e2c189980a5..301495f728a 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,6 +22,7 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import os from collections import namedtuple import imp import platform as py_platform @@ -32,7 +33,10 @@ import llnl.util.tty as tty import spack +import spack.compilers from spack.util.naming import mod_to_class +from spack.util.environment import get_path +from spack.util.multiproc import parmap import spack.error as serr @@ -143,6 +147,7 @@ def operating_system(self, name): return self.operating_sys.get(name, None) + @classmethod def detect(self): """ Subclass is responsible for implementing this method. @@ -170,10 +175,9 @@ class OperatingSystem(object): find compilers we call find_compilers method for each operating system """ - def __init__(self, name, version, compiler_strategy="PATH"): + def __init__(self, name, version): self.name = name self.version = version - self.compiler_strategy = compiler_strategy def __str__(self): return self.name + self.version @@ -182,13 +186,96 @@ def __repr__(self): return self.__str__() def _cmp_key(self): - return (self.name, self.version, self.compiler_strategy) + return (self.name, self.version) + + + def find_compilers(self, *paths): + """ + Return a list of compilers found in the suppied paths. + This invokes the find() method for each Compiler class, + and appends the compilers detected to a list. + """ + if not paths: + paths = get_path('PATH') + # Make sure path elements exist, and include /bin directories + # under prefixes. + filtered_path = [] + for p in paths: + # Eliminate symlinks and just take the real directories. + p = os.path.realpath(p) + if not os.path.isdir(p): + continue + filtered_path.append(p) + + # Check for a bin directory, add it if it exists + bin = join_path(p, 'bin') + if os.path.isdir(bin): + filtered_path.append(os.path.realpath(bin)) + + # Once the paths are cleaned up, do a search for each type of + # compiler. We can spawn a bunch of parallel searches to reduce + # the overhead of spelunking all these directories. + types = spack.compilers.all_compiler_types() + compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *filtered_path), types) + + # ensure all the version calls we made are cached in the parent + # process, as well. This speeds up Spack a lot. + clist = reduce(lambda x,y: x+y, compiler_lists) + return clist + + def find_compiler(self, cmp_cls, *path): + """Try to find the given type of compiler in the user's + environment. For each set of compilers found, this returns + compiler objects with the cc, cxx, f77, fc paths and the + version filled in. + + This will search for compilers with the names in cc_names, + cxx_names, etc. and it will group them if they have common + prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would + be grouped with g++-mp-4.7 and gfortran-mp-4.7. + """ + dicts = parmap( + lambda t: cmp_cls._find_matches_in_path(*t), + [(cmp_cls.cc_names, cmp_cls.cc_version) + tuple(path), + (cmp_cls.cxx_names, cmp_cls.cxx_version) + tuple(path), + (cmp_cls.f77_names, cmp_cls.f77_version) + tuple(path), + (cmp_cls.fc_names, cmp_cls.fc_version) + tuple(path)]) + + all_keys = set() + for d in dicts: + all_keys.update(d) + + compilers = {} + for k in all_keys: + ver, pre, suf = k + + # Skip compilers with unknown version. + if ver == 'unknown': + continue + + paths = tuple(pn[k] if k in pn else None for pn in dicts) + spec = spack.spec.CompilerSpec(cmp_cls.name, ver) + + if ver in compilers: + prev = compilers[ver] + + # prefer the one with more compilers. + prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] + newcount = len([p for p in paths if p is not None]) + prevcount = len([p for p in prev_paths if p is not None]) + + # Don't add if it's not an improvement over prev compiler. + if newcount <= prevcount: + continue + + compilers[ver] = cmp_cls(spec, self, paths) + + return list(compilers.values()) def to_dict(self): d = {} d['name'] = self.name d['version'] = self.version - d['compiler_strategy'] = self.compiler_strategy return d @@ -261,7 +348,6 @@ def _operating_system_from_dict(os_dict): operating_system = OperatingSystem.__new__(OperatingSystem) operating_system.name = os_dict['name'] operating_system.version = os_dict['version'] - operating_system.compiler_strategy = os_dict['compiler_strategy'] return operating_system def arch_from_dict(d): diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 12a529d070a..82d0792e264 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -169,9 +169,8 @@ def set_compiler_environment_variables(pkg): os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) - if compiler.strategy == 'MODULES': - for mod in compiler.modules: - load_module(mod) + for mod in compiler.modules: + load_module(mod) def set_build_environment_variables(pkg): """This ensures a clean install environment when we build packages. diff --git a/lib/spack/spack/cmd/compiler.py b/lib/spack/spack/cmd/compiler.py index a7de61e1e53..e3d311b6b80 100644 --- a/lib/spack/spack/cmd/compiler.py +++ b/lib/spack/spack/cmd/compiler.py @@ -80,7 +80,7 @@ def compiler_add(args): if not paths: paths = get_path('PATH') - compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) + compilers = [c for c in spack.compilers.find_compilers(*paths) if c.spec not in spack.compilers.all_compilers(scope=args.scope)] if compilers: @@ -125,6 +125,8 @@ def compiler_info(args): print "\tcxx = %s" % c.cxx print "\tf77 = %s" % c.f77 print "\tfc = %s" % c.fc + print "\tmodules = %s" % c.modules + print "\toperating system = %s" % c.operating_system def compiler_list(args): diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index c285c8a29d5..8b973d81905 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -106,14 +106,14 @@ class Compiler(object): PrgEnv_compiler = None - def __init__(self, cspec, strategy, paths, modules=None): + def __init__(self, cspec, operating_system, paths, modules=[], alias=None): def check(exe): if exe is None: return None _verify_executables(exe) return exe - self.strategy = strategy + self.operating_system = operating_system self.cc = check(paths[0]) self.cxx = check(paths[1]) @@ -126,62 +126,33 @@ def check(exe): self.spec = cspec self.modules = modules - + self.alias = alias @property def version(self): return self.spec.version - # - # Compiler classes have methods for querying the version of - # specific compiler executables. This is used when discovering compilers. - # - # Compiler *instances* are just data objects, and can only be - # constructed from an actual set of executables. - # - - @classmethod - def default_version(cls, cc): - """Override just this to override all compiler version functions.""" - return dumpversion(cc) - - @classmethod - def cc_version(cls, cc): - return cls.default_version(cc) - - @classmethod - def cxx_version(cls, cxx): - return cls.default_version(cxx) - - @classmethod - def f77_version(cls, f77): - return cls.default_version(f77) - - @classmethod - def fc_version(cls, fc): - return cls.default_version(fc) - @classmethod def _find_matches_in_path(cls, compiler_names, detect_version, *path): """Finds compilers in the paths supplied. - - Looks for all combinations of ``compiler_names`` with the - ``prefixes`` and ``suffixes`` defined for this compiler - class. If any compilers match the compiler_names, - prefixes, or suffixes, uses ``detect_version`` to figure - out what version the compiler is. - - This returns a dict with compilers grouped by (prefix, - suffix, version) tuples. This can be further organized by - find(). - """ + + Looks for all combinations of ``compiler_names`` with the + ``prefixes`` and ``suffixes`` defined for this compiler + class. If any compilers match the compiler_names, + prefixes, or suffixes, uses ``detect_version`` to figure + out what version the compiler is. + + This returns a dict with compilers grouped by (prefix, + suffix, version) tuples. This can be further organized by + find(). + """ if not path: path = get_path('PATH') - + prefixes = [''] + cls.prefixes suffixes = [''] + cls.suffixes - + checks = [] for directory in path: if not (os.path.isdir(directory) and @@ -219,89 +190,34 @@ def check(key): successful = [key for key in parmap(check, checks) if key is not None] return dict(((v, p, s), path) for v, p, s, path in successful) - @classmethod - def find(cls, *path): - compilers = [] - platform = spack.architecture.sys_type() - strategies = [o.compiler_strategy for o in platform.operating_sys.values()] - if 'PATH' in strategies: - compilers.extend(cls.find_in_path(*path)) - if 'MODULES' in strategies: - compilers.extend(cls.find_in_modules()) - return compilers - + # + # Compiler classes have methods for querying the version of + # specific compiler executables. This is used when discovering compilers. + # + # Compiler *instances* are just data objects, and can only be + # constructed from an actual set of executables. + # @classmethod - def find_in_path(cls, *path): - """Try to find this type of compiler in the user's - environment. For each set of compilers found, this returns - compiler objects with the cc, cxx, f77, fc paths and the - version filled in. - - This will search for compilers with the names in cc_names, - cxx_names, etc. and it will group them if they have common - prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would - be grouped with g++-mp-4.7 and gfortran-mp-4.7. - """ - dicts = parmap( - lambda t: cls._find_matches_in_path(*t), - [(cls.cc_names, cls.cc_version) + tuple(path), - (cls.cxx_names, cls.cxx_version) + tuple(path), - (cls.f77_names, cls.f77_version) + tuple(path), - (cls.fc_names, cls.fc_version) + tuple(path)]) - - all_keys = set() - for d in dicts: - all_keys.update(d) - - compilers = {} - for k in all_keys: - ver, pre, suf = k - - # Skip compilers with unknown version. - if ver == 'unknown': - continue - - paths = tuple(pn[k] if k in pn else None for pn in dicts) - spec = spack.spec.CompilerSpec(cls.name, ver) - - if ver in compilers: - prev = compilers[ver] - - # prefer the one with more compilers. - prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] - newcount = len([p for p in paths if p is not None]) - prevcount = len([p for p in prev_paths if p is not None]) - - # Don't add if it's not an improvement over prev compiler. - if newcount <= prevcount: - continue - - compilers[ver] = cls(spec, 'PATH', paths) - - return list(compilers.values()) - + def default_version(cls, cc): + """Override just this to override all compiler version functions.""" + return dumpversion(cc) @classmethod - def find_in_modules(cls): - compilers = [] - if cls.PrgEnv: - if not cls.PrgEnv_compiler: - tty.die('Must supply PrgEnv_compiler with PrgEnv') + def cc_version(cls, cc): + return cls.default_version(cc) - modulecmd = which('modulecmd') - modulecmd.add_default_arg('python') - - output = modulecmd('avail', cls.PrgEnv_compiler, output=str, error=str) - matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output) - for name, version in matches: - v = version - comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES', - ['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v]) + @classmethod + def cxx_version(cls, cxx): + return cls.default_version(cxx) - compilers.append(comp) + @classmethod + def f77_version(cls, f77): + return cls.default_version(f77) - return compilers + @classmethod + def fc_version(cls, fc): + return cls.default_version(fc) def __repr__(self): @@ -311,12 +227,8 @@ def __repr__(self): def __str__(self): """Return a string represntation of the compiler toolchain.""" - if self.strategy is 'MODULES': - return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc, self.modules)))) - else: - return "%s(%s)" % ( - self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc)))) + return "%s(%s)" % ( + self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules, str(self.operating_system))))) class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index caf438e232f..8ef6e976ba8 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -29,6 +29,10 @@ import os import platform import copy +import hashlib +import base64 +import yaml +import sys from llnl.util.lang import memoized, list_modules from llnl.util.filesystem import join_path @@ -46,8 +50,8 @@ from spack.util.environment import get_path _imported_compilers_module = 'spack.compilers' -_required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] -_optional_instance_vars = ['modules', 'strategy'] +_path_instance_vars = ['cc', 'cxx', 'f77', 'fc'] +_other_instance_vars = ['modules', 'operating_system'] _default_order = [] # TODO: customize order in config file @@ -67,93 +71,85 @@ def converter(cspec_like, *args, **kwargs): def _to_dict(compiler): """Return a dict version of compiler suitable to insert in YAML.""" + d = {} + d['spec'] = str(compiler.spec) + d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) + d['operating_system'] = compiler.operating_system.to_dict() + d['modules'] = compiler.modules + + if not compiler.alias: + yaml_text = yaml.dump( + d, default_flow_style=True, width=sys.maxint) + sha = hashlib.sha1(yaml_text) + compiler.alias = base64.b32encode(sha.digest()).lower()[:8] return { - str(compiler.spec) : dict( - (attr, getattr(compiler, attr, None)) - for attr in _required_instance_vars + _optional_instance_vars) + compiler.alias: d } -def get_compiler_config(arch=None, scope=None): +def get_compiler_config(scope=None): """Return the compiler configuration for the specified architecture. """ - # Check whether we're on a front-end (native) architecture. - my_arch = spack.architecture.Arch() - if arch is None: - arch = my_arch - if isinstance(arch, basestring) and arch == 'all': - name = 'all' - else: - name = arch.platform.name - def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config[name] = {} - compilers = find_compilers(*get_path('PATH')) + config = {} + compilers = find_compilers() for compiler in compilers: - config[name].update(_to_dict(compiler)) + config.update(_to_dict(compiler)) spack.config.update_config('compilers', config, scope=scope) config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. - if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: - if scope is None: - # We know no compilers were configured in any scope. +# if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: + if scope is None: + # We know no compilers were configured in any scope. + init_compiler_config() + elif scope == 'user': + # Check the site config and update the user config if + # nothing is configured at the site level. + site_config = spack.config.get_config('compilers', scope='site') + if not site_config: init_compiler_config() - elif scope == 'user': - # Check the site config and update the user config if - # nothing is configured at the site level. - site_config = spack.config.get_config('compilers', scope='site') - if not site_config: - init_compiler_config() - return config[name] if name in config else {} + return config -def add_compilers_to_config(compilers, arch=None, scope=None): +def add_compilers_to_config(compilers, scope=None): """Add compilers to the config for the specified architecture. Arguments: - compilers: a list of Compiler objects. - - arch: arch to add compilers for. - scope: configuration scope to modify. """ - if arch is None: - arch = spack.architecture.Arch() - - compiler_config = get_compiler_config(arch, scope) + compiler_config = get_compiler_config(scope) for compiler in compilers: - compiler_config[str(compiler.spec)] = dict( - (c, getattr(compiler, c, "None")) - for c in _required_instance_vars + _optional_instance_vars) + compiler_config = _to_dict(compiler) - update = { arch.platform.name : compiler_config } - spack.config.update_config('compilers', update, scope) + spack.config.update_config('compilers', compiler_config, scope) @_auto_compiler_spec -def remove_compiler_from_config(compiler_spec, arch=None, scope=None): +def remove_compiler_from_config(compiler_spec, scope=None): """Remove compilers from the config, by spec. Arguments: - compiler_specs: a list of CompilerSpec objects. - - arch: arch to add compilers for. - scope: configuration scope to modify. """ - if arch is None: - arch = spack.architecture.Arch() + compiler_config = get_compiler_config(scope) + matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] + if len(matches) == 1: + del compiler_config[matches[0][0]] + else: + CompilerSpecInsufficientlySpecificError(compiler_spec) - compiler_config = get_compiler_config(arch, scope) - del compiler_config[str(compiler_spec)] - update = { arch : compiler_config } - - spack.config.update_config('compilers', update, scope) + spack.config.update_config('compilers', compiler_config, scope) _cache_config_file = {} -def all_compilers_config(arch=None, scope=None): +def all_compilers_config(scope=None): """Return a set of specs for all the compiler versions currently available to build with. These are instances of CompilerSpec. """ @@ -161,20 +157,16 @@ def all_compilers_config(arch=None, scope=None): global _cache_config_file #Create a cache of the config file so we don't load all the time. if not _cache_config_file: - arch_config = get_compiler_config(arch, scope) - # Merge 'all' compilers with arch-specific ones. - # Arch-specific compilers have higher precedence. - _cache_config_file = get_compiler_config('all', scope=scope) - _cache_config_file = spack.config._merge_yaml(_cache_config_file, arch_config) + _cache_config_file = get_compiler_config(scope) return _cache_config_file else: return _cache_config_file -def all_compilers(arch=None, scope=None): +def all_compilers(scope=None): # Return compiler specs from the merged config. - return [spack.spec.CompilerSpec(s) - for s in all_compilers_config(arch, scope)] + return [spack.spec.CompilerSpec(s['spec']) + for s in all_compilers_config(scope).values()] def default_compiler(): @@ -189,37 +181,19 @@ def default_compiler(): return sorted(versions)[-1] -def find_compilers(*path): +def find_compilers(): """Return a list of compilers found in the suppied paths. - This invokes the find() method for each Compiler class, - and appends the compilers detected to a list. + This invokes the find_compilers() method for each operating + system associated with the host platform, and appends + the compilers detected to a list. """ - # Make sure path elements exist, and include /bin directories - # under prefixes. - filtered_path = [] - for p in path: - # Eliminate symlinks and just take the real directories. - p = os.path.realpath(p) - if not os.path.isdir(p): - continue - filtered_path.append(p) - - # Check for a bin directory, add it if it exists - bin = join_path(p, 'bin') - if os.path.isdir(bin): - filtered_path.append(os.path.realpath(bin)) - - # Once the paths are cleaned up, do a search for each type of - # compiler. We can spawn a bunch of parallel searches to reduce - # the overhead of spelunking all these directories. - types = all_compiler_types() - compiler_lists = parmap(lambda cls: cls.find(*filtered_path), types) - - # ensure all the version calls we made are cached in the parent - # process, as well. This speeds up Spack a lot. - clist = reduce(lambda x,y: x+y, compiler_lists) - return clist + # Find compilers for each operating system class + oss = all_os_classes() + compiler_lists = [] + for os in oss: + compiler_lists.extend(os.find_compilers()) + return compiler_lists def supported_compilers(): """Return a set of names of compilers supported by Spack. @@ -237,47 +211,60 @@ def supported(compiler_spec): @_auto_compiler_spec -def find(compiler_spec, arch=None, scope=None): +def find(compiler_spec, scope=None): """Return specs of available compilers that match the supplied compiler spec. Return an list if nothing found.""" - return [c for c in all_compilers(arch, scope) if c.satisfies(compiler_spec)] + return [c for c in all_compilers(scope) if c.satisfies(compiler_spec)] @_auto_compiler_spec -def compilers_for_spec(compiler_spec, arch=None, scope=None): +def compilers_for_spec(compiler_spec, scope=None): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ - config = all_compilers_config(arch, scope) + config = all_compilers_config(scope) - def get_compiler(cspec): - items = config[str(cspec)] + def get_compilers(cspec): + compilers = [] - if not all(n in items for n in _required_instance_vars): - raise InvalidCompilerConfigurationError(cspec) + for aka, cmp in config.items(): + if cmp['spec'] != str(cspec): + continue + items = cmp + alias = aka + + if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): + raise InvalidCompilerConfigurationError(cspec) - cls = class_for_compiler_name(cspec.name) + cls = class_for_compiler_name(cspec.name) - strategy = items['strategy'] - if not strategy: - raise InvalidCompilerConfigurationError(cspec) + compiler_paths = [] + for c in _path_instance_vars: + compiler_path = items['paths'][c] + if compiler_path != "None": + compiler_paths.append(compiler_path) + else: + compiler_paths.append(None) - compiler_paths = [] - for c in _required_instance_vars: - compiler_path = items[c] - if compiler_path != "None": - compiler_paths.append(compiler_path) + mods = items.get('modules') + if mods == 'None': + mods = [] + + if 'operating_system' in items: + operating_system = spack.architecture._operating_system_from_dict( items['operating_system'] ) else: - compiler_paths.append(None) + operating_system = None - if 'modules' not in items: - items['modules'] = None - mods = items['modules'] + compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias)) - return cls(cspec, strategy, compiler_paths, mods) + return compilers - matches = find(compiler_spec, arch, scope) - return [get_compiler(cspec) for cspec in matches] + matches = set(find(compiler_spec, scope)) + compilers = [] + for cspec in matches: + compilers.extend(get_compilers(cspec)) + return compilers +# return [get_compilers(cspec) for cspec in matches] @_auto_compiler_spec @@ -285,8 +272,9 @@ def compiler_for_spec(compiler_spec, operating_system): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" assert(compiler_spec.concrete) + compilers = [c for c in compilers_for_spec(compiler_spec) - if c.strategy == operating_system.compiler_strategy] + if c.operating_system == operating_system] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, operating_system) if len(compilers) > 1: @@ -308,8 +296,20 @@ def class_for_compiler_name(compiler_name): return cls +def all_os_classes(): + """ + Return the list of classes for all operating systems available on + this platform + """ + classes = [] + + platform = spack.architecture.sys_type() + for os_class in platform.operating_sys.values(): + classes.append(os_class) + + return classes + def all_compiler_types(): -# return [class_for_compiler_name(c) for c in ['gcc']] return [class_for_compiler_name(c) for c in supported_compilers()] @@ -318,7 +318,7 @@ def __init__(self, compiler_spec): super(InvalidCompilerConfigurationError, self).__init__( "Invalid configuration for [compiler \"%s\"]: " % compiler_spec, "Compiler configuration must contain entries for all compilers: %s" - % _required_instance_vars) + % _path_instance_vars) class NoCompilersError(spack.error.SpackError): diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 3422e86ae83..58ea83ac4ac 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -301,15 +301,11 @@ def concretize_compiler(self, spec): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - filter(lambda c: c.strategy == architecture.platform_os.compiler_strategy, compilers) -#if architecture.platform_os.compiler_strategy == 'PATH': - # filter(lambda c: not c.modules, compilers) - #if architecture.platform_os.compiler_strategy == 'MODULES': - # filter(lambda c: c.modules, compilers) + filter(lambda c: c.operating_system == architecture.platform_os, compilers) return compilers - all_compilers = spack.compilers.all_compilers(spec.architecture) + all_compilers = spack.compilers.all_compilers() if (spec.compiler and spec.compiler.concrete and diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index f978ed12b09..49f2b90bfc6 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -151,17 +151,16 @@ 'default': {}, 'additionalProperties': False, 'patternProperties': { - r'\w[\w-]*': { # architecture + r'\w[\w-]*': { # alias 'type': 'object', 'additionalProperties': False, - 'patternProperties': { - r'\w[\w-]*@\w[\w-]*': { # compiler spec + 'required': ['paths', 'spec', 'modules', 'operating_system'], + 'properties': { + 'paths': { 'type': 'object', - 'additionalProperties': False, 'required': ['cc', 'cxx', 'f77', 'fc'], - 'properties': { - 'strategy': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, + 'additionalProperties': False, + 'properties': { 'cc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'cxx': { 'anyOf': [ {'type' : 'string' }, @@ -169,13 +168,21 @@ 'f77': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, - {'type' : 'null' }]}, - 'modules': { 'anyOf': [ {'type' : 'string'}, - {'type' : 'null' }, - {'type': 'array'}, - ]} - },},},},},},},}, - + {'type' : 'null' }]}}}, + 'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*' + 'operating_system': { + 'type': 'object', + 'required': ['name', 'version'], + 'additionalProperties': False, + 'properties': { + 'name': {'type': 'string'}, + 'version': {'type': 'string'} + }}, + 'modules': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }, + {'type': 'array'}, + ]} + },},},},},}, 'mirrors': { '$schema': 'http://json-schema.org/schema#', 'title': 'Spack mirror configuration file schema', diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 9a0bf6c1942..c160a60be82 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -1,4 +1,11 @@ +import re +import os + from spack.architecture import OperatingSystem +from spack.util.executable import * +import spack.spec +from spack.util.multiproc import parmap +import spack.compilers class Cnl(OperatingSystem): """ Compute Node Linux (CNL) is the operating system used for the Cray XC @@ -10,4 +17,46 @@ class Cnl(OperatingSystem): def __init__(self): name = 'CNL' version = '10' - super(Cnl, self).__init__(name, version, "MODULES") + super(Cnl, self).__init__(name, version) + + + def find_compilers(self, *paths): + types = spack.compilers.all_compiler_types() + compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) + + # ensure all the version calls we made are cached in the parent + # process, as well. This speeds up Spack a lot. + clist = reduce(lambda x,y: x+y, compiler_lists) + return clist + + + def find_compiler(self, cmp_cls, *paths): + compilers = [] + if cmp_cls.PrgEnv: + if not cmp_cls.PrgEnv_compiler: + tty.die('Must supply PrgEnv_compiler with PrgEnv') + + modulecmd = which('modulecmd') + modulecmd.add_default_arg('python') + + # Save the environment variable to restore later + old_modulepath = os.environ['MODULEPATH'] + # if given any explicit paths, search them for module files too + if paths: + module_paths = ':' + ':'.join(p for p in paths) + os.environ['MODULEPATH'] = module_paths + + output = modulecmd('avail', cmp_cls.PrgEnv_compiler, output=str, error=str) + matches = re.findall(r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output) + for name, version in matches: + v = version + comp = cmp_cls(spack.spec.CompilerSpec(name + '@' + v), self, + ['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name +'/' + v]) + + compilers.append(comp) + + # Restore modulepath environment variable + if paths: + os.environ['MODULEPATH'] = old_modulepath + + return compilers diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 1ca3e58831d..4cba2da9a5b 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -12,8 +12,18 @@ from spack.platforms.bgq import Bgq from spack.platforms.darwin import Darwin -class ArchitectureTest(unittest.TestCase): +from spack.test.mock_packages_test import * +#class ArchitectureTest(unittest.TestCase): +class ArchitectureTest(MockPackagesTest): + + def setUp(self): + super(ArchitectureTest, self).setUp() + self.platform = sys_type() + + def tearDown(self): + super(ArchitectureTest, self).tearDown() + def test_dict_functions_for_architecture(self): arch = Arch() arch.platform_os = arch.platform.operating_system('default_os') @@ -34,13 +44,13 @@ def test_dict_functions_for_architecture(self): self.assertTrue( isinstance(new_arch.target, Target) ) - def test_platform_class_and_compiler_strategies(self): - a = CrayXc() - t = a.operating_system('default_os') - self.assertEquals(t.compiler_strategy, 'MODULES') - b = Linux() - s = b.operating_system('default_os') - self.assertEquals(s.compiler_strategy, 'PATH') +# def test_platform_class_and_compiler_strategies(self): +# a = CrayXc() +# t = a.operating_system('default_os') +# self.assertEquals(t.compiler_strategy, 'MODULES') +# b = Linux() +# s = b.operating_system('default_os') +# self.assertEquals(s.compiler_strategy, 'PATH') def test_sys_type(self): output_platform_class = sys_type() @@ -56,16 +66,13 @@ def test_sys_type(self): self.assertEqual(str(output_platform_class), str(my_platform_class)) - def setUp(self): - self.platform = sys_type() - def test_user_front_end_input(self): """Test when user inputs just frontend that both the frontend target and frontend operating system match """ frontend_os = self.platform.operating_system("frontend") frontend_target = self.platform.target("frontend") - frontend_spec = Spec("zlib=frontend") + frontend_spec = Spec("libelf=frontend") frontend_spec.concretize() self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) self.assertEqual(frontend_target, frontend_spec.architecture.target) @@ -76,7 +83,7 @@ def test_user_back_end_input(self): """ backend_os = self.platform.operating_system("backend") backend_target = self.platform.target("backend") - backend_spec = Spec("zlib=backend") + backend_spec = Spec("libelf=backend") backend_spec.concretize() self.assertEqual(backend_os, backend_spec.architecture.platform_os) self.assertEqual(backend_target, backend_spec.architecture.target) @@ -85,7 +92,7 @@ def test_user_defaults(self): default_os = self.platform.operating_system("default_os") default_target = self.platform.target("default_target") - default_spec = Spec("zlib") # default is no args + default_spec = Spec("libelf") # default is no args default_spec.concretize() self.assertEqual(default_os, default_spec.architecture.platform_os) self.assertEqual(default_target, default_spec.architecture.target) @@ -103,11 +110,11 @@ def test_user_input_combination(self): for arch in combinations: o,t = arch arch_spec = "-".join(arch) - spec = Spec("zlib=%s" % arch_spec) + spec = Spec("libelf=%s" % arch_spec) spec.concretize() results.append(spec.architecture.platform_os == self.platform.operating_system(o)) results.append(spec.architecture.target == self.platform.target(t)) res = all(results) - print res + self.assertTrue(res) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 0562d2d6209..2a8d642584d 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -33,43 +33,91 @@ # Some sample compiler config data a_comps = { - "all": { - "gcc@4.7.3" : { + 'gcc473': { + 'paths': { "cc" : "/gcc473", "cxx": "/g++473", "f77": None, - "fc" : None }, - "gcc@4.5.0" : { + "fc" : None + }, + 'modules': None, + 'spec': 'gcc@4.7.3', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'gcc450': { + 'paths': { "cc" : "/gcc450", "cxx": "/g++450", - "f77": "/gfortran", - "fc" : "/gfortran" }, - "clang@3.3" : { + "f77": 'gfortran', + "fc" : 'gfortran' + }, + 'modules': None, + 'spec': 'gcc@4.5.0', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'clang33': { + 'paths': { "cc" : "", "cxx": "", - "f77": "", - "fc" : "" } - } + "f77": '', + "fc" : '' }, + 'modules': None, + 'spec': 'clang@3.3', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + } } b_comps = { - "all": { - "icc@10.0" : { + 'icc100': { + 'paths': { "cc" : "/icc100", - "cxx": "/icc100", + "cxx": "/icp100", "f77": None, - "fc" : None }, - "icc@11.1" : { + "fc" : None + }, + 'modules': None, + 'spec': 'icc@10.0', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'icc111': { + 'paths': { "cc" : "/icc111", "cxx": "/icp111", - "f77": "/ifort", - "fc" : "/ifort" }, - "clang@3.3" : { - "cc" : "/clang", - "cxx": "/clang++", - "f77": None, - "fc" : None} - } + "f77": 'ifort', + "fc" : 'ifort' + }, + 'modules': None, + 'spec': 'icc@11.1', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + }, + 'clang33': { + 'paths': { + "cc" : "", + "cxx": "", + "f77": '', + "fc" : '' }, + 'modules': None, + 'spec': 'clang@3.3', + 'operating_system': { + 'name': 'CNL', + 'version': '10' + } + } } class ConfigTest(MockPackagesTest): @@ -96,7 +144,6 @@ def check_config(self, comps, *compiler_names): actual = config['all'][key][c] self.assertEqual(expected, actual) - def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. spack.config.update_config('compilers', a_comps, 'test_low_priority') diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 8f04076f194..612008ac589 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -36,21 +36,50 @@ mock_compiler_config = """\ compilers: - all: - clang@3.3: + clang3.3CNL: + spec: clang@3.3 + operating_system: + name: CNL + version: '10' + paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None - strategy: PATH - modules: None - gcc@4.5.0: + modules: 'None' + clang3.3RHL: + spec: clang@3.3 + operating_system: + name: redhat + version: '6.7' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' + gcc4.5.0CNL: + paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - strategy: PATH - modules: None + operating_system: + name: CNL + version: '10' + spec: gcc@4.5.0 + modules: 'None' + gcc4.5.0RHL: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: RHL + version: '6.7' + spec: gcc@4.5.0 + modules: 'None' """ mock_packages_config = """\ diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index eee65c5c09a..4ed7de9c7d8 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -243,7 +243,7 @@ def test_unsatisfiable_target(self): if len(platform.targets) > 1: first = platform.targets.values()[0].name second = platform.targets.values()[1].name - set_pkg_dep('mpileaks', 'mpich='+first) + self.set_pkg_dep('mpileaks', 'mpich='+first) spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) From 2d1430da130e33fdd8d8345ce9b4cbe3dbdc2871 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 7 May 2016 18:24:24 -0400 Subject: [PATCH 216/375] Address all coments in @trws's latest comment in PR #869. I addressed them by factoring the code better to follow the visitor pattern. This will allow actions to be easily added in the future. These may not even be file sytsem views. One could add actions to generate shell init scripts, JSON DAG-dumpers, GraphViz DOT file generators, etc (yes, some of these are alread in there - just to give the idea). Also added is a top-level test $ source share/spack/setup-env.sh $ ./share/spack/examples/test_view.sh Read the top of that script first. --- lib/spack/spack/cmd/view.py | 318 +++++++++++------- share/spack/examples/test_view.sh | 31 ++ .../builtin.mock/packages/test-a/package.py | 1 + .../builtin.mock/packages/test-b/package.py | 1 + .../builtin.mock/packages/test-c/package.py | 1 + .../builtin.mock/packages/test-d/package.py | 1 + 6 files changed, 231 insertions(+), 122 deletions(-) create mode 100755 share/spack/examples/test_view.sh diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 1bdaaecadc3..31937b39a20 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -1,8 +1,3 @@ -''' -Produce a file-system "view" of a Spack DAG. - -Concept from Nix, implemented by brett.viren@gmail.com ca 2016. -''' ############################################################################## # Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. @@ -27,6 +22,45 @@ # along with this program; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +'''Produce a "view" of a Spack DAG. + +A "view" is the product of applying a function on a set of package specs. + +This set consists of: + +- specs resolved from the package names given by the user (the seeds) +- all depenencies of the seeds unless user specifies `--no-depenencies` +- less any specs with names matching the regular expressions given by `--exclude` + +The `view` command provides a number of functions (the "actions"): + +- symlink :: a file system view which is a directory hierarchy that is + the union of the hierarchies of the installed packages in the DAG + where installed files are referenced via symlinks. + +- hardlink :: like the symlink view but hardlinks are used + +- statlink :: a view producing a status report of a symlink or + hardlink view. + + +The file system view concept is imspired by Nix, implemented by +brett.viren@gmail.com ca 2016. + +''' +# Implementation notes: +# +# This is implemented as a visitor pattern on the set of package specs. +# +# The command line ACTION maps to a visitor_*() function which takes +# the set of package specs and any args which may be specific to the +# ACTION. +# +# To add a new view: +# 1. add a new cmd line args sub parser ACTION +# 2. add any action-specific options/arguments, most likely a list of specs. +# 3. add a visitor_MYACTION() function +# 4. add any visitor_MYALIAS assignments to match any command line aliases import os import re @@ -38,33 +72,48 @@ description = "Produce a single-rooted directory view of a spec." -def setup_parser(subparser): - setup_parser.parser = subparser +def setup_parser(sp): + setup_parser.parser = sp - sp = subparser.add_subparsers(metavar='ACTION', dest='action') + sp.add_argument('-v','--verbose', action='store_true', default=False, + help="Display verbose output.") + sp.add_argument('-e','--exclude', action='append', default=[], + help="Exclude packages with names matching the given regex pattern.") + sp.add_argument('-d', '--dependencies', choices=['true','false','yes','no'], + default='true', + help="Follow dependencies.") + + + ssp = sp.add_subparsers(metavar='ACTION', dest='action') # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. - sps = [ - sp.add_parser('add', aliases=['link'], - help='Add packages to the view, create view if needed.'), - sp.add_parser('remove', aliases=['rm'], - help='Remove packages from the view, and view if empty.'), - sp.add_parser('status', aliases=['check'], - help='Check status of packages in the view.') + file_system_view_actions = [ + ssp.add_parser('symlink', aliases=['add','soft'], + help='Add package files to a filesystem view via symbolic links.'), + ssp.add_parser('hardlink', aliases=['hard'], + help='Add packages files to a filesystem via via hard links.'), + ssp.add_parser('remove', aliases=['rm'], + help='Remove packages from a filesystem view.'), + ssp.add_parser('statlink', aliases=['status','check'], + help='Check status of packages in a filesystem view.') ] - # All these options and arguments are common to every action. - for p in sps: - p.add_argument('-e','--exclude', action='append', default=[], - help="exclude any packages which the given re pattern") - p.add_argument('--no-dependencies', action='store_true', default=False, - help="just operate on named packages and do not follow dependencies") - p.add_argument('prefix', nargs=1, - help="Path to a top-level directory to receive the view.") - p.add_argument('specs', nargs=argparse.REMAINDER, - help="specs of packages to expose in the view.") + for act in file_system_view_actions: + act.add_argument('path', nargs=1, + help="Path to file system view directory.") + act.add_argument('specs', metavar='spec', nargs='+', + help="Seed specs of the packages to view.") + ## Other VIEW ACTIONS might be added here. + ## Some ideas are the following (and some are redundant with existing cmds) + ## A JSON view that dumps a DAG to a JSON file + ## A DOT view that dumps to a GraphViz file + ## A SHELL INIT view that dumps bash/csh script setting up to use packages in the view + return + + +### Util functions def assuredir(path): 'Assure path exists as a directory' @@ -79,7 +128,6 @@ def relative_to(prefix, path): reldir = reldir[1:] return reldir - def transform_path(spec, path, prefix=None): 'Return the a relative path corresponding to given path spec.prefix' if os.path.isabs(path): @@ -92,66 +140,6 @@ def transform_path(spec, path, prefix=None): path = os.path.join(prefix, path) return path -def action_status(spec, prefix): - 'Check status of view in prefix against spec' - dotspack = os.path.join(prefix, '.spack', spec.name) - if os.path.exists(os.path.join(dotspack)): - tty.info("Package added: %s"%spec.name) - return - tty.info("Package missing: %s"%spec.name) - return - -def action_remove(spec, prefix): - 'Remove any files found in spec from prefix and purge empty directories.' - - if not os.path.exists(prefix): - return - - dotspack = transform_path(spec, '.spack', prefix) - if not os.path.exists(dotspack): - tty.info("Skipping nonexistent package %s"%spec.name) - return - - tty.info("remove %s"%spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): - if not filenames: - continue - - targdir = transform_path(spec, dirpath, prefix) - for fname in filenames: - src = os.path.join(dirpath, fname) - dst = os.path.join(targdir, fname) - if not os.path.exists(dst): - #tty.warn("Skipping nonexistent file for view: %s" % dst) - continue - os.unlink(dst) - -def action_link(spec, prefix): - 'Symlink all files in `spec` into directory `prefix`.' - - dotspack = transform_path(spec, '.spack', prefix) - if os.path.exists(dotspack): - tty.warn("Skipping previously added package %s"%spec.name) - return - - tty.info("link %s" % spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): - if not filenames: - continue # avoid explicitly making empty dirs - - targdir = transform_path(spec, dirpath, prefix) - assuredir(targdir) - - for fname in filenames: - src = os.path.join(dirpath, fname) - dst = os.path.join(targdir, fname) - if os.path.exists(dst): - if '.spack' in dst.split(os.path.sep): - continue # silence these - tty.warn("Skipping existing file for view: %s" % dst) - continue - os.symlink(src,dst) - def purge_empty_directories(path): 'Ascend up from the leaves accessible from `path` and remove empty directories.' for dirpath, subdirs, files in os.walk(path, topdown=False): @@ -160,58 +148,144 @@ def purge_empty_directories(path): try: os.rmdir(sdp) except OSError: - #tty.warn("Not removing directory with contents: %s" % sdp) pass - - - -def view_action(action, parser, args): - 'The view command parameterized by the action.' - to_exclude = [re.compile(e) for e in args.exclude] +def filter_exclude(specs, exclude): + 'Filter specs given sequence of exclude regex' + to_exclude = [re.compile(e) for e in exclude] def exclude(spec): for e in to_exclude: if e.match(spec.name): return True return False + return [s for s in specs if not exclude(s)] - specs = spack.cmd.parse_specs(args.specs, normalize=True, concretize=True) - if not specs: - parser.print_help() - return 1 - - prefix = args.prefix[0] - assuredir(prefix) - +def flatten(seeds, descend=True): + 'Normalize and flattend seed specs and descend hiearchy' flat = set() - for spec in specs: - if args.no_dependencies: + for spec in seeds: + if not descend: flat.add(spec) continue flat.update(spec.normalized().traverse()) + return flat - for spec in flat: - if exclude(spec): - tty.info('Skipping excluded package: "%s"' % spec.name) + +### Action-specific helpers + +def check_one(spec, path, verbose=False): + 'Check status of view in path against spec' + dotspack = os.path.join(path, '.spack', spec.name) + if os.path.exists(os.path.join(dotspack)): + tty.info('Package in view: "%s"'%spec.name) + return + tty.info('Package not in view: "%s"'%spec.name) + return + +def remove_one(spec, path, verbose=False): + 'Remove any files found in `spec` from `path` and purge empty directories.' + + if not os.path.exists(path): + return # done, short circuit + + dotspack = transform_path(spec, '.spack', path) + if not os.path.exists(dotspack): + if verbose: + tty.info('Skipping nonexistent package: "%s"'%spec.name) + return + + if verbose: + tty.info('Removing package: "%s"'%spec.name) + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: continue - if not os.path.exists(spec.prefix): - tty.warn('Skipping unknown package: %s in %s' % (spec.name, spec.prefix)) - continue - action(spec, prefix) + targdir = transform_path(spec, dirpath, path) + for fname in filenames: + dst = os.path.join(targdir, fname) + if not os.path.exists(dst): + continue + os.unlink(dst) - if args.action in ['remove','rm']: - purge_empty_directories(prefix) +def link_one(spec, path, link = os.symlink, verbose=False): + 'Link all files in `spec` into directory `path`.' + dotspack = transform_path(spec, '.spack', path) + if os.path.exists(dotspack): + tty.warn('Skipping existing package: "%s"'%spec.name) + return + + if verbose: + tty.info('Linking package: "%s"' % spec.name) + for dirpath,dirnames,filenames in os.walk(spec.prefix): + if not filenames: + continue # avoid explicitly making empty dirs + + targdir = transform_path(spec, dirpath, path) + assuredir(targdir) + + for fname in filenames: + src = os.path.join(dirpath, fname) + dst = os.path.join(targdir, fname) + if os.path.exists(dst): + if '.spack' in dst.split(os.path.sep): + continue # silence these + tty.warn("Skipping existing file: %s" % dst) + continue + link(src,dst) + + +### The canonically named visitor_* functions and their alias maps. +### One for each action. + +def visitor_symlink(specs, args): + 'Symlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, verbose=args.verbose) +visitor_add = visitor_symlink +visitor_soft = visitor_symlink + +def visitor_hardlink(specs, args): + 'Hardlink all files found in specs' + path = args.path[0] + assuredir(path) + for spec in specs: + link_one(spec, path, os.link, verbose=args.verbose) +visitor_hard = visitor_hardlink + +def visitor_remove(specs, args): + 'Remove all files and directories found in specs from args.path' + path = args.path[0] + for spec in specs: + remove_one(spec, path, verbose=args.verbose) + purge_empty_directories(path) +visitor_rm = visitor_remove + +def visitor_statlink(specs, args): + 'Give status of view in args.path relative to specs' + path = args.path[0] + for spec in specs: + check_one(spec, path, verbose=args.verbose) +visitor_status = visitor_statlink +visitor_check = visitor_statlink + + +# Finally, the actual "view" command. There should be no need to +# modify anything below when new actions are added. def view(parser, args): - 'The view command.' - action = { - 'add': action_link, - 'link': action_link, - 'remove': action_remove, - 'rm': action_remove, - 'status': action_status, - 'check': action_status - }[args.action] - view_action(action, parser, args) + 'Produce a view of a set of packages.' + + # Process common args + seeds = [spack.cmd.disambiguate_spec(s) for s in args.specs] + specs = flatten(seeds, args.dependencies.lower() in ['yes','true']) + specs = filter_exclude(specs, args.exclude) + + # Execute the visitation. + try: + visitor = globals()['visitor_' + args.action] + except KeyError: + tty.error('Unknown action: "%s"' % args.action) + visitor(specs, args) diff --git a/share/spack/examples/test_view.sh b/share/spack/examples/test_view.sh new file mode 100755 index 00000000000..b3e71021924 --- /dev/null +++ b/share/spack/examples/test_view.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# This will install a few bogus/test packages in order to test the +# `spack view` command. It assumes you have "spack" in your path. + +# It makes sub-directories in your CWD and installs and uninstalls +# Spack packages named test-*. + +set -x +set -e + +view="spack -m view -v" +for variant in +nom ~nom+var +nom+var +do + spack -m uninstall -f -a -y test-d + spack -m install test-d$variant + testdir=test_view + rm -rf $testdir + echo "hardlink may fail if Spack install area and CWD are not same FS" + for action in symlink hardlink + do + $view --dependencies=no $action $testdir test-d + $view -e test-a -e test-b $action $testdir test-d + $view $action $testdir test-d + $view status $testdir test-d + $view -d false remove $testdir test-a + $view remove $testdir test-d + rmdir $testdir # should not fail + done +done +echo "Warnings about skipping existing in the above are okay" diff --git a/var/spack/repos/builtin.mock/packages/test-a/package.py b/var/spack/repos/builtin.mock/packages/test-a/package.py index 2f72370580a..ff45344b7af 100644 --- a/var/spack/repos/builtin.mock/packages/test-a/package.py +++ b/var/spack/repos/builtin.mock/packages/test-a/package.py @@ -9,6 +9,7 @@ class TestA(Package): """The test-a package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') diff --git a/var/spack/repos/builtin.mock/packages/test-b/package.py b/var/spack/repos/builtin.mock/packages/test-b/package.py index db64b0a5565..f6d30bb85ff 100644 --- a/var/spack/repos/builtin.mock/packages/test-b/package.py +++ b/var/spack/repos/builtin.mock/packages/test-b/package.py @@ -9,6 +9,7 @@ class TestB(Package): """The test-b package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') diff --git a/var/spack/repos/builtin.mock/packages/test-c/package.py b/var/spack/repos/builtin.mock/packages/test-c/package.py index 0b030374669..641e46ee507 100644 --- a/var/spack/repos/builtin.mock/packages/test-c/package.py +++ b/var/spack/repos/builtin.mock/packages/test-c/package.py @@ -9,6 +9,7 @@ class TestC(Package): """The test-c package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') diff --git a/var/spack/repos/builtin.mock/packages/test-d/package.py b/var/spack/repos/builtin.mock/packages/test-d/package.py index 5cb7dcb2cb2..b46c680eddd 100644 --- a/var/spack/repos/builtin.mock/packages/test-d/package.py +++ b/var/spack/repos/builtin.mock/packages/test-d/package.py @@ -9,6 +9,7 @@ class TestD(Package): """The test-d package""" url = 'file://'+source + homepage = "http://www.example.com/" version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') From f1900f6a7b080a6b45392d356749e2a98de46398 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sun, 8 May 2016 10:25:21 -0400 Subject: [PATCH 217/375] Add a 'print' view allowing genreation of arbitrary strings based on format using package/spec parameters. --- lib/spack/spack/cmd/view.py | 64 +++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 31937b39a20..b504dfd2b15 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -38,11 +38,12 @@ the union of the hierarchies of the installed packages in the DAG where installed files are referenced via symlinks. -- hardlink :: like the symlink view but hardlinks are used +- hardlink :: like the symlink view but hardlinks are used. - statlink :: a view producing a status report of a symlink or hardlink view. +- format :: a view printing one string per spec following a given format. The file system view concept is imspired by Nix, implemented by brett.viren@gmail.com ca 2016. @@ -64,6 +65,7 @@ import os import re +import sys import argparse import spack @@ -86,6 +88,9 @@ def setup_parser(sp): ssp = sp.add_subparsers(metavar='ACTION', dest='action') + specs_opts = dict(metavar='spec', nargs='+', + help="Seed specs of the packages to view.") + # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. file_system_view_actions = [ @@ -102,9 +107,15 @@ def setup_parser(sp): for act in file_system_view_actions: act.add_argument('path', nargs=1, help="Path to file system view directory.") - act.add_argument('specs', metavar='spec', nargs='+', - help="Seed specs of the packages to view.") + act.add_argument('specs', **specs_opts) + # The formatted print action. + act = ssp.add_parser('print', + help="Print a string to stdout based on given format") + act.add_argument('format', nargs=1, + help="Format describing per-package printout.") + act.add_argument('specs', **specs_opts) + ## Other VIEW ACTIONS might be added here. ## Some ideas are the following (and some are redundant with existing cmds) ## A JSON view that dumps a DAG to a JSON file @@ -170,6 +181,39 @@ def flatten(seeds, descend=True): flat.update(spec.normalized().traverse()) return flat +def spec2dict(spec): + 'Convert info in a spec into a simple dictionary.' + + # Expclitly convert instead of just returning spec.__dict__ as + # some things need processing or are properties. + # + pkg = spec.package + ret = dict(name = spec.name, + short = spec.short_spec, + cshort = spec.cshort_spec, # color + root = spec.root, + prefix = spec.prefix, + version = spec.version, + variants = spec.variants, + namespace = spec.namespace, + compiler = spec.compiler, + architecture = spec.architecture, + dependencies = ','.join(spec.dependencies.keys()), + dependents = ','.join(spec.dependents.keys()), + external = spec.external or "False", + hash = spec.dag_hash(), + + # package related: + url = pkg.url, + stage = pkg.stage.path, + installed = pkg.installed, + installed_dependents = ','.join([s.name for s in pkg.installed_dependents]), + build_log = pkg.build_log_path, + rpath = ':'.join(pkg.rpath), + + # ... + ) + return ret ### Action-specific helpers @@ -270,6 +314,20 @@ def visitor_statlink(specs, args): visitor_status = visitor_statlink visitor_check = visitor_statlink +def visitor_print(specs, args): + 'Print a string for each spec using args.format.' + fmt = args.format[0] + for spec in specs: + kwds = spec2dict(spec) + try: + string = fmt.format(**kwds) + except KeyError: + tty.error("Format error, use keywords: %s" % (', '.join(kwds.keys()), )) + raise + # argparser escapes these + string = string.replace(r'\n', '\n').replace(r'\t', '\t') + sys.stdout.write(string) + # Finally, the actual "view" command. There should be no need to # modify anything below when new actions are added. From ab41fd6692feb9cdaa29dedf81689e3ff04a5f26 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 12 May 2016 16:21:06 -0500 Subject: [PATCH 218/375] Add bertini package Also add missing dependency. Flex would not build for me without bison. It complained about not being able to find `yacc`. --- .../repos/builtin/packages/bertini/package.py | 49 +++++++++++++++++++ .../repos/builtin/packages/flex/package.py | 2 + 2 files changed, 51 insertions(+) create mode 100644 var/spack/repos/builtin/packages/bertini/package.py diff --git a/var/spack/repos/builtin/packages/bertini/package.py b/var/spack/repos/builtin/packages/bertini/package.py new file mode 100644 index 00000000000..67e4a945ff5 --- /dev/null +++ b/var/spack/repos/builtin/packages/bertini/package.py @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Bertini(Package): + """Bertini is a general-purpose solver, written in C, that was created + for research about polynomial continuation. It solves for the numerical + solution of systems of polynomial equations using homotopy continuation.""" + + homepage = "https://bertini.nd.edu/" + url = "https://bertini.nd.edu/BertiniSource_v1.5.tar.gz" + + version('1.5', 'e3f6cc6e7f9a0cf1d73185e8671af707') + + variant('mpi', default=True, description='Compile in parallel') + + depends_on('flex') + depends_on('bison') + depends_on('gmp') + depends_on('mpfr') + depends_on('mpi', when='+mpi') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py index 926651010f7..99ccf700242 100644 --- a/var/spack/repos/builtin/packages/flex/package.py +++ b/var/spack/repos/builtin/packages/flex/package.py @@ -33,6 +33,8 @@ class Flex(Package): version('2.6.0', '5724bcffed4ebe39e9b55a9be80859ec') version('2.5.39', 'e133e9ead8ec0a58d81166b461244fde') + depends_on('bison') + def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 088dce709e85d0f522ac2d90c22618a20d421cbc Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Thu, 12 May 2016 16:33:37 -0500 Subject: [PATCH 219/375] Make flake8 happy --- var/spack/repos/builtin/packages/bertini/package.py | 1 + var/spack/repos/builtin/packages/flex/package.py | 1 + 2 files changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/bertini/package.py b/var/spack/repos/builtin/packages/bertini/package.py index 67e4a945ff5..8d7da705e45 100644 --- a/var/spack/repos/builtin/packages/bertini/package.py +++ b/var/spack/repos/builtin/packages/bertini/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class Bertini(Package): """Bertini is a general-purpose solver, written in C, that was created for research about polynomial continuation. It solves for the numerical diff --git a/var/spack/repos/builtin/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py index 99ccf700242..f41af061a52 100644 --- a/var/spack/repos/builtin/packages/flex/package.py +++ b/var/spack/repos/builtin/packages/flex/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class Flex(Package): """Flex is a tool for generating scanners.""" From 4c9dd028d2305360329767831e65bafe403c6e8a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 12 May 2016 17:20:48 -0700 Subject: [PATCH 220/375] Fixed error causing hang in env/cc --- lib/spack/env/cc | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 4a3e6eddc94..91916c45323 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -153,6 +153,29 @@ fi # Save original command for debug logging input_command="$@" +# +# Filter '.' and Spack environment directories out of PATH so that +# this script doesn't just call itself +# +IFS=':' read -ra env_path <<< "$PATH" +IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" +spack_env_dirs+=(".") +PATH="" +for dir in "${env_path[@]}"; do + remove="" + for rm_dir in "${spack_env_dirs[@]}"; do + if [ "$dir" = "$rm_dir" ]; then remove=True; fi + done + if [ -z "$remove" ]; then + if [ -z "$PATH" ]; then + PATH="$dir" + else + PATH="$PATH:$dir" + fi + fi +done +export PATH + if [ "$mode" == vcheck ] ; then exec ${command} "$@" fi @@ -317,29 +340,6 @@ unset LD_LIBRARY_PATH unset LD_RUN_PATH unset DYLD_LIBRARY_PATH -# -# Filter '.' and Spack environment directories out of PATH so that -# this script doesn't just call itself -# -IFS=':' read -ra env_path <<< "$PATH" -IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" -spack_env_dirs+=(".") -PATH="" -for dir in "${env_path[@]}"; do - remove="" - for rm_dir in "${spack_env_dirs[@]}"; do - if [ "$dir" = "$rm_dir" ]; then remove=True; fi - done - if [ -z "$remove" ]; then - if [ -z "$PATH" ]; then - PATH="$dir" - else - PATH="$PATH:$dir" - fi - fi -done -export PATH - full_command=("$command") full_command+=("${args[@]}") From b8a91db08948440100a2d4d6a0cad01f1eb9b33a Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Thu, 12 May 2016 20:04:45 -0700 Subject: [PATCH 221/375] Intel software packages and license enhancements --- lib/spack/spack/hooks/licensing.py | 5 +- lib/spack/spack/package.py | 10 +- .../repos/builtin/packages/daal/package.py | 27 ++++ .../packages/intel-parallelstudio/package.py | 112 +++++++++++++++++ .../repos/builtin/packages/intel/package.py | 117 ++++++++++++++++++ .../repos/builtin/packages/ipp/package.py | 25 ++++ .../repos/builtin/packages/mkl/package.py | 27 ++++ 7 files changed, 319 insertions(+), 4 deletions(-) create mode 100644 var/spack/repos/builtin/packages/daal/package.py create mode 100644 var/spack/repos/builtin/packages/intel-parallelstudio/package.py create mode 100644 var/spack/repos/builtin/packages/intel/package.py create mode 100644 var/spack/repos/builtin/packages/ipp/package.py create mode 100644 var/spack/repos/builtin/packages/mkl/package.py diff --git a/lib/spack/spack/hooks/licensing.py b/lib/spack/spack/hooks/licensing.py index 0f63b0e05ab..f5d3ebd98c2 100644 --- a/lib/spack/spack/hooks/licensing.py +++ b/lib/spack/spack/hooks/licensing.py @@ -26,7 +26,7 @@ import spack import llnl.util.tty as tty -from llnl.util.filesystem import join_path +from llnl.util.filesystem import join_path, mkdirp def pre_install(pkg): @@ -154,6 +154,9 @@ def symlink_license(pkg): target = pkg.global_license_file for filename in pkg.license_files: link_name = join_path(pkg.prefix, filename) + license_dir = os.path.split(link_name)[0] + if not os.path.exists(license_dir): + mkdirp(license_dir) if os.path.exists(target): os.symlink(target, link_name) tty.msg("Added local symlink %s to global license file" % diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 81673411275..cda6a4be188 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -397,14 +397,18 @@ def __init__(self, spec): if self.is_extension: spack.repo.get(self.extendee_spec)._check_extendable() + @property + def global_license_dir(self): + """Returns the path where a global license file should be stored.""" + spack_root = ancestor(__file__, 4) + return join_path(spack_root, 'etc', 'spack', 'licenses') + @property def global_license_file(self): """Returns the path where a global license file should be stored.""" if not self.license_files: return - spack_root = ancestor(__file__, 4) - global_license_dir = join_path(spack_root, 'etc', 'spack', 'licenses') - return join_path(global_license_dir, self.name, + return join_path(self.global_license_dir, self.name, os.path.basename(self.license_files[0])) @property diff --git a/var/spack/repos/builtin/packages/daal/package.py b/var/spack/repos/builtin/packages/daal/package.py new file mode 100644 index 00000000000..16ae048a4cb --- /dev/null +++ b/var/spack/repos/builtin/packages/daal/package.py @@ -0,0 +1,27 @@ +from spack import * +import sys, os, re + +from spack.pkg.builtin.intel import IntelInstaller + +class Daal(IntelInstaller): + """Intel Data Analytics Accesleration Library. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/daal" + + version('2016.2.181', 'aad2aa70e5599ebfe6f85b29d8719d46', + url="file://%s/l_daal_2016.2.181.tgz" % os.getcwd()) + version('2016.3.210', 'ad747c0dd97dace4cad03cf2266cad28', + url="file://%s/l_daal_2016.3.210.tgz" % os.getcwd()) + + def install(self, spec, prefix): + + self.intel_prefix = os.path.join(prefix, "pkg") + IntelInstaller.install(self, spec, prefix) + + daal_dir = os.path.join(self.intel_prefix, "daal") + for f in os.listdir(daal_dir): + os.symlink(os.path.join(daal_dir, f), os.path.join(self.prefix, f)) diff --git a/var/spack/repos/builtin/packages/intel-parallelstudio/package.py b/var/spack/repos/builtin/packages/intel-parallelstudio/package.py new file mode 100644 index 00000000000..127a89654df --- /dev/null +++ b/var/spack/repos/builtin/packages/intel-parallelstudio/package.py @@ -0,0 +1,112 @@ +from spack import * +import sys, os, re + +from spack.pkg.builtin.intel import IntelInstaller, filter_pick + +class IntelParallelstudio(IntelInstaller): + """Intel Parallel Studio. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-parallel-studio-xe" + + # TODO: can also try the online installer (will download files on demand) + version('composer.2016.2', '1133fb831312eb519f7da897fec223fa', + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" % os.getcwd()) + version('professional.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + version('cluster.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + version('composer.2016.3', '3208eeabee951fc27579177b593cefe9', + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" % os.getcwd()) + version('professional.2016.3', 'eda19bb0d0d19709197ede58f13443f3', + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + version('cluster.2016.3', 'eda19bb0d0d19709197ede58f13443f3', + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + + variant('rpath', default=True, description="Add rpath to .cfg files") + variant('all', default=False, description="Install all files associated with the requested edition") + variant('mpi', default=True, description="Install the Intel MPI library and ITAC tool") + variant('mkl', default=True, description="Install the Intel MKL library") + variant('daal', default=True, description="Install the Intel DAAL libraries") + variant('ipp', default=True, description="Install the Intel IPP libraries") + variant('tools', default=True, description="Install the Intel Advisor, VTune Amplifier, and Inspector tools") + + provides('mpi', when='@cluster:+mpi') + provides('mkl', when='+mkl') + provides('daal', when='+daal') + provides('ipp', when='+ipp') + + def install(self, spec, prefix): + + # remove the installation DB, otherwise it will try to install into location of other Intel builds + try: + os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) + except OSError: + pass # if the file does not exist + + base_components = "ALL" # when in doubt, install everything + mpi_components = "" + mkl_components = "" + daal_components = "" + ipp_components = "" + tools_components = "" + + if spec.satisfies('+all'): + base_components = "ALL" + else: + with open("pset/mediaconfig.xml", "r") as f: + lines = f.readlines() + all_components = [] + for line in lines: + if line.find('') != -1: + component = line[line.find('') + 6:line.find('')] + all_components.append(component) + base_components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) + mpi_components = filter_pick(all_components, re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) + mkl_components = filter_pick(all_components, re.compile('(mkl)').search) + daal_components = filter_pick(all_components, re.compile('(daal)').search) + ipp_components = filter_pick(all_components, re.compile('(ipp)').search) + tool_components = filter_pick(all_components, re.compile('(gdb|vtune|inspector|advisor)').search) + + components = base_components + if not spec.satisfies('+all'): + if spec.satisfies('+mpi') and 'cluster' in str(spec.version): + components += mpi_components + if spec.satisfies('+mkl'): + components += mkl_components + if spec.satisfies('+daal'): + components += daal_components + if spec.satisfies('+ipp'): + components += ipp_components + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')): + components += tool_components + + self.intel_components = ';'.join(components) + IntelInstaller.install(self, spec, prefix) + + absbindir = os.path.split(os.path.realpath(os.path.join(self.prefix.bin, "icc")))[0] + abslibdir = os.path.split(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a")))[0] + + # symlink or copy? + os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic")) + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')): + os.mkdir(os.path.join(self.prefix, "inspector_xe/licenses")) + os.symlink(self.global_license_file, os.path.join(self.prefix, "inspector_xe/licenses", "license.lic")) + os.mkdir(os.path.join(self.prefix, "advisor_xe/licenses")) + os.symlink(self.global_license_file, os.path.join(self.prefix, "advisor_xe/licenses", "license.lic")) + os.mkdir(os.path.join(self.prefix, "vtune_amplifier_xe/licenses")) + os.symlink(self.global_license_file, os.path.join(self.prefix, "vtune_amplifier_xe/licenses", "license.lic")) + + if (spec.satisfies('+all') or spec.satisfies('+mpi')) and spec.satisfies('@cluster'): + os.symlink(self.global_license_file, os.path.join(self.prefix, "itac_latest", "license.lic")) + + if spec.satisfies('+rpath'): + for compiler_command in ["icc", "icpc", "ifort"]: + cfgfilename = os.path.join(absbindir, "%s.cfg" %(compiler_command)) + with open(cfgfilename, "w") as f: + f.write('-Xlinker -rpath -Xlinker %s\n' %(abslibdir)) + + os.symlink(os.path.join(self.prefix.man, "common", "man1"), os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/intel/package.py b/var/spack/repos/builtin/packages/intel/package.py new file mode 100644 index 00000000000..e51cc773af4 --- /dev/null +++ b/var/spack/repos/builtin/packages/intel/package.py @@ -0,0 +1,117 @@ +from spack import * +import sys, os, re + +def filter_pick(input_list, regex_filter): + return [l for l in input_list for m in (regex_filter(l),) if m] + +def unfilter_pick(input_list, regex_filter): + return [l for l in input_list for m in (regex_filter(l),) if not m] + +def get_all_components(): + all_components = [] + with open("pset/mediaconfig.xml", "r") as f: + lines = f.readlines() + for line in lines: + if line.find('') != -1: + component = line[line.find('') + 6:line.find('')] + all_components.append(component) + return all_components + +class IntelInstaller(Package): + """Base package containing common methods for installing Intel software""" + + homepage = "https://software.intel.com/en-us" + intel_components = "ALL" + license_required = True + license_comment = '#' + license_files = ['Licenses/license.lic'] + license_vars = ['INTEL_LICENSE_FILE'] + license_url = 'https://software.intel.com/en-us/articles/intel-license-manager-faq' + + @property + def global_license_file(self): + """Returns the path where a global license file should be stored.""" + if not self.license_files: + return + return join_path(self.global_license_dir, "intel", + os.path.basename(self.license_files[0])) + + def install(self, spec, prefix): + + # remove the installation DB, otherwise it will try to install into location of other Intel builds + try: + os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) + except OSError: + pass # if the file does not exist + + if not hasattr(self, "intel_prefix"): + self.intel_prefix = self.prefix + + silent_config_filename = 'silent.cfg' + with open(silent_config_filename, 'w') as f: + f.write(""" +ACCEPT_EULA=accept +PSET_MODE=install +CONTINUE_WITH_INSTALLDIR_OVERWRITE=yes +PSET_INSTALL_DIR=%s +ACTIVATION_LICENSE_FILE=%s +ACTIVATION_TYPE=license_file +PHONEHOME_SEND_USAGE_DATA=no +COMPONENTS=%s +""" %(self.intel_prefix, self.global_license_file, self.intel_components)) + + install_script = which("install.sh") + install_script('--silent', silent_config_filename) + + +class Intel(IntelInstaller): + """Intel Compilers. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-parallel-studio-xe" + + # TODO: can also try the online installer (will download files on demand) + version('16.0.2', '1133fb831312eb519f7da897fec223fa', + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" % os.getcwd()) + version('16.0.3', '3208eeabee951fc27579177b593cefe9', + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" % os.getcwd()) + + variant('rpath', default=True, description="Add rpath to .cfg files") + + def install(self, spec, prefix): + + # remove the installation DB, otherwise it will try to install into location of other Intel builds + try: + os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) + except OSError: + pass # if the file does not exist + + components = [] + with open("pset/mediaconfig.xml", "r") as f: + lines = f.readlines() + all_components = [] + for line in lines: + if line.find('') != -1: + component = line[line.find('') + 6:line.find('')] + all_components.append(component) + components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) + + self.intel_components = ';'.join(components) + IntelInstaller.install(self, spec, prefix) + + absbindir = os.path.split(os.path.realpath(os.path.join(self.prefix.bin, "icc")))[0] + abslibdir = os.path.split(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a")))[0] + + # symlink or copy? + os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic")) + + if spec.satisfies('+rpath'): + for compiler_command in ["icc", "icpc", "ifort"]: + cfgfilename = os.path.join(absbindir, "%s.cfg" %(compiler_command)) + with open(cfgfilename, "w") as f: + f.write('-Xlinker -rpath -Xlinker %s\n' %(abslibdir)) + + os.symlink(os.path.join(self.prefix.man, "common", "man1"), os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/ipp/package.py b/var/spack/repos/builtin/packages/ipp/package.py new file mode 100644 index 00000000000..23c944c2d5a --- /dev/null +++ b/var/spack/repos/builtin/packages/ipp/package.py @@ -0,0 +1,25 @@ +from spack import * +import sys, os, re + +from spack.pkg.builtin.intel import IntelInstaller + +class Ipp(IntelInstaller): + """Intel Integrated Performance Primitives. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-ipp" + + version('9.0.3.210', '0e1520dd3de7f811a6ef6ebc7aa429a3', + url="file://%s/l_ipp_9.0.3.210.tgz" % os.getcwd()) + + def install(self, spec, prefix): + + self.intel_prefix = os.path.join(prefix, "pkg") + IntelInstaller.install(self, spec, prefix) + + ipp_dir = os.path.join(self.intel_prefix, "ipp") + for f in os.listdir(ipp_dir): + os.symlink(os.path.join(ipp_dir, f), os.path.join(self.prefix, f)) diff --git a/var/spack/repos/builtin/packages/mkl/package.py b/var/spack/repos/builtin/packages/mkl/package.py new file mode 100644 index 00000000000..1d4553b7f50 --- /dev/null +++ b/var/spack/repos/builtin/packages/mkl/package.py @@ -0,0 +1,27 @@ +from spack import * +import os, re, sys + +from spack.pkg.builtin.intel import IntelInstaller + +class Mkl(IntelInstaller): + """Intel Math Kernel Library. + + Note: You will have to add the download file to a + mirror so that Spack can find it. For instructions on how to set up a + mirror, see http://software.llnl.gov/spack/mirrors.html""" + + homepage = "https://software.intel.com/en-us/intel-mkl" + + version('11.3.2.181', '536dbd82896d6facc16de8f961d17d65', + url="file://%s/l_mkl_11.3.2.181.tgz" % os.getcwd()) + version('11.3.3.210', 'f72546df27f5ebb0941b5d21fd804e34', + url="file://%s/l_mkl_11.3.3.210.tgz" % os.getcwd()) + + def install(self, spec, prefix): + + self.intel_prefix = os.path.join(prefix, "pkg") + IntelInstaller.install(self, spec, prefix) + + mkl_dir = os.path.join(self.intel_prefix, "mkl") + for f in os.listdir(mkl_dir): + os.symlink(os.path.join(mkl_dir, f), os.path.join(self.prefix, f)) From f43cab0951e5023736d3e332d747e0db212acb72 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 13 May 2016 14:39:20 -0700 Subject: [PATCH 222/375] fixed pkgconfig issue --- lib/spack/spack/build_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 82d0792e264..8016bc4151a 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -232,7 +232,7 @@ def add_env_path(path): pcdir = join_path(p, libdir, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) - path_set("PKG_CONFIG_PATH", pkg_config_dirs) + path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) From fce6ecb05c2aa8f025c27f4245994efef2987b38 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 13 May 2016 15:04:06 -0700 Subject: [PATCH 223/375] Changed _proper_compiler_spec to return filtered list. Apparently filter returns a list rather than filtering the original list. Lead to compiler finding problems if user specified frontend compilers --- lib/spack/spack/concretize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 58ea83ac4ac..9ad5271542c 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -301,8 +301,9 @@ def concretize_compiler(self, spec): # Should think whether this can be more efficient def _proper_compiler_style(cspec, architecture): compilers = spack.compilers.compilers_for_spec(cspec) - filter(lambda c: c.operating_system == architecture.platform_os, compilers) - return compilers + return filter(lambda c: c.operating_system == + architecture.platform_os, compilers) + #return compilers all_compilers = spack.compilers.all_compilers() From e9b71872a8d590d342e4cfcc78ff9855cff7fc24 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Fri, 13 May 2016 15:16:07 -0700 Subject: [PATCH 224/375] addressed PR#946 comments from @adamjstewart --- lib/spack/spack/hooks/licensing.py | 2 +- lib/spack/spack/package.py | 6 ++-- .../repos/builtin/packages/daal/package.py | 2 +- .../package.py | 36 +++++++------------ .../repos/builtin/packages/intel/package.py | 17 ++++----- 5 files changed, 24 insertions(+), 39 deletions(-) rename var/spack/repos/builtin/packages/{intel-parallelstudio => intel-parallel-studio}/package.py (73%) diff --git a/lib/spack/spack/hooks/licensing.py b/lib/spack/spack/hooks/licensing.py index f5d3ebd98c2..9010b841549 100644 --- a/lib/spack/spack/hooks/licensing.py +++ b/lib/spack/spack/hooks/licensing.py @@ -154,7 +154,7 @@ def symlink_license(pkg): target = pkg.global_license_file for filename in pkg.license_files: link_name = join_path(pkg.prefix, filename) - license_dir = os.path.split(link_name)[0] + license_dir = os.path.dirname(link_name) if not os.path.exists(license_dir): mkdirp(license_dir) if os.path.exists(target): diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index cda6a4be188..fa3ece47f71 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -399,13 +399,15 @@ def __init__(self, spec): @property def global_license_dir(self): - """Returns the path where a global license file should be stored.""" + """Returns the directory where global license files for all + packages are stored.""" spack_root = ancestor(__file__, 4) return join_path(spack_root, 'etc', 'spack', 'licenses') @property def global_license_file(self): - """Returns the path where a global license file should be stored.""" + """Returns the path where a global license file for this + particular package should be stored.""" if not self.license_files: return return join_path(self.global_license_dir, self.name, diff --git a/var/spack/repos/builtin/packages/daal/package.py b/var/spack/repos/builtin/packages/daal/package.py index 16ae048a4cb..9fdbd7bf5ec 100644 --- a/var/spack/repos/builtin/packages/daal/package.py +++ b/var/spack/repos/builtin/packages/daal/package.py @@ -4,7 +4,7 @@ from spack.pkg.builtin.intel import IntelInstaller class Daal(IntelInstaller): - """Intel Data Analytics Accesleration Library. + """Intel Data Analytics Acceleration Library. Note: You will have to add the download file to a mirror so that Spack can find it. For instructions on how to set up a diff --git a/var/spack/repos/builtin/packages/intel-parallelstudio/package.py b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py similarity index 73% rename from var/spack/repos/builtin/packages/intel-parallelstudio/package.py rename to var/spack/repos/builtin/packages/intel-parallel-studio/package.py index 127a89654df..3a44cb32a3f 100644 --- a/var/spack/repos/builtin/packages/intel-parallelstudio/package.py +++ b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py @@ -1,9 +1,9 @@ from spack import * import sys, os, re -from spack.pkg.builtin.intel import IntelInstaller, filter_pick +from spack.pkg.builtin.intel import IntelInstaller, filter_pick, get_all_components -class IntelParallelstudio(IntelInstaller): +class IntelParallelStudio(IntelInstaller): """Intel Parallel Studio. Note: You will have to add the download file to a @@ -41,12 +41,6 @@ class IntelParallelstudio(IntelInstaller): def install(self, spec, prefix): - # remove the installation DB, otherwise it will try to install into location of other Intel builds - try: - os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) - except OSError: - pass # if the file does not exist - base_components = "ALL" # when in doubt, install everything mpi_components = "" mkl_components = "" @@ -57,19 +51,13 @@ def install(self, spec, prefix): if spec.satisfies('+all'): base_components = "ALL" else: - with open("pset/mediaconfig.xml", "r") as f: - lines = f.readlines() - all_components = [] - for line in lines: - if line.find('') != -1: - component = line[line.find('') + 6:line.find('')] - all_components.append(component) - base_components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) - mpi_components = filter_pick(all_components, re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) - mkl_components = filter_pick(all_components, re.compile('(mkl)').search) - daal_components = filter_pick(all_components, re.compile('(daal)').search) - ipp_components = filter_pick(all_components, re.compile('(ipp)').search) - tool_components = filter_pick(all_components, re.compile('(gdb|vtune|inspector|advisor)').search) + all_components = get_all_components() + base_components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) + mpi_components = filter_pick(all_components, re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) + mkl_components = filter_pick(all_components, re.compile('(mkl)').search) + daal_components = filter_pick(all_components, re.compile('(daal)').search) + ipp_components = filter_pick(all_components, re.compile('(ipp)').search) + tool_components = filter_pick(all_components, re.compile('(gdb|vtune|inspector|advisor)').search) components = base_components if not spec.satisfies('+all'): @@ -87,10 +75,10 @@ def install(self, spec, prefix): self.intel_components = ';'.join(components) IntelInstaller.install(self, spec, prefix) - absbindir = os.path.split(os.path.realpath(os.path.join(self.prefix.bin, "icc")))[0] - abslibdir = os.path.split(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a")))[0] + absbindir = os.path.dirname(os.path.realpath(os.path.join(self.prefix.bin, "icc"))) + abslibdir = os.path.dirname(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a"))) - # symlink or copy? + relbindir = absbindir.strip(os.path.commonprefix([self.prefix, absbindir])) os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic")) if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')): os.mkdir(os.path.join(self.prefix, "inspector_xe/licenses")) diff --git a/var/spack/repos/builtin/packages/intel/package.py b/var/spack/repos/builtin/packages/intel/package.py index e51cc773af4..a547ab56db6 100644 --- a/var/spack/repos/builtin/packages/intel/package.py +++ b/var/spack/repos/builtin/packages/intel/package.py @@ -2,12 +2,15 @@ import sys, os, re def filter_pick(input_list, regex_filter): + """Returns the items in input_list that are found in the regex_filter""" return [l for l in input_list for m in (regex_filter(l),) if m] def unfilter_pick(input_list, regex_filter): + """Returns the items in input_list that are not found in the regex_filter""" return [l for l in input_list for m in (regex_filter(l),) if not m] def get_all_components(): + """Returns a list of all the components associated with the downloaded Intel package""" all_components = [] with open("pset/mediaconfig.xml", "r") as f: lines = f.readlines() @@ -39,10 +42,8 @@ def global_license_file(self): def install(self, spec, prefix): # remove the installation DB, otherwise it will try to install into location of other Intel builds - try: + if os.path.exists(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")): os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) - except OSError: - pass # if the file does not exist if not hasattr(self, "intel_prefix"): self.intel_prefix = self.prefix @@ -90,14 +91,8 @@ def install(self, spec, prefix): pass # if the file does not exist components = [] - with open("pset/mediaconfig.xml", "r") as f: - lines = f.readlines() - all_components = [] - for line in lines: - if line.find('') != -1: - component = line[line.find('') + 6:line.find('')] - all_components.append(component) - components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) + all_components = get_all_components() + components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) self.intel_components = ';'.join(components) IntelInstaller.install(self, spec, prefix) From e3cd0a67d020b73e5fdc6cfeb4ee874d49f2c305 Mon Sep 17 00:00:00 2001 From: "Gregory L. Lee" Date: Fri, 13 May 2016 16:12:48 -0700 Subject: [PATCH 225/375] address flake8 issues --- .../repos/builtin/packages/daal/package.py | 3 +- .../packages/intel-parallel-studio/package.py | 93 ++++++++++++------- .../repos/builtin/packages/intel/package.py | 60 +++++++----- .../repos/builtin/packages/ipp/package.py | 3 +- .../repos/builtin/packages/mkl/package.py | 3 +- 5 files changed, 103 insertions(+), 59 deletions(-) diff --git a/var/spack/repos/builtin/packages/daal/package.py b/var/spack/repos/builtin/packages/daal/package.py index 9fdbd7bf5ec..e13dd80e196 100644 --- a/var/spack/repos/builtin/packages/daal/package.py +++ b/var/spack/repos/builtin/packages/daal/package.py @@ -1,8 +1,9 @@ from spack import * -import sys, os, re +import os from spack.pkg.builtin.intel import IntelInstaller + class Daal(IntelInstaller): """Intel Data Analytics Acceleration Library. diff --git a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py index 3a44cb32a3f..db6cc888ddf 100644 --- a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py +++ b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py @@ -1,8 +1,10 @@ from spack import * -import sys, os, re +import os +import re from spack.pkg.builtin.intel import IntelInstaller, filter_pick, get_all_components + class IntelParallelStudio(IntelInstaller): """Intel Parallel Studio. @@ -14,25 +16,31 @@ class IntelParallelStudio(IntelInstaller): # TODO: can also try the online installer (will download files on demand) version('composer.2016.2', '1133fb831312eb519f7da897fec223fa', - url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" + % os.getcwd()) version('professional.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', - url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) version('cluster.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', - url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) version('composer.2016.3', '3208eeabee951fc27579177b593cefe9', - url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" + % os.getcwd()) version('professional.2016.3', 'eda19bb0d0d19709197ede58f13443f3', - url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) version('cluster.2016.3', 'eda19bb0d0d19709197ede58f13443f3', - url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) variant('rpath', default=True, description="Add rpath to .cfg files") - variant('all', default=False, description="Install all files associated with the requested edition") - variant('mpi', default=True, description="Install the Intel MPI library and ITAC tool") + variant('all', default=False, + description="Install all files associated with the requested edition") + variant('mpi', default=True, + description="Install the Intel MPI library and ITAC tool") variant('mkl', default=True, description="Install the Intel MKL library") - variant('daal', default=True, description="Install the Intel DAAL libraries") + variant('daal', + default=True, description="Install the Intel DAAL libraries") variant('ipp', default=True, description="Install the Intel IPP libraries") - variant('tools', default=True, description="Install the Intel Advisor, VTune Amplifier, and Inspector tools") + variant('tools', default=True, description="""Install the Intel Advisor,\ +VTune Amplifier, and Inspector tools""") provides('mpi', when='@cluster:+mpi') provides('mkl', when='+mkl') @@ -41,23 +49,29 @@ class IntelParallelStudio(IntelInstaller): def install(self, spec, prefix): - base_components = "ALL" # when in doubt, install everything + base_components = "ALL" # when in doubt, install everything mpi_components = "" mkl_components = "" daal_components = "" ipp_components = "" - tools_components = "" if spec.satisfies('+all'): base_components = "ALL" else: all_components = get_all_components() - base_components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) - mpi_components = filter_pick(all_components, re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) - mkl_components = filter_pick(all_components, re.compile('(mkl)').search) - daal_components = filter_pick(all_components, re.compile('(daal)').search) - ipp_components = filter_pick(all_components, re.compile('(ipp)').search) - tool_components = filter_pick(all_components, re.compile('(gdb|vtune|inspector|advisor)').search) + base_components = filter_pick(all_components, + re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' + ).search) + mpi_components = filter_pick(all_components, + re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) + mkl_components = filter_pick(all_components, + re.compile('(mkl)').search) + daal_components = filter_pick(all_components, + re.compile('(daal)').search) + ipp_components = filter_pick(all_components, + re.compile('(ipp)').search) + tool_components = filter_pick(all_components, + re.compile('(gdb|vtune|inspector|advisor)').search) components = base_components if not spec.satisfies('+all'): @@ -69,32 +83,45 @@ def install(self, spec, prefix): components += daal_components if spec.satisfies('+ipp'): components += ipp_components - if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')): + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or\ + spec.satisfies('@professional')): components += tool_components self.intel_components = ';'.join(components) IntelInstaller.install(self, spec, prefix) - absbindir = os.path.dirname(os.path.realpath(os.path.join(self.prefix.bin, "icc"))) - abslibdir = os.path.dirname(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a"))) + absbindir = os.path.dirname(os.path.realpath(os.path.join( + self.prefix.bin, "icc"))) + abslibdir = os.path.dirname(os.path.realpath(os.path.join + (self.prefix.lib, "intel64", "libimf.a"))) - relbindir = absbindir.strip(os.path.commonprefix([self.prefix, absbindir])) - os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic")) - if spec.satisfies('+tools') and (spec.satisfies('@cluster') or spec.satisfies('@professional')): + relbindir = absbindir.strip(os.path.commonprefix([self.prefix, + absbindir])) + os.symlink(self.global_license_file, os.path.join(absbindir, + "license.lic")) + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or\ + spec.satisfies('@professional')): os.mkdir(os.path.join(self.prefix, "inspector_xe/licenses")) - os.symlink(self.global_license_file, os.path.join(self.prefix, "inspector_xe/licenses", "license.lic")) + os.symlink(self.global_license_file, os.path.join( + self.prefix, "inspector_xe/licenses", "license.lic")) os.mkdir(os.path.join(self.prefix, "advisor_xe/licenses")) - os.symlink(self.global_license_file, os.path.join(self.prefix, "advisor_xe/licenses", "license.lic")) + os.symlink(self.global_license_file, os.path.join( + self.prefix, "advisor_xe/licenses", "license.lic")) os.mkdir(os.path.join(self.prefix, "vtune_amplifier_xe/licenses")) - os.symlink(self.global_license_file, os.path.join(self.prefix, "vtune_amplifier_xe/licenses", "license.lic")) + os.symlink(self.global_license_file, os.path.join( + self.prefix, "vtune_amplifier_xe/licenses", "license.lic")) - if (spec.satisfies('+all') or spec.satisfies('+mpi')) and spec.satisfies('@cluster'): - os.symlink(self.global_license_file, os.path.join(self.prefix, "itac_latest", "license.lic")) + if (spec.satisfies('+all') or spec.satisfies('+mpi')) and\ + spec.satisfies('@cluster'): + os.symlink(self.global_license_file, os.path.join( + self.prefix, "itac_latest", "license.lic")) if spec.satisfies('+rpath'): for compiler_command in ["icc", "icpc", "ifort"]: - cfgfilename = os.path.join(absbindir, "%s.cfg" %(compiler_command)) + cfgfilename = os.path.join(absbindir, "%s.cfg" %\ + compiler_command) with open(cfgfilename, "w") as f: - f.write('-Xlinker -rpath -Xlinker %s\n' %(abslibdir)) + f.write('-Xlinker -rpath -Xlinker %s\n' % abslibdir) - os.symlink(os.path.join(self.prefix.man, "common", "man1"), os.path.join(self.prefix.man, "man1")) + os.symlink(os.path.join(self.prefix.man, "common", "man1"), + os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/intel/package.py b/var/spack/repos/builtin/packages/intel/package.py index a547ab56db6..2d6929fc732 100644 --- a/var/spack/repos/builtin/packages/intel/package.py +++ b/var/spack/repos/builtin/packages/intel/package.py @@ -1,16 +1,22 @@ from spack import * -import sys, os, re +import os +import re + def filter_pick(input_list, regex_filter): """Returns the items in input_list that are found in the regex_filter""" return [l for l in input_list for m in (regex_filter(l),) if m] + def unfilter_pick(input_list, regex_filter): - """Returns the items in input_list that are not found in the regex_filter""" + """Returns the items in input_list that are not found in the + regex_filter""" return [l for l in input_list for m in (regex_filter(l),) if not m] + def get_all_components(): - """Returns a list of all the components associated with the downloaded Intel package""" + """Returns a list of all the components associated with the downloaded + Intel package""" all_components = [] with open("pset/mediaconfig.xml", "r") as f: lines = f.readlines() @@ -20,6 +26,7 @@ def get_all_components(): all_components.append(component) return all_components + class IntelInstaller(Package): """Base package containing common methods for installing Intel software""" @@ -29,7 +36,8 @@ class IntelInstaller(Package): license_comment = '#' license_files = ['Licenses/license.lic'] license_vars = ['INTEL_LICENSE_FILE'] - license_url = 'https://software.intel.com/en-us/articles/intel-license-manager-faq' + license_url = \ + 'https://software.intel.com/en-us/articles/intel-license-manager-faq' @property def global_license_file(self): @@ -41,9 +49,12 @@ def global_license_file(self): def install(self, spec, prefix): - # remove the installation DB, otherwise it will try to install into location of other Intel builds - if os.path.exists(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")): - os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) + # Remove the installation DB, otherwise it will try to install into + # location of other Intel builds + if os.path.exists(os.path.join(os.environ["HOME"], "intel", + "intel_sdp_products.db")): + os.remove(os.path.join(os.environ["HOME"], "intel", + "intel_sdp_products.db")) if not hasattr(self, "intel_prefix"): self.intel_prefix = self.prefix @@ -58,6 +69,7 @@ def install(self, spec, prefix): ACTIVATION_LICENSE_FILE=%s ACTIVATION_TYPE=license_file PHONEHOME_SEND_USAGE_DATA=no +CONTINUE_WITH_OPTIONAL_ERROR=yes COMPONENTS=%s """ %(self.intel_prefix, self.global_license_file, self.intel_components)) @@ -76,37 +88,39 @@ class Intel(IntelInstaller): # TODO: can also try the online installer (will download files on demand) version('16.0.2', '1133fb831312eb519f7da897fec223fa', - url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz"\ + % os.getcwd()) version('16.0.3', '3208eeabee951fc27579177b593cefe9', - url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz"\ + % os.getcwd()) variant('rpath', default=True, description="Add rpath to .cfg files") def install(self, spec, prefix): - - # remove the installation DB, otherwise it will try to install into location of other Intel builds - try: - os.remove(os.path.join(os.environ["HOME"], "intel", "intel_sdp_products.db")) - except OSError: - pass # if the file does not exist - components = [] all_components = get_all_components() - components = filter_pick(all_components, re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)').search) + components = filter_pick(all_components, + re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' + ).search) self.intel_components = ';'.join(components) IntelInstaller.install(self, spec, prefix) - absbindir = os.path.split(os.path.realpath(os.path.join(self.prefix.bin, "icc")))[0] - abslibdir = os.path.split(os.path.realpath(os.path.join(self.prefix.lib, "intel64", "libimf.a")))[0] + absbindir = os.path.split(os.path.realpath(os.path.join( + self.prefix.bin, "icc")))[0] + abslibdir = os.path.split(os.path.realpath(os.path.join( + self.prefix.lib, "intel64", "libimf.a")))[0] # symlink or copy? - os.symlink(self.global_license_file, os.path.join(absbindir, "license.lic")) + os.symlink(self.global_license_file, os.path.join(absbindir, + "license.lic")) if spec.satisfies('+rpath'): for compiler_command in ["icc", "icpc", "ifort"]: - cfgfilename = os.path.join(absbindir, "%s.cfg" %(compiler_command)) + cfgfilename = os.path.join(absbindir, "%s.cfg" %\ + compiler_command) with open(cfgfilename, "w") as f: - f.write('-Xlinker -rpath -Xlinker %s\n' %(abslibdir)) + f.write('-Xlinker -rpath -Xlinker %s\n' % abslibdir) - os.symlink(os.path.join(self.prefix.man, "common", "man1"), os.path.join(self.prefix.man, "man1")) + os.symlink(os.path.join(self.prefix.man, "common", "man1"), + os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/ipp/package.py b/var/spack/repos/builtin/packages/ipp/package.py index 23c944c2d5a..2bd931d5bd8 100644 --- a/var/spack/repos/builtin/packages/ipp/package.py +++ b/var/spack/repos/builtin/packages/ipp/package.py @@ -1,8 +1,9 @@ from spack import * -import sys, os, re +import os from spack.pkg.builtin.intel import IntelInstaller + class Ipp(IntelInstaller): """Intel Integrated Performance Primitives. diff --git a/var/spack/repos/builtin/packages/mkl/package.py b/var/spack/repos/builtin/packages/mkl/package.py index 1d4553b7f50..454e78d29ca 100644 --- a/var/spack/repos/builtin/packages/mkl/package.py +++ b/var/spack/repos/builtin/packages/mkl/package.py @@ -1,8 +1,9 @@ from spack import * -import os, re, sys +import os from spack.pkg.builtin.intel import IntelInstaller + class Mkl(IntelInstaller): """Intel Math Kernel Library. From 8ddc1f89778335dda3d1dd213e75bc3a92d99712 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 09:11:04 -0400 Subject: [PATCH 226/375] Move from str.format() to string.Template. --- lib/spack/spack/cmd/view.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index b504dfd2b15..caf14bfa534 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -317,16 +317,19 @@ def visitor_statlink(specs, args): def visitor_print(specs, args): 'Print a string for each spec using args.format.' fmt = args.format[0] + from string import Template + t = Template(fmt) + for spec in specs: kwds = spec2dict(spec) try: - string = fmt.format(**kwds) + text = t.substitute(kwds) except KeyError: tty.error("Format error, use keywords: %s" % (', '.join(kwds.keys()), )) raise # argparser escapes these - string = string.replace(r'\n', '\n').replace(r'\t', '\t') - sys.stdout.write(string) + text = text.replace(r'\n', '\n').replace(r'\t', '\t') + sys.stdout.write(text) # Finally, the actual "view" command. There should be no need to From 8078845273021dae01e3fdde2a5e3b10aea2f845 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 09:11:41 -0400 Subject: [PATCH 227/375] Remove problematic tests, deal with this issue outside of PR #869. --- share/spack/examples/test_view.sh | 31 ---------------- .../builtin.mock/packages/test-a/package.py | 30 ---------------- .../packages/test-a/test-a-0.0.tar.gz | Bin 112 -> 0 bytes .../builtin.mock/packages/test-b/package.py | 32 ----------------- .../packages/test-b/test-b-0.0.tar.gz | Bin 112 -> 0 bytes .../builtin.mock/packages/test-c/package.py | 32 ----------------- .../packages/test-c/test-c-0.0.tar.gz | Bin 112 -> 0 bytes .../builtin.mock/packages/test-d/package.py | 33 ------------------ .../packages/test-d/test-d-0.0.tar.gz | Bin 112 -> 0 bytes 9 files changed, 158 deletions(-) delete mode 100755 share/spack/examples/test_view.sh delete mode 100644 var/spack/repos/builtin.mock/packages/test-a/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz delete mode 100644 var/spack/repos/builtin.mock/packages/test-b/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz delete mode 100644 var/spack/repos/builtin.mock/packages/test-c/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz delete mode 100644 var/spack/repos/builtin.mock/packages/test-d/package.py delete mode 100644 var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz diff --git a/share/spack/examples/test_view.sh b/share/spack/examples/test_view.sh deleted file mode 100755 index b3e71021924..00000000000 --- a/share/spack/examples/test_view.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# This will install a few bogus/test packages in order to test the -# `spack view` command. It assumes you have "spack" in your path. - -# It makes sub-directories in your CWD and installs and uninstalls -# Spack packages named test-*. - -set -x -set -e - -view="spack -m view -v" -for variant in +nom ~nom+var +nom+var -do - spack -m uninstall -f -a -y test-d - spack -m install test-d$variant - testdir=test_view - rm -rf $testdir - echo "hardlink may fail if Spack install area and CWD are not same FS" - for action in symlink hardlink - do - $view --dependencies=no $action $testdir test-d - $view -e test-a -e test-b $action $testdir test-d - $view $action $testdir test-d - $view status $testdir test-d - $view -d false remove $testdir test-a - $view remove $testdir test-d - rmdir $testdir # should not fail - done -done -echo "Warnings about skipping existing in the above are okay" diff --git a/var/spack/repos/builtin.mock/packages/test-a/package.py b/var/spack/repos/builtin.mock/packages/test-a/package.py deleted file mode 100644 index ff45344b7af..00000000000 --- a/var/spack/repos/builtin.mock/packages/test-a/package.py +++ /dev/null @@ -1,30 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-a-0.0.tar.gz') - -class TestA(Package): - """The test-a package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-a') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-a/test-a-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX diff --git a/var/spack/repos/builtin.mock/packages/test-b/package.py b/var/spack/repos/builtin.mock/packages/test-b/package.py deleted file mode 100644 index f6d30bb85ff..00000000000 --- a/var/spack/repos/builtin.mock/packages/test-b/package.py +++ /dev/null @@ -1,32 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-b-0.0.tar.gz') - -class TestB(Package): - """The test-b package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - depends_on('test-a') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-b') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-b/test-b-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX diff --git a/var/spack/repos/builtin.mock/packages/test-c/package.py b/var/spack/repos/builtin.mock/packages/test-c/package.py deleted file mode 100644 index 641e46ee507..00000000000 --- a/var/spack/repos/builtin.mock/packages/test-c/package.py +++ /dev/null @@ -1,32 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-c-0.0.tar.gz') - -class TestC(Package): - """The test-c package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - depends_on('test-a+var',when='+var') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-c') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-c/test-c-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX diff --git a/var/spack/repos/builtin.mock/packages/test-d/package.py b/var/spack/repos/builtin.mock/packages/test-d/package.py deleted file mode 100644 index b46c680eddd..00000000000 --- a/var/spack/repos/builtin.mock/packages/test-d/package.py +++ /dev/null @@ -1,33 +0,0 @@ -from spack import * - -import os - -mydir = os.path.dirname(__file__) -source = os.path.join(mydir,'test-d-0.0.tar.gz') - -class TestD(Package): - """The test-d package""" - - url = 'file://'+source - homepage = "http://www.example.com/" - - version('0.0', '4e823d0af4154fcf52b75dad41b7fd63') - - variant('nom', default=True, description='Nominal variant') - variant('var', default=False, description='Variant variant') - - depends_on('test-b') - depends_on('test-c') - - def install(self, spec, prefix): - bindir = os.path.join(prefix,'bin') - os.makedirs(bindir) - script = os.path.join(bindir, 'test-d') - with open(script,'w') as fp: - fp.write("""#!/bin/bash -echo 'name: %s' -echo 'prefix: %s' -echo 'spec: %s' - """ % (spec.name, prefix, spec)) - - os.chmod(script, 0555) diff --git a/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz b/var/spack/repos/builtin.mock/packages/test-d/test-d-0.0.tar.gz deleted file mode 100644 index 3588e05765cd799dfe19e1f77e057bf7508ec47e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112 zcmb2|=3uzVq!!M={Pvt8-ysJPmJ7D0<~95|Kkb{1f}-PrX$3W18BI)~SE|#F^vw8N zv-i!f&C_|0CdssyKPx$LK4$72|C!Dyr^A#M>z4NF`mcRfZGSIIclq(}ch}1?z`(y3 OtXZo*ykgK`U;qF>c`_gX From c98afe2205ec90bc98c7f1c33b3e8e05403c7c22 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 13:14:51 -0400 Subject: [PATCH 228/375] Pick a better name for the 'spec' parameters. --- lib/spack/spack/cmd/view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index caf14bfa534..8767459c084 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -189,8 +189,8 @@ def spec2dict(spec): # pkg = spec.package ret = dict(name = spec.name, - short = spec.short_spec, - cshort = spec.cshort_spec, # color + spec = spec.short_spec, + colorspec = spec.cshort_spec, # color root = spec.root, prefix = spec.prefix, version = spec.version, From b3ede099e2d2dd684ebe478dd711a979ebb3bf98 Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Sat, 14 May 2016 13:15:25 -0400 Subject: [PATCH 229/375] Document command. --- lib/spack/docs/basic_usage.rst | 233 +++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 29791d98c4a..e340881a930 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -330,6 +330,125 @@ of libelf would look like this: The full spec syntax is discussed in detail in :ref:`sec-specs`. +``spack view print`` +~~~~~~~~~~~~~~~~~~~~~~ + +``spack view print`` is a subcommand of ``spack view`` which displays information about one or more installed packages and their dependencies using a user-provided format string. The string can reverence variables in a shell-like manner for example ``$variable`` or ``${variable}text``. It can also include ``\t`` for tabs and ``\n`` for new lines. + +Some of the supported variables are: + +``name`` + The package name. + +``version`` + The package version. + +``spec`` + The package specification. + +``root`` + The root specification. + +``prefix`` + The installation directory. + +``variants`` + The collection of variants, if any. + +``namespace`` + The package repository name space. + +``compiler`` + The compiler \`name@versoin\` used to build the package. + +``architecture`` + The architecture targeted by the compiler. + +``dependencies`` + A comma-separated list of names of packages on which the package depends. + +``dependents`` + A comma-separated list of names of packages which depend on the package. + +``hash`` + The Spack hash for the package. + +``url`` + The source URL for the package. + +``stage`` + The directory for staging the build. + +``build_log`` + The path to the build log file. + +``rpath`` + The colon-separated library \`RPATH\` used in building the package. + +Here are some example uses of `spack view print`. A simple line-oriented report of information can be produced: + +.. code-block:: sh + + $ spack view print '$name\t$version\t$hash\n' cmake@3.5.2 + ncurses 6.0 bvbu4ixbnvtodpik4qzljlx3ukpyfrcz + zlib 1.2.8 ckki7zlryxrsetfqkgoxxahlhqqjni7n + openssl 1.0.2g 6zbar63sciso253nptxyrnhupymo7oyi + cmake 3.5.2 wprvmoczkpw4tiy5ybuk5zr7saus2d7g + +There are better ways to do this but a slow-and-dirty shell init procedure can be formed: + +.. code-block:: sh + + $ spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4 + export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/bin:$PATH" + export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/bin:$PATH" + $ eval $(spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4) + +Or, maybe you have some external application that can chew on Spack data in which case you can dump it to some easily parsed markup syntax such as YAML: + +.. code-block:: sh + + + $ spack view print '${name}:\n - version: ${version}\n - url : ${url}\n - spec: ${spec}\n - prefix: ${prefix}\n - root : ${root}\n - stage : ${stage}\n - log: ${build_log}\n' m4 + +Which might produce something like: + +.. code-block:: yaml + + libsigsegv: + - version: 2.10 + - url : ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz + - spec: libsigsegv@2.10%gcc@5.2.1=linux-x86_64-h6hsv76 + - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 + - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 + - stage : /spack/var/spack/stage/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 + - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/.spack/build.out + m4: + - version: 1.4.17 + - url : ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz + - spec: m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64-a4ikhdd + - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp + - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 + - stage : /spack/var/spack/stage/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp + - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/.spack/build.out + +Or, maybe you want to do something with information about package dependencies by generating some ready-to-import Python code holding the tree: + +.. code-block:: sh + + $ spack view print '$name = dict(name = "$name", parents = [$dependencies], children = [$dependents])\n' cmake@3.5.2 + +Producing this Python code + +.. code-block:: python + + ncurses = dict(name = "ncurses", parents = []]) + zlib = dict(name = "zlib", parents = []]) + openssl = dict(name = "openssl", parents = [zlib]]) + cmake = dict(name = "cmake", parents = [ncurses,openssl]]) + + + Compiler configuration ----------------------------------- @@ -1035,6 +1154,120 @@ regenerate all module and dotkit files from scratch: .. _extensions: +Filesystem Views +------------------------------- + +.. Maybe this is not the right location for this documentation. + +The Spack installation area allows for many package installation trees +to coexist and gives the user choices as to what versions and variants +of packages to use. To use them, the user must rely on a way to +aggregate a subset of those packages. The section on Environment +Modules gives one good way to do that which relies on setting various +environment variables. An alternative way to aggregate is through +**filesystem views**. + +A filesystem view is a single directory tree which is the union of the +directory hierarchies of the individual package installation trees +that have been included. The files of the view's installed packages +are brought into the view by symbolic or hard links back to their +location in the original Spack installation area. As the view is +formed, any clashes due to a file having the exact same path in its +package installation tree are handled in a first-come-first-served +basis and a warning is printed. Packages and their dependencies can +be both added and removed. During removal, empty directories will be +purged. These operations can be limited to pertain to just the +packages listed by the user or to exclude specific dependencies and +they allow for software installed outside of Spack to coexist inside +the filesystem view tree. + +By its nature, a filesystem view represents a particular choice of one +set of packages among all the versions and variants that are available +in the Spack installation area. It is thus equivalent to the +directory hiearchy that might exist under ``/usr/local``. While this +limits a view to including only one version/variant of any package, it +provides the benefits of having a simpler and traditional layout which +may be used without any particular knowledge that its packages were +built by Spack. + +Views can be used for a variety of purposes including: + +- A central installation in a traditional layout, eg ``/usr/local`` maintained over time by the sysadmin. +- A self-contained installation area which may for the basis of a top-level atomic versioning scheme, eg ``/opt/pro`` vs ``/opt/dev``. +- Providing an atomic and monolithic binary distribution, eg for delivery as a single tarball. +- Producing ephemeral testing or developing environments. + +Using Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~ + +A filesystem view is created and packages are linked in by the ``spack +view`` command's ``symlink`` and ``hardlink`` sub-commands. The +``spack view remove`` command can be used to unlink some or all of the +filesystem view. + +The following example creates a filesystem view based +on an installed ``cmake`` package and then removes from the view the +files in the ``cmake`` package while retaining its dependencies. + +.. code-block:: sh + + + $ spack view -v symlink myview cmake@3.5.2 + ==> Linking package: "ncurses" + ==> Linking package: "zlib" + ==> Linking package: "openssl" + ==> Linking package: "cmake" + + $ ls myview/ + bin doc etc include lib share + + $ ls myview/bin/ + captoinfo clear cpack ctest infotocap openssl tabs toe tset + ccmake cmake c_rehash infocmp ncurses6-config reset tic tput + + $ spack view -v -d false rm myview cmake@3.5.2 + ==> Removing package: "cmake" + + $ ls myview/bin/ + captoinfo c_rehash infotocap openssl tabs toe tset + clear infocmp ncurses6-config reset tic tput + + +Limitations of Filesystem Views +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section describes some limitations that should be considered in +using filesystems views. + +Filesystem views are merely organizational. The binary executable +programs, shared libraries and other build products found in a view +are mere links into the "real" Spack installation area. If a view is +built with symbolic links it requires the Spack-installed package to +be kept in place. Building a view with hardlinks removes this +requirement but any internal paths (eg, rpath or ``#!`` interpreter +specifications) will still require the Spack-installed package files +to be in place. + +.. FIXME: reference the relocation work of Hegner and Gartung. + +As described above, when a view is built only a single instance of a +file may exist in the unified filesystem tree. If more than one +package provides a file at the same path (relative to its own root) +then it is the first package added to the view that "wins". A warning +is printed and it is up to the user to determine if the conflict +matters. + +It is up to the user to assure a consistent view is produced. In +particular if the user excludes packages, limits the following of +dependencies or removes packages the view may become inconsistent. In +particular, if two packages require the same sub-tree of dependencies, +removing one package (recursively) will remove its dependencies and +leave the other package broken. + + + + + Extensions & Python support ------------------------------------ From c2ca5f44b4091821fbda36ce2dce77af3e4087f5 Mon Sep 17 00:00:00 2001 From: Greg Lee Date: Sun, 15 May 2016 08:19:55 -0700 Subject: [PATCH 230/375] flake8 fixes --- .../packages/intel-parallel-studio/package.py | 77 ++++++++++--------- .../repos/builtin/packages/intel/package.py | 23 +++--- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py index db6cc888ddf..a176af101d3 100644 --- a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py +++ b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py @@ -2,7 +2,8 @@ import os import re -from spack.pkg.builtin.intel import IntelInstaller, filter_pick, get_all_components +from spack.pkg.builtin.intel import IntelInstaller, filter_pick, \ + get_all_components class IntelParallelStudio(IntelInstaller): @@ -16,28 +17,28 @@ class IntelParallelStudio(IntelInstaller): # TODO: can also try the online installer (will download files on demand) version('composer.2016.2', '1133fb831312eb519f7da897fec223fa', - url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" # NOQA: ignore=E501 % os.getcwd()) version('professional.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', - url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) # NOQA: ignore=E501 version('cluster.2016.2', '70be832f2d34c9bf596a5e99d5f2d832', - url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update2.tgz" % os.getcwd()) # NOQA: ignore=E501 version('composer.2016.3', '3208eeabee951fc27579177b593cefe9', - url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" # NOQA: ignore=E501 % os.getcwd()) version('professional.2016.3', 'eda19bb0d0d19709197ede58f13443f3', - url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) # NOQA: ignore=E501 version('cluster.2016.3', 'eda19bb0d0d19709197ede58f13443f3', - url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) + url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) # NOQA: ignore=E501 variant('rpath', default=True, description="Add rpath to .cfg files") variant('all', default=False, - description="Install all files associated with the requested edition") + description="Install all files with the requested edition") variant('mpi', default=True, - description="Install the Intel MPI library and ITAC tool") + description="Install the Intel MPI library and ITAC tool") variant('mkl', default=True, description="Install the Intel MKL library") variant('daal', - default=True, description="Install the Intel DAAL libraries") + default=True, description="Install the Intel DAAL libraries") variant('ipp', default=True, description="Install the Intel IPP libraries") variant('tools', default=True, description="""Install the Intel Advisor,\ VTune Amplifier, and Inspector tools""") @@ -59,19 +60,21 @@ def install(self, spec, prefix): base_components = "ALL" else: all_components = get_all_components() - base_components = filter_pick(all_components, - re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' - ).search) - mpi_components = filter_pick(all_components, - re.compile('(icsxe|imb|mpi|itac|intel-tc|clck)').search) - mkl_components = filter_pick(all_components, - re.compile('(mkl)').search) - daal_components = filter_pick(all_components, - re.compile('(daal)').search) - ipp_components = filter_pick(all_components, - re.compile('(ipp)').search) - tool_components = filter_pick(all_components, - re.compile('(gdb|vtune|inspector|advisor)').search) + regex = '(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' + base_components = \ + filter_pick(all_components, re.compile(regex).search) + regex = '(icsxe|imb|mpi|itac|intel-tc|clck)' + mpi_components = \ + filter_pick(all_components, re.compile(regex).search) + mkl_components = \ + filter_pick(all_components, re.compile('(mkl)').search) + daal_components = \ + filter_pick(all_components, re.compile('(daal)').search) + ipp_components = \ + filter_pick(all_components, re.compile('(ipp)').search) + regex = '(gdb|vtune|inspector|advisor)' + tool_components = \ + filter_pick(all_components, re.compile(regex).search) components = base_components if not spec.satisfies('+all'): @@ -83,8 +86,8 @@ def install(self, spec, prefix): components += daal_components if spec.satisfies('+ipp'): components += ipp_components - if spec.satisfies('+tools') and (spec.satisfies('@cluster') or\ - spec.satisfies('@professional')): + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or + spec.satisfies('@professional')): components += tool_components self.intel_components = ';'.join(components) @@ -93,14 +96,12 @@ def install(self, spec, prefix): absbindir = os.path.dirname(os.path.realpath(os.path.join( self.prefix.bin, "icc"))) abslibdir = os.path.dirname(os.path.realpath(os.path.join - (self.prefix.lib, "intel64", "libimf.a"))) + (self.prefix.lib, "intel64", "libimf.a"))) - relbindir = absbindir.strip(os.path.commonprefix([self.prefix, - absbindir])) os.symlink(self.global_license_file, os.path.join(absbindir, - "license.lic")) - if spec.satisfies('+tools') and (spec.satisfies('@cluster') or\ - spec.satisfies('@professional')): + "license.lic")) + if spec.satisfies('+tools') and (spec.satisfies('@cluster') or + spec.satisfies('@professional')): os.mkdir(os.path.join(self.prefix, "inspector_xe/licenses")) os.symlink(self.global_license_file, os.path.join( self.prefix, "inspector_xe/licenses", "license.lic")) @@ -111,17 +112,17 @@ def install(self, spec, prefix): os.symlink(self.global_license_file, os.path.join( self.prefix, "vtune_amplifier_xe/licenses", "license.lic")) - if (spec.satisfies('+all') or spec.satisfies('+mpi')) and\ - spec.satisfies('@cluster'): - os.symlink(self.global_license_file, os.path.join( - self.prefix, "itac_latest", "license.lic")) + if (spec.satisfies('+all') or spec.satisfies('+mpi')) and \ + spec.satisfies('@cluster'): + os.symlink(self.global_license_file, os.path.join( + self.prefix, "itac_latest", "license.lic")) if spec.satisfies('+rpath'): for compiler_command in ["icc", "icpc", "ifort"]: - cfgfilename = os.path.join(absbindir, "%s.cfg" %\ - compiler_command) + cfgfilename = os.path.join(absbindir, "%s.cfg" % + compiler_command) with open(cfgfilename, "w") as f: f.write('-Xlinker -rpath -Xlinker %s\n' % abslibdir) os.symlink(os.path.join(self.prefix.man, "common", "man1"), - os.path.join(self.prefix.man, "man1")) + os.path.join(self.prefix.man, "man1")) diff --git a/var/spack/repos/builtin/packages/intel/package.py b/var/spack/repos/builtin/packages/intel/package.py index 2d6929fc732..ec3192380ae 100644 --- a/var/spack/repos/builtin/packages/intel/package.py +++ b/var/spack/repos/builtin/packages/intel/package.py @@ -52,9 +52,9 @@ def install(self, spec, prefix): # Remove the installation DB, otherwise it will try to install into # location of other Intel builds if os.path.exists(os.path.join(os.environ["HOME"], "intel", - "intel_sdp_products.db")): + "intel_sdp_products.db")): os.remove(os.path.join(os.environ["HOME"], "intel", - "intel_sdp_products.db")) + "intel_sdp_products.db")) if not hasattr(self, "intel_prefix"): self.intel_prefix = self.prefix @@ -71,7 +71,7 @@ def install(self, spec, prefix): PHONEHOME_SEND_USAGE_DATA=no CONTINUE_WITH_OPTIONAL_ERROR=yes COMPONENTS=%s -""" %(self.intel_prefix, self.global_license_file, self.intel_components)) +""" % (self.intel_prefix, self.global_license_file, self.intel_components)) install_script = which("install.sh") install_script('--silent', silent_config_filename) @@ -88,10 +88,10 @@ class Intel(IntelInstaller): # TODO: can also try the online installer (will download files on demand) version('16.0.2', '1133fb831312eb519f7da897fec223fa', - url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz"\ + url="file://%s/parallel_studio_xe_2016_composer_edition_update2.tgz" # NOQA: ignore=E501 % os.getcwd()) version('16.0.3', '3208eeabee951fc27579177b593cefe9', - url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz"\ + url="file://%s/parallel_studio_xe_2016_composer_edition_update3.tgz" # NOQA: ignore=E501 % os.getcwd()) variant('rpath', default=True, description="Add rpath to .cfg files") @@ -99,9 +99,8 @@ class Intel(IntelInstaller): def install(self, spec, prefix): components = [] all_components = get_all_components() - components = filter_pick(all_components, - re.compile('(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' - ).search) + regex = '(comp|openmp|intel-tbb|icc|ifort|psxe|icsxe-pset)' + components = filter_pick(all_components, re.compile(regex).search) self.intel_components = ';'.join(components) IntelInstaller.install(self, spec, prefix) @@ -113,14 +112,14 @@ def install(self, spec, prefix): # symlink or copy? os.symlink(self.global_license_file, os.path.join(absbindir, - "license.lic")) + "license.lic")) if spec.satisfies('+rpath'): for compiler_command in ["icc", "icpc", "ifort"]: - cfgfilename = os.path.join(absbindir, "%s.cfg" %\ - compiler_command) + cfgfilename = os.path.join(absbindir, "%s.cfg" % + compiler_command) with open(cfgfilename, "w") as f: f.write('-Xlinker -rpath -Xlinker %s\n' % abslibdir) os.symlink(os.path.join(self.prefix.man, "common", "man1"), - os.path.join(self.prefix.man, "man1")) + os.path.join(self.prefix.man, "man1")) From 77a34ebdf94dd39be338ab85deb45e564c0a563b Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Mon, 16 May 2016 10:08:59 -0400 Subject: [PATCH 231/375] flake8-clean. --- lib/spack/spack/cmd/view.py | 159 +++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 75 deletions(-) diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 8767459c084..4861383eaa6 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -29,14 +29,17 @@ This set consists of: - specs resolved from the package names given by the user (the seeds) + - all depenencies of the seeds unless user specifies `--no-depenencies` -- less any specs with names matching the regular expressions given by `--exclude` + +- less any specs with names matching the regular expressions given by + `--exclude` The `view` command provides a number of functions (the "actions"): - symlink :: a file system view which is a directory hierarchy that is the union of the hierarchies of the installed packages in the DAG - where installed files are referenced via symlinks. + where installed files are referenced via symlinks. - hardlink :: like the symlink view but hardlinks are used. @@ -66,71 +69,70 @@ import os import re import sys -import argparse - import spack import spack.cmd import llnl.util.tty as tty description = "Produce a single-rooted directory view of a spec." + def setup_parser(sp): setup_parser.parser = sp - sp.add_argument('-v','--verbose', action='store_true', default=False, - help="Display verbose output.") - sp.add_argument('-e','--exclude', action='append', default=[], - help="Exclude packages with names matching the given regex pattern.") - sp.add_argument('-d', '--dependencies', choices=['true','false','yes','no'], - default='true', - help="Follow dependencies.") - + sp.add_argument( + '-v', '--verbose', action='store_true', default=False, + help="Display verbose output.") + sp.add_argument( + '-e', '--exclude', action='append', default=[], + help="Exclude packages with names matching the given regex pattern.") + sp.add_argument( + '-d', '--dependencies', choices=['true', 'false', 'yes', 'no'], + default='true', + help="Follow dependencies.") ssp = sp.add_subparsers(metavar='ACTION', dest='action') - specs_opts = dict(metavar='spec', nargs='+', + specs_opts = dict(metavar='spec', nargs='+', help="Seed specs of the packages to view.") # The action parameterizes the command but in keeping with Spack # patterns we make it a subcommand. file_system_view_actions = [ - ssp.add_parser('symlink', aliases=['add','soft'], - help='Add package files to a filesystem view via symbolic links.'), - ssp.add_parser('hardlink', aliases=['hard'], - help='Add packages files to a filesystem via via hard links.'), - ssp.add_parser('remove', aliases=['rm'], - help='Remove packages from a filesystem view.'), - ssp.add_parser('statlink', aliases=['status','check'], - help='Check status of packages in a filesystem view.') + ssp.add_parser( + 'symlink', aliases=['add', 'soft'], + help='Add package files to a filesystem view via symbolic links.'), + ssp.add_parser( + 'hardlink', aliases=['hard'], + help='Add packages files to a filesystem via via hard links.'), + ssp.add_parser( + 'remove', aliases=['rm'], + help='Remove packages from a filesystem view.'), + ssp.add_parser( + 'statlink', aliases=['status', 'check'], + help='Check status of packages in a filesystem view.') ] # All these options and arguments are common to every action. for act in file_system_view_actions: act.add_argument('path', nargs=1, help="Path to file system view directory.") act.add_argument('specs', **specs_opts) - + # The formatted print action. act = ssp.add_parser('print', help="Print a string to stdout based on given format") - act.add_argument('format', nargs=1, + act.add_argument('format', nargs=1, help="Format describing per-package printout.") act.add_argument('specs', **specs_opts) - ## Other VIEW ACTIONS might be added here. - ## Some ideas are the following (and some are redundant with existing cmds) - ## A JSON view that dumps a DAG to a JSON file - ## A DOT view that dumps to a GraphViz file - ## A SHELL INIT view that dumps bash/csh script setting up to use packages in the view return -### Util functions - def assuredir(path): 'Assure path exists as a directory' if not os.path.exists(path): os.makedirs(path) - + + def relative_to(prefix, path): 'Return end of `path` relative to `prefix`' assert 0 == path.find(prefix) @@ -139,31 +141,36 @@ def relative_to(prefix, path): reldir = reldir[1:] return reldir + def transform_path(spec, path, prefix=None): 'Return the a relative path corresponding to given path spec.prefix' if os.path.isabs(path): path = relative_to(spec.prefix, path) subdirs = path.split(os.path.sep) if subdirs[0] == '.spack': - lst = ['.spack',spec.name]+subdirs[1:] + lst = ['.spack', spec.name] + subdirs[1:] path = os.path.join(*lst) if prefix: path = os.path.join(prefix, path) return path + def purge_empty_directories(path): - 'Ascend up from the leaves accessible from `path` and remove empty directories.' + '''Ascend up from the leaves accessible from `path` + and remove empty directories.''' for dirpath, subdirs, files in os.walk(path, topdown=False): for sd in subdirs: - sdp = os.path.join(dirpath,sd) + sdp = os.path.join(dirpath, sd) try: os.rmdir(sdp) except OSError: pass + def filter_exclude(specs, exclude): 'Filter specs given sequence of exclude regex' to_exclude = [re.compile(e) for e in exclude] + def exclude(spec): for e in to_exclude: if e.match(spec.name): @@ -171,6 +178,7 @@ def exclude(spec): return False return [s for s in specs if not exclude(s)] + def flatten(seeds, descend=True): 'Normalize and flattend seed specs and descend hiearchy' flat = set() @@ -181,6 +189,7 @@ def flatten(seeds, descend=True): flat.update(spec.normalized().traverse()) return flat + def spec2dict(spec): 'Convert info in a spec into a simple dictionary.' @@ -188,44 +197,45 @@ def spec2dict(spec): # some things need processing or are properties. # pkg = spec.package - ret = dict(name = spec.name, - spec = spec.short_spec, - colorspec = spec.cshort_spec, # color - root = spec.root, - prefix = spec.prefix, - version = spec.version, - variants = spec.variants, - namespace = spec.namespace, - compiler = spec.compiler, - architecture = spec.architecture, - dependencies = ','.join(spec.dependencies.keys()), - dependents = ','.join(spec.dependents.keys()), - external = spec.external or "False", - hash = spec.dag_hash(), + inst_deps = ','.join([s.name for s in pkg.installed_dependents]), + ret = dict(name=spec.name, + spec=spec.short_spec, + colorspec=spec.cshort_spec, # color + root=spec.root, + prefix=spec.prefix, + version=spec.version, + variants=spec.variants, + namespace=spec.namespace, + compiler=spec.compiler, + architecture=spec.architecture, + dependencies=','.join(spec.dependencies.keys()), + dependents=','.join(spec.dependents.keys()), + external=spec.external or "False", + hash=spec.dag_hash(), # package related: - url = pkg.url, - stage = pkg.stage.path, - installed = pkg.installed, - installed_dependents = ','.join([s.name for s in pkg.installed_dependents]), - build_log = pkg.build_log_path, - rpath = ':'.join(pkg.rpath), + url=pkg.url, + stage=pkg.stage.path, + installed=pkg.installed, + installed_dependents=inst_deps, + build_log=pkg.build_log_path, + rpath=':'.join(pkg.rpath), # ... ) return ret -### Action-specific helpers def check_one(spec, path, verbose=False): 'Check status of view in path against spec' dotspack = os.path.join(path, '.spack', spec.name) if os.path.exists(os.path.join(dotspack)): - tty.info('Package in view: "%s"'%spec.name) + tty.info('Package in view: "%s"' % spec.name) return - tty.info('Package not in view: "%s"'%spec.name) + tty.info('Package not in view: "%s"' % spec.name) return + def remove_one(spec, path, verbose=False): 'Remove any files found in `spec` from `path` and purge empty directories.' @@ -235,12 +245,12 @@ def remove_one(spec, path, verbose=False): dotspack = transform_path(spec, '.spack', path) if not os.path.exists(dotspack): if verbose: - tty.info('Skipping nonexistent package: "%s"'%spec.name) + tty.info('Skipping nonexistent package: "%s"' % spec.name) return if verbose: - tty.info('Removing package: "%s"'%spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): + tty.info('Removing package: "%s"' % spec.name) + for dirpath, dirnames, filenames in os.walk(spec.prefix): if not filenames: continue targdir = transform_path(spec, dirpath, path) @@ -250,17 +260,18 @@ def remove_one(spec, path, verbose=False): continue os.unlink(dst) -def link_one(spec, path, link = os.symlink, verbose=False): + +def link_one(spec, path, link=os.symlink, verbose=False): 'Link all files in `spec` into directory `path`.' dotspack = transform_path(spec, '.spack', path) if os.path.exists(dotspack): - tty.warn('Skipping existing package: "%s"'%spec.name) + tty.warn('Skipping existing package: "%s"' % spec.name) return if verbose: tty.info('Linking package: "%s"' % spec.name) - for dirpath,dirnames,filenames in os.walk(spec.prefix): + for dirpath, dirnames, filenames in os.walk(spec.prefix): if not filenames: continue # avoid explicitly making empty dirs @@ -275,12 +286,9 @@ def link_one(spec, path, link = os.symlink, verbose=False): continue # silence these tty.warn("Skipping existing file: %s" % dst) continue - link(src,dst) + link(src, dst) -### The canonically named visitor_* functions and their alias maps. -### One for each action. - def visitor_symlink(specs, args): 'Symlink all files found in specs' path = args.path[0] @@ -290,6 +298,7 @@ def visitor_symlink(specs, args): visitor_add = visitor_symlink visitor_soft = visitor_symlink + def visitor_hardlink(specs, args): 'Hardlink all files found in specs' path = args.path[0] @@ -298,6 +307,7 @@ def visitor_hardlink(specs, args): link_one(spec, path, os.link, verbose=args.verbose) visitor_hard = visitor_hardlink + def visitor_remove(specs, args): 'Remove all files and directories found in specs from args.path' path = args.path[0] @@ -306,6 +316,7 @@ def visitor_remove(specs, args): purge_empty_directories(path) visitor_rm = visitor_remove + def visitor_statlink(specs, args): 'Give status of view in args.path relative to specs' path = args.path[0] @@ -314,6 +325,7 @@ def visitor_statlink(specs, args): visitor_status = visitor_statlink visitor_check = visitor_statlink + def visitor_print(specs, args): 'Print a string for each spec using args.format.' fmt = args.format[0] @@ -325,25 +337,22 @@ def visitor_print(specs, args): try: text = t.substitute(kwds) except KeyError: - tty.error("Format error, use keywords: %s" % (', '.join(kwds.keys()), )) + tty.error("Format error, use keywords: %s" % + (', '.join(kwds.keys()), )) raise # argparser escapes these text = text.replace(r'\n', '\n').replace(r'\t', '\t') sys.stdout.write(text) -# Finally, the actual "view" command. There should be no need to -# modify anything below when new actions are added. - - def view(parser, args): 'Produce a view of a set of packages.' - # Process common args + # Process common args seeds = [spack.cmd.disambiguate_spec(s) for s in args.specs] - specs = flatten(seeds, args.dependencies.lower() in ['yes','true']) + specs = flatten(seeds, args.dependencies.lower() in ['yes', 'true']) specs = filter_exclude(specs, args.exclude) - + # Execute the visitation. try: visitor = globals()['visitor_' + args.action] From 7705603f7302f7e7c1076d3aff53cb1829fb8e74 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 16 May 2016 09:43:46 -0700 Subject: [PATCH 232/375] Added more compilers to mock packages tests --- lib/spack/spack/test/mock_packages_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 612008ac589..81b56daacd2 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -47,6 +47,17 @@ f77: None fc: None modules: 'None' + clang3.3SUSE: + spec: clang@3.3 + operating_system: + name: SuSE + version: '11' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' clang3.3RHL: spec: clang@3.3 operating_system: @@ -69,6 +80,17 @@ version: '10' spec: gcc@4.5.0 modules: 'None' + gcc4.5.0SUSE: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: SuSE + version: '11' + spec: gcc@4.5.0 + modules: 'None' gcc4.5.0RHL: paths: cc: /path/to/gcc From 0fd9cdb861b9b794cc6aa20cd5ecca1989a43820 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 16 May 2016 09:56:07 -0700 Subject: [PATCH 233/375] Fixed config tests for new compiler config format --- lib/spack/spack/test/config.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 2a8d642584d..daa4c572bd9 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -138,11 +138,17 @@ def check_config(self, comps, *compiler_names): """Check that named compilers in comps match Spack's config.""" config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] - for key in compiler_names: - for c in compiler_list: - expected = comps['all'][key][c] - actual = config['all'][key][c] - self.assertEqual(expected, actual) + param_list = ['modules', 'paths', 'spec', 'operating_system'] + for alias, compiler in config.items(): + if compiler['spec'] in compiler_names: + for p in param_list: + expected = comps[alias][p] + actual = config[alias][p] + self.assertEqual(expected, actual) + for c in compiler_list: + expected = comps[alias]['paths'][c] + actual = config[alias]['paths'][c] + self.assertEqual(expected, actual) def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. From 7b9b18f03c7654271b865e4a3a0117124331aa0c Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 17 May 2016 13:14:59 -0500 Subject: [PATCH 234/375] Add HPL benchmark package --- .../repos/builtin/packages/hpl/package.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 var/spack/repos/builtin/packages/hpl/package.py diff --git a/var/spack/repos/builtin/packages/hpl/package.py b/var/spack/repos/builtin/packages/hpl/package.py new file mode 100644 index 00000000000..1fe3e6ea323 --- /dev/null +++ b/var/spack/repos/builtin/packages/hpl/package.py @@ -0,0 +1,111 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * +import os +import platform + + +class Hpl(Package): + """HPL is a software package that solves a (random) dense linear system + in double precision (64 bits) arithmetic on distributed-memory computers. + It can thus be regarded as a portable as well as freely available + implementation of the High Performance Computing Linpack Benchmark.""" + + homepage = "http://www.netlib.org/benchmark/hpl/" + url = "http://www.netlib.org/benchmark/hpl/hpl-2.2.tar.gz" + + version('2.2', '0eb19e787c3dc8f4058db22c9e0c5320') + + variant('openmp', default=False, description='Enable OpenMP support') + + depends_on('mpi@1.1:') + depends_on('blas') + + def configure(self, spec, arch): + # List of configuration options + # Order is important + config = [] + + # OpenMP support + if '+openmp' in spec: + config.append( + 'OMP_DEFS = {0}'.format(self.compiler.openmp_flag) + ) + + config.extend([ + # Shell + 'SHELL = /bin/sh', + 'CD = cd', + 'CP = cp', + 'LN_S = ln -fs', + 'MKDIR = mkdir -p', + 'RM = /bin/rm -f', + 'TOUCH = touch', + # Platform identifier + 'ARCH = {0}'.format(arch), + # HPL Directory Structure / HPL library + 'TOPdir = {0}'.format(os.getcwd()), + 'INCdir = $(TOPdir)/include', + 'BINdir = $(TOPdir)/bin/$(ARCH)', + 'LIBdir = $(TOPdir)/lib/$(ARCH)', + 'HPLlib = $(LIBdir)/libhpl.a', + # Message Passing library (MPI) + 'MPinc = -I{0}'.format(spec['mpi'].prefix.include), + 'MPlib = -L{0}'.format(spec['mpi'].prefix.lib), + # Linear Algebra library (BLAS or VSIPL) + 'LAinc = {0}'.format(spec['blas'].prefix.include), + 'LAlib = {0}'.format(spec['blas'].blas_shared_lib), + # F77 / C interface + 'F2CDEFS = -DAdd_ -DF77_INTEGER=int -DStringSunStyle', + # HPL includes / libraries / specifics + 'HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) ' + + '-I$(LAinc) -I$(MPinc)', + 'HPL_LIBS = $(HPLlib) $(LAlib) $(MPlib)', + 'HPL_OPTS = -DHPL_DETAILED_TIMING -DHPL_PROGRESS_REPORT', + 'HPL_DEFS = $(F2CDEFS) $(HPL_OPTS) $(HPL_INCLUDES)', + # Compilers / linkers - Optimization flags + 'CC = {0}'.format(spec['mpi'].mpicc), + 'CCNOOPT = $(HPL_DEFS)', + 'CCFLAGS = $(HPL_DEFS) -O3', + 'LINKER = $(CC)', + 'LINKFLAGS = $(CCFLAGS) $(OMP_DEFS)', + 'ARCHIVER = ar', + 'ARFLAGS = r', + 'RANLIB = echo' + ]) + + # Write configuration options to include file + with open('Make.{0}'.format(arch), 'w') as makefile: + for var in config: + makefile.write('{0}\n'.format(var)) + + + def install(self, spec, prefix): + arch = '{0}-{1}'.format(platform.system(), platform.processor()) + + self.configure(spec, arch) + + make('arch={0}'.format(arch)) + install_tree(join_path('bin', arch), prefix.bin) From 631cbdbda89a1a227dcb0277478195a950984240 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 17 May 2016 13:22:00 -0500 Subject: [PATCH 235/375] Flake8 --- var/spack/repos/builtin/packages/hpl/package.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/hpl/package.py b/var/spack/repos/builtin/packages/hpl/package.py index 1fe3e6ea323..3a469d842e2 100644 --- a/var/spack/repos/builtin/packages/hpl/package.py +++ b/var/spack/repos/builtin/packages/hpl/package.py @@ -81,7 +81,7 @@ def configure(self, spec, arch): 'F2CDEFS = -DAdd_ -DF77_INTEGER=int -DStringSunStyle', # HPL includes / libraries / specifics 'HPL_INCLUDES = -I$(INCdir) -I$(INCdir)/$(ARCH) ' + - '-I$(LAinc) -I$(MPinc)', + '-I$(LAinc) -I$(MPinc)', 'HPL_LIBS = $(HPLlib) $(LAlib) $(MPlib)', 'HPL_OPTS = -DHPL_DETAILED_TIMING -DHPL_PROGRESS_REPORT', 'HPL_DEFS = $(F2CDEFS) $(HPL_OPTS) $(HPL_INCLUDES)', @@ -101,7 +101,6 @@ def configure(self, spec, arch): for var in config: makefile.write('{0}\n'.format(var)) - def install(self, spec, prefix): arch = '{0}-{1}'.format(platform.system(), platform.processor()) From f16228d811e4be68f4e4704ff3c2eaee86c936d6 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 17 May 2016 13:36:38 -0500 Subject: [PATCH 236/375] Add more install directories --- var/spack/repos/builtin/packages/hpl/package.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/var/spack/repos/builtin/packages/hpl/package.py b/var/spack/repos/builtin/packages/hpl/package.py index 3a469d842e2..3d6e9963235 100644 --- a/var/spack/repos/builtin/packages/hpl/package.py +++ b/var/spack/repos/builtin/packages/hpl/package.py @@ -102,9 +102,16 @@ def configure(self, spec, arch): makefile.write('{0}\n'.format(var)) def install(self, spec, prefix): + # Arch used for file naming purposes only arch = '{0}-{1}'.format(platform.system(), platform.processor()) + # Generate Makefile include self.configure(spec, arch) make('arch={0}'.format(arch)) + + # Manual installation install_tree(join_path('bin', arch), prefix.bin) + install_tree(join_path('lib', arch), prefix.lib) + install_tree(join_path('include', arch), prefix.include) + install_tree('man', prefix.man) From 582d01784aa44024da598187c654947bfd2e55de Mon Sep 17 00:00:00 2001 From: Greg Lee Date: Tue, 17 May 2016 21:18:34 -0700 Subject: [PATCH 237/375] added newdtags variant, default to false (i.e., remove --enable-new-dtags) --- .../packages/intel-parallel-studio/package.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py index a176af101d3..493ca16417c 100644 --- a/var/spack/repos/builtin/packages/intel-parallel-studio/package.py +++ b/var/spack/repos/builtin/packages/intel-parallel-studio/package.py @@ -32,6 +32,8 @@ class IntelParallelStudio(IntelInstaller): url="file://%s/parallel_studio_xe_2016_update3.tgz" % os.getcwd()) # NOQA: ignore=E501 variant('rpath', default=True, description="Add rpath to .cfg files") + variant('newdtags', default=False, + description="Allow use of --enable-new-dtags in MPI wrappers") variant('all', default=False, description="Install all files with the requested edition") variant('mpi', default=True, @@ -116,6 +118,20 @@ def install(self, spec, prefix): spec.satisfies('@cluster'): os.symlink(self.global_license_file, os.path.join( self.prefix, "itac_latest", "license.lic")) + if spec.satisfies('~newdtags'): + wrappers = ["mpif77", "mpif77", "mpif90", "mpif90", + "mpigcc", "mpigcc", "mpigxx", "mpigxx", + "mpiicc", "mpiicc", "mpiicpc", "mpiicpc", + "mpiifort", "mpiifort"] + wrapper_paths = [] + for root, dirs, files in os.walk(spec.prefix): + for name in files: + if name in wrappers: + wrapper_paths.append(os.path.join(spec.prefix, + root, name)) + for wrapper in wrapper_paths: + filter_file(r'-Xlinker --enable-new-dtags', r' ', + wrapper) if spec.satisfies('+rpath'): for compiler_command in ["icc", "icpc", "ifort"]: From 434ff28af453619b9c164121d6ef51ba14673e3a Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 18 May 2016 15:14:21 -0500 Subject: [PATCH 238/375] Add STREAM benchmark package --- .../repos/builtin/packages/stream/package.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 var/spack/repos/builtin/packages/stream/package.py diff --git a/var/spack/repos/builtin/packages/stream/package.py b/var/spack/repos/builtin/packages/stream/package.py new file mode 100644 index 00000000000..8b3f32af8a8 --- /dev/null +++ b/var/spack/repos/builtin/packages/stream/package.py @@ -0,0 +1,62 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Stream(Package): + """The STREAM benchmark is a simple synthetic benchmark program that + measures sustainable memory bandwidth (in MB/s) and the corresponding + computation rate for simple vector kernels.""" + + homepage = "https://www.cs.virginia.edu/stream/ref.html" + + version('5.10', git='https://github.com/jeffhammond/STREAM.git') + + variant('openmp', default=False, description='Build with OpenMP support') + + def patch(self): + makefile = FileFilter('Makefile') + + # Use the Spack compiler wrappers + makefile.filter('CC = .*', 'CC = cc') + makefile.filter('FC = .*', 'FC = f77') + + cflags = '-O2' + fflags = '-O2' + if '+openmp' in self.spec: + cflags += ' ' + self.compiler.openmp_flag + fflags += ' ' + self.compiler.openmp_flag + + # Set the appropriate flags for this compiler + makefile.filter('CFLAGS = .*', 'CFLAGS = {0}'.format(cflags)) + makefile.filter('FFLAGS = .*', 'FFLAGS = {0}'.format(fflags)) + + def install(self, spec, prefix): + make() + + # Manual installation + mkdir(prefix.bin) + install('stream_c.exe', prefix.bin) + install('stream_f.exe', prefix.bin) From 80310a3b7c09c8b59b3414b0c20aed6c36620675 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 16:50:50 -0700 Subject: [PATCH 239/375] Finished merge on these files --- lib/spack/env/cc | 44 +++++++-------- lib/spack/spack/compilers/pgi.py | 13 ++--- lib/spack/spack/config.py | 46 +++++++++------ lib/spack/spack/spec.py | 97 ++++++++++++++++---------------- 4 files changed, 105 insertions(+), 95 deletions(-) diff --git a/lib/spack/env/cc b/lib/spack/env/cc index 9758b74f376..1e405ae6e9a 100755 --- a/lib/spack/env/cc +++ b/lib/spack/env/cc @@ -174,6 +174,28 @@ if [[ -z $command ]]; then die "ERROR: Compiler '$SPACK_COMPILER_SPEC' does not support compiling $language programs." fi +# +# Filter '.' and Spack environment directories out of PATH so that +# this script doesn't just call itself +# +IFS=':' read -ra env_path <<< "$PATH" +IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" +spack_env_dirs+=("" ".") +PATH="" +for dir in "${env_path[@]}"; do + addpath=true + for env_dir in "${spack_env_dirs[@]}"; do + if [[ $dir == $env_dir ]]; then + addpath=false + break + fi + done + if $addpath; then + PATH="${PATH:+$PATH:}$dir" + fi +done +export PATH + if [[ $mode == vcheck ]]; then exec ${command} "$@" fi @@ -286,28 +308,6 @@ unset LD_LIBRARY_PATH unset LD_RUN_PATH unset DYLD_LIBRARY_PATH -# -# Filter '.' and Spack environment directories out of PATH so that -# this script doesn't just call itself -# -IFS=':' read -ra env_path <<< "$PATH" -IFS=':' read -ra spack_env_dirs <<< "$SPACK_ENV_PATH" -spack_env_dirs+=("" ".") -PATH="" -for dir in "${env_path[@]}"; do - addpath=true - for env_dir in "${spack_env_dirs[@]}"; do - if [[ $dir == $env_dir ]]; then - addpath=false - break - fi - done - if $addpath; then - PATH="${PATH:+$PATH:}$dir" - fi -done -export PATH - full_command=("$command" "${args[@]}") # In test command mode, write out full command for Spack tests. diff --git a/lib/spack/spack/compilers/pgi.py b/lib/spack/spack/compilers/pgi.py index d3fad15b0c4..6d36d8bfa6d 100644 --- a/lib/spack/spack/compilers/pgi.py +++ b/lib/spack/spack/compilers/pgi.py @@ -44,7 +44,12 @@ class Pgi(Compiler): 'f77' : 'pgi/pgfortran', 'fc' : 'pgi/pgfortran' } -#ifdef NEW + + + PrgEnv = 'PrgEnv-pgi' + PrgEnv_compiler = 'pgi' + + @property def openmp_flag(self): return "-mp" @@ -53,12 +58,6 @@ def openmp_flag(self): def cxx11_flag(self): return "-std=c++11" - -#else /* not NEW */ - PrgEnv = 'PrgEnv-pgi' - PrgEnv_compiler = 'pgi' - -#endif /* not NEW */ @classmethod def default_version(cls, comp): """The '-V' option works for all the PGI compilers. diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3d6c175c7c3..ec04c817873 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -167,6 +167,18 @@ 'f77': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cxxflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'fflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cppflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldlibs': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}}}, 'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*' 'operating_system': { @@ -234,15 +246,11 @@ 'items' : { 'type' : 'string' } }, #compiler specs 'buildable': { 'type': 'boolean', -#ifdef NEW 'default': True, -#else /* not NEW */ - 'default': False, }, - 'module': { - 'anyOf' : [{'type': 'string'}, - {'type': 'null'}] -#endif /* not NEW */ + 'modules': { + 'type' : 'object', + 'default' : {}, }, 'providers': { 'type': 'object', @@ -687,7 +695,8 @@ def spec_externals(spec): external_specs = [] pkg_paths = allpkgs.get(name, {}).get('paths', None) - if not pkg_paths: + pkg_modules = allpkgs.get(name, {}).get('modules', None) + if (not pkg_paths) and (not pkg_modules): return [] for external_spec, path in pkg_paths.iteritems(): @@ -695,20 +704,21 @@ def spec_externals(spec): # skip entries without paths (avoid creating extra Specs) continue -#ifdef NEW external_spec = spack.spec.Spec(external_spec, external=path) if external_spec.satisfies(spec): external_specs.append(external_spec) + + for external_spec, module in pkg_modules.iteritems(): + if not module: + continue + + path = get_path_from_module(module) + + external_spec = spack.spec.Spec(external_spec, external=path, external_module=module) + if external_spec.satisfies(spec): + external_specs.append(external_spec) + return external_specs -#else /* not NEW */ - module = allpkgs.get(pkg, {}).get('module', None) - if not path: - if not module: - continue - path = get_path_from_module(module) - spec_locations.append( (spack.spec.Spec(pkg), path, module) ) - return spec_locations -#endif /* not NEW */ def is_spec_buildable(spec): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f71d56435e7..e61ffc0912c 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -497,14 +497,10 @@ def __init__(self, spec_like, *dep_like, **kwargs): # package.py files for. self._normal = kwargs.get('normal', False) self._concrete = kwargs.get('concrete', False) -#ifdef NEW # Allow a spec to be constructed with an external path. self.external = kwargs.get('external', None) -#else /* not NEW */ - self.external = None - self.external_module = None -#endif /* not NEW */ + self.external_module = kwargs.get('external_module', None) # This allows users to construct a spec DAG with literals. # Note that given two specs a and b, Spec(a) copies a, but @@ -538,8 +534,10 @@ def _add_flag(self, name, value): Known flags currently include "arch" """ valid_flags = FlagMap.valid_compiler_flags() - if name == 'arch': - self._set_architecture(value) + if name == 'os' or name == 'operating_system': + self._set_os(value) + elif name == 'target': + self._set_target(value) elif name in valid_flags: assert(self.compiler_flags is not None) self.compiler_flags[name] = value.split() @@ -553,12 +551,13 @@ def _set_compiler(self, compiler): self.compiler = compiler - def _set_architecture(self, architecture): - """Called by the parser to set the architecture.""" - if self.architecture: raise DuplicateArchitectureError( - "Spec for '%s' cannot have two architectures." % self.name) - self.architecture = architecture + def _set_os(self, value): + """Called by the parser to set the architecture operating system""" + self.architecture.platform_os = self.architecture.platform.operating_system(value) + def _set_target(self, value): + """Called by the parser to set the architecture target""" + self.architecture.target = self.architecture.platform.target(value) def _add_dependency(self, spec): """Called by the parser to add another spec as a dependency.""" @@ -793,9 +792,9 @@ def to_node_dict(self): if self.architecture: # TODO: Fix the target.to_dict to account for the tuple # Want it to be a dict of dicts - d['architecture'] = self.architecture.to_dict() + d['arch'] = self.architecture.to_dict() else: - d['architecture'] = None + d['arch'] = None if self.compiler: d.update(self.compiler.to_dict()) @@ -824,17 +823,12 @@ def from_node_dict(node): spec = Spec(name) spec.namespace = node.get('namespace', None) spec.versions = VersionList.from_dict(node) -#ifdef NEW - spec.architecture = node['arch'] if 'hash' in node: spec._hash = node['hash'] -#else /* not NEW */ - # TODO: Need to fix the architecture.Target.from_dict - spec.architecture = spack.architecture.arch_from_dict(node['architecture']) + spec.architecture = spack.architecture.arch_from_dict(node['arch']) -#endif /* not NEW */ if node['compiler'] is None: spec.compiler = None else: @@ -1423,9 +1417,19 @@ def constrain(self, other, deps=True): # TODO: Check out the logic here if self.architecture is not None and other.architecture is not None: - if self.architecture != other.architecture: - raise UnsatisfiableArchitectureSpecError(self.architecture, - other.architecture) + if self.architecture.platform is not None and other.architecture.platform is not None: + if self.architecture.platform != other.architecture.platform: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + if self.architecture.platform_os is not None and other.architecture.platform_os is not None: + if self.architecture.platform_os != other.architecture.platform_os: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + if self.architecture.target is not None and other.architecture.target is not None: + if self.architecture.target != other.architecture.target: + raise UnsatisfiableArchitectureSpecError(self.architecture, + other.architecture) + changed = False if self.compiler is not None and other.compiler is not None: @@ -1440,7 +1444,14 @@ def constrain(self, other, deps=True): changed |= self.compiler_flags.constrain(other.compiler_flags) old = self.architecture - self.architecture = self.architecture or other.architecture + if self.architecture is None or other.architecture is None: + self.architecture = self.architecture or other.architecture + elif self.architecture.platform is None or other.architecture.platform is None: + self.architecture.platform = self.architecture.platform or other.architecture.platform + elif self.architecture.platform_os is None of other.architecture.platform_os is None: + self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os + elif self.architecture.target is None or other.architecture.target is None: + self.architecture.target = self.architecture.target or other.architecture.target changed |= (self.architecture != old) if deps: @@ -1572,16 +1583,18 @@ def satisfies(self, other, deps=True, strict=False): # Architecture satisfaction is currently just string equality. # If not strict, None means unconstrained. - if isinstance(self.architecture, basestring): - self.add_architecture_from_string(self.architecture) - if isinstance(other.architecture, basestring): - other.add_architecture_from_string(other.architecture) + # TODO: Need to make sure that comparisons can be made via classes if self.architecture and other.architecture: - if self.architecture != other.architecture: + if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or + (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or + (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): return False - elif strict and (other.architecture and not self.architecture): + elif strict and ((other.architecture and not self.architecture) or + (other.architecture.platform and not self.architecture.platform) or + (other.architecture.platform_os and not self.architecture.platform_os) or + (other.architecture.target and not self.architecture.target)): return False if not self.compiler_flags.satisfies(other.compiler_flags, strict=strict): @@ -1663,7 +1676,7 @@ def _dup(self, other, **kwargs): self.architecture != other.architecture and self.compiler != other.compiler and \ self.variants != other.variants and self._normal != other._normal and \ self.concrete != other.concrete and self.external != other.external and \ - self.external_module != other.external_module) + self.external_module != other.external_module and self.compiler_flags != other.compiler_flags) # Local node attributes get copied first. self.name = other.name @@ -1677,12 +1690,9 @@ def _dup(self, other, **kwargs): self.variants = other.variants.copy() self.variants.spec = self self.external = other.external - self.namespace = other.namespace -#ifdef NEW - self._hash = other._hash -#else /* not NEW */ self.external_module = other.external_module -#endif /* not NEW */ + self.namespace = other.namespace + self._hash = other._hash # If we copy dependencies, preserve DAG structure in the new spec if kwargs.get('deps', True): @@ -1983,7 +1993,7 @@ def write(s, c): write(fmt % str(self.variants), '+') elif named_str == 'ARCHITECTURE': if self.architecture: - write(fmt % str(self.architecture), '=') + write(fmt % str(self.architecture), ' arch=') elif named_str == 'SHA1': if self.dependencies: out.write(fmt % str(self.dag_hash(7))) @@ -2202,14 +2212,11 @@ def spec(self, name, check_valid_token = False): spec.name = spec_name spec.versions = VersionList() spec.variants = VariantMap(spec) - spec.architecture = None + spec.architecture = spack.architecture.Arch() spec.compiler = None spec.external = None -#ifdef NEW - spec.compiler_flags = FlagMap(spec) -#else /* not NEW */ spec.external_module = None -#endif /* not NEW */ + spec.compiler_flags = FlagMap(spec) spec.dependents = DependencyMap() spec.dependencies = DependencyMap() spec.namespace = spec_namespace @@ -2284,12 +2291,6 @@ def variant(self,name=None): self.check_identifier() return self.token.value - def architecture(self): - #TODO: Make this work properly as a subcase of variant (includes adding names to grammar) - self.expect(ID) - return self.token.value - - def version(self): start = None end = None From faa0f2a13c01a9f279d59912cff5d00c499cb990 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:01:59 -0700 Subject: [PATCH 240/375] got rid of ifdef --- lib/spack/spack/cmd/find.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index c768faea914..8b91f45cfd6 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -239,8 +239,5 @@ def find(parser, args): mode=args.mode, long=args.long, very_long=args.very_long, -#ifdef NEW show_flags=args.show_flags) -#else /* not NEW */ namespace=args.namespace) -#endif /* not NEW */ From 9f8ff32bcce99c8a188f285e235e9c32ee245f97 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:05:11 -0700 Subject: [PATCH 241/375] Got rid of ifdefs, changed parameters for Compiler: added kwargs --- lib/spack/spack/compiler.py | 69 +++++++++---------------------------- 1 file changed, 17 insertions(+), 52 deletions(-) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index dfae72010af..ce4555bc565 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -101,7 +101,6 @@ def cc_rpath_arg(self): def cxx_rpath_arg(self): return '-Wl,-rpath,' -#ifdef NEW @property def f77_rpath_arg(self): return '-Wl,-rpath,' @@ -109,45 +108,19 @@ def f77_rpath_arg(self): @property def fc_rpath_arg(self): return '-Wl,-rpath,' -#else /* not NEW */ # Cray PrgEnv name that can be used to load this compiler PrgEnv = None - # Name of module used to switch versions of this compiler PrgEnv_compiler = None -#endif /* not NEW */ -#ifdef NEW - - def __init__(self, cspec, cc, cxx, f77, fc, **kwargs): -#else /* not NEW */ - - def __init__(self, cspec, operating_system, paths, modules=[], alias=None): -#endif /* not NEW */ + def __init__(self, cspec, operating_system, + paths, modules=[], alias=None, **kwargs): def check(exe): if exe is None: return None _verify_executables(exe) return exe -#ifdef NEW - self.cc = check(cc) - self.cxx = check(cxx) - self.f77 = check(f77) - self.fc = check(fc) - - # Unfortunately have to make sure these params are accepted - # in the same order they are returned by sorted(flags) - # in compilers/__init__.py - self.flags = {} - for flag in spack.spec.FlagMap.valid_compiler_flags(): - value = kwargs.get(flag, None) - if value is not None: - self.flags[flag] = value.split() - -#else /* not NEW */ - self.operating_system = operating_system - self.cc = check(paths[0]) self.cxx = check(paths[1]) if len(paths) > 2: @@ -157,7 +130,21 @@ def check(exe): else: self.fc = check(paths[3]) -#endif /* not NEW */ + #self.cc = check(cc) + #self.cxx = check(cxx) + #self.f77 = check(f77) + #self.fc = check(fc) + + # Unfortunately have to make sure these params are accepted + # in the same order they are returned by sorted(flags) + # in compilers/__init__.py + self.flags = {} + for flag in spack.spec.FlagMap.valid_compiler_flags(): + value = kwargs.get(flag, None) + if value is not None: + self.flags[flag] = value.split() + + self.operating_system = operating_system self.spec = cspec self.modules = modules self.alias = alias @@ -287,28 +274,6 @@ def check(key): successful.reverse() return dict(((v, p, s), path) for v, p, s, path in successful) - @classmethod - def default_version(cls, cc): - """Override just this to override all compiler version functions.""" - return dumpversion(cc) - - @classmethod - def cc_version(cls, cc): - return cls.default_version(cc) - - @classmethod - def cxx_version(cls, cxx): - return cls.default_version(cxx) - - @classmethod - def f77_version(cls, f77): - return cls.default_version(f77) - - @classmethod - def fc_version(cls, fc): - return cls.default_version(fc) - - def __repr__(self): """Return a string representation of the compiler toolchain.""" return self.__str__() From b4b794223065b3301da9f89eab536bf8d162c150 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:07:23 -0700 Subject: [PATCH 242/375] Moved searching of flags into get compilers in compilers_for_spec --- lib/spack/spack/compilers/__init__.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 000ef2058e9..3432c142bf8 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -254,17 +254,18 @@ def get_compilers(cspec): else: operating_system = None - compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias)) + flags = {} + for f in spack.spec.FlagMap.valid_compiler_flags(): + if f in items: + flags[f] = items[f] -#ifdef NEW - flags = {} - for f in spack.spec.FlagMap.valid_compiler_flags(): - if f in items: - flags[f] = items[f] - return cls(cspec, *compiler_paths, **flags) -#else /* not NEW */ - return compilers -#endif /* not NEW */ + compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias, **flags)) + +##ifdef NEW +# return cls(cspec, *compiler_paths, **flags) +##else /* not NEW */ +# return compilers +##endif /* not NEW */ matches = set(find(compiler_spec, scope)) compilers = [] From deb5011d08467c0a19b52e5ee9fefa3fb0854b89 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:14:16 -0700 Subject: [PATCH 243/375] more merge work --- lib/spack/spack/build_environment.py | 23 ++++++++--------------- lib/spack/spack/spec.py | 10 ++++++++++ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 4974cc1e762..7aeea5c6723 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -213,13 +213,12 @@ def set_compiler_environment_variables(pkg): if flags[flag] != []: env.set('SPACK_' + flag.upper(), ' '.join(f for f in flags[flag])) -#ifdef NEW env.set('SPACK_COMPILER_SPEC', str(pkg.spec.compiler)) - return env -#else /* not NEW */ + for mod in compiler.modules: load_module(mod) -#endif /* not NEW */ + + return env def set_build_environment_variables(pkg, env): @@ -283,16 +282,13 @@ def set_build_environment_variables(pkg, env): pcdir = join_path(p, maybe, 'pkgconfig') if os.path.isdir(pcdir): pkg_config_dirs.append(pcdir) -#ifdef NEW - env.set_path('PKG_CONFIG_PATH', pkg_config_dirs) - return env -#else /* not NEW */ - path_put_first("PKG_CONFIG_PATH", pkg_config_dirs) + env.prepend_path('PKG_CONFIG_PATH', pkg_config_dirs) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) -#endif /* not NEW */ + + return env def set_module_variables_for_package(pkg, module): @@ -368,16 +364,13 @@ def set_module_variables_for_package(pkg, module): def get_rpaths(pkg): """Get a list of all the rpaths for a package.""" - for spec in pkg.spec.traverse(root=False): - if spec.external_module: - load_module(spec.external_module) - spec.external = get_path_from_module(spec.external_module) - rpaths = [pkg.prefix.lib, pkg.prefix.lib64] rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib)) rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) + for mod in pkg.spec.compiler.modules: + rpaths.append(get_path_for_module(mod)) return rpaths diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index e61ffc0912c..9d217d17ca4 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -120,6 +120,7 @@ from spack.util.string import * from spack.util.prefix import Prefix from spack.virtual import ProviderIndex +from spack.build_environment import get_path_from_module, load_module # Valid pattern for an identifier in Spack identifier_re = r'\w[\w-]*' @@ -1085,6 +1086,15 @@ def concretize(self): if s.namespace is None: s.namespace = spack.repo.repo_for_pkg(s.name).namespace + + for s in self.traverse(root=False): + if spec.external_module: + compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) + for mod in compiler.modules: + load_module(mod) + + spec.external = get_path_from_module(spec.external_module) + # Mark everything in the spec as concrete, as well. self._mark_concrete() From b1e5eafb80c8cd87f4d496fe7fdb7a74a77da603 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:18:26 -0700 Subject: [PATCH 244/375] more merge work --- lib/spack/spack/test/config.py | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 7d71cc1c7ff..8fffc094378 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -142,19 +142,6 @@ def check_config(self, comps, *compiler_names): """Check that named compilers in comps match Spack's config.""" config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] -#ifdef NEW - for key in compiler_names: - for c in compiler_list: - expected = comps[arch][key][c] - actual = config[arch][key][c] - self.assertEqual(expected, actual) - - def test_write_list_in_memory(self): - spack.config.update_config('repos', repos_low, 'test_low_priority') - spack.config.update_config('repos', repos_high, 'test_high_priority') - config = spack.config.get_config('repos') - self.assertEqual(config, repos_high+repos_low) -#else /* not NEW */ param_list = ['modules', 'paths', 'spec', 'operating_system'] for alias, compiler in config.items(): if compiler['spec'] in compiler_names: @@ -166,7 +153,12 @@ def test_write_list_in_memory(self): expected = comps[alias]['paths'][c] actual = config[alias]['paths'][c] self.assertEqual(expected, actual) -#endif /* not NEW */ + + def test_write_list_in_memory(self): + spack.config.update_config('repos', repos_low, 'test_low_priority') + spack.config.update_config('repos', repos_high, 'test_high_priority') + config = spack.config.get_config('repos') + self.assertEqual(config, repos_high+repos_low) def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. @@ -187,9 +179,8 @@ def test_write_key_to_disk(self): spack.config.clear_config_caches() # Same check again, to ensure consistency. -#ifdef NEW - self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') + self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') + self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') def test_write_to_same_priority_file(self): # Write b_comps in the same file as a_comps. @@ -200,9 +191,5 @@ def test_write_to_same_priority_file(self): spack.config.clear_config_caches() # Same check again, to ensure consistency. - self.check_config(a_comps, 'x86_64_E5v2_IntelIB', 'gcc@4.7.3', 'gcc@4.5.0') - self.check_config(b_comps, 'x86_64_E5v3', 'icc@10.0', 'icc@11.1', 'clang@3.3') -#else /* not NEW */ self.check_config(a_comps, 'gcc@4.7.3', 'gcc@4.5.0') self.check_config(b_comps, 'icc@10.0', 'icc@11.1', 'clang@3.3') -#endif /* not NEW */ From 5417f1cdc676ab6c244cce1f3594a9620ba24b16 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:21:03 -0700 Subject: [PATCH 245/375] more merge work --- lib/spack/spack/test/__init__.py | 40 ++--------------------------- lib/spack/spack/test/multimethod.py | 22 ++-------------- 2 files changed, 4 insertions(+), 58 deletions(-) diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index cac7c2f0585..480e6290e77 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -31,8 +31,8 @@ from llnl.util.tty.colify import colify from spack.test.tally_plugin import Tally """Names of tests to be included in Spack's test suite""" -#ifdef NEW -test_names = ['versions', 'url_parse', 'url_substitution', 'packages', 'stage', + +test_names = ['architecture', 'versions', 'url_parse', 'url_substitution', 'packages', 'stage', 'spec_syntax', 'spec_semantics', 'spec_dag', 'concretize', 'multimethod', 'install', 'package_sanity', 'config', 'directory_layout', 'pattern', 'python_version', 'git_fetch', @@ -41,42 +41,6 @@ 'make_executable', 'configure_guess', 'lock', 'database', 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.uninstall', 'cmd.test_install'] -#else /* not NEW */ -test_names = ['architecture', - 'versions', - 'url_parse', - 'url_substitution', - 'packages', - 'stage', - 'spec_syntax', - 'spec_semantics', - 'spec_dag', - 'concretize', - 'multimethod', - 'install', - 'package_sanity', - 'config', - 'directory_layout', - 'pattern', - 'python_version', - 'git_fetch', - 'svn_fetch', - 'hg_fetch', - 'mirror', - 'url_extrapolate', - 'cc', - 'link_tree', - 'spec_yaml', - 'optional_deps', - 'make_executable', - 'configure_guess', - 'unit_install', - 'lock', - 'database', - 'namespace_trie', - 'yaml', - 'sbang'] -#endif /* not NEW */ def list_tests(): diff --git a/lib/spack/spack/test/multimethod.py b/lib/spack/spack/test/multimethod.py index 7636bd6c38b..034e6b3923c 100644 --- a/lib/spack/spack/test/multimethod.py +++ b/lib/spack/spack/test/multimethod.py @@ -92,36 +92,18 @@ def test_default_works(self): self.assertEqual(pkg.has_a_default(), 'default') -#ifdef NEW - def test_architecture_match(self): - pkg = spack.repo.get('multimethod arch=x86_64') - self.assertEqual(pkg.different_by_architecture(), 'x86_64') - - pkg = spack.repo.get('multimethod arch=ppc64') - self.assertEqual(pkg.different_by_architecture(), 'ppc64') - - pkg = spack.repo.get('multimethod arch=ppc32') - self.assertEqual(pkg.different_by_architecture(), 'ppc32') - - pkg = spack.repo.get('multimethod arch=arm64') - self.assertEqual(pkg.different_by_architecture(), 'arm64') - - pkg = spack.repo.get('multimethod arch=macos') - self.assertRaises(NoSuchMethodError, pkg.different_by_architecture) -#else /* not NEW */ def test_target_match(self): platform = spack.architecture.sys_type() targets = platform.targets.values() for target in targets[:-1]: - pkg = spack.repo.get('multimethod='+target.name) + pkg = spack.repo.get('multimethod target='+target.name) self.assertEqual(pkg.different_by_target(), target.name) - pkg = spack.repo.get('multimethod='+targets[-1].name) + pkg = spack.repo.get('multimethod target='+targets[-1].name) if len(targets) == 1: self.assertEqual(pkg.different_by_target(), targets[-1].name) else: self.assertRaises(NoSuchMethodError, pkg.different_by_target) -#endif /* not NEW */ def test_dependency_match(self): From 6926f4d0da05ee2839e455af32d52098d9828be7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:23:10 -0700 Subject: [PATCH 246/375] Changed arch tests to new spec target= os= --- lib/spack/spack/test/spec_semantics.py | 109 ++++++++++++------------- 1 file changed, 50 insertions(+), 59 deletions(-) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index b6c9d612936..8cdb91e2060 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import unittest +import spack.architecture from spack.spec import * from spack.test.mock_packages_test import * @@ -138,23 +139,31 @@ def test_satisfies_compiler_version(self): self.check_unsatisfiable('foo %gcc@4.7', '%gcc@4.7.3') -#ifdef NEW def test_satisfies_architecture(self): - self.check_satisfies('foo arch=chaos_5_x86_64_ib', ' arch=chaos_5_x86_64_ib') - self.check_satisfies('foo arch=bgqos_0', ' arch=bgqos_0') + platform = self.architecture.sys_type() + if platform.name == 'crayxc': + self.check_satisfies('foo target=frontend os=frontend', 'target=frontend os=frontend') + self.check_satisfies('foo target=backend os=backend', 'target=backend', 'os=backend') + self.check_satisfies('foo target=default_target os=default_os','target=default_target os=default_os') - self.check_unsatisfiable('foo arch=bgqos_0', ' arch=chaos_5_x86_64_ib') - self.check_unsatisfiable('foo arch=chaos_5_x86_64_ib', ' arch=bgqos_0') -#else /* not NEW */ - def test_satisfies_target(self): - platform = spack.architecture.sys_type() - targets = platform.targets.values() - for target in targets: - self.check_satisfies('foo='+target.name, '='+target.name) -#endif /* not NEW */ - for i in range(1,len(targets)): - self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) +#ifdef NEW + #def test_satisfies_architecture(self): + # self.check_satisfies('foo arch=chaos_5_x86_64_ib', ' arch=chaos_5_x86_64_ib') + # self.check_satisfies('foo arch=bgqos_0', ' arch=bgqos_0') + + # self.check_unsatisfiable('foo arch=bgqos_0', ' arch=chaos_5_x86_64_ib') + # self.check_unsatisfiable('foo arch=chaos_5_x86_64_ib', ' arch=bgqos_0') +#els#e /* not NEW */ + #def test_satisfies_target(self): + # platform = spack.architecture.sys_type() + # targets = platform.targets.values() + # for target in targets: + # self.check_satisfies('foo='+target.name, '='+target.name) +#end#if /* not NEW */ + + # for i in range(1,len(targets)): + # self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) def test_satisfies_dependencies(self): self.check_satisfies('mpileaks^mpich', '^mpich') @@ -355,22 +364,29 @@ def test_constrain_variants(self): self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo') -#ifdef NEW def test_constrain_compiler_flags(self): self.check_constrain('libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3"', 'libelf cppflags="-Wall"') self.check_constrain('libelf cflags="-O3" cppflags="-Wall"', 'libelf cflags="-O3"', 'libelf cflags="-O3" cppflags="-Wall"') - def test_constrain_arch(self): - self.check_constrain('libelf arch=bgqos_0', 'libelf arch=bgqos_0', 'libelf arch=bgqos_0') - self.check_constrain('libelf arch=bgqos_0', 'libelf', 'libelf arch=bgqos_0') -#else /* not NEW */ - def test_constrain_target(self): - platform = spack.architecture.sys_type() - target = platform.target('default_target').name - self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) - self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) -#endif /* not NEW */ + def test_constrain_architecture(self): + self.check_constrain('libelf target=default_target os=default_os', + 'libelf target=default_target os=default_os', + 'libelf target=default_target os=default_os') + self.check_constrain('libelf target=default_target os=default_os', + 'libelf', + 'libelf target=default_target os=default_os') + + #def test_constrain_arch(self): + # self.check_constrain('libelf arch=bgqos_0', 'libelf arch=bgqos_0', 'libelf arch=bgqos_0') + # self.check_constrain('libelf arch=bgqos_0', 'libelf', 'libelf arch=bgqos_0') +#els#e /* not NEW */ + #def test_constrain_target(self): + # platform = spack.architecture.sys_type() + # target = platform.target('default_target').name + # self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) + # self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) +#end#if /* not NEW */ def test_constrain_compiler(self): @@ -384,20 +400,11 @@ def test_invalid_constraint(self): self.check_invalid_constraint('libelf+debug', 'libelf~debug') self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') -#ifdef NEW self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - - self.check_invalid_constraint('libelf arch=bgqos_0', 'libelf arch=x86_54') -#else /* not NEW */ - - platform = spack.architecture.sys_type() - targets = platform.targets.values() - if len(targets) > 1: - self.check_invalid_constraint('libelf='+targets[0].name, 'libelf='+targets[1].name) -#endif /* not NEW */ - + self.check_invalid_constraint('libelf target=default_target os=default_os', + 'libelf target=x86_64 os=ubuntu') def test_constrain_changed(self): self.check_constrain_changed('libelf', '@1.0') @@ -406,14 +413,13 @@ def test_constrain_changed(self): self.check_constrain_changed('libelf%gcc', '%gcc@4.5') self.check_constrain_changed('libelf', '+debug') self.check_constrain_changed('libelf', '~debug') -#ifdef NEW self.check_constrain_changed('libelf', 'debug=2') self.check_constrain_changed('libelf', 'cppflags="-O3"') self.check_constrain_changed('libelf', ' arch=bgqos_0') -#else /* not NEW */ + platform = spack.architecture.sys_type() - self.check_constrain_changed('libelf', '='+platform.target('default_target').name) -#endif /* not NEW */ + self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name) + self.check_constrain_changed('libelf', 'os='+platform.operating_system('default_os').name) def test_constrain_not_changed(self): @@ -424,17 +430,12 @@ def test_constrain_not_changed(self): self.check_constrain_not_changed('libelf%gcc@4.5', '%gcc@4.5') self.check_constrain_not_changed('libelf+debug', '+debug') self.check_constrain_not_changed('libelf~debug', '~debug') -#ifdef NEW self.check_constrain_not_changed('libelf debug=2', 'debug=2') self.check_constrain_not_changed('libelf cppflags="-O3"', 'cppflags="-O3"') - self.check_constrain_not_changed('libelf arch=bgqos_0', ' arch=bgqos_0') -#else /* not NEW */ + platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - self.check_constrain_not_changed('libelf='+default_target, '='+default_target) -#endif /* not NEW */ - self.check_constrain_not_changed('libelf^foo', 'libelf^foo') - self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar') + self.check_constrain_not_changed('libelf target='+default_target, 'target='+default_target) def test_constrain_dependency_changed(self): @@ -444,14 +445,9 @@ def test_constrain_dependency_changed(self): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') -#ifdef NEW - self.check_constrain_changed('libelf^foo', 'libelf^foo cppflags="-O3"') - self.check_constrain_changed('libelf^foo', 'libelf^foo arch=bgqos_0') -#else /* not NEW */ platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - self.check_constrain_changed('libelf^foo', 'libelf^foo='+default_target) -#endif /* not NEW */ + self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) def test_constrain_dependency_not_changed(self): @@ -461,13 +457,8 @@ def test_constrain_dependency_not_changed(self): self.check_constrain_not_changed('libelf^foo%gcc@4.5', 'libelf^foo%gcc@4.5') self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug') self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug') -#ifdef NEW self.check_constrain_not_changed('libelf^foo cppflags="-O3"', 'libelf^foo cppflags="-O3"') - self.check_constrain_not_changed('libelf^foo arch=bgqos_0', 'libelf^foo arch=bgqos_0') -#else /* not NEW */ platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - self.check_constrain_not_changed('libelf^foo='+default_target, 'libelf^foo='+default_target) - -#endif /* not NEW */ + self.check_constrain_not_changed('libelf^foo target='+default_target, 'libelf^foo target='+default_target) From b25da51638c6c3abd97a1f77cc4feffd144a62d1 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Wed, 18 May 2016 17:26:06 -0700 Subject: [PATCH 247/375] Changed unsatisfiable architecture test --- lib/spack/spack/test/spec_dag.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 3c005c4eadc..99a986a528a 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -29,6 +29,7 @@ spack/lib/spack/spack/test/mock_packages """ import spack +import spack.architecture import spack.package from llnl.util.lang import list_modules @@ -240,21 +241,12 @@ def test_unsatisfiable_compiler_version(self): self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize) -#ifdef NEW def test_unsatisfiable_architecture(self): - self.set_pkg_dep('mpileaks', 'mpich arch=bgqos_0') - spec = Spec('mpileaks ^mpich arch=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) -#else /* not NEW */ - def test_unsatisfiable_target(self): platform = spack.architecture.sys_type() - if len(platform.targets) > 1: - first = platform.targets.values()[0].name - second = platform.targets.values()[1].name - self.set_pkg_dep('mpileaks', 'mpich='+first) - spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') - self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) -#endif /* not NEW */ + + self.set_pkg_dep('mpileaks', 'mpich target=%s' % platform.target('default_target')) + spec = Spec('mpileaks ^mpich target=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') + self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) def test_invalid_dep(self): From 01d5ffcd87d4801874dc810fb8123447971b867d Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 17:42:11 -0700 Subject: [PATCH 248/375] Fixed the first set of merge bugs --- lib/spack/spack/architecture.py | 8 ++++---- lib/spack/spack/cmd/find.py | 2 +- lib/spack/spack/compilers/__init__.py | 6 +----- lib/spack/spack/concretize.py | 11 ++++++----- lib/spack/spack/spec.py | 2 +- lib/spack/spack/test/architecture.py | 7 +++---- 6 files changed, 16 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index d7f908cfb1e..41778795c38 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -323,8 +323,8 @@ def to_dict(self): target = self.target d['platform'] = self.platform.name - d['platform_os'] = self.platform_os.to_dict() - d['target'] = self.target.to_dict() + d['platform_os'] = self.platform_os.to_dict() if self.platform_os else None + d['target'] = self.target.to_dict() if self.target else None return d @@ -362,8 +362,8 @@ def arch_from_dict(d): os_dict = d['platform_os'] target_dict = d['target'] - target = _target_from_dict(target_dict) - platform_os = _operating_system_from_dict(os_dict) + target = _target_from_dict(target_dict) if os_dict else None + platform_os = _operating_system_from_dict(os_dict) if os_dict else None arch.target = target arch.platform_os = platform_os diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 8b91f45cfd6..c2bba13dc8e 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -239,5 +239,5 @@ def find(parser, args): mode=args.mode, long=args.long, very_long=args.very_long, - show_flags=args.show_flags) + show_flags=args.show_flags, namespace=args.namespace) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 3432c142bf8..8fe7a17116b 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -261,11 +261,7 @@ def get_compilers(cspec): compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias, **flags)) -##ifdef NEW -# return cls(cspec, *compiler_paths, **flags) -##else /* not NEW */ -# return compilers -##endif /* not NEW */ + return compilers matches = set(find(compiler_spec, scope)) compilers = [] diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 72b184dd8f5..f38afd38dcc 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -282,11 +282,8 @@ def _proper_compiler_style(cspec, architecture): return False #Find the another spec that has a compiler, or the root if none do -#ifdef NEW other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler) -#else /* not NEW */ - other_spec = find_spec(spec, lambda(x) : x.compiler) -#endif /* not NEW */ + if not other_spec: other_spec = spec.root other_compiler = other_spec.compiler @@ -321,6 +318,10 @@ def concretize_compiler_flags(self, spec): compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ + if not spec.architecture.platform_os: + #Although this usually means changed, this means awaiting other changes + return True + ret = False for flag in spack.spec.FlagMap.valid_compiler_flags(): try: @@ -352,7 +353,7 @@ def concretize_compiler_flags(self, spec): # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. - compiler = spack.compilers.compiler_for_spec(spec.compiler) + compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) for flag in compiler.flags: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = compiler.flags[flag] diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 9d217d17ca4..7c04baaa91d 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1458,7 +1458,7 @@ def constrain(self, other, deps=True): self.architecture = self.architecture or other.architecture elif self.architecture.platform is None or other.architecture.platform is None: self.architecture.platform = self.architecture.platform or other.architecture.platform - elif self.architecture.platform_os is None of other.architecture.platform_os is None: + elif self.architecture.platform_os is None or other.architecture.platform_os is None: self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os elif self.architecture.target is None or other.architecture.target is None: self.architecture.target = self.architecture.target or other.architecture.target diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 4cba2da9a5b..2927e468a05 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -72,7 +72,7 @@ def test_user_front_end_input(self): """ frontend_os = self.platform.operating_system("frontend") frontend_target = self.platform.target("frontend") - frontend_spec = Spec("libelf=frontend") + frontend_spec = Spec("libelf os=frontend target=frontend") frontend_spec.concretize() self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) self.assertEqual(frontend_target, frontend_spec.architecture.target) @@ -83,7 +83,7 @@ def test_user_back_end_input(self): """ backend_os = self.platform.operating_system("backend") backend_target = self.platform.target("backend") - backend_spec = Spec("libelf=backend") + backend_spec = Spec("libelf os=backend target=backend") backend_spec.concretize() self.assertEqual(backend_os, backend_spec.architecture.platform_os) self.assertEqual(backend_target, backend_spec.architecture.target) @@ -109,8 +109,7 @@ def test_user_input_combination(self): results = [] for arch in combinations: o,t = arch - arch_spec = "-".join(arch) - spec = Spec("libelf=%s" % arch_spec) + spec = Spec("libelf os=%s target=%s" % (o, t)) spec.concretize() results.append(spec.architecture.platform_os == self.platform.operating_system(o)) results.append(spec.architecture.target == self.platform.target(t)) From 63459ab0c78512906c35c5d5f9fbd5a072a3203a Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 18 May 2016 18:21:52 -0700 Subject: [PATCH 249/375] Fixed some of the bugs --- lib/spack/spack/spec.py | 24 +++++++++-------- lib/spack/spack/test/spec_semantics.py | 37 +++++++++++--------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 7c04baaa91d..f88475d1c82 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1088,12 +1088,12 @@ def concretize(self): for s in self.traverse(root=False): - if spec.external_module: - compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) + if s.external_module: + compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture.platform_os) for mod in compiler.modules: load_module(mod) - spec.external = get_path_from_module(spec.external_module) + s.external = get_path_from_module(s.external_module) # Mark everything in the spec as concrete, as well. self._mark_concrete() @@ -1426,6 +1426,7 @@ def constrain(self, other, deps=True): other.variants[v]) # TODO: Check out the logic here + print self.architecture, other.architecture, "^^^^^^^^^^^^^^^^^^^^^^^" if self.architecture is not None and other.architecture is not None: if self.architecture.platform is not None and other.architecture.platform is not None: if self.architecture.platform != other.architecture.platform: @@ -1453,16 +1454,17 @@ def constrain(self, other, deps=True): changed |= self.compiler_flags.constrain(other.compiler_flags) - old = self.architecture + old = str(self.architecture) if self.architecture is None or other.architecture is None: self.architecture = self.architecture or other.architecture - elif self.architecture.platform is None or other.architecture.platform is None: - self.architecture.platform = self.architecture.platform or other.architecture.platform - elif self.architecture.platform_os is None or other.architecture.platform_os is None: - self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os - elif self.architecture.target is None or other.architecture.target is None: - self.architecture.target = self.architecture.target or other.architecture.target - changed |= (self.architecture != old) + else: + if self.architecture.platform is None or other.architecture.platform is None: + self.architecture.platform = self.architecture.platform or other.architecture.platform + if self.architecture.platform_os is None or other.architecture.platform_os is None: + self.architecture.platform_os = self.architecture.platform_os or other.architecture.platform_os + if self.architecture.target is None or other.architecture.target is None: + self.architecture.target = self.architecture.target or other.architecture.target + changed |= (str(self.architecture) != old) if deps: changed |= self._constrain_dependencies(other) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 8cdb91e2060..9bd32a3d10d 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -140,7 +140,7 @@ def test_satisfies_compiler_version(self): def test_satisfies_architecture(self): - platform = self.architecture.sys_type() + platform = spack.architecture.sys_type() if platform.name == 'crayxc': self.check_satisfies('foo target=frontend os=frontend', 'target=frontend os=frontend') self.check_satisfies('foo target=backend os=backend', 'target=backend', 'os=backend') @@ -377,34 +377,28 @@ def test_constrain_architecture(self): 'libelf', 'libelf target=default_target os=default_os') - #def test_constrain_arch(self): - # self.check_constrain('libelf arch=bgqos_0', 'libelf arch=bgqos_0', 'libelf arch=bgqos_0') - # self.check_constrain('libelf arch=bgqos_0', 'libelf', 'libelf arch=bgqos_0') -#els#e /* not NEW */ - #def test_constrain_target(self): - # platform = spack.architecture.sys_type() - # target = platform.target('default_target').name - # self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target) - # self.check_constrain('libelf='+target, 'libelf', 'libelf='+target) -#end#if /* not NEW */ - - def test_constrain_compiler(self): self.check_constrain('libelf %gcc@4.4.7', 'libelf %gcc@4.4.7', 'libelf %gcc@4.4.7') self.check_constrain('libelf %gcc@4.4.7', 'libelf', 'libelf %gcc@4.4.7') def test_invalid_constraint(self): - self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') - self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') +# self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') +# self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') - self.check_invalid_constraint('libelf+debug', 'libelf~debug') - self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') - self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') +# self.check_invalid_constraint('libelf+debug', 'libelf~debug') +# self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') +# self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') - self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - self.check_invalid_constraint('libelf target=default_target os=default_os', - 'libelf target=x86_64 os=ubuntu') +# self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') + platform = spack.architecture.sys_type() + if len(platform.operating_sys.keys()) > 1 or len(platform.targets.keys()) > 1: + os1 = platform.operating_sys.keys()[0] + os2 = platform.operating_sys.keys()[-1] + target1 = platform.targets.keys()[0] + target2 = platform.targets.keys()[-1] + self.check_invalid_constraint('libelf target=%s os=%s' % (target1, os1), + 'libelf target=%s os=%s' % (target2, os2)) def test_constrain_changed(self): self.check_constrain_changed('libelf', '@1.0') @@ -447,6 +441,7 @@ def test_constrain_dependency_changed(self): self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') platform = spack.architecture.sys_type() default_target = platform.target('default_target').name + print default_target self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) From 126fc10b9dc56a51eb75bb7cfb88619b7029097c Mon Sep 17 00:00:00 2001 From: Brett Viren Date: Thu, 19 May 2016 08:35:54 -0600 Subject: [PATCH 250/375] Remove `view print` for now. --- lib/spack/docs/basic_usage.rst | 118 --------------------------------- lib/spack/spack/cmd/view.py | 72 +------------------- 2 files changed, 3 insertions(+), 187 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 4af4387846b..e108e393d71 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -330,124 +330,6 @@ of libelf would look like this: The full spec syntax is discussed in detail in :ref:`sec-specs`. -``spack view print`` -~~~~~~~~~~~~~~~~~~~~~~ - -``spack view print`` is a subcommand of ``spack view`` which displays information about one or more installed packages and their dependencies using a user-provided format string. The string can reverence variables in a shell-like manner for example ``$variable`` or ``${variable}text``. It can also include ``\t`` for tabs and ``\n`` for new lines. - -Some of the supported variables are: - -``name`` - The package name. - -``version`` - The package version. - -``spec`` - The package specification. - -``root`` - The root specification. - -``prefix`` - The installation directory. - -``variants`` - The collection of variants, if any. - -``namespace`` - The package repository name space. - -``compiler`` - The compiler \`name@versoin\` used to build the package. - -``architecture`` - The architecture targeted by the compiler. - -``dependencies`` - A comma-separated list of names of packages on which the package depends. - -``dependents`` - A comma-separated list of names of packages which depend on the package. - -``hash`` - The Spack hash for the package. - -``url`` - The source URL for the package. - -``stage`` - The directory for staging the build. - -``build_log`` - The path to the build log file. - -``rpath`` - The colon-separated library \`RPATH\` used in building the package. - -Here are some example uses of `spack view print`. A simple line-oriented report of information can be produced: - -.. code-block:: sh - - $ spack view print '$name\t$version\t$hash\n' cmake@3.5.2 - ncurses 6.0 bvbu4ixbnvtodpik4qzljlx3ukpyfrcz - zlib 1.2.8 ckki7zlryxrsetfqkgoxxahlhqqjni7n - openssl 1.0.2g 6zbar63sciso253nptxyrnhupymo7oyi - cmake 3.5.2 wprvmoczkpw4tiy5ybuk5zr7saus2d7g - -There are better ways to do this but a slow-and-dirty shell init procedure can be formed: - -.. code-block:: sh - - $ spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4 - export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/bin:$PATH" - export PATH="/spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/bin:$PATH" - $ eval $(spack view print 'export PATH="${prefix}/bin:$$PATH"\n' m4) - -Or, maybe you have some external application that can chew on Spack data in which case you can dump it to some easily parsed markup syntax such as YAML: - -.. code-block:: sh - - - $ spack view print '${name}:\n - version: ${version}\n - url : ${url}\n - spec: ${spec}\n - prefix: ${prefix}\n - root : ${root}\n - stage : ${stage}\n - log: ${build_log}\n' m4 - -Which might produce something like: - -.. code-block:: yaml - - libsigsegv: - - version: 2.10 - - url : ftp://ftp.gnu.org/gnu/libsigsegv/libsigsegv-2.10.tar.gz - - spec: libsigsegv@2.10%gcc@5.2.1=linux-x86_64-h6hsv76 - - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 - - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 - - stage : /spack/var/spack/stage/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2 - - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/libsigsegv-2.10-h6hsv76hffcjoe3nsaihzxemniwiedu2/.spack/build.out - m4: - - version: 1.4.17 - - url : ftp://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz - - spec: m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64-a4ikhdd - - prefix: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp - - root : m4@1.4.17%gcc@5.2.1+sigsegv=linux-x86_64^libsigsegv@2.10%gcc@5.2.1=linux-x86_64 - - stage : /spack/var/spack/stage/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp - - log: /spack/opt/spack/linux-x86_64/gcc-5.2.1/m4-1.4.17-a4ikhddnk2zonr66mbwaqb226uhigcrp/.spack/build.out - -Or, maybe you want to do something with information about package dependencies by generating some ready-to-import Python code holding the tree: - -.. code-block:: sh - - $ spack view print '$name = dict(name = "$name", parents = [$dependencies], children = [$dependents])\n' cmake@3.5.2 - -Producing this Python code - -.. code-block:: python - - ncurses = dict(name = "ncurses", parents = []]) - zlib = dict(name = "zlib", parents = []]) - openssl = dict(name = "openssl", parents = [zlib]]) - cmake = dict(name = "cmake", parents = [ncurses,openssl]]) - - Compiler configuration diff --git a/lib/spack/spack/cmd/view.py b/lib/spack/spack/cmd/view.py index 4861383eaa6..8f1fc9be740 100644 --- a/lib/spack/spack/cmd/view.py +++ b/lib/spack/spack/cmd/view.py @@ -24,9 +24,8 @@ ############################################################################## '''Produce a "view" of a Spack DAG. -A "view" is the product of applying a function on a set of package specs. - -This set consists of: +A "view" is file hierarchy representing the union of a number of +Spack-installed package file hierarchies. The union is formed from: - specs resolved from the package names given by the user (the seeds) @@ -35,7 +34,7 @@ - less any specs with names matching the regular expressions given by `--exclude` -The `view` command provides a number of functions (the "actions"): +The `view` can be built and tore down via a number of methods (the "actions"): - symlink :: a file system view which is a directory hierarchy that is the union of the hierarchies of the installed packages in the DAG @@ -46,8 +45,6 @@ - statlink :: a view producing a status report of a symlink or hardlink view. -- format :: a view printing one string per spec following a given format. - The file system view concept is imspired by Nix, implemented by brett.viren@gmail.com ca 2016. @@ -68,7 +65,6 @@ import os import re -import sys import spack import spack.cmd import llnl.util.tty as tty @@ -117,13 +113,6 @@ def setup_parser(sp): help="Path to file system view directory.") act.add_argument('specs', **specs_opts) - # The formatted print action. - act = ssp.add_parser('print', - help="Print a string to stdout based on given format") - act.add_argument('format', nargs=1, - help="Format describing per-package printout.") - act.add_argument('specs', **specs_opts) - return @@ -190,42 +179,6 @@ def flatten(seeds, descend=True): return flat -def spec2dict(spec): - 'Convert info in a spec into a simple dictionary.' - - # Expclitly convert instead of just returning spec.__dict__ as - # some things need processing or are properties. - # - pkg = spec.package - inst_deps = ','.join([s.name for s in pkg.installed_dependents]), - ret = dict(name=spec.name, - spec=spec.short_spec, - colorspec=spec.cshort_spec, # color - root=spec.root, - prefix=spec.prefix, - version=spec.version, - variants=spec.variants, - namespace=spec.namespace, - compiler=spec.compiler, - architecture=spec.architecture, - dependencies=','.join(spec.dependencies.keys()), - dependents=','.join(spec.dependents.keys()), - external=spec.external or "False", - hash=spec.dag_hash(), - - # package related: - url=pkg.url, - stage=pkg.stage.path, - installed=pkg.installed, - installed_dependents=inst_deps, - build_log=pkg.build_log_path, - rpath=':'.join(pkg.rpath), - - # ... - ) - return ret - - def check_one(spec, path, verbose=False): 'Check status of view in path against spec' dotspack = os.path.join(path, '.spack', spec.name) @@ -326,25 +279,6 @@ def visitor_statlink(specs, args): visitor_check = visitor_statlink -def visitor_print(specs, args): - 'Print a string for each spec using args.format.' - fmt = args.format[0] - from string import Template - t = Template(fmt) - - for spec in specs: - kwds = spec2dict(spec) - try: - text = t.substitute(kwds) - except KeyError: - tty.error("Format error, use keywords: %s" % - (', '.join(kwds.keys()), )) - raise - # argparser escapes these - text = text.replace(r'\n', '\n').replace(r'\t', '\t') - sys.stdout.write(text) - - def view(parser, args): 'Produce a view of a set of packages.' From 19c8a52fe1c25639c47620bc07f0824c59c921f7 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 19 May 2016 09:53:49 -0700 Subject: [PATCH 251/375] fixed spec syntax test --- lib/spack/spack/test/spec_syntax.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index c4e4c9cdfe5..ae19de177bd 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -58,7 +58,7 @@ class SpecSyntaxTest(unittest.TestCase): # ================================================================================ # Parse checks # ================================================================================ - def check_parse(self, expected, spec=None): + def check_parse(self, expected, spec=None, remove_arch=True): """Assert that the provided spec is able to be parsed. If this is called with one argument, it assumes that the string is canonical (i.e., no spaces and ~ instead of - for variants) and that it @@ -70,6 +70,13 @@ def check_parse(self, expected, spec=None): if spec is None: spec = expected output = spack.spec.parse(spec) + + # Remove architectures that get added by parser. + if remove_arch: + for spec in output: + for s in spec.traverse(): + s.architecture = None + parsed = (" ".join(str(spec) for spec in output)) self.assertEqual(expected, parsed) From 36450b9bc9f41ac0f70696352327d8f07d4ee3e3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 19 May 2016 14:25:50 -0700 Subject: [PATCH 252/375] Adding link_paths to craype compilers --- lib/spack/spack/compilers/craype.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/compilers/craype.py b/lib/spack/spack/compilers/craype.py index e8ae284f5bb..4ba8b110ecc 100644 --- a/lib/spack/spack/compilers/craype.py +++ b/lib/spack/spack/compilers/craype.py @@ -47,10 +47,11 @@ class Craype(Compiler): PrgEnv = 'PrgEnv-cray' PrgEnv_compiler = 'craype' -# @property -# def cxx11_flag(self): -# return "-hstd=c++11" - + link_paths = { 'cc' : 'cc', + 'cxx' : 'c++', + 'f77' : 'f77', + 'fc' : 'fc'} + @classmethod def default_version(cls, comp): return get_compiler_version(comp, r'([Vv]ersion).*(\d+(\.\d+)+)') From 175a042fd332c13200fb30f7f834541c3bbaa6c4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 19 May 2016 16:42:13 -0700 Subject: [PATCH 253/375] Changed rpaths method so that it only grabs rpaths using the compiler module name instead of PrgEnv-x module name --- lib/spack/spack/build_environment.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 7aeea5c6723..4f4ca26cbc2 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -113,7 +113,6 @@ def __call__(self, *args, **kwargs): return super(MakeExecutable, self).__call__(*args, **kwargs) - def load_module(mod): """Takes a module name and removes modules until it is possible to load that module. It then loads the provided module. Depends on the @@ -145,7 +144,6 @@ def get_path_from_module(mod): # Read the module text = modulecmd('show', mod, output=str, error=str).split('\n') - # If it lists its package directory, return that for line in text: if line.find(mod.upper()+'_DIR') >= 0: @@ -166,15 +164,14 @@ def get_path_from_module(mod): # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that for line in text: - if line.find('LD_LIBRARY_PATH') >= 0: + if line.find('LD_LIBRARY_PATH') >= 0: words = line.split() path = words[2] return path[:path.find('/lib')] - # Unable to find module path return None -def set_compiler_environment_variables(pkg): +def set_compiler_environment_variables(pkg, env): assert(pkg.spec.concrete) compiler = pkg.compiler flags = pkg.spec.compiler_flags @@ -276,14 +273,12 @@ def set_build_environment_variables(pkg, env): env.set(SPACK_DEBUG_LOG_DIR, spack.spack_working_dir) # Add any pkgconfig directories to PKG_CONFIG_PATH - pkg_config_dirs = [] - for p in dep_prefixes: - for maybe in ('lib', 'lib64', 'share'): - pcdir = join_path(p, maybe, 'pkgconfig') + for pre in dep_prefixes: + for directory in ('lib', 'lib64', 'share'): + pcdir = join_path(pre, directory, 'pkgconfig') if os.path.isdir(pcdir): - pkg_config_dirs.append(pcdir) - - env.prepend_path('PKG_CONFIG_PATH', pkg_config_dirs) + #pkg_config_dirs.append(pcdir) + env.prepend_path('PKG_CONFIG_PATH',pcdir) if pkg.spec.architecture.target.module_name: load_module(pkg.spec.architecture.target.module_name) @@ -369,8 +364,9 @@ def get_rpaths(pkg): if os.path.isdir(d.prefix.lib)) rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values() if os.path.isdir(d.prefix.lib64)) - for mod in pkg.spec.compiler.modules: - rpaths.append(get_path_for_module(mod)) + # Second module is our compiler mod name. We use that to get rpaths from + # module show output. + rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths From b485ca8198ab677e1e0a3616f5d0d25d5024b3ce Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 20 May 2016 10:32:46 +0200 Subject: [PATCH 254/375] find : added unit tests, exits with 1 if -E and -e are given --- lib/spack/spack/cmd/find.py | 41 ++++++++++++++--------- lib/spack/spack/test/__init__.py | 2 +- lib/spack/spack/test/cmd/find.py | 57 ++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 17 deletions(-) create mode 100644 lib/spack/spack/test/cmd/find.py diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index c22268d5345..9d6d6be1ea2 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -180,6 +180,29 @@ def fmt(s): "deps, short)." % mode) # NOQA: ignore=E501 +def query_arguments(args): + # Check arguments + if args.explicit and args.implicit: + tty.error('You can\'t pass -E and -e options simultaneously.') + raise SystemExit(1) + + # Set up query arguments. + installed, known = True, any + if args.only_missing: + installed = False + elif args.missing: + installed = any + if args.unknown: + known = False + explicit = any + if args.explicit: + explicit = True + if args.implicit: + explicit = False + q_args = {'installed': installed, 'known': known, "explicit": explicit} + return q_args + + def find(parser, args): # Filter out specs that don't exist. query_specs = spack.cmd.parse_specs(args.query_specs) @@ -194,22 +217,7 @@ def find(parser, args): if not query_specs: return - # Set up query arguments. - installed, known = True, any - if args.only_missing: - installed = False - elif args.missing: - installed = any - if args.unknown: - known = False - - explicit = any - if args.explicit: - explicit = False - if args.implicit: - explicit = True - - q_args = {'installed': installed, 'known': known, "explicit": explicit} + q_args = query_arguments(args) # Get all the specs the user asked for if not query_specs: @@ -229,3 +237,4 @@ def find(parser, args): long=args.long, very_long=args.very_long, show_flags=args.show_flags) + diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 1668e271faf..891dc873fd1 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -38,7 +38,7 @@ 'svn_fetch', 'hg_fetch', 'mirror', 'modules', 'url_extrapolate', 'cc', 'link_tree', 'spec_yaml', 'optional_deps', 'make_executable', 'configure_guess', 'lock', 'database', - 'namespace_trie', 'yaml', 'sbang', 'environment', + 'namespace_trie', 'yaml', 'sbang', 'environment', 'cmd.find', 'cmd.uninstall', 'cmd.test_install'] diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py new file mode 100644 index 00000000000..c9909c81c50 --- /dev/null +++ b/lib/spack/spack/test/cmd/find.py @@ -0,0 +1,57 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + + +import spack.cmd.find +import unittest + + +class Bunch(object): + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +class FindTest(unittest.TestCase): + def test_query_arguments(self): + query_arguments = spack.cmd.find.query_arguments + # Default arguments + args = Bunch(only_missing=False, missing=False, unknown=False, explicit=False, implicit=False) + q_args = query_arguments(args) + self.assertTrue('installed' in q_args) + self.assertTrue('known' in q_args) + self.assertTrue('explicit' in q_args) + self.assertEqual(q_args['installed'], True) + self.assertEqual(q_args['known'], any) + self.assertEqual(q_args['explicit'], any) + # Check that explicit works correctly + args.explicit = True + q_args = query_arguments(args) + self.assertEqual(q_args['explicit'], True) + args.explicit = False + args.implicit = True + q_args = query_arguments(args) + self.assertEqual(q_args['explicit'], False) + args.explicit = True + self.assertRaises(SystemExit, query_arguments, args) From 7303c387e87c783b2cc5b980456ae21a8bbe7ed4 Mon Sep 17 00:00:00 2001 From: alalazo Date: Fri, 20 May 2016 10:34:11 +0200 Subject: [PATCH 255/375] autopep8 : fixed style --- lib/spack/spack/cmd/find.py | 1 - lib/spack/spack/test/cmd/find.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 9d6d6be1ea2..93c10a910fd 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -237,4 +237,3 @@ def find(parser, args): long=args.long, very_long=args.very_long, show_flags=args.show_flags) - diff --git a/lib/spack/spack/test/cmd/find.py b/lib/spack/spack/test/cmd/find.py index c9909c81c50..371e9650e04 100644 --- a/lib/spack/spack/test/cmd/find.py +++ b/lib/spack/spack/test/cmd/find.py @@ -29,15 +29,18 @@ class Bunch(object): + def __init__(self, **kwargs): self.__dict__.update(kwargs) class FindTest(unittest.TestCase): + def test_query_arguments(self): query_arguments = spack.cmd.find.query_arguments # Default arguments - args = Bunch(only_missing=False, missing=False, unknown=False, explicit=False, implicit=False) + args = Bunch(only_missing=False, missing=False, + unknown=False, explicit=False, implicit=False) q_args = query_arguments(args) self.assertTrue('installed' in q_args) self.assertTrue('known' in q_args) From 09eed08b9c94a41c46664ca275602deacef40346 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Wed, 18 May 2016 13:41:39 -0400 Subject: [PATCH 256/375] Auto-detect verbs location --- .../repos/builtin/packages/openmpi/package.py | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 0e3185db258..4e465e1784d 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -27,6 +27,26 @@ from spack import * +def _verbs_dir(): + """ + Try to find the directory where the OpenFabrics verbs package is + installed. Return None if not found. + """ + try: + # Try to locate Verbs by looking for a utility in the path + ibv_devices = which("ibv_devices") + # Run it (silently) to ensure it works + ibv_devices(output=str, error=str) + # Get path to executable + path = ibv_devices.exe[0] + # Remove executable name and "bin" directory + path = os.path.dirname(path) + path = os.path.dirname(path) + return path + except: + return None + + class Openmpi(Package): """Open MPI is a project combining technologies and resources from several other projects (FT-MPI, LA-MPI, LAM/MPI, and PACX-MPI) @@ -54,7 +74,7 @@ class Openmpi(Package): variant('psm', default=False, description='Build support for the PSM library.') variant('psm2', default=False, description='Build support for the Intel PSM2 library.') variant('pmi', default=False, description='Build support for PMI-based launchers') - variant('verbs', default=False, description='Build support for OpenFabrics verbs.') + variant('verbs', default=_verbs_dir() is not None, description='Build support for OpenFabrics verbs.') variant('mxm', default=False, description='Build Mellanox Messaging support') variant('thread_multiple', default=False, description='Enable MPI_THREAD_MULTIPLE support') @@ -113,7 +133,6 @@ def install(self, spec, prefix): # Fabrics '--with-psm' if '+psm' in spec else '--without-psm', '--with-psm2' if '+psm2' in spec else '--without-psm2', - ('--with-%s' % self.verbs) if '+verbs' in spec else ('--without-%s' % self.verbs), '--with-mxm' if '+mxm' in spec else '--without-mxm', # Other options '--enable-mpi-thread-multiple' if '+thread_multiple' in spec else '--disable-mpi-thread-multiple', @@ -121,6 +140,14 @@ def install(self, spec, prefix): '--with-sqlite3' if '+sqlite3' in spec else '--without-sqlite3', '--enable-vt' if '+vt' in spec else '--disable-vt' ]) + if '+verbs' in spec: + path = _verbs_dir() + if path is not None: + config_args.append('--with-%s=%s' % (self.verbs, path)) + else: + config_args.append('--with-%s' % self.verbs) + else: + config_args.append('--without-%s' % self.verbs) # TODO: use variants for this, e.g. +lanl, +llnl, etc. # use this for LANL builds, but for LLNL builds, we need: From c3767d593d04f990b4344540c95e07f172d54c42 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 20 May 2016 15:48:13 -0700 Subject: [PATCH 257/375] compiler list is sorted in descending order, in order to get most up to date version we check from the beginning of the list --- lib/spack/spack/concretize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f38afd38dcc..1c451296c60 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -302,10 +302,10 @@ def _proper_compiler_style(cspec, architecture): raise UnavailableCompilerVersionError(other_compiler) # copy concrete version into other_compiler - index = len(matches)-1 + index = 0 while not _proper_compiler_style(matches[index], spec.architecture): - index -= 1 - if index == 0: + index += 1 + if index == len(matches) - 1: raise NoValidVersionError(spec) spec.compiler = matches[index].copy() assert(spec.compiler.concrete) From 404b7c7c4fe721f3b7236f6d02fb1330d4245858 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Mon, 16 May 2016 09:06:03 -0700 Subject: [PATCH 258/375] initial rust support --- .../builtin/packages/rust-bindgen/package.py | 13 ++++ .../repos/builtin/packages/rust/package.py | 62 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 var/spack/repos/builtin/packages/rust-bindgen/package.py create mode 100644 var/spack/repos/builtin/packages/rust/package.py diff --git a/var/spack/repos/builtin/packages/rust-bindgen/package.py b/var/spack/repos/builtin/packages/rust-bindgen/package.py new file mode 100644 index 00000000000..8d48ea096e1 --- /dev/null +++ b/var/spack/repos/builtin/packages/rust-bindgen/package.py @@ -0,0 +1,13 @@ +from spack import * +import os + + +class RustBindgen(Package): + """The rust programming language toolchain""" + homepage = "http://www.rust-lang.org" + url = "https://github.com/crabtw/rust-bindgen" + + extends("rust") + + def install(self, spec, prefix): + cargo('install', '--root', prefix) diff --git a/var/spack/repos/builtin/packages/rust/package.py b/var/spack/repos/builtin/packages/rust/package.py new file mode 100644 index 00000000000..abc10dc047c --- /dev/null +++ b/var/spack/repos/builtin/packages/rust/package.py @@ -0,0 +1,62 @@ +from spack import * +import os + + +def get_submodules(): + git = which('git') + git('submodule', 'update', '--init', '--recursive') + +class Rust(Package): + """The rust programming language toolchain""" + homepage = "http://www.rust-lang.org" + url = "https://github.com/rust-lang/rust" + + version('1.8.0', tag='1.8.0', git="https://github.com/rust-lang/rust") + + resource(name='cargo', + git="https://github.com/rust-lang/cargo.git", + tag='0.10.0', + destination='cargo') + + extendable = True + + # Rust + depends_on("curl") + depends_on("git") + depends_on("cmake") + depends_on("python@:2.8") + + # Cargo + depends_on("openssl") + + def install(self, spec, prefix): + get_submodules() + configure('--prefix=%s' % prefix) + + make() + make("install") + + # Install cargo, rust package manager + with working_dir(os.path.join('cargo', 'cargo')): + get_submodules() + configure('--prefix=' + prefix, + '--local-rust-root=' + prefix) + + make() + make("install") + + def setup_dependent_package(self, module, ext_spec): + """ + Called before python modules' install() methods. + + In most cases, extensions will only need to have one or two lines:: + + cargo('build') + cargo('install', '--root', prefix) + + or + + cargo('install', '--root', prefix) + """ + # Rust extension builds can have a global cargo executable function + module.cargo = Executable(join_path(self.spec.prefix.bin, 'cargo')) From 425beb4d97d2a54f082c6ffbc90d05ef7d7ab7c5 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 18 May 2016 10:57:11 -0700 Subject: [PATCH 259/375] updates for rust-bindgen --- var/spack/repos/builtin/packages/rust-bindgen/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/rust-bindgen/package.py b/var/spack/repos/builtin/packages/rust-bindgen/package.py index 8d48ea096e1..a2164f11c56 100644 --- a/var/spack/repos/builtin/packages/rust-bindgen/package.py +++ b/var/spack/repos/builtin/packages/rust-bindgen/package.py @@ -7,6 +7,8 @@ class RustBindgen(Package): homepage = "http://www.rust-lang.org" url = "https://github.com/crabtw/rust-bindgen" + version('0.16', tag='0.16', git='https://github.com/crabtw/rust-bindgen') + extends("rust") def install(self, spec, prefix): From 9bd4bc02d6dfa60485e193d994a504c01e4d51e1 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Thu, 19 May 2016 13:59:40 -0700 Subject: [PATCH 260/375] fixing warning for package fetch --- lib/spack/spack/fetch_strategy.py | 33 +++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 7c8cebe0c9c..1953d7c1b38 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -1,4 +1,4 @@ -############################################################################## +# # Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # @@ -21,7 +21,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## +# """ Fetch strategies are used to download source code into a staging area in order to build it. They need to define the following methods: @@ -75,11 +75,13 @@ def wrapper(self, *args, **kwargs): class FetchStrategy(object): + """Superclass of all fetch strategies.""" enabled = False # Non-abstract subclasses should be enabled. required_attributes = None # Attributes required in version() args. class __metaclass__(type): + """This metaclass registers all fetch strategies in a list.""" def __init__(cls, name, bases, dict): @@ -126,6 +128,7 @@ def matches(cls, args): @pattern.composite(interface=FetchStrategy) class FetchStrategyComposite(object): + """ Composite for a FetchStrategy object. Implements the GoF composite pattern. """ @@ -134,6 +137,7 @@ class FetchStrategyComposite(object): class URLFetchStrategy(FetchStrategy): + """FetchStrategy that pulls source code from a URL for an archive, checks the archive against a checksum,and decompresses the archive. """ @@ -235,12 +239,13 @@ def fetch(self): # redirects properly. content_types = re.findall(r'Content-Type:[^\r\n]+', headers) if content_types and 'text/html' in content_types[-1]: - tty.warn( - "The contents of " + self.archive_file + " look like HTML.", - "The checksum will likely be bad. If it is, you can use", - "'spack clean ' to remove the bad archive, then fix", - "your internet gateway issue and install again.") - + tty.warn("The contents of ", + (self.archive_file if self.archive_file is not None + else "the archive"), + " look like HTML.", + "The checksum will likely be bad. If it is, you can use", + "'spack clean ' to remove the bad archive, then", + "fix your internet gateway issue and install again.") if save_file: os.rename(partial_file, save_file) @@ -353,6 +358,7 @@ def __str__(self): class VCSFetchStrategy(FetchStrategy): + def __init__(self, name, *rev_types, **kwargs): super(VCSFetchStrategy, self).__init__() self.name = name @@ -407,6 +413,7 @@ def __repr__(self): class GoFetchStrategy(VCSFetchStrategy): + """ Fetch strategy that employs the `go get` infrastructure Use like this in a package: @@ -466,6 +473,7 @@ def __str__(self): class GitFetchStrategy(VCSFetchStrategy): + """ Fetch strategy that gets source code from a git repository. Use like this in a package: @@ -586,6 +594,7 @@ def __str__(self): class SvnFetchStrategy(VCSFetchStrategy): + """Fetch strategy that gets source code from a subversion repository. Use like this in a package: @@ -662,6 +671,7 @@ def __str__(self): class HgFetchStrategy(VCSFetchStrategy): + """ Fetch strategy that gets source code from a Mercurial repository. Use like this in a package: @@ -806,11 +816,13 @@ def for_package_version(pkg, version): class FetchError(spack.error.SpackError): + def __init__(self, msg, long_msg=None): super(FetchError, self).__init__(msg, long_msg) class FailedDownloadError(FetchError): + """Raised wen a download fails.""" def __init__(self, url, msg=""): @@ -820,16 +832,19 @@ def __init__(self, url, msg=""): class NoArchiveFileError(FetchError): + def __init__(self, msg, long_msg): super(NoArchiveFileError, self).__init__(msg, long_msg) class NoDigestError(FetchError): + def __init__(self, msg, long_msg=None): super(NoDigestError, self).__init__(msg, long_msg) class InvalidArgsError(FetchError): + def __init__(self, pkg, version): msg = ("Could not construct a fetch strategy for package %s at " "version %s") @@ -838,6 +853,7 @@ def __init__(self, pkg, version): class ChecksumError(FetchError): + """Raised when archive fails to checksum.""" def __init__(self, message, long_msg=None): @@ -845,6 +861,7 @@ def __init__(self, message, long_msg=None): class NoStageError(FetchError): + """Raised when fetch operations are called before set_stage().""" def __init__(self, method): From 00de72272d2ac4fd949c43cdaa9487defc7875c8 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sun, 22 May 2016 11:14:46 -0700 Subject: [PATCH 261/375] rust compiler/builder package, new llvm variants --- .../repos/builtin/packages/llvm/package.py | 34 +++++++++++++++---- .../builtin/packages/rust-bindgen/package.py | 5 ++- .../repos/builtin/packages/rust/package.py | 5 +-- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index c090c131c6a..c32f66590a6 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -23,7 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os, shutil +import os, glob class Llvm(Package): @@ -46,7 +46,9 @@ class Llvm(Package): variant('libcxx', default=True, description="Build the LLVM C++ standard library") variant('compiler-rt', default=True, description="Build the LLVM compiler runtime, including sanitizers") variant('gold', default=True, description="Add support for LTO with the gold linker plugin") - + variant('shared_libs', default=False, description="Build all components as shared libraries, faster, less memory to build, less stable") + variant('link_dylib', default=False, description="Build and link the libLLVM shared library rather than static") + variant('all_targets', default=True, description="Build all supported targets, default targets ,NVPTX,AMDGPU,CppBackend") # Build dependency depends_on('cmake @2.8.12.2:') @@ -257,6 +259,28 @@ def install(self, spec, prefix): if '+compiler-rt' not in spec: cmake_args.append('-DLLVM_EXTERNAL_COMPILER_RT_BUILD:Bool=OFF') + if '+shared_libs' in spec: + cmake_args.append('-DBUILD_SHARED_LIBS:Bool=ON') + + if '+link_dylib' in spec: + cmake_args.append('-DLLVM_LINK_LLVM_DYLIB:Bool=ON') + + if '+all_targets' not in spec: # all is default on cmake + targets = ['CppBackend', 'NVPTX', 'AMDGPU'] + if 'x86' in spec.architecture.lower(): + targets.append('X86') + elif 'arm' in spec.architecture.lower(): + targets.append('ARM') + elif 'aarch64' in spec.architecture.lower(): + targets.append('AArch64') + elif 'sparc' in spec.architecture.lower(): + targets.append('sparc') + elif ('ppc' in spec.architecture.lower() or + 'power' in spec.architecture.lower()): + targets.append('PowerPC') + + cmake_args.append('-DLLVM_TARGETS_TO_BUILD:Bool=' + ';'.join(targets)) + if '+clang' not in spec: if '+clang_extra' in spec: raise SpackException('The clang_extra variant requires the clang variant to be selected') @@ -267,7 +291,5 @@ def install(self, spec, prefix): cmake(*cmake_args) make() make("install") - query_path = os.path.join('bin', 'clang-query') - # Manually install clang-query, because llvm doesn't... - if os.path.exists(query_path): - shutil.copy(query_path, os.path.join(prefix, 'bin')) + cp = which('cp') + cp('-a', 'bin/', prefix) diff --git a/var/spack/repos/builtin/packages/rust-bindgen/package.py b/var/spack/repos/builtin/packages/rust-bindgen/package.py index a2164f11c56..854016d12a3 100644 --- a/var/spack/repos/builtin/packages/rust-bindgen/package.py +++ b/var/spack/repos/builtin/packages/rust-bindgen/package.py @@ -10,6 +10,9 @@ class RustBindgen(Package): version('0.16', tag='0.16', git='https://github.com/crabtw/rust-bindgen') extends("rust") + depends_on("llvm") def install(self, spec, prefix): - cargo('install', '--root', prefix) + env = dict(os.environ) + env['LIBCLANG_PATH'] = os.path.join(spec['llvm'].prefix, 'lib') + cargo('install', '--root', prefix, env=env) diff --git a/var/spack/repos/builtin/packages/rust/package.py b/var/spack/repos/builtin/packages/rust/package.py index abc10dc047c..65f81ce5342 100644 --- a/var/spack/repos/builtin/packages/rust/package.py +++ b/var/spack/repos/builtin/packages/rust/package.py @@ -21,6 +21,7 @@ class Rust(Package): extendable = True # Rust + depends_on("llvm") depends_on("curl") depends_on("git") depends_on("cmake") @@ -30,8 +31,8 @@ class Rust(Package): depends_on("openssl") def install(self, spec, prefix): - get_submodules() - configure('--prefix=%s' % prefix) + configure('--prefix=%s' % prefix, + '--llvm-root=' + spec['llvm'].prefix) make() make("install") From 6c352132d0dd2c0f05270fa7888d47ca37547674 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 23 May 2016 14:28:00 -0700 Subject: [PATCH 262/375] Started writing documentation for architecture. Might need a couple more drafts --- lib/spack/spack/architecture.py | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 41778795c38..45b4cc228d7 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,6 +22,58 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +""" +This module contains all the elements that are required to create an +architecture object. These include, the target processor, the operating system, +and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. + +On a multiple architecture machine, the architecture spec field can be set to +build a package against any target and operating system that is present on the +platform. On Cray platforms or any other architecture that has different front and +back end environments, the operating system will determine the method of compiler +detection. + +There are two different types of compiler detection: + 1. Through the $PATH env variable (front-end detection) + 2. Through the tcl module system. (back-end detection) + +Depending on which operating system is specified, the compiler will be detected +using one of those methods. + +For platforms such as linux and darwin, the operating system is autodetected and +the target is set to be x86_64. + +The command line syntax for specifying an architecture is as follows: + + target= os= + +If the user wishes to use the defaults, either target or os can be left out of +the command line and Spack will concretize using the default. These defaults are +set in the 'platforms/' directory which contains the different subclasses for +platforms. If the machine has multiple architectures, the user can +also enter front-end, or fe or back-end or be. These settings will concretize +to their respective front-end and back-end targets and operating systems. +Additional platforms can be added by creating a subclass of Platform +and adding it inside the platform directory. + +Platforms are an abstract class that are extended by subclasses. If the user +wants to add a new type of platform (such as cray_xe), they can create a subclass +and set all the class attributes such as priority, front_target ,back_target, +front_os, back_os. Platforms also contain a priority class attribute. A lower +number signifies higher priority. These numbers are arbitrarily set and can be +changed though often there isn't much need unless a new platform is added and +the user wants that to be detected first. + +Targets are created inside the platform subclasses. Most architecture (like linux, +and darwin) will have only one target (x86_64) but in the case of Cray machines, +there is both a frontend and backend processor. The user can specify which targets +are present on front-end and back-end architecture + +Depending on the platform, operating systems are either auto-detected or are +set. The user can set the front-end and back-end operating setting by the class +attributes front_os and back_os. The operating system as described earlier, will +be responsible for compiler detection. +""" import os from collections import namedtuple import imp From d7612e7aaace3b0fa483f17840e20cde5b28f430 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 23 May 2016 16:54:41 -0700 Subject: [PATCH 263/375] Fixed errors caught by spec and concretize tests --- lib/spack/spack/architecture.py | 78 ++++++++++++++++++++----- lib/spack/spack/concretize.py | 23 ++++++-- lib/spack/spack/spec.py | 81 +++++++++++++++++++++----- lib/spack/spack/test/architecture.py | 2 + lib/spack/spack/test/concretize.py | 4 +- lib/spack/spack/test/spec_semantics.py | 14 ++--- lib/spack/spack/test/spec_syntax.py | 6 -- 7 files changed, 160 insertions(+), 48 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 41778795c38..e4b3dbf9c79 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -105,12 +105,25 @@ def __init__(self, name): self.operating_sys = {} self.name = name + def to_dict(self): + n = {} + n['targets'] = dict((name, target.to_dict()) for (name, target) in self.targets.items()) + n['operating_systems'] = dict((name, os.to_dict()) for (name, os) in self.operating_sys.items()) + n['priority'] = self.priority + n['default_front_end_target'] = self.front_end + n['default_back_end_target'] = self.back_end + n['default_target'] = self.default + n['default_front_end_os'] = self.front_os + n['default_back_end_os'] = self.back_os + n['default_os'] = self.default_os + return {self.name: n} + def add_target(self, name, target): """Used by the platform specific subclass to list available targets. Raises an error if the platform specifies a name that is reserved by spack as an alias. """ - if name in ['front_end', 'fe', 'back_end', 'be', 'default']: + if name in ['frontend', 'fe', 'backend', 'be', 'default_target']: raise ValueError( "%s is a spack reserved alias " "and cannot be the name of a target" % name) @@ -135,6 +148,10 @@ def add_operating_system(self, name, os_class): """ Add the operating_system class object into the platform.operating_sys dictionary """ + if name in ['frontend', 'fe', 'backend', 'be', 'default_os']: + raise ValueError( + "%s is a spack reserved alias " + "and cannot be the name of an OS" % name) self.operating_sys[name] = os_class def operating_system(self, name): @@ -284,15 +301,18 @@ def to_dict(self): class Arch(object): "Architecture is now a class to help with setting attributes" - def __init__(self, platform_os=None, target=None): - self.platform = sys_type() - if platform_os: - platform_os = self.platform.operating_system(platform_os) + def __init__(self, platform=None, platform_os=None, target=None): + self.platform = platform + if platform and platform_os: + platform_os = self.platform.operating_system(platform_os) self.platform_os = platform_os - if target: + if platform and target: target = self.platform.target(target) self.target = target + # Hooks for parser to use when platform is set after target or os + self.target_string = None + self.os_string = None @property def concrete(self): @@ -302,16 +322,19 @@ def concrete(self): def __str__(self): - if self.platform.name == 'darwin': - os_name = self.platform_os.name - else: - os_name = str(self.platform_os) + if self.platform or self.platform_os or self.target: + if self.platform.name == 'darwin': + os_name = self.platform_os.name + else: + os_name = str(self.platform_os) - return (str(self.platform) +"-"+ - os_name + "-" + str(self.target)) + return (str(self.platform) +"-"+ + os_name + "-" + str(self.target)) + else: + return '' def _cmp_key(self): - platform = self.platform.name + platform = self.platform.name if isinstance(self.platform, Platform) else self.platform os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os target = self.target.name if isinstance(self.target, Target) else self.target return (platform, os, target) @@ -322,7 +345,7 @@ def to_dict(self): platform_os = self.platform_os target = self.target - d['platform'] = self.platform.name + d['platform'] = self.platform.to_dict() if self.platform else None d['platform_os'] = self.platform_os.to_dict() if self.platform_os else None d['target'] = self.target.to_dict() if self.target else None @@ -350,6 +373,27 @@ def _operating_system_from_dict(os_dict): operating_system.version = os_dict['version'] return operating_system +def _platform_from_dict(platform_dict): + """ Constructs a platform from a dictionary. """ + platform = Platform.__new__(Platform) + name, p_dict = platform_dict.items()[0] + platform.name = name + platform.targets = {} + for name, t_dict in p_dict['targets'].items(): + platform.add_target(name, _target_from_dict(t_dict)) + platform.operating_sys = {} + for name, o_dict in p_dict['operating_systems'].items(): + platform.add_operating_system(name, _operating_system_from_dict(o_dict)) + platform.priority = p_dict['priority'] + platform.front_end = p_dict['default_front_end_target'] + platform.back_end = p_dict['default_back_end_target'] + platform.default = p_dict['default_target'] + platform.front_os = p_dict['default_front_end_os'] + platform.back_os = p_dict['default_back_end_os'] + platform.default_os = p_dict['default_os'] + + return platform + def arch_from_dict(d): """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict helper methods to recreate the arch tuple from the dictionary read from @@ -359,14 +403,20 @@ def arch_from_dict(d): if d is None: return None + platform_dict = d['platform'] os_dict = d['platform_os'] target_dict = d['target'] + platform = _platform_from_dict(platform_dict) if platform_dict else None target = _target_from_dict(target_dict) if os_dict else None platform_os = _operating_system_from_dict(os_dict) if os_dict else None + arch.platform = platform arch.target = target arch.platform_os = platform_os + arch.os_string = None + arch.target_string = None + return arch @memoized diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index f38afd38dcc..1f5c07779e3 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -198,7 +198,7 @@ def _concretize_operating_system(self, spec): spec.architecture.platform_os = spec.root.architecture.platform_os else: spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') - return True #changed + return True #changed def _concretize_target(self, spec): platform = spec.architecture.platform @@ -210,7 +210,19 @@ def _concretize_target(self, spec): spec.architecture.target = spec.root.architecture.target else: spec.architecture.target = spec.architecture.platform.target('default_target') - return True #changed + print spec.architecture, spec.architecture.platform, spec.architecture.platform_os, spec.architecture.target + return True #changed + + def _concretize_platform(self, spec): + if spec.architecture.platform is not None and isinstance( + spec.architecture.platform, spack.architecture.Platform): + return False + if spec.root.architecture and spec.root.architecture.platform: + if isinstance(spec.root.architecture.platform,spack.architecture.Platform): + spec.architecture.platform = spec.root.architecture.platform + else: + spec.architecture.platform = spack.architecture.sys_type() + return True #changed? def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the @@ -227,10 +239,11 @@ def concretize_architecture(self, spec): # Set the architecture to all defaults spec.architecture = spack.architecture.Arch() return True - + # Concretize the operating_system and target based of the spec - ret = any((self._concretize_operating_system(spec), - self._concretize_target(spec))) + ret = any((self._concretize_platform(spec), + self._concretize_operating_system(spec), + self._concretize_target(spec))) return ret diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index f88475d1c82..d15598405fc 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -99,6 +99,7 @@ import itertools import hashlib import base64 +import imp from StringIO import StringIO from operator import attrgetter import yaml @@ -107,6 +108,7 @@ import llnl.util.tty as tty from llnl.util.lang import * from llnl.util.tty.color import * +from llnl.util.filesystem import join_path import spack import spack.architecture @@ -119,6 +121,7 @@ from spack.version import * from spack.util.string import * from spack.util.prefix import Prefix +from spack.util.naming import mod_to_class from spack.virtual import ProviderIndex from spack.build_environment import get_path_from_module, load_module @@ -535,10 +538,24 @@ def _add_flag(self, name, value): Known flags currently include "arch" """ valid_flags = FlagMap.valid_compiler_flags() - if name == 'os' or name == 'operating_system': - self._set_os(value) +# if name == 'arch' or name == 'architecture': +# platform, op_sys, target = value.split('-') +# print platform, op_sys, target, '+++++++' +# self._set_platform(platform) +# self._set_os(op_sys) +# self._set_target(target) + if name == 'platform': + self._set_platform(value) + elif name == 'os' or name == 'operating_system': + if self.architecture.platform: + self._set_os(value) + else: + self.architecture.os_string = value elif name == 'target': - self._set_target(value) + if self.architecture.platform: + self._set_target(value) + else: + self.architecture.target_string = value elif name in valid_flags: assert(self.compiler_flags is not None) self.compiler_flags[name] = value.split() @@ -551,6 +568,39 @@ def _set_compiler(self, compiler): "Spec for '%s' cannot have two compilers." % self.name) self.compiler = compiler + def _set_platform(self, value): + """Called by the parser to set the architecture platform""" + if isinstance(value, basestring): + mod_path = spack.platform_path + mod_string = 'spack.platformss' + names = list_modules(mod_path) + if value in names: + # Create a platform object from the name + mod_name = mod_string + value + path = join_path(mod_path, value) + '.py' + mod = imp.load_source(mod_name, path) + class_name = mod_to_class(value) + if not hasattr(mod, class_name): + tty.die('No class %s defined in %s' % (class_name, mod_name)) + cls = getattr(mod, class_name) + if not inspect.isclass(cls): + tty.die('%s.%s is not a class' % (mod_name, class_name)) + platform = cls() + else: + tty.die("No platform class %s defined." % value) + else: + # The value is a platform + platform = value + + self.architecture.platform = platform + + # Set os and target if we previously got strings for them + if self.architecture.os_string: + self._set_os(self.architecture.os_string) + self.architecture.os_string = None + if self.architecture.target_string: + self._set_target(self.architecture.target_string) + self.architecture.target_string = None def _set_os(self, value): """Called by the parser to set the architecture operating system""" @@ -1016,6 +1066,7 @@ def _expand_virtual_packages(self): changed = True spec.dependencies = DependencyMap() replacement.dependencies = DependencyMap() + replacement.architecture = self.architecture # TODO: could this and the stuff in _dup be cleaned up? def feq(cfield, sfield): @@ -1426,7 +1477,6 @@ def constrain(self, other, deps=True): other.variants[v]) # TODO: Check out the logic here - print self.architecture, other.architecture, "^^^^^^^^^^^^^^^^^^^^^^^" if self.architecture is not None and other.architecture is not None: if self.architecture.platform is not None and other.architecture.platform is not None: if self.architecture.platform != other.architecture.platform: @@ -1831,8 +1881,7 @@ def _cmp_node(self): self.variants, self.architecture, self.compiler, - self.compiler_flags, - self.dag_hash()) + self.compiler_flags) def eq_node(self, other): @@ -1946,7 +1995,7 @@ def write(s, c): if self.variants: write(fmt % str(self.variants), c) elif c == '=': - if self.architecture: + if self.architecture and str(self.architecture): write(fmt % (' arch' + c + str(self.architecture)), c) elif c == '#': out.write('-' + fmt % (self.dag_hash(7))) @@ -2004,7 +2053,7 @@ def write(s, c): if self.variants: write(fmt % str(self.variants), '+') elif named_str == 'ARCHITECTURE': - if self.architecture: + if self.architecture and str(self.architecture): write(fmt % str(self.architecture), ' arch=') elif named_str == 'SHA1': if self.dependencies: @@ -2054,13 +2103,13 @@ def __cmp__(self, other): self.variants, other.variants) #Target - if self.target != other.target: - return spack.pkgsort.target_compare(pkgname, - self.target, other.target) + if self.architecture != other.architecture: + return spack.pkgsort.architecture_compare(pkgname, + self.architecture, other.architecture) #Dependency is not configurable - if self.dep_hash() != other.dep_hash(): - return -1 if self.dep_hash() < other.dep_hash() else 1 + if self.dependencies != other.dependencies: + return -1 if self.dependencies < other.dependencies else 1 #Equal specs return 0 @@ -2181,6 +2230,11 @@ def do_parse(self): raise SpecParseError(e) + # If the spec has an os or a target and no platform, give it the default platform + for spec in specs: + for s in spec.traverse(): + if s.architecture.os_string or s.architecture.target_string: + s._set_platform(spack.architecture.sys_type()) return specs @@ -2401,7 +2455,6 @@ class SpecError(spack.error.SpackError): def __init__(self, message): super(SpecError, self).__init__(message) - class SpecParseError(SpecError): """Wrapper for ParseError for when we're parsing specs.""" def __init__(self, parse_error): diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 2927e468a05..f5b10684355 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -26,12 +26,14 @@ def tearDown(self): def test_dict_functions_for_architecture(self): arch = Arch() + arch.platform = spack.architecture.sys_type() arch.platform_os = arch.platform.operating_system('default_os') arch.target = arch.platform.target('default_target') d = arch.to_dict() new_arch = spack.architecture.arch_from_dict(d) + self.assertEqual(arch, new_arch) self.assertTrue( isinstance(arch, Arch) ) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 49281b9a0c4..963481054e5 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -55,8 +55,8 @@ def check_spec(self, abstract, concrete): if abstract.compiler and abstract.compiler.concrete: self.assertEqual(abstract.compiler, concrete.compiler) - if abstract.architecture and abstract.architecture.target.concrete: - self.assertEqual(abstract.target, concrete.target) + if abstract.architecture and abstract.architecture.concrete: + self.assertEqual(abstract.architecture, concrete.architecture) def check_concretize(self, abstract_spec): diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 9bd32a3d10d..45c89100d43 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -383,14 +383,14 @@ def test_constrain_compiler(self): def test_invalid_constraint(self): -# self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') -# self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') + self.check_invalid_constraint('libelf@0:2.0', 'libelf@2.1:3') + self.check_invalid_constraint('libelf@0:2.5%gcc@4.8:4.9', 'libelf@2.1:3%gcc@4.5:4.7') -# self.check_invalid_constraint('libelf+debug', 'libelf~debug') -# self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') -# self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') + self.check_invalid_constraint('libelf+debug', 'libelf~debug') + self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo') + self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') -# self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') + self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') platform = spack.architecture.sys_type() if len(platform.operating_sys.keys()) > 1 or len(platform.targets.keys()) > 1: os1 = platform.operating_sys.keys()[0] @@ -439,9 +439,9 @@ def test_constrain_dependency_changed(self): self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5') self.check_constrain_changed('libelf^foo', 'libelf^foo+debug') self.check_constrain_changed('libelf^foo', 'libelf^foo~debug') + platform = spack.architecture.sys_type() default_target = platform.target('default_target').name - print default_target self.check_constrain_changed('libelf^foo', 'libelf^foo target='+default_target) diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index ae19de177bd..4a534d7b5ce 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -71,12 +71,6 @@ def check_parse(self, expected, spec=None, remove_arch=True): spec = expected output = spack.spec.parse(spec) - # Remove architectures that get added by parser. - if remove_arch: - for spec in output: - for s in spec.traverse(): - s.architecture = None - parsed = (" ".join(str(spec) for spec in output)) self.assertEqual(expected, parsed) From 1da6bbd14667b50f8c970022614baf39e059a779 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 23 May 2016 17:25:54 -0700 Subject: [PATCH 264/375] Fixed a couple more bugs --- lib/spack/spack/build_environment.py | 3 ++- lib/spack/spack/concretize.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 4f4ca26cbc2..073f73a91a1 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -366,7 +366,8 @@ def get_rpaths(pkg): if os.path.isdir(d.prefix.lib64)) # Second module is our compiler mod name. We use that to get rpaths from # module show output. - rpaths.append(get_path_from_module(pkg.compiler.modules[1])) + if pkg.compiler.modules and len(pkg.compiler.modules > 1): + rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 7ea8ea69287..7a24c1135e3 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -210,7 +210,6 @@ def _concretize_target(self, spec): spec.architecture.target = spec.root.architecture.target else: spec.architecture.target = spec.architecture.platform.target('default_target') - print spec.architecture, spec.architecture.platform, spec.architecture.platform_os, spec.architecture.target return True #changed def _concretize_platform(self, spec): From 513aae5ef8d610644e8f2d2c855e92a080573f5b Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Mon, 23 May 2016 17:39:06 -0700 Subject: [PATCH 265/375] fixed bug where earlier test was blowing away environment, causing an error on which('modulecmd') --- lib/spack/spack/test/environment.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index ded1539e18e..a0d959db2f1 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -24,17 +24,20 @@ ############################################################################## import unittest import os +import copy from spack.environment import EnvironmentModifications class EnvironmentTest(unittest.TestCase): def setUp(self): - os.environ.clear() os.environ['UNSET_ME'] = 'foo' os.environ['EMPTY_PATH_LIST'] = '' os.environ['PATH_LIST'] = '/path/second:/path/third' os.environ['REMOVE_PATH_LIST'] = '/a/b:/duplicate:/a/c:/remove/this:/a/d:/duplicate/:/f/g' + def tearDown(self): + pass + def test_set(self): env = EnvironmentModifications() env.set('A', 'dummy value') From 9c95ec0b299f7f0eb83362318f77e31701708902 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Mon, 2 May 2016 16:41:55 -0700 Subject: [PATCH 266/375] Updated and added new versions to the METIS/ParMETIS packages. - Added a 'url_for_version' function to the METIS/ParMETIS packages. - Added installation support for METIS@5.0.2 and ParMETIS@4.0.2. - Changed the 'double' variant to 'real64' to make it more consistent with the 'idx64' variant naming. - Removed an unnecessary dependency on gdb. --- .../repos/builtin/packages/metis/package.py | 35 ++++++++----------- .../builtin/packages/parmetis/package.py | 18 +++++----- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index 061179b78e5..daeaf6245b1 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -33,30 +33,30 @@ class Metis(Package): recursive-bisection, multilevel k-way, and multi-constraint partitioning schemes. """ - homepage = 'http://glaros.dtc.umn.edu/gkhome/metis/metis/overview' - url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz" + homepage = "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview" + base_url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis" - version('5.1.0', '5465e67079419a69e0116de24fce58fe', - url='http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz') - version('4.0.3', '5efa35de80703c1b2c4d0de080fafbcf4e0d363a21149a1ad2f96e0144841a55', - url='http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/OLD/metis-4.0.3.tar.gz') + version('5.1.0', '5465e67079419a69e0116de24fce58fe') + version('5.0.2', 'acb521a4e8c2e6dd559a7f9abd0468c5') + version('4.0.3', 'd3848b454532ef18dc83e4fb160d1e10') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds the library in debug mode') variant('gdb', default=False, description='Enables gdb support') variant('idx64', default=False, description='Use int64_t as default index type') - variant('double', default=False, description='Use double precision floating point types') + variant('real64', default=False, description='Use double precision floating point types') depends_on('cmake @2.8:', when='@5:') # build-time dependency - depends_on('gdb', when='+gdb') patch('install_gklib_defs_rename.patch', when='@5:') + def url_for_version(self, version): + version_dir = 'OLD/' if version < Version('4.0.3') else '' + return '%s/%smetis-%s.tar.gz' % (Metis.base_url, version_dir, version) @when('@4:4.0.3') def install(self, spec, prefix): - if '+gdb' in spec: raise InstallError('gdb support not implemented in METIS 4!') if '+idx64' in spec: @@ -128,7 +128,6 @@ def install(self, spec, prefix): @when('@5:') def install(self, spec, prefix): - options = [] options.extend(std_cmake_args) @@ -140,20 +139,15 @@ def install(self, spec, prefix): if '+shared' in spec: options.append('-DSHARED:BOOL=ON') - if '+debug' in spec: - options.extend(['-DDEBUG:BOOL=ON', - '-DCMAKE_BUILD_TYPE:STRING=Debug']) - + options.extend(['-DDEBUG:BOOL=ON', '-DCMAKE_BUILD_TYPE:STRING=Debug']) if '+gdb' in spec: options.append('-DGDB:BOOL=ON') metis_header = join_path(source_directory, 'include', 'metis.h') - if '+idx64' in spec: filter_file('IDXTYPEWIDTH 32', 'IDXTYPEWIDTH 64', metis_header) - - if '+double' in spec: + if '+real64' in spec: filter_file('REALTYPEWIDTH 32', 'REALTYPEWIDTH 64', metis_header) # Make clang 7.3 happy. @@ -166,9 +160,10 @@ def install(self, spec, prefix): with working_dir(build_directory, create=True): cmake(source_directory, *options) make() - make("install") + make('install') + # now run some tests: - for f in ["4elt", "copter2", "mdual"]: + for f in ['4elt', 'copter2', 'mdual']: graph = join_path(source_directory,'graphs','%s.graph' % f) Executable(join_path(prefix.bin,'graphchk'))(graph) Executable(join_path(prefix.bin,'gpmetis'))(graph,'2') @@ -182,6 +177,6 @@ def install(self, spec, prefix): # install GKlib headers, which will be needed for ParMETIS GKlib_dist = join_path(prefix.include,'GKlib') mkdirp(GKlib_dist) - fs = glob.glob(join_path(source_directory,'GKlib',"*.h")) + fs = glob.glob(join_path(source_directory,'GKlib','*.h')) for f in fs: install(f, GKlib_dist) diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index 2dead4a76ad..a8456083b47 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -31,10 +31,12 @@ class Parmetis(Package): ParMETIS is an MPI-based parallel library that implements a variety of algorithms for partitioning unstructured graphs, meshes, and for computing fill-reducing orderings of sparse matrices. """ + homepage = 'http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview' - url = 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis/parmetis-4.0.3.tar.gz' + base_url = 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis' version('4.0.3', 'f69c479586bf6bb7aff6a9bc0c739628') + version('4.0.2', '0912a953da5bb9b5e5e10542298ffdce') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds the library in debug mode') @@ -42,17 +44,18 @@ class Parmetis(Package): depends_on('cmake @2.8:') # build dependency depends_on('mpi') - - patch('enable_external_metis.patch') depends_on('metis@5:') + patch('enable_external_metis.patch') # bug fixes from PETSc developers # https://bitbucket.org/petsc/pkg-parmetis/commits/1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b/raw/ patch('pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch') # https://bitbucket.org/petsc/pkg-parmetis/commits/82409d68aa1d6cbc70740d0f35024aae17f7d5cb/raw/ patch('pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch') - depends_on('gdb', when='+gdb') + def url_for_version(self, version): + version_dir = 'OLD/' if version < Version('3.2.0') else '' + return '%s/%sparmetis-%s.tar.gz' % (Parmetis.base_url, version_dir, version) def install(self, spec, prefix): options = [] @@ -71,18 +74,15 @@ def install(self, spec, prefix): if '+shared' in spec: options.append('-DSHARED:BOOL=ON') - if '+debug' in spec: - options.extend(['-DDEBUG:BOOL=ON', - '-DCMAKE_BUILD_TYPE:STRING=Debug']) - + options.extend(['-DDEBUG:BOOL=ON', '-DCMAKE_BUILD_TYPE:STRING=Debug']) if '+gdb' in spec: options.append('-DGDB:BOOL=ON') with working_dir(build_directory, create=True): cmake(source_directory, *options) make() - make("install") + make('install') # The shared library is not installed correctly on Darwin; correct this if (sys.platform == 'darwin') and ('+shared' in spec): From 0ef6843d85a1b74659ed72bda18a72613276d4ec Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Mon, 9 May 2016 15:37:00 -0700 Subject: [PATCH 267/375] Improved MPI path specification for ParMETIS using PR #893 features. --- var/spack/repos/builtin/packages/parmetis/package.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index a8456083b47..956a035f7be 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -65,12 +65,10 @@ def install(self, spec, prefix): source_directory = self.stage.source_path metis_source = join_path(source_directory, 'metis') - # FIXME : Once a contract is defined, MPI compilers should be retrieved indirectly via spec['mpi'] in case - # FIXME : they use a non-standard name options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=spec['metis'].prefix.include), '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=spec['metis'].prefix), - '-DCMAKE_C_COMPILER:STRING=mpicc', - '-DCMAKE_CXX_COMPILER:STRING=mpicxx']) + '-DCMAKE_C_COMPILER:STRING={mpicc}'.format(mpicc=spec['mpi'].mpicc), + '-DCMAKE_CXX_COMPILER:STRING={mpicxx}'.format(mpicxx=spec['mpi'].mpicxx)]) if '+shared' in spec: options.append('-DSHARED:BOOL=ON') From 064d3584c0f95f19f159f0d7bc51c5172278638f Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Thu, 19 May 2016 09:33:47 -0700 Subject: [PATCH 268/375] Updating 'metis' and 'parmetis' to conform to PEP8 standards. --- .../repos/builtin/packages/metis/package.py | 93 ++++++++++--------- .../builtin/packages/parmetis/package.py | 34 +++---- 2 files changed, 65 insertions(+), 62 deletions(-) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index daeaf6245b1..94e6fce6db4 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -24,14 +24,17 @@ ############################################################################## from spack import * -import glob, sys, os +import glob +import sys +import os + class Metis(Package): - """ - METIS is a set of serial programs for partitioning graphs, partitioning finite element meshes, and producing fill - reducing orderings for sparse matrices. The algorithms implemented in METIS are based on the multilevel - recursive-bisection, multilevel k-way, and multi-constraint partitioning schemes. - """ + """METIS is a set of serial programs for partitioning graphs, partitioning + finite element meshes, and producing fill reducing orderings for sparse + matrices. The algorithms implemented in METIS are based on the + multilevel recursive-bisection, multilevel k-way, and multi-constraint + partitioning schemes.""" homepage = "http://glaros.dtc.umn.edu/gkhome/metis/metis/overview" base_url = "http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis" @@ -47,22 +50,20 @@ class Metis(Package): variant('idx64', default=False, description='Use int64_t as default index type') variant('real64', default=False, description='Use double precision floating point types') - depends_on('cmake @2.8:', when='@5:') # build-time dependency + depends_on('cmake@2.8:', when='@5:') # build-time dependency patch('install_gklib_defs_rename.patch', when='@5:') def url_for_version(self, version): - version_dir = 'OLD/' if version < Version('4.0.3') else '' - return '%s/%smetis-%s.tar.gz' % (Metis.base_url, version_dir, version) + verdir = 'OLD/' if version < Version('4.0.3') else '' + return '%s/%smetis-%s.tar.gz' % (Metis.base_url, verdir, version) - @when('@4:4.0.3') + @when('@4') def install(self, spec, prefix): - if '+gdb' in spec: - raise InstallError('gdb support not implemented in METIS 4!') - if '+idx64' in spec: - raise InstallError('idx64 option not implemented in METIS 4!') - if '+double' in spec: - raise InstallError('double option not implemented for METIS 4!') + unsupp_vars = [v for v in ('+gdb', '+idx64', '+real64') if v in spec] + if unsupp_vars: + msg = 'Given variants %s are unsupported by METIS 4!' % unsupp_vars + raise InstallError(msg) options = ['COPTIONS=-fPIC'] if '+debug' in spec: @@ -70,9 +71,10 @@ def install(self, spec, prefix): make(*options) mkdir(prefix.bin) - for x in ('pmetis', 'kmetis', 'oemetis', 'onmetis', 'partnmesh', - 'partdmesh', 'mesh2nodal', 'mesh2dual', 'graphchk'): - install(x, prefix.bin) + binfiles = ('pmetis', 'kmetis', 'oemetis', 'onmetis', 'partnmesh', + 'partdmesh', 'mesh2nodal', 'mesh2dual', 'graphchk') + for binfile in binfiles: + install(binfile, prefix.bin) mkdir(prefix.lib) install('libmetis.a', prefix.lib) @@ -82,12 +84,10 @@ def install(self, spec, prefix): install(h, prefix.include) mkdir(prefix.share) - for f in (join_path(*p) - for p in (('Programs', 'io.c'), - ('Test','mtest.c'), - ('Graphs','4elt.graph'), - ('Graphs', 'metis.mesh'), - ('Graphs', 'test.mgraph'))): + sharefiles = (('Programs', 'io.c'), ('Test', 'mtest.c'), + ('Graphs', '4elt.graph'), ('Graphs', 'metis.mesh'), + ('Graphs', 'test.mgraph')) + for sharefile in tuple(join_path(*sf) for sf in sharefiles): install(f, prefix.share) if '+shared' in spec: @@ -100,10 +100,10 @@ def install(self, spec, prefix): load_flag = '-Wl,-whole-archive' no_load_flag = '-Wl,-no-whole-archive' - os.system(spack_cc + ' -fPIC -shared ' + load_flag + - ' libmetis.a ' + no_load_flag + ' -o libmetis.' + - lib_dsuffix) - install('libmetis.' + lib_dsuffix, prefix.lib) + flags = (self.compiler.cc, load_flag, no_load_flag, lib_dsuffix) + build_cmd = '%s -fPIC -shared %s libmetis.a %s libmetis.%s' % flags + os.system(build_cmd) + install('libmetis.%s' % lib_dsuffix, prefix.lib) # Set up and run tests on installation symlink(join_path(prefix.share, 'io.c'), 'io.c') @@ -125,7 +125,6 @@ def install(self, spec, prefix): os.system(join_path(prefix.bin, 'partdmesh') + metis_mesh + ' 10') os.system(join_path(prefix.bin, 'mesh2dual') + metis_mesh) - @when('@5:') def install(self, spec, prefix): options = [] @@ -134,13 +133,14 @@ def install(self, spec, prefix): build_directory = join_path(self.stage.path, 'spack-build') source_directory = self.stage.source_path - options.append('-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=source_directory)) + options.append('-DGKLIB_PATH:PATH=%s/GKlib' % source_directory) options.append('-DCMAKE_INSTALL_NAME_DIR:PATH=%s/lib' % prefix) if '+shared' in spec: options.append('-DSHARED:BOOL=ON') if '+debug' in spec: - options.extend(['-DDEBUG:BOOL=ON', '-DCMAKE_BUILD_TYPE:STRING=Debug']) + options.extend(['-DDEBUG:BOOL=ON', + '-DCMAKE_BUILD_TYPE:STRING=Debug']) if '+gdb' in spec: options.append('-DGDB:BOOL=ON') @@ -153,9 +153,10 @@ def install(self, spec, prefix): # Make clang 7.3 happy. # Prevents "ld: section __DATA/__thread_bss extends beyond end of file" # See upstream LLVM issue https://llvm.org/bugs/show_bug.cgi?id=27059 - # Adopted from https://github.com/Homebrew/homebrew-science/blob/master/metis.rb + # and https://github.com/Homebrew/homebrew-science/blob/master/metis.rb if spec.satisfies('%clang@7.3.0'): - filter_file('#define MAX_JBUFS 128', '#define MAX_JBUFS 24', join_path(source_directory, 'GKlib', 'error.c')) + filter_file('#define MAX_JBUFS 128', '#define MAX_JBUFS 24', + join_path(source_directory, 'GKlib', 'error.c')) with working_dir(build_directory, create=True): cmake(source_directory, *options) @@ -164,19 +165,19 @@ def install(self, spec, prefix): # now run some tests: for f in ['4elt', 'copter2', 'mdual']: - graph = join_path(source_directory,'graphs','%s.graph' % f) - Executable(join_path(prefix.bin,'graphchk'))(graph) - Executable(join_path(prefix.bin,'gpmetis'))(graph,'2') - Executable(join_path(prefix.bin,'ndmetis'))(graph) + graph = join_path(source_directory, 'graphs', '%s.graph' % f) + Executable(join_path(prefix.bin, 'graphchk'))(graph) + Executable(join_path(prefix.bin, 'gpmetis'))(graph, '2') + Executable(join_path(prefix.bin, 'ndmetis'))(graph) - graph = join_path(source_directory,'graphs','test.mgraph') - Executable(join_path(prefix.bin,'gpmetis'))(graph,'2') - graph = join_path(source_directory,'graphs','metis.mesh') - Executable(join_path(prefix.bin,'mpmetis'))(graph,'2') + graph = join_path(source_directory, 'graphs', 'test.mgraph') + Executable(join_path(prefix.bin, 'gpmetis'))(graph, '2') + graph = join_path(source_directory, 'graphs', 'metis.mesh') + Executable(join_path(prefix.bin, 'mpmetis'))(graph, '2') # install GKlib headers, which will be needed for ParMETIS - GKlib_dist = join_path(prefix.include,'GKlib') + GKlib_dist = join_path(prefix.include, 'GKlib') mkdirp(GKlib_dist) - fs = glob.glob(join_path(source_directory,'GKlib','*.h')) - for f in fs: - install(f, GKlib_dist) + hfiles = glob.glob(join_path(source_directory, 'GKlib', '*.h')) + for hfile in hfiles: + install(hfile, GKlib_dist) diff --git a/var/spack/repos/builtin/packages/parmetis/package.py b/var/spack/repos/builtin/packages/parmetis/package.py index 956a035f7be..9b36f273e4c 100644 --- a/var/spack/repos/builtin/packages/parmetis/package.py +++ b/var/spack/repos/builtin/packages/parmetis/package.py @@ -26,11 +26,11 @@ from spack import * import sys + class Parmetis(Package): - """ - ParMETIS is an MPI-based parallel library that implements a variety of algorithms for partitioning unstructured - graphs, meshes, and for computing fill-reducing orderings of sparse matrices. - """ + """ParMETIS is an MPI-based parallel library that implements a variety of + algorithms for partitioning unstructured graphs, meshes, and for + computing fill-reducing orderings of sparse matrices.""" homepage = 'http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview' base_url = 'http://glaros.dtc.umn.edu/gkhome/fetch/sw/parmetis' @@ -42,20 +42,20 @@ class Parmetis(Package): variant('debug', default=False, description='Builds the library in debug mode') variant('gdb', default=False, description='Enables gdb support') - depends_on('cmake @2.8:') # build dependency + depends_on('cmake@2.8:') # build dependency depends_on('mpi') depends_on('metis@5:') patch('enable_external_metis.patch') # bug fixes from PETSc developers - # https://bitbucket.org/petsc/pkg-parmetis/commits/1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b/raw/ + # https://bitbucket.org/petsc/pkg-parmetis/commits/1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b/raw/ # NOQA: ignore=E501 patch('pkg-parmetis-1c1a9fd0f408dc4d42c57f5c3ee6ace411eb222b.patch') - # https://bitbucket.org/petsc/pkg-parmetis/commits/82409d68aa1d6cbc70740d0f35024aae17f7d5cb/raw/ + # https://bitbucket.org/petsc/pkg-parmetis/commits/82409d68aa1d6cbc70740d0f35024aae17f7d5cb/raw/ # NOQA: ignore=E501 patch('pkg-parmetis-82409d68aa1d6cbc70740d0f35024aae17f7d5cb.patch') def url_for_version(self, version): - version_dir = 'OLD/' if version < Version('3.2.0') else '' - return '%s/%sparmetis-%s.tar.gz' % (Parmetis.base_url, version_dir, version) + verdir = 'OLD/' if version < Version('3.2.0') else '' + return '%s/%sparmetis-%s.tar.gz' % (Parmetis.base_url, verdir, version) def install(self, spec, prefix): options = [] @@ -63,17 +63,19 @@ def install(self, spec, prefix): build_directory = join_path(self.stage.path, 'spack-build') source_directory = self.stage.source_path - metis_source = join_path(source_directory, 'metis') - options.extend(['-DGKLIB_PATH:PATH={metis_source}/GKlib'.format(metis_source=spec['metis'].prefix.include), - '-DMETIS_PATH:PATH={metis_source}'.format(metis_source=spec['metis'].prefix), - '-DCMAKE_C_COMPILER:STRING={mpicc}'.format(mpicc=spec['mpi'].mpicc), - '-DCMAKE_CXX_COMPILER:STRING={mpicxx}'.format(mpicxx=spec['mpi'].mpicxx)]) + options.extend([ + '-DGKLIB_PATH:PATH=%s/GKlib' % spec['metis'].prefix.include, + '-DMETIS_PATH:PATH=%s' % spec['metis'].prefix, + '-DCMAKE_C_COMPILER:STRING=%s' % spec['mpi'].mpicc, + '-DCMAKE_CXX_COMPILER:STRING=%s' % spec['mpi'].mpicxx + ]) if '+shared' in spec: options.append('-DSHARED:BOOL=ON') if '+debug' in spec: - options.extend(['-DDEBUG:BOOL=ON', '-DCMAKE_BUILD_TYPE:STRING=Debug']) + options.extend(['-DDEBUG:BOOL=ON', + '-DCMAKE_BUILD_TYPE:STRING=Debug']) if '+gdb' in spec: options.append('-DGDB:BOOL=ON') @@ -82,6 +84,6 @@ def install(self, spec, prefix): make() make('install') - # The shared library is not installed correctly on Darwin; correct this + # The shared library is not installed correctly on Darwin; fix this if (sys.platform == 'darwin') and ('+shared' in spec): fix_darwin_install_name(prefix.lib) From 1bca69f27281d3bb707bcba41b4f332ab3988509 Mon Sep 17 00:00:00 2001 From: Greg Lee Date: Tue, 24 May 2016 14:54:09 -0700 Subject: [PATCH 269/375] updated launchmon url and version --- .../builtin/packages/launchmon/package.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/var/spack/repos/builtin/packages/launchmon/package.py b/var/spack/repos/builtin/packages/launchmon/package.py index f38bc38202b..f2568100059 100644 --- a/var/spack/repos/builtin/packages/launchmon/package.py +++ b/var/spack/repos/builtin/packages/launchmon/package.py @@ -24,29 +24,19 @@ ############################################################################## from spack import * + class Launchmon(Package): """Software infrastructure that enables HPC run-time tools to co-locate tool daemons with a parallel job.""" - homepage = "http://sourceforge.net/projects/launchmon" - url = "http://downloads.sourceforge.net/project/launchmon/launchmon/1.0.1%20release/launchmon-1.0.1.tar.gz" + homepage = "https://github.com/LLNL/LaunchMON" + url = "https://github.com/LLNL/LaunchMON/releases/download/v1.0.2/launchmon-v1.0.2.tar.gz" # NOQA: ignore=E501 - version('1.0.1', '2f12465803409fd07f91174a4389eb2b') - version('1.0.1-2', git='https://github.com/llnl/launchmon.git', commit='ff7e22424b8f375318951eb1c9282fcbbfa8aadf') + version('1.0.2', '8d6ba77a0ec2eff2fde2c5cc8fa7ff7a') depends_on('autoconf') depends_on('automake') depends_on('libtool') - - def patch(self): - # This patch makes libgcrypt compile correctly with newer gcc versions. - mf = FileFilter('tools/libgcrypt/tests/Makefile.in') - mf.filter(r'(basic_LDADD\s*=\s*.*)', r'\1 -lgpg-error') - mf.filter(r'(tsexp_LDADD\s*=\s*.*)', r'\1 -lgpg-error') - mf.filter(r'(keygen_LDADD\s*=\s*.*)', r'\1 -lgpg-error') - mf.filter(r'(benchmark_LDADD\s*=\s*.*)', r'\1 -lgpg-error') - - def install(self, spec, prefix): configure( "--prefix=" + prefix, From b5d0a388604f3c33bbbfdc1baff98f54ccdede5b Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 24 May 2016 15:03:21 -0700 Subject: [PATCH 270/375] Fixed a few linking/testing bugs in the METIS@:4 install method. --- .../repos/builtin/packages/metis/package.py | 52 +++++++++++-------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index 94e6fce6db4..eb0bdf28de1 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -58,8 +58,9 @@ def url_for_version(self, version): verdir = 'OLD/' if version < Version('4.0.3') else '' return '%s/%smetis-%s.tar.gz' % (Metis.base_url, verdir, version) - @when('@4') + @when('@:4') def install(self, spec, prefix): + # Process library spec and options unsupp_vars = [v for v in ('+gdb', '+idx64', '+real64') if v in spec] if unsupp_vars: msg = 'Given variants %s are unsupported by METIS 4!' % unsupp_vars @@ -70,6 +71,7 @@ def install(self, spec, prefix): options.append('OPTFLAGS=-g -O0') make(*options) + # Compile and install library files mkdir(prefix.bin) binfiles = ('pmetis', 'kmetis', 'oemetis', 'onmetis', 'partnmesh', 'partdmesh', 'mesh2nodal', 'mesh2dual', 'graphchk') @@ -88,7 +90,7 @@ def install(self, spec, prefix): ('Graphs', '4elt.graph'), ('Graphs', 'metis.mesh'), ('Graphs', 'test.mgraph')) for sharefile in tuple(join_path(*sf) for sf in sharefiles): - install(f, prefix.share) + install(sharefile, prefix.share) if '+shared' in spec: if sys.platform == 'darwin': @@ -100,30 +102,38 @@ def install(self, spec, prefix): load_flag = '-Wl,-whole-archive' no_load_flag = '-Wl,-no-whole-archive' - flags = (self.compiler.cc, load_flag, no_load_flag, lib_dsuffix) - build_cmd = '%s -fPIC -shared %s libmetis.a %s libmetis.%s' % flags - os.system(build_cmd) + os.system('%s -fPIC -shared %s libmetis.a %s -o libmetis.%s' % \ + (self.compiler.cc, load_flag, no_load_flag, lib_dsuffix)) install('libmetis.%s' % lib_dsuffix, prefix.lib) # Set up and run tests on installation + inc_flags = '-I%s' % prefix.include + lib_flags = '-L%s' % prefix.lib + if '+shared' in spec: + lib_flags += ' -Wl,-rpath=%s' % prefix.lib + symlink(join_path(prefix.share, 'io.c'), 'io.c') symlink(join_path(prefix.share, 'mtest.c'), 'mtest.c') - os.system(spack_cc + ' -I%s' % prefix.include + ' -c io.c') - os.system(spack_cc + ' -I%s' % prefix.include + - ' -L%s' % prefix.lib + ' -lmetis mtest.c io.o -o mtest') - _4eltgraph = join_path(prefix.share, '4elt.graph') - test_mgraph = join_path(prefix.share, 'test.mgraph') - metis_mesh = join_path(prefix.share, 'metis.mesh') - kmetis = join_path(prefix.bin, 'kmetis') - os.system('./mtest ' + _4eltgraph) - os.system(kmetis + ' ' + _4eltgraph + ' 40') - os.system(join_path(prefix.bin, 'onmetis') + ' ' + _4eltgraph) - os.system(join_path(prefix.bin, 'pmetis') + ' ' + test_mgraph + ' 2') - os.system(kmetis + ' ' + test_mgraph + ' 2') - os.system(kmetis + ' ' + test_mgraph + ' 5') - os.system(join_path(prefix.bin, 'partnmesh') + metis_mesh + ' 10') - os.system(join_path(prefix.bin, 'partdmesh') + metis_mesh + ' 10') - os.system(join_path(prefix.bin, 'mesh2dual') + metis_mesh) + + os.system('%s %s -c io.c' % (self.compiler.cc, inc_flags)) + os.system('%s %s %s mtest.c io.o -o mtest -lmetis -lm' % \ + (self.compiler.cc, inc_flags, lib_flags)) + + test_bin = lambda testname: join_path(prefix.bin, testname) + test_graph = lambda graphname: join_path(prefix.share, graphname) + + graph = test_graph('4elt.graph') + os.system('./mtest %s' % graph) + os.system('%s %s 40' % (test_bin('kmetis'), graph)) + os.system('%s %s' % (test_bin('onmetis'), graph)) + graph = test_graph('test.mgraph') + os.system('%s %s 2' % (test_bin('pmetis'), graph)) + os.system('%s %s 2' % (test_bin('kmetis'), graph)) + os.system('%s %s 5' % (test_bin('kmetis'), graph)) + graph = test_graph('metis.mesh') + os.system('%s %s 10' % (test_bin('partnmesh'), graph)) + os.system('%s %s 10' % (test_bin('partdmesh'), graph)) + os.system('%s %s' % (test_bin('mesh2dual'), graph)) @when('@5:') def install(self, spec, prefix): From c9ba73d931439afd0670bcf6498154724d77cd45 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 24 May 2016 16:05:29 -0700 Subject: [PATCH 271/375] Added staging support and improved error handling for METIS@:4 installs. --- .../repos/builtin/packages/metis/package.py | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index eb0bdf28de1..65595229c02 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -72,6 +72,8 @@ def install(self, spec, prefix): make(*options) # Compile and install library files + ccompile = Executable(self.compiler.cc) + mkdir(prefix.bin) binfiles = ('pmetis', 'kmetis', 'oemetis', 'onmetis', 'partnmesh', 'partdmesh', 'mesh2nodal', 'mesh2dual', 'graphchk') @@ -86,8 +88,7 @@ def install(self, spec, prefix): install(h, prefix.include) mkdir(prefix.share) - sharefiles = (('Programs', 'io.c'), ('Test', 'mtest.c'), - ('Graphs', '4elt.graph'), ('Graphs', 'metis.mesh'), + sharefiles = (('Graphs', '4elt.graph'), ('Graphs', 'metis.mesh'), ('Graphs', 'test.mgraph')) for sharefile in tuple(join_path(*sf) for sf in sharefiles): install(sharefile, prefix.share) @@ -102,28 +103,20 @@ def install(self, spec, prefix): load_flag = '-Wl,-whole-archive' no_load_flag = '-Wl,-no-whole-archive' - os.system('%s -fPIC -shared %s libmetis.a %s -o libmetis.%s' % \ - (self.compiler.cc, load_flag, no_load_flag, lib_dsuffix)) - install('libmetis.%s' % lib_dsuffix, prefix.lib) + ccompile('-fPIC', '-shared', load_flag, 'libmetis.a', no_load_flag, + '-o', '%s/libmetis.%s' % (prefix.lib, lib_dsuffix)) # Set up and run tests on installation - inc_flags = '-I%s' % prefix.include - lib_flags = '-L%s' % prefix.lib - if '+shared' in spec: - lib_flags += ' -Wl,-rpath=%s' % prefix.lib - - symlink(join_path(prefix.share, 'io.c'), 'io.c') - symlink(join_path(prefix.share, 'mtest.c'), 'mtest.c') - - os.system('%s %s -c io.c' % (self.compiler.cc, inc_flags)) - os.system('%s %s %s mtest.c io.o -o mtest -lmetis -lm' % \ - (self.compiler.cc, inc_flags, lib_flags)) + ccompile('-I%s' % prefix.include, '-L%s' % prefix.lib, + '-Wl,-rpath=%s' % (prefix.lib if '+shared' in spec else ''), + join_path('Programs', 'io.o'), join_path('Test', 'mtest.c'), + '-o', '%s/mtest' % prefix.bin, '-lmetis', '-lm') test_bin = lambda testname: join_path(prefix.bin, testname) test_graph = lambda graphname: join_path(prefix.share, graphname) graph = test_graph('4elt.graph') - os.system('./mtest %s' % graph) + os.system('%s %s' % (test_bin('mtest'), graph)) os.system('%s %s 40' % (test_bin('kmetis'), graph)) os.system('%s %s' % (test_bin('onmetis'), graph)) graph = test_graph('test.mgraph') From 483ec0d76275521579b2d1c93620a37841159e36 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Tue, 24 May 2016 16:13:52 -0700 Subject: [PATCH 272/375] Added preliminary improved testing code for METIS@:4. --- .../repos/builtin/packages/metis/package.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index 65595229c02..5ccc04bd5ef 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -128,6 +128,25 @@ def install(self, spec, prefix): os.system('%s %s 10' % (test_bin('partdmesh'), graph)) os.system('%s %s' % (test_bin('mesh2dual'), graph)) + # TODO: The following code should replace the testing code in the + # block above since it causes installs to fail when one or more of the + # Metis tests fail, but it currently doesn't work because the 'mtest', + # 'onmetis', and 'partnmesh' tests return error codes that trigger + # false positives for failure. + """ + Executable(test_bin('mtest'))(test_graph('4elt.graph')) + Executable(test_bin('kmetis'))(test_graph('4elt.graph'), '40') + Executable(test_bin('onmetis'))(test_graph('4elt.graph')) + + Executable(test_bin('pmetis'))(test_graph('test.mgraph'), '2') + Executable(test_bin('kmetis'))(test_graph('test.mgraph'), '2') + Executable(test_bin('kmetis'))(test_graph('test.mgraph'), '5') + + Executable(test_bin('partnmesh'))(test_graph('metis.mesh'), '10') + Executable(test_bin('partdmesh'))(test_graph('metis.mesh'), '10') + Executable(test_bin('mesh2dual'))(test_graph('metis.mesh')) + """ + @when('@5:') def install(self, spec, prefix): options = [] From 74dc7ffe4d9824f916c37b151e9ebe63a184c705 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 10:56:12 -0700 Subject: [PATCH 273/375] updated documentation for cflags PR --- lib/spack/docs/basic_usage.rst | 155 +++++++++++++++++++++-------- lib/spack/docs/configuration.rst | 12 +-- lib/spack/docs/features.rst | 17 +++- lib/spack/docs/packaging_guide.rst | 48 +++++---- 4 files changed, 158 insertions(+), 74 deletions(-) diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 6efed836219..2eed9dddd4e 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -102,8 +102,8 @@ that the packages is installed: ==> adept-utils is already installed in /home/gamblin2/spack/opt/chaos_5_x86_64_ib/gcc@4.4.7/adept-utils@1.0-5adef8da. ==> Trying to fetch from https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz ######################################################################## 100.0% - ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7=chaos_5_x86_64_ib-59f6ad23/mpileaks-1.0.tar.gz - ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7=chaos_5_x86_64_ib-59f6ad23. + ==> Staging archive: /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=chaos_5_x86_64_ib-59f6ad23/mpileaks-1.0.tar.gz + ==> Created stage in /home/gamblin2/spack/var/spack/stage/mpileaks@1.0%gcc@4.4.7 arch=chaos_5_x86_64_ib-59f6ad23. ==> No patches needed for mpileaks. ==> Building mpileaks. @@ -132,10 +132,10 @@ sites, as installing a version that one user needs will not disrupt existing installations for other users. In addition to different versions, Spack can customize the compiler, -compile-time options (variants), and platform (for cross compiles) of -an installation. Spack is unique in that it can also configure the -*dependencies* a package is built with. For example, two -configurations of the same version of a package, one built with boost +compile-time options (variants), compiler flags, and platform (for +cross compiles) of an installation. Spack is unique in that it can +also configure the *dependencies* a package is built with. For example, +two configurations of the same version of a package, one built with boost 1.39.0, and the other version built with version 1.43.0, can coexist. This can all be done on the command line using the *spec* syntax. @@ -334,6 +334,11 @@ of libelf would look like this: -- chaos_5_x86_64_ib / gcc@4.4.7 -------------------------------- libdwarf@20130729-d9b90962 +We can also search for packages that have a certain attribute. For example, +``spack find -l libdwarf +debug`` will show only installations of libdwarf +with the 'debug' compile-time option enabled, while ``spack find -l +debug`` +will find every installed package with a 'debug' compile-time option enabled. + The full spec syntax is discussed in detail in :ref:`sec-specs`. @@ -463,6 +468,26 @@ For compilers, like ``clang``, that do not support Fortran, put Once you save the file, the configured compilers will show up in the list displayed by ``spack compilers``. +You can also add compiler flags to manually configured compilers. The +valid flags are ``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, +``ldflags``, and ``ldlibs``. For example,:: + + ... + chaos_5_x86_64_ib: + ... + intel@15.0.0: + cc: /usr/local/bin/icc-15.0.024-beta + cxx: /usr/local/bin/icpc-15.0.024-beta + f77: /usr/local/bin/ifort-15.0.024-beta + fc: /usr/local/bin/ifort-15.0.024-beta + cppflags: -O3 -fPIC + ... + +These flags will be treated by spack as if they were enterred from +the command line each time this compiler is used. The compiler wrappers +then inject those flags into the compiler command. Compiler flags +enterred from the command line will be discussed in more detail in the +following section. .. _sec-specs: @@ -480,7 +505,7 @@ the full syntax of specs. Here is an example of a much longer spec than we've seen thus far:: - mpileaks @1.2:1.4 %gcc@4.7.5 +debug -qt =bgqos_0 ^callpath @1.1 %gcc@4.7.2 + mpileaks @1.2:1.4 %gcc@4.7.5 +debug -qt arch=bgq_os ^callpath @1.1 %gcc@4.7.2 If provided to ``spack install``, this will install the ``mpileaks`` library at some version between ``1.2`` and ``1.4`` (inclusive), @@ -498,8 +523,12 @@ More formally, a spec consists of the following pieces: * ``%`` Optional compiler specifier, with an optional compiler version (``gcc`` or ``gcc@4.7.3``) * ``+`` or ``-`` or ``~`` Optional variant specifiers (``+debug``, - ``-qt``, or ``~qt``) -* ``=`` Optional architecture specifier (``bgqos_0``) + ``-qt``, or ``~qt``) for boolean variants +* ``name=`` Optional variant specifiers that are not restricted to +boolean variants +* ``name=`` Optional compiler flag specifiers. Valid flag names are +``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``, and ``ldlibs``. +* ``arch=`` Optional architecture specifier (``arch=bgq_os``) * ``^`` Dependency specs (``^callpath@1.1``) There are two things to notice here. The first is that specs are @@ -579,7 +608,7 @@ compilers, variants, and architectures just like any other spec. Specifiers are associated with the nearest package name to their left. For example, above, ``@1.1`` and ``%gcc@4.7.2`` associates with the ``callpath`` package, while ``@1.2:1.4``, ``%gcc@4.7.5``, ``+debug``, -``-qt``, and ``=bgqos_0`` all associate with the ``mpileaks`` package. +``-qt``, and ``arch=bgq_os`` all associate with the ``mpileaks`` package. In the diagram above, ``mpileaks`` depends on ``mpich`` with an unspecified version, but packages can depend on other packages with @@ -635,22 +664,25 @@ based on site policies. Variants ~~~~~~~~~~~~~~~~~~~~~~~ -.. Note:: - - Variants are not yet supported, but will be in the next Spack - release (0.9), due in Q2 2015. - -Variants are named options associated with a particular package, and -they can be turned on or off. For example, above, supplying -``+debug`` causes ``mpileaks`` to be built with debug flags. The -names of particular variants available for a package depend on what -was provided by the package author. ``spack info `` will +Variants are named options associated with a particular package. They are +optional, as each package must provide default values for each variant it +makes available. Variants can be specified using +a flexible parameter syntax ``name=``. For example, +``spack install libelf debug=True`` will install libelf build with debug +flags. The names of particular variants available for a package depend on +what was provided by the package author. ``spack into `` will provide information on what build variants are available. -Depending on the package a variant may be on or off by default. For -``mpileaks`` here, ``debug`` is off by default, and we turned it on -with ``+debug``. If a package is on by default you can turn it off by -either adding ``-name`` or ``~name`` to the spec. +For compatibility with earlier versions, variants which happen to be +boolean in nature can be specified by a syntax that represents turning +options on and off. For example, in the previous spec we could have +supplied ``libelf +debug`` with the same effect of enabling the debug +compile time option for the libelf package. + +Depending on the package a variant may have any default value. For +``libelf`` here, ``debug`` is ``False`` by default, and we turned it on +with ``debug=True`` or ``+debug``. If a package is ``True`` by default +you can turn it off by either adding ``-name`` or ``~name`` to the spec. There are two syntaxes here because, depending on context, ``~`` and ``-`` may mean different things. In most shells, the following will @@ -662,7 +694,7 @@ result in the shell performing home directory substitution: mpileaks~debug # use this instead If there is a user called ``debug``, the ``~`` will be incorrectly -expanded. In this situation, you would want to write ``mpileaks +expanded. In this situation, you would want to write ``libelf -debug``. However, ``-`` can be ambiguous when included after a package name without spaces: @@ -677,12 +709,35 @@ package, not a request for ``mpileaks`` built without ``debug`` options. In this scenario, you should write ``mpileaks~debug`` to avoid ambiguity. -When spack normalizes specs, it prints them out with no spaces and -uses only ``~`` for disabled variants. We allow ``-`` and spaces on -the command line is provided for convenience and legibility. +When spack normalizes specs, it prints them out with no spaces boolean +variants using the backwards compatibility syntax and uses only ``~`` +for disabled boolean variants. We allow ``-`` and spaces on the command +line is provided for convenience and legibility. -Architecture specifier +Compiler Flags +~~~~~~~~~~~~~~~~~~~~~~~ + +Compiler flags are specified using the same syntax as non-boolean variants, +but fulfill a different purpose. While the function of a variant is set by +the package, compiler flags are used by the compiler wrappers to inject +flags into the compile line of the build. Additionally, compiler flags are +inherited by dependencies. ``spack install libdwarf cppflags=\"-g\"`` will +install both libdwarf and libelf with the ``-g`` flag injected into their +compile line. + +Notice that the value of the compiler flags must be escape quoted on the +command line. From within python files, the same spec would be specified +``libdwarf cppflags="-g"``. This is necessary because of how the shell +handles the quote symbols. + +The six compiler flags are injected in the order of implicit make commands +in gnu autotools. If all flags are set, the order is +``$cppflags $cflags|$cxxflags $ldflags command $ldlibs`` for C and C++ and +``$fflags $cppflags $ldflags command $ldlibs`` for fortran. + + +Architecture specifiers ~~~~~~~~~~~~~~~~~~~~~~~ .. Note:: @@ -690,12 +745,9 @@ Architecture specifier Architecture specifiers are part of specs but are not yet functional. They will be in Spack version 1.0, due in Q3 2015. -The architecture specifier starts with a ``=`` and also comes after -some package name within a spec. It allows a user to specify a -particular architecture for the package to be built. This is mostly -used for architectures that need cross-compilation, and in most cases, -users will not need to specify the architecture when they install a -package. +The architecture specifier looks identical to a variant specifier for a +non-boolean variant. The architecture can be specified only using the +reserved name ``arch`` (``arch=bgq_os``). .. _sec-virtual-dependencies: @@ -773,6 +825,23 @@ any MPI implementation will do. If another package depends on error. Likewise, if you try to plug in some package that doesn't provide MPI, Spack will raise an error. +Specifying Specs by Hash +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Complicated specs can become cumbersome to enter on the command line, +especially when many of the qualifications are necessary to +distinguish between similar installs, for example when using the +``uninstall`` command. To avoid this, when referencing an existing spec, +Spack allows you to reference specs by their hash. We previously +discussed the spec hash that Spack computes. In place of a spec in any +command, substitute ``/`` where ```` is any amount from +the beginning of a spec hash. If the given spec hash is sufficient +to be unique, Spack will replace the reference with the spec to which +it refers. Otherwise, it will prompt for a more qualified hash. + +Note that this will not work to reinstall a depencency uninstalled by +``spack uninstall -f``. + .. _spack-providers: ``spack providers`` @@ -1002,8 +1071,8 @@ than one installed package matches it), then Spack will warn you: $ spack load libelf ==> Error: Multiple matches for spec libelf. Choose one: - libelf@0.8.13%gcc@4.4.7=chaos_5_x86_64_ib - libelf@0.8.13%intel@15.0.0=chaos_5_x86_64_ib + libelf@0.8.13%gcc@4.4.7 arch=chaos_5_x86_64_ib + libelf@0.8.13%intel@15.0.0 arch=chaos_5_x86_64_ib You can either type the ``spack load`` command again with a fully qualified argument, or you can add just enough extra constraints to @@ -1282,7 +1351,7 @@ You can find extensions for your Python installation like this: .. code-block:: sh $ spack extensions python - ==> python@2.7.8%gcc@4.4.7=chaos_5_x86_64_ib-703c7a96 + ==> python@2.7.8%gcc@4.4.7 arch=chaos_5_x86_64_ib-703c7a96 ==> 36 extensions: geos py-ipython py-pexpect py-pyside py-sip py-basemap py-libxml2 py-pil py-pytz py-six @@ -1372,9 +1441,9 @@ installation: .. code-block:: sh $ spack activate py-numpy - ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7=chaos_5_x86_64_ib-3c74eb69 for python@2.7.8%gcc@4.4.7. - ==> Activated extension py-nose@1.3.4%gcc@4.4.7=chaos_5_x86_64_ib-5f70f816 for python@2.7.8%gcc@4.4.7. - ==> Activated extension py-numpy@1.9.1%gcc@4.4.7=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-setuptools@11.3.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-3c74eb69 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-nose@1.3.4%gcc@4.4.7 arch=chaos_5_x86_64_ib-5f70f816 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. Several things have happened here. The user requested that ``py-numpy`` be activated in the ``python`` installation it was built @@ -1389,7 +1458,7 @@ packages listed as activated: .. code-block:: sh $ spack extensions python - ==> python@2.7.8%gcc@4.4.7=chaos_5_x86_64_ib-703c7a96 + ==> python@2.7.8%gcc@4.4.7 arch=chaos_5_x86_64_ib-703c7a96 ==> 36 extensions: geos py-ipython py-pexpect py-pyside py-sip py-basemap py-libxml2 py-pil py-pytz py-six @@ -1437,7 +1506,7 @@ dependencies, you can use ``spack activate -f``: .. code-block:: sh $ spack activate -f py-numpy - ==> Activated extension py-numpy@1.9.1%gcc@4.4.7=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. + ==> Activated extension py-numpy@1.9.1%gcc@4.4.7 arch=chaos_5_x86_64_ib-66733244 for python@2.7.8%gcc@4.4.7. .. _spack-deactivate: diff --git a/lib/spack/docs/configuration.rst b/lib/spack/docs/configuration.rst index c0b79a7f444..c613071c65b 100644 --- a/lib/spack/docs/configuration.rst +++ b/lib/spack/docs/configuration.rst @@ -70,9 +70,9 @@ directory. Here's an example of an external configuration: packages: openmpi: paths: - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1 arch=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel This example lists three installations of OpenMPI, one built with gcc, one built with gcc and debug information, and another built with Intel. @@ -108,9 +108,9 @@ be: packages: openmpi: paths: - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 - openmpi@1.4.3%gcc@4.4.7=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug - openmpi@1.6.5%intel@10.1=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel + openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib: /opt/openmpi-1.4.3 + openmpi@1.4.3%gcc@4.4.7 arch=chaos_5_x86_64_ib+debug: /opt/openmpi-1.4.3-debug + openmpi@1.6.5%intel@10.1 arch=chaos_5_x86_64_ib: /opt/openmpi-1.6.5-intel buildable: False The addition of the ``buildable`` flag tells Spack that it should never build diff --git a/lib/spack/docs/features.rst b/lib/spack/docs/features.rst index 0998ba8da4d..27a3b4b4354 100644 --- a/lib/spack/docs/features.rst +++ b/lib/spack/docs/features.rst @@ -31,14 +31,21 @@ platform, all on the command line. # Specify a compiler (and its version), with % $ spack install mpileaks@1.1.2 %gcc@4.7.3 - # Add special compile-time options with + + # Add special compile-time options by name + $ spack install mpileaks@1.1.2 %gcc@4.7.3 debug=True + + # Add special boolean compile-time options with + $ spack install mpileaks@1.1.2 %gcc@4.7.3 +debug - # Cross-compile for a different architecture with = - $ spack install mpileaks@1.1.2 =bgqos_0 + # Add compiler flags using the conventional names + $ spack install mpileaks@1.1.2 %gcc@4.7.3 cppflags=\"-O3 -floop-block\" -Users can specify as many or few options as they care about. Spack -will fill in the unspecified values with sensible defaults. + # Cross-compile for a different architecture with arch= + $ spack install mpileaks@1.1.2 arch=bgqos_0 + +Users can specify as many or few options as they care about. Spack +will fill in the unspecified values with sensible defaults. The two listed +syntaxes for variants are identical when the value is boolean. Customize dependencies diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 63c411ffb58..1f83f611b06 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -1221,11 +1221,13 @@ just as easily provide a version range: depends_on("libelf@0.8.2:0.8.4:") -Or a requirement for a particular variant: +Or a requirement for a particular variant or compiler flags: .. code-block:: python depends_on("libelf@0.8+debug") + depends_on('libelf debug=True') + depends_on('libelf cppflags="-fPIC") Both users *and* package authors can use the same spec syntax to refer to different package configurations. Users use the spec syntax on the @@ -1623,21 +1625,21 @@ the user runs ``spack install`` and the time the ``install()`` method is called. The concretized version of the spec above might look like this:: - mpileaks@2.3%gcc@4.7.3=linux-ppc64 - ^callpath@1.0%gcc@4.7.3+debug=linux-ppc64 - ^dyninst@8.1.2%gcc@4.7.3=linux-ppc64 - ^libdwarf@20130729%gcc@4.7.3=linux-ppc64 - ^libelf@0.8.11%gcc@4.7.3=linux-ppc64 - ^mpich@3.0.4%gcc@4.7.3=linux-ppc64 + mpileaks@2.3%gcc@4.7.3 arch=linux-ppc64 + ^callpath@1.0%gcc@4.7.3+debug arch=linux-ppc64 + ^dyninst@8.1.2%gcc@4.7.3 arch=linux-ppc64 + ^libdwarf@20130729%gcc@4.7.3 arch=linux-ppc64 + ^libelf@0.8.11%gcc@4.7.3 arch=linux-ppc64 + ^mpich@3.0.4%gcc@4.7.3 arch=linux-ppc64 .. graphviz:: digraph { - "mpileaks@2.3\n%gcc@4.7.3\n=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n=linux-ppc64" - "mpileaks@2.3\n%gcc@4.7.3\n=linux-ppc64" -> "callpath@1.0\n%gcc@4.7.3+debug\n=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n=linux-ppc64" - "callpath@1.0\n%gcc@4.7.3+debug\n=linux-ppc64" -> "dyninst@8.1.2\n%gcc@4.7.3\n=linux-ppc64" - "dyninst@8.1.2\n%gcc@4.7.3\n=linux-ppc64" -> "libdwarf@20130729\n%gcc@4.7.3\n=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n=linux-ppc64" - "dyninst@8.1.2\n%gcc@4.7.3\n=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n=linux-ppc64" + "mpileaks@2.3\n%gcc@4.7.3\n arch=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n arch=linux-ppc64" + "mpileaks@2.3\n%gcc@4.7.3\n arch=linux-ppc64" -> "callpath@1.0\n%gcc@4.7.3+debug\n arch=linux-ppc64" -> "mpich@3.0.4\n%gcc@4.7.3\n arch=linux-ppc64" + "callpath@1.0\n%gcc@4.7.3+debug\n arch=linux-ppc64" -> "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" + "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" -> "libdwarf@20130729\n%gcc@4.7.3\n arch=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n arch=linux-ppc64" + "dyninst@8.1.2\n%gcc@4.7.3\n arch=linux-ppc64" -> "libelf@0.8.11\n%gcc@4.7.3\n arch=linux-ppc64" } Here, all versions, compilers, and platforms are filled in, and there @@ -1666,9 +1668,9 @@ running ``spack spec``. For example: ^libdwarf ^libelf - dyninst@8.0.1%gcc@4.7.3=linux-ppc64 - ^libdwarf@20130729%gcc@4.7.3=linux-ppc64 - ^libelf@0.8.13%gcc@4.7.3=linux-ppc64 + dyninst@8.0.1%gcc@4.7.3 arch=linux-ppc64 + ^libdwarf@20130729%gcc@4.7.3 arch=linux-ppc64 + ^libelf@0.8.13%gcc@4.7.3 arch=linux-ppc64 This is useful when you want to know exactly what Spack will do when you ask for a particular spec. @@ -1908,6 +1910,12 @@ the command line. ``$rpath_flag`` can be overriden on a compiler specific basis in ``lib/spack/spack/compilers/$compiler.py``. +The compiler wrappers also pass the compiler flags specified by the user from +the command line (``cflags``, ``cxxflags``, ``fflags``, ``cppflags``, ``ldflags``, +and/or ``ldlibs``). They do not override the canonical autotools flags with the +same names (but in ALL-CAPS) that may be passed into the build by particularly +challenging package scripts. + Compiler flags ~~~~~~~~~~~~~~ In rare circumstances such as compiling and running small unit tests, a package @@ -2154,12 +2162,12 @@ example: def install(self, prefix): # Do default install - @when('=chaos_5_x86_64_ib') + @when('arch=chaos_5_x86_64_ib') def install(self, prefix): # This will be executed instead of the default install if # the package's sys_type() is chaos_5_x86_64_ib. - @when('=bgqos_0") + @when('arch=bgqos_0") def install(self, prefix): # This will be executed if the package's sys_type is bgqos_0 @@ -2749,11 +2757,11 @@ build it: $ spack stage libelf ==> Trying to fetch from http://www.mr511.de/software/libelf-0.8.13.tar.gz ######################################################################## 100.0% - ==> Staging archive: /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3=linux-ppc64/libelf-0.8.13.tar.gz - ==> Created stage in /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3=linux-ppc64. + ==> Staging archive: /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64/libelf-0.8.13.tar.gz + ==> Created stage in /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64. $ spack cd libelf $ pwd - /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3=linux-ppc64/libelf-0.8.13 + /Users/gamblin2/src/spack/var/spack/stage/libelf@0.8.13%gcc@4.8.3 arch=linux-ppc64/libelf-0.8.13 ``spack cd`` here changed he current working directory to the directory containing the expanded ``libelf`` source code. There are a From 3a68dd2011488acdff4e3c8cb4690718e87623e6 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 12:22:16 -0700 Subject: [PATCH 274/375] (hopefully) final fixes of the merge --- lib/spack/spack/architecture.py | 13 +++++++++++-- lib/spack/spack/spec.py | 9 +++++++++ lib/spack/spack/test/modules.py | 19 +++++++++++-------- lib/spack/spack/test/spec_dag.py | 4 ++-- lib/spack/spack/test/spec_semantics.py | 17 +++++------------ 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 05dc969ebee..e9dd489e547 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -233,8 +233,17 @@ def __str__(self): return self.name def _cmp_key(self): - return (self.name, (_cmp_key(t) for t in self.targets.values()), - (_cmp_key(o) for o in self.operating_sys.values())) + t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) + o_keys = ''.join(str(o._cmp_key()) for o in sorted(self.operating_sys.values())) + return (self.name, + self.default, + self.front_end, + self.back_end, + self.default_os, + self.front_os, + self.back_os, + t_keys, + o_keys) @key_ordering class OperatingSystem(object): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index d15598405fc..5d82af4af6a 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1649,9 +1649,18 @@ def satisfies(self, other, deps=True, strict=False): # TODO: Need to make sure that comparisons can be made via classes if self.architecture and other.architecture: + print self.architecture, other.architecture if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): + d1 = self.architecture.platform.to_dict() + d2 = other.architecture.platform.to_dict() + print d1 + print d2 + print d1==d2 + print self.architecture.platform == other.architecture.platform + print self.architecture.platform._cmp_key() + print other.architecture.platform._cmp_key() return False elif strict and ((other.architecture and not self.architecture) or (other.architecture.platform and not self.architecture.platform) or diff --git a/lib/spack/spack/test/modules.py b/lib/spack/spack/test/modules.py index c73badf8f24..582e067860f 100644 --- a/lib/spack/spack/test/modules.py +++ b/lib/spack/spack/test/modules.py @@ -73,7 +73,7 @@ def mock_open(filename, mode): 'all': { 'filter': {'environment_blacklist': ['CMAKE_PREFIX_PATH']} }, - 'arch=x86-linux': { + 'platform=test target=x86_64': { 'environment': {'set': {'FOO': 'foo'}, 'unset': ['BAR']} } @@ -116,6 +116,7 @@ def tearDown(self): def get_modulefile_content(self, spec): spec.concretize() + print spec, '&&&&&' generator = spack.modules.TclModule(spec) generator.write() content = FILE_REGISTRY[generator.file_name].split('\n') @@ -123,27 +124,28 @@ def get_modulefile_content(self, spec): def test_simple_case(self): spack.modules.CONFIGURATION = configuration_autoload_direct - spec = spack.spec.Spec('mpich@3.0.4 arch=x86-linux') + spec = spack.spec.Spec('mpich@3.0.4') content = self.get_modulefile_content(spec) self.assertTrue('module-whatis "mpich @3.0.4"' in content) def test_autoload(self): spack.modules.CONFIGURATION = configuration_autoload_direct - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 2) self.assertEqual(len([x for x in content if 'module load ' in x]), 2) spack.modules.CONFIGURATION = configuration_autoload_all - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 5) self.assertEqual(len([x for x in content if 'module load ' in x]), 5) def test_alter_environment(self): spack.modules.CONFIGURATION = configuration_alter_environment - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks platform=test target=x86_64') content = self.get_modulefile_content(spec) + print content self.assertEqual( len([x for x in content @@ -152,8 +154,9 @@ def test_alter_environment(self): len([x for x in content if 'setenv FOO "foo"' in x]), 1) self.assertEqual(len([x for x in content if 'unsetenv BAR' in x]), 1) - spec = spack.spec.Spec('libdwarf arch=x64-linux') + spec = spack.spec.Spec('libdwarf %clang platform=test target=x86_32') content = self.get_modulefile_content(spec) + print content self.assertEqual( len([x for x in content @@ -164,14 +167,14 @@ def test_alter_environment(self): def test_blacklist(self): spack.modules.CONFIGURATION = configuration_blacklist - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual(len([x for x in content if 'is-loaded' in x]), 1) self.assertEqual(len([x for x in content if 'module load ' in x]), 1) def test_conflicts(self): spack.modules.CONFIGURATION = configuration_conflicts - spec = spack.spec.Spec('mpileaks arch=x86-linux') + spec = spack.spec.Spec('mpileaks') content = self.get_modulefile_content(spec) self.assertEqual( len([x for x in content if x.startswith('conflict')]), 2) diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index 99a986a528a..712f07ac4d3 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -244,8 +244,8 @@ def test_unsatisfiable_compiler_version(self): def test_unsatisfiable_architecture(self): platform = spack.architecture.sys_type() - self.set_pkg_dep('mpileaks', 'mpich target=%s' % platform.target('default_target')) - spec = Spec('mpileaks ^mpich target=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf') + self.set_pkg_dep('mpileaks', 'mpich platform=test target=be') + spec = Spec('mpileaks ^mpich platform=test target=fe ^callpath ^dyninst ^libelf ^libdwarf') self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 45c89100d43..aa99ad71ed6 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -141,10 +141,9 @@ def test_satisfies_compiler_version(self): def test_satisfies_architecture(self): platform = spack.architecture.sys_type() - if platform.name == 'crayxc': - self.check_satisfies('foo target=frontend os=frontend', 'target=frontend os=frontend') - self.check_satisfies('foo target=backend os=backend', 'target=backend', 'os=backend') - self.check_satisfies('foo target=default_target os=default_os','target=default_target os=default_os') + self.check_satisfies('foo platform=test target=frontend os=frontend', 'platform=test target=frontend os=frontend') + self.check_satisfies('foo platform=test target=backend os=backend', 'platform=test target=backend', 'platform=test os=backend') + self.check_satisfies('foo platform=test target=default_target os=default_os','platform=test target=default_target os=default_os') #ifdef NEW @@ -391,14 +390,8 @@ def test_invalid_constraint(self): self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - platform = spack.architecture.sys_type() - if len(platform.operating_sys.keys()) > 1 or len(platform.targets.keys()) > 1: - os1 = platform.operating_sys.keys()[0] - os2 = platform.operating_sys.keys()[-1] - target1 = platform.targets.keys()[0] - target2 = platform.targets.keys()[-1] - self.check_invalid_constraint('libelf target=%s os=%s' % (target1, os1), - 'libelf target=%s os=%s' % (target2, os2)) + self.check_invalid_constraint('libelf platform=test target=be os=be', + 'libelf target=fe os=fe') def test_constrain_changed(self): self.check_constrain_changed('libelf', '@1.0') From 734fb459a45896ee41881ca906643bd362be3e37 Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Wed, 25 May 2016 12:27:52 -0700 Subject: [PATCH 275/375] Fixing a small bug with metis@4+shared installs on the Mac platform. --- .../repos/builtin/packages/metis/package.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/metis/package.py b/var/spack/repos/builtin/packages/metis/package.py index 5ccc04bd5ef..c4f2afaff25 100644 --- a/var/spack/repos/builtin/packages/metis/package.py +++ b/var/spack/repos/builtin/packages/metis/package.py @@ -94,17 +94,19 @@ def install(self, spec, prefix): install(sharefile, prefix.share) if '+shared' in spec: + shared_flags = ['-fPIC', '-shared'] if sys.platform == 'darwin': - lib_dsuffix = 'dylib' - load_flag = '-Wl,-all_load' - no_load_flag = '' + shared_suffix = 'dylib' + shared_flags.extend(['-Wl,-all_load', 'libmetis.a']) else: - lib_dsuffix = 'so' - load_flag = '-Wl,-whole-archive' - no_load_flag = '-Wl,-no-whole-archive' + shared_suffix = 'so' + shared_flags.extend(['-Wl,-whole-archive', 'libmetis.a', + '-Wl,-no-whole-archive']) - ccompile('-fPIC', '-shared', load_flag, 'libmetis.a', no_load_flag, - '-o', '%s/libmetis.%s' % (prefix.lib, lib_dsuffix)) + shared_out = '%s/libmetis.%s' % (prefix.lib, shared_suffix) + shared_flags.extend(['-o', shared_out]) + + ccompile(*shared_flags) # Set up and run tests on installation ccompile('-I%s' % prefix.include, '-L%s' % prefix.lib, @@ -128,7 +130,7 @@ def install(self, spec, prefix): os.system('%s %s 10' % (test_bin('partdmesh'), graph)) os.system('%s %s' % (test_bin('mesh2dual'), graph)) - # TODO: The following code should replace the testing code in the + # FIXME: The following code should replace the testing code in the # block above since it causes installs to fail when one or more of the # Metis tests fail, but it currently doesn't work because the 'mtest', # 'onmetis', and 'partnmesh' tests return error codes that trigger From 0f40174723612c60ca7bdc0ed0ce1e96daa643bd Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 12:28:59 -0700 Subject: [PATCH 276/375] added test platform --- lib/spack/spack/platforms/test.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lib/spack/spack/platforms/test.py diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py new file mode 100644 index 00000000000..3a4f9d5f2d3 --- /dev/null +++ b/lib/spack/spack/platforms/test.py @@ -0,0 +1,28 @@ +import subprocess +from spack.architecture import Platform, Target +from spack.operating_systems.linux_distro import LinuxDistro +from spack.operating_systems.cnl import Cnl + + +class Test(Platform): + priority = 1000000 + front_end = 'x86_32' + back_end = 'x86_64' + default = 'x86_64' + + back_os = 'CNL' + default_os = 'CNL' + + def __init__(self): + super(Test, self).__init__('test') + self.add_target(self.default, Target(self.default)) + self.add_target(self.front_end, Target(self.front_end)) + + self.add_operating_system(self.default_os, Cnl()) + linux_dist = LinuxDistro() + self.front_os = linux_dist.name + self.add_operating_system(self.front_os, linux_dist) + + @classmethod + def detect(self): + return True From 8f99334f11edd4b56f3e9bc88d2f8666cf0a37a1 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 16:06:14 -0700 Subject: [PATCH 277/375] Fixed test config to work on yosemite --- lib/spack/spack/architecture.py | 38 +++++++++++----------- lib/spack/spack/test/mock_packages_test.py | 24 +++++++++++++- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e9dd489e547..5d16a2f5f56 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -22,26 +22,26 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -""" -This module contains all the elements that are required to create an -architecture object. These include, the target processor, the operating system, -and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. +""" +This module contains all the elements that are required to create an +architecture object. These include, the target processor, the operating system, +and the architecture platform (i.e. cray, darwin, linux, bgq, etc) classes. -On a multiple architecture machine, the architecture spec field can be set to -build a package against any target and operating system that is present on the +On a multiple architecture machine, the architecture spec field can be set to +build a package against any target and operating system that is present on the platform. On Cray platforms or any other architecture that has different front and back end environments, the operating system will determine the method of compiler -detection. +detection. -There are two different types of compiler detection: +There are two different types of compiler detection: 1. Through the $PATH env variable (front-end detection) 2. Through the tcl module system. (back-end detection) Depending on which operating system is specified, the compiler will be detected -using one of those methods. +using one of those methods. For platforms such as linux and darwin, the operating system is autodetected and -the target is set to be x86_64. +the target is set to be x86_64. The command line syntax for specifying an architecture is as follows: @@ -52,27 +52,27 @@ set in the 'platforms/' directory which contains the different subclasses for platforms. If the machine has multiple architectures, the user can also enter front-end, or fe or back-end or be. These settings will concretize -to their respective front-end and back-end targets and operating systems. +to their respective front-end and back-end targets and operating systems. Additional platforms can be added by creating a subclass of Platform -and adding it inside the platform directory. +and adding it inside the platform directory. Platforms are an abstract class that are extended by subclasses. If the user wants to add a new type of platform (such as cray_xe), they can create a subclass -and set all the class attributes such as priority, front_target ,back_target, +and set all the class attributes such as priority, front_target ,back_target, front_os, back_os. Platforms also contain a priority class attribute. A lower number signifies higher priority. These numbers are arbitrarily set and can be changed though often there isn't much need unless a new platform is added and -the user wants that to be detected first. +the user wants that to be detected first. Targets are created inside the platform subclasses. Most architecture (like linux, and darwin) will have only one target (x86_64) but in the case of Cray machines, there is both a frontend and backend processor. The user can specify which targets are present on front-end and back-end architecture -Depending on the platform, operating systems are either auto-detected or are +Depending on the platform, operating systems are either auto-detected or are set. The user can set the front-end and back-end operating setting by the class attributes front_os and back_os. The operating system as described earlier, will -be responsible for compiler detection. +be responsible for compiler detection. """ import os from collections import namedtuple @@ -216,7 +216,7 @@ def operating_system(self, name): return self.operating_sys.get(name, None) - + @classmethod def detect(self): """ Subclass is responsible for implementing this method. @@ -235,7 +235,7 @@ def __str__(self): def _cmp_key(self): t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) o_keys = ''.join(str(o._cmp_key()) for o in sorted(self.operating_sys.values())) - return (self.name, + return (self.name, self.default, self.front_end, self.back_end, @@ -385,7 +385,7 @@ def concrete(self): def __str__(self): if self.platform or self.platform_os or self.target: if self.platform.name == 'darwin': - os_name = self.platform_os.name + os_name = self.platform_os.name if self.platform_os else "None" else: os_name = str(self.platform_os) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 79dbdb89a6e..0cdf01f1b9b 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -69,6 +69,17 @@ f77: None fc: None modules: 'None' + clang3.3OSX: + spec: clang@3.3 + operating_system: + name: yosemite + version: '10.10' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' gcc4.5.0CNL: paths: cc: /path/to/gcc @@ -98,10 +109,21 @@ f77: /path/to/gfortran fc: /path/to/gfortran operating_system: - name: RHL + name: redhat version: '6.7' spec: gcc@4.5.0 modules: 'None' + gcc4.5.0OSX: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: yosemite + version: '10.10' + spec: gcc@4.5.0 + modules: 'None' """ mock_packages_config = """\ From c30fe932d96b4a6fbe8845a2f6bdbfb7567ab1d2 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 25 May 2016 16:34:45 -0700 Subject: [PATCH 278/375] fixed mock config to use truncated versions of linux distro versions --- lib/spack/spack/test/mock_packages_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 0cdf01f1b9b..23b1cf52cc9 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -62,7 +62,7 @@ spec: clang@3.3 operating_system: name: redhat - version: '6.7' + version: '6' paths: cc: /path/to/clang cxx: /path/to/clang++ @@ -110,7 +110,7 @@ fc: /path/to/gfortran operating_system: name: redhat - version: '6.7' + version: '6' spec: gcc@4.5.0 modules: 'None' gcc4.5.0OSX: From 0036483dedfcb75c6b893db7108a7f2b6a4c6187 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 26 May 2016 10:25:16 +0200 Subject: [PATCH 279/375] dealii: don't hardcode blas and mpi; minor fixes to dependencies --- .../repos/builtin/packages/dealii/package.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 49dc971d3a8..87fe4b5811a 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -50,8 +50,8 @@ class Dealii(Package): depends_on ("blas") # Boost 1.58 is blacklisted, see https://github.com/dealii/dealii/issues/1591 # require at least 1.59 - depends_on ("boost@1.59.0:", when='~mpi') - depends_on ("boost@1.59.0:+mpi", when='+mpi') + depends_on ("boost@1.59.0:+thread+system+serialization+iostreams", when='~mpi') + depends_on ("boost@1.59.0:+mpi+thread+system+serialization+iostreams", when='+mpi') depends_on ("bzip2") depends_on ("cmake") depends_on ("lapack") @@ -63,10 +63,11 @@ class Dealii(Package): # optional dependencies depends_on ("mpi", when="+mpi") depends_on ("arpack-ng+mpi", when='+arpack+mpi') - depends_on ("doxygen", when='+doc') + depends_on ("doxygen+graphviz", when='+doc') + depends_on ("graphviz", when='+doc') depends_on ("gsl", when='@8.5.0:+gsl') depends_on ("gsl", when='@dev+gsl') - depends_on ("hdf5+mpi~cxx", when='+hdf5+mpi') #FIXME NetCDF declares dependency with ~cxx, why? + depends_on ("hdf5+mpi", when='+hdf5+mpi') depends_on ("metis@5:", when='+metis') depends_on ("netcdf+mpi", when="+netcdf+mpi") depends_on ("netcdf-cxx", when='+netcdf+mpi') @@ -104,8 +105,8 @@ def install(self, spec, prefix): (spec['lapack'].prefix.include, spec['blas'].prefix.include), '-DLAPACK_LIBRARIES=%s;%s' % - (join_path(spec['lapack'].prefix.lib,'liblapack.%s' % dsuf), # FIXME don't hardcode names - join_path(spec['blas'].prefix.lib,'libblas.%s' % dsuf)), # FIXME don't hardcode names + (spec['lapack'].lapack_shared_lib, + spec['blas'].blas_shared_lib), '-DMUPARSER_DIR=%s ' % spec['muparser'].prefix, '-DUMFPACK_DIR=%s' % spec['suite-sparse'].prefix, '-DTBB_DIR=%s' % spec['tbb'].prefix, @@ -116,9 +117,9 @@ def install(self, spec, prefix): if '+mpi' in spec: options.extend([ '-DDEAL_II_WITH_MPI:BOOL=ON', - '-DCMAKE_C_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpicc'), # FIXME: avoid hardcoding mpi wrappers names - '-DCMAKE_CXX_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpic++'), - '-DCMAKE_Fortran_COMPILER=%s' % join_path(self.spec['mpi'].prefix.bin, 'mpif90'), + '-DCMAKE_C_COMPILER=%s' % spec['mpi'].mpicc, + '-DCMAKE_CXX_COMPILER=%s' % spec['mpi'].mpicxx, + '-DCMAKE_Fortran_COMPILER=%s' % spec['mpi'].mpifc, ]) else: options.extend([ From 3b675d8b700369a5b5282e56ecaf6c559e592551 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 26 May 2016 11:09:14 -0700 Subject: [PATCH 280/375] Changed comparison operator so that we don't get caught in an infinite loop --- lib/spack/spack/concretize.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 7a24c1135e3..e30cf329f1e 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -330,6 +330,8 @@ def concretize_compiler_flags(self, spec): compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ + + if not spec.architecture.platform_os: #Although this usually means changed, this means awaiting other changes return True @@ -340,9 +342,9 @@ def concretize_compiler_flags(self, spec): nearest = next(p for p in spec.traverse(direction='parents') if ((p.compiler == spec.compiler and p is not spec) and flag in p.compiler_flags)) - if ((not flag in spec.compiler_flags) or - sorted(spec.compiler_flags[flag]) != sorted(nearest.compiler_flags[flag])): - if flag in spec.compiler_flags: + if not flag in spec.compiler_flags or \ + not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): + if flag in spec.compiler_flag: spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | set(nearest.compiler_flags[flag])) else: From b968603a57a4085b05318c295443d370f4b4c270 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 11:14:05 -0700 Subject: [PATCH 281/375] made yaml format backwards compatible --- lib/spack/spack/architecture.py | 34 +++++++++++-------- lib/spack/spack/operating_systems/pre_v1.py | 17 ++++++++++ .../spack/platforms/spack_compatibility.py | 23 +++++++++++++ 3 files changed, 60 insertions(+), 14 deletions(-) create mode 100644 lib/spack/spack/operating_systems/pre_v1.py create mode 100644 lib/spack/spack/platforms/spack_compatibility.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 5d16a2f5f56..86733ba4189 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -90,7 +90,7 @@ from spack.util.environment import get_path from spack.util.multiproc import parmap import spack.error as serr - +import spack.platforms class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): @@ -462,21 +462,27 @@ def arch_from_dict(d): """ arch = Arch() - if d is None: - return None - platform_dict = d['platform'] - os_dict = d['platform_os'] - target_dict = d['target'] + if isinstance(d, basestring): + # We have an old spec using a string for the architecture + arch.platform = spack.platforms.spack_compatibility.SpackCompatibility() + arch.platform_os = arch.platform.operating_system('default') + arch.target = Target(d) - platform = _platform_from_dict(platform_dict) if platform_dict else None - target = _target_from_dict(target_dict) if os_dict else None - platform_os = _operating_system_from_dict(os_dict) if os_dict else None - arch.platform = platform - arch.target = target - arch.platform_os = platform_os + arch.os_string = None + arch.target_string = None + else: + if d is None: + return None + platform_dict = d['platform'] + os_dict = d['platform_os'] + target_dict = d['target'] - arch.os_string = None - arch.target_string = None + arch.platform = _platform_from_dict(platform_dict) if platform_dict else None + arch.target = _target_from_dict(target_dict) if os_dict else None + arch.platform_os = _operating_system_from_dict(os_dict) if os_dict else None + + arch.os_string = None + arch.target_string = None return arch diff --git a/lib/spack/spack/operating_systems/pre_v1.py b/lib/spack/spack/operating_systems/pre_v1.py new file mode 100644 index 00000000000..71ee557ac1d --- /dev/null +++ b/lib/spack/spack/operating_systems/pre_v1.py @@ -0,0 +1,17 @@ +import re +import os + +from spack.architecture import OperatingSystem + + +class PreV1(OperatingSystem): + """ Compute Node Linux (CNL) is the operating system used for the Cray XC + series super computers. It is a very stripped down version of GNU/Linux. + Any compilers found through this operating system will be used with + modules. If updated, user must make sure that version and name are + updated to indicate that OS has been upgraded (or downgraded) + """ + def __init__(self): + name = 'PreVersion1.0' + version = '1.0' + super(PreV1, self).__init__(name, version) diff --git a/lib/spack/spack/platforms/spack_compatibility.py b/lib/spack/spack/platforms/spack_compatibility.py new file mode 100644 index 00000000000..86cbfb3044a --- /dev/null +++ b/lib/spack/spack/platforms/spack_compatibility.py @@ -0,0 +1,23 @@ +import subprocess +from spack.architecture import Platform +from spack.operating_systems.pre_v1 + +class SpackCompatibility(Platform): + priority = 9999 + + # We don't use the normal target getters for this platform + # Instead, targets are added directly when parsing the yaml + + # OS is the spack backwards compatibility os. + front_os = 'PreVersion1.0' + back_os = 'PreVersion1.0' + default_os = 'PreVersion1.0' + + def __init__(self): + super(SpackCompatibility, self).__init__('spack_compatibility') + sc_os = spack.operating_systems.pre_v1.PreV1() + self.add_operating_system(sc_os.name, sc_os) + + @classmethod + def detect(self): + return True From c0661744cccffc6ca11a82cea840e7be63736d37 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 11:38:40 -0700 Subject: [PATCH 282/375] fixed compatibility work --- lib/spack/spack/architecture.py | 5 ++-- lib/spack/spack/operating_systems/pre_v1.py | 17 -------------- .../spack/platforms/spack_compatibility.py | 23 ------------------- 3 files changed, 2 insertions(+), 43 deletions(-) delete mode 100644 lib/spack/spack/operating_systems/pre_v1.py delete mode 100644 lib/spack/spack/platforms/spack_compatibility.py diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 86733ba4189..a0ef4f14da5 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -90,7 +90,6 @@ from spack.util.environment import get_path from spack.util.multiproc import parmap import spack.error as serr -import spack.platforms class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): @@ -464,8 +463,8 @@ def arch_from_dict(d): if isinstance(d, basestring): # We have an old spec using a string for the architecture - arch.platform = spack.platforms.spack_compatibility.SpackCompatibility() - arch.platform_os = arch.platform.operating_system('default') + arch.platform = Platform('spack_compatibility') + arch.platform_os = OperatingSystem('pre_version', '1.0') arch.target = Target(d) arch.os_string = None diff --git a/lib/spack/spack/operating_systems/pre_v1.py b/lib/spack/spack/operating_systems/pre_v1.py deleted file mode 100644 index 71ee557ac1d..00000000000 --- a/lib/spack/spack/operating_systems/pre_v1.py +++ /dev/null @@ -1,17 +0,0 @@ -import re -import os - -from spack.architecture import OperatingSystem - - -class PreV1(OperatingSystem): - """ Compute Node Linux (CNL) is the operating system used for the Cray XC - series super computers. It is a very stripped down version of GNU/Linux. - Any compilers found through this operating system will be used with - modules. If updated, user must make sure that version and name are - updated to indicate that OS has been upgraded (or downgraded) - """ - def __init__(self): - name = 'PreVersion1.0' - version = '1.0' - super(PreV1, self).__init__(name, version) diff --git a/lib/spack/spack/platforms/spack_compatibility.py b/lib/spack/spack/platforms/spack_compatibility.py deleted file mode 100644 index 86cbfb3044a..00000000000 --- a/lib/spack/spack/platforms/spack_compatibility.py +++ /dev/null @@ -1,23 +0,0 @@ -import subprocess -from spack.architecture import Platform -from spack.operating_systems.pre_v1 - -class SpackCompatibility(Platform): - priority = 9999 - - # We don't use the normal target getters for this platform - # Instead, targets are added directly when parsing the yaml - - # OS is the spack backwards compatibility os. - front_os = 'PreVersion1.0' - back_os = 'PreVersion1.0' - default_os = 'PreVersion1.0' - - def __init__(self): - super(SpackCompatibility, self).__init__('spack_compatibility') - sc_os = spack.operating_systems.pre_v1.PreV1() - self.add_operating_system(sc_os.name, sc_os) - - @classmethod - def detect(self): - return True From 88bec814eb45ff4997229e092f234d8ce7c76664 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 12:10:41 -0700 Subject: [PATCH 283/375] Allow 'arch=' syntax for specs to allow copying from output --- lib/spack/spack/concretize.py | 2 +- lib/spack/spack/platforms/cray_xc.py | 12 ++++++------ lib/spack/spack/platforms/linux.py | 8 ++++---- lib/spack/spack/spec.py | 18 +++++++++++------- lib/spack/spack/test/spec_semantics.py | 1 - 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index e30cf329f1e..5d9715feed4 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -344,7 +344,7 @@ def concretize_compiler_flags(self, spec): and flag in p.compiler_flags)) if not flag in spec.compiler_flags or \ not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): - if flag in spec.compiler_flag: + if flag in spec.compiler_flags: spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | set(nearest.compiler_flags[flag])) else: diff --git a/lib/spack/spack/platforms/cray_xc.py b/lib/spack/spack/platforms/cray_xc.py index 4843a47c622..e710303e23a 100644 --- a/lib/spack/spack/platforms/cray_xc.py +++ b/lib/spack/spack/platforms/cray_xc.py @@ -9,16 +9,16 @@ class CrayXc(Platform): back_end = 'ivybridge' default = 'ivybridge' - front_os = "SuSE" - back_os = "CNL" - default_os = "CNL" + front_os = "SuSE11" + back_os = "CNL10" + default_os = "CNL10" def __init__(self): ''' Since cori doesn't have ivybridge as a front end it's better if we use CRAY_CPU_TARGET as the default. This will ensure that if we're on a XC-40 or XC-30 then we can detect the target ''' - super(CrayXc, self).__init__('crayxc') + super(CrayXc, self).__init__('cray_xc') # Handle the default here so we can check for a key error if 'CRAY_CPU_TARGET' in os.environ: @@ -37,8 +37,8 @@ def __init__(self): self.add_target('haswell', Target('haswell','craype-haswell')) - self.add_operating_system('SuSE', LinuxDistro()) - self.add_operating_system('CNL', Cnl()) + self.add_operating_system('SuSE11', LinuxDistro()) + self.add_operating_system('CNL10', Cnl()) @classmethod def detect(self): diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 18050ac79e1..ebe018812ac 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -12,10 +12,10 @@ def __init__(self): super(Linux, self).__init__('linux') self.add_target(self.default, Target(self.default)) linux_dist = LinuxDistro() - self.default_os = linux_dist.name - self.front_os = linux_dist.name - self.back_os = linux_dist.name - self.add_operating_system(linux_dist.name, linux_dist) + self.default_os = str(linux_dist) + self.front_os = self.default_os + self.back_os = self.default._os + self.add_operating_system(str(linux_dist), linux_dist) @classmethod def detect(self): diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 5d82af4af6a..58d76fd7a92 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -538,13 +538,17 @@ def _add_flag(self, name, value): Known flags currently include "arch" """ valid_flags = FlagMap.valid_compiler_flags() -# if name == 'arch' or name == 'architecture': -# platform, op_sys, target = value.split('-') -# print platform, op_sys, target, '+++++++' -# self._set_platform(platform) -# self._set_os(op_sys) -# self._set_target(target) - if name == 'platform': + if name == 'arch' or name == 'architecture': + platform, op_sys, target = value.split('-') + assert(self.architecture.platform is None) + assert(self.architecture.platform_os is None) + assert(self.architecture.target is None) + assert(self.architecture.os_string is None) + assert(self.architecture.target_string is None) + self._set_platform(platform) + self._set_os(op_sys) + self._set_target(target) + elif name == 'platform': self._set_platform(value) elif name == 'os' or name == 'operating_system': if self.architecture.platform: diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index aa99ad71ed6..0b2111bfe85 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -402,7 +402,6 @@ def test_constrain_changed(self): self.check_constrain_changed('libelf', '~debug') self.check_constrain_changed('libelf', 'debug=2') self.check_constrain_changed('libelf', 'cppflags="-O3"') - self.check_constrain_changed('libelf', ' arch=bgqos_0') platform = spack.architecture.sys_type() self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name) From 189e8b3f349c8102772fa995fd8ee79df3c242c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 13:17:01 -0700 Subject: [PATCH 284/375] fixed compiler config init logic --- lib/spack/spack/compilers/__init__.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 8fe7a17116b..ea79f655b00 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -102,15 +102,18 @@ def init_compiler_config(): # configured. Avoid updating automatically if there ARE site # compilers configured but no user ones. # if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: - if scope is None: - # We know no compilers were configured in any scope. - init_compiler_config() - elif scope == 'user': - # Check the site config and update the user config if - # nothing is configured at the site level. - site_config = spack.config.get_config('compilers', scope='site') - if not site_config: + if not config: + if scope is None: + # We know no compilers were configured in any scope. init_compiler_config() + config = spack.config.get_config('compilers', scope=scope) + elif scope == 'user': + # Check the site config and update the user config if + # nothing is configured at the site level. + site_config = spack.config.get_config('compilers', scope='site') + if not site_config: + init_compiler_config() + config = spack.config.get_config('compilers', scope=scope) return config From 9c1da4fa4cdfe32d8a7a0026f457066b0cac7e8e Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 26 May 2016 13:58:17 -0700 Subject: [PATCH 285/375] Corrected parantheses wrapping --- lib/spack/spack/build_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 073f73a91a1..4483b9d1329 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -366,7 +366,7 @@ def get_rpaths(pkg): if os.path.isdir(d.prefix.lib64)) # Second module is our compiler mod name. We use that to get rpaths from # module show output. - if pkg.compiler.modules and len(pkg.compiler.modules > 1): + if pkg.compiler.modules and len(pkg.compiler.modules) > 1: rpaths.append(get_path_from_module(pkg.compiler.modules[1])) return rpaths From 040c122719c7f4cb97ce14668111e0c66125c8c3 Mon Sep 17 00:00:00 2001 From: Denis Davydov Date: Thu, 26 May 2016 20:54:38 +0200 Subject: [PATCH 286/375] dealii: formatting --- .../repos/builtin/packages/dealii/package.py | 134 ++++++++++-------- 1 file changed, 73 insertions(+), 61 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 87fe4b5811a..867d317e4d8 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -25,8 +25,10 @@ from spack import * import sys + class Dealii(Package): - """C++ software library providing well-documented tools to build finite element codes for a broad variety of PDEs.""" + """C++ software library providing well-documented tools to build finite + element codes for a broad variety of PDEs.""" homepage = "https://www.dealii.org" url = "https://github.com/dealii/dealii/releases/download/v8.4.0/dealii-8.4.0.tar.gz" @@ -36,7 +38,7 @@ class Dealii(Package): variant('mpi', default=True, description='Compile with MPI') variant('arpack', default=True, description='Compile with Arpack and PArpack (only with MPI)') variant('doc', default=False, description='Compile with documentation') - variant('gsl' , default=True, description='Compile with GSL') + variant('gsl', default=True, description='Compile with GSL') variant('hdf5', default=True, description='Compile with HDF5 (only with MPI)') variant('metis', default=True, description='Compile with Metis') variant('netcdf', default=True, description='Compile with Netcdf (only with MPI)') @@ -47,39 +49,40 @@ class Dealii(Package): variant('trilinos', default=True, description='Compile with Trilinos (only with MPI)') # required dependencies, light version - depends_on ("blas") - # Boost 1.58 is blacklisted, see https://github.com/dealii/dealii/issues/1591 - # require at least 1.59 - depends_on ("boost@1.59.0:+thread+system+serialization+iostreams", when='~mpi') - depends_on ("boost@1.59.0:+mpi+thread+system+serialization+iostreams", when='+mpi') - depends_on ("bzip2") - depends_on ("cmake") - depends_on ("lapack") - depends_on ("muparser") - depends_on ("suite-sparse") - depends_on ("tbb") - depends_on ("zlib") + depends_on("blas") + # Boost 1.58 is blacklisted, see + # https://github.com/dealii/dealii/issues/1591 + # Require at least 1.59 + depends_on("boost@1.59.0:+thread+system+serialization+iostreams", when='~mpi') # NOQA: ignore=E501 + depends_on("boost@1.59.0:+mpi+thread+system+serialization+iostreams", when='+mpi') # NOQA: ignore=E501 + depends_on("bzip2") + depends_on("cmake") + depends_on("lapack") + depends_on("muparser") + depends_on("suite-sparse") + depends_on("tbb") + depends_on("zlib") # optional dependencies - depends_on ("mpi", when="+mpi") - depends_on ("arpack-ng+mpi", when='+arpack+mpi') - depends_on ("doxygen+graphviz", when='+doc') - depends_on ("graphviz", when='+doc') - depends_on ("gsl", when='@8.5.0:+gsl') - depends_on ("gsl", when='@dev+gsl') - depends_on ("hdf5+mpi", when='+hdf5+mpi') - depends_on ("metis@5:", when='+metis') - depends_on ("netcdf+mpi", when="+netcdf+mpi") - depends_on ("netcdf-cxx", when='+netcdf+mpi') - depends_on ("oce", when='+oce') - depends_on ("p4est", when='+p4est+mpi') - depends_on ("petsc+mpi", when='+petsc+mpi') - depends_on ("slepc", when='+slepc+petsc+mpi') - depends_on ("trilinos", when='+trilinos+mpi') + depends_on("mpi", when="+mpi") + depends_on("arpack-ng+mpi", when='+arpack+mpi') + depends_on("doxygen+graphviz", when='+doc') + depends_on("graphviz", when='+doc') + depends_on("gsl", when='@8.5.0:+gsl') + depends_on("gsl", when='@dev+gsl') + depends_on("hdf5+mpi", when='+hdf5+mpi') + depends_on("metis@5:", when='+metis') + depends_on("netcdf+mpi", when="+netcdf+mpi") + depends_on("netcdf-cxx", when='+netcdf+mpi') + depends_on("oce", when='+oce') + depends_on("p4est", when='+p4est+mpi') + depends_on("petsc+mpi", when='+petsc+mpi') + depends_on("slepc", when='+slepc+petsc+mpi') + depends_on("trilinos", when='+trilinos+mpi') # developer dependnecies - depends_on ("numdiff", when='@dev') - depends_on ("astyle@2.04", when='@dev') + depends_on("numdiff", when='@dev') + depends_on("astyle@2.04", when='@dev') def install(self, spec, prefix): options = [] @@ -97,9 +100,9 @@ def install(self, spec, prefix): '-DDEAL_II_WITH_THREADS:BOOL=ON', '-DBOOST_DIR=%s' % spec['boost'].prefix, '-DBZIP2_DIR=%s' % spec['bzip2'].prefix, - # CMake's FindBlas/Lapack may pickup system's blas/lapack instead of Spack's. - # Be more specific to avoid this. - # Note that both lapack and blas are provided in -DLAPACK_XYZ variables + # CMake's FindBlas/Lapack may pickup system's blas/lapack instead + # of Spack's. Be more specific to avoid this. + # Note that both lapack and blas are provided in -DLAPACK_XYZ. '-DLAPACK_FOUND=true', '-DLAPACK_INCLUDE_DIRS=%s;%s' % (spec['lapack'].prefix.include, @@ -107,7 +110,7 @@ def install(self, spec, prefix): '-DLAPACK_LIBRARIES=%s;%s' % (spec['lapack'].lapack_shared_lib, spec['blas'].blas_shared_lib), - '-DMUPARSER_DIR=%s ' % spec['muparser'].prefix, + '-DMUPARSER_DIR=%s' % spec['muparser'].prefix, '-DUMFPACK_DIR=%s' % spec['suite-sparse'].prefix, '-DTBB_DIR=%s' % spec['tbb'].prefix, '-DZLIB_DIR=%s' % spec['zlib'].prefix @@ -126,24 +129,25 @@ def install(self, spec, prefix): '-DDEAL_II_WITH_MPI:BOOL=OFF', ]) - # Optional dependencies for which librariy names are the same as CMake variables - for library in ('gsl','hdf5','p4est','petsc','slepc','trilinos','metis'): + # Optional dependencies for which librariy names are the same as CMake + # variables: + for library in ('gsl', 'hdf5', 'p4est', 'petsc', 'slepc', 'trilinos', 'metis'): # NOQA: ignore=E501 if library in spec: options.extend([ - '-D{library}_DIR={value}'.format(library=library.upper(), value=spec[library].prefix), - '-DDEAL_II_WITH_{library}:BOOL=ON'.format(library=library.upper()) + '-D%s_DIR=%s' % (library.upper(), spec[library].prefix), + '-DDEAL_II_WITH_%s:BOOL=ON' % library.upper() ]) else: options.extend([ - '-DDEAL_II_WITH_{library}:BOOL=OFF'.format(library=library.upper()) + '-DDEAL_II_WITH_%s:BOOL=OFF' % library.upper() ]) # doxygen options.extend([ - '-DDEAL_II_COMPONENT_DOCUMENTATION=%s' % ('ON' if '+doc' in spec else 'OFF'), + '-DDEAL_II_COMPONENT_DOCUMENTATION=%s' % + ('ON' if '+doc' in spec else 'OFF'), ]) - # arpack if '+arpack' in spec: options.extend([ @@ -161,11 +165,13 @@ def install(self, spec, prefix): options.extend([ '-DNETCDF_FOUND=true', '-DNETCDF_LIBRARIES=%s;%s' % - (join_path(spec['netcdf-cxx'].prefix.lib,'libnetcdf_c++.%s' % dsuf), - join_path(spec['netcdf'].prefix.lib,'libnetcdf.%s' % dsuf)), + (join_path(spec['netcdf-cxx'].prefix.lib, + 'libnetcdf_c++.%s' % dsuf), + join_path(spec['netcdf'].prefix.lib, + 'libnetcdf.%s' % dsuf)), '-DNETCDF_INCLUDE_DIRS=%s;%s' % (spec['netcdf-cxx'].prefix.include, - spec['netcdf'].prefix.include), + spec['netcdf'].prefix.include), ]) else: options.extend([ @@ -201,7 +207,7 @@ def install(self, spec, prefix): with working_dir('examples/step-3'): cmake('.') make('release') - make('run',parallel=False) + make('run', parallel=False) # An example which uses Metis + PETSc # FIXME: switch step-18 to MPI @@ -214,7 +220,7 @@ def install(self, spec, prefix): if '^petsc' in spec and '^metis' in spec: cmake('.') make('release') - make('run',parallel=False) + make('run', parallel=False) # take step-40 which can use both PETSc and Trilinos # FIXME: switch step-40 to MPI run @@ -223,43 +229,49 @@ def install(self, spec, prefix): print('========== Step-40 PETSc ============') print('=====================================') # list the number of cycles to speed up - filter_file(r'(const unsigned int n_cycles = 8;)', ('const unsigned int n_cycles = 2;'), 'step-40.cc') + filter_file(r'(const unsigned int n_cycles = 8;)', + ('const unsigned int n_cycles = 2;'), 'step-40.cc') cmake('.') if '^petsc' in spec: make('release') - make('run',parallel=False) + make('run', parallel=False) print('=====================================') print('========= Step-40 Trilinos ==========') print('=====================================') # change Linear Algebra to Trilinos - filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') + filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', + ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') if '^trilinos+hypre' in spec: make('release') - make('run',parallel=False) + make('run', parallel=False) print('=====================================') print('=== Step-40 Trilinos SuperluDist ====') print('=====================================') # change to direct solvers - filter_file(r'(LA::SolverCG solver\(solver_control\);)', ('TrilinosWrappers::SolverDirect::AdditionalData data(false,"Amesos_Superludist"); TrilinosWrappers::SolverDirect solver(solver_control,data);'), 'step-40.cc') - filter_file(r'(LA::MPI::PreconditionAMG preconditioner;)', (''), 'step-40.cc') - filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', (''), 'step-40.cc') - filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', (''), 'step-40.cc') - filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') + filter_file(r'(LA::SolverCG solver\(solver_control\);)', ('TrilinosWrappers::SolverDirect::AdditionalData data(false,"Amesos_Superludist"); TrilinosWrappers::SolverDirect solver(solver_control,data);'), 'step-40.cc') # NOQA: ignore=E501 + filter_file(r'(LA::MPI::PreconditionAMG preconditioner;)', + (''), 'step-40.cc') + filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', + (''), 'step-40.cc') + filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', + (''), 'step-40.cc') + filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') # NOQA: ignore=E501 filter_file(r'(preconditioner\);)', (''), 'step-40.cc') if '^trilinos+superlu-dist' in spec: make('release') - make('run',paralle=False) + make('run', paralle=False) print('=====================================') print('====== Step-40 Trilinos MUMPS =======') print('=====================================') # switch to Mumps - filter_file(r'(Amesos_Superludist)', ('Amesos_Mumps'), 'step-40.cc') + filter_file(r'(Amesos_Superludist)', + ('Amesos_Mumps'), 'step-40.cc') if '^trilinos+mumps' in spec: make('release') - make('run',parallel=False) + make('run', parallel=False) print('=====================================') print('============ Step-36 ================') @@ -268,7 +280,7 @@ def install(self, spec, prefix): if 'slepc' in spec: cmake('.') make('release') - make('run',parallel=False) + make('run', parallel=False) print('=====================================') print('============ Step-54 ================') @@ -277,7 +289,7 @@ def install(self, spec, prefix): if 'oce' in spec: cmake('.') make('release') - make('run',parallel=False) + make('run', parallel=False) def setup_environment(self, spack_env, env): env.set('DEAL_II_DIR', self.prefix) From e95eedb165c54e00d22bd6d5acfdf8c7c55d140e Mon Sep 17 00:00:00 2001 From: Joseph Ciurej Date: Thu, 26 May 2016 14:27:15 -0700 Subject: [PATCH 287/375] Added a style checking exemption for lambda expressions. --- .flake8 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index a1e2fcc1f8a..12ae2efa80d 100644 --- a/.flake8 +++ b/.flake8 @@ -8,6 +8,9 @@ # - E221: multiple spaces before operator # - E241: multiple spaces after ‘,’ # +# Let people use terse Python features: +# - E731 : lambda expressions +# # Spack allows wildcard imports: # - F403: disable wildcard import # @@ -16,5 +19,5 @@ # - F999: name name be undefined or undefined from star imports. # [flake8] -ignore = E221,E241,F403,F821,F999 +ignore = E221,E241,E731,F403,F821,F999 max-line-length = 79 From a2f90453f4d1ac1372f717911bd08e33d801ff31 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Thu, 26 May 2016 20:30:05 -0700 Subject: [PATCH 288/375] Fixed backwards compatibility for compilers.yaml config file --- bin/spack | 3 ++ lib/spack/spack/yaml_version_check.py | 55 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 lib/spack/spack/yaml_version_check.py diff --git a/bin/spack b/bin/spack index 3544feb10ac..9b1276a8666 100755 --- a/bin/spack +++ b/bin/spack @@ -138,6 +138,9 @@ def main(): import spack.util.debug as debug debug.register_interrupt_handler() + from spack.yaml_version_check import check_yaml_versions + check_yaml_versions() + spack.spack_working_dir = working_dir if args.mock: from spack.repository import RepoPath diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py new file mode 100644 index 00000000000..7e7bae4edf9 --- /dev/null +++ b/lib/spack/spack/yaml_version_check.py @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Yaml Version Check is a module for ensuring that config file +formats are compatible with the current version of Spack.""" +import os.path +import os +import llnl.util.tty as tty +import spack.util.spack_yaml as syaml +import spack.config + + +def check_yaml_versions(): + check_compiler_yaml_version() + +def check_compiler_yaml_version(): + config_scopes = spack.config.config_scopes + for scope in config_scopes.values(): + file_name = os.path.join(scope.path, 'compilers.yaml') + data = None + if os.path.isfile(file_name): + with open(file_name) as f: + data = syaml.load(f) + + if data: + compilers = data['compilers'].items() + if len(compilers) > 0: + if 'operating_system' not in compilers[0][1]: + new_file = os.path.join(scope.path, '_old_compilers.yaml') + tty.warn('%s in out of date compilers format. ' + 'Moved to %s. Spack automatically generate ' + 'a compilers config file ' + % (file_name, new_file)) + os.rename(file_name, new_file) From dd5a6914f161f2237a90df856126a905e305d80f Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 27 May 2016 12:56:43 -0500 Subject: [PATCH 289/375] Undo changes to flex --- var/spack/repos/builtin/packages/flex/package.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py index f41af061a52..926651010f7 100644 --- a/var/spack/repos/builtin/packages/flex/package.py +++ b/var/spack/repos/builtin/packages/flex/package.py @@ -24,7 +24,6 @@ ############################################################################## from spack import * - class Flex(Package): """Flex is a tool for generating scanners.""" @@ -34,8 +33,6 @@ class Flex(Package): version('2.6.0', '5724bcffed4ebe39e9b55a9be80859ec') version('2.5.39', 'e133e9ead8ec0a58d81166b461244fde') - depends_on('bison') - def install(self, spec, prefix): configure("--prefix=%s" % prefix) From 023504ed645fecc7e7c12948dac890b65124c880 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 27 May 2016 12:26:59 -0700 Subject: [PATCH 290/375] Fixed the external modules bug --- lib/spack/spack/spec.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 58d76fd7a92..bbc1abfa9ea 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1065,7 +1065,7 @@ def _expand_virtual_packages(self): continue # If replacement is external then trim the dependencies - if replacement.external: + if replacement.external or replacement.external_module: if (spec.dependencies): changed = True spec.dependencies = DependencyMap() @@ -1082,7 +1082,8 @@ def feq(cfield, sfield): feq(replacement.architecture, spec.architecture) and feq(replacement.dependencies, spec.dependencies) and feq(replacement.variants, spec.variants) and - feq(replacement.external, spec.external)): + feq(replacement.external, spec.external) and + feq(replacement.external_module, spec.external_module)): continue # Refine this spec to the candidate. This uses @@ -1653,18 +1654,9 @@ def satisfies(self, other, deps=True, strict=False): # TODO: Need to make sure that comparisons can be made via classes if self.architecture and other.architecture: - print self.architecture, other.architecture if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): - d1 = self.architecture.platform.to_dict() - d2 = other.architecture.platform.to_dict() - print d1 - print d2 - print d1==d2 - print self.architecture.platform == other.architecture.platform - print self.architecture.platform._cmp_key() - print other.architecture.platform._cmp_key() return False elif strict and ((other.architecture and not self.architecture) or (other.architecture.platform and not self.architecture.platform) or From f49644cdea3b00b057f33bf0c609e3a83eb3cd65 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 27 May 2016 12:37:58 -0700 Subject: [PATCH 291/375] Fixed typo in linux platform --- lib/spack/spack/platforms/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index ebe018812ac..4d8adac3849 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -14,7 +14,7 @@ def __init__(self): linux_dist = LinuxDistro() self.default_os = str(linux_dist) self.front_os = self.default_os - self.back_os = self.default._os + self.back_os = self.default_os self.add_operating_system(str(linux_dist), linux_dist) @classmethod From 0ae1298d2584403256d5ff02de039e8195116cfa Mon Sep 17 00:00:00 2001 From: Greg Lee Date: Fri, 27 May 2016 12:43:13 -0700 Subject: [PATCH 292/375] updated libdwarf version and dwarfdump build --- var/spack/repos/builtin/packages/libdwarf/package.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/var/spack/repos/builtin/packages/libdwarf/package.py b/var/spack/repos/builtin/packages/libdwarf/package.py index 3f5a72116e9..c9c6e9404f6 100644 --- a/var/spack/repos/builtin/packages/libdwarf/package.py +++ b/var/spack/repos/builtin/packages/libdwarf/package.py @@ -41,12 +41,10 @@ class Libdwarf(Package): MIPS/IRIX C compiler.""" homepage = "http://www.prevanders.net/dwarf.html" - url = "http://www.prevanders.net/libdwarf-20130729.tar.gz" + url = "http://www.prevanders.net/libdwarf-20160507.tar.gz" list_url = homepage - version('20130729', '4cc5e48693f7b93b7aa0261e63c0e21d') - version('20130207', '64b42692e947d5180e162e46c689dfbf') - version('20130126', 'ded74a5e90edb5a12aac3c29d260c5db') + version('20160507', 'ae32d6f9ece5daf05e2d4b14822ea811') depends_on("libelf") @@ -69,7 +67,7 @@ def install(self, spec, prefix): install('libdwarf.h', prefix.include) install('dwarf.h', prefix.include) - with working_dir('dwarfdump2'): + with working_dir('dwarfdump'): configure("--prefix=" + prefix) # This makefile has strings of copy commands that From becec8ac7e17a9d56f1d5b0640c1f56be8f64a7f Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:12:53 -0700 Subject: [PATCH 293/375] Added external module check in normalize to avoid infinite loop --- lib/spack/spack/spec.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index a505b3c12e2..f8bad3ff03f 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1002,11 +1002,12 @@ def _expand_virtual_packages(self): """ # Make an index of stuff this spec already provides self_index = ProviderIndex(self.traverse(), restrict=True) - changed = False done = False + while not done: done = True + for spec in list(self.traverse()): replacement = None if spec.virtual: @@ -1329,7 +1330,7 @@ def _normalize_helper(self, visited, spec_deps, provider_index): # if we descend into a virtual spec, there's nothing more # to normalize. Concretize will finish resolving it later. - if self.virtual or self.external: + if self.virtual or self.external or self.external_module: return False # Combine constraints from package deps with constraints from @@ -1376,7 +1377,6 @@ def normalize(self, force=False): # Ensure first that all packages & compilers in the DAG exist. self.validate_names() - # Get all the dependencies into one DependencyMap spec_deps = self.flat_dependencies(copy=False) From 98a4a9f2cc67c141deb7903f577c60c21912667c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:13:22 -0700 Subject: [PATCH 294/375] Adding test for external modules --- lib/spack/spack/test/concretize.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 963481054e5..77ff026c7aa 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -253,6 +253,13 @@ def test_external_package(self): self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) + def test_external_package_module(self): + spec = Spec('externalmodule') + spec.concretize() + self.assertEqual(spec['externalmodule'].external_module, 'external-module') + self.assertFalse('externalprereq' in spec) + self.assertTrue(spec['externalmodule'].compiler.satisfies('gcc')) + def test_nobuild_package(self): got_error = False spec = Spec('externaltool%clang') From f0a9c45207eabc81b5f50b68f6dc02f807f1a221 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:13:57 -0700 Subject: [PATCH 295/375] Adding external to mock config --- lib/spack/spack/test/mock_packages_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 23b1cf52cc9..5561027aac9 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -137,6 +137,10 @@ paths: externalvirtual@2.0%clang@3.3: /path/to/external_virtual_clang externalvirtual@1.0%gcc@4.5.0: /path/to/external_virtual_gcc + externalmodule: + buildable: False + modules: + externalmodule@1.0%gcc@4.5.0: external-module """ class MockPackagesTest(unittest.TestCase): From 33e1dcc4763024db87e709b3db6ff4ee12d43dc5 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 10:14:40 -0700 Subject: [PATCH 296/375] Adding mock package file for external module --- .../packages/externalmodule/package.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 var/spack/repos/builtin.mock/packages/externalmodule/package.py diff --git a/var/spack/repos/builtin.mock/packages/externalmodule/package.py b/var/spack/repos/builtin.mock/packages/externalmodule/package.py new file mode 100644 index 00000000000..f7b0da3fd91 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/externalmodule/package.py @@ -0,0 +1,37 @@ + +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Externalmodule(Package): + homepage = "http://somewhere.com" + url = "http://somewhere.com/module-1.0.tar.gz" + + version('1.0', '1234567890abcdef1234567890abcdef') + + depends_on('externalprereq') + + def install(self, spec, prefix): + pass From 69585cb6e3b391d2771387830067d893f2e7ec32 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 12:39:43 -0700 Subject: [PATCH 297/375] Added elcap compilers --- lib/spack/spack/test/mock_packages_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 5561027aac9..f02bfb7a079 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -124,6 +124,28 @@ version: '10.10' spec: gcc@4.5.0 modules: 'None' + gcc4.5.0ELCAP: + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: /path/to/gfortran + fc: /path/to/gfortran + operating_system: + name: elcapitan + version: '10.11' + spec: gcc@4.5.0 + modules: 'None' + clang3.3ELCAP: + spec: clang@3.3 + operating_system: + name: elcapitan + version: '10.11' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' """ mock_packages_config = """\ From 22ca72e7b9870dddfd66760658a9221f4f24ffe3 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Sat, 28 May 2016 12:40:29 -0700 Subject: [PATCH 298/375] Added conditional skipIf to test_external_modules, darwin machines don't have tcl modules so skip this test and let user know why --- lib/spack/spack/test/concretize.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 77ff026c7aa..cf2daa4be2a 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -23,6 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import spack +import spack.architecture from spack.spec import Spec, CompilerSpec from spack.version import ver from spack.concretize import find_spec @@ -252,7 +253,8 @@ def test_external_package(self): self.assertFalse('externalprereq' in spec) self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) - + @unittest.skipIf(spack.architecture.sys_type().name == 'darwin', + "No tcl modules on darwin machines") def test_external_package_module(self): spec = Spec('externalmodule') spec.concretize() @@ -269,7 +271,7 @@ def test_nobuild_package(self): got_error = True self.assertTrue(got_error) - + def test_external_and_virtual(self): spec = Spec('externaltest') spec.concretize() From e8b4d5fb6ffe18bcb93a86d00fbfb1ed37d0db3b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sat, 28 May 2016 22:01:58 -0700 Subject: [PATCH 299/375] Performance boost: reduce instantiations of re.Scanner - Lexer is the same for every Spec parser in spack, so don't build it every time. - This improves time to import package.py files a lot, as a Lexer doesn't have to be constructed for every spc in the packages. - To concretize dealii: - Before: ~20 sec - After: ~6 sec --- lib/spack/spack/spec.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 7c09af4c210..16b61236a96 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -2015,10 +2015,13 @@ def __init__(self): (r'\s+', lambda scanner, val: None)]) +# Lexer is always the same for every parser. +_lexer = SpecLexer() + class SpecParser(spack.parse.Parser): def __init__(self): - super(SpecParser, self).__init__(SpecLexer()) + super(SpecParser, self).__init__(_lexer) self.previous = None def do_parse(self): From c2ac47235fd02e34c4ea5ac9d9c65a8b313ffa41 Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Sun, 29 May 2016 13:22:19 +0200 Subject: [PATCH 300/375] additional versions for Dealii Included additional versions for Deal.II package. --- var/spack/repos/builtin/packages/dealii/package.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 867d317e4d8..9c09b205a85 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -30,9 +30,13 @@ class Dealii(Package): """C++ software library providing well-documented tools to build finite element codes for a broad variety of PDEs.""" homepage = "https://www.dealii.org" - url = "https://github.com/dealii/dealii/releases/download/v8.4.0/dealii-8.4.0.tar.gz" + url = "https://github.com/dealii/dealii/releases/download/v8.4.1/dealii-8.4.1.tar.gz" + version('8.4.1', 'efbaf16f9ad59cfccad62302f36c3c1d') version('8.4.0', 'ac5dbf676096ff61e092ce98c80c2b00') + version('8.3.0', 'fc6cdcb16309ef4bea338a4f014de6fa') + version('8.2.1', '71c728dbec14f371297cd405776ccf08') + version('8.1.0', 'aa8fadc2ce5eb674f44f997461bf668d') version('dev', git='https://github.com/dealii/dealii.git') variant('mpi', default=True, description='Compile with MPI') From d7e84fe07c748206afa25e8e45a2ebfac71c7cdf Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Mon, 30 May 2016 14:03:27 +0200 Subject: [PATCH 301/375] dealii, test for specific version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Specified that test case “step-40” is only to be run when using dealii version 8.4.0 or later --- .../repos/builtin/packages/dealii/package.py | 93 ++++++++++--------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 9c09b205a85..c8710590756 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -228,54 +228,55 @@ def install(self, spec, prefix): # take step-40 which can use both PETSc and Trilinos # FIXME: switch step-40 to MPI run - with working_dir('examples/step-40'): - print('=====================================') - print('========== Step-40 PETSc ============') - print('=====================================') - # list the number of cycles to speed up - filter_file(r'(const unsigned int n_cycles = 8;)', - ('const unsigned int n_cycles = 2;'), 'step-40.cc') - cmake('.') - if '^petsc' in spec: - make('release') - make('run', parallel=False) + if spec.satisfies('@8.4.0:'): + with working_dir('examples/step-40'): + print('=====================================') + print('========== Step-40 PETSc ============') + print('=====================================') + # list the number of cycles to speed up + filter_file(r'(const unsigned int n_cycles = 8;)', + ('const unsigned int n_cycles = 2;'), 'step-40.cc') + cmake('.') + if '^petsc' in spec: + make('release') + make('run', parallel=False) + + print('=====================================') + print('========= Step-40 Trilinos ==========') + print('=====================================') + # change Linear Algebra to Trilinos + filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', + ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') + if '^trilinos+hypre' in spec: + make('release') + make('run', parallel=False) - print('=====================================') - print('========= Step-40 Trilinos ==========') - print('=====================================') - # change Linear Algebra to Trilinos - filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', - ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') - if '^trilinos+hypre' in spec: - make('release') - make('run', parallel=False) + print('=====================================') + print('=== Step-40 Trilinos SuperluDist ====') + print('=====================================') + # change to direct solvers + filter_file(r'(LA::SolverCG solver\(solver_control\);)', ('TrilinosWrappers::SolverDirect::AdditionalData data(false,"Amesos_Superludist"); TrilinosWrappers::SolverDirect solver(solver_control,data);'), 'step-40.cc') # NOQA: ignore=E501 + filter_file(r'(LA::MPI::PreconditionAMG preconditioner;)', + (''), 'step-40.cc') + filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', + (''), 'step-40.cc') + filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', + (''), 'step-40.cc') + filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') # NOQA: ignore=E501 + filter_file(r'(preconditioner\);)', (''), 'step-40.cc') + if '^trilinos+superlu-dist' in spec: + make('release') + make('run', paralle=False) - print('=====================================') - print('=== Step-40 Trilinos SuperluDist ====') - print('=====================================') - # change to direct solvers - filter_file(r'(LA::SolverCG solver\(solver_control\);)', ('TrilinosWrappers::SolverDirect::AdditionalData data(false,"Amesos_Superludist"); TrilinosWrappers::SolverDirect solver(solver_control,data);'), 'step-40.cc') # NOQA: ignore=E501 - filter_file(r'(LA::MPI::PreconditionAMG preconditioner;)', - (''), 'step-40.cc') - filter_file(r'(LA::MPI::PreconditionAMG::AdditionalData data;)', - (''), 'step-40.cc') - filter_file(r'(preconditioner.initialize\(system_matrix, data\);)', - (''), 'step-40.cc') - filter_file(r'(solver\.solve \(system_matrix, completely_distributed_solution, system_rhs,)', ('solver.solve (system_matrix, completely_distributed_solution, system_rhs);'), 'step-40.cc') # NOQA: ignore=E501 - filter_file(r'(preconditioner\);)', (''), 'step-40.cc') - if '^trilinos+superlu-dist' in spec: - make('release') - make('run', paralle=False) - - print('=====================================') - print('====== Step-40 Trilinos MUMPS =======') - print('=====================================') - # switch to Mumps - filter_file(r'(Amesos_Superludist)', - ('Amesos_Mumps'), 'step-40.cc') - if '^trilinos+mumps' in spec: - make('release') - make('run', parallel=False) + print('=====================================') + print('====== Step-40 Trilinos MUMPS =======') + print('=====================================') + # switch to Mumps + filter_file(r'(Amesos_Superludist)', + ('Amesos_Mumps'), 'step-40.cc') + if '^trilinos+mumps' in spec: + make('release') + make('run', parallel=False) print('=====================================') print('============ Step-36 ================') From 4f591a6fe3f2fc1477d5b740d1eb9e56a37d56b3 Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Mon, 30 May 2016 16:42:24 +0200 Subject: [PATCH 302/375] Step-40 test for 8.3.0 and earlier Integrated part of step-40 to be tested for earlier versions of dealii --- .../repos/builtin/packages/dealii/package.py | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index c8710590756..e83347409ee 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -228,29 +228,33 @@ def install(self, spec, prefix): # take step-40 which can use both PETSc and Trilinos # FIXME: switch step-40 to MPI run - if spec.satisfies('@8.4.0:'): - with working_dir('examples/step-40'): - print('=====================================') - print('========== Step-40 PETSc ============') - print('=====================================') - # list the number of cycles to speed up - filter_file(r'(const unsigned int n_cycles = 8;)', - ('const unsigned int n_cycles = 2;'), 'step-40.cc') - cmake('.') - if '^petsc' in spec: - make('release') - make('run', parallel=False) - - print('=====================================') - print('========= Step-40 Trilinos ==========') - print('=====================================') - # change Linear Algebra to Trilinos + with working_dir('examples/step-40'): + print('=====================================') + print('========== Step-40 PETSc ============') + print('=====================================') + # list the number of cycles to speed up + filter_file(r'(const unsigned int n_cycles = 8;)', + ('const unsigned int n_cycles = 2;'), 'step-40.cc') + cmake('.') + if '^petsc' in spec: + make('release') + make('run', parallel=False) + + print('=====================================') + print('========= Step-40 Trilinos ==========') + print('=====================================') + # change Linear Algebra to Trilinos + if spec.satisfies('@8.4.0:'): filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') - if '^trilinos+hypre' in spec: - make('release') - make('run', parallel=False) + else: + filter_file(r'(#define USE_PETSC_LA.*)', + ('// #define USE_PETSC_LA'), 'step-40.cc') + if '^trilinos+hypre' in spec: + make('release') + make('run', parallel=False) + if spec.satisfies('@8.4.0:'): print('=====================================') print('=== Step-40 Trilinos SuperluDist ====') print('=====================================') From c4d5bd13c7596577bef119dcac2b83e213a92ce5 Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Mon, 30 May 2016 18:11:34 +0200 Subject: [PATCH 303/375] dealii, Added comment Added comment for why step-40 is done differently for dealii 8.4.0 and later --- var/spack/repos/builtin/packages/dealii/package.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index e83347409ee..2b148da0743 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -244,6 +244,8 @@ def install(self, spec, prefix): print('========= Step-40 Trilinos ==========') print('=====================================') # change Linear Algebra to Trilinos + # The below filter_file should be different for versions + # before and after 8.4.0 if spec.satisfies('@8.4.0:'): filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') @@ -254,6 +256,8 @@ def install(self, spec, prefix): make('release') make('run', parallel=False) + # the rest of the tests on step 40 only works for + # dealii version 8.4.0 and after if spec.satisfies('@8.4.0:'): print('=====================================') print('=== Step-40 Trilinos SuperluDist ====') From f96c97902452761cbf2bff64d3efa1240df70771 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:05:49 -0700 Subject: [PATCH 304/375] Added a conditional to check if package is external --- lib/spack/spack/concretize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 5d9715feed4..58811c3419c 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -84,7 +84,8 @@ def _valid_virtuals_and_externals(self, spec): raise NoBuildError(spec) def cmp_externals(a, b): - if a.name != b.name: + if a.name != b.name and (not a.external or a.external_module and + not b.external and b.external_module): # We're choosing between different providers, so # maintain order from provider sort return candidates.index(a) - candidates.index(b) From 08a9d80ebcd239790d28904787ea5f6a9bf5fa5d Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:07:20 -0700 Subject: [PATCH 305/375] Changed platform to py_platform to avoid name collisions. The naming collisions resulted in an error on linux machines --- lib/spack/spack/test/architecture.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index f5b10684355..3d08269deef 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -3,7 +3,7 @@ """ import unittest import os -import platform +import platform as py_platform import spack from spack.architecture import * from spack.spec import * @@ -61,9 +61,9 @@ def test_sys_type(self): my_platform_class = CrayXc() elif os.path.exists('/bgsys'): my_platform_class = Bgq() - elif 'Linux' in platform.system(): + elif 'Linux' in py_platform.system(): my_platform_class = Linux() - elif 'Darwin' in platform.system(): + elif 'Darwin' in py_platform.system(): my_platform_class = Darwin() self.assertEqual(str(output_platform_class), str(my_platform_class)) From 716991775d6f6541e2acf41d51ff528e4c3412c7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:09:41 -0700 Subject: [PATCH 306/375] Added extra check on linux machines, since most linux machines don't have module system --- lib/spack/spack/test/concretize.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index cf2daa4be2a..67b20b4084c 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -253,8 +253,9 @@ def test_external_package(self): self.assertFalse('externalprereq' in spec) self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) - @unittest.skipIf(spack.architecture.sys_type().name == 'darwin', - "No tcl modules on darwin machines") + @unittest.skipIf(spack.architecture.sys_type().name == 'darwin' or + spack.architecture.sys_type().name == 'linux', + "No tcl modules on darwin/linux machines") def test_external_package_module(self): spec = Spec('externalmodule') spec.concretize() From b14ba3125033dc93afb9662ba87c2ca0acbbbde4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 30 May 2016 13:22:08 -0700 Subject: [PATCH 307/375] Quick fix for mock compilers.yaml to work on linux machines. --- lib/spack/spack/test/mock_packages_test.py | 35 +++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index f02bfb7a079..561f319b16a 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -34,8 +34,41 @@ from spack.repository import RepoPath from spack.spec import Spec +platform = spack.architecture.sys_type() + +linux_os_name = 'debian' +linux_os_version = '6' + +if platform.name == 'linux': + global linux_os_name, linux_os_version + linux_os = platform.operating_system("default_os") + linux_os_name = linux_os.name + linux_os_version = linux_os.version + mock_compiler_config = """\ compilers: + clang3.3GENLINUX: + spec: clang@3.3 + operating_system: + name: {0} + version: '{1}' + paths: + cc: /path/to/clang + cxx: /path/to/clang++ + f77: None + fc: None + modules: 'None' + gcc4.5GENLINUX: + spec: gcc@4.5.0 + operating_system: + name: {0} + version: '{1}' + paths: + cc: /path/to/gcc + cxx: /path/to/g++ + f77: None + fc: None + modules: 'None' clang3.3CNL: spec: clang@3.3 operating_system: @@ -146,7 +179,7 @@ f77: None fc: None modules: 'None' -""" +""".format(linux_os_name, linux_os_version) mock_packages_config = """\ packages: From 24ee32d7b0cb209f8a17f6df7e8715fa508d0f5d Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 30 May 2016 22:02:22 -0700 Subject: [PATCH 308/375] More flexible reading of specs from DB, formatting. --- lib/spack/spack/database.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e768ddf5feb..38bb7541e00 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -214,9 +214,10 @@ def _read_spec_from_yaml(self, hash_key, installs, parent_key=None): # Add dependencies from other records in the install DB to # form a full spec. - for dep_hash in spec_dict[spec.name]['dependencies'].values(): - child = self._read_spec_from_yaml(dep_hash, installs, hash_key) - spec._add_dependency(child) + if 'dependencies' in spec_dict[spec.name]: + for dep_hash in spec_dict[spec.name]['dependencies'].values(): + child = self._read_spec_from_yaml(dep_hash, installs, hash_key) + spec._add_dependency(child) # Specs from the database need to be marked concrete because # they represent actual installations. @@ -289,7 +290,8 @@ def check(cond, msg): except Exception as e: tty.warn("Invalid database reecord:", "file: %s" % self._index_path, - "hash: %s" % hash_key, "cause: %s" % str(e)) + "hash: %s" % hash_key, + "cause: %s: %s" % (type(e).__name__, str(e))) raise self._data = data From e82fb5b58f4ccdf7a4bfcda0331b4649dec3de75 Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Tue, 31 May 2016 08:44:46 +0200 Subject: [PATCH 309/375] Fixed spacing Replaced tabs with spaces --- var/spack/repos/builtin/packages/dealii/package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 2b148da0743..35658c75a0f 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -244,8 +244,8 @@ def install(self, spec, prefix): print('========= Step-40 Trilinos ==========') print('=====================================') # change Linear Algebra to Trilinos - # The below filter_file should be different for versions - # before and after 8.4.0 + # The below filter_file should be different for versions + # before and after 8.4.0 if spec.satisfies('@8.4.0:'): filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') @@ -256,8 +256,8 @@ def install(self, spec, prefix): make('release') make('run', parallel=False) - # the rest of the tests on step 40 only works for - # dealii version 8.4.0 and after + # the rest of the tests on step 40 only works for + # dealii version 8.4.0 and after if spec.satisfies('@8.4.0:'): print('=====================================') print('=== Step-40 Trilinos SuperluDist ====') From 6f96484f83dff008ef8712c63202ee140000ff22 Mon Sep 17 00:00:00 2001 From: Ali Dorostkar Date: Tue, 31 May 2016 09:58:29 +0200 Subject: [PATCH 310/375] Spacing correction More spacing correction --- var/spack/repos/builtin/packages/dealii/package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/dealii/package.py b/var/spack/repos/builtin/packages/dealii/package.py index 35658c75a0f..54b6426d363 100644 --- a/var/spack/repos/builtin/packages/dealii/package.py +++ b/var/spack/repos/builtin/packages/dealii/package.py @@ -244,8 +244,8 @@ def install(self, spec, prefix): print('========= Step-40 Trilinos ==========') print('=====================================') # change Linear Algebra to Trilinos - # The below filter_file should be different for versions - # before and after 8.4.0 + # The below filter_file should be different for versions + # before and after 8.4.0 if spec.satisfies('@8.4.0:'): filter_file(r'(\/\/ #define FORCE_USE_OF_TRILINOS.*)', ('#define FORCE_USE_OF_TRILINOS'), 'step-40.cc') @@ -256,8 +256,8 @@ def install(self, spec, prefix): make('release') make('run', parallel=False) - # the rest of the tests on step 40 only works for - # dealii version 8.4.0 and after + # the rest of the tests on step 40 only works for + # dealii version 8.4.0 and after if spec.satisfies('@8.4.0:'): print('=====================================') print('=== Step-40 Trilinos SuperluDist ====') From 7bdf63a0fa87cce6f9237ffae4a8903311327d54 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 31 May 2016 03:01:05 -0700 Subject: [PATCH 311/375] Fix some bugs in architecture. --- lib/spack/spack/architecture.py | 14 ++++++-------- lib/spack/spack/operating_systems/linux_distro.py | 13 ++++++++++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a0ef4f14da5..88cebc09432 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -487,25 +487,23 @@ def arch_from_dict(d): @memoized def all_platforms(): - modules = [] - + classes = [] mod_path = spack.platform_path - mod_string = "spack.platformss" + parent_module = "spack.platforms" for name in list_modules(mod_path): - mod_name = mod_string + name - path = join_path(mod_path, name) + ".py" - mod = imp.load_source(mod_name, path) + mod_name = '%s.%s' % (parent_module, name) class_name = mod_to_class(name) + mod = __import__(mod_name, fromlist=[class_name]) if not hasattr(mod, class_name): tty.die('No class %s defined in %s' % (class_name, mod_name)) cls = getattr(mod, class_name) if not inspect.isclass(cls): tty.die('%s.%s is not a class' % (mod_name, class_name)) - modules.append(cls) + classes.append(cls) - return modules + return classes @memoized def sys_type(): diff --git a/lib/spack/spack/operating_systems/linux_distro.py b/lib/spack/spack/operating_systems/linux_distro.py index d0f24d842f9..2e3c72719b2 100644 --- a/lib/spack/spack/operating_systems/linux_distro.py +++ b/lib/spack/spack/operating_systems/linux_distro.py @@ -1,3 +1,4 @@ +import re import platform as py_platform from spack.architecture import OperatingSystem @@ -9,7 +10,13 @@ class LinuxDistro(OperatingSystem): platform.dist() """ def __init__(self): - name = py_platform.dist()[0] - version = py_platform.dist()[1].split(".")[0] # Grabs major version from tuple + distname, version, _ = py_platform.linux_distribution( + full_distribution_name=False) - super(LinuxDistro, self).__init__(name, version) + # Grabs major version from tuple on redhat; on other platforms + # grab the first legal identifier in the version field. On + # debian you get things like 'wheezy/sid'; sid means unstable. + # We just record 'wheezy' and don't get quite so detailed. + version = re.split(r'[^\w-]', version)[0] + + super(LinuxDistro, self).__init__(distname, version) From 564e7f5e5bba55d3f4f96916c054d3c3d01a35cf Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Tue, 31 May 2016 09:32:24 -0500 Subject: [PATCH 312/375] Don't build in parallel --- var/spack/repos/builtin/packages/hpl/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/var/spack/repos/builtin/packages/hpl/package.py b/var/spack/repos/builtin/packages/hpl/package.py index 3d6e9963235..efd5c8bb1da 100644 --- a/var/spack/repos/builtin/packages/hpl/package.py +++ b/var/spack/repos/builtin/packages/hpl/package.py @@ -43,6 +43,8 @@ class Hpl(Package): depends_on('mpi@1.1:') depends_on('blas') + parallel = False + def configure(self, spec, arch): # List of configuration options # Order is important From c03c689cb225a3191a16d65d57ad69a70914f238 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 31 May 2016 12:18:17 -0400 Subject: [PATCH 313/375] Update HDF5 to version 1.10.0-patch1 --- var/spack/repos/builtin/packages/hdf5/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/hdf5/package.py b/var/spack/repos/builtin/packages/hdf5/package.py index 21137ef356b..e46f432be50 100644 --- a/var/spack/repos/builtin/packages/hdf5/package.py +++ b/var/spack/repos/builtin/packages/hdf5/package.py @@ -38,6 +38,7 @@ class Hdf5(Package): list_url = "http://www.hdfgroup.org/ftp/HDF5/releases" list_depth = 3 + version('1.10.0-patch1', '9180ff0ef8dc2ef3f61bd37a7404f295') version('1.10.0', 'bdc935337ee8282579cd6bc4270ad199') version('1.8.16', 'b8ed9a36ae142317f88b0c7ef4b9c618', preferred=True) version('1.8.15', '03cccb5b33dbe975fdcd8ae9dc021f24') From 817f43f92766bb37877888043eba6cecd9491f2e Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Tue, 31 May 2016 13:39:25 -0700 Subject: [PATCH 314/375] Fixed config format to allow default compiler flags --- lib/spack/spack/config.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index ec37bd290c9..88544aa7bbc 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -167,6 +167,18 @@ {'type' : 'null' }]}, 'fc': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}, + 'fflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cppflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'cxxflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldflags': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, + 'ldlibs': { 'anyOf': [ {'type' : 'string' }, + {'type' : 'null' }]}, },},},},},},},}, 'mirrors': { From cb673765c27049c965e76935e193f86660ffaf02 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 03:02:48 +0200 Subject: [PATCH 315/375] Adding a version to tetgen --- var/spack/repos/builtin/packages/tetgen/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/tetgen/package.py b/var/spack/repos/builtin/packages/tetgen/package.py index 5e87ed7fbab..1c536197ce5 100644 --- a/var/spack/repos/builtin/packages/tetgen/package.py +++ b/var/spack/repos/builtin/packages/tetgen/package.py @@ -34,6 +34,7 @@ class Tetgen(Package): url = "http://www.tetgen.org/files/tetgen1.4.3.tar.gz" version('1.4.3', 'd6a4bcdde2ac804f7ec66c29dcb63c18') + version('1.5.0', '3b9fd9cdec121e52527b0308f7aad5c1', url='http://www.tetgen.org/1.5/src/tetgen1.5.0.tar.gz') # TODO: Make this a build dependency once build dependencies are supported # (see: https://github.com/LLNL/spack/pull/378). From 58052e6632a91a5d4549ba811ae0405942ccc0b3 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Tue, 31 May 2016 22:13:11 +0200 Subject: [PATCH 316/375] Adding package fenics --- .../repos/builtin/packages/fenics/package.py | 175 ++++++++++++++++++ .../repos/builtin/packages/py-ply/package.py | 38 ++++ 2 files changed, 213 insertions(+) create mode 100644 var/spack/repos/builtin/packages/fenics/package.py create mode 100644 var/spack/repos/builtin/packages/py-ply/package.py diff --git a/var/spack/repos/builtin/packages/fenics/package.py b/var/spack/repos/builtin/packages/fenics/package.py new file mode 100644 index 00000000000..72282994f56 --- /dev/null +++ b/var/spack/repos/builtin/packages/fenics/package.py @@ -0,0 +1,175 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class Fenics(Package): + """FEniCS is organized as a collection of interoperable components + that together form the FEniCS Project. These components include + the problem-solving environment DOLFIN, the form compiler FFC, the + finite element tabulator FIAT, the just-in-time compiler Instant, + the code generation interface UFC, the form language UFL and a + range of additional components.""" + + homepage = "http://fenicsproject.org/" + url = "https://bitbucket.org/fenics-project/dolfin/downloads/dolfin-1.6.0.tar.gz" + + base_url = "https://bitbucket.org/fenics-project/{pkg}/downloads/{pkg}-{version}.tar.gz" # NOQA: ignore E501 + + variant('hdf5', default=True, description='Compile with HDF5') + variant('parmetis', default=True, description='Compile with ParMETIS') + variant('scotch', default=True, description='Compile with Scotch') + variant('petsc', default=True, description='Compile with PETSc') + variant('slepc', default=True, description='Compile with SLEPc') + variant('trilinos', default=True, description='Compile with Trilinos') + variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') + variant('vtk', default=False, description='Compile with VTK') + variant('qt', default=False, description='Compile with QT') + variant('mpi', default=True, description='Enables the distributed memory support') + variant('openmp', default=True, description='Enables the shared memory support') + variant('shared', default=True, description='Enables the build of shared libraries') + variant('debug', default=False, description='Builds a debug version of the libraries') + + # not part of spack list for now + # variant('petsc4py', default=True, description='Uses PETSc4py') + # variant('slepc4py', default=True, description='Uses SLEPc4py') + # variant('pastix', default=True, description='Compile with Pastix') + + extends('python') + + depends_on('py-numpy') + depends_on('py-ply') + depends_on('py-six') + depends_on('py-sphinx@1.0.1:', when='+doc') + depends_on('eigen@3.2.0:') + depends_on('boost') + depends_on('mpi', when='+mpi') + depends_on('hdf5', when='+hdf5') + depends_on('parmetis@4.0.2:^metis+real64', when='+parmetis') + depends_on('scotch~metis', when='+scotch~mpi') + depends_on('scotch+mpi~metis', when='+scotch+mpi') + depends_on('petsc@3.4:', when='+petsc') + depends_on('slepc@3.4:', when='+slepc') + depends_on('trilinos', when='+trilinos') + depends_on('vtk', when='+vtk') + depends_on('suite-sparse', when='+suite-sparse') + depends_on('qt', when='+qt') + + # This are the build dependencies + depends_on('py-setuptools') + depends_on('cmake@2.8.12:') + depends_on('swig@3.0.3:') + + releases = [ + { + 'version': '1.6.0', + 'md5': '35cb4baf7ab4152a40fb7310b34d5800', + 'resources': { + 'ffc': '358faa3e9da62a1b1a717070217b793e', + 'fiat': 'f4509d05c911fd93cea8d288a78a6c6f', + 'instant': '5f2522eb032a5bebbad6597b6fe0732a', + 'ufl': 'c40c5f04eaa847377ab2323122284016', + } + }, + { + 'version': '1.5.0', + 'md5': '9b589a3534299a5e6d22c13c5eb30bb8', + 'resources': { + 'ffc': '343f6d30e7e77d329a400fd8e73e0b63', + 'fiat': 'da3fa4dd8177bb251e7f68ec9c7cf6c5', + 'instant': 'b744023ded27ee9df4a8d8c6698c0d58', + 'ufl': '130d7829cf5a4bd5b52bf6d0955116fd', + } + }, + ] + + for release in releases: + version(release['version'], release['md5'], url=base_url.format(pkg='dolfin', version=release['version'])) + for name, md5 in release['resources'].items(): + resource(name=name, + url=base_url.format(pkg=name, **release), + md5=md5, + destination='depends', + when='@{version}'.format(**release), + placement=name) + + def cmake_is_on(self, option): + return 'ON' if option in self.spec else 'OFF' + + def install(self, spec, prefix): + for package in ['ufl', 'ffc', 'fiat', 'instant']: + with working_dir(join_path('depends', package)): + python('setup.py', 'install', '--prefix=%s' % prefix) + + cmake_args.extend([ + '-DCMAKE_BUILD_TYPE:STRING={0}'.format( + 'Debug' if '+debug' in spec else 'RelWithDebInfo'), + '-DBUILD_SHARED_LIBS:BOOL={0}'.format( + self.cmake_is_on('+shared')), + '-DDOLFIN_SKIP_BUILD_TESTS:BOOL=ON', + '-DDOLFIN_ENABLE_OPENMP:BOOL={0}'.format( + self.cmake_is_on('+openmp')), + '-DDOLFIN_ENABLE_CHOLMOD:BOOL={0}'.format( + self.cmake_is_on('suite-sparse')), + '-DDOLFIN_ENABLE_HDF5:BOOL={0}'.format( + self.cmake_is_on('hdf5')), + '-DDOLFIN_ENABLE_MPI:BOOL={0}'.format( + self.cmake_is_on('mpi')), + '-DDOLFIN_ENABLE_PARMETIS:BOOL={0}'.format( + self.cmake_is_on('parmetis')), + '-DDOLFIN_ENABLE_PASTIX:BOOL={0}'.format( + self.cmake_is_on('pastix')), + '-DDOLFIN_ENABLE_PETSC:BOOL={0}'.format( + self.cmake_is_on('petsc')), + '-DDOLFIN_ENABLE_PETSC4PY:BOOL={0}'.format( + self.cmake_is_on('py-petsc4py')), + '-DDOLFIN_ENABLE_PYTHON:BOOL={0}'.format( + self.cmake_is_on('python')), + '-DDOLFIN_ENABLE_QT:BOOL={0}'.format( + self.cmake_is_on('qt')), + '-DDOLFIN_ENABLE_SCOTCH:BOOL={0}'.format( + self.cmake_is_on('scotch')), + '-DDOLFIN_ENABLE_SLEPC:BOOL={0}'.format( + self.cmake_is_on('slepc')), + '-DDOLFIN_ENABLE_SLEPC4PY:BOOL={0}'.format( + self.cmake_is_on('py-slepc4py')), + '-DDOLFIN_ENABLE_SPHINX:BOOL={0}'.format( + self.cmake_is_on('py-sphinx')), + '-DDOLFIN_ENABLE_TRILINOS:BOOL={0}'.format( + self.cmake_is_on('trilinos')), + '-DDOLFIN_ENABLE_UMFPACK:BOOL={0}'.format( + self.cmake_is_on('suite-sparse')), + '-DDOLFIN_ENABLE_VTK:BOOL={0}'.format( + self.cmake_is_on('vtk')), + '-DDOLFIN_ENABLE_ZLIB:BOOL={0}'.format( + self.cmake_is_on('zlib')), + ]) + cmake_args.extend(std_cmake_args) + + with working_dir('build', create=True): + cmake('..', *cmake_args) + + make() + make('install') diff --git a/var/spack/repos/builtin/packages/py-ply/package.py b/var/spack/repos/builtin/packages/py-ply/package.py new file mode 100644 index 00000000000..47cd3b5dc84 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-ply/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyPly(Package): + """PLY is nothing more than a straightforward lex/yacc implementation.""" + homepage = "http://www.dabeaz.com/ply" + url = "http://www.dabeaz.com/ply/ply-3.8.tar.gz" + + version('3.8', '94726411496c52c87c2b9429b12d5c50') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix=%s' % prefix) From 97bad2c1ac33139f35094727a269001e4815c227 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 03:25:37 +0200 Subject: [PATCH 317/375] Corrections for flake8 --- var/spack/repos/builtin/packages/tetgen/package.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/tetgen/package.py b/var/spack/repos/builtin/packages/tetgen/package.py index 1c536197ce5..c301a5b4e5b 100644 --- a/var/spack/repos/builtin/packages/tetgen/package.py +++ b/var/spack/repos/builtin/packages/tetgen/package.py @@ -24,11 +24,13 @@ ############################################################################## from spack import * + class Tetgen(Package): - """TetGen is a program and library that can be used to generate tetrahedral - meshes for given 3D polyhedral domains. TetGen generates exact constrained - Delaunay tetrahedralizations, boundary conforming Delaunay meshes, and - Voronoi paritions.""" + """TetGen is a program and library that can be used to generate + tetrahedral meshes for given 3D polyhedral domains. TetGen + generates exact constrained Delaunay tetrahedralizations, + boundary conforming Delaunay meshes, and Voronoi paritions. + """ homepage = "http://www.tetgen.org" url = "http://www.tetgen.org/files/tetgen1.4.3.tar.gz" From d802144f39366a2cbd67554de5a099ed8ce80598 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 03:31:29 +0200 Subject: [PATCH 318/375] Introduced a bug by correcting flake8 warnings --- var/spack/repos/builtin/packages/fenics/package.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/fenics/package.py b/var/spack/repos/builtin/packages/fenics/package.py index 72282994f56..08452376567 100644 --- a/var/spack/repos/builtin/packages/fenics/package.py +++ b/var/spack/repos/builtin/packages/fenics/package.py @@ -123,7 +123,7 @@ def install(self, spec, prefix): with working_dir(join_path('depends', package)): python('setup.py', 'install', '--prefix=%s' % prefix) - cmake_args.extend([ + cmake_args = [ '-DCMAKE_BUILD_TYPE:STRING={0}'.format( 'Debug' if '+debug' in spec else 'RelWithDebInfo'), '-DBUILD_SHARED_LIBS:BOOL={0}'.format( @@ -165,7 +165,8 @@ def install(self, spec, prefix): self.cmake_is_on('vtk')), '-DDOLFIN_ENABLE_ZLIB:BOOL={0}'.format( self.cmake_is_on('zlib')), - ]) + ] + cmake_args.extend(std_cmake_args) with working_dir('build', create=True): From 0451c8e55f35c58adef120a986e8782abb3155eb Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 15:58:00 +0200 Subject: [PATCH 319/375] Addapting the mvapich2@2.1: configure options for slurm --- .../builtin/packages/mvapich2/package.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index f4997bdfa19..e871d0cd2b8 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -124,10 +124,19 @@ def set_process_manager(self, spec, configure_args): raise RuntimeError(" %s : 'slurm' cannot be activated together with other process managers" % self.name) process_manager_options = [] + # See: http://slurm.schedmd.com/mpi_guide.html#mvapich2 if self.enabled(Mvapich2.SLURM) in spec: - process_manager_options = [ - "--with-pm=slurm" - ] + if self.version > Version('2.0'): + process_manager_options = [ + "--with-pmi=pmi2", + "--with-pm=slurm" + ] + else: + process_manager_options = [ + "--with-pmi=slurm", + "--with-pm=no" + ] + elif has_slurm_incompatible_variants: pms = [] # The variant name is equal to the process manager name in the configuration options @@ -164,6 +173,11 @@ def set_network_type(self, spec, configure_args): configure_args.extend(network_options) + def setup_environment(self, spack_env, run_env): + if self.enabled(Mvapich2.SLURM) in self.spec and \ + self.version > Version('2.0'): + run_env.set('SLURM_MPI_TYPE', 'pmi2') + def setup_dependent_environment(self, spack_env, run_env, extension_spec): spack_env.set('MPICH_CC', spack_cc) spack_env.set('MPICH_CXX', spack_cxx) @@ -176,7 +190,7 @@ def setup_dependent_package(self, module, dep_spec): self.spec.mpicxx = join_path(self.prefix.bin, 'mpicxx') self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') - + def install(self, spec, prefix): # we'll set different configure flags depending on our environment configure_args = [ From 3ab8468e732a9ca68e4a846e1ae6496757d565f2 Mon Sep 17 00:00:00 2001 From: Nicolas Richart Date: Wed, 1 Jun 2016 16:43:08 +0200 Subject: [PATCH 320/375] Correcting the flake8 warnings --- .../builtin/packages/mvapich2/package.py | 72 ++++++++++++------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/var/spack/repos/builtin/packages/mvapich2/package.py b/var/spack/repos/builtin/packages/mvapich2/package.py index e871d0cd2b8..d1944023d17 100644 --- a/var/spack/repos/builtin/packages/mvapich2/package.py +++ b/var/spack/repos/builtin/packages/mvapich2/package.py @@ -25,6 +25,7 @@ from spack import * import os + class Mvapich2(Package): """MVAPICH2 is an MPI implementation for Infiniband networks.""" homepage = "http://mvapich.cse.ohio-state.edu/" @@ -43,8 +44,9 @@ class Mvapich2(Package): variant('debug', default=False, description='Enables debug information and error messages at run-time') ########## - # TODO : Process managers should be grouped into the same variant, as soon as variant capabilities will be extended - # See https://groups.google.com/forum/#!topic/spack/F8-f8B4_0so + # TODO : Process managers should be grouped into the same variant, + # as soon as variant capabilities will be extended See + # https://groups.google.com/forum/#!topic/spack/F8-f8B4_0so SLURM = 'slurm' HYDRA = 'hydra' GFORKER = 'gforker' @@ -57,7 +59,8 @@ class Mvapich2(Package): ########## ########## - # TODO : Network types should be grouped into the same variant, as soon as variant capabilities will be extended + # TODO : Network types should be grouped into the same variant, as + # soon as variant capabilities will be extended PSM = 'psm' SOCK = 'sock' NEMESISIBTCP = 'nemesisibtcp' @@ -84,8 +87,8 @@ def url_for_version(self, version): @staticmethod def enabled(x): - """ - Given a variant name returns the string that means the variant is enabled + """Given a variant name returns the string that means the variant is + enabled :param x: variant name :return: @@ -93,8 +96,8 @@ def enabled(x): return '+' + x def set_build_type(self, spec, configure_args): - """ - Appends to configure_args the flags that depends only on the build type (i.e. release or debug) + """Appends to configure_args the flags that depends only on the build + type (i.e. release or debug) :param spec: spec :param configure_args: list of current configure arguments @@ -104,7 +107,8 @@ def set_build_type(self, spec, configure_args): "--disable-fast", "--enable-error-checking=runtime", "--enable-error-messages=all", - "--enable-g=dbg", "--enable-debuginfo" # Permits debugging with TotalView + # Permits debugging with TotalView + "--enable-g=dbg", "--enable-debuginfo" ] else: build_type_options = ["--enable-fast=all"] @@ -112,16 +116,22 @@ def set_build_type(self, spec, configure_args): configure_args.extend(build_type_options) def set_process_manager(self, spec, configure_args): - """ - Appends to configure_args the flags that will enable the appropriate process managers + """Appends to configure_args the flags that will enable the + appropriate process managers :param spec: spec :param configure_args: list of current configure arguments """ - # Check that slurm variant is not activated together with other pm variants - has_slurm_incompatible_variants = any(self.enabled(x) in spec for x in Mvapich2.SLURM_INCOMPATIBLE_PMS) - if self.enabled(Mvapich2.SLURM) in spec and has_slurm_incompatible_variants: - raise RuntimeError(" %s : 'slurm' cannot be activated together with other process managers" % self.name) + # Check that slurm variant is not activated together with + # other pm variants + has_slurm_incompatible_variants = \ + any(self.enabled(x) in spec + for x in Mvapich2.SLURM_INCOMPATIBLE_PMS) + + if self.enabled(Mvapich2.SLURM) in spec and \ + has_slurm_incompatible_variants: + raise RuntimeError(" %s : 'slurm' cannot be activated \ + together with other process managers" % self.name) process_manager_options = [] # See: http://slurm.schedmd.com/mpi_guide.html#mvapich2 @@ -139,7 +149,8 @@ def set_process_manager(self, spec, configure_args): elif has_slurm_incompatible_variants: pms = [] - # The variant name is equal to the process manager name in the configuration options + # The variant name is equal to the process manager name in + # the configuration options for x in Mvapich2.SLURM_INCOMPATIBLE_PMS: if self.enabled(x) in spec: pms.append(x) @@ -155,7 +166,9 @@ def set_network_type(self, spec, configure_args): if self.enabled(x) in spec: count += 1 if count > 1: - raise RuntimeError('network variants are mutually exclusive (only one can be selected at a time)') + raise RuntimeError('network variants are mutually exclusive \ + (only one can be selected at a time)') + network_options = [] # From here on I can suppose that only one variant has been selected if self.enabled(Mvapich2.PSM) in spec: @@ -177,7 +190,7 @@ def setup_environment(self, spack_env, run_env): if self.enabled(Mvapich2.SLURM) in self.spec and \ self.version > Version('2.0'): run_env.set('SLURM_MPI_TYPE', 'pmi2') - + def setup_dependent_environment(self, spack_env, run_env, extension_spec): spack_env.set('MPICH_CC', spack_cc) spack_env.set('MPICH_CXX', spack_cxx) @@ -190,9 +203,10 @@ def setup_dependent_package(self, module, dep_spec): self.spec.mpicxx = join_path(self.prefix.bin, 'mpicxx') self.spec.mpifc = join_path(self.prefix.bin, 'mpif90') self.spec.mpif77 = join_path(self.prefix.bin, 'mpif77') - + def install(self, spec, prefix): - # we'll set different configure flags depending on our environment + # we'll set different configure flags depending on our + # environment configure_args = [ "--prefix=%s" % prefix, "--enable-shared", @@ -222,7 +236,6 @@ def install(self, spec, prefix): self.filter_compilers() - def filter_compilers(self): """Run after install to make the MPI compilers use the compilers that Spack built the package with. @@ -242,8 +255,17 @@ def filter_compilers(self): spack_f77 = os.environ['F77'] spack_fc = os.environ['FC'] - kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True } - filter_file('CC="%s"' % spack_cc , 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) - filter_file('CXX="%s"'% spack_cxx, 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) - filter_file('F77="%s"'% spack_f77, 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) - filter_file('FC="%s"' % spack_fc , 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) + kwargs = { + 'ignore_absent': True, + 'backup': False, + 'string': True + } + + filter_file('CC="%s"' % spack_cc, + 'CC="%s"' % self.compiler.cc, mpicc, **kwargs) + filter_file('CXX="%s"' % spack_cxx, + 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs) + filter_file('F77="%s"' % spack_f77, + 'F77="%s"' % self.compiler.f77, mpif77, **kwargs) + filter_file('FC="%s"' % spack_fc, + 'FC="%s"' % self.compiler.fc, mpif90, **kwargs) From 9d04dccb79c606b3d3ca033b5365d31d9a1b0442 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 2 Jun 2016 01:20:40 -0700 Subject: [PATCH 321/375] Update version to correspond to master. --- lib/spack/spack/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index c7a155597fe..8c6e0ba5277 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -105,7 +105,7 @@ # Version information from spack.version import Version -spack_version = Version("0.9") +spack_version = Version("0.9.1") # # Executables used by Spack From 24d160e93ed80052ad30f99ce36b34de2872a888 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 3 Jun 2016 15:57:54 -0700 Subject: [PATCH 322/375] Changed how architecture looks like in spec.yaml. Now it's only three strings, platform, os, target in spec.yaml. Also did some flake8 formatting changes --- lib/spack/spack/architecture.py | 210 ++++++++++++++++---------------- 1 file changed, 102 insertions(+), 108 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index a0ef4f14da5..ee0d2173c77 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -29,8 +29,9 @@ On a multiple architecture machine, the architecture spec field can be set to build a package against any target and operating system that is present on the -platform. On Cray platforms or any other architecture that has different front and -back end environments, the operating system will determine the method of compiler +platform. On Cray platforms or any other architecture that has different front +and back end environments, the operating system will determine the method of +compiler detection. There are two different types of compiler detection: @@ -40,44 +41,42 @@ Depending on which operating system is specified, the compiler will be detected using one of those methods. -For platforms such as linux and darwin, the operating system is autodetected and -the target is set to be x86_64. +For platforms such as linux and darwin, the operating system is autodetected +and the target is set to be x86_64. The command line syntax for specifying an architecture is as follows: target= os= If the user wishes to use the defaults, either target or os can be left out of -the command line and Spack will concretize using the default. These defaults are -set in the 'platforms/' directory which contains the different subclasses for -platforms. If the machine has multiple architectures, the user can +the command line and Spack will concretize using the default. These defaults +are set in the 'platforms/' directory which contains the different subclasses +for platforms. If the machine has multiple architectures, the user can also enter front-end, or fe or back-end or be. These settings will concretize to their respective front-end and back-end targets and operating systems. Additional platforms can be added by creating a subclass of Platform and adding it inside the platform directory. Platforms are an abstract class that are extended by subclasses. If the user -wants to add a new type of platform (such as cray_xe), they can create a subclass -and set all the class attributes such as priority, front_target ,back_target, -front_os, back_os. Platforms also contain a priority class attribute. A lower -number signifies higher priority. These numbers are arbitrarily set and can be -changed though often there isn't much need unless a new platform is added and -the user wants that to be detected first. +wants to add a new type of platform (such as cray_xe), they can create a +subclass and set all the class attributes such as priority, front_target, +back_target, front_os, back_os. Platforms also contain a priority class +attribute. A lower number signifies higher priority. These numbers are +arbitrarily set and can be changed though often there isn't much need unless a +new platform is added and the user wants that to be detected first. -Targets are created inside the platform subclasses. Most architecture (like linux, -and darwin) will have only one target (x86_64) but in the case of Cray machines, -there is both a frontend and backend processor. The user can specify which targets -are present on front-end and back-end architecture +Targets are created inside the platform subclasses. Most architecture +(like linux, and darwin) will have only one target (x86_64) but in the case of +Cray machines, there is both a frontend and backend processor. The user can +specify which targets are present on front-end and back-end architecture Depending on the platform, operating systems are either auto-detected or are set. The user can set the front-end and back-end operating setting by the class -attributes front_os and back_os. The operating system as described earlier, will -be responsible for compiler detection. +attributes front_os and back_os. The operating system as described earlier, +will be responsible for compiler detection. """ import os -from collections import namedtuple import imp -import platform as py_platform import inspect from llnl.util.lang import memoized, list_modules, key_ordering @@ -91,6 +90,7 @@ from spack.util.multiproc import parmap import spack.error as serr + class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): super(InvalidSysTypeError, self).__init__( @@ -128,12 +128,6 @@ def __repr__(self): def __str__(self): return self.name - def to_dict(self): - d = {} - d['name'] = self.name - d['module_name'] = self.module_name - - return d @key_ordering class Platform(object): @@ -142,7 +136,7 @@ class Platform(object): is returned """ - priority = None # Subclass needs to set this number. This controls order in which platform is detected. + priority = None # Subclass sets number. Controls detection order front_end = None back_end = None default = None # The default back end target. On cray ivybridge @@ -156,19 +150,6 @@ def __init__(self, name): self.operating_sys = {} self.name = name - def to_dict(self): - n = {} - n['targets'] = dict((name, target.to_dict()) for (name, target) in self.targets.items()) - n['operating_systems'] = dict((name, os.to_dict()) for (name, os) in self.operating_sys.items()) - n['priority'] = self.priority - n['default_front_end_target'] = self.front_end - n['default_back_end_target'] = self.back_end - n['default_target'] = self.default - n['default_front_end_os'] = self.front_os - n['default_back_end_os'] = self.back_os - n['default_os'] = self.default_os - return {self.name: n} - def add_target(self, name, target): """Used by the platform specific subclass to list available targets. Raises an error if the platform specifies a name @@ -215,7 +196,6 @@ def operating_system(self, name): return self.operating_sys.get(name, None) - @classmethod def detect(self): """ Subclass is responsible for implementing this method. @@ -232,8 +212,10 @@ def __str__(self): return self.name def _cmp_key(self): - t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) - o_keys = ''.join(str(o._cmp_key()) for o in sorted(self.operating_sys.values())) + t_keys = ''.join(str(t._cmp_key()) for t in + sorted(self.targets.values())) + o_keys = ''.join(str(o._cmp_key()) for o in + sorted(self.operating_sys.values())) return (self.name, self.default, self.front_end, @@ -244,6 +226,7 @@ def _cmp_key(self): t_keys, o_keys) + @key_ordering class OperatingSystem(object): """ Operating System will be like a class similar to platform extended @@ -265,7 +248,6 @@ def __repr__(self): def _cmp_key(self): return (self.name, self.version) - def find_compilers(self, *paths): """ Return a list of compilers found in the suppied paths. @@ -293,11 +275,13 @@ def find_compilers(self, *paths): # compiler. We can spawn a bunch of parallel searches to reduce # the overhead of spelunking all these directories. types = spack.compilers.all_compiler_types() - compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *filtered_path), types) + compiler_lists = parmap(lambda cmp_cls: + self.find_compiler(cmp_cls, *filtered_path), + types) # ensure all the version calls we made are cached in the parent # process, as well. This speeds up Spack a lot. - clist = reduce(lambda x,y: x+y, compiler_lists) + clist = reduce(lambda x, y: x+y, compiler_lists) return clist def find_compiler(self, cmp_cls, *path): @@ -338,7 +322,7 @@ def find_compiler(self, cmp_cls, *path): # prefer the one with more compilers. prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] - newcount = len([p for p in paths if p is not None]) + newcount = len([p for p in paths if p is not None]) prevcount = len([p for p in prev_paths if p is not None]) # Don't add if it's not an improvement over prev compiler. @@ -349,14 +333,7 @@ def find_compiler(self, cmp_cls, *path): return list(compilers.values()) - def to_dict(self): - d = {} - d['name'] = self.name - d['version'] = self.version - return d - -#NOTE: Key error caused because Architecture has no comparison method @key_ordering class Arch(object): "Architecture is now a class to help with setting attributes" @@ -376,10 +353,11 @@ def __init__(self, platform=None, platform_os=None, target=None): @property def concrete(self): - return all( (self.platform is not None, isinstance(self.platform, Platform), - self.platform_os is not None, isinstance(self.platform_os, OperatingSystem), - self.target is not None, isinstance(self.target, Target) ) ) - + return all((self.platform is not None, + isinstance(self.platform, Platform), + self.platform_os is not None, + isinstance(self.platform_os, OperatingSystem), + self.target is not None, isinstance(self.target, Target))) def __str__(self): if self.platform or self.platform_os or self.target: @@ -388,71 +366,75 @@ def __str__(self): else: os_name = str(self.platform_os) - return (str(self.platform) +"-"+ + return (str(self.platform) + "-" + os_name + "-" + str(self.target)) else: return '' def _cmp_key(self): - platform = self.platform.name if isinstance(self.platform, Platform) else self.platform - os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os - target = self.target.name if isinstance(self.target, Target) else self.target - return (platform, os, target) + if isinstance(self.platform, Platform): + platform = self.platform.name + else: + platform = self.platform + if isinstance(self.platform_os, OperatingSystem): + platform_os = self.platform_os.name + else: + platform_os = self.platform_os + if isinstance(self.target, Target): + target = self.target.name + else: + target = self.target + print (platform, platform_os, target) + return (platform, platform_os, target) def to_dict(self): d = {} - platform = self.platform - platform_os = self.platform_os - target = self.target - - d['platform'] = self.platform.to_dict() if self.platform else None - d['platform_os'] = self.platform_os.to_dict() if self.platform_os else None - d['target'] = self.target.to_dict() if self.target else None + d['platform'] = str(self.platform) if self.platform else None + d['platform_os'] = str(self.platform_os) if self.platform_os else None + d['target'] = str(self.target) if self.target else None return d -def _target_from_dict(target_dict): +def _target_from_dict(target_name, platform=None): """ Creates new instance of target and assigns all the attributes of that target from the dictionary """ - target = Target.__new__(Target) - target.name = target_dict['name'] - target.module_name = target_dict['module_name'] - if 'platform_name' in target_dict: - target.platform_name = target_dict['platform_name'] - return target + if not platform: + platform = sys_type() + return platform.target(target_name) -def _operating_system_from_dict(os_dict): + +def _operating_system_from_dict(os_name, platform=None): """ uses platform's operating system method to grab the constructed operating systems that are valid on the platform. """ -# NOTE: Might need a better way to create operating system objects - operating_system = OperatingSystem.__new__(OperatingSystem) - operating_system.name = os_dict['name'] - operating_system.version = os_dict['version'] - return operating_system + if not platform: + platform = sys_type() + if isinstance(os_name, spack.util.spack_yaml.syaml_dict): + name = os_name['name'] + version = os_name['version'] + return platform.operating_system(name+version) + else: + return platform.operating_system(os_name) -def _platform_from_dict(platform_dict): + +def _platform_from_dict(platform_name): """ Constructs a platform from a dictionary. """ - platform = Platform.__new__(Platform) - name, p_dict = platform_dict.items()[0] - platform.name = name - platform.targets = {} - for name, t_dict in p_dict['targets'].items(): - platform.add_target(name, _target_from_dict(t_dict)) - platform.operating_sys = {} - for name, o_dict in p_dict['operating_systems'].items(): - platform.add_operating_system(name, _operating_system_from_dict(o_dict)) - platform.priority = p_dict['priority'] - platform.front_end = p_dict['default_front_end_target'] - platform.back_end = p_dict['default_back_end_target'] - platform.default = p_dict['default_target'] - platform.front_os = p_dict['default_front_end_os'] - platform.back_os = p_dict['default_back_end_os'] - platform.default_os = p_dict['default_os'] + platform_path = spack.platform_path + mod_string = "spack.platforms" + + for p in list_modules(platform_path): + if platform_name == p: + mod_name = mod_string + platform_name + path = join_path(platform_path, platform_name) + ".py" + mod = imp.load_source(mod_name, path) + platform_class = mod_to_class(platform_name) + cls = getattr(mod, platform_class) + platform = cls() + return platform + return None - return platform def arch_from_dict(d): """ Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict @@ -472,19 +454,30 @@ def arch_from_dict(d): else: if d is None: return None - platform_dict = d['platform'] - os_dict = d['platform_os'] - target_dict = d['target'] + platform_name = d['platform'] + os_name = d['platform_os'] + target_name = d['target'] - arch.platform = _platform_from_dict(platform_dict) if platform_dict else None - arch.target = _target_from_dict(target_dict) if os_dict else None - arch.platform_os = _operating_system_from_dict(os_dict) if os_dict else None + if platform_name: + arch.platform = _platform_from_dict(platform_name) + else: + arch.platform = None + if target_name: + arch.target = _target_from_dict(target_name, arch.platform) + else: + arch.target = None + if os_name: + arch.platform_os = _operating_system_from_dict(os_name, + arch.platform) + else: + arch.platform_os = None arch.os_string = None arch.target_string = None return arch + @memoized def all_platforms(): modules = [] @@ -507,6 +500,7 @@ def all_platforms(): return modules + @memoized def sys_type(): """ Gather a list of all available subclasses of platforms. From 30c204a0a5ca116d2e6ab1ab177e103a5857ca4e Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Fri, 3 Jun 2016 16:16:59 -0700 Subject: [PATCH 323/375] Made a merge commit earlier with Todd fixing all_platforms, but changed how architecture looks in spec.yaml. Just three strings. Also made some flake8 formatting changes --- lib/spack/spack/architecture.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index e74e72d6cda..818731cbf8f 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -384,7 +384,6 @@ def _cmp_key(self): target = self.target.name else: target = self.target - print (platform, platform_os, target) return (platform, platform_os, target) def to_dict(self): @@ -421,19 +420,10 @@ def _operating_system_from_dict(os_name, platform=None): def _platform_from_dict(platform_name): """ Constructs a platform from a dictionary. """ - platform_path = spack.platform_path - mod_string = "spack.platforms" - - for p in list_modules(platform_path): - if platform_name == p: - mod_name = mod_string + platform_name - path = join_path(platform_path, platform_name) + ".py" - mod = imp.load_source(mod_name, path) - platform_class = mod_to_class(platform_name) - cls = getattr(mod, platform_class) - platform = cls() - return platform - return None + platform_list = all_platforms() + for p in platform_list: + if platform_name.replace("_", "").lower() == p.__name__.lower(): + return p() def arch_from_dict(d): From f751d681771499b1da46216f46beb0a074fa73e7 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:03:32 -0700 Subject: [PATCH 324/375] Passing architecture to functions to find compiler. We can use the platform name if provided to find the operating system. Also changed error to operating system instead of target since operating system is now in charge of compiler finding. --- lib/spack/spack/compilers/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index ea79f655b00..d923d77da8d 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -220,10 +220,11 @@ def find(compiler_spec, scope=None): @_auto_compiler_spec -def compilers_for_spec(compiler_spec, scope=None): +def compilers_for_spec(compiler_spec, scope=None, **kwargs): """This gets all compilers that satisfy the supplied CompilerSpec. Returns an empty list if none are found. """ + platform = kwargs.get("platform", None) config = all_compilers_config(scope) def get_compilers(cspec): @@ -253,7 +254,7 @@ def get_compilers(cspec): mods = [] if 'operating_system' in items: - operating_system = spack.architecture._operating_system_from_dict( items['operating_system'] ) + operating_system = spack.architecture._operating_system_from_dict(items['operating_system'], platform) else: operating_system = None @@ -275,12 +276,13 @@ def get_compilers(cspec): @_auto_compiler_spec -def compiler_for_spec(compiler_spec, operating_system): +def compiler_for_spec(compiler_spec, arch): """Get the compiler that satisfies compiler_spec. compiler_spec must be concrete.""" + operating_system = arch.platform_os assert(compiler_spec.concrete) - compilers = [c for c in compilers_for_spec(compiler_spec) + compilers = [c for c in compilers_for_spec(compiler_spec, platform=arch.platform) if c.operating_system == operating_system] if len(compilers) < 1: raise NoCompilerForSpecError(compiler_spec, operating_system) @@ -334,7 +336,7 @@ def __init__(self): class NoCompilerForSpecError(spack.error.SpackError): def __init__(self, compiler_spec, target): - super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s" % ( + super(NoCompilerForSpecError, self).__init__("No compilers for operating system %s satisfy spec %s" % ( target, compiler_spec)) class CompilerSpecInsufficientlySpecificError(spack.error.SpackError): From c7b48f6fef92167b55d901e47a50e3f704fd65ec Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:04:24 -0700 Subject: [PATCH 325/375] Passing full arch for proper compiler spec so that we can use the platform to get the operating system --- lib/spack/spack/concretize.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/spack/spack/concretize.py b/lib/spack/spack/concretize.py index 58811c3419c..1f37455c770 100644 --- a/lib/spack/spack/concretize.py +++ b/lib/spack/spack/concretize.py @@ -280,10 +280,12 @@ def concretize_compiler(self, spec): # Only use a matching compiler if it is of the proper style # Takes advantage of the proper logic already existing in compiler_for_spec # Should think whether this can be more efficient - def _proper_compiler_style(cspec, architecture): - compilers = spack.compilers.compilers_for_spec(cspec) - return filter(lambda c: c.operating_system == - architecture.platform_os, compilers) + def _proper_compiler_style(cspec, arch): + platform = arch.platform + compilers = spack.compilers.compilers_for_spec(cspec, + platform=platform) + return filter(lambda c: c.operating_system == + arch.platform_os, compilers) #return compilers @@ -368,7 +370,7 @@ def concretize_compiler_flags(self, spec): # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. - compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture.platform_os) + compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture) for flag in compiler.flags: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = compiler.flags[flag] From 5b023bb0a1b2eef9d950cb660b466d2ab0e04349 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:05:14 -0700 Subject: [PATCH 326/375] Passing through full arch instead of just operating sysystem so that we can use the platform to get the operating system --- lib/spack/spack/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 0edb70e4ca6..1a6c289bc7a 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -682,7 +682,7 @@ def compiler(self): if not self.spec.concrete: raise ValueError("Can only get a compiler for a concrete package.") return spack.compilers.compiler_for_spec(self.spec.compiler, - self.spec.architecture.platform_os) + self.spec.architecture) def url_version(self, version): """ From 864191b6ed9fbd95e549b2f4852dc92113c6a190 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:06:58 -0700 Subject: [PATCH 327/375] Got rid of unnecessary global keyword --- lib/spack/spack/test/mock_packages_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 561f319b16a..9c586bd8ce8 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -40,7 +40,6 @@ linux_os_version = '6' if platform.name == 'linux': - global linux_os_name, linux_os_version linux_os = platform.operating_system("default_os") linux_os_name = linux_os.name linux_os_version = linux_os.version From 835982faedc0d728356600836caf2dd4690a9c5b Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 09:08:59 -0700 Subject: [PATCH 328/375] Changed OS name to CNL10 so that we properly find the compilers for the test platform --- lib/spack/spack/platforms/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/platforms/test.py b/lib/spack/spack/platforms/test.py index 3a4f9d5f2d3..8fa2585a7a1 100644 --- a/lib/spack/spack/platforms/test.py +++ b/lib/spack/spack/platforms/test.py @@ -10,8 +10,8 @@ class Test(Platform): back_end = 'x86_64' default = 'x86_64' - back_os = 'CNL' - default_os = 'CNL' + back_os = 'CNL10' + default_os = 'CNL10' def __init__(self): super(Test, self).__init__('test') From 196737c2173a7b5748b3ad59e349611ec4f6b23c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Mon, 6 Jun 2016 10:29:19 -0700 Subject: [PATCH 329/375] Changed to passing full architecture spec to function rather than just platform_os so we can grab the operating system from the platform getters --- lib/spack/spack/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 89c03a13b76..7718adb72cd 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -1124,7 +1124,7 @@ def concretize(self): for s in self.traverse(root=False): if s.external_module: - compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture.platform_os) + compiler = spack.compilers.compiler_for_spec(s.compiler, s.architecture) for mod in compiler.modules: load_module(mod) From 4925be0bc4b63571eb9a9cb63e65409bebcc2c66 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 7 Jun 2016 13:22:47 -0700 Subject: [PATCH 330/375] Load external modules --- lib/spack/spack/build_environment.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 4483b9d1329..7c65091d49c 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -384,6 +384,13 @@ def parent_class_modules(cls): return result +def load_external_modules(pkg): + """ traverse the spec list and find any specs that have external modules. + """ + for dep in list(pkg.spec.traverse()): + if dep.external_module: + load_module(dep.external_module) + def setup_package(pkg): """Execute all environment setup routines.""" spack_env = EnvironmentModifications() @@ -407,7 +414,7 @@ def setup_package(pkg): set_compiler_environment_variables(pkg, spack_env) set_build_environment_variables(pkg, spack_env) - + load_external_modules(pkg) # traverse in postorder so package can use vars from its dependencies spec = pkg.spec for dspec in pkg.spec.traverse(order='post', root=False): From 5715799d4ee1c39d11106bb78a034e6affac474c Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Tue, 7 Jun 2016 14:15:28 -0700 Subject: [PATCH 331/375] Added to_dict back into operating system --- lib/spack/spack/architecture.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 818731cbf8f..1f25bc14ea8 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -195,6 +195,7 @@ def operating_system(self, name): name = self.back_os return self.operating_sys.get(name, None) + @classmethod def detect(self): @@ -333,6 +334,12 @@ def find_compiler(self, cmp_cls, *path): return list(compilers.values()) + def to_dict(self): + d = {} + d['name'] = self.name + d['version'] = self.version + + return d @key_ordering class Arch(object): @@ -410,7 +417,8 @@ def _operating_system_from_dict(os_name, platform=None): """ if not platform: platform = sys_type() - if isinstance(os_name, spack.util.spack_yaml.syaml_dict): + if isinstance(os_name, spack.util.spack_yaml.syaml_dict) or \ + isinstance(os_name, dict): name = os_name['name'] version = os_name['version'] return platform.operating_system(name+version) From 25b2f7b1d75d404b4e06443969cdcb3e125123e6 Mon Sep 17 00:00:00 2001 From: Greg Lee Date: Tue, 7 Jun 2016 20:02:38 -0700 Subject: [PATCH 332/375] put older versions back + version-specific dwarfdump dir --- var/spack/repos/builtin/packages/libdwarf/package.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/libdwarf/package.py b/var/spack/repos/builtin/packages/libdwarf/package.py index c9c6e9404f6..0e2e702b1cc 100644 --- a/var/spack/repos/builtin/packages/libdwarf/package.py +++ b/var/spack/repos/builtin/packages/libdwarf/package.py @@ -23,7 +23,6 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os # Only build certain parts of dwarf because the other ones break. dwarf_dirs = ['libdwarf', 'dwarfdump2'] @@ -45,7 +44,9 @@ class Libdwarf(Package): list_url = homepage version('20160507', 'ae32d6f9ece5daf05e2d4b14822ea811') - + version('20130729', '4cc5e48693f7b93b7aa0261e63c0e21d') + version('20130207', '64b42692e947d5180e162e46c689dfbf') + version('20130126', 'ded74a5e90edb5a12aac3c29d260c5db') depends_on("libelf") parallel = False @@ -67,7 +68,11 @@ def install(self, spec, prefix): install('libdwarf.h', prefix.include) install('dwarf.h', prefix.include) - with working_dir('dwarfdump'): + if spec.satisfies('@20130126:20130729'): + dwarfdump_dir = 'dwarfdump2' + else: + dwarfdump_dir = 'dwarfdump' + with working_dir(dwarfdump_dir): configure("--prefix=" + prefix) # This makefile has strings of copy commands that From bb57b82794bf8434904235af42ad15914eff24ee Mon Sep 17 00:00:00 2001 From: Greg Lee Date: Tue, 7 Jun 2016 20:15:02 -0700 Subject: [PATCH 333/375] flake8 fixes --- var/spack/repos/builtin/packages/libdwarf/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/libdwarf/package.py b/var/spack/repos/builtin/packages/libdwarf/package.py index 0e2e702b1cc..594271f655f 100644 --- a/var/spack/repos/builtin/packages/libdwarf/package.py +++ b/var/spack/repos/builtin/packages/libdwarf/package.py @@ -27,6 +27,7 @@ # Only build certain parts of dwarf because the other ones break. dwarf_dirs = ['libdwarf', 'dwarfdump2'] + class Libdwarf(Package): """The DWARF Debugging Information Format is of interest to programmers working on compilers and debuggers (and any one @@ -51,7 +52,6 @@ class Libdwarf(Package): parallel = False - def install(self, spec, prefix): # dwarf build does not set arguments for ar properly make.add_default_arg('ARFLAGS=rcs') From f4a39457de662376b5f9da300a06e2dff91b1aeb Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 8 Jun 2016 15:35:15 +0200 Subject: [PATCH 334/375] lmod : updated package and dependencies --- lib/spack/spack/version.py | 11 +++++ .../repos/builtin/packages/lmod/package.py | 16 +++---- .../packages/lua-luafilesystem/package.py | 46 +++++++++++++++++++ .../repos/builtin/packages/lua/package.py | 6 ++- 4 files changed, 69 insertions(+), 10 deletions(-) create mode 100644 var/spack/repos/builtin/packages/lua-luafilesystem/package.py diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index 247f6d2362f..d86bcef3e06 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -124,6 +124,17 @@ def __init__(self, string): # last element of separators is '' self.separators = tuple(re.split(segment_regex, string)[1:-1]) + @property + def dotted(self): + return '.'.join(str(x) for x in self.version) + + @property + def underscored(self): + return '_'.join(str(x) for x in self.version) + + @property + def dashed(self): + return '-'.join(str(x) for x in self.version) def up_to(self, index): """Return a version string up to the specified component, exclusive. diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index 0a8b9b4577d..9084c348812 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -34,17 +34,17 @@ class Lmod(Package): variable. Modulefiles for Library packages provide environment variables that specify where the library and header files can be found. """ - homepage = "https://www.tacc.utexas.edu/research-development/tacc-projects/lmod" - url = "http://sourceforge.net/projects/lmod/files/Lmod-6.0.1.tar.bz2/download" + homepage = 'https://www.tacc.utexas.edu/research-development/tacc-projects/lmod' + url = 'https://github.com/TACC/Lmod/archive/6.4.1.tar.gz' + version('6.4.1', '7978ba777c8aa41a4d8c05fec5f780f4') + version('6.3.7', '0fa4d5a24c41cae03776f781aa2dedc1', preferred=True) version('6.0.1', '91abf52fe5033bd419ffe2842ebe7af9') - depends_on("lua@5.2:") + depends_on('lua@5.2:') + depends_on('lua-luaposix') + depends_on('lua-luafilesystem') def install(self, spec, prefix): - # Add our lua to PATH - os.environ['PATH'] = spec['lua'].prefix.bin + ';' + os.environ['PATH'] - configure('--prefix=%s' % prefix) - make() - make("install") + make('install') diff --git a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py new file mode 100644 index 00000000000..14278bbe724 --- /dev/null +++ b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class LuaLuafilesystem(Package): + """ + LuaFileSystem is a Lua library developed to complement the set of functions related to file + systems offered by the standard Lua distribution. + + LuaFileSystem offers a portable way to access the underlying directory structure and file attributes. LuaFileSystem + is free software and uses the same license as Lua 5.1 + """ + homepage = 'http://keplerproject.github.io/luafilesystem' + url = 'https://github.com/keplerproject/luafilesystem/archive/v_1_6_3.tar.gz' + + version('1_6_3', 'd0552c7e5a082f5bb2865af63fb9dc95') + + extends('lua') + + def install(self, spec, prefix): + version = self.spec.version + rockspec_format = join_path(self.stage.path, 'luafilesystem-v_{version.underscored}', 'rockspecs', 'luafilesystem-{version.dotted}-1.rockspec') + luarocks('--tree=' + prefix, 'install', rockspec_format.format(version=self.spec.version)) \ No newline at end of file diff --git a/var/spack/repos/builtin/packages/lua/package.py b/var/spack/repos/builtin/packages/lua/package.py index 170f90516ab..320e25899ea 100644 --- a/var/spack/repos/builtin/packages/lua/package.py +++ b/var/spack/repos/builtin/packages/lua/package.py @@ -105,6 +105,9 @@ def setup_dependent_environment(self, spack_env, run_env, extension_spec): spack_env.set('LUA_PATH', ';'.join(lua_patterns), separator=';') spack_env.set('LUA_CPATH', ';'.join(lua_cpatterns), separator=';') + # Add LUA to PATH for dependent packages + spack_env.prepend_path('PATH', self.prefix.bin) + # For run time environment set only the path for extension_spec and # prepend it to LUAPATH if extension_spec.package.extends(self.spec): @@ -153,5 +156,4 @@ def setup_dependent_package(self, module, ext_spec): """ # Lua extension builds can have lua and luarocks executable functions module.lua = Executable(join_path(self.spec.prefix.bin, 'lua')) - module.luarocks = Executable(join_path(self.spec.prefix.bin, - 'luarocks')) + module.luarocks = Executable(join_path(self.spec.prefix.bin, 'luarocks')) From 4a61efe641c9bf987a62bff729334c7cce3f141b Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 8 Jun 2016 16:21:09 +0200 Subject: [PATCH 335/375] lmod : fixed issues with LUA_PATH during installation --- var/spack/repos/builtin/packages/lmod/package.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index 9084c348812..e50c2cfc4bf 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -23,7 +23,7 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os + class Lmod(Package): """ @@ -38,13 +38,19 @@ class Lmod(Package): url = 'https://github.com/TACC/Lmod/archive/6.4.1.tar.gz' version('6.4.1', '7978ba777c8aa41a4d8c05fec5f780f4') - version('6.3.7', '0fa4d5a24c41cae03776f781aa2dedc1', preferred=True) + version('6.3.7', '0fa4d5a24c41cae03776f781aa2dedc1') version('6.0.1', '91abf52fe5033bd419ffe2842ebe7af9') depends_on('lua@5.2:') depends_on('lua-luaposix') depends_on('lua-luafilesystem') + parallel = False + + def setup_environment(self, spack_env, run_env): + stage_lua_path = join_path(self.stage.path, 'Lmod-{version}', 'src', '?.lua') + spack_env.append_path('LUA_PATH', stage_lua_path.format(version=self.version), separator=';') + def install(self, spec, prefix): configure('--prefix=%s' % prefix) make('install') From 564483936f32abfd5ee79dea6263af4bd6e9bb9d Mon Sep 17 00:00:00 2001 From: alalazo Date: Wed, 8 Jun 2016 16:33:49 +0200 Subject: [PATCH 336/375] qa : fixed flake8 issues --- lib/spack/spack/version.py | 139 ++++++------------ .../repos/builtin/packages/lmod/package.py | 8 +- .../packages/lua-luafilesystem/package.py | 19 ++- .../repos/builtin/packages/lua/package.py | 5 +- 4 files changed, 64 insertions(+), 107 deletions(-) diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index d86bcef3e06..858d5814728 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -43,16 +43,16 @@ intersection concrete """ -import os -import sys import re from bisect import bisect_left from functools import wraps + from functools_backport import total_ordering # Valid version characters VALID_VERSION = r'[A-Za-z0-9_.-]' + def int_if_int(string): """Convert a string to int if possible. Otherwise, return a string.""" try: @@ -62,10 +62,11 @@ def int_if_int(string): def coerce_versions(a, b): - """Convert both a and b to the 'greatest' type between them, in this order: + """ + Convert both a and b to the 'greatest' type between them, in this order: Version < VersionRange < VersionList - This is used to simplify comparison operations below so that we're always - comparing things that are of the same type. + This is used to simplify comparison operations below so that we're always + comparing things that are of the same type. """ order = (Version, VersionRange, VersionList) ta, tb = type(a), type(b) @@ -105,6 +106,7 @@ def coercing_method(a, b, *args, **kwargs): @total_ordering class Version(object): """Class to represent versions""" + def __init__(self, string): string = str(string) @@ -142,15 +144,12 @@ def up_to(self, index): """ return '.'.join(str(x) for x in self[:index]) - def lowest(self): return self - def highest(self): return self - @coerced def satisfies(self, other): """A Version 'satisfies' another if it is at least as specific and has a @@ -158,11 +157,10 @@ def satisfies(self, other): gcc@4.7 so that when a user asks to build with gcc@4.7, we can find a suitable compiler. """ - nself = len(self.version) + nself = len(self.version) nother = len(other.version) return nother <= nself and self.version[:nother] == other.version - def wildcard(self): """Create a regex that will match variants of this version string.""" def a_or_n(seg): @@ -192,28 +190,22 @@ def a_or_n(seg): wc += '(?:[a-z]|alpha|beta)?)?' * (len(segments) - 1) return wc - def __iter__(self): return iter(self.version) - def __getitem__(self, idx): return tuple(self.version[idx]) - def __repr__(self): return self.string - def __str__(self): return self.string - @property def concrete(self): return self - @coerced def __lt__(self, other): """Version comparison is designed for consistency with the way RPM @@ -246,28 +238,23 @@ def __lt__(self, other): # If the common prefix is equal, the one with more segments is bigger. return len(self.version) < len(other.version) - @coerced def __eq__(self, other): return (other is not None and type(other) == Version and self.version == other.version) - def __ne__(self, other): return not (self == other) - def __hash__(self): return hash(self.version) - @coerced def __contains__(self, other): if other is None: return False return other.version[:len(self.version)] == self.version - def is_predecessor(self, other): """True if the other version is the immediate predecessor of this one. That is, NO versions v exist such that: @@ -280,16 +267,13 @@ def is_predecessor(self, other): ol = other.version[-1] return type(sl) == int and type(ol) == int and (ol - sl == 1) - def is_successor(self, other): return other.is_predecessor(self) - @coerced def overlaps(self, other): return self in other or other in self - @coerced def union(self, other): if self == other or other in self: @@ -299,7 +283,6 @@ def union(self, other): else: return VersionList([self, other]) - @coerced def intersection(self, other): if self == other: @@ -310,6 +293,7 @@ def intersection(self, other): @total_ordering class VersionRange(object): + def __init__(self, start, end): if isinstance(start, basestring): start = Version(start) @@ -321,15 +305,12 @@ def __init__(self, start, end): if start and end and end < start: raise ValueError("Invalid Version range: %s" % self) - def lowest(self): return self.start - def highest(self): return self.end - @coerced def __lt__(self, other): """Sort VersionRanges lexicographically so that they are ordered first @@ -342,28 +323,24 @@ def __lt__(self, other): s, o = self, other if s.start != o.start: - return s.start is None or (o.start is not None and s.start < o.start) + return s.start is None or (o.start is not None and s.start < o.start) # NOQA: ignore=E501 return (s.end != o.end and o.end is None or (s.end is not None and s.end < o.end)) - @coerced def __eq__(self, other): return (other is not None and type(other) == VersionRange and self.start == other.start and self.end == other.end) - def __ne__(self, other): return not (self == other) - @property def concrete(self): return self.start if self.start == self.end else None - @coerced def __contains__(self, other): if other is None: @@ -384,57 +361,55 @@ def __contains__(self, other): other.end in self.end))) return in_upper - @coerced def satisfies(self, other): - """A VersionRange satisfies another if some version in this range - would satisfy some version in the other range. To do this it must - either: - a) Overlap with the other range - b) The start of this range satisfies the end of the other range. + """ + A VersionRange satisfies another if some version in this range + would satisfy some version in the other range. To do this it must + either: + a) Overlap with the other range + b) The start of this range satisfies the end of the other range. - This is essentially the same as overlaps(), but overlaps assumes - that its arguments are specific. That is, 4.7 is interpreted as - 4.7.0.0.0.0... . This funciton assumes that 4.7 woudl be satisfied - by 4.7.3.5, etc. + This is essentially the same as overlaps(), but overlaps assumes + that its arguments are specific. That is, 4.7 is interpreted as + 4.7.0.0.0.0... . This funciton assumes that 4.7 woudl be satisfied + by 4.7.3.5, etc. - Rationale: - If a user asks for gcc@4.5:4.7, and a package is only compatible with - gcc@4.7.3:4.8, then that package should be able to build under the - constraints. Just using overlaps() would not work here. + Rationale: + If a user asks for gcc@4.5:4.7, and a package is only compatible with + gcc@4.7.3:4.8, then that package should be able to build under the + constraints. Just using overlaps() would not work here. - Note that we don't need to check whether the end of this range - would satisfy the start of the other range, because overlaps() - already covers that case. + Note that we don't need to check whether the end of this range + would satisfy the start of the other range, because overlaps() + already covers that case. - Note further that overlaps() is a symmetric operation, while - satisfies() is not. + Note further that overlaps() is a symmetric operation, while + satisfies() is not. """ return (self.overlaps(other) or # if either self.start or other.end are None, then this can't # satisfy, or overlaps() would've taken care of it. self.start and other.end and self.start.satisfies(other.end)) - @coerced def overlaps(self, other): - return ((self.start == None or other.end is None or + return ((self.start is None or other.end is None or self.start <= other.end or other.end in self.start or self.start in other.end) and - (other.start is None or self.end == None or + (other.start is None or self.end is None or other.start <= self.end or other.start in self.end or self.end in other.start)) - @coerced def union(self, other): if not self.overlaps(other): if (self.end is not None and other.start is not None and - self.end.is_predecessor(other.start)): + self.end.is_predecessor(other.start)): return VersionRange(self.start, other.end) if (other.end is not None and self.start is not None and - other.end.is_predecessor(self.start)): + other.end.is_predecessor(self.start)): return VersionRange(other.start, self.end) return VersionList([self, other]) @@ -453,13 +428,12 @@ def union(self, other): else: end = self.end # TODO: See note in intersection() about < and in discrepancy. - if not other.end in self.end: + if other.end not in self.end: if end in other.end or other.end > self.end: end = other.end return VersionRange(start, end) - @coerced def intersection(self, other): if self.overlaps(other): @@ -481,7 +455,7 @@ def intersection(self, other): # 1.6 < 1.6.5 = True (lexicographic) # Should 1.6 NOT be less than 1.6.5? Hm. # Here we test (not end in other.end) first to avoid paradox. - if other.end is not None and not end in other.end: + if other.end is not None and end not in other.end: if other.end < end or other.end in end: end = other.end @@ -490,15 +464,12 @@ def intersection(self, other): else: return VersionList() - def __hash__(self): return hash((self.start, self.end)) - def __repr__(self): return self.__str__() - def __str__(self): out = "" if self.start: @@ -512,6 +483,7 @@ def __str__(self): @total_ordering class VersionList(object): """Sorted, non-redundant list of Versions and VersionRanges.""" + def __init__(self, vlist=None): self.versions = [] if vlist is not None: @@ -526,7 +498,6 @@ def __init__(self, vlist=None): for v in vlist: self.add(ver(v)) - def add(self, version): if type(version) in (Version, VersionRange): # This normalizes single-value version ranges. @@ -535,9 +506,9 @@ def add(self, version): i = bisect_left(self, version) - while i-1 >= 0 and version.overlaps(self[i-1]): - version = version.union(self[i-1]) - del self.versions[i-1] + while i - 1 >= 0 and version.overlaps(self[i - 1]): + version = version.union(self[i - 1]) + del self.versions[i - 1] i -= 1 while i < len(self) and version.overlaps(self[i]): @@ -553,7 +524,6 @@ def add(self, version): else: raise TypeError("Can't add %s to VersionList" % type(version)) - @property def concrete(self): if len(self) == 1: @@ -561,11 +531,9 @@ def concrete(self): else: return None - def copy(self): return VersionList(self) - def lowest(self): """Get the lowest version in the list.""" if not self: @@ -573,7 +541,6 @@ def lowest(self): else: return self[0].lowest() - def highest(self): """Get the highest version in the list.""" if not self: @@ -581,7 +548,6 @@ def highest(self): else: return self[-1].highest() - @coerced def overlaps(self, other): if not other or not self: @@ -597,14 +563,12 @@ def overlaps(self, other): o += 1 return False - def to_dict(self): """Generate human-readable dict for YAML.""" if self.concrete: - return { 'version' : str(self[0]) } + return {'version': str(self[0])} else: - return { 'versions' : [str(v) for v in self] } - + return {'versions': [str(v) for v in self]} @staticmethod def from_dict(dictionary): @@ -616,7 +580,6 @@ def from_dict(dictionary): else: raise ValueError("Dict must have 'version' or 'versions' in it.") - @coerced def satisfies(self, other, strict=False): """A VersionList satisfies another if some version in the list @@ -644,20 +607,17 @@ def satisfies(self, other, strict=False): o += 1 return False - @coerced def update(self, other): for v in other.versions: self.add(v) - @coerced def union(self, other): result = self.copy() result.update(other) return result - @coerced def intersection(self, other): # TODO: make this faster. This is O(n^2). @@ -667,7 +627,6 @@ def intersection(self, other): result.add(s.intersection(o)) return result - @coerced def intersect(self, other): """Intersect this spec's list with other. @@ -689,50 +648,40 @@ def __contains__(self, other): if i == 0: if version not in self[0]: return False - elif all(version not in v for v in self[i-1:]): + elif all(version not in v for v in self[i - 1:]): return False return True - def __getitem__(self, index): return self.versions[index] - def __iter__(self): return iter(self.versions) - def __reversed__(self): return reversed(self.versions) - def __len__(self): return len(self.versions) - @coerced def __eq__(self, other): return other is not None and self.versions == other.versions - def __ne__(self, other): return not (self == other) - @coerced def __lt__(self, other): return other is not None and self.versions < other.versions - def __hash__(self): return hash(tuple(self.versions)) - def __str__(self): return ",".join(str(v) for v in self.versions) - def __repr__(self): return str(self.versions) @@ -741,7 +690,7 @@ def _string_to_version(string): """Converts a string to a Version, VersionList, or VersionRange. This is private. Client code should use ver(). """ - string = string.replace(' ','') + string = string.replace(' ', '') if ',' in string: return VersionList(string.split(',')) @@ -749,7 +698,7 @@ def _string_to_version(string): elif ':' in string: s, e = string.split(':') start = Version(s) if s else None - end = Version(e) if e else None + end = Version(e) if e else None return VersionRange(start, end) else: diff --git a/var/spack/repos/builtin/packages/lmod/package.py b/var/spack/repos/builtin/packages/lmod/package.py index e50c2cfc4bf..7d75866d52f 100644 --- a/var/spack/repos/builtin/packages/lmod/package.py +++ b/var/spack/repos/builtin/packages/lmod/package.py @@ -34,7 +34,7 @@ class Lmod(Package): variable. Modulefiles for Library packages provide environment variables that specify where the library and header files can be found. """ - homepage = 'https://www.tacc.utexas.edu/research-development/tacc-projects/lmod' + homepage = 'https://www.tacc.utexas.edu/research-development/tacc-projects/lmod' # NOQA: ignore=E501 url = 'https://github.com/TACC/Lmod/archive/6.4.1.tar.gz' version('6.4.1', '7978ba777c8aa41a4d8c05fec5f780f4') @@ -48,8 +48,10 @@ class Lmod(Package): parallel = False def setup_environment(self, spack_env, run_env): - stage_lua_path = join_path(self.stage.path, 'Lmod-{version}', 'src', '?.lua') - spack_env.append_path('LUA_PATH', stage_lua_path.format(version=self.version), separator=';') + stage_lua_path = join_path( + self.stage.path, 'Lmod-{version}', 'src', '?.lua') + spack_env.append_path('LUA_PATH', stage_lua_path.format( + version=self.version), separator=';') def install(self, spec, prefix): configure('--prefix=%s' % prefix) diff --git a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py index 14278bbe724..a61b9dd6753 100644 --- a/var/spack/repos/builtin/packages/lua-luafilesystem/package.py +++ b/var/spack/repos/builtin/packages/lua-luafilesystem/package.py @@ -27,11 +27,13 @@ class LuaLuafilesystem(Package): """ - LuaFileSystem is a Lua library developed to complement the set of functions related to file - systems offered by the standard Lua distribution. + LuaFileSystem is a Lua library developed to complement the set of + functions related to file systems offered by the standard Lua distribution. - LuaFileSystem offers a portable way to access the underlying directory structure and file attributes. LuaFileSystem - is free software and uses the same license as Lua 5.1 + LuaFileSystem offers a portable way to access the underlying directory + structure and file attributes. + + LuaFileSystem is free software and uses the same license as Lua 5.1 """ homepage = 'http://keplerproject.github.io/luafilesystem' url = 'https://github.com/keplerproject/luafilesystem/archive/v_1_6_3.tar.gz' @@ -41,6 +43,9 @@ class LuaLuafilesystem(Package): extends('lua') def install(self, spec, prefix): - version = self.spec.version - rockspec_format = join_path(self.stage.path, 'luafilesystem-v_{version.underscored}', 'rockspecs', 'luafilesystem-{version.dotted}-1.rockspec') - luarocks('--tree=' + prefix, 'install', rockspec_format.format(version=self.spec.version)) \ No newline at end of file + rockspec_fmt = join_path(self.stage.path, + 'luafilesystem-v_{version.underscored}', + 'rockspecs', + 'luafilesystem-{version.dotted}-1.rockspec') + luarocks('--tree=' + prefix, 'install', + rockspec_fmt.format(version=self.spec.version)) diff --git a/var/spack/repos/builtin/packages/lua/package.py b/var/spack/repos/builtin/packages/lua/package.py index 320e25899ea..8ce4a7b94da 100644 --- a/var/spack/repos/builtin/packages/lua/package.py +++ b/var/spack/repos/builtin/packages/lua/package.py @@ -57,7 +57,7 @@ class Lua(Package): placement='luarocks') def install(self, spec, prefix): - if spec.satisfies("arch=darwin-i686") or spec.satisfies("arch=darwin-x86_64"): + if spec.satisfies("arch=darwin-i686") or spec.satisfies("arch=darwin-x86_64"): # NOQA: ignore=E501 target = 'macosx' else: target = 'linux' @@ -156,4 +156,5 @@ def setup_dependent_package(self, module, ext_spec): """ # Lua extension builds can have lua and luarocks executable functions module.lua = Executable(join_path(self.spec.prefix.bin, 'lua')) - module.luarocks = Executable(join_path(self.spec.prefix.bin, 'luarocks')) + module.luarocks = Executable( + join_path(self.spec.prefix.bin, 'luarocks')) From 42a10d56905727529065d2c492f4c8dc9e4f73cb Mon Sep 17 00:00:00 2001 From: James Riley Wynne III Date: Wed, 8 Jun 2016 12:43:05 -0400 Subject: [PATCH 337/375] Fixes issue #438 --- var/spack/repos/builtin/packages/boost/package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 2f2965eb12b..6da86d6da4e 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -109,6 +109,7 @@ class Boost(Package): variant('multithreaded', default=True, description="Build multi-threaded versions of libraries") variant('singlethreaded', default=True, description="Build single-threaded versions of libraries") variant('icu_support', default=False, description="Include ICU support (for regex/locale libraries)") + variant('graph', default=False, description="Build the Boost Graph library") depends_on('icu', when='+icu_support') depends_on('python', when='+python') @@ -235,6 +236,10 @@ def install(self, spec, prefix): withLibs.remove('chrono') if not spec.satisfies('@1.43.0:'): withLibs.remove('random') + + if '+graph' in spec and '+mpi' in spec: + withLibs.remove('graph') + withLibs.append('graph_parallel') # to make Boost find the user-config.jam env['BOOST_BUILD_PATH'] = './' From 4b9707f8ea3ef947239f6f269aa57849295dd294 Mon Sep 17 00:00:00 2001 From: James Riley Wynne III Date: Wed, 8 Jun 2016 14:36:45 -0400 Subject: [PATCH 338/375] Fixed flake8 formatting problems --- .../repos/builtin/packages/boost/package.py | 74 ++++++++++--------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 6da86d6da4e..8a49672824d 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -27,7 +27,7 @@ import sys import os -import sys + class Boost(Package): """Boost provides free peer-reviewed portable C++ source @@ -75,23 +75,23 @@ class Boost(Package): version('1.34.0', 'ed5b9291ffad776f8757a916e1726ad0') default_install_libs = set(['atomic', - 'chrono', - 'date_time', - 'filesystem', - 'graph', - 'iostreams', - 'locale', - 'log', - 'math', - 'program_options', - 'random', - 'regex', - 'serialization', - 'signals', - 'system', - 'test', - 'thread', - 'wave']) + 'chrono', + 'date_time', + 'filesystem', + 'graph', + 'iostreams', + 'locale', + 'log', + 'math', + 'program_options', + 'random', + 'regex', + 'serialization', + 'signals', + 'system', + 'test', + 'thread', + 'wave']) # mpi/python are not installed by default because they pull in many # dependencies and/or because there is a great deal of customization @@ -121,12 +121,15 @@ class Boost(Package): patch('boost_11856.patch', when='@1.60.0%gcc@4.4.7') def url_for_version(self, version): - """Handle Boost's weird URLs, which write the version two different ways.""" + """ + Handle Boost's weird URLs, + which write the version two different ways. + """ parts = [str(p) for p in Version(version)] dots = ".".join(parts) underscores = "_".join(parts) - return "http://downloads.sourceforge.net/project/boost/boost/%s/boost_%s.tar.bz2" % ( - dots, underscores) + return "http://downloads.sourceforge.net/project/boost" \ + "/boost/%s/boost_%s.tar.bz2" % (dots, underscores) def determine_toolset(self, spec): if spec.satisfies("arch=darwin-x86_64"): @@ -150,20 +153,20 @@ def determine_bootstrap_options(self, spec, withLibs, options): if '+python' in spec: options.append('--with-python=%s' % - join_path(spec['python'].prefix.bin, 'python')) + join_path(spec['python'].prefix.bin, 'python')) with open('user-config.jam', 'w') as f: compiler_wrapper = join_path(spack.build_env_path, 'c++') f.write("using {0} : : {1} ;\n".format(boostToolsetId, - compiler_wrapper)) + compiler_wrapper)) if '+mpi' in spec: f.write('using mpi : %s ;\n' % - join_path(spec['mpi'].prefix.bin, 'mpicxx')) + join_path(spec['mpi'].prefix.bin, 'mpicxx')) if '+python' in spec: f.write('using python : %s : %s ;\n' % - (spec['python'].version, - join_path(spec['python'].prefix.bin, 'python'))) + (spec['python'].version, + join_path(spec['python'].prefix.bin, 'python'))) def determine_b2_options(self, spec, options): if '+debug' in spec: @@ -179,8 +182,7 @@ def determine_b2_options(self, spec, options): '-s', 'BZIP2_INCLUDE=%s' % spec['bzip2'].prefix.include, '-s', 'BZIP2_LIBPATH=%s' % spec['bzip2'].prefix.lib, '-s', 'ZLIB_INCLUDE=%s' % spec['zlib'].prefix.include, - '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib, - ]) + '-s', 'ZLIB_LIBPATH=%s' % spec['zlib'].prefix.lib]) linkTypes = ['static'] if '+shared' in spec: @@ -192,7 +194,8 @@ def determine_b2_options(self, spec, options): if '+singlethreaded' in spec: threadingOpts.append('single') if not threadingOpts: - raise RuntimeError("At least one of {singlethreaded, multithreaded} must be enabled") + raise RuntimeError("""At least one of {singlethreaded, + multithreaded} must be enabled""") options.extend([ 'toolset=%s' % self.determine_toolset(spec), @@ -203,9 +206,9 @@ def determine_b2_options(self, spec, options): def install(self, spec, prefix): # On Darwin, Boost expects the Darwin libtool. However, one of the - # dependencies may have pulled in Spack's GNU libtool, and these two are - # not compatible. We thus create a symlink to Darwin's libtool and add - # it at the beginning of PATH. + # dependencies may have pulled in Spack's GNU libtool, and these two + # are not compatible. We thus create a symlink to Darwin's libtool + # and add it at the beginning of PATH. if sys.platform == 'darwin': newdir = os.path.abspath('darwin-libtool') mkdirp(newdir) @@ -218,7 +221,8 @@ def install(self, spec, prefix): withLibs.append(lib) if not withLibs: # if no libraries are specified for compilation, then you dont have - # to configure/build anything, just copy over to the prefix directory. + # to configure/build anything, just copy over to the prefix + # directory. src = join_path(self.stage.source_path, 'boost') mkdirp(join_path(prefix, 'include')) dst = join_path(prefix, 'include', 'boost') @@ -236,7 +240,6 @@ def install(self, spec, prefix): withLibs.remove('chrono') if not spec.satisfies('@1.43.0:'): withLibs.remove('random') - if '+graph' in spec and '+mpi' in spec: withLibs.remove('graph') withLibs.append('graph_parallel') @@ -264,6 +267,7 @@ def install(self, spec, prefix): for threadingOpt in threadingOpts: b2('install', 'threading=%s' % threadingOpt, *b2_options) - # The shared libraries are not installed correctly on Darwin; correct this + # The shared libraries are not installed correctly + # on Darwin; correct this if (sys.platform == 'darwin') and ('+shared' in spec): fix_darwin_install_name(prefix.lib) From e3ca5e66765d06eca9ec40fed215bd82c4bdd49c Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 8 Jun 2016 16:23:04 -0600 Subject: [PATCH 339/375] Add patch to compile pcre with Intel 16.0.3 compiler --- var/spack/repos/builtin/packages/pcre/intel.patch | 12 ++++++++++++ var/spack/repos/builtin/packages/pcre/package.py | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 var/spack/repos/builtin/packages/pcre/intel.patch diff --git a/var/spack/repos/builtin/packages/pcre/intel.patch b/var/spack/repos/builtin/packages/pcre/intel.patch new file mode 100644 index 00000000000..f160f55e1bd --- /dev/null +++ b/var/spack/repos/builtin/packages/pcre/intel.patch @@ -0,0 +1,12 @@ +diff -up pcre-8.38/pcrecpp.cc.intel pcre-8.38/pcrecpp.cc +--- pcre-8.38/pcrecpp.cc.intel 2014-09-15 07:48:59.000000000 -0600 ++++ pcre-8.38/pcrecpp.cc 2016-06-08 16:16:56.702721214 -0600 +@@ -66,7 +66,7 @@ Arg RE::no_arg((void*)NULL); + // inclusive test if we ever needed it. (Note that not only the + // __attribute__ syntax, but also __USER_LABEL_PREFIX__, are + // gnu-specific.) +-#if defined(__GNUC__) && __GNUC__ >= 3 && defined(__ELF__) ++#if defined(__GNUC__) && __GNUC__ >= 3 && defined(__ELF__) && !defined(__INTEL_COMPILER) + # define ULP_AS_STRING(x) ULP_AS_STRING_INTERNAL(x) + # define ULP_AS_STRING_INTERNAL(x) #x + # define USER_LABEL_PREFIX_STR ULP_AS_STRING(__USER_LABEL_PREFIX__) diff --git a/var/spack/repos/builtin/packages/pcre/package.py b/var/spack/repos/builtin/packages/pcre/package.py index 7a9f3b911d0..b75a50736af 100644 --- a/var/spack/repos/builtin/packages/pcre/package.py +++ b/var/spack/repos/builtin/packages/pcre/package.py @@ -34,6 +34,8 @@ class Pcre(Package): version('8.36', 'b767bc9af0c20bc9c1fe403b0d41ad97') version('8.38', '00aabbfe56d5a48b270f999b508c5ad2') + patch("intel.patch") + def install(self, spec, prefix): configure("--prefix=%s" % prefix) make() From 454acf58af0ea22b6f14f7527226234303d40e99 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 8 Jun 2016 16:49:42 -0600 Subject: [PATCH 340/375] Add additional blank line for flake8 --- var/spack/repos/builtin/packages/pcre/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/var/spack/repos/builtin/packages/pcre/package.py b/var/spack/repos/builtin/packages/pcre/package.py index b75a50736af..8e0f83110e9 100644 --- a/var/spack/repos/builtin/packages/pcre/package.py +++ b/var/spack/repos/builtin/packages/pcre/package.py @@ -24,6 +24,7 @@ ############################################################################## from spack import * + class Pcre(Package): """The PCRE package contains Perl Compatible Regular Expression libraries. These are useful for implementing regular expression From 915115386af09583dc7dfff094b259990179bb4a Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Thu, 9 Jun 2016 18:30:28 -0600 Subject: [PATCH 341/375] Package serf also depends on zlib. + Add zlib as a required dependency. + Point scons build system to the spack provided zlib installation. --- var/spack/repos/builtin/packages/serf/package.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/serf/package.py b/var/spack/repos/builtin/packages/serf/package.py index 3b1d08889ca..7ca72096aac 100644 --- a/var/spack/repos/builtin/packages/serf/package.py +++ b/var/spack/repos/builtin/packages/serf/package.py @@ -36,6 +36,7 @@ class Serf(Package): depends_on('scons') depends_on('expat') depends_on('openssl') + depends_on('zlib') def install(self, spec, prefix): scons = which("scons") @@ -44,8 +45,10 @@ def install(self, spec, prefix): options.append('APR=%s' % spec['apr'].prefix) options.append('APU=%s' % spec['apr-util'].prefix) options.append('OPENSSL=%s' % spec['openssl'].prefix) - options.append('LINKFLAGS=-L%s/lib' % spec['expat'].prefix) - options.append('CPPFLAGS=-I%s/include' % spec['expat'].prefix) + options.append('LINKFLAGS=-L%s/lib -L%s/lib' % + ( spec['expat'].prefix, spec['zlib'].prefix )) + options.append('CPPFLAGS=-I%s/include -I%s/include' % + ( spec['expat'].prefix, spec['zlib'].prefix )) scons(*options) scons('install') From 883f601bca7c6c8f26937b295dadfd2f6a4f023e Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Thu, 9 Jun 2016 19:18:25 -0600 Subject: [PATCH 342/375] Fix style issues. --- var/spack/repos/builtin/packages/serf/package.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/serf/package.py b/var/spack/repos/builtin/packages/serf/package.py index 7ca72096aac..817db682413 100644 --- a/var/spack/repos/builtin/packages/serf/package.py +++ b/var/spack/repos/builtin/packages/serf/package.py @@ -24,8 +24,10 @@ ############################################################################## from spack import * + class Serf(Package): - """Apache Serf - a high performance C-based HTTP client library built upon the Apache Portable Runtime (APR) library""" + """Apache Serf - a high performance C-based HTTP client library + built upon the Apache Portable Runtime (APR) library""" homepage = 'https://serf.apache.org/' url = 'https://archive.apache.org/dist/serf/serf-1.3.8.tar.bz2' @@ -46,9 +48,9 @@ def install(self, spec, prefix): options.append('APU=%s' % spec['apr-util'].prefix) options.append('OPENSSL=%s' % spec['openssl'].prefix) options.append('LINKFLAGS=-L%s/lib -L%s/lib' % - ( spec['expat'].prefix, spec['zlib'].prefix )) + (spec['expat'].prefix, spec['zlib'].prefix)) options.append('CPPFLAGS=-I%s/include -I%s/include' % - ( spec['expat'].prefix, spec['zlib'].prefix )) + (spec['expat'].prefix, spec['zlib'].prefix)) scons(*options) scons('install') From 992bcac7949883b74a9250d5835ac6d9de28a6c0 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Fri, 10 Jun 2016 16:26:02 -0700 Subject: [PATCH 343/375] changed compiler yaml format --- lib/spack/spack/compilers/__init__.py | 34 ++++----- lib/spack/spack/config.py | 24 +++---- lib/spack/spack/test/config.py | 82 +++++++++------------- lib/spack/spack/test/mock_packages_test.py | 72 +++++++------------ lib/spack/spack/yaml_version_check.py | 4 +- 5 files changed, 86 insertions(+), 130 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index d923d77da8d..3cf8f2297fc 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -73,16 +73,14 @@ def _to_dict(compiler): d = {} d['spec'] = str(compiler.spec) d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) - d['operating_system'] = compiler.operating_system.to_dict() + d['operating_system'] = str(compiler.operating_system) d['modules'] = compiler.modules - if not compiler.alias: - yaml_text = yaml.dump( - d, default_flow_style=True, width=sys.maxint) - sha = hashlib.sha1(yaml_text) - compiler.alias = base64.b32encode(sha.digest()).lower()[:8] + if compiler.alias: + d['alias'] = compiler.alias + return { - compiler.alias: d + 'compiler': d } @@ -91,11 +89,11 @@ def get_compiler_config(scope=None): """ def init_compiler_config(): """Compiler search used when Spack has no compilers.""" - config = {} compilers = find_compilers() + compilers_dict = [] for compiler in compilers: - config.update(_to_dict(compiler)) - spack.config.update_config('compilers', config, scope=scope) + compilers_dict.append(_to_dict(compiler)) + spack.config.update_config('compilers', compilers_dict, scope=scope) config = spack.config.get_config('compilers', scope=scope) # Update the configuration if there are currently no compilers @@ -127,7 +125,7 @@ def add_compilers_to_config(compilers, scope=None): """ compiler_config = get_compiler_config(scope) for compiler in compilers: - compiler_config = _to_dict(compiler) + compiler_config.append(_to_dict(compiler)) spack.config.update_config('compilers', compiler_config, scope) @@ -167,8 +165,8 @@ def all_compilers_config(scope=None): def all_compilers(scope=None): # Return compiler specs from the merged config. - return [spack.spec.CompilerSpec(s['spec']) - for s in all_compilers_config(scope).values()] + return [spack.spec.CompilerSpec(s['compiler']['spec']) + for s in all_compilers_config(scope)] def default_compiler(): @@ -230,11 +228,10 @@ def compilers_for_spec(compiler_spec, scope=None, **kwargs): def get_compilers(cspec): compilers = [] - for aka, cmp in config.items(): - if cmp['spec'] != str(cspec): + for items in config: + if items['compiler']['spec'] != str(cspec): continue - items = cmp - alias = aka + items = items['compiler'] if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): raise InvalidCompilerConfigurationError(cspec) @@ -258,6 +255,9 @@ def get_compilers(cspec): else: operating_system = None + + alias = items['alias'] if 'alias' in items else None + flags = {} for f in spack.spec.FlagMap.valid_compiler_flags(): if f in items: diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index ec04c817873..e51016998c9 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -146,11 +146,9 @@ 'additionalProperties': False, 'patternProperties': { 'compilers:?': { # optional colon for overriding site config. - 'type': 'object', - 'default': {}, - 'additionalProperties': False, - 'patternProperties': { - r'\w[\w-]*': { # alias + 'type': 'array', + 'items': { + 'compiler': { 'type': 'object', 'additionalProperties': False, 'required': ['paths', 'spec', 'modules', 'operating_system'], @@ -180,15 +178,10 @@ {'type' : 'null' }]}, 'ldlibs': { 'anyOf': [ {'type' : 'string' }, {'type' : 'null' }]}}}, - 'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*' - 'operating_system': { - 'type': 'object', - 'required': ['name', 'version'], - 'additionalProperties': False, - 'properties': { - 'name': {'type': 'string'}, - 'version': {'type': 'string'} - }}, + 'spec': { 'type': 'string'}, + 'operating_system': { 'type': 'string'}, + 'alias': { 'anyOf': [ {'type' : 'string'}, + {'type' : 'null' }]}, 'modules': { 'anyOf': [ {'type' : 'string'}, {'type' : 'null' }, {'type': 'array'}, @@ -591,8 +584,7 @@ def they_are(t): # Source list is prepended (for precedence) if they_are(list): - seen = set(source) - dest[:] = source + [x for x in dest if x not in seen] + dest[:] = source + [x for x in dest if x not in source] return dest # Source dict is merged into dest. diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 8fffc094378..252d77e66be 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -32,8 +32,8 @@ from spack.test.mock_packages_test import * # Some sample compiler config data -a_comps = { - 'gcc473': { +a_comps = [ + {'compiler': { 'paths': { "cc" : "/gcc473", "cxx": "/g++473", @@ -42,12 +42,9 @@ }, 'modules': None, 'spec': 'gcc@4.7.3', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'gcc450': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "/gcc450", "cxx": "/g++450", @@ -56,12 +53,9 @@ }, 'modules': None, 'spec': 'gcc@4.5.0', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'clang33': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "", "cxx": "", @@ -69,15 +63,12 @@ "fc" : '' }, 'modules': None, 'spec': 'clang@3.3', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - } -} + 'operating_system': 'CNL10' + }} +] -b_comps = { - 'icc100': { +b_comps = [ + {'compiler': { 'paths': { "cc" : "/icc100", "cxx": "/icp100", @@ -86,12 +77,9 @@ }, 'modules': None, 'spec': 'icc@10.0', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'icc111': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "/icc111", "cxx": "/icp111", @@ -100,12 +88,9 @@ }, 'modules': None, 'spec': 'icc@11.1', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - }, - 'clang33': { + 'operating_system': 'CNL10' + }}, + {'compiler': { 'paths': { "cc" : "", "cxx": "", @@ -113,12 +98,9 @@ "fc" : '' }, 'modules': None, 'spec': 'clang@3.3', - 'operating_system': { - 'name': 'CNL', - 'version': '10' - } - } -} + 'operating_system': 'CNL10' + }} +] # Some Sample repo data repos_low = [ "/some/path" ] @@ -143,15 +125,21 @@ def check_config(self, comps, *compiler_names): config = spack.config.get_config('compilers') compiler_list = ['cc', 'cxx', 'f77', 'fc'] param_list = ['modules', 'paths', 'spec', 'operating_system'] - for alias, compiler in config.items(): - if compiler['spec'] in compiler_names: + for compiler in config: + conf = compiler['compiler'] + if conf['spec'] in compiler_names: + comp = None + for c in comps: + if c['compiler']['spec'] == conf['spec']: + comp = c['compiler'] + break + if not comp: + self.fail('Bad config spec') for p in param_list: - expected = comps[alias][p] - actual = config[alias][p] - self.assertEqual(expected, actual) + self.assertEqual(conf[p], comp[p]) for c in compiler_list: - expected = comps[alias]['paths'][c] - actual = config[alias]['paths'][c] + expected = comp['paths'][c] + actual = conf['paths'][c] self.assertEqual(expected, actual) def test_write_list_in_memory(self): diff --git a/lib/spack/spack/test/mock_packages_test.py b/lib/spack/spack/test/mock_packages_test.py index 9c586bd8ce8..a56bd8ebdc9 100644 --- a/lib/spack/spack/test/mock_packages_test.py +++ b/lib/spack/spack/test/mock_packages_test.py @@ -46,132 +46,108 @@ mock_compiler_config = """\ compilers: - clang3.3GENLINUX: +- compiler: spec: clang@3.3 - operating_system: - name: {0} - version: '{1}' + operating_system: {0}{1} paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - gcc4.5GENLINUX: +- compiler: spec: gcc@4.5.0 - operating_system: - name: {0} - version: '{1}' + operating_system: {0}{1} paths: cc: /path/to/gcc cxx: /path/to/g++ f77: None fc: None modules: 'None' - clang3.3CNL: +- compiler: spec: clang@3.3 - operating_system: - name: CNL - version: '10' + operating_system: CNL10 paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - clang3.3SUSE: +- compiler: spec: clang@3.3 - operating_system: - name: SuSE - version: '11' + operating_system: SuSE11 paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - clang3.3RHL: +- compiler: spec: clang@3.3 - operating_system: - name: redhat - version: '6' + operating_system: redhat6 paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - clang3.3OSX: +- compiler: spec: clang@3.3 - operating_system: - name: yosemite - version: '10.10' + operating_system: yosemite paths: cc: /path/to/clang cxx: /path/to/clang++ f77: None fc: None modules: 'None' - gcc4.5.0CNL: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: CNL - version: '10' + operating_system: CNL10 spec: gcc@4.5.0 modules: 'None' - gcc4.5.0SUSE: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: SuSE - version: '11' + operating_system: SuSE11 spec: gcc@4.5.0 modules: 'None' - gcc4.5.0RHL: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: redhat - version: '6' + operating_system: redhat6 spec: gcc@4.5.0 modules: 'None' - gcc4.5.0OSX: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: yosemite - version: '10.10' + operating_system: yosemite spec: gcc@4.5.0 modules: 'None' - gcc4.5.0ELCAP: +- compiler: paths: cc: /path/to/gcc cxx: /path/to/g++ f77: /path/to/gfortran fc: /path/to/gfortran - operating_system: - name: elcapitan - version: '10.11' + operating_system: elcapitan spec: gcc@4.5.0 modules: 'None' - clang3.3ELCAP: +- compiler: spec: clang@3.3 - operating_system: - name: elcapitan - version: '10.11' + operating_system: elcapitan paths: cc: /path/to/clang cxx: /path/to/clang++ diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py index 7e7bae4edf9..5930eefafaa 100644 --- a/lib/spack/spack/yaml_version_check.py +++ b/lib/spack/spack/yaml_version_check.py @@ -44,9 +44,9 @@ def check_compiler_yaml_version(): data = syaml.load(f) if data: - compilers = data['compilers'].items() + compilers = data['compilers'] if len(compilers) > 0: - if 'operating_system' not in compilers[0][1]: + if 'operating_system' not in compilers[0]['compiler']: new_file = os.path.join(scope.path, '_old_compilers.yaml') tty.warn('%s in out of date compilers format. ' 'Moved to %s. Spack automatically generate ' From eb9b00b210b3c58dc1ea4fdf2b59cd1922266561 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 11 Jun 2016 12:44:21 +0200 Subject: [PATCH 344/375] versions : added unit tests for the new properties --- lib/spack/spack/test/versions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index a026403e2ef..892073feab8 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -423,3 +423,11 @@ def test_satisfaction_with_lists(self): self.assert_satisfies('4.8.0', '4.2, 4.3:4.8') self.assert_satisfies('4.8.2', '4.2, 4.3:4.8') + + def test_formatted_strings(self): + versions = '1.2.3', '1_2_3', '1-2-3' + for item in versions: + v = Version(item) + self.assertEqual(v.dotted, '1.2.3') + self.assertEqual(v.dashed, '1-2-3') + self.assertEqual(v.underscored, '1_2_3') \ No newline at end of file From 3737169450228643848572ba2605d9f20aba2039 Mon Sep 17 00:00:00 2001 From: alalazo Date: Sat, 11 Jun 2016 12:57:46 +0200 Subject: [PATCH 345/375] qa : fixed style and a bug reported by flake8 lib/spack/spack/test/versions.py:313: [F811] redefinition of unused 'test_union_with_containment' from line 306 Flake8 found errors. --- lib/spack/spack/test/versions.py | 48 ++------------------------------ 1 file changed, 3 insertions(+), 45 deletions(-) diff --git a/lib/spack/spack/test/versions.py b/lib/spack/spack/test/versions.py index 892073feab8..4624f901c8b 100644 --- a/lib/spack/spack/test/versions.py +++ b/lib/spack/spack/test/versions.py @@ -43,7 +43,6 @@ def assert_ver_lt(self, a, b): self.assertFalse(a > b) self.assertFalse(a >= b) - def assert_ver_gt(self, a, b): a, b = ver(a), ver(b) self.assertTrue(a > b) @@ -53,7 +52,6 @@ def assert_ver_gt(self, a, b): self.assertFalse(a < b) self.assertFalse(a <= b) - def assert_ver_eq(self, a, b): a, b = ver(a), ver(b) self.assertFalse(a > b) @@ -63,55 +61,43 @@ def assert_ver_eq(self, a, b): self.assertFalse(a < b) self.assertTrue(a <= b) - def assert_in(self, needle, haystack): self.assertTrue(ver(needle) in ver(haystack)) - def assert_not_in(self, needle, haystack): self.assertFalse(ver(needle) in ver(haystack)) - def assert_canonical(self, canonical_list, version_list): self.assertEqual(ver(canonical_list), ver(version_list)) - def assert_overlaps(self, v1, v2): self.assertTrue(ver(v1).overlaps(ver(v2))) - def assert_no_overlap(self, v1, v2): self.assertFalse(ver(v1).overlaps(ver(v2))) - def assert_satisfies(self, v1, v2): self.assertTrue(ver(v1).satisfies(ver(v2))) - def assert_does_not_satisfy(self, v1, v2): self.assertFalse(ver(v1).satisfies(ver(v2))) - def check_intersection(self, expected, a, b): self.assertEqual(ver(expected), ver(a).intersection(ver(b))) - def check_union(self, expected, a, b): self.assertEqual(ver(expected), ver(a).union(ver(b))) - def test_two_segments(self): self.assert_ver_eq('1.0', '1.0') self.assert_ver_lt('1.0', '2.0') self.assert_ver_gt('2.0', '1.0') - def test_three_segments(self): self.assert_ver_eq('2.0.1', '2.0.1') self.assert_ver_lt('2.0', '2.0.1') self.assert_ver_gt('2.0.1', '2.0') - def test_alpha(self): # TODO: not sure whether I like this. 2.0.1a is *usually* # TODO: less than 2.0.1, but special-casing it makes version @@ -120,7 +106,6 @@ def test_alpha(self): self.assert_ver_gt('2.0.1a', '2.0.1') self.assert_ver_lt('2.0.1', '2.0.1a') - def test_patch(self): self.assert_ver_eq('5.5p1', '5.5p1') self.assert_ver_lt('5.5p1', '5.5p2') @@ -129,7 +114,6 @@ def test_patch(self): self.assert_ver_lt('5.5p1', '5.5p10') self.assert_ver_gt('5.5p10', '5.5p1') - def test_num_alpha_with_no_separator(self): self.assert_ver_lt('10xyz', '10.1xyz') self.assert_ver_gt('10.1xyz', '10xyz') @@ -137,7 +121,6 @@ def test_num_alpha_with_no_separator(self): self.assert_ver_lt('xyz10', 'xyz10.1') self.assert_ver_gt('xyz10.1', 'xyz10') - def test_alpha_with_dots(self): self.assert_ver_eq('xyz.4', 'xyz.4') self.assert_ver_lt('xyz.4', '8') @@ -145,30 +128,25 @@ def test_alpha_with_dots(self): self.assert_ver_lt('xyz.4', '2') self.assert_ver_gt('2', 'xyz.4') - def test_nums_and_patch(self): self.assert_ver_lt('5.5p2', '5.6p1') self.assert_ver_gt('5.6p1', '5.5p2') self.assert_ver_lt('5.6p1', '6.5p1') self.assert_ver_gt('6.5p1', '5.6p1') - def test_rc_versions(self): self.assert_ver_gt('6.0.rc1', '6.0') self.assert_ver_lt('6.0', '6.0.rc1') - def test_alpha_beta(self): self.assert_ver_gt('10b2', '10a1') self.assert_ver_lt('10a2', '10b2') - def test_double_alpha(self): self.assert_ver_eq('1.0aa', '1.0aa') self.assert_ver_lt('1.0a', '1.0aa') self.assert_ver_gt('1.0aa', '1.0a') - def test_padded_numbers(self): self.assert_ver_eq('10.0001', '10.0001') self.assert_ver_eq('10.0001', '10.1') @@ -176,24 +154,20 @@ def test_padded_numbers(self): self.assert_ver_lt('10.0001', '10.0039') self.assert_ver_gt('10.0039', '10.0001') - def test_close_numbers(self): self.assert_ver_lt('4.999.9', '5.0') self.assert_ver_gt('5.0', '4.999.9') - def test_date_stamps(self): self.assert_ver_eq('20101121', '20101121') self.assert_ver_lt('20101121', '20101122') self.assert_ver_gt('20101122', '20101121') - def test_underscores(self): self.assert_ver_eq('2_0', '2_0') self.assert_ver_eq('2.0', '2_0') self.assert_ver_eq('2_0', '2.0') - def test_rpm_oddities(self): self.assert_ver_eq('1b.fc17', '1b.fc17') self.assert_ver_lt('1b.fc17', '1.fc17') @@ -202,7 +176,6 @@ def test_rpm_oddities(self): self.assert_ver_gt('1g.fc17', '1.fc17') self.assert_ver_lt('1.fc17', '1g.fc17') - # Stuff below here is not taken from RPM's tests and is # unique to spack def test_version_ranges(self): @@ -214,7 +187,6 @@ def test_version_ranges(self): self.assert_ver_lt('1.2:1.4', '1.5:1.6') self.assert_ver_gt('1.5:1.6', '1.2:1.4') - def test_contains(self): self.assert_in('1.3', '1.2:1.4') self.assert_in('1.2.5', '1.2:1.4') @@ -233,7 +205,6 @@ def test_contains(self): self.assert_in('1.4.1', '1.2.7:1.4') self.assert_not_in('1.4.1', '1.2.7:1.4.0') - def test_in_list(self): self.assert_in('1.2', ['1.5', '1.2', '1.3']) self.assert_in('1.2.5', ['1.5', '1.2:1.3']) @@ -245,7 +216,6 @@ def test_in_list(self): self.assert_not_in('1.2.5:1.5', ['1.5', '1.2:1.3']) self.assert_not_in('1.1:1.2.5', ['1.5', '1.2:1.3']) - def test_ranges_overlap(self): self.assert_overlaps('1.2', '1.2') self.assert_overlaps('1.2.1', '1.2.1') @@ -262,7 +232,6 @@ def test_ranges_overlap(self): self.assert_overlaps(':', '1.6:1.9') self.assert_overlaps('1.6:1.9', ':') - def test_overlap_with_containment(self): self.assert_in('1.6.5', '1.6') self.assert_in('1.6.5', ':1.6') @@ -273,7 +242,6 @@ def test_overlap_with_containment(self): self.assert_not_in(':1.6', '1.6.5') self.assert_in('1.6.5', ':1.6') - def test_lists_overlap(self): self.assert_overlaps('1.2b:1.7,5', '1.6:1.9,1') self.assert_overlaps('1,2,3,4,5', '3,4,5,6,7') @@ -287,7 +255,6 @@ def test_lists_overlap(self): self.assert_no_overlap('1,2,3,4,5', '6,7') self.assert_no_overlap('1,2,3,4,5', '6:7') - def test_canonicalize_list(self): self.assert_canonical(['1.2', '1.3', '1.4'], ['1.2', '1.3', '1.3', '1.4']) @@ -316,7 +283,6 @@ def test_canonicalize_list(self): self.assert_canonical([':'], [':,1.3, 1.3.1,1.3.9,1.4 : 1.5 , 1.3 : 1.4']) - def test_intersection(self): self.check_intersection('2.5', '1.0:2.5', '2.5:3.0') @@ -325,12 +291,11 @@ def test_intersection(self): self.check_intersection('0:1', ':', '0:1') self.check_intersection(['1.0', '2.5:2.7'], - ['1.0:2.7'], ['2.5:3.0','1.0']) + ['1.0:2.7'], ['2.5:3.0', '1.0']) self.check_intersection(['2.5:2.7'], - ['1.1:2.7'], ['2.5:3.0','1.0']) + ['1.1:2.7'], ['2.5:3.0', '1.0']) self.check_intersection(['0:1'], [':'], ['0:1']) - def test_intersect_with_containment(self): self.check_intersection('1.6.5', '1.6.5', ':1.6') self.check_intersection('1.6.5', ':1.6', '1.6.5') @@ -338,7 +303,6 @@ def test_intersect_with_containment(self): self.check_intersection('1.6:1.6.5', ':1.6.5', '1.6') self.check_intersection('1.6:1.6.5', '1.6', ':1.6.5') - def test_union_with_containment(self): self.check_union(':1.6', '1.6.5', ':1.6') self.check_union(':1.6', ':1.6', '1.6.5') @@ -346,8 +310,6 @@ def test_union_with_containment(self): self.check_union(':1.6', ':1.6.5', '1.6') self.check_union(':1.6', '1.6', ':1.6.5') - - def test_union_with_containment(self): self.check_union(':', '1.0:', ':2.0') self.check_union('1:4', '1:3', '2:4') @@ -356,7 +318,6 @@ def test_union_with_containment(self): # Tests successor/predecessor case. self.check_union('1:4', '1:2', '3:4') - def test_basic_version_satisfaction(self): self.assert_satisfies('4.7.3', '4.7.3') @@ -372,7 +333,6 @@ def test_basic_version_satisfaction(self): self.assert_does_not_satisfy('4.8', '4.9') self.assert_does_not_satisfy('4', '4.9') - def test_basic_version_satisfaction_in_lists(self): self.assert_satisfies(['4.7.3'], ['4.7.3']) @@ -388,7 +348,6 @@ def test_basic_version_satisfaction_in_lists(self): self.assert_does_not_satisfy(['4.8'], ['4.9']) self.assert_does_not_satisfy(['4'], ['4.9']) - def test_version_range_satisfaction(self): self.assert_satisfies('4.7b6', '4.3:4.7') self.assert_satisfies('4.3.0', '4.3:4.7') @@ -400,7 +359,6 @@ def test_version_range_satisfaction(self): self.assert_satisfies('4.7b6', '4.3:4.7') self.assert_does_not_satisfy('4.8.0', '4.3:4.7') - def test_version_range_satisfaction_in_lists(self): self.assert_satisfies(['4.7b6'], ['4.3:4.7']) self.assert_satisfies(['4.3.0'], ['4.3:4.7']) @@ -430,4 +388,4 @@ def test_formatted_strings(self): v = Version(item) self.assertEqual(v.dotted, '1.2.3') self.assertEqual(v.dashed, '1-2-3') - self.assertEqual(v.underscored, '1_2_3') \ No newline at end of file + self.assertEqual(v.underscored, '1_2_3') From 953b65abccf392e2a565fe9c1b3c1101f0cb5b66 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Sat, 11 Jun 2016 16:41:39 -0400 Subject: [PATCH 346/375] Do not pass "/usr" or "/usr/local" as path to configure --- var/spack/repos/builtin/packages/openmpi/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py index 4e465e1784d..0638628a6c7 100644 --- a/var/spack/repos/builtin/packages/openmpi/package.py +++ b/var/spack/repos/builtin/packages/openmpi/package.py @@ -142,7 +142,7 @@ def install(self, spec, prefix): ]) if '+verbs' in spec: path = _verbs_dir() - if path is not None: + if path is not None and path not in ('/usr', '/usr/local'): config_args.append('--with-%s=%s' % (self.verbs, path)) else: config_args.append('--with-%s' % self.verbs) From b89bcdff62e10223a737c0ac26feb996c8202941 Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Sun, 12 Jun 2016 19:48:45 -0600 Subject: [PATCH 347/375] Make Boost an optional feature of Trilinos. I have been unable to build Boost with some toolset/platform combinations. The users I support don't need the Boost features in Trilinos, so I have been manually hacking trilinos/package.py to disable boost. I would like to propose that these changes be added to the released version. + Add boost as a variant, default to true so there is no change to existing behavior. + Move the CMake configure options related to Boost into a conditional block that is triggered by "if '+boost' in spec". --- .../repos/builtin/packages/trilinos/package.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 1eaec864050..28c0bea7c62 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -58,11 +58,12 @@ class Trilinos(Package): variant('python', default=False, description='Build python wrappers') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds a debug version of the libraries') + variant('boost', default=True, description='Compile with Boost') # Everything should be compiled with -fpic depends_on('blas') depends_on('lapack') - depends_on('boost') + depends_on('boost', when='+boost') depends_on('matio') depends_on('glm') depends_on('swig') @@ -121,9 +122,6 @@ def install(self, spec, prefix): '-DTPL_ENABLE_LAPACK=ON', '-DLAPACK_LIBRARY_NAMES=lapack', '-DLAPACK_LIBRARY_DIRS=%s' % spec['lapack'].prefix, - '-DTPL_ENABLE_Boost:BOOL=ON', - '-DBoost_INCLUDE_DIRS:PATH=%s' % spec['boost'].prefix.include, - '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib, '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', '-DTrilinos_ENABLE_CXX11:BOOL=ON', '-DTPL_ENABLE_Netcdf:BOOL=ON', @@ -131,6 +129,14 @@ def install(self, spec, prefix): '-DTPL_ENABLE_HDF5:BOOL=%s' % ('ON' if '+hdf5' in spec else 'OFF'), ]) + if '+boost' in spec: + options.extend([ + '-DTPL_ENABLE_Boost:BOOL=ON', + '-DBoost_INCLUDE_DIRS:PATH=%s' % spec['boost'].prefix.include, + '-DBoost_LIBRARY_DIRS:PATH=%s' % spec['boost'].prefix.lib + ]) + else: + options.extend(['-DTPL_ENABLE_Boost:BOOL=OFF']) # Fortran lib libgfortran = os.path.dirname (os.popen('%s --print-file-name libgfortran.a' % join_path(mpi_bin,'mpif90') ).read()) options.extend([ From 31d2c386875364a9524727279575955ec32972cd Mon Sep 17 00:00:00 2001 From: "Kelly (KT) Thompson" Date: Sun, 12 Jun 2016 19:56:29 -0600 Subject: [PATCH 348/375] Update formatting to meet code style requirements. --- .../builtin/packages/trilinos/package.py | 193 ++++++++++-------- 1 file changed, 113 insertions(+), 80 deletions(-) diff --git a/var/spack/repos/builtin/packages/trilinos/package.py b/var/spack/repos/builtin/packages/trilinos/package.py index 28c0bea7c62..6913d79dcc9 100644 --- a/var/spack/repos/builtin/packages/trilinos/package.py +++ b/var/spack/repos/builtin/packages/trilinos/package.py @@ -23,18 +23,23 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import os, sys, glob +import os +import sys -# Trilinos is complicated to build, as an inspiration a couple of links to other repositories which build it: +# Trilinos is complicated to build, as an inspiration a couple of links to +# other repositories which build it: # https://github.com/hpcugent/easybuild-easyblocks/blob/master/easybuild/easyblocks/t/trilinos.py#L111 # https://github.com/koecher/candi/blob/master/deal.II-toolchain/packages/trilinos.package # https://gitlab.com/configurations/cluster-config/blob/master/trilinos.sh -# https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb -# and some relevant documentation/examples: +# https://github.com/Homebrew/homebrew-science/blob/master/trilinos.rb and some +# relevant documentation/examples: # https://github.com/trilinos/Trilinos/issues/175 + + class Trilinos(Package): - """The Trilinos Project is an effort to develop algorithms and enabling technologies within an object-oriented - software framework for the solution of large-scale, complex multi-physics engineering and scientific problems. + """The Trilinos Project is an effort to develop algorithms and enabling + technologies within an object-oriented software framework for the solution + of large-scale, complex multi-physics engineering and scientific problems. A unique design feature of Trilinos is its focus on packages. """ homepage = "https://trilinos.org/" @@ -54,7 +59,8 @@ class Trilinos(Package): variant('hypre', default=True, description='Compile with Hypre preconditioner') variant('hdf5', default=True, description='Compile with HDF5') variant('suite-sparse', default=True, description='Compile with SuiteSparse solvers') - # not everyone has py-numpy activated, keep it disabled by default to avoid configure errors + # not everyone has py-numpy activated, keep it disabled by default to avoid + # configure errors variant('python', default=False, description='Build python wrappers') variant('shared', default=True, description='Enables the build of shared libraries') variant('debug', default=False, description='Builds a debug version of the libraries') @@ -67,37 +73,37 @@ class Trilinos(Package): depends_on('matio') depends_on('glm') depends_on('swig') - depends_on('metis@5:',when='+metis') - depends_on('suite-sparse',when='+suite-sparse') + depends_on('metis@5:', when='+metis') + depends_on('suite-sparse', when='+suite-sparse') # MPI related dependencies depends_on('mpi') depends_on('netcdf+mpi') - depends_on('parmetis',when='+metis') - # Trilinos' Tribits config system is limited which makes it - # very tricky to link Amesos with static MUMPS, see + depends_on('parmetis', when='+metis') + # Trilinos' Tribits config system is limited which makes it very tricky to + # link Amesos with static MUMPS, see # https://trilinos.org/docs/dev/packages/amesos2/doc/html/classAmesos2_1_1MUMPS.html - # One could work it out by getting linking flags from mpif90 --showme:link (or alike) - # and adding results to -DTrilinos_EXTRA_LINK_FLAGS - # together with Blas and Lapack and ScaLAPACK and Blacs and -lgfortran and - # it may work at the end. But let's avoid all this by simply using shared libs - depends_on('mumps@5.0:+mpi+shared',when='+mumps') - depends_on('scalapack',when='+mumps') - depends_on('superlu-dist',when='+superlu-dist') - depends_on('hypre~internal-superlu',when='+hypre') - depends_on('hdf5+mpi',when='+hdf5') - - depends_on('python',when='+python') + # One could work it out by getting linking flags from mpif90 --showme:link + # (or alike) and adding results to -DTrilinos_EXTRA_LINK_FLAGS together + # with Blas and Lapack and ScaLAPACK and Blacs and -lgfortran and it may + # work at the end. But let's avoid all this by simply using shared libs + depends_on('mumps@5.0:+mpi+shared', when='+mumps') + depends_on('scalapack', when='+mumps') + depends_on('superlu-dist', when='+superlu-dist') + depends_on('hypre~internal-superlu', when='+hypre') + depends_on('hdf5+mpi', when='+hdf5') + depends_on('python', when='+python') patch('umfpack_from_suitesparse.patch') # check that the combination of variants makes sense def variants_check(self): if '+superlu-dist' in self.spec and self.spec.satisfies('@:11.4.3'): - # For Trilinos v11 we need to force SuperLUDist=OFF, - # since only the deprecated SuperLUDist v3.3 together with an Amesos patch - # is working. - raise RuntimeError('The superlu-dist variant can only be used with Trilinos @12.0.1:') + # For Trilinos v11 we need to force SuperLUDist=OFF, since only the + # deprecated SuperLUDist v3.3 together with an Amesos patch is + # working. + raise RuntimeError('The superlu-dist variant can only be used' + + ' with Trilinos @12.0.1:') def install(self, spec, prefix): self.variants_check() @@ -107,27 +113,32 @@ def install(self, spec, prefix): options.extend(std_cmake_args) mpi_bin = spec['mpi'].prefix.bin - options.extend(['-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', - '-DTrilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=ON', - '-DTrilinos_VERBOSE_CONFIGURE:BOOL=OFF', - '-DTrilinos_ENABLE_TESTS:BOOL=OFF', - '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', - '-DCMAKE_BUILD_TYPE:STRING=%s' % ('DEBUG' if '+debug' in spec else 'RELEASE'), - '-DBUILD_SHARED_LIBS:BOOL=%s' % ('ON' if '+shared' in spec else 'OFF'), - '-DTPL_ENABLE_MPI:BOOL=ON', - '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, - '-DTPL_ENABLE_BLAS=ON', - '-DBLAS_LIBRARY_NAMES=blas', # FIXME: don't hardcode names - '-DBLAS_LIBRARY_DIRS=%s' % spec['blas'].prefix.lib, - '-DTPL_ENABLE_LAPACK=ON', - '-DLAPACK_LIBRARY_NAMES=lapack', - '-DLAPACK_LIBRARY_DIRS=%s' % spec['lapack'].prefix, - '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', - '-DTrilinos_ENABLE_CXX11:BOOL=ON', - '-DTPL_ENABLE_Netcdf:BOOL=ON', - '-DTPL_ENABLE_HYPRE:BOOL=%s' % ('ON' if '+hypre' in spec else 'OFF'), - '-DTPL_ENABLE_HDF5:BOOL=%s' % ('ON' if '+hdf5' in spec else 'OFF'), - ]) + options.extend([ + '-DTrilinos_ENABLE_ALL_PACKAGES:BOOL=ON', + '-DTrilinos_ENABLE_ALL_OPTIONAL_PACKAGES:BOOL=ON', + '-DTrilinos_VERBOSE_CONFIGURE:BOOL=OFF', + '-DTrilinos_ENABLE_TESTS:BOOL=OFF', + '-DTrilinos_ENABLE_EXAMPLES:BOOL=OFF', + '-DCMAKE_BUILD_TYPE:STRING=%s' % ( + 'DEBUG' if '+debug' in spec else 'RELEASE'), + '-DBUILD_SHARED_LIBS:BOOL=%s' % ( + 'ON' if '+shared' in spec else 'OFF'), + '-DTPL_ENABLE_MPI:BOOL=ON', + '-DMPI_BASE_DIR:PATH=%s' % spec['mpi'].prefix, + '-DTPL_ENABLE_BLAS=ON', + '-DBLAS_LIBRARY_NAMES=blas', # FIXME: don't hardcode names + '-DBLAS_LIBRARY_DIRS=%s' % spec['blas'].prefix.lib, + '-DTPL_ENABLE_LAPACK=ON', + '-DLAPACK_LIBRARY_NAMES=lapack', + '-DLAPACK_LIBRARY_DIRS=%s' % spec['lapack'].prefix, + '-DTrilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON', + '-DTrilinos_ENABLE_CXX11:BOOL=ON', + '-DTPL_ENABLE_Netcdf:BOOL=ON', + '-DTPL_ENABLE_HYPRE:BOOL=%s' % ( + 'ON' if '+hypre' in spec else 'OFF'), + '-DTPL_ENABLE_HDF5:BOOL=%s' % ( + 'ON' if '+hdf5' in spec else 'OFF'), + ]) if '+boost' in spec: options.extend([ @@ -137,29 +148,40 @@ def install(self, spec, prefix): ]) else: options.extend(['-DTPL_ENABLE_Boost:BOOL=OFF']) + # Fortran lib - libgfortran = os.path.dirname (os.popen('%s --print-file-name libgfortran.a' % join_path(mpi_bin,'mpif90') ).read()) + libgfortran = os.path.dirname(os.popen( + '%s --print-file-name libgfortran.a' % + join_path(mpi_bin, 'mpif90')).read()) options.extend([ - '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % libgfortran, + '-DTrilinos_EXTRA_LINK_FLAGS:STRING=-L%s/ -lgfortran' % ( + libgfortran), '-DTrilinos_ENABLE_Fortran=ON' ]) # for build-debug only: - #options.extend([ - # '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' - #]) + # options.extend([ + # '-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE' + # ]) # suite-sparse related if '+suite-sparse' in spec: options.extend([ - '-DTPL_ENABLE_Cholmod:BOOL=OFF', # FIXME: Trilinos seems to be looking for static libs only, patch CMake TPL file? - #'-DTPL_ENABLE_Cholmod:BOOL=ON', - #'-DCholmod_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, - #'-DCholmod_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, + # FIXME: Trilinos seems to be looking for static libs only, + # patch CMake TPL file? + '-DTPL_ENABLE_Cholmod:BOOL=OFF', + # '-DTPL_ENABLE_Cholmod:BOOL=ON', + # '-DCholmod_LIBRARY_DIRS:PATH=%s' % ( + # spec['suite-sparse'].prefix.lib, + # '-DCholmod_INCLUDE_DIRS:PATH=%s' % ( + # spec['suite-sparse'].prefix.include, '-DTPL_ENABLE_UMFPACK:BOOL=ON', - '-DUMFPACK_LIBRARY_DIRS:PATH=%s' % spec['suite-sparse'].prefix.lib, - '-DUMFPACK_INCLUDE_DIRS:PATH=%s' % spec['suite-sparse'].prefix.include, - '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;suitesparseconfig' + '-DUMFPACK_LIBRARY_DIRS:PATH=%s' % ( + spec['suite-sparse'].prefix.lib), + '-DUMFPACK_INCLUDE_DIRS:PATH=%s' % ( + spec['suite-sparse'].prefix.include), + '-DUMFPACK_LIBRARY_NAMES=umfpack;amd;colamd;cholmod;' + + 'suitesparseconfig' ]) else: options.extend([ @@ -175,9 +197,11 @@ def install(self, spec, prefix): '-DMETIS_LIBRARY_NAMES=metis', '-DTPL_METIS_INCLUDE_DIRS=%s' % spec['metis'].prefix.include, '-DTPL_ENABLE_ParMETIS:BOOL=ON', - '-DParMETIS_LIBRARY_DIRS=%s;%s' % (spec['parmetis'].prefix.lib,spec['metis'].prefix.lib), + '-DParMETIS_LIBRARY_DIRS=%s;%s' % ( + spec['parmetis'].prefix.lib, spec['metis'].prefix.lib), '-DParMETIS_LIBRARY_NAMES=parmetis;metis', - '-DTPL_ParMETIS_INCLUDE_DIRS=%s' % spec['parmetis'].prefix.include + '-DTPL_ParMETIS_INCLUDE_DIRS=%s' % ( + spec['parmetis'].prefix.include) ]) else: options.extend([ @@ -190,11 +214,14 @@ def install(self, spec, prefix): options.extend([ '-DTPL_ENABLE_MUMPS:BOOL=ON', '-DMUMPS_LIBRARY_DIRS=%s' % spec['mumps'].prefix.lib, - '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord', # order is important! + # order is important! + '-DMUMPS_LIBRARY_NAMES=dmumps;mumps_common;pord', '-DTPL_ENABLE_SCALAPACK:BOOL=ON', - '-DSCALAPACK_LIBRARY_NAMES=scalapack' # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 + # FIXME: for MKL it's mkl_scalapack_lp64;mkl_blacs_mpich_lp64 + '-DSCALAPACK_LIBRARY_NAMES=scalapack' ]) - # see https://github.com/trilinos/Trilinos/blob/master/packages/amesos/README-MUMPS + # see + # https://github.com/trilinos/Trilinos/blob/master/packages/amesos/README-MUMPS cxx_flags.extend([ '-DMUMPS_5_0' ]) @@ -207,16 +234,20 @@ def install(self, spec, prefix): # superlu-dist: if '+superlu-dist' in spec: # Amesos, conflicting types of double and complex SLU_D - # see https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html - # and https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html + # see + # https://trilinos.org/pipermail/trilinos-users/2015-March/004731.html + # and + # https://trilinos.org/pipermail/trilinos-users/2015-March/004802.html options.extend([ '-DTeuchos_ENABLE_COMPLEX:BOOL=OFF', '-DKokkosTSQR_ENABLE_Complex:BOOL=OFF' ]) options.extend([ '-DTPL_ENABLE_SuperLUDist:BOOL=ON', - '-DSuperLUDist_LIBRARY_DIRS=%s' % spec['superlu-dist'].prefix.lib, - '-DSuperLUDist_INCLUDE_DIRS=%s' % spec['superlu-dist'].prefix.include + '-DSuperLUDist_LIBRARY_DIRS=%s' % + spec['superlu-dist'].prefix.lib, + '-DSuperLUDist_INCLUDE_DIRS=%s' % + spec['superlu-dist'].prefix.include ]) if spec.satisfies('^superlu-dist@4.0:'): options.extend([ @@ -227,7 +258,6 @@ def install(self, spec, prefix): '-DTPL_ENABLE_SuperLUDist:BOOL=OFF', ]) - # python if '+python' in spec: options.extend([ @@ -254,23 +284,26 @@ def install(self, spec, prefix): '-DTrilinos_ENABLE_FEI=OFF' ]) - with working_dir('spack-build', create=True): cmake('..', *options) make() make('install') - # When trilinos is built with Python, libpytrilinos is included through - # cmake configure files. Namely, Trilinos_LIBRARIES in TrilinosConfig.cmake - # contains pytrilinos. This leads to a run-time error: - # Symbol not found: _PyBool_Type - # and prevents Trilinos to be used in any C++ code, which links executable - # against the libraries listed in Trilinos_LIBRARIES. - # See https://github.com/Homebrew/homebrew-science/issues/2148#issuecomment-103614509 + # When trilinos is built with Python, libpytrilinos is included + # through cmake configure files. Namely, Trilinos_LIBRARIES in + # TrilinosConfig.cmake contains pytrilinos. This leads to a + # run-time error: Symbol not found: _PyBool_Type and prevents + # Trilinos to be used in any C++ code, which links executable + # against the libraries listed in Trilinos_LIBRARIES. See + # https://github.com/Homebrew/homebrew-science/issues/2148#issuecomment-103614509 # A workaround it to remove PyTrilinos from the COMPONENTS_LIST : if '+python' in self.spec: - filter_file(r'(SET\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)', (r'\1\3'), '%s/cmake/Trilinos/TrilinosConfig.cmake' % prefix.lib) + filter_file(r'(SET\(COMPONENTS_LIST.*)(PyTrilinos;)(.*)', + (r'\1\3'), + '%s/cmake/Trilinos/TrilinosConfig.cmake' % + prefix.lib) - # The shared libraries are not installed correctly on Darwin; correct this + # The shared libraries are not installed correctly on Darwin; + # correct this if (sys.platform == 'darwin') and ('+shared' in spec): fix_darwin_install_name(prefix.lib) From 52160bff187a7d33f6e6041417ced2968e9c13ea Mon Sep 17 00:00:00 2001 From: David Boehme Date: Mon, 13 Jun 2016 11:47:28 -0700 Subject: [PATCH 349/375] Use https for github --- var/spack/repos/builtin/packages/caliper/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/spack/repos/builtin/packages/caliper/package.py b/var/spack/repos/builtin/packages/caliper/package.py index 4b8fe0d8afd..a424c73859f 100644 --- a/var/spack/repos/builtin/packages/caliper/package.py +++ b/var/spack/repos/builtin/packages/caliper/package.py @@ -34,7 +34,7 @@ class Caliper(Package): homepage = "https://github.com/LLNL/Caliper" url = "" - version('master', git='ssh://git@github.com:LLNL/Caliper.git') + version('master', git='https://github.com/LLNL/Caliper.git') variant('mpi', default=False, description='Enable MPI function wrappers.') From e6ec3729260b666483b02e38501671bf32ec3af5 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 14 Jun 2016 09:59:48 -0400 Subject: [PATCH 350/375] Add packages c-blosc and hdf5-blosc c-blosc provides the Blosc compression filter, and hdf5-blosc provides a plugin for HDF5 to use that filter. --- .../repos/builtin/packages/c-blosc/package.py | 47 +++++++++++ .../builtin/packages/hdf5-blosc/package.py | 77 +++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 var/spack/repos/builtin/packages/c-blosc/package.py create mode 100644 var/spack/repos/builtin/packages/hdf5-blosc/package.py diff --git a/var/spack/repos/builtin/packages/c-blosc/package.py b/var/spack/repos/builtin/packages/c-blosc/package.py new file mode 100644 index 00000000000..16e8c89811c --- /dev/null +++ b/var/spack/repos/builtin/packages/c-blosc/package.py @@ -0,0 +1,47 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +from spack import * + +class CBlosc(Package): + """Blosc, an extremely fast, multi-threaded, meta-compressor library""" + homepage = "http://www.blosc.org" + url = "https://github.com/Blosc/c-blosc/archive/v1.9.2.tar.gz" + + version('1.9.2', 'dd2d83069d74b36b8093f1c6b49defc5') + version('1.9.1', '7d708d3daadfacf984a87b71b1734ce2') + version('1.9.0', 'e4c1dc8e2c468e5cfa2bf05eeee5357a') + version('1.8.1', 'd73d5be01359cf271e9386c90dcf5b05') + version('1.8.0', '5b92ecb287695ba20cc33d30bf221c4f') + + depends_on("cmake") + depends_on("snappy") + depends_on("zlib") + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py new file mode 100644 index 00000000000..897c079e4e7 --- /dev/null +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -0,0 +1,77 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import os +import shutil +import sys + +from spack import * + +class Hdf5Blosc(Package): + """Blosc filter for HDF5""" + homepage = "https://github.com/Blosc/hdf5-blosc" + url = "https://github.com/Blosc/hdf5-blosc/archive/master.zip" + + version('master', '02c04acbf4bec66ec8a35bf157d1c9de') + + depends_on("c-blosc") + depends_on("hdf5") + depends_on("libtool") + + parallel = False + + def install(self, spec, prefix): + cmake('.', *std_cmake_args) + + make() + make("install") + if sys.platform == 'darwin': + fix_darwin_install_name(prefix.lib) + + # Build and install plugin manually + # The plugin requires at least HDF5 1.8.11: + if spec['hdf5'].satisfies('@1.8.11:'): + # The plugin does not yet work with Windows: + if sys.platform not in ('win32', 'cygwin'): + # cc = which('cc') + libtool = Executable(join_path(spec['libtool'].prefix.bin, + 'libtool')) + with working_dir('src'): + libtool('--mode=compile', '--tag=CC', + 'cc', '-g', '-O', '-c', 'blosc_plugin.c') + libtool('--mode=link', '--tag=CC', + 'cc', '-g', '-O', + '-rpath', prefix.lib, + '-o', 'libblosc_plugin.la', 'blosc_plugin.lo') + shlibext = 'so' if sys.platform!='darwin' else 'dylib' + shlib0 = 'libblosc_plugin.0.%s' % shlibext + shutil.copyfile(join_path('.libs', shlib0), + join_path(prefix.lib, shlib0)) + shlib = 'libblosc_plugin.%s' % shlibext + os.symlink(shlib0, join_path(prefix.lib, shlib)) + + def setup_dependent_environment(self, spack_env, run_env, dependent_spec): + spack_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) + run_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) From ca5180f7a24d08d49f046a4efa8ef5b48201211d Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 14 Jun 2016 17:03:00 -0400 Subject: [PATCH 351/375] Correct Blosc install on Darwin --- .../repos/builtin/packages/c-blosc/package.py | 4 + .../builtin/packages/hdf5-blosc/package.py | 177 +++++++++++++++--- 2 files changed, 153 insertions(+), 28 deletions(-) diff --git a/var/spack/repos/builtin/packages/c-blosc/package.py b/var/spack/repos/builtin/packages/c-blosc/package.py index 16e8c89811c..dee332be14c 100644 --- a/var/spack/repos/builtin/packages/c-blosc/package.py +++ b/var/spack/repos/builtin/packages/c-blosc/package.py @@ -23,6 +23,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## +import sys + from spack import * class CBlosc(Package): @@ -45,3 +47,5 @@ def install(self, spec, prefix): make() make("install") + if sys.platform == 'darwin': + fix_darwin_install_name(prefix.lib) diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py index 897c079e4e7..680bc41c41a 100644 --- a/var/spack/repos/builtin/packages/hdf5-blosc/package.py +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -43,35 +43,156 @@ class Hdf5Blosc(Package): parallel = False def install(self, spec, prefix): - cmake('.', *std_cmake_args) + # The included cmake recipe doesn"t work for Darwin + # cmake(".", *std_cmake_args) + # + # make() + # make("install") + # if sys.platform == "darwin": + # fix_darwin_install_name(prefix.lib) - make() - make("install") - if sys.platform == 'darwin': - fix_darwin_install_name(prefix.lib) + libtool = Executable(join_path(spec["libtool"].prefix.bin, "libtool")) + if "+mpi" in spec["hdf5"]: + cc = "mpicc" + else: + cc = "cc" + shlibext = "so" if sys.platform!="darwin" else "dylib" + mkdirp(prefix.include) + mkdirp(prefix.lib) - # Build and install plugin manually - # The plugin requires at least HDF5 1.8.11: - if spec['hdf5'].satisfies('@1.8.11:'): - # The plugin does not yet work with Windows: - if sys.platform not in ('win32', 'cygwin'): - # cc = which('cc') - libtool = Executable(join_path(spec['libtool'].prefix.bin, - 'libtool')) - with working_dir('src'): - libtool('--mode=compile', '--tag=CC', - 'cc', '-g', '-O', '-c', 'blosc_plugin.c') - libtool('--mode=link', '--tag=CC', - 'cc', '-g', '-O', - '-rpath', prefix.lib, - '-o', 'libblosc_plugin.la', 'blosc_plugin.lo') - shlibext = 'so' if sys.platform!='darwin' else 'dylib' - shlib0 = 'libblosc_plugin.0.%s' % shlibext - shutil.copyfile(join_path('.libs', shlib0), - join_path(prefix.lib, shlib0)) - shlib = 'libblosc_plugin.%s' % shlibext - os.symlink(shlib0, join_path(prefix.lib, shlib)) + # Build and install filter + with working_dir("src"): + libtool("--mode=compile", "--tag=CC", + "cc", "-g", "-O", + "-c", "blosc_filter.c") + libtool("--mode=link", "--tag=CC", + "cc", "-g", "-O", + "-rpath", prefix.lib, + "-o", "libblosc_filter.la", + "blosc_filter.lo", + "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + shlib0 = "libblosc_filter.0.%s" % shlibext + shlib = "libblosc_filter.%s" % shlibext + shutil.copyfile(join_path(".libs", shlib0), + join_path(prefix.lib, shlib0)) + os.symlink(shlib0, join_path(prefix.lib, shlib)) + + # Build and install plugin + # The plugin requires at least HDF5 1.8.11: + if spec["hdf5"].satisfies("@1.8.11:"): + libtool("--mode=compile", "--tag=CC", + "cc", "-g", "-O", + "-c", "blosc_plugin.c") + libtool("--mode=link", "--tag=CC", + "cc", "-g", "-O", + "-rpath", prefix.lib, + "-o", "libblosc_plugin.la", + "blosc_plugin.lo", + "-L%s" % prefix.lib, "-lblosc_filter", + "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + shlib0 = "libblosc_plugin.0.%s" % shlibext + shlib = "libblosc_plugin.%s" % shlibext + shutil.copyfile(join_path(".libs", shlib0), + join_path(prefix.lib, shlib0)) + os.symlink(shlib0, join_path(prefix.lib, shlib)) + + self.check_install(spec) + def check_install(self, spec): + "Build and run a small program to test the installed HDF5 Blosc plugin" + print "Checking HDF5-Blosc plugin..." + checkdir = "spack-check" + with working_dir(checkdir, create=True): + source = r"""\ +#include +#include +#include +#include + +#define FILTER_BLOSC 32001 /* Blosc filter ID registered with the HDF group */ + +int main(int argc, char **argv) { + herr_t herr; + hid_t file = H5Fcreate("file.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(file >= 0); + hsize_t dims[3] = {10, 10, 10}; + hid_t space = H5Screate_simple(3, dims, NULL); + assert(space >= 0); + hid_t create_proplist = H5Pcreate(H5P_DATASET_CREATE); + assert(create_proplist >= 0); + herr = H5Pset_chunk(create_proplist, 3, dims); + assert(herr >= 0); + herr = H5Pset_filter(create_proplist, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, 0, + NULL); + assert(herr >= 0); + htri_t all_filters_avail = H5Pall_filters_avail(create_proplist); + assert(all_filters_avail > 0); + hid_t dataset = H5Dcreate(file, "dataset", H5T_NATIVE_DOUBLE, space, + H5P_DEFAULT, create_proplist, H5P_DEFAULT); + assert(dataset >= 0); + double data[10][10][10]; + for (int k=0; k<10; ++k) { + for (int j=0; j<10; ++j) { + for (int i=0; i<10; ++i) { + data[k][j][i] = 1.0 / (1.0 + i + j + k); + } + } + } + herr = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, space, space, H5P_DEFAULT, + &data[0][0][0]); + assert(herr >= 0); + herr = H5Pclose(create_proplist); + assert(herr >= 0); + herr = H5Dclose(dataset); + assert(herr >= 0); + herr = H5Sclose(space); + assert(herr >= 0); + herr = H5Fclose(file); + assert(herr >= 0); + printf("Done.\n"); + return 0; +} +""" + expected = """\ +Done. +""" + with open("check.c", "w") as f: + f.write(source) + if "+mpi" in spec["hdf5"]: + cc = which("mpicc") + else: + cc = which("cc") + # TODO: Automate these path and library settings + cc("-c", "-I%s" % spec["hdf5"].prefix.include, "check.c") + cc("-o", "check", "check.o", + "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") + try: + check = Executable("./check") + output = check(return_output=True) + except: + output = "" + success = output == expected + if not success: + print "Produced output does not match expected output." + print "Expected output:" + print "-"*80 + print expected + print "-"*80 + print "Produced output:" + print "-"*80 + print output + print "-"*80 + print "Environment:" + env = which("env") + env() + raise RuntimeError("HDF5 Blosc plugin check failed") + shutil.rmtree(checkdir) + + def setup_environment(self, spack_env, run_env): + spack_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) + run_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) def setup_dependent_environment(self, spack_env, run_env, dependent_spec): - spack_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) - run_env.append_path('HDF5_PLUGIN_PATH', self.spec.prefix.lib) + spack_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) + run_env.append_path("HDF5_PLUGIN_PATH", self.spec.prefix.lib) From 05d7378da57f51a26830a61a88228fb8fc7d3108 Mon Sep 17 00:00:00 2001 From: Erik Schnetter Date: Tue, 14 Jun 2016 17:13:20 -0400 Subject: [PATCH 352/375] Install shared libraries correctly on Linux --- .../builtin/packages/hdf5-blosc/package.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/var/spack/repos/builtin/packages/hdf5-blosc/package.py b/var/spack/repos/builtin/packages/hdf5-blosc/package.py index 680bc41c41a..50f380083cf 100644 --- a/var/spack/repos/builtin/packages/hdf5-blosc/package.py +++ b/var/spack/repos/builtin/packages/hdf5-blosc/package.py @@ -29,6 +29,21 @@ from spack import * +def _install_shlib(name, src, dst): + """Install a shared library from directory src to directory dst""" + if sys.platform == "darwin": + shlib0 = name + ".0.dylib" + shlib = name + ".dylib" + shutil.copyfile(join_path(src, shlib0), join_path(dst, shlib0)) + os.symlink(shlib0, join_path(dst, shlib)) + else: + shlib000 = name + ".so.0.0.0" + shlib0 = name + ".so.0" + shlib = name + ".dylib" + shutil.copyfile(join_path(src, shlib000), join_path(dst, shlib000)) + os.symlink(shlib000, join_path(dst, shlib0)) + os.symlink(shlib0, join_path(dst, shlib)) + class Hdf5Blosc(Package): """Blosc filter for HDF5""" homepage = "https://github.com/Blosc/hdf5-blosc" @@ -72,11 +87,7 @@ def install(self, spec, prefix): "blosc_filter.lo", "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") - shlib0 = "libblosc_filter.0.%s" % shlibext - shlib = "libblosc_filter.%s" % shlibext - shutil.copyfile(join_path(".libs", shlib0), - join_path(prefix.lib, shlib0)) - os.symlink(shlib0, join_path(prefix.lib, shlib)) + _install_shlib("libblosc_filter", ".libs", prefix.lib) # Build and install plugin # The plugin requires at least HDF5 1.8.11: @@ -92,13 +103,10 @@ def install(self, spec, prefix): "-L%s" % prefix.lib, "-lblosc_filter", "-L%s" % spec["c-blosc"].prefix.lib, "-lblosc", "-L%s" % spec["hdf5"].prefix.lib, "-lhdf5") - shlib0 = "libblosc_plugin.0.%s" % shlibext - shlib = "libblosc_plugin.%s" % shlibext - shutil.copyfile(join_path(".libs", shlib0), - join_path(prefix.lib, shlib0)) - os.symlink(shlib0, join_path(prefix.lib, shlib)) + _install_shlib("libblosc_plugin", ".libs", prefix.lib) self.check_install(spec) + def check_install(self, spec): "Build and run a small program to test the installed HDF5 Blosc plugin" print "Checking HDF5-Blosc plugin..." From 36decbfa69e4eda472556ad6e813e14366443de2 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 15 Jun 2016 12:21:05 -0400 Subject: [PATCH 353/375] tty: do not break long words These "words" tend to be file paths which are best left unbroken. Fixes #1057. --- lib/spack/llnl/util/tty/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/spack/llnl/util/tty/__init__.py b/lib/spack/llnl/util/tty/__init__.py index c638b113fdf..ee81e11a207 100644 --- a/lib/spack/llnl/util/tty/__init__.py +++ b/lib/spack/llnl/util/tty/__init__.py @@ -64,12 +64,14 @@ def info(message, *args, **kwargs): format = kwargs.get('format', '*b') stream = kwargs.get('stream', sys.stdout) wrap = kwargs.get('wrap', False) + break_long_words = kwargs.get('break_long_words', False) cprint("@%s{==>} %s" % (format, cescape(str(message))), stream=stream) for arg in args: if wrap: lines = textwrap.wrap( - str(arg), initial_indent=indent, subsequent_indent=indent) + str(arg), initial_indent=indent, subsequent_indent=indent, + break_long_words=break_long_words) for line in lines: stream.write(line + '\n') else: From 716f12dd580cb6f78f085d94e5a45679ba6c8041 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Wed, 15 Jun 2016 12:48:40 -0400 Subject: [PATCH 354/375] reindex: preserve 'explicit' flags Look up in the old database whether the spec was explicit or not and preserve it into the new database. Fixes #1050. --- lib/spack/spack/database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/spack/spack/database.py b/lib/spack/spack/database.py index e768ddf5feb..59d577f0c7c 100644 --- a/lib/spack/spack/database.py +++ b/lib/spack/spack/database.py @@ -309,7 +309,11 @@ def reindex(self, directory_layout): for spec in directory_layout.all_specs(): # Create a spec for each known package and add it. path = directory_layout.path_for_spec(spec) - self._add(spec, path, directory_layout) + old_info = old_data.get(spec.dag_hash()) + explicit = False + if old_info is not None: + explicit = old_info.explicit + self._add(spec, path, directory_layout, explicit=explicit) self._check_ref_counts() From 2fc9ac4036eb017164cc6ac6f19e50dab5008759 Mon Sep 17 00:00:00 2001 From: Gregory Becker Date: Wed, 15 Jun 2016 16:39:39 -0700 Subject: [PATCH 355/375] Two minor fixes --- lib/spack/spack/compilers/__init__.py | 10 +++++----- lib/spack/spack/yaml_version_check.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index ea79f655b00..c76dd252d08 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -74,7 +74,7 @@ def _to_dict(compiler): d['spec'] = str(compiler.spec) d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars ) d['operating_system'] = compiler.operating_system.to_dict() - d['modules'] = compiler.modules + d['modules'] = compiler.modules if compiler.modules else [] if not compiler.alias: yaml_text = yaml.dump( @@ -141,11 +141,11 @@ def remove_compiler_from_config(compiler_spec, scope=None): - scope: configuration scope to modify. """ compiler_config = get_compiler_config(scope) - matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] + matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec] if len(matches) == 1: del compiler_config[matches[0][0]] else: - CompilerSpecInsufficientlySpecificError(compiler_spec) + CompilerSpecInsufficientlySpecificError(compiler_spec) spack.config.update_config('compilers', compiler_config, scope) @@ -234,7 +234,7 @@ def get_compilers(cspec): continue items = cmp alias = aka - + if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)): raise InvalidCompilerConfigurationError(cspec) @@ -309,7 +309,7 @@ def all_os_classes(): this platform """ classes = [] - + platform = spack.architecture.sys_type() for os_class in platform.operating_sys.values(): classes.append(os_class) diff --git a/lib/spack/spack/yaml_version_check.py b/lib/spack/spack/yaml_version_check.py index 7e7bae4edf9..2d3d78ed399 100644 --- a/lib/spack/spack/yaml_version_check.py +++ b/lib/spack/spack/yaml_version_check.py @@ -46,10 +46,10 @@ def check_compiler_yaml_version(): if data: compilers = data['compilers'].items() if len(compilers) > 0: - if 'operating_system' not in compilers[0][1]: + if (not isinstance(compilers, list)) or 'operating_system' not in compilers[0][1]: new_file = os.path.join(scope.path, '_old_compilers.yaml') - tty.warn('%s in out of date compilers format. ' + tty.warn('%s in out of date compilers format. ' 'Moved to %s. Spack automatically generate ' - 'a compilers config file ' + 'a compilers config file ' % (file_name, new_file)) os.rename(file_name, new_file) From c90cc465f581514d18fc4fc95649f015cb178079 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 00:54:59 -0700 Subject: [PATCH 356/375] Add `__contains__` to arch to satsify old packages. --- lib/spack/spack/architecture.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 1f25bc14ea8..df0ab5c14de 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -195,7 +195,7 @@ def operating_system(self, name): name = self.back_os return self.operating_sys.get(name, None) - + @classmethod def detect(self): @@ -206,12 +206,15 @@ def detect(self): """ raise NotImplementedError() + def __repr__(self): return self.__str__() + def __str__(self): return self.name + def _cmp_key(self): t_keys = ''.join(str(t._cmp_key()) for t in sorted(self.targets.values())) @@ -338,7 +341,6 @@ def to_dict(self): d = {} d['name'] = self.name d['version'] = self.version - return d @key_ordering @@ -378,6 +380,11 @@ def __str__(self): else: return '' + + def __contains__(self, string): + return string in str(self) + + def _cmp_key(self): if isinstance(self.platform, Platform): platform = self.platform.name From f3d6818d5c09766946652266188c7210c56f027c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 00:55:39 -0700 Subject: [PATCH 357/375] be more tolerant when parsing new specs. --- lib/spack/spack/spec.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 7718adb72cd..54219ec1b4e 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -526,7 +526,12 @@ def _add_flag(self, name, value): """ valid_flags = FlagMap.valid_compiler_flags() if name == 'arch' or name == 'architecture': - platform, op_sys, target = value.split('-') + parts = value.split('-') + if len(parts) == 3: + platform, op_sys, target = parts + else: + platform, op_sys, target = None, None, value + assert(self.architecture.platform is None) assert(self.architecture.platform_os is None) assert(self.architecture.target is None) @@ -596,11 +601,13 @@ def _set_platform(self, value): def _set_os(self, value): """Called by the parser to set the architecture operating system""" - self.architecture.platform_os = self.architecture.platform.operating_system(value) + if self.architecture.platform: + self.architecture.platform_os = self.architecture.platform.operating_system(value) def _set_target(self, value): """Called by the parser to set the architecture target""" - self.architecture.target = self.architecture.platform.target(value) + if self.architecture.platform: + self.architecture.target = self.architecture.platform.target(value) def _add_dependency(self, spec): """Called by the parser to add another spec as a dependency.""" @@ -1063,7 +1070,7 @@ def feq(cfield, sfield): feq(replacement.architecture, spec.architecture) and feq(replacement.dependencies, spec.dependencies) and feq(replacement.variants, spec.variants) and - feq(replacement.external, spec.external) and + feq(replacement.external, spec.external) and feq(replacement.external_module, spec.external_module)): continue # Refine this spec to the candidate. This uses @@ -1468,7 +1475,7 @@ def constrain(self, other, deps=True): if self.architecture.target != other.architecture.target: raise UnsatisfiableArchitectureSpecError(self.architecture, other.architecture) - + changed = False if self.compiler is not None and other.compiler is not None: @@ -1619,7 +1626,7 @@ def satisfies(self, other, deps=True, strict=False): # Architecture satisfaction is currently just string equality. # If not strict, None means unconstrained. if self.architecture and other.architecture: - if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or + if ((self.architecture.platform and other.architecture.platform and self.architecture.platform != other.architecture.platform) or (self.architecture.platform_os and other.architecture.platform_os and self.architecture.platform_os != other.architecture.platform_os) or (self.architecture.target and other.architecture.target and self.architecture.target != other.architecture.target)): return False From 40d25d90629f41114c2e0b86bb659f0f4dec7bb5 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 00:56:53 -0700 Subject: [PATCH 358/375] Add libxcb dependencies. Fix up packages to get py-basemap and stat to build. --- .../builtin/packages/binutils/package.py | 3 +- .../repos/builtin/packages/boost/package.py | 2 +- .../packages/libpthread-stubs/package.py | 39 ++++++++++++++++ .../repos/builtin/packages/libxau/package.py | 44 +++++++++++++++++++ .../repos/builtin/packages/libxcb/package.py | 7 +-- .../repos/builtin/packages/openssl/package.py | 2 +- .../packages/xorg-util-macros/package.py | 38 ++++++++++++++++ .../repos/builtin/packages/xproto/package.py | 41 +++++++++++++++++ 8 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 var/spack/repos/builtin/packages/libpthread-stubs/package.py create mode 100644 var/spack/repos/builtin/packages/libxau/package.py create mode 100644 var/spack/repos/builtin/packages/xorg-util-macros/package.py create mode 100644 var/spack/repos/builtin/packages/xproto/package.py diff --git a/var/spack/repos/builtin/packages/binutils/package.py b/var/spack/repos/builtin/packages/binutils/package.py index 9e4cc98ae6b..5f305abb028 100644 --- a/var/spack/repos/builtin/packages/binutils/package.py +++ b/var/spack/repos/builtin/packages/binutils/package.py @@ -30,8 +30,9 @@ class Binutils(Package): url="https://ftp.gnu.org/gnu/binutils/binutils-2.25.tar.bz2" + # 2.26 is incompatible with py-pillow build for some reason. version('2.26', '64146a0faa3b411ba774f47d41de239f') - version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66') + version('2.25', 'd9f3303f802a5b6b0bb73a335ab89d66', preferred=True) version('2.24', 'e0f71a7b2ddab0f8612336ac81d9636b') version('2.23.2', '4f8fa651e35ef262edc01d60fb45702e') version('2.20.1', '2b9dc8f2b7dbd5ec5992c6e29de0b764') diff --git a/var/spack/repos/builtin/packages/boost/package.py b/var/spack/repos/builtin/packages/boost/package.py index 2f2965eb12b..1f216a146ab 100644 --- a/var/spack/repos/builtin/packages/boost/package.py +++ b/var/spack/repos/builtin/packages/boost/package.py @@ -128,7 +128,7 @@ def url_for_version(self, version): dots, underscores) def determine_toolset(self, spec): - if spec.satisfies("arch=darwin-x86_64"): + if spec.satisfies("platform=darwin"): return 'darwin' toolsets = {'g++': 'gcc', diff --git a/var/spack/repos/builtin/packages/libpthread-stubs/package.py b/var/spack/repos/builtin/packages/libpthread-stubs/package.py new file mode 100644 index 00000000000..4bcca43c246 --- /dev/null +++ b/var/spack/repos/builtin/packages/libpthread-stubs/package.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class LibpthreadStubs(Package): + """The libpthread-stubs package provides weak aliases for pthread + functions not provided in libc or otherwise available by + default. """ + homepage = "http://xcb.freedesktop.org/" + url = "http://xcb.freedesktop.org/dist/libpthread-stubs-0.1.tar.bz2" + + version('0.3', 'e8fa31b42e13f87e8f5a7a2b731db7ee') + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxau/package.py b/var/spack/repos/builtin/packages/libxau/package.py new file mode 100644 index 00000000000..55816ecdbd4 --- /dev/null +++ b/var/spack/repos/builtin/packages/libxau/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Libxau(Package): + """The libXau package contains a library implementing the X11 + Authorization Protocol. This is useful for restricting client + access to the display.""" + homepage = "http://xcb.freedesktop.org/" + url = "http://ftp.x.org/pub/individual/lib/libXau-1.0.8.tar.bz2" + + version('1.0.8', '685f8abbffa6d145c0f930f00703b21b') + + depends_on('xproto') + + def install(self, spec, prefix): + # FIXME: Modify the configure line to suit your build system here. + configure('--prefix=%s' % prefix) + + # FIXME: Add logic to build and install here + make() + make("install") diff --git a/var/spack/repos/builtin/packages/libxcb/package.py b/var/spack/repos/builtin/packages/libxcb/package.py index 0f39bb0f1d0..746d4567e20 100644 --- a/var/spack/repos/builtin/packages/libxcb/package.py +++ b/var/spack/repos/builtin/packages/libxcb/package.py @@ -35,18 +35,19 @@ class Libxcb(Package): version('1.11', '1698dd837d7e6e94d029dbe8b3a82deb') version('1.11.1', '118623c15a96b08622603a71d8789bf3') + depends_on("python") depends_on("xcb-proto") depends_on("pkg-config") - - # depends_on('pthread') # Ubuntu: apt-get install libpthread-stubs0-dev - # depends_on('xau') # Ubuntu: apt-get install libxau-dev + depends_on("libpthread-stubs") + depends_on('libxau') def patch(self): filter_file('typedef struct xcb_auth_info_t {', 'typedef struct {', 'src/xcb.h') def install(self, spec, prefix): + env['PKG_CONFIG_PATH'] = env['PKG_CONFIG_PATH'] + ':/usr/lib64/pkgconfig' configure("--prefix=%s" % prefix) make() diff --git a/var/spack/repos/builtin/packages/openssl/package.py b/var/spack/repos/builtin/packages/openssl/package.py index 34ab0703ad6..377bffe7234 100644 --- a/var/spack/repos/builtin/packages/openssl/package.py +++ b/var/spack/repos/builtin/packages/openssl/package.py @@ -100,7 +100,7 @@ def install(self, spec, prefix): # in the environment, then this will override what is set in the # Makefile, leading to build errors. env.pop('APPS', None) - if spec.satisfies("arch=darwin-x86_64") or spec.satisfies("arch=ppc64"): + if spec.satisfies("target=x86_64") or spec.satisfies("target=ppc64"): # This needs to be done for all 64-bit architectures (except Linux, # where it happens automatically?) env['KERNEL_BITS'] = '64' diff --git a/var/spack/repos/builtin/packages/xorg-util-macros/package.py b/var/spack/repos/builtin/packages/xorg-util-macros/package.py new file mode 100644 index 00000000000..963d93442f9 --- /dev/null +++ b/var/spack/repos/builtin/packages/xorg-util-macros/package.py @@ -0,0 +1,38 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class XorgUtilMacros(Package): + """The util-macros package contains the m4 macros used by all of the Xorg packages.""" + + homepage = "http://www.example.com" + url = "http://ftp.x.org/pub/individual/util/util-macros-1.19.0.tar.bz2" + + version('1.19.0', '1cf984125e75f8204938d998a8b6c1e1') + + def install(self, spec, prefix): + configure("--prefix=%s" % prefix) + make() + make("install") diff --git a/var/spack/repos/builtin/packages/xproto/package.py b/var/spack/repos/builtin/packages/xproto/package.py new file mode 100644 index 00000000000..7be6defb83b --- /dev/null +++ b/var/spack/repos/builtin/packages/xproto/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + +class Xproto(Package): + """The Xorg protocol headers provide the header files required to + build the system, and to allow other applications to build against + the installed X Window system.""" + homepage = "http://www.x.org/" + url = "https://www.x.org/archive//individual/proto/xproto-7.0.29.tar.gz" + + version('7.0.29', '16a78dd2c5ad73011105c96235f6a0af') + + depends_on("xorg-util-macros") + + def install(self, spec, prefix): + configure('--prefix=%s' % prefix) + make() + make("install") From 76dfaa71a7f59d8c94c415819c57cb191a0e4a4a Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 01:47:20 -0700 Subject: [PATCH 359/375] fix up packages for new architecture. --- var/spack/repos/builtin/packages/espresso/package.py | 2 +- .../repos/builtin/packages/libpciaccess/package.py | 2 +- var/spack/repos/builtin/packages/llvm/package.py | 12 ++++++------ var/spack/repos/builtin/packages/lua/package.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index ef6a3ccc7b0..63a55601374 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -87,7 +87,7 @@ def install(self, spec, prefix): configure(*options) make('all') - if spec.architecture.startswith('darwin'): + if spec.satisfies('platform=darwin'): mkdirp(prefix.bin) for filename in glob("bin/*.x"): install(filename, prefix.bin) diff --git a/var/spack/repos/builtin/packages/libpciaccess/package.py b/var/spack/repos/builtin/packages/libpciaccess/package.py index 42e8711a7df..91cef95cec2 100644 --- a/var/spack/repos/builtin/packages/libpciaccess/package.py +++ b/var/spack/repos/builtin/packages/libpciaccess/package.py @@ -37,7 +37,7 @@ class Libpciaccess(Package): def install(self, spec, prefix): # libpciaccess does not support OS X - if spec.satisfies('arch=darwin-x86_64'): + if spec.satisfies('platform=darwin'): # create a dummy directory mkdir(prefix.lib) return diff --git a/var/spack/repos/builtin/packages/llvm/package.py b/var/spack/repos/builtin/packages/llvm/package.py index c32f66590a6..7a6ea7401ce 100644 --- a/var/spack/repos/builtin/packages/llvm/package.py +++ b/var/spack/repos/builtin/packages/llvm/package.py @@ -267,16 +267,16 @@ def install(self, spec, prefix): if '+all_targets' not in spec: # all is default on cmake targets = ['CppBackend', 'NVPTX', 'AMDGPU'] - if 'x86' in spec.architecture.lower(): + if 'x86' in spec.architecture.target.lower(): targets.append('X86') - elif 'arm' in spec.architecture.lower(): + elif 'arm' in spec.architecture.target.lower(): targets.append('ARM') - elif 'aarch64' in spec.architecture.lower(): + elif 'aarch64' in spec.architecture.target.lower(): targets.append('AArch64') - elif 'sparc' in spec.architecture.lower(): + elif 'sparc' in spec.architecture.target.lower(): targets.append('sparc') - elif ('ppc' in spec.architecture.lower() or - 'power' in spec.architecture.lower()): + elif ('ppc' in spec.architecture.target.lower() or + 'power' in spec.architecture.target.lower()): targets.append('PowerPC') cmake_args.append('-DLLVM_TARGETS_TO_BUILD:Bool=' + ';'.join(targets)) diff --git a/var/spack/repos/builtin/packages/lua/package.py b/var/spack/repos/builtin/packages/lua/package.py index 170f90516ab..1424cf1a17c 100644 --- a/var/spack/repos/builtin/packages/lua/package.py +++ b/var/spack/repos/builtin/packages/lua/package.py @@ -57,7 +57,7 @@ class Lua(Package): placement='luarocks') def install(self, spec, prefix): - if spec.satisfies("arch=darwin-i686") or spec.satisfies("arch=darwin-x86_64"): + if spec.satisfies("platform=darwin"): target = 'macosx' else: target = 'linux' From 88b671f8b1a82ec9d9365c296d77e251a821a66c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 02:06:19 -0700 Subject: [PATCH 360/375] Just call old os "unknown" --- lib/spack/spack/architecture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index df0ab5c14de..4485bbe8401 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -451,7 +451,7 @@ def arch_from_dict(d): if isinstance(d, basestring): # We have an old spec using a string for the architecture arch.platform = Platform('spack_compatibility') - arch.platform_os = OperatingSystem('pre_version', '1.0') + arch.platform_os = OperatingSystem('unknown', '') arch.target = Target(d) arch.os_string = None From 8d9ffcf5650adaca13e85b158022c8b8fed72caf Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 16 Jun 2016 11:22:26 +0200 Subject: [PATCH 361/375] espresso : updated to 5.4.0, added Parallel=False --- var/spack/repos/builtin/packages/espresso/package.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index ef6a3ccc7b0..7359978ce1c 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -34,6 +34,11 @@ class Espresso(Package): homepage = 'http://quantum-espresso.org' url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' + version( + '5.4.0', + '8bb78181b39bd084ae5cb7a512c1cfe7', + url='http://www.qe-forge.org/gf/download/frsrelease/211/968/espresso-5.4.0.tar.gz' + ) version('5.3.0', '6848fcfaeb118587d6be36bd10b7f2c3') variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') @@ -49,6 +54,10 @@ class Espresso(Package): depends_on('fftw+mpi', when='+mpi') depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation + # Spurious problems running in parallel the Makefile + # generated by qe configure + parallel = False + def check_variants(self, spec): error = 'you cannot ask for \'+{variant}\' when \'+mpi\' is not active' if '+scalapack' in spec and '~mpi' in spec: From a1cbb73f8b18f30b30e4ba68e54dfc859d6cd0fa Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 16 Jun 2016 11:24:18 +0200 Subject: [PATCH 362/375] qa : fixed flake8 issues --- .../repos/builtin/packages/espresso/package.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/var/spack/repos/builtin/packages/espresso/package.py b/var/spack/repos/builtin/packages/espresso/package.py index 7359978ce1c..30966d8de98 100644 --- a/var/spack/repos/builtin/packages/espresso/package.py +++ b/var/spack/repos/builtin/packages/espresso/package.py @@ -26,10 +26,13 @@ import os + class Espresso(Package): """ - QE is an integrated suite of Open-Source computer codes for electronic-structure calculations and materials - modeling at the nanoscale. It is based on density-functional theory, plane waves, and pseudopotentials. + QE is an integrated suite of Open-Source computer codes for + electronic-structure calculations and materials modeling at + the nanoscale. It is based on density-functional theory, plane + waves, and pseudopotentials. """ homepage = 'http://quantum-espresso.org' url = 'http://www.qe-forge.org/gf/download/frsrelease/204/912/espresso-5.3.0.tar.gz' @@ -41,10 +44,10 @@ class Espresso(Package): ) version('5.3.0', '6848fcfaeb118587d6be36bd10b7f2c3') - variant('mpi', default=True, description='Build Quantum-ESPRESSO with mpi support') + variant('mpi', default=True, description='Builds with mpi support') variant('openmp', default=False, description='Enables openMP support') variant('scalapack', default=True, description='Enables scalapack support') - variant('elpa', default=True, description='Use elpa as an eigenvalue solver') + variant('elpa', default=True, description='Uses elpa as an eigenvalue solver') depends_on('blas') depends_on('lapack') @@ -52,7 +55,8 @@ class Espresso(Package): depends_on('mpi', when='+mpi') depends_on('fftw~mpi', when='~mpi') depends_on('fftw+mpi', when='+mpi') - depends_on('scalapack', when='+scalapack+mpi') # TODO : + mpi needed to avoid false dependencies installation + # TODO : + mpi needed to avoid false dependencies installation + depends_on('scalapack', when='+scalapack+mpi') # Spurious problems running in parallel the Makefile # generated by qe configure @@ -102,4 +106,3 @@ def install(self, spec, prefix): install(filename, prefix.bin) else: make('install') - From 17b868381f95fa324f7c6327c0977b303975de76 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 02:55:33 -0700 Subject: [PATCH 363/375] Fixes #460: Do not show variants by default in spack find. This does two things: 1. By default `spack find` no longer shows variants. You have to supply `-v` to get that 2. This improves the `colify` implementation so that it no longer pads the rightmost column. This avoids the issue where if one spec was too long in the output, *all* specs would have space padding added to that width, and it would look like the output of `spack find` was double spaced. This no longer happens -- the one bad line wraps around and the other lines are now single-spaced when you use `-v` with boost. --- lib/spack/llnl/util/tty/colify.py | 9 +++++++-- lib/spack/spack/cmd/find.py | 19 +++++++++++++------ lib/spack/spack/cmd/uninstall.py | 13 ++++++++++--- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/lib/spack/llnl/util/tty/colify.py b/lib/spack/llnl/util/tty/colify.py index 429ba458826..81a83691d7a 100644 --- a/lib/spack/llnl/util/tty/colify.py +++ b/lib/spack/llnl/util/tty/colify.py @@ -198,8 +198,13 @@ def colify(elts, **options): for col in xrange(cols): elt = col * rows + row width = config.widths[col] + cextra(elts[elt]) - fmt = '%%-%ds' % width - output.write(fmt % elts[elt]) + if col < cols - 1: + fmt = '%%-%ds' % width + output.write(fmt % elts[elt]) + else: + # Don't pad the rightmost column (sapces can wrap on + # small teriminals if one line is overlong) + output.write(elts[elt]) output.write("\n") row += 1 diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py index 9649bc74354..3ec671f93f6 100644 --- a/lib/spack/spack/cmd/find.py +++ b/lib/spack/spack/cmd/find.py @@ -86,6 +86,11 @@ def setup_parser(subparser): action='store_true', dest='missing', help='Show missing dependencies as well as installed specs.') + subparser.add_argument( + '-v', '--variants', + action='store_true', + dest='variants', + help='Show variants in output (can be long)') subparser.add_argument('-M', '--only-missing', action='store_true', dest='only_missing', @@ -107,6 +112,8 @@ def display_specs(specs, **kwargs): mode = kwargs.get('mode', 'short') hashes = kwargs.get('long', False) namespace = kwargs.get('namespace', False) + flags = kwargs.get('show_flags', False) + variants = kwargs.get('variants', False) hlen = 7 if kwargs.get('very_long', False): @@ -114,10 +121,9 @@ def display_specs(specs, **kwargs): hlen = None nfmt = '.' if namespace else '_' - format_string = '$%s$@$+' % nfmt - flags = kwargs.get('show_flags', False) - if flags: - format_string = '$%s$@$%%+$+' % nfmt + ffmt = '$%+' if flags else '' + vfmt = '$+' if variants else '' + format_string = '$%s$@%s%s' % (nfmt, ffmt, vfmt) # Make a dict with specs keyed by architecture and compiler. index = index_by(specs, ('architecture', 'compiler')) @@ -163,7 +169,7 @@ def fmt(s): string = "" if hashes: string += gray_hash(s, hlen) + ' ' - string += s.format('$-%s$@$+' % nfmt, color=True) + string += s.format('$-%s$@%s' % (nfmt, vfmt), color=True) return string @@ -238,4 +244,5 @@ def find(parser, args): long=args.long, very_long=args.very_long, show_flags=args.show_flags, - namespace=args.namespace) + namespace=args.namespace, + variants=args.variants) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index 9fdf3045b28..a6f08d09ed8 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -39,6 +39,13 @@ b) use spack uninstall -a to uninstall ALL matching specs. """ +# Arguments for display_specs when we find ambiguity +display_args = { + 'long': True, + 'show_flags': True, + 'variants':True +} + def ask_for_confirmation(message): while True: @@ -92,7 +99,7 @@ def concretize_specs(specs, allow_multiple_matches=False, force=False): if not allow_multiple_matches and len(matching) > 1: tty.error("%s matches multiple packages:" % spec) print() - display_specs(matching, long=True, show_flags=True) + display_specs(matching, **display_args) print() has_errors = True @@ -172,7 +179,7 @@ def uninstall(parser, args): tty.error("Will not uninstall %s" % spec.format("$_$@$%@$#", color=True)) print('') print("The following packages depend on it:") - display_specs(lst, long=True) + display_specs(lst, **display_args) print('') has_error = True elif args.dependents: @@ -186,7 +193,7 @@ def uninstall(parser, args): if not args.yes_to_all: tty.msg("The following packages will be uninstalled : ") print('') - display_specs(uninstall_list, long=True, show_flags=True) + display_specs(uninstall_list, **display_args) print('') ask_for_confirmation('Do you want to proceed ? ') From 6b037706023c6cc138709d4cbca91d81d32cc4c4 Mon Sep 17 00:00:00 2001 From: Mario Melara Date: Thu, 16 Jun 2016 09:44:37 -0700 Subject: [PATCH 364/375] Added str method to mac_osx so that whenever we try to access darwins different operating systems we can grab the name correctly. This avoids having to worry whether we have a version attached to it or not. --- lib/spack/spack/operating_systems/mac_osx.py | 9 ++++----- lib/spack/spack/platforms/darwin.py | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_osx.py index a9de03d2cc7..3cf9b4e12e8 100644 --- a/lib/spack/spack/operating_systems/mac_osx.py +++ b/lib/spack/spack/operating_systems/mac_osx.py @@ -21,9 +21,8 @@ def __init__(self): "10.11": "elcapitan"} mac_ver = py_platform.mac_ver()[0][:-2] - try: - name = mac_releases[mac_ver] - except KeyError: - name = "mac_os" - + name = mac_releases.get(mac_ver, "mac_osx") super(MacOsx, self).__init__(name, mac_ver) + + def __str__(self): + return self.name diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 7901f82d06a..8c8af83d55f 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -13,11 +13,11 @@ def __init__(self): self.add_target(self.default, Target(self.default)) mac_os = MacOsx() - self.default_os = mac_os.name - self.front_os = mac_os.name - self.back_os = mac_os.name + self.default_os = str(mac_os) + self.front_os = str(mac_os) + self.back_os = str(mac_os) - self.add_operating_system(mac_os.name, mac_os) + self.add_operating_system(str(mac_os), mac_os) @classmethod def detect(self): From a5aa159d9d0fc97cc9ae882e7325f19438ce597a Mon Sep 17 00:00:00 2001 From: alalazo Date: Thu, 16 Jun 2016 19:23:16 +0200 Subject: [PATCH 365/375] flake8: ignore F405 --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 12ae2efa80d..286522bc48a 100644 --- a/.flake8 +++ b/.flake8 @@ -19,5 +19,5 @@ # - F999: name name be undefined or undefined from star imports. # [flake8] -ignore = E221,E241,E731,F403,F821,F999 +ignore = E221,E241,E731,F403,F821,F999,F405 max-line-length = 79 From c7dfa1699a2898250e51bdcccc4eb568161a7c1c Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 10:43:12 -0700 Subject: [PATCH 366/375] rename Mac OS X to macos --- .../{mac_osx.py => mac_os.py} | 21 ++++++++++--------- lib/spack/spack/platforms/darwin.py | 6 +++--- lib/spack/spack/test/operating_system.py | 13 +++++------- 3 files changed, 19 insertions(+), 21 deletions(-) rename lib/spack/spack/operating_systems/{mac_osx.py => mac_os.py} (53%) diff --git a/lib/spack/spack/operating_systems/mac_osx.py b/lib/spack/spack/operating_systems/mac_os.py similarity index 53% rename from lib/spack/spack/operating_systems/mac_osx.py rename to lib/spack/spack/operating_systems/mac_os.py index 3cf9b4e12e8..f35b3ca5771 100644 --- a/lib/spack/spack/operating_systems/mac_osx.py +++ b/lib/spack/spack/operating_systems/mac_os.py @@ -1,28 +1,29 @@ import platform as py_platform from spack.architecture import OperatingSystem -class MacOsx(OperatingSystem): - """ This class represents the MAC_OSX operating system. This will be auto - detected using the python platform.mac_ver. The MAC_OSX platform - will be represented using the major version operating system name, i.e - el capitan, yosemite...etc. +class MacOs(OperatingSystem): + """This class represents the macOS operating system. This will be + auto detected using the python platform.mac_ver. The macOS + platform will be represented using the major version operating + system name, i.e el capitan, yosemite...etc. """ - + def __init__(self): """ Autodetects the mac version from a dictionary. Goes back as far as 10.6 snowleopard. If the user has an older mac then the version will just be a generic mac_os. """ - mac_releases = {'10.6': "snowleopard", + mac_releases = {'10.6': "snowleopard", "10.7": "lion", "10.8": "mountainlion", "10.9": "mavericks", "10.10": "yosemite", - "10.11": "elcapitan"} + "10.11": "elcapitan", + "10.12": "sierra"} mac_ver = py_platform.mac_ver()[0][:-2] - name = mac_releases.get(mac_ver, "mac_osx") - super(MacOsx, self).__init__(name, mac_ver) + name = mac_releases.get(mac_ver, "macos") + super(MacOs, self).__init__(name, mac_ver) def __str__(self): return self.name diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 8c8af83d55f..d47dd640f99 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,6 +1,6 @@ import subprocess from spack.architecture import Platform, Target -from spack.operating_systems.mac_osx import MacOsx +from spack.operating_systems.mac_os import MacOs class Darwin(Platform): priority = 89 @@ -11,8 +11,8 @@ class Darwin(Platform): def __init__(self): super(Darwin, self).__init__('darwin') self.add_target(self.default, Target(self.default)) - mac_os = MacOsx() - + mac_os = MacOs() + self.default_os = str(mac_os) self.front_os = str(mac_os) self.back_os = str(mac_os) diff --git a/lib/spack/spack/test/operating_system.py b/lib/spack/spack/test/operating_system.py index 205408db3ff..ed5f6ff8adc 100644 --- a/lib/spack/spack/test/operating_system.py +++ b/lib/spack/spack/test/operating_system.py @@ -10,7 +10,7 @@ from spack.platforms.linux import Linux from spack.platforms.darwin import Darwin from spack.operating_system.linux_distro import LinuxDistro -from spack.operating_system.mac_osx import MacOSX +from spack.operating_system.mac_os import MacOs from spack.operating_system.cnl import ComputeNodeLinux class TestOperatingSystem(unittest.TestCase): @@ -27,12 +27,12 @@ def setUp(self): def test_cray_front_end_operating_system(self): self.assertIsInstance(self.cray_operating_sys, LinuxDistro) - + def test_cray_front_end_compiler_strategy(self): self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH") def test_cray_back_end_operating_system(self): - self.assertIsInstance(self.cray_back_os,ComputeNodeLinux) + self.assertIsInstance(self.cray_back_os,ComputeNodeLinux) def test_cray_back_end_compiler_strategy(self): self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES") @@ -43,16 +43,13 @@ def test_linux_operating_system(self): def test_linux_compiler_strategy(self): self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH") - + def test_cray_front_end_compiler_list(self): """ Operating systems will now be in charge of finding compilers. So, depending on which operating system you want to build for - or which operating system you are on, then you could detect + or which operating system you are on, then you could detect compilers in a certain way. Cray linux environment on the front end is just a regular linux distro whereas the Cray linux compute node is a stripped down version which modules are important """ self.assertEquals(True, False) - - - From 8867adf9428dedd42dfc92efece2a3f7498400e0 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 10:43:36 -0700 Subject: [PATCH 367/375] simplify instance check. --- lib/spack/spack/architecture.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 4485bbe8401..cbac7b41d65 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -424,8 +424,7 @@ def _operating_system_from_dict(os_name, platform=None): """ if not platform: platform = sys_type() - if isinstance(os_name, spack.util.spack_yaml.syaml_dict) or \ - isinstance(os_name, dict): + if isinstance(os_name, dict): name = os_name['name'] version = os_name['version'] return platform.operating_system(name+version) From f60ae61652397c614b2e6100eb17dcdc5a99164b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 10:44:15 -0700 Subject: [PATCH 368/375] remove dead code, cleanup. --- lib/spack/spack/test/architecture.py | 19 +++------- lib/spack/spack/test/spec_semantics.py | 49 +++++++++++--------------- 2 files changed, 26 insertions(+), 42 deletions(-) diff --git a/lib/spack/spack/test/architecture.py b/lib/spack/spack/test/architecture.py index 3d08269deef..a6847c57445 100644 --- a/lib/spack/spack/test/architecture.py +++ b/lib/spack/spack/test/architecture.py @@ -23,7 +23,7 @@ def setUp(self): def tearDown(self): super(ArchitectureTest, self).tearDown() - + def test_dict_functions_for_architecture(self): arch = Arch() arch.platform = spack.architecture.sys_type() @@ -46,14 +46,6 @@ def test_dict_functions_for_architecture(self): self.assertTrue( isinstance(new_arch.target, Target) ) -# def test_platform_class_and_compiler_strategies(self): -# a = CrayXc() -# t = a.operating_system('default_os') -# self.assertEquals(t.compiler_strategy, 'MODULES') -# b = Linux() -# s = b.operating_system('default_os') -# self.assertEquals(s.compiler_strategy, 'PATH') - def test_sys_type(self): output_platform_class = sys_type() my_arch_class = None @@ -101,12 +93,12 @@ def test_user_defaults(self): def test_user_input_combination(self): os_list = self.platform.operating_sys.keys() - target_list = self.platform.targets.keys() + target_list = self.platform.targets.keys() additional = ["fe", "be", "frontend", "backend"] os_list.extend(additional) - target_list.extend(additional) - + target_list.extend(additional) + combinations = itertools.product(os_list, target_list) results = [] for arch in combinations: @@ -117,5 +109,4 @@ def test_user_input_combination(self): results.append(spec.architecture.target == self.platform.target(t)) res = all(results) - self.assertTrue(res) - + self.assertTrue(res) diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 0b2111bfe85..9876bfd5a80 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -108,7 +108,8 @@ def test_satisfies_namespace(self): def test_satisfies_namespaced_dep(self): - """Ensure spec from same or unspecified namespace satisfies namespace constraint.""" + """Ensure spec from same or unspecified namespace satisfies namespace + constraint.""" self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpich') self.check_satisfies('mpileaks ^builtin.mock.mpich', '^mpi') @@ -141,29 +142,17 @@ def test_satisfies_compiler_version(self): def test_satisfies_architecture(self): platform = spack.architecture.sys_type() - self.check_satisfies('foo platform=test target=frontend os=frontend', 'platform=test target=frontend os=frontend') - self.check_satisfies('foo platform=test target=backend os=backend', 'platform=test target=backend', 'platform=test os=backend') - self.check_satisfies('foo platform=test target=default_target os=default_os','platform=test target=default_target os=default_os') + self.check_satisfies( + 'foo platform=test target=frontend os=frontend', + 'platform=test target=frontend os=frontend') + self.check_satisfies( + 'foo platform=test target=backend os=backend', + 'platform=test target=backend', 'platform=test os=backend') + self.check_satisfies( + 'foo platform=test target=default_target os=default_os', + 'platform=test target=default_target os=default_os') -#ifdef NEW - #def test_satisfies_architecture(self): - # self.check_satisfies('foo arch=chaos_5_x86_64_ib', ' arch=chaos_5_x86_64_ib') - # self.check_satisfies('foo arch=bgqos_0', ' arch=bgqos_0') - - # self.check_unsatisfiable('foo arch=bgqos_0', ' arch=chaos_5_x86_64_ib') - # self.check_unsatisfiable('foo arch=chaos_5_x86_64_ib', ' arch=bgqos_0') -#els#e /* not NEW */ - #def test_satisfies_target(self): - # platform = spack.architecture.sys_type() - # targets = platform.targets.values() - # for target in targets: - # self.check_satisfies('foo='+target.name, '='+target.name) -#end#if /* not NEW */ - - # for i in range(1,len(targets)): - # self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name) - def test_satisfies_dependencies(self): self.check_satisfies('mpileaks^mpich', '^mpich') self.check_satisfies('mpileaks^zmpi', '^zmpi') @@ -176,10 +165,14 @@ def test_satisfies_dependency_versions(self): self.check_satisfies('mpileaks^mpich@2.0', '^mpich@1:3') self.check_unsatisfiable('mpileaks^mpich@1.2', '^mpich@2.0') - self.check_satisfies('mpileaks^mpich@2.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') - self.check_unsatisfiable('mpileaks^mpich@4.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') - self.check_unsatisfiable('mpileaks^mpich@2.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') - self.check_unsatisfiable('mpileaks^mpich@4.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') + self.check_satisfies( + 'mpileaks^mpich@2.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') + self.check_unsatisfiable( + 'mpileaks^mpich@4.0^callpath@1.5', '^mpich@1:3^callpath@1.4:1.6') + self.check_unsatisfiable( + 'mpileaks^mpich@2.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') + self.check_unsatisfiable( + 'mpileaks^mpich@4.0^callpath@1.7', '^mpich@1:3^callpath@1.4:1.6') def test_satisfies_virtual_dependencies(self): @@ -390,7 +383,7 @@ def test_invalid_constraint(self): self.check_invalid_constraint('libelf debug=2', 'libelf debug=1') self.check_invalid_constraint('libelf cppflags="-O3"', 'libelf cppflags="-O2"') - self.check_invalid_constraint('libelf platform=test target=be os=be', + self.check_invalid_constraint('libelf platform=test target=be os=be', 'libelf target=fe os=fe') def test_constrain_changed(self): @@ -402,7 +395,7 @@ def test_constrain_changed(self): self.check_constrain_changed('libelf', '~debug') self.check_constrain_changed('libelf', 'debug=2') self.check_constrain_changed('libelf', 'cppflags="-O3"') - + platform = spack.architecture.sys_type() self.check_constrain_changed('libelf', 'target='+platform.target('default_target').name) self.check_constrain_changed('libelf', 'os='+platform.operating_system('default_os').name) From 005827ab234e4fdb8d8e298baa99573bb570e4c8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Thu, 16 Jun 2016 12:12:29 -0700 Subject: [PATCH 369/375] Replace 2.6-incompatible skipIf with a conditional. - skipIf doesn't work with Python 2.6. --- lib/spack/spack/test/concretize.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 67b20b4084c..ab201f406aa 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -253,10 +253,14 @@ def test_external_package(self): self.assertFalse('externalprereq' in spec) self.assertTrue(spec['externaltool'].compiler.satisfies('gcc')) - @unittest.skipIf(spack.architecture.sys_type().name == 'darwin' or - spack.architecture.sys_type().name == 'linux', - "No tcl modules on darwin/linux machines") + def test_external_package_module(self): + # No tcl modules on darwin/linux machines + # TODO: improved way to check for this. + if (spack.architecture.sys_type().name == 'darwin' or + spack.architecture.sys_type().name == 'linux'): + return + spec = Spec('externalmodule') spec.concretize() self.assertEqual(spec['externalmodule'].external_module, 'external-module') @@ -272,7 +276,7 @@ def test_nobuild_package(self): got_error = True self.assertTrue(got_error) - + def test_external_and_virtual(self): spec = Spec('externaltest') spec.concretize() From a660c60eb1997317fe0e99054a99cc1cea1d3a56 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 10 Jun 2016 09:24:52 -0500 Subject: [PATCH 370/375] Add py-sncosmo package --- .../builtin/packages/py-astropy/package.py | 41 ++++++++------ .../builtin/packages/py-emcee/package.py | 41 ++++++++++++++ .../repos/builtin/packages/py-h5py/package.py | 31 +++++++---- .../builtin/packages/py-iminuit/package.py | 54 +++++++++++++++++++ .../builtin/packages/py-nestle/package.py | 44 +++++++++++++++ .../builtin/packages/py-sncosmo/package.py | 51 ++++++++++++++++++ 6 files changed, 237 insertions(+), 25 deletions(-) create mode 100644 var/spack/repos/builtin/packages/py-emcee/package.py create mode 100644 var/spack/repos/builtin/packages/py-iminuit/package.py create mode 100644 var/spack/repos/builtin/packages/py-nestle/package.py create mode 100644 var/spack/repos/builtin/packages/py-sncosmo/package.py diff --git a/var/spack/repos/builtin/packages/py-astropy/package.py b/var/spack/repos/builtin/packages/py-astropy/package.py index 86875bbcaea..09d1688f4b7 100644 --- a/var/spack/repos/builtin/packages/py-astropy/package.py +++ b/var/spack/repos/builtin/packages/py-astropy/package.py @@ -24,29 +24,40 @@ ############################################################################## from spack import * -class PyAstropy(Package): - """ - The Astropy Project is a community effort to develop a single core - package for Astronomy in Python and foster interoperability between - Python astronomy packages. - """ - homepage = 'http://www.astropy.org/' +class PyAstropy(Package): + """The Astropy Project is a community effort to develop a single core + package for Astronomy in Python and foster interoperability between + Python astronomy packages.""" + + homepage = 'http://www.astropy.org/' + url = 'https://pypi.python.org/packages/source/a/astropy/astropy-1.1.2.tar.gz' + + version('1.1.2', 'cbe32023b5b1177d1e2498a0d00cda51') version('1.1.post1', 'b52919f657a37d45cc45f5cb0f58c44d') - def url_for_version(self, version): - return 'https://pypi.python.org/packages/source/a/astropy/astropy-{0}.tar.gz'.format(version) - - extends('python') + # Required dependencies + extends('python@2.6.5:2.7,3.3:') + depends_on('py-numpy@1.6.0:') + # Optional dependencies depends_on('cfitsio') depends_on('expat') depends_on('py-h5py') - depends_on('py-numpy') + depends_on('py-pyyaml') depends_on('py-scipy') + depends_on('py-matplotlib') + depends_on('py-pytz') + depends_on('py-scikit-image') + depends_on('py-pandas') + + # TODO: Add packages + # depends_on('py-beautiful-soup') + # depends_on('py-xmllint') + # depends_on('py-wcsaxes') + # depends_on('py-objgraph') def install(self, spec, prefix): python('setup.py', 'build', '--use-system-cfitsio', - '--use-system-expat') - python('setup.py', 'install', '--prefix=' + prefix) - + '--use-system-expat') + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-emcee/package.py b/var/spack/repos/builtin/packages/py-emcee/package.py new file mode 100644 index 00000000000..d57ef4bd76c --- /dev/null +++ b/var/spack/repos/builtin/packages/py-emcee/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyEmcee(Package): + """emcee is an MIT licensed pure-Python implementation of Goodman & Weare's + Affine Invariant Markov chain Monte Carlo (MCMC) Ensemble sampler.""" + + homepage = "http://dan.iel.fm/emcee/current/" + url = "https://pypi.python.org/packages/source/e/emcee/emcee-2.1.0.tar.gz" + + version('2.1.0', 'c6b6fad05c824d40671d4a4fc58dfff7') + + extends('python') + depends_on('py-numpy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index a8eb027ba56..a92fc3c9c7f 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -23,23 +23,34 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * -import re + class PyH5py(Package): - """The h5py package provides both a high- and low-level interface to the HDF5 library from Python.""" + """The h5py package provides both a high- and low-level interface to the + HDF5 library from Python.""" + homepage = "https://pypi.python.org/pypi/h5py" url = "https://pypi.python.org/packages/source/h/h5py/h5py-2.4.0.tar.gz" - version('2.4.0', '80c9a94ae31f84885cc2ebe1323d6758') + version('2.6.0', 'ec476211bd1de3f5ac150544189b0bf4') version('2.5.0', '6e4301b5ad5da0d51b0a1e5ac19e3b74') + version('2.4.0', '80c9a94ae31f84885cc2ebe1323d6758') - extends('python', ignore=lambda f: re.match(r'bin/cy*', f)) - depends_on('hdf5') - depends_on('py-numpy') + variant('mpi', default=False, description='Build with MPI support') + + extends('python', ignore=r'bin/cy.*') + depends_on('hdf5@1.8.4:+mpi', when='+mpi') + depends_on('hdf5@1.8.4:~mpi', when='~mpi') + depends_on('py-numpy@1.6.1:') depends_on('py-cython') - depends_on('py-six') - depends_on('py-pkgconfig') + depends_on('mpi', when='+mpi') def install(self, spec, prefix): - python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix) - python('setup.py', 'install', '--prefix=%s' % prefix) + python('setup.py', 'configure', + '--hdf5={0}'.format(spec['hdf5'].prefix)) + + if '+mpi' in spec: + env['CC'] = spec['mpi'].mpicc + python('setup.py', 'configure', '--mpi') + + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-iminuit/package.py b/var/spack/repos/builtin/packages/py-iminuit/package.py new file mode 100644 index 00000000000..5a42268a73b --- /dev/null +++ b/var/spack/repos/builtin/packages/py-iminuit/package.py @@ -0,0 +1,54 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyIminuit(Package): + """Interactive IPython-Friendly Minimizer based on SEAL Minuit2.""" + + homepage = "https://pypi.python.org/pypi/iminuit" + url = "https://pypi.python.org/packages/source/i/iminuit/iminuit-1.2.tar.gz" + + version('1.2', '4701ec472cae42015e26251703e6e984') + + # Required dependencies + extends('python@2.7:2.8,3.4:') + depends_on('py-setuptools') + + # Optional dependencies + depends_on('py-numpy') + depends_on('py-matplotlib') + + def install(self, spec, prefix): + site_packages = '{0}/lib/python{1}/site-packages'.format( + prefix, spec['python'].version.up_to(2)) + + # site-packages directory must already exist + mkdirp(site_packages) + + # PYTHONPATH must include site-packages directory + env['PYTHONPATH'] += ':{0}'.format(site_packages) + + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-nestle/package.py b/var/spack/repos/builtin/packages/py-nestle/package.py new file mode 100644 index 00000000000..38fa0561dbd --- /dev/null +++ b/var/spack/repos/builtin/packages/py-nestle/package.py @@ -0,0 +1,44 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyNestle(Package): + """Nested sampling algorithms for evaluating Bayesian evidence.""" + + homepage = "http://kbarbary.github.io/nestle/" + url = "https://pypi.python.org/packages/source/n/nestle/nestle-0.1.1.tar.gz" + + version('0.1.1', '4875c0f9a0a8e263c1d7f5fa6ce604c5') + + # Required dependencies + extends('python') + depends_on('py-numpy@1.6:') + + # Optional dependencies + depends_on('py-scipy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-sncosmo/package.py b/var/spack/repos/builtin/packages/py-sncosmo/package.py new file mode 100644 index 00000000000..d40623ffa17 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-sncosmo/package.py @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PySncosmo(Package): + """SNCosmo is a Python library for high-level supernova cosmology + analysis.""" + + homepage = "http://sncosmo.readthedocs.io/" + url = "https://pypi.python.org/packages/source/s/sncosmo/sncosmo-1.2.0.tar.gz" + + version('1.2.0', '028e6d1dc84ab1c17d2f3b6378b2cb1e') + + # Required dependencies + # py-sncosmo binaries are duplicates of those from py-astropy + extends('python@2.6:2.7,3.3:', ignore=r'bin/*') + depends_on('py-numpy') + depends_on('py-scipy') + depends_on('py-astropy') + + # Recommended dependencies + depends_on('py-matplotlib') + depends_on('py-iminuit') + depends_on('py-emcee') + depends_on('py-nestle') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) From 4eeaff175cc6c9fd15a7d2455e1e0663ad147aca Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 10 Jun 2016 14:29:57 -0500 Subject: [PATCH 371/375] Allow packages to build --- .../repos/builtin/packages/py-astropy/package.py | 3 ++- var/spack/repos/builtin/packages/py-h5py/package.py | 4 +++- .../repos/builtin/packages/py-iminuit/package.py | 12 ++---------- .../repos/builtin/packages/py-sncosmo/package.py | 3 ++- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-astropy/package.py b/var/spack/repos/builtin/packages/py-astropy/package.py index 09d1688f4b7..bac794e551b 100644 --- a/var/spack/repos/builtin/packages/py-astropy/package.py +++ b/var/spack/repos/builtin/packages/py-astropy/package.py @@ -60,4 +60,5 @@ class PyAstropy(Package): def install(self, spec, prefix): python('setup.py', 'build', '--use-system-cfitsio', '--use-system-expat') - python('setup.py', 'install', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--single-version-externally-managed', + '--root=/', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index a92fc3c9c7f..1b8675ecf2d 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -39,6 +39,7 @@ class PyH5py(Package): variant('mpi', default=False, description='Build with MPI support') extends('python', ignore=r'bin/cy.*') + depends_on('hdf5@1.8.4:+mpi', when='+mpi') depends_on('hdf5@1.8.4:~mpi', when='~mpi') depends_on('py-numpy@1.6.1:') @@ -53,4 +54,5 @@ def install(self, spec, prefix): env['CC'] = spec['mpi'].mpicc python('setup.py', 'configure', '--mpi') - python('setup.py', 'install', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--single-version-externally-managed', + '--root=/', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-iminuit/package.py b/var/spack/repos/builtin/packages/py-iminuit/package.py index 5a42268a73b..557cf19c198 100644 --- a/var/spack/repos/builtin/packages/py-iminuit/package.py +++ b/var/spack/repos/builtin/packages/py-iminuit/package.py @@ -42,13 +42,5 @@ class PyIminuit(Package): depends_on('py-matplotlib') def install(self, spec, prefix): - site_packages = '{0}/lib/python{1}/site-packages'.format( - prefix, spec['python'].version.up_to(2)) - - # site-packages directory must already exist - mkdirp(site_packages) - - # PYTHONPATH must include site-packages directory - env['PYTHONPATH'] += ':{0}'.format(site_packages) - - python('setup.py', 'install', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--single-version-externally-managed', + '--root=/', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-sncosmo/package.py b/var/spack/repos/builtin/packages/py-sncosmo/package.py index d40623ffa17..58c4625a4f3 100644 --- a/var/spack/repos/builtin/packages/py-sncosmo/package.py +++ b/var/spack/repos/builtin/packages/py-sncosmo/package.py @@ -48,4 +48,5 @@ class PySncosmo(Package): depends_on('py-nestle') def install(self, spec, prefix): - python('setup.py', 'install', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--single-version-externally-managed', + '--root=/', '--prefix={0}'.format(prefix)) From b2aa78d6ab569f5dac65b665c2f9b2962b72fdbb Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 10 Jun 2016 17:08:50 -0500 Subject: [PATCH 372/375] Remove version constraints --- var/spack/repos/builtin/packages/py-astropy/package.py | 7 +++---- var/spack/repos/builtin/packages/py-h5py/package.py | 3 +-- var/spack/repos/builtin/packages/py-iminuit/package.py | 5 ++--- var/spack/repos/builtin/packages/py-nestle/package.py | 2 +- var/spack/repos/builtin/packages/py-sncosmo/package.py | 5 ++--- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-astropy/package.py b/var/spack/repos/builtin/packages/py-astropy/package.py index bac794e551b..e3776caf707 100644 --- a/var/spack/repos/builtin/packages/py-astropy/package.py +++ b/var/spack/repos/builtin/packages/py-astropy/package.py @@ -37,8 +37,8 @@ class PyAstropy(Package): version('1.1.post1', 'b52919f657a37d45cc45f5cb0f58c44d') # Required dependencies - extends('python@2.6.5:2.7,3.3:') - depends_on('py-numpy@1.6.0:') + extends('python') + depends_on('py-numpy') # Optional dependencies depends_on('cfitsio') @@ -60,5 +60,4 @@ class PyAstropy(Package): def install(self, spec, prefix): python('setup.py', 'build', '--use-system-cfitsio', '--use-system-expat') - python('setup.py', 'install', '--single-version-externally-managed', - '--root=/', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index 1b8675ecf2d..1379513197f 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -54,5 +54,4 @@ def install(self, spec, prefix): env['CC'] = spec['mpi'].mpicc python('setup.py', 'configure', '--mpi') - python('setup.py', 'install', '--single-version-externally-managed', - '--root=/', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-iminuit/package.py b/var/spack/repos/builtin/packages/py-iminuit/package.py index 557cf19c198..ffc5a5bd14a 100644 --- a/var/spack/repos/builtin/packages/py-iminuit/package.py +++ b/var/spack/repos/builtin/packages/py-iminuit/package.py @@ -34,7 +34,7 @@ class PyIminuit(Package): version('1.2', '4701ec472cae42015e26251703e6e984') # Required dependencies - extends('python@2.7:2.8,3.4:') + extends('python') depends_on('py-setuptools') # Optional dependencies @@ -42,5 +42,4 @@ class PyIminuit(Package): depends_on('py-matplotlib') def install(self, spec, prefix): - python('setup.py', 'install', '--single-version-externally-managed', - '--root=/', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-nestle/package.py b/var/spack/repos/builtin/packages/py-nestle/package.py index 38fa0561dbd..16506e3eca0 100644 --- a/var/spack/repos/builtin/packages/py-nestle/package.py +++ b/var/spack/repos/builtin/packages/py-nestle/package.py @@ -35,7 +35,7 @@ class PyNestle(Package): # Required dependencies extends('python') - depends_on('py-numpy@1.6:') + depends_on('py-numpy') # Optional dependencies depends_on('py-scipy') diff --git a/var/spack/repos/builtin/packages/py-sncosmo/package.py b/var/spack/repos/builtin/packages/py-sncosmo/package.py index 58c4625a4f3..8976413ab95 100644 --- a/var/spack/repos/builtin/packages/py-sncosmo/package.py +++ b/var/spack/repos/builtin/packages/py-sncosmo/package.py @@ -36,7 +36,7 @@ class PySncosmo(Package): # Required dependencies # py-sncosmo binaries are duplicates of those from py-astropy - extends('python@2.6:2.7,3.3:', ignore=r'bin/*') + extends('python', ignore=r'bin/*') depends_on('py-numpy') depends_on('py-scipy') depends_on('py-astropy') @@ -48,5 +48,4 @@ class PySncosmo(Package): depends_on('py-nestle') def install(self, spec, prefix): - python('setup.py', 'install', '--single-version-externally-managed', - '--root=/', '--prefix={0}'.format(prefix)) + python('setup.py', 'install', '--prefix={0}'.format(prefix)) From 24f65c502e54cc8767d47bafa827c7434c81bd07 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Mon, 13 Jun 2016 12:31:18 -0500 Subject: [PATCH 373/375] Add additional Astropy dependencies --- .../builtin/packages/py-astropy/package.py | 12 +++--- .../packages/py-beautifulsoup4/package.py | 41 ++++++++++++++++++ .../builtin/packages/py-wcsaxes/package.py | 43 +++++++++++++++++++ 3 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 var/spack/repos/builtin/packages/py-beautifulsoup4/package.py create mode 100644 var/spack/repos/builtin/packages/py-wcsaxes/package.py diff --git a/var/spack/repos/builtin/packages/py-astropy/package.py b/var/spack/repos/builtin/packages/py-astropy/package.py index e3776caf707..a9962777dc0 100644 --- a/var/spack/repos/builtin/packages/py-astropy/package.py +++ b/var/spack/repos/builtin/packages/py-astropy/package.py @@ -41,21 +41,19 @@ class PyAstropy(Package): depends_on('py-numpy') # Optional dependencies - depends_on('cfitsio') - depends_on('expat') depends_on('py-h5py') + depends_on('py-beautifulsoup4') depends_on('py-pyyaml') depends_on('py-scipy') + depends_on('libxml2') depends_on('py-matplotlib') depends_on('py-pytz') depends_on('py-scikit-image') depends_on('py-pandas') - # TODO: Add packages - # depends_on('py-beautiful-soup') - # depends_on('py-xmllint') - # depends_on('py-wcsaxes') - # depends_on('py-objgraph') + # System dependencies + depends_on('cfitsio') + depends_on('expat') def install(self, spec, prefix): python('setup.py', 'build', '--use-system-cfitsio', diff --git a/var/spack/repos/builtin/packages/py-beautifulsoup4/package.py b/var/spack/repos/builtin/packages/py-beautifulsoup4/package.py new file mode 100644 index 00000000000..64368fe70d9 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-beautifulsoup4/package.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyBeautifulsoup4(Package): + """Beautiful Soup is a Python library for pulling data out of HTML and + XML files. It works with your favorite parser to provide idiomatic ways + of navigating, searching, and modifying the parse tree.""" + + homepage = "https://www.crummy.com/software/BeautifulSoup" + url = "https://pypi.python.org/packages/source/b/beautifulsoup4/beautifulsoup4-4.4.1.tar.gz" + + version('4.4.1', '8fbd9a7cac0704645fa20d1419036815') + + extends('python') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-wcsaxes/package.py b/var/spack/repos/builtin/packages/py-wcsaxes/package.py new file mode 100644 index 00000000000..b0adbe36580 --- /dev/null +++ b/var/spack/repos/builtin/packages/py-wcsaxes/package.py @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/llnl/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack import * + + +class PyWcsaxes(Package): + """WCSAxes is a framework for making plots of Astronomical data + in Matplotlib.""" + + homepage = "http://wcsaxes.readthedocs.io/en/latest/index.html" + url = "https://github.com/astrofrog/wcsaxes/archive/v0.8.tar.gz" + + version('0.8', 'de1c60fdae4c330bf5ddb9f1ab5ab920') + + extends('python', ignore=r'bin/pbr') + depends_on('py-numpy') + depends_on('py-matplotlib') + depends_on('py-astropy') + + def install(self, spec, prefix): + python('setup.py', 'install', '--prefix={0}'.format(prefix)) From 80d444624b433d1059bed6580c24fb87db331016 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Wed, 15 Jun 2016 16:42:50 -0500 Subject: [PATCH 374/375] Fix activation of sites.py for sncosmo --- var/spack/repos/builtin/packages/py-h5py/package.py | 13 ++++++++++--- .../repos/builtin/packages/py-iminuit/package.py | 1 + .../repos/builtin/packages/py-sncosmo/package.py | 2 +- var/spack/repos/builtin/packages/python/package.py | 7 +++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/var/spack/repos/builtin/packages/py-h5py/package.py b/var/spack/repos/builtin/packages/py-h5py/package.py index 1379513197f..0180b658f5b 100644 --- a/var/spack/repos/builtin/packages/py-h5py/package.py +++ b/var/spack/repos/builtin/packages/py-h5py/package.py @@ -38,14 +38,21 @@ class PyH5py(Package): variant('mpi', default=False, description='Build with MPI support') - extends('python', ignore=r'bin/cy.*') + extends('python') + # Build dependencies + depends_on('py-cython@0.19:') + depends_on('pkg-config') depends_on('hdf5@1.8.4:+mpi', when='+mpi') depends_on('hdf5@1.8.4:~mpi', when='~mpi') - depends_on('py-numpy@1.6.1:') - depends_on('py-cython') depends_on('mpi', when='+mpi') + # Build and runtime dependencies + depends_on('py-numpy@1.6.1:') + + # Runtime dependencies + depends_on('py-six') + def install(self, spec, prefix): python('setup.py', 'configure', '--hdf5={0}'.format(spec['hdf5'].prefix)) diff --git a/var/spack/repos/builtin/packages/py-iminuit/package.py b/var/spack/repos/builtin/packages/py-iminuit/package.py index ffc5a5bd14a..47751a702d1 100644 --- a/var/spack/repos/builtin/packages/py-iminuit/package.py +++ b/var/spack/repos/builtin/packages/py-iminuit/package.py @@ -40,6 +40,7 @@ class PyIminuit(Package): # Optional dependencies depends_on('py-numpy') depends_on('py-matplotlib') + depends_on('py-cython') def install(self, spec, prefix): python('setup.py', 'install', '--prefix={0}'.format(prefix)) diff --git a/var/spack/repos/builtin/packages/py-sncosmo/package.py b/var/spack/repos/builtin/packages/py-sncosmo/package.py index 8976413ab95..82ae2a2e69c 100644 --- a/var/spack/repos/builtin/packages/py-sncosmo/package.py +++ b/var/spack/repos/builtin/packages/py-sncosmo/package.py @@ -36,7 +36,7 @@ class PySncosmo(Package): # Required dependencies # py-sncosmo binaries are duplicates of those from py-astropy - extends('python', ignore=r'bin/*') + extends('python', ignore=r'bin/.*') depends_on('py-numpy') depends_on('py-scipy') depends_on('py-astropy') diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index 86c12498e1f..d6d5fac4b1c 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -167,14 +167,13 @@ def python_ignore(self, ext_pkg, args): ignore_arg = args.get('ignore', lambda f: False) # Always ignore easy-install.pth, as it needs to be merged. - patterns = [r'easy-install\.pth$'] + patterns = [r'site-packages/easy-install\.pth$'] # Ignore pieces of setuptools installed by other packages. if ext_pkg.name != 'py-setuptools': - patterns.append(r'/site[^/]*\.pyc?$') - patterns.append(r'setuptools\.pth') patterns.append(r'bin/easy_install[^/]*$') - patterns.append(r'setuptools.*egg$') + patterns.append(r'site-packages/setuptools\.pth$') + patterns.append(r'site-packages/site[^/]*\.pyc?$') if ext_pkg.name != 'py-numpy': patterns.append(r'bin/f2py$') From 536fa73d5f1e0a676d3db68097bbdbab8ba424cd Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Fri, 17 Jun 2016 13:33:09 -0500 Subject: [PATCH 375/375] Python flake8 overhaul --- .../repos/builtin/packages/python/package.py | 188 +++++++++++------- 1 file changed, 113 insertions(+), 75 deletions(-) diff --git a/var/spack/repos/builtin/packages/python/package.py b/var/spack/repos/builtin/packages/python/package.py index d6d5fac4b1c..b6a62bf7420 100644 --- a/var/spack/repos/builtin/packages/python/package.py +++ b/var/spack/repos/builtin/packages/python/package.py @@ -22,9 +22,6 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import functools -import glob -import inspect import os import re from contextlib import closing @@ -37,11 +34,10 @@ class Python(Package): """The Python programming language.""" + homepage = "http://www.python.org" url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz" - extendable = True - version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe') version('3.5.0', 'a56c0c0b45d75a0ec9c6dee933c41c36') version('2.7.11', '6b6076ec9e93f05dd63e47eb9c15728b', preferred=True) @@ -49,6 +45,8 @@ class Python(Package): version('2.7.9', '5eebcaa0030dc4061156d3429657fb83') version('2.7.8', 'd4bca0159acb0b44a781292b5231936f') + extendable = True + depends_on("openssl") depends_on("bzip2") depends_on("readline") @@ -64,39 +62,63 @@ def install(self, spec, prefix): # Rest of install is pretty standard except setup.py needs to # be able to read the CPPFLAGS and LDFLAGS as it scans for the # library and headers to build - configure_args= [ - "--prefix=%s" % prefix, - "--with-threads", - "--enable-shared", - "CPPFLAGS=-I%s/include -I%s/include -I%s/include -I%s/include -I%s/include -I%s/include" % ( - spec['openssl'].prefix, spec['bzip2'].prefix, - spec['readline'].prefix, spec['ncurses'].prefix, - spec['sqlite'].prefix, spec['zlib'].prefix), - "LDFLAGS=-L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib -L%s/lib" % ( - spec['openssl'].prefix, spec['bzip2'].prefix, - spec['readline'].prefix, spec['ncurses'].prefix, - spec['sqlite'].prefix, spec['zlib'].prefix) - ] + cppflags = ' -I'.join([ + spec['openssl'].prefix.include, spec['bzip2'].prefix.include, + spec['readline'].prefix.include, spec['ncurses'].prefix.include, + spec['sqlite'].prefix.include, spec['zlib'].prefix.include + ]) + + ldflags = ' -L'.join([ + spec['openssl'].prefix.lib, spec['bzip2'].prefix.lib, + spec['readline'].prefix.lib, spec['ncurses'].prefix.lib, + spec['sqlite'].prefix.lib, spec['zlib'].prefix.lib + ]) + + config_args = [ + "--prefix={0}".format(prefix), + "--with-threads", + "--enable-shared", + "CPPFLAGS=-I{0}".format(cppflags), + "LDFLAGS=-L{0}".format(ldflags) + ] + if spec.satisfies('@3:'): - configure_args.append('--without-ensurepip') - configure(*configure_args) + config_args.append('--without-ensurepip') + + configure(*config_args) + make() make("install") - # Modify compiler paths in configuration files. This is necessary for - # building site packages outside of spack - filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', - (r'\4\5'), - join_path(prefix.lib, 'python%d.%d' % self.version[:2], '_sysconfigdata.py')) + self.filter_compilers(spec, prefix) - python3_version = '' + def filter_compilers(self, spec, prefix): + """Run after install to tell the configuration files and Makefiles + to use the compilers that Spack built the package with. + + If this isn't done, they'll have CC and CXX set to Spack's generic + cc and c++. We want them to be bound to whatever compiler + they were built with.""" + + kwargs = {'ignore_absent': True, 'backup': False, 'string': True} + + dirname = join_path(prefix.lib, + 'python{0}'.format(self.version.up_to(2))) + + config = 'config' if spec.satisfies('@3:'): - python3_version = '-%d.%dm' % self.version[:2] - makefile_filepath = join_path(prefix.lib, 'python%d.%d' % self.version[:2], 'config%s' % python3_version, 'Makefile') - filter_file(r'([/s]=?)([\S=]*)/lib/spack/env(/[^\s/]*)?/(\S*)(\s)', - (r'\4\5'), - makefile_filepath) + config = 'config-{0}m'.format(self.version.up_to(2)) + files = [ + '_sysconfigdata.py', + join_path(config, 'Makefile') + ] + + for filename in files: + filter_file(env['CC'], self.compiler.cc, + join_path(dirname, filename), **kwargs) + filter_file(env['CXX'], self.compiler.cxx, + join_path(dirname, filename), **kwargs) # ======================================================================== # Set up environment to make install easy for python extensions. @@ -104,57 +126,59 @@ def install(self, spec, prefix): @property def python_lib_dir(self): - return os.path.join('lib', 'python%d.%d' % self.version[:2]) - + return join_path('lib', 'python{0}'.format(self.version.up_to(2))) @property def python_include_dir(self): - return os.path.join('include', 'python%d.%d' % self.version[:2]) - + return join_path('include', 'python{0}'.format(self.version.up_to(2))) @property def site_packages_dir(self): - return os.path.join(self.python_lib_dir, 'site-packages') - + return join_path(self.python_lib_dir, 'site-packages') def setup_dependent_environment(self, spack_env, run_env, extension_spec): - # TODO: do this only for actual extensions. + """Set PYTHONPATH to include site-packages dir for the + extension and any other python extensions it depends on.""" - # Set PYTHONPATH to include site-packages dir for the - # extension and any other python extensions it depends on. python_paths = [] for d in extension_spec.traverse(): if d.package.extends(self.spec): - python_paths.append(os.path.join(d.prefix, self.site_packages_dir)) + python_paths.append(join_path(d.prefix, + self.site_packages_dir)) pythonpath = ':'.join(python_paths) spack_env.set('PYTHONPATH', pythonpath) - # For run time environment set only the path for extension_spec and prepend it to PYTHONPATH + # For run time environment set only the path for + # extension_spec and prepend it to PYTHONPATH if extension_spec.package.extends(self.spec): - run_env.prepend_path('PYTHONPATH', os.path.join(extension_spec.prefix, self.site_packages_dir)) - + run_env.prepend_path('PYTHONPATH', join_path( + extension_spec.prefix, self.site_packages_dir)) def setup_dependent_package(self, module, ext_spec): - """ - Called before python modules' install() methods. + """Called before python modules' install() methods. In most cases, extensions will only need to have one line:: - python('setup.py', 'install', '--prefix=%s' % prefix) - """ + python('setup.py', 'install', '--prefix={0}'.format(prefix))""" + # Python extension builds can have a global python executable function - if self.version >= Version("3.0.0") and self.version < Version("4.0.0"): - module.python = Executable(join_path(self.spec.prefix.bin, 'python3')) + if Version("3.0.0") <= self.version < Version("4.0.0"): + module.python = Executable(join_path(self.spec.prefix.bin, + 'python3')) else: - module.python = Executable(join_path(self.spec.prefix.bin, 'python')) + module.python = Executable(join_path(self.spec.prefix.bin, + 'python')) # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs. - module.python_lib_dir = os.path.join(ext_spec.prefix, self.python_lib_dir) - module.python_include_dir = os.path.join(ext_spec.prefix, self.python_include_dir) - module.site_packages_dir = os.path.join(ext_spec.prefix, self.site_packages_dir) + module.python_lib_dir = join_path(ext_spec.prefix, + self.python_lib_dir) + module.python_include_dir = join_path(ext_spec.prefix, + self.python_include_dir) + module.site_packages_dir = join_path(ext_spec.prefix, + self.site_packages_dir) - # Make the site packages directory for extensions, if it does not exist already. + # Make the site packages directory for extensions if ext_spec.package.is_extension: mkdirp(module.site_packages_dir) @@ -170,21 +194,25 @@ def python_ignore(self, ext_pkg, args): patterns = [r'site-packages/easy-install\.pth$'] # Ignore pieces of setuptools installed by other packages. + # Must include directory name or it will remove all site*.py files. if ext_pkg.name != 'py-setuptools': - patterns.append(r'bin/easy_install[^/]*$') - patterns.append(r'site-packages/setuptools\.pth$') - patterns.append(r'site-packages/site[^/]*\.pyc?$') + patterns.extend([ + r'bin/easy_install[^/]*$', + r'site-packages/setuptools[^/]*\.egg$', + r'site-packages/setuptools\.pth$', + r'site-packages/site[^/]*\.pyc?$', + r'site-packages/__pycache__/site[^/]*\.pyc?$' + ]) if ext_pkg.name != 'py-numpy': patterns.append(r'bin/f2py$') return match_predicate(ignore_arg, patterns) - def write_easy_install_pth(self, exts): paths = [] for ext in sorted(exts.values()): - ext_site_packages = os.path.join(ext.prefix, self.site_packages_dir) - easy_pth = "%s/easy-install.pth" % ext_site_packages + ext_site_packages = join_path(ext.prefix, self.site_packages_dir) + easy_pth = join_path(ext_site_packages, "easy-install.pth") if not os.path.isfile(easy_pth): continue @@ -194,15 +222,18 @@ def write_easy_install_pth(self, exts): line = line.rstrip() # Skip lines matching these criteria - if not line: continue - if re.search(r'^(import|#)', line): continue - if (ext.name != 'py-setuptools' and - re.search(r'setuptools.*egg$', line)): continue + if not line: + continue + if re.search(r'^(import|#)', line): + continue + if ((ext.name != 'py-setuptools' and + re.search(r'setuptools.*egg$', line))): + continue paths.append(line) - site_packages = os.path.join(self.prefix, self.site_packages_dir) - main_pth = "%s/easy-install.pth" % site_packages + site_packages = join_path(self.prefix, self.site_packages_dir) + main_pth = join_path(site_packages, "easy-install.pth") if not paths: if os.path.isfile(main_pth): @@ -210,15 +241,22 @@ def write_easy_install_pth(self, exts): else: with closing(open(main_pth, 'w')) as f: - f.write("import sys; sys.__plen = len(sys.path)\n") + f.write(""" +import sys +sys.__plen = len(sys.path) +""") for path in paths: - f.write("%s\n" % path) - f.write("import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; " - "p=getattr(sys,'__egginsert',0); sys.path[p:p]=new; sys.__egginsert = p+len(new)\n") - + f.write("{0}\n".format(path)) + f.write(""" +new = sys.path[sys.__plen:] +del sys.path[sys.__plen:] +p = getattr(sys, '__egginsert', 0) +sys.path[p:p] = new +sys.__egginsert = p + len(new) +""") def activate(self, ext_pkg, **args): - ignore=self.python_ignore(ext_pkg, args) + ignore = self.python_ignore(ext_pkg, args) args.update(ignore=ignore) super(Python, self).activate(ext_pkg, **args) @@ -227,12 +265,12 @@ def activate(self, ext_pkg, **args): exts[ext_pkg.name] = ext_pkg.spec self.write_easy_install_pth(exts) - def deactivate(self, ext_pkg, **args): args.update(ignore=self.python_ignore(ext_pkg, args)) super(Python, self).deactivate(ext_pkg, **args) exts = spack.install_layout.extension_map(self.spec) - if ext_pkg.name in exts: # Make deactivate idempotent. + # Make deactivate idempotent + if ext_pkg.name in exts: del exts[ext_pkg.name] self.write_easy_install_pth(exts)