A new package: CNTK. (#3578)
This commit is contained in:
健美猫
Adam J. Stewart
parent
eef7d17946
commit
dd5f4cb251
401
var/spack/repos/builtin/packages/cntk/build.patch
Normal file
401
var/spack/repos/builtin/packages/cntk/build.patch
Normal file
@ -0,0 +1,401 @@
|
||||
From 6ee89684a8e7b509f977bab00573c73563df4bc6 Mon Sep 17 00:00:00 2001
|
||||
From: Jianwen WEI <weijianwen@gmail.com>
|
||||
Date: Wed, 5 Apr 2017 16:52:54 +0800
|
||||
Subject: [PATCH 1/1] Refine the build process to allow reuse pre-installed
|
||||
packages: multiverso 1bit-sgd kaldi openfst.
|
||||
|
||||
---
|
||||
Makefile | 69 +++++++++++++++---------------
|
||||
configure | 144 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
|
||||
2 files changed, 178 insertions(+), 35 deletions(-)
|
||||
|
||||
diff --git a/Makefile b/Makefile
|
||||
index 0586fd94b..bbd75e08c 100644
|
||||
--- a/Makefile
|
||||
+++ b/Makefile
|
||||
@@ -25,14 +25,18 @@
|
||||
# If not specified, GPU will not be enabled
|
||||
# CUB_PATH= path to NVIDIA CUB installation, so $(CUB_PATH)/cub/cub.cuh exists
|
||||
# defaults to /usr/local/cub-1.4.1
|
||||
-# CUDNN_PATH= path to NVIDIA cuDNN installation so $(CUDNN_PATH)/cuda/include/cudnn.h exists
|
||||
+# CUDNN_PATH= path to NVIDIA cuDNN installation so $(CUDNN_PATH)/include/cudnn.h exists
|
||||
# CuDNN version needs to be 5.0 or higher.
|
||||
# KALDI_PATH= Path to Kaldi
|
||||
# If not specified, Kaldi plugins will not be built
|
||||
+# OPENFST_PATH= Path to OpenFST
|
||||
+# OpenFST is required by Kaldi Reader.
|
||||
# OPENCV_PATH= path to OpenCV 3.1.0 installation, so $(OPENCV_PATH) exists
|
||||
# defaults to /usr/local/opencv-3.1.0
|
||||
# PROTOBUF_PATH= path to Protocol Buffers 3.1.0 installation, so $(PROTOBUF_PATH) exists
|
||||
# defaults to /usr/local/protobuf-3.1.0
|
||||
+# MULTIVERSO_PATH= path to Multiverso installation or source code, so $(MULTIVERSO_PATH) exists
|
||||
+# defaults to Source/Multiverso
|
||||
# LIBZIP_PATH= path to libzip installation, so $(LIBZIP_PATH) exists
|
||||
# defaults to /usr/local/
|
||||
# BOOST_PATH= path to Boost installation, so $(BOOST_PATH)/include/boost/test/unit_test.hpp
|
||||
@@ -149,8 +153,8 @@ ifdef CUDA_PATH
|
||||
|
||||
# Set up cuDNN if needed
|
||||
ifdef CUDNN_PATH
|
||||
- INCLUDEPATH += $(CUDNN_PATH)/cuda/include
|
||||
- LIBPATH += $(CUDNN_PATH)/cuda/lib64
|
||||
+ INCLUDEPATH += $(CUDNN_PATH)/include
|
||||
+ LIBPATH += $(CUDNN_PATH)/lib64
|
||||
LIBS_LIST += cudnn
|
||||
COMMON_FLAGS +=-DUSE_CUDNN
|
||||
endif
|
||||
@@ -188,16 +192,11 @@ ifeq ("$(MATHLIB)","openblas")
|
||||
CPPFLAGS += -DUSE_OPENBLAS
|
||||
endif
|
||||
|
||||
-
|
||||
+# Set up kaldi and openfst for Kaldi2Reader if needed
|
||||
ifdef KALDI_PATH
|
||||
- ########## Copy includes and defines from $(KALDI_PATH)/src/kaldi.mk ##########
|
||||
- FSTROOT = $(KALDI_PATH)/tools/openfst
|
||||
- ATLASINC = $(KALDI_PATH)/tools/ATLAS/include
|
||||
-
|
||||
- INCLUDEPATH += $(KALDI_PATH)/src $(ATLASINC) $(FSTROOT)/include
|
||||
+ INCLUDEPATH += $(KALDI_PATH)/include $(OPENFST_PATH)/incldue
|
||||
CPPFLAGS += -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_ATLAS -DHAVE_OPENFST_GE_10400
|
||||
-
|
||||
- KALDI_LIBPATH += $(KALDI_PATH)/src/lib
|
||||
+ KALDI_LIBPATH += $(KALDI_PATH)/lib $(OPENFST_PATH)/lib
|
||||
KALDI_LIBS_LIST := kaldi-util kaldi-matrix kaldi-base kaldi-hmm kaldi-cudamatrix kaldi-nnet kaldi-lat
|
||||
KALDI_LIBS := $(addprefix -l,$(KALDI_LIBS_LIST))
|
||||
endif
|
||||
@@ -316,7 +315,7 @@ SRC += $(PP_SRC)
|
||||
$(PERF_PROFILER_LIB): $(PP_OBJ)
|
||||
@echo $(SEPARATOR)
|
||||
@echo creating $@ for $(ARCH) with build type $(BUILDTYPE)
|
||||
- @mkdir -p $(dir $@)
|
||||
+ -@mkdir -p $(dir $@)
|
||||
$(CXX) $(LDFLAGS) -shared $(patsubst %,$(RPATH)%, $(ORIGINDIR)) -o $@ $^
|
||||
|
||||
|
||||
@@ -920,17 +919,13 @@ endif
|
||||
|
||||
ifeq ("$(CNTK_ENABLE_1BitSGD)","true")
|
||||
|
||||
-ifeq (,$(wildcard Source/1BitSGD/*.h))
|
||||
- $(error Build with 1bit-SGD was requested but cannot find the code. Please check https://github.com/Microsoft/CNTK/wiki/Enabling-1bit-SGD for instructions)
|
||||
-endif
|
||||
-
|
||||
- INCLUDEPATH += $(SOURCEDIR)/1BitSGD
|
||||
+ INCLUDEPATH += $(ONEBITSGD_PATH)
|
||||
|
||||
COMMON_FLAGS += -DCNTK_PARALLEL_TRAINING_SUPPORT
|
||||
# temporarily adding to 1bit, need to work with others to fix it
|
||||
endif
|
||||
|
||||
-
|
||||
+
|
||||
########################################
|
||||
# ASGD(multiverso) setup
|
||||
########################################
|
||||
@@ -938,13 +933,9 @@ endif
|
||||
|
||||
ifeq ("$(CNTK_ENABLE_ASGD)","true")
|
||||
|
||||
-ifeq (,$(wildcard Source/Multiverso/include/multiverso/*.h))
|
||||
- $(error Build with Multiverso was requested but cannot find the code. Please check https://github.com/Microsoft/CNTK/wiki/Multiple-GPUs-and-machines#24-data-parallel-asgd to learn more.)
|
||||
-endif
|
||||
-
|
||||
lMULTIVERSO:=-lmultiverso
|
||||
|
||||
-INCLUDEPATH += $(SOURCEDIR)/Multiverso/include
|
||||
+INCLUDEPATH += $(MULTIVERSO_PATH)/include
|
||||
COMMON_FLAGS += -DASGD_PARALLEL_SUPPORT
|
||||
|
||||
MULTIVERSO_LIB:=$(LIBDIR)/libmultiverso.so
|
||||
@@ -958,11 +949,16 @@ MULTIVERSO_CMAKE_BUILDTYPE=Debug
|
||||
endif
|
||||
|
||||
# TODO need to align Multiverso OpenMP with the one we use (libiomp). For now, disabled.
|
||||
-$(MULTIVERSO_LIB):
|
||||
+$(MULTIVERSO_LIB):
|
||||
+ifneq ("$(wildcard $(MULTIVERSO_PATH)/lib/libmultiverso.so)","")
|
||||
+ # reuse pre-installed multiverso library to $(LIBDIR) then skip building $(MULTIVERSO_LIB) when possible
|
||||
+ @mkdir -p $(LIBDIR)
|
||||
+ cp $(MULTIVERSO_PATH)/lib/libmultiverso.so $(LIBDIR)
|
||||
+else
|
||||
@echo "Build Multiverso lib"
|
||||
@mkdir -p $(LIBDIR)
|
||||
@mkdir -p $(BINDIR)
|
||||
- @mkdir -p $(SOURCEDIR)/Multiverso/build/$(BUILDTYPE)
|
||||
+ @mkdir -p $(MULTIVERSO_PATH)/build/$(BUILDTYPE)
|
||||
@cmake -DCMAKE_VERBOSE_MAKEFILE=TRUE \
|
||||
-DCMAKE_CXX_COMPILER=$(CXX) \
|
||||
-DOpenMP_CXX_FLAGS="" \
|
||||
@@ -976,17 +972,22 @@ $(MULTIVERSO_LIB):
|
||||
-DCMAKE_BUILD_TYPE=$(MULTIVERSO_CMAKE_BUILDTYPE) \
|
||||
-B./Source/Multiverso/build/$(BUILDTYPE) -H./Source/Multiverso
|
||||
@make VERBOSE=1 -C ./Source/Multiverso/build/$(BUILDTYPE) -j multiverso
|
||||
+endif
|
||||
|
||||
UNITTEST_MULTIVERSO_SRC = \
|
||||
- $(SOURCEDIR)/Multiverso/Test/unittests/test_array.cpp \
|
||||
- $(SOURCEDIR)/Multiverso/Test/unittests/test_blob.cpp \
|
||||
- $(SOURCEDIR)/Multiverso/Test/unittests/test_kv.cpp \
|
||||
- $(SOURCEDIR)/Multiverso/Test/unittests/test_message.cpp \
|
||||
- $(SOURCEDIR)/Multiverso/Test/unittests/test_multiverso.cpp \
|
||||
- $(SOURCEDIR)/Multiverso/Test/unittests/test_node.cpp \
|
||||
- $(SOURCEDIR)/Multiverso/Test/unittests/test_sync.cpp \
|
||||
-
|
||||
-UNITTEST_MULTIVERSO_OBJ := $(patsubst %.cpp, $(OBJDIR)/%.o, $(UNITTEST_MULTIVERSO_SRC))
|
||||
+ $(MULTIVERSO_PATH)/Test/unittests/test_array.cpp \
|
||||
+ $(MULTIVERSO_PATH)/Test/unittests/test_blob.cpp \
|
||||
+ $(MULTIVERSO_PATH)/Test/unittests/test_kv.cpp \
|
||||
+ $(MULTIVERSO_PATH)/Test/unittests/test_message.cpp \
|
||||
+ $(MULTIVERSO_PATH)/Test/unittests/test_multiverso.cpp \
|
||||
+ $(MULTIVERSO_PATH)/Test/unittests/test_node.cpp \
|
||||
+ $(MULTIVERSO_PATH)/Test/unittests/test_sync.cpp \
|
||||
+
|
||||
+UNITTEST_MULTIVERSO_OBJ := $(patsubst $(MULTIVERSO_PATH)/Test/unittests/%.cpp, $(OBJDIR)/Source/Multiverso/Test/unittests/%.o, $(UNITTEST_MULTIVERSO_SRC))
|
||||
+
|
||||
+$(OBJDIR)/Source/Multiverso/Test/unittests/%.o: $(MULTIVERSO_PATH)/Test/unittests/%.cpp
|
||||
+ -@ mkdir -p $(OBJDIR)/Source/Multiverso/Test/unittests
|
||||
+ $(CXX) -c $< -o $@ $(COMMON_FLAGS) $(CPPFLAGS) $(CXXFLAGS) $(INCLUDEPATH:%=-I%)
|
||||
|
||||
UNITTEST_MULTIVERSO := $(BINDIR)/multiversotests
|
||||
|
||||
diff --git a/configure b/configure
|
||||
index 2c2e5cee7..c9c0072c9 100755
|
||||
--- a/configure
|
||||
+++ b/configure
|
||||
@@ -41,6 +41,14 @@ boost_check=include/boost/test/unit_test.hpp
|
||||
protobuf_path=
|
||||
protobuf_check=lib/libprotobuf.a
|
||||
|
||||
+# Multiverso library
|
||||
+multiverso_path=
|
||||
+multiverso_check=include/multiverso/multiverso.h
|
||||
+
|
||||
+# 1bitsgd library
|
||||
+onebitsgd_path=
|
||||
+onebitsgd_check=BlockMomentumSGD.h
|
||||
+
|
||||
# MPI library
|
||||
mpi_path=
|
||||
mpi_check=include/mpi.h
|
||||
@@ -49,6 +57,10 @@ have_kaldi=no
|
||||
kaldi_path=
|
||||
kaldi_check=src/kaldi.mk
|
||||
|
||||
+have_openfst=no
|
||||
+openfst_path=
|
||||
+openfst_check=include/fst/fst.h
|
||||
+
|
||||
have_buildtype=no
|
||||
buildtype=
|
||||
default_buildtype=release
|
||||
@@ -67,7 +79,7 @@ cub_check=cub/cub.cuh
|
||||
|
||||
have_cudnn=no
|
||||
cudnn_path=
|
||||
-cudnn_check=cuda/include/cudnn.h
|
||||
+cudnn_check=include/cudnn.h
|
||||
|
||||
have_opencv=no
|
||||
opencv_path=
|
||||
@@ -112,6 +124,7 @@ default_boost="boost-1.60.0"
|
||||
default_cudas="cuda-8.0 cuda-7.5"
|
||||
default_nccls="nccl"
|
||||
default_kaldis="kaldi-trunk kaldi-c024e8aa"
|
||||
+default_openfst="openfst-1.6"
|
||||
default_gdk_includes="include/nvidia/gdk"
|
||||
default_gdk_nvml_libs="src/gdk/nvml/lib"
|
||||
default_cubs="cub-1.4.1"
|
||||
@@ -121,6 +134,8 @@ default_protobuf="protobuf-3.1.0"
|
||||
default_libzips="libzip-1.1.2"
|
||||
default_swig="swig-3.0.10"
|
||||
default_mpi="mpi"
|
||||
+default_multiverso=Source/Multiverso
|
||||
+default_onebitsgd=Source/1BitSGD
|
||||
|
||||
function default_paths ()
|
||||
{
|
||||
@@ -178,6 +193,16 @@ function find_protobuf ()
|
||||
find_dir "$default_protobuf" "$protobuf_check"
|
||||
}
|
||||
|
||||
+function find_multiverso ()
|
||||
+{
|
||||
+ find_dir "$default_multiverso" "$multiverso_check"
|
||||
+}
|
||||
+
|
||||
+function find_onebitsgd ()
|
||||
+{
|
||||
+ find_dir "$default_onebitsgd" "$onebitsgd_check"
|
||||
+}
|
||||
+
|
||||
function find_nccl ()
|
||||
{
|
||||
find_dir "$default_nccls" "$nccl_check"
|
||||
@@ -193,6 +218,11 @@ function find_kaldi ()
|
||||
find_dir "$default_kaldis" "$kaldi_check"
|
||||
}
|
||||
|
||||
+function find_openfst ()
|
||||
+{
|
||||
+ find_dir "$default_openfst" "$openfst_check"
|
||||
+}
|
||||
+
|
||||
function find_gdk_include ()
|
||||
{
|
||||
find_dir "$default_gdk_includes" "$gdk_include_check"
|
||||
@@ -352,11 +382,14 @@ function show_help ()
|
||||
echo " --with-openblas[=directory] (experimental) $(show_default $(find_openblas))"
|
||||
echo " --with-buildtype=(debug|release) $(show_default $default_buildtype)"
|
||||
echo " --with-kaldi[=directory] $(show_default $(find_kaldi))"
|
||||
+ echo " --with-openfst[=directory] $(show_default $(find_openfst))"
|
||||
echo " --with-opencv[=directory] $(show_default $(find_opencv))"
|
||||
echo " --with-libzip[=directory] $(show_default $(find_libzip))"
|
||||
echo " --with-code-coverage[=(yes|no)] $(show_default ${default_use_code_coverage})"
|
||||
echo " --with-boost[=directory] $(show_default $(find_boost))"
|
||||
echo " --with-protobuf[=directory] $(show_default $(find_protobuf))"
|
||||
+ echo " --with-multiverso[=directory] $(show_default $(find_multiverso))"
|
||||
+ echo " --with-1bitsgd[=directory] $(show_default $(find_onebitsgd))"
|
||||
echo " --with-py-versions=(space-separated list of 27, 34, 35)"
|
||||
echo " --with-py27-path[=directory] $(show_default $(find_python 27))"
|
||||
echo " --with-py34-path[=directory] $(show_default $(find_python 34))"
|
||||
@@ -751,6 +784,46 @@ do
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
+ --with-multiverso*)
|
||||
+ if test x$optarg = x
|
||||
+ then
|
||||
+ multiverso_path=$(find_multiverso)
|
||||
+ if test x$multiverso_path = x
|
||||
+ then
|
||||
+ echo "Cannot find Multiverso directory"
|
||||
+ echo "Please specify a value for --with-multiverso"
|
||||
+ exit 1
|
||||
+ fi
|
||||
+ else
|
||||
+ if test $(check_dir $optarg $multiverso_check) = yes
|
||||
+ then
|
||||
+ multiverso_path=$optarg
|
||||
+ else
|
||||
+ echo "Invalid Multiverso directory $optarg"
|
||||
+ exit 1
|
||||
+ fi
|
||||
+ fi
|
||||
+ ;;
|
||||
+ --with-1bitsgd*)
|
||||
+ if test x$optarg = x
|
||||
+ then
|
||||
+ onebitsgd_path=$(find_onebitsgd)
|
||||
+ if test x$onebitsgd_path = x
|
||||
+ then
|
||||
+ echo "Cannot find 1BitSGD directory"
|
||||
+ echo "Please specify a value for --with-1bitsgd"
|
||||
+ exit 1
|
||||
+ fi
|
||||
+ else
|
||||
+ if test $(check_dir $optarg $onebitsgd_check) = yes
|
||||
+ then
|
||||
+ onebitsgd_path=$optarg
|
||||
+ else
|
||||
+ echo "Invalid 1BitSGD directory $optarg"
|
||||
+ exit 1
|
||||
+ fi
|
||||
+ fi
|
||||
+ ;;
|
||||
--with-buildtype*)
|
||||
have_buildtype=yes
|
||||
case $optarg in
|
||||
@@ -784,6 +857,27 @@ do
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
+ --with-openfst*)
|
||||
+ have_openfst=yes
|
||||
+ if test x$optarg = x
|
||||
+ then
|
||||
+ openfst_path=$(find_openfst)
|
||||
+ if test x$openfst_path = x
|
||||
+ then
|
||||
+ echo "Cannot find openfst directory"
|
||||
+ echo "Please specify a value for --with-openfst"
|
||||
+ exit 1
|
||||
+ fi
|
||||
+ else
|
||||
+ if test $(check_dir $optarg $openfst_check)
|
||||
+ then
|
||||
+ openfst_path=$optarg
|
||||
+ else
|
||||
+ echo "Invalid openfst directory $optarg"
|
||||
+ exit 1
|
||||
+ fi
|
||||
+ fi
|
||||
+ ;;
|
||||
--with-opencv*)
|
||||
have_opencv=yes
|
||||
if test x$optarg = x
|
||||
@@ -1049,6 +1143,48 @@ then
|
||||
fi
|
||||
fi
|
||||
|
||||
+if test x$kaldi_path != x && test x$openfst_path = x
|
||||
+then
|
||||
+ openfst_path=$(find_openfst)
|
||||
+ if test x$openfst_path = x
|
||||
+ then
|
||||
+ echo CNTK KaldiReader requires OpenFST library. Please check
|
||||
+ echo https://github.com/Microsoft/CNTK/blob/master/Source/Readers/KaldiReaderReadme
|
||||
+ echo to learn more.
|
||||
+ exit 1
|
||||
+ else
|
||||
+ echo Found OpenFST library at $openfst_path
|
||||
+ fi
|
||||
+fi
|
||||
+
|
||||
+if test $enable_1bitsgd = yes && test x$onebitsgd_path = x
|
||||
+then
|
||||
+ onebitsgd_path=$(find_onebitsgd)
|
||||
+ if test x$onebitsgd_path = x
|
||||
+ then
|
||||
+ echo Cannot locate 1BitSGD library. See
|
||||
+ echo https://github.com/Microsoft/CNTK/wiki/Enabling-1bit-SGD
|
||||
+ echo for installation instructions.
|
||||
+ exit 1
|
||||
+ else
|
||||
+ echo Found 1BitSGD library at $onebitsgd_path
|
||||
+ fi
|
||||
+fi
|
||||
+
|
||||
+if test $enable_asgd = yes && test x$multiverso_path = x
|
||||
+then
|
||||
+ multiverso_path=$(find_multiverso)
|
||||
+ if test x$multiverso_path = x
|
||||
+ then
|
||||
+ echo Build with Multiverso was requested but cannot find the code. Please check
|
||||
+ echo https://github.com/Microsoft/CNTK/wiki/Multiple-GPUs-and-machines#24-data-parallel-asgd
|
||||
+ echo to learn more.
|
||||
+ exit 1
|
||||
+ else
|
||||
+ echo Found Multiverso library at $multiverso_path
|
||||
+ fi
|
||||
+fi
|
||||
+
|
||||
if test x$mpi_path = x
|
||||
then
|
||||
mpi_path=$(find_mpi)
|
||||
@@ -1123,6 +1259,12 @@ fi
|
||||
if test x$protobuf_path != x; then
|
||||
echo PROTOBUF_PATH=$protobuf_path >> $config
|
||||
fi
|
||||
+if test x$multiverso_path != x; then
|
||||
+ echo MULTIVERSO_PATH=$multiverso_path >> $config
|
||||
+fi
|
||||
+if test x$onebitsgd_path != x; then
|
||||
+ echo ONEBITSGD_PATH=$onebitsgd_path >> $config
|
||||
+fi
|
||||
if test x$mpi_path != x; then
|
||||
echo MPI_PATH=$mpi_path >> $config
|
||||
fi
|
||||
--
|
||||
2.11.1
|
||||
|
||||
@ -0,0 +1,32 @@
|
||||
From a5cdb4fdce55813b79e3ee6300f282ba6d7b6b85 Mon Sep 17 00:00:00 2001
|
||||
From: Jianwen WEI <weijianwen@gmail.com>
|
||||
Date: Fri, 14 Apr 2017 12:40:44 +0800
|
||||
Subject: [PATCH 1/1] Replace ATLAS with OpenBLAS in KaldiReader.
|
||||
|
||||
---
|
||||
Makefile | 4 ++--
|
||||
1 file changed, 2 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/Makefile b/Makefile
|
||||
index bbd75e08c..909457963 100644
|
||||
--- a/Makefile
|
||||
+++ b/Makefile
|
||||
@@ -189,13 +189,13 @@ ifeq ("$(MATHLIB)","openblas")
|
||||
INCLUDEPATH += $(OPENBLAS_PATH)/include
|
||||
LIBPATH += $(OPENBLAS_PATH)/lib
|
||||
LIBS_LIST += openblas m pthread
|
||||
- CPPFLAGS += -DUSE_OPENBLAS
|
||||
+ CPPFLAGS += -DUSE_OPENBLAS -DHAVE_OPENBLAS
|
||||
endif
|
||||
|
||||
# Set up kaldi and openfst for Kaldi2Reader if needed
|
||||
ifdef KALDI_PATH
|
||||
INCLUDEPATH += $(KALDI_PATH)/include $(OPENFST_PATH)/incldue
|
||||
- CPPFLAGS += -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_ATLAS -DHAVE_OPENFST_GE_10400
|
||||
+ CPPFLAGS += -DKALDI_DOUBLEPRECISION=0 -DHAVE_POSIX_MEMALIGN -DHAVE_EXECINFO_H=1 -DHAVE_CXXABI_H -DHAVE_OPENFST_GE_10400
|
||||
KALDI_LIBPATH += $(KALDI_PATH)/lib $(OPENFST_PATH)/lib
|
||||
KALDI_LIBS_LIST := kaldi-util kaldi-matrix kaldi-base kaldi-hmm kaldi-cudamatrix kaldi-nnet kaldi-lat
|
||||
KALDI_LIBS := $(addprefix -l,$(KALDI_LIBS_LIST))
|
||||
--
|
||||
2.11.1
|
||||
|
||||
117
var/spack/repos/builtin/packages/cntk/package.py
Normal file
117
var/spack/repos/builtin/packages/cntk/package.py
Normal file
@ -0,0 +1,117 @@
|
||||
##############################################################################
|
||||
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
|
||||
# Produced at the Lawrence Livermore National Laboratory.
|
||||
#
|
||||
# This file is part of Spack.
|
||||
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
|
||||
# LLNL-CODE-647188
|
||||
#
|
||||
# For details, see https://github.com/llnl/spack
|
||||
# Please also see the LICENSE file for our notice and the LGPL.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License (as
|
||||
# published by the Free Software Foundation) version 2.1, February 1999.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
|
||||
# conditions of the GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
##############################################################################
|
||||
from spack import *
|
||||
|
||||
|
||||
class Cntk(Package):
|
||||
"""The Microsoft Cognitive Toolkit is a unified deep-learning toolkit
|
||||
that describes neural networks as a series of computational steps
|
||||
via a directed graph."""
|
||||
|
||||
homepage = "https://www.microsoft.com/en-us/research/product/cognitive-toolkit"
|
||||
url = "https://github.com/Microsoft/CNTK/archive/v2.0.beta15.0.tar.gz"
|
||||
|
||||
version('master', git='https://github.com/Microsoft/CNTK.git', branch='master')
|
||||
version('2.0.rc1', url='https://github.com/Microsoft/CNTK/archive/v2.0.rc1.tar.gz')
|
||||
|
||||
variant('opencv', default=False, description="Enable OpenCV support.")
|
||||
variant('kaldi', default=False, description="Enable Kaldi support.")
|
||||
variant('asgd', default=True, description="Enable DataParallelASGD powered by Multiverso.")
|
||||
variant('1bitsgd', default=False, description="Enable 1bitsgd support.")
|
||||
variant('cuda', default=False, description="Enable CUDA support.")
|
||||
variant('debug', default=False, description="Debug build.")
|
||||
|
||||
depends_on('libzip')
|
||||
depends_on('openblas')
|
||||
depends_on('mpi')
|
||||
depends_on('boost')
|
||||
depends_on('protobuf~shared')
|
||||
# CNTK depends on kaldi@c02e8.
|
||||
# See https://github.com/Microsoft/CNTK/blob/master/Tools/docker/CNTK-CPUOnly-Image/Dockerfile#L105-L125
|
||||
depends_on('kaldi@c024e8', when='+kaldi')
|
||||
depends_on('opencv', when='+opencv')
|
||||
depends_on('cuda', when='+cuda')
|
||||
depends_on('cub@1.4.1', when='+cuda')
|
||||
depends_on('cudnn@5.1', when='+cuda')
|
||||
depends_on('nccl', when='+cuda')
|
||||
depends_on('cntk1bitsgd@c8b77d', when='+1bitsgd')
|
||||
depends_on('multiverso@143187', when='+asgd')
|
||||
|
||||
# Patch CNTN's build process to use libs installed outside CNTK source tree
|
||||
# multiverso, kaldi, openfst
|
||||
patch('build.patch')
|
||||
# Patch to fix BLAS inconsistency between CNTK and KaldiReader
|
||||
patch('kaldireader-openblas.patch')
|
||||
|
||||
def install(self, spec, prefix):
|
||||
args = []
|
||||
|
||||
args.append('--with-mpi=' + spec['mpi'].prefix)
|
||||
args.append('--with-openblas=' + spec['openblas'].prefix)
|
||||
args.append('--with-libzip=' + spec['libzip'].prefix)
|
||||
args.append('--with-boost=' + spec['boost'].prefix)
|
||||
args.append('--with-protobuf=' + spec['protobuf'].prefix)
|
||||
|
||||
if '+debug' in spec:
|
||||
args.append('--with-buildtype=debug')
|
||||
else:
|
||||
args.append('--with-buildtype=release')
|
||||
|
||||
if '+1bitsgd' in spec:
|
||||
args.append('--1bitsgd=yes')
|
||||
args.append('--with-1bitsgd={0}/include'
|
||||
.format(spec['cntk1bitsgd'].prefix))
|
||||
|
||||
if '+asgd' in spec:
|
||||
args.append('--asgd=yes')
|
||||
args.append('--with-multiverso={0}'
|
||||
.format(spec['multiverso'].prefix))
|
||||
else:
|
||||
args.append('--asgd=no')
|
||||
|
||||
if '+opencv' in spec:
|
||||
args.append('--with-opencv=' + spec['opencv'].prefix)
|
||||
|
||||
if '+kaldi' in spec:
|
||||
args.append('--with-kaldi=' + spec['kaldi'].prefix)
|
||||
args.append('--with-openfst=' + spec['openfst'].prefix)
|
||||
|
||||
if '+cuda' in spec:
|
||||
args.append('--cuda=yes')
|
||||
args.append('--with-cuda={0}'.format(spec['cuda'].prefix))
|
||||
args.append('--with-cub={0}'
|
||||
.format(spec['cub'].prefix.include))
|
||||
args.append('--with-cudnn={0}'
|
||||
.format(spec['cudnn'].prefix))
|
||||
args.append('--with-nccl={0}'.format(spec['nccl'].prefix))
|
||||
|
||||
configure(*args)
|
||||
|
||||
make()
|
||||
|
||||
install_tree('bin', prefix.bin)
|
||||
install_tree('lib', prefix.lib)
|
||||
install_tree('Examples', join_path(prefix, 'Examples'))
|
||||
install_tree('Tutorials', join_path(prefix, 'Tutorials'))
|
||||
44
var/spack/repos/builtin/packages/cntk1bitsgd/package.py
Normal file
44
var/spack/repos/builtin/packages/cntk1bitsgd/package.py
Normal file
@ -0,0 +1,44 @@
|
||||
##############################################################################
|
||||
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
|
||||
# Produced at the Lawrence Livermore National Laboratory.
|
||||
#
|
||||
# This file is part of Spack.
|
||||
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
|
||||
# LLNL-CODE-647188
|
||||
#
|
||||
# For details, see https://github.com/llnl/spack
|
||||
# Please also see the LICENSE file for our notice and the LGPL.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License (as
|
||||
# published by the Free Software Foundation) version 2.1, February 1999.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
|
||||
# conditions of the GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
##############################################################################
|
||||
from spack import *
|
||||
from os import listdir
|
||||
|
||||
|
||||
class Cntk1bitsgd(Package):
|
||||
"""CNTK1bitSGD is the header-only
|
||||
1-bit stochastic gradient descent (1bit-SGD) library for
|
||||
the Computational Network Toolkit (CNTK)."""
|
||||
|
||||
homepage = "https://github.com/CNTK-components/CNTK1bitSGD"
|
||||
|
||||
version('master', git='https://github.com/CNTK-components/CNTK1bitSGD.git')
|
||||
version('c8b77d', git='https://github.com/CNTK-components/CNTK1bitSGD.git',
|
||||
commit='c8b77d6e325a4786547b27624890276c1483aed1')
|
||||
|
||||
def install(self, spec, prefix):
|
||||
mkdirp(prefix.include)
|
||||
for file in listdir('.'):
|
||||
if file.endswith('.h'):
|
||||
install(file, prefix.include)
|
||||
@ -0,0 +1,28 @@
|
||||
From 478bcfecadafb09aa73bacab35d932851b323142 Mon Sep 17 00:00:00 2001
|
||||
From: Jianwen WEI <weijianwen@gmail.com>
|
||||
Date: Mon, 10 Apr 2017 15:29:50 +0800
|
||||
Subject: [PATCH 1/1] Patch 1431875.
|
||||
|
||||
---
|
||||
CMakeLists.txt | 5 +++--
|
||||
1 file changed, 3 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/CMakeLists.txt b/CMakeLists.txt
|
||||
index 6a824f1..100b21c 100644
|
||||
--- a/CMakeLists.txt
|
||||
+++ b/CMakeLists.txt
|
||||
@@ -30,9 +30,10 @@ ADD_SUBDIRECTORY(Test/unittests)
|
||||
ADD_SUBDIRECTORY(Applications/WordEmbedding)
|
||||
ADD_SUBDIRECTORY(Applications/LogisticRegression)
|
||||
|
||||
-if(USE_INSTALL_MULTIVERSO)
|
||||
+if(INSTALL_MULTIVERSO)
|
||||
install (DIRECTORY ${PROJECT_SOURCE_DIR}/include/multiverso DESTINATION include)
|
||||
-endif(USE_INSTALL_MULTIVERSO)
|
||||
+ install (DIRECTORY ${PROJECT_SOURCE_DIR}/Test DESTINATION .)
|
||||
+endif(INSTALL_MULTIVERSO)
|
||||
|
||||
|
||||
# uninstall target
|
||||
--
|
||||
2.11.1
|
||||
48
var/spack/repos/builtin/packages/multiverso/package.py
Normal file
48
var/spack/repos/builtin/packages/multiverso/package.py
Normal file
@ -0,0 +1,48 @@
|
||||
##############################################################################
|
||||
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
|
||||
# Produced at the Lawrence Livermore National Laboratory.
|
||||
#
|
||||
# This file is part of Spack.
|
||||
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
|
||||
# LLNL-CODE-647188
|
||||
#
|
||||
# For details, see https://github.com/llnl/spack
|
||||
# Please also see the LICENSE file for our notice and the LGPL.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License (as
|
||||
# published by the Free Software Foundation) version 2.1, February 1999.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
|
||||
# conditions of the GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
##############################################################################
|
||||
from spack import *
|
||||
|
||||
|
||||
class Multiverso(CMakePackage):
|
||||
"""Multiverso is a parameter server based framework for
|
||||
training machine learning models on big data with numbers of machines."""
|
||||
|
||||
homepage = "https://github.com/Microsoft/Multiverso"
|
||||
url = "https://github.com/Microsoft/Multiverso/archive/v0.2.tar.gz"
|
||||
|
||||
version('master', git='https://github.com/Microsoft/Multiverso.git',
|
||||
branch='master')
|
||||
version('143187', git='https://github.com/Microsoft/Multiverso.git',
|
||||
commit='143187575d1cfa410100037b8aea2e767e0af637')
|
||||
version('0.2', '483ca7524fea14a311389e421f2bc098')
|
||||
|
||||
depends_on('mpi')
|
||||
depends_on('boost')
|
||||
|
||||
patch('cmake-143187.patch', when='@143187')
|
||||
|
||||
def cmake_args(self):
|
||||
spec = self.spec
|
||||
return ['-DBOOST_ROOT:PATH=%s' % spec['boost'].prefix]
|
||||
Loading…
Reference in New Issue
Block a user