zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Oneapi packages: update URLs, environment management, and dependencies (#22202)

Browse Source 
* Replace URL computation in base IntelOneApiPackage class with
  defining URLs in component packages (this is expected to be
  simpler for now)
* Add component_dir property that all oneAPI component packages must
  define. This property names a directory that should exist after
  installation completes (useful for making sure the install was
  successful) and also defines the search location for the
  component's environment update script.
* Add needed dependencies for components (e.g. intel-oneapi-dnn
  requires intel-oneapi-tbb). The compilers provided by
  intel-oneapi-compilers need some components under certain
  circumstances (e.g. when enabling SYCL support) but these were
  omitted since the libraries should only be linked when a
  dependent package requests that feature
* Remove individual setup_run_environment implementations and use
  IntelOneApiPackage superclass method which sources vars.sh 
  (located in a subdirectory of component_dir)
* Add documentation for IntelOneApiPackge build system

Co-authored-by: Vasily Danilin <vasily.danilin@yandex.ru>
This commit is contained in:
Robert Cohn
2021-03-22 20:35:45 -04:00
committed by GitHub
parent 138312efab
commit f57626a7c4
14 changed files with 340 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
var/spack/repos/builtin/packages/intel-oneapi-ccl/package.py
View File

@@ -3,12 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from sys import platform
from spack import *
releases = {
'2021.1.1': {'irc_id': '17391', 'build': '54'}}
class IntelOneapiCcl(IntelOneApiLibraryPackage):
"""Intel oneAPI CCL."""
@@ -17,11 +15,14 @@ class IntelOneapiCcl(IntelOneApiLibraryPackage):
homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/oneccl.html'
version('2021.1.1', sha256='de732df57a03763a286106c8b885fd60e83d17906936a8897a384b874e773f49', expand=False)
depends_on('intel-oneapi-mpi')
def __init__(self, spec):
self.component_info(dir_name='ccl',
components='intel.oneapi.lin.ccl.devel',
releases=releases,
url_name='oneapi_ccl')
super(IntelOneapiCcl, self).__init__(spec)
if platform == 'linux':
version('2021.1.1',
sha256='de732df57a03763a286106c8b885fd60e83d17906936a8897a384b874e773f49',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17391/l_oneapi_ccl_p_2021.1.1.54_offline.sh',
expand=False)
@property
def component_dir(self):
return 'ccl'

102
var/spack/repos/builtin/packages/intel-oneapi-compilers/package.py
View File

@@ -5,82 +5,78 @@
import glob
import subprocess
from os import path
from sys import platform
from spack import *
releases = {'2021.1.0':
{'irc_id': '17427', 'build': '2684'}}
class IntelOneapiCompilers(IntelOneApiPackage):
"""Intel oneAPI compilers.
"""Intel OneAPI compilers
Contains icc, icpc, icx, icpx, dpcpp, ifort, ifx.
Provides Classic and Beta compilers for: Fortran, C, C++"""
"""
maintainers = ['rscohn2']
homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/dpc-compiler.html'
version('2021.1.0', sha256='666b1002de3eab4b6f3770c42bcf708743ac74efeba4c05b0834095ef27a11b9', expand=False)
homepage = "https://software.intel.com/content/www/us/en/develop/tools/oneapi.html"
depends_on('patchelf', type='build')
def __init__(self, spec):
self.component_info(
dir_name='compiler',
components=('intel.oneapi.lin.dpcpp-cpp-compiler-pro'
':intel.oneapi.lin.ifort-compiler'),
releases=releases,
url_name='HPCKit')
super(IntelOneapiCompilers, self).__init__(spec)
if platform == 'linux':
version('2021.1.2',
sha256='68d6cb638091990e578e358131c859f3bbbbfbf975c581fd0b4b4d36476d6f0a',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17513/l_dpcpp-cpp-compiler_p_2021.1.2.63_offline.sh',
expand=False)
resource(name='fortran-installer',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17508/l_fortran-compiler_p_2021.1.2.62_offline.sh',
sha256='29345145268d08a59fa7eb6e58c7522768466dd98f6d9754540d1a0803596829',
expand=False,
placement='fortran-installer',
when='@2021.1.2')
def _join_prefix(self, path):
return join_path(self.prefix, 'compiler', 'latest', 'linux', path)
@property
def component_dir(self):
return 'compiler'
def _join_prefix(self, p):
return path.join(self.prefix, 'compiler', 'latest', 'linux', p)
def _ld_library_path(self):
dirs = ['lib',
'lib/x64',
'lib/emu',
'lib/oclfpga/host/linux64/lib',
'lib/oclfpga/linux64/lib',
'compiler/lib/intel64_lin',
'compiler/lib']
path.join('lib', 'x64'),
path.join('lib', 'emu'),
path.join('lib', 'oclfpga', 'host', 'linux64', 'lib'),
path.join('lib', 'oclfpga', 'linux64', 'lib'),
path.join('compiler', 'lib', 'intel64_lin'),
path.join('compiler', 'lib')]
for dir in dirs:
yield self._join_prefix(dir)
def install(self, spec, prefix):
# For quick turnaround debugging, comment out line below and
# use the copy instead
# install cpp
# Copy instead of install to speed up debugging
# subprocess.run(f'cp -r /opt/intel/oneapi/compiler {prefix}', shell=True)
super(IntelOneapiCompilers, self).install(spec, prefix)
# Copy installed compiler instead of running the installer
# from shutil import copytree
# copytree('/opt/intel/oneapi/compiler', join_path(prefix, 'compiler'),
# symlinks=True)
# install fortran
super(IntelOneapiCompilers, self).install(
spec,
prefix,
installer_path=glob.glob(path.join('fortran-installer', '*'))[0])
# Some installers have a bug and do not return an error code when failing
if not path.isfile(path.join(prefix, 'compiler', 'latest', 'linux',
'bin', 'intel64', 'ifort')):
raise RuntimeError('install failed')
# set rpath so 'spack compiler add' can check version strings
# without setting LD_LIBRARY_PATH
rpath = ':'.join(self._ld_library_path())
patch_dirs = ['compiler/lib/intel64_lin',
'compiler/lib/intel64',
patch_dirs = [path.join('compiler', 'lib', 'intel64_lin'),
path.join('compiler', 'lib', 'intel64'),
'bin']
for pd in patch_dirs:
for file in glob.glob(self._join_prefix(join_path(pd, '*'))):
patchables = glob.glob(self._join_prefix(path.join(pd, '*')))
patchables.append(self._join_prefix(path.join('lib', 'icx-lto.so')))
for file in patchables:
# Try to patch all files, patchelf will do nothing if
# file should not be patched
subprocess.call(['patchelf', '--set-rpath', rpath, file])
def setup_run_environment(self, env):
env.prepend_path('PATH', self._join_prefix('bin'))
env.prepend_path('CPATH', self._join_prefix('include'))
env.prepend_path('LIBRARY_PATH', self._join_prefix('lib'))
for dir in self._ld_library_path():
env.prepend_path('LD_LIBRARY_PATH', dir)
env.set('CC', self._join_prefix('bin/icx'))
env.set('CXX', self._join_prefix('bin/icpx'))
env.set('FC', self._join_prefix('bin/ifx'))
# Set these so that MPI wrappers will pick up these compilers
# when this module is loaded.
env.set('I_MPI_CC', 'icx')
env.set('I_MPI_CXX', 'icpx')
env.set('I_MPI_FC', 'ifx')

24
var/spack/repos/builtin/packages/intel-oneapi-dal/package.py
View File

@@ -4,10 +4,9 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
from sys import platform
releases = {
'2021.1.1': {'irc_id': '17443', 'build': '79'}}
from spack import *
class IntelOneapiDal(IntelOneApiLibraryPackage):
@@ -17,11 +16,16 @@ class IntelOneapiDal(IntelOneApiLibraryPackage):
homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onedal.html'
version('2021.1.1', sha256='6e0e24bba462e80f0fba5a46e95cf0cca6cf17948a7753f8e396ddedd637544e', expand=False)
if platform == 'linux':
version('2021.1.1',
sha256='6e0e24bba462e80f0fba5a46e95cf0cca6cf17948a7753f8e396ddedd637544e',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17443/l_daal_oneapi_p_2021.1.1.79_offline.sh',
expand=False)
def __init__(self, spec):
self.component_info(dir_name='dal',
components='intel.oneapi.lin.dal.devel',
releases=releases,
url_name='daal_oneapi')
super(IntelOneapiDal, self).__init__(spec)
depends_on('intel-oneapi-tbb')
provides('daal')
@property
def component_dir(self):
return 'dal'

22
var/spack/repos/builtin/packages/intel-oneapi-dnn/package.py
View File

@@ -4,10 +4,9 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
from sys import platform
releases = {
'2021.1.1': {'irc_id': '17385', 'build': '55'}}
from spack import *
class IntelOneapiDnn(IntelOneApiLibraryPackage):
@@ -17,11 +16,14 @@ class IntelOneapiDnn(IntelOneApiLibraryPackage):
homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onednn.html'
version('2021.1.1', sha256='24002c57bb8931a74057a471a5859d275516c331fd8420bee4cae90989e77dc3', expand=False)
if platform == 'linux':
version('2021.1.1',
sha256='24002c57bb8931a74057a471a5859d275516c331fd8420bee4cae90989e77dc3',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17385/l_onednn_p_2021.1.1.55_offline.sh',
expand=False)
def __init__(self, spec):
self.component_info(dir_name='dnn',
components='intel.oneapi.lin.dnnl.devel',
releases=releases,
url_name='onednn')
super(IntelOneapiDnn, self).__init__(spec)
depends_on('intel-oneapi-tbb')
@property
def component_dir(self):
return 'dnnl'

21
var/spack/repos/builtin/packages/intel-oneapi-ipp/package.py
View File

@@ -3,12 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from sys import platform
from spack import *
releases = {
'2021.1.1': {'irc_id': '17436', 'build': '47'}}
class IntelOneapiIpp(IntelOneApiLibraryPackage):
"""Intel oneAPI IPP."""
@@ -17,13 +15,16 @@ class IntelOneapiIpp(IntelOneApiLibraryPackage):
homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/ipp.html'
version('2021.1.1', sha256='2656a3a7f1f9f1438cbdf98fd472a213c452754ef9476dd65190a7d46618ba86', expand=False)
if platform == 'linux':
version('2021.1.1',
sha256='2656a3a7f1f9f1438cbdf98fd472a213c452754ef9476dd65190a7d46618ba86',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17436/l_ipp_oneapi_p_2021.1.1.47_offline.sh',
expand=False)
depends_on('intel-oneapi-tbb')
provides('ipp')
def __init__(self, spec):
self.component_info(dir_name='ipp',
components='intel.oneapi.lin.ipp.devel',
releases=releases,
url_name='ipp_oneapi')
super(IntelOneapiIpp, self).__init__(spec)
@property
def component_dir(self):
return 'ipp'

20
var/spack/repos/builtin/packages/intel-oneapi-ippcp/package.py
View File

@@ -4,10 +4,9 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
from sys import platform
releases = {
'2021.1.1': {'irc_id': '17415', 'build': '54'}}
from spack import *
class IntelOneapiIppcp(IntelOneApiLibraryPackage):
@@ -17,11 +16,12 @@ class IntelOneapiIppcp(IntelOneApiLibraryPackage):
homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/ipp.html'
version('2021.1.1', sha256='c0967afae22c7a223ec42542bcc702121064cd3d8f680eff36169c94f964a936', expand=False)
if platform == 'linux':
version('2021.1.1',
sha256='c0967afae22c7a223ec42542bcc702121064cd3d8f680eff36169c94f964a936',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17415/l_ippcp_oneapi_p_2021.1.1.54_offline.sh',
expand=False)
def __init__(self, spec):
self.component_info(dir_name='ippcp',
components='intel.oneapi.lin.ippcp.devel',
releases=releases,
url_name='ippcp_oneapi')
super(IntelOneapiIppcp, self).__init__(spec)
@property
def component_dir(self):
return 'ippcp'

44
var/spack/repos/builtin/packages/intel-oneapi-mkl/package.py
View File

@@ -4,10 +4,9 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
from sys import platform
releases = {
'2021.1.1': {'irc_id': '17402', 'build': '52'}}
from spack import *
class IntelOneapiMkl(IntelOneApiLibraryPackage):
@@ -17,7 +16,13 @@ class IntelOneapiMkl(IntelOneApiLibraryPackage):
homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onemkl.html'
version('2021.1.1', sha256='818b6bd9a6c116f4578cda3151da0612ec9c3ce8b2c8a64730d625ce5b13cc0c', expand=False)
if platform == 'linux':
version('2021.1.1',
sha256='818b6bd9a6c116f4578cda3151da0612ec9c3ce8b2c8a64730d625ce5b13cc0c',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17402/l_onemkl_p_2021.1.1.52_offline.sh',
expand=False)
depends_on('intel-oneapi-tbb')
provides('fftw-api@3')
provides('scalapack')
@@ -25,31 +30,6 @@ class IntelOneapiMkl(IntelOneApiLibraryPackage):
provides('lapack')
provides('blas')
def __init__(self, spec):
self.component_info(dir_name='mkl',
components='intel.oneapi.lin.mkl.devel',
releases=releases,
url_name='onemkl')
super(IntelOneapiMkl, self).__init__(spec)
def _join_prefix(self, path):
return join_path(self.prefix, 'mkl', 'latest', path)
def _ld_library_path(self):
dirs = ['lib/intel64']
for dir in dirs:
yield self._join_prefix(dir)
def _library_path(self):
dirs = ['lib/intel64']
for dir in dirs:
yield self._join_prefix(dir)
def setup_run_environment(self, env):
env.prepend_path('PATH', self._join_prefix('bin/intel64'))
env.prepend_path('CPATH', self._join_prefix('include'))
for dir in self._library_path():
env.prepend_path('LIBRARY_PATH', dir)
for dir in self._ld_library_path():
env.prepend_path('LD_LIBRARY_PATH', dir)
env.set('MKLROOT', join_path(self.prefix, 'mkl', 'latest'))
@property
def component_dir(self):
return 'mkl'

58
var/spack/repos/builtin/packages/intel-oneapi-mpi/package.py
View File

@@ -4,13 +4,13 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from os import path
import subprocess
from sys import platform
from spack import *
releases = {
'2021.1.1': {'irc_id': '17397', 'build': '76'}}
class IntelOneapiMpi(IntelOneApiLibraryPackage):
"""Intel oneAPI MPI."""
@@ -19,18 +19,19 @@ class IntelOneapiMpi(IntelOneApiLibraryPackage):
homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/mpi-library.html'
version('2021.1.1', sha256='8b7693a156c6fc6269637bef586a8fd3ea6610cac2aae4e7f48c1fbb601625fe', expand=False)
if platform == 'linux':
version('2021.1.1',
sha256='8b7693a156c6fc6269637bef586a8fd3ea6610cac2aae4e7f48c1fbb601625fe',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17397/l_mpi_oneapi_p_2021.1.1.76_offline.sh',
expand=False)
provides('mpi@:3')
depends_on('patchelf', type='build')
def __init__(self, spec):
self.component_info(dir_name='mpi',
components='intel.oneapi.lin.mpi.devel',
releases=releases,
url_name='mpi_oneapi')
super(IntelOneapiMpi, self).__init__(spec)
@property
def component_dir(self):
return 'mpi'
def setup_dependent_package(self, module, dep_spec):
dir = join_path(self.prefix, 'mpi', 'latest', 'bin')
@@ -49,8 +50,10 @@ def setup_dependent_build_environment(self, env, dependent_spec):
@property
def libs(self):
libs = []
for dir in ['lib/release_mt', 'lib', 'libfabric/lib']:
lib_path = '{0}/{1}/latest/{2}'.format(self.prefix, self._dir_name, dir)
for dir in [path.join('lib', 'release_mt'),
'lib',
path.join('libfabric', 'lib')]:
lib_path = path.join(self.prefix, 'mpi', 'latest', dir)
ldir = find_libraries('*', root=lib_path, shared=True, recursive=False)
libs += ldir
return libs
@@ -58,38 +61,11 @@ def libs(self):
def _join_prefix(self, path):
return join_path(self.prefix, 'mpi', 'latest', path)
def _ld_library_path(self):
dirs = ['lib',
'lib/release',
'libfabric/lib']
for dir in dirs:
yield self._join_prefix(dir)
def _library_path(self):
dirs = ['lib',
'lib/release',
'libfabric/lib']
for dir in dirs:
yield self._join_prefix(dir)
def install(self, spec, prefix):
super(IntelOneapiMpi, self).install(spec, prefix)
# need to patch libmpi.so so it can always find libfabric
libfabric_rpath = self._join_prefix('libfabric/lib')
libfabric_rpath = self._join_prefix(path.join('libfabric', 'lib'))
for lib_version in ['debug', 'release', 'release_mt', 'debug_mt']:
file = self._join_prefix('lib/' + lib_version + '/libmpi.so')
file = self._join_prefix(path.join('lib', lib_version, 'libmpi.so'))
subprocess.call(['patchelf', '--set-rpath', libfabric_rpath, file])
def setup_run_environment(self, env):
env.prepend_path('PATH', self._join_prefix('bin'))
env.prepend_path('CPATH', self._join_prefix('include'))
for dir in self._library_path():
env.prepend_path('LIBRARY_PATH', dir)
for dir in self._ld_library_path():
env.prepend_path('LD_LIBRARY_PATH', dir)
# so wrappers know where MPI lives
mpi_root = join_path(prefix, 'mpi', 'latest')
env.set('I_MPI_ROOT', mpi_root)
# set this so that wrappers can find libfabric providers
env.set('FI_PROVIDER_PATH', self._join_prefix('libfabric/lib/prov'))

40
var/spack/repos/builtin/packages/intel-oneapi-tbb/package.py
View File

@@ -4,10 +4,9 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
from sys import platform
releases = {
'2021.1.1': {'irc_id': '17378', 'build': '119'}}
from spack import *
class IntelOneapiTbb(IntelOneApiLibraryPackage):
@@ -17,33 +16,14 @@ class IntelOneapiTbb(IntelOneApiLibraryPackage):
homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onetbb.html'
version('2021.1.1', sha256='535290e3910a9d906a730b24af212afa231523cf13a668d480bade5f2a01b53b', expand=False)
if platform == 'linux':
version('2021.1.1',
sha256='535290e3910a9d906a730b24af212afa231523cf13a668d480bade5f2a01b53b',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17378/l_tbb_oneapi_p_2021.1.1.119_offline.sh',
expand=False)
provides('tbb')
def __init__(self, spec):
self.component_info(dir_name='tbb',
components='intel.oneapi.lin.tbb.devel',
releases=releases,
url_name='tbb_oneapi')
super(IntelOneapiTbb, self).__init__(spec)
def _join_prefix(self, path):
return join_path(self.prefix, 'tbb', 'latest', path)
def _ld_library_path(self):
dirs = ['lib/intel64/gcc4.8']
for dir in dirs:
yield self._join_prefix(dir)
def _library_path(self):
dirs = ['lib/intel64/gcc4.8']
for dir in dirs:
yield self._join_prefix(dir)
def setup_run_environment(self, env):
for dir in self._library_path():
env.prepend_path('LIBRARY_PATH', dir)
for dir in self._ld_library_path():
env.prepend_path('LD_LIBRARY_PATH', dir)
env.set('TBBROOT', join_path(self.prefix, 'tbb', 'latest'))
@property
def component_dir(self):
return 'tbb'

19
var/spack/repos/builtin/packages/intel-oneapi-vpl/package.py
View File

@@ -3,12 +3,10 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from sys import platform
from spack import *
releases = {
'2021.1.1': {'irc_id': '17418', 'build': '66'}}
class IntelOneapiVpl(IntelOneApiLibraryPackage):
"""Intel oneAPI VPL."""
@@ -17,11 +15,12 @@ class IntelOneapiVpl(IntelOneApiLibraryPackage):
homepage = 'https://software.intel.com/content/www/us/en/develop/tools/oneapi/components/onevpl.html'
version('2021.1.1', sha256='0fec42545b30b7bb2e4e33deb12ab27a02900f5703153d9601673a8ce43082ed', expand=False)
if platform == 'linux':
version('2021.1.1',
sha256='0fec42545b30b7bb2e4e33deb12ab27a02900f5703153d9601673a8ce43082ed',
url='https://registrationcenter-download.intel.com/akdlm/irc_nas/17418/l_oneVPL_p_2021.1.1.66_offline.sh',
expand=False)
def __init__(self, spec):
self.component_info(dir_name='vpl',
components='intel.oneapi.lin.vpl.devel',
releases=releases,
url_name='oneVPL')
super(IntelOneapiVpl, self).__init__(spec)
@property
def component_dir(self):
return 'vpl'