Python: improve site_packages_dir handling (#28346)

* Python: improve site_packages_dir handling

* Replace all site_packages_dir with purelib/platlib

lib/spack/spack/bootstrap.py

@@ -78,13 +78,10 @@ def _try_import_from_store(module, query_spec, query_info=None):
 
     for candidate_spec in installed_specs:
         pkg = candidate_spec['python'].package
-        purelib = pkg.config_vars['python_lib']['false']['false']
+        module_paths = {
-        platlib = pkg.config_vars['python_lib']['true']['false']
+            os.path.join(candidate_spec.prefix, pkg.purelib),
-
+            os.path.join(candidate_spec.prefix, pkg.platlib),
-        module_paths = [
+        }
-            os.path.join(candidate_spec.prefix, purelib),
-            os.path.join(candidate_spec.prefix, platlib),
-        ]
         sys.path.extend(module_paths)
 
         try:

lib/spack/spack/build_systems/python.py

@@ -128,22 +128,24 @@ def import_modules(self):
             list: list of strings of module names
         """
         modules = []
-        root = os.path.join(
+        pkg = self.spec['python'].package
-            self.prefix,
-            self.spec['python'].package.config_vars['python_lib']['true']['false'],
-        )
 
-        # Some Python libraries are packages: collections of modules
+        # Packages may be installed in platform-specific or platform-independent
-        # distributed in directories containing __init__.py files
+        # site-packages directories
-        for path in find(root, '__init__.py', recursive=True):
+        for directory in {pkg.platlib, pkg.purelib}:
-            modules.append(path.replace(root + os.sep, '', 1).replace(
+            root = os.path.join(self.prefix, directory)
-                os.sep + '__init__.py', '').replace('/', '.'))
 
-        # Some Python libraries are modules: individual *.py files
+            # Some Python libraries are packages: collections of modules
-        # found in the site-packages directory
+            # distributed in directories containing __init__.py files
-        for path in find(root, '*.py', recursive=False):
+            for path in find(root, '__init__.py', recursive=True):
-            modules.append(path.replace(root + os.sep, '', 1).replace(
+                modules.append(path.replace(root + os.sep, '', 1).replace(
-                '.py', '').replace('/', '.'))
+                    os.sep + '__init__.py', '').replace('/', '.'))
+
+            # Some Python libraries are modules: individual *.py files
+            # found in the site-packages directory
+            for path in find(root, '*.py', recursive=False):
+                modules.append(path.replace(root + os.sep, '', 1).replace(
+                    '.py', '').replace('/', '.'))
 
         modules = [mod for mod in modules if re.match('[a-zA-Z0-9._]+$', mod)]
 
@@ -258,18 +260,13 @@ def install_args(self, spec, prefix):
         # Get all relative paths since we set the root to `prefix`
         # We query the python with which these will be used for the lib and inc
         # directories. This ensures we use `lib`/`lib64` as expected by python.
-        pure_site_packages_dir = spec['python'].package.config_vars[
+        pkg = spec['python'].package
-            'python_lib']['false']['false']
-        plat_site_packages_dir = spec['python'].package.config_vars[
-            'python_lib']['true']['false']
-        inc_dir = spec['python'].package.config_vars['python_inc']['true']
-
         args += ['--root=%s' % prefix,
-                 '--install-purelib=%s' % pure_site_packages_dir,
+                 '--install-purelib=%s' % pkg.purelib,
-                 '--install-platlib=%s' % plat_site_packages_dir,
+                 '--install-platlib=%s' % pkg.platlib,
                  '--install-scripts=bin',
                  '--install-data=',
-                 '--install-headers=%s' % inc_dir
+                 '--install-headers=%s' % pkg.include,
                  ]
 
         return args

lib/spack/spack/build_systems/sip.py

@@ -67,7 +67,7 @@ def import_modules(self):
         modules = []
         root = os.path.join(
             self.prefix,
-            self.spec['python'].package.config_vars['python_lib']['true']['false'],
+            self.spec['python'].package.platlib,
         )
 
         # Some Python libraries are packages: collections of modules
@@ -114,7 +114,7 @@ def configure(self, spec, prefix):
             '--sip-incdir', join_path(spec['py-sip'].prefix.include,
                                       python_include_dir),
             '--bindir', prefix.bin,
-            '--destdir', inspect.getmodule(self).site_packages_dir,
+            '--destdir', inspect.getmodule(self).python_platlib,
         ])
 
         self.python(configure, *args)
@@ -167,7 +167,7 @@ def extend_path_setup(self):
         module = self.spec['py-sip'].variants['module'].value
         if module != 'sip':
             module = module.split('.')[0]
-            with working_dir(inspect.getmodule(self).site_packages_dir):
+            with working_dir(inspect.getmodule(self).python_platlib):
                 with open(os.path.join(module, '__init__.py'), 'a') as f:
                     f.write('from pkgutil import extend_path\n')
                     f.write('__path__ = extend_path(__path__, __name__)\n')

var/spack/repos/builtin/packages/ascent/package.py

@@ -392,7 +392,7 @@ def hostconfig(self, spec, prefix):
             try:
                 cfg.write("# python module install dir\n")
                 cfg.write(cmake_cache_entry("PYTHON_MODULE_INSTALL_PREFIX",
-                          site_packages_dir))
+                          python_platlib))
             except NameError:
                 # spack's  won't exist in a subclass
                 pass

var/spack/repos/builtin/packages/bart/package.py

@@ -64,18 +64,15 @@ def edit(self, spec, prefix):
             env['GPUARCH_FLAGS'] = ' '.join(self.cuda_flags(cuda_arch))
 
     def install(self, spec, prefix):
-        python_dir = join_path(prefix,
-                               spec['python'].package.site_packages_dir)
-
         make('install')
 
         install_tree('scripts', prefix.scripts)
         install_tree('matlab', prefix.matlab)
         install('startup.m', prefix)
 
-        install('python/bart.py', python_dir)
+        install('python/bart.py', python_platlib)
-        install('python/cfl.py', python_dir)
+        install('python/cfl.py', python_platlib)
-        install('python/wslsupport.py', python_dir)
+        install('python/wslsupport.py', python_platlib)
 
         if '^python@3:' in spec:
             install('python/bartview3.py', join_path(prefix.bin, 'bartview'))

var/spack/repos/builtin/packages/bohrium/package.py

@@ -234,8 +234,7 @@ def check_install(self):
 
         # Add the PYTHONPATH to bohrium to the PYTHONPATH environment
         pythonpaths = [p for p in os.environ["PYTHONPATH"].split(":")]
-        pythonpaths.append(join_path(self.prefix,
+        pythonpaths.append(python_platlib)
-                                     spec['python'].package.site_packages_dir))
         test_env["PYTHONPATH"] = ":".join(pythonpaths)
 
         # Collect the stacks which should be available:

var/spack/repos/builtin/packages/clingo/package.py

@@ -68,12 +68,10 @@ def cmake_python_hints(self):
         """Return standard CMake defines to ensure that the
         current spec is the one found by CMake find_package(Python, ...)
         """
-        python_spec = self.spec['python']
+        python = self.spec['python']
-        include_dir = join_path(
-            python_spec.prefix, python_spec.package.config_vars['python_inc']['false'])
         return [
-            self.define('Python_EXECUTABLE', str(python_spec.command)),
+            self.define('Python_EXECUTABLE', str(python.command)),
-            self.define('Python_INCLUDE_DIR', include_dir)
+            self.define('Python_INCLUDE_DIR', python.package.config_vars['include'])
         ]
 
     @property

var/spack/repos/builtin/packages/conduit/package.py

@@ -444,7 +444,7 @@ def hostconfig(self, spec, prefix):
             try:
                 cfg.write("# python module install dir\n")
                 cfg.write(cmake_cache_entry("PYTHON_MODULE_INSTALL_PREFIX",
-                          site_packages_dir))
+                          python_platlib))
             except NameError:
                 # spack's  won't exist in a subclass
                 pass

var/spack/repos/builtin/packages/elemental/package.py

@@ -167,6 +167,6 @@ def cmake_args(self):
 
         if '+python' in spec:
             args.extend([
-                '-DPYTHON_SITE_PACKAGES:STRING={0}'.format(site_packages_dir)])
+                '-DPYTHON_SITE_PACKAGES:STRING={0}'.format(python_platlib)])
 
         return args

var/spack/repos/builtin/packages/faiss/package.py

@@ -125,4 +125,4 @@ def _fix_makefile(self):
 
     def setup_run_environment(self, env):
         if '+python' in self.spec:
-            env.prepend_path('PYTHONPATH', site_packages_dir)
+            env.prepend_path('PYTHONPATH', python_platlib)

var/spack/repos/builtin/packages/flann/package.py

@@ -89,7 +89,7 @@ def patch(self):
         # Fix the install location so that spack activate works
         if '+python' in self.spec:
             filter_file("share/flann/python",
-                        site_packages_dir,
+                        python_platlib,
                         "src/python/CMakeLists.txt")
         # Hack. Don't install setup.py
         filter_file("install( FILES",

var/spack/repos/builtin/packages/gdl/package.py

@@ -144,7 +144,7 @@ def post_install(self):
             src = os.path.join(
                 self.spec.prefix.lib,
                 'site-python')
-            dst = site_packages_dir
+            dst = python_platlib
             if os.path.isdir(src):
                 if not os.path.isdir(dst):
                     mkdirp(dst)

var/spack/repos/builtin/packages/hepmc3/package.py

@@ -51,11 +51,10 @@ def cmake_args(self):
 
         if self.spec.satisfies('+python'):
             py_ver = spec['python'].version.up_to(2)
-            py_sitepkg = join_path(self.prefix, site_packages_dir)
             args.extend([
                 '-DHEPMC3_PYTHON_VERSIONS={0}'.format(py_ver),
                 '-DHEPMC3_Python_SITEARCH{0}={1}'.format(
-                    py_ver.joined, py_sitepkg)
+                    py_ver.joined, python_platlib)
             ])
 
         if self.spec.satisfies('+rootio'):

var/spack/repos/builtin/packages/hoomd-blue/package.py

@@ -60,12 +60,10 @@ class HoomdBlue(CMakePackage):
 
     def cmake_args(self):
         spec = self.spec
-        install_dir = spec['python'].package.site_packages_dir
-        install_path = os.path.join(spec.prefix, install_dir)
 
         cmake_args = [
             '-DPYTHON_EXECUTABLE={0}'.format(spec['python'].command.path),
-            '-DCMAKE_INSTALL_PREFIX={0}'.format(install_path)
+            '-DCMAKE_INSTALL_PREFIX={0}'.format(python_platlib)
         ]
 
         # MPI support

var/spack/repos/builtin/packages/libxml2/package.py

@@ -62,7 +62,7 @@ def configure_args(self):
         if '+python' in spec:
             args.extend([
                 '--with-python={0}'.format(spec['python'].home),
-                '--with-python-install-dir={0}'.format(site_packages_dir)
+                '--with-python-install-dir={0}'.format(python_platlib)
             ])
         else:
             args.append('--without-python')

var/spack/repos/builtin/packages/llvm-doe/package.py

@@ -639,10 +639,10 @@ def post_install(self):
                 ninja()
                 ninja("install")
         if "+python" in self.spec:
-            install_tree("llvm/bindings/python", site_packages_dir)
+            install_tree("llvm/bindings/python", python_platlib)
 
             if "+clang" in self.spec:
-                install_tree("clang/bindings/python", site_packages_dir)
+                install_tree("clang/bindings/python", python_platlib)
 
         with working_dir(self.build_directory):
             install_tree("bin", join_path(self.prefix, "libexec", "llvm"))

var/spack/repos/builtin/packages/llvm/package.py

@@ -702,10 +702,10 @@ def post_install(self):
                 ninja()
                 ninja("install")
         if "+python" in self.spec:
-            install_tree("llvm/bindings/python", site_packages_dir)
+            install_tree("llvm/bindings/python", python_platlib)
 
             if "+clang" in self.spec:
-                install_tree("clang/bindings/python", site_packages_dir)
+                install_tree("clang/bindings/python", python_platlib)
 
         with working_dir(self.build_directory):
             install_tree("bin", join_path(self.prefix, "libexec", "llvm"))

var/spack/repos/builtin/packages/lvarray/package.py

@@ -131,8 +131,8 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None):
           subclasses this package provide a specific site packages
           dir when calling this function. `py_site_pkgs_dir` should
           be an absolute path or `None`.
-          This is necessary because the spack `site_packages_dir`
+          This is necessary because the spack `python_purelib` and `python_platlib`
-          var will not exist in the base class. For more details
+          vars will not exist in the base class. For more details
           on this issue see: https://github.com/spack/spack/issues/6261
         """
 

var/spack/repos/builtin/packages/migraphx/package.py

@@ -64,11 +64,9 @@ def cmake_python_hints(self):
         """Include the python include path to the
         CMake based on current spec
         """
-        python_spec = self.spec['python']
+        python = self.spec['python']
-        include_dir = join_path(
-            python_spec.prefix, python_spec.package.config_vars['python_inc']['false'])
         return [
-            self.define('Python_INCLUDE_DIR', include_dir)
+            self.define('Python_INCLUDE_DIR', python.package.config_vars['include'])
         ]
 
     def cmake_args(self):

var/spack/repos/builtin/packages/openturns/package.py

@@ -53,7 +53,7 @@ def cmake_args(self):
                 # By default picks up the system python not the Spack build
                 '-DPYTHON_EXECUTABLE={0}'.format(spec['python'].command.path),
                 # By default installs to the python prefix
-                '-DPYTHON_SITE_PACKAGES={0}'.format(site_packages_dir),
+                '-DPYTHON_SITE_PACKAGES={0}'.format(python_platlib),
             ])
 
         return args

var/spack/repos/builtin/packages/openvdb/package.py

@@ -70,7 +70,7 @@ def cmake_args(self):
 
     # The python extension is being put in the wrong directory
     # by OpenVDB's cmake, instead it needs to be in
-    # site_packages_dir. For RHEL systems we seem to get the
+    # python_platlib. For RHEL systems we seem to get the
     # dso in lib64/ instead of lib/
     @run_after('install')
     def post_install(self):
@@ -87,5 +87,4 @@ def post_install(self):
             src = prefix.lib.join(pyver).join(pyso)
             if not os.path.isfile(src):
                 src = prefix.lib64.join(pyver).join(pyso)
-            mkdirp(site_packages_dir)
+            os.rename(src, os.path.join(python_platlib, pyso))
-            os.rename(src, os.path.join(site_packages_dir, pyso))

var/spack/repos/builtin/packages/pagmo/package.py

@@ -93,7 +93,7 @@ def cmake_args(self):
                 # By default picks up the system python not the Spack build
                 '-DPYTHON_EXECUTABLE={0}'.format(spec['python'].command.path),
                 # By default installs to the python prefix not the pagmo prefix
-                '-DPYTHON_MODULES_DIR={0}'.format(site_packages_dir),
+                '-DPYTHON_MODULES_DIR={0}'.format(python_platlib),
             ])
 
         return args

var/spack/repos/builtin/packages/precice/package.py

@@ -143,7 +143,7 @@ def variant_bool(feature, on='ON', off='OFF'):
             python_include = spec['python'].headers.directories[0]
             numpy_include = join_path(
                 spec['py-numpy'].prefix,
-                spec['python'].package.site_packages_dir,
+                spec['python'].package.platlib,
                 'numpy', 'core', 'include')
             if xsdk_mode:
                 cmake_args.append('-DTPL_ENABLE_PYTHON:BOOL=ON')

var/spack/repos/builtin/packages/py-flit-core/package.py

@@ -29,4 +29,4 @@ def build(self, spec, prefix):
     def install(self, spec, prefix):
         wheel = glob.glob(os.path.join('flit_core', 'dist', '*.whl'))[0]
         with zipfile.ZipFile(wheel) as f:
-            f.extractall(site_packages_dir)
+            f.extractall(python_purelib)

var/spack/repos/builtin/packages/py-fury/package.py

@@ -31,7 +31,7 @@ class PyFury(PythonPackage):
     def install_test(self):
         with working_dir('spack-test', create=True):
             pytest = which('pytest')
-            pytest(join_path(self.prefix, site_packages_dir, 'fury'),
+            pytest(join_path(python_purelib, 'fury'),
                    # 'Some warning' is not propagated to __warningregistry__ so
                    # that the test fails, disable it for now
                    # running all tests manually after the package is installed

var/spack/repos/builtin/packages/py-gpaw/package.py

@@ -58,7 +58,7 @@ def patch(self):
         python_include = spec['python'].headers.directories[0]
         numpy_include = join_path(
             spec['py-numpy'].prefix,
-            spec['python'].package.site_packages_dir,
+            spec['python'].package.platlib,
             'numpy', 'core', 'include')
 
         libs = blas.libs + lapack.libs + libxc.libs

var/spack/repos/builtin/packages/py-ninja/package.py

@@ -27,7 +27,7 @@ def installit(self):
                                    'ninja_syntax.py')
         bin_file = os.path.join(self.spec['ninja'].prefix.bin,
                                 'ninja')
-        dst = os.path.join(site_packages_dir,
+        dst = os.path.join(python_platlib,
                            'ninja')
         dstbin = os.path.join(dst, 'data', 'bin')
         mkdirp(dstbin)

var/spack/repos/builtin/packages/py-protobuf/package.py

@@ -86,6 +86,4 @@ def install_args(self, spec, prefix):
     def fix_import_error(self):
         if str(self.spec['python'].version.up_to(1)) == '2':
             touch = which('touch')
-            touch(self.prefix + '/' +
+            touch(join_path(python_platlib, 'google', '__init__.py'))
-                  self.spec['python'].package.site_packages_dir +
-                  '/google/__init__.py')

var/spack/repos/builtin/packages/py-pymol/package.py

@@ -48,10 +48,7 @@ def install_launcher(self):
         binpath = self.prefix.bin
         mkdirp(self.prefix.bin)
         fname = join_path(binpath, 'pymol')
-        script = join_path(self.prefix,
+        script = join_path(python_platlib, 'pymol', '__init__.py')
-                           self.spec['python'].package.site_packages_dir,
-                           'pymol',
-                           '__init__.py')
 
         shebang = '#!/bin/sh\n'
         fdata = 'exec {0} {1} \"$@\"'.format(self.spec['python'].command,

var/spack/repos/builtin/packages/py-pyqt4/package.py

@@ -36,7 +36,7 @@ def configure_args(self):
         args = [
             '--pyuic4-interpreter', self.spec['python'].command.path,
             '--sipdir', self.prefix.share.sip.PyQt4,
-            '--stubsdir', join_path(site_packages_dir, 'PyQt4')
+            '--stubsdir', join_path(python_platlib, 'PyQt4')
         ]
         if '+qsci_api' in self.spec:
             args.extend(['--qsci-api',

var/spack/repos/builtin/packages/py-pyqt5/package.py

@@ -39,10 +39,7 @@ def configure_args(self):
             '--sipdir', self.prefix.share.sip.PyQt5,
             '--designer-plugindir', self.prefix.plugins.designer,
             '--qml-plugindir', self.prefix.plugins.PyQt5,
-            '--stubsdir', join_path(
+            '--stubsdir', join_path(python_platlib, 'PyQt5'),
-                self.prefix,
-                self.spec['python'].package.site_packages_dir,
-                'PyQt5'),
         ]
         if '+qsci_api' in self.spec:
             args.extend(['--qsci-api',

var/spack/repos/builtin/packages/py-pyside/package.py

@@ -38,10 +38,8 @@ class PyPyside(PythonPackage):
     def patch(self):
         """Undo PySide RPATH handling and add Spack RPATH."""
         # Figure out the special RPATH
-        pypkg = self.spec['python'].package
         rpath = self.rpath
-        rpath.append(os.path.join(
+        rpath.append(os.path.join(python_platlib, 'PySide'))
-            self.prefix, pypkg.site_packages_dir, 'PySide'))
 
         # Fix subprocess.mswindows check for Python 3.5
         # https://github.com/pyside/pyside-setup/pull/55

var/spack/repos/builtin/packages/py-python-meep/package.py

@@ -41,7 +41,7 @@ def common_args(self, spec, prefix):
             spec['meep'].prefix.include,
             os.path.join(
                 spec['py-numpy'].prefix,
-                spec['python'].package.python_include_dir
+                spec['python'].package.include
             )
         ]
 

var/spack/repos/builtin/packages/py-ruamel-yaml/package.py

@@ -31,6 +31,4 @@ class PyRuamelYaml(PythonPackage):
     def fix_import_error(self):
         if str(self.spec['python'].version.up_to(1)) == '2':
             touch = which('touch')
-            touch(self.prefix + '/' +
+            touch(join_path(python_purelib, 'ruamel', '__init__.py'))
-                  self.spec['python'].package.site_packages_dir +
-                  '/ruamel/__init__.py')

var/spack/repos/builtin/packages/py-scikit-learn/package.py

@@ -100,4 +100,4 @@ def install_test(self):
         # https://scikit-learn.org/stable/developers/advanced_installation.html#testing
         with working_dir('spack-test', create=True):
             pytest = which('pytest')
-            pytest(join_path(self.prefix, site_packages_dir, 'sklearn'))
+            pytest(join_path(self.prefix, python_purelib, 'sklearn'))

var/spack/repos/builtin/packages/py-shiboken/package.py

@@ -26,10 +26,8 @@ def patch(self):
         """Undo Shiboken RPATH handling and add Spack RPATH."""
         # Add Spack's standard CMake args to the sub-builds.
         # They're called BY setup.py so we have to patch it.
-        pypkg = self.spec['python'].package
         rpath = self.rpath
-        rpath.append(os.path.join(
+        rpath.append(os.path.join(python_platlib, 'Shiboken'))
-            self.prefix, pypkg.site_packages_dir, 'Shiboken'))
 
         filter_file(
             r'OPTION_CMAKE,',

var/spack/repos/builtin/packages/py-sip/package.py

@@ -52,10 +52,10 @@ def configure(self, spec, prefix):
             args = [
                 '--sip-module={0}'.format(spec.variants['module'].value),
                 '--bindir={0}'.format(prefix.bin),
-                '--destdir={0}'.format(site_packages_dir),
+                '--destdir={0}'.format(python_platlib),
-                '--incdir={0}'.format(python_include_dir),
+                '--incdir={0}'.format(spec['python'].package.include),
                 '--sipdir={0}'.format(prefix.share.sip),
-                '--stubsdir={0}'.format(site_packages_dir),
+                '--stubsdir={0}'.format(python_platlib),
             ]
 
             python('configure.py', *args)
@@ -75,7 +75,7 @@ def extend_path_setup(self):
             module = self.spec.variants['module'].value
             if module != 'sip':
                 module = module.split('.')[0]
-                with working_dir(site_packages_dir):
+                with working_dir(python_platlib):
                     with open(os.path.join(module, '__init__.py'), 'w') as f:
                         f.write('from pkgutil import extend_path\n')
                         f.write('__path__ = extend_path(__path__, __name__)\n')

var/spack/repos/builtin/packages/py-tensorflow/package.py

@@ -310,7 +310,7 @@ def setup_build_environment(self, env):
         env.set('PYTHON_BIN_PATH', spec['python'].command.path)
 
         # Please input the desired Python library path to use
-        env.set('PYTHON_LIB_PATH', site_packages_dir)
+        env.set('PYTHON_LIB_PATH', python_platlib)
 
         # Ensure swig is in PATH or set SWIG_PATH
         env.set('SWIG_PATH', spec['swig'].prefix.bin.swig)

var/spack/repos/builtin/packages/py-torch/package.py

@@ -207,14 +207,12 @@ class PyTorch(PythonPackage, CudaPackage):
 
     @property
     def libs(self):
-        root = join_path(self.prefix, self.spec['python'].package.site_packages_dir,
+        root = join_path(python_platlib, 'torch', 'lib')
-                         'torch', 'lib')
         return find_libraries('libtorch', root)
 
     @property
     def headers(self):
-        root = join_path(self.prefix, self.spec['python'].package.site_packages_dir,
+        root = join_path(python_platlib, 'torch', 'include')
-                         'torch', 'include')
         headers = find_all_headers(root)
         headers.directories = [root]
         return headers

var/spack/repos/builtin/packages/py-xdot/package.py

@@ -40,8 +40,7 @@ def post_install(self):
             join_path(spec['atk'].prefix.lib, 'girepository-1.0'),
             join_path(spec['gdk-pixbuf'].prefix.lib, 'girepository-1.0'),
             join_path(spec['gtkplus'].prefix.lib, 'girepository-1.0'))
-        dst = join_path(spec.prefix, spec['python'].package.site_packages_dir,
+        dst = join_path(python_platlib, 'xdot', '__init__.py')
-                        'xdot/__init__.py')
         filter_file("import sys",
                     "import sys\nimport os\nos.environ['GI_TYPELIB_PATH']" +
                     " = '%s'" % repo_paths, dst)

var/spack/repos/builtin/packages/python/package.py

@@ -15,7 +15,6 @@
 
 from spack import *
 from spack.build_environment import dso_suffix
-from spack.compilers import NoCompilerForSpecError
 from spack.util.environment import is_system_path
 from spack.util.prefix import Prefix
 
@@ -716,40 +715,29 @@ def print_string(self, string):
     def config_vars(self):
         """Return a set of variable definitions associated with a Python installation.
 
-        Wrapper around various ``distutils.sysconfig`` functions.
+        Wrapper around various ``sysconfig`` functions. To see these variables on the
+        command line, run:
+
+        .. code-block:: console
+
+           $ python -m sysconfig
 
         Returns:
             dict: variable definitions
         """
-        # TODO: distutils is deprecated in Python 3.10 and will be removed in
-        # Python 3.12, find a different way to access this information.
-        # Also, calling the python executable disallows us from cross-compiling,
-        # so we want to try to avoid that if possible.
         cmd = """
 import json
-from distutils.sysconfig import (
+from sysconfig import (
     get_config_vars,
     get_config_h_filename,
     get_makefile_filename,
-    get_python_inc,
+    get_paths,
-    get_python_lib,
 )
 
 config = get_config_vars()
 config['config_h_filename'] = get_config_h_filename()
 config['makefile_filename'] = get_makefile_filename()
-config['python_inc'] = {}
+config.update(get_paths())
-config['python_lib'] = {}
-
-for plat_specific in [True, False]:
-    plat_key = str(plat_specific).lower()
-    config['python_inc'][plat_key] = get_python_inc(plat_specific, prefix='')
-    config['python_lib'][plat_key] = {}
-    for standard_lib in [True, False]:
-        lib_key = str(standard_lib).lower()
-        config['python_lib'][plat_key][lib_key] = get_python_lib(
-            plat_specific, standard_lib, prefix=''
-        )
 
 %s
 """ % self.print_string("json.dumps(config)")
@@ -759,16 +747,10 @@ def config_vars(self):
         if dag_hash not in self._config_vars:
             # Default config vars
             version = self.version.up_to(2)
-            try:
-                cc = self.compiler.cc
-                cxx = self.compiler.cxx
-            except (TypeError, NoCompilerForSpecError):
-                cc = 'cc'
-                cxx = 'c++'
-
             config = {
-                'CC': cc,
+                # get_config_vars
-                'CXX': cxx,
+                'CC': 'cc',
+                'CXX': 'c++',
                 'INCLUDEPY': self.prefix.include.join('python{}').format(version),
                 'LIBDEST': self.prefix.lib.join('python{}').format(version),
                 'LIBDIR': self.prefix.lib,
@@ -776,34 +758,31 @@ def config_vars(self):
                     'config-{0}-{1}').format(version, sys.platform),
                 'LDLIBRARY': 'libpython{}.{}'.format(version, dso_suffix),
                 'LIBRARY': 'libpython{}.a'.format(version),
-                'LDSHARED': cc,
+                'LDSHARED': 'cc',
-                'LDCXXSHARED': cxx,
+                'LDCXXSHARED': 'c++',
                 'PYTHONFRAMEWORKPREFIX': '/System/Library/Frameworks',
+                'base': self.prefix,
+                'installed_base': self.prefix,
+                'installed_platbase': self.prefix,
+                'platbase': self.prefix,
                 'prefix': self.prefix,
+                # get_config_h_filename
                 'config_h_filename': self.prefix.include.join('python{}').join(
                     'pyconfig.h').format(version),
+                # get_makefile_filename
                 'makefile_filename': self.prefix.lib.join('python{0}').join(
                     'config-{0}-{1}').Makefile.format(version, sys.platform),
-                'python_inc': {
+                # get_paths
-                    # plat_specific
+                'data': self.prefix,
-                    'true': os.path.join('include64', 'python{}'.format(version)),
+                'include': self.prefix.include.join('python{}'.format(version)),
-                    'false': os.path.join('include', 'python{}'.format(version)),
+                'platinclude': self.prefix.include64.join('python{}'.format(version)),
-                },
+                'platlib': self.prefix.lib64.join(
-                'python_lib': {
+                    'python{}'.format(version)).join('site-packages'),
-                    # plat_specific
+                'platstdlib': self.prefix.lib64.join('python{}'.format(version)),
-                    'true': {
+                'purelib': self.prefix.lib.join(
-                        # standard_lib
+                    'python{}'.format(version)).join('site-packages'),
-                        'true': os.path.join('lib64', 'python{}'.format(version)),
+                'scripts': self.prefix.bin,
-                        'false': os.path.join(
+                'stdlib': self.prefix.lib.join('python{}'.format(version)),
-                            'lib64', 'python{}'.format(version), 'site-packages'),
-                    },
-                    'false': {
-                        # standard_lib
-                        'true': os.path.join('lib', 'python{}'.format(version)),
-                        'false': os.path.join(
-                            'lib', 'python{}'.format(version), 'site-packages'),
-                    },
-                },
             }
 
             try:
@@ -906,66 +885,68 @@ def headers(self):
         headers.directories = [os.path.dirname(config_h)]
         return headers
 
-    @property
+    # https://docs.python.org/3/library/sysconfig.html#installation-paths
-    def python_include_dir(self):
-        """Directory for the include files.
-
-        On most systems, and for Spack-installed Python, this will look like:
-
-            ``include/pythonX.Y``
-
-        However, some systems append a ``m`` to the end of this path.
-
-        Returns:
-            str: include files directory
-        """
-        return self.config_vars['python_inc']['false']
 
     @property
-    def python_lib_dir(self):
+    def platlib(self):
-        """Directory for the standard library.
+        """Directory for site-specific, platform-specific files.
 
-        On most systems, and for Spack-installed Python, this will look like:
+        Exact directory depends on platform/OS/Python version. Examples include:
 
-            ``lib/pythonX.Y``
+        * ``lib/pythonX.Y/site-packages`` on most POSIX systems
-
+        * ``lib64/pythonX.Y/site-packages`` on RHEL/CentOS/Fedora with system Python
-        On RHEL/CentOS/Fedora, when using the system Python, this will look like:
+        * ``lib/pythonX/dist-packages`` on Debian/Ubuntu with system Python
-
+        * ``lib/python/site-packages`` on macOS with framework Python
-            ``lib64/pythonX.Y``
+        * ``Lib/site-packages`` on Windows
-
-        On Debian/Ubuntu, when using the system Python, this will look like:
-
-            ``lib/pythonX``
 
         Returns:
-            str: standard library directory
+            str: platform-specific site-packages directory
         """
-        return self.config_vars['python_lib']['false']['true']
+        return self.config_vars['platlib'].replace(
+            self.config_vars['platbase'] + os.sep, ''
+        )
 
     @property
-    def site_packages_dir(self):
+    def purelib(self):
-        """Directory where third-party extensions should be installed.
+        """Directory for site-specific, non-platform-specific files.
 
-        On most systems, and for Spack-installed Python, this will look like:
+        Exact directory depends on platform/OS/Python version. Examples include:
 
-            ``lib/pythonX.Y/site-packages``
+        * ``lib/pythonX.Y/site-packages`` on most POSIX systems
-
+        * ``lib/pythonX/dist-packages`` on Debian/Ubuntu with system Python
-        On RHEL/CentOS/Fedora, when using the system Python, this will look like:
+        * ``lib/python/site-packages`` on macOS with framework Python
-
+        * ``Lib/site-packages`` on Windows
-            ``lib64/pythonX.Y/site-packages``
-
-        On Debian/Ubuntu, when using the system Python, this will look like:
-
-            ``lib/pythonX/dist-packages``
 
         Returns:
-            str: site-packages directory
+            str: platform-independent site-packages directory
         """
-        return self.config_vars['python_lib']['false']['false']
+        return self.config_vars['purelib'].replace(
+            self.config_vars['base'] + os.sep, ''
+        )
+
+    @property
+    def include(self):
+        """Directory for non-platform-specific header files.
+
+        Exact directory depends on platform/Python version/ABI flags. Examples include:
+
+        * ``include/pythonX.Y`` on most POSIX systems
+        * ``include/pythonX.Yd`` for debug builds
+        * ``include/pythonX.Ym`` for malloc builds
+        * ``include/pythonX.Yu`` for wide unicode builds
+        * ``include`` on macOS with framework Python
+        * ``Include`` on Windows
+
+        Returns:
+            str: platform-independent header file directory
+        """
+        return self.config_vars['include'].replace(
+            self.config_vars['installed_base'] + os.sep, ''
+        )
 
     @property
     def easy_install_file(self):
-        return join_path(self.site_packages_dir, "easy-install.pth")
+        return join_path(self.purelib, "easy-install.pth")
 
     def setup_run_environment(self, env):
         env.prepend_path('CPATH', os.pathsep.join(
@@ -985,10 +966,28 @@ def setup_dependent_build_environment(self, env, dependent_spec):
         if not is_system_path(path):
             env.prepend_path('PATH', path)
 
-        for d in dependent_spec.traverse(deptype=('build', 'run', 'test'), root=True):
+        # Add installation prefix to PYTHONPATH, needed to run import tests
-            if d.package.extends(self.spec):
+        prefixes = set()
-                env.prepend_path('PYTHONPATH', join_path(
+        if dependent_spec.package.extends(self.spec):
-                    d.prefix, self.site_packages_dir))
+            prefixes.add(dependent_spec.prefix)
+
+        # Add direct build/run/test dependencies to PYTHONPATH,
+        # needed to build the package and to run import tests
+        for direct_dep in dependent_spec.dependencies(deptype=('build', 'run', 'test')):
+            if direct_dep.package.extends(self.spec):
+                prefixes.add(direct_dep.prefix)
+
+                # Add recursive run dependencies of all direct dependencies,
+                # needed by direct dependencies at run-time
+                for indirect_dep in direct_dep.traverse(deptype='run'):
+                    if indirect_dep.package.extends(self.spec):
+                        prefixes.add(indirect_dep.prefix)
+
+        for prefix in prefixes:
+            # Packages may be installed in platform-specific or platform-independent
+            # site-packages directories
+            for directory in {self.platlib, self.purelib}:
+                env.prepend_path('PYTHONPATH', os.path.join(prefix, directory))
 
         # We need to make sure that the extensions are compiled and linked with
         # the Spack wrapper. Paths to the executables that are used for these
@@ -1051,8 +1050,12 @@ def setup_dependent_run_environment(self, env, dependent_spec):
         """
         for d in dependent_spec.traverse(deptype=('run'), root=True):
             if d.package.extends(self.spec):
-                env.prepend_path('PYTHONPATH', join_path(
+                # Packages may be installed in platform-specific or platform-independent
-                    d.prefix, self.site_packages_dir))
+                # site-packages directories
+                for directory in {self.platlib, self.purelib}:
+                    env.prepend_path(
+                        'PYTHONPATH', os.path.join(d.prefix, directory)
+                    )
 
     def setup_dependent_package(self, module, dependent_spec):
         """Called before python modules' install() methods."""
@@ -1061,19 +1064,15 @@ def setup_dependent_package(self, module, dependent_spec):
         module.setup_py = Executable(
             self.command.path + ' setup.py --no-user-cfg')
 
-        # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs.
+        module.python_platlib = join_path(dependent_spec.prefix, self.platlib)
-        module.python_lib_dir = join_path(dependent_spec.prefix,
+        module.python_purelib = join_path(dependent_spec.prefix, self.purelib)
-                                          self.python_lib_dir)
-        module.python_include_dir = join_path(dependent_spec.prefix,
-                                              self.python_include_dir)
-        module.site_packages_dir = join_path(dependent_spec.prefix,
-                                             self.site_packages_dir)
 
         self.spec.home = self.home
 
         # Make the site packages directory for extensions
         if dependent_spec.package.is_extension:
-            mkdirp(module.site_packages_dir)
+            mkdirp(module.python_platlib)
+            mkdirp(module.python_purelib)
 
     # ========================================================================
     # Handle specifics of activating and deactivating python modules.

var/spack/repos/builtin/packages/qscintilla/package.py

@@ -87,10 +87,7 @@ def make_qsci(self):
                 pyqtx = 'PyQt5'
 
             with working_dir(join_path(self.stage.source_path, 'Python')):
-                pydir = join_path(
+                pydir = join_path(python_platlib, pyqtx)
-                    self.prefix,
-                    self.spec['python'].package.site_packages_dir,
-                    pyqtx)
                 mkdirp(os.path.join(self.prefix.share.sip, pyqtx))
                 python = self.spec['python'].command
                 python('configure.py', '--pyqt=' + pyqtx,
@@ -144,7 +141,7 @@ def extend_path_setup(self):
         module = self.spec['py-sip'].variants['module'].value
         if module != 'sip':
             module = module.split('.')[0]
-            with working_dir(site_packages_dir):
+            with working_dir(python_platlib):
                 with open(os.path.join(module, '__init__.py'), 'w') as f:
                     f.write('from pkgutil import extend_path\n')
                     f.write('__path__ = extend_path(__path__, __name__)\n')

var/spack/repos/builtin/packages/sollve/package.py

@@ -306,7 +306,7 @@ def post_install(self):
         if '+clang' in self.spec and '+python' in self.spec:
             install_tree(
                 'tools/clang/bindings/python/clang',
-                join_path(site_packages_dir, 'clang'))
+                join_path(python_platlib, 'clang'))
 
         with working_dir(self.build_directory):
             install_tree('bin', self.prefix.libexec.llvm)

var/spack/repos/builtin/packages/sz/package.py

@@ -93,7 +93,7 @@ def cmake_args(self):
 
         if "+python" in self.spec:
             args.append("-DBUILD_PYTHON_WRAPPER=ON")
-            args.append("-DSZ_PYTHON_SITELIB={0}".format(site_packages_dir))
+            args.append("-DSZ_PYTHON_SITELIB={0}".format(python_platlib))
         else:
             args.append("-DBUILD_PYTHON_WRAPPER=OFF")
 

var/spack/repos/builtin/packages/vapor/package.py

@@ -42,12 +42,12 @@ def cmake_args(self):
             python = self.spec['python']
             f.write('set (PYTHONVERSION {0})\n'.format(python.version.up_to(2)))
             f.write('set (PYTHONDIR {0})\n'.format(python.home))
-            f.write('set (PYTHONPATH {0})\n'.format(python.package.site_packages_dir))
+            f.write('set (PYTHONPATH {0})\n'.format(python.package.platlib))
             # install expects the share/images directory to install below this path
             f.write('set (THIRD_PARTY_DIR {0})\n'.format(self.stage.source_path))
             numpy_include = join_path(
                 self.spec['py-numpy'].prefix,
-                self.spec['python'].package.site_packages_dir,
+                self.spec['python'].package.platlib,
                 'numpy', 'core', 'include')
             f.write('set (THIRD_PARTY_INC_DIR "{0}")\n'.format(numpy_include))
 

var/spack/repos/builtin/packages/vigra/package.py

@@ -71,7 +71,7 @@ def cmake_args(self):
                 '-DBoost_DIR={0}'.format(spec['boost'].prefix),
                 '-DBoost_INCLUDE_DIR={0}'.format(spec['boost'].prefix.include),
                 '-DBoost_PYTHON_LIBRARY={0}'.format(boost_python_lib),
-                '-DVIGRANUMPY_INSTALL_DIR={0}'.format(site_packages_dir)
+                '-DVIGRANUMPY_INSTALL_DIR={0}'.format(python_platlib)
             ])
         if '+fftw' in spec:
             args.extend([