Merge pull request #2506 from skosukhin/pr_python

A couple of updates for python package.

var/spack/repos/builtin/packages/python/package.py

@@ -22,6 +22,7 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+import ast
 import os
 import re
 from contextlib import closing
@@ -29,15 +30,17 @@
 import spack
 import llnl.util.tty as tty
 from llnl.util.lang import match_predicate
+from llnl.util.filesystem import force_remove
 from spack import *
 from spack.util.environment import *
+import spack.util.spack_json as sjson
 
 
 class Python(Package):
     """The Python programming language."""
 
     homepage = "http://www.python.org"
-    url      = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz"
+    url = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz"
 
     version('3.5.2', '3fe8434643a78630c61c6464fe2e7e72')
     version('3.5.1', 'be78e48cdfc1a7ad90efff146dce6cfe')
@@ -54,7 +57,7 @@ class Python(Package):
 
     extendable = True
 
-    variant('tk',   default=False, description='Provide support for Tkinter')
+    variant('tk', default=False, description='Provide support for Tkinter')
     variant('ucs4', default=False,
             description='Enable UCS4 (wide) unicode strings')
     # From https://docs.python.org/2/c-api/unicode.html: Python's default
@@ -71,11 +74,15 @@ class Python(Package):
     depends_on("ncurses")
     depends_on("sqlite")
     depends_on("zlib")
-    depends_on("tk",  when="+tk")
+    depends_on("tk", when="+tk")
     depends_on("tcl", when="+tk")
 
     patch('ncurses.patch')
 
+    _DISTUTIL_VARS_TO_SAVE = ['LDSHARED']
+    _DISTUTIL_CACHE_FILENAME = 'sysconfig.json'
+    _distutil_vars = None
+
     @when('@2.7,3.4:')
     def patch(self):
         # NOTE: Python's default installation procedure makes it possible for a
@@ -132,19 +139,19 @@ def install(self, spec, prefix):
         make()
         make('install')
 
-        self.filter_compilers(spec, prefix)
+        self._save_distutil_vars(prefix)
+
+        self.filter_compilers(prefix)
 
         # TODO:
         # On OpenSuse 13, python uses <prefix>/lib64/python2.7/lib-dynload/*.so
         # instead of <prefix>/lib/python2.7/lib-dynload/*.so. Oddly enough the
         # result is that Python can not find modules like cPickle. A workaround
         # for now is to symlink to `lib`:
-        src = os.path.join(prefix,
+        src = os.path.join(prefix.lib64,
-                           'lib64',
                            'python{0}'.format(self.version.up_to(2)),
                            'lib-dynload')
-        dst = os.path.join(prefix,
+        dst = os.path.join(prefix.lib,
-                           'lib',
                            'python{0}'.format(self.version.up_to(2)),
                            'lib-dynload')
         if os.path.isdir(src) and not os.path.isdir(dst):
@@ -174,7 +181,101 @@ def install(self, spec, prefix):
     #            >>> import Tkinter
     #            >>> Tkinter._test()
 
-    def filter_compilers(self, spec, prefix):
+    def _save_distutil_vars(self, prefix):
+        """
+        Run before changing automatically generated contents of the
+        _sysconfigdata.py, which is used by distutils to figure out what
+        executables to use while compiling and linking extensions. If we build
+        extensions with spack those executables should be spack's wrappers.
+        Spack partially covers this by setting environment variables that
+        are also accounted for by distutils. Currently there is one more known
+        variable that must be set, which is LDSHARED, so the method saves its
+        autogenerated value to pass it to the dependant package's setup script.
+        """
+
+        self._distutil_vars = {}
+
+        input_filename = None
+        for filename in [join_path(lib_dir,
+                                   'python{0}'.format(self.version.up_to(2)),
+                                   '_sysconfigdata.py')
+                         for lib_dir in [prefix.lib, prefix.lib64]]:
+            if os.path.isfile(filename):
+                input_filename = filename
+                break
+
+        if not input_filename:
+            return
+
+        input_dict = None
+        try:
+            with open(input_filename) as input_file:
+                match = re.search(r'build_time_vars\s*=\s*(?P<dict>{.*})',
+                                  input_file.read(),
+                                  flags=re.DOTALL)
+
+                if match:
+                    input_dict = ast.literal_eval(match.group('dict'))
+        except (IOError, SyntaxError):
+            pass
+
+        if not input_dict:
+            tty.warn('Failed to find \'build_time_vars\' dictionary in file '
+                     '\'%s\'. This might cause the extensions that are '
+                     'installed with distutils to call compilers directly '
+                     'avoiding Spack\'s wrappers.' % input_filename)
+            return
+
+        for var_name in Python._DISTUTIL_VARS_TO_SAVE:
+            if var_name in input_dict:
+                self._distutil_vars[var_name] = input_dict[var_name]
+            else:
+                tty.warn('Failed to find key \'%s\' in \'build_time_vars\' '
+                         'dictionary in file \'%s\'. This might cause the '
+                         'extensions that are installed with distutils to '
+                         'call compilers directly avoiding Spack\'s wrappers.'
+                         % (var_name, input_filename))
+
+        if len(self._distutil_vars) > 0:
+            output_filename = None
+            try:
+                output_filename = join_path(
+                    spack.store.layout.metadata_path(self.spec),
+                    Python._DISTUTIL_CACHE_FILENAME)
+                with open(output_filename, 'w') as output_file:
+                    sjson.dump(self._distutil_vars, output_file)
+            except:
+                tty.warn('Failed to save metadata for distutils. This might '
+                         'cause the extensions that are installed with '
+                         'distutils to call compilers directly avoiding '
+                         'Spack\'s wrappers.')
+                # We make the cache empty if we failed to save it to file
+                # to provide the same behaviour as in the case when the cache
+                # is initialized by the method load_distutils_data().
+                self._distutil_vars = {}
+                if output_filename:
+                    force_remove(output_filename)
+
+    def _load_distutil_vars(self):
+        # We update and keep the cache unchanged only if the package is
+        # installed.
+        if not self._distutil_vars and self.installed:
+            try:
+                input_filename = join_path(
+                    spack.store.layout.metadata_path(self.spec),
+                    Python._DISTUTIL_CACHE_FILENAME)
+                if os.path.isfile(input_filename):
+                    with open(input_filename) as input_file:
+                        self._distutil_vars = sjson.load(input_file)
+            except:
+                pass
+
+            if not self._distutil_vars:
+                self._distutil_vars = {}
+
+        return self._distutil_vars
+
+    def filter_compilers(self, prefix):
         """Run after install to tell the configuration files and Makefiles
         to use the compilers that Spack built the package with.
 
@@ -184,23 +285,21 @@ def filter_compilers(self, spec, prefix):
 
         kwargs = {'ignore_absent': True, 'backup': False, 'string': True}
 
-        dirname = join_path(prefix.lib,
+        lib_dirnames = [
-                            'python{0}'.format(self.version.up_to(2)))
+            join_path(lib_dir, 'python{0}'.format(self.version.up_to(2))) for
+            lib_dir in [prefix.lib, prefix.lib64]]
 
-        config = 'config'
+        config_dirname = 'config-{0}m'.format(
-        if spec.satisfies('@3:'):
+            self.version.up_to(2)) if self.spec.satisfies('@3:') else 'config'
-            config = 'config-{0}m'.format(self.version.up_to(2))
 
-        files = [
+        rel_filenames = ['_sysconfigdata.py',
-            '_sysconfigdata.py',
+                         join_path(config_dirname, 'Makefile')]
-            join_path(config, 'Makefile')
-        ]
 
-        for filename in files:
+        abs_filenames = [join_path(dirname, filename) for dirname in
-            filter_file(env['CC'],  self.compiler.cc,
+                         lib_dirnames for filename in rel_filenames]
-                        join_path(dirname, filename), **kwargs)
+
-            filter_file(env['CXX'], self.compiler.cxx,
+        filter_file(env['CC'], self.compiler.cc, *abs_filenames, **kwargs)
-                        join_path(dirname, filename), **kwargs)
+        filter_file(env['CXX'], self.compiler.cxx, *abs_filenames, **kwargs)
 
     # ========================================================================
     # Set up environment to make install easy for python extensions.
@@ -272,13 +371,19 @@ def setup_dependent_package(self, module, ext_spec):
         module.python = Executable(python_path)
         module.setup_py = Executable(python_path + ' setup.py --no-user-cfg')
 
+        distutil_vars = self._load_distutil_vars()
+
+        if distutil_vars:
+            for key, value in distutil_vars.iteritems():
+                module.setup_py.add_default_env(key, value)
+
         # Add variables for lib/pythonX.Y and lib/pythonX.Y/site-packages dirs.
-        module.python_lib_dir     = join_path(ext_spec.prefix,
+        module.python_lib_dir = join_path(ext_spec.prefix,
-                                              self.python_lib_dir)
+                                          self.python_lib_dir)
         module.python_include_dir = join_path(ext_spec.prefix,
                                               self.python_include_dir)
-        module.site_packages_dir  = join_path(ext_spec.prefix,
+        module.site_packages_dir = join_path(ext_spec.prefix,
-                                              self.site_packages_dir)
+                                             self.site_packages_dir)
 
         # Make the site packages directory for extensions
         if ext_spec.package.is_extension:
@@ -330,8 +435,8 @@ def write_easy_install_pth(self, exts):
                         continue
                     if re.search(r'^(import|#)', line):
                         continue
-                    if ((ext.name != 'py-setuptools' and
+                    if (ext.name != 'py-setuptools' and
-                         re.search(r'setuptools.*egg$', line))):
+                            re.search(r'setuptools.*egg$', line)):
                         continue
 
                     paths.append(line)