Use byte-encoded UTF-8 for sourced environment in Python 2 (#3489)

- Fixes recurring errors on develop with unicode commit characters.
- still Python3-proof: python3 will use str instead of bytestrings.

lib/spack/spack/environment.py

@@ -26,6 +26,7 @@
 import inspect
 import json
 import os
+import sys
 import os.path
 import subprocess
 
@@ -268,8 +269,8 @@ def from_sourcing_files(*args, **kwargs):
         :param \*args: list of files to be sourced
         :rtype: instance of EnvironmentModifications
         """
-
         env = EnvironmentModifications()
+
         # Check if the files are actually there
         files = [line.split(' ')[0] for line in args]
         non_existing = [file for file in files if not os.path.isfile(file)]
@@ -277,6 +278,7 @@ def from_sourcing_files(*args, **kwargs):
             message = 'trying to source non-existing files\n'
             message += '\n'.join(non_existing)
             raise RuntimeError(message)
+
         # Relevant kwd parameters and formats
         info = dict(kwargs)
         info.setdefault('shell', '/bin/bash')
@@ -309,11 +311,17 @@ def from_sourcing_files(*args, **kwargs):
         if proc.returncode != 0:
             raise RuntimeError('sourcing files returned a non-zero exit code')
         output = ''.join([line for line in proc.stdout])
-        # Construct a dictionary with all the variables in the new environment
-        after_source_env = dict(
-            (k, v.decode('utf8')) for k, v in json.loads(output).items())
-        this_environment = dict(
-            (k, v.decode('utf8')) for k, v in os.environ.items())
+
+        # Construct a dictionaries of the environment before and after
+        # sourcing the files, so that we can diff them.
+        this_environment = dict(os.environ)
+        after_source_env = json.loads(output)
+
+        # If we're in python2, convert to str objects instead of unicode
+        # like json gives us.  We can't put unicode in os.environ anyway.
+        if sys.version_info[0] < 3:
+            after_source_env = dict((k.encode('utf-8'), v.encode('utf-8'))
+                                    for k, v in after_source_env.items())
 
         # Filter variables that are not related to sourcing a file
         to_be_filtered = 'SHLVL', '_', 'PWD', 'OLDPWD'
@@ -364,6 +372,7 @@ def return_separator_if_any(first_value, second_value):
                 start = modified_list.index(remaining_list[0])
                 end = modified_list.index(remaining_list[-1])
                 search = sep.join(modified_list[start:end + 1])
+
                 if search not in current:
                     # We just need to set the variable to the new value
                     env.set(x, after_source_env[x])

lib/spack/spack/test/data/sourceme_unicode.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+##############################################################################
+# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+# Set an environment variable with some unicode in it to ensure that
+# Spack can decode it.
+#
+# This has caused squashed commits on develop to break, as some
+# committers use unicode in their messages, and Travis sets the
+# current commit message in an environment variable.
+export UNICODE_VAR='don\xe2\x80\x99t'

lib/spack/spack/test/environment.py

@@ -96,7 +96,8 @@ def files_to_be_sourced():
     files = [
         os.path.join(datadir, 'sourceme_first.sh'),
         os.path.join(datadir, 'sourceme_second.sh'),
-        os.path.join(datadir, 'sourceme_parameters.sh intel64')
+        os.path.join(datadir, 'sourceme_parameters.sh intel64'),
+        os.path.join(datadir, 'sourceme_unicode.sh')
     ]
 
     return files
@@ -230,7 +231,6 @@ def test_source_files(files_to_be_sourced):
     """Tests the construction of a list of environment modifications that are
     the result of sourcing a file.
     """
-
     env = EnvironmentModifications.from_sourcing_files(*files_to_be_sourced)
     modifications = env.group_by_name()
 
@@ -238,7 +238,7 @@ def test_source_files(files_to_be_sourced):
     # spurious entries for things like PS1
     #
     # TODO: figure out how to make a bit more robust.
-    assert len(modifications) >= 4
+    assert len(modifications) >= 5
 
     # Set new variables
     assert len(modifications['NEW_VAR']) == 1