bugfix: handle unicode properly in spack.util.executable (#10186)

- When returning string output, use text_type and decode utf-8 in Python
  2 instead of using `str`

- This properly handles unicode, whereas before we would pass bad strings
  to colify in `spack blame` when reading git output

- add a test that round-trips some unicode through an Executable object

lib/spack/spack/test/util/executable.py

@@ -0,0 +1,34 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import sys
+
+import llnl.util.filesystem as fs
+
+import spack.util.executable as ex
+
+
+def test_read_unicode(tmpdir):
+    script_name = 'print_unicode.py'
+
+    with tmpdir.as_cwd():
+
+        # make a script that prints some unicode
+        with open(script_name, 'w') as f:
+            f.write('''#!{0}
+from __future__ import print_function
+import sys
+if sys.version_info < (3, 0, 0):
+    reload(sys)
+    sys.setdefaultencoding('utf8')
+print(u'\\xc3')
+'''.format(sys.executable))
+
+        # make it executable
+        fs.set_executable(script_name)
+
+        # read the unicode back in and see whether things work
+        script = ex.Executable('./%s' % script_name)
+        assert u'\xc3' == script(output=str).strip()

lib/spack/spack/util/executable.py

@@ -6,8 +6,7 @@
 import os
 import re
 import subprocess
-from six import string_types
-import sys
+from six import string_types, text_type
 
 import llnl.util.tty as tty
 
@@ -171,9 +170,9 @@ def streamify(arg, mode):
             if output is str or error is str:
                 result = ''
                 if output is str:
-                    result += to_str(out)
+                    result += text_type(out.decode('utf-8'))
                 if error is str:
-                    result += to_str(err)
+                    result += text_type(err.decode('utf-8'))
 
             rc = self.returncode = proc.returncode
             if fail_on_error and rc != 0 and (rc not in ignore_errors):
@@ -224,20 +223,6 @@ def __str__(self):
         return ' '.join(self.exe)
 
 
-def to_str(content):
-    """Produce a str type from the content of a process stream obtained with
-       Popen.communicate.
-    """
-    # Prior to python3, Popen.communicate returns a str type. For python3 it
-    # returns a bytes type. In the case of python3 we decode the
-    # byte string to produce a str type. This will generate junk if the
-    # encoding is not UTF-8 (which includes ASCII).
-    if sys.version_info < (3, 0, 0):
-        return content
-    else:
-        return content.decode('utf-8')
-
-
 def which(*args, **kwargs):
     """Finds an executable in the path like command-line which.