Resolve Python2/Python3 unicode issues by using str()

- Remove ascii encoding assumption from spack_yaml
- proc.communicate() returns bytes; convert to str before adding.
- Fix various byte string/unicode issues for Python 2/3 support
- Need to decode subprocess output as utf-8 in from_sourcing_files.
- Fix comments in strify()

lib/spack/spack/environment.py

@@ -310,7 +310,7 @@ def from_sourcing_files(*args, **kwargs):
         proc.wait()
         if proc.returncode != 0:
             raise RuntimeError('sourcing files returned a non-zero exit code')
-        output = ''.join([line for line in proc.stdout])
+        output = ''.join([line.decode('utf-8') for line in proc.stdout])
 
         # Construct a dictionaries of the environment before and after
         # sourcing the files, so that we can diff them.

lib/spack/spack/stage.py

@@ -226,7 +226,7 @@ def __init__(
         self._lock = None
         if lock:
             if self.name not in Stage.stage_locks:
-                sha1 = hashlib.sha1(self.name).digest()
+                sha1 = hashlib.sha1(self.name.encode('utf-8')).digest()
                 lock_id = prefix_bits(sha1, bit_length(sys.maxsize))
                 stage_lock_path = join_path(spack.stage_path, '.lock')
 

lib/spack/spack/test/architecture.py

@@ -138,8 +138,8 @@ def test_user_defaults(config):
 
 def test_user_input_combination(config):
     platform = spack.architecture.platform()
-    os_list = platform.operating_sys.keys()
+    os_list = list(platform.operating_sys.keys())
-    target_list = platform.targets.keys()
+    target_list = list(platform.targets.keys())
     additional = ["fe", "be", "frontend", "backend"]
 
     os_list.extend(additional)

lib/spack/spack/util/crypto.py

@@ -22,6 +22,7 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
+import sys
 import hashlib
 
 """Set of acceptable hashes that Spack will use."""
@@ -104,11 +105,16 @@ def check(self, filename):
 
 def prefix_bits(byte_array, bits):
     """Return the first <bits> bits of a byte array as an integer."""
+    if sys.version_info < (3,):
+        b2i = ord          # In Python 2, indexing byte_array gives str
+    else:
+        b2i = lambda b: b  # In Python 3, indexing byte_array gives int
+
     result = 0
     n = 0
     for i, b in enumerate(byte_array):
         n += 8
-        result = (result << 8) | ord(b)
+        result = (result << 8) | b2i(b)
         if n >= bits:
             break
 

lib/spack/spack/util/executable.py

@@ -178,9 +178,9 @@ def streamify(arg, mode):
             if output is str or error is str:
                 result = ''
                 if output is str:
-                    result += out
+                    result += out.decode('utf-8')
                 if error is str:
-                    result += err
+                    result += err.decode('utf-8')
                 return result
 
         except OSError as e:

lib/spack/spack/util/spack_json.py

@@ -68,6 +68,7 @@ def _byteify(data, ignore_dicts=False):
         return dict((_byteify(key, ignore_dicts=True),
                      _byteify(value, ignore_dicts=True)) for key, value in
                     iteritems(data))
+
     # if it's anything else, return it in its original form
     return data
 

lib/spack/spack/util/spack_yaml.py

@@ -85,11 +85,6 @@ class OrderedLineLoader(Loader):
 
     def construct_yaml_str(self, node):
         value = self.construct_scalar(node)
-        try:
-            value = value.encode('ascii')
-        except UnicodeEncodeError:
-            pass
-
         value = syaml_str(value)
 
         mark(value, node)
@@ -181,7 +176,7 @@ def represent_mapping(self, tag, mapping, flow_style=None):
             # if it's a syaml_dict, preserve OrderedDict order.
             # Otherwise do the default thing.
             sort = not isinstance(mapping, syaml_dict)
-            mapping = mapping.items()
+            mapping = list(mapping.items())
             if sort:
                 mapping.sort()