Convert Python 2 idioms to Python 2/3-compatible ones.

- convert print, StringIO, except as, octals, izip
- convert print statement to print function
- convert StringIO to six.StringIO
  - remove usage of csv reader in Spec, in favor of simple regex
  - csv reader only does byte strings
- convert 0755 octal literals to 0o755
- convert `except Foo, e` to `except Foo as e`
- fix a few places `str` is used.
  - may need to switch everything to str later.
- convert iteritems usages to use six.iteritems
- fix urllib and HTMLParser
- port metaclasses to use six.with_metaclass
- More octal literal conversions for Python 2/3
- Fix a new octal literal.
- Convert `basestring` to `six.string_types`
- Convert xrange -> range
- Fix various issues with encoding, iteritems, and Python3 semantics.
- Convert contextlib.nested to explicitly nexted context managers.
- Convert use of filter() to list comprehensions.
- Replace reduce() with list comprehensions.
-  Clean up composite: replace inspect.ismethod() with callable()
- Python 3 doesn't have "method" objects; inspect.ismethod returns False.
- Need to use callable in Composite to make it work.
- Update colify to use future division.
- Fix zip() usages that need to be lists.
- Python3: Use line-buffered logging instead of unbuffered.
- Python3 raises an error with unbuffered I/O
  - See https://bugs.python.org/issue17404

lib/spack/llnl/util/filesystem.py

@@ -175,9 +175,9 @@ def change_sed_delimiter(old_delim, new_delim, *filenames):
 def set_install_permissions(path):
     """Set appropriate permissions on the installed file."""
     if os.path.isdir(path):
-        os.chmod(path, 0755)
+        os.chmod(path, 0o755)
     else:
-        os.chmod(path, 0644)
+        os.chmod(path, 0o644)
 
 
 def copy_mode(src, dest):

lib/spack/llnl/util/lang.py

@@ -27,6 +27,7 @@
 import functools
 import collections
 import inspect
+from six import string_types
 
 # Ignore emacs backups when listing modules
 ignore_modules = [r'^\.#', '~$']
@@ -80,7 +81,7 @@ def index_by(objects, *funcs):
         return objects
 
     f = funcs[0]
-    if isinstance(f, basestring):
+    if isinstance(f, str):
         f = lambda x: getattr(x, funcs[0])
     elif isinstance(f, tuple):
         f = lambda x: tuple(getattr(x, p) for p in funcs[0])
@@ -326,7 +327,7 @@ def match_predicate(*args):
     """
     def match(string):
         for arg in args:
-            if isinstance(arg, basestring):
+            if isinstance(arg, string_types):
                 if re.search(arg, string):
                     return True
             elif isinstance(arg, list) or isinstance(arg, tuple):

lib/spack/llnl/util/tty/__init__.py

@@ -29,7 +29,7 @@
 import termios
 import struct
 import traceback
-from StringIO import StringIO
+from six import StringIO
 
 from llnl.util.tty.color import *
 
@@ -93,7 +93,7 @@ def msg(message, *args, **kwargs):
     else:
         cwrite("@*b{%s==>} %s" % (st_text, cescape(message)))
     for arg in args:
-        print indent + str(arg)
+        print(indent + str(arg))
 
 
 def info(message, *args, **kwargs):
@@ -201,7 +201,7 @@ def get_yes_or_no(prompt, **kwargs):
         if not ans:
             result = default_value
             if result is None:
-                print "Please enter yes or no."
+                print("Please enter yes or no.")
         else:
             if ans == 'y' or ans == 'yes':
                 result = True
@@ -239,7 +239,7 @@ def hline(label=None, **kwargs):
     out.write(label)
     out.write(suffix)
 
-    print out.getvalue()
+    print(out.getvalue())
 
 
 def terminal_size():

lib/spack/llnl/util/tty/colify.py

@@ -25,9 +25,11 @@
 """
 Routines for printing columnar output.  See colify() for more information.
 """
+from __future__ import division
+
 import os
 import sys
-from StringIO import StringIO
+from six import StringIO
 
 from llnl.util.tty import terminal_size
 from llnl.util.tty.color import clen, cextra
@@ -64,18 +66,18 @@ def config_variable_cols(elts, console_width, padding, cols=0):
     # Get a bound on the most columns we could possibly have.
     # 'clen' ignores length of ansi color sequences.
     lengths = [clen(e) for e in elts]
-    max_cols = max(1, console_width / (min(lengths) + padding))
+    max_cols = max(1, console_width // (min(lengths) + padding))
     max_cols = min(len(elts), max_cols)
 
     # Range of column counts to try.  If forced, use the supplied value.
-    col_range = [cols] if cols else xrange(1, max_cols + 1)
+    col_range = [cols] if cols else range(1, max_cols + 1)
 
     # Determine the most columns possible for the console width.
     configs = [ColumnConfig(c) for c in col_range]
     for i, length in enumerate(lengths):
         for conf in configs:
             if conf.valid:
-                col = i / ((len(elts) + conf.cols - 1) / conf.cols)
+                col = i // ((len(elts) + conf.cols - 1) // conf.cols)
                 p = padding if col < (conf.cols - 1) else 0
 
                 if conf.widths[col] < (length + p):
@@ -107,7 +109,7 @@ def config_uniform_cols(elts, console_width, padding, cols=0):
     # 'clen' ignores length of ansi color sequences.
     max_len = max(clen(e) for e in elts) + padding
     if cols == 0:
-        cols = max(1, console_width / max_len)
+        cols = max(1, console_width // max_len)
         cols = min(len(elts), cols)
 
     config = ColumnConfig(cols)
@@ -193,12 +195,12 @@ def colify(elts, **options):
         raise ValueError("method must be one of: " + allowed_methods)
 
     cols = config.cols
-    rows = (len(elts) + cols - 1) / cols
+    rows = (len(elts) + cols - 1) // cols
     rows_last_col = len(elts) % rows
 
-    for row in xrange(rows):
+    for row in range(rows):
         output.write(" " * indent)
-        for col in xrange(cols):
+        for col in range(cols):
             elt = col * rows + row
             width = config.widths[col] + cextra(elts[elt])
             if col < cols - 1:
@@ -233,7 +235,7 @@ def colify_table(table, **options):
     columns = len(table[0])
 
     def transpose():
-        for i in xrange(columns):
+        for i in range(columns):
             for row in table:
                 yield row[i]
 

lib/spack/llnl/util/tty/log.py

@@ -165,8 +165,12 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self.p.join(60.0)  # 1 minute to join the child
 
     def _spawn_writing_daemon(self, read, input_stream):
-        # Parent: read from child, skip the with block.
-        read_file = os.fdopen(read, 'r', 0)
+        # This is the Parent: read from child, skip the with block.
+
+        # Use line buffering (3rd param = 1) since Python 3 has a bug
+        # that prevents unbuffered text I/O.
+        read_file = os.fdopen(read, 'r', 1)
+
         with open(self.filename, 'w') as log_file:
             with keyboard_input(input_stream):
                 while True: