From f624ce083408d5db90ffa0714a3d2398a30a2fe3 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Fri, 9 Apr 2021 18:10:01 -0700 Subject: [PATCH] Build process output: handle UTF-8 for python 3.x to 3.7 (#22888) We set LC_ALL=C to encourage a build process to generate ASCII output (so our logger daemon can decode it). Most packages respect this but it appears that intel-oneapi-compilers does not in some cases (see #22813). This reads the output of the build process as UTF-8, which still works if the build process respects LC_ALL=C but also works if the process generates UTF-8 output. For Python >= 3.7 all files are opened with UTF-8 encoding by default. Python 2 does not support the encoding argument on 'open', so to support Python 2 the files would have to be opened in byte mode and explicitly decoded (as a side note, this would be the only way to handle other encodings without being informed of them in advance). --- lib/spack/llnl/util/tty/log.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/spack/llnl/util/tty/log.py b/lib/spack/llnl/util/tty/log.py index ef14baeacf9..337eb0ac689 100644 --- a/lib/spack/llnl/util/tty/log.py +++ b/lib/spack/llnl/util/tty/log.py @@ -321,7 +321,10 @@ def __init__(self, file_like): def unwrap(self): if self.open: if self.file_like: - self.file = open(self.file_like, 'w') + if sys.version_info < (3,): + self.file = open(self.file_like, 'w') + else: + self.file = open(self.file_like, 'w', encoding='utf-8') else: self.file = StringIO() return self.file @@ -722,7 +725,11 @@ def _writer_daemon(stdin_multiprocess_fd, read_multiprocess_fd, write_fd, echo, # Use line buffering (3rd param = 1) since Python 3 has a bug # that prevents unbuffered text I/O. - in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1) + if sys.version_info < (3,): + in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1) + else: + # Python 3.x before 3.7 does not open with UTF-8 encoding by default + in_pipe = os.fdopen(read_multiprocess_fd.fd, 'r', 1, encoding='utf-8') if stdin_multiprocess_fd: stdin = os.fdopen(stdin_multiprocess_fd.fd)