Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/openjd/adaptor_runtime/process/_logging_subprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,13 @@ def __init__(
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
encoding=encoding,
# Decode the subprocess output leniently. DCC batch processes may emit
# locale-encoded (non-UTF-8) bytes on stdout/stderr; with the default strict
# error handling a single undecodable byte raises UnicodeDecodeError in the
# stream-reader thread, killing it. Once the reader thread is gone the pipe is
# no longer drained, the subprocess blocks on a full stdout pipe, and the job
# deadlocks. Replacing undecodable bytes keeps the reader alive.
errors="replace",

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider errors="backslashreplace" instead of "replace" (suggestion). It keeps the reader alive exactly the same way, but renders the offending byte as \xc7 instead of �, preserving the original byte value in logs. Genuinely useful when debugging which codepage a DCC is emitting (the very scenario motivating this PR), and it's what CPython itself uses for its own stderr. Only the handler and the two test assertions would change.

Would recommend doing this. We can push up a follow up PR :)

cwd=startup_directory,
)
if OSName.is_windows(): # pragma: is-posix
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,37 @@ def test_log_levels(self, log_level: int, caplog):

assert any(r.message == message and r.levelno == _STDERR_LEVEL for r in records)

@pytest.mark.timeout(10)
def test_non_utf8_output_does_not_kill_reader(self, caplog):
"""
A subprocess that writes a non-UTF-8 (locale-encoded) byte to stdout must not kill the
stream-reader thread. If the reader thread dies, lines emitted after the undecodable byte
are never logged and the stdout pipe stops being drained, which can deadlock the subprocess.
This is a regression test for that failure mode; the undecodable byte must be tolerated and
subsequent output must still be logged.
"""
# GIVEN
caplog.set_level(_STDOUT_LEVEL)
# 0xc7 is "Ç" in cp1252 and is not a valid standalone UTF-8 byte.
script = (
"import sys; "
"sys.stdout.buffer.write(b'before\\n'); "
"sys.stdout.buffer.write(b'bad \\xc7 byte\\n'); "
"sys.stdout.buffer.write(b'after\\n'); "
"sys.stdout.buffer.flush()"
)

# WHEN
p = LoggingSubprocess(args=[sys.executable, "-c", script])
p.wait()
p._cleanup_io_threads()

# THEN
# "after" is only logged if the reader thread survived the undecodable byte.
assert "after" in caplog.text
# The undecodable byte is replaced rather than raising.
assert "�" in caplog.text

def test_executable_not_found(self):
"""When calling LoggingSubprocess with a missing executable, FileNotFoundError will be raised"""
args = ["missing_executable"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def test_process_creation(self, mock_popen: mock.Mock, mock_stream_logger: mock.
popen_params = dict(
args=args,
encoding="utf-8",
errors="replace",
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
Expand Down Expand Up @@ -482,6 +483,7 @@ def test_startup_directory_default(self, mock_popen_autospec: mock.Mock):
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf-8",
errors="replace",
cwd=None,
)
if OSName.is_windows():
Expand All @@ -504,6 +506,7 @@ def test_start_directory(self, mock_popen_autospec: mock.Mock):
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf-8",
errors="replace",
cwd="startup_dir",
)
if OSName.is_windows():
Expand Down
Loading