https://github.com/python/cpython/commit/cb1dc91dcb1eb6637d1c79b34a0bab728939f717
commit: cb1dc91dcb1eb6637d1c79b34a0bab728939f717
branch: main
author: Malcolm Smith <[email protected]>
committer: freakboy3742 <[email protected]>
date: 2026-02-03T16:37:34+08:00
summary:
gh-144415: Android testbed fixes (#142912)
Modifies handling of `.gz` files in Android app payloads, and ensures that
when the Android testbed streams logs, stream flushes aren't treated as
newlines. This improves the output of test suites that use "one dot per test"
progress indicators.
files:
A Misc/NEWS.d/next/Tests/2026-02-03-07-57-24.gh-issue-144415.U3L15r.rst
M Android/android.py
M Android/testbed/app/build.gradle.kts
M Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt
M Lib/_android_support.py
M Lib/test/test_android.py
diff --git a/Android/android.py b/Android/android.py
index d1a10be776ed16..629696be3db300 100755
--- a/Android/android.py
+++ b/Android/android.py
@@ -15,6 +15,7 @@
from asyncio import wait_for
from contextlib import asynccontextmanager
from datetime import datetime, timezone
+from enum import IntEnum, auto
from glob import glob
from os.path import abspath, basename, relpath
from pathlib import Path
@@ -61,6 +62,19 @@
hidden_output = []
+# Based on android/log.h in the NDK.
+class LogPriority(IntEnum):
+ UNKNOWN = 0
+ DEFAULT = auto()
+ VERBOSE = auto()
+ DEBUG = auto()
+ INFO = auto()
+ WARN = auto()
+ ERROR = auto()
+ FATAL = auto()
+ SILENT = auto()
+
+
def log_verbose(context, line, stream=sys.stdout):
if context.verbose:
stream.write(line)
@@ -505,21 +519,23 @@ async def logcat_task(context, initial_devices):
pid = await wait_for(find_pid(serial), startup_timeout)
# `--pid` requires API level 24 or higher.
- args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"]
+ #
+ # `--binary` mode is used in order to detect which messages end with a
+ # newline, which most of the other modes don't indicate (except `--format
+ # long`). For example, every time pytest runs a test, it prints a "." and
+ # flushes the stream. Each "." becomes a separate log message, but we
should
+ # show them all on the same line.
+ args = [adb, "-s", serial, "logcat", "--pid", pid, "--binary"]
logcat_started = False
async with async_process(
- *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+ *args, stdout=subprocess.PIPE, stderr=None
) as process:
- while line := (await process.stdout.readline()).decode(*DECODE_ARGS):
- if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL):
+ while True:
+ try:
+ priority, tag, message = await read_logcat(process.stdout)
logcat_started = True
- level, message = match.groups()
- else:
- # If the regex doesn't match, this is either a logcat startup
- # error, or the second or subsequent line of a multi-line
- # message. Python won't produce multi-line messages, but other
- # components might.
- level, message = None, line
+ except asyncio.IncompleteReadError:
+ break
# Exclude high-volume messages which are rarely useful.
if context.verbose < 2 and "from python test_syslog" in message:
@@ -527,25 +543,23 @@ async def logcat_task(context, initial_devices):
# Put high-level messages on stderr so they're highlighted in the
# buildbot logs. This will include Python's own stderr.
- stream = (
- sys.stderr
- if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka
ASSERT)
- else sys.stdout
- )
-
- # To simplify automated processing of the output, e.g. a buildbot
- # posting a failure notice on a GitHub PR, we strip the level and
- # tag indicators from Python's stdout and stderr.
- for prefix in ["python.stdout: ", "python.stderr: "]:
- if message.startswith(prefix):
- global python_started
- python_started = True
- stream.write(message.removeprefix(prefix))
- break
+ stream = sys.stderr if priority >= LogPriority.WARN else sys.stdout
+
+ # The app's stdout and stderr should be passed through
transparently
+ # to our own corresponding streams.
+ if tag in ["python.stdout", "python.stderr"]:
+ global python_started
+ python_started = True
+ stream.write(message)
+ stream.flush()
else:
# Non-Python messages add a lot of noise, but they may
- # sometimes help explain a failure.
- log_verbose(context, line, stream)
+ # sometimes help explain a failure. Format them in the same way
+ # as `logcat --format tag`.
+ formatted = f"{priority.name[0]}/{tag}: {message}"
+ if not formatted.endswith("\n"):
+ formatted += "\n"
+ log_verbose(context, formatted, stream)
# If the device disconnects while logcat is running, which always
# happens in --managed mode, some versions of adb return non-zero.
@@ -556,6 +570,44 @@ async def logcat_task(context, initial_devices):
raise CalledProcessError(status, args)
+# Read one binary log message from the given StreamReader. The message format
is
+# described at https://android.stackexchange.com/a/74660. All supported
versions
+# of Android use format version 2 or later.
+async def read_logcat(stream):
+ async def read_bytes(size):
+ return await stream.readexactly(size)
+
+ async def read_int(size):
+ return int.from_bytes(await read_bytes(size), "little")
+
+ payload_len = await read_int(2)
+ if payload_len < 2:
+ # 1 byte for priority, 1 byte for null terminator of tag.
+ raise ValueError(f"payload length {payload_len} is too short")
+
+ header_len = await read_int(2)
+ if header_len < 4:
+ raise ValueError(f"header length {header_len} is too short")
+ await read_bytes(header_len - 4) # Ignore other header fields.
+
+ priority_int = await read_int(1)
+ try:
+ priority = LogPriority(priority_int)
+ except ValueError:
+ priority = LogPriority.UNKNOWN
+
+ payload_fields = (await read_bytes(payload_len - 1)).split(b"\0")
+ if len(payload_fields) < 2:
+ raise ValueError(
+ f"payload {payload!r} does not contain at least 2 "
+ f"null-separated fields"
+ )
+ tag, message, *_ = [
+ field.decode(*DECODE_ARGS) for field in payload_fields
+ ]
+ return priority, tag, message
+
+
def stop_app(serial):
run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False)
diff --git a/Android/testbed/app/build.gradle.kts
b/Android/testbed/app/build.gradle.kts
index 4f184cf42d5b5a..53cdc591fa35fd 100644
--- a/Android/testbed/app/build.gradle.kts
+++ b/Android/testbed/app/build.gradle.kts
@@ -94,10 +94,13 @@ android {
}
// This controls the API level of the maxVersion managed emulator,
which is used
- // by CI and cibuildwheel. 34 takes up too much disk space (#142289),
35 has
- // issues connecting to the internet (#142387), and 36 and later are
not
- // available as aosp_atd images yet.
- targetSdk = 33
+ // by CI and cibuildwheel.
+ // * 33 has excessive buffering in the logcat client
+ //
(https://cs.android.com/android/_/android/platform/system/logging/+/d340721894f223327339010df59b0ac514308826).
+ // * 34 consumes too much disk space on GitHub Actions (#142289).
+ // * 35 has issues connecting to the internet (#142387).
+ // * 36 and later are not available as aosp_atd images yet.
+ targetSdk = 32
versionCode = 1
versionName = "1.0"
@@ -130,9 +133,10 @@ android {
path("src/main/c/CMakeLists.txt")
}
- // Set this property to something non-empty, otherwise it'll use the
default
- // list, which ignores asset directories beginning with an underscore.
- aaptOptions.ignoreAssetsPattern = ".git"
+ // Set this property to something nonexistent but non-empty. Otherwise
it'll use the
+ // default list, which ignores asset directories beginning with an
underscore, and
+ // maybe also other files required by tests.
+ aaptOptions.ignoreAssetsPattern = "android-testbed-dont-ignore-anything"
compileOptions {
sourceCompatibility = JavaVersion.VERSION_1_8
@@ -234,6 +238,12 @@ androidComponents.onVariants { variant ->
from(cwd)
}
}
+
+ // A filename ending with .gz will be automatically decompressed
+ // while building the APK. Avoid this by adding a dash to the end,
+ // and add an extra dash to any filenames that already end with
one.
+ // This will be undone in MainActivity.kt.
+ rename(""".*(\.gz|-)""", "$0-")
}
}
diff --git
a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt
b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt
index 5727b0fe6c30c0..dc49cdb9a9f739 100644
--- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt
+++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt
@@ -80,7 +80,9 @@ class PythonTestRunner(val context: Context) {
continue
}
input.use {
- File(targetSubdir, name).outputStream().use { output ->
+ // Undo the .gz workaround from build.gradle.kts.
+ val outputName = name.replace(Regex("""(.*)-"""), "$1")
+ File(targetSubdir, outputName).outputStream().use { output ->
input.copyTo(output)
}
}
diff --git a/Lib/_android_support.py b/Lib/_android_support.py
index a439d03a144dd2..320dab52acdc0b 100644
--- a/Lib/_android_support.py
+++ b/Lib/_android_support.py
@@ -168,6 +168,13 @@ def write(self, prio, tag, message):
# message.
message = message.replace(b"\x00", b"\xc0\x80")
+ # On API level 30 and higher, Logcat will strip any number of leading
+ # newlines. This is visible in all `logcat` modes, even --binary. Work
+ # around this by adding a leading space, which shouldn't make any
+ # difference to the log's usability.
+ if message.startswith(b"\n"):
+ message = b" " + message
+
with self._lock:
now = time()
self._bucket_level += (
diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py
index c6c4a15a7ee34d..31daafbc3d6300 100644
--- a/Lib/test/test_android.py
+++ b/Lib/test/test_android.py
@@ -1,5 +1,4 @@
import io
-import platform
import queue
import re
import subprocess
@@ -17,8 +16,6 @@
if sys.platform != "android":
raise unittest.SkipTest("Android-specific")
-api_level = platform.android_ver().api_level
-
# (name, level, fileno)
STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)]
@@ -179,14 +176,18 @@ def write(s, lines=None, *, write_len=None):
# Multi-line messages. Avoid identical consecutive lines, as
# they may activate "chatty" filtering and break the tests.
- write("\nx", [""])
+ #
+ # Additional spaces will appear in the output where necessary
to
+ # protect leading newlines.
+ write("\nx", [" "])
write("\na\n", ["x", "a"])
- write("\n", [""])
+ write("\n", [" "])
+ write("\n\n", [" ", " "])
write("b\n", ["b"])
- write("c\n\n", ["c", ""])
+ write("c\n\n", ["c", " "])
write("d\ne", ["d"])
write("xx", [])
- write("f\n\ng", ["exxf", ""])
+ write("f\n\ng", ["exxf", " "])
write("\n", ["g"])
# Since this is a line-based logging system, line buffering
@@ -197,15 +198,16 @@ def write(s, lines=None, *, write_len=None):
# However, buffering can be turned off completely if you want a
# flush after every write.
with self.reconfigure(stream, write_through=True):
- write("\nx", ["", "x"])
- write("\na\n", ["", "a"])
- write("\n", [""])
+ write("\nx", [" ", "x"])
+ write("\na\n", [" ", "a"])
+ write("\n", [" "])
+ write("\n\n", [" ", " "])
write("b\n", ["b"])
- write("c\n\n", ["c", ""])
+ write("c\n\n", ["c", " "])
write("d\ne", ["d", "e"])
write("xx", ["xx"])
- write("f\n\ng", ["f", "", "g"])
- write("\n", [""])
+ write("f\n\ng", ["f", " ", "g"])
+ write("\n", [" "])
# "\r\n" should be translated into "\n".
write("hello\r\n", ["hello"])
@@ -325,19 +327,16 @@ def write(b, lines=None, *, write_len=None):
# currently use `logcat -v tag`, which shows each line as if it
# was a separate log entry, but strips a single trailing
# newline.
- #
- # On newer versions of Android, all three of the above tools
(or
- # maybe Logcat itself) will also strip any number of leading
- # newlines.
- write(b"\nx", ["", "x"] if api_level < 30 else ["x"])
- write(b"\na\n", ["", "a"] if api_level < 30 else ["a"])
- write(b"\n", [""])
+ write(b"\nx", [" ", "x"])
+ write(b"\na\n", [" ", "a"])
+ write(b"\n", [" "])
+ write(b"\n\n", [" ", ""])
write(b"b\n", ["b"])
write(b"c\n\n", ["c", ""])
write(b"d\ne", ["d", "e"])
write(b"xx", ["xx"])
write(b"f\n\ng", ["f", "", "g"])
- write(b"\n", [""])
+ write(b"\n", [" "])
# "\r\n" should be translated into "\n".
write(b"hello\r\n", ["hello"])
diff --git
a/Misc/NEWS.d/next/Tests/2026-02-03-07-57-24.gh-issue-144415.U3L15r.rst
b/Misc/NEWS.d/next/Tests/2026-02-03-07-57-24.gh-issue-144415.U3L15r.rst
new file mode 100644
index 00000000000000..b3a8d46329679e
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2026-02-03-07-57-24.gh-issue-144415.U3L15r.rst
@@ -0,0 +1,3 @@
+The Android testbed now distinguishes between stdout/stderr messages which
+were triggered by a newline, and those triggered by a manual call to
+``flush``. This fixes logging of progress indicators and similar content.
_______________________________________________
Python-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3//lists/python-checkins.python.org
Member address: [email protected]
