https://github.com/python/cpython/commit/ebc047f7179923a644416ebfe233109e1b9bbbc6 commit: ebc047f7179923a644416ebfe233109e1b9bbbc6 branch: 3.13 author: Miss Islington (bot) <[email protected]> committer: freakboy3742 <[email protected]> date: 2026-02-03T09:28:43Z summary:
[3.13] gh-144415: Android testbed fixes (GH-142912) (#144417) Modifies handling of `.gz` files in Android app payloads, and ensures that when the Android testbed streams logs, stream flushes aren't treated as newlines. This improves the output of test suites that use "one dot per test" progress indicators. (cherry picked from commit cb1dc91dcb1eb6637d1c79b34a0bab728939f717) Co-authored-by: Malcolm Smith <[email protected]> files: A Misc/NEWS.d/next/Tests/2026-02-03-07-57-24.gh-issue-144415.U3L15r.rst M Android/android.py M Android/testbed/app/build.gradle.kts M Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt M Lib/_android_support.py M Lib/test/test_android.py diff --git a/Android/android.py b/Android/android.py index 50a0da47530a4c..2ca844dda788cd 100755 --- a/Android/android.py +++ b/Android/android.py @@ -15,6 +15,7 @@ from asyncio import wait_for from contextlib import asynccontextmanager from datetime import datetime, timezone +from enum import IntEnum, auto from glob import glob from os.path import abspath, basename, relpath from pathlib import Path @@ -61,6 +62,19 @@ hidden_output = [] +# Based on android/log.h in the NDK. +class LogPriority(IntEnum): + UNKNOWN = 0 + DEFAULT = auto() + VERBOSE = auto() + DEBUG = auto() + INFO = auto() + WARN = auto() + ERROR = auto() + FATAL = auto() + SILENT = auto() + + def log_verbose(context, line, stream=sys.stdout): if context.verbose: stream.write(line) @@ -505,21 +519,23 @@ async def logcat_task(context, initial_devices): pid = await wait_for(find_pid(serial), startup_timeout) # `--pid` requires API level 24 or higher. - args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] + # + # `--binary` mode is used in order to detect which messages end with a + # newline, which most of the other modes don't indicate (except `--format + # long`). For example, every time pytest runs a test, it prints a "." and + # flushes the stream. Each "." becomes a separate log message, but we should + # show them all on the same line. + args = [adb, "-s", serial, "logcat", "--pid", pid, "--binary"] logcat_started = False async with async_process( - *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + *args, stdout=subprocess.PIPE, stderr=None ) as process: - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): + while True: + try: + priority, tag, message = await read_logcat(process.stdout) logcat_started = True - level, message = match.groups() - else: - # If the regex doesn't match, this is either a logcat startup - # error, or the second or subsequent line of a multi-line - # message. Python won't produce multi-line messages, but other - # components might. - level, message = None, line + except asyncio.IncompleteReadError: + break # Exclude high-volume messages which are rarely useful. if context.verbose < 2 and "from python test_syslog" in message: @@ -527,25 +543,23 @@ async def logcat_task(context, initial_devices): # Put high-level messages on stderr so they're highlighted in the # buildbot logs. This will include Python's own stderr. - stream = ( - sys.stderr - if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka ASSERT) - else sys.stdout - ) - - # To simplify automated processing of the output, e.g. a buildbot - # posting a failure notice on a GitHub PR, we strip the level and - # tag indicators from Python's stdout and stderr. - for prefix in ["python.stdout: ", "python.stderr: "]: - if message.startswith(prefix): - global python_started - python_started = True - stream.write(message.removeprefix(prefix)) - break + stream = sys.stderr if priority >= LogPriority.WARN else sys.stdout + + # The app's stdout and stderr should be passed through transparently + # to our own corresponding streams. + if tag in ["python.stdout", "python.stderr"]: + global python_started + python_started = True + stream.write(message) + stream.flush() else: # Non-Python messages add a lot of noise, but they may - # sometimes help explain a failure. - log_verbose(context, line, stream) + # sometimes help explain a failure. Format them in the same way + # as `logcat --format tag`. + formatted = f"{priority.name[0]}/{tag}: {message}" + if not formatted.endswith("\n"): + formatted += "\n" + log_verbose(context, formatted, stream) # If the device disconnects while logcat is running, which always # happens in --managed mode, some versions of adb return non-zero. @@ -556,6 +570,44 @@ async def logcat_task(context, initial_devices): raise CalledProcessError(status, args) +# Read one binary log message from the given StreamReader. The message format is +# described at https://android.stackexchange.com/a/74660. All supported versions +# of Android use format version 2 or later. +async def read_logcat(stream): + async def read_bytes(size): + return await stream.readexactly(size) + + async def read_int(size): + return int.from_bytes(await read_bytes(size), "little") + + payload_len = await read_int(2) + if payload_len < 2: + # 1 byte for priority, 1 byte for null terminator of tag. + raise ValueError(f"payload length {payload_len} is too short") + + header_len = await read_int(2) + if header_len < 4: + raise ValueError(f"header length {header_len} is too short") + await read_bytes(header_len - 4) # Ignore other header fields. + + priority_int = await read_int(1) + try: + priority = LogPriority(priority_int) + except ValueError: + priority = LogPriority.UNKNOWN + + payload_fields = (await read_bytes(payload_len - 1)).split(b"\0") + if len(payload_fields) < 2: + raise ValueError( + f"payload {payload!r} does not contain at least 2 " + f"null-separated fields" + ) + tag, message, *_ = [ + field.decode(*DECODE_ARGS) for field in payload_fields + ] + return priority, tag, message + + def stop_app(serial): run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False) diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index 4f184cf42d5b5a..53cdc591fa35fd 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -94,10 +94,13 @@ android { } // This controls the API level of the maxVersion managed emulator, which is used - // by CI and cibuildwheel. 34 takes up too much disk space (#142289), 35 has - // issues connecting to the internet (#142387), and 36 and later are not - // available as aosp_atd images yet. - targetSdk = 33 + // by CI and cibuildwheel. + // * 33 has excessive buffering in the logcat client + // (https://cs.android.com/android/_/android/platform/system/logging/+/d340721894f223327339010df59b0ac514308826). + // * 34 consumes too much disk space on GitHub Actions (#142289). + // * 35 has issues connecting to the internet (#142387). + // * 36 and later are not available as aosp_atd images yet. + targetSdk = 32 versionCode = 1 versionName = "1.0" @@ -130,9 +133,10 @@ android { path("src/main/c/CMakeLists.txt") } - // Set this property to something non-empty, otherwise it'll use the default - // list, which ignores asset directories beginning with an underscore. - aaptOptions.ignoreAssetsPattern = ".git" + // Set this property to something nonexistent but non-empty. Otherwise it'll use the + // default list, which ignores asset directories beginning with an underscore, and + // maybe also other files required by tests. + aaptOptions.ignoreAssetsPattern = "android-testbed-dont-ignore-anything" compileOptions { sourceCompatibility = JavaVersion.VERSION_1_8 @@ -234,6 +238,12 @@ androidComponents.onVariants { variant -> from(cwd) } } + + // A filename ending with .gz will be automatically decompressed + // while building the APK. Avoid this by adding a dash to the end, + // and add an extra dash to any filenames that already end with one. + // This will be undone in MainActivity.kt. + rename(""".*(\.gz|-)""", "$0-") } } diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index 5727b0fe6c30c0..dc49cdb9a9f739 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -80,7 +80,9 @@ class PythonTestRunner(val context: Context) { continue } input.use { - File(targetSubdir, name).outputStream().use { output -> + // Undo the .gz workaround from build.gradle.kts. + val outputName = name.replace(Regex("""(.*)-"""), "$1") + File(targetSubdir, outputName).outputStream().use { output -> input.copyTo(output) } } diff --git a/Lib/_android_support.py b/Lib/_android_support.py index a439d03a144dd2..320dab52acdc0b 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -168,6 +168,13 @@ def write(self, prio, tag, message): # message. message = message.replace(b"\x00", b"\xc0\x80") + # On API level 30 and higher, Logcat will strip any number of leading + # newlines. This is visible in all `logcat` modes, even --binary. Work + # around this by adding a leading space, which shouldn't make any + # difference to the log's usability. + if message.startswith(b"\n"): + message = b" " + message + with self._lock: now = time() self._bucket_level += ( diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 6e6768e37e8e2c..93e07acc86b236 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -183,14 +183,18 @@ def write(s, lines=None, *, write_len=None): # Multi-line messages. Avoid identical consecutive lines, as # they may activate "chatty" filtering and break the tests. - write("\nx", [""]) + # + # Additional spaces will appear in the output where necessary to + # protect leading newlines. + write("\nx", [" "]) write("\na\n", ["x", "a"]) - write("\n", [""]) + write("\n", [" "]) + write("\n\n", [" ", " "]) write("b\n", ["b"]) - write("c\n\n", ["c", ""]) + write("c\n\n", ["c", " "]) write("d\ne", ["d"]) write("xx", []) - write("f\n\ng", ["exxf", ""]) + write("f\n\ng", ["exxf", " "]) write("\n", ["g"]) # Since this is a line-based logging system, line buffering @@ -201,15 +205,16 @@ def write(s, lines=None, *, write_len=None): # However, buffering can be turned off completely if you want a # flush after every write. with self.reconfigure(stream, write_through=True): - write("\nx", ["", "x"]) - write("\na\n", ["", "a"]) - write("\n", [""]) + write("\nx", [" ", "x"]) + write("\na\n", [" ", "a"]) + write("\n", [" "]) + write("\n\n", [" ", " "]) write("b\n", ["b"]) - write("c\n\n", ["c", ""]) + write("c\n\n", ["c", " "]) write("d\ne", ["d", "e"]) write("xx", ["xx"]) - write("f\n\ng", ["f", "", "g"]) - write("\n", [""]) + write("f\n\ng", ["f", " ", "g"]) + write("\n", [" "]) # "\r\n" should be translated into "\n". write("hello\r\n", ["hello"]) @@ -329,19 +334,16 @@ def write(b, lines=None, *, write_len=None): # currently use `logcat -v tag`, which shows each line as if it # was a separate log entry, but strips a single trailing # newline. - # - # On newer versions of Android, all three of the above tools (or - # maybe Logcat itself) will also strip any number of leading - # newlines. - write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) - write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) - write(b"\n", [""]) + write(b"\nx", [" ", "x"]) + write(b"\na\n", [" ", "a"]) + write(b"\n", [" "]) + write(b"\n\n", [" ", ""]) write(b"b\n", ["b"]) write(b"c\n\n", ["c", ""]) write(b"d\ne", ["d", "e"]) write(b"xx", ["xx"]) write(b"f\n\ng", ["f", "", "g"]) - write(b"\n", [""]) + write(b"\n", [" "]) # "\r\n" should be translated into "\n". write(b"hello\r\n", ["hello"]) diff --git a/Misc/NEWS.d/next/Tests/2026-02-03-07-57-24.gh-issue-144415.U3L15r.rst b/Misc/NEWS.d/next/Tests/2026-02-03-07-57-24.gh-issue-144415.U3L15r.rst new file mode 100644 index 00000000000000..b3a8d46329679e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-02-03-07-57-24.gh-issue-144415.U3L15r.rst @@ -0,0 +1,3 @@ +The Android testbed now distinguishes between stdout/stderr messages which +were triggered by a newline, and those triggered by a manual call to +``flush``. This fixes logging of progress indicators and similar content. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
