feat(scripts): device-testing perf harness
Add a cross-platform, standard-library-only Python package under scripts/device-testing/ that replicates LibreMail's on-device performance-test scenarios and logging capture, codifying the methodology run by hand on 2026-07-05 (Pixel 10 Pro XL). Modules: - breadcrumbs.py: a pure, unit-tested parser for the ImapPerf / MailReader / Reader / MailBackfiller breadcrumbs, plus open-correlation that reproduces the manual timing-tables.md figures exactly. - adb.py: a safety-guarded adb wrapper -- an allow-list of adb subcommands and a deny-list + assertions on shell commands. The only sanctioned app-state mutation is clearing LibreMail's own cache/ (exact-match); no pm clear / uninstall / data wipe, and no touching databases/ files/ shared_prefs/ datastore/ can be constructed. - uidump.py: uiautomator XML parser + screen recognition (mailbox rows with the cached "Available offline" flag, reader, and the keyguard / foreign-app guards). - scenarios.py: cold-open, message-open (uncached), back-nav, prefetch A/B (fetch-policy toggle) and cross-provider, each keyguard-guarded and driven through the guarded wrapper. - report.py + perf_harness.py: aggregates, a timing-tables.md renderer mirroring the manual write-up, and the CLI (timestamped run dir with the raw logcat, a filtered breadcrumb extract, and the tables). --dry-run prints the exact command plan without touching device state. Tests (stdlib unittest, 68 cases) validate the parser, guardrails, UI recognition and report against the manual run's real captures (fixtures include a verbatim perf-extract slice and the reader/lockscreen/alarm dumps). Track the *.log fixture past the gitignore *.log rule via a scoped negation. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Test package for the LibreMail device-testing perf harness."""
|
||||
@@ -0,0 +1,17 @@
|
||||
07-05 17:55:38.343 10261 10423 D ImapPerf: prefetch-body connect=857ms work=2150ms live=3
|
||||
07-05 17:55:54.352 10261 10425 D ImapPerf: backfill-page connect=4366ms work=71989ms live=2
|
||||
07-05 17:57:38.981 10261 10425 I MailReader: openMessage imap:94058a folder=INBOX fetchedBody=false took=21ms
|
||||
07-05 17:57:39.041 10261 10261 D Reader : reader ready took=82ms html=true inline=0
|
||||
07-05 18:04:37.001 10261 10408 I MailBackfiller: backfill slice done: moreWork=true
|
||||
07-05 18:04:37.002 10261 10408 I MailBackfiller: backfill slice: maxBatches=20
|
||||
07-05 18:07:16.094 10261 15306 D ImapPerf: body-fetch select=2566ms body=14253ms flag=2356ms rfc822=60457B chars=51610 att=0
|
||||
07-05 18:07:18.435 10261 15306 D ImapPerf: body-fetch connect=2624ms work=26211ms live=3
|
||||
07-05 18:07:20.817 10261 15306 I MailReader: openMessage imap:94058a folder=INBOX fetchedBody=true took=31227ms
|
||||
07-05 18:07:21.206 10261 10261 D Reader : reader ready took=31617ms html=true inline=0
|
||||
07-05 18:07:32.607 10261 10414 D ImapPerf: backfill-page connect=656ms work=3038ms live=3
|
||||
07-05 18:36:37.959 19255 21059 D MailBackfiller: backfill outlook:6b54d6 folder=INBOX pages=20 complete=false
|
||||
07-05 18:36:54.941 19255 19314 D ImapPerf: body-fetch select=185ms body=1355ms flag=78ms rfc822=189086B chars=82986 att=0
|
||||
07-05 18:36:55.002 19255 19314 D ImapPerf: body-fetch connect=772ms work=2018ms live=2
|
||||
07-05 18:36:55.140 19255 21060 I MailReader: openMessage outlook:6b54d6 folder=INBOX fetchedBody=true took=2934ms
|
||||
07-05 18:36:55.144 19255 19255 D Reader : reader ready took=2939ms html=true inline=0
|
||||
07-05 18:37:11.704 19255 19314 D ImapPerf: imap connect=3156ms work=8032ms live=2
|
||||
@@ -0,0 +1 @@
|
||||
<?xml version='1.0' encoding='UTF-8' standalone='yes' ?><hierarchy rotation="0"><node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.google.android.deskclock" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1344,2992]" drawing-order="0" hint=""><node index="0" text="" resource-id="" class="android.widget.LinearLayout" package="com.google.android.deskclock" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1344,2992]" drawing-order="1" hint=""><node index="0" text="" resource-id="" class="android.widget.FrameLayout" package="com.google.android.deskclock" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1344,2992]" drawing-order="2" hint=""><node index="0" text="" resource-id="com.google.android.deskclock:id/action_bar_root" class="android.widget.LinearLayout" package="com.google.android.deskclock" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1344,2992]" drawing-order="1" hint=""><node index="0" text="" resource-id="android:id/content" class="android.widget.FrameLayout" package="com.google.android.deskclock" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1344,2992]" drawing-order="2" hint=""><node index="0" text="" resource-id="com.google.android.deskclock:id/content" class="android.view.ViewGroup" package="com.google.android.deskclock" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[0,0][1344,2992]" drawing-order="1" hint=""><node index="0" text="" resource-id="com.google.android.deskclock:id/digital_clock" class="android.widget.FrameLayout" package="com.google.android.deskclock" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[54,1047][1290,1586]" drawing-order="18" hint=""><node index="0" text="18:00" resource-id="com.google.android.deskclock:id/auto_sizing_text_clock" class="android.widget.TextView" package="com.google.android.deskclock" content-desc="18:00" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[54,1047][1290,1586]" drawing-order="1" hint="" /></node><node index="1" text="Bae check in" resource-id="com.google.android.deskclock:id/title" class="android.widget.TextView" package="com.google.android.deskclock" content-desc="" checkable="false" checked="false" clickable="false" enabled="true" focusable="false" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[54,1646][1290,1960]" drawing-order="19" hint="" /><node index="2" text="Snooze" resource-id="com.google.android.deskclock:id/button_tap_snooze" class="android.widget.Button" package="com.google.android.deskclock" content-desc="" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[126,2274][669,2562]" drawing-order="39" hint="" /><node index="3" text="Stop" resource-id="com.google.android.deskclock:id/button_tap_dismiss" class="android.widget.Button" package="com.google.android.deskclock" content-desc="" checkable="false" checked="false" clickable="true" enabled="true" focusable="true" focused="false" scrollable="false" long-clickable="false" password="false" selected="false" bounds="[675,2274][1218,2562]" drawing-order="40" hint="" /></node></node></node></node></node></node></hierarchy>
|
||||
@@ -0,0 +1 @@
|
||||
<?xml version='1.0' encoding='UTF-8' standalone='yes' ?><hierarchy rotation="0"><node index="0" text="" resource-id="com.android.systemui:id/legacy_window_root" class="android.widget.FrameLayout" package="com.android.systemui" content-desc="" clickable="false" enabled="true" long-clickable="false" scrollable="false" bounds="[0,0][1344,2992]"><node index="0" text="" resource-id="com.android.systemui:id/keyguard_root_view" class="android.view.ViewGroup" package="com.android.systemui" content-desc="" clickable="false" enabled="true" long-clickable="false" scrollable="false" bounds="[0,0][1344,2992]"><node index="0" text="" resource-id="com.android.systemui:id/accessibility_actions_view" class="android.view.View" package="com.android.systemui" content-desc="Lock screen" clickable="false" enabled="true" long-clickable="true" scrollable="false" bounds="[0,216][1344,2992]" /><node index="1" text="18 02" resource-id="" class="android.widget.TextView" package="com.android.systemui" content-desc="18:02" clickable="false" enabled="true" long-clickable="false" scrollable="false" bounds="[100,454][583,620]" /><node index="2" text="" resource-id="com.android.systemui:id/device_entry_icon_fg" class="android.widget.ImageView" package="com.android.systemui" content-desc="Fingerprint sensor" clickable="false" enabled="true" long-clickable="false" scrollable="false" bounds="[561,2126][791,2356]" /><node index="3" text="Charged" resource-id="com.android.systemui:id/keyguard_indication_text" class="android.widget.TextView" package="com.android.systemui" content-desc="" clickable="false" enabled="true" long-clickable="false" scrollable="false" bounds="[16,2860][1344,2925]" /></node></node></hierarchy>
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Tests for the adb safety guardrails -- the harness must refuse dangerous commands."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import adb # noqa: E402
|
||||
from adb import Adb, AdbSafetyError, assert_safe # noqa: E402
|
||||
|
||||
PKG = "org.libremail.app"
|
||||
|
||||
|
||||
class TestAllowedCommands(unittest.TestCase):
|
||||
def _ok(self, args):
|
||||
try:
|
||||
assert_safe(PKG, args)
|
||||
except AdbSafetyError as exc: # pragma: no cover - failure path
|
||||
self.fail(f"assert_safe wrongly rejected {args}: {exc}")
|
||||
|
||||
def test_lifecycle_and_input_allowed(self):
|
||||
self._ok(["shell", "am", "force-stop", PKG])
|
||||
self._ok(["shell", "am", "start", "-W", "-n", f"{PKG}/org.libremail.MainActivity"])
|
||||
self._ok(["shell", "input", "tap", "672", "504"])
|
||||
self._ok(["shell", "input", "swipe", "672", "2000", "672", "900", "400"])
|
||||
self._ok(["shell", "input", "keyevent", "KEYCODE_BACK"])
|
||||
self._ok(["shell", "input", "keyevent", "KEYCODE_WAKEUP"])
|
||||
self._ok(["shell", "uiautomator", "dump", "/dev/tty"])
|
||||
self._ok(["shell", "svc", "power", "stayon", "true"])
|
||||
self._ok(["shell", "dumpsys", "gfxinfo", PKG])
|
||||
self._ok(["shell", "settings", "get", "global", "airplane_mode_on"])
|
||||
self._ok(["shell", "logcat", "-c"])
|
||||
self._ok(["logcat", "-b", "all", "-v", "threadtime"])
|
||||
self._ok(["devices"])
|
||||
self._ok(["get-state"])
|
||||
self._ok(["install", "-r", "app-debug.apk"])
|
||||
|
||||
def test_sanctioned_cache_clear_allowed(self):
|
||||
self._ok(["shell", "run-as", PKG, "sh", "-c", "rm -rf cache/*"])
|
||||
self._ok(["shell", "run-as", PKG, "sh", "-c", "rm -rf cache"])
|
||||
self._ok(["shell", "run-as", PKG, "ls", "-la", "cache"])
|
||||
|
||||
def test_input_swipe_not_confused_with_wipe(self):
|
||||
# 'swipe' contains the substring 'wipe'; it must still be allowed.
|
||||
self._ok(["shell", "input", "swipe", "0", "0", "0", "500", "300"])
|
||||
|
||||
|
||||
class TestDeniedCommands(unittest.TestCase):
|
||||
def _deny(self, args):
|
||||
with self.assertRaises(AdbSafetyError):
|
||||
assert_safe(PKG, args)
|
||||
|
||||
def test_top_level_allowlist(self):
|
||||
self._deny(["uninstall", PKG])
|
||||
self._deny(["root"])
|
||||
self._deny(["remount"])
|
||||
self._deny(["reboot"])
|
||||
self._deny(["disable-verity"])
|
||||
self._deny(["emu", "kill"])
|
||||
self._deny(["push", "x", "/data"])
|
||||
self._deny(["pull", "/data/data/org.libremail.app/databases/x"])
|
||||
|
||||
def test_pm_clear_and_uninstall_denied(self):
|
||||
self._deny(["shell", "pm", "clear", PKG])
|
||||
self._deny(["shell", "pm", "uninstall", PKG])
|
||||
self._deny(["shell", "pm", "disable", PKG])
|
||||
|
||||
def test_reboot_and_root_via_shell_denied(self):
|
||||
self._deny(["shell", "reboot"])
|
||||
self._deny(["shell", "svc", "power", "reboot"])
|
||||
self._deny(["shell", "su", "-c", "reboot"])
|
||||
|
||||
def test_no_touching_private_dirs(self):
|
||||
self._deny(["shell", "run-as", PKG, "sh", "-c", "rm -rf databases"])
|
||||
self._deny(["shell", "run-as", PKG, "sh", "-c", "rm -rf shared_prefs"])
|
||||
self._deny(["shell", "run-as", PKG, "sh", "-c", "rm -rf datastore"])
|
||||
self._deny(["shell", "run-as", PKG, "cat", "databases/libremail.db"])
|
||||
self._deny(["shell", "run-as", PKG, "ls", "files/datastore"])
|
||||
|
||||
def test_rm_outside_cache_denied(self):
|
||||
self._deny(["shell", "run-as", PKG, "sh", "-c", "rm -rf /sdcard/x"])
|
||||
self._deny(["shell", "run-as", PKG, "sh", "-c", "rm -rf cache/../databases"])
|
||||
self._deny(["shell", "rm", "-rf", "/data/local/tmp"])
|
||||
|
||||
def test_output_redirect_denied(self):
|
||||
self._deny(["shell", "echo", "x", ">", "/sdcard/y"])
|
||||
|
||||
def test_run_as_foreign_package_denied(self):
|
||||
self._deny(["shell", "run-as", "com.other.app", "ls", "cache"])
|
||||
|
||||
def test_am_foreign_targets_denied(self):
|
||||
self._deny(["shell", "am", "force-stop", "com.other.app"])
|
||||
self._deny(["shell", "am", "start", "-n", "com.other.app/.Main"])
|
||||
|
||||
def test_empty_command_denied(self):
|
||||
self._deny([])
|
||||
|
||||
|
||||
class TestAdbWrapperBuildsSafeArgv(unittest.TestCase):
|
||||
"""In dry-run the wrapper still validates and returns the argv it would run."""
|
||||
|
||||
def setUp(self):
|
||||
self.adb = Adb(serial="SERIAL123", package=PKG, dry_run=True)
|
||||
|
||||
def test_force_stop_argv(self):
|
||||
self.assertEqual(self.adb.force_stop().args, ["shell", "am", "force-stop", PKG])
|
||||
|
||||
def test_start_activity_argv(self):
|
||||
args = self.adb.start_activity().args
|
||||
self.assertEqual(args[:5], ["shell", "am", "start", "-W", "-n"])
|
||||
self.assertTrue(args[5].startswith(PKG + "/"))
|
||||
|
||||
def test_clear_cache_argv_is_sanctioned(self):
|
||||
self.assertEqual(
|
||||
self.adb.clear_cache().args,
|
||||
["shell", "run-as", PKG, "sh", "-c", "rm -rf cache/*"],
|
||||
)
|
||||
|
||||
def test_argv_includes_serial(self):
|
||||
full = self.adb._argv(["devices"])
|
||||
self.assertEqual(full, ["adb", "-s", "SERIAL123", "devices"])
|
||||
|
||||
def test_wrapper_rejects_unsafe_low_level_call(self):
|
||||
with self.assertRaises(AdbSafetyError):
|
||||
self.adb.run(["shell", "pm", "clear", PKG])
|
||||
|
||||
|
||||
class TestPureParsers(unittest.TestCase):
|
||||
def test_parse_devices(self):
|
||||
out = (
|
||||
"List of devices attached\n"
|
||||
"5C310DLCQ000G3\tdevice\n"
|
||||
"emulator-5554\tdevice\n"
|
||||
"0123456789\toffline\n"
|
||||
)
|
||||
self.assertEqual(adb.parse_devices(out), ["5C310DLCQ000G3", "emulator-5554"])
|
||||
|
||||
def test_parse_devices_empty(self):
|
||||
self.assertEqual(adb.parse_devices("List of devices attached\n"), [])
|
||||
|
||||
def test_parse_am_start(self):
|
||||
out = (
|
||||
"Starting: Intent { cmp=org.libremail.app/org.libremail.MainActivity }\n"
|
||||
"Status: ok\n"
|
||||
"LaunchState: COLD\n"
|
||||
"TotalTime: 384\n"
|
||||
"WaitTime: 386\n"
|
||||
)
|
||||
fields = adb.parse_am_start(out)
|
||||
self.assertEqual(fields["TotalTime"], 384)
|
||||
self.assertEqual(fields["WaitTime"], 386)
|
||||
|
||||
def test_parse_am_start_missing(self):
|
||||
self.assertEqual(adb.parse_am_start("Status: ok\n"), {})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,222 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Unit tests for the breadcrumb parser -- validated against the manual run's real captures.
|
||||
|
||||
The fixture ``fixtures/perf-extract-sample.log`` is a verbatim slice of the manual run's
|
||||
``perf-extract-ALL.log`` (plus the Outlook control), so these tests assert that the parser
|
||||
reproduces the exact figures in the hand-written ``timing-tables.md``.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import breadcrumbs as bc # noqa: E402
|
||||
|
||||
FIXTURE = os.path.join(os.path.dirname(__file__), "fixtures", "perf-extract-sample.log")
|
||||
|
||||
|
||||
def _load_fixture_lines():
|
||||
with open(FIXTURE, encoding="utf-8") as fh:
|
||||
return fh.read().splitlines()
|
||||
|
||||
|
||||
class TestLogcatLine(unittest.TestCase):
|
||||
def test_parses_threadtime_fields(self):
|
||||
line = "07-05 18:07:20.817 10261 15306 I MailReader: openMessage imap:94058a folder=INBOX fetchedBody=true took=31227ms"
|
||||
parsed = bc.parse_logcat_line(line)
|
||||
self.assertIsNotNone(parsed)
|
||||
self.assertEqual(parsed.date, "07-05")
|
||||
self.assertEqual(parsed.time, "18:07:20.817")
|
||||
self.assertEqual(parsed.pid, 10261)
|
||||
self.assertEqual(parsed.tid, 15306)
|
||||
self.assertEqual(parsed.level, "I")
|
||||
self.assertEqual(parsed.tag, "MailReader")
|
||||
self.assertTrue(parsed.message.startswith("openMessage imap:94058a"))
|
||||
self.assertEqual(parsed.timestamp, "07-05 18:07:20.817")
|
||||
|
||||
def test_tolerates_space_padded_tag(self):
|
||||
# logcat pads the tag column, e.g. "Reader :".
|
||||
line = "07-05 17:57:39.041 10261 10261 D Reader : reader ready took=82ms html=true inline=0"
|
||||
parsed = bc.parse_logcat_line(line)
|
||||
self.assertEqual(parsed.tag, "Reader")
|
||||
self.assertEqual(parsed.message, "reader ready took=82ms html=true inline=0")
|
||||
|
||||
def test_non_threadtime_returns_none(self):
|
||||
self.assertIsNone(bc.parse_logcat_line("--------- beginning of main"))
|
||||
self.assertIsNone(bc.parse_logcat_line(""))
|
||||
|
||||
def test_is_perf_line(self):
|
||||
self.assertTrue(
|
||||
bc.is_perf_line("07-05 17:55:38.343 10261 10423 D ImapPerf: prefetch-body connect=857ms work=2150ms live=3")
|
||||
)
|
||||
self.assertFalse(
|
||||
bc.is_perf_line("07-05 15:00:06.896 15192 15217 I MailSyncer: sync all: 1 accounts")
|
||||
)
|
||||
self.assertFalse(bc.is_perf_line("garbage"))
|
||||
|
||||
|
||||
class TestMessageParsers(unittest.TestCase):
|
||||
def test_imap_perf_generic_op(self):
|
||||
ev = bc.parse_imap_perf("prefetch-body connect=857ms work=2150ms live=3")
|
||||
self.assertIsInstance(ev, bc.ImapPerfOp)
|
||||
self.assertEqual((ev.op, ev.connect_ms, ev.work_ms, ev.live), ("prefetch-body", 857, 2150, 3))
|
||||
|
||||
def test_imap_perf_backfill_page(self):
|
||||
ev = bc.parse_imap_perf("backfill-page connect=4366ms work=71989ms live=2")
|
||||
self.assertEqual((ev.op, ev.connect_ms, ev.work_ms, ev.live), ("backfill-page", 4366, 71989, 2))
|
||||
|
||||
def test_imap_perf_body_fetch_op(self):
|
||||
ev = bc.parse_imap_perf("body-fetch connect=2624ms work=26211ms live=3")
|
||||
self.assertIsInstance(ev, bc.ImapPerfOp)
|
||||
self.assertEqual(ev.op, "body-fetch")
|
||||
|
||||
def test_body_fetch_detail(self):
|
||||
ev = bc.parse_imap_perf(
|
||||
"body-fetch select=2566ms body=14253ms flag=2356ms rfc822=60457B chars=51610 att=0"
|
||||
)
|
||||
self.assertIsInstance(ev, bc.BodyFetch)
|
||||
self.assertEqual(ev.select_ms, 2566)
|
||||
self.assertEqual(ev.body_ms, 14253)
|
||||
self.assertEqual(ev.flag_ms, 2356)
|
||||
self.assertEqual(ev.rfc822_bytes, 60457)
|
||||
self.assertEqual(ev.chars, 51610)
|
||||
self.assertEqual(ev.att, 0)
|
||||
|
||||
def test_body_fetch_throughput_matches_manual_table(self):
|
||||
# timing-tables.md reports 4.2 KB/s for A1 and 139.5 KB/s for O1.
|
||||
a1 = bc.parse_imap_perf(
|
||||
"body-fetch select=2566ms body=14253ms flag=2356ms rfc822=60457B chars=51610 att=0"
|
||||
)
|
||||
self.assertAlmostEqual(a1.body_kb_per_s, 4.24, places=2)
|
||||
o1 = bc.parse_imap_perf(
|
||||
"body-fetch select=185ms body=1355ms flag=78ms rfc822=189086B chars=82986 att=0"
|
||||
)
|
||||
self.assertAlmostEqual(o1.body_kb_per_s, 139.5, places=1)
|
||||
|
||||
def test_open_message_uncached(self):
|
||||
ev = bc.parse_open_message("openMessage imap:94058a folder=INBOX fetchedBody=true took=31227ms")
|
||||
self.assertEqual(ev.account_ref, "imap:94058a")
|
||||
self.assertEqual(ev.folder, "INBOX")
|
||||
self.assertTrue(ev.fetched_body)
|
||||
self.assertEqual(ev.took_ms, 31227)
|
||||
|
||||
def test_open_message_cached(self):
|
||||
ev = bc.parse_open_message("openMessage outlook:6b54d6 folder=INBOX fetchedBody=false took=4ms")
|
||||
self.assertEqual(ev.account_ref, "outlook:6b54d6")
|
||||
self.assertFalse(ev.fetched_body)
|
||||
self.assertEqual(ev.took_ms, 4)
|
||||
|
||||
def test_reader_ready(self):
|
||||
ev = bc.parse_reader_ready("reader ready took=31617ms html=true inline=0")
|
||||
self.assertEqual(ev.took_ms, 31617)
|
||||
self.assertTrue(ev.html)
|
||||
self.assertEqual(ev.inline, 0)
|
||||
|
||||
def test_backfill_progress(self):
|
||||
ev = bc.parse_backfill("backfill outlook:6b54d6 folder=INBOX pages=20 complete=false")
|
||||
self.assertIsInstance(ev, bc.BackfillProgress)
|
||||
self.assertEqual(ev.account_ref, "outlook:6b54d6")
|
||||
self.assertEqual(ev.pages, 20)
|
||||
self.assertFalse(ev.complete)
|
||||
|
||||
def test_backfill_slice_start_and_done(self):
|
||||
start = bc.parse_backfill("backfill slice: maxBatches=20")
|
||||
self.assertIsInstance(start, bc.BackfillSliceStart)
|
||||
self.assertEqual(start.max_batches, 20)
|
||||
done = bc.parse_backfill("backfill slice done: moreWork=true")
|
||||
self.assertIsInstance(done, bc.BackfillSliceDone)
|
||||
self.assertTrue(done.more_work)
|
||||
|
||||
def test_unparseable_messages_return_none(self):
|
||||
self.assertIsNone(bc.parse_imap_perf("nonsense here"))
|
||||
self.assertIsNone(bc.parse_open_message("openMessage missing fields"))
|
||||
self.assertIsNone(bc.parse_reader_ready("reader not ready"))
|
||||
self.assertIsNone(bc.parse_backfill("backfill mystery"))
|
||||
|
||||
|
||||
class TestParseBreadcrumbDispatch(unittest.TestCase):
|
||||
def test_dispatch_attaches_source_line(self):
|
||||
line = "07-05 18:07:18.435 10261 15306 D ImapPerf: body-fetch connect=2624ms work=26211ms live=3"
|
||||
ev = bc.parse_breadcrumb(line)
|
||||
self.assertIsInstance(ev, bc.ImapPerfOp)
|
||||
self.assertIsNotNone(ev.line)
|
||||
self.assertEqual(ev.line.pid, 10261)
|
||||
self.assertEqual(ev.timestamp, "07-05 18:07:18.435")
|
||||
|
||||
def test_dispatch_ignores_non_perf_tags(self):
|
||||
self.assertIsNone(
|
||||
bc.parse_breadcrumb("07-05 15:00:06.896 15192 15217 I MailSyncer: sync all: 1 accounts")
|
||||
)
|
||||
|
||||
|
||||
class TestFixtureStream(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.lines = _load_fixture_lines()
|
||||
self.events = list(bc.iter_events(self.lines))
|
||||
|
||||
def test_all_fixture_lines_are_perf_lines(self):
|
||||
self.assertEqual(sum(1 for line in self.lines if bc.is_perf_line(line)), len(self.lines))
|
||||
|
||||
def test_every_line_parses(self):
|
||||
self.assertEqual(len(self.events), len(self.lines))
|
||||
|
||||
def test_event_type_mix(self):
|
||||
kinds = [type(e).__name__ for e in self.events]
|
||||
# 3 prefetch/backfill-page ops + 2 body-fetch ops + 1 imap op = 6.
|
||||
self.assertEqual(kinds.count("ImapPerfOp"), 6)
|
||||
self.assertEqual(kinds.count("BodyFetch"), 2)
|
||||
self.assertEqual(kinds.count("OpenMessage"), 3)
|
||||
self.assertEqual(kinds.count("ReaderReady"), 3)
|
||||
self.assertEqual(kinds.count("BackfillProgress"), 1)
|
||||
self.assertEqual(kinds.count("BackfillSliceStart"), 1)
|
||||
self.assertEqual(kinds.count("BackfillSliceDone"), 1)
|
||||
|
||||
|
||||
class TestCorrelateOpens(unittest.TestCase):
|
||||
"""correlate_opens must reproduce the manual timing-tables rows exactly."""
|
||||
|
||||
def setUp(self):
|
||||
events = list(bc.iter_events(_load_fixture_lines()))
|
||||
self.samples = bc.correlate_opens(events)
|
||||
|
||||
def test_three_opens_detected(self):
|
||||
self.assertEqual(len(self.samples), 3)
|
||||
|
||||
def test_cached_gmail_open(self):
|
||||
s = self.samples[0]
|
||||
self.assertEqual(s.account_ref, "imap:94058a")
|
||||
self.assertTrue(s.cached)
|
||||
self.assertEqual(s.took_ms, 21)
|
||||
self.assertIsNone(s.body_fetch)
|
||||
self.assertEqual(s.reader_ready.took_ms, 82)
|
||||
|
||||
def test_gmail_A1_row_matches_table1(self):
|
||||
s = self.samples[1]
|
||||
self.assertEqual(s.account_ref, "imap:94058a")
|
||||
self.assertFalse(s.cached)
|
||||
self.assertEqual(s.took_ms, 31227)
|
||||
self.assertEqual(s.reader_ready.took_ms, 31617)
|
||||
self.assertEqual(s.rfc822_bytes, 60457)
|
||||
self.assertEqual(s.body_fetch.select_ms, 2566)
|
||||
self.assertEqual(s.body_fetch.body_ms, 14253)
|
||||
self.assertEqual(s.body_fetch.flag_ms, 2356)
|
||||
self.assertEqual(s.body_fetch_op.connect_ms, 2624)
|
||||
self.assertEqual(s.body_fetch_op.work_ms, 26211)
|
||||
self.assertEqual(s.body_fetch_op.live, 3)
|
||||
self.assertAlmostEqual(s.body_kb_per_s, 4.24, places=2)
|
||||
|
||||
def test_outlook_O1_row_matches_table2(self):
|
||||
s = self.samples[2]
|
||||
self.assertEqual(s.account_ref, "outlook:6b54d6")
|
||||
self.assertFalse(s.cached)
|
||||
self.assertEqual(s.took_ms, 2934)
|
||||
self.assertEqual(s.reader_ready.took_ms, 2939)
|
||||
self.assertEqual(s.rfc822_bytes, 189086)
|
||||
self.assertEqual(s.body_fetch_op.connect_ms, 772)
|
||||
self.assertAlmostEqual(s.body_kb_per_s, 139.5, places=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,109 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Tests for aggregation and the timing-tables markdown renderers."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import breadcrumbs as bc # noqa: E402
|
||||
import report # noqa: E402
|
||||
|
||||
FIXTURE = os.path.join(os.path.dirname(__file__), "fixtures", "perf-extract-sample.log")
|
||||
|
||||
|
||||
class TestAggregate(unittest.TestCase):
|
||||
def test_basic_stats(self):
|
||||
agg = report.aggregate([10, 20, 30])
|
||||
self.assertEqual(agg.n, 3)
|
||||
self.assertEqual(agg.mean, 20)
|
||||
self.assertEqual(agg.median, 20)
|
||||
self.assertEqual((agg.minimum, agg.maximum), (10, 30))
|
||||
|
||||
def test_reproduces_manual_gmail_median(self):
|
||||
# timing-tables.md Condition A openMessage values -> median ~32.1 s.
|
||||
agg = report.aggregate([31227, 32059, 31721, 32148, 32949, 39728])
|
||||
self.assertEqual(agg.median, 32103.5)
|
||||
|
||||
def test_empty(self):
|
||||
agg = report.aggregate([])
|
||||
self.assertEqual(agg.n, 0)
|
||||
self.assertIsNone(agg.mean)
|
||||
self.assertIsNone(agg.median)
|
||||
|
||||
|
||||
class TestMdTable(unittest.TestCase):
|
||||
def test_table_shape(self):
|
||||
table = report.md_table(["a", "b"], [[1, 2], [3, None]])
|
||||
lines = table.splitlines()
|
||||
self.assertEqual(lines[0], "| a | b |")
|
||||
self.assertEqual(lines[1], "| --- | --- |")
|
||||
self.assertEqual(lines[2], "| 1 | 2 |")
|
||||
self.assertEqual(lines[3], "| 3 | |") # None renders empty
|
||||
|
||||
|
||||
class TestColdOpen(unittest.TestCase):
|
||||
def test_render_includes_mean_row(self):
|
||||
samples = [
|
||||
report.ColdOpenSample(run=1, total_time_ms=523, wait_time_ms=526),
|
||||
report.ColdOpenSample(run=2, total_time_ms=384, wait_time_ms=386),
|
||||
]
|
||||
md = report.render_cold_open(samples)
|
||||
self.assertIn("523", md)
|
||||
self.assertIn("**mean**", md)
|
||||
self.assertIn("454", md) # mean total = 453.5 -> 454
|
||||
|
||||
|
||||
class TestMessageOpen(unittest.TestCase):
|
||||
def setUp(self):
|
||||
with open(FIXTURE, encoding="utf-8") as fh:
|
||||
events = list(bc.iter_events(fh.read().splitlines()))
|
||||
self.samples = bc.correlate_opens(events)
|
||||
|
||||
def test_from_open_sample_maps_fields(self):
|
||||
row = report.ReaderOpenRow.from_open_sample(self.samples[1], index=1, label="A1")
|
||||
self.assertEqual(row.took_ms, 31227)
|
||||
self.assertEqual(row.reader_ready_ms, 31617)
|
||||
self.assertEqual(row.rfc822_bytes, 60457)
|
||||
self.assertEqual(row.connect_ms, 2624)
|
||||
self.assertEqual(row.select_ms, 2566)
|
||||
self.assertFalse(row.cached)
|
||||
|
||||
def test_render_message_open_table_and_aggregate(self):
|
||||
rows = [
|
||||
report.ReaderOpenRow.from_open_sample(s, index=i, label=f"m{i}")
|
||||
for i, s in enumerate(self.samples, start=1)
|
||||
]
|
||||
md = report.render_message_open("Message open (uncached)", rows)
|
||||
self.assertIn("31227 ms", md)
|
||||
self.assertIn("2934 ms", md)
|
||||
self.assertIn("60457", md)
|
||||
self.assertIn("4.2", md) # Gmail throughput
|
||||
self.assertIn("139.5", md) # Outlook throughput
|
||||
self.assertIn("n=2", md) # two uncached opens among the three
|
||||
|
||||
def test_skipped_row_renders(self):
|
||||
rows = [report.ReaderOpenRow(index=1, label="x", skipped=True, reason="keyguard")]
|
||||
md = report.render_message_open("t", rows)
|
||||
self.assertIn("SKIPPED: keyguard", md)
|
||||
|
||||
|
||||
class TestBackNav(unittest.TestCase):
|
||||
def test_caveat_present(self):
|
||||
samples = [report.BackNavSample(index=1, back_ms=3484)]
|
||||
md = report.render_back_nav(samples)
|
||||
self.assertIn("Caveat", md)
|
||||
self.assertIn("3484 ms", md)
|
||||
|
||||
|
||||
class TestDocument(unittest.TestCase):
|
||||
def test_build_document(self):
|
||||
doc = report.build_document({"scenario": "cold-open"}, ["## Section\n\nbody\n"])
|
||||
self.assertIn("# LibreMail device perf run", doc)
|
||||
self.assertIn("**scenario:** cold-open", doc)
|
||||
self.assertIn("## Section", doc)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,108 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
"""Tests for the uiautomator-dump parser and screen/keyguard recognition.
|
||||
|
||||
Fixtures are cleaned copies of the manual run's real dumps: the reader (``ui_reader.xml``),
|
||||
a deskclock alarm (``ui_alarm.xml``) and the keyguard (``ui_lockscreen.xml``) are the
|
||||
foreign-screen negatives the harness must skip; ``ui_mailbox.xml`` is a compact, structurally
|
||||
faithful mailbox list.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import uidump # noqa: E402
|
||||
|
||||
FIX = os.path.join(os.path.dirname(__file__), "fixtures")
|
||||
APP = "org.libremail.app"
|
||||
|
||||
|
||||
def load(name):
|
||||
with open(os.path.join(FIX, name), encoding="utf-8") as fh:
|
||||
return uidump.parse_dump(fh.read())
|
||||
|
||||
|
||||
class TestBounds(unittest.TestCase):
|
||||
def test_parse_bounds(self):
|
||||
self.assertEqual(uidump.parse_bounds("[0,392][1344,617]"), (0, 392, 1344, 617))
|
||||
|
||||
def test_parse_bounds_none(self):
|
||||
self.assertIsNone(uidump.parse_bounds(""))
|
||||
self.assertIsNone(uidump.parse_bounds("not-bounds"))
|
||||
|
||||
|
||||
class TestMailbox(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.root = load("ui_mailbox.xml")
|
||||
|
||||
def test_recognised_as_app_not_lockscreen(self):
|
||||
self.assertEqual(uidump.foreground_package(self.root), APP)
|
||||
self.assertFalse(uidump.is_lockscreen(self.root))
|
||||
self.assertTrue(uidump.is_app_foreground(self.root, APP))
|
||||
self.assertFalse(uidump.is_reader(self.root, APP))
|
||||
|
||||
def test_finds_three_message_rows(self):
|
||||
rows = uidump.find_message_rows(self.root, APP)
|
||||
self.assertEqual(len(rows), 3)
|
||||
|
||||
def test_row_centers_and_cached_flags(self):
|
||||
rows = uidump.find_message_rows(self.root, APP)
|
||||
self.assertEqual(rows[0].center, (672, 504))
|
||||
self.assertFalse(rows[0].cached)
|
||||
self.assertEqual(rows[1].center, (672, 732))
|
||||
self.assertFalse(rows[1].cached)
|
||||
# Row 2 carries the "Available offline" content-desc -> cached.
|
||||
self.assertEqual(rows[2].center, (672, 988))
|
||||
self.assertTrue(rows[2].cached)
|
||||
|
||||
def test_row_label_drops_monogram(self):
|
||||
rows = uidump.find_message_rows(self.root, APP)
|
||||
self.assertIn("github-actions[bot]", rows[0].label)
|
||||
self.assertNotIn(" G ", rows[0].label) # single-letter avatar filtered out
|
||||
|
||||
def test_settings_entry_tappable(self):
|
||||
node = uidump.find_by_text(self.root, "Settings")
|
||||
self.assertIsNotNone(node)
|
||||
anchor = node.first_clickable_ancestor()
|
||||
self.assertIsNotNone(anchor)
|
||||
self.assertEqual(anchor.center, (1014, 2728))
|
||||
|
||||
def test_top_bar_content_descs(self):
|
||||
self.assertIsNotNone(uidump.find_by_content_desc(self.root, "Show folders"))
|
||||
self.assertIsNotNone(uidump.find_by_content_desc(self.root, "Search"))
|
||||
|
||||
|
||||
class TestReader(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.root = load("ui_reader.xml")
|
||||
|
||||
def test_recognised_as_reader(self):
|
||||
self.assertTrue(uidump.is_app_foreground(self.root, APP))
|
||||
self.assertTrue(uidump.is_reader(self.root, APP))
|
||||
|
||||
def test_progress_bar_signals_loading(self):
|
||||
self.assertTrue(uidump.has_progress_bar(self.root))
|
||||
|
||||
def test_no_message_rows_in_reader(self):
|
||||
self.assertEqual(uidump.find_message_rows(self.root, APP), [])
|
||||
|
||||
|
||||
class TestForeignScreensAreGuarded(unittest.TestCase):
|
||||
def test_lockscreen_detected(self):
|
||||
root = load("ui_lockscreen.xml")
|
||||
self.assertTrue(uidump.is_lockscreen(root))
|
||||
self.assertFalse(uidump.is_app_foreground(root, APP))
|
||||
self.assertFalse(uidump.is_reader(root, APP))
|
||||
|
||||
def test_alarm_is_foreign_app(self):
|
||||
root = load("ui_alarm.xml")
|
||||
self.assertEqual(uidump.foreground_package(root), "com.google.android.deskclock")
|
||||
self.assertFalse(uidump.is_app_foreground(root, APP))
|
||||
self.assertFalse(uidump.is_lockscreen(root))
|
||||
self.assertFalse(uidump.is_reader(root, APP))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user