JMR-dev/LibreMail
1
0
Fork 0
Code Issues 40 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

perf(db): covering index for unified-inbox summary scans

Browse Source 
The paged "All inboxes" query (MessageDao.pagingUnifiedFolderSummaries:
WHERE folder = ? AND inInbox = 1 ORDER BY timestampMillis DESC) had no
folder-leading index, so it SCANned the whole messages table via
index_messages_timestampMillis and filtered folder/inInbox per row.

Add a (folder, inInbox, timestampMillis) index so the two equality
predicates become an index seek and the ORDER BY is supplied by the
index. EXPLAIN QUERY PLAN for the query goes from
  SCAN messages USING INDEX index_messages_timestampMillis
to
  SEARCH messages USING INDEX index_messages_folder_inInbox_timestampMillis (folder=? AND inInbox=?)
with no temp B-tree sort.

Pure additive index (no column/table change): bump the Room DB to v20
with MIGRATION_19_20 (CREATE INDEX IF NOT EXISTS), register it in
DatabaseModule, export 20.json, and add a MigrationTest that runs the
migration and asserts the index shape plus the SEARCH plan on real
Android SQLite.

Closes #187

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
JMR-dev
2026-07-04 00:14:39 -05:00
co-authored by Claude Opus 4.8
parent e587ba0d82
commit 5a3669f017
6 changed files with 610 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/schemas/org.libremail.data.local.LibreMailDatabase/20.json
+506
View File
@@ -0,0 +1,506 @@
{
"formatVersion": 1,
"database": {
"version": 20,
"identityHash": "8264768635364869a347064a0864df9c",
"entities": [
{
"tableName": "messages",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT NOT NULL, `sender` TEXT NOT NULL, `senderEmail` TEXT NOT NULL, `subject` TEXT NOT NULL, `snippet` TEXT NOT NULL, `body` TEXT NOT NULL, `isHtml` INTEGER NOT NULL, `timestampMillis` INTEGER NOT NULL, `isRead` INTEGER NOT NULL, `isStarred` INTEGER NOT NULL, `folder` TEXT NOT NULL DEFAULT 'INBOX', `inInbox` INTEGER NOT NULL, `bodyFetched` INTEGER NOT NULL, `uid` INTEGER NOT NULL DEFAULT 0, `senderFold` TEXT NOT NULL DEFAULT '', `senderEmailFold` TEXT NOT NULL DEFAULT '', `subjectFold` TEXT NOT NULL DEFAULT '', `snippetFold` TEXT NOT NULL DEFAULT '', PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "sender",
"columnName": "sender",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "senderEmail",
"columnName": "senderEmail",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "subject",
"columnName": "subject",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "snippet",
"columnName": "snippet",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "body",
"columnName": "body",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "isHtml",
"columnName": "isHtml",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "timestampMillis",
"columnName": "timestampMillis",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isRead",
"columnName": "isRead",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isStarred",
"columnName": "isStarred",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "folder",
"columnName": "folder",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "'INBOX'"
},
{
"fieldPath": "inInbox",
"columnName": "inInbox",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "bodyFetched",
"columnName": "bodyFetched",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "uid",
"columnName": "uid",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "0"
},
{
"fieldPath": "senderFold",
"columnName": "senderFold",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "senderEmailFold",
"columnName": "senderEmailFold",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "subjectFold",
"columnName": "subjectFold",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "snippetFold",
"columnName": "snippetFold",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
},
"indices": [
{
"name": "index_messages_accountId",
"unique": false,
"columnNames": [
"accountId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_messages_accountId` ON `${TABLE_NAME}` (`accountId`)"
},
{
"name": "index_messages_timestampMillis",
"unique": false,
"columnNames": [
"timestampMillis"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_messages_timestampMillis` ON `${TABLE_NAME}` (`timestampMillis`)"
},
{
"name": "index_messages_accountId_folder_uid",
"unique": false,
"columnNames": [
"accountId",
"folder",
"uid"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_messages_accountId_folder_uid` ON `${TABLE_NAME}` (`accountId`, `folder`, `uid`)"
},
{
"name": "index_messages_folder_inInbox_timestampMillis",
"unique": false,
"columnNames": [
"folder",
"inInbox",
"timestampMillis"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_messages_folder_inInbox_timestampMillis` ON `${TABLE_NAME}` (`folder`, `inInbox`, `timestampMillis`)"
}
]
},
{
"tableName": "attachments",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`messageId` TEXT NOT NULL, `partIndex` INTEGER NOT NULL, `filename` TEXT NOT NULL, `mimeType` TEXT NOT NULL, `sizeBytes` INTEGER NOT NULL, `contentId` TEXT, PRIMARY KEY(`messageId`, `partIndex`), FOREIGN KEY(`messageId`) REFERENCES `messages`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
"fields": [
{
"fieldPath": "messageId",
"columnName": "messageId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "partIndex",
"columnName": "partIndex",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "filename",
"columnName": "filename",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "mimeType",
"columnName": "mimeType",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "sizeBytes",
"columnName": "sizeBytes",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "contentId",
"columnName": "contentId",
"affinity": "TEXT"
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"messageId",
"partIndex"
]
},
"indices": [
{
"name": "index_attachments_messageId",
"unique": false,
"columnNames": [
"messageId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_attachments_messageId` ON `${TABLE_NAME}` (`messageId`)"
}
],
"foreignKeys": [
{
"table": "messages",
"onDelete": "CASCADE",
"onUpdate": "NO ACTION",
"columns": [
"messageId"
],
"referencedColumns": [
"id"
]
}
]
},
{
"tableName": "outbox",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT NOT NULL, `toAddresses` TEXT NOT NULL, `ccAddresses` TEXT NOT NULL, `bccAddresses` TEXT NOT NULL DEFAULT '', `subject` TEXT NOT NULL, `body` TEXT NOT NULL, `createdAt` INTEGER NOT NULL, `lastError` TEXT, `bodyHtml` TEXT, `attachments` TEXT NOT NULL DEFAULT '', PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "toAddresses",
"columnName": "toAddresses",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "ccAddresses",
"columnName": "ccAddresses",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "bccAddresses",
"columnName": "bccAddresses",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "subject",
"columnName": "subject",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "body",
"columnName": "body",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "createdAt",
"columnName": "createdAt",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "lastError",
"columnName": "lastError",
"affinity": "TEXT"
},
{
"fieldPath": "bodyHtml",
"columnName": "bodyHtml",
"affinity": "TEXT"
},
{
"fieldPath": "attachments",
"columnName": "attachments",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
}
},
{
"tableName": "drafts",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT, `toAddresses` TEXT NOT NULL, `ccAddresses` TEXT NOT NULL, `bccAddresses` TEXT NOT NULL DEFAULT '', `subject` TEXT NOT NULL, `body` TEXT NOT NULL, `updatedAt` INTEGER NOT NULL, `attachments` TEXT NOT NULL, `bodyHtml` TEXT, PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "TEXT"
},
{
"fieldPath": "toAddresses",
"columnName": "toAddresses",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "ccAddresses",
"columnName": "ccAddresses",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "bccAddresses",
"columnName": "bccAddresses",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "subject",
"columnName": "subject",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "body",
"columnName": "body",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "updatedAt",
"columnName": "updatedAt",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "attachments",
"columnName": "attachments",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "bodyHtml",
"columnName": "bodyHtml",
"affinity": "TEXT"
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
}
},
{
"tableName": "folders",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`accountId` TEXT NOT NULL, `fullName` TEXT NOT NULL, `displayName` TEXT NOT NULL, `role` TEXT NOT NULL, `selectable` INTEGER NOT NULL, `sortOrder` INTEGER NOT NULL, `specialUse` INTEGER NOT NULL DEFAULT 0, `hierarchyDelimiter` TEXT, PRIMARY KEY(`accountId`, `fullName`))",
"fields": [
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "fullName",
"columnName": "fullName",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "displayName",
"columnName": "displayName",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "role",
"columnName": "role",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "selectable",
"columnName": "selectable",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "sortOrder",
"columnName": "sortOrder",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "specialUse",
"columnName": "specialUse",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "0"
},
{
"fieldPath": "hierarchyDelimiter",
"columnName": "hierarchyDelimiter",
"affinity": "TEXT"
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"accountId",
"fullName"
]
}
},
{
"tableName": "backfill_progress",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`accountId` TEXT NOT NULL, `folder` TEXT NOT NULL, `nextBeforeUid` INTEGER NOT NULL, `complete` INTEGER NOT NULL, PRIMARY KEY(`accountId`, `folder`))",
"fields": [
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "folder",
"columnName": "folder",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "nextBeforeUid",
"columnName": "nextBeforeUid",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "complete",
"columnName": "complete",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"accountId",
"folder"
]
}
}
],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '8264768635364869a347064a0864df9c')"
]
}
}
app/src/androidTest/kotlin/org/libremail/data/local/MigrationTest.kt
+66
View File
@@ -63,6 +63,56 @@ class MigrationTest {
}
}
/**
* v19 -> v20 (issue #187): the unified-inbox covering index appears over exactly
* `(folder, inInbox, timestampMillis)`, cached rows survive, and the paged "All inboxes" query now
* plans as a bounded `SEARCH` on that index with no temp B-tree sort (instead of a whole-table
* `SCAN`). Asserting the query plan on the real Android SQLite proves the index is genuinely
* covering the filter+order, not merely present.
*/
@Test
fun migrate19To20_addsUnifiedInboxCoveringIndexUsedByTheSummaryScan() {
helper.createDatabase(TEST_DB, 19).apply {
// A representative spread: two synced INBOX rows, plus a transient search hit (inInbox = 0)
// — all must survive the pure additive index migration. Fold columns default to ''.
execSQL(
"INSERT INTO messages (id, accountId, sender, senderEmail, subject, snippet, body, isHtml, " +
"timestampMillis, isRead, isStarred, folder, inInbox, bodyFetched, uid) VALUES " +
"('a:INBOX:2', 'a', 'Ada', 'ada@example.org', 'Hi', '', '', 0, 2000, 0, 0, 'INBOX', 1, 1, 2), " +
"('b:INBOX:1', 'b', 'Bob', 'bob@example.org', 'Yo', '', '', 0, 1000, 0, 0, 'INBOX', 1, 1, 1), " +
"('a:INBOX:9', 'a', 'Cy', 'cy@example.org', 'Q', '', '', 0, 3000, 0, 0, 'INBOX', 0, 0, 9)",
)
close()
}
val db = helper.runMigrationsAndValidate(TEST_DB, 20, true, MIGRATION_19_20)
// The index exists over exactly (folder, inInbox, timestampMillis), in that order.
assertEquals(
"19->20 must create the (folder, inInbox, timestampMillis) unified-inbox covering index",
listOf("folder", "inInbox", "timestampMillis"),
db.indexColumns("index_messages_folder_inInbox_timestampMillis"),
)
// The cached rows are untouched by the additive migration.
assertEquals("19->20 must not touch the mail cache", 3, db.count("messages"))
// The production pagingUnifiedFolderSummaries query now SEARCHes the new index and drops the
// temp B-tree sort (before this index it SCANned index_messages_timestampMillis whole-table).
val plan = db.queryPlan(
"SELECT id, accountId, sender, senderEmail, subject, snippet, timestampMillis, isRead, " +
"isStarred, folder, inInbox, bodyFetched FROM messages " +
"WHERE folder = 'INBOX' AND inInbox = 1 ORDER BY timestampMillis DESC",
)
assertTrue(
"the unified-inbox summary query must SEARCH the covering index, not SCAN; plan was $plan",
plan.any { it.contains("SEARCH") && it.contains("index_messages_folder_inInbox_timestampMillis") },
)
assertTrue(
"the covering index must supply the ordering (no temp B-tree sort); plan was $plan",
plan.none { it.contains("TEMP B-TREE") },
)
db.close()
}
/** v11 -> v12 (PR #54): `folders.specialUse` appears defaulting to 0 and existing data survives. */
@Test
fun migrate11To12_defaultsExistingFoldersToNotSpecialUse() {
@@ -558,6 +608,22 @@ class MigrationTest {
c.getInt(0)
}
/** Column names of [index], in index (seqno) order — empty if the index does not exist. */
private fun SupportSQLiteDatabase.indexColumns(index: String): List<String> =
query("PRAGMA index_info(`$index`)").use { c ->
buildList {
// PRAGMA index_info rows are (seqno, cid, name); the cursor yields them in seqno order.
while (c.moveToNext()) add(c.getString(2))
}
}
/** The human-readable `detail` step of each `EXPLAIN QUERY PLAN [sql]` row (the last column). */
private fun SupportSQLiteDatabase.queryPlan(sql: String): List<String> = query("EXPLAIN QUERY PLAN $sql").use { c ->
buildList {
while (c.moveToNext()) add(c.getString(c.columnCount - 1))
}
}
private companion object {
const val TEST_DB = "migration-test.db"
app/src/main/kotlin/org/libremail/data/local/LibreMailDatabase.kt
+1 -1
View File
@@ -35,7 +35,7 @@ import org.libremail.data.local.entity.OutboxEntity
FolderEntity::class,
BackfillProgressEntity::class,
],
version = 19,
version = 20,
exportSchema = true,
)
abstract class LibreMailDatabase : RoomDatabase() {
app/src/main/kotlin/org/libremail/data/local/Migrations.kt
+22
View File
@@ -380,3 +380,25 @@ val MIGRATION_18_19 = object : Migration(18, 19) {
)
}
}
/**
* v19 -> v20: covering index for the unified-inbox summary scan (issue #187; preserves existing data
* — a pure additive index, no column/table change or data transformation). The paged "All inboxes"
* query [org.libremail.data.local.dao.MessageDao.pagingUnifiedFolderSummaries] filters
* `folder = ? AND inInbox = 1 ORDER BY timestampMillis DESC`, but no index led with `folder`, so the
* planner walked the whole table via `index_messages_timestampMillis` in timestamp order and filtered
* `folder`/`inInbox` per row (a full `SCAN`, verified via `EXPLAIN QUERY PLAN`). The
* `(folder, inInbox, timestampMillis)` index turns the two equality predicates into an index seek and
* supplies the `timestampMillis` ordering, so the scan becomes a bounded `SEARCH … USING INDEX
* index_messages_folder_inInbox_timestampMillis (folder=? AND inInbox=?)` with no temp B-tree sort.
* `CREATE INDEX IF NOT EXISTS` is idempotent, and the name/columns match the Room `@Index` on
* [org.libremail.data.local.entity.MessageEntity] so the migrated schema validates against 20.json.
*/
val MIGRATION_19_20 = object : Migration(19, 20) {
override fun migrate(db: SupportSQLiteDatabase) {
db.execSQL(
"CREATE INDEX IF NOT EXISTS `index_messages_folder_inInbox_timestampMillis` " +
"ON `messages` (`folder`, `inInbox`, `timestampMillis`)",
)
}
}
app/src/main/kotlin/org/libremail/data/local/entity/MessageEntity.kt
+13 -1
View File
@@ -11,7 +11,19 @@ import androidx.room.PrimaryKey
// The (accountId, folder, uid) index serves the folder-scoped UID probes the backfill/reconcile
// hot paths run on every page/sync: MIN(uid) (lowestSyncedUid) and the uid >= window bound
// (deleteSyncedInWindowNotIn / syncedIdsBeyondCountInFolder).
indices = [Index("accountId"), Index("timestampMillis"), Index("accountId", "folder", "uid")],
//
// The (folder, inInbox, timestampMillis) index serves the unified-inbox summary scan (issue #187):
// MessageDao.pagingUnifiedFolderSummaries filters `folder = ? AND inInbox = 1 ORDER BY
// timestampMillis DESC` with no folder-leading index, so it SCANned the whole table via
// index_messages_timestampMillis and filtered per row. This index makes the two equalities an
// index seek and supplies the timestampMillis ordering, turning the SCAN into a bounded SEARCH
// with no temp B-tree sort (verified via EXPLAIN QUERY PLAN).
indices = [
Index("accountId"),
Index("timestampMillis"),
Index("accountId", "folder", "uid"),
Index("folder", "inInbox", "timestampMillis"),
],
)
data class MessageEntity(
@PrimaryKey val id: String,
app/src/main/kotlin/org/libremail/di/DatabaseModule.kt
+2
View File
@@ -25,6 +25,7 @@ import org.libremail.data.local.MIGRATION_15_16
import org.libremail.data.local.MIGRATION_16_17
import org.libremail.data.local.MIGRATION_17_18
import org.libremail.data.local.MIGRATION_18_19
import org.libremail.data.local.MIGRATION_19_20
import org.libremail.data.local.MIGRATION_1_2
import org.libremail.data.local.MIGRATION_2_3
import org.libremail.data.local.MIGRATION_3_4
@@ -69,6 +70,7 @@ object DatabaseModule {
MIGRATION_16_17,
MIGRATION_17_18,
MIGRATION_18_19,
MIGRATION_19_20,
)
// No destructive fallback: the migration chain is complete, and silently dropping the
// mail/message tables would lose cached data. A missing migration should fail loudly in