JMR-dev/LibreMail
1
0
Fork 0
Code Issues 40 Pull Requests 2 Actions Packages Projects Releases 2 Wiki Activity

Merge pull request #285 from JMR-dev/perf-187-covering-index

Browse Source 
perf(db): covering index for unified-inbox summary scans
This commit was merged in pull request #285.
This commit is contained in:
Jason Ross
2026-07-05 14:31:22 -05:00
committed by GitHub
parent c3933a3612 1cd0d66fa4
commit 6118b6ded0
7 changed files with 611 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/schemas/org.libremail.data.local.LibreMailDatabase/20.json
+506
View File
@@ -0,0 +1,506 @@
{
"formatVersion": 1,
"database": {
"version": 20,
"identityHash": "8264768635364869a347064a0864df9c",
"entities": [
{
"tableName": "messages",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT NOT NULL, `sender` TEXT NOT NULL, `senderEmail` TEXT NOT NULL, `subject` TEXT NOT NULL, `snippet` TEXT NOT NULL, `body` TEXT NOT NULL, `isHtml` INTEGER NOT NULL, `timestampMillis` INTEGER NOT NULL, `isRead` INTEGER NOT NULL, `isStarred` INTEGER NOT NULL, `folder` TEXT NOT NULL DEFAULT 'INBOX', `inInbox` INTEGER NOT NULL, `bodyFetched` INTEGER NOT NULL, `uid` INTEGER NOT NULL DEFAULT 0, `senderFold` TEXT NOT NULL DEFAULT '', `senderEmailFold` TEXT NOT NULL DEFAULT '', `subjectFold` TEXT NOT NULL DEFAULT '', `snippetFold` TEXT NOT NULL DEFAULT '', PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "sender",
"columnName": "sender",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "senderEmail",
"columnName": "senderEmail",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "subject",
"columnName": "subject",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "snippet",
"columnName": "snippet",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "body",
"columnName": "body",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "isHtml",
"columnName": "isHtml",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "timestampMillis",
"columnName": "timestampMillis",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isRead",
"columnName": "isRead",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "isStarred",
"columnName": "isStarred",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "folder",
"columnName": "folder",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "'INBOX'"
},
{
"fieldPath": "inInbox",
"columnName": "inInbox",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "bodyFetched",
"columnName": "bodyFetched",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "uid",
"columnName": "uid",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "0"
},
{
"fieldPath": "senderFold",
"columnName": "senderFold",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "senderEmailFold",
"columnName": "senderEmailFold",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "subjectFold",
"columnName": "subjectFold",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "snippetFold",
"columnName": "snippetFold",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
},
"indices": [
{
"name": "index_messages_accountId",
"unique": false,
"columnNames": [
"accountId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_messages_accountId` ON `${TABLE_NAME}` (`accountId`)"
},
{
"name": "index_messages_timestampMillis",
"unique": false,
"columnNames": [
"timestampMillis"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_messages_timestampMillis` ON `${TABLE_NAME}` (`timestampMillis`)"
},
{
"name": "index_messages_accountId_folder_uid",
"unique": false,
"columnNames": [
"accountId",
"folder",
"uid"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_messages_accountId_folder_uid` ON `${TABLE_NAME}` (`accountId`, `folder`, `uid`)"
},
{
"name": "index_messages_folder_inInbox_timestampMillis",
"unique": false,
"columnNames": [
"folder",
"inInbox",
"timestampMillis"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_messages_folder_inInbox_timestampMillis` ON `${TABLE_NAME}` (`folder`, `inInbox`, `timestampMillis`)"
}
]
},
{
"tableName": "attachments",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`messageId` TEXT NOT NULL, `partIndex` INTEGER NOT NULL, `filename` TEXT NOT NULL, `mimeType` TEXT NOT NULL, `sizeBytes` INTEGER NOT NULL, `contentId` TEXT, PRIMARY KEY(`messageId`, `partIndex`), FOREIGN KEY(`messageId`) REFERENCES `messages`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
"fields": [
{
"fieldPath": "messageId",
"columnName": "messageId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "partIndex",
"columnName": "partIndex",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "filename",
"columnName": "filename",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "mimeType",
"columnName": "mimeType",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "sizeBytes",
"columnName": "sizeBytes",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "contentId",
"columnName": "contentId",
"affinity": "TEXT"
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"messageId",
"partIndex"
]
},
"indices": [
{
"name": "index_attachments_messageId",
"unique": false,
"columnNames": [
"messageId"
],
"orders": [],
"createSql": "CREATE INDEX IF NOT EXISTS `index_attachments_messageId` ON `${TABLE_NAME}` (`messageId`)"
}
],
"foreignKeys": [
{
"table": "messages",
"onDelete": "CASCADE",
"onUpdate": "NO ACTION",
"columns": [
"messageId"
],
"referencedColumns": [
"id"
]
}
]
},
{
"tableName": "outbox",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT NOT NULL, `toAddresses` TEXT NOT NULL, `ccAddresses` TEXT NOT NULL, `bccAddresses` TEXT NOT NULL DEFAULT '', `subject` TEXT NOT NULL, `body` TEXT NOT NULL, `createdAt` INTEGER NOT NULL, `lastError` TEXT, `bodyHtml` TEXT, `attachments` TEXT NOT NULL DEFAULT '', PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "toAddresses",
"columnName": "toAddresses",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "ccAddresses",
"columnName": "ccAddresses",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "bccAddresses",
"columnName": "bccAddresses",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "subject",
"columnName": "subject",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "body",
"columnName": "body",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "createdAt",
"columnName": "createdAt",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "lastError",
"columnName": "lastError",
"affinity": "TEXT"
},
{
"fieldPath": "bodyHtml",
"columnName": "bodyHtml",
"affinity": "TEXT"
},
{
"fieldPath": "attachments",
"columnName": "attachments",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
}
},
{
"tableName": "drafts",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT, `toAddresses` TEXT NOT NULL, `ccAddresses` TEXT NOT NULL, `bccAddresses` TEXT NOT NULL DEFAULT '', `subject` TEXT NOT NULL, `body` TEXT NOT NULL, `updatedAt` INTEGER NOT NULL, `attachments` TEXT NOT NULL, `bodyHtml` TEXT, PRIMARY KEY(`id`))",
"fields": [
{
"fieldPath": "id",
"columnName": "id",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "TEXT"
},
{
"fieldPath": "toAddresses",
"columnName": "toAddresses",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "ccAddresses",
"columnName": "ccAddresses",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "bccAddresses",
"columnName": "bccAddresses",
"affinity": "TEXT",
"notNull": true,
"defaultValue": "''"
},
{
"fieldPath": "subject",
"columnName": "subject",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "body",
"columnName": "body",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "updatedAt",
"columnName": "updatedAt",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "attachments",
"columnName": "attachments",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "bodyHtml",
"columnName": "bodyHtml",
"affinity": "TEXT"
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"id"
]
}
},
{
"tableName": "folders",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`accountId` TEXT NOT NULL, `fullName` TEXT NOT NULL, `displayName` TEXT NOT NULL, `role` TEXT NOT NULL, `selectable` INTEGER NOT NULL, `sortOrder` INTEGER NOT NULL, `specialUse` INTEGER NOT NULL DEFAULT 0, `hierarchyDelimiter` TEXT, PRIMARY KEY(`accountId`, `fullName`))",
"fields": [
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "fullName",
"columnName": "fullName",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "displayName",
"columnName": "displayName",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "role",
"columnName": "role",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "selectable",
"columnName": "selectable",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "sortOrder",
"columnName": "sortOrder",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "specialUse",
"columnName": "specialUse",
"affinity": "INTEGER",
"notNull": true,
"defaultValue": "0"
},
{
"fieldPath": "hierarchyDelimiter",
"columnName": "hierarchyDelimiter",
"affinity": "TEXT"
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"accountId",
"fullName"
]
}
},
{
"tableName": "backfill_progress",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`accountId` TEXT NOT NULL, `folder` TEXT NOT NULL, `nextBeforeUid` INTEGER NOT NULL, `complete` INTEGER NOT NULL, PRIMARY KEY(`accountId`, `folder`))",
"fields": [
{
"fieldPath": "accountId",
"columnName": "accountId",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "folder",
"columnName": "folder",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "nextBeforeUid",
"columnName": "nextBeforeUid",
"affinity": "INTEGER",
"notNull": true
},
{
"fieldPath": "complete",
"columnName": "complete",
"affinity": "INTEGER",
"notNull": true
}
],
"primaryKey": {
"autoGenerate": false,
"columnNames": [
"accountId",
"folder"
]
}
}
],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '8264768635364869a347064a0864df9c')"
]
}
}
app/src/androidTest/kotlin/org/libremail/data/local/DatabaseEncryptionTest.kt
+1 -1
View File
@@ -149,7 +149,7 @@ class DatabaseEncryptionTest {
} finally {
encrypted.close()
}
assertEquals("Room's schema version must survive the plaintext -> encrypted conversion", 19, version)
assertEquals("Room's schema version must survive the plaintext -> encrypted conversion", 20, version)
}
@Test
app/src/androidTest/kotlin/org/libremail/data/local/MigrationTest.kt
+66
View File
@@ -64,6 +64,56 @@ class MigrationTest {
}
}
/**
* v19 -> v20 (issue #187): the unified-inbox covering index appears over exactly
* `(folder, inInbox, timestampMillis)`, cached rows survive, and the paged "All inboxes" query now
* plans as a bounded `SEARCH` on that index with no temp B-tree sort (instead of a whole-table
* `SCAN`). Asserting the query plan on the real Android SQLite proves the index is genuinely
* covering the filter+order, not merely present.
*/
@Test
fun migrate19To20_addsUnifiedInboxCoveringIndexUsedByTheSummaryScan() {
helper.createDatabase(TEST_DB, 19).apply {
// A representative spread: two synced INBOX rows, plus a transient search hit (inInbox = 0)
// — all must survive the pure additive index migration. Fold columns default to ''.
execSQL(
"INSERT INTO messages (id, accountId, sender, senderEmail, subject, snippet, body, isHtml, " +
"timestampMillis, isRead, isStarred, folder, inInbox, bodyFetched, uid) VALUES " +
"('a:INBOX:2', 'a', 'Ada', 'ada@example.org', 'Hi', '', '', 0, 2000, 0, 0, 'INBOX', 1, 1, 2), " +
"('b:INBOX:1', 'b', 'Bob', 'bob@example.org', 'Yo', '', '', 0, 1000, 0, 0, 'INBOX', 1, 1, 1), " +
"('a:INBOX:9', 'a', 'Cy', 'cy@example.org', 'Q', '', '', 0, 3000, 0, 0, 'INBOX', 0, 0, 9)",
)
close()
}
val db = helper.runMigrationsAndValidate(TEST_DB, 20, true, MIGRATION_19_20)
// The index exists over exactly (folder, inInbox, timestampMillis), in that order.
assertEquals(
"19->20 must create the (folder, inInbox, timestampMillis) unified-inbox covering index",
listOf("folder", "inInbox", "timestampMillis"),
db.indexColumns("index_messages_folder_inInbox_timestampMillis"),
)
// The cached rows are untouched by the additive migration.
assertEquals("19->20 must not touch the mail cache", 3, db.count("messages"))
// The production pagingUnifiedFolderSummaries query now SEARCHes the new index and drops the
// temp B-tree sort (before this index it SCANned index_messages_timestampMillis whole-table).
val plan = db.queryPlan(
"SELECT id, accountId, sender, senderEmail, subject, snippet, timestampMillis, isRead, " +
"isStarred, folder, inInbox, bodyFetched FROM messages " +
"WHERE folder = 'INBOX' AND inInbox = 1 ORDER BY timestampMillis DESC",
)
assertTrue(
"the unified-inbox summary query must SEARCH the covering index, not SCAN; plan was $plan",
plan.any { it.contains("SEARCH") && it.contains("index_messages_folder_inInbox_timestampMillis") },
)
assertTrue(
"the covering index must supply the ordering (no temp B-tree sort); plan was $plan",
plan.none { it.contains("TEMP B-TREE") },
)
db.close()
}
/** v11 -> v12 (PR #54): `folders.specialUse` appears defaulting to 0 and existing data survives. */
@Test
fun migrate11To12_defaultsExistingFoldersToNotSpecialUse() {
@@ -576,6 +626,22 @@ class MigrationTest {
c.getInt(0)
}
/** Column names of [index], in index (seqno) order — empty if the index does not exist. */
private fun SupportSQLiteDatabase.indexColumns(index: String): List<String> =
query("PRAGMA index_info(`$index`)").use { c ->
buildList {
// PRAGMA index_info rows are (seqno, cid, name); the cursor yields them in seqno order.
while (c.moveToNext()) add(c.getString(2))
}
}
/** The human-readable `detail` step of each `EXPLAIN QUERY PLAN [sql]` row (the last column). */
private fun SupportSQLiteDatabase.queryPlan(sql: String): List<String> = query("EXPLAIN QUERY PLAN $sql").use { c ->
buildList {
while (c.moveToNext()) add(c.getString(c.columnCount - 1))
}
}
private companion object {
const val TEST_DB = "migration-test.db"
app/src/main/kotlin/org/libremail/data/local/LibreMailDatabase.kt
+1 -1
View File
@@ -35,7 +35,7 @@ import org.libremail.data.local.entity.OutboxEntity
FolderEntity::class,
BackfillProgressEntity::class,
],
version = 19,
version = 20,
exportSchema = true,
)
abstract class LibreMailDatabase : RoomDatabase() {
app/src/main/kotlin/org/libremail/data/local/Migrations.kt
+22
View File
@@ -380,3 +380,25 @@ val MIGRATION_18_19 = object : Migration(18, 19) {
)
}
}
/**
* v19 -> v20: covering index for the unified-inbox summary scan (issue #187; preserves existing data
* — a pure additive index, no column/table change or data transformation). The paged "All inboxes"
* query [org.libremail.data.local.dao.MessageDao.pagingUnifiedFolderSummaries] filters
* `folder = ? AND inInbox = 1 ORDER BY timestampMillis DESC`, but no index led with `folder`, so the
* planner walked the whole table via `index_messages_timestampMillis` in timestamp order and filtered
* `folder`/`inInbox` per row (a full `SCAN`, verified via `EXPLAIN QUERY PLAN`). The
* `(folder, inInbox, timestampMillis)` index turns the two equality predicates into an index seek and
* supplies the `timestampMillis` ordering, so the scan becomes a bounded `SEARCH … USING INDEX
* index_messages_folder_inInbox_timestampMillis (folder=? AND inInbox=?)` with no temp B-tree sort.
* `CREATE INDEX IF NOT EXISTS` is idempotent, and the name/columns match the Room `@Index` on
* [org.libremail.data.local.entity.MessageEntity] so the migrated schema validates against 20.json.
*/
val MIGRATION_19_20 = object : Migration(19, 20) {
override fun migrate(db: SupportSQLiteDatabase) {
db.execSQL(
"CREATE INDEX IF NOT EXISTS `index_messages_folder_inInbox_timestampMillis` " +
"ON `messages` (`folder`, `inInbox`, `timestampMillis`)",
)
}
}
app/src/main/kotlin/org/libremail/data/local/entity/MessageEntity.kt
+13 -1
View File
@@ -11,7 +11,19 @@ import androidx.room.PrimaryKey
// The (accountId, folder, uid) index serves the folder-scoped UID probes the backfill/reconcile
// hot paths run on every page/sync: MIN(uid) (lowestSyncedUid) and the uid >= window bound
// (deleteSyncedInWindowNotIn / syncedIdsBeyondCountInFolder).
indices = [Index("accountId"), Index("timestampMillis"), Index("accountId", "folder", "uid")],
//
// The (folder, inInbox, timestampMillis) index serves the unified-inbox summary scan (issue #187):
// MessageDao.pagingUnifiedFolderSummaries filters `folder = ? AND inInbox = 1 ORDER BY
// timestampMillis DESC` with no folder-leading index, so it SCANned the whole table via
// index_messages_timestampMillis and filtered per row. This index makes the two equalities an
// index seek and supplies the timestampMillis ordering, turning the SCAN into a bounded SEARCH
// with no temp B-tree sort (verified via EXPLAIN QUERY PLAN).
indices = [
Index("accountId"),
Index("timestampMillis"),
Index("accountId", "folder", "uid"),
Index("folder", "inInbox", "timestampMillis"),
],
)
data class MessageEntity(
@PrimaryKey val id: String,
app/src/main/kotlin/org/libremail/di/DatabaseModule.kt
+2
View File
@@ -26,6 +26,7 @@ import org.libremail.data.local.MIGRATION_15_16
import org.libremail.data.local.MIGRATION_16_17
import org.libremail.data.local.MIGRATION_17_18
import org.libremail.data.local.MIGRATION_18_19
import org.libremail.data.local.MIGRATION_19_20
import org.libremail.data.local.MIGRATION_1_2
import org.libremail.data.local.MIGRATION_2_3
import org.libremail.data.local.MIGRATION_3_4
@@ -73,6 +74,7 @@ object DatabaseModule {
MIGRATION_16_17,
MIGRATION_17_18,
MIGRATION_18_19,
MIGRATION_19_20,
)
@Provides