diff --git a/app/schemas/org.libremail.data.local.LibreMailDatabase/20.json b/app/schemas/org.libremail.data.local.LibreMailDatabase/20.json new file mode 100644 index 0000000..965b494 --- /dev/null +++ b/app/schemas/org.libremail.data.local.LibreMailDatabase/20.json @@ -0,0 +1,506 @@ +{ + "formatVersion": 1, + "database": { + "version": 20, + "identityHash": "8264768635364869a347064a0864df9c", + "entities": [ + { + "tableName": "messages", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT NOT NULL, `sender` TEXT NOT NULL, `senderEmail` TEXT NOT NULL, `subject` TEXT NOT NULL, `snippet` TEXT NOT NULL, `body` TEXT NOT NULL, `isHtml` INTEGER NOT NULL, `timestampMillis` INTEGER NOT NULL, `isRead` INTEGER NOT NULL, `isStarred` INTEGER NOT NULL, `folder` TEXT NOT NULL DEFAULT 'INBOX', `inInbox` INTEGER NOT NULL, `bodyFetched` INTEGER NOT NULL, `uid` INTEGER NOT NULL DEFAULT 0, `senderFold` TEXT NOT NULL DEFAULT '', `senderEmailFold` TEXT NOT NULL DEFAULT '', `subjectFold` TEXT NOT NULL DEFAULT '', `snippetFold` TEXT NOT NULL DEFAULT '', PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sender", + "columnName": "sender", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "senderEmail", + "columnName": "senderEmail", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "subject", + "columnName": "subject", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "snippet", + "columnName": "snippet", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "body", + "columnName": "body", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isHtml", + "columnName": "isHtml", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "timestampMillis", + "columnName": "timestampMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isRead", + "columnName": "isRead", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isStarred", + "columnName": "isStarred", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "folder", + "columnName": "folder", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'INBOX'" + }, + { + "fieldPath": "inInbox", + "columnName": "inInbox", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "bodyFetched", + "columnName": "bodyFetched", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "senderFold", + "columnName": "senderFold", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "senderEmailFold", + "columnName": "senderEmailFold", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "subjectFold", + "columnName": "subjectFold", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "snippetFold", + "columnName": "snippetFold", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_messages_accountId", + "unique": false, + "columnNames": [ + "accountId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_messages_accountId` ON `${TABLE_NAME}` (`accountId`)" + }, + { + "name": "index_messages_timestampMillis", + "unique": false, + "columnNames": [ + "timestampMillis" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_messages_timestampMillis` ON `${TABLE_NAME}` (`timestampMillis`)" + }, + { + "name": "index_messages_accountId_folder_uid", + "unique": false, + "columnNames": [ + "accountId", + "folder", + "uid" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_messages_accountId_folder_uid` ON `${TABLE_NAME}` (`accountId`, `folder`, `uid`)" + }, + { + "name": "index_messages_folder_inInbox_timestampMillis", + "unique": false, + "columnNames": [ + "folder", + "inInbox", + "timestampMillis" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_messages_folder_inInbox_timestampMillis` ON `${TABLE_NAME}` (`folder`, `inInbox`, `timestampMillis`)" + } + ] + }, + { + "tableName": "attachments", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`messageId` TEXT NOT NULL, `partIndex` INTEGER NOT NULL, `filename` TEXT NOT NULL, `mimeType` TEXT NOT NULL, `sizeBytes` INTEGER NOT NULL, `contentId` TEXT, PRIMARY KEY(`messageId`, `partIndex`), FOREIGN KEY(`messageId`) REFERENCES `messages`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "messageId", + "columnName": "messageId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "partIndex", + "columnName": "partIndex", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "filename", + "columnName": "filename", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "mimeType", + "columnName": "mimeType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sizeBytes", + "columnName": "sizeBytes", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "contentId", + "columnName": "contentId", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "messageId", + "partIndex" + ] + }, + "indices": [ + { + "name": "index_attachments_messageId", + "unique": false, + "columnNames": [ + "messageId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_attachments_messageId` ON `${TABLE_NAME}` (`messageId`)" + } + ], + "foreignKeys": [ + { + "table": "messages", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "messageId" + ], + "referencedColumns": [ + "id" + ] + } + ] + }, + { + "tableName": "outbox", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT NOT NULL, `toAddresses` TEXT NOT NULL, `ccAddresses` TEXT NOT NULL, `bccAddresses` TEXT NOT NULL DEFAULT '', `subject` TEXT NOT NULL, `body` TEXT NOT NULL, `createdAt` INTEGER NOT NULL, `lastError` TEXT, `bodyHtml` TEXT, `attachments` TEXT NOT NULL DEFAULT '', PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "toAddresses", + "columnName": "toAddresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ccAddresses", + "columnName": "ccAddresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "bccAddresses", + "columnName": "bccAddresses", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "subject", + "columnName": "subject", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "body", + "columnName": "body", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastError", + "columnName": "lastError", + "affinity": "TEXT" + }, + { + "fieldPath": "bodyHtml", + "columnName": "bodyHtml", + "affinity": "TEXT" + }, + { + "fieldPath": "attachments", + "columnName": "attachments", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "drafts", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT, `toAddresses` TEXT NOT NULL, `ccAddresses` TEXT NOT NULL, `bccAddresses` TEXT NOT NULL DEFAULT '', `subject` TEXT NOT NULL, `body` TEXT NOT NULL, `updatedAt` INTEGER NOT NULL, `attachments` TEXT NOT NULL, `bodyHtml` TEXT, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT" + }, + { + "fieldPath": "toAddresses", + "columnName": "toAddresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ccAddresses", + "columnName": "ccAddresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "bccAddresses", + "columnName": "bccAddresses", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "subject", + "columnName": "subject", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "body", + "columnName": "body", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "updatedAt", + "columnName": "updatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "attachments", + "columnName": "attachments", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "bodyHtml", + "columnName": "bodyHtml", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "folders", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`accountId` TEXT NOT NULL, `fullName` TEXT NOT NULL, `displayName` TEXT NOT NULL, `role` TEXT NOT NULL, `selectable` INTEGER NOT NULL, `sortOrder` INTEGER NOT NULL, `specialUse` INTEGER NOT NULL DEFAULT 0, `hierarchyDelimiter` TEXT, PRIMARY KEY(`accountId`, `fullName`))", + "fields": [ + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fullName", + "columnName": "fullName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "displayName", + "columnName": "displayName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "role", + "columnName": "role", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "selectable", + "columnName": "selectable", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "sortOrder", + "columnName": "sortOrder", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "specialUse", + "columnName": "specialUse", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + }, + { + "fieldPath": "hierarchyDelimiter", + "columnName": "hierarchyDelimiter", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "accountId", + "fullName" + ] + } + }, + { + "tableName": "backfill_progress", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`accountId` TEXT NOT NULL, `folder` TEXT NOT NULL, `nextBeforeUid` INTEGER NOT NULL, `complete` INTEGER NOT NULL, PRIMARY KEY(`accountId`, `folder`))", + "fields": [ + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "folder", + "columnName": "folder", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "nextBeforeUid", + "columnName": "nextBeforeUid", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "complete", + "columnName": "complete", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "accountId", + "folder" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '8264768635364869a347064a0864df9c')" + ] + } +} \ No newline at end of file diff --git a/app/src/androidTest/kotlin/org/libremail/data/local/DatabaseEncryptionTest.kt b/app/src/androidTest/kotlin/org/libremail/data/local/DatabaseEncryptionTest.kt index a4b8c30..17876f2 100644 --- a/app/src/androidTest/kotlin/org/libremail/data/local/DatabaseEncryptionTest.kt +++ b/app/src/androidTest/kotlin/org/libremail/data/local/DatabaseEncryptionTest.kt @@ -149,7 +149,7 @@ class DatabaseEncryptionTest { } finally { encrypted.close() } - assertEquals("Room's schema version must survive the plaintext -> encrypted conversion", 19, version) + assertEquals("Room's schema version must survive the plaintext -> encrypted conversion", 20, version) } @Test diff --git a/app/src/androidTest/kotlin/org/libremail/data/local/MigrationTest.kt b/app/src/androidTest/kotlin/org/libremail/data/local/MigrationTest.kt index 95803c7..5b15b78 100644 --- a/app/src/androidTest/kotlin/org/libremail/data/local/MigrationTest.kt +++ b/app/src/androidTest/kotlin/org/libremail/data/local/MigrationTest.kt @@ -64,6 +64,56 @@ class MigrationTest { } } + /** + * v19 -> v20 (issue #187): the unified-inbox covering index appears over exactly + * `(folder, inInbox, timestampMillis)`, cached rows survive, and the paged "All inboxes" query now + * plans as a bounded `SEARCH` on that index with no temp B-tree sort (instead of a whole-table + * `SCAN`). Asserting the query plan on the real Android SQLite proves the index is genuinely + * covering the filter+order, not merely present. + */ + @Test + fun migrate19To20_addsUnifiedInboxCoveringIndexUsedByTheSummaryScan() { + helper.createDatabase(TEST_DB, 19).apply { + // A representative spread: two synced INBOX rows, plus a transient search hit (inInbox = 0) + // — all must survive the pure additive index migration. Fold columns default to ''. + execSQL( + "INSERT INTO messages (id, accountId, sender, senderEmail, subject, snippet, body, isHtml, " + + "timestampMillis, isRead, isStarred, folder, inInbox, bodyFetched, uid) VALUES " + + "('a:INBOX:2', 'a', 'Ada', 'ada@example.org', 'Hi', '', '', 0, 2000, 0, 0, 'INBOX', 1, 1, 2), " + + "('b:INBOX:1', 'b', 'Bob', 'bob@example.org', 'Yo', '', '', 0, 1000, 0, 0, 'INBOX', 1, 1, 1), " + + "('a:INBOX:9', 'a', 'Cy', 'cy@example.org', 'Q', '', '', 0, 3000, 0, 0, 'INBOX', 0, 0, 9)", + ) + close() + } + + val db = helper.runMigrationsAndValidate(TEST_DB, 20, true, MIGRATION_19_20) + + // The index exists over exactly (folder, inInbox, timestampMillis), in that order. + assertEquals( + "19->20 must create the (folder, inInbox, timestampMillis) unified-inbox covering index", + listOf("folder", "inInbox", "timestampMillis"), + db.indexColumns("index_messages_folder_inInbox_timestampMillis"), + ) + // The cached rows are untouched by the additive migration. + assertEquals("19->20 must not touch the mail cache", 3, db.count("messages")) + // The production pagingUnifiedFolderSummaries query now SEARCHes the new index and drops the + // temp B-tree sort (before this index it SCANned index_messages_timestampMillis whole-table). + val plan = db.queryPlan( + "SELECT id, accountId, sender, senderEmail, subject, snippet, timestampMillis, isRead, " + + "isStarred, folder, inInbox, bodyFetched FROM messages " + + "WHERE folder = 'INBOX' AND inInbox = 1 ORDER BY timestampMillis DESC", + ) + assertTrue( + "the unified-inbox summary query must SEARCH the covering index, not SCAN; plan was $plan", + plan.any { it.contains("SEARCH") && it.contains("index_messages_folder_inInbox_timestampMillis") }, + ) + assertTrue( + "the covering index must supply the ordering (no temp B-tree sort); plan was $plan", + plan.none { it.contains("TEMP B-TREE") }, + ) + db.close() + } + /** v11 -> v12 (PR #54): `folders.specialUse` appears defaulting to 0 and existing data survives. */ @Test fun migrate11To12_defaultsExistingFoldersToNotSpecialUse() { @@ -576,6 +626,22 @@ class MigrationTest { c.getInt(0) } + /** Column names of [index], in index (seqno) order — empty if the index does not exist. */ + private fun SupportSQLiteDatabase.indexColumns(index: String): List = + query("PRAGMA index_info(`$index`)").use { c -> + buildList { + // PRAGMA index_info rows are (seqno, cid, name); the cursor yields them in seqno order. + while (c.moveToNext()) add(c.getString(2)) + } + } + + /** The human-readable `detail` step of each `EXPLAIN QUERY PLAN [sql]` row (the last column). */ + private fun SupportSQLiteDatabase.queryPlan(sql: String): List = query("EXPLAIN QUERY PLAN $sql").use { c -> + buildList { + while (c.moveToNext()) add(c.getString(c.columnCount - 1)) + } + } + private companion object { const val TEST_DB = "migration-test.db" diff --git a/app/src/main/kotlin/org/libremail/data/local/LibreMailDatabase.kt b/app/src/main/kotlin/org/libremail/data/local/LibreMailDatabase.kt index 8354a5d..5267905 100644 --- a/app/src/main/kotlin/org/libremail/data/local/LibreMailDatabase.kt +++ b/app/src/main/kotlin/org/libremail/data/local/LibreMailDatabase.kt @@ -35,7 +35,7 @@ import org.libremail.data.local.entity.OutboxEntity FolderEntity::class, BackfillProgressEntity::class, ], - version = 19, + version = 20, exportSchema = true, ) abstract class LibreMailDatabase : RoomDatabase() { diff --git a/app/src/main/kotlin/org/libremail/data/local/Migrations.kt b/app/src/main/kotlin/org/libremail/data/local/Migrations.kt index 5b7d62c..7eacbf6 100644 --- a/app/src/main/kotlin/org/libremail/data/local/Migrations.kt +++ b/app/src/main/kotlin/org/libremail/data/local/Migrations.kt @@ -380,3 +380,25 @@ val MIGRATION_18_19 = object : Migration(18, 19) { ) } } + +/** + * v19 -> v20: covering index for the unified-inbox summary scan (issue #187; preserves existing data + * — a pure additive index, no column/table change or data transformation). The paged "All inboxes" + * query [org.libremail.data.local.dao.MessageDao.pagingUnifiedFolderSummaries] filters + * `folder = ? AND inInbox = 1 ORDER BY timestampMillis DESC`, but no index led with `folder`, so the + * planner walked the whole table via `index_messages_timestampMillis` in timestamp order and filtered + * `folder`/`inInbox` per row (a full `SCAN`, verified via `EXPLAIN QUERY PLAN`). The + * `(folder, inInbox, timestampMillis)` index turns the two equality predicates into an index seek and + * supplies the `timestampMillis` ordering, so the scan becomes a bounded `SEARCH … USING INDEX + * index_messages_folder_inInbox_timestampMillis (folder=? AND inInbox=?)` with no temp B-tree sort. + * `CREATE INDEX IF NOT EXISTS` is idempotent, and the name/columns match the Room `@Index` on + * [org.libremail.data.local.entity.MessageEntity] so the migrated schema validates against 20.json. + */ +val MIGRATION_19_20 = object : Migration(19, 20) { + override fun migrate(db: SupportSQLiteDatabase) { + db.execSQL( + "CREATE INDEX IF NOT EXISTS `index_messages_folder_inInbox_timestampMillis` " + + "ON `messages` (`folder`, `inInbox`, `timestampMillis`)", + ) + } +} diff --git a/app/src/main/kotlin/org/libremail/data/local/entity/MessageEntity.kt b/app/src/main/kotlin/org/libremail/data/local/entity/MessageEntity.kt index 97b0fae..6119060 100644 --- a/app/src/main/kotlin/org/libremail/data/local/entity/MessageEntity.kt +++ b/app/src/main/kotlin/org/libremail/data/local/entity/MessageEntity.kt @@ -11,7 +11,19 @@ import androidx.room.PrimaryKey // The (accountId, folder, uid) index serves the folder-scoped UID probes the backfill/reconcile // hot paths run on every page/sync: MIN(uid) (lowestSyncedUid) and the uid >= window bound // (deleteSyncedInWindowNotIn / syncedIdsBeyondCountInFolder). - indices = [Index("accountId"), Index("timestampMillis"), Index("accountId", "folder", "uid")], + // + // The (folder, inInbox, timestampMillis) index serves the unified-inbox summary scan (issue #187): + // MessageDao.pagingUnifiedFolderSummaries filters `folder = ? AND inInbox = 1 ORDER BY + // timestampMillis DESC` with no folder-leading index, so it SCANned the whole table via + // index_messages_timestampMillis and filtered per row. This index makes the two equalities an + // index seek and supplies the timestampMillis ordering, turning the SCAN into a bounded SEARCH + // with no temp B-tree sort (verified via EXPLAIN QUERY PLAN). + indices = [ + Index("accountId"), + Index("timestampMillis"), + Index("accountId", "folder", "uid"), + Index("folder", "inInbox", "timestampMillis"), + ], ) data class MessageEntity( @PrimaryKey val id: String, diff --git a/app/src/main/kotlin/org/libremail/di/DatabaseModule.kt b/app/src/main/kotlin/org/libremail/di/DatabaseModule.kt index 367ae82..e852240 100644 --- a/app/src/main/kotlin/org/libremail/di/DatabaseModule.kt +++ b/app/src/main/kotlin/org/libremail/di/DatabaseModule.kt @@ -26,6 +26,7 @@ import org.libremail.data.local.MIGRATION_15_16 import org.libremail.data.local.MIGRATION_16_17 import org.libremail.data.local.MIGRATION_17_18 import org.libremail.data.local.MIGRATION_18_19 +import org.libremail.data.local.MIGRATION_19_20 import org.libremail.data.local.MIGRATION_1_2 import org.libremail.data.local.MIGRATION_2_3 import org.libremail.data.local.MIGRATION_3_4 @@ -73,6 +74,7 @@ object DatabaseModule { MIGRATION_16_17, MIGRATION_17_18, MIGRATION_18_19, + MIGRATION_19_20, ) @Provides