diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 72c7c2f..e2fab8e 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -109,6 +109,9 @@ android { includeInBundle = false } + // Ship the exported Room schemas as androidTest assets so MigrationTestHelper can load them. + sourceSets.getByName("androidTest").assets.srcDir("$projectDir/schemas") + packaging { resources { // Angus Mail / Jakarta Activation (added later) ship duplicate META-INF entries. @@ -150,7 +153,7 @@ android { } } -// Export Room schemas so migrations can be validated by instrumented MigrationTestHelper tests. +// Export Room schemas so the instrumented MigrationTest can replay and validate each migration. ksp { arg("room.schemaLocation", "$projectDir/schemas") } diff --git a/app/schemas/org.libremail.data.local.LibreMailDatabase/14.json b/app/schemas/org.libremail.data.local.LibreMailDatabase/14.json new file mode 100644 index 0000000..20e918a --- /dev/null +++ b/app/schemas/org.libremail.data.local.LibreMailDatabase/14.json @@ -0,0 +1,671 @@ +{ + "formatVersion": 1, + "database": { + "version": 14, + "identityHash": "e4f7ef1e0d780324d6eec3efced768ae", + "entities": [ + { + "tableName": "accounts", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `email` TEXT NOT NULL, `displayName` TEXT NOT NULL, `authType` TEXT NOT NULL, `imap_host` TEXT NOT NULL, `imap_port` INTEGER NOT NULL, `imap_security` TEXT NOT NULL, `smtp_host` TEXT NOT NULL, `smtp_port` INTEGER NOT NULL, `smtp_security` TEXT NOT NULL, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "email", + "columnName": "email", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "displayName", + "columnName": "displayName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "authType", + "columnName": "authType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "imap.host", + "columnName": "imap_host", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "imap.port", + "columnName": "imap_port", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "imap.security", + "columnName": "imap_security", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "smtp.host", + "columnName": "smtp_host", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "smtp.port", + "columnName": "smtp_port", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "smtp.security", + "columnName": "smtp_security", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "account_settings", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`accountId` TEXT NOT NULL, `signature` TEXT NOT NULL, `signatureEnabled` INTEGER NOT NULL, `notificationsEnabled` INTEGER NOT NULL, `retentionCount` INTEGER, `retentionMonths` INTEGER, PRIMARY KEY(`accountId`), FOREIGN KEY(`accountId`) REFERENCES `accounts`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "signature", + "columnName": "signature", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "signatureEnabled", + "columnName": "signatureEnabled", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "notificationsEnabled", + "columnName": "notificationsEnabled", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "retentionCount", + "columnName": "retentionCount", + "affinity": "INTEGER" + }, + { + "fieldPath": "retentionMonths", + "columnName": "retentionMonths", + "affinity": "INTEGER" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "accountId" + ] + }, + "foreignKeys": [ + { + "table": "accounts", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "accountId" + ], + "referencedColumns": [ + "id" + ] + } + ] + }, + { + "tableName": "messages", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT NOT NULL, `sender` TEXT NOT NULL, `senderEmail` TEXT NOT NULL, `subject` TEXT NOT NULL, `snippet` TEXT NOT NULL, `body` TEXT NOT NULL, `isHtml` INTEGER NOT NULL, `timestampMillis` INTEGER NOT NULL, `isRead` INTEGER NOT NULL, `isStarred` INTEGER NOT NULL, `folder` TEXT NOT NULL DEFAULT 'INBOX', `inInbox` INTEGER NOT NULL, `bodyFetched` INTEGER NOT NULL, `uid` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sender", + "columnName": "sender", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "senderEmail", + "columnName": "senderEmail", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "subject", + "columnName": "subject", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "snippet", + "columnName": "snippet", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "body", + "columnName": "body", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isHtml", + "columnName": "isHtml", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "timestampMillis", + "columnName": "timestampMillis", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isRead", + "columnName": "isRead", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "isStarred", + "columnName": "isStarred", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "folder", + "columnName": "folder", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "'INBOX'" + }, + { + "fieldPath": "inInbox", + "columnName": "inInbox", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "bodyFetched", + "columnName": "bodyFetched", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "uid", + "columnName": "uid", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_messages_accountId", + "unique": false, + "columnNames": [ + "accountId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_messages_accountId` ON `${TABLE_NAME}` (`accountId`)" + }, + { + "name": "index_messages_timestampMillis", + "unique": false, + "columnNames": [ + "timestampMillis" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_messages_timestampMillis` ON `${TABLE_NAME}` (`timestampMillis`)" + }, + { + "name": "index_messages_accountId_folder_uid", + "unique": false, + "columnNames": [ + "accountId", + "folder", + "uid" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_messages_accountId_folder_uid` ON `${TABLE_NAME}` (`accountId`, `folder`, `uid`)" + } + ] + }, + { + "tableName": "credentials", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`accountId` TEXT NOT NULL, `encryptedSecret` TEXT NOT NULL, PRIMARY KEY(`accountId`))", + "fields": [ + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "encryptedSecret", + "columnName": "encryptedSecret", + "affinity": "TEXT", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "accountId" + ] + } + }, + { + "tableName": "attachments", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`messageId` TEXT NOT NULL, `partIndex` INTEGER NOT NULL, `filename` TEXT NOT NULL, `mimeType` TEXT NOT NULL, `sizeBytes` INTEGER NOT NULL, PRIMARY KEY(`messageId`, `partIndex`), FOREIGN KEY(`messageId`) REFERENCES `messages`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "messageId", + "columnName": "messageId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "partIndex", + "columnName": "partIndex", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "filename", + "columnName": "filename", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "mimeType", + "columnName": "mimeType", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "sizeBytes", + "columnName": "sizeBytes", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "messageId", + "partIndex" + ] + }, + "indices": [ + { + "name": "index_attachments_messageId", + "unique": false, + "columnNames": [ + "messageId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_attachments_messageId` ON `${TABLE_NAME}` (`messageId`)" + } + ], + "foreignKeys": [ + { + "table": "messages", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "messageId" + ], + "referencedColumns": [ + "id" + ] + } + ] + }, + { + "tableName": "outbox", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT NOT NULL, `toAddresses` TEXT NOT NULL, `ccAddresses` TEXT NOT NULL, `bccAddresses` TEXT NOT NULL DEFAULT '', `subject` TEXT NOT NULL, `body` TEXT NOT NULL, `createdAt` INTEGER NOT NULL, `lastError` TEXT, `bodyHtml` TEXT, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "toAddresses", + "columnName": "toAddresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ccAddresses", + "columnName": "ccAddresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "bccAddresses", + "columnName": "bccAddresses", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "subject", + "columnName": "subject", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "body", + "columnName": "body", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "createdAt", + "columnName": "createdAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "lastError", + "columnName": "lastError", + "affinity": "TEXT" + }, + { + "fieldPath": "bodyHtml", + "columnName": "bodyHtml", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "drafts", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT, `toAddresses` TEXT NOT NULL, `ccAddresses` TEXT NOT NULL, `bccAddresses` TEXT NOT NULL DEFAULT '', `subject` TEXT NOT NULL, `body` TEXT NOT NULL, `updatedAt` INTEGER NOT NULL, `attachments` TEXT NOT NULL, `bodyHtml` TEXT, PRIMARY KEY(`id`))", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT" + }, + { + "fieldPath": "toAddresses", + "columnName": "toAddresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "ccAddresses", + "columnName": "ccAddresses", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "bccAddresses", + "columnName": "bccAddresses", + "affinity": "TEXT", + "notNull": true, + "defaultValue": "''" + }, + { + "fieldPath": "subject", + "columnName": "subject", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "body", + "columnName": "body", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "updatedAt", + "columnName": "updatedAt", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "attachments", + "columnName": "attachments", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "bodyHtml", + "columnName": "bodyHtml", + "affinity": "TEXT" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + } + }, + { + "tableName": "folders", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`accountId` TEXT NOT NULL, `fullName` TEXT NOT NULL, `displayName` TEXT NOT NULL, `role` TEXT NOT NULL, `selectable` INTEGER NOT NULL, `sortOrder` INTEGER NOT NULL, `specialUse` INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(`accountId`, `fullName`))", + "fields": [ + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "fullName", + "columnName": "fullName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "displayName", + "columnName": "displayName", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "role", + "columnName": "role", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "selectable", + "columnName": "selectable", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "sortOrder", + "columnName": "sortOrder", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "specialUse", + "columnName": "specialUse", + "affinity": "INTEGER", + "notNull": true, + "defaultValue": "0" + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "accountId", + "fullName" + ] + } + }, + { + "tableName": "signatures", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` TEXT NOT NULL, `accountId` TEXT NOT NULL, `name` TEXT NOT NULL, `contentHtml` TEXT NOT NULL, `isDefault` INTEGER NOT NULL, PRIMARY KEY(`id`), FOREIGN KEY(`accountId`) REFERENCES `accounts`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )", + "fields": [ + { + "fieldPath": "id", + "columnName": "id", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "name", + "columnName": "name", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "contentHtml", + "columnName": "contentHtml", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "isDefault", + "columnName": "isDefault", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "id" + ] + }, + "indices": [ + { + "name": "index_signatures_accountId", + "unique": false, + "columnNames": [ + "accountId" + ], + "orders": [], + "createSql": "CREATE INDEX IF NOT EXISTS `index_signatures_accountId` ON `${TABLE_NAME}` (`accountId`)" + } + ], + "foreignKeys": [ + { + "table": "accounts", + "onDelete": "CASCADE", + "onUpdate": "NO ACTION", + "columns": [ + "accountId" + ], + "referencedColumns": [ + "id" + ] + } + ] + }, + { + "tableName": "backfill_progress", + "createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`accountId` TEXT NOT NULL, `folder` TEXT NOT NULL, `nextBeforeUid` INTEGER NOT NULL, `complete` INTEGER NOT NULL, PRIMARY KEY(`accountId`, `folder`))", + "fields": [ + { + "fieldPath": "accountId", + "columnName": "accountId", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "folder", + "columnName": "folder", + "affinity": "TEXT", + "notNull": true + }, + { + "fieldPath": "nextBeforeUid", + "columnName": "nextBeforeUid", + "affinity": "INTEGER", + "notNull": true + }, + { + "fieldPath": "complete", + "columnName": "complete", + "affinity": "INTEGER", + "notNull": true + } + ], + "primaryKey": { + "autoGenerate": false, + "columnNames": [ + "accountId", + "folder" + ] + } + } + ], + "setupQueries": [ + "CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)", + "INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, 'e4f7ef1e0d780324d6eec3efced768ae')" + ] + } +} \ No newline at end of file diff --git a/app/src/androidTest/kotlin/org/libremail/data/local/LibreMailDatabaseTest.kt b/app/src/androidTest/kotlin/org/libremail/data/local/LibreMailDatabaseTest.kt index 909467e..16cbccf 100644 --- a/app/src/androidTest/kotlin/org/libremail/data/local/LibreMailDatabaseTest.kt +++ b/app/src/androidTest/kotlin/org/libremail/data/local/LibreMailDatabaseTest.kt @@ -22,9 +22,9 @@ import org.libremail.data.local.entity.MessageEntity import org.libremail.data.local.entity.ServerConfigEmbedded /** - * Schema-behavior tests on the real (v7) Room database. (Migrations from versions before - * exportSchema was enabled can't be replayed with MigrationTestHelper, since their schema JSONs - * were never exported; exportSchema is now on so future migrations can be tested.) + * Schema-behavior tests on a fresh in-memory database at the current version. The migration DDL + * itself is exercised by [MigrationTest], which replays the schema chain exported to app/schemas. + * (Migrations from before v7 predate schema export, so they can't be replayed there.) */ @RunWith(AndroidJUnit4::class) class LibreMailDatabaseTest { diff --git a/app/src/androidTest/kotlin/org/libremail/data/local/MigrationTest.kt b/app/src/androidTest/kotlin/org/libremail/data/local/MigrationTest.kt new file mode 100644 index 0000000..076e821 --- /dev/null +++ b/app/src/androidTest/kotlin/org/libremail/data/local/MigrationTest.kt @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.libremail.data.local + +import androidx.room.migration.Migration +import androidx.room.testing.MigrationTestHelper +import androidx.sqlite.db.SupportSQLiteDatabase +import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import java.lang.reflect.Modifier + +/** + * Replays the app's Room migrations against the schema JSONs exported to `app/schemas` (shipped to + * the test APK as assets). DatabaseModule deliberately registers no destructive fallback — dropping + * tables would lose stored accounts and credentials — so a drifted migration crashes upgrading + * users at first database open; these tests make such drift fail in CI instead (issue #63). + * + * The migration list is discovered from Migrations.kt and the target version from the newest + * exported schema, so a future migration is covered automatically: author the `Migration`, register + * it in DatabaseModule, commit the new schema JSON, and [migratingFromV7ReplaysEveryMigrationAndPreservesData] + * replays it with no edit here. (Only v7+ can be replayed — older versions predate schema export.) + */ +@RunWith(AndroidJUnit4::class) +class MigrationTest { + + @get:Rule + val helper = MigrationTestHelper( + InstrumentationRegistry.getInstrumentation(), + LibreMailDatabase::class.java, + emptyList(), + FrameworkSQLiteOpenHelperFactory(), + ) + + /** v11 -> v12 (PR #54): `folders.specialUse` appears defaulting to 0 and existing data survives. */ + @Test + fun migrate11To12_defaultsExistingFoldersToNotSpecialUse() { + helper.createDatabase(TEST_DB, 11).apply { + insertAccount() + execSQL( + "INSERT INTO folders (accountId, fullName, displayName, role, selectable, sortOrder) VALUES " + + "('acct', 'INBOX', 'INBOX', 'INBOX', 1, 0), " + + "('acct', '[Gmail]/Sent Mail', 'Sent Mail', 'SENT', 1, 1)", + ) + execSQL( + "INSERT INTO messages (id, accountId, sender, senderEmail, subject, snippet, body, isHtml, " + + "timestampMillis, isRead, isStarred, folder, inInbox, bodyFetched) " + + "VALUES ('acct:INBOX:1', 'acct', 'Ada', 'ada@example.org', 'Hi', '', '', 0, 1000, 0, 0, " + + "'INBOX', 1, 0)", + ) + close() + } + + val db = helper.runMigrationsAndValidate(TEST_DB, 12, true, MIGRATION_11_12) + + db.query("SELECT fullName, specialUse FROM folders ORDER BY sortOrder").use { c -> + assertTrue(c.moveToFirst()) + assertEquals("INBOX", c.getString(0)) + assertEquals(0, c.getInt(1)) + assertTrue(c.moveToNext()) + assertEquals("[Gmail]/Sent Mail", c.getString(0)) + assertEquals(0, c.getInt(1)) + assertFalse("both pre-upgrade folder rows must survive, and nothing else", c.moveToNext()) + } + // The folders' account and cached mail are untouched. + assertEquals(1, db.count("accounts")) + assertEquals(1, db.count("messages")) + db.close() + } + + /** + * A gap in the chain (or a migration whose target schema was never committed) crashes upgrading + * users, so fail fast with a readable message before replaying anything. + */ + @Test + fun migrationsFormOneUnbrokenChainUpToTheLatestExportedSchema() { + assertEquals( + "Migrations.kt must chain every version step up to the newest exported schema " + + "(DatabaseModule registers no destructive fallback, so a gap crashes upgrades)", + (1 until latestExportedSchemaVersion()).map { it to it + 1 }, + allAppMigrations.map { it.startVersion to it.endVersion }, + ) + } + + /** + * Creates a database at v7 (the oldest exported schema), fills it like a used install, then + * replays every migration one step at a time — `runMigrationsAndValidate` diffs the migrated + * schema against each version's exported JSON, so a failure names the exact step that drifted. + */ + @Test + fun migratingFromV7ReplaysEveryMigrationAndPreservesData() { + helper.createDatabase(TEST_DB, OLDEST_EXPORTED_SCHEMA).apply { + seedVersion7Cache() + close() + } + + var open: SupportSQLiteDatabase? = null + for (migration in allAppMigrations.filter { it.startVersion >= OLDEST_EXPORTED_SCHEMA }) { + open?.close() + val stepDb = helper.runMigrationsAndValidate(TEST_DB, migration.endVersion, true, migration) + stepDb.writeMidChainData() + open = stepDb + } + val db = checkNotNull(open) { "no migration starts at v$OLDEST_EXPORTED_SCHEMA" } + + assertEquals("the chain must end at the newest exported schema", latestExportedSchemaVersion(), db.version) + db.assertVersion7CacheSurvived() + db.assertMigrationBackfillsApplied() + db.close() + } + + /** The newest schema JSON exported to app/schemas (shipped to the test APK as assets). */ + private fun latestExportedSchemaVersion(): Int { + val schemaFolder = checkNotNull(LibreMailDatabase::class.java.canonicalName) + val versions = InstrumentationRegistry.getInstrumentation().context.assets.list(schemaFolder) + .orEmpty() + .mapNotNull { it.removeSuffix(".json").toIntOrNull() } + check(versions.isNotEmpty()) { "no exported schemas under androidTest assets/$schemaFolder" } + return versions.max() + } + + /** The accounts table has kept this shape since before v7, so every test can share one insert. */ + private fun SupportSQLiteDatabase.insertAccount() { + execSQL( + "INSERT INTO accounts (id, email, displayName, authType, imap_host, imap_port, imap_security, " + + "smtp_host, smtp_port, smtp_security) VALUES ('acct', 'ada@example.org', 'Ada', " + + "'PASSWORD_IMAP', 'imap.example.org', 993, 'SSL_TLS', 'smtp.example.org', 465, 'SSL_TLS')", + ) + } + + /** Fills a v7 database the way a used install would look (columns exactly as in 7.json). */ + private fun SupportSQLiteDatabase.seedVersion7Cache() { + insertAccount() + execSQL("INSERT INTO credentials (accountId, encryptedSecret) VALUES ('acct', 'sealed-secret')") + execSQL( + "INSERT INTO messages (id, accountId, sender, senderEmail, subject, snippet, body, isHtml, " + + "timestampMillis, isRead, isStarred, inInbox, bodyFetched) VALUES " + + "('acct:1', 'acct', 'Ada', 'ada@example.org', 'Analytical engines', 'Dear Charles', " + + "'Dear Charles, the mill works.', 0, 1000, 1, 0, 1, 1), " + + "('acct:2', 'acct', 'Charles', 'charles@example.org', 'Re: engines', '', '', 0, 2000, 0, 1, 1, 0)", + ) + execSQL( + "INSERT INTO attachments (messageId, partIndex, filename, mimeType, sizeBytes) " + + "VALUES ('acct:1', 0, 'notes.pdf', 'application/pdf', 2048)", + ) + execSQL( + "INSERT INTO outbox (id, accountId, toAddresses, ccAddresses, subject, body, createdAt, " + + "lastError) VALUES ('out-1', 'acct', 'charles@example.org', '', 'Queued', 'Body', 3000, NULL)", + ) + execSQL( + "INSERT INTO drafts (id, accountId, toAddresses, ccAddresses, subject, body, updatedAt, " + + "attachments) VALUES ('draft-1', 'acct', 'charles@example.org', '', 'Draft', 'Text', 4000, '')", + ) + } + + /** Rows a user would write at intermediate versions; they must survive the rest of the chain. */ + private fun SupportSQLiteDatabase.writeMidChainData() { + when (version) { + // v8 is the first version with a folders table; 11->12 must stamp this row specialUse = 0. + 8 -> execSQL( + "INSERT INTO folders (accountId, fullName, displayName, role, selectable, sortOrder) " + + "VALUES ('acct', 'INBOX', 'INBOX', 'INBOX', 1, 0)", + ) + // A signature saved by a v9 user: 10->11 must carry it into the new signatures table. + 9 -> execSQL( + "UPDATE account_settings SET signature = 'Cheers,' || char(10) || 'Ada' " + + "WHERE accountId = 'acct'", + ) + } + } + + /** Every row cached at v7 must still be present and correct at the end of the chain. */ + private fun SupportSQLiteDatabase.assertVersion7CacheSurvived() { + assertEquals(1, count("accounts")) + assertEquals(2, count("messages")) + assertEquals(1, count("outbox")) + assertEquals(1, count("drafts")) + query("SELECT folder, subject, isRead FROM messages WHERE id = 'acct:1'").use { c -> + assertTrue("message cached at v7 must survive", c.moveToFirst()) + assertEquals("7->8 files pre-upgrade messages under INBOX", "INBOX", c.getString(0)) + assertEquals("Analytical engines", c.getString(1)) + assertEquals(1, c.getInt(2)) + } + query("SELECT encryptedSecret FROM credentials WHERE accountId = 'acct'").use { c -> + assertTrue("stored credentials must never be dropped by a migration", c.moveToFirst()) + assertEquals("sealed-secret", c.getString(0)) + } + query("SELECT filename FROM attachments WHERE messageId = 'acct:1'").use { c -> + assertTrue("attachment rows must survive the 6->7 style table rebuilds", c.moveToFirst()) + assertEquals("notes.pdf", c.getString(0)) + } + } + + /** Columns and rows created by the migrations themselves must hold their documented defaults. */ + private fun SupportSQLiteDatabase.assertMigrationBackfillsApplied() { + // 8->9 backfills one default settings row per existing account. + query("SELECT signatureEnabled, notificationsEnabled FROM account_settings").use { c -> + assertTrue("8->9 must backfill a settings row for the v7 account", c.moveToFirst()) + assertEquals(1, c.getInt(0)) + assertEquals(1, c.getInt(1)) + } + // 9->10 adds bcc columns defaulting to ''; 10->11 adds nullable bodyHtml. + query("SELECT bccAddresses, bodyHtml FROM outbox WHERE id = 'out-1'").use { c -> + assertTrue(c.moveToFirst()) + assertEquals("", c.getString(0)) + assertTrue("bodyHtml must default to null (plaintext-only)", c.isNull(1)) + } + query("SELECT bccAddresses, bodyHtml FROM drafts WHERE id = 'draft-1'").use { c -> + assertTrue(c.moveToFirst()) + assertEquals("", c.getString(0)) + assertTrue(c.isNull(1)) + } + // 10->11 turns the signature written at v9 into that account's default rich-text signature. + query("SELECT name, contentHtml, isDefault FROM signatures WHERE accountId = 'acct'").use { c -> + assertTrue("10->11 must backfill the legacy per-account signature", c.moveToFirst()) + assertEquals("Signature", c.getString(0)) + assertEquals("Cheers,
Ada", c.getString(1)) + assertEquals(1, c.getInt(2)) + assertFalse("exactly one signature row must be backfilled", c.moveToNext()) + } + // 11->12 stamps the folder cached at v8 as not special-use. + query("SELECT specialUse FROM folders WHERE fullName = 'INBOX'").use { c -> + assertTrue("folder cached at v8 must survive to the newest version", c.moveToFirst()) + assertEquals(0, c.getInt(0)) + } + } + + private fun SupportSQLiteDatabase.count(table: String): Int = query("SELECT COUNT(*) FROM $table").use { c -> + c.moveToFirst() + c.getInt(0) + } + + private companion object { + const val TEST_DB = "migration-test.db" + + /** The oldest schema in app/schemas; earlier versions predate schema export. */ + const val OLDEST_EXPORTED_SCHEMA = 7 + + /** + * Every migration the app ships, pulled from Migrations.kt's file facade so a newly added + * migration is replayed automatically. DatabaseModule.provideDatabase registers this same + * set of top-level vals. + */ + val allAppMigrations: List = run { + val migrationsFile = checkNotNull(MIGRATION_1_2.javaClass.enclosingClass) { + "expected MIGRATION_1_2 to be a top-level val in Migrations.kt" + } + migrationsFile.methods + .filter { Modifier.isStatic(it.modifiers) && it.parameterCount == 0 } + .filter { Migration::class.java.isAssignableFrom(it.returnType) } + .map { it.invoke(null) as Migration } + .sortedBy { it.startVersion } + } + } +} diff --git a/app/src/main/kotlin/org/libremail/data/Snippet.kt b/app/src/main/kotlin/org/libremail/data/Snippet.kt new file mode 100644 index 0000000..5fe9718 --- /dev/null +++ b/app/src/main/kotlin/org/libremail/data/Snippet.kt @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.libremail.data + +import org.libremail.mail.HtmlToText + +/** + * Derives the one-line mailbox preview snippet that is persisted alongside a fetched body. + * + * HTML bodies are reduced to readable text via [HtmlToText] (script/style *content* dropped, tags + * stripped, entities decoded) before the whitespace collapsing. Plain-text bodies get no markup + * handling at all — literal `<`/`>` characters survive — only whitespace collapsing. Both paths end + * with the same [MAX_LENGTH] cap. + * + * Derivation runs once, when a body is fetched and cached (plus the one-off migration backfill) — + * never per mailbox-list row. + */ +object Snippet { + + const val MAX_LENGTH = 140 + + private val WHITESPACE = Regex("\\s+") + + fun of(body: String, isHtml: Boolean): String { + val text = if (isHtml) HtmlToText.convert(body) else body + return text.replace(WHITESPACE, " ").trim().take(MAX_LENGTH) + } +} diff --git a/app/src/main/kotlin/org/libremail/data/local/LibreMailDatabase.kt b/app/src/main/kotlin/org/libremail/data/local/LibreMailDatabase.kt index befa2e8..7173749 100644 --- a/app/src/main/kotlin/org/libremail/data/local/LibreMailDatabase.kt +++ b/app/src/main/kotlin/org/libremail/data/local/LibreMailDatabase.kt @@ -37,7 +37,7 @@ import org.libremail.data.local.entity.SignatureEntity SignatureEntity::class, BackfillProgressEntity::class, ], - version = 13, + version = 14, exportSchema = true, ) abstract class LibreMailDatabase : RoomDatabase() { diff --git a/app/src/main/kotlin/org/libremail/data/local/Migrations.kt b/app/src/main/kotlin/org/libremail/data/local/Migrations.kt index 5a6a32b..f4b3392 100644 --- a/app/src/main/kotlin/org/libremail/data/local/Migrations.kt +++ b/app/src/main/kotlin/org/libremail/data/local/Migrations.kt @@ -3,6 +3,7 @@ package org.libremail.data.local import androidx.room.migration.Migration import androidx.sqlite.db.SupportSQLiteDatabase +import org.libremail.data.Snippet /** v1 -> v2: add the encrypted-credentials table (preserves existing accounts/messages). */ val MIGRATION_1_2 = object : Migration(1, 2) { @@ -267,3 +268,28 @@ val MIGRATION_12_13 = object : Migration(12, 13) { ) } } + +/** + * v13 -> v14: data-only, no schema change (preserves existing data). Re-derives the persisted + * `snippet` of every message with a cached body using [Snippet.of], which — unlike the derivation + * it replaces — respects `isHtml`: HTML rows lose leaked `

Visible

", + isHtml = true, + ) + assertEquals("Visible", snippet) + } + + @Test + fun `html entities are decoded including numeric references`() { + assertEquals( + "Tom & Jerry — \"friends\" don’t fight", + Snippet.of("Tom & Jerry — "friends" don’t fight", isHtml = true), + ) + } + + @Test + fun `html nbsp becomes a plain space and runs of whitespace collapse`() { + assertEquals("a b c", Snippet.of("a b\n\t c", isHtml = true)) + } + + @Test + fun `html snippet is capped after markup is removed`() { + val body = "

" + "x".repeat(500) + "

" + val snippet = Snippet.of(body, isHtml = true) + assertEquals("x".repeat(Snippet.MAX_LENGTH), snippet) + } + + @Test + fun `plain text keeps literal angle brackets`() { + assertEquals( + "From : 3 < 5 and x > y", + Snippet.of("From : 3 < 5 and x > y", isHtml = false), + ) + } + + @Test + fun `plain text is not entity-decoded`() { + assertEquals("Fish & chips", Snippet.of("Fish & chips", isHtml = false)) + } + + @Test + fun `plain text collapses whitespace and trims`() { + assertEquals("one two three", Snippet.of(" one\r\n two\t\tthree \n", isHtml = false)) + } + + @Test + fun `plain text is capped at the max length`() { + assertEquals("y".repeat(Snippet.MAX_LENGTH), Snippet.of("y".repeat(1_000), isHtml = false)) + } +} diff --git a/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplTest.kt b/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplTest.kt index 5773d14..712d003 100644 --- a/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplTest.kt +++ b/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplTest.kt @@ -125,6 +125,46 @@ class MailRepositoryImplTest { coVerify { imapClient.fetchBodyMarkingSeen(any(), "Archive", "5") } } + @Test + fun `openMessage derives a readable plain-text snippet from an HTML body`() = runTest { + val id = "acct:INBOX:20" + coEvery { messageDao.getById(id) } returns messageEntity(id, "INBOX") + coEvery { accountDao.getById("acct") } returns accountEntity() + coEvery { connectionFactory.imapParamsFor(any()) } returns imapParams() + coEvery { imapClient.fetchBodyMarkingSeen(any(), "INBOX", "20") } returns MessageContent( + "

Tom & Jerry say "hi"

", + isHtml = true, + ) + val snippet = slot() + coEvery { messageDao.updateBody(id, any(), any(), capture(snippet)) } just Runs + coEvery { messageDao.setRead(id, true) } just Runs + + repository.openMessage(id) + + // Style content must not leak and entities must be decoded (the derivation honors isHtml). + assertEquals("Tom & Jerry say \"hi\"", snippet.captured) + } + + @Test + fun `prefetchMessage leaves a plain-text body's literal angle brackets in the snippet`() = runTest { + val cache = Files.createTempDirectory("attach").toFile() + every { context.cacheDir } returns cache + val id = "acct:INBOX:21" + coEvery { messageDao.getById(id) } returns messageEntity(id, "INBOX") + coEvery { accountDao.getById("acct") } returns accountEntity() + coEvery { connectionFactory.imapParamsFor(any()) } returns imapParams() + coEvery { imapClient.fetchBodyPeek(any(), "INBOX", "21") } returns + MessageContent("Reply to : 3 < 5", isHtml = false) + val snippet = slot() + coEvery { messageDao.updateBody(id, any(), any(), capture(snippet)) } just Runs + coEvery { attachmentDao.getForMessage(id) } returns emptyList() + + repository.prefetchMessage(id) + + // No tag stripping for plain text — only whitespace collapsing (and the length cap) applies. + assertEquals("Reply to : 3 < 5", snippet.captured) + } + @Test fun `archive moves messages to the account's archive folder and drops the local rows`() = runTest { val id = "acct:INBOX:5" diff --git a/app/src/test/kotlin/org/libremail/mail/HtmlToTextTest.kt b/app/src/test/kotlin/org/libremail/mail/HtmlToTextTest.kt index a5e67be..cecf675 100644 --- a/app/src/test/kotlin/org/libremail/mail/HtmlToTextTest.kt +++ b/app/src/test/kotlin/org/libremail/mail/HtmlToTextTest.kt @@ -32,6 +32,22 @@ class HtmlToTextTest { assertFalse(converted.contains("alert")) } + @Test + fun `decodes numeric character references in decimal and hex`() { + assertEquals("It’s fine", HtmlToText.convert("It’s fine")) + assertEquals("It’s fine", HtmlToText.convert("It’s fine")) + } + + @Test + fun `decoding is single-pass so a decoded character is never re-decoded`() { + assertEquals("<b>", HtmlToText.convert("&lt;b&gt;")) + } + + @Test + fun `unknown or malformed references are left as-is`() { + assertEquals("&bogus; &#; stays", HtmlToText.convert("&bogus; &#; stays")) + } + @Test fun `collapses excess whitespace`() { val converted = HtmlToText.convert("

Hello there

\n\n\n

bye

")