diff --git a/app/src/androidTest/kotlin/org/libremail/data/sync/GmailBandwidthTrackerInstrumentedTest.kt b/app/src/androidTest/kotlin/org/libremail/data/sync/GmailBandwidthTrackerInstrumentedTest.kt new file mode 100644 index 0000000..feb2db3 --- /dev/null +++ b/app/src/androidTest/kotlin/org/libremail/data/sync/GmailBandwidthTrackerInstrumentedTest.kt @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.libremail.data.sync + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.libremail.domain.model.Account +import org.libremail.domain.model.MailProvider + +/** + * On-device proof of issue #361's Gmail bandwidth pacing across the CI API matrix. Production feeds + * [GmailBandwidthTracker] from `MailRepositoryImpl.prefetchMessage`, which can race across + * concurrently-syncing accounts under the real dispatcher, so this proves the tracker's + * concurrent-map-backed accounting holds up under genuine concurrent updates on real threads rather + * than coroutines-test virtual time (the JVM [GmailBandwidthTrackerTest] covers the day-rollover and + * threshold-crossing logic in detail). Also proves [GmailSyncLimits.appliesTo] resolves the real + * [MailProvider] presets identically to production. Deliberately mock-free — no `mockk`, no framework + * `Context` — the tracker's only collaborator is the real wall clock (mirrors + * `BackfillPacerInstrumentedTest`'s mock-free idiom). + */ +@RunWith(AndroidJUnit4::class) +class GmailBandwidthTrackerInstrumentedTest { + + @Test + fun concurrentDownloadsForOneAccountAllLandWithoutLosingAnUpdate() = runBlocking { + val tracker = GmailBandwidthTracker() + val perTask = 1_000L + + val jobs = (1..CONCURRENT_TASKS).map { + async(Dispatchers.Default) { tracker.recordDownload("acct", perTask) } + } + jobs.awaitAll() + + assertEquals(CONCURRENT_TASKS * perTask, tracker.bytesDownloadedToday("acct")) + } + + @Test + fun accountsStayIsolatedUnderConcurrentRecording() = runBlocking { + val tracker = GmailBandwidthTracker() + + val heavy = async(Dispatchers.Default) { + tracker.recordDownload("heavy", GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES) + } + val light = async(Dispatchers.Default) { tracker.recordDownload("light", 1L) } + heavy.await() + light.await() + + assertTrue(tracker.isOverDailyBudget("heavy")) + assertFalse(tracker.isOverDailyBudget("light")) + } + + @Test + fun appliesToResolvesTheRealGmailPresetOnDevice() { + val gmail = MailProvider.GMAIL.createAccount("user@gmail.com") + val outlook = Account.outlook("user@outlook.com") + + assertTrue(GmailSyncLimits.appliesTo(gmail)) + assertFalse(GmailSyncLimits.appliesTo(outlook)) + } + + private companion object { + const val CONCURRENT_TASKS = 50 + } +} diff --git a/app/src/main/kotlin/org/libremail/data/repository/MailRepositoryImpl.kt b/app/src/main/kotlin/org/libremail/data/repository/MailRepositoryImpl.kt index 1a0480a..de70bbd 100644 --- a/app/src/main/kotlin/org/libremail/data/repository/MailRepositoryImpl.kt +++ b/app/src/main/kotlin/org/libremail/data/repository/MailRepositoryImpl.kt @@ -39,6 +39,8 @@ import org.libremail.data.local.toOutgoingAttachments import org.libremail.data.local.toOutgoingAttachmentsJson import org.libremail.data.settings.AccountSettingsRepository import org.libremail.data.settings.SignatureRepository +import org.libremail.data.sync.GmailBandwidthTracker +import org.libremail.data.sync.GmailSyncLimits import org.libremail.data.sync.InteractiveImapGate import org.libremail.data.sync.MailConnectionFactory import org.libremail.data.sync.SendScheduler @@ -81,6 +83,7 @@ class MailRepositoryImpl @Inject constructor( private val signatureRepository: SignatureRepository, private val attachmentUriGrants: AttachmentUriGrants, private val interactiveGate: InteractiveImapGate, + private val bandwidthTracker: GmailBandwidthTracker, ) : MailRepository { // Application-lifetime scope for fire-and-forget server pushes that must outlive the caller — e.g. @@ -253,7 +256,7 @@ class MailRepositoryImpl @Inject constructor( // fetch just omits that image, leaving a broken rather than failing the open. val file = runCatching { ensureAttachmentFile(messageId, routing.accountId, routing.folder, row.partIndex, row.filename) - }.getOrNull() ?: return@mapNotNull null + }.getOrNull()?.file ?: return@mapNotNull null InlineImage(contentId = row.contentId!!, mimeType = row.mimeType, bytes = file.readBytes()) } } @@ -275,11 +278,18 @@ class MailRepositoryImpl @Inject constructor( routing.folder, partIndex, meta?.filename ?: "attachment", - ) + ).file } } } + /** + * [ensureAttachmentFile]'s outcome: the cached [file] plus the bytes actually pulled over the + * network THIS call — `0` on a cache hit. [downloadedBytes] feeds Gmail's bandwidth accounting + * (issue #361, see [prefetchMessage]); a cache hit costs nothing so it must not be double-counted. + */ + private class AttachmentFetch(val file: File, val downloadedBytes: Long) + /** * Returns the on-disk file for one attachment part, downloading and caching it on first use so it * then opens instantly and offline. Takes the message's already-resolved account/folder so a batch @@ -292,16 +302,16 @@ class MailRepositoryImpl @Inject constructor( folder: String, partIndex: Int, filename: String, - ): File { + ): AttachmentFetch { val target = attachmentFile(messageId, partIndex, filename) // Reuse a previously downloaded (or pre-fetched) file so it opens instantly and offline. - if (target.exists() && target.length() > 0L) return target + if (target.exists() && target.length() > 0L) return AttachmentFetch(target, downloadedBytes = 0L) val account = accountDao.getById(accountId)?.toDomain() ?: error("Account not found") val params = connectionFactory.imapParamsFor(account) val downloaded = imapClient.fetchAttachment(params, folder, uidOf(messageId), partIndex) target.parentFile?.mkdirs() target.outputStream().use { it.write(downloaded.bytes) } - return target + return AttachmentFetch(target, downloadedBytes = downloaded.bytes.size.toLong()) } override suspend fun downloadedAttachmentParts(messageId: String): Set = withContext(Dispatchers.IO) { @@ -317,12 +327,17 @@ class MailRepositoryImpl @Inject constructor( override suspend fun prefetchMessage(messageId: String): Result = runCatching { val routing = messageDao.getRouting(messageId) ?: return@runCatching val account = accountDao.getById(routing.accountId)?.toDomain() ?: return@runCatching + // Bytes actually pulled over the network this call (0 on an all-cache-hit prefetch), fed to + // Gmail's daily download-budget tracker below (issue #361) — the proactive pacing that composes + // with #360's reactive AccountThrottleGate and #356's BackfillPacer without modifying either. + var downloadedBytes = 0L // Cache the body (peek, so prefetching never marks the message read) and its attachment metadata. if (!routing.bodyFetched) { val params = connectionFactory.imapParamsFor(account) val content = imapClient.fetchBodyPeek(params, routing.folder, uidOf(messageId)) messageDao.updateBody(messageId, content.body, content.isHtml, Snippet.of(content.body, content.isHtml)) attachmentDao.replaceForMessage(messageId, content.attachments.map { it.toEntity(messageId) }) + downloadedBytes += content.body.toByteArray(Charsets.UTF_8).size.toLong() } // Auto-download every attachment's bytes into the persistent per-part cache (skips ones present). // This is BACKGROUND work driven by the backfill, so it goes straight to ensureAttachmentFile and @@ -338,7 +353,13 @@ class MailRepositoryImpl @Inject constructor( attachment.partIndex, attachment.filename, ) - } + }.getOrNull()?.let { downloadedBytes += it.downloadedBytes } + } + // Gmail-specific bandwidth accounting (issue #361): only tracked for Gmail, since that is the + // only provider whose daily download budget is enforced today (see GmailSyncLimits.appliesTo / + // MailBackfiller.prefetchIfEnabled / MailSyncer.prefetchIfEnabled for the deferral this feeds). + if (downloadedBytes > 0L && GmailSyncLimits.appliesTo(account)) { + bandwidthTracker.recordDownload(account.id, downloadedBytes) } } diff --git a/app/src/main/kotlin/org/libremail/data/sync/GmailBandwidthTracker.kt b/app/src/main/kotlin/org/libremail/data/sync/GmailBandwidthTracker.kt new file mode 100644 index 0000000..bb556e8 --- /dev/null +++ b/app/src/main/kotlin/org/libremail/data/sync/GmailBandwidthTracker.kt @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.libremail.data.sync + +import org.libremail.reporting.AppLog +import org.libremail.reporting.accountLogRef +import java.util.concurrent.ConcurrentHashMap +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Per-account, per-day running total of bytes downloaded by background prefetch, tracked against + * Gmail's documented [GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES] (issue #361). The stateful + * counterpart to the pure [GmailSyncLimits]: + * [org.libremail.data.repository.MailRepositoryImpl.prefetchMessage] feeds it bytes actually pulled + * over the network via [recordDownload], and [MailBackfiller] / [MailSyncer] consult + * [isOverDailyBudget] before starting a fresh prefetch batch for an account, deferring the rest of the + * day's prefetch once the budget is reached. + * + * **Proactive, not reactive — orthogonal to [AccountThrottleGate].** The #360 gate only fires once a + * provider actually rejects a request; this tracker heads that off by pacing our OWN traffic against a + * budget Gmail documents but does not necessarily announce hitting. It has the same relationship to + * [AccountThrottleGate] that [InteractiveImapGate] already documents having with it: a separate, + * composing mechanism, not a duplicate or a replacement. + * + * **Self-healing.** A day boundary (a wall-clock day number derived from [nowMillis]) resets an + * account's tracked total, so a deferred account automatically resumes full prefetch the next day with + * no explicit reset needed — mirrors how [AccountThrottleGate]'s backoff window elapses on its own. + * + * **Interactive traffic is deliberately NOT tracked here.** Only background prefetch (issue #361's + * "sync/backfill" scope) feeds this tracker — opening a message, downloading a tapped attachment, and + * loading inline images are never deferred by a budget (the same interactive-priority principle + * #355/#360 already apply), so counting their bytes here would only make the tracker's *deferral* + * decision — which exclusively affects background prefetch — less representative of what it can + * actually still influence. + * + * State lives only in-process (a `@Singleton`); a process restart clears it, which simply means a + * fresh process re-earns its budget for the (partial) remainder of the day — a conservative direction + * to fail in, same as [AccountThrottleGate]'s reset-on-restart. Every log line is PII-free: + * [accountLogRef] for the account, and byte counts only. + */ +@Singleton +class GmailBandwidthTracker internal constructor(private val nowMillis: () -> Long) { + /** Production wiring: the real wall clock. */ + @Inject constructor() : this(nowMillis = System::currentTimeMillis) + + /** One account's running download total for [dayEpoch] (whole days since the epoch). */ + private data class Window(val dayEpoch: Long, val bytes: Long) + + private val windows = ConcurrentHashMap() + + /** + * Adds [bytes] to [accountId]'s running total for today, starting a fresh window if the day has + * rolled over since the last call (yesterday's total is simply discarded, not carried forward). A + * no-op for `bytes <= 0`. Logs once, PII-free, the moment this call carries the account from under + * budget to at-or-over it — not on every call, so an account that stays over budget for the rest of + * a sync/backfill pass doesn't spam the log. + */ + fun recordDownload(accountId: String, bytes: Long) { + if (bytes <= 0L) return + val day = currentDayEpoch() + val before = windows[accountId]?.takeIf { it.dayEpoch == day }?.bytes ?: 0L + val updated = windows.compute(accountId) { _, previous -> + val carried = if (previous != null && previous.dayEpoch == day) previous.bytes else 0L + Window(day, carried + bytes) + }!! + val budget = GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES + if (before < budget && updated.bytes >= budget) { + AppLog.w(TAG, "daily download budget reached ${accountLogRef(accountId)}") + } + } + + /** [accountId]'s tracked download bytes so far today, or 0 when untracked or the day has rolled over. */ + fun bytesDownloadedToday(accountId: String): Long { + val window = windows[accountId] ?: return 0L + return if (window.dayEpoch == currentDayEpoch()) window.bytes else 0L + } + + /** True once [accountId]'s tracked downloads for today reach [GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES]. */ + fun isOverDailyBudget(accountId: String): Boolean = + bytesDownloadedToday(accountId) >= GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES + + private fun currentDayEpoch(): Long = nowMillis() / MILLIS_PER_DAY + + private companion object { + const val TAG = "GmailBandwidth" + const val MILLIS_PER_DAY = 24 * 60 * 60 * 1000L + } +} diff --git a/app/src/main/kotlin/org/libremail/data/sync/GmailSyncLimits.kt b/app/src/main/kotlin/org/libremail/data/sync/GmailSyncLimits.kt new file mode 100644 index 0000000..804a2f4 --- /dev/null +++ b/app/src/main/kotlin/org/libremail/data/sync/GmailSyncLimits.kt @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.libremail.data.sync + +import org.libremail.domain.model.Account +import org.libremail.domain.model.MailProvider + +/** + * Gmail's documented IMAP connection and bandwidth ceilings (issue #361) — the Gmail-specific config + * that feeds the shared, provider-agnostic pacing machinery ([AccountThrottleGate]'s reactive backoff, + * issue #360; [BackfillPacer]'s proactive inter-slice cooldown, issue #356) instead of reinventing + * either. Per-account, per-provider connection/bandwidth caps were deliberately deferred out of both + * (see [org.libremail.mail.ImapConnectionCache]'s "separate effort #356/#360-#364" note); this is + * Gmail's slice of that follow-up. Kept additive and provider-scoped — the sibling Yahoo (#362), iCloud + * (#363), and Outlook/Graph (#364) tickets land their own provider config independently. + * + * Values are Google's documented Gmail IMAP limits (as referenced by issue #361): + * - 15 max simultaneous IMAP connections per account. + * - 2,500 MB/day download, 500 MB/day upload. + * - 10,000 messages per label, 10,000 labels. + * + * Pure data + provider detection only — no state, no logging (mirrors how [ThrottleBackoff] and + * [ThrottleClassifier] stay pure while [AccountThrottleGate] carries the state and logging). The + * stateful counterpart that actually tracks bytes against [DAILY_DOWNLOAD_BUDGET_BYTES] is + * [GmailBandwidthTracker]. + */ +object GmailSyncLimits { + /** Gmail's documented simultaneous-IMAP-connection ceiling, per account. */ + const val MAX_IMAP_CONNECTIONS = 15 + + /** + * Connections proactively reserved for interactive use (never spent by background sync/backfill), + * mirroring the "interactive-request priority over backfill" lever from the #360 umbrella issue. + * LibreMail's IMAP layer already keeps at most ONE reused connection per account + * ([org.libremail.mail.ImapConnectionCache], issue #125/#357) plus one dedicated IMAP-IDLE + * connection ([org.libremail.mail.ImapClient.idle]) — 2 total, well inside the resulting headroom + * regardless of this value — so today this is documented config for any future pooling work rather + * than something that needs active enforcement (see `GmailSyncLimitsTest` for the invariant that + * ties the two together). + */ + const val INTERACTIVE_RESERVED_CONNECTIONS = 1 + + /** [MAX_IMAP_CONNECTIONS] minus [INTERACTIVE_RESERVED_CONNECTIONS] — background sync/backfill's budget. */ + const val MAX_BACKGROUND_IMAP_CONNECTIONS = MAX_IMAP_CONNECTIONS - INTERACTIVE_RESERVED_CONNECTIONS + + private const val BYTES_PER_MB = 1024L * 1024L + + /** + * Gmail's documented daily download budget. [GmailBandwidthTracker] accumulates bytes actually + * pulled by background prefetch (see + * [org.libremail.data.repository.MailRepositoryImpl.prefetchMessage]) against this; [MailBackfiller] + * and [MailSyncer] defer further prefetch for an account once it is reached, resuming automatically + * the next day. Interactive fetches (opening a message, downloading a tapped attachment, loading + * inline images) are never gated by it — the same interactive-priority principle #355/#360 already + * apply elsewhere. + */ + const val DAILY_DOWNLOAD_BUDGET_BYTES = 2_500L * BYTES_PER_MB + + /** + * Gmail's documented daily upload budget (SMTP send). Captured here for completeness against + * issue #361's documented limits; sending/composing is a separate path from this issue's + * sync/backfill scope, so it is not enforced by this change. + */ + const val DAILY_UPLOAD_BUDGET_BYTES = 500L * BYTES_PER_MB + + /** Gmail's documented per-label message ceiling. */ + const val MAX_MESSAGES_PER_LABEL = 10_000 + + /** Gmail's documented total-labels ceiling. */ + const val MAX_LABELS = 10_000 + + /** + * True when [account]'s IMAP host resolves to [MailProvider.GMAIL] (including its legacy + * `imap.googlemail.com` alias) — the single place issue #361's caps decide "is this Gmail". + */ + fun appliesTo(account: Account): Boolean = MailProvider.forImapHost(account.imap.host) == MailProvider.GMAIL +} diff --git a/app/src/main/kotlin/org/libremail/data/sync/MailBackfiller.kt b/app/src/main/kotlin/org/libremail/data/sync/MailBackfiller.kt index 42bb8f0..00001e2 100644 --- a/app/src/main/kotlin/org/libremail/data/sync/MailBackfiller.kt +++ b/app/src/main/kotlin/org/libremail/data/sync/MailBackfiller.kt @@ -59,6 +59,7 @@ class MailBackfiller @Inject constructor( private val maintenanceGate: MailMaintenanceGate, private val throttleGate: AccountThrottleGate, private val interactiveGate: InteractiveImapGate, + private val bandwidthTracker: GmailBandwidthTracker, ) { /** One folder's slice outcome: pages fetched, and whether an immediate follow-up slice has work to do. */ private data class FolderResult(val batches: Int, val moreWork: Boolean) @@ -214,7 +215,7 @@ class MailBackfiller @Inject constructor( } beforeUid = nextBeforeUid backfillProgressDao.upsert(BackfillProgressEntity(account.id, folder, beforeUid, complete = false)) - prefetchIfEnabled(entities.map { it.id }) + prefetchIfEnabled(account, entities.map { it.id }) // Breathe between pages so a large mailbox doesn't hammer the server. delay(BACKFILL_BATCH_DELAY_MS) } @@ -293,7 +294,7 @@ class MailBackfiller @Inject constructor( * Best-effort and cancellable between messages so an interruption stops promptly; anything not * fetched is filled in lazily when the message is opened. */ - private suspend fun prefetchIfEnabled(ids: List) { + private suspend fun prefetchIfEnabled(account: Account, ids: List) { // Debug-only fetch gate (issue #393): pause proactive body prefetch so a later open is a genuine // uncached fetch. Header paging above is untouched (its own gate is the BackfillWorker entry), so // history still lands; a skipped body is filled in lazily on open. Compiled out of release @@ -308,6 +309,17 @@ class MailBackfiller @Inject constructor( battery = batteryStatusProvider.current(), ) if (!shouldPrefetch) return + // Gmail-specific proactive bandwidth pacing (#361): once this account's tracked downloads for + // today reach Gmail's documented daily budget, defer body/attachment prefetch for the rest of + // the day instead of continuing to spend it — header paging above is unaffected, and a fresh + // cycle resumes automatically once the day rolls over (GmailBandwidthTracker). Orthogonal to + // the #360 throttle skip above (which only fires once the provider actually rejects a request) + // and #356's BackfillPacer (which paces slice cadence, not bytes) — same graceful-degradation + // shape as both, composing rather than duplicating either. + if (GmailSyncLimits.appliesTo(account) && bandwidthTracker.isOverDailyBudget(account.id)) { + AppLog.i(TAG, "prefetch deferred ${accountLogRef(account.id)}: Gmail daily download budget reached") + return + } for (id in ids) { currentCoroutineContext().ensureActive() mailRepository.prefetchMessage(id) diff --git a/app/src/main/kotlin/org/libremail/data/sync/MailSyncer.kt b/app/src/main/kotlin/org/libremail/data/sync/MailSyncer.kt index e2f0814..426860c 100644 --- a/app/src/main/kotlin/org/libremail/data/sync/MailSyncer.kt +++ b/app/src/main/kotlin/org/libremail/data/sync/MailSyncer.kt @@ -41,6 +41,7 @@ class MailSyncer @Inject constructor( private val notifier: MailNotifier, private val mailRepository: MailRepository, private val throttleGate: AccountThrottleGate, + private val bandwidthTracker: GmailBandwidthTracker, ) : Syncer { // Serializes all syncing: syncAll/syncAccount/syncFolder are invoked concurrently by the periodic // worker, pull-to-refresh, one-shot syncs, folder opens, and one IDLE watcher per account. Without @@ -187,6 +188,17 @@ class MailSyncer @Inject constructor( battery = batteryStatusProvider.current(), ) if (!shouldPrefetch) return + // Gmail-specific proactive bandwidth pacing (#361): once this account's tracked downloads for + // today reach Gmail's documented daily budget, defer body/attachment prefetch for the rest of + // the day instead of continuing to spend it — header sync above is unaffected, and a fresh + // cycle resumes automatically once the day rolls over (GmailBandwidthTracker). Orthogonal to + // #360's reactive AccountThrottleGate (only fires once the provider actually rejects a request) + // and #356's BackfillPacer (paces backfill slice cadence, not bytes) — same graceful-degradation + // shape as both, composing rather than duplicating either. + if (GmailSyncLimits.appliesTo(account) && bandwidthTracker.isOverDailyBudget(account.id)) { + AppLog.i(TAG, "prefetch deferred ${accountLogRef(account.id)}: Gmail daily download budget reached") + return + } for (id in messageDao.getUnfetchedIds(account.id, folder)) { currentCoroutineContext().ensureActive() mailRepository.prefetchMessage(id) // best-effort; swallows its own per-message failures diff --git a/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryGrantsTest.kt b/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryGrantsTest.kt index 805955c..fcc9f83 100644 --- a/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryGrantsTest.kt +++ b/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryGrantsTest.kt @@ -16,6 +16,7 @@ import org.libremail.data.local.dao.DraftDao import org.libremail.data.local.dao.OutboxDao import org.libremail.data.local.entity.DraftEntity import org.libremail.data.local.entity.OutboxEntity +import org.libremail.data.sync.GmailBandwidthTracker import org.libremail.data.sync.InteractiveImapGate import java.nio.file.Files @@ -47,6 +48,7 @@ class MailRepositoryGrantsTest { signatureRepository = mockk(relaxed = true), attachmentUriGrants = attachmentUriGrants, interactiveGate = InteractiveImapGate(), + bandwidthTracker = GmailBandwidthTracker(), ) @Test diff --git a/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplCoverageTest.kt b/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplCoverageTest.kt index 46afc61..6690a04 100644 --- a/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplCoverageTest.kt +++ b/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplCoverageTest.kt @@ -40,6 +40,7 @@ import org.libremail.data.local.entity.OutboxEntity import org.libremail.data.local.entity.ServerConfigEmbedded import org.libremail.data.settings.AccountSettingsRepository import org.libremail.data.settings.SignatureRepository +import org.libremail.data.sync.GmailBandwidthTracker import org.libremail.data.sync.InteractiveImapGate import org.libremail.data.sync.MailConnectionFactory import org.libremail.data.sync.SendScheduler @@ -99,6 +100,7 @@ class MailRepositoryImplCoverageTest { signatureRepository = signatureRepository, attachmentUriGrants = mockk(relaxed = true), interactiveGate = InteractiveImapGate(), + bandwidthTracker = GmailBandwidthTracker(), ) // openMessage now breadcrumbs via AppLog (issue #358); android.util.Log is a no-op stub under plain diff --git a/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplTest.kt b/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplTest.kt index 36a22fc..50413b5 100644 --- a/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplTest.kt +++ b/app/src/test/kotlin/org/libremail/data/repository/MailRepositoryImplTest.kt @@ -40,6 +40,7 @@ import org.libremail.data.local.entity.MessageSummary import org.libremail.data.local.entity.ServerConfigEmbedded import org.libremail.data.settings.AccountSettingsRepository import org.libremail.data.settings.SignatureRepository +import org.libremail.data.sync.GmailBandwidthTracker import org.libremail.data.sync.InteractiveImapGate import org.libremail.data.sync.MailConnectionFactory import org.libremail.domain.model.AccountSettings @@ -78,6 +79,9 @@ class MailRepositoryImplTest { // A real gate (cheap, no deps) so tests can observe the interactive-fetch counter it raises (#355). private val interactiveGate = InteractiveImapGate() + + // A real tracker (cheap, no deps) so tests can observe Gmail's daily download-budget accounting (#361). + private val bandwidthTracker = GmailBandwidthTracker() private val repository = MailRepositoryImpl( context = context, messageDao = messageDao, @@ -94,6 +98,7 @@ class MailRepositoryImplTest { // Grant-release wiring (deleteDraft / cancelOutboxMessage) is covered by MailRepositoryGrantsTest. attachmentUriGrants = mockk(relaxed = true), interactiveGate = interactiveGate, + bandwidthTracker = bandwidthTracker, ) // openMessage now breadcrumbs via AppLog (issue #358); android.util.Log is a no-op stub under plain @@ -411,6 +416,48 @@ class MailRepositoryImplTest { assertEquals("Reply to : 3 < 5", snippet.captured) } + // --- issue #361: Gmail bandwidth-aware prefetch pacing ---------------------------------------- + + @Test + fun `prefetchMessage records the fetched body and attachment bytes for a gmail account`() = runTest { + val cache = Files.createTempDirectory("attach").toFile() + every { context.cacheDir } returns cache + val id = "acct:INBOX:22" + val body = "Hello world" + coEvery { messageDao.getRouting(id) } returns messageRouting(id, "INBOX") + coEvery { accountDao.getById("acct") } returns + accountEntity().copy(imap = ServerConfigEmbedded("imap.gmail.com", 993, "SSL_TLS")) + coEvery { connectionFactory.imapParamsFor(any()) } returns imapParams() + coEvery { imapClient.fetchBodyPeek(any(), "INBOX", "22") } returns MessageContent(body, isHtml = false) + coEvery { messageDao.updateBody(id, any(), any(), any()) } just Runs + coEvery { attachmentDao.getForMessage(id) } returns listOf(attachmentEntity(id, 0, "photo.jpg")) + coEvery { imapClient.fetchAttachment(any(), "INBOX", "22", 0) } returns + DownloadedAttachment("photo.jpg", "image/jpeg", ByteArray(500)) + + repository.prefetchMessage(id) + + val expectedBytes = body.toByteArray(Charsets.UTF_8).size + 500 + assertEquals(expectedBytes.toLong(), bandwidthTracker.bytesDownloadedToday("acct")) + } + + @Test + fun `prefetchMessage does not track bytes for a non-gmail account`() = runTest { + val cache = Files.createTempDirectory("attach").toFile() + every { context.cacheDir } returns cache + val id = "acct:INBOX:23" + coEvery { messageDao.getRouting(id) } returns messageRouting(id, "INBOX") + coEvery { accountDao.getById("acct") } returns accountEntity() // non-Gmail host (imap.example.org) + coEvery { connectionFactory.imapParamsFor(any()) } returns imapParams() + coEvery { imapClient.fetchBodyPeek(any(), "INBOX", "23") } returns + MessageContent("Hello world", isHtml = false) + coEvery { messageDao.updateBody(id, any(), any(), any()) } just Runs + coEvery { attachmentDao.getForMessage(id) } returns emptyList() + + repository.prefetchMessage(id) + + assertEquals(0L, bandwidthTracker.bytesDownloadedToday("acct")) + } + @Test fun `archive moves messages to the account's archive folder and drops the local rows`() = runTest { val id = "acct:INBOX:5" diff --git a/app/src/test/kotlin/org/libremail/data/sync/GmailBandwidthTrackerTest.kt b/app/src/test/kotlin/org/libremail/data/sync/GmailBandwidthTrackerTest.kt new file mode 100644 index 0000000..0ce7c4b --- /dev/null +++ b/app/src/test/kotlin/org/libremail/data/sync/GmailBandwidthTrackerTest.kt @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.libremail.data.sync + +import android.util.Log +import io.mockk.every +import io.mockk.mockkStatic +import io.mockk.unmockkAll +import org.junit.After +import org.junit.Before +import org.junit.Test +import org.libremail.reporting.AppLog +import org.libremail.reporting.RingLogBuffer +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * [GmailBandwidthTracker] (issue #361) must accumulate an account's daily download bytes, isolate + * accounts from one another, roll its window over at the day boundary, report the daily-budget + * threshold accurately, and log only a PII-free, once-per-crossing breadcrumb. Mirrors + * [AccountThrottleGateTest]'s virtual-clock idiom (a plain injected `nowMillis` instead of + * coroutines-test virtual time, since the tracker itself is not a suspend API). + */ +class GmailBandwidthTrackerTest { + + private val logBuffer = RingLogBuffer() + + /** A manual clock for the day-rollover tests; [tracker] reads it live, so tests advance it by hand. */ + private var now = 0L + + private fun tracker() = GmailBandwidthTracker(nowMillis = { now }) + + @Before + fun setUp() { + // AppLog forwards to android.util.Log, a throwing no-op stub under plain JVM unit tests. + mockkStatic(Log::class) + every { Log.w(any(), any()) } returns 0 + AppLog.install(logBuffer) + } + + @After + fun tearDown() = unmockkAll() + + @Test + fun `bytes accumulate across calls for the same account and day`() { + val tracker = tracker() + + tracker.recordDownload("acct", 100L) + tracker.recordDownload("acct", 250L) + + assertEquals(350L, tracker.bytesDownloadedToday("acct")) + } + + @Test + fun `an untracked account reports zero bytes and is not over budget`() { + val tracker = tracker() + + assertEquals(0L, tracker.bytesDownloadedToday("acct")) + assertFalse(tracker.isOverDailyBudget("acct")) + } + + @Test + fun `crossing the daily download budget marks the account over budget`() { + val tracker = tracker() + + tracker.recordDownload("acct", GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES - 1) + assertFalse(tracker.isOverDailyBudget("acct"), "one byte under budget must not trip it") + + tracker.recordDownload("acct", 1L) + assertTrue(tracker.isOverDailyBudget("acct"), "reaching the budget exactly must trip it") + } + + @Test + fun `a tracked account never affects another account's budget`() { + val tracker = tracker() + + tracker.recordDownload("heavy", GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES) + + assertTrue(tracker.isOverDailyBudget("heavy")) + assertFalse(tracker.isOverDailyBudget("light")) + assertEquals(0L, tracker.bytesDownloadedToday("light")) + } + + @Test + fun `a new day resets the tracked total instead of carrying it forward`() { + val tracker = tracker() + val oneDayMs = 24 * 60 * 60 * 1000L + + tracker.recordDownload("acct", GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES) + assertTrue(tracker.isOverDailyBudget("acct")) + + now += oneDayMs + assertFalse(tracker.isOverDailyBudget("acct"), "a new day must clear yesterday's total") + assertEquals(0L, tracker.bytesDownloadedToday("acct")) + + tracker.recordDownload("acct", 10L) + assertEquals(10L, tracker.bytesDownloadedToday("acct"), "today's total starts fresh, not carried over") + } + + @Test + fun `recording zero or negative bytes is a no-op`() { + val tracker = tracker() + + tracker.recordDownload("acct", 0L) + tracker.recordDownload("acct", -5L) + + assertEquals(0L, tracker.bytesDownloadedToday("acct")) + } + + @Test + fun `crossing the budget logs exactly once and stays PII-free`() { + val tracker = tracker() + val accountId = "imap:user@example.org" + + tracker.recordDownload(accountId, GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES) // crosses + tracker.recordDownload(accountId, 10L) // still over budget; must not log again + + val messages = logBuffer.snapshot().map { it.message } + assertEquals(1, messages.count { it.contains("daily download budget reached") }) + messages.forEach { assertFalse(it.contains("user@example.org"), it) } + } + + @Test + fun `staying under budget never logs`() { + val tracker = tracker() + + tracker.recordDownload("acct", GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES - 1) + + assertTrue(logBuffer.snapshot().isEmpty()) + } +} diff --git a/app/src/test/kotlin/org/libremail/data/sync/GmailSyncLimitsTest.kt b/app/src/test/kotlin/org/libremail/data/sync/GmailSyncLimitsTest.kt new file mode 100644 index 0000000..3ed6094 --- /dev/null +++ b/app/src/test/kotlin/org/libremail/data/sync/GmailSyncLimitsTest.kt @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package org.libremail.data.sync + +import org.junit.Test +import org.libremail.domain.model.Account +import org.libremail.domain.model.MailProvider +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * Locks down Gmail's documented IMAP connection/bandwidth ceilings (issue #361) — easy to mistype and + * painful to debug on-device, so the exact figures are asserted here rather than trusted to a code + * review (mirrors [org.libremail.domain.model.MailProviderTest]'s rationale for the provider presets). + */ +class GmailSyncLimitsTest { + + @Test + fun `documented connection and bandwidth ceilings match Gmail's published limits`() { + assertEquals(15, GmailSyncLimits.MAX_IMAP_CONNECTIONS) + assertEquals(1, GmailSyncLimits.INTERACTIVE_RESERVED_CONNECTIONS) + assertEquals(14, GmailSyncLimits.MAX_BACKGROUND_IMAP_CONNECTIONS) + assertEquals(2_500L * 1024 * 1024, GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES) + assertEquals(500L * 1024 * 1024, GmailSyncLimits.DAILY_UPLOAD_BUDGET_BYTES) + assertEquals(10_000, GmailSyncLimits.MAX_MESSAGES_PER_LABEL) + assertEquals(10_000, GmailSyncLimits.MAX_LABELS) + } + + /** + * Ties Gmail's documented ceiling to today's actual architecture: [org.libremail.mail.ImapConnectionCache] + * (#125/#357) keeps at most ONE reused connection per account, plus `ImapClient.idle`'s own dedicated + * IDLE connection — 2 total, regardless of provider. Asserting that invariant against the real + * headroom-adjusted cap means a future change that grows per-account concurrency (e.g. a real + * connection pool) trips this test well before it could ever approach Gmail's actual ceiling. + */ + @Test + fun `today's architecture keeps concurrent connections per account well inside the background budget`() { + val knownConcurrentConnectionsPerAccount = 2 // one reused IMAP connection + one dedicated IDLE connection + assertTrue(knownConcurrentConnectionsPerAccount <= GmailSyncLimits.MAX_BACKGROUND_IMAP_CONNECTIONS) + } + + @Test + fun `appliesTo is true for a gmail account, including the legacy googlemail host`() { + val gmail = MailProvider.GMAIL.createAccount("user@gmail.com") + assertTrue(GmailSyncLimits.appliesTo(gmail)) + + val legacyHost = gmail.copy(imap = gmail.imap.copy(host = "imap.googlemail.com")) + assertTrue(GmailSyncLimits.appliesTo(legacyHost)) + } + + @Test + fun `appliesTo is false for a non-gmail account`() { + assertFalse(GmailSyncLimits.appliesTo(MailProvider.YAHOO.createAccount("user@yahoo.com"))) + assertFalse(GmailSyncLimits.appliesTo(MailProvider.ICLOUD.createAccount("user@icloud.com"))) + assertFalse(GmailSyncLimits.appliesTo(MailProvider.AOL.createAccount("user@aol.com"))) + assertFalse(GmailSyncLimits.appliesTo(Account.outlook("user@outlook.com"))) + } +} diff --git a/app/src/test/kotlin/org/libremail/data/sync/MailBackfillerTest.kt b/app/src/test/kotlin/org/libremail/data/sync/MailBackfillerTest.kt index 4a9ad75..b9948e1 100644 --- a/app/src/test/kotlin/org/libremail/data/sync/MailBackfillerTest.kt +++ b/app/src/test/kotlin/org/libremail/data/sync/MailBackfillerTest.kt @@ -606,6 +606,57 @@ class MailBackfillerTest { coVerify(atLeast = 1) { imapClient.fetchOlderThan(any(), any(), any(), any()) } } + // --- issue #361: Gmail bandwidth-aware prefetch pacing ---------------------------------------- + + /** + * Once a Gmail account's tracked downloads for today reach the documented daily budget, backfill's + * body/attachment prefetch is deferred for the rest of the day — but header paging (the history + * itself) is untouched, the same "prefetch-only" gating shape as the low-battery (#89) and + * fetch-gate (#393) tests above. + */ + @Test + fun `gmail prefetch defers once the daily download budget is reached, but header paging is unaffected`() = runTest { + appendMessages(60) + seedForegroundWindow() + val gmailAccount = accountEntity.copy(imap = ServerConfigEmbedded("imap.gmail.com", 993, "SSL_TLS")) + val tracker = GmailBandwidthTracker().apply { + recordDownload("acct", GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES) + } + + backfiller( + AccountSettings("acct"), + fetchPolicy = FetchPolicy.ALWAYS, + bandwidthTracker = tracker, + account = gmailAccount, + ).runBackfill() + + assertEquals(60, distinctCachedUids().size, "header paging itself is not budget-gated") + coVerify(exactly = 0) { requireNotNull(lastMailRepository).prefetchMessage(any()) } + assertTrue( + logBuffer.snapshot().any { it.message.startsWith("prefetch deferred acct:") }, + "a PII-free deferral breadcrumb is recorded", + ) + } + + /** + * The Gmail bandwidth budget is provider-scoped, not a blanket cap: an over-budget tracker entry + * for the same account id must not affect a non-Gmail account's prefetch. + */ + @Test + fun `a non-gmail account's prefetch is unaffected by an over-budget gmail bandwidth tracker`() = runTest { + appendMessages(60) + seedForegroundWindow() + val tracker = GmailBandwidthTracker().apply { + recordDownload("acct", GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES) + } + + // account defaults to the fixture's non-Gmail (127.0.0.1) host. + backfiller(AccountSettings("acct"), fetchPolicy = FetchPolicy.ALWAYS, bandwidthTracker = tracker) + .runBackfill() + + coVerify(atLeast = 1) { requireNotNull(lastMailRepository).prefetchMessage(any()) } + } + // --- issue #329: AppLog breadcrumbs --------------------------------------------------------- @Test @@ -686,9 +737,11 @@ class MailBackfillerTest { imapClient: ImapClient = client, throttleGate: AccountThrottleGate = AccountThrottleGate(), interactiveGate: InteractiveImapGate = InteractiveImapGate(), + bandwidthTracker: GmailBandwidthTracker = GmailBandwidthTracker(), + account: AccountEntity = accountEntity, ): MailBackfiller { val accountDao = mockk() - coEvery { accountDao.getAll() } returns listOf(accountEntity) + coEvery { accountDao.getAll() } returns listOf(account) val messageDao = mockk(relaxed = true) coEvery { messageDao.insertNew(any()) } answers { @@ -743,6 +796,7 @@ class MailBackfillerTest { maintenanceGate = MailMaintenanceGate(), throttleGate = throttleGate, interactiveGate = interactiveGate, + bandwidthTracker = bandwidthTracker, ).also { lastMessageDao = messageDao lastMailRepository = mailRepository diff --git a/app/src/test/kotlin/org/libremail/data/sync/MailMaintenanceGateTest.kt b/app/src/test/kotlin/org/libremail/data/sync/MailMaintenanceGateTest.kt index 410f941..50790f1 100644 --- a/app/src/test/kotlin/org/libremail/data/sync/MailMaintenanceGateTest.kt +++ b/app/src/test/kotlin/org/libremail/data/sync/MailMaintenanceGateTest.kt @@ -202,6 +202,7 @@ class MailMaintenanceGateTest { maintenanceGate = gate, throttleGate = AccountThrottleGate(), interactiveGate = InteractiveImapGate(), + bandwidthTracker = GmailBandwidthTracker(), ) } diff --git a/app/src/test/kotlin/org/libremail/data/sync/MailSyncConcurrencyTest.kt b/app/src/test/kotlin/org/libremail/data/sync/MailSyncConcurrencyTest.kt index c659e8e..10c0a8c 100644 --- a/app/src/test/kotlin/org/libremail/data/sync/MailSyncConcurrencyTest.kt +++ b/app/src/test/kotlin/org/libremail/data/sync/MailSyncConcurrencyTest.kt @@ -331,6 +331,7 @@ class MailSyncConcurrencyTest { notifier = mockk(relaxed = true), mailRepository = mockk(relaxed = true), throttleGate = AccountThrottleGate(), + bandwidthTracker = GmailBandwidthTracker(), ) } @@ -362,6 +363,7 @@ class MailSyncConcurrencyTest { maintenanceGate = MailMaintenanceGate(), throttleGate = AccountThrottleGate(), interactiveGate = InteractiveImapGate(), + bandwidthTracker = GmailBandwidthTracker(), ) } diff --git a/app/src/test/kotlin/org/libremail/data/sync/MailSyncerTest.kt b/app/src/test/kotlin/org/libremail/data/sync/MailSyncerTest.kt index 1f5e611..0e36516 100644 --- a/app/src/test/kotlin/org/libremail/data/sync/MailSyncerTest.kt +++ b/app/src/test/kotlin/org/libremail/data/sync/MailSyncerTest.kt @@ -95,9 +95,11 @@ class MailSyncerTest { battery: BatteryStatus = BatteryStatus(percent = 100, isCharging = false), fetched: List = emptyList(), throttleGate: AccountThrottleGate = AccountThrottleGate(), + bandwidthTracker: GmailBandwidthTracker = GmailBandwidthTracker(), + accountEntity: AccountEntity = account, ): MailSyncer { val accountDao = mockk() - coEvery { accountDao.getById("acct") } returns account + coEvery { accountDao.getById("acct") } returns accountEntity val messageDao = mockk(relaxed = true) coEvery { messageDao.getSyncedIds(any(), any()) } returns emptyList() coEvery { messageDao.getUnfetchedIds("acct", "INBOX") } returns listOf("acct:INBOX:1") @@ -124,6 +126,7 @@ class MailSyncerTest { notifier = mockk(relaxed = true), mailRepository = mailRepository, throttleGate = throttleGate, + bandwidthTracker = bandwidthTracker, ) } @@ -282,6 +285,43 @@ class MailSyncerTest { coVerify { repo.prefetchMessage("acct:INBOX:1") } } + // --- issue #361: Gmail bandwidth-aware prefetch pacing ---------------------------------------- + + @Test + fun `gmail prefetch defers once the daily download budget is reached, leaving the header sync untouched`() = + runTest { + val repo = mockk(relaxed = true) + val gmailAccount = account.copy(imap = ServerConfigEmbedded("imap.gmail.com", 993, "SSL_TLS")) + val tracker = GmailBandwidthTracker().apply { + recordDownload("acct", GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES) + } + + val result = syncer( + FetchPolicy.ALWAYS, + repo, + accountEntity = gmailAccount, + bandwidthTracker = tracker, + ).syncFolder("acct", "INBOX") + + assertEquals(0, result.getOrNull()) // header sync still ran and succeeded + coVerify(exactly = 0) { repo.prefetchMessage(any()) } + assertTrue(logBuffer.snapshot().any { it.message.startsWith("prefetch deferred acct:") }) + } + + @Test + fun `a non-gmail account's prefetch is unaffected by an over-budget gmail bandwidth tracker`() = runTest { + val repo = mockk() + coEvery { repo.prefetchMessage(any()) } returns Result.success(Unit) + val tracker = GmailBandwidthTracker().apply { + recordDownload("acct", GmailSyncLimits.DAILY_DOWNLOAD_BUDGET_BYTES) + } + + // accountEntity defaults to the fixture's non-Gmail (imap.example.org) host. + syncer(FetchPolicy.ALWAYS, repo, bandwidthTracker = tracker).syncFolder("acct", "INBOX") + + coVerify { repo.prefetchMessage("acct:INBOX:1") } + } + @Test fun `notifies for new mail when both global and per-account notifications are enabled`() = runTest { val notifier = mockk(relaxed = true) @@ -340,6 +380,7 @@ class MailSyncerTest { notifier = notifier, mailRepository = mockk(relaxed = true), throttleGate = AccountThrottleGate(), + bandwidthTracker = GmailBandwidthTracker(), ) } @@ -423,6 +464,7 @@ class MailSyncerTest { notifier = mockk(relaxed = true), mailRepository = mockk(relaxed = true), throttleGate = AccountThrottleGate(), + bandwidthTracker = GmailBandwidthTracker(), ) } diff --git a/config/detekt/detekt.yml b/config/detekt/detekt.yml index 6458d26..3bb9f85 100644 --- a/config/detekt/detekt.yml +++ b/config/detekt/detekt.yml @@ -75,6 +75,9 @@ style: - '**/data/sync/AccountThrottleGateTest.kt' # #356 backfill pacer: cooldown/cap/skip breadcrumbs through AppLog, so this suite mockkStatic(Log) too. - '**/data/sync/BackfillPacerTest.kt' + # #361 Gmail bandwidth tracker: the daily-budget-crossing breadcrumb goes through AppLog, so this + # suite mockkStatic(Log) too. + - '**/data/sync/GmailBandwidthTrackerTest.kt' # Reader-path perf logging (issue #358): the repository's openMessage and the reader ViewModel # log via AppLog, so their unit tests mockkStatic(Log) too. - '**/data/repository/MailRepositoryImplTest.kt'