From d3c73d77f9ad7fa523062bdca4d808e964826675 Mon Sep 17 00:00:00 2001 From: Tom Levy Date: Thu, 21 Mar 2024 16:38:58 +0000 Subject: [PATCH] Make import a bit faster by bulk-inserting SMS messages Importing is very slow when there are a lot of messages. The new code is a bit faster, but unfortunately still rather slow. There isn't much more that we can do because the majority of the time is spent in the bulkInsert() call waiting for the Telephony provider, so it's outside our control. The new code has mostly the same behaviour as the old code, but there are some differences: - The error handling isn't perfect. If one message in a batch causes an error, then the other messages in that batch may not be imported. (The previous code tried to import every single message even if some of them caused exceptions.) It's possible to change the code so that it re-tries the messages individually when a batch fails, but that's complicated. - Existing messages are skipped as before, however if the backup file itself contains duplicate messages then they might not be de-duplicated (it depends on whether they get processed in the same batch). - If there are errors while querying for existing messages, they are displayed using a toast. The old code (which still exists but is now unused) silently ignores errors in the queryCursor() call; it's not clear to me whether that's intentional. The new code is more complicated, so it's important to know if it fails, and the batching significantly reduces the number of toasts that might be shown. This commit doesn't use bulk-insert for MMS, because that's more complicated and because MMS messages are typically large so the overhead of single insertion is less significant. Fixes #TODO. --- .../messages/helpers/MessagesImporter.kt | 28 ++++++---- .../messages/helpers/MessagesWriter.kt | 55 +++++++++++++++++-- 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/app/src/main/kotlin/org/fossify/messages/helpers/MessagesImporter.kt b/app/src/main/kotlin/org/fossify/messages/helpers/MessagesImporter.kt index 9faa06a63..feb95c111 100644 --- a/app/src/main/kotlin/org/fossify/messages/helpers/MessagesImporter.kt +++ b/app/src/main/kotlin/org/fossify/messages/helpers/MessagesImporter.kt @@ -61,18 +61,26 @@ class MessagesImporter(private val activity: SimpleActivity) { fun restoreMessages(messagesBackup: List, callback: (ImportResult) -> Unit) { ensureBackgroundThread { try { - messagesBackup.forEach { message -> - try { - if (message.backupType == BackupType.SMS && config.importSms) { - messageWriter.writeSmsMessage(message as SmsBackup) - messagesImported++ - } else if (message.backupType == BackupType.MMS && config.importMms) { - messageWriter.writeMmsMessage(message as MmsBackup) + if (config.importSms) { + messagesBackup.filterIsInstance().chunked(999) { chunk -> + try { + messageWriter.bulkWriteSmsMessages(chunk) + messagesImported += chunk.size + } catch (e: Exception) { + activity.showErrorToast(e) + messagesFailed += chunk.size + } + } + } + if (config.importMms) { + messagesBackup.filterIsInstance().forEach { mmsBackup -> + try { + messageWriter.writeMmsMessage(mmsBackup) messagesImported++ + } catch (e: Exception) { + activity.showErrorToast(e) + messagesFailed++ } - } catch (e: Exception) { - activity.showErrorToast(e) - messagesFailed++ } } refreshMessages() diff --git a/app/src/main/kotlin/org/fossify/messages/helpers/MessagesWriter.kt b/app/src/main/kotlin/org/fossify/messages/helpers/MessagesWriter.kt index 52f474fce..da140bf33 100644 --- a/app/src/main/kotlin/org/fossify/messages/helpers/MessagesWriter.kt +++ b/app/src/main/kotlin/org/fossify/messages/helpers/MessagesWriter.kt @@ -1,6 +1,7 @@ package org.fossify.messages.helpers import android.annotation.SuppressLint +import android.content.ContentValues import android.content.Context import android.net.Uri import android.provider.Telephony.Mms @@ -19,16 +20,22 @@ import org.fossify.messages.models.SmsBackup class MessagesWriter(private val context: Context) { private val INVALID_ID = -1L private val contentResolver = context.contentResolver + private val threadIdCache = HashMap() fun writeSmsMessage(smsBackup: SmsBackup) { - val contentValues = smsBackup.toContentValues() - val threadId = Utils.getOrCreateThreadId(context, smsBackup.address) - contentValues.put(Sms.THREAD_ID, threadId) if (!smsExist(smsBackup)) { - contentResolver.insert(Sms.CONTENT_URI, contentValues) + contentResolver.insert(Sms.CONTENT_URI, smsToContentValuesWithThreadId(smsBackup)) } } + fun bulkWriteSmsMessages(smsBackups: List) { + // the batch size must be at most 999 (see bulkSmsExist) + val exist = bulkSmsExist(smsBackups) + val newSmsBackups = smsBackups.filterIndexed { i, _ -> !exist[i] } + val contentValues = newSmsBackups.map { smsToContentValuesWithThreadId(it) }.toTypedArray() + contentResolver.bulkInsert(Sms.CONTENT_URI, contentValues) + } + private fun smsExist(smsBackup: SmsBackup): Boolean { val uri = Sms.CONTENT_URI val projection = arrayOf(Sms._ID) @@ -41,6 +48,46 @@ class MessagesWriter(private val context: Context) { return exists } + private fun bulkSmsExist(smsBackups: List): BooleanArray { + // the number of messages must be at most 999, otherwise this might fail with: + // android.database.sqlite.SQLiteException: too many SQL variables (code 1) + // (it's a limit from older versions of SQLite, increased in https://sqlite.org/src/info/2def75693a8ae002) + // + // it's a tricky to make bulk queries with ContentResolver.query() + // our approach is to query multiple timestamps in a single query + // and then check the address and type columns separately + // (the timestamps should be mostly unique, so this is efficient) + val dates = smsBackups.map { it.date }.distinct() + val existingMessages = HashSet>() // a message is represented as a (date, address, type) triple + if (!dates.isEmpty()) { + val uri = Sms.CONTENT_URI + val projection = arrayOf(Sms.DATE, Sms.ADDRESS, Sms.TYPE) + val selectionParams = "?" + ",?".repeat(dates.size - 1) + val selection = "${Sms.DATE} IN (${selectionParams})" + val selectionArgs = dates.map { it.toString() }.toTypedArray() + context.queryCursor(uri, projection, selection, selectionArgs, showErrors = true) { + val date = it.getLong(0) + val address = it.getString(1) + val type = it.getInt(2) + existingMessages.add(Triple(date, address, type)) + } + } + return smsBackups.map { existingMessages.contains(Triple(it.date, it.address, it.type)) }.toBooleanArray() + } + + private fun smsToContentValuesWithThreadId(smsBackup: SmsBackup): ContentValues { + val contentValues = smsBackup.toContentValues() + val threadId = getOrCreateThreadId(smsBackup.address) + contentValues.put(Sms.THREAD_ID, threadId) + return contentValues + } + + private fun getOrCreateThreadId(recipient: String): Long { + return threadIdCache.getOrPut(recipient) { + Utils.getOrCreateThreadId(context, recipient) + } + } + fun writeMmsMessage(mmsBackup: MmsBackup) { // 1. write mms msg, get the msg_id, check if mms exists before writing // 2. write parts - parts depend on the msg id, check if part exist before writing, write data if it is a non-text part