Use Hive CE database to replace Hive database

Signed-off-by: dab246 <tdvu@linagora.com>
This commit is contained in:
dab246
2025-06-18 00:57:30 +07:00
committed by Dat H. Pham
parent 23019ae030
commit 45cd1c3613
35 changed files with 163 additions and 81 deletions
+33
View File
@@ -0,0 +1,33 @@
# 66. Synchronize caching
Date: 2025-06-18
## Status
Accepted
## Context
Synchronizing server and database data is quite troublesome,
when we use many protocols to receive mails (FCM, WebSocket, Memory operator).
In addition, we are using `Hive` database to store data,
it does not support multiple isolate to access the database.
This leads to inconsistent database data, sometimes corrupted and lost data.
## Decision
To overcome this problem we need a database that can support multiple isolate.
So we need to use [Hive_CE](https://pub.dev/packages/hive_ce) to replace `Hive` database.
Hive CE is a spiritual continuation of Hive v2 with the following new features, such as:
- Isolate support through IsolatedHive
- Automatic type adapter generation using the GenerateAdapters annotation
- No more manually adding annotations to every type and field
- Generate adapters for classes outside the current package
- A HiveRegistrar extension that lets you register all your generated adapters in one call
- Extends the maximum type ID from 223 to 65439
...
## Consequences
- Data is always synchronized with the latest
@@ -0,0 +1,17 @@
import 'package:tmail_ui_user/features/base/upgradeable/upgrade_database_steps.dart';
import 'package:tmail_ui_user/features/caching/caching_manager.dart';
class UpgradeHiveDatabaseStepsV16 extends UpgradeDatabaseSteps {
final CachingManager _cachingManager;
UpgradeHiveDatabaseStepsV16(this._cachingManager);
@override
Future<void> onUpgrade(int oldVersion, int newVersion) async {
if (oldVersion > 0 && oldVersion < newVersion && newVersion == 17) {
await _cachingManager.clearAll();
}
}
}
-17
View File
@@ -87,23 +87,6 @@ class CachingManager {
], eagerError: true);
}
Future<void> clearData() async {
await Future.wait([
_stateCacheClient.clearAllData(),
_mailboxCacheClient.clearAllData(),
_emailCacheClient.clearAllData(),
_fcmCacheClient.clearAllData(),
_firebaseRegistrationCacheClient.clearAllData(),
_localSpamReportManager.clear(),
if (PlatformInfo.isMobile)
...[
_newEmailHiveCacheClient.clearAllData(),
_openedEmailHiveCacheClient.clearAllData(),
_sendingEmailCacheManager.clearAllSendingEmails(),
]
], eagerError: true);
}
Future<void> clearEmailAndStateCache({AccountId? accountId, UserName? userName}) {
log('CachingManager::clearEmailAndStateCache:userName = $userName');
if (accountId != null && userName != null) {
@@ -1,4 +1,4 @@
class CacheVersion {
static const int hiveDBVersion = 16;
static const int hiveDBVersion = 17;
}
@@ -0,0 +1,19 @@
import 'dart:ui' as flutter;
import 'package:hive_ce/hive.dart';
class FcmIsolateNameServer extends IsolateNameServer {
const FcmIsolateNameServer();
@override
dynamic lookupPortByName(String name) =>
flutter.IsolateNameServer.lookupPortByName(name);
@override
bool registerPortWithName(dynamic port, String name) =>
flutter.IsolateNameServer.registerPortWithName(port, name);
@override
bool removePortNameMapping(String name) =>
flutter.IsolateNameServer.removePortNameMapping(name);
}
@@ -2,7 +2,7 @@ import 'dart:typed_data';
import 'package:core/presentation/extensions/map_extensions.dart';
import 'package:core/utils/app_logger.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/config/hive_cache_config.dart';
import 'package:tmail_ui_user/features/caching/utils/cache_utils.dart';
@@ -14,20 +14,20 @@ abstract class HiveCacheClient<T> {
Future<Uint8List?> _getEncryptionKey() => HiveCacheConfig.instance.getEncryptionKey();
Future<Box<T>> openBox() async {
if (Hive.isBoxOpen(tableName)) {
return Hive.box<T>(tableName);
Future<IsolatedBox<T>> openBox() async {
if (IsolatedHive.isBoxOpen(tableName)) {
return IsolatedHive.box<T>(tableName);
} else {
return Hive.openBox<T>(tableName);
return IsolatedHive.openBox<T>(tableName);
}
}
Future<Box<T>> openBoxEncryption() async {
Future<IsolatedBox<T>> openBoxEncryption() async {
final encryptionKey = await _getEncryptionKey();
if (Hive.isBoxOpen(tableName)) {
return Hive.box<T>(tableName);
if (IsolatedHive.isBoxOpen(tableName)) {
return IsolatedHive.box<T>(tableName);
} else {
return Hive.openBox<T>(tableName,
return IsolatedHive.openBox<T>(tableName,
encryptionCipher:
encryptionKey != null ? HiveAesCipher(encryptionKey) : null);
}
@@ -66,7 +66,8 @@ abstract class HiveCacheClient<T> {
Future<List<T>> getAll() {
return Future.sync(() async {
final boxItem = encryption ? await openBoxEncryption() : await openBox();
return boxItem.values.toList();
final items = await boxItem.values;
return items.toList();
}).catchError((error) {
throw error;
});
@@ -75,7 +76,8 @@ abstract class HiveCacheClient<T> {
Future<List<T>> getListByNestedKey(String nestedKey) {
return Future.sync(() async {
final boxItem = encryption ? await openBoxEncryption() : await openBox();
final listItem = boxItem.toMap()
final mapItems = await boxItem.toMap();
final listItem = mapItems
.where((key, value) => _matchedNestedKey(key, nestedKey))
.values
.toList();
@@ -89,7 +91,8 @@ abstract class HiveCacheClient<T> {
Future<List<T>> getValuesByListKey(List<String> listKeys) {
return Future.sync(() async {
final boxItem = encryption ? await openBoxEncryption() : await openBox();
return boxItem.toMap()
final mapItems = await boxItem.toMap();
return mapItems
.where((key, value) => listKeys.contains(key))
.values
.toList();
@@ -154,7 +157,7 @@ abstract class HiveCacheClient<T> {
}
Future<void> deleteBox() {
return Hive.deleteBoxFromDisk(tableName);
return IsolatedHive.deleteBoxFromDisk(tableName);
}
Future<void> clearAllData() {
@@ -171,7 +174,8 @@ abstract class HiveCacheClient<T> {
Future<void> clearAllDataContainKey(String nestedKey) {
return Future.sync(() async {
final boxItem = encryption ? await openBoxEncryption() : await openBox();
final listKeys = boxItem.toMap()
final mapItems = await boxItem.toMap();
final listKeys = mapItems
.where((key, value) => _matchedNestedKey(key, nestedKey))
.keys;
log('$runtimeType::clearAllDataContainKey:listKeys: ${listKeys.length}');
@@ -182,8 +186,8 @@ abstract class HiveCacheClient<T> {
}
Future<void> closeBox() async {
if (Hive.isBoxOpen(tableName)) {
await Hive.box<T>(tableName).close();
if (IsolatedHive.isBoxOpen(tableName)) {
await IsolatedHive.box<T>(tableName).close();
}
return Future.value();
}
@@ -4,7 +4,7 @@ import 'dart:typed_data';
import 'package:core/utils/app_logger.dart';
import 'package:core/utils/platform_info.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:path_provider/path_provider.dart' as path_provider;
import 'package:tmail_ui_user/features/base/upgradeable/upgrade_hive_database_steps_v10.dart';
import 'package:tmail_ui_user/features/base/upgradeable/upgrade_hive_database_steps_v11.dart';
@@ -16,6 +16,7 @@ import 'package:tmail_ui_user/features/base/upgradeable/upgrade_hive_database_st
import 'package:tmail_ui_user/features/base/upgradeable/upgrade_hive_database_steps_v7.dart';
import 'package:tmail_ui_user/features/caching/caching_manager.dart';
import 'package:tmail_ui_user/features/caching/config/cache_version.dart';
import 'package:tmail_ui_user/features/caching/config/fcm_isolate_nam_server.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
import 'package:tmail_ui_user/features/home/data/model/session_hive_obj.dart';
import 'package:tmail_ui_user/features/login/data/local/encryption_key_cache_manager.dart';
@@ -55,14 +56,17 @@ class HiveCacheConfig {
}
Future<void> initializeDatabase({String? databasePath}) async {
if (databasePath != null) {
Hive.init(databasePath);
} else {
if (PlatformInfo.isMobile) {
Directory directory = await path_provider.getApplicationDocumentsDirectory();
Hive.init(directory.path);
}
if (databasePath == null && PlatformInfo.isMobile) {
Directory directory = await path_provider.getApplicationDocumentsDirectory();
databasePath = directory.path;
}
if (databasePath == null) return;
await IsolatedHive.init(
databasePath,
isolateNameServer: const FcmIsolateNameServer(),
);
}
Future<void> onUpgradeDatabase(CachingManager cachingManager) async {
@@ -78,6 +82,7 @@ class HiveCacheConfig {
await UpgradeHiveDatabaseStepsV14(cachingManager).onUpgrade(oldVersion, newVersion);
await UpgradeHiveDatabaseStepsV15(cachingManager).onUpgrade(oldVersion, newVersion);
await UpgradeHiveDatabaseStepsV16(cachingManager).onUpgrade(oldVersion, newVersion);
await UpgradeHiveDatabaseStepsV16(cachingManager).onUpgrade(oldVersion, newVersion);
if (oldVersion != newVersion) {
await cachingManager.storeCacheVersion(newVersion);
@@ -201,12 +206,12 @@ class HiveCacheConfig {
}
void registerCacheAdapter<T>(TypeAdapter<T> typeAdapter, int typeId) {
if (!Hive.isAdapterRegistered(typeId)) {
Hive.registerAdapter<T>(typeAdapter);
if (!IsolatedHive.isAdapterRegistered(typeId)) {
IsolatedHive.registerAdapter<T>(typeAdapter);
}
}
Future<void> closeHive() async {
return await Hive.close();
return await IsolatedHive.close();
}
}
@@ -1,5 +1,5 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'session_hive_obj.g.dart';
@@ -1,5 +1,5 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'account_cache.g.dart';
@@ -1,5 +1,5 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'authentication_info_cache.g.dart';
@@ -1,5 +1,5 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'encryption_key_cache.g.dart';
@@ -1,5 +1,5 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'oidc_configuration_cache.g.dart';
@@ -1,5 +1,5 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:jmap_dart_client/jmap/core/extensions/date_time_extension.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
import 'package:tmail_ui_user/features/login/domain/model/recent_login_url.dart';
@@ -1,5 +1,5 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:jmap_dart_client/jmap/core/extensions/date_time_extension.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
import 'package:tmail_ui_user/features/login/domain/model/recent_login_username.dart';
@@ -1,5 +1,5 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'token_oidc_cache.g.dart';
@@ -1,6 +1,6 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
import 'package:tmail_ui_user/features/mailbox/data/model/mailbox_rights_cache.dart';
@@ -1,6 +1,6 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'mailbox_rights_cache.g.dart';
@@ -1,6 +1,6 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
import 'package:tmail_ui_user/features/mailbox/data/model/state_type.dart';
@@ -1,5 +1,5 @@
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:jmap_dart_client/jmap/account_id.dart';
import 'package:jmap_dart_client/jmap/core/user_name.dart';
import 'package:model/extensions/account_id_extensions.dart';
@@ -1,6 +1,6 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:jmap_dart_client/jmap/core/extensions/date_time_extension.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
import 'package:tmail_ui_user/features/mailbox_dashboard/domain/model/recent_search.dart';
@@ -36,7 +36,7 @@ abstract class WorkerQueue<A> {
}
void _handleTaskExecuteCompleted(dynamic value) {
log('WorkerQueue<$workerName>::_handleTaskExecuteCompleted(): $value');
log('WorkerQueue<$workerName>::_handleTaskExecuteCompleted():');
completer?.complete();
_releaseCompleter();
if (queue.isNotEmpty) {
@@ -25,7 +25,7 @@ class NewEmailCacheManager {
DetailedEmailHiveCache detailedEmailCache
) async {
final listDetailedEmails = await getAllDetailedEmails(accountId, userName);
log('NewEmailCacheManager::storeDetailedNewEmail():listDetailedEmails: $listDetailedEmails');
log('NewEmailCacheManager::storeDetailedNewEmail(): length of listDetailedEmails is ${listDetailedEmails.length}');
if (listDetailedEmails.length >= CachingConstants.maxNumberNewEmailsForOffline) {
final lastElementsListEmail = listDetailedEmails.sublist(CachingConstants.maxNumberNewEmailsForOffline - 1);
for (var email in lastElementsListEmail) {
@@ -34,10 +34,8 @@ class NewEmailCacheManager {
}
await removeDetailedEmail(accountId, userName, email.emailId);
}
log('NewEmailCacheManager::storeDetailedNewEmail(): DELETE COMPLETED');
}
await insertDetailedEmail(accountId, userName, detailedEmailCache);
log('NewEmailCacheManager::storeDetailedNewEmail(): INSERT COMPLETED');
return detailedEmailCache;
}
@@ -1,6 +1,6 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'attachment_hive_cache.g.dart';
@@ -1,6 +1,6 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
import 'package:tmail_ui_user/features/offline_mode/model/email_header_hive_cache.dart';
@@ -1,6 +1,6 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'email_header_hive_cache.g.dart';
@@ -1,6 +1,6 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'sending_email_hive_cache.g.dart';
@@ -1,6 +1,6 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'firebase_registration_cache.g.dart';
@@ -183,6 +183,7 @@ class WebSocketController extends PushBaseController {
}
void _enableWebSocketPush() {
log('WebSocketController::_enableWebSocketPush:');
_webSocketChannel?.sink.add(jsonEncode(WebSocketPushEnableRequest.toJson(
dataTypes: [TypeName.emailType, TypeName.mailboxType]
)));
@@ -197,6 +198,7 @@ class WebSocketController extends PushBaseController {
}
void _listenToWebSocket() {
log('WebSocketController::_listenToWebSocket:');
_webSocketSubscription = _webSocketChannel?.stream.listen(
(data) {
log('WebSocketController::_listenToWebSocket(): $data');
@@ -225,6 +227,7 @@ class WebSocketController extends PushBaseController {
}
void _initStateChangeDeouncerTimer() {
log('WebSocketController::_initStateChangeDeouncerTimer:');
_stateChangeDebouncer = Debouncer<StateChange?>(
const Duration(milliseconds: FcmUtils.durationMessageComing),
initialValue: null,
@@ -56,16 +56,12 @@ extension EmailCacheExtension on EmailCache {
bool expireTimeCaching(EmailCleanupRule cleanupRule) {
final currentTime = DateTime.now();
log('EmailCacheExtension::expireTimeCaching():currentTime: $currentTime | receivedAt: $receivedAt');
if (receivedAt != null) {
final countDay = currentTime.daysBetween(receivedAt!.toLocal());
log('EmailCacheExtension::expireTimeCaching():countDay: $countDay | cleanupRule.cachingEmailPeriod.countDay: ${cleanupRule.cachingEmailPeriod.countDay}');
if (countDay >= cleanupRule.cachingEmailPeriod.countDay) {
log('EmailCacheExtension::expireTimeCaching():true');
return true;
}
}
log('EmailCacheExtension::expireTimeCaching():false');
return false;
}
}
@@ -1,5 +1,5 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
part 'email_address_hive_cache.g.dart';
@@ -1,6 +1,6 @@
import 'package:equatable/equatable.dart';
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/utils/caching_constants.dart';
import 'package:tmail_ui_user/features/thread/data/model/email_address_hive_cache.dart';
@@ -43,7 +43,6 @@ class LocalizationService extends Translations {
static Locale getLocaleFromLanguage({String? langCode}) {
try {
final languageCacheManager = getBinding<LanguageCacheManager>();
log('LocalizationService::_getLocaleFromLanguage:languageCacheManager: $languageCacheManager');
final localeStored = languageCacheManager?.getStoredLanguage();
log('LocalizationService::_getLocaleFromLanguage():localeStored: $localeStored');
final localeSelected = supportedLocales.firstWhereOrNull(
+20 -3
View File
@@ -1224,13 +1224,22 @@ packages:
source: hosted
version: "2.1.0"
hive:
dependency: "direct overridden"
description:
path: "overrides/hive"
ref: "18f92b53295e9eb77ebd4830d905a72cd404a126"
resolved-ref: "18f92b53295e9eb77ebd4830d905a72cd404a126"
url: "https://github.com/IO-Design-Team/hive_ce"
source: git
version: "0.0.0"
hive_ce:
dependency: "direct main"
description:
name: hive
sha256: "8dcf6db979d7933da8217edcec84e9df1bdb4e4edc7fc77dbd5aa74356d6d941"
name: hive_ce
sha256: "708bb39050998707c5d422752159f91944d3c81ab42d80e1bd0ee37d8e130658"
url: "https://pub.dev"
source: hosted
version: "2.2.3"
version: "2.11.3"
hive_generator:
dependency: "direct dev"
description:
@@ -1344,6 +1353,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.0.4"
isolate_channel:
dependency: transitive
description:
name: isolate_channel
sha256: f3d36f783b301e6b312c3450eeb2656b0e7d1db81331af2a151d9083a3f6b18d
url: "https://pub.dev"
source: hosted
version: "0.2.2+1"
jmap_dart_client:
dependency: "direct main"
description:
+9 -1
View File
@@ -154,7 +154,7 @@ dependencies:
share_plus: 7.2.1
hive: 2.2.3
hive_ce: 2.11.3
pointer_interceptor: 0.10.1+2
@@ -309,6 +309,14 @@ dependency_overrides:
url: https://github.com/linagora/flutter_file_picker.git
ref: v0.8.5-twake-mail
# TODO: Due conflict version of hive_ce_generator with other dependencies
# So we will remove it when found the appropriate version
hive:
git:
url: https://github.com/IO-Design-Team/hive_ce
ref: 18f92b53295e9eb77ebd4830d905a72cd404a126
path: overrides/hive
# For information on the generic Dart part of this file, see the
# following page: https://dart.dev/tools/pub/pubspec
@@ -1,4 +1,4 @@
import 'package:hive/hive.dart';
import 'package:hive_ce/hive.dart';
import 'package:tmail_ui_user/features/caching/clients/account_cache_client.dart';
import 'package:tmail_ui_user/features/login/data/model/account_cache.dart';
@@ -83,12 +83,12 @@ class MemoryAccountCacheClient implements AccountCacheClient {
}
@override
Future<Box<AccountCache>> openBox() {
Future<IsolatedBox<AccountCache>> openBox() {
throw UnimplementedError();
}
@override
Future<Box<AccountCache>> openBoxEncryption() {
Future<IsolatedBox<AccountCache>> openBoxEncryption() {
throw UnimplementedError();
}