feat: use sqlite for logging (#20414)

* feat: use drift for logging

* fix: tests

* feat: use the truncate limit from constants.ts as default

* chore: move setupAll to top level and restructure

* chore: code review changes

* fix: inherits

* feat: raise log line limit to 2000

* limit getAll to 250 lines

* delete DLog and make LogRepository not a singleton

* fix: drift build settings and `make migration`

* fix: tests

* remove sensitive log

---------

Co-authored-by: Alex <[email protected]>

Author: bwees

mobile/build.yaml

@@ -19,6 +19,7 @@ targets:
            - lib/infrastructure/entities/*.dart
            - lib/infrastructure/entities/*.drift
            - lib/infrastructure/repositories/db.repository.dart
+           - lib/infrastructure/repositories/logger_db.repository.dart
        drift_dev:modular:
          enabled: true
          options: *drift_options

mobile/lib/constants/constants.dart

@@ -3,7 +3,7 @@ const double downloadCompleted = -1;
 const double downloadFailed = -2;
 
 // Number of log entries to retain on app start
-const int kLogTruncateLimit = 250;
+const int kLogTruncateLimit = 2000;
 
 // Sync
 const int kSyncEventBatchSize = 5000;

mobile/lib/domain/services/hash.service.dart

@@ -6,7 +6,6 @@ import 'package:immich_mobile/infrastructure/repositories/local_album.repository
 import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
 import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
 import 'package:immich_mobile/platform/native_sync_api.g.dart';
-import 'package:immich_mobile/presentation/pages/dev/dev_logger.dart';
 import 'package:logging/logging.dart';
 
 class HashService {
@@ -46,7 +45,6 @@ class HashService {
 
     stopwatch.stop();
     _log.info("Hashing took - ${stopwatch.elapsedMilliseconds}ms");
-    DLog.log("Hashing took - ${stopwatch.elapsedMilliseconds}ms");
   }
 
   /// Processes a list of [LocalAsset]s, storing their hash and updating the assets in the DB
@@ -101,7 +99,6 @@ class HashService {
     }
 
     _log.fine("Hashed ${hashed.length}/${toHash.length} assets");
-    DLog.log("Hashed ${hashed.length}/${toHash.length} assets");
 
     await _localAssetRepository.updateHashes(hashed);
     await _storageRepository.clearCache();

mobile/lib/domain/services/local_sync.service.dart

@@ -6,7 +6,6 @@ import 'package:immich_mobile/domain/models/album/local_album.model.dart';
 import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
 import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart';
 import 'package:immich_mobile/platform/native_sync_api.g.dart';
-import 'package:immich_mobile/presentation/pages/dev/dev_logger.dart';
 import 'package:immich_mobile/utils/diff.dart';
 import 'package:logging/logging.dart';
 import 'package:platform/platform.dart';
@@ -30,19 +29,17 @@ class LocalSyncService {
     try {
       if (full || await _nativeSyncApi.shouldFullSync()) {
         _log.fine("Full sync request from ${full ? "user" : "native"}");
-        DLog.log("Full sync request from ${full ? "user" : "native"}");
         return await fullSync();
       }
 
       final delta = await _nativeSyncApi.getMediaChanges();
       if (!delta.hasChanges) {
         _log.fine("No media changes detected. Skipping sync");
-        DLog.log("No media changes detected. Skipping sync");
         return;
       }
 
-      DLog.log("Delta updated: ${delta.updates.length}");
-      DLog.log("Delta deleted: ${delta.deletes.length}");
+      _log.fine("Delta updated: ${delta.updates.length}");
+      _log.fine("Delta deleted: ${delta.deletes.length}");
 
       final deviceAlbums = await _nativeSyncApi.getAlbums();
       await _localAlbumRepository.updateAll(deviceAlbums.toLocalAlbums());
@@ -83,7 +80,6 @@ class LocalSyncService {
     } finally {
       stopwatch.stop();
       _log.info("Device sync took - ${stopwatch.elapsedMilliseconds}ms");
-      DLog.log("Device sync took - ${stopwatch.elapsedMilliseconds}ms");
     }
   }
 
@@ -106,7 +102,6 @@ class LocalSyncService {
       await _nativeSyncApi.checkpointSync();
       stopwatch.stop();
       _log.info("Full device sync took - ${stopwatch.elapsedMilliseconds}ms");
-      DLog.log("Full device sync took - ${stopwatch.elapsedMilliseconds}ms");
     } catch (e, s) {
       _log.severe("Error performing full device sync", e, s);
     }
@@ -150,7 +145,6 @@ class LocalSyncService {
       // Faster path - only new assets added
       if (await checkAddition(dbAlbum, deviceAlbum)) {
         _log.fine("Fast synced device album ${dbAlbum.name}");
-        DLog.log("Fast synced device album ${dbAlbum.name}");
         return true;
       }
 

mobile/lib/domain/services/log.service.dart

@@ -14,7 +14,7 @@ import 'package:logging/logging.dart';
 /// writes them to a persistent [ILogRepository], and manages log levels
 /// via [IStoreRepository]
 class LogService {
-  final IsarLogRepository _logRepository;
+  final LogRepository _logRepository;
   final IsarStoreRepository _storeRepository;
 
   final List<LogMessage> _msgBuffer = [];
@@ -37,7 +37,7 @@ class LogService {
   }
 
   static Future<LogService> init({
-    required IsarLogRepository logRepository,
+    required LogRepository logRepository,
     required IsarStoreRepository storeRepository,
     bool shouldBuffer = true,
   }) async {
@@ -50,7 +50,7 @@ class LogService {
   }
 
   static Future<LogService> create({
-    required IsarLogRepository logRepository,
+    required LogRepository logRepository,
     required IsarStoreRepository storeRepository,
     bool shouldBuffer = true,
   }) async {
@@ -85,7 +85,7 @@ class LogService {
 
     if (_shouldBuffer) {
       _msgBuffer.add(record);
-      _flushTimer ??= Timer(const Duration(seconds: 5), () => unawaited(flushBuffer()));
+      _flushTimer ??= Timer(const Duration(seconds: 5), () => unawaited(_flushBuffer()));
     } else {
       unawaited(_logRepository.insert(record));
     }
@@ -108,20 +108,18 @@ class LogService {
     await _logRepository.deleteAll();
   }
 
-  void flush() {
+  Future<void> flush() {
     _flushTimer?.cancel();
-    // TODO: Rename enable this after moving to sqlite - #16504
-    // await _flushBufferToDatabase();
+    return _flushBuffer();
   }
 
   Future<void> dispose() {
     _flushTimer?.cancel();
     _logSubscription.cancel();
-    return flushBuffer();
+    return _flushBuffer();
   }
 
-  // TOOD: Move this to private once Isar is removed
-  Future<void> flushBuffer() async {
+  Future<void> _flushBuffer() async {
     _flushTimer = null;
     final buffer = [..._msgBuffer];
     _msgBuffer.clear();

mobile/lib/domain/services/sync_stream.service.dart

@@ -3,7 +3,6 @@ import 'dart:async';
 import 'package:immich_mobile/domain/models/sync_event.model.dart';
 import 'package:immich_mobile/infrastructure/repositories/sync_api.repository.dart';
 import 'package:immich_mobile/infrastructure/repositories/sync_stream.repository.dart';
-import 'package:immich_mobile/presentation/pages/dev/dev_logger.dart';
 import 'package:logging/logging.dart';
 import 'package:openapi/api.dart';
 
@@ -26,7 +25,6 @@ class SyncStreamService {
 
   Future<void> sync() {
     _logger.info("Remote sync request for user");
-    DLog.log("Remote sync request for user");
     // Start the sync stream and handle events
     return _syncApiRepository.streamChanges(_handleEvents);
   }

mobile/lib/infrastructure/entities/log.entity.dart

@@ -1,47 +1,29 @@
-import 'package:immich_mobile/domain/models/log.model.dart';
-import 'package:isar/isar.dart';
+import 'package:drift/drift.dart';
+import 'package:immich_mobile/infrastructure/entities/log.entity.drift.dart';
+import 'package:immich_mobile/domain/models/log.model.dart' as domain;
 
-part 'log.entity.g.dart';
+class LogMessageEntity extends Table {
+  const LogMessageEntity();
 
-@Collection(inheritance: false)
-class LoggerMessage {
-  final Id id = Isar.autoIncrement;
-  final String message;
-  final String? details;
-  @Enumerated(EnumType.ordinal)
-  final LogLevel level;
-  final DateTime createdAt;
-  final String? context1;
-  final String? context2;
+  @override
+  String get tableName => 'logger_messages';
 
-  const LoggerMessage({
-    required this.message,
-    required this.details,
-    this.level = LogLevel.info,
-    required this.createdAt,
-    required this.context1,
-    required this.context2,
-  });
-
-  LogMessage toDto() {
-    return LogMessage(
-      message: message,
-      level: level,
-      createdAt: createdAt,
-      logger: context1,
-      error: details,
-      stack: context2,
-    );
-  }
+  IntColumn get id => integer().autoIncrement()();
+  TextColumn get message => text()();
+  TextColumn get details => text().nullable()();
+  IntColumn get level => intEnum<domain.LogLevel>()();
+  DateTimeColumn get createdAt => dateTime()();
+  TextColumn get logger => text().nullable()();
+  TextColumn get stack => text().nullable()();
+}
 
-  static LoggerMessage fromDto(LogMessage log) {
-    return LoggerMessage(
-      message: log.message,
-      details: log.error,
-      level: log.level,
-      createdAt: log.createdAt,
-      context1: log.logger,
-      context2: log.stack,
-    );
-  }
+extension LogMessageEntityDataDomainEx on LogMessageEntityData {
+  domain.LogMessage toDto() => domain.LogMessage(
+    message: message,
+    level: level,
+    createdAt: createdAt,
+    logger: logger,
+    error: details,
+    stack: stack,
+  );
 }