TF-4265 Add include list to support custom keywords detection

lib/features/composer/presentation/manager/attachment_text_detector.dart

@@ -1,5 +1,6 @@
 import 'package:flutter/foundation.dart';
 import 'package:tmail_ui_user/features/composer/presentation/manager/detection_params.dart';
+import 'package:tmail_ui_user/features/composer/presentation/manager/exclude_list_filter.dart';
 import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart';
 import 'package:tmail_ui_user/main/localizations/language_code_constants.dart';
 
@@ -50,8 +51,9 @@ class AttachmentTextDetector {
     ],
   };
 
-  static RegExp? _combinedRegExp;
+  /// Cached RegExp for the default case (when no include list is provided).
-  /// SYNC/ASYNC EXECUTION THRESHOLD
+  static RegExp? _cachedDefaultRegExp;
+  /// Threshold to switch from Sync to Async execution.
   ///
   /// Value: 20,000 characters (approx. 4-5 pages of text).
   ///
@@ -114,55 +116,61 @@ class AttachmentTextDetector {
     return result;
   }
 
-  /// Initializes the single optimized RegExp pattern.
+  /// Builds the Regex pattern.
-  static void _initializeRegExp() {
+  /// Merges [defaultKeywords] with [includeList] to create the search scope.
-    if (_combinedRegExp != null) return;
+  static RegExp _buildRegExp(List<String> additionalKeywords) {
+    // 1. Get all default keywords
+    final defaultKeywords = _keywordsByLang.values.expand((l) => l).toList();
 
-    final allKeywords =
+    // 2. Merge with Include List (Add user-defined keywords)
-        _keywordsByLang.values.expand((l) => l).toSet().toList();
+    final allKeywords = {...defaultKeywords, ...additionalKeywords}.toList();
 
-    // Sort by length descending (Longest first).
+    // 3. Sort by length descending.
-    // Crucial to ensure "attachment" is matched before "attach".
+    // Crucial to match "attachment" (long) before "attach" (short).
     allKeywords.sort((a, b) => b.length.compareTo(a.length));
 
-    // LOGIC:
+    // 4. Create Pattern: (keyword1|keyword2)(?![\p{L}])
-    // 1. RegExp.escape: Neutralizes special characters.
+    // (?![\p{L}]) ensures we don't match substrings inside other words (e.g., "filetage").
-    // 2. No Nested Quantifiers: Prevents catastrophic backtracking.
-    // 3. Linear Scan: Uses logical OR (|) for a single-pass scan.
     final pattern = allKeywords.map(RegExp.escape).join('|');
 
-    // Pattern: (keyword1|keyword2)(?![\p{L}])
+    return RegExp(
-    // (?![\p{L}]): Negative lookahead to ensure the next char is NOT a letter.
-    // Allows: numbers (file123), punctuation (file.), spaces (file ).
-    // Rejects: letter extensions (filetage).
-    _combinedRegExp = RegExp(
       '($pattern)(?![\\p{L}])',
       unicode: true,
       caseSensitive: false,
     );
   }
 
-  static List<String> _processMatchedKeywordsInIsolate(DetectionParams params) {
+  /// The core logic function.
-    // Re-initialize RegExp (Isolates do not share static memory with the main app)
+  /// 1. Builds/Retrieves the Regex.
-    _initializeRegExp();
+  /// 2. Scans the text.
-
+  /// 3. Filters results using Exclude List.
+  static List<String> _processInIsolate(DetectionParams params) {
     final text = params.text;
-    final filters = params.filters;
-
     if (text.isEmpty) return [];
 
-    final matches = _combinedRegExp!.allMatches(text);
+    RegExp regex;
+
+    // Use cached Regex if there are no custom keywords.
+    if (params.includeList.isEmpty) {
+      _cachedDefaultRegExp ??= _buildRegExp([]);
+      regex = _cachedDefaultRegExp!;
+    } else {
+      // Rebuild Regex if Include List is present.
+      regex = _buildRegExp(params.includeList);
+    }
+
+    final matches = regex.allMatches(text);
     final result = <String>{};
 
     for (final match in matches) {
       final keyword = match.group(0)!.toLowerCase();
 
-      // Apply Filter Pipeline
+      // Check if the found keyword should be excluded based on context.
       bool isAccepted = true;
-      for (final filter in filters) {
+      for (final filter in params.filters) {
         if (!filter.isValid(text, match)) {
           isAccepted = false;
-          break; // Stop checking other filters to save CPU
+          break;
         }
       }
 
@@ -174,25 +182,34 @@ class AttachmentTextDetector {
     return result.toList();
   }
 
-  /// Smart keyword detection (Hybrid Sync/Async).
+  /// Detects keywords in the text.
-  /// Automatically decides whether to run on the UI Thread or a Background Thread.
+  ///
-  /// [forceSync]: If true, it will be MANDATORY to run on the UI Thread (ignoring the threshold).
+  /// [includeList]: Adds NEW keywords to the search (e.g., 'invoice').
-  /// Use this parameter for unit tests to benchmark the algorithm more accurately.
+  /// [excludeList]: BLOCKS specific tokens from results (e.g., 'invoice-draft').
+  /// [forceSync]: If true, forces execution on the UI thread (for testing).
   static Future<List<String>> matchedKeywordsUnique(
     String text, {
-    List<KeywordFilter> filters = const [],
+    List<String> includeList = const [],
+    List<String> excludeList = const [],
     bool forceSync = false,
   }) async {
-    // If text is short, run Synchronously to avoid Isolate overhead cost.
+    // Prepare filters (Exclude List)
+    final filters = <KeywordFilter>[];
+    if (excludeList.isNotEmpty) {
+      filters.add(ExcludeListFilter(excludeList));
+    }
+
+    final params = DetectionParams(
+      text: text,
+      includeList: includeList, // Passed to build the Regex
+      filters: filters, // Passed to filter the matches
+    );
+
+    // Decision: Run Sync (fast) or Async (safe)
     if (forceSync || text.length < _kAsyncExecutionThreshold) {
-      return _processMatchedKeywordsInIsolate(
+      return _processInIsolate(params);
-        DetectionParams(text: text, filters: filters),
+    } else {
-      );
+      return await compute(_processInIsolate, params);
-    } else { // If text is long, use `compute` to offload to an Isolate, preventing UI freeze.
-      return await compute(
-        _processMatchedKeywordsInIsolate,
-        DetectionParams(text: text, filters: filters),
-      );
     }
   }
 }

lib/features/composer/presentation/manager/detection_params.dart

@@ -1,10 +1,15 @@
 import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart';
 
-/// Data Transfer Object used to package data sent to the Isolate.
+///  Simple DTO to pass parameters to the Isolate.
-/// Since Isolates cannot share memory, we need a simple object to pass parameters.
+/// Isolates cannot share memory, so we package everything here.
 class DetectionParams {
   final String text;
-  final List<KeywordFilter> filters;
+  final List<String> includeList; // Keywords to ADD to the search.
+  final List<KeywordFilter> filters; // Filters to BLOCK specific matches.
 
-  DetectionParams({required this.text, required this.filters});
+  DetectionParams({
+    required this.text,
+    required this.includeList,
+    required this.filters,
+  });
 }

lib/features/composer/presentation/manager/exclude_list_filter.dart

@@ -1,55 +1,28 @@
 import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart';
+import 'package:tmail_ui_user/features/composer/presentation/mixin/token_extraction_mixin.dart';
 
-/// Implementation of Exclude List Filter.
+/// Filter that rejects keywords found in a specific blocklist (Exclude List).
-/// Optimized for memory usage by avoiding unnecessary String allocations.
+class ExcludeListFilter with TokenExtractionMixin implements KeywordFilter {
-class ExcludeListFilter implements KeywordFilter {
   final Set<String> _excludes;
 
-  /// Constructor takes a List for convenience, but converts to Set for O(1) lookup.
   ExcludeListFilter(List<String> rawExcludes)
       : _excludes = rawExcludes.map((e) => e.toLowerCase()).toSet();
 
-  /// Static RegExp to check for whitespace.
-  /// Using a static instance avoids re-compilation in loops.
-  static final RegExp _whitespaceRegExp = RegExp(r'\s');
-
   @override
   bool isValid(String fullText, Match match) {
     if (_excludes.isEmpty) return true;
 
-    // Extract the full token surrounding the match
+    // Get the full word context (e.g., "file-246")
-    final token = _getSurroundingToken(fullText, match.start, match.end);
+    final token = getSurroundingToken(fullText, match.start, match.end);
     final lowerToken = token.toLowerCase();
 
-    // Exact match with the token (e.g., "file-246")
+    // Check 1: Exact match block
     if (_excludes.contains(lowerToken)) return false;
 
-    // Match with trailing punctuation removed (e.g., "file246." -> "file246")
+    // Check 2: Block even if it has trailing punctuation (e.g., "file-246.")
     final cleanToken = lowerToken.replaceAll(RegExp(r'[^\w\s]+$'), '');
     if (_excludes.contains(cleanToken)) return false;
 
     return true;
   }
-
-  /// Extracts the surrounding token without creating garbage strings.
-  /// Instead of using .trim() inside a loop (which allocates memory),
-  /// we check characters directly.
-  String _getSurroundingToken(String text, int matchStart, int matchEnd) {
-    int start = matchStart;
-    int end = matchEnd;
-
-    // Expand to the left until whitespace or start of string
-    while (start > 0) {
-      // Check the character before the current start index
-      if (_whitespaceRegExp.hasMatch(text[start - 1])) break;
-      start--;
-    }
-
-    // Expand to the right until whitespace or end of string
-    while (end < text.length) {
-      if (_whitespaceRegExp.hasMatch(text[end])) break;
-      end++;
-    }
-    return text.substring(start, end);
-  }
 }

lib/features/composer/presentation/manager/keyword_filter.dart

@@ -1,6 +1,5 @@
-/// Common interface for all keyword filters.
+/// Interface for defining custom keyword validation logic.
 abstract class KeywordFilter {
-  /// Returns `true` if the keyword is valid.
+  /// Returns `true` to keep the keyword, `false` to reject it.
-  /// Returns `false` if the keyword should be rejected.
   bool isValid(String fullText, Match match);
 }

lib/features/composer/presentation/mixin/token_extraction_mixin.dart

@@ -0,0 +1,24 @@
+/// Helper mixin to extract the full token surrounding a keyword match.
+/// Example: Extracts "file-246" from text when the keyword is "file".
+mixin TokenExtractionMixin {
+  static final RegExp _whitespaceRegExp = RegExp(r'\s');
+
+  String getSurroundingToken(String text, int matchStart, int matchEnd) {
+    int start = matchStart;
+    int end = matchEnd;
+
+    // Expand left until whitespace
+    while (start > 0) {
+      if (_whitespaceRegExp.hasMatch(text[start - 1])) break;
+      start--;
+    }
+
+    // Expand right until whitespace
+    while (end < text.length) {
+      if (_whitespaceRegExp.hasMatch(text[end])) break;
+      end++;
+    }
+
+    return text.substring(start, end);
+  }
+}

test/features/composer/presentation/manager/attachment_text_detector_test.dart

@@ -1,7 +1,6 @@
 import 'package:core/utils/app_logger.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:tmail_ui_user/features/composer/presentation/manager/attachment_text_detector.dart';
-import 'package:tmail_ui_user/features/composer/presentation/manager/exclude_list_filter.dart';
 import 'package:tmail_ui_user/main/localizations/language_code_constants.dart';
 
 /// Helper: generate email about [targetLength] characters long
@@ -924,35 +923,27 @@ void main() {
     });
 
     test('Exclude Filter: Should exclude blacklisted tokens', () async {
-      final excludeFilter = ExcludeListFilter(['file-246', 'my_file']);
-
       final result = await AttachmentTextDetector.matchedKeywordsUnique(
         'Check file-246 and my_file please.',
-        filters: [excludeFilter],
+        excludeList: ['file-246', 'my_file'],
       );
 
       expect(result, isEmpty);
     });
 
     test('Exclude Filter: Should NOT exclude partial matches', () async {
-      final excludeFilter = ExcludeListFilter(['file-246']);
-
       final result = await AttachmentTextDetector.matchedKeywordsUnique(
         'Check file-999.',
-        filters: [excludeFilter],
+        excludeList: ['file-246'],
       );
 
       expect(result, contains('file'));
     });
 
     test('Multiple Filters Integration (Future Proof)', () async {
-      final excludeFilter = ExcludeListFilter(['file-bad']);
-
       final result = await AttachmentTextDetector.matchedKeywordsUnique(
         'file-good vs file-bad',
-        filters: [
+        excludeList: ['file-bad'],
-          excludeFilter,
-        ],
       );
 
       expect(result, contains('file'));
@@ -1000,11 +991,9 @@ void main() {
 
       final longText = sb.toString();
 
-      final excludeFilter = ExcludeListFilter(['file-246']);
-
       final result = await AttachmentTextDetector.matchedKeywordsUnique(
         longText,
-        filters: [excludeFilter],
+        excludeList: ['file-246'],
       );
 
       expect(result, contains('file'));
@@ -1023,4 +1012,230 @@ void main() {
       }
     });
   });
+
+  group('AttachmentTextDetector (Include/Exclude) filters', () {
+    group('Strict Logic Tests (Include/Exclude)', () {
+      test(
+          'IncludeList: Should strictly accept listed tokens and reject others', () async {
+        const text = "Please see attachment-vip and delete file-trash.";
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          text,
+          includeList: ['attachment-vip'],
+          forceSync: true,
+        );
+
+        expect(result, contains('attachment-vip'));
+      });
+
+      test('Edge Case: Punctuation & Case Sensitivity', () async {
+        const text = "Check FILE-VIP. Also check file-vip, and\nfile-vip!";
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          text,
+          includeList: ['file-vip'],
+          forceSync: true,
+        );
+
+        expect(result, contains('file-vip'));
+        expect(result.length, 1);
+      });
+
+      test('Edge Case: Conflict (Include vs Exclude)', () async {
+        const text = "This is a file-report.";
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          text,
+          includeList: ['file-report'],
+          excludeList: ['file-report'],
+          forceSync: true,
+        );
+
+        expect(result, isEmpty);
+      });
+
+      test('Edge Case: Weird separators (Tabs, Double Spaces)', () async {
+        const text = "Open\tfile-vip  and   file-bad";
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          text,
+          includeList: ['file-vip'],
+          forceSync: true,
+        );
+
+        expect(result, contains('file'));
+
+        const text2 = "Open\tattachment-vip  and   file-bad";
+        final result2 = await AttachmentTextDetector.matchedKeywordsUnique(
+          text2,
+          includeList: ['attachment-vip'],
+          forceSync: true,
+        );
+
+        expect(result2, contains('attachment-vip'));
+      });
+
+      test('Include List should ADD keywords to detection (Fixing previous issue)', () async {
+        const text = "Please check the invoice-2024.";
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          text,
+          includeList: ['invoice'],
+          forceSync: true,
+        );
+
+        expect(result, contains('invoice'));
+      });
+
+      test('Logic Flow: Include List (Add) -> Exclude List (Block)', () async {
+        const text = "This is a secret-code.";
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          text,
+          includeList: ['secret'],
+          excludeList: ['secret-code'],
+          forceSync: true,
+        );
+
+        expect(result, isEmpty);
+      });
+
+      test('Logic Flow: Include List (Add) -> Exclude List (Pass)', () async {
+        const text = "This is a secret-message.";
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          text,
+          includeList: ['secret'],
+          excludeList: ['secret-code'],
+          forceSync: true,
+        );
+
+        expect(result, contains('secret'));
+      });
+    });
+
+    group('Performance & Load Tests', () {
+      test('Benchmark: 100k chars with complex filters (Force Sync)', () async {
+        final sb = StringBuffer();
+        for (int i = 0; i < 5000; i++) {
+          sb.write("random text ");
+          if (i % 100 == 0) sb.write("file-vip ");
+          if (i % 100 == 1) sb.write("file-trash ");
+        }
+        final bigText = sb.toString();
+
+        final stopwatch = Stopwatch()
+          ..start();
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          bigText,
+          includeList: ['file-vip'],
+          forceSync: true,
+        );
+
+        stopwatch.stop();
+        log('Algorithm Time (100k chars): ${stopwatch.elapsedMilliseconds}ms');
+
+        expect(result, contains('file'));
+        expect(stopwatch.elapsedMilliseconds, lessThan(100));
+      });
+
+      test('Integration: Large Text via Isolate (Async)', () async {
+        final bigText = "file-vip " * 5000; // ~45k chars (> 20k threshold)
+
+        final stopwatch = Stopwatch()
+          ..start();
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          bigText,
+          includeList: ['file-vip'],
+        );
+
+        stopwatch.stop();
+        log('Total Async Time (incl. Isolate spawn): ${stopwatch
+            .elapsedMilliseconds}ms');
+
+        expect(result, contains('file-vip'));
+        expect(stopwatch.elapsedMilliseconds, lessThan(300));
+      });
+
+      test('Safety: Input with potential catastrophic backtracking', () async {
+        final trickyText = "${"a" * 50000} file ${"b" * 50000}";
+
+        final stopwatch = Stopwatch()
+          ..start();
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+            trickyText, forceSync: true);
+        stopwatch.stop();
+
+        expect(result, contains('file'));
+        expect(stopwatch.elapsedMilliseconds, lessThan(100));
+      });
+    });
+
+    group('Real-world Context Tests', () {
+      test(
+          'IncludeList: Should only highlight official documents (e.g., invoice-2024)',
+          () async {
+        const text =
+            "Please review the invoice-2024 and ignore the invoice-draft.";
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          text,
+          includeList: ['invoice-2024'],
+          forceSync: true,
+        );
+
+        expect(result, contains('invoice-2024'));
+      });
+
+      test('ExcludeList: Should ignore false positives like email signatures',
+          () async {
+        const text = "See attached image. Best regards, signature-logo.";
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          text,
+          excludeList: ['signature-logo'],
+          forceSync: true,
+        );
+
+        expect(result, containsAll(['attached']));
+
+        const textOnlySignature = "Best regards, signature-logo.";
+        final resultSignature =
+            await AttachmentTextDetector.matchedKeywordsUnique(
+          textOnlySignature,
+          excludeList: ['signature-logo'],
+          forceSync: true,
+        );
+        expect(resultSignature, isEmpty);
+      });
+
+      test('Benchmark: Processing large email logs', () async {
+        final sb = StringBuffer();
+        for (int i = 0; i < 5000; i++) {
+          sb.write("System check... ");
+          if (i % 50 == 0) sb.write("contract-signed ");
+          if (i % 50 == 1) sb.write("css-style-v2 ");
+        }
+        final bigLog = sb.toString();
+
+        final stopwatch = Stopwatch()..start();
+
+        final result = await AttachmentTextDetector.matchedKeywordsUnique(
+          bigLog,
+          includeList: ['contract-signed'],
+          excludeList: ['css-style-v2'],
+          forceSync: true,
+        );
+
+        stopwatch.stop();
+
+        log('Processing Time: ${stopwatch.elapsedMilliseconds}ms');
+
+        expect(result, contains('contract-signed'));
+        expect(stopwatch.elapsedMilliseconds, lessThan(100));
+      });
+    });
+  });
 }