diff --git a/lib/features/composer/presentation/manager/attachment_text_detector.dart b/lib/features/composer/presentation/manager/attachment_text_detector.dart index 1cc7ac353..eb3d2647a 100644 --- a/lib/features/composer/presentation/manager/attachment_text_detector.dart +++ b/lib/features/composer/presentation/manager/attachment_text_detector.dart @@ -1,5 +1,6 @@ import 'package:flutter/foundation.dart'; import 'package:tmail_ui_user/features/composer/presentation/manager/detection_params.dart'; +import 'package:tmail_ui_user/features/composer/presentation/manager/exclude_list_filter.dart'; import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart'; import 'package:tmail_ui_user/main/localizations/language_code_constants.dart'; @@ -50,8 +51,9 @@ class AttachmentTextDetector { ], }; - static RegExp? _combinedRegExp; - /// SYNC/ASYNC EXECUTION THRESHOLD + /// Cached RegExp for the default case (when no include list is provided). + static RegExp? _cachedDefaultRegExp; + /// Threshold to switch from Sync to Async execution. /// /// Value: 20,000 characters (approx. 4-5 pages of text). /// @@ -114,55 +116,61 @@ class AttachmentTextDetector { return result; } - /// Initializes the single optimized RegExp pattern. - static void _initializeRegExp() { - if (_combinedRegExp != null) return; + /// Builds the Regex pattern. + /// Merges [defaultKeywords] with [includeList] to create the search scope. + static RegExp _buildRegExp(List additionalKeywords) { + // 1. Get all default keywords + final defaultKeywords = _keywordsByLang.values.expand((l) => l).toList(); - final allKeywords = - _keywordsByLang.values.expand((l) => l).toSet().toList(); + // 2. Merge with Include List (Add user-defined keywords) + final allKeywords = {...defaultKeywords, ...additionalKeywords}.toList(); - // Sort by length descending (Longest first). - // Crucial to ensure "attachment" is matched before "attach". + // 3. Sort by length descending. + // Crucial to match "attachment" (long) before "attach" (short). allKeywords.sort((a, b) => b.length.compareTo(a.length)); - // LOGIC: - // 1. RegExp.escape: Neutralizes special characters. - // 2. No Nested Quantifiers: Prevents catastrophic backtracking. - // 3. Linear Scan: Uses logical OR (|) for a single-pass scan. + // 4. Create Pattern: (keyword1|keyword2)(?![\p{L}]) + // (?![\p{L}]) ensures we don't match substrings inside other words (e.g., "filetage"). final pattern = allKeywords.map(RegExp.escape).join('|'); - // Pattern: (keyword1|keyword2)(?![\p{L}]) - // (?![\p{L}]): Negative lookahead to ensure the next char is NOT a letter. - // Allows: numbers (file123), punctuation (file.), spaces (file ). - // Rejects: letter extensions (filetage). - _combinedRegExp = RegExp( + return RegExp( '($pattern)(?![\\p{L}])', unicode: true, caseSensitive: false, ); } - static List _processMatchedKeywordsInIsolate(DetectionParams params) { - // Re-initialize RegExp (Isolates do not share static memory with the main app) - _initializeRegExp(); - + /// The core logic function. + /// 1. Builds/Retrieves the Regex. + /// 2. Scans the text. + /// 3. Filters results using Exclude List. + static List _processInIsolate(DetectionParams params) { final text = params.text; - final filters = params.filters; - if (text.isEmpty) return []; - final matches = _combinedRegExp!.allMatches(text); + RegExp regex; + + // Use cached Regex if there are no custom keywords. + if (params.includeList.isEmpty) { + _cachedDefaultRegExp ??= _buildRegExp([]); + regex = _cachedDefaultRegExp!; + } else { + // Rebuild Regex if Include List is present. + regex = _buildRegExp(params.includeList); + } + + final matches = regex.allMatches(text); final result = {}; for (final match in matches) { final keyword = match.group(0)!.toLowerCase(); - // Apply Filter Pipeline + // Check if the found keyword should be excluded based on context. bool isAccepted = true; - for (final filter in filters) { + for (final filter in params.filters) { if (!filter.isValid(text, match)) { isAccepted = false; - break; // Stop checking other filters to save CPU + break; } } @@ -174,25 +182,34 @@ class AttachmentTextDetector { return result.toList(); } - /// Smart keyword detection (Hybrid Sync/Async). - /// Automatically decides whether to run on the UI Thread or a Background Thread. - /// [forceSync]: If true, it will be MANDATORY to run on the UI Thread (ignoring the threshold). - /// Use this parameter for unit tests to benchmark the algorithm more accurately. + /// Detects keywords in the text. + /// + /// [includeList]: Adds NEW keywords to the search (e.g., 'invoice'). + /// [excludeList]: BLOCKS specific tokens from results (e.g., 'invoice-draft'). + /// [forceSync]: If true, forces execution on the UI thread (for testing). static Future> matchedKeywordsUnique( String text, { - List filters = const [], + List includeList = const [], + List excludeList = const [], bool forceSync = false, }) async { - // If text is short, run Synchronously to avoid Isolate overhead cost. + // Prepare filters (Exclude List) + final filters = []; + if (excludeList.isNotEmpty) { + filters.add(ExcludeListFilter(excludeList)); + } + + final params = DetectionParams( + text: text, + includeList: includeList, // Passed to build the Regex + filters: filters, // Passed to filter the matches + ); + + // Decision: Run Sync (fast) or Async (safe) if (forceSync || text.length < _kAsyncExecutionThreshold) { - return _processMatchedKeywordsInIsolate( - DetectionParams(text: text, filters: filters), - ); - } else { // If text is long, use `compute` to offload to an Isolate, preventing UI freeze. - return await compute( - _processMatchedKeywordsInIsolate, - DetectionParams(text: text, filters: filters), - ); + return _processInIsolate(params); + } else { + return await compute(_processInIsolate, params); } } } diff --git a/lib/features/composer/presentation/manager/detection_params.dart b/lib/features/composer/presentation/manager/detection_params.dart index 467eca492..384949048 100644 --- a/lib/features/composer/presentation/manager/detection_params.dart +++ b/lib/features/composer/presentation/manager/detection_params.dart @@ -1,10 +1,15 @@ import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart'; -/// Data Transfer Object used to package data sent to the Isolate. -/// Since Isolates cannot share memory, we need a simple object to pass parameters. +/// Simple DTO to pass parameters to the Isolate. +/// Isolates cannot share memory, so we package everything here. class DetectionParams { final String text; - final List filters; + final List includeList; // Keywords to ADD to the search. + final List filters; // Filters to BLOCK specific matches. - DetectionParams({required this.text, required this.filters}); + DetectionParams({ + required this.text, + required this.includeList, + required this.filters, + }); } \ No newline at end of file diff --git a/lib/features/composer/presentation/manager/exclude_list_filter.dart b/lib/features/composer/presentation/manager/exclude_list_filter.dart index f0393eeee..a83069a82 100644 --- a/lib/features/composer/presentation/manager/exclude_list_filter.dart +++ b/lib/features/composer/presentation/manager/exclude_list_filter.dart @@ -1,55 +1,28 @@ import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart'; +import 'package:tmail_ui_user/features/composer/presentation/mixin/token_extraction_mixin.dart'; -/// Implementation of Exclude List Filter. -/// Optimized for memory usage by avoiding unnecessary String allocations. -class ExcludeListFilter implements KeywordFilter { +/// Filter that rejects keywords found in a specific blocklist (Exclude List). +class ExcludeListFilter with TokenExtractionMixin implements KeywordFilter { final Set _excludes; - /// Constructor takes a List for convenience, but converts to Set for O(1) lookup. ExcludeListFilter(List rawExcludes) : _excludes = rawExcludes.map((e) => e.toLowerCase()).toSet(); - /// Static RegExp to check for whitespace. - /// Using a static instance avoids re-compilation in loops. - static final RegExp _whitespaceRegExp = RegExp(r'\s'); - @override bool isValid(String fullText, Match match) { if (_excludes.isEmpty) return true; - // Extract the full token surrounding the match - final token = _getSurroundingToken(fullText, match.start, match.end); + // Get the full word context (e.g., "file-246") + final token = getSurroundingToken(fullText, match.start, match.end); final lowerToken = token.toLowerCase(); - // Exact match with the token (e.g., "file-246") + // Check 1: Exact match block if (_excludes.contains(lowerToken)) return false; - // Match with trailing punctuation removed (e.g., "file246." -> "file246") + // Check 2: Block even if it has trailing punctuation (e.g., "file-246.") final cleanToken = lowerToken.replaceAll(RegExp(r'[^\w\s]+$'), ''); if (_excludes.contains(cleanToken)) return false; return true; } - - /// Extracts the surrounding token without creating garbage strings. - /// Instead of using .trim() inside a loop (which allocates memory), - /// we check characters directly. - String _getSurroundingToken(String text, int matchStart, int matchEnd) { - int start = matchStart; - int end = matchEnd; - - // Expand to the left until whitespace or start of string - while (start > 0) { - // Check the character before the current start index - if (_whitespaceRegExp.hasMatch(text[start - 1])) break; - start--; - } - - // Expand to the right until whitespace or end of string - while (end < text.length) { - if (_whitespaceRegExp.hasMatch(text[end])) break; - end++; - } - return text.substring(start, end); - } } diff --git a/lib/features/composer/presentation/manager/keyword_filter.dart b/lib/features/composer/presentation/manager/keyword_filter.dart index 6543b27bd..46d639aa6 100644 --- a/lib/features/composer/presentation/manager/keyword_filter.dart +++ b/lib/features/composer/presentation/manager/keyword_filter.dart @@ -1,6 +1,5 @@ -/// Common interface for all keyword filters. +/// Interface for defining custom keyword validation logic. abstract class KeywordFilter { - /// Returns `true` if the keyword is valid. - /// Returns `false` if the keyword should be rejected. + /// Returns `true` to keep the keyword, `false` to reject it. bool isValid(String fullText, Match match); } \ No newline at end of file diff --git a/lib/features/composer/presentation/mixin/token_extraction_mixin.dart b/lib/features/composer/presentation/mixin/token_extraction_mixin.dart new file mode 100644 index 000000000..17b4e2435 --- /dev/null +++ b/lib/features/composer/presentation/mixin/token_extraction_mixin.dart @@ -0,0 +1,24 @@ +/// Helper mixin to extract the full token surrounding a keyword match. +/// Example: Extracts "file-246" from text when the keyword is "file". +mixin TokenExtractionMixin { + static final RegExp _whitespaceRegExp = RegExp(r'\s'); + + String getSurroundingToken(String text, int matchStart, int matchEnd) { + int start = matchStart; + int end = matchEnd; + + // Expand left until whitespace + while (start > 0) { + if (_whitespaceRegExp.hasMatch(text[start - 1])) break; + start--; + } + + // Expand right until whitespace + while (end < text.length) { + if (_whitespaceRegExp.hasMatch(text[end])) break; + end++; + } + + return text.substring(start, end); + } +} \ No newline at end of file diff --git a/test/features/composer/presentation/manager/attachment_text_detector_test.dart b/test/features/composer/presentation/manager/attachment_text_detector_test.dart index 7c17f3ed9..23c9b5905 100644 --- a/test/features/composer/presentation/manager/attachment_text_detector_test.dart +++ b/test/features/composer/presentation/manager/attachment_text_detector_test.dart @@ -1,7 +1,6 @@ import 'package:core/utils/app_logger.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:tmail_ui_user/features/composer/presentation/manager/attachment_text_detector.dart'; -import 'package:tmail_ui_user/features/composer/presentation/manager/exclude_list_filter.dart'; import 'package:tmail_ui_user/main/localizations/language_code_constants.dart'; /// Helper: generate email about [targetLength] characters long @@ -924,35 +923,27 @@ void main() { }); test('Exclude Filter: Should exclude blacklisted tokens', () async { - final excludeFilter = ExcludeListFilter(['file-246', 'my_file']); - final result = await AttachmentTextDetector.matchedKeywordsUnique( 'Check file-246 and my_file please.', - filters: [excludeFilter], + excludeList: ['file-246', 'my_file'], ); expect(result, isEmpty); }); test('Exclude Filter: Should NOT exclude partial matches', () async { - final excludeFilter = ExcludeListFilter(['file-246']); - final result = await AttachmentTextDetector.matchedKeywordsUnique( 'Check file-999.', - filters: [excludeFilter], + excludeList: ['file-246'], ); expect(result, contains('file')); }); test('Multiple Filters Integration (Future Proof)', () async { - final excludeFilter = ExcludeListFilter(['file-bad']); - final result = await AttachmentTextDetector.matchedKeywordsUnique( 'file-good vs file-bad', - filters: [ - excludeFilter, - ], + excludeList: ['file-bad'], ); expect(result, contains('file')); @@ -1000,11 +991,9 @@ void main() { final longText = sb.toString(); - final excludeFilter = ExcludeListFilter(['file-246']); - final result = await AttachmentTextDetector.matchedKeywordsUnique( longText, - filters: [excludeFilter], + excludeList: ['file-246'], ); expect(result, contains('file')); @@ -1023,4 +1012,230 @@ void main() { } }); }); + + group('AttachmentTextDetector (Include/Exclude) filters', () { + group('Strict Logic Tests (Include/Exclude)', () { + test( + 'IncludeList: Should strictly accept listed tokens and reject others', () async { + const text = "Please see attachment-vip and delete file-trash."; + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + text, + includeList: ['attachment-vip'], + forceSync: true, + ); + + expect(result, contains('attachment-vip')); + }); + + test('Edge Case: Punctuation & Case Sensitivity', () async { + const text = "Check FILE-VIP. Also check file-vip, and\nfile-vip!"; + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + text, + includeList: ['file-vip'], + forceSync: true, + ); + + expect(result, contains('file-vip')); + expect(result.length, 1); + }); + + test('Edge Case: Conflict (Include vs Exclude)', () async { + const text = "This is a file-report."; + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + text, + includeList: ['file-report'], + excludeList: ['file-report'], + forceSync: true, + ); + + expect(result, isEmpty); + }); + + test('Edge Case: Weird separators (Tabs, Double Spaces)', () async { + const text = "Open\tfile-vip and file-bad"; + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + text, + includeList: ['file-vip'], + forceSync: true, + ); + + expect(result, contains('file')); + + const text2 = "Open\tattachment-vip and file-bad"; + final result2 = await AttachmentTextDetector.matchedKeywordsUnique( + text2, + includeList: ['attachment-vip'], + forceSync: true, + ); + + expect(result2, contains('attachment-vip')); + }); + + test('Include List should ADD keywords to detection (Fixing previous issue)', () async { + const text = "Please check the invoice-2024."; + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + text, + includeList: ['invoice'], + forceSync: true, + ); + + expect(result, contains('invoice')); + }); + + test('Logic Flow: Include List (Add) -> Exclude List (Block)', () async { + const text = "This is a secret-code."; + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + text, + includeList: ['secret'], + excludeList: ['secret-code'], + forceSync: true, + ); + + expect(result, isEmpty); + }); + + test('Logic Flow: Include List (Add) -> Exclude List (Pass)', () async { + const text = "This is a secret-message."; + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + text, + includeList: ['secret'], + excludeList: ['secret-code'], + forceSync: true, + ); + + expect(result, contains('secret')); + }); + }); + + group('Performance & Load Tests', () { + test('Benchmark: 100k chars with complex filters (Force Sync)', () async { + final sb = StringBuffer(); + for (int i = 0; i < 5000; i++) { + sb.write("random text "); + if (i % 100 == 0) sb.write("file-vip "); + if (i % 100 == 1) sb.write("file-trash "); + } + final bigText = sb.toString(); + + final stopwatch = Stopwatch() + ..start(); + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + bigText, + includeList: ['file-vip'], + forceSync: true, + ); + + stopwatch.stop(); + log('Algorithm Time (100k chars): ${stopwatch.elapsedMilliseconds}ms'); + + expect(result, contains('file')); + expect(stopwatch.elapsedMilliseconds, lessThan(100)); + }); + + test('Integration: Large Text via Isolate (Async)', () async { + final bigText = "file-vip " * 5000; // ~45k chars (> 20k threshold) + + final stopwatch = Stopwatch() + ..start(); + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + bigText, + includeList: ['file-vip'], + ); + + stopwatch.stop(); + log('Total Async Time (incl. Isolate spawn): ${stopwatch + .elapsedMilliseconds}ms'); + + expect(result, contains('file-vip')); + expect(stopwatch.elapsedMilliseconds, lessThan(300)); + }); + + test('Safety: Input with potential catastrophic backtracking', () async { + final trickyText = "${"a" * 50000} file ${"b" * 50000}"; + + final stopwatch = Stopwatch() + ..start(); + final result = await AttachmentTextDetector.matchedKeywordsUnique( + trickyText, forceSync: true); + stopwatch.stop(); + + expect(result, contains('file')); + expect(stopwatch.elapsedMilliseconds, lessThan(100)); + }); + }); + + group('Real-world Context Tests', () { + test( + 'IncludeList: Should only highlight official documents (e.g., invoice-2024)', + () async { + const text = + "Please review the invoice-2024 and ignore the invoice-draft."; + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + text, + includeList: ['invoice-2024'], + forceSync: true, + ); + + expect(result, contains('invoice-2024')); + }); + + test('ExcludeList: Should ignore false positives like email signatures', + () async { + const text = "See attached image. Best regards, signature-logo."; + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + text, + excludeList: ['signature-logo'], + forceSync: true, + ); + + expect(result, containsAll(['attached'])); + + const textOnlySignature = "Best regards, signature-logo."; + final resultSignature = + await AttachmentTextDetector.matchedKeywordsUnique( + textOnlySignature, + excludeList: ['signature-logo'], + forceSync: true, + ); + expect(resultSignature, isEmpty); + }); + + test('Benchmark: Processing large email logs', () async { + final sb = StringBuffer(); + for (int i = 0; i < 5000; i++) { + sb.write("System check... "); + if (i % 50 == 0) sb.write("contract-signed "); + if (i % 50 == 1) sb.write("css-style-v2 "); + } + final bigLog = sb.toString(); + + final stopwatch = Stopwatch()..start(); + + final result = await AttachmentTextDetector.matchedKeywordsUnique( + bigLog, + includeList: ['contract-signed'], + excludeList: ['css-style-v2'], + forceSync: true, + ); + + stopwatch.stop(); + + log('Processing Time: ${stopwatch.elapsedMilliseconds}ms'); + + expect(result, contains('contract-signed')); + expect(stopwatch.elapsedMilliseconds, lessThan(100)); + }); + }); + }); }