eduvia/workavia-mail-front
11
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

TF-4265 Add include list to support custom keywords detection

Browse Source
This commit is contained in:
dab246
2026-01-30 14:01:36 +07:00
committed by Dat H. Pham
parent 1071bb49cf
commit 42fb24e6e5
6 changed files with 331 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/features/composer/presentation/manager/attachment_text_detector.dart
+59 -42
View File
@@ -1,5 +1,6 @@
import 'package:flutter/foundation.dart';
import 'package:tmail_ui_user/features/composer/presentation/manager/detection_params.dart';
import 'package:tmail_ui_user/features/composer/presentation/manager/exclude_list_filter.dart';
import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart';
import 'package:tmail_ui_user/main/localizations/language_code_constants.dart';
@@ -50,8 +51,9 @@ class AttachmentTextDetector {
],
};
static RegExp? _combinedRegExp;
/// SYNC/ASYNC EXECUTION THRESHOLD
/// Cached RegExp for the default case (when no include list is provided).
static RegExp? _cachedDefaultRegExp;
/// Threshold to switch from Sync to Async execution.
///
/// Value: 20,000 characters (approx. 4-5 pages of text).
///
@@ -114,55 +116,61 @@ class AttachmentTextDetector {
return result;
}
/// Initializes the single optimized RegExp pattern.
static void _initializeRegExp() {
if (_combinedRegExp != null) return;
/// Builds the Regex pattern.
/// Merges [defaultKeywords] with [includeList] to create the search scope.
static RegExp _buildRegExp(List<String> additionalKeywords) {
// 1. Get all default keywords
final defaultKeywords = _keywordsByLang.values.expand((l) => l).toList();
final allKeywords =
_keywordsByLang.values.expand((l) => l).toSet().toList();
// 2. Merge with Include List (Add user-defined keywords)
final allKeywords = {...defaultKeywords, ...additionalKeywords}.toList();
// Sort by length descending (Longest first).
// Crucial to ensure "attachment" is matched before "attach".
// 3. Sort by length descending.
// Crucial to match "attachment" (long) before "attach" (short).
allKeywords.sort((a, b) => b.length.compareTo(a.length));
// LOGIC:
// 1. RegExp.escape: Neutralizes special characters.
// 2. No Nested Quantifiers: Prevents catastrophic backtracking.
// 3. Linear Scan: Uses logical OR (|) for a single-pass scan.
// 4. Create Pattern: (keyword1|keyword2)(?![\p{L}])
// (?![\p{L}]) ensures we don't match substrings inside other words (e.g., "filetage").
final pattern = allKeywords.map(RegExp.escape).join('|');
// Pattern: (keyword1|keyword2)(?![\p{L}])
// (?![\p{L}]): Negative lookahead to ensure the next char is NOT a letter.
// Allows: numbers (file123), punctuation (file.), spaces (file ).
// Rejects: letter extensions (filetage).
_combinedRegExp = RegExp(
return RegExp(
'($pattern)(?![\\p{L}])',
unicode: true,
caseSensitive: false,
);
}
static List<String> _processMatchedKeywordsInIsolate(DetectionParams params) {
// Re-initialize RegExp (Isolates do not share static memory with the main app)
_initializeRegExp();
/// The core logic function.
/// 1. Builds/Retrieves the Regex.
/// 2. Scans the text.
/// 3. Filters results using Exclude List.
static List<String> _processInIsolate(DetectionParams params) {
final text = params.text;
final filters = params.filters;
if (text.isEmpty) return [];
final matches = _combinedRegExp!.allMatches(text);
RegExp regex;
// Use cached Regex if there are no custom keywords.
if (params.includeList.isEmpty) {
_cachedDefaultRegExp ??= _buildRegExp([]);
regex = _cachedDefaultRegExp!;
} else {
// Rebuild Regex if Include List is present.
regex = _buildRegExp(params.includeList);
}
final matches = regex.allMatches(text);
final result = <String>{};
for (final match in matches) {
final keyword = match.group(0)!.toLowerCase();
// Apply Filter Pipeline
// Check if the found keyword should be excluded based on context.
bool isAccepted = true;
for (final filter in filters) {
for (final filter in params.filters) {
if (!filter.isValid(text, match)) {
isAccepted = false;
break; // Stop checking other filters to save CPU
break;
}
}
@@ -174,25 +182,34 @@ class AttachmentTextDetector {
return result.toList();
}
/// Smart keyword detection (Hybrid Sync/Async).
/// Automatically decides whether to run on the UI Thread or a Background Thread.
/// [forceSync]: If true, it will be MANDATORY to run on the UI Thread (ignoring the threshold).
/// Use this parameter for unit tests to benchmark the algorithm more accurately.
/// Detects keywords in the text.
///
/// [includeList]: Adds NEW keywords to the search (e.g., 'invoice').
/// [excludeList]: BLOCKS specific tokens from results (e.g., 'invoice-draft').
/// [forceSync]: If true, forces execution on the UI thread (for testing).
static Future<List<String>> matchedKeywordsUnique(
String text, {
List<KeywordFilter> filters = const [],
List<String> includeList = const [],
List<String> excludeList = const [],
bool forceSync = false,
}) async {
// If text is short, run Synchronously to avoid Isolate overhead cost.
// Prepare filters (Exclude List)
final filters = <KeywordFilter>[];
if (excludeList.isNotEmpty) {
filters.add(ExcludeListFilter(excludeList));
}
final params = DetectionParams(
text: text,
includeList: includeList, // Passed to build the Regex
filters: filters, // Passed to filter the matches
);
// Decision: Run Sync (fast) or Async (safe)
if (forceSync || text.length < _kAsyncExecutionThreshold) {
return _processMatchedKeywordsInIsolate(
DetectionParams(text: text, filters: filters),
);
} else { // If text is long, use `compute` to offload to an Isolate, preventing UI freeze.
return await compute(
_processMatchedKeywordsInIsolate,
DetectionParams(text: text, filters: filters),
);
return _processInIsolate(params);
} else {
return await compute(_processInIsolate, params);
}
}
}
lib/features/composer/presentation/manager/detection_params.dart
+9 -4
View File
@@ -1,10 +1,15 @@
import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart';
/// Data Transfer Object used to package data sent to the Isolate.
/// Since Isolates cannot share memory, we need a simple object to pass parameters.
/// Simple DTO to pass parameters to the Isolate.
/// Isolates cannot share memory, so we package everything here.
class DetectionParams {
final String text;
final List<KeywordFilter> filters;
final List<String> includeList; // Keywords to ADD to the search.
final List<KeywordFilter> filters; // Filters to BLOCK specific matches.
DetectionParams({required this.text, required this.filters});
DetectionParams({
required this.text,
required this.includeList,
required this.filters,
});
}
lib/features/composer/presentation/manager/exclude_list_filter.dart
+7 -34
View File
@@ -1,55 +1,28 @@
import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart';
import 'package:tmail_ui_user/features/composer/presentation/mixin/token_extraction_mixin.dart';
/// Implementation of Exclude List Filter.
/// Optimized for memory usage by avoiding unnecessary String allocations.
class ExcludeListFilter implements KeywordFilter {
/// Filter that rejects keywords found in a specific blocklist (Exclude List).
class ExcludeListFilter with TokenExtractionMixin implements KeywordFilter {
final Set<String> _excludes;
/// Constructor takes a List for convenience, but converts to Set for O(1) lookup.
ExcludeListFilter(List<String> rawExcludes)
: _excludes = rawExcludes.map((e) => e.toLowerCase()).toSet();
/// Static RegExp to check for whitespace.
/// Using a static instance avoids re-compilation in loops.
static final RegExp _whitespaceRegExp = RegExp(r'\s');
@override
bool isValid(String fullText, Match match) {
if (_excludes.isEmpty) return true;
// Extract the full token surrounding the match
final token = _getSurroundingToken(fullText, match.start, match.end);
// Get the full word context (e.g., "file-246")
final token = getSurroundingToken(fullText, match.start, match.end);
final lowerToken = token.toLowerCase();
// Exact match with the token (e.g., "file-246")
// Check 1: Exact match block
if (_excludes.contains(lowerToken)) return false;
// Match with trailing punctuation removed (e.g., "file246." -> "file246")
// Check 2: Block even if it has trailing punctuation (e.g., "file-246.")
final cleanToken = lowerToken.replaceAll(RegExp(r'[^\w\s]+$'), '');
if (_excludes.contains(cleanToken)) return false;
return true;
}
/// Extracts the surrounding token without creating garbage strings.
/// Instead of using .trim() inside a loop (which allocates memory),
/// we check characters directly.
String _getSurroundingToken(String text, int matchStart, int matchEnd) {
int start = matchStart;
int end = matchEnd;
// Expand to the left until whitespace or start of string
while (start > 0) {
// Check the character before the current start index
if (_whitespaceRegExp.hasMatch(text[start - 1])) break;
start--;
}
// Expand to the right until whitespace or end of string
while (end < text.length) {
if (_whitespaceRegExp.hasMatch(text[end])) break;
end++;
}
return text.substring(start, end);
}
}
lib/features/composer/presentation/manager/keyword_filter.dart
+2 -3
View File
@@ -1,6 +1,5 @@
/// Common interface for all keyword filters.
/// Interface for defining custom keyword validation logic.
abstract class KeywordFilter {
/// Returns `true` if the keyword is valid.
/// Returns `false` if the keyword should be rejected.
/// Returns `true` to keep the keyword, `false` to reject it.
bool isValid(String fullText, Match match);
}
lib/features/composer/presentation/mixin/token_extraction_mixin.dart
+24
View File
@@ -0,0 +1,24 @@
/// Helper mixin to extract the full token surrounding a keyword match.
/// Example: Extracts "file-246" from text when the keyword is "file".
mixin TokenExtractionMixin {
static final RegExp _whitespaceRegExp = RegExp(r'\s');
String getSurroundingToken(String text, int matchStart, int matchEnd) {
int start = matchStart;
int end = matchEnd;
// Expand left until whitespace
while (start > 0) {
if (_whitespaceRegExp.hasMatch(text[start - 1])) break;
start--;
}
// Expand right until whitespace
while (end < text.length) {
if (_whitespaceRegExp.hasMatch(text[end])) break;
end++;
}
return text.substring(start, end);
}
}
test/features/composer/presentation/manager/attachment_text_detector_test.dart
+230 -15
View File
@@ -1,7 +1,6 @@
import 'package:core/utils/app_logger.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:tmail_ui_user/features/composer/presentation/manager/attachment_text_detector.dart';
import 'package:tmail_ui_user/features/composer/presentation/manager/exclude_list_filter.dart';
import 'package:tmail_ui_user/main/localizations/language_code_constants.dart';
/// Helper: generate email about [targetLength] characters long
@@ -924,35 +923,27 @@ void main() {
});
test('Exclude Filter: Should exclude blacklisted tokens', () async {
final excludeFilter = ExcludeListFilter(['file-246', 'my_file']);
final result = await AttachmentTextDetector.matchedKeywordsUnique(
'Check file-246 and my_file please.',
filters: [excludeFilter],
excludeList: ['file-246', 'my_file'],
);
expect(result, isEmpty);
});
test('Exclude Filter: Should NOT exclude partial matches', () async {
final excludeFilter = ExcludeListFilter(['file-246']);
final result = await AttachmentTextDetector.matchedKeywordsUnique(
'Check file-999.',
filters: [excludeFilter],
excludeList: ['file-246'],
);
expect(result, contains('file'));
});
test('Multiple Filters Integration (Future Proof)', () async {
final excludeFilter = ExcludeListFilter(['file-bad']);
final result = await AttachmentTextDetector.matchedKeywordsUnique(
'file-good vs file-bad',
filters: [
excludeFilter,
],
excludeList: ['file-bad'],
);
expect(result, contains('file'));
@@ -1000,11 +991,9 @@ void main() {
final longText = sb.toString();
final excludeFilter = ExcludeListFilter(['file-246']);
final result = await AttachmentTextDetector.matchedKeywordsUnique(
longText,
filters: [excludeFilter],
excludeList: ['file-246'],
);
expect(result, contains('file'));
@@ -1023,4 +1012,230 @@ void main() {
}
});
});
group('AttachmentTextDetector (Include/Exclude) filters', () {
group('Strict Logic Tests (Include/Exclude)', () {
test(
'IncludeList: Should strictly accept listed tokens and reject others', () async {
const text = "Please see attachment-vip and delete file-trash.";
final result = await AttachmentTextDetector.matchedKeywordsUnique(
text,
includeList: ['attachment-vip'],
forceSync: true,
);
expect(result, contains('attachment-vip'));
});
test('Edge Case: Punctuation & Case Sensitivity', () async {
const text = "Check FILE-VIP. Also check file-vip, and\nfile-vip!";
final result = await AttachmentTextDetector.matchedKeywordsUnique(
text,
includeList: ['file-vip'],
forceSync: true,
);
expect(result, contains('file-vip'));
expect(result.length, 1);
});
test('Edge Case: Conflict (Include vs Exclude)', () async {
const text = "This is a file-report.";
final result = await AttachmentTextDetector.matchedKeywordsUnique(
text,
includeList: ['file-report'],
excludeList: ['file-report'],
forceSync: true,
);
expect(result, isEmpty);
});
test('Edge Case: Weird separators (Tabs, Double Spaces)', () async {
const text = "Open\tfile-vip and file-bad";
final result = await AttachmentTextDetector.matchedKeywordsUnique(
text,
includeList: ['file-vip'],
forceSync: true,
);
expect(result, contains('file'));
const text2 = "Open\tattachment-vip and file-bad";
final result2 = await AttachmentTextDetector.matchedKeywordsUnique(
text2,
includeList: ['attachment-vip'],
forceSync: true,
);
expect(result2, contains('attachment-vip'));
});
test('Include List should ADD keywords to detection (Fixing previous issue)', () async {
const text = "Please check the invoice-2024.";
final result = await AttachmentTextDetector.matchedKeywordsUnique(
text,
includeList: ['invoice'],
forceSync: true,
);
expect(result, contains('invoice'));
});
test('Logic Flow: Include List (Add) -> Exclude List (Block)', () async {
const text = "This is a secret-code.";
final result = await AttachmentTextDetector.matchedKeywordsUnique(
text,
includeList: ['secret'],
excludeList: ['secret-code'],
forceSync: true,
);
expect(result, isEmpty);
});
test('Logic Flow: Include List (Add) -> Exclude List (Pass)', () async {
const text = "This is a secret-message.";
final result = await AttachmentTextDetector.matchedKeywordsUnique(
text,
includeList: ['secret'],
excludeList: ['secret-code'],
forceSync: true,
);
expect(result, contains('secret'));
});
});
group('Performance & Load Tests', () {
test('Benchmark: 100k chars with complex filters (Force Sync)', () async {
final sb = StringBuffer();
for (int i = 0; i < 5000; i++) {
sb.write("random text ");
if (i % 100 == 0) sb.write("file-vip ");
if (i % 100 == 1) sb.write("file-trash ");
}
final bigText = sb.toString();
final stopwatch = Stopwatch()
..start();
final result = await AttachmentTextDetector.matchedKeywordsUnique(
bigText,
includeList: ['file-vip'],
forceSync: true,
);
stopwatch.stop();
log('Algorithm Time (100k chars): ${stopwatch.elapsedMilliseconds}ms');
expect(result, contains('file'));
expect(stopwatch.elapsedMilliseconds, lessThan(100));
});
test('Integration: Large Text via Isolate (Async)', () async {
final bigText = "file-vip " * 5000; // ~45k chars (> 20k threshold)
final stopwatch = Stopwatch()
..start();
final result = await AttachmentTextDetector.matchedKeywordsUnique(
bigText,
includeList: ['file-vip'],
);
stopwatch.stop();
log('Total Async Time (incl. Isolate spawn): ${stopwatch
.elapsedMilliseconds}ms');
expect(result, contains('file-vip'));
expect(stopwatch.elapsedMilliseconds, lessThan(300));
});
test('Safety: Input with potential catastrophic backtracking', () async {
final trickyText = "${"a" * 50000} file ${"b" * 50000}";
final stopwatch = Stopwatch()
..start();
final result = await AttachmentTextDetector.matchedKeywordsUnique(
trickyText, forceSync: true);
stopwatch.stop();
expect(result, contains('file'));
expect(stopwatch.elapsedMilliseconds, lessThan(100));
});
});
group('Real-world Context Tests', () {
test(
'IncludeList: Should only highlight official documents (e.g., invoice-2024)',
() async {
const text =
"Please review the invoice-2024 and ignore the invoice-draft.";
final result = await AttachmentTextDetector.matchedKeywordsUnique(
text,
includeList: ['invoice-2024'],
forceSync: true,
);
expect(result, contains('invoice-2024'));
});
test('ExcludeList: Should ignore false positives like email signatures',
() async {
const text = "See attached image. Best regards, signature-logo.";
final result = await AttachmentTextDetector.matchedKeywordsUnique(
text,
excludeList: ['signature-logo'],
forceSync: true,
);
expect(result, containsAll(['attached']));
const textOnlySignature = "Best regards, signature-logo.";
final resultSignature =
await AttachmentTextDetector.matchedKeywordsUnique(
textOnlySignature,
excludeList: ['signature-logo'],
forceSync: true,
);
expect(resultSignature, isEmpty);
});
test('Benchmark: Processing large email logs', () async {
final sb = StringBuffer();
for (int i = 0; i < 5000; i++) {
sb.write("System check... ");
if (i % 50 == 0) sb.write("contract-signed ");
if (i % 50 == 1) sb.write("css-style-v2 ");
}
final bigLog = sb.toString();
final stopwatch = Stopwatch()..start();
final result = await AttachmentTextDetector.matchedKeywordsUnique(
bigLog,
includeList: ['contract-signed'],
excludeList: ['css-style-v2'],
forceSync: true,
);
stopwatch.stop();
log('Processing Time: ${stopwatch.elapsedMilliseconds}ms');
expect(result, contains('contract-signed'));
expect(stopwatch.elapsedMilliseconds, lessThan(100));
});
});
});
}