TF-4265 Add exclude list support to AttachmentTextDetector
This commit is contained in:
@@ -886,7 +886,7 @@ class ComposerController extends BaseController
|
|||||||
_sendButtonState = ButtonState.enabled;
|
_sendButtonState = ButtonState.enabled;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
final attachmentKeywords = validateAttachmentReminder(
|
final attachmentKeywords = await validateAttachmentReminder(
|
||||||
emailSubject: subjectEmail.value ?? '',
|
emailSubject: subjectEmail.value ?? '',
|
||||||
emailContent: emailContent,
|
emailContent: emailContent,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -9,14 +9,15 @@ import 'package:tmail_ui_user/main/routes/route_navigation.dart';
|
|||||||
|
|
||||||
extension AttachmentDetectionExtension on ComposerController {
|
extension AttachmentDetectionExtension on ComposerController {
|
||||||
|
|
||||||
List<String> validateAttachmentReminder({
|
Future<List<String>> validateAttachmentReminder({
|
||||||
required String emailContent,
|
required String emailContent,
|
||||||
required String emailSubject,
|
required String emailSubject,
|
||||||
}) {
|
}) async {
|
||||||
try {
|
try {
|
||||||
final fullContent = '$emailSubject $emailContent';
|
final fullContent = '$emailSubject $emailContent';
|
||||||
final plainText = HtmlUtils.extractPlainText(fullContent);
|
final plainText = HtmlUtils.extractPlainText(fullContent);
|
||||||
final keywords = AttachmentTextDetector.matchedKeywordsUnique(plainText);
|
final keywords =
|
||||||
|
await AttachmentTextDetector.matchedKeywordsUnique(plainText);
|
||||||
if (keywords.isEmpty) {
|
if (keywords.isEmpty) {
|
||||||
return [];
|
return [];
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -1,3 +1,6 @@
|
|||||||
|
import 'package:flutter/foundation.dart';
|
||||||
|
import 'package:tmail_ui_user/features/composer/presentation/manager/detection_params.dart';
|
||||||
|
import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart';
|
||||||
import 'package:tmail_ui_user/main/localizations/language_code_constants.dart';
|
import 'package:tmail_ui_user/main/localizations/language_code_constants.dart';
|
||||||
|
|
||||||
class AttachmentTextDetector {
|
class AttachmentTextDetector {
|
||||||
@@ -47,6 +50,22 @@ class AttachmentTextDetector {
|
|||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
static RegExp? _combinedRegExp;
|
||||||
|
/// SYNC/ASYNC EXECUTION THRESHOLD
|
||||||
|
///
|
||||||
|
/// Value: 20,000 characters (approx. 4-5 pages of text).
|
||||||
|
///
|
||||||
|
/// Technical Rationale:
|
||||||
|
/// 1. Frame Budget (16ms): To maintain 60fps, operations on the UI thread must stay under 16ms.
|
||||||
|
/// 2. Isolate Overhead: `compute` takes about 2-4ms just to copy data and start the thread.
|
||||||
|
/// 3. Real-world Benchmark:
|
||||||
|
/// - For < 20k chars, Sync execution is usually faster (< 10ms) than the Async overhead.
|
||||||
|
/// - For > 20k chars, Regex processing might exceed 16ms on low-end devices, causing jank.
|
||||||
|
///
|
||||||
|
/// -> Below 20k: Run Sync (Instant response).
|
||||||
|
/// -> Above 20k: Run Async (Safety for UI).
|
||||||
|
static const int _kAsyncExecutionThreshold = 20000;
|
||||||
|
|
||||||
/// Detect if the text contains keywords suggesting there is an attachment.
|
/// Detect if the text contains keywords suggesting there is an attachment.
|
||||||
/// [lang] is the language code (`en`, `fr`, `ru`, `vi`, `ar`, ...).
|
/// [lang] is the language code (`en`, `fr`, `ru`, `vi`, `ar`, ...).
|
||||||
static bool containsAttachmentKeyword(String text, {required String lang}) {
|
static bool containsAttachmentKeyword(String text, {required String lang}) {
|
||||||
@@ -95,48 +114,85 @@ class AttachmentTextDetector {
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store a single combined RegExp pattern
|
/// Initializes the single optimized RegExp pattern.
|
||||||
static RegExp? _combinedRegExp;
|
|
||||||
|
|
||||||
/// Initializes a single optimized RegExp that contains all keywords.
|
|
||||||
static void _initializeRegExp() {
|
static void _initializeRegExp() {
|
||||||
if (_combinedRegExp != null) return;
|
if (_combinedRegExp != null) return;
|
||||||
|
|
||||||
// Flatten all keywords from all languages into a single list
|
final allKeywords =
|
||||||
final allKeywords = _keywordsByLang.values.expand((list) => list).toSet().toList();
|
_keywordsByLang.values.expand((l) => l).toSet().toList();
|
||||||
|
|
||||||
// Sort keywords by length descending (longest first).
|
// Sort by length descending (Longest first).
|
||||||
|
// Crucial to ensure "attachment" is matched before "attach".
|
||||||
allKeywords.sort((a, b) => b.length.compareTo(a.length));
|
allKeywords.sort((a, b) => b.length.compareTo(a.length));
|
||||||
|
|
||||||
// Create a pattern like: (attachment|attach|file|files)(?![\p{L}])
|
// LOGIC:
|
||||||
// Join all keywords with OR operator (|)
|
// 1. RegExp.escape: Neutralizes special characters.
|
||||||
final patternString = allKeywords.map(RegExp.escape).join('|');
|
// 2. No Nested Quantifiers: Prevents catastrophic backtracking.
|
||||||
|
// 3. Linear Scan: Uses logical OR (|) for a single-pass scan.
|
||||||
|
final pattern = allKeywords.map(RegExp.escape).join('|');
|
||||||
|
|
||||||
|
// Pattern: (keyword1|keyword2)(?![\p{L}])
|
||||||
|
// (?![\p{L}]): Negative lookahead to ensure the next char is NOT a letter.
|
||||||
|
// Allows: numbers (file123), punctuation (file.), spaces (file ).
|
||||||
|
// Rejects: letter extensions (filetage).
|
||||||
_combinedRegExp = RegExp(
|
_combinedRegExp = RegExp(
|
||||||
'($patternString)(?![\\p{L}])',
|
'($pattern)(?![\\p{L}])',
|
||||||
unicode: true,
|
unicode: true,
|
||||||
caseSensitive: false, // Let Regex handle case-insensitivity
|
caseSensitive: false,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Detects unique keywords in the provided text based on predefined language lists.
|
static List<String> _processMatchedKeywordsInIsolate(DetectionParams params) {
|
||||||
static List<String> matchedKeywordsUnique(String text) {
|
// Re-initialize RegExp (Isolates do not share static memory with the main app)
|
||||||
if (text.isEmpty) return [];
|
|
||||||
|
|
||||||
// Ensure RegExp are initialized
|
|
||||||
_initializeRegExp();
|
_initializeRegExp();
|
||||||
|
|
||||||
|
final text = params.text;
|
||||||
|
final filters = params.filters;
|
||||||
|
|
||||||
|
if (text.isEmpty) return [];
|
||||||
|
|
||||||
final matches = _combinedRegExp!.allMatches(text);
|
final matches = _combinedRegExp!.allMatches(text);
|
||||||
final result = <String>{};
|
final result = <String>{};
|
||||||
|
|
||||||
for (final match in matches) {
|
for (final match in matches) {
|
||||||
// match.group(0) returns the actual text found (e.g., "FILE", "File").
|
final keyword = match.group(0)!.toLowerCase();
|
||||||
// We convert it to lowercase to normalize the result list.
|
|
||||||
if (match.group(0) != null) {
|
// Apply Filter Pipeline
|
||||||
result.add(match.group(0)!.toLowerCase());
|
bool isAccepted = true;
|
||||||
|
for (final filter in filters) {
|
||||||
|
if (!filter.isValid(text, match)) {
|
||||||
|
isAccepted = false;
|
||||||
|
break; // Stop checking other filters to save CPU
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isAccepted) {
|
||||||
|
result.add(keyword);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return result.toList();
|
return result.toList();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Smart keyword detection (Hybrid Sync/Async).
|
||||||
|
/// Automatically decides whether to run on the UI Thread or a Background Thread.
|
||||||
|
/// [forceSync]: If true, it will be MANDATORY to run on the UI Thread (ignoring the threshold).
|
||||||
|
/// Use this parameter for unit tests to benchmark the algorithm more accurately.
|
||||||
|
static Future<List<String>> matchedKeywordsUnique(
|
||||||
|
String text, {
|
||||||
|
List<KeywordFilter> filters = const [],
|
||||||
|
bool forceSync = false,
|
||||||
|
}) async {
|
||||||
|
// If text is short, run Synchronously to avoid Isolate overhead cost.
|
||||||
|
if (forceSync || text.length < _kAsyncExecutionThreshold) {
|
||||||
|
return _processMatchedKeywordsInIsolate(
|
||||||
|
DetectionParams(text: text, filters: filters),
|
||||||
|
);
|
||||||
|
} else { // If text is long, use `compute` to offload to an Isolate, preventing UI freeze.
|
||||||
|
return await compute(
|
||||||
|
_processMatchedKeywordsInIsolate,
|
||||||
|
DetectionParams(text: text, filters: filters),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart';
|
||||||
|
|
||||||
|
/// Data Transfer Object used to package data sent to the Isolate.
|
||||||
|
/// Since Isolates cannot share memory, we need a simple object to pass parameters.
|
||||||
|
class DetectionParams {
|
||||||
|
final String text;
|
||||||
|
final List<KeywordFilter> filters;
|
||||||
|
|
||||||
|
DetectionParams({required this.text, required this.filters});
|
||||||
|
}
|
||||||
@@ -0,0 +1,55 @@
|
|||||||
|
import 'package:tmail_ui_user/features/composer/presentation/manager/keyword_filter.dart';
|
||||||
|
|
||||||
|
/// Implementation of Exclude List Filter.
|
||||||
|
/// Optimized for memory usage by avoiding unnecessary String allocations.
|
||||||
|
class ExcludeListFilter implements KeywordFilter {
|
||||||
|
final Set<String> _excludes;
|
||||||
|
|
||||||
|
/// Constructor takes a List for convenience, but converts to Set for O(1) lookup.
|
||||||
|
ExcludeListFilter(List<String> rawExcludes)
|
||||||
|
: _excludes = rawExcludes.map((e) => e.toLowerCase()).toSet();
|
||||||
|
|
||||||
|
/// Static RegExp to check for whitespace.
|
||||||
|
/// Using a static instance avoids re-compilation in loops.
|
||||||
|
static final RegExp _whitespaceRegExp = RegExp(r'\s');
|
||||||
|
|
||||||
|
@override
|
||||||
|
bool isValid(String fullText, Match match) {
|
||||||
|
if (_excludes.isEmpty) return true;
|
||||||
|
|
||||||
|
// Extract the full token surrounding the match
|
||||||
|
final token = _getSurroundingToken(fullText, match.start, match.end);
|
||||||
|
final lowerToken = token.toLowerCase();
|
||||||
|
|
||||||
|
// Exact match with the token (e.g., "file-246")
|
||||||
|
if (_excludes.contains(lowerToken)) return false;
|
||||||
|
|
||||||
|
// Match with trailing punctuation removed (e.g., "file246." -> "file246")
|
||||||
|
final cleanToken = lowerToken.replaceAll(RegExp(r'[^\w\s]+$'), '');
|
||||||
|
if (_excludes.contains(cleanToken)) return false;
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extracts the surrounding token without creating garbage strings.
|
||||||
|
/// Instead of using .trim() inside a loop (which allocates memory),
|
||||||
|
/// we check characters directly.
|
||||||
|
String _getSurroundingToken(String text, int matchStart, int matchEnd) {
|
||||||
|
int start = matchStart;
|
||||||
|
int end = matchEnd;
|
||||||
|
|
||||||
|
// Expand to the left until whitespace or start of string
|
||||||
|
while (start > 0) {
|
||||||
|
// Check the character before the current start index
|
||||||
|
if (_whitespaceRegExp.hasMatch(text[start - 1])) break;
|
||||||
|
start--;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Expand to the right until whitespace or end of string
|
||||||
|
while (end < text.length) {
|
||||||
|
if (_whitespaceRegExp.hasMatch(text[end])) break;
|
||||||
|
end++;
|
||||||
|
}
|
||||||
|
return text.substring(start, end);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
/// Common interface for all keyword filters.
|
||||||
|
abstract class KeywordFilter {
|
||||||
|
/// Returns `true` if the keyword is valid.
|
||||||
|
/// Returns `false` if the keyword should be rejected.
|
||||||
|
bool isValid(String fullText, Match match);
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import 'package:core/utils/app_logger.dart';
|
import 'package:core/utils/app_logger.dart';
|
||||||
import 'package:flutter_test/flutter_test.dart';
|
import 'package:flutter_test/flutter_test.dart';
|
||||||
import 'package:tmail_ui_user/features/composer/presentation/manager/attachment_text_detector.dart';
|
import 'package:tmail_ui_user/features/composer/presentation/manager/attachment_text_detector.dart';
|
||||||
|
import 'package:tmail_ui_user/features/composer/presentation/manager/exclude_list_filter.dart';
|
||||||
import 'package:tmail_ui_user/main/localizations/language_code_constants.dart';
|
import 'package:tmail_ui_user/main/localizations/language_code_constants.dart';
|
||||||
|
|
||||||
/// Helper: generate email about [targetLength] characters long
|
/// Helper: generate email about [targetLength] characters long
|
||||||
@@ -17,6 +18,8 @@ String generateLongEmail(int targetLength, {bool includeKeywords = true}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void main() {
|
void main() {
|
||||||
|
TestWidgetsFlutterBinding.ensureInitialized();
|
||||||
|
|
||||||
group('AttachmentTextDetector.containsAttachmentKeyword', () {
|
group('AttachmentTextDetector.containsAttachmentKeyword', () {
|
||||||
test('English - should detect "attach" and "attachment"', () {
|
test('English - should detect "attach" and "attachment"', () {
|
||||||
expect(
|
expect(
|
||||||
@@ -266,14 +269,14 @@ void main() {
|
|||||||
});
|
});
|
||||||
|
|
||||||
group('AttachmentTextDetector.matchedKeywordsUnique', () {
|
group('AttachmentTextDetector.matchedKeywordsUnique', () {
|
||||||
test('should return unique matches across multiple languages', () {
|
test('should return unique matches across multiple languages', () async {
|
||||||
const email = """
|
const email = """
|
||||||
Please see the attached document.
|
Please see the attached document.
|
||||||
Vui lòng xem tài liệu đính kèm.
|
Vui lòng xem tài liệu đính kèm.
|
||||||
Смотрите приложение с отчетом.
|
Смотрите приложение с отчетом.
|
||||||
""";
|
""";
|
||||||
|
|
||||||
final matches = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final matches = await AttachmentTextDetector.matchedKeywordsUnique(email);
|
||||||
|
|
||||||
expect(matches,
|
expect(matches,
|
||||||
containsAll(['tài liệu', 'đính kèm', 'приложение']));
|
containsAll(['tài liệu', 'đính kèm', 'приложение']));
|
||||||
@@ -281,14 +284,14 @@ void main() {
|
|||||||
reason: 'No duplicates allowed');
|
reason: 'No duplicates allowed');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('should return unique matches even if repeated multiple times', () {
|
test('should return unique matches even if repeated multiple times', () async {
|
||||||
const email = """
|
const email = """
|
||||||
file file file attach attach attach
|
file file file attach attach attach
|
||||||
đính kèm đính kèm
|
đính kèm đính kèm
|
||||||
приложение приложение
|
приложение приложение
|
||||||
""";
|
""";
|
||||||
|
|
||||||
final matches = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final matches = await AttachmentTextDetector.matchedKeywordsUnique(email);
|
||||||
|
|
||||||
expect(
|
expect(
|
||||||
matches, containsAll(['file', 'attach', 'đính kèm', 'приложение']));
|
matches, containsAll(['file', 'attach', 'đính kèm', 'приложение']));
|
||||||
@@ -296,15 +299,15 @@ void main() {
|
|||||||
reason: 'Duplicates must be removed');
|
reason: 'Duplicates must be removed');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('should return empty list if no keywords present', () {
|
test('should return empty list if no keywords present', () async {
|
||||||
const email = "This email does not mention anything related to document.";
|
const email = "This email does not mention anything related to document.";
|
||||||
final matches = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final matches = await AttachmentTextDetector.matchedKeywordsUnique(email);
|
||||||
expect(matches, isEmpty);
|
expect(matches, isEmpty);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('should match Russian word with both ё and е', () {
|
test('should match Russian word with both ё and е', () async {
|
||||||
const email = "Вот наш отчёт и вот наш снова.";
|
const email = "Вот наш отчёт и вот наш снова.";
|
||||||
final matches = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final matches = await AttachmentTextDetector.matchedKeywordsUnique(email);
|
||||||
|
|
||||||
expect(matches, containsAll(['отчёт']));
|
expect(matches, containsAll(['отчёт']));
|
||||||
});
|
});
|
||||||
@@ -384,25 +387,25 @@ void main() {
|
|||||||
|
|
||||||
/// Test input validation and edge cases
|
/// Test input validation and edge cases
|
||||||
group('AttachmentTextDetector Input Validation', () {
|
group('AttachmentTextDetector Input Validation', () {
|
||||||
test('should handle empty input gracefully', () {
|
test('should handle empty input gracefully', () async {
|
||||||
expect(AttachmentTextDetector.containsAnyAttachmentKeyword(''), isFalse);
|
expect(AttachmentTextDetector.containsAnyAttachmentKeyword(''), isFalse);
|
||||||
expect(AttachmentTextDetector.matchedKeywordsUnique(''), isEmpty);
|
expect((await AttachmentTextDetector.matchedKeywordsUnique('')), isEmpty);
|
||||||
expect(AttachmentTextDetector.matchedKeywordsAll(''), isEmpty);
|
expect(AttachmentTextDetector.matchedKeywordsAll(''), isEmpty);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('should handle special characters and symbols', () {
|
test('should handle special characters and symbols', () async {
|
||||||
const email = "attach@#\$%^&*()_+ pièce jointe!@#\$%";
|
const email = "attach@#\$%^&*()_+ pièce jointe!@#\$%";
|
||||||
expect(
|
expect(
|
||||||
AttachmentTextDetector.containsAnyAttachmentKeyword(email), isTrue);
|
AttachmentTextDetector.containsAnyAttachmentKeyword(email), isTrue);
|
||||||
final matches = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final matches = await AttachmentTextDetector.matchedKeywordsUnique(email);
|
||||||
expect(matches, containsAll(['attach', 'pièce jointe']));
|
expect(matches, containsAll(['attach', 'pièce jointe']));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('should handle whitespace-only input', () {
|
test('should handle whitespace-only input', () async {
|
||||||
const email = " \n\t\r ";
|
const email = " \n\t\r ";
|
||||||
expect(
|
expect(
|
||||||
AttachmentTextDetector.containsAnyAttachmentKeyword(email), isFalse);
|
AttachmentTextDetector.containsAnyAttachmentKeyword(email), isFalse);
|
||||||
expect(AttachmentTextDetector.matchedKeywordsUnique(email), isEmpty);
|
expect((await AttachmentTextDetector.matchedKeywordsUnique(email)), isEmpty);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('should handle newlines and tabs in content', () {
|
test('should handle newlines and tabs in content', () {
|
||||||
@@ -504,11 +507,11 @@ void main() {
|
|||||||
|
|
||||||
/// Test Unicode and encoding edge cases
|
/// Test Unicode and encoding edge cases
|
||||||
group('AttachmentTextDetector Unicode Handling', () {
|
group('AttachmentTextDetector Unicode Handling', () {
|
||||||
test('should handle emoji and special Unicode characters', () {
|
test('should handle emoji and special Unicode characters', () async {
|
||||||
const email = "📎 attachment 📄 file 🔗 pièce jointe";
|
const email = "📎 attachment 📄 file 🔗 pièce jointe";
|
||||||
expect(
|
expect(
|
||||||
AttachmentTextDetector.containsAnyAttachmentKeyword(email), isTrue);
|
AttachmentTextDetector.containsAnyAttachmentKeyword(email), isTrue);
|
||||||
final matches = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final matches = await AttachmentTextDetector.matchedKeywordsUnique(email);
|
||||||
expect(matches, containsAll(['attachment', 'file', 'pièce jointe']));
|
expect(matches, containsAll(['attachment', 'file', 'pièce jointe']));
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -538,7 +541,7 @@ void main() {
|
|||||||
|
|
||||||
/// Test real-world scenarios and integration cases
|
/// Test real-world scenarios and integration cases
|
||||||
group('AttachmentTextDetector Real-World Scenarios', () {
|
group('AttachmentTextDetector Real-World Scenarios', () {
|
||||||
test('should handle common email signatures and footers', () {
|
test('should handle common email signatures and footers', () async {
|
||||||
const email = """
|
const email = """
|
||||||
Please review the attached document.
|
Please review the attached document.
|
||||||
|
|
||||||
@@ -547,7 +550,7 @@ void main() {
|
|||||||
|
|
||||||
This email and any attachments are confidential and may be privileged.
|
This email and any attachments are confidential and may be privileged.
|
||||||
""";
|
""";
|
||||||
final matches = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final matches = await AttachmentTextDetector.matchedKeywordsUnique(email);
|
||||||
expect(matches, containsAll(['attached', 'attachments']));
|
expect(matches, containsAll(['attached', 'attachments']));
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -558,7 +561,7 @@ void main() {
|
|||||||
AttachmentTextDetector.containsAnyAttachmentKeyword(email), isTrue);
|
AttachmentTextDetector.containsAnyAttachmentKeyword(email), isTrue);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('should handle quoted email content', () {
|
test('should handle quoted email content', () async {
|
||||||
const email = """
|
const email = """
|
||||||
Hi John,
|
Hi John,
|
||||||
|
|
||||||
@@ -570,11 +573,11 @@ void main() {
|
|||||||
|
|
||||||
Thanks!
|
Thanks!
|
||||||
""";
|
""";
|
||||||
final matches = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final matches = await AttachmentTextDetector.matchedKeywordsUnique(email);
|
||||||
expect(matches, containsAll(['attachment', 'attached', 'file']));
|
expect(matches, containsAll(['attachment', 'attached', 'file']));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('should handle multiple languages in single email', () {
|
test('should handle multiple languages in single email', () async {
|
||||||
const email = """
|
const email = """
|
||||||
English: Please see the attached file.
|
English: Please see the attached file.
|
||||||
French: Veuillez voir le fichier joint.
|
French: Veuillez voir le fichier joint.
|
||||||
@@ -584,7 +587,7 @@ void main() {
|
|||||||
final matchesAll = AttachmentTextDetector.matchedKeywordsAll(email);
|
final matchesAll = AttachmentTextDetector.matchedKeywordsAll(email);
|
||||||
expect(matchesAll.keys, containsAll(['en', 'fr', 'vi', 'ru']));
|
expect(matchesAll.keys, containsAll(['en', 'fr', 'vi', 'ru']));
|
||||||
|
|
||||||
final matchesUnique = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final matchesUnique = await AttachmentTextDetector.matchedKeywordsUnique(email);
|
||||||
expect(
|
expect(
|
||||||
matchesUnique,
|
matchesUnique,
|
||||||
containsAll([
|
containsAll([
|
||||||
@@ -622,10 +625,13 @@ void main() {
|
|||||||
returnsNormally);
|
returnsNormally);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('should handle repeated keyword patterns efficiently', () {
|
test('should handle repeated keyword patterns efficiently', () async {
|
||||||
final email = "attach " * 10000; // 10K repetitions
|
final email = "attach " * 10000; // 10K repetitions, ~70k chars
|
||||||
final sw = Stopwatch()..start();
|
final sw = Stopwatch()..start();
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
|
email,
|
||||||
|
forceSync: true,
|
||||||
|
);
|
||||||
sw.stop();
|
sw.stop();
|
||||||
|
|
||||||
expect(result, equals(['attach']));
|
expect(result, equals(['attach']));
|
||||||
@@ -644,7 +650,7 @@ void main() {
|
|||||||
lessThan(500)); // Should complete within 500ms
|
lessThan(500)); // Should complete within 500ms
|
||||||
});
|
});
|
||||||
|
|
||||||
test('should handle many different keywords in large text', () {
|
test('should handle many different keywords in large text', () async {
|
||||||
final email = """
|
final email = """
|
||||||
attach attachment file document report
|
attach attachment file document report
|
||||||
pièce jointe fichier joint document joint
|
pièce jointe fichier joint document joint
|
||||||
@@ -654,7 +660,7 @@ void main() {
|
|||||||
1000; // Repeat 1000 times
|
1000; // Repeat 1000 times
|
||||||
|
|
||||||
final sw = Stopwatch()..start();
|
final sw = Stopwatch()..start();
|
||||||
final matches = AttachmentTextDetector.matchedKeywordsUnique(email);
|
final matches = await AttachmentTextDetector.matchedKeywordsUnique(email);
|
||||||
sw.stop();
|
sw.stop();
|
||||||
|
|
||||||
expect(matches.length, greaterThan(10));
|
expect(matches.length, greaterThan(10));
|
||||||
@@ -714,9 +720,9 @@ void main() {
|
|||||||
expect(result.keys.length, equals(1));
|
expect(result.keys.length, equals(1));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('matchedKeywordsUnique should return unique Arabic keywords only', () {
|
test('matchedKeywordsUnique should return unique Arabic keywords only', () async {
|
||||||
const text = 'مرفق ملف مرفق تقرير ملف';
|
const text = 'مرفق ملف مرفق تقرير ملف';
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(text);
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(text);
|
||||||
|
|
||||||
expect(result, containsAll(['مرفق', 'ملف', 'تقرير']));
|
expect(result, containsAll(['مرفق', 'ملف', 'تقرير']));
|
||||||
// Ensure no duplicates
|
// Ensure no duplicates
|
||||||
@@ -753,10 +759,10 @@ void main() {
|
|||||||
expect(result[LanguageCodeConstants.french], contains('pièce jointe'));
|
expect(result[LanguageCodeConstants.french], contains('pièce jointe'));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('matchedKeywordsUnique should return unique keywords from all languages', () {
|
test('matchedKeywordsUnique should return unique keywords from all languages', () async {
|
||||||
const text = 'Attached báo cáo مرفق pièce jointe file مستند attach đính kèm';
|
const text = 'Attached báo cáo مرفق pièce jointe file مستند attach đính kèm';
|
||||||
|
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(text);
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(text);
|
||||||
|
|
||||||
expect(result, containsAll([
|
expect(result, containsAll([
|
||||||
'attach',
|
'attach',
|
||||||
@@ -775,27 +781,27 @@ void main() {
|
|||||||
|
|
||||||
group('AttachmentTextDetector.matchedKeywordsUnique', () {
|
group('AttachmentTextDetector.matchedKeywordsUnique', () {
|
||||||
group('Logic Tests', () {
|
group('Logic Tests', () {
|
||||||
test('Basic Match: Should detect simple keywords', () {
|
test('Basic Match: Should detect simple keywords', () async {
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
'Please find the attachment.');
|
'Please find the attachment.');
|
||||||
expect(result, contains('attachment'));
|
expect(result, contains('attachment'));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Case Insensitivity: Should detect mixed case keywords', () {
|
test('Case Insensitivity: Should detect mixed case keywords', () async {
|
||||||
final result =
|
final result =
|
||||||
AttachmentTextDetector.matchedKeywordsUnique('FiLe is here');
|
await AttachmentTextDetector.matchedKeywordsUnique('FiLe is here');
|
||||||
expect(result, contains('file'));
|
expect(result, contains('file'));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Unique Results: Should not return duplicates', () {
|
test('Unique Results: Should not return duplicates', () async {
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
'File here, file there, FILE everywhere.');
|
'File here, file there, FILE everywhere.');
|
||||||
expect(result.length, 1);
|
expect(result.length, 1);
|
||||||
expect(result, contains('file'));
|
expect(result, contains('file'));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Multiple Keywords: Should detect different keywords', () {
|
test('Multiple Keywords: Should detect different keywords', () async {
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
'See file and attachment');
|
'See file and attachment');
|
||||||
expect(result, containsAll(['file', 'attachment']));
|
expect(result, containsAll(['file', 'attachment']));
|
||||||
});
|
});
|
||||||
@@ -804,40 +810,40 @@ void main() {
|
|||||||
group('Suffix & Boundary Tests (Crucial)', () {
|
group('Suffix & Boundary Tests (Crucial)', () {
|
||||||
test(
|
test(
|
||||||
'Invalid Suffix: Should IGNORE words extended by letters (e.g., filetage)',
|
'Invalid Suffix: Should IGNORE words extended by letters (e.g., filetage)',
|
||||||
() {
|
() async {
|
||||||
// "filetage" contains "file", but followed by 't' (letter) -> Should fail
|
// "filetage" contains "file", but followed by 't' (letter) -> Should fail
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
'This is a filetage system.');
|
'This is a filetage system.');
|
||||||
expect(result, isEmpty);
|
expect(result, isEmpty);
|
||||||
});
|
});
|
||||||
|
|
||||||
test(
|
test(
|
||||||
'Valid Suffix (Number): Should ACCEPT words followed by numbers (e.g., file123)',
|
'Valid Suffix (Number): Should ACCEPT words followed by numbers (e.g., file123)',
|
||||||
() {
|
() async {
|
||||||
final result =
|
final result =
|
||||||
AttachmentTextDetector.matchedKeywordsUnique('Check file123 now.');
|
await AttachmentTextDetector.matchedKeywordsUnique('Check file123 now.');
|
||||||
expect(result, contains('file'));
|
expect(result, contains('file'));
|
||||||
});
|
});
|
||||||
|
|
||||||
test(
|
test(
|
||||||
'Valid Suffix (Punctuation): Should ACCEPT words followed by punctuation',
|
'Valid Suffix (Punctuation): Should ACCEPT words followed by punctuation',
|
||||||
() {
|
() async {
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
'Is this a file? Yes, file.');
|
'Is this a file? Yes, file.');
|
||||||
expect(result, contains('file'));
|
expect(result, contains('file'));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Valid Suffix (Whitespace): Should ACCEPT words followed by space',
|
test('Valid Suffix (Whitespace): Should ACCEPT words followed by space',
|
||||||
() {
|
() async {
|
||||||
final result =
|
final result =
|
||||||
AttachmentTextDetector.matchedKeywordsUnique('file name');
|
await AttachmentTextDetector.matchedKeywordsUnique('file name');
|
||||||
expect(result, contains('file'));
|
expect(result, contains('file'));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Valid Suffix (End of String): Should ACCEPT word at the very end',
|
test('Valid Suffix (End of String): Should ACCEPT word at the very end',
|
||||||
() {
|
() async {
|
||||||
final result =
|
final result =
|
||||||
AttachmentTextDetector.matchedKeywordsUnique('Open file');
|
await AttachmentTextDetector.matchedKeywordsUnique('Open file');
|
||||||
expect(result, contains('file'));
|
expect(result, contains('file'));
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -845,10 +851,10 @@ void main() {
|
|||||||
group('Complex & Unicode Tests', () {
|
group('Complex & Unicode Tests', () {
|
||||||
test(
|
test(
|
||||||
'Longest Match Priority: Should match "attachment" instead of "attach"',
|
'Longest Match Priority: Should match "attachment" instead of "attach"',
|
||||||
() {
|
() async {
|
||||||
// Because we sort by length desc, "attachment" comes before "attach" in regex
|
// Because we sort by length desc, "attachment" comes before "attach" in regex
|
||||||
final result =
|
final result =
|
||||||
AttachmentTextDetector.matchedKeywordsUnique('See attachment.');
|
await AttachmentTextDetector.matchedKeywordsUnique('See attachment.');
|
||||||
|
|
||||||
expect(result, contains('attachment'));
|
expect(result, contains('attachment'));
|
||||||
// Note: "attachment" contains "attach", but since it consumes the text,
|
// Note: "attachment" contains "attach", but since it consumes the text,
|
||||||
@@ -857,27 +863,27 @@ void main() {
|
|||||||
expect(result, isNot(contains('attach')));
|
expect(result, isNot(contains('attach')));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Vietnamese Support: Should detect keywords with accents', () {
|
test('Vietnamese Support: Should detect keywords with accents', () async {
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
'Gửi tài liệu đính kèm.');
|
'Gửi tài liệu đính kèm.');
|
||||||
expect(result, containsAll(['tài liệu', 'đính kèm']));
|
expect(result, containsAll(['tài liệu', 'đính kèm']));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Russian Support: Should detect Cyrillic characters', () {
|
test('Russian Support: Should detect Cyrillic characters', () async {
|
||||||
final result =
|
final result =
|
||||||
AttachmentTextDetector.matchedKeywordsUnique('Это файл.');
|
await AttachmentTextDetector.matchedKeywordsUnique('Это файл.');
|
||||||
expect(result, contains('файл'));
|
expect(result, contains('файл'));
|
||||||
});
|
});
|
||||||
|
|
||||||
test('Arabic Support: Should detect Arabic characters', () {
|
test('Arabic Support: Should detect Arabic characters', () async {
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
'هذا ملف مهم'); // "This is an important file"
|
'هذا ملف مهم'); // "This is an important file"
|
||||||
expect(result, contains('ملف'));
|
expect(result, contains('ملف'));
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
group('Performance Benchmark', () {
|
group('Performance Benchmark', () {
|
||||||
test('Large Text Performance: Should process 100k chars under 50ms', () {
|
test('Large Text Performance: Should process 100k chars under 50ms', () async {
|
||||||
// Generate a large text (~100k characters)
|
// Generate a large text (~100k characters)
|
||||||
final buffer = StringBuffer();
|
final buffer = StringBuffer();
|
||||||
for (int i = 0; i < 5000; i++) {
|
for (int i = 0; i < 5000; i++) {
|
||||||
@@ -887,11 +893,14 @@ void main() {
|
|||||||
}
|
}
|
||||||
if (i % 150 == 0) buffer.write('filetage '); // Insert trap word
|
if (i % 150 == 0) buffer.write('filetage '); // Insert trap word
|
||||||
}
|
}
|
||||||
final largeText = buffer.toString();
|
final largeText = buffer.toString(); // ~215k chars
|
||||||
|
|
||||||
// Measure execution time
|
// Measure execution time
|
||||||
final stopwatch = Stopwatch()..start();
|
final stopwatch = Stopwatch()..start();
|
||||||
final result = AttachmentTextDetector.matchedKeywordsUnique(largeText);
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
|
largeText,
|
||||||
|
forceSync: true,
|
||||||
|
);
|
||||||
stopwatch.stop();
|
stopwatch.stop();
|
||||||
|
|
||||||
log('Performance result: Found ${result.length} keywords in ${stopwatch.elapsedMilliseconds}ms for ${largeText.length} chars.');
|
log('Performance result: Found ${result.length} keywords in ${stopwatch.elapsedMilliseconds}ms for ${largeText.length} chars.');
|
||||||
@@ -906,4 +915,112 @@ void main() {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
group('AttachmentTextDetector with exclude filter', () {
|
||||||
|
test('Basic detection without filters', () async {
|
||||||
|
final result =
|
||||||
|
await AttachmentTextDetector.matchedKeywordsUnique('Check file attachment');
|
||||||
|
expect(result, containsAll(['file', 'attachment']));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Exclude Filter: Should exclude blacklisted tokens', () async {
|
||||||
|
final excludeFilter = ExcludeListFilter(['file-246', 'my_file']);
|
||||||
|
|
||||||
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
|
'Check file-246 and my_file please.',
|
||||||
|
filters: [excludeFilter],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result, isEmpty);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Exclude Filter: Should NOT exclude partial matches', () async {
|
||||||
|
final excludeFilter = ExcludeListFilter(['file-246']);
|
||||||
|
|
||||||
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
|
'Check file-999.',
|
||||||
|
filters: [excludeFilter],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result, contains('file'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Multiple Filters Integration (Future Proof)', () async {
|
||||||
|
final excludeFilter = ExcludeListFilter(['file-bad']);
|
||||||
|
|
||||||
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
|
'file-good vs file-bad',
|
||||||
|
filters: [
|
||||||
|
excludeFilter,
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result, contains('file'));
|
||||||
|
expect(result, isNot(contains('file-bad')));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
group('AttachmentTextDetector Sync vs Async Logic', () {
|
||||||
|
test('Should run SYNC for short text and detect keywords', () async {
|
||||||
|
const shortText = "Please check the attached file for details.";
|
||||||
|
|
||||||
|
expect(shortText.length, lessThan(20000));
|
||||||
|
|
||||||
|
final result =
|
||||||
|
await AttachmentTextDetector.matchedKeywordsUnique(shortText);
|
||||||
|
|
||||||
|
expect(result, containsAll(['attached', 'file']));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Should run ASYNC (via compute) for large text', () async {
|
||||||
|
final sb = StringBuffer();
|
||||||
|
sb.write("nonsense " * 3000);
|
||||||
|
sb.write(" attachment ");
|
||||||
|
sb.write("garbage " * 1000);
|
||||||
|
|
||||||
|
final longText = sb.toString();
|
||||||
|
|
||||||
|
expect(longText.length, greaterThan(20000));
|
||||||
|
|
||||||
|
final result =
|
||||||
|
await AttachmentTextDetector.matchedKeywordsUnique(longText);
|
||||||
|
|
||||||
|
expect(result, contains('attachment'));
|
||||||
|
expect(result, isNot(contains('nonsense')));
|
||||||
|
});
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Should apply Filters correctly even when running in Isolate (Long text)',
|
||||||
|
() async {
|
||||||
|
final sb = StringBuffer();
|
||||||
|
sb.write("padding " * 3000);
|
||||||
|
sb.write(" file-246 ");
|
||||||
|
sb.write(" file-ok ");
|
||||||
|
sb.write("padding " * 1000);
|
||||||
|
|
||||||
|
final longText = sb.toString();
|
||||||
|
|
||||||
|
final excludeFilter = ExcludeListFilter(['file-246']);
|
||||||
|
|
||||||
|
final result = await AttachmentTextDetector.matchedKeywordsUnique(
|
||||||
|
longText,
|
||||||
|
filters: [excludeFilter],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(result, contains('file'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Should handle multiple async calls concurrently', () async {
|
||||||
|
final longText = "file " * 4000; // ~20k chars
|
||||||
|
|
||||||
|
final futures = List.generate(
|
||||||
|
10, (_) => AttachmentTextDetector.matchedKeywordsUnique(longText));
|
||||||
|
|
||||||
|
final results = await Future.wait(futures);
|
||||||
|
|
||||||
|
for (final res in results) {
|
||||||
|
expect(res, contains('file'));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user