Use html parser instead of regex to clean composer input
Before sending input to the LLM, we clean the composer input to remove html tags. We relied on regex that we not complete enough and could be bypassed easily. So let's use the html parser available in Dart instead. I also added some tests.
This commit is contained in:
@@ -4,6 +4,7 @@ import 'dart:typed_data';
|
||||
import 'package:core/utils/app_logger.dart';
|
||||
import 'package:core/domain/exceptions/string_exception.dart';
|
||||
import 'package:core/utils/mail/named_address.dart';
|
||||
import 'package:html/dom.dart';
|
||||
import 'package:html/parser.dart';
|
||||
import 'package:http_parser/http_parser.dart';
|
||||
|
||||
@@ -198,15 +199,29 @@ class StringConvert {
|
||||
}
|
||||
|
||||
static String convertHtmlContentToTextContent(String htmlContent) {
|
||||
String textContent = htmlContent.replaceAll(RegExp(r'<[^>]*>'), '');
|
||||
final document = parse(htmlContent);
|
||||
|
||||
textContent = textContent
|
||||
.replaceAll(' ', ' ')
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll(''', "'");
|
||||
// Each paragraph is surrounded by block tags so we add a /n for each block tag
|
||||
// Even <br> are surrounded by block tags so we can ignore <br> and treat them
|
||||
// as paragraph
|
||||
const blockTags = [
|
||||
'p',
|
||||
'div',
|
||||
'li',
|
||||
'section',
|
||||
'article',
|
||||
'header',
|
||||
'footer',
|
||||
'h1','h2','h3','h4','h5','h6'
|
||||
];
|
||||
|
||||
for (final tag in blockTags) {
|
||||
document.querySelectorAll(tag).forEach((element) {
|
||||
element.append(Text('\n'));
|
||||
});
|
||||
}
|
||||
|
||||
final String textContent = document.body?.text ?? '';
|
||||
|
||||
return textContent.trim();
|
||||
}
|
||||
|
||||
@@ -816,4 +816,20 @@ void main() {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
group('StringConvert.convertHtmlContentToTextContent', () {
|
||||
test('should preserve line breaks between paragraphs', () {
|
||||
const htmlContent = '<p>First paragraph</p><p>Second paragraph</p><p>Third paragraph</p>';
|
||||
final result = StringConvert.convertHtmlContentToTextContent(htmlContent);
|
||||
|
||||
expect(result, 'First paragraph\nSecond paragraph\nThird paragraph');
|
||||
});
|
||||
|
||||
test('should preserve line breaks for br', () {
|
||||
const htmlContent = '<p>First paragraph</p><div><br></div><p>Second paragraph</p>';
|
||||
final result = StringConvert.convertHtmlContentToTextContent(htmlContent);
|
||||
|
||||
expect(result, 'First paragraph\n\nSecond paragraph');
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user