Use html parser instead of regex to clean composer input

Before sending input to the LLM, we clean the composer input to
remove html tags. We relied on regex that we not complete enough and
could be bypassed easily. So let's use the html parser available in
Dart instead.

I also added some tests.
This commit is contained in:
Théo Poizat
2025-12-15 08:22:16 +01:00
committed by Dat H. Pham
parent f77580ed44
commit a9cfe86a59
2 changed files with 39 additions and 8 deletions
+23 -8
View File
@@ -4,6 +4,7 @@ import 'dart:typed_data';
import 'package:core/utils/app_logger.dart';
import 'package:core/domain/exceptions/string_exception.dart';
import 'package:core/utils/mail/named_address.dart';
import 'package:html/dom.dart';
import 'package:html/parser.dart';
import 'package:http_parser/http_parser.dart';
@@ -198,15 +199,29 @@ class StringConvert {
}
static String convertHtmlContentToTextContent(String htmlContent) {
String textContent = htmlContent.replaceAll(RegExp(r'<[^>]*>'), '');
final document = parse(htmlContent);
textContent = textContent
.replaceAll('&nbsp;', ' ')
.replaceAll('&amp;', '&')
.replaceAll('&lt;', '<')
.replaceAll('&gt;', '>')
.replaceAll('&quot;', '"')
.replaceAll('&#39;', "'");
// Each paragraph is surrounded by block tags so we add a /n for each block tag
// Even <br> are surrounded by block tags so we can ignore <br> and treat them
// as paragraph
const blockTags = [
'p',
'div',
'li',
'section',
'article',
'header',
'footer',
'h1','h2','h3','h4','h5','h6'
];
for (final tag in blockTags) {
document.querySelectorAll(tag).forEach((element) {
element.append(Text('\n'));
});
}
final String textContent = document.body?.text ?? '';
return textContent.trim();
}
+16
View File
@@ -816,4 +816,20 @@ void main() {
);
});
});
group('StringConvert.convertHtmlContentToTextContent', () {
test('should preserve line breaks between paragraphs', () {
const htmlContent = '<p>First paragraph</p><p>Second paragraph</p><p>Third paragraph</p>';
final result = StringConvert.convertHtmlContentToTextContent(htmlContent);
expect(result, 'First paragraph\nSecond paragraph\nThird paragraph');
});
test('should preserve line breaks for br', () {
const htmlContent = '<p>First paragraph</p><div><br></div><p>Second paragraph</p>';
final result = StringConvert.convertHtmlContentToTextContent(htmlContent);
expect(result, 'First paragraph\n\nSecond paragraph');
});
});
}