commit ce90a17: [Fix] Fix tokenization near exceptions

[Vsevolod Stakhov]

Author: Vsevolod Stakhov
Date: 2021-06-16 16:40:53 +0100
URL: https://github.com/rspamd/rspamd/commit/ce90a17cc548782f9fa204148750f2c5c192e8d7

[Fix] Fix tokenization near exceptions

---
 src/libstat/tokenizers/tokenizers.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 8d6d93add..2dd4a6f5a 100644
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -459,8 +459,8 @@ start_over:
 									ex->pos)) {
 								token.original.begin = text + last;
 								token.original.len = ex->pos - last;
-								token.flags = 0;
-								g_array_append_val (res, token);
+								token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT |
+											  RSPAMD_STAT_TOKEN_FLAG_UTF;
 							}
 
 							/* Process the current exception */