commit 69aeac8: [Minor] Langdet: Exclude exceptions (e.g. urls)

[Vsevolod Stakhov]

Author: Vsevolod Stakhov
Date: 2019-08-02 18:28:11 +0100
URL: https://github.com/rspamd/rspamd/commit/69aeac8388061295ac17518a223fe36efda4c2b1

[Minor] Langdet: Exclude exceptions (e.g. urls)

---
 src/libmime/lang_detection.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c
index 276771778..74c6f7247 100644
--- a/src/libmime/lang_detection.c
+++ b/src/libmime/lang_detection.c
@@ -939,6 +939,7 @@ rspamd_language_detector_random_select (GArray *ucs_tokens, guint nwords,
 			/* Filter bad tokens */
 
 			if (tok->unicode.len >= 2 &&
+					!(tok->flags & RSPAMD_STAT_TOKEN_FLAG_EXCEPTION) &&
 					u_isalpha (tok->unicode.begin[0]) &&
 					u_isalpha (tok->unicode.begin[tok->unicode.len - 1])) {
 				offsets_out[out_idx] = sel;