commit b6dbe34: [Minor] Core: Improve raw files scan

[Vsevolod Stakhov]

Author: Vsevolod Stakhov
Date: 2019-01-05 14:50:01 +0000
URL: https://github.com/rspamd/rspamd/commit/b6dbe34858bf747446e1d4796e8d7e8dfebce6ff

[Minor] Core: Improve raw files scan

---
 src/libmime/message.c     | 43 ++++++++++++++++++++++++++++++++++++++++---
 src/libmime/mime_parser.c |  2 +-
 src/libmime/mime_parser.h |  3 +++
 3 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/src/libmime/message.c b/src/libmime/message.c
index b75eb6b68..a15485339 100644
--- a/src/libmime/message.c
+++ b/src/libmime/message.c
@@ -712,7 +712,8 @@ rspamd_message_process_text_part_maybe (struct rspamd_task *task,
 	gboolean found_html = FALSE, found_txt = FALSE;
 	enum rspamd_action_type act;
 
-	if (IS_CT_TEXT (mime_part->ct)) {
+	if (IS_CT_TEXT (mime_part->ct) && (!mime_part->detected_ct ||
+									   IS_CT_TEXT (mime_part->detected_ct))) {
 		html_tok.begin = "html";
 		html_tok.len = 4;
 		xhtml_tok.begin = "xhtml";
@@ -899,6 +900,7 @@ rspamd_message_from_data (struct rspamd_task *task, const guchar *start,
 	const char *mb = NULL;
 	gchar *mid;
 	rspamd_ftok_t srch, *tok;
+	gchar cdbuf[1024];
 
 	g_assert (start != NULL);
 
@@ -923,11 +925,26 @@ rspamd_message_from_data (struct rspamd_task *task, const guchar *start,
 			srch.len = strlen (mb);
 			ct = rspamd_content_type_parse (srch.begin, srch.len,
 					task->task_pool);
-			msg_warn_task ("construct fake mime of type: %s", mb);
 
 			if (!part->ct) {
+				msg_info_task ("construct fake mime of type: %s", mb);
 				part->ct = ct;
 			}
+			else {
+				/* Check sanity */
+				if (IS_CT_TEXT (part->ct)) {
+					RSPAMD_FTOK_FROM_STR (&srch, "application");
+
+					if (rspamd_ftok_cmp (&ct->type, &srch) == 0) {
+						msg_info_task ("construct fake mime of type: %s", mb);
+						part->ct = ct;
+					}
+				}
+				else {
+					msg_info_task ("construct fake mime of type: %T/%T, detected %s",
+							&part->ct->type, &part->ct->subtype, mb);
+				}
+			}
 
 			part->detected_ct = ct;
 		}
@@ -937,6 +954,27 @@ rspamd_message_from_data (struct rspamd_task *task, const guchar *start,
 	part->raw_data.len = len;
 	part->parsed_data.begin = start;
 	part->parsed_data.len = len;
+	part->id = task->parts->len;
+	part->raw_headers =  g_hash_table_new_full (rspamd_strcase_hash,
+			rspamd_strcase_equal, NULL, rspamd_ptr_array_free_hard);
+	part->headers_order = g_queue_new ();
+
+
+
+	tok = rspamd_task_get_request_header (task, "Filename");
+
+	if (tok) {
+		rspamd_snprintf (cdbuf, sizeof (cdbuf), "inline; filename=\"%T\"", tok);
+	}
+	else {
+		rspamd_snprintf (cdbuf, sizeof (cdbuf), "inline");
+	}
+
+	part->cd = rspamd_content_disposition_parse (cdbuf, strlen (cdbuf),
+			task->task_pool);
+
+	g_ptr_array_add (task->parts, part);
+	rspamd_mime_parser_calc_digest (part);
 
 	/* Generate message ID */
 	mid = rspamd_mime_message_id_generate ("localhost.localdomain");
@@ -1041,7 +1079,6 @@ rspamd_message_parse (struct rspamd_task *task)
 		}
 	}
 	else {
-		task->flags &= ~RSPAMD_TASK_FLAG_MIME;
 		rspamd_message_from_data (task, p, len);
 	}
 
diff --git a/src/libmime/mime_parser.c b/src/libmime/mime_parser.c
index 242c656f0..1dc3d85d8 100644
--- a/src/libmime/mime_parser.c
+++ b/src/libmime/mime_parser.c
@@ -405,7 +405,7 @@ rspamd_mime_part_get_cd (struct rspamd_task *task, struct rspamd_mime_part *part
 	part->cd = cd;
 }
 
-static void
+void
 rspamd_mime_parser_calc_digest (struct rspamd_mime_part *part)
 {
 	/* Blake2b applied to string 'rspamd' */
diff --git a/src/libmime/mime_parser.h b/src/libmime/mime_parser.h
index c0b7dec7e..5164ea4d5 100644
--- a/src/libmime/mime_parser.h
+++ b/src/libmime/mime_parser.h
@@ -19,6 +19,7 @@
 #include "config.h"
 
 struct rspamd_task;
+struct rspamd_mime_part;
 
 enum rspamd_mime_parse_error {
 	RSPAMD_MIME_PARSE_OK = 0,
@@ -29,4 +30,6 @@ enum rspamd_mime_parse_error {
 enum rspamd_mime_parse_error rspamd_mime_parse_task (struct rspamd_task *task,
 		GError **err);
 
+void rspamd_mime_parser_calc_digest (struct rspamd_mime_part *part);
+
 #endif /* SRC_LIBMIME_MIME_PARSER_H_ */