commit c3c8d4d: [Fix] Fix empty prefilters that require mime structures

[Vsevolod Stakhov]

Author: Vsevolod Stakhov
Date: 2019-10-13 09:34:27 +0100
URL: https://github.com/rspamd/rspamd/commit/c3c8d4d13302095c558e74f233269a3277de91c3 (HEAD -> master)

[Fix] Fix empty prefilters that require mime structures

---
 src/plugins/lua/asn.lua       |  3 +--
 src/plugins/lua/ratelimit.lua | 20 ++++++++++++++++----
 src/plugins/lua/settings.lua  | 12 ++++++++++++
 src/plugins/lua/whitelist.lua |  6 +++---
 4 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/src/plugins/lua/asn.lua b/src/plugins/lua/asn.lua
index 7f236ca99..ea698f5aa 100644
--- a/src/plugins/lua/asn.lua
+++ b/src/plugins/lua/asn.lua
@@ -126,10 +126,9 @@ end
 if configure_asn_module() then
   local id = rspamd_config:register_symbol({
     name = 'ASN_CHECK',
-    type = 'prefilter,nostat',
+    type = 'prefilter,nostat,empty',
     callback = asn_check,
     priority = 8,
-    flags = 'empty',
   })
   if options['symbol'] then
     rspamd_config:register_symbol({
diff --git a/src/plugins/lua/ratelimit.lua b/src/plugins/lua/ratelimit.lua
index bd77b1ad6..dbaffd975 100644
--- a/src/plugins/lua/ratelimit.lua
+++ b/src/plugins/lua/ratelimit.lua
@@ -396,11 +396,13 @@ local keywords = {
     ['get_value'] = function(task)
       return task:get_principal_recipient()
     end,
+    need_process = true,
   },
   ['digest'] = {
     ['get_value'] = function(task)
       return task:get_digest()
     end,
+    need_process = true,
   },
   ['attachments'] = {
     ['get_value'] = function(task)
@@ -419,6 +421,7 @@ local keywords = {
 
       return nil
     end,
+    need_process = true,
   },
   ['files'] = {
     ['get_value'] = function(task)
@@ -438,6 +441,7 @@ local keywords = {
 
       return nil
     end,
+    need_process = true,
   },
 }
 
@@ -445,12 +449,18 @@ local function gen_rate_key(task, rtype, bucket)
   local key_t = {tostring(lua_util.round(100000.0 / bucket.burst))}
   local key_keywords = lua_util.str_split(rtype, '_')
   local have_user = false
+  local message_processed = false
 
   for _, v in ipairs(key_keywords) do
     local ret
-
     if keywords[v] and type(keywords[v]['get_value']) == 'function' then
-      ret = keywords[v]['get_value'](task)
+      if keywords[v].need_process and not message_processed then
+        lua_util.debugm(N, task, 'process message as %s needs mime elts',
+            v)
+        message_processed = task:process_message()
+        message_processed = true
+      end
+      ret = keywords[v].get_value(task)
     end
     if not ret then return nil end
     if v == 'user' then have_user = true end
@@ -495,6 +505,9 @@ local function limit_to_prefixes(task, k, v, prefixes)
   local n = 0
   for _,bucket in ipairs(v.buckets) do
     if v.selector then
+      lua_util.debugm(N, task, 'process message as selectors need mime elts',
+          v)
+      task:process_message()
       local selectors = lua_selectors.process_selectors(task, v.selector)
       if selectors then
         local combined = lua_selectors.combine_selectors(task, selectors, ':')
@@ -855,11 +868,10 @@ if opts then
     lua_util.disable_module(N, "redis")
   else
     local s = {
-      type = 'prefilter,nostat',
+      type = 'prefilter,nostat,empty',
       name = 'RATELIMIT_CHECK',
       priority = 7,
       callback = ratelimit_cb,
-      flags = 'empty',
     }
 
     if settings.symbol then
diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua
index 0f895dc0b..98e46a2e6 100644
--- a/src/plugins/lua/settings.lua
+++ b/src/plugins/lua/settings.lua
@@ -256,6 +256,7 @@ end
 
 -- Check limit for a task
 local function check_settings(task)
+  local message_processed = false
   local function check_specific_setting(rule, matched)
     local res = false
 
@@ -263,6 +264,13 @@ local function check_settings(task)
       local elt = rule.checks[atom]
 
       if elt then
+        if elt.need_process and not message_processed then
+          lua_util.debugm(N, task, 'process message as %s needs mime elts',
+              atom)
+          message_processed = task:process_message()
+          message_processed = true
+        end
+
         local input = elt.extract(task)
         if not input then return false end
 
@@ -616,6 +624,7 @@ local function process_settings_table(tbl, allow_ids, mempool)
           extract = function(task)
             return task:get_from(2)
           end,
+          need_process = true,
         }
       end
     end
@@ -630,6 +639,7 @@ local function process_settings_table(tbl, allow_ids, mempool)
           extract = function(task)
             return task:get_recipients(2)
           end,
+          need_process = true,
         }
       end
     end
@@ -773,6 +783,7 @@ local function process_settings_table(tbl, allow_ids, mempool)
                 return v
               end,
               extract = extractor_func(k),
+              need_process = true,
             }
 
             local skey = process_compound_condition(cond, table_element,
@@ -815,6 +826,7 @@ local function process_settings_table(tbl, allow_ids, mempool)
             end, values)
           end,
           extract = sel,
+          need_process = true,
         }
         local skey = process_compound_condition(cond, 'selector', elt.selector)
         lua_util.debugm(N, rspamd_config, 'added selector condition to "%s": %s',
diff --git a/src/plugins/lua/whitelist.lua b/src/plugins/lua/whitelist.lua
index 11c01134b..0d18a5d87 100644
--- a/src/plugins/lua/whitelist.lua
+++ b/src/plugins/lua/whitelist.lua
@@ -389,9 +389,9 @@ local configure_whitelist_module = function()
           return
         end
 
-        local flags = 'nice,empty'
-        if rule['blacklist'] then
-          flags = 'empty'
+        local flags
+        if not rule['blacklist'] then
+          flags = 'nice'
         end
 
         local id = rspamd_config:register_symbol({