commit 1d577bf: [Project] More fixes to follow the C module semantics

[Vsevolod Stakhov]

Author: Vsevolod Stakhov
Date: 2019-12-02 17:02:08 +0000
URL: https://github.com/rspamd/rspamd/commit/1d577bf39b986cdee3dcaaced23c38ffadf62d5e

[Project] More fixes to follow the C module semantics

---
 src/plugins/lua/dmarc.lua | 29 ++++++----------------------
 src/plugins/lua/spf.lua   | 48 ++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 49 insertions(+), 28 deletions(-)

diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua
index 08d9732d9..e6a520e8e 100644
--- a/src/plugins/lua/dmarc.lua
+++ b/src/plugins/lua/dmarc.lua
@@ -23,8 +23,7 @@ local rspamd_url = require "rspamd_url"
 local rspamd_util = require "rspamd_util"
 local rspamd_redis = require "lua_redis"
 local lua_util = require "lua_util"
-local check_local = false
-local check_authed = false
+local auth_and_local_conf
 
 if confighelp then
   return
@@ -567,8 +566,7 @@ local function dmarc_callback(task)
     return
   end
 
-  if ((not check_authed and task:get_user()) or
-      (not check_local and ip_addr and ip_addr:is_local())) then
+  if lua_util.is_skip_local_or_authed(task, auth_and_local_conf, ip_addr) then
     rspamd_logger.infox(task, "skip DMARC checks for local networks and authorized users")
     return
   end
@@ -709,29 +707,14 @@ local function dmarc_callback(task)
 end
 
 
-local function try_opts(where)
-  local ret = false
-  local opts = rspamd_config:get_all_opt(where)
-  if type(opts) == 'table' then
-    if type(opts['check_local']) == 'boolean' then
-      check_local = opts['check_local']
-      ret = true
-    end
-    if type(opts['check_authed']) == 'boolean' then
-      check_authed = opts['check_authed']
-      ret = true
-    end
-  end
-
-  return ret
-end
-
-if not try_opts(N) then try_opts('options') end
-
 local opts = rspamd_config:get_all_opt('dmarc')
 if not opts or type(opts) ~= 'table' then
   return
 end
+
+auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N,
+    false, false)
+
 no_sampling_domains = rspamd_map_add(N, 'no_sampling_domains', 'map', 'Domains not to apply DMARC sampling to')
 no_reporting_domains = rspamd_map_add(N, 'no_reporting_domains', 'map', 'Domains not to apply DMARC reporting to')
 
diff --git a/src/plugins/lua/spf.lua b/src/plugins/lua/spf.lua
index 62f147f91..e48c8e9ce 100644
--- a/src/plugins/lua/spf.lua
+++ b/src/plugins/lua/spf.lua
@@ -18,6 +18,7 @@ local N = "spf"
 local lua_util = require "lua_util"
 local rspamd_spf = require "rspamd_spf"
 local bit = require "bit"
+local rspamd_logger = require "rspamd_logger"
 
 if confighelp then
   rspamd_config:add_example(nil, N,
@@ -66,6 +67,8 @@ local default_config = {
 }
 
 local local_config = rspamd_config:get_all_opt('spf')
+local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, N,
+    false, false)
 
 if local_config then
   local_config = lua_util.override_defaults(default_config, local_config)
@@ -74,6 +77,9 @@ else
 end
 
 local function spf_check_callback(task)
+
+  local ip = task:get_from_ip()
+
   local function flag_to_symbol(fl)
     if bit.band(fl, rspamd_spf.flags.temp_fail) ~= 0 then
       return local_config.symbols.dnsfail
@@ -103,8 +109,8 @@ local function spf_check_callback(task)
   local function spf_resolved_cb(record, flags, err)
     lua_util.debugm(N, task, 'got spf results: %s flags, %s err',
         flags, err)
+
     if record then
-      local ip = task:get_from_ip()
       local result, flag_or_policy, error_or_addr = record:check_ip(ip)
 
       lua_util.debugm(N, task,
@@ -116,6 +122,12 @@ local function spf_check_callback(task)
         local opt = string.format('%s%s', code, error_or_addr.str or '???')
         if bit.band(flags, rspamd_spf.flags.cached) ~= 0 then
           opt = opt .. ':c'
+          rspamd_logger.infox(task,
+              "use cached record for %s (0x%s) in LRU cache for %s seconds",
+              record:get_domain(),
+              record:get_digest(),
+              record:get_ttl() - math.floor(task:get_timeval(true) -
+                  record:get_timestamp()));
         end
         task:insert_result(sym, 1.0, opt)
       else
@@ -126,13 +138,32 @@ local function spf_check_callback(task)
       local sym = flag_to_symbol(flags)
       task:insert_result(sym, 1.0, err)
     end
+  end
+
+  if ip then
+    if local_config.whitelist and ip and local_config.whitelist:get_key(ip) then
+      rspamd_logger.infox(task, 'whitelisted SPF checks from %s',
+          tostring(ip))
+      return
+    end
 
-    local dmarc_checks = task:get_mempool():get_variable('dmarc_checks', 'double') or 0
-    dmarc_checks = dmarc_checks + 1
-    task:get_mempool():set_variable('dmarc_checks', dmarc_checks)
+    if lua_util.is_skip_local_or_authed(task, auth_and_local_conf, ip) then
+      rspamd_logger.infox(task, 'skip SPF checks for local networks and authorized users')
+      return
+    end
+
+    rspamd_spf.resolve(task, spf_resolved_cb)
+  else
+    lua_util.debugm(N, task, "spf checks are not possible as no source IP address is defined")
   end
 
-  rspamd_spf.resolve(task, spf_resolved_cb)
+  -- FIXME: we actually need to set this variable when we really checked SPF
+  -- However, the old C module has set it all the times
+  -- Hence, we follow the same rule for now. It should be better designed at some day
+  local mpool = task:get_mempool()
+  local dmarc_checks = mpool:get_variable('dmarc_checks', 'double') or 0
+  dmarc_checks = dmarc_checks + 1
+  mpool:set_variable('dmarc_checks', dmarc_checks)
 end
 
 -- Register all symbols and init rspamd_spf library
@@ -146,6 +177,13 @@ local sym_id = rspamd_config:register_symbol{
   callback = spf_check_callback
 }
 
+if local_config.whitelist then
+  local lua_maps = require "lua_maps"
+
+  local_config.whitelist = lua_maps.map_add_from_ucl(local_config.whitelist,
+      "radix", "SPF whitelist map")
+end
+
 for _,sym in pairs(local_config.symbols) do
   rspamd_config:register_symbol{
     name = sym,