commit ce21f8f: [Rework] Phishing: Split from redirectors usage

[Vsevolod Stakhov]

Author: Vsevolod Stakhov
Date: 2021-02-25 19:17:47 +0000
URL: https://github.com/rspamd/rspamd/commit/ce21f8fe16a45a5a409d3c9f8217eba2de1fbe14 (HEAD -> master)

[Rework] Phishing: Split from redirectors usage

---
 conf/modules.d/phishing.conf      | 19 +++++++++++++------
 conf/scores.d/phishing_group.conf |  4 ++++
 src/plugins/lua/phishing.lua      | 30 +++++++-----------------------
 3 files changed, 24 insertions(+), 29 deletions(-)

diff --git a/conf/modules.d/phishing.conf b/conf/modules.d/phishing.conf
index d828f9fa5..b82854297 100644
--- a/conf/modules.d/phishing.conf
+++ b/conf/modules.d/phishing.conf
@@ -22,12 +22,19 @@ phishing {
   phishtank_enabled = true;
 
   # Make exclusions for known redirectors
-  redirector_domains = [
-    "https://maps.rspamd.com/rspamd/redirectors.inc.zst:REDIRECTOR_FALSE",
-    "$LOCAL_CONFDIR/local.d/maps.d/redirectors.inc:LOCAL_REDIRECTOR_FALSE",
-    "$LOCAL_CONFDIR/local.d/redirectors.inc:LOCAL_REDIRECTOR_FALSE",
-    "fallback+file://${CONFDIR}/maps.d/redirectors.inc:REDIRECTOR_FALSE"
-  ];
+  exceptions = {
+    REDIRECTOR_FALSE = [
+      "https://maps.rspamd.com/rspamd/redirectors.inc.zst",
+      "$LOCAL_CONFDIR/local.d/maps.d/redirectors.inc",
+      "$LOCAL_CONFDIR/local.d/redirectors.inc",
+      "fallback+file://${CONFDIR}/maps.d/redirectors.inc"
+    ];
+    PHISHED_WHITELISTED = [
+      "glob;https://maps.rspamd.com/rspamd/phishing_whitelist.inc.zst",
+      "glob;$LOCAL_CONFDIR/local.d/maps.d/phishing_whitelist.inc",
+      "glob;$LOCAL_CONFDIR/local.d/phishing_whitelist.inc",
+    ];
+  };
 
   .include(try=true,priority=5) "${DBDIR}/dynamic/phishing.conf"
   .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/phishing.conf"
diff --git a/conf/scores.d/phishing_group.conf b/conf/scores.d/phishing_group.conf
index c7368ef72..9e5c4a325 100644
--- a/conf/scores.d/phishing_group.conf
+++ b/conf/scores.d/phishing_group.conf
@@ -41,4 +41,8 @@ symbols = {
         weight = 0.0;
         description = "Phishing exclusion symbol for known redirectors";
     }
+    PHISHED_WHITELISTED {
+        weight = 0.0;
+        description = "Phishing exclusion symbol for known exceptions";
+    }
 }
\ No newline at end of file
diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua
index 89d69cb5e..289aebac2 100644
--- a/src/plugins/lua/phishing.lua
+++ b/src/plugins/lua/phishing.lua
@@ -33,7 +33,8 @@ local phishtank_symbol = 'PHISHED_PHISHTANK'
 local generic_service_name = 'generic service'
 local domains = nil
 local strict_domains = {}
-local redirector_domains = {}
+local exceptions_maps = {}
+local exclude_domains = {}
 local generic_service_map = nil
 local openphish_map = 'https://www.openphish.com/feed.txt'
 local phishtank_suffix = 'phishtank.rspamd.com'
@@ -268,7 +269,7 @@ local function phishing_cb(task)
           end
         end
 
-        if not found_in_map(redirector_domains) then
+        if not found_in_map(exceptions_maps) then
           if not found_in_map(strict_domains, purl, 1.0) then
             if domains then
               if domains:get_key(ptld) then
@@ -286,32 +287,15 @@ end
 
 local function phishing_map(mapname, phishmap, id)
   if opts[mapname] then
-    local xd = {}
+    local xd
     if type(opts[mapname]) == 'table' then
       xd = opts[mapname]
     else
-      xd[1] = opts[mapname]
+      rspamd_logger.errx(rspamd_config, 'invalid exception table')
     end
 
-    local found_maps = {}
 
-    for _,d in ipairs(xd) do
-      local s = string.find(d, ':[^:]+$')
-      if s then
-        local sym = string.sub(d, s + 1, -1)
-        local map = string.sub(d, 1, s - 1)
-
-        if found_maps[sym] then
-          table.insert(found_maps[sym], map)
-        else
-          found_maps[sym] = {map}
-        end
-      else
-        rspamd_logger.infox(rspamd_config, mapname .. ' option must be in format <map>:<symbol>')
-      end
-    end
-
-    for sym,urls in pairs(found_maps) do
+    for sym,urls in pairs(xd) do
       local rmap = rspamd_config:add_map ({
         type = 'set',
         url = urls,
@@ -539,5 +523,5 @@ if opts then
     })
   end
   phishing_map('strict_domains', strict_domains, id)
-  phishing_map('redirector_domains', redirector_domains, id)
+  phishing_map('exceptions', exceptions_maps, id)
 end