commit 327f406: [Rework] Implement settings expressions
Vsevolod Stakhov
vsevolod at highsecure.ru
Tue Jun 25 17:42:03 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-06-25 17:43:06 +0100
URL: https://github.com/rspamd/rspamd/commit/327f40670c931e39a8643af9539fde9625925cec
[Rework] Implement settings expressions
---
src/plugins/lua/settings.lua | 705 +++++++++++++++++++++++++------------------
1 file changed, 406 insertions(+), 299 deletions(-)
diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua
index de9ac5c65..f269a4018 100644
--- a/src/plugins/lua/settings.lua
+++ b/src/plugins/lua/settings.lua
@@ -31,6 +31,7 @@ local lua_selectors = require "lua_selectors"
local lua_settings = require "lua_settings"
local ucl = require "ucl"
local fun = require "fun"
+local rspamd_mempool = require "rspamd_mempool"
local redis_params
@@ -181,218 +182,98 @@ local function check_query_settings(task)
return false
end
--- Check limit for a task
-local function check_settings(task)
- local function check_addr_setting(rule, addr)
- local function check_specific_addr(elt)
- if rule['name'] then
- if rspamd_maps.rspamd_maybe_check_map(rule['name'], elt['addr']) then
- return true
- end
- end
- if rule['user'] then
- if rspamd_maps.rspamd_maybe_check_map(rule['user'], elt['user']) then
- return true
- end
- end
- if rule['domain'] and elt['domain'] then
- if rspamd_maps.rspamd_maybe_check_map(rule['domain'], elt['domain']) then
- return true
- end
- end
- if rule['regexp'] then
- if rule['regexp']:match(elt['addr']) then
- return true
- end
+local function check_addr_setting(expected, addr)
+ local function check_specific_addr(elt)
+ if expected.name then
+ if rspamd_maps.rspamd_maybe_check_map(expected.name, elt.addr) then
+ return true
end
- return false
end
-
- for _, e in ipairs(addr) do
- if check_specific_addr(e) then
+ if expected.user then
+ if rspamd_maps.rspamd_maybe_check_map(expected.user, elt.user) then
return true
end
end
-
- return false
- end
-
- local function check_ip_setting(rule, ip)
- if not rule[2] then
- if rspamd_maps.rspamd_maybe_check_map(rule[1], ip:to_string()) then
+ if expected.domain and elt.domain then
+ if rspamd_maps.rspamd_maybe_check_map(expected.domain, elt.domain) then
return true
end
- else
- if rule[2] ~= 0 then
- local nip = ip:apply_mask(rule[2])
- if nip and nip:to_string() == rule[1]:to_string() then
- return true
- end
- elseif ip:to_string() == rule[1]:to_string() then
+ end
+ if expected.regexp then
+ if expected.regexp:match(elt.addr) then
return true
end
end
-
return false
end
- local function check_specific_setting(rule_name, rule, data, matched)
- local res = false
-
- local function ip_valid(ip)
- return ip:is_valid()
- end
-
- local function not_empty(s)
- return #s > 0
- end
-
- local function generic_check(value, to_check, check_func, what, valid_func)
- if not to_check then return true end
-
- if type(value) == 'function' then
- value = value()
- end
-
- if value then
- if valid_func then
- if not valid_func(value) then
- return false
- end
- end
-
- if not check_func then
- check_func = function(a, b) return a == b end
- end
-
- local ret = fun.any(function(d)
- return check_func(d, value)
- end, to_check)
- if ret then
- res = true
- matched[#matched + 1] = what
- else
- return false
- end
- else
- return false
- end
-
+ for _, e in ipairs(addr) do
+ if check_specific_addr(e) then
return true
end
+ end
- if not generic_check(data.ip, rule.ip,
- check_ip_setting, 'ip', ip_valid) then
- return nil
- end
-
- if not generic_check(data.client_ip, rule.client_ip,
- check_ip_setting, 'client_ip', ip_valid) then
- return nil
- end
-
- if not generic_check(data.from, rule.from,
- check_addr_setting, 'from') then
- return nil
- end
-
- if not generic_check(data.from_mime, rule.from_mime,
- check_addr_setting, 'from_mime') then
- return nil
- end
+ return false
+end
- if not generic_check(data.rcpt, rule.rcpt,
- check_addr_setting, 'rcpt') then
- return nil
+local function check_string_setting(expected, str)
+ if expected.regexp then
+ if expected.regexp:match(str) then
+ return true
end
-
- if not generic_check(data.rcpt_mime, rule.rcpt_mime,
- check_addr_setting, 'rcpt_mime') then
- return nil
+ elseif expected.check then
+ if rspamd_maps.rspamd_maybe_check_map(expected.check, str) then
+ return true
end
+ end
+ return false
+end
- if not generic_check(data.user, rule.user,
- check_addr_setting, 'user') then
- return nil
+local function check_ip_setting(expected, ip)
+ if not expected[2] then
+ if rspamd_maps.rspamd_maybe_check_map(expected[1], ip:to_string()) then
+ return true
end
-
- if not generic_check(data.hostname, rule.hostname,
- check_addr_setting, 'hostname', not_empty) then
- return nil
+ else
+ if expected[2] ~= 0 then
+ local nip = ip:apply_mask(expected[2])
+ if nip and nip:to_string() == expected[1]:to_string() then
+ return true
+ end
+ elseif ip:to_string() == expected[1]:to_string() then
+ return true
end
+ end
- -- Non generic checks
+ return false
+end
- if rule.authenticated then
- if data.user[1] then
- res = true
- matched[#matched + 1] = 'authenticated'
- end
- if not res then
- return nil
- end
- end
+-- Check limit for a task
+local function check_settings(task)
+ local function check_specific_setting(rule, matched)
+ local res = false
- if rule['local'] then
- if not data.ip or not data.ip:is_valid() then
- return nil
- end
+ local function process_atom(atom)
+ local elt = rule.checks[atom]
- if data.ip:is_local() then
- matched[#matched + 1] = 'local'
- res = true
- else
- return nil
- end
- end
+ if elt then
+ local input = elt.extract(task)
+ if not input then return false end
- if rule.request_header then
- for hname, pattern in pairs(rule.request_header) do
- local hvalue = task:get_request_header(hname)
- res = (hvalue and pattern:match(hvalue))
- if res then
- matched[#matched + 1] = 'req_header: ' .. hname
- break
+ if elt.check(input) then
+ matched[#matched] = atom
+ return 1.0
end
+ else
+ rspamd_logger.errx(task, 'error in settings: check %s is not defined!', atom)
end
- if not res then
- return nil
- end
- end
- if rule.header then
- for _,elt in ipairs(rule.header) do
- for hname,patterns in pairs(elt) do
- for _,pattern in ipairs(patterns) do
- local hvalue = task:get_header(hname)
- res = (hvalue and pattern:match(hvalue))
- if res then
- matched[#matched + 1] = 'header: ' .. hname
- break
- end
- end
- if res then
- break
- end
- end
- if res then
- break
- end
- end
- if not res then
- return nil
- end
+ return 0
end
- if rule.selector then
- res = fun.all(function(s) return s(task) end, rule.selector)
-
- if res then
- matched[#matched + 1] = 'selector'
- end
- end
+ res = rule.expression and rule.expression:process(process_atom)
- if res then
+ if res and res > 0 then
if rule['whitelist'] then
rule['apply'] = {whitelist = true}
end
@@ -413,31 +294,6 @@ local function check_settings(task)
return
end
- lua_util.debugm(N, task, "check for settings")
- local data = {
- ip = task:get_from_ip(),
- client_ip = task:get_client_ip(),
- from = task:get_from(1),
- from_mime = task:get_from(2),
- rcpt = task:get_recipients(1),
- rcpt_mime = task:get_recipients(2),
- hostname = task:get_hostname() or '',
- user = {}
- }
-
- local uname = task:get_user()
- if uname then
- data.user[1] = {}
- local localpart, domainpart = string.gmatch(uname, "(.+)@(.+)")()
- if localpart then
- data.user[1]["user"] = localpart
- data.user[1]["domain"] = domainpart
- data.user[1]["addr"] = uname
- else
- data.user[1]["user"] = uname
- data.user[1]["addr"] = uname
- end
- end
-- Match rules according their order
local applied = false
@@ -445,10 +301,12 @@ local function check_settings(task)
if not applied and settings[pri] then
for _,s in ipairs(settings[pri]) do
local matched = {}
- local result = check_specific_setting(s.name, s.rule, data, matched)
+ lua_util.debugm(N, task, "check for settings element %s; %s",
+ s.name, s.rule.expression)
+ local result = check_specific_setting(s.rule, matched)
-- Can use xor here but more complicated for reading
- if (result and not s.rule.inverse) or (not result and s.rule.inverse) then
+ if result then
if s.rule['apply'] then
if s.rule.id then
-- Extract static settings
@@ -487,7 +345,7 @@ local function check_settings(task)
end
-- Process settings based on their priority
-local function process_settings_table(tbl, allow_ids)
+local function process_settings_table(tbl, allow_ids, mempool)
local get_priority = function(elt)
local pri_tonum = function(p)
if p then
@@ -514,13 +372,13 @@ local function process_settings_table(tbl, allow_ids)
local process_setting_elt = function(name, elt)
lua_util.debugm(N, rspamd_config, 'process settings "%s"', name)
- -- Process IP address
- local function process_ip(ip)
+ -- Process IP address: converted to a table {ip, mask}
+ local function process_ip_condition(ip)
local out = {}
if type(ip) == "table" then
for _,v in ipairs(ip) do
- table.insert(out, process_ip(v))
+ table.insert(out, process_ip_condition(v))
end
elseif type(ip) == "string" then
local slash = string.find(ip, '/')
@@ -555,11 +413,16 @@ local function process_settings_table(tbl, allow_ids)
return out
end
- local function process_addr(addr)
+ -- Process email like condition, converted to a table with fields:
+ -- name - full email (surprise!)
+ -- user - user part
+ -- domain - domain part
+ -- regexp - full email regexp (yes, it sucks)
+ local function process_email_condition(addr)
local out = {}
if type(addr) == "table" then
for _,v in ipairs(addr) do
- table.insert(out, process_addr(v))
+ table.insert(out, process_email_condition(v))
end
elseif type(addr) == "string" then
if string.sub(addr, 1, 4) == "map:" then
@@ -599,7 +462,43 @@ local function process_settings_table(tbl, allow_ids)
return out
end
- local check_table = function(chk_elt, out)
+ -- Convert a plain string condition to a table:
+ -- check - string to match
+ -- regexp - regexp to match
+ local function process_string_condition(addr)
+ local out = {}
+ if type(addr) == "table" then
+ for _,v in ipairs(addr) do
+ table.insert(out, process_string_condition(v))
+ end
+ elseif type(addr) == "string" then
+ if string.sub(addr, 1, 4) == "map:" then
+ -- It is map, don't apply any extra logic
+ out['check'] = addr
+ else
+ local start = string.sub(addr, 1, 1)
+ if start == '/' then
+ -- It is a regexp
+ local re = rspamd_regexp.create(addr)
+ if re then
+ out['regexp'] = re
+ else
+ rspamd_logger.errx(rspamd_config, "bad regexp: " .. addr)
+ return nil
+ end
+
+ else
+ out['check'] = addr
+ end
+ end
+ else
+ return nil
+ end
+
+ return out
+ end
+
+ local convert_to_table = function(chk_elt, out)
if type(chk_elt) == 'string' then
return {out}
end
@@ -607,135 +506,262 @@ local function process_settings_table(tbl, allow_ids)
return out
end
+ -- Used to create a checking closure: if value matches expected somehow, return true
+ local function gen_check_closure(expected, check_func)
+ return function(value)
+ if not value then return false end
+
+ if type(value) == 'function' then
+ value = value()
+ end
+
+ if value then
+
+ if not check_func then
+ check_func = function(a, b) return a == b end
+ end
+
+ local ret = fun.any(function(d)
+ return check_func(d, value)
+ end, expected)
+ if ret then
+ return true
+ end
+ end
+
+ return false
+ end
+ end
+
local out = {}
+ local checks = {}
if elt['ip'] then
- local ip = process_ip(elt['ip'])
+ local ips_table = process_ip_condition(elt['ip'])
- if ip then
+ if ips_table then
lua_util.debugm(N, rspamd_config, 'added ip condition to "%s": %s',
- name, ip)
- out['ip'] = check_table(elt['ip'], ip)
+ name, ips_table)
+ checks.ip = {
+ check = gen_check_closure(convert_to_table(elt.ip, ips_table), check_ip_setting),
+ extract = function(task)
+ local ip = task:get_from_ip()
+ if ip:is_valid() then return ip end
+ return nil
+ end,
+ }
end
end
if elt['client_ip'] then
- local ip = process_ip(elt['client_ip'])
+ local client_ips_table = process_ip_condition(elt['client_ip'])
- if ip then
+ if client_ips_table then
lua_util.debugm(N, rspamd_config, 'added client_ip condition to "%s": %s',
- name, ip)
- out['client_ip'] = check_table(elt['client_ip'], ip)
+ name, client_ips_table)
+ checks.client_ip = {
+ check = gen_check_closure(convert_to_table(elt.client_ip, client_ips_table),
+ check_ip_setting),
+ extract = function(task)
+ local ip = task:get_client_ip()
+ if ip:is_valid() then return ip end
+ return nil
+ end,
+ }
end
end
if elt['from'] then
- local from = process_addr(elt['from'])
+ local from_condition = process_email_condition(elt['from'])
- if from then
+ if from_condition then
lua_util.debugm(N, rspamd_config, 'added from condition to "%s": %s',
- name, from)
- out['from'] = check_table(elt['from'], from)
+ name, from_condition)
+ checks.from = {
+ check = gen_check_closure(convert_to_table(elt.from, from_condition),
+ check_addr_setting),
+ extract = function(task)
+ return task:get_from(1)
+ end,
+ }
end
end
if elt['rcpt'] then
- local rcpt = process_addr(elt['rcpt'])
- if rcpt then
+ local rcpt_condition = process_email_condition(elt['rcpt'])
+ if rcpt_condition then
lua_util.debugm(N, rspamd_config, 'added rcpt condition to "%s": %s',
- name, rcpt)
- out['rcpt'] = check_table(elt['rcpt'], rcpt)
+ name, rcpt_condition)
+ checks.rcpt = {
+ check = gen_check_closure(convert_to_table(elt.rcpt, rcpt_condition),
+ check_addr_setting),
+ extract = function(task)
+ return task:get_recipients(1)
+ end,
+ }
end
end
if elt['from_mime'] then
- local from_mime = process_addr(elt['from_mime'])
+ local from_mime_condition = process_email_condition(elt['from_mime'])
- if from_mime then
+ if from_mime_condition then
lua_util.debugm(N, rspamd_config, 'added from_mime condition to "%s": %s',
- name, from_mime)
- out['from_mime'] = check_table(elt['from_mime'], from_mime)
+ name, from_mime_condition)
+ checks.from_mime = {
+ check = gen_check_closure(convert_to_table(elt.from_mime, from_mime_condition),
+ check_addr_setting),
+ extract = function(task)
+ return task:get_from(2)
+ end,
+ }
end
end
if elt['rcpt_mime'] then
- local rcpt_mime = process_addr(elt['rcpt_mime'])
- if rcpt_mime then
- lua_util.debugm(N, rspamd_config, 'added rcpt_mime condition to "%s": %s',
- name, rcpt_mime)
- out['rcpt_mime'] = check_table(elt['rcpt_mime'], rcpt_mime)
+ local rcpt_mime_condition = process_email_condition(elt['rcpt'])
+ if rcpt_mime_condition then
+ lua_util.debugm(N, rspamd_config, 'added rcpt mime condition to "%s": %s',
+ name, rcpt_mime_condition)
+ checks.rcpt_mime = {
+ check = gen_check_closure(convert_to_table(elt.rcpt_mime, rcpt_mime_condition),
+ check_addr_setting),
+ extract = function(task)
+ return task:get_recipients(2)
+ end,
+ }
end
end
if elt['user'] then
- local user = process_addr(elt['user'])
- if user then
+ local user_condition = process_email_condition(elt['user'])
+ if user_condition then
lua_util.debugm(N, rspamd_config, 'added user condition to "%s": %s',
- name, user)
- out['user'] = check_table(elt['user'], user)
+ name, user_condition)
+ checks.user = {
+ check = gen_check_closure(convert_to_table(elt.user, user_condition),
+ check_addr_setting),
+ extract = function(task)
+ local uname = task:get_user()
+ local user = {}
+ if uname then
+ user[1] = {}
+ local localpart, domainpart = string.gmatch(uname, "(.+)@(.+)")()
+ if localpart then
+ user[1]["user"] = localpart
+ user[1]["domain"] = domainpart
+ user[1]["addr"] = uname
+ else
+ user[1]["user"] = uname
+ user[1]["addr"] = uname
+ end
+
+ return user
+ end
+
+ return nil
+ end,
+ }
end
end
if elt['hostname'] then
- local hostname = process_addr(elt['hostname'])
- if hostname then
+ local hostname_condition = process_string_condition(elt['hostname'])
+ if hostname_condition then
lua_util.debugm(N, rspamd_config, 'added hostname condition to "%s": %s',
- name, hostname)
- out['hostname'] = check_table(elt['hostname'], hostname)
+ name, hostname_condition)
+ checks.hostname = {
+ check = gen_check_closure(convert_to_table(elt.hostname, hostname_condition),
+ check_string_setting),
+ extract = function(task)
+ return task:get_hostname() or ''
+ end,
+ }
end
end
if elt['authenticated'] then
lua_util.debugm(N, rspamd_config, 'added authenticated condition to "%s"',
name)
- out['authenticated'] = true
+ checks.authenticated = {
+ check = function(value) if value then return true end return false end,
+ extract = function(task)
+ return task:get_user()
+ end
+ }
end
if elt['local'] then
- out['local'] = true
lua_util.debugm(N, rspamd_config, 'added local condition to "%s"',
name)
- end
- if elt['inverse'] then
- lua_util.debugm(N, rspamd_config, 'added inverse condition to "%s"',
- name)
- out['inverse'] = true
- end
- if elt['request_header'] then
- local rho = {}
- for k, v in pairs(elt['request_header']) do
- local re = rspamd_regexp.create(v)
- if re then
- rho[k] = re
+ checks['local'] = {
+ check = function(value) if value then return true end return false end,
+ extract = function(task)
+ local ip = task:get_from_ip()
+ if not ip or not ip:is_valid() then
+ return nil
+ end
+
+ if ip:is_local() then
+ return true
+ else
+ return nil
+ end
end
- end
- lua_util.debugm(N, rspamd_config, 'added request_header condition to "%s": %s',
- name, rho)
- out['request_header'] = rho
+ }
end
- if elt['header'] then
- if not elt['header'][1] and next(elt['header']) then
- elt['header'] = {elt['header']}
- end
- for _, e in ipairs(elt['header']) do
- local rho = {}
- for k, v in pairs(e) do
- if type(v) ~= 'table' then
- v = {v}
- end
- for _, r in ipairs(v) do
- local re = rspamd_regexp.get_cached(r)
- if not re then
*** OUTPUT TRUNCATED, 211 LINES SKIPPED ***
More information about the Commits
mailing list