commit 8e65fac: [Rework] Move common and rarely used dmarc code to the library
Vsevolod Stakhov
vsevolod at highsecure.ru
Mon Aug 2 11:14:04 UTC 2021
Author: Vsevolod Stakhov
Date: 2021-08-02 12:09:10 +0100
URL: https://github.com/rspamd/rspamd/commit/8e65fac07f092bc0450bcaa611fccde3dd890fa6 (HEAD -> master)
[Rework] Move common and rarely used dmarc code to the library
---
lualib/plugins/dmarc.lua | 233 ++++++++++++++++++++++++++++++++++++++++++++++
src/plugins/lua/dmarc.lua | 225 +++-----------------------------------------
2 files changed, 246 insertions(+), 212 deletions(-)
diff --git a/lualib/plugins/dmarc.lua b/lualib/plugins/dmarc.lua
new file mode 100644
index 000000000..9477f3c0d
--- /dev/null
+++ b/lualib/plugins/dmarc.lua
@@ -0,0 +1,233 @@
+--[[
+Copyright (c) 2011-2016, Vsevolod Stakhov <vsevolod at highsecure.ru>
+Copyright (c) 2015-2016, Andrew Lewis <nerf at judo.za.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Common dmarc stuff
+local rspamd_logger = require "rspamd_logger"
+local N = "dmarc"
+
+local exports = {}
+
+exports.default_settings = {
+ auth_and_local_conf = false,
+ symbols = {
+ spf_allow_symbol = 'R_SPF_ALLOW',
+ spf_deny_symbol = 'R_SPF_FAIL',
+ spf_softfail_symbol = 'R_SPF_SOFTFAIL',
+ spf_neutral_symbol = 'R_SPF_NEUTRAL',
+ spf_tempfail_symbol = 'R_SPF_DNSFAIL',
+ spf_permfail_symbol = 'R_SPF_PERMFAIL',
+ spf_na_symbol = 'R_SPF_NA',
+
+ dkim_allow_symbol = 'R_DKIM_ALLOW',
+ dkim_deny_symbol = 'R_DKIM_REJECT',
+ dkim_tempfail_symbol = 'R_DKIM_TEMPFAIL',
+ dkim_na_symbol = 'R_DKIM_NA',
+ dkim_permfail_symbol = 'R_DKIM_PERMFAIL',
+
+ -- DMARC symbols
+ allow = 'DMARC_POLICY_ALLOW',
+ badpolicy = 'DMARC_BAD_POLICY',
+ dnsfail = 'DMARC_DNSFAIL',
+ na = 'DMARC_NA',
+ reject = 'DMARC_POLICY_REJECT',
+ softfail = 'DMARC_POLICY_SOFTFAIL',
+ quarantine = 'DMARC_POLICY_QUARANTINE',
+ },
+ no_sampling_domains = nil,
+ no_reporting_domains = nil,
+ reporting = {
+ report_local_controller = false, -- Store reports for local/controller scans (for testing only)
+ redis_keys = {
+ index_prefix = 'dmarc_idx',
+ report_prefix = 'dmarc_rpt',
+ join_char = ';',
+ },
+ enabled = false,
+ max_entries = 1000,
+ keys_expire = 172800,
+ only_domains = nil,
+ },
+ actions = {},
+}
+
+
+-- Returns a key used to be inserted into dmarc report sample
+exports.dmarc_report = function (task, settings, data)
+ local rspamd_lua_utils = require "lua_util"
+ local E = {}
+
+ local ip = task:get_from_ip()
+ if ip and not ip:is_valid() then
+ rspamd_logger.infox(task, 'cannot store dmarc report for %s: no valid source IP',
+ data.domain)
+ return nil
+ end
+
+ ip = ip:to_string()
+
+ if rspamd_lua_utils.is_rspamc_or_controller(task) and not settings.reporting.report_local_controller then
+ rspamd_logger.infox(task, 'cannot store dmarc report for %s from IP %s: has come from controller/rspamc',
+ data.domain, ip)
+ return
+ end
+
+ local dkim_pass = table.concat(data.dkim_results.pass or E, '|')
+ local dkim_fail = table.concat(data.dkim_results.fail or E, '|')
+ local dkim_temperror = table.concat(data.dkim_results.temperror or E, '|')
+ local dkim_permerror = table.concat(data.dkim_results.permerror or E, '|')
+ local disposition_to_return = data.disposition
+ local res = table.concat({
+ ip, data.spf_ok, data.dkim_ok,
+ disposition_to_return, (data.sampled_out and 'sampled_out' or ''), data.domain,
+ dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, data.spf_domain, data.spf_result}, ',')
+
+ return res
+end
+
+
+exports.gen_munging_callback = function(munging_opts, settings)
+ local lua_util = require "lua_util"
+ local rspamd_util = require "rspamd_util"
+ local lua_mime = require "lua_mime"
+ return function (task)
+ if munging_opts.mitigate_allow_only then
+ if not task:has_symbol(settings.symbols.allow) then
+ lua_util.debugm(N, task, 'skip munging, no %s symbol',
+ settings.symbols.allow)
+ -- Excepted
+ return
+ end
+ else
+ local has_dmarc = task:has_symbol(settings.symbols.allow) or
+ task:has_symbol(settings.symbols.quarantine) or
+ task:has_symbol(settings.symbols.reject) or
+ task:has_symbol(settings.symbols.softfail)
+
+ if not has_dmarc then
+ lua_util.debugm(N, task, 'skip munging, no %s symbol',
+ settings.symbols.allow)
+ -- Excepted
+ return
+ end
+ end
+ if munging_opts.mitigate_strict_only then
+ local s = task:get_symbol(settings.symbols.allow) or {[1] = {}}
+ local sopts = s[1].options or {}
+
+ local seen_strict
+ for _,o in ipairs(sopts) do
+ if o == 'reject' or o == 'quarantine' then
+ seen_strict = true
+ break
+ end
+ end
+
+ if not seen_strict then
+ lua_util.debugm(N, task, 'skip munging, no strict policy found in %s',
+ settings.symbols.allow)
+ -- Excepted
+ return
+ end
+ end
+ if munging_opts.munge_map_condition then
+ local accepted,trace = munging_opts.munge_map_condition:process(task)
+ if not accepted then
+ lua_util.debugm(task, 'skip munging, maps condition not satisified: (%s)',
+ trace)
+ -- Excepted
+ return
+ end
+ end
+ -- Now, look for domain for munging
+ local mr = task:get_recipients({ 'mime', 'orig'})
+ local rcpt_found
+ if mr then
+ for _,r in ipairs(mr) do
+ if r.domain and munging_opts.list_map:get_key(r.addr) then
+ rcpt_found = r
+ break
+ end
+ end
+ end
+
+ if not rcpt_found then
+ lua_util.debugm(task, 'skip munging, recipients are not in list_map')
+ -- Excepted
+ return
+ end
+
+ local from = task:get_from({ 'mime', 'orig'})
+
+ if not from or not from[1] then
+ lua_util.debugm(task, 'skip munging, from is bad')
+ -- Excepted
+ return
+ end
+
+ from = from[1]
+ local via_user = rcpt_found.user
+ local via_addr = rcpt_found.addr
+ local via_name
+
+ if from.name then
+ via_name = string.format('%s via %s', from.name, via_user)
+ else
+ via_name = string.format('%s via %s', from.user or 'unknown', via_user)
+ end
+
+ local hdr_encoded = rspamd_util.fold_header('From',
+ rspamd_util.mime_header_encode(string.format('%s <%s>',
+ via_name, via_addr)))
+ local orig_from_encoded = rspamd_util.fold_header('X-Original-From',
+ rspamd_util.mime_header_encode(string.format('%s <%s>',
+ from.name or '', from.addr)))
+ local add_hdrs = {
+ ['From'] = { order = 1, value = hdr_encoded },
+ }
+ local remove_hdrs = {['From'] = 0}
+
+ local nreply = from.addr
+ if munging_opts.reply_goes_to_list then
+ -- Reply-to goes to the list
+ nreply = via_addr
+ end
+
+ if task:has_header('Reply-To') then
+ -- If we have reply-to header, then we need to insert an additional
+ -- address there
+ local orig_reply = task:get_header_full('Reply-To')[1]
+ if orig_reply.value then
+ nreply = string.format('%s, %s', orig_reply.value, nreply)
+ end
+ remove_hdrs['Reply-To'] = 1
+ end
+
+ add_hdrs['Reply-To'] = {order = 0, value = nreply}
+
+ add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded}
+ lua_mime.modify_headers(task, {
+ remove = remove_hdrs,
+ add = add_hdrs
+ })
+ lua_util.debugm(N, task, 'munged DMARC header for %s: %s -> %s',
+ from.domain, hdr_encoded, from.addr)
+ rspamd_logger.infox(task, 'munged DMARC header for %s', from.addr)
+ task:insert_result('DMARC_MUNGED', 1.0, from.addr)
+ end
+end
+
+return exports
\ No newline at end of file
diff --git a/src/plugins/lua/dmarc.lua b/src/plugins/lua/dmarc.lua
index 417bd89eb..0209dedb8 100644
--- a/src/plugins/lua/dmarc.lua
+++ b/src/plugins/lua/dmarc.lua
@@ -21,6 +21,7 @@ local rspamd_logger = require "rspamd_logger"
local rspamd_util = require "rspamd_util"
local lua_redis = require "lua_redis"
local lua_util = require "lua_util"
+local dmarc_common = require "plugins/dmarc"
if confighelp then
return
@@ -28,48 +29,7 @@ end
local N = 'dmarc'
-local settings = {
- auth_and_local_conf = false,
- symbols = {
- spf_allow_symbol = 'R_SPF_ALLOW',
- spf_deny_symbol = 'R_SPF_FAIL',
- spf_softfail_symbol = 'R_SPF_SOFTFAIL',
- spf_neutral_symbol = 'R_SPF_NEUTRAL',
- spf_tempfail_symbol = 'R_SPF_DNSFAIL',
- spf_permfail_symbol = 'R_SPF_PERMFAIL',
- spf_na_symbol = 'R_SPF_NA',
-
- dkim_allow_symbol = 'R_DKIM_ALLOW',
- dkim_deny_symbol = 'R_DKIM_REJECT',
- dkim_tempfail_symbol = 'R_DKIM_TEMPFAIL',
- dkim_na_symbol = 'R_DKIM_NA',
- dkim_permfail_symbol = 'R_DKIM_PERMFAIL',
-
- -- DMARC symbols
- allow = 'DMARC_POLICY_ALLOW',
- badpolicy = 'DMARC_BAD_POLICY',
- dnsfail = 'DMARC_DNSFAIL',
- na = 'DMARC_NA',
- reject = 'DMARC_POLICY_REJECT',
- softfail = 'DMARC_POLICY_SOFTFAIL',
- quarantine = 'DMARC_POLICY_QUARANTINE',
- },
- no_sampling_domains = nil,
- no_reporting_domains = nil,
- reporting = {
- report_local_controller = false, -- Store reports for local/controller scans (for testing only)
- redis_keys = {
- index_prefix = 'dmarc_idx',
- report_prefix = 'dmarc_rpt',
- join_char = ';',
- },
- enabled = false,
- max_entries = 1000,
- keys_expire = 172800,
- only_domains = nil,
- },
- actions = {},
-}
+local settings = dmarc_common.default_settings
local redis_params = nil
@@ -132,40 +92,6 @@ local function dmarc_key_value_case(elts)
return result
end
-
--- Returns a key used to be inserted into dmarc report sample
-local function dmarc_report(task, spf_ok, dkim_ok, disposition,
- sampled_out, hfromdom, spfdom, dres, spf_result)
- local rspamd_lua_utils = require "lua_util"
-
- local ip = task:get_from_ip()
- if ip and not ip:is_valid() then
- rspamd_logger.infox(task, 'cannot store dmarc report for %s: no valid source IP',
- hfromdom)
- return nil
- end
-
- ip = ip:to_string()
-
- if rspamd_lua_utils.is_rspamc_or_controller(task) and not settings.reporting.report_local_controller then
- rspamd_logger.infox(task, 'cannot store dmarc report for %s from IP %s: has come from controller/rspamc',
- hfromdom, ip)
- return
- end
-
- local dkim_pass = table.concat(dres.pass or E, '|')
- local dkim_fail = table.concat(dres.fail or E, '|')
- local dkim_temperror = table.concat(dres.temperror or E, '|')
- local dkim_permerror = table.concat(dres.permerror or E, '|')
- local disposition_to_return = (disposition == "softfail") and "none" or disposition
- local res = table.concat({
- ip, spf_ok, dkim_ok,
- disposition_to_return, (sampled_out and 'sampled_out' or ''), hfromdom,
- dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, spfdom, spf_result}, ',')
-
- return res
-end
-
local function maybe_force_action(task, disposition)
if disposition then
local force_action = settings.actions[disposition]
@@ -503,15 +429,16 @@ local function dmarc_validate_policy(task, policy, hdrfromdom, dmarc_esld)
local dmarc_domain_key = table.concat(
{settings.reporting.redis_keys.report_prefix, hdrfromdom, policy.rua, period},
settings.reporting.redis_keys.join_char)
- local report_data = dmarc_report(task,
- spf_ok and 'pass' or 'fail',
- dkim_ok and 'pass' or 'fail',
- disposition,
- sampled_out,
- hdrfromdom,
- spf_domain,
- dkim_results,
- spf_result)
+ local report_data = dmarc_common.dmarc_report(task, settings, {
+ spf_ok = spf_ok and 'pass' or 'fail',
+ dkim_ok = dkim_ok and 'pass' or 'fail',
+ disposition = (disposition == "softfail") and "none" or disposition,
+ sampled_out = sampled_out,
+ domain = hdrfromdom,
+ spf_domain = spf_domain,
+ dkim_results = dkim_results,
+ spf_result = spf_result
+ })
local idx_key = table.concat({settings.reporting.redis_keys.index_prefix, period},
@@ -806,7 +733,6 @@ rspamd_config:register_dependency('DMARC_CHECK', settings.symbols['dkim_allow_sy
-- DMARC munging support
if settings.munging then
local lua_maps_expressions = require "lua_maps_expressions"
- local lua_mime = require "lua_mime"
local munging_defaults = {
reply_goes_to_list = false,
@@ -839,131 +765,6 @@ if settings.munging then
munging_opts.munge_map_condition, N)
end
- local function dmarc_munge_callback(task)
- if munging_opts.mitigate_allow_only then
- if not task:has_symbol(settings.symbols.allow) then
- lua_util.debugm(N, task, 'skip munging, no %s symbol',
- settings.symbols.allow)
- -- Excepted
- return
- end
- else
- local has_dmarc = task:has_symbol(settings.symbols.allow) or
- task:has_symbol(settings.symbols.quarantine) or
- task:has_symbol(settings.symbols.reject) or
- task:has_symbol(settings.symbols.softfail)
-
- if not has_dmarc then
- lua_util.debugm(N, task, 'skip munging, no %s symbol',
- settings.symbols.allow)
- -- Excepted
- return
- end
- end
- if munging_opts.mitigate_strict_only then
- local s = task:get_symbol(settings.symbols.allow) or {[1] = {}}
- local sopts = s[1].options or {}
-
- local seen_strict
- for _,o in ipairs(sopts) do
- if o == 'reject' or o == 'quarantine' then
- seen_strict = true
- break
- end
- end
-
- if not seen_strict then
- lua_util.debugm(N, task, 'skip munging, no strict policy found in %s',
- settings.symbols.allow)
- -- Excepted
- return
- end
- end
- if munging_opts.munge_map_condition then
- local accepted,trace = munging_opts.munge_map_condition:process(task)
- if not accepted then
- lua_util.debugm(task, 'skip munging, maps condition not satisified: (%s)',
- trace)
- -- Excepted
- return
- end
- end
- -- Now, look for domain for munging
- local mr = task:get_recipients({ 'mime', 'orig'})
- local rcpt_found
- if mr then
- for _,r in ipairs(mr) do
- if r.domain and munging_opts.list_map:get_key(r.addr) then
- rcpt_found = r
- break
- end
- end
- end
-
- if not rcpt_found then
- lua_util.debugm(task, 'skip munging, recipients are not in list_map')
- -- Excepted
- return
- end
-
- local from = task:get_from({ 'mime', 'orig'})
-
- if not from or not from[1] then
- lua_util.debugm(task, 'skip munging, from is bad')
- -- Excepted
- return
- end
-
- from = from[1]
- local via_user = rcpt_found.user
- local via_addr = rcpt_found.addr
- local via_name
-
- if from.name then
- via_name = string.format('%s via %s', from.name, via_user)
- else
- via_name = string.format('%s via %s', from.user or 'unknown', via_user)
- end
-
- local hdr_encoded = rspamd_util.fold_header('From',
- rspamd_util.mime_header_encode(string.format('%s <%s>',
- via_name, via_addr)))
- local orig_from_encoded = rspamd_util.fold_header('X-Original-From',
- rspamd_util.mime_header_encode(string.format('%s <%s>',
- from.name or '', from.addr)))
- local add_hdrs = {
- ['From'] = { order = 1, value = hdr_encoded },
- }
- local remove_hdrs = {['From'] = 0}
-
- local nreply = from.addr
- if munging_opts.reply_goes_to_list then
- -- Reply-to goes to the list
- nreply = via_addr
- end
-
- if task:has_header('Reply-To') then
- -- If we have reply-to header, then we need to insert an additional
- -- address there
- local orig_reply = task:get_header_full('Reply-To')[1]
- if orig_reply.value then
- nreply = string.format('%s, %s', orig_reply.value, nreply)
- end
- remove_hdrs['Reply-To'] = 1
- end
-
- add_hdrs['Reply-To'] = {order = 0, value = nreply}
-
- add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded}
- lua_mime.modify_headers(task, {
- remove = remove_hdrs,
- add = add_hdrs
- })
- lua_util.debugm(N, task, 'munged DMARC header for %s: %s -> %s',
- from.domain, hdr_encoded, from.addr)
- rspamd_logger.infox(task, 'munged DMARC header for %s', from.addr)
- task:insert_result('DMARC_MUNGED', 1.0, from.addr)
- end
rspamd_config:register_symbol({
name = 'DMARC_MUNGED',
@@ -972,7 +773,7 @@ if settings.munging then
score = 0,
group = 'policies',
groups = {'dmarc'},
- callback = dmarc_munge_callback
+ callback = dmarc_common.gen_munging_callback(munging_opts, settings)
})
rspamd_config:register_dependency('DMARC_MUNGED', 'DMARC_CHECK')
More information about the Commits
mailing list