commit cb1fb98: [Minor] lua_scanners - simplify need_check and dynamic_scan
Carsten Rosenberg
c.rosenberg at heinlein-support.de
Fri Oct 4 10:42:13 UTC 2019
Author: Carsten Rosenberg
Date: 2019-09-26 22:56:36 +0200
URL: https://github.com/rspamd/rspamd/commit/cb1fb98db95bb42ba43c98b8c390748f7ac9c300
[Minor] lua_scanners - simplify need_check and dynamic_scan
---
lualib/lua_scanners/clamav.lua | 19 ++-------
lualib/lua_scanners/common.lua | 79 ++++++++++++++++++++++++++++++++----
lualib/lua_scanners/dcc.lua | 28 ++++---------
lualib/lua_scanners/fprot.lua | 19 ++-------
lualib/lua_scanners/icap.lua | 17 ++------
lualib/lua_scanners/kaspersky_av.lua | 19 ++-------
lualib/lua_scanners/oletools.lua | 29 +++++--------
lualib/lua_scanners/savapi.lua | 21 +++-------
lualib/lua_scanners/sophos.lua | 21 +++-------
lualib/lua_scanners/spamassassin.lua | 22 +++-------
lualib/lua_scanners/vadesecure.lua | 11 ++---
11 files changed, 125 insertions(+), 160 deletions(-)
diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua
index 01386cfe7..7771261e1 100644
--- a/lualib/lua_scanners/clamav.lua
+++ b/lualib/lua_scanners/clamav.lua
@@ -151,19 +151,11 @@ local function clamav_check(task, content, digest, rule)
end
end
if cached then
- common.save_av_cache(task, digest, rule, cached)
+ common.save_cache(task, digest, rule, cached)
end
end
end
- if rule.dynamic_scan then
- local pre_check, pre_check_msg = common.check_metric_results(task, rule)
- if pre_check then
- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
- return true
- end
- end
-
tcp.request({
task = task,
host = addr:to_string(),
@@ -175,13 +167,10 @@ local function clamav_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
- if common.check_av_cache(task, digest, rule, clamav_check_uncached) then
- return
- else
- clamav_check_uncached()
- end
+ if common.need_check(task, content, rule, digest) then
+ clamav_check_uncached()
end
+
end
return {
diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua
index 65dd4aef8..dcd31db30 100644
--- a/lualib/lua_scanners/common.lua
+++ b/lualib/lua_scanners/common.lua
@@ -125,11 +125,68 @@ local function message_not_too_large(task, content, rule)
return true
end
-local function need_av_check(task, content, rule)
- return message_not_too_large(task, content, rule)
+local function message_not_too_small(task, content, rule)
+ local min_size = tonumber(rule.min_size)
+ if not min_size then return true end
+ if #content < min_size then
+ rspamd_logger.infox(task, "skip %s check as it is too small: %s (%s is allowed)",
+ rule.log_prefix, #content, min_size)
+ return false
+ end
+ return true
+end
+
+local function message_min_words(task, rule)
+ if rule.text_part_min_words then
+ local text_parts_empty = false
+ local text_parts = task:get_text_parts()
+
+ local filter_func = function(p)
+ return p:get_words_count() <= tonumber(rule.text_part_min_words)
+ end
+
+ fun.each(function(p)
+ text_parts_empty = true
+ rspamd_logger.infox(task, '%s: #words is less then text_part_min_words: %s',
+ rule.log_prefix, rule.text_part_min_words)
+ end, fun.filter(filter_func, text_parts))
+
+ return text_parts_empty
+ else
+ return true
+ end
end
-local function check_av_cache(task, digest, rule, fn)
+local function dynamic_scan(task, rule)
+ if rule.dynamic_scan then
+ if rule.action ~= 'reject' then
+ local metric_result = task:get_metric_score('default')
+ local metric_action = task:get_metric_action('default')
+ local has_pre_result = task:has_pre_result()
+ -- ToDo: needed?
+ -- Sometimes leads to FPs
+ --if rule.symbol_type == 'postfilter' and metric_action == 'reject' then
+ -- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, "result is already reject")
+ -- return false
+ --elseif metric_result[1] > metric_result[2]*2 then
+ if metric_result[1] > metric_result[2]*2 then
+ rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'score > 2 * reject_level: ' .. metric_result[1])
+ return false
+ elseif has_pre_result and metric_action == 'reject' then
+ rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'pre_result reject is set')
+ return false
+ else
+ return true, 'undecided'
+ end
+ else
+ return true, 'dynamic_scan is not possible with config `action=reject;`'
+ end
+ else
+ return true
+ end
+end
+
+local function check_cache(task, digest, rule, fn)
local key = digest
local function redis_av_cb(err, data)
@@ -173,7 +230,15 @@ local function check_av_cache(task, digest, rule, fn)
return false
end
-local function save_av_cache(task, digest, rule, to_save, dyn_weight)
+local function need_check(task, content, rule, digest)
+ return check_cache(task, digest, rule) and
+ message_not_too_large(task, content, rule) and
+ message_not_too_small(task, content, rule) and
+ message_min_words(task, rule) and
+ dynamic_scan(task, rule)
+end
+
+local function save_cache(task, digest, rule, to_save, dyn_weight)
local key = digest
if not dyn_weight then dyn_weight = 1.0 end
@@ -363,9 +428,9 @@ end
exports.log_clean = log_clean
exports.yield_result = yield_result
exports.match_patterns = match_patterns
-exports.need_av_check = need_av_check
-exports.check_av_cache = check_av_cache
-exports.save_av_cache = save_av_cache
+exports.need_check = need_check
+exports.check_cache = check_cache
+exports.save_cache = save_cache
exports.create_regex_table = create_regex_table
exports.check_parts_match = check_parts_match
exports.check_metric_results = check_metric_results
diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua
index e26e666f9..85d72dd8b 100644
--- a/lualib/lua_scanners/dcc.lua
+++ b/lualib/lua_scanners/dcc.lua
@@ -193,7 +193,7 @@ local function dcc_check(task, content, digest, rule)
if (result == 'R') then
-- Reject
common.yield_result(task, rule, info, rule.default_score)
- common.save_av_cache(task, digest, rule, info, rule.default_score)
+ common.save_cache(task, digest, rule, info, rule.default_score)
elseif (result == 'T') then
-- Temporary failure
rspamd_logger.warnx(task, 'DCC returned a temporary failure result: %s', result)
@@ -248,9 +248,9 @@ local function dcc_check(task, content, digest, rule)
task:insert_result(rule.symbol_bulk,
score,
opts)
- common.save_av_cache(task, digest, rule, opts, score)
+ common.save_cache(task, digest, rule, opts, score)
else
- common.save_av_cache(task, digest, rule, 'OK')
+ common.save_cache(task, digest, rule, 'OK')
if rule.log_clean then
rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
rule.log_prefix, info)
@@ -261,7 +261,7 @@ local function dcc_check(task, content, digest, rule)
end
elseif result == 'G' then
-- do nothing
- common.save_av_cache(task, digest, rule, 'OK')
+ common.save_cache(task, digest, rule, 'OK')
if rule.log_clean then
rspamd_logger.infox(task, '%s: clean, returned result G - info: %s', rule.log_prefix, info)
else
@@ -269,7 +269,7 @@ local function dcc_check(task, content, digest, rule)
end
elseif result == 'S' then
-- do nothing
- common.save_av_cache(task, digest, rule, 'OK')
+ common.save_cache(task, digest, rule, 'OK')
if rule.log_clean then
rspamd_logger.infox(task, '%s: clean, returned result S - info: %s', rule.log_prefix, info)
else
@@ -284,14 +284,6 @@ local function dcc_check(task, content, digest, rule)
end
end
- if rule.dynamic_scan then
- local pre_check, pre_check_msg = common.check_metric_results(task, rule)
- if pre_check then
- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
- return true
- end
- end
-
tcp.request({
task = task,
host = addr:to_string(),
@@ -305,13 +297,11 @@ local function dcc_check(task, content, digest, rule)
fuz2_max = 999999,
})
end
- if common.need_av_check(task, content, rule) then
- if common.check_av_cache(task, digest, rule, dcc_check_uncached) then
- return
- else
- dcc_check_uncached()
- end
+
+ if common.need_check(task, content, rule, digest) then
+ dcc_check_uncached()
end
+
end
return {
diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua
index 907fab139..30d179ec6 100644
--- a/lualib/lua_scanners/fprot.lua
+++ b/lualib/lua_scanners/fprot.lua
@@ -144,19 +144,11 @@ local function fprot_check(task, content, digest, rule)
end
end
if cached then
- common.save_av_cache(task, digest, rule, cached)
+ common.save_cache(task, digest, rule, cached)
end
end
end
- if rule.dynamic_scan then
- local pre_check, pre_check_msg = common.check_metric_results(task, rule)
- if pre_check then
- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
- return true
- end
- end
-
tcp.request({
task = task,
host = addr:to_string(),
@@ -168,13 +160,10 @@ local function fprot_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
- if common.check_av_cache(task, digest, rule, fprot_check_uncached) then
- return
- else
- fprot_check_uncached()
- end
+ if common.need_check(task, content, rule, digest) then
+ fprot_check_uncached()
end
+
end
return {
diff --git a/lualib/lua_scanners/icap.lua b/lualib/lua_scanners/icap.lua
index 908c2c1f6..c7b495c29 100644
--- a/lualib/lua_scanners/icap.lua
+++ b/lualib/lua_scanners/icap.lua
@@ -354,14 +354,6 @@ local function icap_check(task, content, digest, rule)
end
end
- if rule.dynamic_scan then
- local pre_check, pre_check_msg = common.check_metric_results(task, rule)
- if pre_check then
- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
- return true
- end
- end
-
tcp.request({
task = task,
host = addr:to_string(),
@@ -374,13 +366,10 @@ local function icap_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
- if common.check_av_cache(task, digest, rule, icap_check_uncached) then
- return
- else
- icap_check_uncached()
- end
+ if common.need_check(task, content, rule, digest) then
+ icap_check_uncached()
end
+
end
return {
diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua
index 87411c3b9..cb652f5b6 100644
--- a/lualib/lua_scanners/kaspersky_av.lua
+++ b/lualib/lua_scanners/kaspersky_av.lua
@@ -162,19 +162,11 @@ local function kaspersky_check(task, content, digest, rule)
end
end
if cached then
- common.save_av_cache(task, digest, rule, cached)
+ common.save_cache(task, digest, rule, cached)
end
end
end
- if rule.dynamic_scan then
- local pre_check, pre_check_msg = common.check_metric_results(task, rule)
- if pre_check then
- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
- return true
- end
- end
-
tcp.request({
task = task,
host = addr:to_string(),
@@ -186,13 +178,10 @@ local function kaspersky_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
- if common.check_av_cache(task, digest, rule, kaspersky_check_uncached) then
- return
- else
- kaspersky_check_uncached()
- end
+ if common.need_check(task, content, rule, digest) then
+ kaspersky_check_uncached()
end
+
end
return {
diff --git a/lualib/lua_scanners/oletools.lua b/lualib/lua_scanners/oletools.lua
index 3daa76713..88ecfdece 100644
--- a/lualib/lua_scanners/oletools.lua
+++ b/lualib/lua_scanners/oletools.lua
@@ -42,6 +42,7 @@ local function oletools_config(opts)
log_clean = false,
retransmits = 2,
cache_expire = 86400, -- expire redis in 1d
+ min_size = 500,
symbol = "OLETOOLS",
message = '${SCANNER}: Oletools threat message found: "${VIRUS}"',
detection_category = "office macro",
@@ -176,7 +177,7 @@ local function oletools_check(task, content, digest, rule)
rspamd_logger.errx(task, '%s: ERROR found: %s', rule.log_prefix,
result[1].error)
if result[1].error == 'File too small' then
- common.save_av_cache(task, digest, rule, 'OK')
+ common.save_cache(task, digest, rule, 'OK')
common.log_clean(task, rule, 'File too small to be scanned for macros')
else
oletools_requery(result[1].error)
@@ -198,7 +199,7 @@ local function oletools_check(task, content, digest, rule)
rspamd_logger.warnx(task, '%s: maybe unhandled python or oletools error', rule.log_prefix)
common.yield_result(task, rule, 'oletools unhandled error', 0.0, 'fail')
elseif type(result[2]['analysis']) ~= 'table' and #result[2]['macros'] == 0 then
- common.save_av_cache(task, digest, rule, 'OK')
+ common.save_cache(task, digest, rule, 'OK')
common.log_clean(task, rule, 'No macro found')
elseif #result[2]['macros'] > 0 then
-- M=Macros, A=Auto-executable, S=Suspicious keywords, I=IOCs,
@@ -257,7 +258,7 @@ local function oletools_check(task, content, digest, rule)
local threat = 'AutoExec + Suspicious (' .. table.concat(analysis_keyword_table, ',') .. ')'
lua_util.debugm(rule.name, task, '%s: threat result: %s', rule.log_prefix, threat)
common.yield_result(task, rule, threat, rule.default_score)
- common.save_av_cache(task, digest, rule, threat, rule.default_score)
+ common.save_cache(task, digest, rule, threat, rule.default_score)
elseif rule.extended == true and #analysis_keyword_table > 0 then
-- report any flags (types) and any most keywords as individual virus name
@@ -276,9 +277,9 @@ local function oletools_check(task, content, digest, rule)
rule.log_prefix, table.concat(analysis_keyword_table, ','))
common.yield_result(task, rule, analysis_keyword_table, rule.default_score)
- common.save_av_cache(task, digest, rule, analysis_keyword_table, rule.default_score)
+ common.save_cache(task, digest, rule, analysis_keyword_table, rule.default_score)
else
- common.save_av_cache(task, digest, rule, 'OK')
+ common.save_cache(task, digest, rule, 'OK')
common.log_clean(task, rule, 'Scanned Macro is OK')
end
@@ -290,14 +291,6 @@ local function oletools_check(task, content, digest, rule)
end
end
- if rule.dynamic_scan then
- local pre_check, pre_check_msg = common.check_metric_results(task, rule)
- if pre_check then
- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
- return true
- end
- end
-
tcp.request({
task = task,
host = addr:to_string(),
@@ -309,13 +302,11 @@ local function oletools_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
- if common.check_av_cache(task, digest, rule, oletools_check_uncached) then
- return
- else
- oletools_check_uncached()
- end
+
+ if common.need_check(task, content, rule, digest) then
+ oletools_check_uncached()
end
+
end
return {
diff --git a/lualib/lua_scanners/savapi.lua b/lualib/lua_scanners/savapi.lua
index 65a9c825c..11f658da3 100644
--- a/lualib/lua_scanners/savapi.lua
+++ b/lualib/lua_scanners/savapi.lua
@@ -127,7 +127,7 @@ local function savapi_check(task, content, digest, rule)
end
common.yield_result(task, rule, vname)
- common.save_av_cache(task, digest, rule, vname)
+ common.save_cache(task, digest, rule, vname)
end
if conn then
conn:close()
@@ -144,7 +144,7 @@ local function savapi_check(task, content, digest, rule)
if rule['log_clean'] then
rspamd_logger.infox(task, '%s: message or mime_part is clean', rule['type'])
end
- common.save_av_cache(task, digest, rule, 'OK')
+ common.save_cache(task, digest, rule, 'OK')
conn:add_write(savapi_fin_cb, 'QUIT\n')
-- Terminal response - infected
@@ -237,14 +237,6 @@ local function savapi_check(task, content, digest, rule)
end
end
- if rule.dynamic_scan then
- local pre_check, pre_check_msg = common.check_metric_results(task, rule)
- if pre_check then
- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
- return true
- end
- end
-
tcp.request({
task = task,
host = addr:to_string(),
@@ -255,13 +247,10 @@ local function savapi_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
- if common.check_av_cache(task, digest, rule, savapi_check_uncached) then
- return
- else
- savapi_check_uncached()
- end
+ if common.need_check(task, content, rule, digest) then
+ savapi_check_uncached()
end
+
end
return {
diff --git a/lualib/lua_scanners/sophos.lua b/lualib/lua_scanners/sophos.lua
index 59facc845..ff47c262d 100644
--- a/lualib/lua_scanners/sophos.lua
+++ b/lualib/lua_scanners/sophos.lua
@@ -125,7 +125,7 @@ local function sophos_check(task, content, digest, rule)
local vname = string.match(data, 'VIRUS (%S+) ')
if vname then
common.yield_result(task, rule, vname)
- common.save_av_cache(task, digest, rule, vname)
+ common.save_cache(task, digest, rule, vname)
else
if string.find(data, 'DONE OK') then
if rule['log_clean'] then
@@ -134,7 +134,7 @@ local function sophos_check(task, content, digest, rule)
lua_util.debugm(rule.name, task,
'%s: message or mime_part is clean', rule.log_prefix)
end
- common.save_av_cache(task, digest, rule, 'OK')
+ common.save_cache(task, digest, rule, 'OK')
-- not finished - continue
elseif string.find(data, 'ACC') or string.find(data, 'OK SSSP') then
conn:add_read(sophos_callback)
@@ -157,14 +157,6 @@ local function sophos_check(task, content, digest, rule)
end
end
- if rule.dynamic_scan then
- local pre_check, pre_check_msg = common.check_metric_results(task, rule)
- if pre_check then
- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
- return true
- end
- end
-
tcp.request({
task = task,
host = addr:to_string(),
@@ -175,13 +167,10 @@ local function sophos_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
- if common.check_av_cache(task, digest, rule, sophos_check_uncached) then
- return
- else
- sophos_check_uncached()
- end
+ if common.need_check(task, content, rule, digest) then
+ sophos_check_uncached()
end
+
end
return {
diff --git a/lualib/lua_scanners/spamassassin.lua b/lualib/lua_scanners/spamassassin.lua
index 860df42dd..dd4c914bb 100644
--- a/lualib/lua_scanners/spamassassin.lua
+++ b/lualib/lua_scanners/spamassassin.lua
@@ -177,14 +177,14 @@ local function spamassassin_check(task, content, digest, rule)
if rule.extended == false then
common.yield_result(task, rule, symbols, spam_score)
- common.save_av_cache(task, digest, rule, symbols, spam_score)
+ common.save_cache(task, digest, rule, symbols, spam_score)
else
local symbols_table = {}
symbols_table = rspamd_str_split(symbols, ",")
lua_util.debugm(rule.N, task, '%s: returned symbols as table: %s', rule.log_prefix, symbols_table)
common.yield_result(task, rule, symbols_table, spam_score)
- common.save_av_cache(task, digest, rule, symbols_table, spam_score)
+ common.save_cache(task, digest, rule, symbols_table, spam_score)
end
else
common.log_clean(task, rule, 'no spam detected - spam score: ' .. spam_score .. ', symbols: ' .. symbols)
@@ -192,14 +192,6 @@ local function spamassassin_check(task, content, digest, rule)
end
end
- if rule.dynamic_scan then
- local pre_check, pre_check_msg = common.check_metric_results(task, rule)
- if pre_check then
- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
- return true
- end
- end
-
tcp.request({
task = task,
host = addr:to_string(),
@@ -209,13 +201,11 @@ local function spamassassin_check(task, content, digest, rule)
callback = spamassassin_callback,
})
end
- if common.need_av_check(task, content, rule) then
- if common.check_av_cache(task, digest, rule, spamassassin_check_uncached) then
- return
- else
- spamassassin_check_uncached()
- end
+
+ if common.need_check(task, content, rule, digest) then
+ spamassassin_check_uncached()
end
+
end
return {
diff --git a/lualib/lua_scanners/vadesecure.lua b/lualib/lua_scanners/vadesecure.lua
index 196d55ba4..0e539ef46 100644
--- a/lualib/lua_scanners/vadesecure.lua
+++ b/lualib/lua_scanners/vadesecure.lua
@@ -305,16 +305,11 @@ local function vade_check(task, content, digest, rule)
end
end
- if rule.dynamic_scan then
- local pre_check, pre_check_msg = common.check_metric_results(task, rule)
- if pre_check then
- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, pre_check_msg)
- return true
- end
+ if common.need_check(task, content, rule, digest) then
+ request_data.callback = vade_callback
+ http.request(request_data)
end
- request_data.callback = vade_callback
- http.request(request_data)
end
return {
More information about the Commits
mailing list