commit e9c96fd: [Minor] Lua_scanners: Add more generic yield result
Vsevolod Stakhov
vsevolod at highsecure.ru
Mon Dec 31 12:07:05 UTC 2018
Author: Vsevolod Stakhov
Date: 2018-12-31 11:25:43 +0000
URL: https://github.com/rspamd/rspamd/commit/e9c96fdb286727b1a95b8484e7d4e2877bbc3629
[Minor] Lua_scanners: Add more generic yield result
---
lualib/lua_scanners/clamav.lua | 7 ++--
lualib/lua_scanners/common.lua | 63 ++++++++++++++++++++++++------------
lualib/lua_scanners/dcc.lua | 6 ++--
lualib/lua_scanners/fprot.lua | 7 ++--
lualib/lua_scanners/kaspersky_av.lua | 5 +--
lualib/lua_scanners/savapi.lua | 5 +--
lualib/lua_scanners/sophos.lua | 5 +--
7 files changed, 62 insertions(+), 36 deletions(-)
diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua
index 26d5e9c81..c7dd08bfc 100644
--- a/lualib/lua_scanners/clamav.lua
+++ b/lualib/lua_scanners/clamav.lua
@@ -26,7 +26,7 @@ local rspamd_util = require "rspamd_util"
local rspamd_logger = require "rspamd_logger"
local common = require "lua_scanners/common"
-local N = "antivirus"
+local N = "clamav"
local default_message = '${SCANNER}: virus found: "${VIRUS}"'
@@ -37,7 +37,8 @@ local function clamav_config(opts)
scan_image_mime = false;
default_port = 3310,
log_clean = false,
- timeout = 15.0, -- FIXME: this will break task_timeout!
+ timeout = 5.0, -- FIXME: this will break task_timeout!
+ detection_category = "virus",
retransmits = 2,
cache_expire = 3600, -- expire redis in one hour
message = default_message,
@@ -149,7 +150,7 @@ local function clamav_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
+ if common.need_av_check(task, content, rule, N) then
if common.check_av_cache(task, digest, rule, clamav_check_uncached, N) then
return
else
diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua
index ad99137a2..605e3bb15 100644
--- a/lualib/lua_scanners/common.lua
+++ b/lualib/lua_scanners/common.lua
@@ -22,6 +22,7 @@ limitations under the License.
local rspamd_logger = require "rspamd_logger"
local lua_util = require "lua_util"
local lua_redis = require "lua_redis"
+local fun = require "fun"
local exports = {}
@@ -46,36 +47,38 @@ local function match_patterns(default_sym, found, patterns)
end
end
-local function yield_result(task, rule, vname, N)
+local function yield_result(task, rule, vname, N, dyn_weight)
local all_whitelisted = true
+ if not dyn_weight then dyn_weight = 1.0 end
if type(vname) == 'string' then
- local symname = match_patterns(rule['symbol'], vname, rule['patterns'])
- if rule['whitelist'] and rule['whitelist']:get_key(vname) then
- rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule['type'], vname)
+ local symname = match_patterns(rule.symbol, vname, rule.patterns)
+ if rule.whitelist and rule.whitelist:get_key(vname) then
+ rspamd_logger.infox(task, '%s: "%s" is in whitelist', N, vname)
return
end
task:insert_result(symname, 1.0, vname)
- rspamd_logger.infox(task, '%s: virus found: "%s"', rule['type'], vname)
+ rspamd_logger.infox(task, '%s: %s found: "%s"', N, rule.detection_category, vname)
elseif type(vname) == 'table' then
for _, vn in ipairs(vname) do
- local symname = match_patterns(rule['symbol'], vn, rule['patterns'])
- if rule['whitelist'] and rule['whitelist']:get_key(vn) then
- rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule['type'], vn)
+ local symname = match_patterns(rule.symbol, vn, rule.patterns)
+ if rule.whitelist and rule.whitelist:get_key(vn) then
+ rspamd_logger.infox(task, '%s: "%s" is in whitelist', N, vn)
else
all_whitelisted = false
- task:insert_result(symname, 1.0, vn)
- rspamd_logger.infox(task, '%s: virus found: "%s"', rule['type'], vn)
+ task:insert_result(symname, dyn_weight, vn)
+ rspamd_logger.infox(task, '%s: %s found: "%s"',
+ N, rule.detection_category, vn)
end
end
end
- if rule['action'] then
+ if rule.action then
if type(vname) == 'table' then
if all_whitelisted then return end
vname = table.concat(vname, '; ')
end
task:set_pre_result(rule['action'],
lua_util.template(rule.message or 'Rejected', {
- SCANNER = rule['type'],
+ SCANNER = N,
VIRUS = vname,
}), N)
end
@@ -85,15 +88,15 @@ local function message_not_too_large(task, content, rule)
local max_size = tonumber(rule.max_size)
if not max_size then return true end
if #content > max_size then
- rspamd_logger.infox(task, "skip %s AV check as it is too large: %s (%s is allowed)",
- rule.type, #content, max_size)
+ rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)",
+ N, #content, max_size)
return false
end
return true
end
-local function need_av_check(task, content, rule)
- return message_not_too_large(task, content, rule)
+local function need_av_check(task, content, rule, N)
+ return message_not_too_large(task, content, rule, N)
end
local function check_av_cache(task, digest, rule, fn, N)
@@ -144,8 +147,8 @@ local function save_av_cache(task, digest, rule, to_save, N)
local function redis_set_cb(err)
-- Do nothing
if err then
- rspamd_logger.errx(task, 'failed to save virus cache for %s -> "%s": %s',
- to_save, key, err)
+ rspamd_logger.errx(task, 'failed to save %s cache for %s -> "%s": %s',
+ rule.detection_category, to_save, key, err)
else
lua_util.debugm(N, task, 'saved cached result for %s: %s',
key, to_save)
@@ -156,8 +159,8 @@ local function save_av_cache(task, digest, rule, to_save, N)
to_save = table.concat(to_save, '\v')
end
- if rule.redis_params then
- key = rule['prefix'] .. key
+ if rule.redis_params and rule.prefix then
+ key = rule.prefix .. key
lua_redis.redis_make_request(task,
rule.redis_params, -- connect params
@@ -165,18 +168,36 @@ local function save_av_cache(task, digest, rule, to_save, N)
true, -- is write
redis_set_cb, --callback
'SETEX', -- command
- { key, rule['cache_expire'], to_save }
+ { key, rule.cache_expire or 0, to_save }
)
end
return false
end
+local function text_parts_min_words(task, min_words)
+ local text_parts_empty = true
+ local text_parts = task:get_text_parts()
+
+ local filter_func = function(p)
+ return p:get_words_count() >= min_words
+ end
+
+ fun.each(function(p)
+ text_parts_empty = false
+ end, fun.filter(filter_func, text_parts))
+
+ return text_parts_empty
+
+end
+
+
exports.yield_result = yield_result
exports.match_patterns = match_patterns
exports.need_av_check = need_av_check
exports.check_av_cache = check_av_cache
exports.save_av_cache = save_av_cache
+exports.text_parts_min_words = text_parts_min_words
setmetatable(exports, {
__call = function(t, override)
diff --git a/lualib/lua_scanners/dcc.lua b/lualib/lua_scanners/dcc.lua
index 43beda6ff..d34bd8425 100644
--- a/lualib/lua_scanners/dcc.lua
+++ b/lualib/lua_scanners/dcc.lua
@@ -16,7 +16,7 @@ limitations under the License.
]]--
--[[[
--- @module fprot
+-- @module dcc
-- This module contains dcc access functions
--]]
@@ -225,7 +225,7 @@ local function dcc_check(task, content, _, rule)
callback = dcc_callback
})
end
- if common.need_av_check(task, content, rule) then
+ if common.need_av_check(task, content, rule, N) then
dcc_check_uncached()
end
end
@@ -280,7 +280,7 @@ local function dcc_config(opts)
end
return {
- type = {'dcc','bulk_scanner', 'scanner'},
+ type = {'dcc','bulk', 'hash', 'scanner'},
description = 'dcc bulk scanner',
configure = dcc_config,
check = dcc_check,
diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua
index 1cb21dd43..27a29a4bc 100644
--- a/lualib/lua_scanners/fprot.lua
+++ b/lualib/lua_scanners/fprot.lua
@@ -25,7 +25,7 @@ local upstream_list = require "rspamd_upstream_list"
local rspamd_logger = require "rspamd_logger"
local common = require "lua_scanners/common"
-local N = "antivirus"
+local N = "fprot"
local default_message = '${SCANNER}: virus found: "${VIRUS}"'
@@ -35,8 +35,9 @@ local function fprot_config(opts)
scan_text_mime = false;
scan_image_mime = false;
default_port = 10200,
- timeout = 15.0, -- FIXME: this will break task_timeout!
+ timeout = 5.0, -- FIXME: this will break task_timeout!
log_clean = false,
+ detection_category = "virus",
retransmits = 2,
cache_expire = 3600, -- expire redis in one hour
message = default_message,
@@ -152,7 +153,7 @@ local function fprot_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
+ if common.need_av_check(task, content, rule, N) then
if common.check_av_cache(task, digest, rule, fprot_check_uncached, N) then
return
else
diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua
index b55b6c24c..e903467c2 100644
--- a/lualib/lua_scanners/kaspersky_av.lua
+++ b/lualib/lua_scanners/kaspersky_av.lua
@@ -26,7 +26,7 @@ local rspamd_util = require "rspamd_util"
local rspamd_logger = require "rspamd_logger"
local common = require "lua_scanners/common"
-local N = "antivirus"
+local N = "kaspersky"
local default_message = '${SCANNER}: virus found: "${VIRUS}"'
@@ -41,6 +41,7 @@ local function kaspersky_config(opts)
retransmits = 1, -- use local files, retransmits are useless
cache_expire = 3600, -- expire redis in one hour
message = default_message,
+ detection_category = "virus",
tmpdir = '/tmp',
prefix = 'rs_ak',
}
@@ -170,7 +171,7 @@ local function kaspersky_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
+ if common.need_av_check(task, content, rule, N) then
if common.check_av_cache(task, digest, rule, kaspersky_check_uncached, N) then
return
else
diff --git a/lualib/lua_scanners/savapi.lua b/lualib/lua_scanners/savapi.lua
index 0cbe9ff48..84452e017 100644
--- a/lualib/lua_scanners/savapi.lua
+++ b/lualib/lua_scanners/savapi.lua
@@ -26,7 +26,7 @@ local rspamd_util = require "rspamd_util"
local rspamd_logger = require "rspamd_logger"
local common = require "lua_scanners/common"
-local N = "antivirus"
+local N = "savapi"
local default_message = '${SCANNER}: virus found: "${VIRUS}"'
@@ -42,6 +42,7 @@ local function savapi_config(opts)
retransmits = 1, -- FIXME: useless, for local files
cache_expire = 3600, -- expire redis in one hour
message = default_message,
+ detection_category = "virus",
tmpdir = '/tmp',
}
@@ -234,7 +235,7 @@ local function savapi_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
+ if common.need_av_check(task, content, rule, N) then
if common.check_av_cache(task, digest, rule, savapi_check_uncached, N) then
return
else
diff --git a/lualib/lua_scanners/sophos.lua b/lualib/lua_scanners/sophos.lua
index ef4acb3aa..c805cc56f 100644
--- a/lualib/lua_scanners/sophos.lua
+++ b/lualib/lua_scanners/sophos.lua
@@ -25,7 +25,7 @@ local upstream_list = require "rspamd_upstream_list"
local rspamd_logger = require "rspamd_logger"
local common = require "lua_scanners/common"
-local N = "antivirus"
+local N = "sophos"
local default_message = '${SCANNER}: virus found: "${VIRUS}"'
@@ -41,6 +41,7 @@ local function sophos_config(opts)
cache_expire = 3600, -- expire redis in one hour
message = default_message,
savdi_report_encrypted = false,
+ detection_category = "virus",
savdi_report_oversize = false,
}
@@ -168,7 +169,7 @@ local function sophos_check(task, content, digest, rule)
})
end
- if common.need_av_check(task, content, rule) then
+ if common.need_av_check(task, content, rule, N) then
if common.check_av_cache(task, digest, rule, sophos_check_uncached, N) then
return
else
More information about the Commits
mailing list