commit f20b2f5: [Rework] Antivirus: Move antivirus definitions to lualib from a plugin
Vsevolod Stakhov
vsevolod at highsecure.ru
Thu Dec 27 18:28:11 UTC 2018
Author: Vsevolod Stakhov
Date: 2018-12-18 16:15:58 +0000
URL: https://github.com/rspamd/rspamd/commit/f20b2f5eebec3fe03aa38c4a866b9a01092a6c7a
[Rework] Antivirus: Move antivirus definitions to lualib from a plugin
---
.../lua/antivirus.lua => lualib/lua_antivirus.lua | 406 +++------
src/plugins/lua/antivirus.lua | 952 +--------------------
2 files changed, 118 insertions(+), 1240 deletions(-)
diff --git a/src/plugins/lua/antivirus.lua b/lualib/lua_antivirus.lua
similarity index 73%
copy from src/plugins/lua/antivirus.lua
copy to lualib/lua_antivirus.lua
index 2aa1f0344..286ef64d0 100644
--- a/src/plugins/lua/antivirus.lua
+++ b/lualib/lua_antivirus.lua
@@ -1,5 +1,5 @@
--[[
-Copyright (c) 2016, Vsevolod Stakhov <vsevolod at highsecure.ru>
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod at highsecure.ru>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -12,64 +12,22 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-]] --
+]]--
-local rspamd_logger = require "rspamd_logger"
-local rspamd_util = require "rspamd_util"
-local rspamd_regexp = require "rspamd_regexp"
+--[[[
+-- @module lua_antivirus
+-- This module contains antivirus access functions
+--]]
+
+local lua_util = require "lua_util"
local tcp = require "rspamd_tcp"
local upstream_list = require "rspamd_upstream_list"
-local lua_util = require "lua_util"
-local fun = require "fun"
-local redis_params
+local rspamd_util = require "rspamd_util"
+local lua_redis = require "lua_redis"
+local rspamd_logger = require "rspamd_logger"
local N = "antivirus"
-if confighelp then
- rspamd_config:add_example(nil, 'antivirus',
- "Check messages for viruses",
- [[
-antivirus {
- # multiple scanners could be checked, for each we create a configuration block with an arbitrary name
- clamav {
- # If set force this action if any virus is found (default unset: no action is forced)
- # action = "reject";
- # If set, then rejection message is set to this value (mention single quotes)
- # message = '${SCANNER}: virus found: "${VIRUS}"';
- # Scan mime_parts seperately - otherwise the complete mail will be transfered to AV Scanner
- #scan_mime_parts = true;
- # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity)
- #scan_text_mime = false;
- #scan_image_mime = false;
- # If `max_size` is set, messages > n bytes in size are not scanned
- max_size = 20000000;
- # symbol to add (add it to metric if you want non-zero weight)
- symbol = "CLAM_VIRUS";
- # type of scanner: "clamav", "fprot", "sophos" or "savapi"
- type = "clamav";
- # For "savapi" you must also specify the following variable
- product_id = 12345;
- # You can enable logging for clean messages
- log_clean = true;
- # servers to query (if port is unspecified, scanner-specific default is used)
- # can be specified multiple times to pool servers
- # can be set to a path to a unix socket
- # Enable this in local.d/antivirus.conf
- servers = "127.0.0.1:3310";
- # if `patterns` is specified virus name will be matched against provided regexes and the related
- # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
- patterns {
- # symbol_name = "pattern";
- JUST_EICAR = "^Eicar-Test-Signature$";
- }
- # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
- whitelist = "/etc/rspamd/antivirus.wl";
- }
-}
-]])
- return
-end
-
local default_message = '${SCANNER}: virus found: "${VIRUS}"'
local function match_patterns(default_sym, found, patterns)
@@ -156,15 +114,15 @@ local function clamav_config(opts)
end
clamav_conf['upstreams'] = upstream_list.create(rspamd_config,
- clamav_conf['servers'],
- clamav_conf.default_port)
+ clamav_conf['servers'],
+ clamav_conf.default_port)
if clamav_conf['upstreams'] then
return clamav_conf
end
rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
- clamav_conf['servers'])
+ clamav_conf['servers'])
return nil
end
@@ -196,15 +154,15 @@ local function fprot_config(opts)
end
fprot_conf['upstreams'] = upstream_list.create(rspamd_config,
- fprot_conf['servers'],
- fprot_conf.default_port)
+ fprot_conf['servers'],
+ fprot_conf.default_port)
if fprot_conf['upstreams'] then
return fprot_conf
end
rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
- fprot_conf['servers'])
+ fprot_conf['servers'])
return nil
end
@@ -238,15 +196,15 @@ local function sophos_config(opts)
end
sophos_conf['upstreams'] = upstream_list.create(rspamd_config,
- sophos_conf['servers'],
- sophos_conf.default_port)
+ sophos_conf['servers'],
+ sophos_conf.default_port)
if sophos_conf['upstreams'] then
return sophos_conf
end
rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
- sophos_conf['servers'])
+ sophos_conf['servers'])
return nil
end
@@ -280,15 +238,15 @@ local function savapi_config(opts)
end
savapi_conf['upstreams'] = upstream_list.create(rspamd_config,
- savapi_conf['servers'],
- savapi_conf.default_port)
+ savapi_conf['servers'],
+ savapi_conf.default_port)
if savapi_conf['upstreams'] then
return savapi_conf
end
rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
- savapi_conf['servers'])
+ savapi_conf['servers'])
return nil
end
@@ -332,7 +290,7 @@ local function message_not_too_large(task, content, rule)
if not max_size then return true end
if #content > max_size then
rspamd_logger.infox("skip %s AV check as it is too large: %s (%s is allowed)",
- rule.type, #content, max_size)
+ rule.type, #content, max_size)
return false
end
return true
@@ -363,17 +321,17 @@ local function check_av_cache(task, digest, rule, fn)
end
end
- if redis_params then
+ if rule.redis_params then
key = rule['prefix'] .. key
- if rspamd_redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- false, -- is write
- redis_av_cb, --callback
- 'GET', -- command
- {key} -- arguments)
+ if lua_redis.redis_make_request(task,
+ rule.redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_av_cb, --callback
+ 'GET', -- command
+ {key} -- arguments)
) then
return true
end
@@ -389,9 +347,10 @@ local function save_av_cache(task, digest, rule, to_save)
-- Do nothing
if err then
rspamd_logger.errx(task, 'failed to save virus cache for %s -> "%s": %s',
- to_save, key, err)
+ to_save, key, err)
else
- lua_util.debugm(N, task, 'saved cached result for %s: %s', key, to_save)
+ lua_util.debugm(N, task, 'saved cached result for %s: %s',
+ key, to_save)
end
end
@@ -399,16 +358,16 @@ local function save_av_cache(task, digest, rule, to_save)
to_save = table.concat(to_save, '\v')
end
- if redis_params then
+ if rule.redis_params then
key = rule['prefix'] .. key
- rspamd_redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- true, -- is write
- redis_set_cb, --callback
- 'SETEX', -- command
- { key, rule['cache_expire'], to_save }
+ lua_redis.redis_make_request(task,
+ rule.redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'SETEX', -- command
+ { key, rule['cache_expire'], to_save }
)
end
@@ -452,8 +411,11 @@ local function fprot_check(task, content, digest, rule)
stop_pattern = '\n'
})
else
- rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
- task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+ rspamd_logger.errx(task,
+ '%s [%s]: failed to scan, maximum retransmits exceed',
+ rule['symbol'], rule['type'])
+ task:insert_result(rule['symbol_fail'], 0.0,
+ 'failed to scan and retransmits exceed')
end
else
upstream:ok()
@@ -463,7 +425,9 @@ local function fprot_check(task, content, digest, rule)
if clean then
cached = 'OK'
if rule['log_clean'] then
- rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+ rspamd_logger.infox(task,
+ '%s [%s]: message or mime_part is clean',
+ rule['symbol'], rule['type'])
end
else
-- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occured
@@ -508,7 +472,7 @@ local function clamav_check(task, content, digest, rule)
local addr = upstream:get_addr()
local retransmits = rule.retransmits
local header = rspamd_util.pack("c9 c1 >I4", "zINSTREAM", "\0",
- #content)
+ #content)
local footer = rspamd_util.pack(">I4", 0)
local function clamav_callback(err, data)
@@ -602,32 +566,32 @@ local function sophos_check(task, content, digest, rule)
local function sophos_callback(err, data, conn)
if err then
- -- set current upstream to fail because an error occurred
- upstream:fail()
+ -- set current upstream to fail because an error occurred
+ upstream:fail()
- -- retry with another upstream until retransmits exceeds
- if retransmits > 0 then
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
- retransmits = retransmits - 1
+ retransmits = retransmits - 1
- -- Select a different upstream!
- upstream = rule.upstreams:get_upstream_round_robin()
- addr = upstream:get_addr()
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
- lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+ lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
- tcp.request({
- task = task,
- host = addr:to_string(),
- port = addr:get_port(),
- timeout = rule['timeout'],
- callback = sophos_callback,
- data = { protocol, streamsize, content, bye }
- })
- else
- rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
- task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
- end
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule['timeout'],
+ callback = sophos_callback,
+ data = { protocol, streamsize, content, bye }
+ })
+ else
+ rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+ end
else
upstream:ok()
data = tostring(data)
@@ -762,11 +726,11 @@ local function savapi_check(task, content, digest, rule)
save_av_cache(task, digest, rule, 'OK')
conn:add_write(savapi_fin_cb, 'QUIT\n')
- -- Terminal response - infected
+ -- Terminal response - infected
elseif string.find(result, '319') then
conn:add_write(savapi_fin_cb, 'QUIT\n')
- -- Non-terminal response
+ -- Non-terminal response
elseif string.find(result, '310') then
local virus
virus = result:match "310.*<<<%s(.*)%s+;.*;.*"
@@ -981,194 +945,42 @@ local function kaspersky_check(task, content, digest, rule)
end
end
-local av_types = {
- clamav = {
- configure = clamav_config,
- check = clamav_check
- },
- fprot = {
- configure = fprot_config,
- check = fprot_check
- },
- sophos = {
- configure = sophos_config,
- check = sophos_check
- },
- savapi = {
- configure = savapi_config,
- check = savapi_check
+local exports = {
+ av_types = {
+ clamav = {
+ configure = clamav_config,
+ check = clamav_check
+ },
+ fprot = {
+ configure = fprot_config,
+ check = fprot_check
+ },
+ sophos = {
+ configure = sophos_config,
+ check = sophos_check
+ },
+ savapi = {
+ configure = savapi_config,
+ check = savapi_check
+ },
+ kaspersky = {
+ configure = kaspersky_config,
+ check = kaspersky_check
+ }
},
- kaspersky = {
- configure = kaspersky_config,
- check = kaspersky_check
- }
+ -- Some utilities
+ match_patterns = match_patterns,
+ check_av_cache = check_av_cache,
+ save_av_cache = save_av_cache,
}
-local function add_antivirus_rule(sym, opts)
- if not opts['type'] then
- rspamd_logger.errx(rspamd_config, 'unknown type for AV rule %s', sym)
- return nil
- end
-
- if not opts['symbol'] then opts['symbol'] = sym:upper() end
- local cfg = av_types[opts['type']]
-
- if not opts['symbol_fail'] then
- opts['symbol_fail'] = string.upper(opts['type']) .. '_FAIL'
- end
-
- -- WORKAROUND for deprecated attachments_only
- if opts['attachments_only'] ~= nil then
- opts['scan_mime_parts'] = opts['attachments_only']
- rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. '..
- 'Please use scan_mime_parts = %s instead', opts['symbol'], opts['type'], opts['attachments_only'])
- end
- -- WORKAROUND for deprecated attachments_only
-
- if not cfg then
- rspamd_logger.errx(rspamd_config, 'unknown antivirus type: %s',
- opts['type'])
- end
-
- local rule = cfg.configure(opts)
- rule.type = opts.type
- rule.symbol_fail = opts.symbol_fail
-
-
- if not rule then
- rspamd_logger.errx(rspamd_config, 'cannot configure %s for %s',
- opts['type'], opts['symbol'])
- return nil
- end
-
- if type(opts['patterns']) == 'table' then
- rule['patterns'] = {}
- if opts['patterns'][1] then
- for i, p in ipairs(opts['patterns']) do
- if type(p) == 'table' then
- local new_set = {}
- for k, v in pairs(p) do
- new_set[k] = rspamd_regexp.create_cached(v)
- end
- rule['patterns'][i] = new_set
- else
- rule['patterns'][i] = {}
- end
- end
- else
- for k, v in pairs(opts['patterns']) do
- rule['patterns'][k] = rspamd_regexp.create_cached(v)
- end
- end
- end
-
- if opts['whitelist'] then
- rule['whitelist'] = rspamd_config:add_hash_map(opts['whitelist'])
- end
-
- return function(task)
- if rule.scan_mime_parts then
- local parts = task:get_parts() or {}
-
- local filter_func = function(p)
- return (rule.scan_image_mime and p:is_image())
- or (rule.scan_text_mime and p:is_text())
- or (p:is_attachment())
- end
-
- fun.each(function(p)
- local content = p:get_content()
-
- if content and #content > 0 then
- cfg.check(task, content, p:get_digest(), rule)
- end
- end, fun.filter(filter_func, parts))
-
- else
- cfg.check(task, task:get_content(), task:get_digest(), rule)
- end
- end
+exports.add_antivirus = function(name, conf_func, check_func)
+ assert(type(conf_func) == 'function' and type(check_func) == 'function',
+ 'bad arguments')
+ exports.av_types[name] = {
+ configure = conf_func,
+ check = check_func,
+ }
end
--- Registration
-local opts = rspamd_config:get_all_opt('antivirus')
-if opts and type(opts) == 'table' then
- redis_params = rspamd_parse_redis_server('antivirus')
- local has_valid = false
- for k, m in pairs(opts) do
- if type(m) == 'table' and m.servers then
- if not m.type then m.type = k end
- local cb = add_antivirus_rule(k, m)
-
- if not cb then
- rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"')
- else
- local id = rspamd_config:register_symbol({
- type = 'normal',
- name = m['symbol'],
- callback = cb,
- score = 0.0,
- group = 'antivirus'
- })
- rspamd_config:register_symbol({
- type = 'virtual',
- name = m['symbol_fail'],
- parent = id,
- score = 0.0,
- group = 'antivirus'
- })
- has_valid = true
- if type(m['patterns']) == 'table' then
- if m['patterns'][1] then
- for _, p in ipairs(m['patterns']) do
- if type(p) == 'table' then
- for sym in pairs(p) do
- rspamd_logger.debugm(N, rspamd_config, 'registering: %1', {
- type = 'virtual',
- name = sym,
- parent = m['symbol'],
- parent_id = id,
- })
- rspamd_config:register_symbol({
- type = 'virtual',
- name = sym,
- parent = id
- })
- end
- end
- end
- else
- for sym in pairs(m['patterns']) do
- rspamd_config:register_symbol({
- type = 'virtual',
- name = sym,
- parent = id
- })
- end
- end
- end
- if m['score'] then
- -- Register metric symbol
- local description = 'antivirus symbol'
- local group = 'antivirus'
- if m['description'] then
- description = m['description']
- end
- if m['group'] then
- group = m['group']
- end
- rspamd_config:set_metric_symbol({
- name = m['symbol'],
- score = m['score'],
- description = description,
- group = group or 'antivirus'
- })
- end
- end
- end
- end
-
- if not has_valid then
- lua_util.disable_module(N, 'config')
- end
-end
+return exports
\ No newline at end of file
diff --git a/src/plugins/lua/antivirus.lua b/src/plugins/lua/antivirus.lua
index 2aa1f0344..ed3d93e79 100644
--- a/src/plugins/lua/antivirus.lua
+++ b/src/plugins/lua/antivirus.lua
@@ -15,12 +15,10 @@ limitations under the License.
]] --
local rspamd_logger = require "rspamd_logger"
-local rspamd_util = require "rspamd_util"
local rspamd_regexp = require "rspamd_regexp"
-local tcp = require "rspamd_tcp"
-local upstream_list = require "rspamd_upstream_list"
local lua_util = require "lua_util"
local fun = require "fun"
+local lua_antivirus = require "lua_antivirus"
local redis_params
local N = "antivirus"
@@ -70,939 +68,6 @@ antivirus {
return
end
-local default_message = '${SCANNER}: virus found: "${VIRUS}"'
-
-local function match_patterns(default_sym, found, patterns)
- if type(patterns) ~= 'table' then return default_sym end
- if not patterns[1] then
- for sym, pat in pairs(patterns) do
- if pat:match(found) then
- return sym
- end
- end
- return default_sym
- else
- for _, p in ipairs(patterns) do
- for sym, pat in pairs(p) do
- if pat:match(found) then
- return sym
- end
- end
- end
- return default_sym
- end
-end
-
-local function yield_result(task, rule, vname)
- local all_whitelisted = true
- if type(vname) == 'string' then
- local symname = match_patterns(rule['symbol'], vname, rule['patterns'])
- if rule['whitelist'] and rule['whitelist']:get_key(vname) then
- rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule['type'], vname)
- return
- end
- task:insert_result(symname, 1.0, vname)
- rspamd_logger.infox(task, '%s: virus found: "%s"', rule['type'], vname)
- elseif type(vname) == 'table' then
- for _, vn in ipairs(vname) do
- local symname = match_patterns(rule['symbol'], vn, rule['patterns'])
- if rule['whitelist'] and rule['whitelist']:get_key(vn) then
- rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule['type'], vn)
- else
- all_whitelisted = false
- task:insert_result(symname, 1.0, vn)
- rspamd_logger.infox(task, '%s: virus found: "%s"', rule['type'], vn)
- end
- end
- end
- if rule['action'] then
- if type(vname) == 'table' then
- if all_whitelisted then return end
- vname = table.concat(vname, '; ')
- end
- task:set_pre_result(rule['action'],
- lua_util.template(rule.message or 'Rejected', {
- SCANNER = rule['type'],
- VIRUS = vname,
- }), N)
- end
-end
-
-local function clamav_config(opts)
- local clamav_conf = {
- scan_mime_parts = true;
- scan_text_mime = false;
- scan_image_mime = false;
- default_port = 3310,
- log_clean = false,
- timeout = 15.0, -- FIXME: this will break task_timeout!
- retransmits = 2,
- cache_expire = 3600, -- expire redis in one hour
- message = default_message,
- }
-
- for k,v in pairs(opts) do
- clamav_conf[k] = v
- end
-
- if not clamav_conf.prefix then
- clamav_conf.prefix = 'rs_cl'
- end
-
- if not clamav_conf['servers'] then
- rspamd_logger.errx(rspamd_config, 'no servers defined')
-
- return nil
- end
-
- clamav_conf['upstreams'] = upstream_list.create(rspamd_config,
- clamav_conf['servers'],
- clamav_conf.default_port)
-
- if clamav_conf['upstreams'] then
- return clamav_conf
- end
-
- rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
- clamav_conf['servers'])
- return nil
-end
*** OUTPUT TRUNCATED, 871 LINES SKIPPED ***
More information about the Commits
mailing list