commit 434255a: [Rework] Lua_scanners: Further library split
Vsevolod Stakhov
vsevolod at highsecure.ru
Thu Dec 27 18:28:14 UTC 2018
Author: Vsevolod Stakhov
Date: 2018-12-26 14:59:14 +0000
URL: https://github.com/rspamd/rspamd/commit/434255a2a71e508d1e1de2bda368572d62be7e3e
[Rework] Lua_scanners: Further library split
---
lualib/lua_scanners/clamav.lua | 167 ++++++
lualib/lua_scanners/common.lua | 199 +++++++
lualib/lua_scanners/fprot.lua | 171 ++++++
lualib/lua_scanners/init.lua | 31 +-
lualib/lua_scanners/kaspersky_av.lua | 188 +++++++
lualib/lua_scanners/lua_antivirus.lua | 986 ----------------------------------
lualib/lua_scanners/savapi.lua | 252 +++++++++
lualib/lua_scanners/sophos.lua | 187 +++++++
src/plugins/lua/antivirus.lua | 4 +-
9 files changed, 1196 insertions(+), 989 deletions(-)
diff --git a/lualib/lua_scanners/clamav.lua b/lualib/lua_scanners/clamav.lua
new file mode 100644
index 000000000..26d5e9c81
--- /dev/null
+++ b/lualib/lua_scanners/clamav.lua
@@ -0,0 +1,167 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod at highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module clamav
+-- This module contains clamav access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "antivirus"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function clamav_config(opts)
+ local clamav_conf = {
+ scan_mime_parts = true;
+ scan_text_mime = false;
+ scan_image_mime = false;
+ default_port = 3310,
+ log_clean = false,
+ timeout = 15.0, -- FIXME: this will break task_timeout!
+ retransmits = 2,
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ }
+
+ for k,v in pairs(opts) do
+ clamav_conf[k] = v
+ end
+
+ if not clamav_conf.prefix then
+ clamav_conf.prefix = 'rs_cl'
+ end
+
+ if not clamav_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ clamav_conf['upstreams'] = upstream_list.create(rspamd_config,
+ clamav_conf['servers'],
+ clamav_conf.default_port)
+
+ if clamav_conf['upstreams'] then
+ return clamav_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ clamav_conf['servers'])
+ return nil
+end
+
+local function clamav_check(task, content, digest, rule)
+ local function clamav_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local header = rspamd_util.pack("c9 c1 >I4", "zINSTREAM", "\0",
+ #content)
+ local footer = rspamd_util.pack(">I4", 0)
+
+ local function clamav_callback(err, data)
+ if err then
+
+ -- set current upstream to fail because an error occurred
+ upstream:fail()
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule['timeout'],
+ callback = clamav_callback,
+ data = { header, content, footer },
+ stop_pattern = '\0'
+ })
+ else
+ rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and retransmits exceed')
+ end
+
+ else
+ upstream:ok()
+ data = tostring(data)
+ local cached
+ lua_util.debugm(N, task, '%s [%s]: got reply: %s', rule['symbol'], rule['type'], data)
+ if data == 'stream: OK' then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+ else
+ lua_util.debugm(N, task, '%s [%s]: message or mime_part is clean', rule['symbol'], rule['type'])
+ end
+ else
+ local vname = string.match(data, 'stream: (.+) FOUND')
+ if vname then
+ common.yield_result(task, rule, vname, N)
+ cached = vname
+ else
+ rspamd_logger.errx(task, 'unhandled response: %s', data)
+ task:insert_result(rule['symbol_fail'], 0.0, 'unhandled response')
+ end
+ end
+ if cached then
+ common.save_av_cache(task, digest, rule, cached, N)
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule['timeout'],
+ callback = clamav_callback,
+ data = { header, content, footer },
+ stop_pattern = '\0'
+ })
+ end
+
+ if common.need_av_check(task, content, rule) then
+ if common.check_av_cache(task, digest, rule, clamav_check_uncached, N) then
+ return
+ else
+ clamav_check_uncached()
+ end
+ end
+end
+
+return {
+ type = 'antivirus',
+ description = 'clamav antivirus',
+ configure = clamav_config,
+ check = clamav_check,
+ name = 'clamav'
+}
\ No newline at end of file
diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua
new file mode 100644
index 000000000..ad99137a2
--- /dev/null
+++ b/lualib/lua_scanners/common.lua
@@ -0,0 +1,199 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod at highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module lua_scanners_common
+-- This module contains common external scanners functions
+--]]
+
+local rspamd_logger = require "rspamd_logger"
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+
+local exports = {}
+
+local function match_patterns(default_sym, found, patterns)
+ if type(patterns) ~= 'table' then return default_sym end
+ if not patterns[1] then
+ for sym, pat in pairs(patterns) do
+ if pat:match(found) then
+ return sym
+ end
+ end
+ return default_sym
+ else
+ for _, p in ipairs(patterns) do
+ for sym, pat in pairs(p) do
+ if pat:match(found) then
+ return sym
+ end
+ end
+ end
+ return default_sym
+ end
+end
+
+local function yield_result(task, rule, vname, N)
+ local all_whitelisted = true
+ if type(vname) == 'string' then
+ local symname = match_patterns(rule['symbol'], vname, rule['patterns'])
+ if rule['whitelist'] and rule['whitelist']:get_key(vname) then
+ rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule['type'], vname)
+ return
+ end
+ task:insert_result(symname, 1.0, vname)
+ rspamd_logger.infox(task, '%s: virus found: "%s"', rule['type'], vname)
+ elseif type(vname) == 'table' then
+ for _, vn in ipairs(vname) do
+ local symname = match_patterns(rule['symbol'], vn, rule['patterns'])
+ if rule['whitelist'] and rule['whitelist']:get_key(vn) then
+ rspamd_logger.infox(task, '%s: "%s" is in whitelist', rule['type'], vn)
+ else
+ all_whitelisted = false
+ task:insert_result(symname, 1.0, vn)
+ rspamd_logger.infox(task, '%s: virus found: "%s"', rule['type'], vn)
+ end
+ end
+ end
+ if rule['action'] then
+ if type(vname) == 'table' then
+ if all_whitelisted then return end
+ vname = table.concat(vname, '; ')
+ end
+ task:set_pre_result(rule['action'],
+ lua_util.template(rule.message or 'Rejected', {
+ SCANNER = rule['type'],
+ VIRUS = vname,
+ }), N)
+ end
+end
+
+local function message_not_too_large(task, content, rule)
+ local max_size = tonumber(rule.max_size)
+ if not max_size then return true end
+ if #content > max_size then
+ rspamd_logger.infox(task, "skip %s AV check as it is too large: %s (%s is allowed)",
+ rule.type, #content, max_size)
+ return false
+ end
+ return true
+end
+
+local function need_av_check(task, content, rule)
+ return message_not_too_large(task, content, rule)
+end
+
+local function check_av_cache(task, digest, rule, fn, N)
+ local key = digest
+
+ local function redis_av_cb(err, data)
+ if data and type(data) == 'string' then
+ -- Cached
+ if data ~= 'OK' then
+ lua_util.debugm(N, task, 'got cached result for %s: %s',
+ key, data)
+ data = lua_util.str_split(data, '\v')
+ yield_result(task, rule, data, N)
+ else
+ lua_util.debugm(N, task, 'got cached result for %s: %s',
+ key, data)
+ end
+ else
+ if err then
+ rspamd_logger.errx(task, 'got error checking cache: %s', err)
+ end
+ fn()
+ end
+ end
+
+ if rule.redis_params then
+
+ key = rule['prefix'] .. key
+
+ if lua_redis.redis_make_request(task,
+ rule.redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_av_cb, --callback
+ 'GET', -- command
+ {key} -- arguments)
+ ) then
+ return true
+ end
+ end
+
+ return false
+end
+
+local function save_av_cache(task, digest, rule, to_save, N)
+ local key = digest
+
+ local function redis_set_cb(err)
+ -- Do nothing
+ if err then
+ rspamd_logger.errx(task, 'failed to save virus cache for %s -> "%s": %s',
+ to_save, key, err)
+ else
+ lua_util.debugm(N, task, 'saved cached result for %s: %s',
+ key, to_save)
+ end
+ end
+
+ if type(to_save) == 'table' then
+ to_save = table.concat(to_save, '\v')
+ end
+
+ if rule.redis_params then
+ key = rule['prefix'] .. key
+
+ lua_redis.redis_make_request(task,
+ rule.redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'SETEX', -- command
+ { key, rule['cache_expire'], to_save }
+ )
+ end
+
+ return false
+end
+
+exports.yield_result = yield_result
+exports.match_patterns = match_patterns
+exports.need_av_check = need_av_check
+exports.check_av_cache = check_av_cache
+exports.save_av_cache = save_av_cache
+
+setmetatable(exports, {
+ __call = function(t, override)
+ for k, v in pairs(t) do
+ if _G[k] ~= nil then
+ local msg = 'function ' .. k .. ' already exists in global scope.'
+ if override then
+ _G[k] = v
+ print('WARNING: ' .. msg .. ' Overwritten.')
+ else
+ print('NOTICE: ' .. msg .. ' Skipped.')
+ end
+ else
+ _G[k] = v
+ end
+ end
+ end,
+})
+
+return exports
\ No newline at end of file
diff --git a/lualib/lua_scanners/fprot.lua b/lualib/lua_scanners/fprot.lua
new file mode 100644
index 000000000..d52af8fea
--- /dev/null
+++ b/lualib/lua_scanners/fprot.lua
@@ -0,0 +1,171 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod at highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module fprot
+-- This module contains fprot access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "antivirus"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function fprot_config(opts)
+ local fprot_conf = {
+ scan_mime_parts = true;
+ scan_text_mime = false;
+ scan_image_mime = false;
+ default_port = 10200,
+ timeout = 15.0, -- FIXME: this will break task_timeout!
+ log_clean = false,
+ retransmits = 2,
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ }
+
+ for k,v in pairs(opts) do
+ fprot_conf[k] = v
+ end
+
+ if not fprot_conf.prefix then
+ fprot_conf.prefix = 'rs_fp'
+ end
+
+ if not fprot_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ fprot_conf['upstreams'] = upstream_list.create(rspamd_config,
+ fprot_conf['servers'],
+ fprot_conf.default_port)
+
+ if fprot_conf['upstreams'] then
+ return fprot_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ fprot_conf['servers'])
+ return nil
+end
+
+local function fprot_check(task, content, digest, rule)
+ local function fprot_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local scan_id = task:get_queue_id()
+ if not scan_id then scan_id = task:get_uid() end
+ local header = string.format('SCAN STREAM %s SIZE %d\n', scan_id,
+ #content)
+ local footer = '\n'
+
+ local function fprot_callback(err, data)
+ if err then
+ -- set current upstream to fail because an error occurred
+ upstream:fail()
+
+ -- retry with another upstream until retransmits exceeds
+ if retransmits > 0 then
+
+ retransmits = retransmits - 1
+
+ -- Select a different upstream!
+ upstream = rule.upstreams:get_upstream_round_robin()
+ addr = upstream:get_addr()
+
+ lua_util.debugm(N, task, '%s [%s]: retry IP: %s', rule['symbol'], rule['type'], addr)
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule['timeout'],
+ callback = fprot_callback,
+ data = { header, content, footer },
+ stop_pattern = '\n'
+ })
+ else
+ rspamd_logger.errx(task,
+ '%s [%s]: failed to scan, maximum retransmits exceed',
+ rule['symbol'], rule['type'])
+ task:insert_result(rule['symbol_fail'], 0.0,
+ 'failed to scan and retransmits exceed')
+ end
+ else
+ upstream:ok()
+ data = tostring(data)
+ local cached
+ local clean = string.match(data, '^0 <clean>')
+ if clean then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task,
+ '%s [%s]: message or mime_part is clean',
+ rule['symbol'], rule['type'])
+ end
+ else
+ -- returncodes: 1: infected, 2: suspicious, 3: both, 4-255: some error occured
+ -- see http://www.f-prot.com/support/helpfiles/unix/appendix_c.html for more detail
+ local vname = string.match(data, '^[1-3] <[%w%s]-: (.-)>')
+ if not vname then
+ rspamd_logger.errx(task, 'Unhandled response: %s', data)
+ else
+ common.yield_result(task, rule, vname, N)
+ cached = vname
+ end
+ end
+ if cached then
+ common.save_av_cache(task, digest, rule, cached, N)
+ end
+ end
+ end
+
+ tcp.request({
+ task = task,
+ host = addr:to_string(),
+ port = addr:get_port(),
+ timeout = rule['timeout'],
+ callback = fprot_callback,
+ data = { header, content, footer },
+ stop_pattern = '\n'
+ })
+ end
+
+ if common.need_av_check(task, content, rule) then
+ if common.check_av_cache(task, digest, rule, fprot_check_uncached, N) then
+ return
+ else
+ fprot_check_uncached()
+ end
+ end
+end
+
+return {
+ type = 'antivirus',
+ description = 'fprot antivirus',
+ configure = fprot_config,
+ check = fprot_check,
+ name = 'fprot'
+}
\ No newline at end of file
diff --git a/lualib/lua_scanners/init.lua b/lualib/lua_scanners/init.lua
index 9937fd73f..149402874 100644
--- a/lualib/lua_scanners/init.lua
+++ b/lualib/lua_scanners/init.lua
@@ -19,8 +19,37 @@ limitations under the License.
-- This module contains external scanners functions
--]]
+local fun = require "fun"
+
local exports = {
- antivirus = require "lua_scanners/lua_antivirus",
}
+local function require_scanner(name)
+ local sc = require ("lua_scanners/" .. name)
+
+ exports[sc.name or name] = sc
+end
+
+require_scanner('clamav')
+require_scanner('fprot')
+require_scanner('kaspersky_av')
+require_scanner('savapi')
+require_scanner('sophos')
+
+exports.add_scanner = function(name, t, conf_func, check_func)
+ assert(type(conf_func) == 'function' and type(check_func) == 'function',
+ 'bad arguments')
+ exports[name] = {
+ type = t,
+ configure = conf_func,
+ check = check_func,
+ }
+end
+
+exports.filter = function(t)
+ return fun.tomap(fun.filter(function(_, elt)
+ return type(elt) == 'table' and elt.type and elt.type == t
+ end, exports))
+end
+
return exports
\ No newline at end of file
diff --git a/lualib/lua_scanners/kaspersky_av.lua b/lualib/lua_scanners/kaspersky_av.lua
new file mode 100644
index 000000000..b55b6c24c
--- /dev/null
+++ b/lualib/lua_scanners/kaspersky_av.lua
@@ -0,0 +1,188 @@
+--[[
+Copyright (c) 2018, Vsevolod Stakhov <vsevolod at highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module kaspersky
+-- This module contains kaspersky antivirus access functions
+--]]
+
+local lua_util = require "lua_util"
+local tcp = require "rspamd_tcp"
+local upstream_list = require "rspamd_upstream_list"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = "antivirus"
+
+local default_message = '${SCANNER}: virus found: "${VIRUS}"'
+
+local function kaspersky_config(opts)
+ local kaspersky_conf = {
+ scan_mime_parts = true;
+ scan_text_mime = false;
+ scan_image_mime = false;
+ product_id = 0,
+ log_clean = false,
+ timeout = 5.0,
+ retransmits = 1, -- use local files, retransmits are useless
+ cache_expire = 3600, -- expire redis in one hour
+ message = default_message,
+ tmpdir = '/tmp',
+ prefix = 'rs_ak',
+ }
+
+ kaspersky_conf = lua_util.override_defaults(kaspersky_conf, opts)
+
+ if not kaspersky_conf['servers'] then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+
+ return nil
+ end
+
+ kaspersky_conf['upstreams'] = upstream_list.create(rspamd_config,
+ kaspersky_conf['servers'], 0)
+
+ if kaspersky_conf['upstreams'] then
+ return kaspersky_conf
+ end
+
+ rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
+ kaspersky_conf['servers'])
+ return nil
+end
+
+local function kaspersky_check(task, content, digest, rule)
+ local function kaspersky_check_uncached ()
+ local upstream = rule.upstreams:get_upstream_round_robin()
+ local addr = upstream:get_addr()
+ local retransmits = rule.retransmits
+ local fname = string.format('%s/%s.tmp',
+ rule.tmpdir, rspamd_util.random_hex(32))
*** OUTPUT TRUNCATED, 1583 LINES SKIPPED ***
More information about the Commits
mailing list