commit 299c314: [Feature] Antivirus: Add preliminary virustotal support
Vsevolod Stakhov
vsevolod at highsecure.ru
Thu Oct 31 16:00:14 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-10-31 15:56:25 +0000
URL: https://github.com/rspamd/rspamd/commit/299c314a12f80f3637d13fbb53c2db1715c57d63 (HEAD -> master)
[Feature] Antivirus: Add preliminary virustotal support
Issue: #3109
---
lualib/lua_scanners/init.lua | 1 +
lualib/lua_scanners/virustotal.lua | 191 +++++++++++++++++++++++++++++++++++++
src/plugins/lua/antivirus.lua | 10 +-
3 files changed, 198 insertions(+), 4 deletions(-)
diff --git a/lualib/lua_scanners/init.lua b/lualib/lua_scanners/init.lua
index b92ba45d9..4c369bb8b 100644
--- a/lualib/lua_scanners/init.lua
+++ b/lualib/lua_scanners/init.lua
@@ -37,6 +37,7 @@ require_scanner('kaspersky_av')
require_scanner('kaspersky_se')
require_scanner('savapi')
require_scanner('sophos')
+require_scanner('virustotal')
-- Other scanners
require_scanner('dcc')
diff --git a/lualib/lua_scanners/virustotal.lua b/lualib/lua_scanners/virustotal.lua
new file mode 100644
index 000000000..9d06f9108
--- /dev/null
+++ b/lualib/lua_scanners/virustotal.lua
@@ -0,0 +1,191 @@
+--[[
+Copyright (c) 2019, Vsevolod Stakhov <vsevolod at highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module virustotal
+-- This module contains Virustotal integaration support
+-- https://www.virustotal.com/
+--]]
+
+local lua_util = require "lua_util"
+local http = require "rspamd_http"
+local rspamd_cryptobox_hash = require "rspamd_cryptobox_hash"
+local rspamd_logger = require "rspamd_logger"
+local common = require "lua_scanners/common"
+
+local N = 'virustotal'
+
+local function virustotal_config(opts)
+
+ local default_conf = {
+ name = N,
+ url = 'https://www.virustotal.com/vtapi/v2/file',
+ timeout = 5.0,
+ log_clean = false,
+ retransmits = 1,
+ cache_expire = 7200, -- expire redis in 2h
+ message = '${SCANNER}: spam message found: "${VIRUS}"',
+ detection_category = "virus",
+ default_score = 1,
+ action = false,
+ scan_mime_parts = true,
+ scan_text_mime = false,
+ scan_image_mime = false,
+ apikey = nil, -- Required to set by user
+ -- Specific for virustotal
+ minimum_engines = 3, -- Minimum required to get scored
+ full_score_engines = 7, -- After this number we set max score
+ }
+
+ default_conf = lua_util.override_defaults(default_conf, opts)
+
+ if not default_conf.prefix then
+ default_conf.prefix = 'rs_' .. default_conf.name .. '_'
+ end
+
+ if not default_conf.log_prefix then
+ if default_conf.name:lower() == default_conf.type:lower() then
+ default_conf.log_prefix = default_conf.name
+ else
+ default_conf.log_prefix = default_conf.name .. ' (' .. default_conf.type .. ')'
+ end
+ end
+
+ if not default_conf.apikey then
+ rspamd_logger.errx(rspamd_config, 'no apikey defined for virustotal, disable checks')
+
+ return nil
+ end
+
+ lua_util.add_debug_alias('external_services', default_conf.name)
+ return default_conf
+end
+
+local function virustotal_check(task, content, digest, rule)
+ local function virustotal_check_uncached()
+ local function make_url(hash)
+ return string.format('%s/report?apikey=%s&resource=%s',
+ rule.url, rule.apikey, hash)
+ end
+
+ local hash = rspamd_cryptobox_hash.create_specific('md5')
+ hash:update(content)
+ hash = hash:hex()
+
+ local url = make_url(hash)
+ lua_util.debugm(N, task, "send request %s", url)
+ local request_data = {
+ task = task,
+ url = url,
+ timeout = rule.timeout,
+ }
+
+ local function vt_http_callback(http_err, code, body, headers)
+ if http_err then
+ rspamd_logger.errx(task, 'HTTP error: %s, body: %s, headers: %s', http_err, body, headers)
+ else
+ local cached
+ -- Parse the response
+ if code ~= 200 then
+ if code == 404 then
+ cached = 'OK'
+ if rule['log_clean'] then
+ rspamd_logger.infox(task, '%s: hash %s clean (not found)',
+ rule.log_prefix, hash)
+ else
+ lua_util.debugm(rule.name, task, '%s: hash %s clean (not found)',
+ rule.log_prefix)
+ end
+ else
+ rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
+ return
+ end
+ else
+ local ucl = require "ucl"
+ local parser = ucl.parser()
+ local res,json_err = parser:parse_string(body)
+
+ lua_util.debugm(rule.name, task, '%s: got reply data: "%s"',
+ rule.log_prefix, body)
+
+ if res then
+ local obj = parser:get_object()
+ if not obj.positives then
+ rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
+ 'no positives element', body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: no `positives` element')
+ return
+ end
+ if obj.positives < rule.minimum_engines then
+ lua_util.debugm(rule.name, task, '%s: hash %s has not enough hits: %s where %s is min',
+ rule.log_prefix, obj.positives, rule.minimum_engines)
+ -- TODO: add proper hashing!
+ cached = 'OK'
+ else
+ local dyn_score
+ if obj.positives > rule.full_score_engines then
+ dyn_score = 1.0
+ else
+ local norm_pos = obj.positives - rule.minimum_engines
+ dyn_score = norm_pos / (rule.full_score_engines - rule.minimum_engines)
+ end
+
+ if dyn_score < 0 or dyn_score > 1 then
+ dyn_score = 1.0
+ end
+ common.yield_result(task, rule, {
+ hash,
+ string.format("%s/%s", obj.positives, obj.total)
+ }, dyn_score)
+ cached = hash
+ end
+ else
+ rspamd_logger.errx(task, 'invalid JSON reply: %s, body: %s, headers: %s',
+ json_err, body, headers)
+ task:insert_result(rule.symbol_fail, 1.0, 'Bad JSON reply: ' .. json_err)
+ return
+ end
+
+ end
+
+ if cached then
+ common.save_cache(task, digest, rule, cached)
+ end
+ end
+ end
+
+ request_data.callback = vt_http_callback
+ http.request(request_data)
+ end
+
+ if common.condition_check_and_continue(task, content, rule, digest,
+ virustotal_check_uncached) then
+ return
+ else
+
+ virustotal_check_uncached()
+ end
+
+end
+
+return {
+ type = 'antivirus',
+ description = 'Virustotal integration',
+ configure = virustotal_config,
+ check = virustotal_check,
+ name = N
+}
diff --git a/src/plugins/lua/antivirus.lua b/src/plugins/lua/antivirus.lua
index 4c89526a5..34b0c6947 100644
--- a/src/plugins/lua/antivirus.lua
+++ b/src/plugins/lua/antivirus.lua
@@ -98,11 +98,13 @@ local function add_antivirus_rule(sym, opts)
if opts.attachments_only ~= nil then
opts.scan_mime_parts = opts.attachments_only
rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. '..
- 'Please use scan_mime_parts = %s instead', opts.symbol, opts.type, opts.attachments_only)
+ 'Please use scan_mime_parts = %s instead', opts.symbol, opts.type, opts.attachments_only)
end
-- WORKAROUND for deprecated attachments_only
local rule = cfg.configure(opts)
+ if not rule then return nil end
+
rule.type = opts.type
rule.symbol_fail = opts.symbol_fail
rule.symbol_encrypted = opts.symbol_encrypted
@@ -110,7 +112,7 @@ local function add_antivirus_rule(sym, opts)
if not rule then
rspamd_logger.errx(rspamd_config, 'cannot configure %s for %s',
- opts.type, opts.symbol)
+ opts.type, opts.symbol)
return nil
end
@@ -143,7 +145,7 @@ if opts and type(opts) == 'table' then
redis_params = rspamd_parse_redis_server(N)
local has_valid = false
for k, m in pairs(opts) do
- if type(m) == 'table' and m.servers then
+ if type(m) == 'table' then
if not m.type then m.type = k end
if not m.name then m.name = k end
local cb = add_antivirus_rule(k, m)
@@ -151,7 +153,7 @@ if opts and type(opts) == 'table' then
if not cb then
rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"')
else
-
+ rspamd_logger.infox(rspamd_config, 'added antivirus engine %s -> %s', k, m.symbol)
local t = {
name = m.symbol,
callback = cb,
More information about the Commits
mailing list