commit a767535: [Slashing] Remove ip_score module
Vsevolod Stakhov
vsevolod at highsecure.ru
Wed Jul 17 16:14:06 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-07-17 17:00:28 +0100
URL: https://github.com/rspamd/rspamd/commit/a7675359a1f7dffa91430d69c17a0a0a364c8fc2
[Slashing] Remove ip_score module
---
conf/modules.d/ip_score.conf | 26 ---
src/plugins/lua/ip_score.lua | 426 -------------------------------------------
2 files changed, 452 deletions(-)
diff --git a/conf/modules.d/ip_score.conf b/conf/modules.d/ip_score.conf
deleted file mode 100644
index 065c6ca2e..000000000
--- a/conf/modules.d/ip_score.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-# Please don't modify this file as your changes might be overwritten with
-# the next update.
-#
-# You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
-# parameters defined on the top level
-#
-# You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
-# parameters defined on the top level
-#
-# For specific modules or configuration you can also modify
-# '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
-# '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
-#
-# See https://rspamd.com/doc/tutorials/writing_rules.html for details
-
-ip_score {
-# servers = "localhost";
-# threshold = 100;
-# reject_score = 3;
-# no_action_score = -2;
-# add_header_score = 1;
-# whitelist = "file:///ip_map";
- .include(try=true,priority=5) "${DBDIR}/dynamic/ip_score.conf"
- .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/ip_score.conf"
- .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/ip_score.conf"
-}
diff --git a/src/plugins/lua/ip_score.lua b/src/plugins/lua/ip_score.lua
deleted file mode 100644
index 9416f4e88..000000000
--- a/src/plugins/lua/ip_score.lua
+++ /dev/null
@@ -1,426 +0,0 @@
---[[
-Copyright (c) 2011-2015, Vsevolod Stakhov <vsevolod at highsecure.ru>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-]]--
-
-if confighelp then
- return
-end
-
--- IP score is a module that set ip score of specific ip, asn, country
-local rspamd_logger = require "rspamd_logger"
-local rspamd_util = require "rspamd_util"
-local lua_util = require "lua_util"
-
--- Default settings
-local redis_params = nil
-local whitelist = nil
-local asn_cc_whitelist = nil
-local check_authed = false
-local check_local = false
-local M = "ip_score"
-local N = M
-
-local options = {
- actions = { -- how each action is treated in scoring
- ['reject'] = 1.0,
- ['add header'] = 0.25,
- ['rewrite subject'] = 0.25,
- ['no action'] = 1.0
- },
- scores = { -- how each component is evaluated
- ['asn'] = 0.4,
- ['country'] = 0.01,
- ['ipnet'] = 0.5,
- ['ip'] = 1.0
- },
- symbol = 'IP_SCORE', -- symbol to be inserted
- hash = 'ip_score', -- hash table in redis used for storing scores
- asn_prefix = 'a:', -- prefix for ASN hashes
- country_prefix = 'c:', -- prefix for country hashes
- ipnet_prefix = 'n:', -- prefix for ipnet hashes
- servers = '', -- list of servers
- lower_bound = 10, -- minimum number of messages to be scored
- metric = 'default',
- old_shard = false,
- min_score = nil,
- max_score = nil,
- score_divisor = 1,
-}
-
-local function ip_score_hash_key(asn, country, ipnet, ip)
- if options.old_shard then
- if country then
- return country
- elseif asn then
- return asn
- elseif ipnet then
- return ipnet
- else
- return ip:to_string()
- end
- end
-
- -- Better sharding
- return string.format('%s:%s:%s:%s', asn, country, ipnet,
- ip:to_string())
-end
-
-local function ip_score_get_task_vars(task)
- local pool = task:get_mempool()
- local asn, country, ipnet
- if pool:get_variable("asn") then
- asn = pool:get_variable("asn")
- end
- if pool:get_variable("country") then
- country = pool:get_variable("country")
- end
- if pool:get_variable("ipnet") then
- ipnet = pool:get_variable("ipnet")
- end
-
- return asn, country, ipnet
-end
-
-local function normalize_score(sc, total, mult)
- if total < options['lower_bound'] then
- return 0
- end
- return mult * rspamd_util.tanh(2.718281 * sc / total)
-end
-
--- Set score based on metric's action
-local ip_score_set = function(task)
- if lua_util.is_rspamc_or_controller(task) then return end
- local function new_score_set(score, old_score, old_total)
- local new_total
- if old_total == -1 or old_total ~= old_total then
- new_total = 1
- else
- new_total = old_total + 1
- end
-
- if score ~= score then
- score = 0
- end
-
- return old_score + score, new_total
- end
-
- local score_set_cb = function(err)
- if err then
- rspamd_logger.infox(task, 'got error while IP score changing: %1', err)
- end
- end
-
- local ip = task:get_from_ip()
- if not check_authed and task:get_user() then
- return
- end
- local action = task:get_metric_action(options['metric'])
- if not ip or not ip:is_valid() then
- return
- end
- if not check_local and ip:is_local() then
- return
- end
-
- local pool = task:get_mempool()
- local asn, country, ipnet = ip_score_get_task_vars(task)
-
- if not pool:has_variable('ip_score') or not asn or not country or not ipnet then
- return
- end
-
- local asn_score,total_asn,
- country_score,total_country,
- ipnet_score,total_ipnet,
- ip_score, total_ip = pool:get_variable('ip_score',
- 'double,double,double,double,double,double,double,double')
- lua_util.debugm(M, task,
- 'raw scores: asn: %s, total_asn: %s, country: %s, ' ..
- 'total_country: %s, ipnet: %s, total_ipnet: %s, ip:%s, total_ip: %s',
- asn_score, total_asn,
- country_score, total_country,
- ipnet_score, total_ipnet,
- ip_score, total_ip)
-
- local score_mult = 0
- if options['actions'][action] then
- score_mult = options['actions'][action]
- end
- local score = task:get_metric_score(options['metric'])[1]
- if action == 'no action' and score > 0 then
- score_mult = 0
- end
-
- score = score_mult * rspamd_util.tanh (2.718281 * (score/options['score_divisor']))
-
- local hkey = ip_score_hash_key(asn, country, ipnet, ip)
-
- asn_score,total_asn = new_score_set(score, asn_score, total_asn)
- country_score,total_country = new_score_set(score, country_score, total_country)
- ipnet_score,total_ipnet = new_score_set(score, ipnet_score, total_ipnet)
- ip_score,total_ip = new_score_set(score, ip_score, total_ip)
- lua_util.debugm(M, task,
- 'processed scores: asn: %s, total_asn: %s, country: %s, total_country: %s,' ..
- ' ipnet: %s, total_ipnet: %s, ip:%s, total_ip: %s',
- asn_score,total_asn,
- country_score,total_country,
- ipnet_score,total_ipnet,
- ip_score, total_ip)
- local redis_args = {options['hash'],
- options['asn_prefix'] .. asn, string.format('%f|%d', asn_score, total_asn),
- options['country_prefix'] .. country, string.format('%f|%d', country_score, total_country),
- options['ipnet_prefix'] .. ipnet, string.format('%f|%d', ipnet_score, total_ipnet),
- ip:to_string(), string.format('%f|%d', ip_score, total_ip)}
-
- local ret = rspamd_redis_make_request(task,
- redis_params, -- connect params
- hkey, -- hash key
- true, -- is write
- score_set_cb, --callback
- 'HMSET', -- command
- redis_args -- arguments
- )
- if not ret then
- rspamd_logger.errx(task, 'error connecting to redis')
- return
- end
-
- -- Now insert final result
- asn_score = normalize_score(asn_score, total_asn, options['scores']['asn'])
- country_score = normalize_score(country_score, total_country,
- options['scores']['country'])
- ipnet_score = normalize_score(ipnet_score, total_ipnet,
- options['scores']['ipnet'])
- ip_score = normalize_score(ip_score, total_ip, options['scores']['ip'])
-
- local total_score = 0.0
- local description_t = {}
-
- if ip_score ~= 0 then
- total_score = total_score + ip_score
- table.insert(description_t,
- 'ip: ' .. '(' .. string.format('%.2f', ip_score * 10) .. ')')
- end
- if ipnet_score ~= 0 then
- total_score = total_score + ipnet_score
- table.insert(description_t,
- 'ipnet: ' .. ipnet .. '(' .. string.format('%.2f', ipnet_score * 10) .. ')')
- end
- if asn_score ~= 0 then
- total_score = total_score + asn_score
- table.insert(description_t,
- 'asn: ' .. asn .. '(' .. string.format('%.2f', asn_score * 10) .. ')')
- end
- if country_score ~= 0 then
- total_score = total_score + country_score
- table.insert(description_t,
- 'country: ' .. country .. '(' .. string.format('%.2f', country_score * 10) .. ')')
- end
-
- if options['max_score'] and (total_score*10) > options['max_score'] then
- total_score = options['max_score']/10
- end
- if options['min_score'] and (total_score*10) < options['min_score'] then
- total_score = options['min_score']/10
- end
-
- if total_score ~= 0 then
- task:insert_result(options['symbol'], total_score, table.concat(description_t, ', '))
- end
-end
-
--- Check score for ip in keystorage
-local ip_score_check = function(task)
- local asn, country, ipnet = ip_score_get_task_vars(task)
- local ip = task:get_from_ip()
-
- local ip_score_redis_cb = function(err, data)
- if err then
- rspamd_logger.errx(task, 'Redis error: %s', err)
- -- XXX: upstreams
- end
- local function calculate_score(score)
- local parts = lua_util.rspamd_str_split(score, '|')
- local rep = tonumber(parts[1])
- local total = tonumber(parts[2])
-
- if rep ~= rep then rep = 0 end
- if total ~= total then total = 0 end
-
- return rep, total
- end
-
- if err then
- -- Key is not found or error occurred
- return
- elseif data then
- -- Scores and total number of messages per bucket
- local asn_score,total_asn,
- country_score,total_country,
- ipnet_score,total_ipnet,
- ip_score, total_ip = 0, -1, 0, -1, 0, -1, 0, -1
- if data[1] and type(data[1]) ~= 'userdata' then
- asn_score,total_asn = calculate_score(data[1])
- end
- if data[2] and type(data[2]) ~= 'userdata' then
- country_score,total_country = calculate_score(data[2])
- end
- if data[3] and type(data[3]) ~= 'userdata' then
- ipnet_score,total_ipnet = calculate_score(data[3])
- end
- if data[4] and type(data[4]) ~= 'userdata' then
- ip_score,total_ip = calculate_score(data[4])
- end
- -- Save everything for the post filter
- task:get_mempool():set_variable('ip_score', asn_score,total_asn,
- country_score,total_country,
- ipnet_score,total_ipnet,
- ip_score, total_ip)
- end
- end
-
- local function create_get_command()
- local cmd = 'HMGET'
-
- local args = {options['hash']}
-
- if asn then
- table.insert(args, options['asn_prefix'] .. asn)
- else
- -- fake arg
- table.insert(args, options['asn_prefix'])
- end
- if country then
- table.insert(args, options['country_prefix'] .. country)
- else
- -- fake arg
- table.insert(args, options['country_prefix'])
- end
- if ipnet then
- table.insert(args, options['ipnet_prefix'] .. ipnet)
- else
- -- fake arg
- table.insert(args, options['ipnet_prefix'])
- end
-
- table.insert(args, ip:to_string())
-
- return cmd, args
- end
-
- if task:get_user() and not check_authed then
- rspamd_logger.infox(task, "skip IP Score for authorized users")
- return
- end
- if ip and ip:is_local() and not check_local then
- rspamd_logger.infox(task, "skip IP Score for local networks")
- return
- end
- if ip:is_valid() then
- -- Check IP whitelist
- if whitelist then
- if whitelist:get_key(task:get_from_ip()) then
- -- Address is whitelisted
- return
- end
- end
- -- Check ASN & country whitelist
- if asn_cc_whitelist then
- if asn_cc_whitelist:get_key(country) then
- return
- end
- if asn_cc_whitelist:get_key(asn) then
- return
- end
- end
-
- local cmd, args = create_get_command()
-
- local ret = rspamd_redis_make_request(task,
- redis_params, -- connect params
- ip_score_hash_key(asn, country, ipnet, ip), -- hash key
- false, -- is write
- ip_score_redis_cb, --callback
- cmd, -- command
- args -- arguments
- )
- if not ret then
- rspamd_logger.errx(task, 'error connecting to redis')
- end
- end
-end
-
-local function try_opts(where)
- local ret = false
- local opts = rspamd_config:get_all_opt(where)
- if type(opts) == 'table' then
- if type(opts['check_local']) == 'boolean' then
- check_local = opts['check_local']
- ret = true
- end
- if type(opts['check_authed']) == 'boolean' then
- check_authed = opts['check_authed']
- ret = true
- end
- end
-
- return ret
-end
-
-if not try_opts(N) then try_opts('options') end
-
--- Configuration options
-local configure_ip_score_module = function()
- local opts = rspamd_config:get_all_opt(N)
-
- if not opts then return end
- for k,v in pairs(opts) do
- options[k] = v
- end
- redis_params = rspamd_parse_redis_server('ip_score')
- if not redis_params then
- rspamd_logger.infox(rspamd_config, 'no servers are specified')
- return
- end
- asn_cc_whitelist = rspamd_map_add('ip_score', 'asn_cc_whitelist', 'map',
- 'IP score whitelisted ASNs/countries')
- whitelist = rspamd_map_add('ip_score', 'whitelist', 'radix',
- 'IP score whitelisted ips')
- return true
-end
-
-
-if not configure_ip_score_module() then return end
-if redis_params then
- -- Register ip_score module
- rspamd_config:register_symbol({
- name = 'IPSCORE_SAVE',
- type = 'postfilter,nostat',
- priority = 5,
- callback = ip_score_set,
- flags = 'empty',
- })
- rspamd_config:register_symbol({
- name = options['symbol'],
- callback = ip_score_check,
- group = 'reputation',
- score = 2.0,
- flags = 'empty',
- })
-else
- lua_util.disable_module(N, "redis")
-end
More information about the Commits
mailing list