commit e4e8e67: [Feature] Add p0f scanner
denpamusic
denpa at netfleet.space
Tue Sep 17 12:21:08 UTC 2019
Author: denpamusic
Date: 2019-09-15 23:15:44 +0300
URL: https://github.com/rspamd/rspamd/commit/e4e8e675b610b49975c8b90d1d207f6f56ac6f93 (refs/pull/3037/head)
[Feature] Add p0f scanner
---
conf/modules.d/{asn.conf => p0f.conf} | 40 +++++--
lualib/lua_scanners/init.lua | 1 +
lualib/lua_scanners/p0f.lua | 210 ++++++++++++++++++++++++++++++++++
src/plugins/lua/milter_headers.lua | 24 ++++
src/plugins/lua/p0f.lua | 117 +++++++++++++++++++
test/functional/cases/161_p0f.robot | 88 ++++++++++++++
test/functional/configs/p0f.conf | 11 ++
test/functional/lib/vars.py | 1 +
test/functional/util/dummy_p0f.py | 98 ++++++++++++++++
9 files changed, 578 insertions(+), 12 deletions(-)
diff --git a/conf/modules.d/asn.conf b/conf/modules.d/p0f.conf
similarity index 51%
copy from conf/modules.d/asn.conf
copy to conf/modules.d/p0f.conf
index 955e6ba33..efeab1a40 100644
--- a/conf/modules.d/asn.conf
+++ b/conf/modules.d/p0f.conf
@@ -13,18 +13,34 @@
#
# See https://rspamd.com/doc/tutorials/writing_rules.html for details
-asn {
- # Provider: just "rspamd" for now
- provider_type = "rspamd";
- # Provider-specific configuration
- provider_info {
- ip4 = "asn.rspamd.com";
- ip6 = "asn6.rspamd.com";
- }
+p0f {
+ # Disable module by default
+ enabled = false;
+
+ # Path to the unix socket that p0f listens on
+ socket = '/tmp/p0f.sock';
+
+ # Connection timeout
+ timeout = 5s;
+
# If defined, insert symbol with lookup results
- # symbol = "ASN";
+ symbol = 'P0F';
+
+ # Patterns to match against results returned by p0f
+ # Symbol will be yielded on OS string, link type or distance matches
+ patterns = {
+ WINDOWS = '^Windows.*';
+ #DSL = '^DSL$';
+ #DISTANCE10 = '^distance:10$';
+ }
+
+ # Cache lifetime in seconds (default - 2 hours)
+ expire = 7200;
+
+ # Cache key prefix
+ prefix = 'p0f';
- .include(try=true,priority=5) "${DBDIR}/dynamic/asn.conf"
- .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/asn.conf"
- .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/asn.conf"
+ .include(try=true,priority=5) "${DBDIR}/dynamic/p0f.conf"
+ .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/p0f.conf"
+ .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/p0f.conf"
}
diff --git a/lualib/lua_scanners/init.lua b/lualib/lua_scanners/init.lua
index 99cec68b3..4a10dc51b 100644
--- a/lualib/lua_scanners/init.lua
+++ b/lualib/lua_scanners/init.lua
@@ -43,6 +43,7 @@ require_scanner('oletools')
require_scanner('icap')
require_scanner('vadesecure')
require_scanner('spamassassin')
+require_scanner('p0f')
exports.add_scanner = function(name, t, conf_func, check_func)
assert(type(conf_func) == 'function' and type(check_func) == 'function',
diff --git a/lualib/lua_scanners/p0f.lua b/lualib/lua_scanners/p0f.lua
new file mode 100644
index 000000000..72093577b
--- /dev/null
+++ b/lualib/lua_scanners/p0f.lua
@@ -0,0 +1,210 @@
+--[[
+Copyright (c) 2019, Vsevolod Stakhov <vsevolod at highsecure.ru>
+Copyright (c) 2019, Denis Paavilainen <denpa at denpa.pro>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+--[[[
+-- @module p0f
+-- This module contains p0f access functions
+--]]
+
+local tcp = require "rspamd_tcp"
+local rspamd_util = require "rspamd_util"
+local rspamd_logger = require "rspamd_logger"
+local lua_redis = require "lua_redis"
+local lua_util = require "lua_util"
+local common = require "lua_scanners/common"
+
+-- SEE: https://github.com/p0f/p0f/blob/v3.06b/docs/README#L317
+local S = {
+ BAD_QUERY = 0x0,
+ OK = 0x10,
+ NO_MATCH = 0x20
+}
+
+local N = 'p0f'
+
+local function p0f_check(task, ip, rule)
+
+ local function ip2bin(addr)
+ addr = addr:to_table()
+
+ for k, v in ipairs(addr) do
+ addr[k] = rspamd_util.pack('B', v)
+ end
+
+ return table.concat(addr)
+ end
+
+ local function trim(...)
+ local vars = {...}
+
+ for k in pairs(vars) do
+ -- skip numbers, trim only strings
+ if tonumber(vars[k]) == nil then
+ vars[k] = string.gsub(vars[k], '[^%w-_\\.\\(\\) ]', '')
+ end
+ end
+
+ return lua_util.unpack(vars)
+ end
+
+ local function parse_p0f_response(data)
+ --[[
+ p0f_api_response[232]: magic, status, first_seen, last_seen, total_conn,
+ uptime_min, up_mod_days, last_nat, last_chg, distance, bad_sw, os_match_q,
+ os_name, os_flavor, http_name, http_flavor, link_type, language
+ ]]--
+
+ data = tostring(data)
+
+ -- API response must be 232 bytes long
+ if (#data < 232) then
+ rspamd_logger.errx(task, 'malformed response from p0f on %s, %s bytes',
+ rule.socket, #data)
+
+ common.yield_result(task, rule, 'Malformed Response: ' .. rule.socket,
+ 0.0, 'fail')
+ return
+ end
+
+ local _, status, _, _, _, uptime_min, _, _, _, distance, _, _, os_name,
+ os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack(
+ 'I4I4I4I4I4I4I4I4I4hbbc32c32c32c32c32c32', data))
+
+ if status ~= S.OK then
+ if status == S.BAD_QUERY then
+ rspamd_logger.errx(task, 'malformed p0f query on %s', rule.socket)
+ common.yield_result(task, rule, 'Malformed Query: ' .. rule.socket,
+ 0.0, 'fail')
+ end
+
+ return
+ end
+
+ local os_string = #os_name == 0 and 'unknown' or os_name .. ' ' .. os_flavor
+
+ task:get_mempool():set_variable('os_fingerprint', os_string, link_type,
+ uptime_min, distance)
+
+ common.yield_result(task, rule, {
+ os_string, link_type, 'distance:' .. distance }, 0.0)
+
+ return data
+ end
+
+ local function make_p0f_request()
+
+ local function check_p0f_cb(err, data)
+
+ local function redis_set_cb(redis_set_err)
+ if redis_set_err then
+ rspamd_logger.errx(task, 'redis received an error: %s', redis_set_err)
+ return
+ end
+ end
+
+ data = parse_p0f_response(data)
+
+ if rule.redis_params then
+ local key = rule.prefix .. ip:to_string()
+ local ret = lua_redis.redis_make_request(task,
+ rule.redis_params,
+ key,
+ true,
+ redis_set_cb,
+ 'SETEX',
+ { key, tostring(rule.expire), data }
+ )
+
+ if not ret then
+ rspamd_logger.warnx(task, 'error connecting to redis')
+ end
+ end
+ end
+
+ local query = rspamd_util.pack('I4 I1 c16', 0x50304601,
+ ip:get_version(), ip2bin(ip))
+
+ tcp.request({
+ host = rule.socket,
+ callback = check_p0f_cb,
+ data = { query },
+ task = task,
+ timeout = rule.timeout
+ })
+ end
+
+ local function redis_get_cb(err, data)
+ if err or type(data) ~= 'string' then
+ make_p0f_request()
+ else
+ parse_p0f_response(data)
+ end
+ end
+
+ local ret = nil
+ if rule.redis_prams then
+ local key = rule.prefix .. ip:to_string()
+ ret = lua_redis.redis_make_request(task,
+ rule.redis_params,
+ key,
+ false,
+ redis_get_cb,
+ 'GET',
+ { key }
+ )
+ end
+
+ if not ret then
+ make_p0f_request() -- fallback to directly querying p0f
+ end
+end
+
+local function p0f_config(opts)
+ local p0f_conf = {
+ name = N,
+ timeout = 5,
+ symbol = 'P0F',
+ symbol_fail = 'P0F_FAIL',
+ patterns = {},
+ expire = 7200,
+ prefix = 'p0f',
+ detection_category = 'fingerprint',
+ message = '${SCANNER}: fingerprint matched: "${VIRUS}"'
+ }
+
+ p0f_conf = lua_util.override_defaults(p0f_conf, opts)
+ p0f_conf.patterns = common.create_regex_table(p0f_conf.patterns)
+
+ if not p0f_conf.log_prefix then
+ p0f_conf.log_prefix = p0f_conf.name
+ end
+
+ if not p0f_conf.socket then
+ rspamd_logger.errx(rspamd_config, 'no servers defined')
+ return nil
+ end
+
+ return p0f_conf
+end
+
+return {
+ type = {N, 'fingerprint', 'scanner'},
+ description = 'passive OS fingerprinter',
+ configure = p0f_config,
+ check = p0f_check,
+ name = N
+}
diff --git a/src/plugins/lua/milter_headers.lua b/src/plugins/lua/milter_headers.lua
index 332625d7c..5d8d24c91 100644
--- a/src/plugins/lua/milter_headers.lua
+++ b/src/plugins/lua/milter_headers.lua
@@ -78,6 +78,10 @@ local settings = {
symbols_fail = {},
symbols = {}, -- needs config
},
+ ['x-os-fingerprint'] = {
+ header = 'X-OS-Fingerprint',
+ remove = 0,
+ },
['x-spamd-bar'] = {
header = 'X-Spamd-Bar',
positive = '+',
@@ -413,6 +417,26 @@ local function milter_headers(task)
end
end
+ routines['x-os-fingerprint'] = function()
+ if skip_wanted('x-os-fingerprint') then return end
+
+ local os_string, link_type, uptime_min, distance =
+ task:get_mempool():get_variable('os_fingerprint',
+ 'string, string, int, int');
+
+ if not os_string then return end
+
+ local value = string.format('%s, (up: %u min), (distance %i, link: %s)',
+ os_string, uptime_min, distance, link_type)
+
+ if settings.routines['x-os-fingerprint'].remove then
+ remove[settings.routines['x-os-fingerprint'].header]
+ = settings.routines['x-os-fingerprint'].remove
+ end
+
+ add_header(settings.routines['x-os-fingerprint'].header, value)
+ end
+
routines['x-spam-status'] = function()
if skip_wanted('x-spam-status') then return end
if not common['metric_score'] then
diff --git a/src/plugins/lua/p0f.lua b/src/plugins/lua/p0f.lua
new file mode 100644
index 000000000..84c525536
--- /dev/null
+++ b/src/plugins/lua/p0f.lua
@@ -0,0 +1,117 @@
+--[[
+Copyright (c) 2019, Vsevolod Stakhov <vsevolod at highsecure.ru>
+Copyright (c) 2019, Denis Paavilainen <denpa at denpa.pro>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+-- Detect remote OS via passive fingerprinting
+
+local lua_util = require "lua_util"
+local lua_redis = require "lua_redis"
+local rspamd_logger = require "rspamd_logger"
+local p0f = require("lua_scanners").filter('p0f').p0f
+
+local N = 'p0f'
+
+if confighelp then
+ rspamd_config:add_example(nil, N,
+ 'Detect remote OS via passive fingerprinting',
+ [[
+p0f {
+ # Enable module
+ enabled = true
+
+ # Path to the unix socket that p0f listens on
+ socket = '/tmp/p0f.sock';
+
+ # Connection timeout
+ timeout = 5s;
+
+ # If defined, insert symbol with lookup results
+ symbol = 'P0F';
+
+ # Patterns to match against results returned by p0f
+ # Symbol will be yielded on OS string, link type or distance matches
+ patterns = {
+ WINDOWS = '^Windows.*';
+ #DSL = '^DSL$';
+ #DISTANCE10 = '^distance:10$';
+ }
+
+ # Cache lifetime in seconds (default - 2 hours)
+ expire = 7200;
+
+ # Cache key prefix
+ prefix = 'p0f';
+}
+]])
+ return
+end
+
+local rule
+
+local function check_p0f(task)
+ local ip = task:get_from_ip()
+
+ if not (ip and ip:is_valid()) or ip:is_local() then
+ return
+ end
+
+ p0f.check(task, ip, rule)
+end
+
+local opts = rspamd_config:get_all_opt(N)
+
+rule = p0f.configure(opts)
+
+if rule then
+ rule.redis_params = lua_redis.parse_redis_server(N)
+
+ local id = rspamd_config:register_symbol({
+ name = 'P0F_CHECK',
+ type = 'prefilter,nostat',
+ callback = check_p0f,
+ priority = 8,
+ flags = 'empty',
+ group = N
+ })
+
+ if rule.symbol then
+ rspamd_config:register_symbol({
+ name = rule.symbol,
+ parent = id,
+ type = 'virtual',
+ flags = 'empty',
+ group = N
+ })
+ end
+
+ for sym in pairs(rule.patterns) do
+ rspamd_logger.debugm(N, rspamd_config, 'registering: %1', {
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ group = N
+ })
+ rspamd_config:register_symbol({
+ type = 'virtual',
+ name = sym,
+ parent = id,
+ group = N
+ })
+ end
+else
+ lua_util.disable_module(N, 'config')
+ rspamd_logger.infox('p0f module not configured');
+end
diff --git a/test/functional/cases/161_p0f.robot b/test/functional/cases/161_p0f.robot
new file mode 100644
index 000000000..9acbf7b2d
--- /dev/null
+++ b/test/functional/cases/161_p0f.robot
@@ -0,0 +1,88 @@
+*** Settings ***
+Suite Setup p0f Setup
+Suite Teardown p0f Teardown
+Library Process
+Library ${TESTDIR}/lib/rspamd.py
+Resource ${TESTDIR}/lib/rspamd.robot
+Variables ${TESTDIR}/lib/vars.py
+
+*** Variables ***
+${CONFIG} ${TESTDIR}/configs/plugins.conf
+${MESSAGE} ${TESTDIR}/messages/spam_message.eml
+${MESSAGE2} ${TESTDIR}/messages/freemail.eml
+${REDIS_SCOPE} Suite
+${RSPAMD_SCOPE} Suite
+${URL_TLD} ${TESTDIR}/../lua/unit/test_tld.dat
+
+*** Test Cases ***
+p0f MISS
+ Run Dummy p0f
+ ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.1
+ Check Rspamc ${result} P0F
+ Check Rspamc ${result} WINDOWS inverse=1
+ Check Rspamc ${result} P0F_FAIL inverse=1
+ Shutdown p0f
+
+p0f HIT
+ Run Dummy p0f ${P0F_SOCKET} windows
+ ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.2
+ Check Rspamc ${result} P0F inverse=1
+ Check Rspamc ${result} ETHER
+ Check Rspamc ${result} DISTGE10
+ Check Rspamc ${result} WINDOWS
+ Shutdown p0f
+
+p0f NOREDIS
+ Shutdown Process With Children ${REDIS_PID}
+ Run Dummy p0f
+ ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.3
+ Check Rspamc ${result} P0F
+ Check Rspamc ${result} ETHER
+ Check Rspamc ${result} DISTGE10
+ Check Rspamc ${result} P0F_FAIL inverse=1
+ Shutdown p0f
+
+p0f NOMATCH
+ Run Dummy p0f ${P0F_SOCKET} windows no_match
+ ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.4
+ Check Rspamc ${result} P0F inverse=1
+ Check Rspamc ${result} WINDOWS inverse=1
+ Shutdown p0f
+
+p0f BADQUERY
+ Run Dummy p0f ${P0F_SOCKET} windows bad_query
+ ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.5
+ Check Rspamc ${result} P0F_FAIL
+ Check Rspamc ${result} Malformed Query
+ Check Rspamc ${result} WINDOWS inverse=1
+ Shutdown p0f
+
+p0f FAILURE
+ Run Dummy p0f ${P0F_SOCKET} windows fail
+ ${result} = Scan Message With Rspamc ${MESSAGE} --ip 1.1.1.6
+ Check Rspamc ${result} P0F_FAIL
+ Check Rspamc ${result} Malformed Response
+ Check Rspamc ${result} WINDOWS inverse=1
+ Shutdown p0f
+
+*** Keywords ***
+p0f Setup
+ ${PLUGIN_CONFIG} = Get File ${TESTDIR}/configs/p0f.conf
+ Set Suite Variable ${PLUGIN_CONFIG}
+ Generic Setup PLUGIN_CONFIG
+ Run Redis
+
+p0f Teardown
+ Normal Teardown
+ Shutdown Process With Children ${REDIS_PID}
+ Shutdown p0f
+ Terminate All Processes kill=True
+
+Shutdown p0f
+ ${p0f_pid} = Get File if exists /tmp/dummy_p0f.pid
+ Run Keyword if ${p0f_pid} Shutdown Process With Children ${p0f_pid}
+
+Run Dummy p0f
+ [Arguments] ${socket}=${P0F_SOCKET} ${os}=linux ${status}=ok
+ ${result} = Start Process ${TESTDIR}/util/dummy_p0f.py ${socket} ${os} ${status}
+ Wait Until Created /tmp/dummy_p0f.pid
diff --git a/test/functional/configs/p0f.conf b/test/functional/configs/p0f.conf
new file mode 100644
index 000000000..69303772a
--- /dev/null
+++ b/test/functional/configs/p0f.conf
@@ -0,0 +1,11 @@
+redis {
+ servers = "${REDIS_ADDR}:${REDIS_PORT}";
+}
+p0f {
+ socket = "${P0F_SOCKET}";
+ patterns {
+ WINDOWS = '^Windows.*';
+ ETHER = '^Ethernet.*';
+ DISTGE10 = '^distance:[0-9]{2}$';
+ }
+}
diff --git a/test/functional/lib/vars.py b/test/functional/lib/vars.py
index 97b53b2e1..4559db205 100644
--- a/test/functional/lib/vars.py
+++ b/test/functional/lib/vars.py
@@ -15,6 +15,7 @@ PORT_PROXY = 56795
PORT_CLAM = 56796
PORT_FPROT = 56797
PORT_FPROT2_DUPLICATE = 56798
+P0F_SOCKET = '/tmp/p0f.sock'
REDIS_ADDR = u'127.0.0.1'
REDIS_PORT = 56379
NGINX_ADDR = u'127.0.0.1'
diff --git a/test/functional/util/dummy_p0f.py b/test/functional/util/dummy_p0f.py
new file mode 100755
index 000000000..e44844812
--- /dev/null
+++ b/test/functional/util/dummy_p0f.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+PID = "/tmp/dummy_p0f.pid"
+
+import os
+import sys
+import struct
+import socket
+import dummy_killer
+try:
+ import SocketServer as socketserver
+except:
+ import socketserver
+
+class MyStreamHandler(socketserver.BaseRequestHandler):
+
+ def handle(self):
+ S = {
+ 'bad_query' : 0x0,
+ 'ok' : 0x10,
+ 'no_match' : 0x20
+ }
+
+ OS = {
+ 'windows' : ('Windows', '7 or 8'),
+ 'linux' : ('Linux', '3.11 and newer')
+ }
+
+ self.data = self.request.recv(21).strip()
+
+ if self.server.p0f_status == 'fail':
+ response = 0
+ else:
+ response = struct.pack(
+ "IbIIIIIIIhbb32s32s32s32s32s32s",
+ 0x50304602, # magic
+ S[self.server.p0f_status], # status
+ 1568493408, # first_seen
+ 1568493408, # last_seen
+ 1, # total_conn
+ 1, # uptime_min
+ 4, # up_mod_days
+ 1568493408, # last_nat
+ 1568493408, # last_chg
+ 10, # distance
+ 0, # bad_sw
+ 0, # os_match_q
+ OS[self.server.p0f_os][0], # os_name
+ OS[self.server.p0f_os][1], # os_flavor
+ '', # http_name
+ '', # http_flavor
+ 'Ethernet or modem', # link_type
+ '' # language
+ )
+
+ self.request.sendall(response)
+ self.request.close()
+
+def cleanup(SOCK):
+ if os.path.exists(SOCK):
+ try:
+ os.unlink(SOCK)
+ except OSError:
+ logging.warning("Could not unlink socket %s", SOCK)
+
+if __name__ == "__main__":
+ SOCK = '/tmp/p0f.sock'
+ p0f_status = 'ok'
+ p0f_os = 'linux'
+
+ alen = len(sys.argv)
+ if alen > 1:
+ SOCK = sys.argv[1]
+ if alen >= 4:
+ p0f_os = sys.argv[2]
+ p0f_status = sys.argv[3]
+ elif alen >= 3:
+ p0f_os = sys.argv[2]
+
+ cleanup(SOCK)
+
+ server = socketserver.UnixStreamServer(SOCK, MyStreamHandler, bind_and_activate=False)
+ server.allow_reuse_address = True
+ server.p0f_status = p0f_status
+ server.p0f_os = p0f_os
+ server.server_bind()
+ server.server_activate()
+
+ dummy_killer.setup_killer(server)
+ dummy_killer.write_pid(PID)
+
+ try:
+ server.handle_request()
+ except socket.error:
+ print "Socket closed"
+
+ server.server_close()
+ cleanup(SOCK)
More information about the Commits
mailing list