From vsevolod at highsecure.ru Wed Dec 1 09:42:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 1 Dec 2021 09:42:04 +0000 (UTC) Subject: commit b805127: [Minor] Update bundled lua-argparse to 0.7.1 Message-ID: <20211201094204.3407E27040@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-01 09:40:56 +0000 URL: https://github.com/rspamd/rspamd/commit/b805127041e641e3c5e4f8cd2eecc50f0df1f1ad (HEAD -> master) [Minor] Update bundled lua-argparse to 0.7.1 --- contrib/DEPENDENCY_INFO.md | 2 +- contrib/lua-argparse/argparse.lua | 38 ++++++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/contrib/DEPENDENCY_INFO.md b/contrib/DEPENDENCY_INFO.md index c4385bdef..7bfc0361d 100644 --- a/contrib/DEPENDENCY_INFO.md +++ b/contrib/DEPENDENCY_INFO.md @@ -11,7 +11,7 @@ | librdns | ? | BSD-2-Clause | YES | | | libucl | ? | BSD-2-Clause | YES | | | replxx | 6d93360 | BSD-2-Clause | YES | libicu usage | -| lua-argparse | 0.7.0 | MIT | NO | | +| lua-argparse | 0.7.1 | MIT | NO | | | lua-bit | 1.0.2 | MIT | YES | build fixes | | lua-fun | ? | MIT | YES | rspamd text | | lua-lpeg | 1.0 | MIT | YES | rspamd text + alloc| diff --git a/contrib/lua-argparse/argparse.lua b/contrib/lua-argparse/argparse.lua index dc6cdb0de..6b5296247 100644 --- a/contrib/lua-argparse/argparse.lua +++ b/contrib/lua-argparse/argparse.lua @@ -130,12 +130,30 @@ local multiname = {"name", function(self, value) for alias in value:gmatch("%S+") do self._name = self._name or alias table.insert(self._aliases, alias) + table.insert(self._public_aliases, alias) + -- If alias contains '_', accept '-' also. + if alias:find("_", 1, true) then + table.insert(self._aliases, (alias:gsub("_", "-"))) + end end -- Do not set _name as with other properties. return true end} +local multiname_hidden = {"hidden_name", function(self, value) + typecheck("hidden_name", {"string"}, value) + + for alias in value:gmatch("%S+") do + table.insert(self._aliases, alias) + if alias:find("_", 1, true) then + table.insert(self._aliases, (alias:gsub("_", "-"))) + end + end + + return true +end} + local function parse_boundaries(str) if tonumber(str) then return tonumber(str), tonumber(str) @@ -257,12 +275,14 @@ local Parser = class({ }) local Command = class({ - _aliases = {} + _aliases = {}, + _public_aliases = {} }, { args = 3, multiname, typechecked("description", "string"), typechecked("epilog", "string"), + multiname_hidden, typechecked("summary", "string"), typechecked("target", "string"), typechecked("usage", "string"), @@ -307,6 +327,7 @@ local Argument = class({ local Option = class({ _aliases = {}, + _public_aliases = {}, _mincount = 0, _overwrite = true }, { @@ -317,6 +338,7 @@ local Option = class({ typechecked("convert", "function", "table"), boundaries("args"), boundaries("count"), + multiname_hidden, typechecked("target", "string"), typechecked("defmode", "string"), typechecked("show_default", "boolean"), @@ -505,22 +527,22 @@ function Option:_get_label_lines() if #argument_list == 0 then -- Don't put aliases for simple flags like `-h` on different lines. - return {table.concat(self._aliases, ", ")} + return {table.concat(self._public_aliases, ", ")} end local longest_alias_length = -1 - for _, alias in ipairs(self._aliases) do + for _, alias in ipairs(self._public_aliases) do longest_alias_length = math.max(longest_alias_length, #alias) end local argument_list_repr = table.concat(argument_list, " ") local lines = {} - for i, alias in ipairs(self._aliases) do + for i, alias in ipairs(self._public_aliases) do local line = (" "):rep(longest_alias_length - #alias) .. alias .. " " .. argument_list_repr - if i ~= #self._aliases then + if i ~= #self._public_aliases then line = line .. "," end @@ -531,7 +553,7 @@ function Option:_get_label_lines() end function Command:_get_label_lines() - return {table.concat(self._aliases, ", ")} + return {table.concat(self._public_aliases, ", ")} end function Argument:_get_description() @@ -569,7 +591,7 @@ end function Option:_get_default_target() local res - for _, alias in ipairs(self._aliases) do + for _, alias in ipairs(self._public_aliases) do if alias:sub(1, 1) == alias:sub(2, 2) then res = alias:sub(3) break @@ -2069,7 +2091,7 @@ end local argparse = {} -argparse.version = "0.7.0" +argparse.version = "0.7.1" setmetatable(argparse, {__call = function(_, ...) return Parser(default_cmdline[0]):add_help(true)(...) From vsevolod at highsecure.ru Wed Dec 1 19:42:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 1 Dec 2021 19:42:04 +0000 (UTC) Subject: commit 78ac4a6: [Minor] Process DNS reply flags Message-ID: <20211201194204.2A97C2708E@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-01 17:00:37 +0000 URL: https://github.com/rspamd/rspamd/commit/78ac4a61f7385eb05bafdbd465918be0b8c727d0 [Minor] Process DNS reply flags --- contrib/librdns/rdns.h | 7 ++++++- contrib/librdns/resolver.c | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/contrib/librdns/rdns.h b/contrib/librdns/rdns.h index 4df1991fd..266ccabc2 100644 --- a/contrib/librdns/rdns.h +++ b/contrib/librdns/rdns.h @@ -129,13 +129,18 @@ enum dns_rcode { RDNS_RC_NOREC = 13 }; +enum dns_reply_flags { + RDNS_AUTH = (1u << 0u), + RDNS_TRUNCATED = (1u << 1u) +}; + struct rdns_reply { struct rdns_request *request; struct rdns_resolver *resolver; struct rdns_reply_entry *entries; const char *requested_name; enum dns_rcode code; - bool authenticated; + uint8_t flags; /* see enum dns_reply_flags */ }; typedef void (*rdns_periodic_callback)(void *user_data); diff --git a/contrib/librdns/resolver.c b/contrib/librdns/resolver.c index 2c402077c..a0f09a294 100644 --- a/contrib/librdns/resolver.c +++ b/contrib/librdns/resolver.c @@ -149,7 +149,7 @@ rdns_make_reply (struct rdns_request *req, enum dns_rcode rcode) rep->entries = NULL; rep->code = rcode; req->reply = rep; - rep->authenticated = false; + rep->flags = 0; rep->requested_name = req->requested_names[0].name; } @@ -223,7 +223,11 @@ rdns_parse_reply (uint8_t *in, int r, struct rdns_request *req, rep = rdns_make_reply (req, header->rcode); if (header->ad) { - rep->authenticated = true; + rep->flags |= RDNS_AUTH; + } + + if (header->tc) { + rep->flags |= RDNS_TRUNCATED; } if (rep == NULL) { From vsevolod at highsecure.ru Wed Dec 1 19:42:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 1 Dec 2021 19:42:05 +0000 (UTC) Subject: commit e0f1732: [Minor] Adopt Rspamd to flags in rdns reply Message-ID: <20211201194205.3B13327090@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-01 19:10:29 +0000 URL: https://github.com/rspamd/rspamd/commit/e0f1732a975260140c235223d126e3e8a849f1e7 [Minor] Adopt Rspamd to flags in rdns reply --- src/lua/lua_dns.c | 5 ++++- src/lua/lua_dns_resolver.c | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lua/lua_dns.c b/src/lua/lua_dns.c index 0fdbe3f70..fe9951e89 100644 --- a/src/lua/lua_dns.c +++ b/src/lua/lua_dns.c @@ -166,9 +166,12 @@ lua_dns_callback (struct rdns_reply *reply, void *arg) else { lua_push_dns_reply (L, reply); - lua_pushboolean (L, reply->authenticated); + lua_pushboolean (L, reply->flags & RDNS_AUTH); lua_setfield (L, -3, "authenticated"); + lua_pushboolean (L, reply->flags & RDNS_TRUNCATED); + lua_setfield (L, -3, "truncated"); + /* result 1 - not and error */ lua_pushboolean (L, true); /* push table into stack, result 2 - results itself */ diff --git a/src/lua/lua_dns_resolver.c b/src/lua/lua_dns_resolver.c index f43267dc6..b5ded396d 100644 --- a/src/lua/lua_dns_resolver.c +++ b/src/lua/lua_dns_resolver.c @@ -169,7 +169,7 @@ lua_dns_resolver_callback (struct rdns_reply *reply, gpointer arg) * 3 - entries | nil * 4 - error | nil * 5 - user_str - * 6 - reply->authenticated + * 6 - reply->flags & RDNS_AUTH * 7 - server */ if (reply->code != RDNS_RC_NOERROR) { @@ -183,7 +183,7 @@ lua_dns_resolver_callback (struct rdns_reply *reply, gpointer arg) lua_pushnil (L); } - lua_pushboolean (L, reply->authenticated); + lua_pushboolean (L, reply->flags & RDNS_AUTH); const gchar *servname = rdns_request_get_server (reply->request); From vsevolod at highsecure.ru Wed Dec 1 19:42:06 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 1 Dec 2021 19:42:06 +0000 (UTC) Subject: commit 670d2ce: [Minor] Spf: Deal with enormously large SPF records Message-ID: <20211201194206.5174327092@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-01 19:25:52 +0000 URL: https://github.com/rspamd/rspamd/commit/670d2ce2758f08bd6d59135a625ef00ecbd7bc22 [Minor] Spf: Deal with enormously large SPF records --- src/libserver/spf.c | 65 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/src/libserver/spf.c b/src/libserver/spf.c index a506d73df..dd85dfbe5 100644 --- a/src/libserver/spf.c +++ b/src/libserver/spf.c @@ -847,15 +847,24 @@ spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) struct spf_addr *addr; struct spf_record *rec; const struct rdns_request_name *req_name; + bool truncated = false; rec = cb->rec; task = rec->task; cb->rec->requests_inflight--; addr = cb->addr; + req_name = rdns_request_get_name (reply->request, NULL); - if (reply->code == RDNS_RC_NOERROR) { - req_name = rdns_request_get_name (reply->request, NULL); + if (reply->flags & RDNS_TRUNCATED) { + /* Do not process truncated DNS replies */ + truncated = true; + msg_warn_spf ("got a truncated record when trying to resolve %s (%s type) for SPF domain %s", + req_name, rdns_str_from_type (req_name->type), + rec->sender_domain); + } + + if (reply->code == RDNS_RC_NOERROR && !truncated) { LL_FOREACH (reply->entries, elt_data) { /* Adjust ttl if a resolved record has lower ttl than spf record itself */ @@ -2434,26 +2443,40 @@ spf_dns_callback (struct rdns_reply *reply, gpointer arg) rec->requests_inflight--; - if (reply->code == RDNS_RC_NOERROR) { - resolved = rspamd_spf_new_addr_list (rec, rec->sender_domain); - if (rec->resolved->len == 1) { - /* Top level resolved element */ - rec->ttl = reply->entries->ttl; - } - } - else if ((reply->code == RDNS_RC_NOREC || reply->code == RDNS_RC_NXDOMAIN) - && rec->dns_requests == 0) { - resolved = rspamd_spf_new_addr_list (rec, rec->sender_domain); - addr = g_malloc0 (sizeof(*addr)); - addr->flags |= RSPAMD_SPF_FLAG_NA; - g_ptr_array_insert (resolved->elts, 0, addr); - } - else if (reply->code != RDNS_RC_NOREC && reply->code != RDNS_RC_NXDOMAIN - && rec->dns_requests == 0) { - resolved = rspamd_spf_new_addr_list (rec, rec->sender_domain); - addr = g_malloc0 (sizeof(*addr)); + if (reply->flags & RDNS_TRUNCATED) { + msg_warn_spf ("got a truncated record when trying to resolve TXT record for %s", + rec->sender_domain); + resolved = rspamd_spf_new_addr_list(rec, rec->sender_domain); + addr = g_malloc0(sizeof(*addr)); addr->flags |= RSPAMD_SPF_FLAG_TEMPFAIL; - g_ptr_array_insert (resolved->elts, 0, addr); + g_ptr_array_insert(resolved->elts, 0, addr); + + rspamd_spf_maybe_return (rec); + + return; + } + else { + if (reply->code == RDNS_RC_NOERROR) { + resolved = rspamd_spf_new_addr_list(rec, rec->sender_domain); + if (rec->resolved->len == 1) { + /* Top level resolved element */ + rec->ttl = reply->entries->ttl; + } + } + else if ((reply->code == RDNS_RC_NOREC || reply->code == RDNS_RC_NXDOMAIN) + && rec->dns_requests == 0) { + resolved = rspamd_spf_new_addr_list(rec, rec->sender_domain); + addr = g_malloc0(sizeof(*addr)); + addr->flags |= RSPAMD_SPF_FLAG_NA; + g_ptr_array_insert(resolved->elts, 0, addr); + } + else if (reply->code != RDNS_RC_NOREC && reply->code != RDNS_RC_NXDOMAIN + && rec->dns_requests == 0) { + resolved = rspamd_spf_new_addr_list(rec, rec->sender_domain); + addr = g_malloc0(sizeof(*addr)); + addr->flags |= RSPAMD_SPF_FLAG_TEMPFAIL; + g_ptr_array_insert(resolved->elts, 0, addr); + } } if (resolved) { From vsevolod at highsecure.ru Wed Dec 1 19:42:07 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 1 Dec 2021 19:42:07 +0000 (UTC) Subject: commit 7bd6659: [Minor] Fix format string Message-ID: <20211201194207.709E827094@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-01 19:41:26 +0000 URL: https://github.com/rspamd/rspamd/commit/7bd6659b969b1c183b11fb19437cb46ca5929297 (HEAD -> master) [Minor] Fix format string --- src/libserver/spf.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/libserver/spf.c b/src/libserver/spf.c index dd85dfbe5..b33f06c01 100644 --- a/src/libserver/spf.c +++ b/src/libserver/spf.c @@ -859,9 +859,17 @@ spf_record_dns_callback (struct rdns_reply *reply, gpointer arg) if (reply->flags & RDNS_TRUNCATED) { /* Do not process truncated DNS replies */ truncated = true; - msg_warn_spf ("got a truncated record when trying to resolve %s (%s type) for SPF domain %s", - req_name, rdns_str_from_type (req_name->type), - rec->sender_domain); + + if (req_name) { + msg_warn_spf ("got a truncated record when trying to resolve %s (%s type) for SPF domain %s", + req_name->name, rdns_str_from_type(req_name->type), + rec->sender_domain); + } + else { + msg_warn_spf ("got a truncated record when trying to resolve ??? " + "(internal error) for SPF domain %s", + rec->sender_domain); + } } if (reply->code == RDNS_RC_NOERROR && !truncated) { From vsevolod at highsecure.ru Thu Dec 2 12:21:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 2 Dec 2021 12:21:04 +0000 (UTC) Subject: commit 60b407b: [Minor] Fix ip validity check Message-ID: <20211202122104.64CBB2711F@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-02 12:17:06 +0000 URL: https://github.com/rspamd/rspamd/commit/60b407b19f54277cb82bda418e2f2439f39a7f3f (HEAD -> master) [Minor] Fix ip validity check --- lualib/plugins/dmarc.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lualib/plugins/dmarc.lua b/lualib/plugins/dmarc.lua index cd1b8e4b8..783447242 100644 --- a/lualib/plugins/dmarc.lua +++ b/lualib/plugins/dmarc.lua @@ -78,7 +78,7 @@ exports.dmarc_report = function (task, settings, data) local E = {} local ip = task:get_from_ip() - if ip and not ip:is_valid() then + if not ip or not ip:is_valid() then rspamd_logger.infox(task, 'cannot store dmarc report for %s: no valid source IP', data.domain) return nil From vsevolod at highsecure.ru Thu Dec 2 20:35:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 2 Dec 2021 20:35:04 +0000 (UTC) Subject: commit a340977: [Minor] Redis_history: Fix docs Message-ID: <20211202203504.3517027162@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-02 20:18:39 +0000 URL: https://github.com/rspamd/rspamd/commit/a3409774a675e2a8d2444097f0d9a83533997628 [Minor] Redis_history: Fix docs Issue: #3997 --- src/plugins/lua/history_redis.lua | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/plugins/lua/history_redis.lua b/src/plugins/lua/history_redis.lua index c14b8d32a..fa9db5b85 100644 --- a/src/plugins/lua/history_redis.lua +++ b/src/plugins/lua/history_redis.lua @@ -19,13 +19,20 @@ if confighelp then "Store history of checks for WebUI using Redis", [[ redis_history { - key_prefix = 'rs_history', # default key name - nrows = 200; # default rows limit - compress = true; # use zstd compression when storing data in redis - subject_privacy = false; # subject privacy is off - subject_privacy_alg = 'blake2'; # default hash-algorithm to obfuscate subject - subject_privacy_prefix = 'obf'; # prefix to show it's obfuscated - subject_privacy_length = 16; # cut the length of the hash + # History key name + key_prefix = 'rs_history'; + # History rows limit + nrows = 200; + # Use zstd compression when storing data in redis + compress = true; + # Obfuscate subjects for privacy + subject_privacy = false; + # Default hash-algorithm to obfuscate subject + subject_privacy_alg = 'blake2'; + # Prefix to show it's obfuscated + subject_privacy_prefix = 'obf'; + # Cut the length of the hash if desired + subject_privacy_length = 16; } ]]) return From vsevolod at highsecure.ru Thu Dec 2 20:35:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 2 Dec 2021 20:35:05 +0000 (UTC) Subject: commit 52d8a02: [Minor] Greylist: Fix docs Message-ID: <20211202203505.4FFB227164@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-02 20:31:40 +0000 URL: https://github.com/rspamd/rspamd/commit/52d8a02f10b6791580859b11b5d44a04c193db96 (HEAD -> master) [Minor] Greylist: Fix docs Issue: #3997 --- src/plugins/lua/greylist.lua | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/src/plugins/lua/greylist.lua b/src/plugins/lua/greylist.lua index 807859bff..652452496 100644 --- a/src/plugins/lua/greylist.lua +++ b/src/plugins/lua/greylist.lua @@ -40,19 +40,32 @@ if confighelp then "Performs adaptive greylisting using Redis", [[ greylist { - expire = 1d; # Buckets expire (1 day by default) - timeout = 5m; # Greylisting timeout - key_prefix = 'rg'; # Redis prefix - max_data_len = 10k; # Use boy hash up to this value of bytes for greylisting - message = 'Try again later'; # Default greylisting message - symbol = 'GREYLIST'; # Append symbol - action = 'soft reject'; # Default action change (for Exim use `greylist`) - whitelist_symbols = []; # Skip greylisting if one of the following symbols has been found - ipv4_mask = 19; # Mask bits for ipv4 - ipv6_mask = 64; # Mask bits for ipv6 - report_time = false; # Tell when greylisting is expired (appended to `message`) - check_local = false; # Greylist local messages - check_authed = false; # Greylist authenticated users + # Buckets expire (1 day by default) + expire = 1d; + # Greylisting timeout + timeout = 5m; + # Redis prefix + key_prefix = 'rg'; + # Use body hash up to this value of bytes for greylisting + max_data_len = 10k; + # Default greylisting message + message = 'Try again later'; + # Append symbol on greylisting + symbol = 'GREYLIST'; + # Default action change (for Exim use `greylist`) + action = 'soft reject'; + # Skip greylisting if one of the following symbols has been found + whitelist_symbols = []; + # Mask bits for ipv4 + ipv4_mask = 19; + # Mask bits for ipv6 + ipv6_mask = 64; + # Tell when greylisting is expired (appended to `message`) + report_time = false; + # Greylist local messages + check_local = false; + # Greylist messages from authenticated users + check_authed = false; } ]]) return From vsevolod at highsecure.ru Fri Dec 3 14:42:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Fri, 3 Dec 2021 14:42:04 +0000 (UTC) Subject: commit 137ab1c: [Feature] Lua_magic: Add a sane CSV heuristic Message-ID: <20211203144204.501A3271FD@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-03 14:37:52 +0000 URL: https://github.com/rspamd/rspamd/commit/137ab1cb787d5a5e95a2fc7d2aa86dc4ab47b9a5 (HEAD -> master) [Feature] Lua_magic: Add a sane CSV heuristic --- lualib/lua_magic/heuristics.lua | 66 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/lualib/lua_magic/heuristics.lua b/lualib/lua_magic/heuristics.lua index d4918e978..7eae38ffc 100644 --- a/lualib/lua_magic/heuristics.lua +++ b/lualib/lua_magic/heuristics.lua @@ -320,6 +320,67 @@ local function detect_archive_flaw(part, arch, log_obj, _) return arch_type:lower(),40 end +local csv_grammar +-- Returns a grammar that will count commas +local function get_csv_grammar() + if not csv_grammar then + local lpeg = require'lpeg' + + local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P'""' / '"')^0) * '"' + + lpeg.C((1 - lpeg.S',\n"')^0) + + csv_grammar = lpeg.Cf(lpeg.Cc(0) * field * lpeg.P( (lpeg.P(',') + + lpeg.P('\t')) * field)^1 * (lpeg.S'\r\n' + -1), + function(acc) return acc + 1 end) + end + + return csv_grammar +end +local function validate_csv(part, content, log_obj) + local max_chunk = 32768 + local chunk = content:sub(1, max_chunk) + + local expected_commas + local matched_lines = 0 + local max_matched_lines = 10 + + lua_util.debugm(N, log_obj, "check for csv pattern") + + for s in chunk:lines() do + local ncommas = get_csv_grammar():match(s) + + if not ncommas then + lua_util.debugm(N, log_obj, "not a csv line at line number %s", + matched_lines) + return false + end + + if expected_commas and ncommas ~= expected_commas then + -- Mismatched commas + lua_util.debugm(N, log_obj, "missmatched commas on line %s: %s != %s", + matched_lines, ncommas, expected_commas) + return false + elseif not expected_commas then + if ncommas == 0 then + lua_util.debugm(N, log_obj, "no commas in the first line") + return false + end + expected_commas = ncommas + end + + matched_lines = matched_lines + 1 + + if matched_lines > max_matched_lines then + break + end + end + + lua_util.debugm(N, log_obj, "csv content is sane: %s fields; %s lines checked", + expected_commas, matched_lines) + + return true +end + exports.mime_part_heuristic = function(part, log_obj, _) if part:is_archive() then local arch = part:get_archive() @@ -452,7 +513,10 @@ exports.text_part_heuristic = function(part, log_obj, _) if weight then if weight >= 40 then - return ext, weight + -- Extra validation for csv extension + if ext ~= 'csv' or validate_csv(part, content, log_obj) then + return ext, weight + end elseif fname and weight >= 20 then return ext, weight end From vsevolod at highsecure.ru Fri Dec 3 21:14:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Fri, 3 Dec 2021 21:14:04 +0000 (UTC) Subject: commit d8efaee: [Minor] Fix alignment issue Message-ID: <20211203211404.2102A27230@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-03 21:13:21 +0000 URL: https://github.com/rspamd/rspamd/commit/d8efaee9fe69c6c72ba53b038b39b41103be8620 (HEAD -> master) [Minor] Fix alignment issue --- src/libutil/str_util.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c index 6cee32243..2d39ccf8a 100644 --- a/src/libutil/str_util.c +++ b/src/libutil/str_util.c @@ -3569,12 +3569,15 @@ rspamd_str_has_8bit_u64 (const guchar *beg, gsize len) guint8 orb = 0; if (len >= 16) { - const guchar *nextd = beg+8; + const guchar *nextd = beg + sizeof(guint64); guint64 n1 = 0, n2 = 0; do { - n1 |= *(const guint64 *)beg; - n2 |= *(const guint64 *)nextd; + guint64 t; + memcpy(&t, beg, sizeof(t)); + n1 |= t; + memcpy(&t, nextd, sizeof(t)); + n2 |= t; beg += 16; nextd += 16; len -= 16; From vsevolod at highsecure.ru Sat Dec 4 14:21:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sat, 4 Dec 2021 14:21:04 +0000 (UTC) Subject: commit 9be5ac3: [Minor] Allow ubsan usage Message-ID: <20211204142104.606C2272C2@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-04 14:12:39 +0000 URL: https://github.com/rspamd/rspamd/commit/9be5ac33b5e08734db941f811267dd396b4b9426 [Minor] Allow ubsan usage --- cmake/Sanitizer.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Sanitizer.cmake b/cmake/Sanitizer.cmake index 4b4df9fae..74d6a4b1c 100644 --- a/cmake/Sanitizer.cmake +++ b/cmake/Sanitizer.cmake @@ -46,7 +46,7 @@ if (SANITIZE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libubsan") set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan") endif () - + else () message (FATAL_ERROR "Unknown sanitizer type: ${SANITIZE}") endif () message (STATUS "Add sanitizer: ${SANITIZE}") From vsevolod at highsecure.ru Sat Dec 4 14:21:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sat, 4 Dec 2021 14:21:05 +0000 (UTC) Subject: commit 03f7369: [Minor] Fix enum handling Message-ID: <20211204142105.74289272C4@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-04 14:20:26 +0000 URL: https://github.com/rspamd/rspamd/commit/03f73692a9cdc4b42aede804faa44572545ebdfc (HEAD -> master) [Minor] Fix enum handling --- src/libserver/html/html.cxx | 2 +- src/libserver/html/html_tag.hxx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libserver/html/html.cxx b/src/libserver/html/html.cxx index a920f9a4f..c782148d2 100644 --- a/src/libserver/html/html.cxx +++ b/src/libserver/html/html.cxx @@ -2017,7 +2017,7 @@ html_process_input(rspamd_mempool_t *pool, /* Propagate styles */ hc->traverse_block_tags([&hc, &pool](const html_tag *tag) -> bool { - if (hc->css_style) { + if (hc->css_style && tag->id > Tag_UNKNOWN && tag->id < Tag_MAX) { auto *css_block = hc->css_style->check_tag_block(tag); if (css_block) { diff --git a/src/libserver/html/html_tag.hxx b/src/libserver/html/html_tag.hxx index 5971ca179..ba086be82 100644 --- a/src/libserver/html/html_tag.hxx +++ b/src/libserver/html/html_tag.hxx @@ -91,7 +91,7 @@ struct html_tag { unsigned int tag_start = 0; unsigned int content_offset = 0; std::uint32_t flags = 0; - tag_id_t id = Tag_UNKNOWN; + std::int32_t id = Tag_UNKNOWN; html_closing_tag closing; std::vector components; From vsevolod at highsecure.ru Sat Dec 4 14:56:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sat, 4 Dec 2021 14:56:04 +0000 (UTC) Subject: commit 50528c6: [Rework] Mempool: Use explicit alignment Message-ID: <20211204145604.25BBE272CE@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-04 14:37:47 +0000 URL: https://github.com/rspamd/rspamd/commit/50528c680dfc61234b9fd8045afc2a23cf61abe1 [Rework] Mempool: Use explicit alignment --- src/libutil/mem_pool.c | 55 ++++++++++++++++++++++++++++---------------------- src/libutil/mem_pool.h | 31 ++++++++++++++++------------ 2 files changed, 49 insertions(+), 37 deletions(-) diff --git a/src/libutil/mem_pool.c b/src/libutil/mem_pool.c index 7ade92440..1b72ed7c4 100644 --- a/src/libutil/mem_pool.c +++ b/src/libutil/mem_pool.c @@ -168,10 +168,10 @@ rspamd_mempool_get_entry (const gchar *loc) } static struct _pool_chain * -rspamd_mempool_chain_new (gsize size, enum rspamd_mempool_chain_type pool_type) +rspamd_mempool_chain_new (gsize size, gsize alignment, enum rspamd_mempool_chain_type pool_type) { struct _pool_chain *chain; - gsize total_size = size + sizeof (struct _pool_chain) + MIN_MEM_ALIGNMENT, + gsize total_size = size + sizeof (struct _pool_chain) + alignment, optimal_size = 0; gpointer map; @@ -223,7 +223,7 @@ rspamd_mempool_chain_new (gsize size, enum rspamd_mempool_chain_type pool_type) optimal_size = sys_alloc_size (total_size); #endif total_size = MAX (total_size, optimal_size); - gint ret = posix_memalign (&map, MIN_MEM_ALIGNMENT, total_size); + gint ret = posix_memalign (&map, alignment, total_size); if (ret != 0 || map == NULL) { g_error ("%s: failed to allocate %"G_GSIZE_FORMAT" bytes: %d - %s", @@ -237,7 +237,7 @@ rspamd_mempool_chain_new (gsize size, enum rspamd_mempool_chain_type pool_type) g_atomic_int_inc (&mem_pool_stat->chunks_allocated); } - chain->pos = align_ptr (chain->begin, MIN_MEM_ALIGNMENT); + chain->pos = align_ptr (chain->begin, alignment); chain->slice_size = total_size - sizeof (struct _pool_chain); return chain; @@ -441,7 +441,7 @@ rspamd_mempool_new_ (gsize size, const gchar *tag, gint flags, const gchar *loc) } static void * -memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, +memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, gsize alignment, enum rspamd_mempool_chain_type pool_type, const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; @@ -468,7 +468,7 @@ rspamd_mempool_notify_alloc_ (rspamd_mempool_t *pool, gsize size, const gchar *l } static void * -memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, +memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, gsize alignment, enum rspamd_mempool_chain_type pool_type, const gchar *loc) { guint8 *tmp; @@ -486,7 +486,13 @@ memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, if (always_malloc && pool_type != RSPAMD_MEMPOOL_SHARED) { void *ptr; - ptr = g_malloc (size); + if (alignment <= G_MEM_ALIGN) { + ptr = g_malloc(size); + } + else { + ptr = g_malloc(size + alignment); + ptr = align_ptr(ptr, alignment); + } POOL_MTX_UNLOCK (); if (pool->priv->trash_stack == NULL) { @@ -505,15 +511,15 @@ memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, free = pool_chain_free (cur); } - if (cur == NULL || free < size) { + if (cur == NULL || free < size + alignment) { if (free < size) { pool->priv->wasted_memory += free; } /* Allocate new chain element */ - if (pool->priv->elt_len >= size + MIN_MEM_ALIGNMENT) { + if (pool->priv->elt_len >= size + alignment) { pool->priv->entry->elts[pool->priv->entry->cur_elts].fragmentation += size; - new = rspamd_mempool_chain_new (pool->priv->elt_len, + new = rspamd_mempool_chain_new (pool->priv->elt_len, alignment, pool_type); } else { @@ -521,7 +527,7 @@ memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, g_atomic_int_add (&mem_pool_stat->fragmented_size, free); pool->priv->entry->elts[pool->priv->entry->cur_elts].fragmentation += free; - new = rspamd_mempool_chain_new (size + pool->priv->elt_len, pool_type); + new = rspamd_mempool_chain_new (alignment, size + pool->priv->elt_len, pool_type); } /* Connect to pool subsystem */ @@ -535,7 +541,7 @@ memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, } /* No need to allocate page */ - tmp = align_ptr (cur->pos, MIN_MEM_ALIGNMENT); + tmp = align_ptr (cur->pos, alignment); cur->pos = tmp + size; POOL_MTX_UNLOCK (); @@ -547,32 +553,32 @@ memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, void * -rspamd_mempool_alloc_ (rspamd_mempool_t * pool, gsize size, const gchar *loc) +rspamd_mempool_alloc_ (rspamd_mempool_t * pool, gsize size, gsize alignment, const gchar *loc) { - return memory_pool_alloc_common (pool, size, RSPAMD_MEMPOOL_NORMAL, loc); + return memory_pool_alloc_common (pool, size, alignment, RSPAMD_MEMPOOL_NORMAL, loc); } void * -rspamd_mempool_alloc0_ (rspamd_mempool_t * pool, gsize size, const gchar *loc) +rspamd_mempool_alloc0_ (rspamd_mempool_t * pool, gsize size, gsize alignment, const gchar *loc) { - void *pointer = rspamd_mempool_alloc_ (pool, size, loc); + void *pointer = rspamd_mempool_alloc_ (pool, size, alignment, loc); memset (pointer, 0, size); return pointer; } void * -rspamd_mempool_alloc0_shared_ (rspamd_mempool_t * pool, gsize size, const gchar *loc) +rspamd_mempool_alloc0_shared_ (rspamd_mempool_t * pool, gsize size, gsize alignment, const gchar *loc) { - void *pointer = rspamd_mempool_alloc_shared (pool, size); + void *pointer = rspamd_mempool_alloc_shared_ (pool, size, alignment, loc); memset (pointer, 0, size); return pointer; } void * -rspamd_mempool_alloc_shared_ (rspamd_mempool_t * pool, gsize size, const gchar *loc) +rspamd_mempool_alloc_shared_ (rspamd_mempool_t * pool, gsize size, gsize alignment, const gchar *loc) { - return memory_pool_alloc_common (pool, size, RSPAMD_MEMPOOL_SHARED, loc); + return memory_pool_alloc_common (pool, size, alignment, RSPAMD_MEMPOOL_SHARED, loc); } @@ -587,7 +593,7 @@ rspamd_mempool_strdup_ (rspamd_mempool_t * pool, const gchar *src, const gchar * } len = strlen (src); - newstr = rspamd_mempool_alloc_ (pool, len + 1, loc); + newstr = rspamd_mempool_alloc_ (pool, len + 1, MIN_MEM_ALIGNMENT, loc); memcpy (newstr, src, len); newstr[len] = '\0'; @@ -604,7 +610,7 @@ rspamd_mempool_fstrdup_ (rspamd_mempool_t * pool, const struct f_str_s *src, return NULL; } - newstr = rspamd_mempool_alloc_ (pool, src->len + 1, loc); + newstr = rspamd_mempool_alloc_ (pool, src->len + 1, MIN_MEM_ALIGNMENT, loc); memcpy (newstr, src->str, src->len); newstr[src->len] = '\0'; @@ -621,7 +627,7 @@ rspamd_mempool_ftokdup_ (rspamd_mempool_t *pool, const rspamd_ftok_t *src, return NULL; } - newstr = rspamd_mempool_alloc_ (pool, src->len + 1, loc); + newstr = rspamd_mempool_alloc_ (pool, src->len + 1, MIN_MEM_ALIGNMENT, loc); memcpy (newstr, src->begin, src->len); newstr[src->len] = '\0'; @@ -638,7 +644,8 @@ rspamd_mempool_add_destructor_full (rspamd_mempool_t * pool, struct _pool_destructors *cur; POOL_MTX_LOCK (); - cur = rspamd_mempool_alloc_ (pool, sizeof (*cur), line); + cur = rspamd_mempool_alloc_ (pool, sizeof (*cur), + RSPAMD_ALIGNOF(struct _pool_destructors), line); cur->func = func; cur->data = data; cur->function = function; diff --git a/src/libutil/mem_pool.h b/src/libutil/mem_pool.h index 8db233d8f..1847980bb 100644 --- a/src/libutil/mem_pool.h +++ b/src/libutil/mem_pool.h @@ -67,7 +67,12 @@ struct f_str_s; #define MEMPOOL_TAG_LEN 20 #define MEMPOOL_UID_LEN 20 /* All pointers are aligned as this variable */ -#define MIN_MEM_ALIGNMENT sizeof (guint64) +#define MIN_MEM_ALIGNMENT G_MEM_ALIGN +#ifndef __cplusplus +#define RSPAMD_ALIGNOF G_ALIGNOF +#else +#define RSPAMD_ALIGNOF(t) alignof(t) +#endif /** * Destructor type definition */ @@ -151,14 +156,14 @@ rspamd_mempool_t *rspamd_mempool_new_ (gsize size, const gchar *tag, gint flags, * @param size bytes to allocate * @return pointer to allocated object */ -void *rspamd_mempool_alloc_ (rspamd_mempool_t *pool, gsize size, const gchar *loc) +void *rspamd_mempool_alloc_ (rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; #define rspamd_mempool_alloc(pool, size) \ - rspamd_mempool_alloc_((pool), (size), (G_STRLOC)) + rspamd_mempool_alloc_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC)) #define rspamd_mempool_alloc_type(pool, type) \ - (type *)(rspamd_mempool_alloc_((pool), sizeof(type), (G_STRLOC))) + (type *)(rspamd_mempool_alloc_((pool), sizeof(type), RSPAMD_ALIGNOF(type), (G_STRLOC))) #define rspamd_mempool_alloc_buffer(pool, buflen) \ - (char *)(rspamd_mempool_alloc_((pool), sizeof(char) * (buflen), (G_STRLOC))) + (char *)(rspamd_mempool_alloc_((pool), sizeof(char) * (buflen), MIN_MEM_ALIGNMENT, (G_STRLOC))) /** * Notify external memory usage for memory pool * @param pool @@ -175,12 +180,12 @@ void rspamd_mempool_notify_alloc_ (rspamd_mempool_t *pool, gsize size, const gch * @param size bytes to allocate * @return pointer to allocated object */ -void *rspamd_mempool_alloc0_ (rspamd_mempool_t *pool, gsize size, const gchar *loc) +void *rspamd_mempool_alloc0_ (rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; #define rspamd_mempool_alloc0(pool, size) \ - rspamd_mempool_alloc0_((pool), (size), (G_STRLOC)) + rspamd_mempool_alloc0_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC)) #define rspamd_mempool_alloc0_type(pool, type) \ - (type *)(rspamd_mempool_alloc0_((pool), sizeof(type), (G_STRLOC))) + (type *)(rspamd_mempool_alloc0_((pool), sizeof(type), RSPAMD_ALIGNOF(type), (G_STRLOC))) /** * Make a copy of string in pool @@ -202,7 +207,7 @@ gchar *rspamd_mempool_strdup_ (rspamd_mempool_t *pool, const gchar *src, const g gchar *rspamd_mempool_fstrdup_ (rspamd_mempool_t *pool, const struct f_str_s *src, const gchar *loc) -RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT); + RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT); #define rspamd_mempool_fstrdup(pool, src) \ rspamd_mempool_fstrdup_ ((pool), (src), G_STRLOC) @@ -226,15 +231,15 @@ RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT); * @param pool memory pool object * @param size bytes to allocate */ -void *rspamd_mempool_alloc_shared_ (rspamd_mempool_t *pool, gsize size, const gchar *loc) +void *rspamd_mempool_alloc_shared_ (rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; #define rspamd_mempool_alloc_shared(pool, size) \ - rspamd_mempool_alloc_shared_((pool), (size), (G_STRLOC)) + rspamd_mempool_alloc_shared_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC)) -void *rspamd_mempool_alloc0_shared_ (rspamd_mempool_t *pool, gsize size, const gchar *loc) +void *rspamd_mempool_alloc0_shared_ (rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; #define rspamd_mempool_alloc0_shared(pool, size) \ - rspamd_mempool_alloc0_shared_((pool), (size), (G_STRLOC)) + rspamd_mempool_alloc0_shared_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC)) /** * Add destructor callback to pool From vsevolod at highsecure.ru Sat Dec 4 14:56:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sat, 4 Dec 2021 14:56:05 +0000 (UTC) Subject: commit f5888ae: [Minor] Fix arguments order Message-ID: <20211204145605.4389E272D0@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-04 14:40:50 +0000 URL: https://github.com/rspamd/rspamd/commit/f5888ae382b9afdc1c1f26fff53e295b6739e9a6 [Minor] Fix arguments order --- src/libutil/mem_pool.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libutil/mem_pool.c b/src/libutil/mem_pool.c index 1b72ed7c4..6bb74b6e7 100644 --- a/src/libutil/mem_pool.c +++ b/src/libutil/mem_pool.c @@ -527,7 +527,8 @@ memory_pool_alloc_common (rspamd_mempool_t * pool, gsize size, gsize alignment, g_atomic_int_add (&mem_pool_stat->fragmented_size, free); pool->priv->entry->elts[pool->priv->entry->cur_elts].fragmentation += free; - new = rspamd_mempool_chain_new (alignment, size + pool->priv->elt_len, pool_type); + new = rspamd_mempool_chain_new (size + pool->priv->elt_len, alignment, + pool_type); } /* Connect to pool subsystem */ From vsevolod at highsecure.ru Sat Dec 4 14:56:06 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sat, 4 Dec 2021 14:56:06 +0000 (UTC) Subject: commit 553bd31: [Minor] Fix some alignment issues Message-ID: <20211204145606.5AEDF272D2@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-04 14:45:30 +0000 URL: https://github.com/rspamd/rspamd/commit/553bd31ac240a9c2eff73133f9427ce090d9a6bb [Minor] Fix some alignment issues --- src/libserver/maps/map_helpers.c | 6 +++--- src/libutil/mem_pool.h | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libserver/maps/map_helpers.c b/src/libserver/maps/map_helpers.c index 9ac84a81e..a29467497 100644 --- a/src/libserver/maps/map_helpers.c +++ b/src/libserver/maps/map_helpers.c @@ -741,7 +741,7 @@ rspamd_map_helper_new_hash (struct rspamd_map *map) NULL, 0); } - htb = rspamd_mempool_alloc0 (pool, sizeof (*htb)); + htb = rspamd_mempool_alloc0_type(pool, struct rspamd_hash_map_helper); htb->htb = kh_init (rspamd_map_hash); htb->pool = pool; htb->map = map; @@ -801,7 +801,7 @@ rspamd_map_helper_new_radix (struct rspamd_map *map) NULL, 0); } - r = rspamd_mempool_alloc0 (pool, sizeof (*r)); + r = rspamd_mempool_alloc0_type (pool, struct rspamd_radix_map_helper); r->trie = radix_create_compressed_with_pool (pool, name); r->htb = kh_init (rspamd_map_hash); r->pool = pool; @@ -855,7 +855,7 @@ rspamd_map_helper_new_regexp (struct rspamd_map *map, pool = rspamd_mempool_new (rspamd_mempool_suggest_size (), map->tag, 0); - re_map = rspamd_mempool_alloc0 (pool, sizeof (*re_map)); + re_map = rspamd_mempool_alloc0_type (pool, struct rspamd_regexp_map_helper); re_map->pool = pool; re_map->values = g_ptr_array_new (); re_map->regexps = g_ptr_array_new (); diff --git a/src/libutil/mem_pool.h b/src/libutil/mem_pool.h index 1847980bb..2db36e03a 100644 --- a/src/libutil/mem_pool.h +++ b/src/libutil/mem_pool.h @@ -161,7 +161,8 @@ void *rspamd_mempool_alloc_ (rspamd_mempool_t *pool, gsize size, gsize alignment #define rspamd_mempool_alloc(pool, size) \ rspamd_mempool_alloc_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC)) #define rspamd_mempool_alloc_type(pool, type) \ - (type *)(rspamd_mempool_alloc_((pool), sizeof(type), RSPAMD_ALIGNOF(type), (G_STRLOC))) + (type *)(rspamd_mempool_alloc_((pool), sizeof(type), \ + MAX(MIN_MEM_ALIGNMENT, RSPAMD_ALIGNOF(type)), (G_STRLOC))) #define rspamd_mempool_alloc_buffer(pool, buflen) \ (char *)(rspamd_mempool_alloc_((pool), sizeof(char) * (buflen), MIN_MEM_ALIGNMENT, (G_STRLOC))) /** @@ -185,7 +186,8 @@ void *rspamd_mempool_alloc0_ (rspamd_mempool_t *pool, gsize size, gsize alignmen #define rspamd_mempool_alloc0(pool, size) \ rspamd_mempool_alloc0_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC)) #define rspamd_mempool_alloc0_type(pool, type) \ - (type *)(rspamd_mempool_alloc0_((pool), sizeof(type), RSPAMD_ALIGNOF(type), (G_STRLOC))) + (type *)(rspamd_mempool_alloc0_((pool), sizeof(type), \ + MAX(MIN_MEM_ALIGNMENT, RSPAMD_ALIGNOF(type)), (G_STRLOC))) /** * Make a copy of string in pool From vsevolod at highsecure.ru Sat Dec 4 14:56:08 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sat, 4 Dec 2021 14:56:08 +0000 (UTC) Subject: commit 23889da: [Minor] Update bundled xxhash Message-ID: <20211204145608.A3433272D4@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-04 14:53:23 +0000 URL: https://github.com/rspamd/rspamd/commit/23889da9ae115fbc23b43b856c06e94c12e3bb82 (HEAD -> master) [Minor] Update bundled xxhash --- contrib/DEPENDENCY_INFO.md | 2 +- contrib/xxhash/CMakeLists.txt | 2 +- contrib/xxhash/xxh3.h | 55 + contrib/xxhash/xxh_x86dispatch.c | 770 ++++++ contrib/xxhash/xxh_x86dispatch.h | 86 + contrib/xxhash/xxhash.c | 972 +------ contrib/xxhash/xxhash.h | 5610 +++++++++++++++++++++++++++++++++++++- src/libcryptobox/cryptobox.c | 2 - 8 files changed, 6464 insertions(+), 1035 deletions(-) diff --git a/contrib/DEPENDENCY_INFO.md b/contrib/DEPENDENCY_INFO.md index 7bfc0361d..f539d3238 100644 --- a/contrib/DEPENDENCY_INFO.md +++ b/contrib/DEPENDENCY_INFO.md @@ -24,7 +24,7 @@ | snowball | ? | BSD-3-Clause | NO | | | t1ha | ? | Zlib | NO | | | uthash | 1.9.8 | BSD | YES | | -| xxhash | ? | BSD | NO | | +| xxhash | 0.8.1 | BSD | NO | | | zstd | 1.4.5 | BSD | NO | | | google-ced | 37529e6 | Apache 2 | YES | build fixes | | kann | ? | MIT | YES | blas/lapack changes| diff --git a/contrib/xxhash/CMakeLists.txt b/contrib/xxhash/CMakeLists.txt index 8caf319e4..fb2a98ce9 100644 --- a/contrib/xxhash/CMakeLists.txt +++ b/contrib/xxhash/CMakeLists.txt @@ -1,4 +1,4 @@ -SET(XXHASHSRC xxhash.c) +SET(XXHASHSRC xxhash.c xxh_x86dispatch.c) ADD_LIBRARY(xxhash STATIC ${XXHASHSRC}) diff --git a/contrib/xxhash/xxh3.h b/contrib/xxhash/xxh3.h new file mode 100644 index 000000000..f7dc1959b --- /dev/null +++ b/contrib/xxhash/xxh3.h @@ -0,0 +1,55 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Development source file for `xxh3` + * Copyright (C) 2019-2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +/* + * Note: This file used to host the source code of XXH3_* variants. + * during the development period. + * The source code is now properly integrated within xxhash.h. + * + * xxh3.h is no longer useful, + * but it is still provided for compatibility with source code + * which used to include it directly. + * + * Programs are now highly discouraged to include xxh3.h. + * Include `xxhash.h` instead, which is the officially supported interface. + * + * In the future, xxh3.h will start to generate warnings, then errors, + * then it will be removed from source package and from include directory. + */ + +/* Simulate the same impact as including the old xxh3.h source file */ + +#define XXH_INLINE_ALL +#include "xxhash.h" diff --git a/contrib/xxhash/xxh_x86dispatch.c b/contrib/xxhash/xxh_x86dispatch.c new file mode 100644 index 000000000..399bad904 --- /dev/null +++ b/contrib/xxhash/xxh_x86dispatch.c @@ -0,0 +1,770 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Copyright (C) 2020 Yann Collet + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + + +/*! + * @file xxh_x86dispatch.c + * + * Automatic dispatcher code for the @ref xxh3_family on x86-based targets. + * + * Optional add-on. + * + * **Compile this file with the default flags for your target.** Do not compile + * with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`, there will be + * an error. See @ref XXH_X86DISPATCH_ALLOW_AVX for details. + * + * @defgroup dispatch x86 Dispatcher + * @{ + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)) +# error "Dispatching is currently only supported on x86 and x86_64." +#endif + +/*! + * @def XXH_X86DISPATCH_ALLOW_AVX + * @brief Disables the AVX sanity check. + * + * Don't compile xxh_x86dispatch.c with options like `-mavx*`, `-march=native`, + * or `/arch:AVX*`. It is intended to be compiled for the minimum target, and + * it selectively enables SSE2, AVX2, and AVX512 when it is needed. + * + * Using this option _globally_ allows this feature, and therefore makes it + * undefined behavior to execute on any CPU without said feature. + * + * Even if the source code isn't directly using AVX intrinsics in a function, + * the compiler can still generate AVX code from autovectorization and by + * "upgrading" SSE2 intrinsics to use the VEX prefixes (a.k.a. AVX128). + * + * Use the same flags that you use to compile the rest of the program; this + * file will safely generate SSE2, AVX2, and AVX512 without these flags. + * + * Define XXH_X86DISPATCH_ALLOW_AVX to ignore this check, and feel free to open + * an issue if there is a target in the future where AVX is a default feature. + */ +#ifdef XXH_DOXYGEN +# define XXH_X86DISPATCH_ALLOW_AVX +#endif + +#if defined(__AVX__) && !defined(XXH_X86DISPATCH_ALLOW_AVX) +# error "Do not compile xxh_x86dispatch.c with AVX enabled! See the comment above." +#endif + +#ifdef __has_include +# define XXH_HAS_INCLUDE(header) __has_include(header) +#else +# define XXH_HAS_INCLUDE(header) 0 +#endif + +/*! + * @def XXH_DISPATCH_SCALAR + * @brief Enables/dispatching the scalar code path. + * + * If this is defined to 0, SSE2 support is assumed. This reduces code size + * when the scalar path is not needed. + * + * This is automatically defined to 0 when... + * - SSE2 support is enabled in the compiler + * - Targeting x86_64 + * - Targeting Android x86 + * - Targeting macOS + */ +#ifndef XXH_DISPATCH_SCALAR +# if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \ + || defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \ + || defined(__ANDROID__) || defined(__APPLEv__) /* Android or macOS */ +# define XXH_DISPATCH_SCALAR 0 /* disable */ +# else +# define XXH_DISPATCH_SCALAR 1 +# endif +#endif +/*! + * @def XXH_DISPATCH_AVX2 + * @brief Enables/disables dispatching for AVX2. + * + * This is automatically detected if it is not defined. + * - GCC 4.7 and later are known to support AVX2, but >4.9 is required for + * to get the AVX2 intrinsics and typedefs without -mavx -mavx2. + * - Visual Studio 2013 Update 2 and later are known to support AVX2. + * - The GCC/Clang internal header `` is detected. While this is + * not allowed to be included directly, it still appears in the builtin + * include path and is detectable with `__has_include`. + * + * @see XXH_AVX2 + */ +#ifndef XXH_DISPATCH_AVX2 +# if (defined(__GNUC__) && (__GNUC__ > 4)) /* GCC 5.0+ */ \ + || (defined(_MSC_VER) && _MSC_VER >= 1900) /* VS 2015+ */ \ + || (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 180030501) /* VS 2013 Update 2 */ \ + || XXH_HAS_INCLUDE() /* GCC/Clang internal header */ +# define XXH_DISPATCH_AVX2 1 /* enable dispatch towards AVX2 */ +# else +# define XXH_DISPATCH_AVX2 0 +# endif +#endif /* XXH_DISPATCH_AVX2 */ + +/*! + * @def XXH_DISPATCH_AVX512 + * @brief Enables/disables dispatching for AVX512. + * + * Automatically detected if one of the following conditions is met: + * - GCC 4.9 and later are known to support AVX512. + * - Visual Studio 2017 and later are known to support AVX2. + * - The GCC/Clang internal header `` is detected. While this + * is not allowed to be included directly, it still appears in the builtin + * include path and is detectable with `__has_include`. + * + * @see XXH_AVX512 + */ +#ifndef XXH_DISPATCH_AVX512 +# if (defined(__GNUC__) \ + && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9))) /* GCC 4.9+ */ \ + || (defined(_MSC_VER) && _MSC_VER >= 1910) /* VS 2017+ */ \ + || XXH_HAS_INCLUDE() /* GCC/Clang internal header */ +# define XXH_DISPATCH_AVX512 1 /* enable dispatch towards AVX512 */ +# else +# define XXH_DISPATCH_AVX512 0 +# endif +#endif /* XXH_DISPATCH_AVX512 */ + +/*! + * @def XXH_TARGET_SSE2 + * @brief Allows a function to be compiled with SSE2 intrinsics. + * + * Uses `__attribute__((__target__("sse2")))` on GCC to allow SSE2 to be used + * even with `-mno-sse2`. + * + * @def XXH_TARGET_AVX2 + * @brief Like @ref XXH_TARGET_SSE2, but for AVX2. + * + * @def XXH_TARGET_AVX512 + * @brief Like @ref XXH_TARGET_SSE2, but for AVX512. + */ +#if defined(__GNUC__) +# include /* SSE2 */ +# if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 +# include /* AVX2, AVX512F */ +# endif +# define XXH_TARGET_SSE2 __attribute__((__target__("sse2"))) +# define XXH_TARGET_AVX2 __attribute__((__target__("avx2"))) +# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f"))) +#elif defined(_MSC_VER) +# include +# define XXH_TARGET_SSE2 +# define XXH_TARGET_AVX2 +# define XXH_TARGET_AVX512 +#else +# error "Dispatching is currently not supported for your compiler." +#endif + +#ifdef XXH_DISPATCH_DEBUG +/* debug logging */ +# include +# define XXH_debugPrint(str) { fprintf(stderr, "DEBUG: xxHash dispatch: %s \n", str); fflush(NULL); } +#else +# define XXH_debugPrint(str) ((void)0) +# undef NDEBUG /* avoid redefinition */ +# define NDEBUG +#endif +#include + +#define XXH_INLINE_ALL +#define XXH_X86DISPATCH +#include "xxhash.h" + +/* + * Support both AT&T and Intel dialects + * + * GCC doesn't convert AT&T syntax to Intel syntax, and will error out if + * compiled with -masm=intel. Instead, it supports dialect switching with + * curly braces: { AT&T syntax | Intel syntax } + * + * Clang's integrated assembler automatically converts AT&T syntax to Intel if + * needed, making the dialect switching useless (it isn't even supported). + * + * Note: Comments are written in the inline assembly itself. + */ +#ifdef __clang__ +# define XXH_I_ATT(intel, att) att "\n\t" +#else +# define XXH_I_ATT(intel, att) "{" att "|" intel "}\n\t" +#endif + +/*! + * @internal + * @brief Runs CPUID. + * + * @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively. + * @param abcd The array to store the result in, `{ eax, ebx, ecx, edx }` + */ +static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd) +{ +#if defined(_MSC_VER) + __cpuidex(abcd, eax, ecx); +#else + xxh_u32 ebx, edx; +# if defined(__i386__) && defined(__PIC__) + __asm__( + "# Call CPUID\n\t" + "#\n\t" + "# On 32-bit x86 with PIC enabled, we are not allowed to overwrite\n\t" + "# EBX, so we use EDI instead.\n\t" + XXH_I_ATT("mov edi, ebx", "movl %%ebx, %%edi") + XXH_I_ATT("cpuid", "cpuid" ) + XXH_I_ATT("xchg edi, ebx", "xchgl %%ebx, %%edi") + : "=D" (ebx), +# else + __asm__( + "# Call CPUID\n\t" + XXH_I_ATT("cpuid", "cpuid") + : "=b" (ebx), +# endif + "+a" (eax), "+c" (ecx), "=d" (edx)); + abcd[0] = eax; + abcd[1] = ebx; + abcd[2] = ecx; + abcd[3] = edx; +#endif +} + +/* + * Modified version of Intel's guide + * https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family + */ + +#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 +/*! + * @internal + * @brief Runs `XGETBV`. + * + * While the CPU may support AVX2, the operating system might not properly save + * the full YMM/ZMM registers. + * + * xgetbv is used for detecting this: Any compliant operating system will define + * a set of flags in the xcr0 register indicating how it saves the AVX registers. + * + * You can manually disable this flag on Windows by running, as admin: + * + * bcdedit.exe /set xsavedisable 1 + * + * and rebooting. Run the same command with 0 to re-enable it. + */ +static xxh_u64 XXH_xgetbv(void) +{ +#if defined(_MSC_VER) + return _xgetbv(0); /* min VS2010 SP1 compiler is required */ +#else + xxh_u32 xcr0_lo, xcr0_hi; + __asm__( + "# Call XGETBV\n\t" + "#\n\t" + "# Older assemblers (e.g. macOS's ancient GAS version) don't support\n\t" + "# the XGETBV opcode, so we encode it by hand instead.\n\t" + "# See for details.\n\t" + ".byte 0x0f, 0x01, 0xd0\n\t" + : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0)); + return xcr0_lo | ((xxh_u64)xcr0_hi << 32); +#endif +} +#endif + +#define XXH_SSE2_CPUID_MASK (1 << 26) +#define XXH_OSXSAVE_CPUID_MASK ((1 << 26) | (1 << 27)) +#define XXH_AVX2_CPUID_MASK (1 << 5) +#define XXH_AVX2_XGETBV_MASK ((1 << 2) | (1 << 1)) +#define XXH_AVX512F_CPUID_MASK (1 << 16) +#define XXH_AVX512F_XGETBV_MASK ((7 << 5) | (1 << 2) | (1 << 1)) + +/*! + * @internal + * @brief Returns the best XXH3 implementation. + * + * Runs various CPUID/XGETBV tests to try and determine the best implementation. + * + * @ret The best @ref XXH_VECTOR implementation. + * @see XXH_VECTOR_TYPES + */ +static int XXH_featureTest(void) +{ + xxh_u32 abcd[4]; + xxh_u32 max_leaves; + int best = XXH_SCALAR; +#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 + xxh_u64 xgetbv_val; +#endif +#if defined(__GNUC__) && defined(__i386__) + xxh_u32 cpuid_supported; + __asm__( + "# For the sake of ruthless backwards compatibility, check if CPUID\n\t" + "# is supported in the EFLAGS on i386.\n\t" + "# This is not necessary on x86_64 - CPUID is mandatory.\n\t" + "# The ID flag (bit 21) in the EFLAGS register indicates support\n\t" + "# for the CPUID instruction. If a software procedure can set and\n\t" + "# clear this flag, the processor executing the procedure supports\n\t" + "# the CPUID instruction.\n\t" + "# \n\t" + "#\n\t" + "# Routine is from .\n\t" + + "# Save EFLAGS\n\t" + XXH_I_ATT("pushfd", "pushfl" ) + "# Store EFLAGS\n\t" + XXH_I_ATT("pushfd", "pushfl" ) + "# Invert the ID bit in stored EFLAGS\n\t" + XXH_I_ATT("xor dword ptr[esp], 0x200000", "xorl $0x200000, (%%esp)") + "# Load stored EFLAGS (with ID bit inverted)\n\t" + XXH_I_ATT("popfd", "popfl" ) + "# Store EFLAGS again (ID bit may or not be inverted)\n\t" + XXH_I_ATT("pushfd", "pushfl" ) + "# eax = modified EFLAGS (ID bit may or may not be inverted)\n\t" + XXH_I_ATT("pop eax", "popl %%eax" ) + "# eax = whichever bits were changed\n\t" + XXH_I_ATT("xor eax, dword ptr[esp]", "xorl (%%esp), %%eax" ) + "# Restore original EFLAGS\n\t" + XXH_I_ATT("popfd", "popfl" ) + "# eax = zero if ID bit can't be changed, else non-zero\n\t" + XXH_I_ATT("and eax, 0x200000", "andl $0x200000, %%eax" ) + : "=a" (cpuid_supported) :: "cc"); + + if (XXH_unlikely(!cpuid_supported)) { + XXH_debugPrint("CPUID support is not detected!"); + return best; + } + +#endif + /* Check how many CPUID pages we have */ + XXH_cpuid(0, 0, abcd); + max_leaves = abcd[0]; + + /* Shouldn't happen on hardware, but happens on some QEMU configs. */ + if (XXH_unlikely(max_leaves == 0)) { + XXH_debugPrint("Max CPUID leaves == 0!"); + return best; + } + + /* Check for SSE2, OSXSAVE and xgetbv */ + XXH_cpuid(1, 0, abcd); + + /* + * Test for SSE2. The check is redundant on x86_64, but it doesn't hurt. + */ + if (XXH_unlikely((abcd[3] & XXH_SSE2_CPUID_MASK) != XXH_SSE2_CPUID_MASK)) + return best; + + XXH_debugPrint("SSE2 support detected."); + + best = XXH_SSE2; +#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 + /* Make sure we have enough leaves */ + if (XXH_unlikely(max_leaves < 7)) + return best; + + /* Test for OSXSAVE and XGETBV */ + if ((abcd[2] & XXH_OSXSAVE_CPUID_MASK) != XXH_OSXSAVE_CPUID_MASK) + return best; + + /* CPUID check for AVX features */ + XXH_cpuid(7, 0, abcd); + + xgetbv_val = XXH_xgetbv(); +#if XXH_DISPATCH_AVX2 + /* Validate that AVX2 is supported by the CPU */ + if ((abcd[1] & XXH_AVX2_CPUID_MASK) != XXH_AVX2_CPUID_MASK) + return best; + + /* Validate that the OS supports YMM registers */ + if ((xgetbv_val & XXH_AVX2_XGETBV_MASK) != XXH_AVX2_XGETBV_MASK) { + XXH_debugPrint("AVX2 supported by the CPU, but not the OS."); + return best; + } + + /* AVX2 supported */ + XXH_debugPrint("AVX2 support detected."); + best = XXH_AVX2; +#endif +#if XXH_DISPATCH_AVX512 + /* Check if AVX512F is supported by the CPU */ + if ((abcd[1] & XXH_AVX512F_CPUID_MASK) != XXH_AVX512F_CPUID_MASK) { + XXH_debugPrint("AVX512F not supported by CPU"); + return best; + } + + /* Validate that the OS supports ZMM registers */ + if ((xgetbv_val & XXH_AVX512F_XGETBV_MASK) != XXH_AVX512F_XGETBV_MASK) { + XXH_debugPrint("AVX512F supported by the CPU, but not the OS."); + return best; + } + + /* AVX512F supported */ + XXH_debugPrint("AVX512F support detected."); + best = XXH_AVX512; +#endif +#endif + return best; +} + + +/* === Vector implementations === */ + +/*! + * @internal + * @brief Defines the various dispatch functions. + * + * TODO: Consolidate? + * + * @param suffix The suffix for the functions, e.g. sse2 or scalar + * @param target XXH_TARGET_* or empty. + */ +#define XXH_DEFINE_DISPATCH_FUNCS(suffix, target) \ + \ +/* === XXH3, default variants === */ \ + \ +XXH_NO_INLINE target XXH64_hash_t \ +XXHL64_default_##suffix(const void* XXH_RESTRICT input, size_t len) \ +{ \ + return XXH3_hashLong_64b_internal( \ + input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ + ); \ +} \ + \ +/* === XXH3, Seeded variants === */ \ + \ +XXH_NO_INLINE target XXH64_hash_t \ +XXHL64_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \ + XXH64_hash_t seed) \ +{ \ + return XXH3_hashLong_64b_withSeed_internal( \ + input, len, seed, XXH3_accumulate_512_##suffix, \ + XXH3_scrambleAcc_##suffix, XXH3_initCustomSecret_##suffix \ + ); \ +} \ + \ +/* === XXH3, Secret variants === */ \ + \ +XXH_NO_INLINE target XXH64_hash_t \ +XXHL64_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \ + const void* secret, size_t secretLen) \ +{ \ + return XXH3_hashLong_64b_internal( \ + input, len, secret, secretLen, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ + ); \ +} \ + \ +/* === XXH3 update variants === */ \ + \ +XXH_NO_INLINE target XXH_errorcode \ +XXH3_update_##suffix(XXH3_state_t* state, const void* input, size_t len) \ +{ \ + return XXH3_update(state, (const xxh_u8*)input, len, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \ +} \ + \ +/* === XXH128 default variants === */ \ + \ +XXH_NO_INLINE target XXH128_hash_t \ +XXHL128_default_##suffix(const void* XXH_RESTRICT input, size_t len) \ +{ \ + return XXH3_hashLong_128b_internal( \ + input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ + ); \ +} \ + \ +/* === XXH128 Secret variants === */ \ + \ +XXH_NO_INLINE target XXH128_hash_t \ +XXHL128_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \ + const void* XXH_RESTRICT secret, size_t secretLen) \ +{ \ + return XXH3_hashLong_128b_internal( \ + input, len, (const xxh_u8*)secret, secretLen, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \ +} \ + \ +/* === XXH128 Seeded variants === */ \ + \ +XXH_NO_INLINE target XXH128_hash_t \ +XXHL128_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \ + XXH64_hash_t seed) \ +{ \ + return XXH3_hashLong_128b_withSeed_internal(input, len, seed, \ + XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix, \ + XXH3_initCustomSecret_##suffix); \ +} + +/* End XXH_DEFINE_DISPATCH_FUNCS */ + +#if XXH_DISPATCH_SCALAR +XXH_DEFINE_DISPATCH_FUNCS(scalar, /* nothing */) +#endif +XXH_DEFINE_DISPATCH_FUNCS(sse2, XXH_TARGET_SSE2) +#if XXH_DISPATCH_AVX2 +XXH_DEFINE_DISPATCH_FUNCS(avx2, XXH_TARGET_AVX2) +#endif +#if XXH_DISPATCH_AVX512 +XXH_DEFINE_DISPATCH_FUNCS(avx512, XXH_TARGET_AVX512) +#endif +#undef XXH_DEFINE_DISPATCH_FUNCS + +/* ==== Dispatchers ==== */ + +typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(const void* XXH_RESTRICT, size_t); + +typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t); + +typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t); + +typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH3_state_t*, const void*, size_t); + +typedef struct { + XXH3_dispatchx86_hashLong64_default hashLong64_default; + XXH3_dispatchx86_hashLong64_withSeed hashLong64_seed; + XXH3_dispatchx86_hashLong64_withSecret hashLong64_secret; + XXH3_dispatchx86_update update; +} XXH_dispatchFunctions_s; + +#define XXH_NB_DISPATCHES 4 + +/*! + * @internal + * @brief Table of dispatchers for @ref XXH3_64bits(). + * + * @pre The indices must match @ref XXH_VECTOR_TYPE. + */ +static const XXH_dispatchFunctions_s XXH_kDispatch[XXH_NB_DISPATCHES] = { +#if XXH_DISPATCH_SCALAR + /* Scalar */ { XXHL64_default_scalar, XXHL64_seed_scalar, XXHL64_secret_scalar, XXH3_update_scalar }, +#else + /* Scalar */ { NULL, NULL, NULL, NULL }, +#endif + /* SSE2 */ { XXHL64_default_sse2, XXHL64_seed_sse2, XXHL64_secret_sse2, XXH3_update_sse2 }, +#if XXH_DISPATCH_AVX2 + /* AVX2 */ { XXHL64_default_avx2, XXHL64_seed_avx2, XXHL64_secret_avx2, XXH3_update_avx2 }, +#else + /* AVX2 */ { NULL, NULL, NULL, NULL }, +#endif +#if XXH_DISPATCH_AVX512 + /* AVX512 */ { XXHL64_default_avx512, XXHL64_seed_avx512, XXHL64_secret_avx512, XXH3_update_avx512 } +#else + /* AVX512 */ { NULL, NULL, NULL, NULL } +#endif +}; +/*! + * @internal + * @brief The selected dispatch table for @ref XXH3_64bits(). + */ +static XXH_dispatchFunctions_s XXH_g_dispatch = { NULL, NULL, NULL, NULL }; *** OUTPUT TRUNCATED, 6938 LINES SKIPPED *** From vsevolod at highsecure.ru Sun Dec 5 18:21:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sun, 5 Dec 2021 18:21:04 +0000 (UTC) Subject: commit 7ad02fc: [Rework] Use xxh3 as a default hash and fix memory/alignment issues Message-ID: <20211205182104.932E8273C3@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-05 18:15:36 +0000 URL: https://github.com/rspamd/rspamd/commit/7ad02fc1f7e786a0db9f1e25f34d2771997a1c57 (HEAD -> master) [Rework] Use xxh3 as a default hash and fix memory/alignment issues --- contrib/xxhash/xxh_x86dispatch.h | 1 - src/libcryptobox/cryptobox.c | 56 +++++++++++++++++++++++++++++++++++----- src/libcryptobox/cryptobox.h | 13 ++++++++-- src/libserver/cfg_utils.c | 2 +- src/libutil/util.c | 1 - src/lua/lua_cryptobox.c | 12 ++++----- 6 files changed, 68 insertions(+), 17 deletions(-) diff --git a/contrib/xxhash/xxh_x86dispatch.h b/contrib/xxhash/xxh_x86dispatch.h index 6bc17bcbb..8e91fcf74 100644 --- a/contrib/xxhash/xxh_x86dispatch.h +++ b/contrib/xxhash/xxh_x86dispatch.h @@ -71,7 +71,6 @@ XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update_dispatch(XXH3_state_t* state, c # undef XXH128 # define XXH128 XXH3_128bits_withSeed_dispatch -# define XXH3_128bits XXH3_128bits_dispatch # undef XXH3_128bits # define XXH3_128bits XXH3_128bits_dispatch # undef XXH3_128bits_withSeed diff --git a/src/libcryptobox/cryptobox.c b/src/libcryptobox/cryptobox.c index d48cee16b..fe4d3df29 100644 --- a/src/libcryptobox/cryptobox.c +++ b/src/libcryptobox/cryptobox.c @@ -27,6 +27,8 @@ #include "base64/base64.h" #include "ottery.h" #include "printf.h" +#define XXH_INLINE_ALL +#define XXH_PRIVATE_API #include "xxhash.h" #define MUM_TARGET_INDEPENDENT_HASH 1 /* For 32/64 bit equal hashes */ #include "../../contrib/mumhash/mum.h" @@ -1461,7 +1463,7 @@ void rspamd_cryptobox_hash_final (rspamd_cryptobox_hash_state_t *p, guchar *out) { crypto_generichash_blake2b_state *st = cryptobox_align_ptr (p, - _Alignof(crypto_generichash_blake2b_state)); + RSPAMD_ALIGNOF(crypto_generichash_blake2b_state)); crypto_generichash_blake2b_final (st, out, crypto_generichash_blake2b_BYTES_MAX); } @@ -1480,6 +1482,8 @@ void rspamd_cryptobox_hash (guchar *out, G_STATIC_ASSERT (sizeof (t1ha_context_t) <= sizeof (((rspamd_cryptobox_fast_hash_state_t *)NULL)->opaque)); +G_STATIC_ASSERT (sizeof (struct XXH3_state_s) <= + sizeof (((rspamd_cryptobox_fast_hash_state_t *)NULL)->opaque)); struct RSPAMD_ALIGNED(16) _mum_iuf { @@ -1491,13 +1495,33 @@ struct RSPAMD_ALIGNED(16) _mum_iuf { unsigned rem; }; +rspamd_cryptobox_fast_hash_state_t* +rspamd_cryptobox_fast_hash_new(void) +{ + rspamd_cryptobox_fast_hash_state_t *nst; + int ret = posix_memalign ((void **)&nst, RSPAMD_ALIGNOF(rspamd_cryptobox_fast_hash_state_t), + sizeof(rspamd_cryptobox_fast_hash_state_t)); + + if (ret != 0) { + abort(); + } + + return nst; +} + +void +rspamd_cryptobox_fast_hash_free(rspamd_cryptobox_fast_hash_state_t *st) +{ + free(st); +} + void rspamd_cryptobox_fast_hash_init (rspamd_cryptobox_fast_hash_state_t *st, guint64 seed) { - t1ha_context_t *rst = (t1ha_context_t *)st->opaque; - st->type = RSPAMD_CRYPTOBOX_T1HA; - t1ha2_init (rst, seed, 0); + XXH3_state_t *rst = (XXH3_state_t *)st->opaque; + st->type = RSPAMD_CRYPTOBOX_XXHASH3; + XXH3_64bits_reset_withSeed (rst, seed); } void @@ -1527,6 +1551,13 @@ rspamd_cryptobox_fast_hash_init_specific (rspamd_cryptobox_fast_hash_state_t *st XXH32_reset (xst, seed); break; } + case RSPAMD_CRYPTOBOX_XXHASH3: + { + XXH3_state_t *xst = (XXH3_state_t *) st->opaque; + st->type = RSPAMD_CRYPTOBOX_XXHASH3; + XXH3_64bits_reset_withSeed (xst, seed); + break; + } case RSPAMD_CRYPTOBOX_MUMHASH: { struct _mum_iuf *iuf = (struct _mum_iuf *) st->opaque; st->type = RSPAMD_CRYPTOBOX_MUMHASH; @@ -1559,6 +1590,12 @@ rspamd_cryptobox_fast_hash_update (rspamd_cryptobox_fast_hash_state_t *st, XXH32_update (xst, data, len); break; } + case RSPAMD_CRYPTOBOX_XXHASH3: + { + XXH3_state_t *xst = (XXH3_state_t *) st->opaque; + XXH3_64bits_update (xst, data, len); + break; + } case RSPAMD_CRYPTOBOX_MUMHASH: { struct _mum_iuf *iuf = (struct _mum_iuf *) st->opaque; gsize drem = len; @@ -1629,6 +1666,11 @@ rspamd_cryptobox_fast_hash_final (rspamd_cryptobox_fast_hash_state_t *st) ret = XXH32_digest (xst); break; } + case RSPAMD_CRYPTOBOX_XXHASH3: { + XXH3_state_t *xst = (XXH3_state_t *) st->opaque; + ret = XXH3_64bits_digest (xst); + break; + } case RSPAMD_CRYPTOBOX_MUMHASH: { struct _mum_iuf *iuf = (struct _mum_iuf *) st->opaque; iuf->h = mum_hash_step (iuf->h, iuf->buf.ll); @@ -1656,14 +1698,14 @@ static inline guint64 rspamd_cryptobox_fast_hash_machdep (const void *data, gsize len, guint64 seed) { - return t1ha2_atonce (data, len, seed); + return XXH3_64bits_withSeed(data, len, seed); } static inline guint64 rspamd_cryptobox_fast_hash_indep (const void *data, gsize len, guint64 seed) { - return t1ha2_atonce (data, len, seed); + return XXH3_64bits_withSeed(data, len, seed); } guint64 @@ -1682,6 +1724,8 @@ rspamd_cryptobox_fast_hash_specific ( switch (type) { case RSPAMD_CRYPTOBOX_XXHASH32: return XXH32 (data, len, seed); + case RSPAMD_CRYPTOBOX_XXHASH3: + return XXH3_64bits_withSeed (data, len, seed); case RSPAMD_CRYPTOBOX_XXHASH64: return XXH64 (data, len, seed); case RSPAMD_CRYPTOBOX_MUMHASH: diff --git a/src/libcryptobox/cryptobox.h b/src/libcryptobox/cryptobox.h index aa93f8972..e7d2dc79a 100644 --- a/src/libcryptobox/cryptobox.h +++ b/src/libcryptobox/cryptobox.h @@ -348,6 +348,7 @@ void rspamd_cryptobox_hash (guchar *out, enum rspamd_cryptobox_fast_hash_type { RSPAMD_CRYPTOBOX_XXHASH64 = 0, RSPAMD_CRYPTOBOX_XXHASH32, + RSPAMD_CRYPTOBOX_XXHASH3, RSPAMD_CRYPTOBOX_MUMHASH, RSPAMD_CRYPTOBOX_T1HA, RSPAMD_CRYPTOBOX_HASHFAST, @@ -355,11 +356,19 @@ enum rspamd_cryptobox_fast_hash_type { }; /* Non crypto hash IUF interface */ -typedef struct rspamd_cryptobox_fast_hash_state_s { - guint64 opaque[11]; +typedef struct CRYPTO_ALIGN(64) rspamd_cryptobox_fast_hash_state_s { + guchar opaque[576]; /* Required for xxhash3 */ enum rspamd_cryptobox_fast_hash_type type; } rspamd_cryptobox_fast_hash_state_t; + +/** + * Creates a new cryptobox state properly aligned + * @return + */ +rspamd_cryptobox_fast_hash_state_t* rspamd_cryptobox_fast_hash_new(void); +void rspamd_cryptobox_fast_hash_free(rspamd_cryptobox_fast_hash_state_t *st); + /** * Init cryptobox hash state using key if needed, `st` must point to the buffer * with at least rspamd_cryptobox_HASHSTATEBYTES bytes length. If keylen == 0, then diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c index 3f699a843..67ceb5df8 100644 --- a/src/libserver/cfg_utils.c +++ b/src/libserver/cfg_utils.c @@ -164,7 +164,7 @@ rspamd_config_new (enum rspamd_config_init_flags flags) rspamd_mempool_t *pool; pool = rspamd_mempool_new (8 * 1024 * 1024, "cfg", 0); - cfg = rspamd_mempool_alloc0 (pool, sizeof (*cfg)); + cfg = rspamd_mempool_alloc0_type(pool, struct rspamd_config); /* Allocate larger pool for cfg */ cfg->cfg_pool = pool; cfg->dns_timeout = 1.0; diff --git a/src/libutil/util.c b/src/libutil/util.c index 27631ae65..2b0dfa9c1 100644 --- a/src/libutil/util.c +++ b/src/libutil/util.c @@ -17,7 +17,6 @@ #include "util.h" #include "unix-std.h" -#include "xxhash.h" #include "ottery.h" #include "cryptobox.h" diff --git a/src/lua/lua_cryptobox.c b/src/lua/lua_cryptobox.c index f16fd8b67..7d1b8e4a9 100644 --- a/src/lua/lua_cryptobox.c +++ b/src/lua/lua_cryptobox.c @@ -1011,7 +1011,7 @@ lua_cryptobox_hash_dtor (struct rspamd_lua_cryptobox_hash *h) free (h->content.h); /* Allocated by posix_memalign */ } else { - g_free (h->content.fh); + rspamd_cryptobox_fast_hash_free (h->content.fh); } g_free (h); @@ -1023,7 +1023,7 @@ rspamd_lua_hash_init_default (struct rspamd_lua_cryptobox_hash *h, { h->type = LUA_CRYPTOBOX_HASH_BLAKE2; if (posix_memalign ((void **)&h->content.h, - _Alignof (rspamd_cryptobox_hash_state_t), + RSPAMD_ALIGNOF(rspamd_cryptobox_hash_state_t), sizeof (*h->content.h)) != 0) { g_assert_not_reached (); } @@ -1128,28 +1128,28 @@ rspamd_lua_hash_create (const gchar *type, const gchar *key, gsize keylen) } else if (g_ascii_strcasecmp (type, "xxh64") == 0) { h->type = LUA_CRYPTOBOX_HASH_XXHASH64; - h->content.fh = g_malloc0 (sizeof (*h->content.fh)); + h->content.fh = rspamd_cryptobox_fast_hash_new (); rspamd_cryptobox_fast_hash_init_specific (h->content.fh, RSPAMD_CRYPTOBOX_XXHASH64, 0); h->out_len = sizeof (guint64); } else if (g_ascii_strcasecmp (type, "xxh32") == 0) { h->type = LUA_CRYPTOBOX_HASH_XXHASH32; - h->content.fh = g_malloc0 (sizeof (*h->content.fh)); + h->content.fh = rspamd_cryptobox_fast_hash_new (); rspamd_cryptobox_fast_hash_init_specific (h->content.fh, RSPAMD_CRYPTOBOX_XXHASH32, 0); h->out_len = sizeof (guint32); } else if (g_ascii_strcasecmp (type, "mum") == 0) { h->type = LUA_CRYPTOBOX_HASH_MUM; - h->content.fh = g_malloc0 (sizeof (*h->content.fh)); + h->content.fh = rspamd_cryptobox_fast_hash_new (); rspamd_cryptobox_fast_hash_init_specific (h->content.fh, RSPAMD_CRYPTOBOX_MUMHASH, 0); h->out_len = sizeof (guint64); } else if (g_ascii_strcasecmp (type, "t1ha") == 0) { h->type = LUA_CRYPTOBOX_HASH_T1HA; - h->content.fh = g_malloc0 (sizeof (*h->content.fh)); + h->content.fh = rspamd_cryptobox_fast_hash_new (); rspamd_cryptobox_fast_hash_init_specific (h->content.fh, RSPAMD_CRYPTOBOX_T1HA, 0); h->out_len = sizeof (guint64); From vsevolod at highsecure.ru Sun Dec 5 18:28:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sun, 5 Dec 2021 18:28:04 +0000 (UTC) Subject: commit 0d4bf7e: [Minor] Add xxh3 support to lua_cryptobox Message-ID: <20211205182804.1F2F5273C6@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-05 18:18:38 +0000 URL: https://github.com/rspamd/rspamd/commit/0d4bf7ef10f5c93f3f0ee828d353bb9da2c27d48 [Minor] Add xxh3 support to lua_cryptobox --- src/lua/lua_cryptobox.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/lua/lua_cryptobox.c b/src/lua/lua_cryptobox.c index 7d1b8e4a9..7c19a7805 100644 --- a/src/lua/lua_cryptobox.c +++ b/src/lua/lua_cryptobox.c @@ -45,6 +45,7 @@ enum lua_cryptobox_hash_type { LUA_CRYPTOBOX_HASH_HMAC, LUA_CRYPTOBOX_HASH_XXHASH64, LUA_CRYPTOBOX_HASH_XXHASH32, + LUA_CRYPTOBOX_HASH_XXHASH3, LUA_CRYPTOBOX_HASH_MUM, LUA_CRYPTOBOX_HASH_T1HA, }; @@ -977,6 +978,7 @@ rspamd_lua_hash_update (struct rspamd_lua_cryptobox_hash *h, break; case LUA_CRYPTOBOX_HASH_XXHASH64: case LUA_CRYPTOBOX_HASH_XXHASH32: + case LUA_CRYPTOBOX_HASH_XXHASH3: case LUA_CRYPTOBOX_HASH_MUM: case LUA_CRYPTOBOX_HASH_T1HA: rspamd_cryptobox_fast_hash_update (h->content.fh, p, len); @@ -1140,6 +1142,13 @@ rspamd_lua_hash_create (const gchar *type, const gchar *key, gsize keylen) RSPAMD_CRYPTOBOX_XXHASH32, 0); h->out_len = sizeof (guint32); } + else if (g_ascii_strcasecmp (type, "xxh3") == 0) { + h->type = LUA_CRYPTOBOX_HASH_XXHASH3; + h->content.fh = rspamd_cryptobox_fast_hash_new (); + rspamd_cryptobox_fast_hash_init_specific (h->content.fh, + RSPAMD_CRYPTOBOX_XXHASH3, 0); + h->out_len = sizeof (guint64); + } else if (g_ascii_strcasecmp (type, "mum") == 0) { h->type = LUA_CRYPTOBOX_HASH_MUM; h->content.fh = rspamd_cryptobox_fast_hash_new (); @@ -1467,6 +1476,10 @@ lua_cryptobox_hash_reset (lua_State *L) rspamd_cryptobox_fast_hash_init_specific (h->content.fh, RSPAMD_CRYPTOBOX_XXHASH32, 0); break; + case LUA_CRYPTOBOX_HASH_XXHASH3: + rspamd_cryptobox_fast_hash_init_specific (h->content.fh, + RSPAMD_CRYPTOBOX_XXHASH3, 0); + break; case LUA_CRYPTOBOX_HASH_MUM: rspamd_cryptobox_fast_hash_init_specific (h->content.fh, RSPAMD_CRYPTOBOX_MUMHASH, 0); @@ -1518,6 +1531,7 @@ lua_cryptobox_hash_finish (struct rspamd_lua_cryptobox_hash *h) break; case LUA_CRYPTOBOX_HASH_XXHASH64: case LUA_CRYPTOBOX_HASH_XXHASH32: + case LUA_CRYPTOBOX_HASH_XXHASH3: case LUA_CRYPTOBOX_HASH_MUM: case LUA_CRYPTOBOX_HASH_T1HA: ll = rspamd_cryptobox_fast_hash_final (h->content.fh); From vsevolod at highsecure.ru Sun Dec 5 18:28:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sun, 5 Dec 2021 18:28:05 +0000 (UTC) Subject: commit 5f34f23: [Test] Add a small test for xxh3 Message-ID: <20211205182805.3355F273C8@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-05 18:22:41 +0000 URL: https://github.com/rspamd/rspamd/commit/5f34f2323a97909a303f12df9c47f291b36a84b5 (HEAD -> master) [Test] Add a small test for xxh3 --- test/functional/lua/hashes.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/functional/lua/hashes.lua b/test/functional/lua/hashes.lua index d934770e5..f02e01c4e 100644 --- a/test/functional/lua/hashes.lua +++ b/test/functional/lua/hashes.lua @@ -40,6 +40,12 @@ rspamd_config:register_symbol({ ['hex'] = 'cf25ddc406c50de0c13de2b79d127646', ['reset'] = true, }, + { + ['init'] = 'hello', + ['specific'] = 'xxh3', + ['str'] = 'hello', + ['hex'] = 'c1156ae6cb7ff175', + } } for _, t in ipairs(test_data) do From vsevolod at highsecure.ru Mon Dec 6 11:35:06 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Mon, 6 Dec 2021 11:35:06 +0000 (UTC) Subject: commit d2bad43: [Minor] Remove unused x86 dispatcher Message-ID: <20211206113506.5388427452@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-06 11:30:19 +0000 URL: https://github.com/rspamd/rspamd/commit/d2bad43bb3559c3a14536ca14274210924906e5e (HEAD -> master) [Minor] Remove unused x86 dispatcher --- contrib/xxhash/CMakeLists.txt | 2 +- contrib/xxhash/xxh_x86dispatch.c | 770 --------------------------------------- contrib/xxhash/xxh_x86dispatch.h | 85 ----- 3 files changed, 1 insertion(+), 856 deletions(-) diff --git a/contrib/xxhash/CMakeLists.txt b/contrib/xxhash/CMakeLists.txt index fb2a98ce9..8caf319e4 100644 --- a/contrib/xxhash/CMakeLists.txt +++ b/contrib/xxhash/CMakeLists.txt @@ -1,4 +1,4 @@ -SET(XXHASHSRC xxhash.c xxh_x86dispatch.c) +SET(XXHASHSRC xxhash.c) ADD_LIBRARY(xxhash STATIC ${XXHASHSRC}) diff --git a/contrib/xxhash/xxh_x86dispatch.c b/contrib/xxhash/xxh_x86dispatch.c deleted file mode 100644 index 399bad904..000000000 --- a/contrib/xxhash/xxh_x86dispatch.c +++ /dev/null @@ -1,770 +0,0 @@ -/* - * xxHash - Extremely Fast Hash algorithm - * Copyright (C) 2020 Yann Collet - * - * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * You can contact the author at: - * - xxHash homepage: https://www.xxhash.com - * - xxHash source repository: https://github.com/Cyan4973/xxHash - */ - - -/*! - * @file xxh_x86dispatch.c - * - * Automatic dispatcher code for the @ref xxh3_family on x86-based targets. - * - * Optional add-on. - * - * **Compile this file with the default flags for your target.** Do not compile - * with flags like `-mavx*`, `-march=native`, or `/arch:AVX*`, there will be - * an error. See @ref XXH_X86DISPATCH_ALLOW_AVX for details. - * - * @defgroup dispatch x86 Dispatcher - * @{ - */ - -#if defined (__cplusplus) -extern "C" { -#endif - -#if !(defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64)) -# error "Dispatching is currently only supported on x86 and x86_64." -#endif - -/*! - * @def XXH_X86DISPATCH_ALLOW_AVX - * @brief Disables the AVX sanity check. - * - * Don't compile xxh_x86dispatch.c with options like `-mavx*`, `-march=native`, - * or `/arch:AVX*`. It is intended to be compiled for the minimum target, and - * it selectively enables SSE2, AVX2, and AVX512 when it is needed. - * - * Using this option _globally_ allows this feature, and therefore makes it - * undefined behavior to execute on any CPU without said feature. - * - * Even if the source code isn't directly using AVX intrinsics in a function, - * the compiler can still generate AVX code from autovectorization and by - * "upgrading" SSE2 intrinsics to use the VEX prefixes (a.k.a. AVX128). - * - * Use the same flags that you use to compile the rest of the program; this - * file will safely generate SSE2, AVX2, and AVX512 without these flags. - * - * Define XXH_X86DISPATCH_ALLOW_AVX to ignore this check, and feel free to open - * an issue if there is a target in the future where AVX is a default feature. - */ -#ifdef XXH_DOXYGEN -# define XXH_X86DISPATCH_ALLOW_AVX -#endif - -#if defined(__AVX__) && !defined(XXH_X86DISPATCH_ALLOW_AVX) -# error "Do not compile xxh_x86dispatch.c with AVX enabled! See the comment above." -#endif - -#ifdef __has_include -# define XXH_HAS_INCLUDE(header) __has_include(header) -#else -# define XXH_HAS_INCLUDE(header) 0 -#endif - -/*! - * @def XXH_DISPATCH_SCALAR - * @brief Enables/dispatching the scalar code path. - * - * If this is defined to 0, SSE2 support is assumed. This reduces code size - * when the scalar path is not needed. - * - * This is automatically defined to 0 when... - * - SSE2 support is enabled in the compiler - * - Targeting x86_64 - * - Targeting Android x86 - * - Targeting macOS - */ -#ifndef XXH_DISPATCH_SCALAR -# if defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) /* SSE2 on by default */ \ - || defined(__x86_64__) || defined(_M_X64) /* x86_64 */ \ - || defined(__ANDROID__) || defined(__APPLEv__) /* Android or macOS */ -# define XXH_DISPATCH_SCALAR 0 /* disable */ -# else -# define XXH_DISPATCH_SCALAR 1 -# endif -#endif -/*! - * @def XXH_DISPATCH_AVX2 - * @brief Enables/disables dispatching for AVX2. - * - * This is automatically detected if it is not defined. - * - GCC 4.7 and later are known to support AVX2, but >4.9 is required for - * to get the AVX2 intrinsics and typedefs without -mavx -mavx2. - * - Visual Studio 2013 Update 2 and later are known to support AVX2. - * - The GCC/Clang internal header `` is detected. While this is - * not allowed to be included directly, it still appears in the builtin - * include path and is detectable with `__has_include`. - * - * @see XXH_AVX2 - */ -#ifndef XXH_DISPATCH_AVX2 -# if (defined(__GNUC__) && (__GNUC__ > 4)) /* GCC 5.0+ */ \ - || (defined(_MSC_VER) && _MSC_VER >= 1900) /* VS 2015+ */ \ - || (defined(_MSC_FULL_VER) && _MSC_FULL_VER >= 180030501) /* VS 2013 Update 2 */ \ - || XXH_HAS_INCLUDE() /* GCC/Clang internal header */ -# define XXH_DISPATCH_AVX2 1 /* enable dispatch towards AVX2 */ -# else -# define XXH_DISPATCH_AVX2 0 -# endif -#endif /* XXH_DISPATCH_AVX2 */ - -/*! - * @def XXH_DISPATCH_AVX512 - * @brief Enables/disables dispatching for AVX512. - * - * Automatically detected if one of the following conditions is met: - * - GCC 4.9 and later are known to support AVX512. - * - Visual Studio 2017 and later are known to support AVX2. - * - The GCC/Clang internal header `` is detected. While this - * is not allowed to be included directly, it still appears in the builtin - * include path and is detectable with `__has_include`. - * - * @see XXH_AVX512 - */ -#ifndef XXH_DISPATCH_AVX512 -# if (defined(__GNUC__) \ - && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9))) /* GCC 4.9+ */ \ - || (defined(_MSC_VER) && _MSC_VER >= 1910) /* VS 2017+ */ \ - || XXH_HAS_INCLUDE() /* GCC/Clang internal header */ -# define XXH_DISPATCH_AVX512 1 /* enable dispatch towards AVX512 */ -# else -# define XXH_DISPATCH_AVX512 0 -# endif -#endif /* XXH_DISPATCH_AVX512 */ - -/*! - * @def XXH_TARGET_SSE2 - * @brief Allows a function to be compiled with SSE2 intrinsics. - * - * Uses `__attribute__((__target__("sse2")))` on GCC to allow SSE2 to be used - * even with `-mno-sse2`. - * - * @def XXH_TARGET_AVX2 - * @brief Like @ref XXH_TARGET_SSE2, but for AVX2. - * - * @def XXH_TARGET_AVX512 - * @brief Like @ref XXH_TARGET_SSE2, but for AVX512. - */ -#if defined(__GNUC__) -# include /* SSE2 */ -# if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 -# include /* AVX2, AVX512F */ -# endif -# define XXH_TARGET_SSE2 __attribute__((__target__("sse2"))) -# define XXH_TARGET_AVX2 __attribute__((__target__("avx2"))) -# define XXH_TARGET_AVX512 __attribute__((__target__("avx512f"))) -#elif defined(_MSC_VER) -# include -# define XXH_TARGET_SSE2 -# define XXH_TARGET_AVX2 -# define XXH_TARGET_AVX512 -#else -# error "Dispatching is currently not supported for your compiler." -#endif - -#ifdef XXH_DISPATCH_DEBUG -/* debug logging */ -# include -# define XXH_debugPrint(str) { fprintf(stderr, "DEBUG: xxHash dispatch: %s \n", str); fflush(NULL); } -#else -# define XXH_debugPrint(str) ((void)0) -# undef NDEBUG /* avoid redefinition */ -# define NDEBUG -#endif -#include - -#define XXH_INLINE_ALL -#define XXH_X86DISPATCH -#include "xxhash.h" - -/* - * Support both AT&T and Intel dialects - * - * GCC doesn't convert AT&T syntax to Intel syntax, and will error out if - * compiled with -masm=intel. Instead, it supports dialect switching with - * curly braces: { AT&T syntax | Intel syntax } - * - * Clang's integrated assembler automatically converts AT&T syntax to Intel if - * needed, making the dialect switching useless (it isn't even supported). - * - * Note: Comments are written in the inline assembly itself. - */ -#ifdef __clang__ -# define XXH_I_ATT(intel, att) att "\n\t" -#else -# define XXH_I_ATT(intel, att) "{" att "|" intel "}\n\t" -#endif - -/*! - * @internal - * @brief Runs CPUID. - * - * @param eax , ecx The parameters to pass to CPUID, %eax and %ecx respectively. - * @param abcd The array to store the result in, `{ eax, ebx, ecx, edx }` - */ -static void XXH_cpuid(xxh_u32 eax, xxh_u32 ecx, xxh_u32* abcd) -{ -#if defined(_MSC_VER) - __cpuidex(abcd, eax, ecx); -#else - xxh_u32 ebx, edx; -# if defined(__i386__) && defined(__PIC__) - __asm__( - "# Call CPUID\n\t" - "#\n\t" - "# On 32-bit x86 with PIC enabled, we are not allowed to overwrite\n\t" - "# EBX, so we use EDI instead.\n\t" - XXH_I_ATT("mov edi, ebx", "movl %%ebx, %%edi") - XXH_I_ATT("cpuid", "cpuid" ) - XXH_I_ATT("xchg edi, ebx", "xchgl %%ebx, %%edi") - : "=D" (ebx), -# else - __asm__( - "# Call CPUID\n\t" - XXH_I_ATT("cpuid", "cpuid") - : "=b" (ebx), -# endif - "+a" (eax), "+c" (ecx), "=d" (edx)); - abcd[0] = eax; - abcd[1] = ebx; - abcd[2] = ecx; - abcd[3] = edx; -#endif -} - -/* - * Modified version of Intel's guide - * https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family - */ - -#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 -/*! - * @internal - * @brief Runs `XGETBV`. - * - * While the CPU may support AVX2, the operating system might not properly save - * the full YMM/ZMM registers. - * - * xgetbv is used for detecting this: Any compliant operating system will define - * a set of flags in the xcr0 register indicating how it saves the AVX registers. - * - * You can manually disable this flag on Windows by running, as admin: - * - * bcdedit.exe /set xsavedisable 1 - * - * and rebooting. Run the same command with 0 to re-enable it. - */ -static xxh_u64 XXH_xgetbv(void) -{ -#if defined(_MSC_VER) - return _xgetbv(0); /* min VS2010 SP1 compiler is required */ -#else - xxh_u32 xcr0_lo, xcr0_hi; - __asm__( - "# Call XGETBV\n\t" - "#\n\t" - "# Older assemblers (e.g. macOS's ancient GAS version) don't support\n\t" - "# the XGETBV opcode, so we encode it by hand instead.\n\t" - "# See for details.\n\t" - ".byte 0x0f, 0x01, 0xd0\n\t" - : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0)); - return xcr0_lo | ((xxh_u64)xcr0_hi << 32); -#endif -} -#endif - -#define XXH_SSE2_CPUID_MASK (1 << 26) -#define XXH_OSXSAVE_CPUID_MASK ((1 << 26) | (1 << 27)) -#define XXH_AVX2_CPUID_MASK (1 << 5) -#define XXH_AVX2_XGETBV_MASK ((1 << 2) | (1 << 1)) -#define XXH_AVX512F_CPUID_MASK (1 << 16) -#define XXH_AVX512F_XGETBV_MASK ((7 << 5) | (1 << 2) | (1 << 1)) - -/*! - * @internal - * @brief Returns the best XXH3 implementation. - * - * Runs various CPUID/XGETBV tests to try and determine the best implementation. - * - * @ret The best @ref XXH_VECTOR implementation. - * @see XXH_VECTOR_TYPES - */ -static int XXH_featureTest(void) -{ - xxh_u32 abcd[4]; - xxh_u32 max_leaves; - int best = XXH_SCALAR; -#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 - xxh_u64 xgetbv_val; -#endif -#if defined(__GNUC__) && defined(__i386__) - xxh_u32 cpuid_supported; - __asm__( - "# For the sake of ruthless backwards compatibility, check if CPUID\n\t" - "# is supported in the EFLAGS on i386.\n\t" - "# This is not necessary on x86_64 - CPUID is mandatory.\n\t" - "# The ID flag (bit 21) in the EFLAGS register indicates support\n\t" - "# for the CPUID instruction. If a software procedure can set and\n\t" - "# clear this flag, the processor executing the procedure supports\n\t" - "# the CPUID instruction.\n\t" - "# \n\t" - "#\n\t" - "# Routine is from .\n\t" - - "# Save EFLAGS\n\t" - XXH_I_ATT("pushfd", "pushfl" ) - "# Store EFLAGS\n\t" - XXH_I_ATT("pushfd", "pushfl" ) - "# Invert the ID bit in stored EFLAGS\n\t" - XXH_I_ATT("xor dword ptr[esp], 0x200000", "xorl $0x200000, (%%esp)") - "# Load stored EFLAGS (with ID bit inverted)\n\t" - XXH_I_ATT("popfd", "popfl" ) - "# Store EFLAGS again (ID bit may or not be inverted)\n\t" - XXH_I_ATT("pushfd", "pushfl" ) - "# eax = modified EFLAGS (ID bit may or may not be inverted)\n\t" - XXH_I_ATT("pop eax", "popl %%eax" ) - "# eax = whichever bits were changed\n\t" - XXH_I_ATT("xor eax, dword ptr[esp]", "xorl (%%esp), %%eax" ) - "# Restore original EFLAGS\n\t" - XXH_I_ATT("popfd", "popfl" ) - "# eax = zero if ID bit can't be changed, else non-zero\n\t" - XXH_I_ATT("and eax, 0x200000", "andl $0x200000, %%eax" ) - : "=a" (cpuid_supported) :: "cc"); - - if (XXH_unlikely(!cpuid_supported)) { - XXH_debugPrint("CPUID support is not detected!"); - return best; - } - -#endif - /* Check how many CPUID pages we have */ - XXH_cpuid(0, 0, abcd); - max_leaves = abcd[0]; - - /* Shouldn't happen on hardware, but happens on some QEMU configs. */ - if (XXH_unlikely(max_leaves == 0)) { - XXH_debugPrint("Max CPUID leaves == 0!"); - return best; - } - - /* Check for SSE2, OSXSAVE and xgetbv */ - XXH_cpuid(1, 0, abcd); - - /* - * Test for SSE2. The check is redundant on x86_64, but it doesn't hurt. - */ - if (XXH_unlikely((abcd[3] & XXH_SSE2_CPUID_MASK) != XXH_SSE2_CPUID_MASK)) - return best; - - XXH_debugPrint("SSE2 support detected."); - - best = XXH_SSE2; -#if XXH_DISPATCH_AVX2 || XXH_DISPATCH_AVX512 - /* Make sure we have enough leaves */ - if (XXH_unlikely(max_leaves < 7)) - return best; - - /* Test for OSXSAVE and XGETBV */ - if ((abcd[2] & XXH_OSXSAVE_CPUID_MASK) != XXH_OSXSAVE_CPUID_MASK) - return best; - - /* CPUID check for AVX features */ - XXH_cpuid(7, 0, abcd); - - xgetbv_val = XXH_xgetbv(); -#if XXH_DISPATCH_AVX2 - /* Validate that AVX2 is supported by the CPU */ - if ((abcd[1] & XXH_AVX2_CPUID_MASK) != XXH_AVX2_CPUID_MASK) - return best; - - /* Validate that the OS supports YMM registers */ - if ((xgetbv_val & XXH_AVX2_XGETBV_MASK) != XXH_AVX2_XGETBV_MASK) { - XXH_debugPrint("AVX2 supported by the CPU, but not the OS."); - return best; - } - - /* AVX2 supported */ - XXH_debugPrint("AVX2 support detected."); - best = XXH_AVX2; -#endif -#if XXH_DISPATCH_AVX512 - /* Check if AVX512F is supported by the CPU */ - if ((abcd[1] & XXH_AVX512F_CPUID_MASK) != XXH_AVX512F_CPUID_MASK) { - XXH_debugPrint("AVX512F not supported by CPU"); - return best; - } - - /* Validate that the OS supports ZMM registers */ - if ((xgetbv_val & XXH_AVX512F_XGETBV_MASK) != XXH_AVX512F_XGETBV_MASK) { - XXH_debugPrint("AVX512F supported by the CPU, but not the OS."); - return best; - } - - /* AVX512F supported */ - XXH_debugPrint("AVX512F support detected."); - best = XXH_AVX512; -#endif -#endif - return best; -} - - -/* === Vector implementations === */ - -/*! - * @internal - * @brief Defines the various dispatch functions. - * - * TODO: Consolidate? - * - * @param suffix The suffix for the functions, e.g. sse2 or scalar - * @param target XXH_TARGET_* or empty. - */ -#define XXH_DEFINE_DISPATCH_FUNCS(suffix, target) \ - \ -/* === XXH3, default variants === */ \ - \ -XXH_NO_INLINE target XXH64_hash_t \ -XXHL64_default_##suffix(const void* XXH_RESTRICT input, size_t len) \ -{ \ - return XXH3_hashLong_64b_internal( \ - input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \ - XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ - ); \ -} \ - \ -/* === XXH3, Seeded variants === */ \ - \ -XXH_NO_INLINE target XXH64_hash_t \ -XXHL64_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \ - XXH64_hash_t seed) \ -{ \ - return XXH3_hashLong_64b_withSeed_internal( \ - input, len, seed, XXH3_accumulate_512_##suffix, \ - XXH3_scrambleAcc_##suffix, XXH3_initCustomSecret_##suffix \ - ); \ -} \ - \ -/* === XXH3, Secret variants === */ \ - \ -XXH_NO_INLINE target XXH64_hash_t \ -XXHL64_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \ - const void* secret, size_t secretLen) \ -{ \ - return XXH3_hashLong_64b_internal( \ - input, len, secret, secretLen, \ - XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ - ); \ -} \ - \ -/* === XXH3 update variants === */ \ - \ -XXH_NO_INLINE target XXH_errorcode \ -XXH3_update_##suffix(XXH3_state_t* state, const void* input, size_t len) \ -{ \ - return XXH3_update(state, (const xxh_u8*)input, len, \ - XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \ -} \ - \ -/* === XXH128 default variants === */ \ - \ -XXH_NO_INLINE target XXH128_hash_t \ -XXHL128_default_##suffix(const void* XXH_RESTRICT input, size_t len) \ -{ \ - return XXH3_hashLong_128b_internal( \ - input, len, XXH3_kSecret, sizeof(XXH3_kSecret), \ - XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix \ - ); \ -} \ - \ -/* === XXH128 Secret variants === */ \ - \ -XXH_NO_INLINE target XXH128_hash_t \ -XXHL128_secret_##suffix(const void* XXH_RESTRICT input, size_t len, \ - const void* XXH_RESTRICT secret, size_t secretLen) \ -{ \ - return XXH3_hashLong_128b_internal( \ - input, len, (const xxh_u8*)secret, secretLen, \ - XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix); \ -} \ - \ -/* === XXH128 Seeded variants === */ \ - \ -XXH_NO_INLINE target XXH128_hash_t \ -XXHL128_seed_##suffix(const void* XXH_RESTRICT input, size_t len, \ - XXH64_hash_t seed) \ -{ \ - return XXH3_hashLong_128b_withSeed_internal(input, len, seed, \ - XXH3_accumulate_512_##suffix, XXH3_scrambleAcc_##suffix, \ - XXH3_initCustomSecret_##suffix); \ -} - -/* End XXH_DEFINE_DISPATCH_FUNCS */ - -#if XXH_DISPATCH_SCALAR -XXH_DEFINE_DISPATCH_FUNCS(scalar, /* nothing */) -#endif -XXH_DEFINE_DISPATCH_FUNCS(sse2, XXH_TARGET_SSE2) -#if XXH_DISPATCH_AVX2 -XXH_DEFINE_DISPATCH_FUNCS(avx2, XXH_TARGET_AVX2) -#endif -#if XXH_DISPATCH_AVX512 -XXH_DEFINE_DISPATCH_FUNCS(avx512, XXH_TARGET_AVX512) -#endif -#undef XXH_DEFINE_DISPATCH_FUNCS - -/* ==== Dispatchers ==== */ - -typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_default)(const void* XXH_RESTRICT, size_t); - -typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t); - -typedef XXH64_hash_t (*XXH3_dispatchx86_hashLong64_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t); - -typedef XXH_errorcode (*XXH3_dispatchx86_update)(XXH3_state_t*, const void*, size_t); - -typedef struct { - XXH3_dispatchx86_hashLong64_default hashLong64_default; - XXH3_dispatchx86_hashLong64_withSeed hashLong64_seed; - XXH3_dispatchx86_hashLong64_withSecret hashLong64_secret; - XXH3_dispatchx86_update update; -} XXH_dispatchFunctions_s; - -#define XXH_NB_DISPATCHES 4 - -/*! - * @internal - * @brief Table of dispatchers for @ref XXH3_64bits(). - * - * @pre The indices must match @ref XXH_VECTOR_TYPE. - */ -static const XXH_dispatchFunctions_s XXH_kDispatch[XXH_NB_DISPATCHES] = { -#if XXH_DISPATCH_SCALAR - /* Scalar */ { XXHL64_default_scalar, XXHL64_seed_scalar, XXHL64_secret_scalar, XXH3_update_scalar }, -#else - /* Scalar */ { NULL, NULL, NULL, NULL }, -#endif - /* SSE2 */ { XXHL64_default_sse2, XXHL64_seed_sse2, XXHL64_secret_sse2, XXH3_update_sse2 }, -#if XXH_DISPATCH_AVX2 - /* AVX2 */ { XXHL64_default_avx2, XXHL64_seed_avx2, XXHL64_secret_avx2, XXH3_update_avx2 }, -#else - /* AVX2 */ { NULL, NULL, NULL, NULL }, -#endif -#if XXH_DISPATCH_AVX512 - /* AVX512 */ { XXHL64_default_avx512, XXHL64_seed_avx512, XXHL64_secret_avx512, XXH3_update_avx512 } -#else - /* AVX512 */ { NULL, NULL, NULL, NULL } -#endif -}; -/*! - * @internal - * @brief The selected dispatch table for @ref XXH3_64bits(). - */ -static XXH_dispatchFunctions_s XXH_g_dispatch = { NULL, NULL, NULL, NULL }; - - -typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_default)(const void* XXH_RESTRICT, size_t); - -typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSeed)(const void* XXH_RESTRICT, size_t, XXH64_hash_t); - -typedef XXH128_hash_t (*XXH3_dispatchx86_hashLong128_withSecret)(const void* XXH_RESTRICT, size_t, const void* XXH_RESTRICT, size_t); - -typedef struct { - XXH3_dispatchx86_hashLong128_default hashLong128_default; - XXH3_dispatchx86_hashLong128_withSeed hashLong128_seed; - XXH3_dispatchx86_hashLong128_withSecret hashLong128_secret; - XXH3_dispatchx86_update update; -} XXH_dispatch128Functions_s; - - -/*! - * @internal - * @brief Table of dispatchers for @ref XXH3_128bits(). - * - * @pre The indices must match @ref XXH_VECTOR_TYPE. - */ -static const XXH_dispatch128Functions_s XXH_kDispatch128[XXH_NB_DISPATCHES] = { -#if XXH_DISPATCH_SCALAR - /* Scalar */ { XXHL128_default_scalar, XXHL128_seed_scalar, XXHL128_secret_scalar, XXH3_update_scalar }, -#else - /* Scalar */ { NULL, NULL, NULL, NULL }, -#endif - /* SSE2 */ { XXHL128_default_sse2, XXHL128_seed_sse2, XXHL128_secret_sse2, XXH3_update_sse2 }, -#if XXH_DISPATCH_AVX2 - /* AVX2 */ { XXHL128_default_avx2, XXHL128_seed_avx2, XXHL128_secret_avx2, XXH3_update_avx2 }, -#else - /* AVX2 */ { NULL, NULL, NULL, NULL }, -#endif -#if XXH_DISPATCH_AVX512 - /* AVX512 */ { XXHL128_default_avx512, XXHL128_seed_avx512, XXHL128_secret_avx512, XXH3_update_avx512 } -#else - /* AVX512 */ { NULL, NULL, NULL, NULL } -#endif -}; - -/*! - * @internal - * @brief The selected dispatch table for @ref XXH3_64bits(). - */ -static XXH_dispatch128Functions_s XXH_g_dispatch128 = { NULL, NULL, NULL, NULL }; - -/*! - * @internal - * @brief Runs a CPUID check and sets the correct dispatch tables. - */ -static void XXH_setDispatch(void) -{ - int vecID = XXH_featureTest(); - XXH_STATIC_ASSERT(XXH_AVX512 == XXH_NB_DISPATCHES-1); - assert(XXH_SCALAR <= vecID && vecID <= XXH_AVX512); -#if !XXH_DISPATCH_SCALAR - assert(vecID != XXH_SCALAR); -#endif -#if !XXH_DISPATCH_AVX512 - assert(vecID != XXH_AVX512); -#endif -#if !XXH_DISPATCH_AVX2 - assert(vecID != XXH_AVX2); -#endif - XXH_g_dispatch = XXH_kDispatch[vecID]; - XXH_g_dispatch128 = XXH_kDispatch128[vecID]; -} - - -/* ==== XXH3 public functions ==== */ - -static XXH64_hash_t -XXH3_hashLong_64b_defaultSecret_selection(const void* input, size_t len, - XXH64_hash_t seed64, const xxh_u8* secret, size_t secretLen) -{ - (void)seed64; (void)secret; (void)secretLen; - if (XXH_g_dispatch.hashLong64_default == NULL) XXH_setDispatch(); - return XXH_g_dispatch.hashLong64_default(input, len); *** OUTPUT TRUNCATED, 189 LINES SKIPPED *** From vsevolod at highsecure.ru Mon Dec 6 11:42:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Mon, 6 Dec 2021 11:42:04 +0000 (UTC) Subject: commit df914aa: [Minor] Polish alignment macros Message-ID: <20211206114204.4C89627454@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-06 11:39:54 +0000 URL: https://github.com/rspamd/rspamd/commit/df914aa64f9f9b6c5a472a438f5d6f7991d26068 (HEAD -> master) [Minor] Polish alignment macros --- config.h.in | 11 +++++++++++ src/libcryptobox/cryptobox.c | 6 +++--- src/libserver/re_cache.c | 2 +- src/libutil/mem_pool.h | 6 +----- src/libutil/multipattern.c | 4 ++-- src/libutil/str_util.h | 2 +- 6 files changed, 19 insertions(+), 12 deletions(-) diff --git a/config.h.in b/config.h.in index d6a3fc1d0..9487fd76e 100644 --- a/config.h.in +++ b/config.h.in @@ -322,6 +322,17 @@ typedef off_t goffset; #endif #endif +#ifndef __cplusplus +# ifdef G_ALIGNOF +# define RSPAMD_ALIGNOF G_ALIGNOF +# else +# define RSPAMD_ALIGNOF(t) _Alignof(t) +# endif +#else +/* glib G_ALIGNOF nor C11 _Alignof are not good enough for C++, nuff said... */ +# define RSPAMD_ALIGNOF(t) alignof(t) +#endif + /* Address sanitizer */ #ifdef __clang__ # if __has_feature(address_sanitizer) diff --git a/src/libcryptobox/cryptobox.c b/src/libcryptobox/cryptobox.c index fe4d3df29..41f13ed73 100644 --- a/src/libcryptobox/cryptobox.c +++ b/src/libcryptobox/cryptobox.c @@ -588,7 +588,7 @@ static gsize rspamd_cryptobox_auth_ctx_len (enum rspamd_cryptobox_mode mode) { if (G_LIKELY (mode == RSPAMD_CRYPTOBOX_MODE_25519)) { - return sizeof (crypto_onetimeauth_state) + _Alignof (crypto_onetimeauth_state); + return sizeof (crypto_onetimeauth_state) + RSPAMD_ALIGNOF(crypto_onetimeauth_state); } else { #ifndef HAVE_USABLE_OPENSSL @@ -1440,7 +1440,7 @@ void rspamd_cryptobox_hash_init (rspamd_cryptobox_hash_state_t *p, const guchar *key, gsize keylen) { crypto_generichash_blake2b_state *st = cryptobox_align_ptr (p, - _Alignof(crypto_generichash_blake2b_state)); + RSPAMD_ALIGNOF(crypto_generichash_blake2b_state)); crypto_generichash_blake2b_init (st, key, keylen, crypto_generichash_blake2b_BYTES_MAX); } @@ -1452,7 +1452,7 @@ void rspamd_cryptobox_hash_update (rspamd_cryptobox_hash_state_t *p, const guchar *data, gsize len) { crypto_generichash_blake2b_state *st = cryptobox_align_ptr (p, - _Alignof(crypto_generichash_blake2b_state)); + RSPAMD_ALIGNOF(crypto_generichash_blake2b_state)); crypto_generichash_blake2b_update (st, data, len); } diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c index 3016d04c5..2c5555154 100644 --- a/src/libserver/re_cache.c +++ b/src/libserver/re_cache.c @@ -406,7 +406,7 @@ rspamd_re_cache_init (struct rspamd_re_cache *cache, struct rspamd_config *cfg) rspamd_regexp_set_cache_id (re, i); if (re_class->st == NULL) { - (void) !posix_memalign ((void **)&re_class->st, _Alignof (rspamd_cryptobox_hash_state_t), + (void) !posix_memalign ((void **)&re_class->st, RSPAMD_ALIGNOF(rspamd_cryptobox_hash_state_t), sizeof (*re_class->st)); g_assert (re_class->st != NULL); rspamd_cryptobox_hash_init (re_class->st, NULL, 0); diff --git a/src/libutil/mem_pool.h b/src/libutil/mem_pool.h index 2db36e03a..c369f116a 100644 --- a/src/libutil/mem_pool.h +++ b/src/libutil/mem_pool.h @@ -68,11 +68,7 @@ struct f_str_s; #define MEMPOOL_UID_LEN 20 /* All pointers are aligned as this variable */ #define MIN_MEM_ALIGNMENT G_MEM_ALIGN -#ifndef __cplusplus -#define RSPAMD_ALIGNOF G_ALIGNOF -#else -#define RSPAMD_ALIGNOF(t) alignof(t) -#endif + /** * Destructor type definition */ diff --git a/src/libutil/multipattern.c b/src/libutil/multipattern.c index 2842b1d59..8f959f486 100644 --- a/src/libutil/multipattern.c +++ b/src/libutil/multipattern.c @@ -256,7 +256,7 @@ rspamd_multipattern_create (enum rspamd_multipattern_flags flags) struct rspamd_multipattern *mp; /* Align due to blake2b state */ - (void) !posix_memalign((void **)&mp, _Alignof (struct rspamd_multipattern), + (void) !posix_memalign((void **)&mp, RSPAMD_ALIGNOF(struct rspamd_multipattern), sizeof (*mp)); g_assert (mp != NULL); memset (mp, 0, sizeof (*mp)); @@ -285,7 +285,7 @@ rspamd_multipattern_create_sized (guint npatterns, struct rspamd_multipattern *mp; /* Align due to blake2b state */ - (void) !posix_memalign((void **)&mp, _Alignof (struct rspamd_multipattern), sizeof (*mp)); + (void) !posix_memalign((void **)&mp, RSPAMD_ALIGNOF(struct rspamd_multipattern), sizeof (*mp)); g_assert (mp != NULL); memset (mp, 0, sizeof (*mp)); mp->flags = flags; diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h index b08dd56ad..47abf062e 100644 --- a/src/libutil/str_util.h +++ b/src/libutil/str_util.h @@ -473,7 +473,7 @@ gsize rspamd_memspn (const gchar *s, const gchar *e, gsize len); * Check if a pointer is aligned; n must be power of two */ #define rspamd_is_aligned(p, n) (((uintptr_t)(p) & ((uintptr_t)(n) - 1)) == 0) -#define rspamd_is_aligned_as(p, v) rspamd_is_aligned(p, _Alignof(__typeof((v)))) +#define rspamd_is_aligned_as(p, v) rspamd_is_aligned(p, RSPAMD_ALIGNOF(__typeof((v)))) gboolean rspamd_str_has_8bit (const guchar *beg, gsize len); struct UConverter; From vsevolod at highsecure.ru Mon Dec 6 11:49:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Mon, 6 Dec 2021 11:49:04 +0000 (UTC) Subject: commit 7a90007: [Minor] Backport ucl fix Message-ID: <20211206114904.3369D27457@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-06 11:45:52 +0000 URL: https://github.com/rspamd/rspamd/commit/7a9000757ddcefbb3013c2218e5d43d92acc7e38 (HEAD -> master) [Minor] Backport ucl fix --- contrib/libucl/ucl_parser.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/contrib/libucl/ucl_parser.c b/contrib/libucl/ucl_parser.c index 1d285f27d..8ccbf05af 100644 --- a/contrib/libucl/ucl_parser.c +++ b/contrib/libucl/ucl_parser.c @@ -850,6 +850,10 @@ ucl_maybe_parse_number (ucl_object_t *obj, dv *= ucl_lex_num_multiplier (*p, false); } p += 2; + if (end - p > 0 && !ucl_lex_is_atom_end (*p)) { + *pos = start; + return EINVAL; + } goto set_obj; } else if (number_bytes || (p[1] == 'b' || p[1] == 'B')) { @@ -860,6 +864,10 @@ ucl_maybe_parse_number (ucl_object_t *obj, } lv *= ucl_lex_num_multiplier (*p, true); p += 2; + if (end - p > 0 && !ucl_lex_is_atom_end (*p)) { + *pos = start; + return EINVAL; + } goto set_obj; } else if (ucl_lex_is_atom_end (p[1])) { @@ -884,6 +892,10 @@ ucl_maybe_parse_number (ucl_object_t *obj, is_time = true; dv *= 60.; p += 3; + if (end - p > 0 && !ucl_lex_is_atom_end (*p)) { + *pos = start; + return EINVAL; + } goto set_obj; } } @@ -896,6 +908,10 @@ ucl_maybe_parse_number (ucl_object_t *obj, lv *= ucl_lex_num_multiplier (*p, number_bytes); } p ++; + if (end - p > 0 && !ucl_lex_is_atom_end (*p)) { + *pos = start; + return EINVAL; + } goto set_obj; } break; From vsevolod at highsecure.ru Mon Dec 6 13:42:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Mon, 6 Dec 2021 13:42:04 +0000 (UTC) Subject: commit 3f57a0d: [Test] Remove broken test Message-ID: <20211206134204.4DF4A27468@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-06 13:31:56 +0000 URL: https://github.com/rspamd/rspamd/commit/3f57a0d2d90a874eee6cb15242304827cc64844a (HEAD -> master) [Test] Remove broken test --- test/lua/unit/selectors.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/lua/unit/selectors.lua b/test/lua/unit/selectors.lua index 270d06941..5eff52528 100644 --- a/test/lua/unit/selectors.lua +++ b/test/lua/unit/selectors.lua @@ -201,9 +201,10 @@ context("Selectors test", function() selector = "specific_urls({need_emails = true, limit = 2})", expect = {{"test at example.net", "http://subdomain.example.net"}}}, - ["specific_urls + emails limit"] = { - selector = "specific_urls({need_emails = true, limit = 1})", - expect = {{"test at example.net"}}}, + -- Broken test as order depends on the hash function internally + --["specific_urls + emails limit"] = { + -- selector = "specific_urls({need_emails = true, limit = 1})", + -- expect = {{"test at example.net"}}}, ["pool_var str, default type"] = { selector = [[pool_var("str_var")]], From vsevolod at highsecure.ru Tue Dec 7 14:07:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 7 Dec 2021 14:07:04 +0000 (UTC) Subject: commit cd9ba26: [Minor] Dmarc_report: Fix automatic dates filling Message-ID: <20211207140704.4BCB327533@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-07 14:05:09 +0000 URL: https://github.com/rspamd/rspamd/commit/cd9ba26ecb13c50fa3200f7d80394cdfd8b0089a (HEAD -> master) [Minor] Dmarc_report: Fix automatic dates filling --- lualib/rspamadm/dmarc_report.lua | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lualib/rspamadm/dmarc_report.lua b/lualib/rspamadm/dmarc_report.lua index 561ea599d..0d4897c27 100644 --- a/lualib/rspamadm/dmarc_report.lua +++ b/lualib/rspamadm/dmarc_report.lua @@ -618,6 +618,10 @@ local function process_report_date(opts, start_time, date) end local function handler(args) + local start_time + -- Preserve start time as report sending might take some time + local start_collection = os.time() + local opts = parser:parse(args) pool = rspamd_mempool.create() @@ -653,7 +657,6 @@ local function handler(args) 'GET', 'rspamd_dmarc_last_collection' }) - local start_time if not ret or not tonumber(results) then start_time = os.time() - 86400 else @@ -663,9 +666,10 @@ local function handler(args) lua_util.debugm(N, 'previous last report date is %s', start_time) if not opts.date or #opts.date == 0 then - local now = os.time() + local now = start_collection opts.date = {} - while now >= start_time do + -- Allow some fuzz when adding dates + while now >= start_time - 60 do table.insert(opts.date, os.date('!%Y%m%d', now)) now = now - 86400 end @@ -689,7 +693,7 @@ local function handler(args) local function finish_cb(nsuccess, nfail) if not opts.no_opt then - lua_util.debugm(N, 'set last report date to %s', os.time()) + lua_util.debugm(N, 'set last report date to %s', start_collection) -- Hack to avoid coroutines + async functions mess: we use async redis call here redis_attrs.callback = function() logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed', @@ -697,7 +701,7 @@ local function handler(args) end lua_redis.request(redis_params, redis_attrs, {'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2, - tostring(os.time())}) + tostring(start_collection)}) else logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed', ndates, nreports, nsuccess, nfail) From vsevolod at highsecure.ru Tue Dec 7 18:49:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 7 Dec 2021 18:49:04 +0000 (UTC) Subject: commit fe5a7f1: [Minor] Try to fix keepalive for SSL connections Message-ID: <20211207184904.6C5342755B@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-07 18:47:35 +0000 URL: https://github.com/rspamd/rspamd/commit/fe5a7f1829f106b5ca9a901121ea9cc04dab3df6 (HEAD -> master) [Minor] Try to fix keepalive for SSL connections --- src/libserver/http/http_connection.c | 39 +++++++++++++++++------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/src/libserver/http/http_connection.c b/src/libserver/http/http_connection.c index e3e32eef8..0a5dee754 100644 --- a/src/libserver/http/http_connection.c +++ b/src/libserver/http/http_connection.c @@ -2303,29 +2303,26 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn return FALSE; } else { - if (priv->ssl) { - /* Cleanup the existing connection */ - rspamd_ssl_connection_free (priv->ssl); - } - - priv->ssl = rspamd_ssl_connection_new (ssl_ctx, priv->ctx->event_loop, - !(msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY), - conn->log_tag); - g_assert (priv->ssl != NULL); + if (!priv->ssl) { + priv->ssl = rspamd_ssl_connection_new (ssl_ctx, priv->ctx->event_loop, + !(msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY), + conn->log_tag); + g_assert (priv->ssl != NULL); - if (!rspamd_ssl_connect_fd (priv->ssl, conn->fd, host, &priv->ev, - priv->timeout, rspamd_http_event_handler, - rspamd_http_ssl_err_handler, conn)) { + if (!rspamd_ssl_connect_fd (priv->ssl, conn->fd, host, &priv->ev, + priv->timeout, rspamd_http_event_handler, + rspamd_http_ssl_err_handler, conn)) { - err = g_error_new (HTTP_ERROR, 400, - "ssl connection error: ssl error=%s, errno=%s", - ERR_error_string (ERR_get_error (), NULL), - strerror (errno)); - rspamd_http_connection_ref (conn); - conn->error_handler (conn, err); - rspamd_http_connection_unref (conn); - g_error_free (err); - return FALSE; + err = g_error_new (HTTP_ERROR, 400, + "ssl connection error: ssl error=%s, errno=%s", + ERR_error_string (ERR_get_error (), NULL), + strerror (errno)); + rspamd_http_connection_ref (conn); + conn->error_handler (conn, err); + rspamd_http_connection_unref (conn); + g_error_free (err); + return FALSE; + } } } } From vsevolod at highsecure.ru Tue Dec 7 21:35:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 7 Dec 2021 21:35:04 +0000 (UTC) Subject: commit df8519e: [Minor] Bimi: Rework redis caching logic Message-ID: <20211207213504.592BA27579@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-07 21:30:56 +0000 URL: https://github.com/rspamd/rspamd/commit/df8519e3234fc78ed59017abdb965ad6c8e341b4 (HEAD -> master) [Minor] Bimi: Rework redis caching logic --- src/plugins/lua/bimi.lua | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/src/plugins/lua/bimi.lua b/src/plugins/lua/bimi.lua index a622aed5e..e7e2527e4 100644 --- a/src/plugins/lua/bimi.lua +++ b/src/plugins/lua/bimi.lua @@ -136,6 +136,8 @@ end local function make_helper_request(task, domain, record, redis_server) local is_sync = settings.helper_sync local helper_url = string.format('%s/v1/check', settings.helper_url) + local redis_key = string.format('%s%s', settings.redis_prefix, + domain) local function http_helper_callback(http_err, code, body, _) if http_err then @@ -163,6 +165,31 @@ local function make_helper_request(task, domain, record, redis_server) lua_util.debugm(N, task, "invalid BIMI for %s: %s", domain, d.error) end + + local ret, upstream + local function redis_set_cb(redis_err, _) + if redis_err then + rspamd_logger.warnx(task, 'cannot get reply from Redis when storing image %s: %s', + upstream:get_addr():to_string(), redis_err) + upstream:fail() + else + lua_util.debugm(N, task, 'stored bimi image in Redis for domain %s; key=%s', + domain, redis_key) + end + end + + ret,_,upstream = lua_redis.redis_make_request(task, + redis_params, -- connect params + redis_key, -- hash key + true, -- is write + redis_set_cb, --callback + 'PSETEX', -- command + {redis_key, tostring(settings.redis_min_expiry * 1000.0), d.content}) + + if not ret then + rspamd_logger.warnx(task, 'cannot make request to Redis when storing image; domain %s', + domain) + end end else -- In async mode we skip request and use merely Redis to insert indicators @@ -174,12 +201,18 @@ local function make_helper_request(task, domain, record, redis_server) local request_data = { url = record.a, sync = is_sync, - redis_server = redis_server, - redis_prefix = settings.redis_prefix, - redis_expiry = settings.redis_min_expiry * 1000.0, -- helper accepts milliseconds domain = domain } + if not is_sync then + -- Allow bimi helper to save data in Redis + request_data.redis_server = redis_server + request_data.redis_prefix = settings.redis_prefix + request_data.redis_expiry = settings.redis_min_expiry * 1000.0 + else + request_data.skip_redis = true + end + local serialised = ucl.to_format(request_data, 'json-compact') lua_util.debugm(N, task, "send request to BIMI helper: %s", serialised) @@ -202,7 +235,7 @@ local function check_bimi_vmc(task, domain, record) local function redis_cached_cb(err, data) if err then rspamd_logger.warnx(task, 'cannot get reply from Redis %s: %s', - upstream:get_addr():to_string()) + upstream:get_addr():to_string(), err) upstream:fail() else if type(data) == 'string' then @@ -233,8 +266,8 @@ local function check_bimi_vmc(task, domain, record) -- We first check Redis and then try to use helper ret,_,upstream = lua_redis.redis_make_request(task, redis_params, -- connect params - nil, -- hash key - true, -- is write + redis_key, -- hash key + false, -- is write redis_cached_cb, --callback 'GET', -- command {redis_key}) From vsevolod at highsecure.ru Wed Dec 8 11:49:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 8 Dec 2021 11:49:04 +0000 (UTC) Subject: commit 707357d: [Rework] Include SSL flag into keepalive hash Message-ID: <20211208114904.52C64275F6@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-08 10:49:29 +0000 URL: https://github.com/rspamd/rspamd/commit/707357d35e12ee833f358418f380b685100e70b9 [Rework] Include SSL flag into keepalive hash --- src/libserver/http/http_connection.c | 4 ++-- src/libserver/http/http_connection.h | 2 +- src/libserver/http/http_context.c | 44 ++++++++++++++++++++++++------------ src/libserver/http/http_context.h | 12 ++++------ src/libserver/http/http_private.h | 1 + 5 files changed, 39 insertions(+), 24 deletions(-) diff --git a/src/libserver/http/http_connection.c b/src/libserver/http/http_connection.c index 0a5dee754..478e00984 100644 --- a/src/libserver/http/http_connection.c +++ b/src/libserver/http/http_connection.c @@ -1255,7 +1255,7 @@ rspamd_http_connection_new_keepalive (struct rspamd_http_context *ctx, ctx = rspamd_http_context_default (); } - conn = rspamd_http_context_check_keepalive (ctx, addr, host); + conn = rspamd_http_context_check_keepalive(ctx, addr, host, false); if (conn) { return conn; @@ -1267,7 +1267,7 @@ rspamd_http_connection_new_keepalive (struct rspamd_http_context *ctx, addr); if (conn) { - rspamd_http_context_prepare_keepalive (ctx, conn, addr, host); + rspamd_http_context_prepare_keepalive(ctx, conn, addr, host, ); } return conn; diff --git a/src/libserver/http/http_connection.h b/src/libserver/http/http_connection.h index 896f83c20..cc7c8a8f1 100644 --- a/src/libserver/http/http_connection.h +++ b/src/libserver/http/http_connection.h @@ -68,7 +68,7 @@ struct rspamd_storage_shmem { */ #define RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE (1 << 3) /** - * Use tls for this message + * Use tls for this message (how the fuck SSL flag could be used PER MESSAGE???) */ #define RSPAMD_HTTP_FLAG_SSL (1 << 4) /** diff --git a/src/libserver/http/http_context.c b/src/libserver/http/http_context.c index 75bfbf2cf..b56f1c4c0 100644 --- a/src/libserver/http/http_context.c +++ b/src/libserver/http/http_context.c @@ -365,21 +365,29 @@ rspamd_http_context_default (void) gint32 rspamd_keep_alive_key_hash (struct rspamd_keepalive_hash_key *k) { - gint32 h; + guint32 h; h = rspamd_inet_address_port_hash (k->addr); if (k->host) { - h = rspamd_cryptobox_fast_hash (k->host, strlen (k->host), h); + h ^= rspamd_cryptobox_fast_hash (k->host, strlen (k->host), h); } - return h; + if (k->is_ssl) { + h = ~h; + } + + return (gint32)h; } bool rspamd_keep_alive_key_equal (struct rspamd_keepalive_hash_key *k1, struct rspamd_keepalive_hash_key *k2) { + if (k1->is_ssl != k2->is_ssl) { + return false; + } + if (k1->host && k2->host) { if (rspamd_inet_address_port_equal (k1->addr, k2->addr)) { return strcmp (k1->host, k2->host) == 0; @@ -393,16 +401,18 @@ rspamd_keep_alive_key_equal (struct rspamd_keepalive_hash_key *k1, return false; } -struct rspamd_http_connection* -rspamd_http_context_check_keepalive (struct rspamd_http_context *ctx, - const rspamd_inet_addr_t *addr, - const gchar *host) +struct rspamd_http_connection * +rspamd_http_context_check_keepalive(struct rspamd_http_context *ctx, + const rspamd_inet_addr_t *addr, + const gchar *host, + bool is_ssl) { struct rspamd_keepalive_hash_key hk, *phk; khiter_t k; hk.addr = (rspamd_inet_addr_t *)addr; hk.host = (gchar *)host; + hk.is_ssl = is_ssl; k = kh_get (rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk); @@ -430,20 +440,23 @@ rspamd_http_context_check_keepalive (struct rspamd_http_context *ctx, if (err != 0) { rspamd_http_connection_unref (conn); - msg_debug_http_context ("invalid reused keepalive element %s (%s); " + msg_debug_http_context ("invalid reused keepalive element %s (%s, ssl=%b); " "%s error; " "%d connections queued", rspamd_inet_address_to_string_pretty (phk->addr), phk->host, + phk->is_ssl, g_strerror (err), conns->length); return NULL; } - msg_debug_http_context ("reused keepalive element %s (%s), %d connections queued", + msg_debug_http_context ("reused keepalive element %s (%s, ssl=%b), %d connections queued", rspamd_inet_address_to_string_pretty (phk->addr), - phk->host, conns->length); + phk->host, + phk->is_ssl, + conns->length); /* We transfer refcount here! */ return conn; @@ -459,16 +472,18 @@ rspamd_http_context_check_keepalive (struct rspamd_http_context *ctx, } void -rspamd_http_context_prepare_keepalive (struct rspamd_http_context *ctx, - struct rspamd_http_connection *conn, - const rspamd_inet_addr_t *addr, - const gchar *host) +rspamd_http_context_prepare_keepalive(struct rspamd_http_context *ctx, + struct rspamd_http_connection *conn, + const rspamd_inet_addr_t *addr, + const gchar *host, + bool is_ssl) { struct rspamd_keepalive_hash_key hk, *phk; khiter_t k; hk.addr = (rspamd_inet_addr_t *)addr; hk.host = (gchar *)host; + hk.is_ssl = is_ssl; k = kh_get (rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk); @@ -487,6 +502,7 @@ rspamd_http_context_prepare_keepalive (struct rspamd_http_context *ctx, phk = g_malloc (sizeof (*phk)); phk->conns = empty_init; phk->host = g_strdup (host); + phk->is_ssl = is_ssl; phk->addr = rspamd_inet_address_copy (addr); kh_put (rspamd_keep_alive_hash, ctx->keep_alive_hash, phk, &r); diff --git a/src/libserver/http/http_context.h b/src/libserver/http/http_context.h index 82ee400b0..f42164dba 100644 --- a/src/libserver/http/http_context.h +++ b/src/libserver/http/http_context.h @@ -74,9 +74,9 @@ struct rspamd_http_context *rspamd_http_context_default (void); * @param host * @return */ -struct rspamd_http_connection *rspamd_http_context_check_keepalive ( - struct rspamd_http_context *ctx, const rspamd_inet_addr_t *addr, - const gchar *host); +struct rspamd_http_connection * +rspamd_http_context_check_keepalive(struct rspamd_http_context *ctx, const rspamd_inet_addr_t *addr, const gchar *host, + bool is_ssl); /** * Prepares keepalive key for a connection by creating a new entry or by reusing existent @@ -86,10 +86,8 @@ struct rspamd_http_connection *rspamd_http_context_check_keepalive ( * @param addr * @param host */ -void rspamd_http_context_prepare_keepalive (struct rspamd_http_context *ctx, - struct rspamd_http_connection *conn, - const rspamd_inet_addr_t *addr, - const gchar *host); +void rspamd_http_context_prepare_keepalive(struct rspamd_http_context *ctx, struct rspamd_http_connection *conn, + const rspamd_inet_addr_t *addr, const gchar *host, bool is_ssl); /** * Pushes a connection to keepalive pool after client request is finished, diff --git a/src/libserver/http/http_private.h b/src/libserver/http/http_private.h index f2270277b..6306d197b 100644 --- a/src/libserver/http/http_private.h +++ b/src/libserver/http/http_private.h @@ -86,6 +86,7 @@ struct rspamd_http_message { struct rspamd_keepalive_hash_key { rspamd_inet_addr_t *addr; gchar *host; + bool is_ssl; GQueue conns; }; From vsevolod at highsecure.ru Wed Dec 8 11:49:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 8 Dec 2021 11:49:05 +0000 (UTC) Subject: commit 315b61b: [Rework] Rework SSL flag operations Message-ID: <20211208114905.76420275F8@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-08 11:31:35 +0000 URL: https://github.com/rspamd/rspamd/commit/315b61b3190f4b10f0710638bfaad27373fb3671 [Rework] Rework SSL flag operations --- src/libserver/http/http_connection.c | 46 +++++++++++++++++++++++++----------- src/libserver/http/http_connection.h | 12 ++++++---- src/libserver/http/http_message.c | 4 ++-- src/libserver/http/http_private.h | 2 +- src/libserver/maps/map.c | 13 ++++++---- src/lua/lua_http.c | 10 ++++++-- 6 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/libserver/http/http_connection.c b/src/libserver/http/http_connection.c index 478e00984..3dfe8e86c 100644 --- a/src/libserver/http/http_connection.c +++ b/src/libserver/http/http_connection.c @@ -1242,12 +1242,13 @@ rspamd_http_connection_new_client (struct rspamd_http_context *ctx, } struct rspamd_http_connection * -rspamd_http_connection_new_keepalive (struct rspamd_http_context *ctx, - rspamd_http_body_handler_t body_handler, - rspamd_http_error_handler_t error_handler, - rspamd_http_finish_handler_t finish_handler, - rspamd_inet_addr_t *addr, - const gchar *host) +rspamd_http_connection_new_client_keepalive (struct rspamd_http_context *ctx, + rspamd_http_body_handler_t body_handler, + rspamd_http_error_handler_t error_handler, + rspamd_http_finish_handler_t finish_handler, + unsigned opts, + rspamd_inet_addr_t *addr, + const gchar *host) { struct rspamd_http_connection *conn; @@ -1255,7 +1256,8 @@ rspamd_http_connection_new_keepalive (struct rspamd_http_context *ctx, ctx = rspamd_http_context_default (); } - conn = rspamd_http_context_check_keepalive(ctx, addr, host, false); + conn = rspamd_http_context_check_keepalive(ctx, addr, host, + opts & RSPAMD_HTTP_CLIENT_SSL); if (conn) { return conn; @@ -1263,11 +1265,12 @@ rspamd_http_connection_new_keepalive (struct rspamd_http_context *ctx, conn = rspamd_http_connection_new_client (ctx, body_handler, error_handler, finish_handler, - RSPAMD_HTTP_CLIENT_SIMPLE|RSPAMD_HTTP_CLIENT_KEEP_ALIVE, + opts|RSPAMD_HTTP_CLIENT_SIMPLE|RSPAMD_HTTP_CLIENT_KEEP_ALIVE, addr); if (conn) { - rspamd_http_context_prepare_keepalive(ctx, conn, addr, host, ); + rspamd_http_context_prepare_keepalive(ctx, conn, addr, host, + opts & RSPAMD_HTTP_CLIENT_SSL); } return conn; @@ -1879,7 +1882,7 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted, "Connection: %s\r\n" "Content-Length: %z\r\n", http_method_str(msg->method), - (msg->flags & RSPAMD_HTTP_FLAG_SSL) ? "https" : "http", + (conn->opts & RSPAMD_HTTP_CLIENT_SSL) ? "https" : "http", host, msg->port, msg->url, @@ -1893,7 +1896,7 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted, "Host: %s\r\n" "Content-Length: %z\r\n", http_method_str(msg->method), - (msg->flags & RSPAMD_HTTP_FLAG_SSL) ? "https" : "http", + (conn->opts & RSPAMD_HTTP_CLIENT_SSL) ? "https" : "http", host, msg->port, msg->url, @@ -1986,6 +1989,16 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn priv->buf->data = rspamd_fstring_sized_new (512); buf = priv->buf->data; + if ((msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL) && !(conn->opts & RSPAMD_HTTP_CLIENT_SSL)) { + err = g_error_new (HTTP_ERROR, 400, + "SSL connection requested but not created properly, internal error"); + rspamd_http_connection_ref (conn); + conn->error_handler (conn, err); + rspamd_http_connection_unref (conn); + g_error_free (err); + return FALSE; + } + if (priv->peer_key && priv->local_key) { priv->msg->peer_key = priv->peer_key; priv->peer_key = NULL; @@ -2282,14 +2295,19 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_RESETED; - if (priv->flags & RSPAMD_HTTP_CONN_FLAG_PROXY) { + if ((priv->flags & RSPAMD_HTTP_CONN_FLAG_PROXY) && (conn->opts & RSPAMD_HTTP_CLIENT_SSL)) { /* We need to disable SSL flag! */ - msg->flags &=~ RSPAMD_HTTP_FLAG_SSL; + err = g_error_new (HTTP_ERROR, 400, "cannot use proxy for SSL connections"); + rspamd_http_connection_ref (conn); + conn->error_handler (conn, err); + rspamd_http_connection_unref (conn); + g_error_free (err); + return FALSE; } rspamd_ev_watcher_stop (priv->ctx->event_loop, &priv->ev); - if (msg->flags & RSPAMD_HTTP_FLAG_SSL) { + if (conn->opts & RSPAMD_HTTP_CLIENT_SSL) { gpointer ssl_ctx = (msg->flags & RSPAMD_HTTP_FLAG_SSL_NOVERIFY) ? priv->ctx->ssl_ctx_noverify : priv->ctx->ssl_ctx; diff --git a/src/libserver/http/http_connection.h b/src/libserver/http/http_connection.h index cc7c8a8f1..029dbc745 100644 --- a/src/libserver/http/http_connection.h +++ b/src/libserver/http/http_connection.h @@ -67,10 +67,6 @@ struct rspamd_storage_shmem { * Store body of the message in an immutable shared memory segment */ #define RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE (1 << 3) -/** - * Use tls for this message (how the fuck SSL flag could be used PER MESSAGE???) - */ -#define RSPAMD_HTTP_FLAG_SSL (1 << 4) /** * Body has been set for a message */ @@ -83,6 +79,10 @@ struct rspamd_storage_shmem { * Body has been set for a message */ #define RSPAMD_HTTP_FLAG_HAS_HOST_HEADER (1 << 7) +/** + * Message is intended for SSL connection + */ +#define RSPAMD_HTTP_FLAG_WANT_SSL (1 << 8) /** * Options for HTTP connection */ @@ -93,6 +93,7 @@ enum rspamd_http_options { RSPAMD_HTTP_CLIENT_SHARED = 1u << 3, /**< Store reply in shared memory */ RSPAMD_HTTP_REQUIRE_ENCRYPTION = 1u << 4, RSPAMD_HTTP_CLIENT_KEEP_ALIVE = 1u << 5, + RSPAMD_HTTP_CLIENT_SSL = 1u << 6u, }; typedef int (*rspamd_http_body_handler_t) (struct rspamd_http_connection *conn, @@ -154,11 +155,12 @@ struct rspamd_http_connection *rspamd_http_connection_new_server ( * @param host * @return */ -struct rspamd_http_connection *rspamd_http_connection_new_keepalive ( +struct rspamd_http_connection *rspamd_http_connection_new_client_keepalive ( struct rspamd_http_context *ctx, rspamd_http_body_handler_t body_handler, rspamd_http_error_handler_t error_handler, rspamd_http_finish_handler_t finish_handler, + unsigned opts, rspamd_inet_addr_t *addr, const gchar *host); diff --git a/src/libserver/http/http_message.c b/src/libserver/http/http_message.c index d15856956..962699a9c 100644 --- a/src/libserver/http/http_message.c +++ b/src/libserver/http/http_message.c @@ -75,7 +75,7 @@ rspamd_http_message_from_url (const gchar *url) if ((pu.field_set & (1 << UF_SCHEMA))) { if (pu.field_data[UF_SCHEMA].len == sizeof ("https") - 1 && memcmp (url + pu.field_data[UF_SCHEMA].off, "https", 5) == 0) { - flags |= RSPAMD_HTTP_FLAG_SSL; + flags |= RSPAMD_HTTP_FLAG_WANT_SSL; } } @@ -97,7 +97,7 @@ rspamd_http_message_from_url (const gchar *url) } else { /* XXX: magic constant */ - if (flags & RSPAMD_HTTP_FLAG_SSL) { + if (flags & RSPAMD_HTTP_FLAG_WANT_SSL) { msg->port = 443; } else { diff --git a/src/libserver/http/http_private.h b/src/libserver/http/http_private.h index 6306d197b..c6a5b497b 100644 --- a/src/libserver/http/http_private.h +++ b/src/libserver/http/http_private.h @@ -86,7 +86,7 @@ struct rspamd_http_message { struct rspamd_keepalive_hash_key { rspamd_inet_addr_t *addr; gchar *host; - bool is_ssl; + gboolean is_ssl; GQueue conns; }; diff --git a/src/libserver/maps/map.c b/src/libserver/maps/map.c index 938f4a47f..531a7ce10 100644 --- a/src/libserver/maps/map.c +++ b/src/libserver/maps/map.c @@ -91,11 +91,6 @@ write_http_request (struct http_callback_data *cbd) struct rspamd_http_message *msg; msg = rspamd_http_new_message (HTTP_REQUEST); - - if (cbd->bk->protocol == MAP_PROTO_HTTPS) { - msg->flags |= RSPAMD_HTTP_FLAG_SSL; - } - if (cbd->check) { msg->method = HTTP_HEAD; } @@ -1268,6 +1263,9 @@ rspamd_map_dns_callback (struct rdns_reply *reply, void *arg) retry: msg_debug_map ("try open http connection to %s", rspamd_inet_address_to_string_pretty (cbd->addr)); + if (cbd->bk->protocol == MAP_PROTO_HTTPS) { + flags |= RSPAMD_HTTP_CLIENT_SSL; + } cbd->conn = rspamd_http_connection_new_client (NULL, NULL, http_map_error, @@ -1792,6 +1790,11 @@ check: strlen (data->host), RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) { rspamd_inet_address_set_port (addr, cbd->data->port); g_ptr_array_add (cbd->addrs, (void *)addr); + + if (bk->protocol == MAP_PROTO_HTTPS) { + flags |= RSPAMD_HTTP_CLIENT_SSL; + } + cbd->conn = rspamd_http_connection_new_client ( NULL, NULL, diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c index 2f1a1c5a7..1fee9e313 100644 --- a/src/lua/lua_http.c +++ b/src/lua/lua_http.c @@ -388,14 +388,20 @@ static gboolean lua_http_make_connection (struct lua_http_cbdata *cbd) { rspamd_inet_address_set_port (cbd->addr, cbd->msg->port); + unsigned http_opts = RSPAMD_HTTP_CLIENT_SIMPLE; + + if (cbd->msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL) { + http_opts |= RSPAMD_HTTP_CLIENT_SSL; + } if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE) { cbd->fd = -1; /* FD is owned by keepalive connection */ - cbd->conn = rspamd_http_connection_new_keepalive ( + cbd->conn = rspamd_http_connection_new_client_keepalive( NULL, /* Default context */ NULL, lua_http_error_handler, lua_http_finish_handler, + http_opts, cbd->addr, cbd->host); } @@ -406,7 +412,7 @@ lua_http_make_connection (struct lua_http_cbdata *cbd) NULL, lua_http_error_handler, lua_http_finish_handler, - RSPAMD_HTTP_CLIENT_SIMPLE, + http_opts, cbd->addr); } From vsevolod at highsecure.ru Wed Dec 8 11:49:06 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 8 Dec 2021 11:49:06 +0000 (UTC) Subject: commit b81a068: [Rework] Allow to restore SSL handlers after keepalive pooling Message-ID: <20211208114906.8812A275FA@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-08 11:37:33 +0000 URL: https://github.com/rspamd/rspamd/commit/b81a068aea8dca451b1f106ef191c0820db52c52 [Rework] Allow to restore SSL handlers after keepalive pooling --- src/libserver/http/http_connection.c | 11 +++++++++++ src/libserver/ssl_util.c | 15 +++++++++++++++ src/libserver/ssl_util.h | 12 ++++++++++++ 3 files changed, 38 insertions(+) diff --git a/src/libserver/http/http_connection.c b/src/libserver/http/http_connection.c index 3dfe8e86c..e82c543c2 100644 --- a/src/libserver/http/http_connection.c +++ b/src/libserver/http/http_connection.c @@ -1260,6 +1260,17 @@ rspamd_http_connection_new_client_keepalive (struct rspamd_http_context *ctx, opts & RSPAMD_HTTP_CLIENT_SSL); if (conn) { + struct rspamd_http_connection_private *priv; + + priv = conn->priv; + + if (priv->ssl) { + rspamd_ssl_connection_restore_handlers (priv->ssl, + rspamd_http_event_handler, + rspamd_http_ssl_err_handler, + conn); + } + return conn; } diff --git a/src/libserver/ssl_util.c b/src/libserver/ssl_util.c index b4e905619..c229b6794 100644 --- a/src/libserver/ssl_util.c +++ b/src/libserver/ssl_util.c @@ -754,6 +754,21 @@ rspamd_ssl_connect_fd (struct rspamd_ssl_connection *conn, gint fd, return TRUE; } +void +rspamd_ssl_connection_restore_handlers (struct rspamd_ssl_connection *conn, + rspamd_ssl_handler_t handler, + rspamd_ssl_error_handler_t err_handler, + gpointer handler_data) +{ + conn->handler = handler; + conn->err_handler = err_handler; + conn->handler_data = handler_data; + + rspamd_ev_watcher_stop (conn->event_loop, conn->ev); + rspamd_ev_watcher_init (conn->ev, conn->fd, EV_WRITE, rspamd_ssl_event_handler, conn); + rspamd_ev_watcher_start (conn->event_loop, conn->ev, conn->ev->timeout); +} + gssize rspamd_ssl_read (struct rspamd_ssl_connection *conn, gpointer buf, gsize buflen) diff --git a/src/libserver/ssl_util.h b/src/libserver/ssl_util.h index f3593387f..f2d591b56 100644 --- a/src/libserver/ssl_util.h +++ b/src/libserver/ssl_util.h @@ -56,6 +56,18 @@ gboolean rspamd_ssl_connect_fd (struct rspamd_ssl_connection *conn, gint fd, rspamd_ssl_handler_t handler, rspamd_ssl_error_handler_t err_handler, gpointer handler_data); +/** + * Restores SSL handlers for the existing ssl connection (e.g. after keepalive) + * @param conn + * @param handler + * @param err_handler + * @param handler_data + */ +void rspamd_ssl_connection_restore_handlers (struct rspamd_ssl_connection *conn, + rspamd_ssl_handler_t handler, + rspamd_ssl_error_handler_t err_handler, + gpointer handler_data); + /** * Perform async read from SSL socket * @param conn From vsevolod at highsecure.ru Wed Dec 8 11:49:07 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 8 Dec 2021 11:49:07 +0000 (UTC) Subject: commit 891ad9e: [Fix] Clear SSL errors Message-ID: <20211208114907.A8FFF275FC@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-08 11:42:17 +0000 URL: https://github.com/rspamd/rspamd/commit/891ad9ef8ce431a24b4b011a63097b57d643c305 (HEAD -> master) [Fix] Clear SSL errors --- src/libserver/dkim.c | 2 ++ src/libserver/ssl_util.c | 2 ++ src/lua/lua_rsa.c | 5 +++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libserver/dkim.c b/src/libserver/dkim.c index 06318c847..a4f77bfea 100644 --- a/src/libserver/dkim.c +++ b/src/libserver/dkim.c @@ -2871,6 +2871,7 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, if (RSA_verify (nid, raw_digest, dlen, ctx->b, ctx->blen, key->key.key_rsa) != 1) { msg_debug_dkim ("headers rsa verify failed"); + ERR_clear_error (); res->rcode = DKIM_REJECT; res->fail_reason = "headers rsa verify failed"; @@ -2898,6 +2899,7 @@ rspamd_dkim_check (rspamd_dkim_context_t *ctx, RSPAMD_DKIM_KEY_ID_LEN, rspamd_dkim_key_id (key), ctx->dkim_header); msg_debug_dkim ("headers ecdsa verify failed"); + ERR_clear_error (); res->rcode = DKIM_REJECT; res->fail_reason = "headers ecdsa verify failed"; } diff --git a/src/libserver/ssl_util.c b/src/libserver/ssl_util.c index c229b6794..319e87a04 100644 --- a/src/libserver/ssl_util.c +++ b/src/libserver/ssl_util.c @@ -666,6 +666,8 @@ rspamd_ssl_connect_fd (struct rspamd_ssl_connection *conn, gint fd, g_assert (conn != NULL); + /* Ensure that we start from the empty SSL errors stack */ + ERR_clear_error (); conn->ssl = SSL_new (conn->ssl_ctx->s); if (hostname) { diff --git a/src/lua/lua_rsa.c b/src/lua/lua_rsa.c index 0d4a268ed..a554cd79b 100644 --- a/src/lua/lua_rsa.c +++ b/src/lua/lua_rsa.c @@ -620,9 +620,10 @@ lua_rsa_verify_memory (lua_State *L) signature->str, signature->len, rsa); if (ret == 0) { - msg_info ("cannot check rsa signature for data: %s", - ERR_error_string (ERR_get_error (), NULL)); lua_pushboolean (L, FALSE); + lua_pushstring (L, ERR_error_string (ERR_get_error (), NULL)); + + return 2; } else { lua_pushboolean (L, TRUE); From vsevolod at highsecure.ru Wed Dec 8 13:14:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 8 Dec 2021 13:14:04 +0000 (UTC) Subject: commit 857e16e: [Fix] Phishing: Deal with phishing + redirected URL Message-ID: <20211208131404.4526B27609@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-08 13:07:44 +0000 URL: https://github.com/rspamd/rspamd/commit/857e16e9308f313e4af482035355e6e99f65a867 (HEAD -> master) [Fix] Phishing: Deal with phishing + redirected URL --- src/plugins/lua/phishing.lua | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/plugins/lua/phishing.lua b/src/plugins/lua/phishing.lua index dac5e92e9..32efc9f7b 100644 --- a/src/plugins/lua/phishing.lua +++ b/src/plugins/lua/phishing.lua @@ -192,8 +192,9 @@ local function phishing_cb(task) end local urls = task:get_urls() or {} - for _,url in ipairs(urls) do + for _,url_iter in ipairs(urls) do local function do_loop_iter() -- to emulate continue + local url = url_iter if generic_service_hash then check_phishing_map(generic_service_data, url, generic_service_symbol) end @@ -206,8 +207,23 @@ local function phishing_cb(task) check_phishing_dns(phishtank_suffix, url, phishtank_symbol) end - if url:is_phished() and not url:is_redirected() then - local purl = url:get_phished() + if url:is_phished() then + local purl + + if url:is_redirected() then + local rspamd_url = require "rspamd_url" + -- Examine the real redirect target instead of the url + local redirected_url = url:get_redirected() + if not redirected_url then + return + end + + purl = rspamd_url.create(task:get_mempool(), url:get_visible()) + url = redirected_url + else + purl = url:get_phished() + end + if not purl then return From vsevolod at highsecure.ru Wed Dec 8 14:07:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 8 Dec 2021 14:07:04 +0000 (UTC) Subject: commit ada2d80: [Minor] Another ubsan catch Message-ID: <20211208140704.3ABB827613@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-08 14:05:35 +0000 URL: https://github.com/rspamd/rspamd/commit/ada2d8000bb8ff179fc0a9811fa69f8a698be22d (HEAD -> master) [Minor] Another ubsan catch --- src/libserver/html/html.cxx | 8 +++++--- src/plugins/lua/multimap.lua | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/libserver/html/html.cxx b/src/libserver/html/html.cxx index c782148d2..79120d109 100644 --- a/src/libserver/html/html.cxx +++ b/src/libserver/html/html.cxx @@ -2254,10 +2254,12 @@ rspamd_html_tag_seen(void *ptr, const gchar *tagname) const gchar * rspamd_html_tag_by_id(gint id) { - const auto *td = rspamd::html::html_tags_defs.by_id(id); + if (id > Tag_UNKNOWN && id < Tag_MAX) { + const auto *td = rspamd::html::html_tags_defs.by_id(id); - if (td != nullptr) { - return td->name.c_str(); + if (td != nullptr) { + return td->name.c_str(); + } } return nullptr; diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 8ae903078..2edc87870 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -821,6 +821,7 @@ local function multimap_callback(task, rule) end else local hv = task:get_header_full(rule['header']) + lua_util.debugm(N, task, "hui: %s", hv) match_list(rule, hv, {'decoded'}) end end, From vsevolod at highsecure.ru Wed Dec 8 14:14:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 8 Dec 2021 14:14:04 +0000 (UTC) Subject: commit 2e8bffa: [Minor] Remove occasional debug Message-ID: <20211208141404.1A3F027616@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-08 14:07:17 +0000 URL: https://github.com/rspamd/rspamd/commit/2e8bffa2f408e1e7372fa72a70520802961444dd (HEAD -> master) [Minor] Remove occasional debug --- src/plugins/lua/multimap.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 2edc87870..8ae903078 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -821,7 +821,6 @@ local function multimap_callback(task, rule) end else local hv = task:get_header_full(rule['header']) - lua_util.debugm(N, task, "hui: %s", hv) match_list(rule, hv, {'decoded'}) end end, From vsevolod at highsecure.ru Wed Dec 8 15:00:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 8 Dec 2021 15:00:04 +0000 (UTC) Subject: commit 75f1f90: [Feature] Aws_s3: Allow to store large parts separately Message-ID: <20211208150004.6F70C2761D@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-08 14:57:57 +0000 URL: https://github.com/rspamd/rspamd/commit/75f1f90326f395b3726257e5b14c9f4c1bb6a41b (HEAD -> master) [Feature] Aws_s3: Allow to store large parts separately --- src/plugins/lua/aws_s3.lua | 71 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/src/plugins/lua/aws_s3.lua b/src/plugins/lua/aws_s3.lua index 1a7873086..ec45dca7c 100644 --- a/src/plugins/lua/aws_s3.lua +++ b/src/plugins/lua/aws_s3.lua @@ -32,6 +32,7 @@ local settings = { s3_timeout = 10, save_raw = true, save_structure = false, + inline_content_limit = nil, } local settings_schema = ts.shape{ @@ -46,6 +47,7 @@ local settings_schema = ts.shape{ zstd_compress = ts.boolean:is_optional(), save_raw = ts.boolean:is_optional(), save_structure = ts.boolean:is_optional(), + inline_content_limit = (ts.integer + ts.string / tonumber):is_optional(), } local function raw_data(task, nonce, queue_id) @@ -66,24 +68,61 @@ local function raw_data(task, nonce, queue_id) return path, content, content_type end +local function gen_ext() + local ext = 'msgpack' + if settings.zstd_compress then + ext = 'msgpack.zst' + end + + return ext +end + +local function convert_to_ref(task, nonce, queue_id, part, external_refs) + local path = string.format('/%s-%s-%s.%s', queue_id, nonce, + rspamd_text.randombytes(8):base32(), gen_ext()) + local content = part.content + + if settings.zstd_compress then + external_refs[path] = rspamd_util.zstd_compress(content) + else + external_refs[path] = content + end + + part.content = nil + part.content_path = path + + return path +end + local function structured_data(task, nonce, queue_id) - local ext, content, content_type + local content, content_type + local external_refs = {} local lua_mime = require "lua_mime" local ucl = require "ucl" + local message_split = lua_mime.message_to_ucl(task) + + if settings.inline_content_limit and settings.inline_content_limit > 0 then + for i,part in ipairs(message_split.parts() or {}) do + if part.content and #part.content >= settings.inline_content_limit then + local ref = convert_to_ref(task, nonce, queue_id, part, external_refs) + lua_util.debugm(N, task, "convert part number %s to a reference %s", + i, ref) + end + end + end + if settings.zstd_compress then - ext = 'msgpack.zst' content = rspamd_util.zstd_compress(ucl.to_format(lua_mime.message_to_ucl(task), 'msgpack')) content_type = 'application/zstd' else - ext = 'msgpack' content = ucl.to_format(lua_mime.message_to_ucl(task), 'msgpack') content_type = 'application/msgpack' end - local path = string.format('/%s-%s.%s', queue_id, nonce, ext) + local path = string.format('/%s-%s.%s', queue_id, nonce, gen_ext()) - return path, content, content_type + return path, content, content_type, external_refs end local function s3_aws_callback(task) @@ -97,7 +136,7 @@ local function s3_aws_callback(task) -- Hack to pass host local aws_host = string.format('%s.%s', settings.s3_bucket, settings.s3_host) - local function gen_s3_http_callback(path) + local function gen_s3_http_callback(path, what) return function (http_err, code, body, headers) if http_err then @@ -108,7 +147,7 @@ local function s3_aws_callback(task) end rspamd_logger.errx(task, 'cannot save %s to AWS S3: %s', path, http_err) else - rspamd_logger.messagex(task, 'saved message successfully in S3 object %s', path) + rspamd_logger.messagex(task, 'saved %s successfully in S3 object %s', what, path) end lua_util.debugm(N, task, 'obj=%s, err=%s, code=%s, body=%s, headers=%s', path, http_err, code, body, headers) @@ -133,13 +172,13 @@ local function s3_aws_callback(task) task = task, method = 'PUT', body = content, - callback = gen_s3_http_callback(path), + callback = gen_s3_http_callback(path, 'raw message'), headers = hdrs, timeout = settings.s3_timeout, }) end if settings.save_structure then - local path, content, content_type = structured_data(task, nonce, queue_id) + local path, content, content_type, external_refs = structured_data(task, nonce, queue_id) local hdrs = lua_aws.aws_request_enrich({ region = settings.s3_region, headers = { @@ -156,10 +195,22 @@ local function s3_aws_callback(task) task = task, method = 'PUT', body = content, - callback = gen_s3_http_callback(path), + callback = gen_s3_http_callback(path, 'structured message'), headers = hdrs, timeout = settings.s3_timeout, }) + + for _,ref in ipairs(external_refs) do + rspamd_http.request({ + url = uri .. ref, + task = task, + method = 'PUT', + body = content, + callback = gen_s3_http_callback(ref, 'part content'), + headers = hdrs, + timeout = settings.s3_timeout, + }) + end end From vsevolod at highsecure.ru Wed Dec 8 15:21:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 8 Dec 2021 15:21:04 +0000 (UTC) Subject: commit e884106: [Minor] Aws_s3: Fix some issues found Message-ID: <20211208152104.7488C27620@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-08 15:20:44 +0000 URL: https://github.com/rspamd/rspamd/commit/e8841063f78a78db360d4bea1ce7e9ce6dc54371 (HEAD -> master) [Minor] Aws_s3: Fix some issues found --- src/plugins/lua/aws_s3.lua | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/src/plugins/lua/aws_s3.lua b/src/plugins/lua/aws_s3.lua index ec45dca7c..78d718c24 100644 --- a/src/plugins/lua/aws_s3.lua +++ b/src/plugins/lua/aws_s3.lua @@ -47,7 +47,7 @@ local settings_schema = ts.shape{ zstd_compress = ts.boolean:is_optional(), save_raw = ts.boolean:is_optional(), save_structure = ts.boolean:is_optional(), - inline_content_limit = (ts.integer + ts.string / tonumber):is_optional(), + inline_content_limit = ts.number:is_optional(), } local function raw_data(task, nonce, queue_id) @@ -68,10 +68,10 @@ local function raw_data(task, nonce, queue_id) return path, content, content_type end -local function gen_ext() - local ext = 'msgpack' +local function gen_ext(base) + local ext = base if settings.zstd_compress then - ext = 'msgpack.zst' + ext = base .. '.zst' end return ext @@ -79,7 +79,7 @@ end local function convert_to_ref(task, nonce, queue_id, part, external_refs) local path = string.format('/%s-%s-%s.%s', queue_id, nonce, - rspamd_text.randombytes(8):base32(), gen_ext()) + rspamd_text.randombytes(8):base32(), gen_ext('raw')) local content = part.content if settings.zstd_compress then @@ -101,9 +101,9 @@ local function structured_data(task, nonce, queue_id) local ucl = require "ucl" local message_split = lua_mime.message_to_ucl(task) - if settings.inline_content_limit and settings.inline_content_limit > 0 then - for i,part in ipairs(message_split.parts() or {}) do + + for i,part in ipairs(message_split.parts or {}) do if part.content and #part.content >= settings.inline_content_limit then local ref = convert_to_ref(task, nonce, queue_id, part, external_refs) lua_util.debugm(N, task, "convert part number %s to a reference %s", @@ -113,14 +113,14 @@ local function structured_data(task, nonce, queue_id) end if settings.zstd_compress then - content = rspamd_util.zstd_compress(ucl.to_format(lua_mime.message_to_ucl(task), 'msgpack')) + content = rspamd_util.zstd_compress(ucl.to_format(message_split, 'msgpack')) content_type = 'application/zstd' else - content = ucl.to_format(lua_mime.message_to_ucl(task), 'msgpack') + content = ucl.to_format(message_split, 'msgpack') content_type = 'application/msgpack' end - local path = string.format('/%s-%s.%s', queue_id, nonce, gen_ext()) + local path = string.format('/%s-%s.%s', queue_id, nonce, gen_ext('msgpack')) return path, content, content_type, external_refs end @@ -200,14 +200,25 @@ local function s3_aws_callback(task) timeout = settings.s3_timeout, }) - for _,ref in ipairs(external_refs) do + for ref,part_content in pairs(external_refs) do + local part_hdrs = lua_aws.aws_request_enrich({ + region = settings.s3_region, + headers = { + ['Content-Type'] = content_type, + ['Host'] = aws_host + }, + uri = ref, + key_id = settings.s3_key_id, + secret_key = settings.s3_secret_key, + method = 'PUT', + }, part_content) rspamd_http.request({ url = uri .. ref, task = task, method = 'PUT', - body = content, + body = part_content, callback = gen_s3_http_callback(ref, 'part content'), - headers = hdrs, + headers = part_hdrs, timeout = settings.s3_timeout, }) end From vsevolod at highsecure.ru Thu Dec 9 11:14:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 9 Dec 2021 11:14:04 +0000 (UTC) Subject: commit 510b61f: [Fix] Output service parts as well Message-ID: <20211209111404.8FCAB276CF@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-09 11:00:14 +0000 URL: https://github.com/rspamd/rspamd/commit/510b61f82bae20b9d8c2b61b51a35d9579c2d347 [Fix] Output service parts as well --- lualib/lua_mime.lua | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lualib/lua_mime.lua b/lualib/lua_mime.lua index 968d38c0a..1646b892c 100644 --- a/lualib/lua_mime.lua +++ b/lualib/lua_mime.lua @@ -609,17 +609,29 @@ exports.message_to_ucl = function(task, stringify_content) local parts = task:get_parts() or E result.parts = {} for _,part in ipairs(parts) do - local l = part:get_length() - if l > 0 then + if not part:is_multipart() and not part:is_message() then local p = { - size = l, + size = part:get_length(), type = string.format('%s/%s', part:get_type()), detected_type = string.format('%s/%s', part:get_detected_type()), filename = part:get_filename(), content = maybe_stringify_f(part:get_content()), headers = part:get_headers(true) or E, - boundary = part:get_enclosing_boundary() + boundary = part:get_enclosing_boundary(), } + table.insert(result.parts, p) + else + -- Service part: multipart container or message/rfc822 + local p = { + type = string.format('%s/%s', part:get_type()), + headers = part:get_headers(true) or E, + boundary = part:get_enclosing_boundary(), + } + + if part:is_multipart() then + p.multipart_boundary = part:get_boundary() + end + table.insert(result.parts, p) end end From vsevolod at highsecure.ru Thu Dec 9 11:14:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 9 Dec 2021 11:14:05 +0000 (UTC) Subject: commit 533d7bb: [Minor] Fix schema and add size in all cases Message-ID: <20211209111405.A2951276D1@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-09 11:02:32 +0000 URL: https://github.com/rspamd/rspamd/commit/533d7bb3646317e7a71894e3d42608270b7fae46 [Minor] Fix schema and add size in all cases --- lualib/lua_mime.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lualib/lua_mime.lua b/lualib/lua_mime.lua index 1646b892c..2128b29a6 100644 --- a/lualib/lua_mime.lua +++ b/lualib/lua_mime.lua @@ -626,6 +626,7 @@ exports.message_to_ucl = function(task, stringify_content) type = string.format('%s/%s', part:get_type()), headers = part:get_headers(true) or E, boundary = part:get_enclosing_boundary(), + size = 0, } if part:is_multipart() then @@ -661,7 +662,8 @@ exports.message_to_ucl_schema = function() local function part_schema() return ts.shape{ - content = ts.string:describe('Decoded content'), + content = ts.string:describe('Decoded content'):is_optional(), + multipart_boundary = ts.string:describe('Multipart service boundary'):is_optional(), size = ts.integer:describe('Size of the part'), type = ts.string:describe('Announced type'):is_optional(), detected_type = ts.string:describe('Detected type'):is_optional(), From vsevolod at highsecure.ru Thu Dec 9 11:14:06 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 9 Dec 2021 11:14:06 +0000 (UTC) Subject: commit 2130677: [Test] Fix string formatting Message-ID: <20211209111406.B5C62276D3@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-09 11:08:54 +0000 URL: https://github.com/rspamd/rspamd/commit/2130677e50a4ccdf104247640d18c8ca1e6e22c8 [Test] Fix string formatting --- test/lua/rspamd_assertions.lua | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/lua/rspamd_assertions.lua b/test/lua/rspamd_assertions.lua index 8e483431d..d0590fdcd 100644 --- a/test/lua/rspamd_assertions.lua +++ b/test/lua/rspamd_assertions.lua @@ -39,15 +39,17 @@ local function format_line(level, key, v_expect, v_actual) local prefix if v_expect == v_actual then prefix = string.rep(' ', level * 2 + 1) - return string.format("%s[%s] = %s", prefix, tostring(key), tostring(v_expect)) + return logger.slog("%s[%s] = %s", prefix, key, v_expect) else prefix = string.rep(' ', level * 2) local ret = {} if v_expect then - ret[#ret + 1] = string.format("-%s[%s] = %s: %s", prefix, tostring(key), type(v_expect), tostring(v_expect)) + ret[#ret + 1] = logger.slog("-%s[%s] = %s: %s", prefix, key, + type(v_expect), v_expect) end if v_actual then - ret[#ret + 1] = string.format("+%s[%s] = %s: %s", prefix, tostring(key), type(v_actual), tostring(v_actual)) + ret[#ret + 1] = logger.slog("+%s[%s] = %s: %s", prefix, + (key), type(v_actual), (v_actual)) end return table.concat(ret, "\n") end From vsevolod at highsecure.ru Thu Dec 9 11:14:07 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 9 Dec 2021 11:14:07 +0000 (UTC) Subject: commit 354ad43: [Test] Fix mime dump test Message-ID: <20211209111407.C8458276D5@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-09 11:09:06 +0000 URL: https://github.com/rspamd/rspamd/commit/354ad43417e0c1264ab198cb1b54de2240a44fa4 (HEAD -> master) [Test] Fix mime dump test --- test/lua/unit/lua_mime.message_to_ucl.lua | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/lua/unit/lua_mime.message_to_ucl.lua b/test/lua/unit/lua_mime.message_to_ucl.lua index d02d2cc7b..e4fe1781a 100644 --- a/test/lua/unit/lua_mime.message_to_ucl.lua +++ b/test/lua/unit/lua_mime.message_to_ucl.lua @@ -59,6 +59,12 @@ context("Task piecewise split", function() local expected_json = [[ { "parts": [ + { + "type": "multipart/alternative", + "multipart_boundary": "_000_6be055295eab48a5af7ad4022f33e2d0_", + "size": 0, + "headers": [] + }, { "content": "Hello world\n\n\n", "size": 14, From vsevolod at highsecure.ru Thu Dec 9 12:00:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 9 Dec 2021 12:00:04 +0000 (UTC) Subject: commit 4fe64bd: [Minor] Fix ssl connection reuse Message-ID: <20211209120004.C4F40276DF@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-09 11:58:55 +0000 URL: https://github.com/rspamd/rspamd/commit/4fe64bdcb51eb13b732bfcab0ffb321b16e91395 (HEAD -> master) [Minor] Fix ssl connection reuse --- src/libserver/http/http_connection.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libserver/http/http_connection.c b/src/libserver/http/http_connection.c index e82c543c2..afd685ae0 100644 --- a/src/libserver/http/http_connection.c +++ b/src/libserver/http/http_connection.c @@ -1264,13 +1264,6 @@ rspamd_http_connection_new_client_keepalive (struct rspamd_http_context *ctx, priv = conn->priv; - if (priv->ssl) { - rspamd_ssl_connection_restore_handlers (priv->ssl, - rspamd_http_event_handler, - rspamd_http_ssl_err_handler, - conn); - } - return conn; } @@ -2353,6 +2346,13 @@ rspamd_http_connection_write_message_common (struct rspamd_http_connection *conn return FALSE; } } + else { + /* Just restore SSL handlers */ + rspamd_ssl_connection_restore_handlers (priv->ssl, + rspamd_http_event_handler, + rspamd_http_ssl_err_handler, + conn); + } } } else { From vsevolod at highsecure.ru Thu Dec 9 14:07:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 9 Dec 2021 14:07:04 +0000 (UTC) Subject: commit 05d6d35: [Feature] Allow to skip DNS resolution for keep-alive connections Message-ID: <20211209140704.55D79276EF@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-09 14:02:18 +0000 URL: https://github.com/rspamd/rspamd/commit/05d6d354ec99e800007828137315b42be332c719 (HEAD -> master) [Feature] Allow to skip DNS resolution for keep-alive connections --- src/libserver/http/http_context.c | 56 ++++++++++++++++++++++++++++++++------- src/libserver/http/http_context.h | 20 +++++++++++--- src/libserver/http/http_private.h | 1 + src/lua/lua_http.c | 42 ++++++++++++++++++----------- 4 files changed, 91 insertions(+), 28 deletions(-) diff --git a/src/libserver/http/http_context.c b/src/libserver/http/http_context.c index b56f1c4c0..2dbaff0a4 100644 --- a/src/libserver/http/http_context.c +++ b/src/libserver/http/http_context.c @@ -365,19 +365,18 @@ rspamd_http_context_default (void) gint32 rspamd_keep_alive_key_hash (struct rspamd_keepalive_hash_key *k) { - guint32 h; + rspamd_cryptobox_fast_hash_state_t hst; - h = rspamd_inet_address_port_hash (k->addr); + rspamd_cryptobox_fast_hash_init (&hst, 0); if (k->host) { - h ^= rspamd_cryptobox_fast_hash (k->host, strlen (k->host), h); + rspamd_cryptobox_fast_hash_update (&hst, k->host, strlen (k->host)); } - if (k->is_ssl) { - h = ~h; - } + rspamd_cryptobox_fast_hash_update (&hst, &k->port, sizeof(k->port)); + rspamd_cryptobox_fast_hash_update (&hst, &k->is_ssl, sizeof(k->is_ssl)); - return (gint32)h; + return rspamd_cryptobox_fast_hash_final (&hst); } bool @@ -389,12 +388,12 @@ rspamd_keep_alive_key_equal (struct rspamd_keepalive_hash_key *k1, } if (k1->host && k2->host) { - if (rspamd_inet_address_port_equal (k1->addr, k2->addr)) { + if (k1->port == k2->port) { return strcmp (k1->host, k2->host) == 0; } } else if (!k1->host && !k2->host) { - return rspamd_inet_address_port_equal (k1->addr, k2->addr); + return (k1->port == k2->port); } /* One has host and another has no host */ @@ -410,8 +409,13 @@ rspamd_http_context_check_keepalive(struct rspamd_http_context *ctx, struct rspamd_keepalive_hash_key hk, *phk; khiter_t k; + if (ctx == NULL) { + ctx = rspamd_http_context_default (); + } + hk.addr = (rspamd_inet_addr_t *)addr; hk.host = (gchar *)host; + hk.port = rspamd_inet_address_get_port (addr); hk.is_ssl = is_ssl; k = kh_get (rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk); @@ -471,6 +475,37 @@ rspamd_http_context_check_keepalive(struct rspamd_http_context *ctx, return NULL; } +const rspamd_inet_addr_t * +rspamd_http_context_has_keepalive(struct rspamd_http_context *ctx, + const gchar *host, + unsigned port, + bool is_ssl) +{ + struct rspamd_keepalive_hash_key hk, *phk; + khiter_t k; + + if (ctx == NULL) { + ctx = rspamd_http_context_default (); + } + + hk.host = (gchar *)host; + hk.port = port; + hk.is_ssl = is_ssl; + + k = kh_get (rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk); + + if (k != kh_end (ctx->keep_alive_hash)) { + phk = kh_key (ctx->keep_alive_hash, k); + GQueue *conns = &phk->conns; + + if (g_queue_get_length(conns) > 0) { + return phk->addr; + } + } + + return NULL; +} + void rspamd_http_context_prepare_keepalive(struct rspamd_http_context *ctx, struct rspamd_http_connection *conn, @@ -484,6 +519,7 @@ rspamd_http_context_prepare_keepalive(struct rspamd_http_context *ctx, hk.addr = (rspamd_inet_addr_t *)addr; hk.host = (gchar *)host; hk.is_ssl = is_ssl; + hk.port = rspamd_inet_address_get_port (addr); k = kh_get (rspamd_keep_alive_hash, ctx->keep_alive_hash, &hk); @@ -504,6 +540,8 @@ rspamd_http_context_prepare_keepalive(struct rspamd_http_context *ctx, phk->host = g_strdup (host); phk->is_ssl = is_ssl; phk->addr = rspamd_inet_address_copy (addr); + phk->port = hk.port; + kh_put (rspamd_keep_alive_hash, ctx->keep_alive_hash, phk, &r); conn->keepalive_hash_key = phk; diff --git a/src/libserver/http/http_context.h b/src/libserver/http/http_context.h index f42164dba..635da7ba5 100644 --- a/src/libserver/http/http_context.h +++ b/src/libserver/http/http_context.h @@ -74,9 +74,23 @@ struct rspamd_http_context *rspamd_http_context_default (void); * @param host * @return */ -struct rspamd_http_connection * -rspamd_http_context_check_keepalive(struct rspamd_http_context *ctx, const rspamd_inet_addr_t *addr, const gchar *host, - bool is_ssl); +struct rspamd_http_connection * rspamd_http_context_check_keepalive(struct rspamd_http_context *ctx, + const rspamd_inet_addr_t *addr, + const gchar *host, + bool is_ssl); + +/** + * Checks if there is a valid keepalive connection + * @param ctx + * @param addr + * @param host + * @param is_ssl + * @return + */ +const rspamd_inet_addr_t *rspamd_http_context_has_keepalive(struct rspamd_http_context *ctx, + const gchar *host, + unsigned port, + bool is_ssl); /** * Prepares keepalive key for a connection by creating a new entry or by reusing existent diff --git a/src/libserver/http/http_private.h b/src/libserver/http/http_private.h index c6a5b497b..0e48d9f3c 100644 --- a/src/libserver/http/http_private.h +++ b/src/libserver/http/http_private.h @@ -87,6 +87,7 @@ struct rspamd_keepalive_hash_key { rspamd_inet_addr_t *addr; gchar *host; gboolean is_ssl; + unsigned port; GQueue conns; }; diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c index 1fee9e313..85aa04e1f 100644 --- a/src/lua/lua_http.c +++ b/src/lua/lua_http.c @@ -389,6 +389,7 @@ lua_http_make_connection (struct lua_http_cbdata *cbd) { rspamd_inet_address_set_port (cbd->addr, cbd->msg->port); unsigned http_opts = RSPAMD_HTTP_CLIENT_SIMPLE; + struct rspamd_http_message *msg = cbd->msg; if (cbd->msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL) { http_opts |= RSPAMD_HTTP_CLIENT_SSL; @@ -1036,20 +1037,6 @@ lua_http_request (lua_State *L) } - const rspamd_ftok_t *host_header_tok = rspamd_http_message_find_header (msg, "Host"); - if (host_header_tok != NULL) { - if (msg->host) { - g_string_free (msg->host, true); - } - msg->host = g_string_new_len (host_header_tok->begin, host_header_tok->len); - cbd->host = msg->host->str; - } - else { - if (msg->host) { - cbd->host = msg->host->str; - } - } - if (body) { if (gzip) { if (rspamd_fstring_gzip (&body)) { @@ -1064,8 +1051,31 @@ lua_http_request (lua_State *L) cbd->session = session; } - if (msg->host && rspamd_parse_inet_address (&cbd->addr, - msg->host->str, msg->host->len, RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) { + bool numeric_ip = false; + + /* Check if we can skip resolving */ + if (msg->host) { + cbd->host = msg->host->str; + + if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE) { + const rspamd_inet_addr_t *ka_addr = rspamd_http_context_has_keepalive(NULL, + msg->host->str, msg->port, msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL); + + if (ka_addr) { + cbd->addr = rspamd_inet_address_copy(ka_addr); + numeric_ip = true; + } + } + + if (!cbd->addr) { + if (rspamd_parse_inet_address (&cbd->addr, + msg->host->str, msg->host->len, RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) { + numeric_ip = true; + } + } + } + + if (numeric_ip) { /* Host is numeric IP, no need to resolve */ gboolean ret; From noreply at github.com Fri Dec 10 16:28:04 2021 From: noreply at github.com (GitHub) Date: Fri, 10 Dec 2021 16:28:04 +0000 (UTC) Subject: commit 559e371: Update clickhouse.lua Message-ID: <20211210162804.2CDE0277D3@lists.rspamd.com> Author: Mehmet Suslu Date: 2021-12-10 18:39:39 +0300 URL: https://github.com/rspamd/rspamd/commit/559e371e0af20afee9191368bccf9892ad7138dd Update clickhouse.lua fix typo collect_garbadge -> collect_garbage --- src/plugins/lua/clickhouse.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/lua/clickhouse.lua b/src/plugins/lua/clickhouse.lua index 97c66946d..7a2e276ee 100644 --- a/src/plugins/lua/clickhouse.lua +++ b/src/plugins/lua/clickhouse.lua @@ -1057,7 +1057,7 @@ local function clickhouse_maybe_send_data_periodic(cfg, ev_base, now) clickhouse_send_data(nil, ev_base, reason, saved_rows, saved_custom) - if settings.collect_garbadge then + if settings.collect_garbage then collectgarbage() end end @@ -1484,7 +1484,7 @@ if opts then clickhouse_send_data(task, nil, 'final collection', saved_rows, saved_custom) - if settings.collect_garbadge then + if settings.collect_garbage then collectgarbage() end end From noreply at github.com Fri Dec 10 16:28:05 2021 From: noreply at github.com (GitHub) Date: Fri, 10 Dec 2021 16:28:05 +0000 (UTC) Subject: commit d910553: Merge pull request #1 from msuslu/msuslu-patch-1 Message-ID: <20211210162805.47660277D5@lists.rspamd.com> Author: Mehmet Suslu Date: 2021-12-10 18:41:04 +0300 URL: https://github.com/rspamd/rspamd/commit/d91055383df657c0defea5f1338ea4c04b4469db (refs/pull/4003/head) Merge pull request #1 from msuslu/msuslu-patch-1 Update clickhouse.lua src/plugins/lua/clickhouse.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) From noreply at github.com Fri Dec 10 16:28:06 2021 From: noreply at github.com (GitHub) Date: Fri, 10 Dec 2021 16:28:06 +0000 (UTC) Subject: commit 0d4c611: Merge pull request #4003 from msuslu/master Message-ID: <20211210162806.65F5C277D7@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-10 16:22:17 +0000 URL: https://github.com/rspamd/rspamd/commit/0d4c611c689bfc434132458239a7395b5204c833 (HEAD -> master) Merge pull request #4003 from msuslu/master Fix Typo at Settings Key src/plugins/lua/clickhouse.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) From vsevolod at highsecure.ru Sat Dec 11 12:35:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sat, 11 Dec 2021 12:35:04 +0000 (UTC) Subject: commit caac14e: [Minor] Use unified method to obtain http host Message-ID: <20211211123504.54BB327885@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-11 12:30:09 +0000 URL: https://github.com/rspamd/rspamd/commit/caac14e056da239aa86b626b1ca8786628558cf0 (HEAD -> master) [Minor] Use unified method to obtain http host --- src/libserver/http/http_message.c | 26 ++++++++++++++++++++++++++ src/libserver/http/http_message.h | 8 ++++++++ src/lua/lua_http.c | 13 ++++++++----- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/libserver/http/http_message.c b/src/libserver/http/http_message.c index 962699a9c..a313283f3 100644 --- a/src/libserver/http/http_message.c +++ b/src/libserver/http/http_message.c @@ -690,4 +690,30 @@ rspamd_http_message_remove_header (struct rspamd_http_message *msg, } return res; +} + +const gchar* +rspamd_http_message_get_http_host (struct rspamd_http_message *msg) +{ + if (msg->flags & RSPAMD_HTTP_FLAG_HAS_HOST_HEADER) { + rspamd_ftok_t srch; + + RSPAMD_FTOK_ASSIGN(&srch, "Host"); + + khiter_t k = kh_get (rspamd_http_headers_hash, msg->headers, &srch); + + if (k != kh_end (msg->headers)) { + return (kh_value (msg->headers, k)->value).begin; + } + else if (msg->host) { + return msg->host->str; + } + } + else { + if (msg->host) { + return msg->host->str; + } + } + + return NULL; } \ No newline at end of file diff --git a/src/libserver/http/http_message.h b/src/libserver/http/http_message.h index e13c7427c..1750c1dd6 100644 --- a/src/libserver/http/http_message.h +++ b/src/libserver/http/http_message.h @@ -229,6 +229,14 @@ void rspamd_http_message_shmem_unref (struct rspamd_storage_shmem *p); */ guint rspamd_http_message_get_flags (struct rspamd_http_message *msg); +/** + * Returns an HTTP hostname for a message, derived from a header if it has it + * or from a url if it doesn't + * @param msg + * @return + */ +const gchar* rspamd_http_message_get_http_host (struct rspamd_http_message *msg); + #ifdef __cplusplus } #endif diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c index 85aa04e1f..e4a3bedb1 100644 --- a/src/lua/lua_http.c +++ b/src/lua/lua_http.c @@ -75,7 +75,7 @@ struct lua_http_cbdata { struct rspamd_cryptobox_pubkey *peer_pk; rspamd_inet_addr_t *addr; gchar *mime_type; - gchar *host; + const gchar *host; gchar *auth; const gchar *url; gsize max_size; @@ -1054,12 +1054,13 @@ lua_http_request (lua_State *L) bool numeric_ip = false; /* Check if we can skip resolving */ - if (msg->host) { - cbd->host = msg->host->str; + cbd->host = rspamd_http_message_get_http_host (msg); + if (cbd->host) { if (cbd->flags & RSPAMD_LUA_HTTP_FLAG_KEEP_ALIVE) { const rspamd_inet_addr_t *ka_addr = rspamd_http_context_has_keepalive(NULL, - msg->host->str, msg->port, msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL); + rspamd_http_message_get_http_host (msg), + msg->port, msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL); if (ka_addr) { cbd->addr = rspamd_inet_address_copy(ka_addr); @@ -1068,8 +1069,10 @@ lua_http_request (lua_State *L) } if (!cbd->addr) { + /* We use msg->host here, not cbd->host ! */ if (rspamd_parse_inet_address (&cbd->addr, - msg->host->str, msg->host->len, RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) { + msg->host->str, msg->host->len, + RSPAMD_INET_ADDRESS_PARSE_DEFAULT)) { numeric_ip = true; } } From vsevolod at highsecure.ru Sun Dec 12 23:49:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sun, 12 Dec 2021 23:49:04 +0000 (UTC) Subject: commit 0f464be: [Minor] Spamtrap: Allow to use multiple recipients Message-ID: <20211212234904.2B225279B3@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-12 23:42:18 +0000 URL: https://github.com/rspamd/rspamd/commit/0f464be6050efa1f57d60e26cd7caaafcdc52b83 (HEAD -> master) [Minor] Spamtrap: Allow to use multiple recipients --- src/plugins/lua/spamtrap.lua | 99 ++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 44 deletions(-) diff --git a/src/plugins/lua/spamtrap.lua b/src/plugins/lua/spamtrap.lua index 6147ad680..594d4f225 100644 --- a/src/plugins/lua/spamtrap.lua +++ b/src/plugins/lua/spamtrap.lua @@ -22,6 +22,7 @@ local redis_params local use_redis = false; local M = 'spamtrap' local lua_util = require "lua_util" +local fun = require "fun" local settings = { symbol = 'SPAMTRAP', @@ -31,6 +32,7 @@ local settings = { fuzzy_flag = 1, fuzzy_weight = 10.0, key_prefix = 'sptr_', + allow_multiple_rcpts = false, } local check_authed = true @@ -41,7 +43,6 @@ local function spamtrap_cb(task) local authed_user = task:get_user() local ip_addr = task:get_ip() local called_for_domain = false - local target if ((not check_authed and authed_user) or (not check_local and ip_addr and ip_addr:is_local())) then @@ -81,60 +82,70 @@ local function spamtrap_cb(task) module = 'spamtrap', flags = act_flags} end + + return true end - local function redis_spamtrap_cb(err, data) - if err ~= nil then - rspamd_logger.errx(task, 'redis_spamtrap_cb received error: %1', err) - return - end + local function gen_redis_spamtrap_cb(target) + return function(err, data) + if err ~= nil then + rspamd_logger.errx(task, 'redis_spamtrap_cb received error: %1', err) + return + end - if data and type(data) ~= 'userdata' then - do_action(target) - else - if not called_for_domain then - -- Recurse for @catchall domain - target = rcpts[1]['domain']:lower() - local key = settings['key_prefix'] .. '@' .. target - local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_spamtrap_cb, -- callback - 'GET', -- command - {key} -- arguments - ) - if not ret then - rspamd_logger.errx(task, "redis request wasn't scheduled") - end - called_for_domain = true + if data and type(data) ~= 'userdata' then + do_action(target) else - lua_util.debugm(M, task, 'skip spamtrap for %s', target) + if not called_for_domain then + -- Recurse for @catchall domain + target = rcpts[1]['domain']:lower() + local key = settings['key_prefix'] .. '@' .. target + local ret = rspamd_redis_make_request(task, + redis_params, -- connect params + key, -- hash key + false, -- is write + gen_redis_spamtrap_cb(target), -- callback + 'GET', -- command + {key} -- arguments + ) + if not ret then + rspamd_logger.errx(task, "redis request wasn't scheduled") + end + called_for_domain = true + else + lua_util.debugm(M, task, 'skip spamtrap for %s', target) + end end end end -- Do not risk a FP by checking for more than one recipient - if rcpts and #rcpts == 1 then - target = rcpts[1]['addr']:lower() + if rcpts and (#rcpts == 1 or (#rcpts > 0 and settings.allow_multiple_rcpts)) then + local targets = fun.map(function(r) return r['addr']:lower() end, rcpts) if use_redis then - local key = settings['key_prefix'] .. target - local ret = rspamd_redis_make_request(task, - redis_params, -- connect params - key, -- hash key - false, -- is write - redis_spamtrap_cb, -- callback - 'GET', -- command - {key} -- arguments - ) - if not ret then - rspamd_logger.errx(task, "redis request wasn't scheduled") - end + fun.each(function(target) + local key = settings['key_prefix'] .. target + local ret = rspamd_redis_make_request(task, + redis_params, -- connect params + key, -- hash key + false, -- is write + gen_redis_spamtrap_cb(target), -- callback + 'GET', -- command + {key} -- arguments + ) + if not ret then + rspamd_logger.errx(task, "redis request wasn't scheduled") + end + end, targets) + elseif settings['map'] then - if settings['map']:get_key(target) then - do_action(target) - else - lua_util.debugm(M, task, 'skip spamtrap for %s', target) + local function check_map_functor(target) + if settings['map']:get_key(target) then + return do_action(target) + end + end + if not fun.any(check_map_functor, targets) then + lua_util.debugm(M, task, 'skip spamtrap') end end end From vsevolod at highsecure.ru Mon Dec 13 14:28:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Mon, 13 Dec 2021 14:28:04 +0000 (UTC) Subject: commit 93430bb: [Minor] Bimi: Fix Redis storage Message-ID: <20211213142804.8B2DF27A2F@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-13 14:25:43 +0000 URL: https://github.com/rspamd/rspamd/commit/93430bbe86b810010cdbac230336eaeb5cc9cfa9 (HEAD -> master) [Minor] Bimi: Fix Redis storage --- src/plugins/lua/bimi.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plugins/lua/bimi.lua b/src/plugins/lua/bimi.lua index e7e2527e4..d8b36d030 100644 --- a/src/plugins/lua/bimi.lua +++ b/src/plugins/lua/bimi.lua @@ -184,7 +184,8 @@ local function make_helper_request(task, domain, record, redis_server) true, -- is write redis_set_cb, --callback 'PSETEX', -- command - {redis_key, tostring(settings.redis_min_expiry * 1000.0), d.content}) + {redis_key, tostring(settings.redis_min_expiry * 1000.0), + ucl.to_format(d, "json-compact")}) if not ret then rspamd_logger.warnx(task, 'cannot make request to Redis when storing image; domain %s', From vsevolod at highsecure.ru Tue Dec 14 15:56:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 14 Dec 2021 15:56:04 +0000 (UTC) Subject: commit 958449c: [Minor] Bimi: Fold header Message-ID: <20211214155604.72A7727B05@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-14 15:54:36 +0000 URL: https://github.com/rspamd/rspamd/commit/958449c027d3a01e113b92d009bd6ec28bfe71df (HEAD -> master) [Minor] Bimi: Fold header --- src/plugins/lua/bimi.lua | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/plugins/lua/bimi.lua b/src/plugins/lua/bimi.lua index d8b36d030..bc3d90be7 100644 --- a/src/plugins/lua/bimi.lua +++ b/src/plugins/lua/bimi.lua @@ -22,6 +22,7 @@ local lua_redis = require "lua_redis" local ucl = require "ucl" local lua_mime = require "lua_mime" local rspamd_http = require "rspamd_http" +local rspamd_util = require "rspamd_util" local settings = { helper_url = "http://127.0.0.1:3030", @@ -108,9 +109,15 @@ local function check_bimi_record(task, rec) end local function insert_bimi_headers(task, domain, bimi_content) + local hdr_name = 'BIMI-Indicator' lua_mime.modify_headers(task, { - remove = {['BIMI-Indicator'] = 0}, - add = {['BIMI-Indicator'] = {order = 0, value = bimi_content}} + remove = {[hdr_name] = 0}, + add = { + [hdr_name] = { + order = 0, + value = rspamd_util.fold_header(hdr_name, bimi_content) + } + } }) task:insert_result('BIMI_VALID', 1.0, {domain}) end From vsevolod at highsecure.ru Tue Dec 14 17:49:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 14 Dec 2021 17:49:04 +0000 (UTC) Subject: commit 568773d: [Minor] Don't use zero terminated strings Message-ID: <20211214174904.4C42E27B17@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-14 16:16:34 +0000 URL: https://github.com/rspamd/rspamd/commit/568773df1dcf5ed043ca0c760eaf53f1705ceb34 [Minor] Don't use zero terminated strings --- src/libutil/str_util.c | 23 ++++++++++++----------- src/libutil/str_util.h | 2 ++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c index 2d39ccf8a..50af10f28 100644 --- a/src/libutil/str_util.c +++ b/src/libutil/str_util.c @@ -1571,18 +1571,18 @@ rspamd_strings_levenshtein_distance (const gchar *s1, gsize s1len, } GString * -rspamd_header_value_fold (const gchar *name, - const gchar *value, - guint fold_max, - enum rspamd_newlines_type how, - const gchar *fold_on_chars) +rspamd_header_value_fold (const gchar *name, gsize name_len, + const gchar *value, + gsize value_len, + guint fold_max, + enum rspamd_newlines_type how, + const gchar *fold_on_chars) { GString *res; const guint default_fold_max = 76; guint cur_len; - const gchar *p, *c; + const gchar *p, *c, *end; guint nspaces = 0; - const gchar *last; gboolean first_token = TRUE; enum { fold_before = 0, @@ -1603,14 +1603,15 @@ rspamd_header_value_fold (const gchar *name, fold_max = default_fold_max; } - res = g_string_sized_new (strlen (value)); + res = g_string_sized_new (value_len); c = value; p = c; + end = value + value_len; /* name: */ - cur_len = strlen (name) + 2; + cur_len = name_len + 2; - while (*p) { + while (p < end) { switch (state) { case read_token: @@ -1697,7 +1698,7 @@ rspamd_header_value_fold (const gchar *name, * Check any spaces that are appended to the result * before folding */ - last = &res->str[res->len - 1]; + const gchar *last = &res->str[res->len - 1]; while (g_ascii_isspace (*last)) { last --; diff --git a/src/libutil/str_util.h b/src/libutil/str_util.h index 47abf062e..199a384ca 100644 --- a/src/libutil/str_util.h +++ b/src/libutil/str_util.h @@ -366,7 +366,9 @@ gint rspamd_strings_levenshtein_distance (const gchar *s1, gsize s1len, * @return new GString with the folded value */ GString *rspamd_header_value_fold (const gchar *name, + gsize name_len, const gchar *value, + gsize value_len, guint fold_max, enum rspamd_newlines_type how, const gchar *fold_on_chars); From vsevolod at highsecure.ru Tue Dec 14 17:49:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 14 Dec 2021 17:49:05 +0000 (UTC) Subject: commit 601e640: [Minor] Allow lua_text in addition to strings Message-ID: <20211214174905.68E1327B19@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-14 16:16:56 +0000 URL: https://github.com/rspamd/rspamd/commit/601e6403a2c8bcd9e73bb683cd26aa5d3a2c06b9 [Minor] Allow lua_text in addition to strings --- src/lua/lua_text.c | 2 +- src/lua/lua_util.c | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/lua/lua_text.c b/src/lua/lua_text.c index d1fd87ded..f88b5c29b 100644 --- a/src/lua/lua_text.c +++ b/src/lua/lua_text.c @@ -292,7 +292,7 @@ lua_check_text_or_string (lua_State * L, gint pos) * Fake static lua_text, we allow to use this function multiple times * by having a small array of static structures. */ - static int cur_txt_idx = 0; + static unsigned cur_txt_idx = 0; static struct rspamd_lua_text fake_text[4]; gsize len; int sel_idx; diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c index 570a51be7..4dd944008 100644 --- a/src/lua/lua_util.c +++ b/src/lua/lua_util.c @@ -1366,11 +1366,12 @@ static gint lua_util_fold_header (lua_State *L) { LUA_TRACE_POINT; - const gchar *name, *value, *how, *stop_chars = NULL; + const gchar *how, *stop_chars = NULL; + struct rspamd_lua_text *name, *value; GString *folded; - name = luaL_checkstring (L, 1); - value = luaL_checkstring (L, 2); + name = lua_check_text_or_string (L, 1); + value = lua_check_text_or_string (L, 2); if (name && value) { @@ -1383,20 +1384,25 @@ lua_util_fold_header (lua_State *L) } if (strcmp (how, "cr") == 0) { - folded = rspamd_header_value_fold (name, value, 0, + folded = rspamd_header_value_fold (name->start, name->len, + value->start, value->len, + 0, RSPAMD_TASK_NEWLINES_CR, stop_chars); } else if (strcmp (how, "lf") == 0) { - folded = rspamd_header_value_fold (name, value, 0, + folded = rspamd_header_value_fold (name->start, name->len, + value->start, value->len, 0, RSPAMD_TASK_NEWLINES_LF, stop_chars); } else { - folded = rspamd_header_value_fold (name, value, 0, + folded = rspamd_header_value_fold (name->start, name->len, + value->start, value->len, 0, RSPAMD_TASK_NEWLINES_CRLF, stop_chars); } } else { - folded = rspamd_header_value_fold (name, value, 0, + folded = rspamd_header_value_fold (name->start, name->len, + value->start, value->len, 0, RSPAMD_TASK_NEWLINES_CRLF, stop_chars); } From vsevolod at highsecure.ru Tue Dec 14 17:49:06 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 14 Dec 2021 17:49:06 +0000 (UTC) Subject: commit 19afb90: [Minor] Update invocation Message-ID: <20211214174906.8904827B1B@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-14 16:25:19 +0000 URL: https://github.com/rspamd/rspamd/commit/19afb90db2d67a43cde5959e211134cf4231f624 [Minor] Update invocation --- src/client/rspamc.c | 4 ++-- src/libserver/protocol.c | 24 ++++++++++++++++-------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/client/rspamc.c b/src/client/rspamc.c index 4bb632121..3760c62e0 100644 --- a/src/client/rspamc.c +++ b/src/client/rspamc.c @@ -1383,8 +1383,8 @@ rspamc_mime_output (FILE *out, ucl_object_t *result, GString *input, g_string_erase (symbuf, symbuf->len - 1, 1); } - folded_symbuf = rspamd_header_value_fold ("X-Spam-Symbols", - symbuf->str, + folded_symbuf = rspamd_header_value_fold ("X-Spam-Symbols", strlen ("X-Spam-Symbols"), + symbuf->str, symbuf->len, 0, nl_type, ","); rspamd_printf_gstring (added_headers, "X-Spam-Symbols: %v%s", folded_symbuf, line_end); diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c index 65d0900df..eb82a743f 100644 --- a/src/libserver/protocol.c +++ b/src/libserver/protocol.c @@ -1512,12 +1512,16 @@ rspamd_protocol_write_ucl (struct rspamd_task *task, if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER || !task->message) { - folded_header = rspamd_header_value_fold ("DKIM-Signature", - dkim_sig->str, 80, RSPAMD_TASK_NEWLINES_LF, NULL); + folded_header = rspamd_header_value_fold ( + "DKIM-Signature", strlen ("DKIM-Signature"), + dkim_sig->str, dkim_sig->len, + 80, RSPAMD_TASK_NEWLINES_LF, NULL); } else { - folded_header = rspamd_header_value_fold ("DKIM-Signature", - dkim_sig->str, 80, + folded_header = rspamd_header_value_fold ( + "DKIM-Signature", strlen ("DKIM-Signature"), + dkim_sig->str, dkim_sig->len, + 80, MESSAGE_FIELD (task, nlines_type), NULL); } @@ -1539,12 +1543,16 @@ rspamd_protocol_write_ucl (struct rspamd_task *task, dkim_sig = (GString *) dkim_sigs->data; if (task->protocol_flags & RSPAMD_TASK_PROTOCOL_FLAG_MILTER) { - folded_header = rspamd_header_value_fold ("DKIM-Signature", - dkim_sig->str, 80, RSPAMD_TASK_NEWLINES_LF, NULL); + folded_header = rspamd_header_value_fold ( + "DKIM-Signature", strlen ("DKIM-Signature"), + dkim_sig->str, dkim_sig->len, + 80, RSPAMD_TASK_NEWLINES_LF, NULL); } else { - folded_header = rspamd_header_value_fold ("DKIM-Signature", - dkim_sig->str, 80, MESSAGE_FIELD (task, nlines_type), + folded_header = rspamd_header_value_fold ( + "DKIM-Signature", strlen ("DKIM-Signature"), + dkim_sig->str, dkim_sig->len, + 80, MESSAGE_FIELD (task, nlines_type), NULL); } From vsevolod at highsecure.ru Tue Dec 14 17:49:07 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 14 Dec 2021 17:49:07 +0000 (UTC) Subject: commit c1f6f95: [Minor] More fixes for headers folding Message-ID: <20211214174907.A3AAE27B1D@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-14 16:40:01 +0000 URL: https://github.com/rspamd/rspamd/commit/c1f6f952f4932089c45d36b71919330e5ee8dfe1 [Minor] More fixes for headers folding --- src/libutil/str_util.c | 109 ++++++++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 56 deletions(-) diff --git a/src/libutil/str_util.c b/src/libutil/str_util.c index 50af10f28..88f9ce3e5 100644 --- a/src/libutil/str_util.c +++ b/src/libutil/str_util.c @@ -1581,7 +1581,7 @@ rspamd_header_value_fold (const gchar *name, gsize name_len, GString *res; const guint default_fold_max = 76; guint cur_len; - const gchar *p, *c, *end; + const gchar *p, *c, *end, *fold_sequence; guint nspaces = 0; gboolean first_token = TRUE; enum { @@ -1603,6 +1603,19 @@ rspamd_header_value_fold (const gchar *name, gsize name_len, fold_max = default_fold_max; } + switch (how) { + case RSPAMD_TASK_NEWLINES_LF: + fold_sequence = "\n\t"; + break; + case RSPAMD_TASK_NEWLINES_CR: + fold_sequence = "\r\t"; + break; + case RSPAMD_TASK_NEWLINES_CRLF: + default: + fold_sequence ="\r\n\t"; + break; + } + res = g_string_sized_new (value_len); c = value; @@ -1653,16 +1666,44 @@ rspamd_header_value_fold (const gchar *name, gsize name_len, state = fold_token; next_state = read_token; } else { - /* Reset line length */ - cur_len = 0; + /* We need to ensure that it is a folding and not something else */ + + const char *t = p; + bool seen_fold = false; - while (g_ascii_isspace (*p)) { - p++; + while (t < end) { + if (*t == ' ' || *t == '\t') { + seen_fold = true; + break; + } + else if (!g_ascii_isspace(*t)) { + break; + } + + t++; } - g_string_append_len (res, c, p - c); - c = p; - first_token = TRUE; + if (seen_fold) { + /* Reset line length */ + cur_len = 0; + + while (g_ascii_isspace (*p)) { + p++; + } + + g_string_append_len(res, c, p - c); + c = p; + first_token = TRUE; + } + else { + /* Not seen folding, inject it */ + g_string_append_len (res, c, p - c); + g_string_append (res, fold_sequence); + p = t; /* Adjust p to ensure that we do not append extra stuff */ + state = read_token; + first_token = TRUE; + c = p; + } } } else if (g_ascii_isspace (*p)) { if (cur_len > fold_max * 0.8 && cur_len < fold_max) { @@ -1707,18 +1748,7 @@ rspamd_header_value_fold (const gchar *name, gsize name_len, } } - switch (how) { - case RSPAMD_TASK_NEWLINES_LF: - g_string_append_len (res, "\n\t", 2); - break; - case RSPAMD_TASK_NEWLINES_CR: - g_string_append_len (res, "\r\t", 2); - break; - case RSPAMD_TASK_NEWLINES_CRLF: - default: - g_string_append_len (res, "\r\n\t", 3); - break; - } + g_string_append (res, fold_sequence); /* Skip space if needed */ if (g_ascii_isspace (*p)) { @@ -1753,18 +1783,7 @@ rspamd_header_value_fold (const gchar *name, gsize name_len, res->len --; } - switch (how) { - case RSPAMD_TASK_NEWLINES_LF: - g_string_append_len (res, "\n\t", 2); - break; - case RSPAMD_TASK_NEWLINES_CR: - g_string_append_len (res, "\r\t", 2); - break; - case RSPAMD_TASK_NEWLINES_CRLF: - default: - g_string_append_len (res, "\r\n\t", 3); - break; - } + g_string_append (res, fold_sequence); } /* Move leftover spaces */ @@ -1816,18 +1835,7 @@ rspamd_header_value_fold (const gchar *name, gsize name_len, if (g_ascii_isspace (*c)) { c ++; } - switch (how) { - case RSPAMD_TASK_NEWLINES_LF: - g_string_append_len (res, "\n\t", 2); - break; - case RSPAMD_TASK_NEWLINES_CR: - g_string_append_len (res, "\r\t", 2); - break; - case RSPAMD_TASK_NEWLINES_CRLF: - default: - g_string_append_len (res, "\r\n\t", 3); - break; - } + g_string_append (res, fold_sequence); g_string_append_len (res, c, p - c); } else { @@ -1846,18 +1854,7 @@ rspamd_header_value_fold (const gchar *name, gsize name_len, else { if (*c != '\r' && *c != '\n') { /* We need to add folding as well */ - switch (how) { - case RSPAMD_TASK_NEWLINES_LF: - g_string_append_len (res, "\n\t", 2); - break; - case RSPAMD_TASK_NEWLINES_CR: - g_string_append_len (res, "\r\t", 2); - break; - case RSPAMD_TASK_NEWLINES_CRLF: - default: - g_string_append_len (res, "\r\n\t", 3); - break; - } + g_string_append (res, fold_sequence); g_string_append_len (res, c, p - c); } else { From vsevolod at highsecure.ru Tue Dec 14 17:49:08 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 14 Dec 2021 17:49:08 +0000 (UTC) Subject: commit 6a32c3a: [Minor] Bimi: Fold huge bimi header Message-ID: <20211214174908.BDA2A27B1F@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-14 16:41:01 +0000 URL: https://github.com/rspamd/rspamd/commit/6a32c3ad6043c433e5c3ca458c4c3a47bcc67641 [Minor] Bimi: Fold huge bimi header --- src/plugins/lua/bimi.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/lua/bimi.lua b/src/plugins/lua/bimi.lua index bc3d90be7..a2768ea26 100644 --- a/src/plugins/lua/bimi.lua +++ b/src/plugins/lua/bimi.lua @@ -110,12 +110,16 @@ end local function insert_bimi_headers(task, domain, bimi_content) local hdr_name = 'BIMI-Indicator' + -- Re-encode base64... + local content = rspamd_util.encode_base64(rspamd_util.decode_base64(bimi_content), + 73, task:get_newlines_type()) lua_mime.modify_headers(task, { remove = {[hdr_name] = 0}, add = { [hdr_name] = { order = 0, - value = rspamd_util.fold_header(hdr_name, bimi_content) + value = rspamd_util.fold_header(hdr_name, content, + task:get_newlines_type()) } } }) From vsevolod at highsecure.ru Tue Dec 14 17:49:09 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 14 Dec 2021 17:49:09 +0000 (UTC) Subject: commit 0ee4250: [Minor] Dmarc: Fold header using specific newline type Message-ID: <20211214174909.DCDE427B21@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-14 16:41:14 +0000 URL: https://github.com/rspamd/rspamd/commit/0ee4250cac7589cea27a2d6770deb68b01a03242 [Minor] Dmarc: Fold header using specific newline type --- lualib/plugins/dmarc.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lualib/plugins/dmarc.lua b/lualib/plugins/dmarc.lua index 783447242..c72b68824 100644 --- a/lualib/plugins/dmarc.lua +++ b/lualib/plugins/dmarc.lua @@ -197,10 +197,10 @@ exports.gen_munging_callback = function(munging_opts, settings) local hdr_encoded = rspamd_util.fold_header('From', rspamd_util.mime_header_encode(string.format('%s <%s>', - via_name, via_addr))) + via_name, via_addr)), task:get_newlines_type()) local orig_from_encoded = rspamd_util.fold_header('X-Original-From', rspamd_util.mime_header_encode(string.format('%s <%s>', - from.name or '', from.addr))) + from.name or '', from.addr)), task:get_newlines_type()) local add_hdrs = { ['From'] = { order = 1, value = hdr_encoded }, } From vsevolod at highsecure.ru Tue Dec 14 17:49:11 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 14 Dec 2021 17:49:11 +0000 (UTC) Subject: commit fd222e4: [Fix] Fix processing captures from pcre2 Message-ID: <20211214174911.05E4227B23@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-14 17:42:35 +0000 URL: https://github.com/rspamd/rspamd/commit/fd222e43f32a87106612c3b5197a9d491256bfe4 (HEAD -> master) [Fix] Fix processing captures from pcre2 --- src/libutil/regexp.c | 82 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/src/libutil/regexp.c b/src/libutil/regexp.c index 5b928bcb4..cd34a5998 100644 --- a/src/libutil/regexp.c +++ b/src/libutil/regexp.c @@ -547,6 +547,7 @@ rspamd_regexp_search (const rspamd_regexp_t *re, const gchar *text, gsize len, const gchar *mt; gsize remain = 0; gint rc, match_flags = 0, *ovec, ncaptures, i; + const int junk = 0xdeadbabe; g_assert (re != NULL); g_assert (text != NULL); @@ -607,6 +608,11 @@ rspamd_regexp_search (const rspamd_regexp_t *re, const gchar *text, gsize len, ncaptures = (re->ncaptures + 1) * 3; ovec = g_alloca (sizeof (gint) * ncaptures); + + for (i = 0; i < ncaptures; i ++) { + ovec[i] = junk; + } + if (!(re->flags & RSPAMD_REGEXP_FLAG_NOOPT)) { #ifdef HAVE_PCRE_JIT # if defined(HAVE_PCRE_JIT_FAST) && !defined(DISABLE_JIT_FAST) @@ -637,11 +643,21 @@ rspamd_regexp_search (const rspamd_regexp_t *re, const gchar *text, gsize len, } if (rc >= 0) { - if (start) { - *start = mt + ovec[0]; + if (rc > 0) { + if (start) { + *start = mt + ovec[0]; + } + if (end) { + *end = mt + ovec[1]; + } } - if (end) { - *end = mt + ovec[1]; + else { + if (start) { + *start = mt; + } + if (end) { + *end = mt + remain; + } } if (captures != NULL && rc >= 1) { @@ -652,9 +668,16 @@ rspamd_regexp_search (const rspamd_regexp_t *re, const gchar *text, gsize len, g_array_set_size (captures, rc); for (i = 0; i < rc; i ++) { - elt = &g_array_index (captures, struct rspamd_re_capture, i); - elt->p = mt + ovec[i * 2]; - elt->len = (mt + ovec[i * 2 + 1]) - elt->p; + if (ovec[i * 2] != junk && ovec[i * 2] >= 0) { + elt = &g_array_index (captures, struct rspamd_re_capture, i); + elt->p = mt + ovec[i * 2]; + elt->len = (mt + ovec[i * 2 + 1]) - elt->p; + } + else { + /* Runtime match returned fewer captures than expected */ + g_array_set_size (captures, i); + break; + } } } @@ -682,7 +705,8 @@ rspamd_regexp_search (const rspamd_regexp_t *re, const gchar *text, gsize len, pcre2_match_context *mcontext; PCRE_T *r; const gchar *mt; - gsize remain = 0, *ovec; + PCRE2_SIZE remain = 0, *ovec; + const PCRE2_SIZE junk = 0xdeadbabeeeeeeeeULL; gint rc, match_flags, novec, i; gboolean ret = FALSE; @@ -731,6 +755,14 @@ rspamd_regexp_search (const rspamd_regexp_t *re, const gchar *text, gsize len, } match_data = pcre2_match_data_create (re->ncaptures + 1, NULL); + novec = pcre2_get_ovector_count (match_data); + ovec = pcre2_get_ovector_pointer (match_data); + + /* Fill ovec with crap, so we can stop if actual matches is less than announced */ + for (i = 0; i < novec; i ++) { + ovec[i * 2] = junk; + ovec[i * 2 + 1] = junk; + } #ifdef HAVE_PCRE_JIT if (!(re->flags & RSPAMD_REGEXP_FLAG_DISABLE_JIT) && can_jit) { @@ -752,14 +784,21 @@ rspamd_regexp_search (const rspamd_regexp_t *re, const gchar *text, gsize len, #endif if (rc >= 0) { - novec = pcre2_get_ovector_count (match_data); - ovec = pcre2_get_ovector_pointer (match_data); - - if (start) { - *start = mt + ovec[0]; + if (novec > 0) { + if (start) { + *start = mt + ovec[0]; + } + if (end) { + *end = mt + ovec[1]; + } } - if (end) { - *end = mt + ovec[1]; + else { + if (start) { + *start = mt; + } + if (end) { + *end = mt + remain; + } } if (captures != NULL && novec >= 1) { @@ -770,10 +809,15 @@ rspamd_regexp_search (const rspamd_regexp_t *re, const gchar *text, gsize len, g_array_set_size (captures, novec); for (i = 0; i < novec; i ++) { - elt = &g_array_index (captures, struct rspamd_re_capture, i); - elt->p = mt + ovec[i * 2]; - elt->len = (mt + ovec[i * 2 + 1]) - elt->p; - + if (ovec[i * 2] != junk && ovec[i * 2] != PCRE2_UNSET) { + elt = &g_array_index (captures, struct rspamd_re_capture, i); + elt->p = mt + ovec[i * 2]; + elt->len = (mt + ovec[i * 2 + 1]) - elt->p; + } + else { + g_array_set_size (captures, i); + break; + } } } From vsevolod at highsecure.ru Wed Dec 15 15:21:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 15 Dec 2021 15:21:04 +0000 (UTC) Subject: commit c20ae89: [Minor] Antivirus: Improve parts selection Message-ID: <20211215152104.34F2127BE4@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-15 15:20:19 +0000 URL: https://github.com/rspamd/rspamd/commit/c20ae890aa62c5323c7dad718ef356e09b00e2a5 (HEAD -> master) [Minor] Antivirus: Improve parts selection --- lualib/lua_scanners/common.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lualib/lua_scanners/common.lua b/lualib/lua_scanners/common.lua index a92d923d0..d052f4552 100644 --- a/lualib/lua_scanners/common.lua +++ b/lualib/lua_scanners/common.lua @@ -459,14 +459,15 @@ local function check_parts_match(task, rule) end if rule.scan_all_mime_parts ~= false then + local is_part_checkable = (p:is_attachment() and (not p:is_image() or rule.scan_image_mime)) if detected_ext then -- We know what to scan! local magic = lua_magic_types[detected_ext] or {} - if p:is_attachment() or magic.av_check ~= false then + if magic.av_check ~= false or is_part_checkable then return true end - elseif p:is_attachment() then + elseif is_part_checkable then -- Just rely on attachment property return true end From vsevolod at highsecure.ru Thu Dec 16 15:56:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 16 Dec 2021 15:56:04 +0000 (UTC) Subject: commit 0983f59: [Minor] Add type safety and fix default font size Message-ID: <20211216155604.7DB1327CAA@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-16 15:37:26 +0000 URL: https://github.com/rspamd/rspamd/commit/0983f59671106a252d6d2f193e22f6e0759f35e1 [Minor] Add type safety and fix default font size --- src/libserver/html/html_block.hxx | 133 +++++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 59 deletions(-) diff --git a/src/libserver/html/html_block.hxx b/src/libserver/html/html_block.hxx index 4a69b7325..aeb802013 100644 --- a/src/libserver/html/html_block.hxx +++ b/src/libserver/html/html_block.hxx @@ -71,7 +71,8 @@ struct html_block { } height_mask = how; } - auto set_width(float w, bool is_percent = false, int how = html_block::set) -> void { + + auto set_width(float w, bool is_percent = false, int how = html_block::set) -> void { w = is_percent ? (-w) : w; if (w < INT16_MIN) { width = INT16_MIN; @@ -84,6 +85,7 @@ struct html_block { } width_mask = how; } + auto set_display(bool v, int how = html_block::set) -> void { if (v) { display = rspamd::css::css_display_value::DISPLAY_INLINE; @@ -93,11 +95,13 @@ struct html_block { } display_mask = how; } - auto set_display(rspamd::css::css_display_value v, int how = html_block::set) -> void { + + auto set_display(rspamd::css::css_display_value v, int how = html_block::set) -> void { display = v; display_mask = how; } - auto set_font_size(float fs, bool is_percent = false, int how = html_block::set) -> void { + + auto set_font_size(float fs, bool is_percent = false, int how = html_block::set) -> void { fs = is_percent ? (-fs) : fs; if (fs < INT8_MIN) { font_size = -100; @@ -111,72 +115,83 @@ struct html_block { font_mask = how; } - /** - * Propagate values from the block if they are not defined by the current block - * @param other - * @return - */ - auto propagate_block(const html_block &other) -> void { - auto simple_prop = [](auto mask_val, auto other_mask, auto &our_val, - auto other_val) constexpr -> int { - if (other_mask && other_mask > mask_val) { - our_val = other_val; - mask_val = html_block::inherited; - } +private: + template + static constexpr auto simple_prop(MT mask_val, MT other_mask, T &our_val, + T other_val) -> MT + { + if (other_mask && other_mask > mask_val) { + our_val = other_val; + mask_val = html_block::inherited; + } - return mask_val; - }; + return mask_val; + } - fg_color_mask = simple_prop(fg_color_mask, other.fg_color_mask, fg_color, other.fg_color); - bg_color_mask = simple_prop(bg_color_mask, other.bg_color_mask, bg_color, other.bg_color); - display_mask = simple_prop(display_mask, other.display_mask, display, other.display); - - /* Sizes are very different - * We can have multiple cases: - * 1) Our size is > 0 and we can use it as is - * 2) Parent size is > 0 and our size is undefined, so propagate parent - * 3) Parent size is < 0 and our size is undefined - propagate parent - * 4) Parent size is > 0 and our size is < 0 - multiply parent by abs(ours) - * 5) Parent size is undefined and our size is < 0 - tricky stuff, assume some defaults - */ - auto size_prop = [](auto mask_val, auto other_mask, auto &our_val, - auto other_val, auto default_val) constexpr -> int { - if (mask_val) { - /* We have our value */ - if (our_val < 0) { - if (other_mask > 0) { - if (other_val >= 0) { - our_val = other_val * (-our_val / 100.0); - } - else { - our_val *= (-other_val / 100.0); - } + /* Sizes propagation logic + * We can have multiple cases: + * 1) Our size is > 0 and we can use it as is + * 2) Parent size is > 0 and our size is undefined, so propagate parent + * 3) Parent size is < 0 and our size is undefined - propagate parent + * 4) Parent size is > 0 and our size is < 0 - multiply parent by abs(ours) + * 5) Parent size is undefined and our size is < 0 - tricky stuff, assume some defaults + */ + template + static constexpr auto size_prop (MT mask_val, MT other_mask, T &our_val, + T other_val, T default_val) -> MT + { + if (mask_val) { + /* We have our value */ + if (our_val < 0) { + if (other_mask > 0) { + if (other_val >= 0) { + our_val = other_val * (-our_val / 100.0); } else { - /* Parent value is not defined and our value is relative */ - our_val = default_val * (-our_val / 100.0); + our_val *= (-other_val / 100.0); } } - else if (other_mask && other_mask > mask_val) { - our_val = other_val; - mask_val = html_block::inherited; + else { + /* Parent value is not defined and our value is relative */ + our_val = default_val * (-our_val / 100.0); } } - else { - /* We propagate parent if defined */ - if (other_mask && other_mask > mask_val) { - our_val = other_val; - mask_val = html_block::inherited; - } - /* Otherwise do nothing */ + else if (other_mask && other_mask > mask_val) { + our_val = other_val; + mask_val = html_block::inherited; } + } + else { + /* We propagate parent if defined */ + if (other_mask && other_mask > mask_val) { + our_val = other_val; + mask_val = html_block::inherited; + } + /* Otherwise do nothing */ + } - return mask_val; - }; - - height_mask = size_prop(height_mask, other.height_mask, height, other.height, 800); - width_mask = size_prop(width_mask, other.width_mask, width, other.width, 1024); - font_mask = size_prop(font_mask, other.font_mask, font_size, other.font_size, 1024); + return mask_val; + } +public: + /** + * Propagate values from the block if they are not defined by the current block + * @param other + * @return + */ + auto propagate_block(const html_block &other) -> void { + fg_color_mask = html_block::simple_prop(fg_color_mask, other.fg_color_mask, + fg_color, other.fg_color); + bg_color_mask = html_block::simple_prop(bg_color_mask, other.bg_color_mask, + bg_color, other.bg_color); + display_mask = html_block::simple_prop(display_mask, other.display_mask, + display, other.display); + + height_mask = html_block::size_prop(height_mask, other.height_mask, + height, other.height, static_cast(800)); + width_mask = html_block::size_prop(width_mask, other.width_mask, + width, other.width, static_cast(1024)); + font_mask = html_block::size_prop(font_mask, other.font_mask, + font_size, other.font_size, static_cast(1024)); } /* From vsevolod at highsecure.ru Thu Dec 16 15:56:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 16 Dec 2021 15:56:05 +0000 (UTC) Subject: commit 84aef02: [Minor] Really set a sane font size Message-ID: <20211216155605.9CAE327CAC@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-16 15:50:52 +0000 URL: https://github.com/rspamd/rspamd/commit/84aef0280c7ea8a18737d4ee5232f4a9bd004d30 (HEAD -> master) [Minor] Really set a sane font size --- src/libserver/html/html_block.hxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libserver/html/html_block.hxx b/src/libserver/html/html_block.hxx index aeb802013..9cdbfd728 100644 --- a/src/libserver/html/html_block.hxx +++ b/src/libserver/html/html_block.hxx @@ -191,7 +191,7 @@ public: width_mask = html_block::size_prop(width_mask, other.width_mask, width, other.width, static_cast(1024)); font_mask = html_block::size_prop(font_mask, other.font_mask, - font_size, other.font_size, static_cast(1024)); + font_size, other.font_size, static_cast(10)); } /* From vsevolod at highsecure.ru Fri Dec 17 12:07:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Fri, 17 Dec 2021 12:07:04 +0000 (UTC) Subject: commit d8519d5: [Minor] Clear up on lua errors Message-ID: <20211217120704.519ED27D4D@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-17 12:04:00 +0000 URL: https://github.com/rspamd/rspamd/commit/d8519d5e744362e7ed950d7095b4269fc230cb1b (HEAD -> master) [Minor] Clear up on lua errors --- src/libserver/re_cache.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c index 2c5555154..cde2e88a4 100644 --- a/src/libserver/re_cache.c +++ b/src/libserver/re_cache.c @@ -570,6 +570,7 @@ rspamd_re_cache_check_lua_condition (struct rspamd_task *task, msg_warn_task ("cannot call for re_cache_check_lua_condition for re %s: %e", rspamd_regexp_get_pattern (re), err); g_error_free (err); + lua_settop (L, text_pos - 1); return TRUE; } From vsevolod at highsecure.ru Fri Dec 17 18:14:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Fri, 17 Dec 2021 18:14:04 +0000 (UTC) Subject: commit 9752886: [Minor] Increase EDNS0 max packet size to couple with some poor TXT records Message-ID: <20211217181404.260AC27D7F@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-17 18:08:24 +0000 URL: https://github.com/rspamd/rspamd/commit/9752886d5dde4b2295d3b7b91e3ea23416eea641 (HEAD -> master) [Minor] Increase EDNS0 max packet size to couple with some poor TXT records --- contrib/librdns/dns_private.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/librdns/dns_private.h b/contrib/librdns/dns_private.h index 76bfba8fe..19729f1e6 100644 --- a/contrib/librdns/dns_private.h +++ b/contrib/librdns/dns_private.h @@ -34,7 +34,7 @@ static const int dns_port = 53; static const int default_io_cnt = 8; -#define UDP_PACKET_SIZE 4096 +#define UDP_PACKET_SIZE (4096 * 2) #define DNS_COMPRESSION_BITS 0xC0 From moiseev at mezonplus.ru Sat Dec 18 19:14:04 2021 From: moiseev at mezonplus.ru (moisseev) Date: Sat, 18 Dec 2021 19:14:04 +0000 (UTC) Subject: commit 6a2db7e: [WebUI] Fix saving credentials in password manager Message-ID: <20211218191404.382A427E4B@lists.rspamd.com> Author: moisseev Date: 2021-12-18 20:44:39 +0300 URL: https://github.com/rspamd/rspamd/commit/6a2db7ef3845584274445b93b4aa98533b6be2e3 (refs/pull/4014/head) [WebUI] Fix saving credentials in password manager for recent browser versions. --- interface/index.html | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/interface/index.html b/interface/index.html index 1b66cf329..2a5e181ae 100644 --- a/interface/index.html +++ b/interface/index.html @@ -589,6 +589,11 @@
+ + Author: Vsevolod Stakhov Date: 2021-12-18 19:12:47 +0000 URL: https://github.com/rspamd/rspamd/commit/c1cce0dd49f51680ea76145e912ee7792d341323 (HEAD -> master) Merge pull request #4014 from moisseev/webui [WebUI] Fix saving credentials in password manager interface/index.html | 5 +++++ 1 file changed, 5 insertions(+) From moiseev at mezonplus.ru Sat Dec 18 19:35:04 2021 From: moiseev at mezonplus.ru (moisseev) Date: Sat, 18 Dec 2021 19:35:04 +0000 (UTC) Subject: commit 7afa069: [Test] Disable prefer-object-has-own ESLint rule Message-ID: <20211218193504.1DF9A27E51@lists.rspamd.com> Author: moisseev Date: 2021-12-18 22:22:24 +0300 URL: https://github.com/rspamd/rspamd/commit/7afa0694dac2e455828cb12e863489ddc0783e4c (refs/pull/4015/head) [Test] Disable prefer-object-has-own ESLint rule as Object.hasOwn(?) is not implemented in Pale Moon. --- .eslintrc.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc.json b/.eslintrc.json index 2dc3ebc51..ff514b13c 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -47,6 +47,7 @@ "padded-blocks": "off", "prefer-arrow-callback": "off", "prefer-destructuring": "off", + "prefer-object-has-own": "off", "prefer-spread": "off", "prefer-template": "off", "quote-props" : ["error", "consistent-as-needed"], From noreply at github.com Sat Dec 18 19:35:05 2021 From: noreply at github.com (GitHub) Date: Sat, 18 Dec 2021 19:35:05 +0000 (UTC) Subject: commit c821bfa: Merge pull request #4015 from moisseev/eslint Message-ID: <20211218193505.3039E27E53@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-18 19:28:05 +0000 URL: https://github.com/rspamd/rspamd/commit/c821bfa3a7fa7b173e415b812fc1f7d3f5eed620 (HEAD -> master) Merge pull request #4015 from moisseev/eslint [Test] Disable prefer-object-has-own ESLint rule .eslintrc.json | 1 + 1 file changed, 1 insertion(+) From jean-louis at dupond.be Sun Dec 19 14:28:04 2021 From: jean-louis at dupond.be (Jean-Louis Dupond) Date: Sun, 19 Dec 2021 14:28:04 +0000 (UTC) Subject: commit 52fd01b: Assign weights to Mailspike Whitelists Message-ID: <20211219142804.4BA1327EEC@lists.rspamd.com> Author: Jean-Louis Dupond Date: 2021-12-16 16:18:42 +0100 URL: https://github.com/rspamd/rspamd/commit/52fd01b1da8d1990e38fefa2b19a2d26bb1b6ccd (refs/pull/4010/head) Assign weights to Mailspike Whitelists Assign a weight to the MAILSPIKE whitelist RBL symbols. Use the same but negative weight as its blacklist counterparts. Also fix some indents in the file --- conf/scores.d/rbl_group.conf | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/conf/scores.d/rbl_group.conf b/conf/scores.d/rbl_group.conf index 690d0ee17..6b3f04de1 100644 --- a/conf/scores.d/rbl_group.conf +++ b/conf/scores.d/rbl_group.conf @@ -117,14 +117,14 @@ symbols = { groups = ["spamhaus"]; } "RBL_SPAMHAUS_BLOCKED_OPENRESOLVER" { - weight = 0.0; - description = "You are querying Spamhaus from an open resolver, please see https://www.spamhaus.org/returnc/pub/"; - groups = ["spamhaus"]; + weight = 0.0; + description = "You are querying Spamhaus from an open resolver, please see https://www.spamhaus.org/returnc/pub/"; + groups = ["spamhaus"]; } "RBL_SPAMHAUS_BLOCKED" { - weight = 0.0; - description = "You are exceeding the query limit, please see https://www.spamhaus.org/returnc/vol/"; - groups = ["spamhaus"]; + weight = 0.0; + description = "You are exceeding the query limit, please see https://www.spamhaus.org/returnc/vol/"; + groups = ["spamhaus"]; } "RECEIVED_SPAMHAUS_SBL" { weight = 1.0; @@ -157,14 +157,14 @@ symbols = { one_shot = true; } "RECEIVED_SPAMHAUS_BLOCKED_OPENRESOLVER" { - weight = 0.0; - description = "You are querying Spamhaus from an open resolver, please see https://www.spamhaus.org/returnc/pub/"; - groups = ["spamhaus"]; + weight = 0.0; + description = "You are querying Spamhaus from an open resolver, please see https://www.spamhaus.org/returnc/pub/"; + groups = ["spamhaus"]; } "RECEIVED_SPAMHAUS_BLOCKED" { - weight = 0.0; - description = "You are exceeding the query limit, please see https://www.spamhaus.org/returnc/vol/"; - groups = ["spamhaus"]; + weight = 0.0; + description = "You are exceeding the query limit, please see https://www.spamhaus.org/returnc/vol/"; + groups = ["spamhaus"]; } "RBL_SENDERSCORE" { @@ -202,17 +202,17 @@ symbols = { groups = ["mailspike"]; } "RWL_MAILSPIKE_GOOD" { - weight = 0.0; + weight = -0.1; description = "From address is listed in RWL - good reputation"; groups = ["mailspike"]; } "RWL_MAILSPIKE_VERYGOOD" { - weight = 0.0; + weight = -0.2; description = "From address is listed in RWL - very good reputation"; groups = ["mailspike"]; } "RWL_MAILSPIKE_EXCELLENT" { - weight = 0.0; + weight = -0.4; description = "From address is listed in RWL - excellent reputation"; groups = ["mailspike"]; } From jean-louis at dupond.be Sun Dec 19 14:28:05 2021 From: jean-louis at dupond.be (Jean-Louis Dupond) Date: Sun, 19 Dec 2021 14:28:05 +0000 (UTC) Subject: commit 9a0bf41: Map application/x-pdf to pdf Message-ID: <20211219142805.5ADC727EEE@lists.rspamd.com> Author: Jean-Louis Dupond Date: 2021-12-17 10:18:35 +0100 URL: https://github.com/rspamd/rspamd/commit/9a0bf41851697b92a0d4bfd3ed3643d3b5f838b9 (refs/pull/4013/head) Map application/x-pdf to pdf Some mails were marked as MIME_BAD_ATTACHMENT (4) [pdf:application/x-pdf] application/x-pdf might be obsolete but still used. --- conf/modules.d/mime_types.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/modules.d/mime_types.conf b/conf/modules.d/mime_types.conf index 376b70171..1f67595a7 100644 --- a/conf/modules.d/mime_types.conf +++ b/conf/modules.d/mime_types.conf @@ -30,7 +30,8 @@ mime_types { ]; pdf = [ "application/octet-stream", - "application/pdf" + "application/pdf", + "application/x-pdf" ]; } From noreply at github.com Sun Dec 19 14:28:06 2021 From: noreply at github.com (GitHub) Date: Sun, 19 Dec 2021 14:28:06 +0000 (UTC) Subject: commit a9c94c9: Merge pull request #4010 from dupondje/mailspike_wl Message-ID: <20211219142806.79B8C27EF0@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-19 14:27:11 +0000 URL: https://github.com/rspamd/rspamd/commit/a9c94c9389fb65c9f48d49ab5c9ffe690d5512cb Merge pull request #4010 from dupondje/mailspike_wl Assign weights to Mailspike Whitelists conf/scores.d/rbl_group.conf | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) From noreply at github.com Sun Dec 19 14:28:07 2021 From: noreply at github.com (GitHub) Date: Sun, 19 Dec 2021 14:28:07 +0000 (UTC) Subject: commit c62978c: Merge pull request #4013 from dupondje/add_pdf_mapping Message-ID: <20211219142807.8B8D927EF2@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-19 14:27:22 +0000 URL: https://github.com/rspamd/rspamd/commit/c62978c254eadf54abb7c584a1dae8144c2325d4 (HEAD -> master) Merge pull request #4013 from dupondje/add_pdf_mapping Map application/x-pdf to pdf conf/modules.d/mime_types.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) From vsevolod at highsecure.ru Mon Dec 20 16:42:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Mon, 20 Dec 2021 16:42:04 +0000 (UTC) Subject: commit a1652ef: [Minor] Enable redis periodics for the fuzzy worker Message-ID: <20211220164204.8DF6227FCB@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-20 16:36:55 +0000 URL: https://github.com/rspamd/rspamd/commit/a1652efaebec969dae1f4924744c11e01221de43 (HEAD -> master) [Minor] Enable redis periodics for the fuzzy worker --- lualib/lua_redis.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lualib/lua_redis.lua b/lualib/lua_redis.lua index 296638078..5bd6ede30 100644 --- a/lualib/lua_redis.lua +++ b/lualib/lua_redis.lua @@ -276,7 +276,7 @@ local function add_redis_sentinels(params) rspamd_config:add_on_load(function(_, ev_base, worker) local initialised = false - if worker:is_scanner() then + if worker:is_scanner() or worker:get_type() == 'fuzzy' then rspamd_config:add_periodic(ev_base, 0.0, function() redis_query_sentinel(ev_base, params, initialised) initialised = true From vsevolod at highsecure.ru Tue Dec 21 21:21:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 21 Dec 2021 21:21:04 +0000 (UTC) Subject: commit dd2b7b4: [Minor] Add unhex utility Message-ID: <20211221212104.7BDEA380B0@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-21 20:21:06 +0000 URL: https://github.com/rspamd/rspamd/commit/dd2b7b4c24e34d77382ab7213d21546d8f3d93e7 [Minor] Add unhex utility --- lualib/lua_util.lua | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lualib/lua_util.lua b/lualib/lua_util.lua index 75d218da3..4b0e50192 100644 --- a/lualib/lua_util.lua +++ b/lualib/lua_util.lua @@ -1473,4 +1473,19 @@ exports.shuffle = function(tbl) return tbl end +-- +local hex_table = {} +for idx = 0, 255 do + hex_table[("%02X"):format(idx)] = string.char(idx) + hex_table[("%02x"):format(idx)] = string.char(idx) +end + +---[[[ +-- @function lua_util.unhex(str) +-- Decode hex encoded string +-- @param {string} str string to decode +-- @return {string} hex decoded string (valid hex pairs are decoded, everything else is printed as is) +--]]] +exports.unhex = function(str) return str:gsub('(..)', hex_table) end + return exports From vsevolod at highsecure.ru Tue Dec 21 21:21:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 21 Dec 2021 21:21:05 +0000 (UTC) Subject: commit 2bcc989: [Minor] Use unhex utility Message-ID: <20211221212105.8CB07380B2@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-21 21:18:17 +0000 URL: https://github.com/rspamd/rspamd/commit/2bcc9897a397de883561ebc19605d667a517c1c3 (HEAD -> master) [Minor] Use unhex utility --- lualib/lua_content/pdf.lua | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lualib/lua_content/pdf.lua b/lualib/lua_content/pdf.lua index b3c1f6002..45c11d3b4 100644 --- a/lualib/lua_content/pdf.lua +++ b/lualib/lua_content/pdf.lua @@ -181,17 +181,14 @@ local function generic_grammar_elts() -- Helper functions local function pdf_hexstring_unescape(s) - local function ue(cc) - return string.char(tonumber(cc, 16)) - end if #s % 2 == 0 then -- Sane hex string - return s:gsub('..', ue) + return lua_util.unhex(s) end -- WTF hex string -- Append '0' to it and unescape... - return s:sub(1, #s - 1):gsub('..' , ue) .. (s:sub(#s) .. '0'):gsub('..' , ue) + return lua_util.unhex(s:sub(1, #s - 1)) .. lua_util.unhex((s:sub(#s) .. '0')) end local function pdf_string_unescape(s) From vsevolod at highsecure.ru Wed Dec 22 20:07:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 22 Dec 2021 20:07:04 +0000 (UTC) Subject: commit 9faf4e7: [Minor] Fix fuzzy hashes exporting Message-ID: <20211222200704.4A25A38173@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-22 20:04:05 +0000 URL: https://github.com/rspamd/rspamd/commit/9faf4e70cee731053c540931c1ab7eb353a5dadc (HEAD -> master) [Minor] Fix fuzzy hashes exporting Suggested by: @dupondje --- src/plugins/lua/metadata_exporter.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/plugins/lua/metadata_exporter.lua b/src/plugins/lua/metadata_exporter.lua index 48a5ffce3..b6ad3363d 100644 --- a/src/plugins/lua/metadata_exporter.lua +++ b/src/plugins/lua/metadata_exporter.lua @@ -91,7 +91,11 @@ local function get_general_metadata(task, flatten, no_content) r.fuzzy = table.concat(fz, ', ') end else - r.fuzzy = 'unknown' + if not flatten then + r.fuzzy = {} + else + r.fuzzy = '' + end end local rcpt = task:get_recipients('smtp') From vsevolod at highsecure.ru Wed Dec 22 20:49:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 22 Dec 2021 20:49:04 +0000 (UTC) Subject: commit 6f2788b: [Fix] Try to revert back maps content on errors properly Message-ID: <20211222204904.4286338179@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-22 20:47:13 +0000 URL: https://github.com/rspamd/rspamd/commit/6f2788b45b5982be1257832cb1c2546397117245 (HEAD -> master) [Fix] Try to revert back maps content on errors properly --- src/libserver/maps/map.c | 10 ++- src/libserver/maps/map.h | 1 + src/libserver/maps/map_helpers.c | 164 +++++++++++++++++++++++++-------------- src/lua/lua_map.c | 96 +++++++++++++---------- 4 files changed, 169 insertions(+), 102 deletions(-) diff --git a/src/libserver/maps/map.c b/src/libserver/maps/map.c index 531a7ce10..4d9b1584b 100644 --- a/src/libserver/maps/map.c +++ b/src/libserver/maps/map.c @@ -983,8 +983,8 @@ rspamd_map_periodic_dtor (struct map_periodic_cbdata *periodic) map = periodic->map; msg_debug_map ("periodic dtor %p", periodic); - if (periodic->need_modify) { - /* We are done */ + if (periodic->need_modify || periodic->cbdata.errored) { + /* Need to notify the real data structure */ periodic->map->fin_callback (&periodic->cbdata, periodic->map->user_data); } else { @@ -1138,7 +1138,6 @@ rspamd_map_schedule_periodic (struct rspamd_map *map, int how) } cbd = g_malloc0 (sizeof (*cbd)); - cbd->cbdata.state = 0; cbd->cbdata.prev_data = *map->user_data; cbd->cbdata.cur_data = NULL; cbd->cbdata.map = map; @@ -2000,7 +1999,7 @@ rspamd_map_process_periodic (struct map_periodic_cbdata *cbd) } if (cbd->errored) { - /* We should not check other backends if some backend has failed */ + /* We should not check other backends if some backend has failed*/ rspamd_map_schedule_periodic (cbd->map, RSPAMD_MAP_SCHEDULE_ERROR); if (cbd->locked) { @@ -2008,6 +2007,9 @@ rspamd_map_process_periodic (struct map_periodic_cbdata *cbd) cbd->locked = FALSE; } + /* Also set error flag for the map consumer */ + cbd->cbdata.errored = true; + msg_debug_map ("unlocked map %s, refcount=%d", cbd->map->name, cbd->ref.refcount); MAP_RELEASE (cbd, "periodic"); diff --git a/src/libserver/maps/map.h b/src/libserver/maps/map.h index 0812e1d44..6d77454fb 100644 --- a/src/libserver/maps/map.h +++ b/src/libserver/maps/map.h @@ -50,6 +50,7 @@ struct rspamd_map; struct map_cb_data { struct rspamd_map *map; gint state; + bool errored; void *prev_data; void *cur_data; }; diff --git a/src/libserver/maps/map_helpers.c b/src/libserver/maps/map_helpers.c index a29467497..4eb6b2fee 100644 --- a/src/libserver/maps/map_helpers.c +++ b/src/libserver/maps/map_helpers.c @@ -940,22 +940,34 @@ rspamd_kv_list_fin (struct map_cb_data *data, void **target) struct rspamd_map *map = data->map; struct rspamd_hash_map_helper *htb; - if (data->cur_data) { - htb = (struct rspamd_hash_map_helper *)data->cur_data; - msg_info_map ("read hash of %d elements from %s", kh_size (htb->htb), - map->name); - data->map->traverse_function = rspamd_map_helper_traverse_hash; - data->map->nelts = kh_size (htb->htb); - data->map->digest = rspamd_cryptobox_fast_hash_final (&htb->hst); + if (data->errored) { + /* Clean up the current data and do not touch prev data */ + if (data->cur_data) { + msg_info_map ("cleanup unfinished new data as error occurred for %s", + map->name); + htb = (struct rspamd_hash_map_helper *) data->cur_data; + rspamd_map_helper_destroy_hash(htb); + data->cur_data = NULL; + } } + else { + if (data->cur_data) { + htb = (struct rspamd_hash_map_helper *) data->cur_data; + msg_info_map ("read hash of %d elements from %s", kh_size(htb->htb), + map->name); + data->map->traverse_function = rspamd_map_helper_traverse_hash; + data->map->nelts = kh_size (htb->htb); + data->map->digest = rspamd_cryptobox_fast_hash_final(&htb->hst); + } - if (target) { - *target = data->cur_data; - } + if (target) { + *target = data->cur_data; + } - if (data->prev_data) { - htb = (struct rspamd_hash_map_helper *)data->prev_data; - rspamd_map_helper_destroy_hash (htb); + if (data->prev_data) { + htb = (struct rspamd_hash_map_helper *) data->prev_data; + rspamd_map_helper_destroy_hash(htb); + } } } @@ -1000,22 +1012,34 @@ rspamd_radix_fin (struct map_cb_data *data, void **target) struct rspamd_map *map = data->map; struct rspamd_radix_map_helper *r; - if (data->cur_data) { - r = (struct rspamd_radix_map_helper *)data->cur_data; - msg_info_map ("read radix trie of %z elements: %s", - radix_get_size (r->trie), radix_get_info (r->trie)); - data->map->traverse_function = rspamd_map_helper_traverse_radix; - data->map->nelts = kh_size (r->htb); - data->map->digest = rspamd_cryptobox_fast_hash_final (&r->hst); + if (data->errored) { + /* Clean up the current data and do not touch prev data */ + if (data->cur_data) { + msg_info_map ("cleanup unfinished new data as error occurred for %s", + map->name); + r = (struct rspamd_radix_map_helper *) data->cur_data; + rspamd_map_helper_destroy_radix(r); + data->cur_data = NULL; + } } + else { + if (data->cur_data) { + r = (struct rspamd_radix_map_helper *) data->cur_data; + msg_info_map ("read radix trie of %z elements: %s", + radix_get_size(r->trie), radix_get_info(r->trie)); + data->map->traverse_function = rspamd_map_helper_traverse_radix; + data->map->nelts = kh_size (r->htb); + data->map->digest = rspamd_cryptobox_fast_hash_final(&r->hst); + } - if (target) { - *target = data->cur_data; - } + if (target) { + *target = data->cur_data; + } - if (data->prev_data) { - r = (struct rspamd_radix_map_helper *)data->prev_data; - rspamd_map_helper_destroy_radix (r); + if (data->prev_data) { + r = (struct rspamd_radix_map_helper *) data->prev_data; + rspamd_map_helper_destroy_radix(r); + } } } @@ -1494,33 +1518,45 @@ rspamd_regexp_list_fin (struct map_cb_data *data, void **target) struct rspamd_regexp_map_helper *re_map = NULL, *old_re_map; struct rspamd_map *map = data->map; - if (data->cur_data) { - re_map = data->cur_data; - rspamd_cryptobox_hash_final (&re_map->hst, re_map->re_digest); - memcpy (&data->map->digest, re_map->re_digest, sizeof (data->map->digest)); - rspamd_re_map_finalize (re_map); - msg_info_map ("read regexp list of %ud elements", - re_map->regexps->len); - data->map->traverse_function = rspamd_map_helper_traverse_regexp; - data->map->nelts = kh_size (re_map->htb); + if (data->errored) { + /* Clean up the current data and do not touch prev data */ + if (data->cur_data) { + msg_info_map ("cleanup unfinished new data as error occurred for %s", + map->name); + re_map = (struct rspamd_regexp_map_helper *)data->cur_data; + rspamd_map_helper_destroy_regexp (re_map); + data->cur_data = NULL; + } } + else { + if (data->cur_data) { + re_map = data->cur_data; + rspamd_cryptobox_hash_final(&re_map->hst, re_map->re_digest); + memcpy(&data->map->digest, re_map->re_digest, sizeof(data->map->digest)); + rspamd_re_map_finalize(re_map); + msg_info_map ("read regexp list of %ud elements", + re_map->regexps->len); + data->map->traverse_function = rspamd_map_helper_traverse_regexp; + data->map->nelts = kh_size (re_map->htb); + } - if (target) { - *target = data->cur_data; - } + if (target) { + *target = data->cur_data; + } - if (data->prev_data) { - old_re_map = data->prev_data; + if (data->prev_data) { + old_re_map = data->prev_data; #ifdef WITH_HYPERSCAN - if (re_map && memcmp (re_map->re_digest, old_re_map->re_digest, - sizeof (re_map->re_digest)) != 0) { - /* Cleanup old stuff */ - rspamd_re_map_cache_cleanup_old (old_re_map); - } + if (re_map && memcmp(re_map->re_digest, old_re_map->re_digest, + sizeof(re_map->re_digest)) != 0) { + /* Cleanup old stuff */ + rspamd_re_map_cache_cleanup_old(old_re_map); + } #endif - rspamd_map_helper_destroy_regexp (old_re_map); + rspamd_map_helper_destroy_regexp(old_re_map); + } } } void @@ -1889,21 +1925,33 @@ rspamd_cdb_list_fin (struct map_cb_data *data, void **target) struct rspamd_map *map = data->map; struct rspamd_cdb_map_helper *cdb_data; - if (data->cur_data) { - cdb_data = (struct rspamd_cdb_map_helper *)data->cur_data; - msg_info_map ("read cdb of %Hz size", cdb_data->total_size); - data->map->traverse_function = NULL; - data->map->nelts = 0; - data->map->digest = rspamd_cryptobox_fast_hash_final (&cdb_data->hst); + if (data->errored) { + /* Clean up the current data and do not touch prev data */ + if (data->cur_data) { + msg_info_map ("cleanup unfinished new data as error occurred for %s", + map->name); + cdb_data = (struct rspamd_cdb_map_helper *) data->cur_data; + rspamd_map_helper_destroy_cdb(cdb_data); + data->cur_data = NULL; + } } + else { + if (data->cur_data) { + cdb_data = (struct rspamd_cdb_map_helper *) data->cur_data; + msg_info_map ("read cdb of %Hz size", cdb_data->total_size); + data->map->traverse_function = NULL; + data->map->nelts = 0; + data->map->digest = rspamd_cryptobox_fast_hash_final(&cdb_data->hst); + } - if (target) { - *target = data->cur_data; - } + if (target) { + *target = data->cur_data; + } - if (data->prev_data) { - cdb_data = (struct rspamd_cdb_map_helper *)data->prev_data; - rspamd_map_helper_destroy_cdb (cdb_data); + if (data->prev_data) { + cdb_data = (struct rspamd_cdb_map_helper *) data->prev_data; + rspamd_map_helper_destroy_cdb(cdb_data); + } } } void diff --git a/src/lua/lua_map.c b/src/lua/lua_map.c index 72bc00fca..923b9adcc 100644 --- a/src/lua/lua_map.c +++ b/src/lua/lua_map.c @@ -430,60 +430,76 @@ lua_map_fin (struct map_cb_data *data, void **target) map = data->map; - if (data->cur_data) { - cbdata = (struct lua_map_callback_data *)data->cur_data; + if (data->errored) { + if (data->cur_data) { + cbdata = (struct lua_map_callback_data *)data->cur_data; + if (cbdata->ref != -1) { + luaL_unref (cbdata->L, LUA_REGISTRYINDEX, cbdata->ref); + } + + if (cbdata->data) { + rspamd_fstring_free (cbdata->data); + } + + data->cur_data = NULL; + } } else { - msg_err_map ("no data read for map"); - return; - } + if (data->cur_data) { + cbdata = (struct lua_map_callback_data *) data->cur_data; + } + else { + msg_err_map ("no data read for map"); + return; + } - if (cbdata->ref == -1) { - msg_err_map ("map has no callback set"); - } - else if (cbdata->data != NULL && cbdata->data->len != 0) { + if (cbdata->ref == -1) { + msg_err_map ("map has no callback set"); + } + else if (cbdata->data != NULL && cbdata->data->len != 0) { - lua_pushcfunction (cbdata->L, &rspamd_lua_traceback); - int err_idx = lua_gettop (cbdata->L); + lua_pushcfunction (cbdata->L, &rspamd_lua_traceback); + int err_idx = lua_gettop(cbdata->L); - lua_rawgeti (cbdata->L, LUA_REGISTRYINDEX, cbdata->ref); + lua_rawgeti(cbdata->L, LUA_REGISTRYINDEX, cbdata->ref); - if (!cbdata->opaque) { - lua_pushlstring (cbdata->L, cbdata->data->str, cbdata->data->len); - } - else { - struct rspamd_lua_text *t; + if (!cbdata->opaque) { + lua_pushlstring(cbdata->L, cbdata->data->str, cbdata->data->len); + } + else { + struct rspamd_lua_text *t; - t = lua_newuserdata (cbdata->L, sizeof (*t)); - rspamd_lua_setclass (cbdata->L, "rspamd{text}", -1); - t->flags = 0; - t->len = cbdata->data->len; - t->start = cbdata->data->str; - } + t = lua_newuserdata(cbdata->L, sizeof(*t)); + rspamd_lua_setclass(cbdata->L, "rspamd{text}", -1); + t->flags = 0; + t->len = cbdata->data->len; + t->start = cbdata->data->str; + } - pmap = lua_newuserdata (cbdata->L, sizeof (void *)); - *pmap = cbdata->lua_map; - rspamd_lua_setclass (cbdata->L, "rspamd{map}", -1); + pmap = lua_newuserdata(cbdata->L, sizeof(void *)); + *pmap = cbdata->lua_map; + rspamd_lua_setclass(cbdata->L, "rspamd{map}", -1); - gint ret = lua_pcall (cbdata->L, 2, 0, err_idx); + gint ret = lua_pcall(cbdata->L, 2, 0, err_idx); - if (ret != 0) { - msg_info_map ("call to %s failed (%d): %s", "map fin function", - ret, - lua_tostring (cbdata->L, -1)); - } + if (ret != 0) { + msg_info_map ("call to %s failed (%d): %s", "map fin function", + ret, + lua_tostring(cbdata->L, -1)); + } - lua_settop (cbdata->L, err_idx - 1); - } + lua_settop(cbdata->L, err_idx - 1); + } - cbdata->data = rspamd_fstring_assign (cbdata->data, "", 0); + cbdata->data = rspamd_fstring_assign(cbdata->data, "", 0); - if (target) { - *target = data->cur_data; - } + if (target) { + *target = data->cur_data; + } - if (data->prev_data) { - data->prev_data = NULL; + if (data->prev_data) { + data->prev_data = NULL; + } } } From kupferschmid at puzzle.ch Fri Dec 24 10:42:04 2021 From: kupferschmid at puzzle.ch (Reto Kupferschmid) Date: Fri, 24 Dec 2021 10:42:04 +0000 (UTC) Subject: commit 4129ef5: [Fix] Avoid overwriting whitelisted_signers_map Message-ID: <20211224104204.55B7C382AE@lists.rspamd.com> Author: Reto Kupferschmid Date: 2021-12-23 22:53:50 +0100 URL: https://github.com/rspamd/rspamd/commit/4129ef5132a00d6a0441a3b9fb387120a195fe0c (refs/pull/4019/head) [Fix] Avoid overwriting whitelisted_signers_map --- lualib/lua_dkim_tools.lua | 2 ++ src/plugins/lua/arc.lua | 30 ++++++++++++------------------ 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/lualib/lua_dkim_tools.lua b/lualib/lua_dkim_tools.lua index 53a8a8f0f..06ab819f8 100644 --- a/lualib/lua_dkim_tools.lua +++ b/lualib/lua_dkim_tools.lua @@ -705,6 +705,8 @@ exports.process_signing_settings = function(N, settings, opts) else logger.errx(rspamd_config, 'cannot load sign condition %s: %s', v, f) end + elseif k == 'whitelisted_signers_map' then + settings[k] = lua_maps.map_add(N, k, 'set', 'ARC trusted signers domains') else settings[k] = v end diff --git a/src/plugins/lua/arc.lua b/src/plugins/lua/arc.lua index eeff889f4..b73dc89e4 100644 --- a/src/plugins/lua/arc.lua +++ b/src/plugins/lua/arc.lua @@ -413,24 +413,6 @@ rspamd_config:register_symbol({ groups = {'arc'}, }) -if settings.whitelisted_signers_map then - local lua_maps = require "lua_maps" - settings.whitelisted_signers_map = lua_maps.map_add_from_ucl(settings.whitelisted_signers_map, - 'set', - 'ARC trusted signers domains') - if settings.whitelisted_signers_map then - arc_symbols.trusted_allow = arc_symbols.trusted_allow or 'ARC_ALLOW_TRUSTED' - rspamd_config:register_symbol({ - name = arc_symbols.trusted_allow, - parent = id, - type = 'virtual', - score = -2.0, - group = 'policies', - groups = {'arc'}, - }) - end -end - rspamd_config:register_dependency('ARC_CALLBACK', 'SPF_CHECK') rspamd_config:register_dependency('ARC_CALLBACK', 'DKIM_CHECK') @@ -741,6 +723,18 @@ if type(settings.forbidden_ids) == 'table' then sym_reg_tbl.forbidden_ids = settings.forbidden_ids end +if settings.whitelisted_signers_map then + arc_symbols.trusted_allow = arc_symbols.trusted_allow or 'ARC_ALLOW_TRUSTED' + rspamd_config:register_symbol({ + name = arc_symbols.trusted_allow, + parent = id, + type = 'virtual', + score = -2.0, + group = 'policies', + groups = {'arc'}, + }) +end + rspamd_config:register_symbol(sym_reg_tbl) -- Do not sign unless checked From noreply at github.com Fri Dec 24 10:42:05 2021 From: noreply at github.com (GitHub) Date: Fri, 24 Dec 2021 10:42:05 +0000 (UTC) Subject: commit 3f578a1: Merge pull request #4019 from rekup/fix/arc_whitelisted_signers_map Message-ID: <20211224104205.65D4D382B0@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-24 10:35:21 +0000 URL: https://github.com/rspamd/rspamd/commit/3f578a1e9c7c087f6dbdefb42d6ca80f76fb0123 (HEAD -> master) Merge pull request #4019 from rekup/fix/arc_whitelisted_signers_map [Fix] Avoid overwriting whitelisted_signers_map lualib/lua_dkim_tools.lua | 2 ++ src/plugins/lua/arc.lua | 30 ++++++++++++------------------ 2 files changed, 14 insertions(+), 18 deletions(-) From vsevolod at highsecure.ru Sat Dec 25 20:35:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sat, 25 Dec 2021 20:35:04 +0000 (UTC) Subject: commit ae24d39: [Minor] Fix division by zero Message-ID: <20211225203504.432ED383BE@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-25 20:02:54 +0000 URL: https://github.com/rspamd/rspamd/commit/ae24d391fb899234fefa49fa6a557e7c77a96522 [Minor] Fix division by zero --- src/plugins/chartable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/chartable.c b/src/plugins/chartable.c index c997862a1..56fdac3a6 100644 --- a/src/plugins/chartable.c +++ b/src/plugins/chartable.c @@ -596,7 +596,7 @@ rspamd_chartable_process_part (struct rspamd_task *task, gdouble cur_score = 0.0; if (part == NULL || part->utf_words == NULL || - part->utf_words->len == 0) { + part->utf_words->len == 0 || part->nwords == 0) { return FALSE; } From vsevolod at highsecure.ru Sat Dec 25 20:35:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sat, 25 Dec 2021 20:35:05 +0000 (UTC) Subject: commit 2d8c7bf: [Minor] More divisions by zero Message-ID: <20211225203505.5CFEF383C0@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-25 20:17:08 +0000 URL: https://github.com/rspamd/rspamd/commit/2d8c7bfb4e011bdcd4a632061e0f759abccff40a [Minor] More divisions by zero --- src/libmime/lang_detection.c | 4 ++++ src/plugins/chartable.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c index ba638982d..d0139919f 100644 --- a/src/libmime/lang_detection.c +++ b/src/libmime/lang_detection.c @@ -1308,6 +1308,10 @@ rspamd_language_detector_cmp_heuristic (gconstpointer a, gconstpointer b, gdouble adj; gdouble proba_adjusted, probb_adjusted, freqa, freqb; + if (cbd->d->total_occurencies == 0) { + return 0; + } + freqa = ((gdouble)canda->elt->occurencies) / (gdouble)cbd->d->total_occurencies; freqb = ((gdouble)candb->elt->occurencies) / diff --git a/src/plugins/chartable.c b/src/plugins/chartable.c index 56fdac3a6..745f51900 100644 --- a/src/plugins/chartable.c +++ b/src/plugins/chartable.c @@ -674,7 +674,7 @@ chartable_symbol_callback (struct rspamd_task *task, ignore_diacritics = TRUE; } - if (task->meta_words != NULL) { + if (task->meta_words != NULL && task->meta_words->len > 0) { rspamd_stat_token_t *w; gdouble cur_score = 0; gsize arlen = task->meta_words->len; From vsevolod at highsecure.ru Sat Dec 25 20:35:06 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sat, 25 Dec 2021 20:35:06 +0000 (UTC) Subject: commit 6bf8a3e: [Minor] Check for NULL pointer Message-ID: <20211225203506.7A00C383C2@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-25 20:22:18 +0000 URL: https://github.com/rspamd/rspamd/commit/6bf8a3e898ea5a307b9492f5298bf17f3520d9ed (HEAD -> master) [Minor] Check for NULL pointer --- src/libmime/mime_headers.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libmime/mime_headers.c b/src/libmime/mime_headers.c index 76ae1770a..b819f0ec5 100644 --- a/src/libmime/mime_headers.c +++ b/src/libmime/mime_headers.c @@ -907,6 +907,10 @@ rspamd_message_get_header_from_hash (struct rspamd_mime_headers_table *hdrs, const gchar *field, gboolean need_modified) { + if (hdrs == NULL) { + return NULL; + } + khiter_t k; khash_t(rspamd_mime_headers_htb) *htb = &hdrs->htb; struct rspamd_mime_header *hdr; From vsevolod at highsecure.ru Sun Dec 26 20:00:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sun, 26 Dec 2021 20:00:04 +0000 (UTC) Subject: commit f91baff: [Minor] Fix another overflow Message-ID: <20211226200004.6348B3847F@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-26 19:56:58 +0000 URL: https://github.com/rspamd/rspamd/commit/f91baff1833441f3d86b63dd282364857eceeda8 [Minor] Fix another overflow --- src/libserver/css/css_tokeniser.cxx | 66 +++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/src/libserver/css/css_tokeniser.cxx b/src/libserver/css/css_tokeniser.cxx index dc570d64b..e44c14011 100644 --- a/src/libserver/css/css_tokeniser.cxx +++ b/src/libserver/css/css_tokeniser.cxx @@ -20,6 +20,7 @@ #include "frozen/unordered_map.h" #include "frozen/string.h" #include +#include namespace rspamd::css { @@ -29,8 +30,8 @@ namespace rspamd::css { * This helper is intended to create tokens either with a tag and value * or with just a tag. */ -template -auto make_token(const Args&... args) -> css_parser_token; +template +auto make_token(const Arg &arg) -> css_parser_token; template<> auto make_token(const std::string_view &s) @@ -76,7 +77,7 @@ auto make_token(const char &c) template<> auto make_token(const float &d) --> css_parser_token + -> css_parser_token { return css_parser_token{css_parser_token::token_type::number_token, d}; } @@ -360,51 +361,52 @@ auto css_tokeniser::consume_number() -> struct css_parser_token } if (i > offset) { - float num; - /* I wish it was supported properly */ //auto conv_res = std::from_chars(&input[offset], &input[i], num); char numbuf[128], *endptr = NULL; rspamd_strlcpy(numbuf, &input[offset], MIN(i - offset + 1, sizeof(numbuf))); - num = g_ascii_strtod(numbuf, &endptr); + auto num = g_ascii_strtod(numbuf, &endptr); + offset = i; - if (endptr && *endptr != '\0') { + if ((endptr && *endptr != '\0') || num >= G_MAXFLOAT || num <= G_MINFLOAT || isnan(num)) { msg_debug_css("invalid number: %s", numbuf); + return make_token(input[i - 1]); } - offset = i; + else { - auto ret = make_token(num); + auto ret = make_token(static_cast(num)); - if (i < input.size()) { - if (input[i] == '%') { - ret.flags |= css_parser_token::number_percent; - i ++; + if (i < input.size()) { + if (input[i] == '%') { + ret.flags |= css_parser_token::number_percent; + i++; - offset = i; - } - else if (is_plain_ident_start(input[i])) { - auto dim_token = consume_ident(); - - if (dim_token.type == css_parser_token::token_type::ident_token) { - if (!ret.adjust_dim(dim_token)) { - auto sv = std::get(dim_token.value); - msg_debug_css("cannot apply dimension from the token %*s; number value = %.1f", - (int)sv.size(), sv.begin(), num); - /* Unconsume ident */ - offset = i; + offset = i; + } + else if (is_plain_ident_start(input[i])) { + auto dim_token = consume_ident(); + + if (dim_token.type == css_parser_token::token_type::ident_token) { + if (!ret.adjust_dim(dim_token)) { + auto sv = std::get(dim_token.value); + msg_debug_css("cannot apply dimension from the token %*s; number value = %.1f", + (int) sv.size(), sv.begin(), num); + /* Unconsume ident */ + offset = i; + } + } + else { + /* We have no option but to uncosume ident token in this case */ + msg_debug_css("got invalid ident like token after number, unconsume it"); } } else { - /* We have no option but to uncosume ident token in this case */ - msg_debug_css("got invalid ident like token after number, unconsume it"); + /* Plain number, nothing to do */ } } - else { - /* Plain number, nothing to do */ - } - } - return ret; + return ret; + } } else { msg_err_css("internal error: invalid number, empty token"); From vsevolod at highsecure.ru Sun Dec 26 20:00:05 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sun, 26 Dec 2021 20:00:05 +0000 (UTC) Subject: commit b4cf342: [Minor] Add namespace Message-ID: <20211226200005.749D638481@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-26 19:59:27 +0000 URL: https://github.com/rspamd/rspamd/commit/b4cf34230c3b811da73bdeeaf626b32a3f4919dc (HEAD -> master) [Minor] Add namespace --- src/libserver/css/css_tokeniser.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libserver/css/css_tokeniser.cxx b/src/libserver/css/css_tokeniser.cxx index e44c14011..fe547d82c 100644 --- a/src/libserver/css/css_tokeniser.cxx +++ b/src/libserver/css/css_tokeniser.cxx @@ -368,7 +368,7 @@ auto css_tokeniser::consume_number() -> struct css_parser_token auto num = g_ascii_strtod(numbuf, &endptr); offset = i; - if ((endptr && *endptr != '\0') || num >= G_MAXFLOAT || num <= G_MINFLOAT || isnan(num)) { + if ((endptr && *endptr != '\0') || num >= G_MAXFLOAT || num <= G_MINFLOAT || std::isnan(num)) { msg_debug_css("invalid number: %s", numbuf); return make_token(input[i - 1]); } From vsevolod at highsecure.ru Sun Dec 26 20:14:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Sun, 26 Dec 2021 20:14:04 +0000 (UTC) Subject: commit a941c55: [Minor] Sigh, one more fix to the conditions Message-ID: <20211226201404.3E62938487@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-26 20:09:14 +0000 URL: https://github.com/rspamd/rspamd/commit/a941c5524852826f904f5f53e3867081cc7c36ec (HEAD -> master) [Minor] Sigh, one more fix to the conditions --- src/libserver/css/css_tokeniser.cxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libserver/css/css_tokeniser.cxx b/src/libserver/css/css_tokeniser.cxx index fe547d82c..737b3460d 100644 --- a/src/libserver/css/css_tokeniser.cxx +++ b/src/libserver/css/css_tokeniser.cxx @@ -363,12 +363,12 @@ auto css_tokeniser::consume_number() -> struct css_parser_token if (i > offset) { /* I wish it was supported properly */ //auto conv_res = std::from_chars(&input[offset], &input[i], num); - char numbuf[128], *endptr = NULL; + char numbuf[128], *endptr = nullptr; rspamd_strlcpy(numbuf, &input[offset], MIN(i - offset + 1, sizeof(numbuf))); auto num = g_ascii_strtod(numbuf, &endptr); offset = i; - if ((endptr && *endptr != '\0') || num >= G_MAXFLOAT || num <= G_MINFLOAT || std::isnan(num)) { + if (fabs (num) >= G_MAXFLOAT || std::isnan(num)) { msg_debug_css("invalid number: %s", numbuf); return make_token(input[i - 1]); } From vsevolod at highsecure.ru Mon Dec 27 18:49:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Mon, 27 Dec 2021 18:49:04 +0000 (UTC) Subject: commit d6b21b8: [Minor] Log events on forced timeout Message-ID: <20211227184904.2E2993853F@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-27 18:45:43 +0000 URL: https://github.com/rspamd/rspamd/commit/d6b21b81428e45f009b1f42714342d68270cb8f9 (HEAD -> master) [Minor] Log events on forced timeout --- src/libserver/async_session.c | 41 ++++++++++++++++++++++++++++++++--------- src/libserver/async_session.h | 2 +- src/libserver/task.c | 4 ++-- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/libserver/async_session.c b/src/libserver/async_session.c index 057604447..f84b8aa5e 100644 --- a/src/libserver/async_session.c +++ b/src/libserver/async_session.c @@ -257,7 +257,7 @@ rspamd_session_destroy (struct rspamd_async_session *session) if (!rspamd_session_blocked (session)) { session->flags |= RSPAMD_SESSION_FLAG_DESTROYING; - rspamd_session_cleanup (session); + rspamd_session_cleanup (session, false); if (session->cleanup != NULL) { session->cleanup (session->user_data); @@ -268,7 +268,7 @@ rspamd_session_destroy (struct rspamd_async_session *session) } void -rspamd_session_cleanup (struct rspamd_async_session *session) +rspamd_session_cleanup (struct rspamd_async_session *session, bool forced_cleanup) { struct rspamd_async_event *ev; @@ -285,15 +285,32 @@ rspamd_session_cleanup (struct rspamd_async_session *session) int ret; if (ev->fin != NULL) { - msg_debug_session ("removed event on destroy: %p, subsystem: %s", - ev->user_data, - ev->subsystem); + if (forced_cleanup) { + msg_info_session ("forced removed event on destroy: %p, subsystem: %s, scheduled from: %s", + ev->user_data, + ev->subsystem, + ev->loc); + } + else { + msg_debug_session("removed event on destroy: %p, subsystem: %s", + ev->user_data, + ev->subsystem); + } ev->fin (ev->user_data); } else { - msg_debug_session ("NOT removed event on destroy - uncancellable: %p, subsystem: %s", - ev->user_data, - ev->subsystem); + if (forced_cleanup) { + msg_info_session ("NOT forced removed event on destroy - uncancellable: " + "%p, subsystem: %s, scheduled from: %s", + ev->user_data, + ev->subsystem, + ev->loc); + } + else { + msg_debug_session("NOT removed event on destroy - uncancellable: %p, subsystem: %s", + ev->user_data, + ev->subsystem); + } /* Assume an event is uncancellable, move it to a new hash table */ kh_put (rspamd_events_hash, uncancellable_events, ev, &ret); } @@ -301,7 +318,13 @@ rspamd_session_cleanup (struct rspamd_async_session *session) kh_destroy (rspamd_events_hash, session->events); session->events = uncancellable_events; - msg_debug_session ("pending %d uncancellable events", kh_size (uncancellable_events)); + if (forced_cleanup) { + msg_info_session ("pending %d uncancellable events", kh_size (uncancellable_events)); + } + else { + msg_debug_session ("pending %d uncancellable events", kh_size (uncancellable_events)); + } + session->flags &= ~RSPAMD_SESSION_FLAG_CLEANUP; } diff --git a/src/libserver/async_session.h b/src/libserver/async_session.h index ad79769e9..5dc4a3b84 100644 --- a/src/libserver/async_session.h +++ b/src/libserver/async_session.h @@ -83,7 +83,7 @@ gboolean rspamd_session_destroy (struct rspamd_async_session *session); /** * Try to remove all events pending */ -void rspamd_session_cleanup (struct rspamd_async_session *session); +void rspamd_session_cleanup (struct rspamd_async_session *session, bool forced_cleanup); /** * Returns mempool associated with async session diff --git a/src/libserver/task.c b/src/libserver/task.c index c9f3fb627..fa97d67a5 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -1911,7 +1911,7 @@ rspamd_task_timeout (EV_P_ ev_timer *w, int revents) ev_timer_again (EV_A_ w); task->processed_stages |= RSPAMD_TASK_STAGE_FILTERS; - rspamd_session_cleanup (task->s); + rspamd_session_cleanup (task->s, true); rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); rspamd_session_pending (task->s); } @@ -1940,7 +1940,7 @@ rspamd_task_timeout (EV_P_ ev_timer *w, int revents) ev_timer_stop (EV_A_ w); task->processed_stages |= RSPAMD_TASK_STAGE_DONE; - rspamd_session_cleanup (task->s); + rspamd_session_cleanup (task->s, true); rspamd_task_process (task, RSPAMD_TASK_PROCESS_ALL); rspamd_session_pending (task->s); } From vsevolod at highsecure.ru Mon Dec 27 23:07:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Mon, 27 Dec 2021 23:07:04 +0000 (UTC) Subject: commit 160184d: [Minor] Refactor argument/member name for better soundness Message-ID: <20211227230704.2CD6B38563@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-27 21:05:35 +0000 URL: https://github.com/rspamd/rspamd/commit/160184d857561076e22fe7de1a685804e44f75a0 (HEAD -> master) [Minor] Refactor argument/member name for better soundness --- src/libserver/async_session.c | 22 +++++++++++----------- src/libserver/async_session.h | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/libserver/async_session.c b/src/libserver/async_session.c index f84b8aa5e..2f084687b 100644 --- a/src/libserver/async_session.c +++ b/src/libserver/async_session.c @@ -50,7 +50,7 @@ static struct rspamd_counter_data events_count; struct rspamd_async_event { const gchar *subsystem; - const gchar *loc; + const gchar *event_source; event_finalizer_t fin; void *user_data; }; @@ -151,7 +151,7 @@ rspamd_session_add_event_full (struct rspamd_async_session *session, event_finalizer_t fin, gpointer user_data, const gchar *subsystem, - const gchar *loc) + const gchar *event_source) { struct rspamd_async_event *new_event; gint ret; @@ -174,14 +174,14 @@ rspamd_session_add_event_full (struct rspamd_async_session *session, new_event->fin = fin; new_event->user_data = user_data; new_event->subsystem = subsystem; - new_event->loc = loc; + new_event->event_source = event_source; msg_debug_session ("added event: %p, pending %d (+1) events, " "subsystem: %s (%s)", user_data, kh_size (session->events), subsystem, - loc); + event_source); kh_put (rspamd_events_hash, session->events, new_event, &ret); g_assert (ret > 0); @@ -193,7 +193,7 @@ void rspamd_session_remove_event_full (struct rspamd_async_session *session, event_finalizer_t fin, void *ud, - const gchar *loc) + const gchar *event_source) { struct rspamd_async_event search_ev, *found_ev; khiter_t k; @@ -215,11 +215,11 @@ rspamd_session_remove_event_full (struct rspamd_async_session *session, if (k == kh_end (session->events)) { gchar t; - msg_err_session ("cannot find event: %p(%p) from %s", fin, ud, loc); + msg_err_session ("cannot find event: %p(%p) from %s", fin, ud, event_source); kh_foreach (session->events, found_ev, t, { msg_err_session ("existing event %s (%s): %p(%p)", found_ev->subsystem, - found_ev->loc, + found_ev->event_source, found_ev->fin, found_ev->user_data); }); @@ -235,8 +235,8 @@ rspamd_session_remove_event_full (struct rspamd_async_session *session, ud, kh_size (session->events), found_ev->subsystem, - loc, - found_ev->loc); + event_source, + found_ev->event_source); kh_del (rspamd_events_hash, session->events, k); /* Remove event */ @@ -289,7 +289,7 @@ rspamd_session_cleanup (struct rspamd_async_session *session, bool forced_cleanu msg_info_session ("forced removed event on destroy: %p, subsystem: %s, scheduled from: %s", ev->user_data, ev->subsystem, - ev->loc); + ev->event_source); } else { msg_debug_session("removed event on destroy: %p, subsystem: %s", @@ -304,7 +304,7 @@ rspamd_session_cleanup (struct rspamd_async_session *session, bool forced_cleanu "%p, subsystem: %s, scheduled from: %s", ev->user_data, ev->subsystem, - ev->loc); + ev->event_source); } else { msg_debug_session("NOT removed event on destroy - uncancellable: %p, subsystem: %s", diff --git a/src/libserver/async_session.h b/src/libserver/async_session.h index 5dc4a3b84..b5323b1df 100644 --- a/src/libserver/async_session.h +++ b/src/libserver/async_session.h @@ -55,7 +55,7 @@ rspamd_session_add_event_full (struct rspamd_async_session *session, event_finalizer_t fin, gpointer user_data, const gchar *subsystem, - const gchar *loc); + const gchar *event_source); #define rspamd_session_add_event(session, fin, user_data, subsystem) \ rspamd_session_add_event_full(session, fin, user_data, subsystem, G_STRLOC) @@ -69,7 +69,7 @@ rspamd_session_add_event_full (struct rspamd_async_session *session, void rspamd_session_remove_event_full (struct rspamd_async_session *session, event_finalizer_t fin, gpointer ud, - const gchar *loc); + const gchar *event_source); #define rspamd_session_remove_event(session, fin, user_data) \ rspamd_session_remove_event_full(session, fin, user_data, G_STRLOC) From vsevolod at highsecure.ru Tue Dec 28 20:21:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Tue, 28 Dec 2021 20:21:04 +0000 (UTC) Subject: commit 3deecad: [Minor] Slightly improve session events logging Message-ID: <20211228202104.288E038615@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-28 20:15:50 +0000 URL: https://github.com/rspamd/rspamd/commit/3deecad150f2ec11679cb85265c55df948e7c2bc (HEAD -> master) [Minor] Slightly improve session events logging --- src/lua/lua_http.c | 14 ++++++++++---- src/lua/lua_tcp.c | 8 +++++++- src/lua/lua_udp.c | 10 ++++++++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/lua/lua_http.c b/src/lua/lua_http.c index e4a3bedb1..ce2a48d63 100644 --- a/src/lua/lua_http.c +++ b/src/lua/lua_http.c @@ -389,7 +389,6 @@ lua_http_make_connection (struct lua_http_cbdata *cbd) { rspamd_inet_address_set_port (cbd->addr, cbd->msg->port); unsigned http_opts = RSPAMD_HTTP_CLIENT_SIMPLE; - struct rspamd_http_message *msg = cbd->msg; if (cbd->msg->flags & RSPAMD_HTTP_FLAG_WANT_SSL) { http_opts |= RSPAMD_HTTP_CLIENT_SSL; @@ -440,9 +439,16 @@ lua_http_make_connection (struct lua_http_cbdata *cbd) } if (cbd->session) { - rspamd_session_add_event (cbd->session, - (event_finalizer_t) lua_http_fin, cbd, - M); + if (cbd->item) { + rspamd_session_add_event_full (cbd->session, + (event_finalizer_t) lua_http_fin, cbd, + M, rspamd_symcache_item_name (cbd->item)); + } + else { + rspamd_session_add_event (cbd->session, + (event_finalizer_t) lua_http_fin, cbd, + M); + } cbd->flags |= RSPAMD_LUA_HTTP_FLAG_RESOLVED; } diff --git a/src/lua/lua_tcp.c b/src/lua/lua_tcp.c index f15e25399..37f535850 100644 --- a/src/lua/lua_tcp.c +++ b/src/lua/lua_tcp.c @@ -1231,7 +1231,13 @@ lua_tcp_register_event (struct lua_tcp_cbdata *cbd) if (cbd->session) { event_finalizer_t fin = IS_SYNC (cbd) ? lua_tcp_void_finalyser : lua_tcp_fin; - cbd->async_ev = rspamd_session_add_event (cbd->session, fin, cbd, M); + if (cbd->item) { + cbd->async_ev = rspamd_session_add_event_full (cbd->session, fin, cbd, M, + rspamd_symcache_item_name (cbd->item)); + } + else { + cbd->async_ev = rspamd_session_add_event (cbd->session, fin, cbd, M); + } if (!cbd->async_ev) { return FALSE; diff --git a/src/lua/lua_udp.c b/src/lua/lua_udp.c index 5d1fd8c62..656d80349 100644 --- a/src/lua/lua_udp.c +++ b/src/lua/lua_udp.c @@ -236,8 +236,14 @@ static gboolean lua_udp_maybe_register_event (struct lua_udp_cbdata *cbd) { if (cbd->s && !cbd->async_ev) { - cbd->async_ev = rspamd_session_add_event (cbd->s, lua_udp_cbd_fin, - cbd, M); + if (cbd->item) { + cbd->async_ev = rspamd_session_add_event_full (cbd->s, lua_udp_cbd_fin, + cbd, M, rspamd_symcache_item_name (cbd->item)); + } + else { + cbd->async_ev = rspamd_session_add_event (cbd->s, lua_udp_cbd_fin, + cbd, M); + } if (!cbd->async_ev) { return FALSE; From vsevolod at highsecure.ru Wed Dec 29 21:35:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Wed, 29 Dec 2021 21:35:04 +0000 (UTC) Subject: commit d2a9a77: [Minor] Set 0600 mode on privkey files by default Message-ID: <20211229213504.2E3FB386EB@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-29 21:32:07 +0000 URL: https://github.com/rspamd/rspamd/commit/d2a9a7765c9d16fff4f567a0c35092da83360365 (HEAD -> master) [Minor] Set 0600 mode on privkey files by default Issue: #4023 --- src/rspamadm/dkim_keygen.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/rspamadm/dkim_keygen.c b/src/rspamadm/dkim_keygen.c index a1e7286ae..318cc924d 100644 --- a/src/rspamadm/dkim_keygen.c +++ b/src/rspamadm/dkim_keygen.c @@ -20,6 +20,8 @@ #include "libcryptobox/cryptobox.h" #include "contrib/libottery/ottery.h" #include "lua/lua_common.h" +#include "unix-std.h" + #include #include #include @@ -108,15 +110,33 @@ rspamd_dkim_generate_rsa_keypair (const gchar *domain, const gchar *selector, g_assert (EVP_PKEY_set1_RSA (pk, r) == 1); if (priv_fname) { - privout = BIO_new_file (priv_fname, "w"); + int fd = open (priv_fname, O_WRONLY | O_CREAT | O_TRUNC, 0600); + + if (fd < 0) { + rspamd_fprintf (stderr, "cannot open output file %s: %s\n", + priv_fname, strerror (errno)); + exit (EXIT_FAILURE); + } + + FILE *fp = fdopen (fd, "w"); + + if (fp == NULL) { + close (fd); + rspamd_fprintf (stderr, "cannot open output file %s: %s\n", + priv_fname, strerror (errno)); + exit (EXIT_FAILURE); + } + + privout = BIO_new_fp (fp, BIO_CLOSE); if (privout == NULL) { + fclose (fp); rspamd_fprintf (stderr, "cannot open output file %s: %s\n", priv_fname, strerror (errno)); exit (EXIT_FAILURE); } } else { - privout = BIO_new_fp (stdout, 0); + privout = BIO_new_fp (stdout, BIO_NOCLOSE); } rc = PEM_write_bio_PrivateKey (privout, pk, NULL, NULL, 0, NULL, NULL); From vsevolod at highsecure.ru Thu Dec 30 21:14:04 2021 From: vsevolod at highsecure.ru (Vsevolod Stakhov) Date: Thu, 30 Dec 2021 21:14:04 +0000 (UTC) Subject: commit 1acb4ba: [Minor] Dkim_keygen: Use 0640 instead of 0600 Message-ID: <20211230211404.2A8A0387C2@lists.rspamd.com> Author: Vsevolod Stakhov Date: 2021-12-30 21:09:15 +0000 URL: https://github.com/rspamd/rspamd/commit/1acb4ba50c7344332a2b35d118c6f34d9f7edaff (HEAD -> master) [Minor] Dkim_keygen: Use 0640 instead of 0600 Suggested by: @moisseev Issue: #4023 --- src/rspamadm/dkim_keygen.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rspamadm/dkim_keygen.c b/src/rspamadm/dkim_keygen.c index 318cc924d..1bbb71775 100644 --- a/src/rspamadm/dkim_keygen.c +++ b/src/rspamadm/dkim_keygen.c @@ -110,7 +110,7 @@ rspamd_dkim_generate_rsa_keypair (const gchar *domain, const gchar *selector, g_assert (EVP_PKEY_set1_RSA (pk, r) == 1); if (priv_fname) { - int fd = open (priv_fname, O_WRONLY | O_CREAT | O_TRUNC, 0600); + int fd = open (priv_fname, O_WRONLY | O_CREAT | O_TRUNC, 0640); if (fd < 0) { rspamd_fprintf (stderr, "cannot open output file %s: %s\n",