commit cd50acb: Merge branch 'master' into lua_string_utils
Miecio Za
miecio at miecio.net
Tue Mar 19 10:42:07 UTC 2019
Author: Miecio Za
Date: 2019-03-18 20:53:02 +0100
URL: https://github.com/rspamd/rspamd/commit/cd50acbfc30ddba123ee85b7d9c2ee7ca9d0c20b (refs/pull/2788/head)
Merge branch 'master' into lua_string_utils
CMakeLists.txt | 14 +-
ChangeLog | 139 ++++++++++
PVS-Studio.cmake | 547 ++++++++++++++++++++++++++++++++++++++++
conf/scores.d/rbl_group.conf | 37 +++
conf/scores.d/surbl_group.conf | 35 +++
src/CMakeLists.txt | 2 +-
src/client/rspamc.c | 35 ++-
src/client/rspamdclient.c | 5 +-
src/client/rspamdclient.h | 4 +-
src/controller.c | 3 +-
src/fuzzy_storage.c | 14 +-
src/libmime/archives.c | 5 +-
src/libserver/dkim.c | 2 +
src/libserver/milter.c | 6 +-
src/libserver/milter.h | 4 +-
src/libserver/rspamd_control.c | 5 +-
src/libserver/rspamd_symcache.c | 1 -
src/libserver/task.c | 2 +-
src/libutil/http_connection.c | 248 +++++++++++-------
src/libutil/http_connection.h | 62 ++++-
src/libutil/http_context.c | 86 ++++++-
src/libutil/http_context.h | 7 +-
src/libutil/http_message.c | 4 +-
src/libutil/http_private.h | 5 +-
src/libutil/http_router.c | 5 +-
src/libutil/logger.c | 2 +-
src/libutil/map.c | 170 ++++++-------
src/libutil/map_private.h | 1 -
src/libutil/mem_pool.c | 18 +-
src/libutil/multipattern.c | 1 -
src/libutil/rrd.c | 13 +-
src/libutil/shingles.c | 4 +
src/libutil/upstream.c | 52 ++--
src/libutil/upstream.h | 4 +
src/lua/lua_common.c | 4 +-
src/lua/lua_config.c | 6 +
src/lua/lua_html.c | 3 -
src/lua/lua_http.c | 27 +-
src/lua/lua_task.c | 49 ++++
src/lua/lua_thread_pool.c | 2 +-
src/lua/lua_udp.c | 2 +-
src/lua/lua_util.c | 4 +-
src/plugins/fuzzy_check.c | 7 +-
src/plugins/lua/greylist.lua | 23 +-
src/plugins/lua/neural.lua | 2 +
src/plugins/lua/replies.lua | 69 +++--
src/plugins/surbl.c | 38 +--
src/rspamadm/control.c | 14 +-
src/rspamadm/rspamadm.c | 3 +-
src/rspamd.c | 2 +-
src/rspamd_proxy.c | 18 +-
src/worker.c | 8 +-
52 files changed, 1438 insertions(+), 385 deletions(-)
diff --combined src/lua/lua_util.c
index 0a1fc2010,dad8c336b..1a37eaef6
--- a/src/lua/lua_util.c
+++ b/src/lua/lua_util.c
@@@ -22,7 -22,6 +22,7 @@@
#include "libmime/email_addr.h"
#include "libmime/content_type.h"
#include "libmime/mime_headers.h"
+#include "libutil/hash.h"
#include "linenoise.h"
#include <math.h>
#include <glob.h>
@@@ -393,23 -392,6 +393,23 @@@ LUA_FUNCTION_DEF (util, normalize_prob)
*/
LUA_FUNCTION_DEF (util, is_utf_spoofed);
+/**
+* @function util.is_utf_outside_range(str, range_start, range_end)
+* Returns true if a string contains chars outside range
+* @param {string} String to check
+* @param {number} start of character range similar to uset_addRange
+* @param {number} end of character range similar to uset_addRange
+* @return {boolean} true if a string contains chars outside selected utf range
+*/
+LUA_FUNCTION_DEF (util, is_utf_outside_range);
+
+/***
+* @function util.get_string_stats(str)
+* Returns table with number of letters and digits in string
+* @return {table} with string stats keys are "digits" and "letters"
+*/
+LUA_FUNCTION_DEF (util, get_string_stats);
+
/***
* @function util.is_valid_utf8(str)
* Returns true if a string is valid UTF8 string
@@@ -633,8 -615,6 +633,8 @@@ static const struct luaL_reg utillib_f[
LUA_INTERFACE_DEF (util, caseless_hash),
LUA_INTERFACE_DEF (util, caseless_hash_fast),
LUA_INTERFACE_DEF (util, is_utf_spoofed),
+ LUA_INTERFACE_DEF (util, is_utf_outside_range),
+ LUA_INTERFACE_DEF (util, get_string_stats),
LUA_INTERFACE_DEF (util, is_valid_utf8),
LUA_INTERFACE_DEF (util, has_obscured_unicode),
LUA_INTERFACE_DEF (util, readline),
@@@ -1197,7 -1177,7 +1197,7 @@@ lua_util_tokenize_text (lua_State *L
{
LUA_TRACE_POINT;
const gchar *in = NULL;
- gsize len, pos, ex_len, i;
+ gsize len = 0, pos, ex_len, i;
GList *exceptions = NULL, *cur;
struct rspamd_lua_text *t;
struct rspamd_process_exception *ex;
@@@ -2263,6 -2243,8 +2263,8 @@@ lua_util_gzip_decompress (lua_State *L
return luaL_error (L, "invalid arguments");
}
+ sz = t->len;
+
memset (&strm, 0, sizeof (strm));
/* windowBits +16 to decode gzip, zlib 1.2.0.4+ */
rc = inflateInit2 (&strm, MAX_WBITS + 16);
@@@ -2459,12 -2441,6 +2461,12 @@@ lua_util_is_utf_spoofed (lua_State *L
uspoof_setChecks (spc_sgl,
USPOOF_INVISIBLE | USPOOF_MIXED_SCRIPT_CONFUSABLE | USPOOF_ANY_CASE,
&uc_err);
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err ("Cannot set proper checks for uspoof: %s", u_errorName (uc_err));
+ lua_pushboolean (L, false);
+ uspoof_close(spc);
+ return 1;
+ }
}
ret = uspoof_checkUTF8 (spc_sgl, s1, l1, NULL, &uc_err);
@@@ -2496,109 -2472,6 +2498,109 @@@
return nres;
}
+static gint
+lua_util_get_string_stats (lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gsize len_of_string;
+ gint num_of_digits = 0, num_of_letters = 0;
+ const gchar *string_to_check = lua_tolstring (L, 1, &len_of_string);
+
+ if (string_to_check) {
+ while (*string_to_check != '\0') {
+ if (g_ascii_isdigit(*string_to_check)) {
+ num_of_digits++;
+ } else if (g_ascii_isalpha(*string_to_check)) {
+ num_of_letters++;
+ }
+ string_to_check++;
+ }
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ lua_createtable(L, 0, 2);
+ lua_pushstring(L, "digits");
+ lua_pushinteger(L, num_of_digits);
+ lua_settable(L, -3);
+ lua_pushstring(L, "letters");
+ lua_pushinteger(L, num_of_letters);
+ lua_settable(L, -3);
+
+ return 1;
+}
+
+
+static gint
+lua_util_is_utf_outside_range(lua_State *L)
+{
+ LUA_TRACE_POINT;
+ gsize len_of_string;
+ gint ret;
+ const gchar *string_to_check = lua_tolstring (L, 1, &len_of_string);
+ guint32 range_start = lua_tointeger (L, 2);
+ guint32 range_end = lua_tointeger (L, 3);
+
+ static rspamd_lru_hash_t *validators;
+
+ if (validators == NULL) {
+ validators = rspamd_lru_hash_new_full(16, g_free, (GDestroyNotify)uspoof_close, g_int64_hash, g_int64_equal);
+ }
+
+ if (string_to_check) {
+ guint64 hash_key = (guint64)range_end << 32 || range_start;
+
+ USpoofChecker *validator = rspamd_lru_hash_lookup(validators, &hash_key, 0);
+
+ UErrorCode uc_err = U_ZERO_ERROR;
+
+ if (validator == NULL) {
+ USet * allowed_chars;
+ guint64 * creation_hash_key = g_malloc(sizeof(guint64));
+ *creation_hash_key = hash_key;
+
+ validator = uspoof_open (&uc_err);
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err ("cannot init spoof checker: %s", u_errorName (uc_err));
+ lua_pushboolean (L, false);
+ uspoof_close(validator);
+ g_free(creation_hash_key);
+ return 1;
+ }
+
+ allowed_chars = uset_openEmpty();
+ uset_addRange(allowed_chars, range_start, range_end);
+ uspoof_setAllowedChars(validator, allowed_chars, &uc_err);
+
+ uspoof_setChecks (validator,
+ USPOOF_CHAR_LIMIT | USPOOF_ANY_CASE, &uc_err);
+
+ uset_close(allowed_chars);
+
+ if (uc_err != U_ZERO_ERROR) {
+ msg_err ("Cannot configure uspoof: %s", u_errorName (uc_err));
+ lua_pushboolean (L, false);
+ uspoof_close(validator);
+ g_free(creation_hash_key);
+ return 1;
+ }
+
+ rspamd_lru_hash_insert(validators, creation_hash_key, validator, 0, 0);
+ }
+
+ ret = uspoof_checkUTF8 (validator, string_to_check, len_of_string, NULL, &uc_err);
+ }
+ else {
+ return luaL_error (L, "invalid arguments");
+ }
+
+ lua_pushboolean (L, !!(ret != 0));
+
+ return 1;
+}
+
+
static gint
lua_util_get_hostname (lua_State *L)
{
More information about the Commits
mailing list