commit 72d0de2: [Project] Implement logic to deal with settings id in the cache
Vsevolod Stakhov
vsevolod at highsecure.ru
Fri Jun 14 19:00:05 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-06-14 14:15:39 +0100
URL: https://github.com/rspamd/rspamd/commit/72d0de26db18763be29faca6c37212887fbf2058
[Project] Implement logic to deal with settings id in the cache
---
src/libserver/cfg_utils.c | 4 +-
src/libserver/protocol.c | 2 +-
src/libserver/rspamd_symcache.c | 177 ++++++++++++++++++++++++++++++++++++----
src/libserver/rspamd_symcache.h | 12 +++
4 files changed, 175 insertions(+), 20 deletions(-)
diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c
index e32ca7020..0c47ec22e 100644
--- a/src/libserver/cfg_utils.c
+++ b/src/libserver/cfg_utils.c
@@ -2416,7 +2416,8 @@ rspamd_config_register_settings_id (struct rspamd_config *cfg,
}
REF_INIT_RETAIN (nelt, rspamd_config_settings_elt_dtor);
- msg_info_config ("replace settings id %d (%s)", id, name);
+ msg_warn_config ("replace settings id %d (%s)", id, name);
+ rspamd_symcache_process_settings_elt (cfg->cache, elt);
DL_APPEND (cfg->setting_ids, nelt);
/*
@@ -2443,6 +2444,7 @@ rspamd_config_register_settings_id (struct rspamd_config *cfg,
msg_info_config ("register new settings id %d (%s)", id, name);
REF_INIT_RETAIN (elt, rspamd_config_settings_elt_dtor);
+ rspamd_symcache_process_settings_elt (cfg->cache, elt);
DL_APPEND (cfg->setting_ids, elt);
}
}
diff --git a/src/libserver/protocol.c b/src/libserver/protocol.c
index 5cd0568da..1ee5a2992 100644
--- a/src/libserver/protocol.c
+++ b/src/libserver/protocol.c
@@ -1554,7 +1554,7 @@ rspamd_protocol_write_log_pipe (struct rspamd_task *task)
struct rspamd_metric_result *mres;
struct rspamd_symbol_result *sym;
gint id, i;
- guint32 sid, n = 0, nextra = 0;
+ guint32 n = 0, nextra = 0;
gsize sz;
GArray *extra;
struct rspamd_protocol_log_symbol_result er;
diff --git a/src/libserver/rspamd_symcache.c b/src/libserver/rspamd_symcache.c
index 22f4c3229..767ecc2c9 100644
--- a/src/libserver/rspamd_symcache.c
+++ b/src/libserver/rspamd_symcache.c
@@ -90,7 +90,8 @@ struct rspamd_symcache_id_list {
guint32 st[4];
struct {
guint32 e; /* First element */
- guint32 dynlen;
+ guint16 len;
+ guint16 allocated;
guint *n;
} dyn;
};
@@ -1400,7 +1401,7 @@ rspamd_symcache_check_id_list (const struct rspamd_symcache_id_list *ls, guint32
guint i;
if (ls->dyn.e == -1) {
- guint *res = bsearch (&id, ls->dyn.n, ls->dyn.dynlen, sizeof (guint32),
+ guint *res = bsearch (&id, ls->dyn.n, ls->dyn.len, sizeof (guint32),
rspamd_id_cmp);
if (res) {
@@ -1438,14 +1439,22 @@ rspamd_symcache_is_item_allowed (struct rspamd_task *task,
return FALSE;
}
- if (item->allowed_ids.st[0] != 0 &&
+ if (!(item->type & SYMBOL_TYPE_EXPLICIT_DISABLE)) {
+ if (item->allowed_ids.st[0] != 0 &&
!rspamd_symcache_check_id_list (&item->allowed_ids,
id)) {
- msg_debug_cache_task ("deny execution of %s as it is not listed as allowed for "
- "settings id %d",
- item->symbol,
- id);
- return FALSE;
+ msg_debug_cache_task ("deny execution of %s as it is not listed "
+ "as allowed for settings id %d",
+ item->symbol,
+ id);
+ return FALSE;
+ }
+ }
+ else {
+ msg_debug_cache_task ("allow execution of %s for "
+ "settings id %d as it can be only disabled explicitly",
+ item->symbol,
+ id);
}
}
@@ -2985,7 +2994,8 @@ rspamd_symcache_set_allowed_settings_ids (struct rspamd_symcache *cache,
item->allowed_ids.dyn.e = -1; /* Flag */
item->allowed_ids.dyn.n = rspamd_mempool_alloc (cache->static_pool,
sizeof (guint32) * nids);
- item->allowed_ids.dyn.dynlen = nids;
+ item->allowed_ids.dyn.len = nids;
+ item->allowed_ids.dyn.allocated = nids;
for (guint i = 0; i < nids; i++) {
item->allowed_ids.dyn.n[i] = ids[i];
@@ -3012,6 +3022,8 @@ rspamd_symcache_set_forbidden_settings_ids (struct rspamd_symcache *cache,
return false;
}
+ g_assert (nids < G_MAXUINT16);
+
if (nids <= G_N_ELEMENTS (item->forbidden_ids.st)) {
/* Use static version */
memset (&item->forbidden_ids, 0, sizeof (item->forbidden_ids));
@@ -3024,7 +3036,8 @@ rspamd_symcache_set_forbidden_settings_ids (struct rspamd_symcache *cache,
item->forbidden_ids.dyn.e = -1; /* Flag */
item->forbidden_ids.dyn.n = rspamd_mempool_alloc (cache->static_pool,
sizeof (guint32) * nids);
- item->forbidden_ids.dyn.dynlen = nids;
+ item->forbidden_ids.dyn.len = nids;
+ item->forbidden_ids.dyn.allocated = nids;
for (guint i = 0; i < nids; i++) {
item->forbidden_ids.dyn.n[i] = ids[i];
@@ -3043,6 +3056,7 @@ rspamd_symcache_get_allowed_settings_ids (struct rspamd_symcache *cache,
guint *nids)
{
struct rspamd_symcache_item *item;
+ guint cnt = 0;
item = rspamd_symcache_find_filter (cache, symbol, true);
@@ -3052,13 +3066,11 @@ rspamd_symcache_get_allowed_settings_ids (struct rspamd_symcache *cache,
if (item->allowed_ids.dyn.e == -1) {
/* Dynamic list */
- *nids = item->allowed_ids.dyn.dynlen;
+ *nids = item->allowed_ids.dyn.len;
return item->allowed_ids.dyn.n;
}
else {
- guint cnt = 0;
-
while (item->allowed_ids.st[cnt] != 0) {
cnt ++;
@@ -3078,6 +3090,7 @@ rspamd_symcache_get_forbidden_settings_ids (struct rspamd_symcache *cache,
guint *nids)
{
struct rspamd_symcache_item *item;
+ guint cnt = 0;
item = rspamd_symcache_find_filter (cache, symbol, true);
@@ -3087,22 +3100,150 @@ rspamd_symcache_get_forbidden_settings_ids (struct rspamd_symcache *cache,
if (item->forbidden_ids.dyn.e == -1) {
/* Dynamic list */
- *nids = item->forbidden_ids.dyn.dynlen;
+ *nids = item->allowed_ids.dyn.len;
- return item->forbidden_ids.dyn.n;
+ return item->allowed_ids.dyn.n;
}
else {
- guint cnt = 0;
-
while (item->forbidden_ids.st[cnt] != 0) {
cnt ++;
g_assert (cnt < G_N_ELEMENTS (item->allowed_ids.st));
}
-
*nids = cnt;
return item->forbidden_ids.st;
}
+}
+
+/* Usable for near-sorted ids list */
+static inline void
+rspamd_ids_insertion_sort (guint *a, guint n)
+{
+ for (guint i = 1; i < n; i++) {
+ guint32 tmp = a[i];
+ guint j = i;
+
+ while (j > 0 && tmp < a[j - 1]) {
+ a[j] = a[j - 1];
+ j --;
+ }
+
+ a[j] = tmp;
+ }
+}
+
+static inline void
+rspamd_symcache_add_id_to_list (rspamd_mempool_t *pool,
+ struct rspamd_symcache_id_list *ls,
+ guint32 id)
+{
+ guint cnt = 0;
+ guint *new_array;
+
+ if (ls->st[0] == -1) {
+ /* Dynamic array */
+ if (ls->dyn.len < ls->dyn.allocated) {
+ /* Trivial, append + qsort */
+ ls->dyn.n[ls->dyn.len++] = id;
+ }
+ else {
+ /* Reallocate */
+ g_assert (ls->dyn.allocated <= G_MAXINT16);
+ ls->dyn.allocated *= 2;
+
+ new_array = rspamd_mempool_alloc (pool, ls->dyn.allocated);
+ memcpy (new_array, ls->dyn.n, ls->dyn.len * sizeof (guint32));
+ ls->dyn.n = new_array;
+ ls->dyn.n[ls->dyn.len++] = id;
+ }
+
+ rspamd_ids_insertion_sort (ls->dyn.n, ls->dyn.len);
+ }
+ else {
+ /* Static part */
+ while (ls->st[cnt] != 0) {
+ cnt ++;
+
+ g_assert (cnt < G_N_ELEMENTS (ls->st));
+ }
+
+
+ if (cnt < G_N_ELEMENTS (ls->st)) {
+ ls->st[cnt] = id;
+ }
+ else {
+ /* Switch to dynamic */
+ new_array = rspamd_mempool_alloc (pool, G_N_ELEMENTS (ls->st) * 2);
+ memcpy (new_array, ls->st, G_N_ELEMENTS (ls->st) * sizeof (guint32));
+ ls->dyn.n = new_array;
+ ls->dyn.e = -1;
+ ls->dyn.allocated = G_N_ELEMENTS (ls->st) * 2;
+ ls->dyn.len = G_N_ELEMENTS (ls->st);
+
+ /* Recursively jump to dynamic branch that will handle insertion + sorting */
+ rspamd_symcache_add_id_to_list (pool, ls, id);
+ }
+ }
+}
+
+void
+rspamd_symcache_process_settings_elt (struct rspamd_symcache *cache,
+ struct rspamd_config_settings_elt *elt)
+{
+ guint32 id = elt->id;
+ ucl_object_iter_t iter;
+ struct rspamd_symcache_item *item;
+ const ucl_object_t *cur;
+
+
+ if (elt->symbols_disabled) {
+ /* Process denied symbols */
+ iter = NULL;
+
+ while ((cur = ucl_object_iterate (elt->symbols_disabled, &iter, true)) != NULL) {
+ item = rspamd_symcache_find_filter (cache,
+ ucl_object_tostring (cur), false);
+
+ if (item->is_virtual) {
+ /*
+ * Virtual symbols are special:
+ * we ignore them in symcache but prevent them from being
+ * inserted.
+ */
+ msg_debug_cache ("skip virtual symbol %s for settings id %d (%s)",
+ ucl_object_tostring (cur), id, elt->name);
+ }
+ else {
+ /* Normal symbol, disable it */
+ rspamd_symcache_add_id_to_list (cache->static_pool,
+ &item->forbidden_ids, id);
+ msg_debug_cache ("deny symbol %s for settings %d (%s)",
+ ucl_object_tostring (cur), id, elt->name);
+ }
+ }
+ }
+ if (elt->symbols_enabled) {
+ iter = NULL;
+
+ while ((cur = ucl_object_iterate (elt->symbols_enabled, &iter, true)) != NULL) {
+ /* Here, we resolve parent and explicitly allow it */
+ item = rspamd_symcache_find_filter (cache,
+ ucl_object_tostring (cur), true);
+
+ if (elt->symbols_disabled &&
+ ucl_object_lookup (elt->symbols_disabled, item->symbol)) {
+ msg_err_cache ("conflict in %s: cannot enable disabled symbol %s, "
+ "wanted to enable symbol %s",
+ elt->name, item->symbol, ucl_object_tostring (cur));
+ }
+ else {
+ rspamd_symcache_add_id_to_list (cache->static_pool,
+ &item->allowed_ids, id);
+ msg_debug_cache ("allow execution of symbol %s for settings %d (%s)",
+ ucl_object_tostring (cur), id, elt->name);
+ }
+ }
+ }
}
\ No newline at end of file
diff --git a/src/libserver/rspamd_symcache.h b/src/libserver/rspamd_symcache.h
index 3be2ab2ec..a440a542b 100644
--- a/src/libserver/rspamd_symcache.h
+++ b/src/libserver/rspamd_symcache.h
@@ -18,6 +18,7 @@
#include "config.h"
#include "ucl.h"
+#include "cfg_file.h"
#include <lua.h>
#include <event.h>
@@ -26,6 +27,7 @@ struct rspamd_config;
struct rspamd_symcache;
struct rspamd_worker;
struct rspamd_symcache_item;
+struct rspamd_config_settings_elt;
typedef void (*symbol_func_t)(struct rspamd_task *task,
struct rspamd_symcache_item *item,
@@ -468,4 +470,14 @@ const guint32* rspamd_symcache_get_forbidden_settings_ids (struct rspamd_symcach
const gchar *symbol,
guint *nids);
+
+/**
+ * Processes settings_elt in cache and converts it to a set of
+ * adjustments for forbidden/allowed settings_ids for each symbol
+ * @param cache
+ * @param elt
+ */
+void rspamd_symcache_process_settings_elt (struct rspamd_symcache *cache,
+ struct rspamd_config_settings_elt *elt);
+
#endif
More information about the Commits
mailing list