commit 957e212: [Rework] Rework expression API
Vsevolod Stakhov
vsevolod at highsecure.ru
Fri May 17 14:49:06 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-05-17 15:46:20 +0100
URL: https://github.com/rspamd/rspamd/commit/957e21258d23679a0b4b9f46a720750ff86d8cf8 (HEAD -> master)
[Rework] Rework expression API
---
src/libmime/mime_expressions.c | 6 +-
src/libserver/composites.c | 15 +--
src/libutil/expression.c | 38 ++++++--
src/libutil/expression.h | 28 ++----
src/lua/lua_expression.c | 204 ++++++++++++++++++++++++++++++-----------
src/plugins/regexp.c | 13 +--
6 files changed, 203 insertions(+), 101 deletions(-)
diff --git a/src/libmime/mime_expressions.c b/src/libmime/mime_expressions.c
index d45dafe5c..4a0071f28 100644
--- a/src/libmime/mime_expressions.c
+++ b/src/libmime/mime_expressions.c
@@ -87,7 +87,7 @@ static gboolean rspamd_has_flag_expr (struct rspamd_task *task,
static rspamd_expression_atom_t * rspamd_mime_expr_parse (const gchar *line, gsize len,
rspamd_mempool_t *pool, gpointer ud, GError **err);
-static gdouble rspamd_mime_expr_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom);
+static gdouble rspamd_mime_expr_process (void *ud, rspamd_expression_atom_t *atom);
static gint rspamd_mime_expr_priority (rspamd_expression_atom_t *atom);
static void rspamd_mime_expr_destroy (rspamd_expression_atom_t *atom);
@@ -1070,9 +1070,9 @@ rspamd_mime_expr_process_function (struct rspamd_function_atom * func,
}
static gdouble
-rspamd_mime_expr_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom)
+rspamd_mime_expr_process (void *ud, rspamd_expression_atom_t *atom)
{
- struct rspamd_task *task = process_data->task;
+ struct rspamd_task *task = (struct rspamd_task *)ud;
struct rspamd_mime_atom *mime_atom;
lua_State *L;
gdouble ret = 0;
diff --git a/src/libserver/composites.c b/src/libserver/composites.c
index b1bb2f694..4d8329376 100644
--- a/src/libserver/composites.c
+++ b/src/libserver/composites.c
@@ -68,7 +68,7 @@ struct symbol_remove_data {
static rspamd_expression_atom_t * rspamd_composite_expr_parse (const gchar *line, gsize len,
rspamd_mempool_t *pool, gpointer ud, GError **err);
-static gdouble rspamd_composite_expr_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom);
+static gdouble rspamd_composite_expr_process (void *ud, rspamd_expression_atom_t *atom);
static gint rspamd_composite_expr_priority (rspamd_expression_atom_t *atom);
static void rspamd_composite_expr_destroy (rspamd_expression_atom_t *atom);
static void composites_foreach_callback (gpointer key, gpointer value, void *data);
@@ -253,10 +253,10 @@ rspamd_composite_process_symbol_removal (rspamd_expression_atom_t *atom,
}
static gdouble
-rspamd_composite_expr_process (struct rspamd_expr_process_data *process_data,
+rspamd_composite_expr_process (void *ud,
rspamd_expression_atom_t *atom)
{
- struct composites_data *cd = process_data->cd;
+ struct composites_data *cd = (struct composites_data *)ud;
const gchar *beg = atom->data, *sym = NULL;
struct rspamd_symbol_result *ms = NULL;
@@ -442,13 +442,8 @@ composites_foreach_callback (gpointer key, gpointer value, void *data)
return;
}
- struct rspamd_expr_process_data process_data;
- memset (&process_data, 0, sizeof process_data);
-
- process_data.flags = RSPAMD_EXPRESSION_FLAG_NOOPT;
- process_data.cd = cd;
-
- rc = rspamd_process_expression (comp->expr, &process_data);
+ rc = rspamd_process_expression (comp->expr, RSPAMD_EXPRESSION_FLAG_NOOPT,
+ cd);
/* Checked bit */
setbit (cd->checked, comp->id * 2);
diff --git a/src/libutil/expression.c b/src/libutil/expression.c
index ecdb2e87a..71414dc66 100644
--- a/src/libutil/expression.c
+++ b/src/libutil/expression.c
@@ -57,6 +57,14 @@ struct rspamd_expression {
guint evals;
};
+struct rspamd_expr_process_data {
+ gpointer *ud;
+ gint flags;
+ /* != NULL if trace is collected */
+ GPtrArray *trace;
+ rspamd_expression_process_cb process_closure;
+};
+
static GQuark
rspamd_expr_quark (void)
{
@@ -1021,7 +1029,7 @@ rspamd_ast_process_node (struct rspamd_expression *expr, GNode *node,
t1 = rspamd_get_ticks (TRUE);
}
- elt->value = process_data->process_closure (process_data, elt->p.atom);
+ elt->value = process_data->process_closure (process_data->ud, elt->p.atom);
if (fabs (elt->value) > 1e-9) {
elt->p.atom->hits ++;
@@ -1103,8 +1111,11 @@ rspamd_ast_cleanup_traverse (GNode *n, gpointer d)
gdouble
rspamd_process_expression_closure (struct rspamd_expression *expr,
rspamd_expression_process_cb cb,
- struct rspamd_expr_process_data *process_data)
+ gint flags,
+ gpointer runtime_ud,
+ GPtrArray **track)
{
+ struct rspamd_expr_process_data pd;
gdouble ret = 0;
g_assert (expr != NULL);
@@ -1113,8 +1124,16 @@ rspamd_process_expression_closure (struct rspamd_expression *expr,
expr->evals ++;
- process_data->process_closure = cb;
- ret = rspamd_ast_process_node (expr, expr->ast, process_data);
+ pd.process_closure = cb;
+ pd.flags = flags;
+ pd.ud = runtime_ud;
+
+ if (track) {
+ pd.trace = g_ptr_array_sized_new (32);
+ *track = pd.trace;
+ }
+
+ ret = rspamd_ast_process_node (expr, expr->ast, &pd);
/* Cleanup */
g_node_traverse (expr->ast, G_IN_ORDER, G_TRAVERSE_ALL, -1,
@@ -1138,18 +1157,21 @@ rspamd_process_expression_closure (struct rspamd_expression *expr,
gdouble
rspamd_process_expression_track (struct rspamd_expression *expr,
- struct rspamd_expr_process_data *process_data)
+ gint flags,
+ gpointer runtime_ud,
+ GPtrArray **track)
{
return rspamd_process_expression_closure (expr,
- expr->subr->process, process_data);
+ expr->subr->process, flags, runtime_ud, track);
}
gdouble
rspamd_process_expression (struct rspamd_expression *expr,
- struct rspamd_expr_process_data *process_data)
+ gint flags,
+ gpointer runtime_ud)
{
return rspamd_process_expression_closure (expr,
- expr->subr->process, process_data);
+ expr->subr->process, flags, runtime_ud, NULL);
}
static gboolean
diff --git a/src/libutil/expression.h b/src/libutil/expression.h
index 4f16812e6..bd336b87b 100644
--- a/src/libutil/expression.h
+++ b/src/libutil/expression.h
@@ -56,24 +56,9 @@ typedef struct rspamd_expression_atom_s {
gint priority;
} rspamd_expression_atom_t;
-struct rspamd_expr_process_data;
-
-typedef gdouble (*rspamd_expression_process_cb)(struct rspamd_expr_process_data *process_data,
+typedef gdouble (*rspamd_expression_process_cb)(gpointer runtime_data,
rspamd_expression_atom_t *atom);
-struct rspamd_expr_process_data {
- /* Current Lua state to run atom processing */
- struct lua_State *L;
- /* Parameter of lua-function processing atom*/
- gint stack_item;
- gint flags;
- /* != NULL if trace is collected */
- GPtrArray *trace;
- struct composites_data *cd;
- struct rspamd_task *task;
- rspamd_expression_process_cb process_closure;
-};
-
struct rspamd_atom_subr {
/* Parses atom from string and returns atom structure */
rspamd_expression_atom_t * (*parse)(const gchar *line, gsize len,
@@ -111,7 +96,8 @@ gboolean rspamd_parse_expression (const gchar *line, gsize len,
* @return the value of expression
*/
gdouble rspamd_process_expression (struct rspamd_expression *expr,
- struct rspamd_expr_process_data *process_data);
+ gint flags,
+ gpointer runtime_ud);
/**
* Process the expression and return its value using atom 'process' functions with the specified data pointer.
@@ -122,7 +108,9 @@ gdouble rspamd_process_expression (struct rspamd_expression *expr,
* @return the value of expression
*/
gdouble rspamd_process_expression_track (struct rspamd_expression *expr,
- struct rspamd_expr_process_data *process_data);
+ gint flags,
+ gpointer runtime_ud,
+ GPtrArray **track);
/**
* Process the expression with the custom processor
@@ -133,7 +121,9 @@ gdouble rspamd_process_expression_track (struct rspamd_expression *expr,
*/
gdouble rspamd_process_expression_closure (struct rspamd_expression *expr,
rspamd_expression_process_cb cb,
- struct rspamd_expr_process_data *process_data);
+ gint flags,
+ gpointer runtime_ud,
+ GPtrArray **track);
/**
* Shows string representation of an expression
* @param expr expression to show
diff --git a/src/lua/lua_expression.c b/src/lua/lua_expression.c
index 0d57a3bd8..d65f8c454 100644
--- a/src/lua/lua_expression.c
+++ b/src/lua/lua_expression.c
@@ -23,10 +23,10 @@
*/
/***
- * @function rspamd_expression.create(line, {parse_func, process_func}, pool)
+ * @function rspamd_expression.create(line, {parse_func, [process_func]}, pool)
* Create expression from the line using atom parsing routines and the specified memory pool
* @param {string} line expression line
- * @param {table} atom_functions parse_atom function and process_atom function
+ * @param {table} atom_functions parse_atom function and optional process_atom function
* @param {rspamd_mempool} memory pool to use for this function
* @return {expr, err} expression object and error message of `expr` is nil
* @example
@@ -60,16 +60,18 @@ LUA_FUNCTION_DEF (expr, create);
LUA_FUNCTION_DEF (expr, to_string);
/***
- * @method rspamd_expression:process(input)
+ * @method rspamd_expression:process([callback, ]input[, flags])
* Executes the expression and pass input to process atom callbacks
+ * @param {function} callback if not specified on process, then callback must be here
* @param {any} input input data for processing callbacks
* @return {number} result of the expression evaluation
*/
LUA_FUNCTION_DEF (expr, process);
/***
- * @method rspamd_expression:process_traced(input)
+ * @method rspamd_expression:process_traced([callback, ]input[, flags])
* Executes the expression and pass input to process atom callbacks. This function also saves the full trace
+ * @param {function} callback if not specified on process, then callback must be here
* @param {any} input input data for processing callbacks
* @return {number, table of matched atoms} result of the expression evaluation
*/
@@ -98,7 +100,7 @@ static const struct luaL_reg exprlib_f[] = {
static rspamd_expression_atom_t * lua_atom_parse (const gchar *line, gsize len,
rspamd_mempool_t *pool, gpointer ud, GError **err);
-static gdouble lua_atom_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom);
+static gdouble lua_atom_process (gpointer runtime_ud, rspamd_expression_atom_t *atom);
static const struct rspamd_atom_subr lua_atom_subr = {
.parse = lua_atom_parse,
@@ -129,6 +131,19 @@ rspamd_lua_expression (lua_State * L, gint pos)
return ud ? *((struct lua_expression **)ud) : NULL;
}
+static void
+lua_expr_dtor (gpointer p)
+{
+ struct lua_expression *e = (struct lua_expression *)p;
+
+ if (e->parse_idx != -1) {
+ luaL_unref (e->L, LUA_REGISTRYINDEX, e->parse_idx);
+ }
+
+ if (e->process_idx != -1) {
+ luaL_unref (e->L, LUA_REGISTRYINDEX, e->process_idx);
+ }
+}
static rspamd_expression_atom_t *
lua_atom_parse (const gchar *line, gsize len,
@@ -165,25 +180,52 @@ lua_atom_parse (const gchar *line, gsize len,
return atom;
}
+struct lua_atom_process_data {
+ lua_State *L;
+ struct lua_expression *e;
+ gint process_cb_pos;
+ gint stack_item;
+};
+
static gdouble
-lua_atom_process (struct rspamd_expr_process_data *process_data, rspamd_expression_atom_t *atom)
+lua_atom_process (gpointer runtime_ud, rspamd_expression_atom_t *atom)
{
- struct lua_expression *e = (struct lua_expression *)atom->data;
+ struct lua_atom_process_data *pd = (struct lua_atom_process_data *)runtime_ud;
gdouble ret = 0;
+ guint nargs;
+ gint err_idx;
+
+ if (pd->stack_item != -1) {
+ nargs = 2;
+ }
+ else {
+ nargs = 1;
+ }
- lua_rawgeti (process_data->L, LUA_REGISTRYINDEX, e->process_idx);
- lua_pushlstring (process_data->L, atom->str, atom->len);
- lua_pushvalue (process_data->L, process_data->stack_item);
+ lua_pushcfunction (pd->L, &rspamd_lua_traceback);
+ err_idx = lua_gettop (pd->L);
- if (lua_pcall (process_data->L, 2, 1, 0) != 0) {
- msg_info ("callback call failed: %s", lua_tostring (process_data->L, -1));
- lua_pop (process_data->L, 1);
+ /* Function position */
+ lua_pushvalue (pd->L, pd->process_cb_pos);
+ /* Atom name */
+ lua_pushlstring (pd->L, atom->str, atom->len);
+
+ /* If we have data passed */
+ if (pd->stack_item != -1) {
+ lua_pushvalue (pd->L, pd->stack_item);
+ }
+
+ if (lua_pcall (pd->L, nargs, 1, err_idx) != 0) {
+ GString *tb = lua_touserdata (pd->L, -1);
+ msg_info ("expression process callback failed: %s", tb->str);
+ g_string_free (tb, TRUE);
}
else {
- ret = lua_tonumber (process_data->L, -1);
- lua_pop (process_data->L, 1);
+ ret = lua_tonumber (pd->L, -1);
}
+ lua_settop (pd->L, err_idx - 1);
+
return ret;
}
@@ -192,20 +234,51 @@ lua_expr_process (lua_State *L)
{
LUA_TRACE_POINT;
struct lua_expression *e = rspamd_lua_expression (L, 1);
+ struct lua_atom_process_data pd;
gdouble res;
- gint flags = 0;
+ gint flags = 0, old_top;
+
+ pd.L = L;
+ pd.e = e;
+ old_top = lua_gettop (L);
+
+ if (e->process_idx == -1) {
+ if (!lua_isfunction (L, 2)) {
+ return luaL_error (L, "expression process is called with no callback function");
+ }
- struct rspamd_expr_process_data process_data;
- memset (&process_data, 0, sizeof process_data);
- process_data.L = L;
- process_data.stack_item = 2;
+ pd.process_cb_pos = 2;
- if (lua_gettop (L) >= 3) {
- process_data.flags = flags;
+ if (lua_type (L, 3) != LUA_TNONE && lua_type (L, 3) != LUA_TNIL) {
+ pd.stack_item = 3;
+ }
+ else {
+ pd.stack_item = -1;
+ }
+
+ if (lua_isnumber (L, 4)) {
+ flags = lua_tointeger (L, 4);
+ }
}
+ else {
+ lua_rawgeti (L, LUA_REGISTRYINDEX, e->process_idx);
+ pd.process_cb_pos = lua_gettop (L);
- res = rspamd_process_expression (e->expr, &process_data);
+ if (lua_type (L, 2) != LUA_TNONE && lua_type (L, 2) != LUA_TNIL) {
+ pd.stack_item = 2;
+ }
+ else {
+ pd.stack_item = -1;
+ }
+ if (lua_isnumber (L, 3)) {
+ flags = lua_tointeger (L, 3);
+ }
+ }
+
+ res = rspamd_process_expression (e->expr, flags, &pd);
+
+ lua_settop (L, old_top);
lua_pushnumber (L, res);
return 1;
@@ -216,39 +289,52 @@ lua_expr_process_traced (lua_State *L)
{
LUA_TRACE_POINT;
struct lua_expression *e = rspamd_lua_expression (L, 1);
- rspamd_expression_atom_t *atom;
- gint res;
- guint i;
- struct rspamd_expr_process_data process_data;
- memset (&process_data, 0, sizeof process_data);
-
- process_data.L = L;
- /*
- * stack:1 - self
- * stack:2 - data, see process_traced() definition for details
- */
- process_data.stack_item = 2;
-
- if (lua_gettop (L) >= 3) {
- process_data.flags = lua_tonumber (L, 3);
+ struct lua_atom_process_data pd;
+ gdouble res;
+ gint flags = 0, old_top;
+ GPtrArray *trace;
+
+ pd.L = L;
+ pd.e = e;
+ old_top = lua_gettop (L);
+
+ if (e->process_idx == -1) {
+ if (!lua_isfunction (L, 2)) {
+ return luaL_error (L, "expression process is called with no callback function");
+ }
+
+ pd.process_cb_pos = 2;
+ pd.stack_item = 3;
+
+ if (lua_isnumber (L, 4)) {
+ flags = lua_tointeger (L, 4);
+ }
}
+ else {
+ lua_rawgeti (L, LUA_REGISTRYINDEX, e->process_idx);
+ pd.process_cb_pos = lua_gettop (L);
+ pd.stack_item = 2;
- process_data.trace = g_ptr_array_sized_new (32);
+ if (lua_isnumber (L, 3)) {
+ flags = lua_tointeger (L, 3);
+ }
+ }
- res = rspamd_process_expression_track (e->expr, &process_data);
+ res = rspamd_process_expression_track (e->expr, flags, &pd, &trace);
+ lua_settop (L, old_top);
lua_pushnumber (L, res);
- lua_createtable (L, process_data.trace->len, 0);
+ lua_createtable (L, trace->len, 0);
- for (i = 0; i < process_data.trace->len; i ++) {
- atom = g_ptr_array_index (process_data.trace, i);
+ for (guint i = 0; i < trace->len; i ++) {
+ struct rspamd_expression_atom_s *atom = g_ptr_array_index (trace, i);
lua_pushlstring (L, atom->str, atom->len);
lua_rawseti (L, -2, i + 1);
}
- g_ptr_array_free (process_data.trace, TRUE);
+ g_ptr_array_free (trace, TRUE);
return 2;
}
@@ -260,6 +346,7 @@ lua_expr_create (lua_State *L)
struct lua_expression *e, **pe;
const char *line;
gsize len;
+ gboolean no_process = FALSE;
GError *err = NULL;
rspamd_mempool_t *pool;
@@ -293,11 +380,16 @@ lua_expr_create (lua_State *L)
lua_gettable (L, -2);
if (lua_type (L, -1) != LUA_TFUNCTION) {
- lua_pop (L, 2);
- lua_pushnil (L);
- lua_pushstring (L, "bad process callback");
-
- return 2;
+ if (lua_type (L, -1) != LUA_TNIL && lua_type (L, -1) != LUA_TNONE) {
+ lua_pop (L, 2);
+ lua_pushnil (L);
+ lua_pushstring (L, "bad process callback");
+
+ return 2;
+ }
+ else {
+ no_process = TRUE;
+ }
}
lua_pop (L, 1);
@@ -312,9 +404,15 @@ lua_expr_create (lua_State *L)
lua_gettable (L, -2);
e->parse_idx = luaL_ref (L, LUA_REGISTRYINDEX);
- lua_pushnumber (L, 2);
- lua_gettable (L, -2);
- e->process_idx = luaL_ref (L, LUA_REGISTRYINDEX);
+ if (!no_process) {
+ lua_pushnumber (L, 2);
+ lua_gettable (L, -2);
+ e->process_idx = luaL_ref (L, LUA_REGISTRYINDEX);
+ }
+ else {
+ e->process_idx = -1;
+ }
+
lua_pop (L, 1); /* Table */
if (!rspamd_parse_expression (line, len, &lua_atom_subr, e, pool, &err,
@@ -322,10 +420,12 @@ lua_expr_create (lua_State *L)
lua_pushnil (L);
lua_pushstring (L, err->message);
g_error_free (err);
+ lua_expr_dtor (e);
return 2;
}
+ rspamd_mempool_add_destructor (pool, lua_expr_dtor, e);
pe = lua_newuserdata (L, sizeof (struct lua_expression *));
rspamd_lua_setclass (L, "rspamd{expr}", -1);
*pe = e;
diff --git a/src/plugins/regexp.c b/src/plugins/regexp.c
index 2fea0be97..6841e452a 100644
--- a/src/plugins/regexp.c
+++ b/src/plugins/regexp.c
@@ -442,7 +442,7 @@ regexp_module_reconfig (struct rspamd_config *cfg)
static gboolean
rspamd_lua_call_expression_func (struct ucl_lua_funcdata *lua_data,
struct rspamd_task *task,
- GArray *args, gint *res,
+ GArray *args, gdouble *res,
const gchar *symbol)
{
lua_State *L = lua_data->L;
@@ -511,7 +511,7 @@ process_regexp_item (struct rspamd_task *task,
void *user_data)
{
struct regexp_module_item *item = user_data;
- gint res = FALSE;
+ gdouble res = FALSE;
/* Non-threaded version */
if (item->lua_function) {
@@ -526,12 +526,7 @@ process_regexp_item (struct rspamd_task *task,
else {
/* Process expression */
if (item->expr) {
- struct rspamd_expr_process_data process_data;
- memset (&process_data, 0, sizeof process_data);
-
- process_data.task = task;
-
- res = rspamd_process_expression (item->expr, &process_data);
+ res = rspamd_process_expression (item->expr, 0, task);
}
else {
msg_warn_task ("FIXME: %s symbol is broken with new expressions",
@@ -539,7 +534,7 @@ process_regexp_item (struct rspamd_task *task,
}
}
- if (res) {
+ if (res != 0) {
rspamd_task_insert_result (task, item->symbol, res, NULL);
}
More information about the Commits
mailing list