commit 812dfbb: [Feature] Preprocess config files using jinja templates
Vsevolod Stakhov
vsevolod at highsecure.ru
Wed Mar 27 14:21:05 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-03-27 14:18:22 +0000
URL: https://github.com/rspamd/rspamd/commit/812dfbbd064daf5f60260b5718bbe3ba4173eabf (HEAD -> master)
[Feature] Preprocess config files using jinja templates
---
src/libserver/cfg_rcl.c | 93 +++++++++++++++++++++++--
src/libserver/cfg_rcl.h | 5 +-
src/lua/lua_common.c | 171 ++++++++++++++++++++++++++++++----------------
src/lua/lua_common.h | 3 +-
src/lua/lua_config.c | 5 +-
src/lua/lua_util.c | 2 +-
src/rspamadm/configdump.c | 6 +-
src/rspamadm/configtest.c | 6 +-
src/rspamadm/rspamadm.c | 12 +++-
src/rspamadm/rspamadm.h | 1 +
src/rspamd.c | 9 ++-
test/rspamd_lua_test.c | 2 +-
12 files changed, 241 insertions(+), 74 deletions(-)
diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c
index c6d2f3572..5c99aed59 100644
--- a/src/libserver/cfg_rcl.c
+++ b/src/libserver/cfg_rcl.c
@@ -3506,7 +3506,7 @@ rspamd_rcl_maybe_apply_lua_transform (struct rspamd_config *cfg)
}
static bool
-rspamd_rcl_decrypt_handler(struct ucl_parser *parser,
+rspamd_rcl_decrypt_handler (struct ucl_parser *parser,
const unsigned char *source, size_t source_len,
unsigned char **destination, size_t *dest_len,
void *user_data)
@@ -3525,6 +3525,64 @@ rspamd_rcl_decrypt_handler(struct ucl_parser *parser,
return true;
}
+static bool
+rspamd_rcl_jinja_handler (struct ucl_parser *parser,
+ const unsigned char *source, size_t source_len,
+ unsigned char **destination, size_t *dest_len,
+ void *user_data)
+{
+ struct rspamd_config *cfg = (struct rspamd_config *)user_data;
+ lua_State *L = cfg->lua_state;
+ gint err_idx;
+
+ lua_pushcfunction (L, &rspamd_lua_traceback);
+ err_idx = lua_gettop (L);
+
+ /* Obtain function */
+ if (!rspamd_lua_require_function (L, "lua_util", "jinja_template")) {
+ msg_err_config ("cannot require lua_util.jinja_template");
+ lua_settop (L, err_idx - 1);
+
+ return false;
+ }
+
+ lua_pushlstring (L, source, source_len);
+ lua_getglobal (L, "rspamd_env");
+ lua_pushboolean (L, false);
+
+ if (lua_pcall (L, 3, 1, err_idx) != 0) {
+ GString *tb;
+
+ tb = lua_touserdata (L, -1);
+ msg_err_config ("cannot call lua try_load_redis_servers script: %s", tb->str);
+ g_string_free (tb, TRUE);
+ lua_settop (L, err_idx - 1);
+
+ return false;
+ }
+
+ if (lua_type (L, -1) == LUA_TSTRING) {
+ const char *ndata;
+ gsize nsize;
+
+ ndata = lua_tolstring (L, -1, &nsize);
+ *destination = UCL_ALLOC (nsize);
+ memcpy (*destination, ndata, nsize);
+ *dest_len = nsize;
+ }
+ else {
+ msg_err_config ("invalid return type when templating jinja %s",
+ lua_typename (L, lua_type (L, -1)));
+ lua_settop (L, err_idx - 1);
+
+ return false;
+ }
+
+ lua_settop (L, err_idx - 1);
+
+ return true;
+}
+
static void
rspamd_rcl_decrypt_free (unsigned char *data, size_t len, void *user_data)
{
@@ -3561,6 +3619,7 @@ rspamd_config_parse_ucl (struct rspamd_config *cfg,
GHashTable *vars,
ucl_include_trace_func_t inc_trace,
void *trace_data,
+ gboolean skip_jinja,
GError **err)
{
struct stat st;
@@ -3652,6 +3711,18 @@ rspamd_config_parse_ucl (struct rspamd_config *cfg,
ucl_parser_add_special_handler (parser, decrypt_handler);
}
+ if (!skip_jinja) {
+ struct ucl_parser_special_handler *jinja_handler;
+
+ jinja_handler = rspamd_mempool_alloc0 (cfg->cfg_pool,
+ sizeof (*jinja_handler));
+ jinja_handler->user_data = cfg;
+ jinja_handler->flags = UCL_SPECIAL_HANDLER_PREPROCESS_ALL;
+ jinja_handler->handler = rspamd_rcl_jinja_handler;
+
+ ucl_parser_add_special_handler (parser, jinja_handler);
+ }
+
if (!ucl_parser_add_chunk (parser, data, st.st_size)) {
g_set_error (err, cfg_rcl_error_quark (), errno,
"ucl parser error: %s", ucl_parser_get_error (parser));
@@ -3670,17 +3741,28 @@ rspamd_config_parse_ucl (struct rspamd_config *cfg,
}
gboolean
-rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
+rspamd_config_read (struct rspamd_config *cfg,
+ const gchar *filename,
rspamd_rcl_section_fin_t logger_fin,
- gpointer logger_ud, GHashTable *vars)
+ gpointer logger_ud,
+ GHashTable *vars,
+ gboolean skip_jinja,
+ gchar **lua_env)
{
GError *err = NULL;
struct rspamd_rcl_section *top, *logger_section;
const ucl_object_t *logger_obj;
- rspamd_lua_set_env (cfg->lua_state, vars);
+ rspamd_lua_set_path (cfg->lua_state, NULL, vars);
+
+ if (!rspamd_lua_set_env (cfg->lua_state, vars, lua_env, &err)) {
+ msg_err_config_forced ("failed to set up environment: %e", err);
+ g_error_free (err);
+
+ return FALSE;
+ }
- if (!rspamd_config_parse_ucl (cfg, filename, vars, NULL, NULL, &err)) {
+ if (!rspamd_config_parse_ucl (cfg, filename, vars, NULL, NULL, skip_jinja, &err)) {
msg_err_config_forced ("failed to load config: %e", err);
g_error_free (err);
@@ -3688,6 +3770,7 @@ rspamd_config_read (struct rspamd_config *cfg, const gchar *filename,
}
top = rspamd_rcl_config_init (cfg, NULL);
+ /* Add new paths if defined in options */
rspamd_lua_set_path (cfg->lua_state, cfg->rcl_obj, vars);
rspamd_lua_set_globals (cfg, cfg->lua_state);
rspamd_mempool_add_destructor (cfg->cfg_pool, rspamd_rcl_section_free, top);
diff --git a/src/libserver/cfg_rcl.h b/src/libserver/cfg_rcl.h
index e2477481e..12a89a673 100644
--- a/src/libserver/cfg_rcl.h
+++ b/src/libserver/cfg_rcl.h
@@ -481,10 +481,13 @@ gboolean rspamd_config_parse_ucl (struct rspamd_config *cfg,
GHashTable *vars,
ucl_include_trace_func_t inc_trace,
void *trace_data,
+ gboolean skip_jinja,
GError **err);
gboolean rspamd_config_read (struct rspamd_config *cfg,
const gchar *filename,
rspamd_rcl_section_fin_t logger_fin,
gpointer logger_ud,
- GHashTable *vars);
+ GHashTable *vars,
+ gboolean skip_jinja,
+ gchar **lua_env);
#endif /* CFG_RCL_H_ */
diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c
index 1c687fb48..5f1be424d 100644
--- a/src/lua/lua_common.c
+++ b/src/lua/lua_common.c
@@ -263,83 +263,76 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars
}
}
- /* Try environment */
- t = getenv ("SHAREDIR");
- if (t) {
- sharedir = t;
- }
-
- t = getenv ("PLUGINSDIR");
- if (t) {
- pluginsdir = t;
- }
-
- t = getenv ("RULESDIR");
- if (t) {
- rulesdir = t;
- }
-
- t = getenv ("LUALIBDIR");
- if (t) {
- lualibdir = t;
- }
-
- t = getenv ("LIBDIR");
- if (t) {
- libdir = t;
- }
-
- t = getenv ("RSPAMD_LIBDIR");
- if (t) {
- libdir = t;
+ if (additional_path) {
+ rspamd_snprintf (path_buf, sizeof (path_buf),
+ "%s;"
+ "%s",
+ additional_path, old_path);
}
-
- if (vars) {
- t = g_hash_table_lookup (vars, "PLUGINSDIR");
+ else {
+ /* Try environment */
+ t = getenv ("SHAREDIR");
if (t) {
- pluginsdir = t;
+ sharedir = t;
}
- t = g_hash_table_lookup (vars, "SHAREDIR");
+ t = getenv ("PLUGINSDIR");
if (t) {
- sharedir = t;
+ pluginsdir = t;
}
- t = g_hash_table_lookup (vars, "RULESDIR");
+ t = getenv ("RULESDIR");
if (t) {
rulesdir = t;
}
- t = g_hash_table_lookup (vars, "LUALIBDIR");
+ t = getenv ("LUALIBDIR");
if (t) {
lualibdir = t;
}
- t = g_hash_table_lookup (vars, "LIBDIR");
+ t = getenv ("LIBDIR");
if (t) {
libdir = t;
}
- t = g_hash_table_lookup (vars, "RSPAMD_LIBDIR");
+ t = getenv ("RSPAMD_LIBDIR");
if (t) {
libdir = t;
}
- }
- if (additional_path) {
- rspamd_snprintf (path_buf, sizeof (path_buf),
- "%s/lua/?.lua;"
- "%s/?.lua;"
- "%s/?.lua;"
- "%s/?/init.lua;"
- "%s;"
- "%s",
- RSPAMD_CONFDIR,
- rulesdir,
- lualibdir, lualibdir,
- additional_path, old_path);
- }
- else {
+ if (vars) {
+ t = g_hash_table_lookup (vars, "PLUGINSDIR");
+ if (t) {
+ pluginsdir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "SHAREDIR");
+ if (t) {
+ sharedir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "RULESDIR");
+ if (t) {
+ rulesdir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "LUALIBDIR");
+ if (t) {
+ lualibdir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "LIBDIR");
+ if (t) {
+ libdir = t;
+ }
+
+ t = g_hash_table_lookup (vars, "RSPAMD_LIBDIR");
+ if (t) {
+ libdir = t;
+ }
+ }
+
rspamd_snprintf (path_buf, sizeof (path_buf),
"%s/lua/?.lua;"
"%s/?.lua;"
@@ -372,11 +365,9 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars
if (additional_path) {
rspamd_snprintf (path_buf, sizeof (path_buf),
"%s/?%s;"
- "%s;"
"%s",
- libdir,
- OS_SO_SUFFIX,
additional_path,
+ OS_SO_SUFFIX,
old_path);
}
else {
@@ -387,6 +378,7 @@ rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj, GHashTable *vars
OS_SO_SUFFIX,
old_path);
}
+
lua_pop (L, 1);
lua_pushstring (L, path_buf);
lua_setfield (L, -2, "cpath");
@@ -537,8 +529,56 @@ rspamd_lua_rspamd_version (lua_State *L)
return 1;
}
-void
-rspamd_lua_set_env (lua_State *L, GHashTable *vars)
+static gboolean
+rspamd_lua_load_env (lua_State *L, const char *fname, gint tbl_pos, GError **err)
+{
+ gint orig_top = lua_gettop (L), err_idx;
+ gboolean ret = TRUE;
+
+ lua_pushcfunction (L, &rspamd_lua_traceback);
+ err_idx = lua_gettop (L);
+
+ if (luaL_loadfile (L, fname) != 0) {
+ g_set_error (err, g_quark_from_static_string ("lua_env"), errno,
+ "cannot load lua file %s: %s",
+ fname,
+ lua_tostring (L, -1));
+ ret = FALSE;
+ }
+
+ if (ret && lua_pcall (L, 0, 1, err_idx) != 0) {
+ GString *tb = lua_touserdata (L, -1);
+ g_set_error (err, g_quark_from_static_string ("lua_env"), errno,
+ "cannot init lua file %s: %s",
+ fname,
+ tb->str);
+ g_string_free (tb, TRUE);
+
+ ret = FALSE;
+ }
+
+ if (ret && lua_type (L, -1) == LUA_TTABLE) {
+ for (lua_pushnil (L); lua_next (L, -2); lua_pop (L, 1)) {
+ lua_pushvalue (L, -2); /* Store key */
+ lua_pushvalue (L, -2); /* Store value */
+ lua_settable (L, tbl_pos);
+ }
+ }
+ else if (ret) {
+ g_set_error (err, g_quark_from_static_string ("lua_env"), errno,
+ "invalid return type when loading env from %s: %s",
+ fname,
+ lua_typename (L, lua_type (L, -1)));
+ ret = FALSE;
+ }
+
+ lua_settop (L, orig_top);
+
+ return ret;
+}
+
+gboolean
+rspamd_lua_set_env (lua_State *L, GHashTable *vars, char **lua_env, GError **err)
{
gint orig_top = lua_gettop (L);
gchar **env = g_get_environ ();
@@ -742,10 +782,23 @@ rspamd_lua_set_env (lua_State *L, GHashTable *vars)
}
}
+ if (lua_env) {
+ gint lim = g_strv_length (lua_env);
+
+ for (gint i = 0; i < lim; i ++) {
+ if (!rspamd_lua_load_env (L, lua_env[i], lua_gettop (L), err)) {
+ return FALSE;
+ }
+ }
+ }
+
lua_setglobal (L, "rspamd_env");
}
lua_settop (L, orig_top);
+ g_strfreev (env);
+
+ return TRUE;
}
void
diff --git a/src/lua/lua_common.h b/src/lua/lua_common.h
index 520f5920d..01b7d3581 100644
--- a/src/lua/lua_common.h
+++ b/src/lua/lua_common.h
@@ -306,7 +306,8 @@ void rspamd_lua_set_path (lua_State *L, const ucl_object_t *cfg_obj,
GHashTable *vars);
/* Set some lua globals */
-void rspamd_lua_set_env (lua_State *L, GHashTable *vars);
+gboolean rspamd_lua_set_env (lua_State *L, GHashTable *vars, char **lua_env,
+ GError **err);
void rspamd_lua_set_globals (struct rspamd_config *cfg, lua_State *L);
struct memory_pool_s * rspamd_lua_check_mempool (lua_State * L, gint pos);
diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c
index 3990e014e..f60fe6995 100644
--- a/src/lua/lua_config.c
+++ b/src/lua/lua_config.c
@@ -3819,7 +3819,7 @@ lua_config_load_ucl (lua_State *L)
cbd.L = L;
if (!rspamd_config_parse_ucl (cfg, filename, paths,
- lua_include_trace_cb, &cbd, &err)) {
+ lua_include_trace_cb, &cbd, lua_toboolean (L, 4), &err)) {
luaL_unref (L, LUA_REGISTRYINDEX, cbd.cbref);
lua_pushboolean (L, false);
lua_pushfstring (L, "failed to load config: %s", err->message);
@@ -3832,7 +3832,8 @@ lua_config_load_ucl (lua_State *L)
luaL_unref (L, LUA_REGISTRYINDEX, cbd.cbref);
}
else {
- if (!rspamd_config_parse_ucl (cfg, filename, paths, NULL, NULL, &err)) {
+ if (!rspamd_config_parse_ucl (cfg, filename, paths, NULL, NULL,
+ lua_toboolean (L, 3), &err)) {
lua_pushboolean (L, false);
lua_pushfstring (L, "failed to load config: %s", err->message);
g_error_free (err);
diff --git a/src/lua/lua_util.c b/src/lua/lua_util.c
index af4673af8..84958ebc0 100644
--- a/src/lua/lua_util.c
+++ b/src/lua/lua_util.c
@@ -717,7 +717,7 @@ lua_util_load_rspamd_config (lua_State *L)
cfg = rspamd_config_new (RSPAMD_CONFIG_INIT_SKIP_LUA);
cfg->lua_state = L;
- if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL)) {
+ if (rspamd_config_read (cfg, cfg_name, NULL, NULL, NULL, FALSE, NULL)) {
msg_err_config ("cannot load config from %s", cfg_name);
lua_pushnil (L);
}
diff --git a/src/rspamadm/configdump.c b/src/rspamadm/configdump.c
index a255994a5..8f38cdf61 100644
--- a/src/rspamadm/configdump.c
+++ b/src/rspamadm/configdump.c
@@ -27,6 +27,7 @@ static gboolean compact = FALSE;
static gboolean show_help = FALSE;
static gboolean show_comments = FALSE;
static gboolean modules_state = FALSE;
+static gboolean skip_template = FALSE;
static gchar *config = NULL;
extern struct rspamd_main *rspamd_main;
/* Defined in modules.c */
@@ -57,6 +58,8 @@ static GOptionEntry entries[] = {
"Show saved comments from the configuration file", NULL },
{"modules-state", 'm', 0, G_OPTION_ARG_NONE, &modules_state,
"Show modules state only", NULL},
+ {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template,
+ "Do not apply Jinja templates", NULL},
{NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
};
@@ -284,7 +287,8 @@ rspamadm_configdump (gint argc, gchar **argv, const struct rspamadm_command *cmd
cfg->compiled_workers = workers;
cfg->cfg_name = config;
- if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) {
+ if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main,
+ ucl_vars, skip_template, lua_env)) {
ret = FALSE;
}
else {
diff --git a/src/rspamadm/configtest.c b/src/rspamadm/configtest.c
index db9a8d604..3b7a6b5b0 100644
--- a/src/rspamadm/configtest.c
+++ b/src/rspamadm/configtest.c
@@ -23,6 +23,7 @@
static gboolean quiet = FALSE;
static gchar *config = NULL;
static gboolean strict = FALSE;
+static gboolean skip_template = FALSE;
extern struct rspamd_main *rspamd_main;
/* Defined in modules.c */
extern module_t *modules[];
@@ -48,6 +49,8 @@ static GOptionEntry entries[] = {
"Config file to test", NULL},
{"strict", 's', 0, G_OPTION_ARG_NONE, &strict,
"Stop on any error in config", NULL},
+ {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template,
+ "Do not apply Jinja templates", NULL},
{NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
};
@@ -141,7 +144,8 @@ rspamadm_configtest (gint argc, gchar **argv, const struct rspamadm_command *cmd
cfg->compiled_workers = workers;
cfg->cfg_name = config;
- if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) {
+ if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main,
+ ucl_vars, skip_template, lua_env)) {
ret = FALSE;
}
else {
diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c
index 281690132..762a74c1f 100644
--- a/src/rspamadm/rspamadm.c
+++ b/src/rspamadm/rspamadm.c
@@ -31,6 +31,7 @@ static gboolean list_commands = FALSE;
static gboolean show_help = FALSE;
static gboolean show_version = FALSE;
GHashTable *ucl_vars = NULL;
+gchar **lua_env = NULL;
struct rspamd_main *rspamd_main = NULL;
struct rspamd_async_session *rspamadm_session = NULL;
lua_State *L = NULL;
@@ -65,6 +66,8 @@ static GOptionEntry entries[] = {
"Show help", NULL},
{"version", 'V', 0, G_OPTION_ARG_NONE, &show_version,
"Show version", NULL},
+ {"lua-env", '\0', 0, G_OPTION_ARG_FILENAME_ARRAY, &lua_env,
+ "Load lua environment from the specified files", NULL},
{NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL}
};
@@ -458,8 +461,15 @@ main (gint argc, gchar **argv, gchar **env)
setproctitle ("rspamdadm");
L = cfg->lua_state;
- rspamd_lua_set_env (L, ucl_vars);
rspamd_lua_set_path (L, NULL, ucl_vars);
+
+ if (!rspamd_lua_set_env (L, ucl_vars, lua_env, &error)) {
+ rspamd_fprintf (stderr, "Cannot load lua environment: %e", error);
+ g_error_free (error);
+
+ exit (EXIT_FAILURE);
+ }
+
rspamd_lua_set_globals (cfg, L);
rspamadm_add_lua_globals ();
diff --git a/src/rspamadm/rspamadm.h b/src/rspamadm/rspamadm.h
index cd01cc86b..e8ed9c873 100644
--- a/src/rspamadm/rspamadm.h
+++ b/src/rspamadm/rspamadm.h
@@ -23,6 +23,7 @@
#include <lualib.h>
extern GHashTable *ucl_vars;
+extern gchar **lua_env;
extern struct rspamd_main *rspamd_main;
GQuark rspamadm_error (void);
diff --git a/src/rspamd.c b/src/rspamd.c
index 631759f05..8b12fa48e 100644
--- a/src/rspamd.c
+++ b/src/rspamd.c
@@ -95,6 +95,8 @@ static gboolean is_insecure = FALSE;
static gboolean gen_keypair = FALSE;
static gboolean encrypt_password = FALSE;
static GHashTable *ucl_vars = NULL;
+static gchar **lua_env = NULL;
+static gboolean skip_template = FALSE;
static gint term_attempts = 0;
@@ -146,6 +148,10 @@ static GOptionEntry entries[] =
"Show version and exit", NULL },
{"var", 0, 0, G_OPTION_ARG_CALLBACK, (gpointer)&rspamd_parse_var,
"Redefine/define environment variable", NULL},
+ {"skip-template", 'T', 0, G_OPTION_ARG_NONE, &skip_template,
+ "Do not apply Jinja templates", NULL},
+ {"lua-env", '\0', 0, G_OPTION_ARG_FILENAME_ARRAY, &lua_env,
+ "Load lua environment from the specified files", NULL},
{ NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL, NULL }
};
@@ -923,7 +929,8 @@ load_rspamd_config (struct rspamd_main *rspamd_main,
cfg->compiled_modules = modules;
cfg->compiled_workers = workers;
- if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main, ucl_vars)) {
+ if (!rspamd_config_read (cfg, cfg->cfg_name, config_logger, rspamd_main,
+ ucl_vars, skip_template, lua_env)) {
return FALSE;
}
diff --git a/test/rspamd_lua_test.c b/test/rspamd_lua_test.c
index 2af0d9aab..ad40ef488 100644
--- a/test/rspamd_lua_test.c
+++ b/test/rspamd_lua_test.c
@@ -64,7 +64,7 @@ rspamd_lua_test_func (void)
glob_t globbuf;
gint i, len;
- rspamd_lua_set_env (L, NULL);
+ rspamd_lua_set_env (L, NULL, NULL, NULL);
rspamd_lua_set_globals (rspamd_main->cfg, L);
if (lua_test_case) {
More information about the Commits
mailing list