commit 3f00d10: [Project] Adopt plugins, workers, rspamadm and rspamc for http_context
Vsevolod Stakhov
vsevolod at highsecure.ru
Tue Feb 19 18:00:11 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-02-19 17:57:13 +0000
URL: https://github.com/rspamd/rspamd/commit/3f00d108df7c99edd5cb45e34d232a80e159caa3 (HEAD -> master)
[Project] Adopt plugins, workers, rspamadm and rspamc for http_context
---
src/client/rspamc.c | 12 +++++++-
src/client/rspamdclient.c | 13 ++++----
src/client/rspamdclient.h | 12 +++++---
src/controller.c | 9 +++---
src/fuzzy_storage.c | 27 ++++++++--------
src/plugins/surbl.c | 5 ++-
src/rspamadm/control.c | 8 ++---
src/rspamadm/lua_repl.c | 5 +--
src/rspamadm/rspamadm.c | 1 +
src/rspamd.h | 1 +
src/rspamd_proxy.c | 78 ++++++++++-------------------------------------
src/worker.c | 13 ++++----
src/worker_private.h | 2 +-
13 files changed, 77 insertions(+), 109 deletions(-)
diff --git a/src/client/rspamc.c b/src/client/rspamc.c
index 588496d46..d9af90b7f 100644
--- a/src/client/rspamc.c
+++ b/src/client/rspamc.c
@@ -67,6 +67,7 @@ static gchar *key = NULL;
static gchar *user_agent = "rspamc";
static GList *children;
static GPatternSpec **exclude_compiled = NULL;
+static struct rspamd_http_context *http_ctx;
static gint retcode = EXIT_SUCCESS;
@@ -1668,7 +1669,7 @@ rspamc_process_input (struct event_base *ev_base, struct rspamc_command *cmd,
}
- conn = rspamd_client_init (ev_base, hostbuf, port, timeout, key);
+ conn = rspamd_client_init (http_ctx, ev_base, hostbuf, port, timeout, key);
if (conn != NULL) {
cbdata = g_malloc0 (sizeof (struct rspamc_callback_data));
@@ -1894,6 +1895,15 @@ main (gint argc, gchar **argv, gchar **env)
rspamd_init_libs ();
ev_base = event_base_new ();
+ struct rspamd_http_context_cfg http_config;
+
+ memset (&http_config, 0, sizeof (http_config));
+ http_config.kp_cache_size_client = 32;
+ http_config.kp_cache_size_server = 0;
+ http_config.user_agent = user_agent;
+ http_ctx = rspamd_http_context_create_config (&http_config,
+ ev_base);
+
/* Ignore sigpipe */
sigemptyset (&sigpipe_act.sa_mask);
sigaddset (&sigpipe_act.sa_mask, SIGPIPE);
diff --git a/src/client/rspamdclient.c b/src/client/rspamdclient.c
index b1b550024..54cd490e6 100644
--- a/src/client/rspamdclient.c
+++ b/src/client/rspamdclient.c
@@ -241,8 +241,9 @@ rspamd_client_finish_handler (struct rspamd_http_connection *conn,
}
struct rspamd_client_connection *
-rspamd_client_init (struct event_base *ev_base, const gchar *name,
- guint16 port, gdouble timeout, const gchar *key)
+rspamd_client_init (struct rspamd_http_context *http_ctx,
+ struct event_base *ev_base, const gchar *name,
+ guint16 port, gdouble timeout, const gchar *key)
{
struct rspamd_client_connection *conn;
gint fd;
@@ -256,14 +257,12 @@ rspamd_client_init (struct event_base *ev_base, const gchar *name,
conn->ev_base = ev_base;
conn->fd = fd;
conn->req_sent = FALSE;
- conn->keys_cache = rspamd_keypair_cache_new (32);
- conn->http_conn = rspamd_http_connection_new (rspamd_client_body_handler,
+ conn->http_conn = rspamd_http_connection_new (http_ctx,
+ rspamd_client_body_handler,
rspamd_client_error_handler,
rspamd_client_finish_handler,
0,
- RSPAMD_HTTP_CLIENT,
- conn->keys_cache,
- NULL);
+ RSPAMD_HTTP_CLIENT);
conn->server_name = g_string_new (name);
if (port != 0) {
diff --git a/src/client/rspamdclient.h b/src/client/rspamdclient.h
index 129ee39a1..c2a3c1886 100644
--- a/src/client/rspamdclient.h
+++ b/src/client/rspamdclient.h
@@ -47,6 +47,7 @@ typedef void (*rspamd_client_callback) (
gdouble send_time,
GError *err);
+struct rspamd_http_context;
/**
* Start rspamd worker or controller command
* @param ev_base event base
@@ -56,11 +57,12 @@ typedef void (*rspamd_client_callback) (
* @return
*/
struct rspamd_client_connection * rspamd_client_init (
- struct event_base *ev_base,
- const gchar *name,
- guint16 port,
- gdouble timeout,
- const gchar *key);
+ struct rspamd_http_context *http_ctx,
+ struct event_base *ev_base,
+ const gchar *name,
+ guint16 port,
+ gdouble timeout,
+ const gchar *key);
/**
*
diff --git a/src/controller.c b/src/controller.c
index b19ac3db7..a6214655a 100644
--- a/src/controller.c
+++ b/src/controller.c
@@ -150,6 +150,7 @@ struct rspamd_controller_worker_ctx {
rspamd_ftok_t cached_password;
rspamd_ftok_t cached_enable_password;
/* HTTP server */
+ struct rspamd_http_context *http_ctx;
struct rspamd_http_connection_router *http;
/* Server's start time */
time_t start_time;
@@ -3707,7 +3708,6 @@ start_controller_worker (struct rspamd_worker *worker)
GHashTableIter iter;
gpointer key, value;
guint i;
- struct rspamd_keypair_cache *cache;
struct timeval stv;
const guint save_stats_interval = 60 * 1000; /* 1 minute */
gpointer m;
@@ -3783,10 +3783,10 @@ start_controller_worker (struct rspamd_worker *worker)
"password");
/* Accept event */
- cache = rspamd_keypair_cache_new (256);
+ ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
ctx->http = rspamd_http_router_new (rspamd_controller_error_handler,
- rspamd_controller_finish_handler, &ctx->io_tv, ctx->ev_base,
- ctx->static_files_dir, cache);
+ rspamd_controller_finish_handler, &ctx->io_tv,
+ ctx->static_files_dir, ctx->http_ctx);
/* Add callbacks for different methods */
rspamd_http_router_add_path (ctx->http,
@@ -3949,6 +3949,7 @@ start_controller_worker (struct rspamd_worker *worker)
g_hash_table_unref (ctx->plugins);
g_hash_table_unref (ctx->custom_commands);
+ rspamd_http_context_free (ctx->http_ctx);
REF_RELEASE (ctx->cfg);
rspamd_log_close (worker->srv->logger, TRUE);
diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index d0a4f3602..a02db290b 100644
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -173,6 +173,7 @@ struct rspamd_fuzzy_storage_ctx {
struct rspamd_cryptobox_keypair *collection_keypair;
struct rspamd_cryptobox_pubkey *collection_sign_key;
gchar *collection_id_file;
+ struct rspamd_http_context *http_ctx;
struct rspamd_keypair_cache *keypair_cache;
rspamd_lru_hash_t *errors_ips;
rspamd_lru_hash_t *ratelimit_buckets;
@@ -616,13 +617,13 @@ rspamd_fuzzy_send_update_mirror (struct rspamd_fuzzy_storage_ctx *ctx,
msg = rspamd_http_new_message (HTTP_REQUEST);
rspamd_printf_fstring (&msg->url, "/update_v1/%s", m->name);
- conn->http_conn = rspamd_http_connection_new (NULL,
+ conn->http_conn = rspamd_http_connection_new (
+ ctx->http_ctx,
+ NULL,
fuzzy_mirror_error_handler,
fuzzy_mirror_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- ctx->keypair_cache,
- NULL);
+ RSPAMD_HTTP_CLIENT);
rspamd_http_connection_set_key (conn->http_conn,
ctx->sync_keypair);
@@ -1994,13 +1995,13 @@ accept_fuzzy_mirror_socket (gint fd, short what, void *arg)
session->name = rspamd_inet_address_to_string (addr);
rspamd_random_hex (session->uid, sizeof (session->uid) - 1);
session->uid[sizeof (session->uid) - 1] = '\0';
- http_conn = rspamd_http_connection_new (NULL,
+ http_conn = rspamd_http_connection_new (
+ ctx->http_ctx,
+ NULL,
rspamd_fuzzy_mirror_error_handler,
rspamd_fuzzy_mirror_finish_handler,
0,
- RSPAMD_HTTP_SERVER,
- ctx->keypair_cache,
- NULL);
+ RSPAMD_HTTP_SERVER);
rspamd_http_connection_set_key (http_conn, ctx->sync_keypair);
session->ctx = ctx;
@@ -3004,6 +3005,8 @@ start_fuzzy (struct rspamd_worker *worker)
ctx->keypair_cache = rspamd_keypair_cache_new (ctx->keypair_cache_size);
}
+ ctx->http_ctx = rspamd_http_context_create (cfg, ctx->ev_base);
+
if (!ctx->collection_mode) {
/*
* Open DB and perform VACUUM
@@ -3058,8 +3061,8 @@ start_fuzzy (struct rspamd_worker *worker)
rspamd_fuzzy_collection_error_handler,
rspamd_fuzzy_collection_finish_handler,
&ctx->stat_tv,
- ctx->ev_base,
- NULL, ctx->keypair_cache);
+ NULL,
+ ctx->http_ctx);
if (ctx->collection_keypair) {
rspamd_http_router_set_key (ctx->collection_rt,
@@ -3202,8 +3205,6 @@ start_fuzzy (struct rspamd_worker *worker)
else if (worker->index == 0) {
gint fd;
- /* Steal keypairs cache... */
- ctx->collection_rt->cache = NULL;
rspamd_http_router_free (ctx->collection_rt);
/* Try to save collection id */
@@ -3240,8 +3241,8 @@ start_fuzzy (struct rspamd_worker *worker)
rspamd_keypair_cache_destroy (ctx->keypair_cache);
}
+ rspamd_http_context_free (ctx->http_ctx);
REF_RELEASE (ctx->cfg);
-
rspamd_log_close (worker->srv->logger, TRUE);
exit (EXIT_SUCCESS);
diff --git a/src/plugins/surbl.c b/src/plugins/surbl.c
index 5949f5bb6..22088c2e5 100644
--- a/src/plugins/surbl.c
+++ b/src/plugins/surbl.c
@@ -1734,12 +1734,11 @@ register_redirector_call (struct rspamd_url *url, struct rspamd_task *task,
param->url = url;
param->task = task;
param->conn = rspamd_http_connection_new (NULL,
+ NULL,
surbl_redirector_error,
surbl_redirector_finish,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- NULL,
- NULL);
+ RSPAMD_HTTP_CLIENT);
param->ctx = surbl_module_ctx;
msg = rspamd_http_new_message (HTTP_REQUEST);
msg->url = rspamd_fstring_assign (msg->url, url->string, url->urllen);
diff --git a/src/rspamadm/control.c b/src/rspamadm/control.c
index 554392e63..c030907f6 100644
--- a/src/rspamadm/control.c
+++ b/src/rspamadm/control.c
@@ -234,13 +234,13 @@ rspamadm_control (gint argc, gchar **argv, const struct rspamadm_command *_cmd)
exit (1);
}
- conn = rspamd_http_connection_new (NULL,
+ conn = rspamd_http_connection_new (
+ NULL, /* Default context */
+ NULL,
rspamd_control_error_handler,
rspamd_control_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- NULL,
- NULL);
+ RSPAMD_HTTP_CLIENT);
msg = rspamd_http_new_message (HTTP_REQUEST);
msg->url = rspamd_fstring_new_init (path, strlen (path));
double_to_tv (timeout, &tv);
diff --git a/src/rspamadm/lua_repl.c b/src/rspamadm/lua_repl.c
index 6248f2aa2..e91bd34fa 100644
--- a/src/rspamadm/lua_repl.c
+++ b/src/rspamadm/lua_repl.c
@@ -808,8 +808,9 @@ rspamadm_lua (gint argc, gchar **argv, const struct rspamadm_command *cmd)
ctx = g_malloc0 (sizeof (*ctx));
http = rspamd_http_router_new (rspamadm_lua_error_handler,
rspamadm_lua_finish_handler,
- NULL, ev_base,
- NULL, NULL);
+ NULL,
+ NULL,
+ rspamd_main->http_ctx);
ctx->L = L;
ctx->rt = http;
rspamd_http_router_add_path (http,
diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c
index ef52af1e3..c0bb4bc72 100644
--- a/src/rspamadm/rspamadm.c
+++ b/src/rspamadm/rspamadm.c
@@ -436,6 +436,7 @@ main (gint argc, gchar **argv, gchar **env)
(void) dns_resolver_init (rspamd_main->logger,
rspamd_main->ev_base,
cfg);
+ rspamd_main->http_ctx = rspamd_http_context_create (cfg, rspamd_main->ev_base);
g_log_set_default_handler (rspamd_glib_log_function, rspamd_main->logger);
g_set_printerr_handler (rspamd_glib_printerr_function);
diff --git a/src/rspamd.h b/src/rspamd.h
index c11090a6e..10d3be9fb 100644
--- a/src/rspamd.h
+++ b/src/rspamd.h
@@ -277,6 +277,7 @@ struct rspamd_main {
gboolean cores_throttling; /**< turn off cores when limits are exceeded */
struct roll_history *history; /**< rolling history */
struct event_base *ev_base;
+ struct rspamd_http_context *http_ctx;
};
enum rspamd_exception_type {
diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c
index 31aeceb12..4ecbcf54b 100644
--- a/src/rspamd_proxy.c
+++ b/src/rspamd_proxy.c
@@ -123,18 +123,14 @@ struct rspamd_proxy_ctx {
struct timeval io_tv;
/* Encryption key for clients */
struct rspamd_cryptobox_keypair *key;
- /* Keys cache */
- struct rspamd_keypair_cache *keys_cache;
+ /* HTTP context */
+ struct rspamd_http_context *http_ctx;
/* Upstreams to use */
GHashTable *upstreams;
/* Mirrors to send traffic to */
GPtrArray *mirrors;
/* Default upstream */
struct rspamd_http_upstream *default_upstream;
- /* Local rotating keypair for upstreams */
- struct rspamd_cryptobox_keypair *local_key;
- struct event rotate_ev;
- gdouble rotate_tm;
lua_State *lua_state;
/* Array of callback functions called on end of scan to compare results */
GArray *cmp_refs;
@@ -746,7 +742,6 @@ init_rspamd_proxy (struct rspamd_config *cfg)
ctx->mirrors = g_ptr_array_new ();
rspamd_mempool_add_destructor (cfg->cfg_pool,
(rspamd_mempool_destruct_t)rspamd_ptr_array_free_hard, ctx->mirrors);
- ctx->rotate_tm = DEFAULT_ROTATION_TIME;
ctx->cfg = cfg;
ctx->lua_state = cfg->lua_state;
ctx->cmp_refs = g_array_new (FALSE, FALSE, sizeof (gint));
@@ -763,15 +758,6 @@ init_rspamd_proxy (struct rspamd_config *cfg)
G_STRUCT_OFFSET (struct rspamd_proxy_ctx, timeout),
RSPAMD_CL_FLAG_TIME_FLOAT,
"IO timeout");
- rspamd_rcl_register_worker_option (cfg,
- type,
- "rotate",
- rspamd_rcl_parse_struct_time,
- ctx,
- G_STRUCT_OFFSET (struct rspamd_proxy_ctx, rotate_tm),
- RSPAMD_CL_FLAG_TIME_FLOAT,
- "Rotation keys time, default: "
- G_STRINGIFY (DEFAULT_ROTATION_TIME) " seconds");
rspamd_rcl_register_worker_option (cfg,
type,
"keypair",
@@ -1418,17 +1404,15 @@ proxy_open_mirror_connections (struct rspamd_proxy_session *session)
rspamd_http_message_add_header (msg, "Settings-ID", m->settings_id);
}
- bk_conn->backend_conn = rspamd_http_connection_new (NULL,
+ bk_conn->backend_conn = rspamd_http_connection_new (
+ session->ctx->http_ctx,
+ NULL,
proxy_backend_mirror_error_handler,
proxy_backend_mirror_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- session->ctx->keys_cache,
- NULL);
+ RSPAMD_HTTP_CLIENT);
if (m->key) {
- rspamd_http_connection_set_key (bk_conn->backend_conn,
- session->ctx->local_key);
msg->peer_key = rspamd_pubkey_ref (m->key);
}
@@ -1851,21 +1835,18 @@ retry:
}
session->master_conn->backend_conn = rspamd_http_connection_new (
+ session->ctx->http_ctx,
NULL,
proxy_backend_master_error_handler,
proxy_backend_master_finish_handler,
RSPAMD_HTTP_CLIENT_SIMPLE,
- RSPAMD_HTTP_CLIENT,
- session->ctx->keys_cache,
- NULL);
+ RSPAMD_HTTP_CLIENT);
session->master_conn->flags &= ~RSPAMD_BACKEND_CLOSED;
session->master_conn->parser_from_ref = backend->parser_from_ref;
session->master_conn->parser_to_ref = backend->parser_to_ref;
if (backend->key) {
msg->peer_key = rspamd_pubkey_ref (backend->key);
- rspamd_http_connection_set_key (session->master_conn->backend_conn,
- session->ctx->local_key);
}
if (backend->settings_id != NULL) {
@@ -2105,13 +2086,13 @@ proxy_accept_socket (gint fd, short what, void *arg)
}
if (!ctx->milter) {
- session->client_conn = rspamd_http_connection_new (NULL,
+ session->client_conn = rspamd_http_connection_new (
+ ctx->http_ctx,
+ NULL,
proxy_client_error_handler,
proxy_client_finish_handler,
0,
- RSPAMD_HTTP_SERVER,
- ctx->keys_cache,
- NULL);
+ RSPAMD_HTTP_SERVER);
if (ctx->key) {
rspamd_http_connection_set_key (session->client_conn, ctx->key);
@@ -2158,24 +2139,6 @@ proxy_accept_socket (gint fd, short what, void *arg)
}
}
-static void
-proxy_rotate_key (gint fd, short what, void *arg)
-{
- struct timeval rot_tv;
- struct rspamd_proxy_ctx *ctx = arg;
- gpointer kp;
-
- double_to_tv (ctx->rotate_tm, &rot_tv);
- rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec);
- event_del (&ctx->rotate_ev);
- event_add (&ctx->rotate_ev, &rot_tv);
-
- kp = ctx->local_key;
- ctx->local_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
- RSPAMD_CRYPTOBOX_MODE_25519);
- rspamd_keypair_unref (kp);
-}
-
static void
adjust_upstreams_limits (struct rspamd_proxy_ctx *ctx)
{
@@ -2205,9 +2168,9 @@ adjust_upstreams_limits (struct rspamd_proxy_ctx *ctx)
}
void
-start_rspamd_proxy (struct rspamd_worker *worker) {
+start_rspamd_proxy (struct rspamd_worker *worker)
+{
struct rspamd_proxy_ctx *ctx = worker->ctx;
- struct timeval rot_tv;
ctx->cfg = worker->srv->cfg;
ctx->ev_base = rspamd_prepare_worker (worker, "rspamd_proxy",
@@ -2222,16 +2185,7 @@ start_rspamd_proxy (struct rspamd_worker *worker) {
rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
ctx->ev_base, ctx->resolver->r);
- /* XXX: stupid default */
- ctx->keys_cache = rspamd_keypair_cache_new (256);
- ctx->local_key = rspamd_keypair_new (RSPAMD_KEYPAIR_KEX,
- RSPAMD_CRYPTOBOX_MODE_25519);
-
- double_to_tv (ctx->rotate_tm, &rot_tv);
- rot_tv.tv_sec += ottery_rand_range (rot_tv.tv_sec);
- event_set (&ctx->rotate_ev, -1, EV_TIMEOUT, proxy_rotate_key, ctx);
- event_base_set (ctx->ev_base, &ctx->rotate_ev);
- event_add (&ctx->rotate_ev, &rot_tv);
+ ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
if (ctx->has_self_scan) {
/* Additional initialisation needed */
@@ -2264,7 +2218,7 @@ start_rspamd_proxy (struct rspamd_worker *worker) {
rspamd_stat_close ();
}
- rspamd_keypair_cache_destroy (ctx->keys_cache);
+ rspamd_http_context_free (ctx->http_ctx);
REF_RELEASE (ctx->cfg);
rspamd_log_close (worker->srv->logger, TRUE);
diff --git a/src/worker.c b/src/worker.c
index 2f7176a8d..8fa02b51a 100644
--- a/src/worker.c
+++ b/src/worker.c
@@ -414,13 +414,13 @@ accept_socket (gint fd, short what, void *arg)
http_opts = RSPAMD_HTTP_REQUIRE_ENCRYPTION;
}
- task->http_conn = rspamd_http_connection_new (rspamd_worker_body_handler,
+ task->http_conn = rspamd_http_connection_new (
+ ctx->http_ctx,
+ rspamd_worker_body_handler,
rspamd_worker_error_handler,
rspamd_worker_finish_handler,
http_opts,
- RSPAMD_HTTP_SERVER,
- ctx->keys_cache,
- NULL);
+ RSPAMD_HTTP_SERVER);
rspamd_http_connection_set_max_size (task->http_conn, task->cfg->max_message);
worker->nconns++;
rspamd_mempool_add_destructor (task->task_pool,
@@ -696,8 +696,7 @@ start_worker (struct rspamd_worker *worker)
rspamd_upstreams_library_config (worker->srv->cfg, ctx->cfg->ups_ctx,
ctx->ev_base, ctx->resolver->r);
- /* XXX: stupid default */
- ctx->keys_cache = rspamd_keypair_cache_new (256);
+ ctx->http_ctx = rspamd_http_context_create (ctx->cfg, ctx->ev_base);
rspamd_worker_init_scanner (worker, ctx->ev_base, ctx->resolver,
&ctx->lang_det);
rspamd_lua_run_postloads (ctx->cfg->lua_state, ctx->cfg, ctx->ev_base,
@@ -707,7 +706,7 @@ start_worker (struct rspamd_worker *worker)
rspamd_worker_block_signals ();
rspamd_stat_close ();
- rspamd_keypair_cache_destroy (ctx->keys_cache);
+ rspamd_http_context_free (ctx->http_ctx);
REF_RELEASE (ctx->cfg);
rspamd_log_close (worker->srv->logger, TRUE);
diff --git a/src/worker_private.h b/src/worker_private.h
index f07a95b41..398c5d23d 100644
--- a/src/worker_private.h
+++ b/src/worker_private.h
@@ -49,7 +49,7 @@ struct rspamd_worker_ctx {
/* Encryption key */
struct rspamd_cryptobox_keypair *key;
/* Keys cache */
- struct rspamd_keypair_cache *keys_cache;
+ struct rspamd_http_context *http_ctx;
/* Language detector */
struct rspamd_lang_detector *lang_det;
};
More information about the Commits
mailing list