commit fab1414: [Minor] Distinguish max_lua_urls and max_urls

[Vsevolod Stakhov]

Author: Vsevolod Stakhov
Date: 2020-01-27 14:08:16 +0000
URL: https://github.com/rspamd/rspamd/commit/fab141472d7aa115b2fc603b9e6723e2dbebda7a (HEAD -> master)

[Minor] Distinguish max_lua_urls and max_urls

---
 conf/options.inc          | 1 +
 src/libserver/cfg_file.h  | 1 +
 src/libserver/cfg_rcl.c   | 6 ++++++
 src/libserver/cfg_utils.c | 1 +
 src/libserver/url.c       | 4 ++--
 5 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/conf/options.inc b/conf/options.inc
index 78fd3bee6..92d6ad7d7 100644
--- a/conf/options.inc
+++ b/conf/options.inc
@@ -28,6 +28,7 @@ check_all_filters = false;
 # Default settings
 dns_max_requests = 64;
 max_lua_urls = 1024;
+max_urls = 10240;
 
 dns {
     timeout = 1s;
diff --git a/src/libserver/cfg_file.h b/src/libserver/cfg_file.h
index de6f37766..a6d37de00 100644
--- a/src/libserver/cfg_file.h
+++ b/src/libserver/cfg_file.h
@@ -469,6 +469,7 @@ struct rspamd_config {
 	guint lua_gc_pause;                                /**< lua gc pause										*/
 	guint full_gc_iters;                            /**< iterations between full gc cycle					*/
 	guint max_lua_urls;                             /**< maximum number of urls to be passed to Lua			*/
+	guint max_urls;                                 /**< maximum number of urls to be processed in general	*/
 	guint max_blas_threads;                         /**< maximum threads for openblas when learning ANN		*/
 	guint max_opts_len;                             /**< maximum length for all options for a symbol		*/
 
diff --git a/src/libserver/cfg_rcl.c b/src/libserver/cfg_rcl.c
index 817f7efc5..ebca34563 100644
--- a/src/libserver/cfg_rcl.c
+++ b/src/libserver/cfg_rcl.c
@@ -2222,6 +2222,12 @@ rspamd_rcl_config_init (struct rspamd_config *cfg, GHashTable *skip_sections)
 				G_STRUCT_OFFSET (struct rspamd_config, max_lua_urls),
 				RSPAMD_CL_FLAG_INT_32,
 				"Maximum count of URLs to pass to Lua to avoid DoS (default: 1024)");
+		rspamd_rcl_add_default_handler (sub,
+				"max_urls",
+				rspamd_rcl_parse_struct_integer,
+				G_STRUCT_OFFSET (struct rspamd_config, max_urls),
+				RSPAMD_CL_FLAG_INT_32,
+				"Maximum count of URLs to process to avoid DoS (default: 10240)");
 		rspamd_rcl_add_default_handler (sub,
 				"max_blas_threads",
 				rspamd_rcl_parse_struct_integer,
diff --git a/src/libserver/cfg_utils.c b/src/libserver/cfg_utils.c
index 37a3d4578..cd85b73ac 100644
--- a/src/libserver/cfg_utils.c
+++ b/src/libserver/cfg_utils.c
@@ -196,6 +196,7 @@ rspamd_config_new (enum rspamd_config_init_flags flags)
 	cfg->log_error_elt_maxlen = 1000;
 	cfg->cache_reload_time = 30.0;
 	cfg->max_lua_urls = 1024;
+	cfg->max_urls = cfg->max_lua_urls * 10;
 	cfg->max_blas_threads = 1;
 	cfg->max_opts_len = 4096;
 
diff --git a/src/libserver/url.c b/src/libserver/url.c
index de895299b..52af7ca27 100644
--- a/src/libserver/url.c
+++ b/src/libserver/url.c
@@ -3108,8 +3108,8 @@ rspamd_url_text_part_callback (struct rspamd_url *url, gsize start_offset,
 
 	if (target_tbl) {
 		/* Also check max urls */
-		if (cbd->task->cfg && cbd->task->cfg->max_lua_urls > 0) {
-			if (g_hash_table_size (target_tbl) > cbd->task->cfg->max_lua_urls) {
+		if (cbd->task->cfg && cbd->task->cfg->max_urls > 0) {
+			if (g_hash_table_size (target_tbl) > cbd->task->cfg->max_urls) {
 				msg_err_task ("part has too many URLs, we cannot process more: "
 							  "%d urls extracted ",
 						(guint)g_hash_table_size (target_tbl));