commit f6cf156: [Feature] Tune memory management in Rspamd and Lua
Vsevolod Stakhov
vsevolod at highsecure.ru
Fri May 10 11:49:03 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-05-10 12:44:33 +0100
URL: https://github.com/rspamd/rspamd/commit/f6cf15626cadecf9e6f7023f6f72432f22624746 (HEAD -> master)
[Feature] Tune memory management in Rspamd and Lua
---
src/libserver/task.c | 44 ++++++++++++++++++++++++++++++++++++++++----
src/lua/lua_common.c | 7 +++++++
2 files changed, 47 insertions(+), 4 deletions(-)
diff --git a/src/libserver/task.c b/src/libserver/task.c
index b69b96341..6ec349580 100644
--- a/src/libserver/task.c
+++ b/src/libserver/task.c
@@ -30,6 +30,14 @@
#include "libmime/lang_detection.h"
#include "libmime/filter_private.h"
+#ifdef WITH_JEMALLOC
+#include <jemalloc/jemalloc.h>
+#else
+# if defined(__GLIBC__) && defined(_GNU_SOURCE)
+# include <malloc.h>
+# endif
+#endif
+
#include <math.h>
/*
@@ -233,6 +241,8 @@ rspamd_task_free (struct rspamd_task *task)
struct rspamd_mime_text_part *tp;
struct rspamd_email_address *addr;
struct rspamd_lua_cached_entry *entry;
+ static guint free_iters = 0;
+ const guint free_iters_limit = 5000;
GHashTableIter it;
gpointer k, v;
guint i;
@@ -341,6 +351,32 @@ rspamd_task_free (struct rspamd_task *task)
g_hash_table_unref (task->lua_cache);
}
+ if (++free_iters > free_iters_limit) {
+ /* Perform more expensive cleanup cycle */
+ gsize allocated = 0, active = 0, metadata = 0,
+ resident = 0, mapped = 0;
+
+#ifdef WITH_JEMALLOC
+ gsize sz = sizeof (gsize);
+ mallctl ("stats.allocated", &allocated, &sz, NULL, 0);
+ mallctl ("stats.active", &active, &sz, NULL, 0);
+ mallctl ("stats.metadata", &metadata, &sz, NULL, 0);
+ mallctl ("stats.resident", &resident, &sz, NULL, 0);
+ mallctl ("stats.mapped", &mapped, &sz, NULL, 0);
+#else
+# if defined(__GLIBC__) && defined(_GNU_SOURCE)
+ malloc_trim (0);
+# endif
+#endif
+ msg_notice_task ("perform full gc cycle; memory stats: "
+ "%z allocated, %z active, %z metadata, %z rezident, %z mapped;"
+ " lua memory: %d kb",
+ allocated, active, metadata, resident, mapped,
+ lua_gc (task->cfg->lua_state, LUA_GCCOUNT, 0));
+ free_iters = 0;
+ lua_gc (task->cfg->lua_state, LUA_GCCOLLECT, 0);
+ }
+
REF_RELEASE (task->cfg);
}
@@ -369,7 +405,7 @@ rspamd_task_unmapper (gpointer ud)
gboolean
rspamd_task_load_message (struct rspamd_task *task,
- struct rspamd_http_message *msg, const gchar *start, gsize len)
+ struct rspamd_http_message *msg, const gchar *start, gsize len)
{
guint control_len, r;
struct ucl_parser *parser;
@@ -446,7 +482,7 @@ rspamd_task_load_message (struct rspamd_task *task,
if (offset > (gulong)st.st_size) {
msg_err_task ("invalid offset %ul (%ul available) for shm "
- "segment %s", offset, st.st_size, fp);
+ "segment %s", offset, st.st_size, fp);
munmap (map, st.st_size);
close (fd);
@@ -463,7 +499,7 @@ rspamd_task_load_message (struct rspamd_task *task,
if (shmem_size > (gulong)st.st_size) {
msg_err_task ("invalid length %ul (%ul available) for %s "
- "segment %s", shmem_size, st.st_size, ft, fp);
+ "segment %s", shmem_size, st.st_size, ft, fp);
munmap (map, st.st_size);
close (fd);
@@ -655,7 +691,7 @@ rspamd_task_load_message (struct rspamd_task *task,
task->flags |= RSPAMD_TASK_FLAG_COMPRESSED;
msg_info_task ("loaded message from zstd compressed stream; "
- "compressed: %ul; uncompressed: %ul",
+ "compressed: %ul; uncompressed: %ul",
(gulong)zin.size, (gulong)zout.pos);
}
diff --git a/src/lua/lua_common.c b/src/lua/lua_common.c
index 1b3e0121d..1a5cf117f 100644
--- a/src/lua/lua_common.c
+++ b/src/lua/lua_common.c
@@ -898,6 +898,7 @@ rspamd_lua_init (bool wipe_mem)
L = luaL_newstate ();
}
+ lua_gc (L, LUA_GCSTOP, 0);
luaL_openlibs (L);
luaopen_logger (L);
luaopen_mempool (L);
@@ -994,6 +995,12 @@ rspamd_lua_init (bool wipe_mem)
lua_setglobal (L, "get_traces");
#endif
+ /* Set up GC */
+ lua_gc (L, LUA_GCCOLLECT, 0);
+ lua_gc (L, LUA_GCSETSTEPMUL, 50);
+ lua_gc (L, LUA_GCSETPAUSE, 400);
+ lua_gc (L, LUA_GCRESTART, 0);
+
return L;
}
More information about the Commits
mailing list