commit d027dca: [Feature] Query sentinel on master errors

[Vsevolod Stakhov]

Author: Vsevolod Stakhov
Date: 2018-12-05 14:41:01 +0000
URL: https://github.com/rspamd/rspamd/commit/d027dca0f2c45caa9d8a26e476d44f94a92f639e

[Feature] Query sentinel on master errors
Issue: #2665

---
 lualib/lua_redis.lua | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/lualib/lua_redis.lua b/lualib/lua_redis.lua
index 658e26242..a5879e130 100644
--- a/lualib/lua_redis.lua
+++ b/lualib/lua_redis.lua
@@ -35,6 +35,7 @@ local common_schema = ts.shape {
   sentinels = (ts.string + ts.array_of(ts.string)):is_optional(),
   sentinel_watch_time = (ts.number + ts.string / lutil.parse_time_interval):is_optional(),
   sentinel_masters_pattern = ts.string:is_optional(),
+  sentinel_master_maxerrors = (ts.number + ts.string / tonumber):is_optional(),
 }
 
 local config_schema =
@@ -181,6 +182,19 @@ local function redis_query_sentinel(ev_base, params, initialised)
             addr:get_addr():to_string(true), write_servers_str)
         params.write_servers = write_upstreams
         params.write_servers_str = write_servers_str
+
+        local queried = false
+
+        local function monitor_failures(up, _, count)
+          if count > params.sentinel_master_maxerrors and not queried then
+            logger.infox(rspamd_config, 'sentinel: master with address %s, caused %s failures, try to query sentinel',
+                up:get_addr():to_string(true), count)
+            queried = true -- Avoid multiple checks caused by this monitor
+            redis_query_sentinel(ev_base, params, true)
+          end
+        end
+
+        write_upstreams:add_watcher('failure', monitor_failures)
       end
     end
 
@@ -212,6 +226,10 @@ local function add_redis_sentinels(params)
     params.sentinel_watch_time = 60 -- Each minute
   end
 
+  if not params.sentinel_master_maxerrors then
+    params.sentinel_master_maxerrors = 2 -- Maximum number of errors before rechecking
+  end
+
   rspamd_config:add_on_load(function(cfg, ev_base, worker)
     local initialised = false
     if worker:is_scanner() then