commit cfc7ee6: Merge pull request #3153 from resec/smembers

[GitHub]

Author: Vsevolod Stakhov
Date: 2020-01-16 12:30:16 +0000
URL: https://github.com/rspamd/rspamd/commit/cfc7ee6d6824f83e63e79642e6bad6e5ee868c95 (HEAD -> master)

Merge pull request #3153 from resec/smembers
[Minor] Eliminate redis SMEMBERS usage

 src/libstat/backends/redis_backend.c | 76 ++++++++++++++++++++++--------------
 1 file changed, 47 insertions(+), 29 deletions(-)

diff --combined src/libstat/backends/redis_backend.c
index 2a2636f5f,ec65a133f..bc245bf0b
--- a/src/libstat/backends/redis_backend.c
+++ b/src/libstat/backends/redis_backend.c
@@@ -846,9 -846,10 +846,10 @@@ rspamd_redis_stat_keys (redisAsyncConte
  {
  	struct rspamd_redis_stat_elt *redis_elt = (struct rspamd_redis_stat_elt *)priv;
  	struct rspamd_redis_stat_cbdata *cbdata;
- 	redisReply *reply = r, *elt;
+ 	redisReply *reply = r, *more_elt, *elts, *elt;
  	gchar **pk, *k;
  	guint i, processed = 0;
+ 	gboolean more = false;
  
  	cbdata = redis_elt->cbdata;
  
@@@ -860,10 -861,17 +861,17 @@@
  
  	if (c->err == 0 && r != NULL) {
  		if (reply->type == REDIS_REPLY_ARRAY) {
- 			g_ptr_array_set_size (cbdata->cur_keys, reply->elements);
+ 			more_elt = reply->element[0];
+ 			elts = reply->element[1];
  
- 			for (i = 0; i < reply->elements; i ++) {
- 				elt = reply->element[i];
+ 			if (more_elt != NULL && more_elt->str != NULL && strcmp (more_elt->str, "0") != 0) {
+ 				more = true;
+ 			}
+ 
+ 			g_ptr_array_set_size (cbdata->cur_keys, elts->elements);
+ 
+ 			for (i = 0; i < elts->elements; i ++) {
+ 				elt = elts->element[i];
  
  				if (elt->type == REDIS_REPLY_STRING) {
  					pk = (gchar **)&g_ptr_array_index (cbdata->cur_keys, i);
@@@ -912,30 -920,40 +920,40 @@@
  			}
  		}
  
- 		/* Set up the required keys */
- 		ucl_object_insert_key (cbdata->cur,
- 				ucl_object_typed_new (UCL_INT), "revision", 0, false);
- 		ucl_object_insert_key (cbdata->cur,
- 				ucl_object_typed_new (UCL_INT), "used", 0, false);
- 		ucl_object_insert_key (cbdata->cur,
- 				ucl_object_typed_new (UCL_INT), "total", 0, false);
- 		ucl_object_insert_key (cbdata->cur,
- 				ucl_object_typed_new (UCL_INT), "size", 0, false);
- 		ucl_object_insert_key (cbdata->cur,
- 				ucl_object_fromstring (cbdata->elt->ctx->stcf->symbol),
- 				"symbol", 0, false);
- 		ucl_object_insert_key (cbdata->cur, ucl_object_fromstring ("redis"),
- 				"type", 0, false);
- 		ucl_object_insert_key (cbdata->cur, ucl_object_fromint (0),
- 				"languages", 0, false);
- 		ucl_object_insert_key (cbdata->cur, ucl_object_fromint (processed),
- 				"users", 0, false);
- 
- 		rspamd_upstream_ok (cbdata->selected);
- 
- 		if (cbdata->inflight == 0) {
- 			rspamd_redis_async_cbdata_cleanup (cbdata);
- 			redis_elt->cbdata = NULL;
+ 		if (more) {
+ 			/* Get more stat keys */
+ 			redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, redis_elt,
+ 					"SSCAN %s_keys %s COUNT 1000",
+ 					cbdata->elt->ctx->stcf->symbol, more_elt->str);
+ 
+ 			cbdata->inflight += 1;
+ 		}
+ 		else {
+ 			/* Set up the required keys */
+ 			ucl_object_insert_key (cbdata->cur,
+ 					ucl_object_typed_new (UCL_INT), "revision", 0, false);
+ 			ucl_object_insert_key (cbdata->cur,
+ 					ucl_object_typed_new (UCL_INT), "used", 0, false);
+ 			ucl_object_insert_key (cbdata->cur,
+ 					ucl_object_typed_new (UCL_INT), "total", 0, false);
+ 			ucl_object_insert_key (cbdata->cur,
+ 					ucl_object_typed_new (UCL_INT), "size", 0, false);
+ 			ucl_object_insert_key (cbdata->cur,
+ 					ucl_object_fromstring (cbdata->elt->ctx->stcf->symbol),
+ 					"symbol", 0, false);
+ 			ucl_object_insert_key (cbdata->cur, ucl_object_fromstring ("redis"),
+ 					"type", 0, false);
+ 			ucl_object_insert_key (cbdata->cur, ucl_object_fromint (0),
+ 					"languages", 0, false);
+ 			ucl_object_insert_key (cbdata->cur, ucl_object_fromint (processed),
+ 					"users", 0, false);
+ 
+ 			rspamd_upstream_ok (cbdata->selected);
+ 
+ 			if (cbdata->inflight == 0) {
+ 				rspamd_redis_async_cbdata_cleanup (cbdata);
+ 				redis_elt->cbdata = NULL;
+ 			}
  		}
  	}
  	else {
@@@ -946,7 -964,7 +964,7 @@@
  			msg_err ("cannot get keys to gather stat: unknown error");
  		}
  
 -		rspamd_upstream_fail (cbdata->selected, FALSE);
 +		rspamd_upstream_fail (cbdata->selected, FALSE, c->errstr);
  		rspamd_redis_async_cbdata_cleanup (cbdata);
  		redis_elt->cbdata = NULL;
  	}
@@@ -1029,7 -1047,7 +1047,7 @@@ rspamd_redis_async_stat_cb (struct rspa
  	/* Get keys in redis that match our symbol */
  	rspamd_redis_maybe_auth (ctx, cbdata->redis);
  	redisAsyncCommand (cbdata->redis, rspamd_redis_stat_keys, redis_elt,
- 			"SMEMBERS %s_keys",
+ 			"SSCAN %s_keys 0 COUNT 1000",
  			ctx->stcf->symbol);
  }
  
@@@ -1106,7 -1124,7 +1124,7 @@@ rspamd_redis_timeout (EV_P_ ev_timer *w
  	msg_err_task_check ("connection to redis server %s timed out",
  			rspamd_upstream_name (rt->selected));
  
 -	rspamd_upstream_fail (rt->selected, FALSE);
 +	rspamd_upstream_fail (rt->selected, FALSE, "timeout");
  
  	if (rt->redis) {
  		redis = rt->redis;
@@@ -1207,7 -1225,7 +1225,7 @@@ rspamd_redis_processed (redisAsyncConte
  				rspamd_upstream_name (rt->selected), c->errstr);
  
  		if (rt->redis) {
 -			rspamd_upstream_fail (rt->selected, FALSE);
 +			rspamd_upstream_fail (rt->selected, FALSE, c->errstr);
  		}
  
  		if (!rt->err) {
@@@ -1335,7 -1353,7 +1353,7 @@@ rspamd_redis_connected (redisAsyncConte
  	else if (rt->has_event) {
  		msg_err_task ("error getting reply from redis server %s: %s",
  				rspamd_upstream_name (rt->selected), c->errstr);
 -		rspamd_upstream_fail (rt->selected, FALSE);
 +		rspamd_upstream_fail (rt->selected, FALSE,  c->errstr);
  
  		if (!rt->err) {
  			g_set_error (&rt->err, rspamd_redis_stat_quark (), c->err,
@@@ -1366,7 -1384,7 +1384,7 @@@ rspamd_redis_learned (redisAsyncContex
  				rspamd_upstream_name (rt->selected), c->errstr);
  
  		if (rt->redis) {
 -			rspamd_upstream_fail (rt->selected, FALSE);
 +			rspamd_upstream_fail (rt->selected, FALSE, c->errstr);
  		}
  
  		if (!rt->err) {