commit 5bc4370: [Feature] Add preliminary exporter to AWS S3

Vsevolod Stakhov vsevolod at highsecure.ru
Thu Aug 12 17:21:05 UTC 2021


Author: Vsevolod Stakhov
Date: 2021-08-12 17:07:48 +0100
URL: https://github.com/rspamd/rspamd/commit/5bc4370297b964b4b80f3465605eecf317a923fa

[Feature] Add preliminary exporter to AWS S3

---
 conf/modules.d/{asn.conf => aws_s3.conf} |  25 ++++----
 src/plugins/lua/aws_s3.lua               | 105 +++++++++++++++++++++++++++++++
 2 files changed, 116 insertions(+), 14 deletions(-)

diff --git a/conf/modules.d/asn.conf b/conf/modules.d/aws_s3.conf
similarity index 57%
copy from conf/modules.d/asn.conf
copy to conf/modules.d/aws_s3.conf
index 61efd5281..2dd966374 100644
--- a/conf/modules.d/asn.conf
+++ b/conf/modules.d/aws_s3.conf
@@ -12,18 +12,15 @@
 #
 # Module documentation can be found at  https://rspamd.com/doc/modules/asn.html
 
-asn {
-  # Provider: just "rspamd" for now
-  provider_type = "rspamd";
-  # Provider-specific configuration
-  provider_info {
-    ip4 = "asn.rspamd.com";
-    ip6 = "asn6.rspamd.com";
-  }
-  # If defined, insert symbol with lookup results
-  # symbol = "ASN";
-
-  .include(try=true,priority=5) "${DBDIR}/dynamic/asn.conf"
-  .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/asn.conf"
-  .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/asn.conf"
+aws_s3 {
+  # Required attributes
+  #s3_bucket = 'xxx';
+  s3_region = 'us-east-1';
+  #s3_secret_key = 'xxx';
+  #s3_key_id = 'xxx';
+  # Enable in local.d/aws_s3.conf
+  enabled = false;
+  .include(try=true,priority=5) "${DBDIR}/dynamic/aws_s3.conf"
+  .include(try=true,priority=1,duplicate=merge) "$LOCAL_CONFDIR/local.d/aws_s3.conf"
+  .include(try=true,priority=10) "$LOCAL_CONFDIR/override.d/aws_s3.conf"
 }
diff --git a/src/plugins/lua/aws_s3.lua b/src/plugins/lua/aws_s3.lua
new file mode 100644
index 000000000..9ea4a58e2
--- /dev/null
+++ b/src/plugins/lua/aws_s3.lua
@@ -0,0 +1,105 @@
+--[[
+Copyright (c) 2021, Vsevolod Stakhov <vsevolod at highsecure.ru>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+]]--
+
+local N = "aws_s3"
+local lua_util = require "lua_util"
+local lua_aws = require "lua_aws"
+local rspamd_logger = require "rspamd_logger"
+local ts = (require "tableshape").types
+local rspamd_text = require "rspamd_text"
+local rspamd_http = require "rspamd_http"
+
+local settings = {
+  s3_bucket = nil,
+  s3_region = 'us-east-1',
+  s3_secret_key = nil,
+  s3_key_id = nil,
+  s3_timeout = 10,
+}
+
+local settings_schema = ts.shape{
+  s3_bucket = ts.string,
+  s3_region = ts.string,
+  s3_secret_key = ts.string,
+  s3_key_id = ts.string,
+  s3_timeout = ts.number + ts.string / lua_util.parse_time_interval,
+  enabled = ts.boolean:is_optional(),
+}
+
+local function s3_aws_callback(task)
+  local uri = string.format('https://%s.s3.amazonaws.com', settings.s3_bucket)
+  -- Create a nonce
+  local nonce = rspamd_text.randombytes(16):base32()
+  local queue_id = task:get_queue_id()
+
+  if not queue_id then
+    queue_id = rspamd_text.randombytes(8):base32()
+  end
+  local path = string.format('/%s-%s', queue_id, nonce)
+  -- Hack to pass host
+  local aws_host = string.format('%s.s3.amazonaws.com', settings.s3_bucket)
+  local hdrs = lua_aws.aws_request_enrich({
+    region = settings.s3_region,
+    headers = {
+      ['Content-Type'] = 'message/rfc-822',
+      ['Host'] = aws_host
+    },
+    uri = path,
+    key_id = settings.s3_key_id,
+    secret_key = settings.s3_secret_key,
+    method = 'PUT',
+  }, task:get_content())
+
+  local function s3_http_callback(http_err, code, body, headers)
+    rspamd_logger.errx('obj=%s, err=%s, code=%s, body=%s, headers=%s',
+      path, http_err, code, body, headers)
+  end
+
+  rspamd_http.request({
+    url = uri .. path,
+    task = task,
+    method = 'PUT',
+    body = task:get_content(),
+    callback = s3_http_callback,
+    headers = hdrs,
+    timeout = settings.s3_timeout,
+  })
+end
+
+local opts = rspamd_config:get_all_opt('aws_s3')
+if not opts then
+  return
+end
+
+settings = lua_util.override_defaults(settings, opts)
+local res,err = settings_schema:transform(settings)
+
+if not res then
+  rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err)
+
+  return
+end
+
+rspamd_logger.infox(rspamd_config, 'enabled AWS s3 dump to %s', res.s3_bucket)
+
+settings = res
+rspamd_config:register_symbol({
+  name = 'EXPORT_AWS_S3',
+  type = 'idempotent',
+  callback = s3_aws_callback,
+  priority = 10,
+  flags = 'empty,explicit_disable,ignore_passthrough,nostat',
+})
\ No newline at end of file


More information about the Commits mailing list