commit 38e0750: Merge branch 'master' into arc-defaults

GitHub noreply at github.com
Thu Oct 10 11:49:11 UTC 2019


Author: Vsevolod Stakhov
Date: 2019-10-10 10:09:21 +0100
URL: https://github.com/rspamd/rspamd/commit/38e0750db654f6909c160b0bd205d251852004a7 (refs/pull/3019/head)

Merge branch 'master' into arc-defaults


 .eslintrc.json                                     |     6 +-
 .gitignore                                         |    24 +
 CMakeLists.txt                                     |    12 +-
 centos/rspamd.spec                                 |    10 +-
 conf/composites.conf                               |     8 +-
 conf/{ => maps.d}/dmarc_whitelist.inc              |     0
 conf/{ => maps.d}/maillist.inc                     |     0
 conf/{ => maps.d}/mid.inc                          |     0
 conf/{ => maps.d}/mime_types.inc                   |     0
 conf/{ => maps.d}/redirectors.inc                  |     0
 conf/{ => maps.d}/spf_dkim_whitelist.inc           |     0
 conf/{ => maps.d}/surbl-whitelist.inc              |     0
 conf/modules.d/antivirus.conf                      |    15 +-
 conf/modules.d/arc.conf                            |    18 +-
 conf/modules.d/asn.conf                            |    15 +-
 conf/modules.d/chartable.conf                      |    15 +-
 conf/modules.d/clickhouse.conf                     |    15 +-
 conf/modules.d/dcc.conf                            |    15 +-
 conf/modules.d/dkim.conf                           |    15 +-
 conf/modules.d/dkim_signing.conf                   |    15 +-
 conf/modules.d/dmarc.conf                          |    15 +-
 conf/modules.d/elastic.conf                        |    14 +
 conf/modules.d/emails.conf                         |    45 +-
 conf/modules.d/external_services.conf              |    15 +-
 conf/modules.d/force_actions.conf                  |    15 +-
 conf/modules.d/forged_recipients.conf              |    15 +-
 conf/modules.d/fuzzy_check.conf                    |    15 +-
 conf/modules.d/greylist.conf                       |    16 +-
 conf/modules.d/hfilter.conf                        |    15 +-
 conf/modules.d/history_redis.conf                  |    15 +-
 conf/modules.d/maillist.conf                       |    15 +-
 conf/modules.d/metadata_exporter.conf              |    15 +-
 conf/modules.d/metric_exporter.conf                |    16 +-
 conf/modules.d/mid.conf                            |    18 +-
 conf/modules.d/milter_headers.conf                 |    15 +-
 conf/modules.d/mime_types.conf                     |    18 +-
 conf/modules.d/multimap.conf                       |    17 +-
 conf/modules.d/mx_check.conf                       |    15 +-
 conf/modules.d/neural.conf                         |    15 +-
 conf/modules.d/once_received.conf                  |    15 +-
 conf/modules.d/p0f.conf                            |    45 +
 conf/modules.d/phishing.conf                       |    18 +-
 conf/modules.d/ratelimit.conf                      |    15 +-
 conf/modules.d/rbl.conf                            |   231 +-
 conf/modules.d/redis.conf                          |    13 +-
 conf/modules.d/regexp.conf                         |    15 +-
 conf/modules.d/replies.conf                        |    15 +-
 conf/modules.d/reputation.conf                     |    19 +-
 conf/modules.d/rspamd_update.conf                  |    18 +-
 conf/modules.d/spamassassin.conf                   |    15 +-
 conf/modules.d/spamtrap.conf                       |    15 +-
 conf/modules.d/spf.conf                            |    16 +-
 conf/modules.d/surbl.conf                          |   121 +-
 conf/modules.d/trie.conf                           |    15 +-
 conf/modules.d/url_redirector.conf                 |    15 +-
 conf/modules.d/url_reputation.conf                 |    48 -
 conf/modules.d/url_tags.conf                       |    29 -
 conf/modules.d/whitelist.conf                      |    27 +-
 conf/options.inc                                   |     2 +-
 conf/scores.d/surbl_group.conf                     |    78 +-
 config.h.in                                        |    52 +-
 contrib/DEPENDENCY_INFO.md                         |     2 +-
 contrib/elastic/rspamd_template.json               |   255 +-
 contrib/lc-btrie/btrie.c                           |     5 +-
 contrib/libev/ev.h                                 |     2 +
 contrib/librdns/util.c                             |     2 +
 contrib/libucl/lua_ucl.c                           |     6 +-
 contrib/linenoise/CMakeLists.txt                   |    10 -
 contrib/linenoise/LICENSE                          |    25 -
 contrib/linenoise/linenoise.c                      |  1194 -
 contrib/linenoise/linenoise.h                      |    76 -
 contrib/lua-argparse/argparse.lua                  |   602 +-
 contrib/lua-lpeg/lptree.c                          |     1 +
 contrib/replxx/CMakeLists.txt                      |    82 +
 contrib/replxx/LICENSE.md                          |    63 +
 contrib/replxx/README.md                           |   119 +
 contrib/replxx/include/replxx.h                    |   454 +
 contrib/replxx/include/replxx.hxx                  |   466 +
 contrib/replxx/src/ConvertUTF.cpp                  |   271 +
 contrib/replxx/src/ConvertUTF.h                    |   139 +
 contrib/replxx/src/conversion.cxx                  |   113 +
 contrib/replxx/src/conversion.hxx                  |    20 +
 contrib/replxx/src/escape.cxx                      |   860 +
 contrib/replxx/src/escape.hxx                      |    37 +
 contrib/replxx/src/history.cxx                     |   148 +
 contrib/replxx/src/history.hxx                     |    73 +
 contrib/replxx/src/io.cxx                          |   674 +
 contrib/replxx/src/io.hxx                          |    79 +
 contrib/replxx/src/killring.hxx                    |    76 +
 contrib/replxx/src/prompt.cxx                      |   150 +
 contrib/replxx/src/prompt.hxx                      |    49 +
 contrib/replxx/src/replxx.cxx                      |   521 +
 contrib/replxx/src/replxx_impl.cxx                 |  1984 ++
 contrib/replxx/src/replxx_impl.hxx                 |   243 +
 contrib/replxx/src/unicodestring.hxx               |   179 +
 contrib/replxx/src/utf8string.hxx                  |    71 +
 contrib/replxx/src/util.cxx                        |   152 +
 contrib/replxx/src/util.hxx                        |    21 +
 contrib/replxx/src/wcwidth.cpp                     |   296 +
 contrib/replxx/src/windows.cxx                     |   158 +
 contrib/replxx/src/windows.hxx                     |    44 +
 contrib/t1ha/t1ha_bits.h                           |     4 -
 doc/Makefile                                       |     2 +-
 interface/index.html                               |     8 +-
 interface/js/app/rspamd.js                         |     4 +-
 interface/js/app/upload.js                         |    11 +-
 interface/js/lib/d3evolution.min.js                |     6 +-
 lualib/lua_cfg_transform.lua                       |   130 +-
 lualib/lua_lexer.lua                               |   161 +
 lualib/lua_magic/heuristics.lua                    |   420 +
 lualib/lua_magic/init.lua                          |   364 +
 lualib/lua_magic/patterns.lua                      |   435 +
 lualib/lua_magic/types.lua                         |   279 +
 lualib/lua_maps.lua                                |    29 +-
 lualib/lua_scanners/clamav.lua                     |    22 +-
 lualib/lua_scanners/common.lua                     |   119 +-
 lualib/lua_scanners/dcc.lua                        |   138 +-
 lualib/lua_scanners/fprot.lua                      |    21 +-
 lualib/lua_scanners/icap.lua                       |   520 +-
 lualib/lua_scanners/init.lua                       |     2 +
 lualib/lua_scanners/kaspersky_av.lua               |    21 +-
 lualib/lua_scanners/kaspersky_se.lua               |   280 +
 lualib/lua_scanners/oletools.lua                   |   393 +-
 lualib/lua_scanners/p0f.lua                        |   209 +
 lualib/lua_scanners/savapi.lua                     |    23 +-
 lualib/lua_scanners/sophos.lua                     |    29 +-
 lualib/lua_scanners/spamassassin.lua               |   134 +-
 lualib/lua_scanners/vadesecure.lua                 |   334 +-
 lualib/lua_selectors/extractors.lua                |     1 +
 lualib/lua_util.lua                                |    69 +-
 package.json                                       |     3 +-
 rules/html.lua                                     |     1 -
 rules/regexp/misc.lua                              |     9 +-
 src/CMakeLists.txt                                 |     6 +-
 src/client/rspamc.c                                |     5 +-
 src/controller.c                                   |    40 +-
 src/fuzzy_storage.c                                |    95 +-
 src/hs_helper.c                                    |   100 +-
 src/libcryptobox/cryptobox.c                       |     9 +
 src/libcryptobox/cryptobox.h                       |     1 +
 src/libmime/content_type.c                         |     2 +-
 src/libmime/images.c                               |    91 +-
 src/libmime/images.h                               |     5 +
 src/libmime/message.c                              |   251 +-
 src/libmime/message.h                              |     4 +-
 src/libmime/mime_headers.c                         |    47 +-
 src/libmime/mime_parser.c                          |    29 +-
 src/libserver/cfg_file.h                           |    19 +-
 src/libserver/cfg_rcl.c                            |    64 +-
 src/libserver/cfg_rcl.h                            |     1 +
 src/libserver/cfg_utils.c                          |    58 +-
 src/libserver/dns.c                                |    21 +-
 src/libserver/dns.h                                |     5 +-
 src/libserver/html.c                               |    24 +-
 src/libserver/milter.c                             |     5 -
 src/libserver/monitored.c                          |    12 +-
 src/libserver/protocol.c                           |    14 +-
 src/libserver/re_cache.c                           |   612 +-
 src/libserver/re_cache.h                           |     9 +-
 src/libserver/rspamd_control.c                     |   151 +-
 src/libserver/rspamd_control.h                     |    46 +-
 src/libserver/rspamd_symcache.c                    |    12 +-
 src/libserver/spf.c                                |    58 +-
 src/libserver/task.c                               |    83 +-
 src/libserver/url.c                                |   113 +-
 src/libserver/url.h                                |     7 +-
 src/libserver/worker_util.c                        |   201 +-
 src/libserver/worker_util.h                        |     8 +
 src/libstat/backends/redis_backend.c               |    12 +-
 src/libstat/classifiers/bayes.c                    |    34 +-
 src/libstat/tokenizers/tokenizers.c                |     4 +-
 src/libutil/addr.c                                 |    75 +-
 src/libutil/addr.h                                 |    16 +-
 src/libutil/http_connection.c                      |    13 +-
 src/libutil/http_context.c                         |     8 +-
 src/libutil/libev_helper.c                         |     6 +-
 src/libutil/logger.c                               |   186 +-
 src/libutil/logger.h                               |    10 -
 src/libutil/map.c                                  |     8 +-
 src/libutil/map_helpers.c                          |     4 +-
 src/libutil/mem_pool.c                             |    29 +-
 src/libutil/multipattern.c                         |    95 +-
 src/libutil/multipattern.h                         |     5 +-
 src/libutil/printf.c                               |     1 +
 src/libutil/regexp.c                               |    57 +-
 src/libutil/regexp.h                               |     5 -
 src/libutil/str_util.c                             |    30 +-
 src/libutil/str_util.h                             |     2 +-
 src/libutil/upstream.c                             |   281 +-
 src/libutil/upstream.h                             |     1 +
 src/libutil/util.c                                 |    39 +-
 src/lua/lua_cdb.c                                  |    14 +-
 src/lua/lua_cfg_file.c                             |    42 +-
 src/lua/lua_common.c                               |   126 +-
 src/lua/lua_common.h                               |    34 +-
 src/lua/lua_config.c                               |     7 +-
 src/lua/lua_cryptobox.c                            |    50 +-
 src/lua/lua_dns_resolver.c                         |    27 +-
 src/lua/lua_expression.c                           |    14 +-
 src/lua/lua_http.c                                 |     3 +-
 src/lua/lua_ip.c                                   |    27 +-
 src/lua/lua_logger.c                               |    53 +-
 src/lua/lua_mempool.c                              |    14 +-
 src/lua/lua_mimepart.c                             |    34 +-
 src/lua/lua_redis.c                                |    18 +-
 src/lua/lua_regexp.c                               |    69 +-
 src/lua/lua_rsa.c                                  |    36 +-
 src/lua/lua_sqlite3.c                              |    22 +-
 src/lua/lua_task.c                                 |   117 +-
 src/lua/lua_tcp.c                                  |     6 +-
 src/lua/lua_text.c                                 |   143 +-
 src/lua/lua_trie.c                                 |   135 +-
 src/lua/lua_udp.c                                  |     3 +-
 src/lua/lua_upstream.c                             |    28 +-
 src/lua/lua_url.c                                  |     9 +-
 src/lua/lua_util.c                                 |   116 +-
 src/lua/lua_worker.c                               |   208 +
 src/plugins/chartable.c                            |    41 +-
 src/plugins/fuzzy_check.c                          |     4 +-
 src/plugins/lua/antivirus.lua                      |    10 +
 src/plugins/lua/clickhouse.lua                     |    35 +-
 src/plugins/lua/dmarc.lua                          |    13 +-
 src/plugins/lua/emails.lua                         |   283 +-
 src/plugins/lua/external_services.lua              |     6 +-
 src/plugins/lua/greylist.lua                       |    12 +-
 src/plugins/lua/history_redis.lua                  |     4 +-
 src/plugins/lua/milter_headers.lua                 |    24 +
 src/plugins/lua/mime_types.lua                     |   146 +-
 src/plugins/lua/multimap.lua                       |    46 +-
 src/plugins/lua/p0f.lua                            |   117 +
 src/plugins/lua/ratelimit.lua                      |    44 +-
 src/plugins/lua/rbl.lua                            |   456 +-
 src/plugins/lua/reputation.lua                     |    26 +-
 src/plugins/lua/whitelist.lua                      |    21 +-
 src/plugins/surbl.c                                |  2286 --
 src/ragel/content_disposition.rl                   |     4 +-
 src/ragel/smtp_ip_parser.rl                        |     3 +-
 src/rspamadm/commands.c                            |     5 +-
 src/rspamadm/configdump.c                          |     4 +-
 src/rspamadm/confighelp.c                          |     2 +
 src/rspamadm/configtest.c                          |    17 +-
 src/rspamadm/control.c                             |     6 +-
 src/rspamadm/dkim_keygen.c                         |     2 +
 src/rspamadm/fuzzy_convert.c                       |     3 +
 src/rspamadm/lua_repl.c                            |   275 +-
 src/rspamadm/pw.c                                  |     3 +
 src/rspamadm/rspamadm.c                            |    44 +-
 src/rspamadm/rspamadm.h                            |     5 +-
 src/rspamadm/signtool.c                            |     3 +
 src/rspamadm/stat_convert.c                        |     3 +
 src/rspamd.c                                       |   124 +-
 src/rspamd.h                                       |    12 +-
 src/rspamd_proxy.c                                 |     6 +-
 src/worker.c                                       |     9 +-
 test/functional/cases/102_multimap.robot           |     5 +
 test/functional/cases/123_whitelist.robot          |    23 +
 test/functional/cases/161_p0f.robot                |   107 +
 .../cases/210_clickhouse/001_migration.robot       |     6 +-
 test/functional/cases/220_http.robot               |     2 +-
 test/functional/cases/230_tcp.robot                |     2 +-
 test/functional/cases/231_tcp_down.robot           |     2 +-
 test/functional/cases/240_redis.robot              |     2 +-
 test/functional/cases/241_redis_is_dead.robot      |     2 +-
 test/functional/cases/250_dns.robot                |     2 +-
 test/functional/cases/280_rules.robot              |     5 +
 test/functional/cases/310_udp.robot                |     2 +-
 test/functional/cases/340_surbl.robot              |    76 +
 test/functional/cases/350_magic.robot              |    71 +
 test/functional/configs/maps/domains.list.2        |    14 +
 test/functional/configs/multimap.conf              |     6 +
 test/functional/configs/p0f.conf                   |    11 +
 test/functional/configs/plugins.conf               |    37 +-
 test/functional/configs/rbl.conf                   |    20 +-
 test/functional/configs/surbl.conf                 |    38 +-
 test/functional/configs/whitelist.conf             |    11 +
 test/functional/lib/vars.py                        |     1 +
 test/functional/lua/magic.lua                      |    14 +
 test/functional/messages/badboundary.eml           |    17 +
 ...{good_dkim_rspamd.eml => no_dkim_valid_spf.eml} |   108 +-
 test/functional/messages/emailbltext.eml           |     4 +
 test/functional/messages/emailbltext2.eml          |     3 +
 test/functional/messages/emailbltext3.eml          |     4 +
 test/functional/messages/freemailcc.eml            |    10 +
 test/functional/messages/gargantua.eml             | 23469 +++++++++++++++++++
 test/functional/messages/mailadr.eml               |     4 +
 test/functional/messages/mailadr2.eml              |     4 +
 test/functional/messages/replyto.eml               |     5 +
 test/functional/messages/replyto2.eml              |     5 +
 test/functional/messages/replytosubdomain.eml      |     5 +
 test/functional/messages/urlimage.eml              |     5 +
 test/functional/messages/urlinsubject.eml          |     7 +
 test/functional/messages/urlinsubjectencoded.eml   |     9 +
 test/functional/messages/whitelist.eml             |     3 +
 test/functional/util/dummy_p0f.py                  |    98 +
 test/lua/unit/addr.lua                             |     4 +-
 test/lua/unit/lua_util.extract_specific_urls.lua   |   246 +-
 test/lua/unit/quoted_printable.lua                 |    19 +-
 test/lua/unit/received.lua                         |    12 +
 test/lua/unit/trie.lua                             |    60 +-
 test/rspamd_test_suite.c                           |     1 -
 test/rspamd_upstream_test.c                        |     6 +-
 utils/CMakeLists.txt                               |     1 -
 utils/rspamd_stats.pl                              |     4 +-
 303 files changed, 40859 insertions(+), 7548 deletions(-)

diff --combined conf/modules.d/arc.conf
index a57cec00d,b69ee7303..48b5e59e9
--- a/conf/modules.d/arc.conf
+++ b/conf/modules.d/arc.conf
@@@ -1,14 -1,16 +1,16 @@@
- # You can modify '$LOCAL_CONFDIR/rspamd.conf.local.override' to redefine
- # parameters defined on the top level
+ # Please don't modify this file as your changes might be overwritten with
+ # the next update.
  #
- # You can modify '$LOCAL_CONFDIR/rspamd.conf.local' to add
- # parameters defined on the top level
+ # You can modify 'local.d/arc.conf' to add and merge
+ # parameters defined inside this section
  #
- # For specific modules or configuration you can also modify
- # '$LOCAL_CONFDIR/local.d/file.conf' - to add your options or rewrite defaults
- # '$LOCAL_CONFDIR/override.d/file.conf' - to override the defaults
+ # You can modify 'override.d/arc.conf' to strictly override all
+ # parameters defined inside this section
  #
- # See https://rspamd.com/doc/tutorials/writing_rules.html for details
+ # See https://rspamd.com/doc/faq.html#what-are-the-locald-and-overrided-directories
+ # for details
+ #
+ # Module documentation can be found at  https://rspamd.com/doc/modules/arc.html
  
  
  # To configure this module, please also check the following document:
@@@ -26,27 -28,25 +28,27 @@@ arc 
    # If false, messages with empty envelope from are not signed
    allow_envfrom_empty = true;
    # If true, envelope/header domain mismatch is ignored
 -  allow_hdrfrom_mismatch = false;
 +  allow_hdrfrom_mismatch = true;
    # If true, multiple from headers are allowed (but only first is used)
    allow_hdrfrom_multiple = false;
    # If true, username does not need to contain matching domain
    allow_username_mismatch = false;
    # If false, messages from authenticated users are not selected for signing
 -  auth_only = true;
 +  auth_only = false;
    # Default path to key, can include '$domain' and '$selector' variables
    #path = "${DBDIR}/arc/$domain.$selector.key";
    # Default selector to use
    selector = "arc";
 +  # If false, inbound messages are not selected for signing
 +  sign_inbound = true;
    # If false, messages from local networks are not selected for signing
 -  sign_local = true;
 +  sign_local = false;
    # Symbol to add when message is signed
    symbol_sign = "ARC_SIGNED";
    # Whether to fallback to global config
    try_fallback = true;
 -  # Domain to use for ARC signing: can be "header" or "envelope"
 -  use_domain = "header";
 +  # Domain to use for DKIM signing: can be "header", "envelope" or "recipient"
 +  use_domain = "recipient";
    # Whether to normalise domains to eSLD
    use_esld = true;
    # Whether to get keys from Redis


More information about the Commits mailing list