commit a841d41: [Rework] Finish http code split and cleanup
Vsevolod Stakhov
vsevolod at highsecure.ru
Fri Feb 15 18:28:07 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-02-15 18:23:40 +0000
URL: https://github.com/rspamd/rspamd/commit/a841d419c9d230681a46900cac8629b576b2dfe8 (HEAD -> master)
[Rework] Finish http code split and cleanup
---
contrib/http-parser/http_parser.c | 3 +-
contrib/http-parser/http_parser.h | 4 +-
src/controller.c | 1 +
src/fuzzy_storage.c | 4 +-
src/libserver/url.c | 1 +
src/libserver/worker_util.c | 1 +
src/libutil/CMakeLists.txt | 3 +
src/libutil/http_connection.c | 1272 +------------------------------------
src/libutil/http_connection.h | 338 +---------
src/libutil/http_message.c | 465 ++++++++++++++
src/libutil/http_message.h | 221 +++++++
src/libutil/http_private.h | 9 +-
src/libutil/http_router.c | 564 ++++++++++++++++
src/libutil/http_router.h | 139 ++++
src/libutil/http_util.c | 247 +++++++
src/libutil/http_util.h | 48 ++
src/plugins/fuzzy_check.c | 1 +
src/rspamadm/lua_repl.c | 1 +
18 files changed, 1738 insertions(+), 1584 deletions(-)
diff --git a/contrib/http-parser/http_parser.c b/contrib/http-parser/http_parser.c
index 0e75d964b..c14ecc034 100644
--- a/contrib/http-parser/http_parser.c
+++ b/contrib/http-parser/http_parser.c
@@ -122,6 +122,7 @@ do { \
#define KEEP_ALIVE "keep-alive"
#define CLOSE "close"
+enum rspamd_http_message_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
static const char *method_strings[] =
{
@@ -1981,7 +1982,7 @@ http_method_str (enum http_method m)
void
-http_parser_init (http_parser *parser, enum http_parser_type t)
+http_parser_init (http_parser *parser, int t)
{
void *data = parser->data; /* preserve application data */
memset(parser, 0, sizeof(*parser));
diff --git a/contrib/http-parser/http_parser.h b/contrib/http-parser/http_parser.h
index e2a0b4985..7b4ac497c 100644
--- a/contrib/http-parser/http_parser.h
+++ b/contrib/http-parser/http_parser.h
@@ -124,8 +124,6 @@ enum http_method
};
-enum http_parser_type { HTTP_REQUEST, HTTP_RESPONSE, HTTP_BOTH };
-
/* Flag values for http_parser.flags field */
enum flags
@@ -280,7 +278,7 @@ struct http_parser_url {
*/
unsigned long http_parser_version(void);
-void http_parser_init(http_parser *parser, enum http_parser_type type);
+void http_parser_init(http_parser *parser, int type);
size_t http_parser_execute(http_parser *parser,
diff --git a/src/controller.c b/src/controller.c
index 6a839b4df..b19ac3db7 100644
--- a/src/controller.c
+++ b/src/controller.c
@@ -21,6 +21,7 @@
#include "libutil/map_helpers.h"
#include "libutil/map_private.h"
#include "libutil/http_private.h"
+#include "libutil/http_router.h"
#include "libstat/stat_api.h"
#include "rspamd.h"
#include "libserver/worker_util.h"
diff --git a/src/fuzzy_storage.c b/src/fuzzy_storage.c
index 36b41113d..bbd6b36ca 100644
--- a/src/fuzzy_storage.c
+++ b/src/fuzzy_storage.c
@@ -34,10 +34,10 @@
#include "libcryptobox/keypairs_cache.h"
#include "libcryptobox/keypair.h"
#include "libserver/rspamd_control.h"
-#include "libutil/map_private.h"
#include "libutil/hash.h"
+#include "libutil/map_private.h"
#include "libutil/http_private.h"
-#include "libutil/hash.h"
+#include "libutil/http_router.h"
#include "unix-std.h"
#include <math.h>
diff --git a/src/libserver/url.c b/src/libserver/url.c
index 0effe4d6b..461c232af 100644
--- a/src/libserver/url.c
+++ b/src/libserver/url.c
@@ -46,6 +46,7 @@
#include "message.h"
#include "multipattern.h"
#include "contrib/uthash/utlist.h"
+#include "contrib/http-parser/http_parser.h"
#include <unicode/utf8.h>
#include <unicode/uchar.h>
diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c
index a0e511929..e10e25bc0 100644
--- a/src/libserver/worker_util.c
+++ b/src/libserver/worker_util.c
@@ -24,6 +24,7 @@
#include "libutil/map.h"
#include "libutil/map_private.h"
#include "libutil/http_private.h"
+#include "libutil/http_router.h"
#ifdef WITH_GPERF_TOOLS
#include <gperftools/profiler.h>
diff --git a/src/libutil/CMakeLists.txt b/src/libutil/CMakeLists.txt
index aef2ed268..4d4f8c7bd 100644
--- a/src/libutil/CMakeLists.txt
+++ b/src/libutil/CMakeLists.txt
@@ -6,7 +6,10 @@ SET(LIBRSPAMDUTILSRC
${CMAKE_CURRENT_SOURCE_DIR}/expression.c
${CMAKE_CURRENT_SOURCE_DIR}/fstring.c
${CMAKE_CURRENT_SOURCE_DIR}/hash.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http_util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http_message.c
${CMAKE_CURRENT_SOURCE_DIR}/http_connection.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http_router.c
${CMAKE_CURRENT_SOURCE_DIR}/logger.c
${CMAKE_CURRENT_SOURCE_DIR}/map.c
${CMAKE_CURRENT_SOURCE_DIR}/map_helpers.c
diff --git a/src/libutil/http_connection.c b/src/libutil/http_connection.c
index a82fc24f7..df283f752 100644
--- a/src/libutil/http_connection.c
+++ b/src/libutil/http_connection.c
@@ -14,8 +14,9 @@
* limitations under the License.
*/
#include "config.h"
-#include "../../contrib/mumhash/mum.h"
+#include "http_connection.h"
#include "http_private.h"
+#include "http_message.h"
#include "utlist.h"
#include "util.h"
#include "printf.h"
@@ -24,11 +25,13 @@
#include "ottery.h"
#include "keypair_private.h"
#include "cryptobox.h"
-#include "unix-std.h"
#include "libutil/ssl_util.h"
-#include "libutil/regexp.h"
#include "libserver/url.h"
+#include "contrib/mumhash/mum.h"
+#include "contrib/http-parser/http_parser.h"
+#include "unix-std.h"
+
#include <openssl/err.h>
#define ENCRYPTED_VERSION " HTTP/1.0"
@@ -71,30 +74,6 @@ struct rspamd_http_connection_private {
gsize wr_total;
};
-enum http_magic_type {
- HTTP_MAGIC_PLAIN = 0,
- HTTP_MAGIC_HTML,
- HTTP_MAGIC_CSS,
- HTTP_MAGIC_JS,
- HTTP_MAGIC_PNG,
- HTTP_MAGIC_JPG
-};
-
-static const struct _rspamd_http_magic {
- const gchar *ext;
- const gchar *ct;
-} http_file_types[] = {
- [HTTP_MAGIC_PLAIN] = { "txt", "text/plain" },
- [HTTP_MAGIC_HTML] = { "html", "text/html" },
- [HTTP_MAGIC_CSS] = { "css", "text/css" },
- [HTTP_MAGIC_JS] = { "js", "application/javascript" },
- [HTTP_MAGIC_PNG] = { "png", "image/png" },
- [HTTP_MAGIC_JPG] = { "jpg", "image/jpeg" },
-};
-
-static const gchar *http_week[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" };
-static const gchar *http_month[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun",
- "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
static const rspamd_ftok_t key_header = {
.begin = "Key",
.len = 3
@@ -108,9 +87,7 @@ static const rspamd_ftok_t last_modified_header = {
.len = 13
};
-static void rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg);
-static gboolean rspamd_http_message_grow_body (struct rspamd_http_message *msg,
- gsize len);
+
#define HTTP_ERROR http_error_quark ()
GQuark
@@ -1767,18 +1744,14 @@ rspamd_http_message_write_header (const gchar* mime_type, gboolean encrypted,
{
gchar datebuf[64];
gint meth_len = 0;
- struct tm t;
if (conn->type == RSPAMD_HTTP_SERVER) {
/* Format reply */
if (msg->method < HTTP_SYMBOLS) {
rspamd_ftok_t status;
- rspamd_gmtime (msg->date, &t);
- rspamd_snprintf (datebuf, sizeof(datebuf),
- "%s, %02d %s %4d %02d:%02d:%02d GMT", http_week[t.tm_wday],
- t.tm_mday, http_month[t.tm_mon], t.tm_year + 1900,
- t.tm_hour, t.tm_min, t.tm_sec);
+ rspamd_http_date_format (datebuf, sizeof (datebuf), msg->date);
+
if (mime_type == NULL) {
mime_type =
encrypted ? "application/octet-stream" : "text/plain";
@@ -2381,449 +2354,6 @@ rspamd_http_connection_write_message_shared (struct rspamd_http_connection *conn
ud, fd, timeout, base, TRUE);
}
-struct rspamd_http_message *
-rspamd_http_new_message (enum http_parser_type type)
-{
- struct rspamd_http_message *new;
-
- new = g_malloc0 (sizeof (struct rspamd_http_message));
-
- if (type == HTTP_REQUEST) {
- new->url = rspamd_fstring_new ();
- }
- else {
- new->url = NULL;
- new->code = 200;
- }
-
- new->port = 80;
- new->type = type;
- new->method = HTTP_INVALID;
-
- REF_INIT_RETAIN (new, rspamd_http_message_free);
-
- return new;
-}
-
-struct rspamd_http_message*
-rspamd_http_message_from_url (const gchar *url)
-{
- struct http_parser_url pu;
- struct rspamd_http_message *msg;
- const gchar *host, *path;
- size_t pathlen, urllen;
- guint flags = 0;
-
- if (url == NULL) {
- return NULL;
- }
-
- urllen = strlen (url);
- memset (&pu, 0, sizeof (pu));
-
- if (http_parser_parse_url (url, urllen, FALSE, &pu) != 0) {
- msg_warn ("cannot parse URL: %s", url);
- return NULL;
- }
-
- if ((pu.field_set & (1 << UF_HOST)) == 0) {
- msg_warn ("no host argument in URL: %s", url);
- return NULL;
- }
-
- if ((pu.field_set & (1 << UF_SCHEMA))) {
- if (pu.field_data[UF_SCHEMA].len == sizeof ("https") - 1 &&
- memcmp (url + pu.field_data[UF_SCHEMA].off, "https", 5) == 0) {
- flags |= RSPAMD_HTTP_FLAG_SSL;
- }
- }
-
- if ((pu.field_set & (1 << UF_PATH)) == 0) {
- path = "/";
- pathlen = 1;
- }
- else {
- path = url + pu.field_data[UF_PATH].off;
- pathlen = urllen - pu.field_data[UF_PATH].off;
- }
-
- msg = rspamd_http_new_message (HTTP_REQUEST);
- host = url + pu.field_data[UF_HOST].off;
- msg->flags = flags;
-
- if ((pu.field_set & (1 << UF_PORT)) != 0) {
- msg->port = pu.port;
- }
- else {
- /* XXX: magic constant */
- if (flags & RSPAMD_HTTP_FLAG_SSL) {
- msg->port = 443;
- }
- else {
- msg->port = 80;
- }
- }
-
- msg->host = rspamd_fstring_new_init (host, pu.field_data[UF_HOST].len);
- msg->url = rspamd_fstring_append (msg->url, path, pathlen);
-
- REF_INIT_RETAIN (msg, rspamd_http_message_free);
-
- return msg;
-}
-
-const gchar *
-rspamd_http_message_get_body (struct rspamd_http_message *msg,
- gsize *blen)
-{
- const gchar *ret = NULL;
-
- if (msg->body_buf.len > 0) {
- ret = msg->body_buf.begin;
- }
-
- if (blen) {
- *blen = msg->body_buf.len;
- }
-
- return ret;
-}
-
-static void
-rspamd_http_shname_dtor (void *p)
-{
- struct rspamd_storage_shmem *n = p;
-
-#ifdef HAVE_SANE_SHMEM
- shm_unlink (n->shm_name);
-#else
- unlink (n->shm_name);
-#endif
- g_free (n->shm_name);
- g_free (n);
-}
-
-struct rspamd_storage_shmem *
-rspamd_http_message_shmem_ref (struct rspamd_http_message *msg)
-{
- if ((msg->flags & RSPAMD_HTTP_FLAG_SHMEM) && msg->body_buf.c.shared.name) {
- REF_RETAIN (msg->body_buf.c.shared.name);
- return msg->body_buf.c.shared.name;
- }
-
- return NULL;
-}
-
-guint
-rspamd_http_message_get_flags (struct rspamd_http_message *msg)
-{
- return msg->flags;
-}
-
-void
-rspamd_http_message_shmem_unref (struct rspamd_storage_shmem *p)
-{
- REF_RELEASE (p);
-}
-
-gboolean
-rspamd_http_message_set_body (struct rspamd_http_message *msg,
- const gchar *data, gsize len)
-{
- union _rspamd_storage_u *storage;
- storage = &msg->body_buf.c;
-
- rspamd_http_message_storage_cleanup (msg);
-
- if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
- storage->shared.name = g_malloc (sizeof (*storage->shared.name));
- REF_INIT_RETAIN (storage->shared.name, rspamd_http_shname_dtor);
-#ifdef HAVE_SANE_SHMEM
-#if defined(__DragonFly__)
- // DragonFly uses regular files for shm. User rspamd is not allowed to create
- // files in the root.
- storage->shared.name->shm_name = g_strdup ("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX");
-#else
- storage->shared.name->shm_name = g_strdup ("/rhm.XXXXXXXXXXXXXXXXXXXX");
-#endif
- storage->shared.shm_fd = rspamd_shmem_mkstemp (storage->shared.name->shm_name);
-#else
- /* XXX: assume that tempdir is /tmp */
- storage->shared.name->shm_name = g_strdup ("/tmp/rhm.XXXXXXXXXXXXXXXXXXXX");
- storage->shared.shm_fd = mkstemp (storage->shared.name->shm_name);
-#endif
-
- if (storage->shared.shm_fd == -1) {
- return FALSE;
- }
-
- if (len != 0 && len != ULLONG_MAX) {
- if (ftruncate (storage->shared.shm_fd, len) == -1) {
- return FALSE;
- }
-
- msg->body_buf.str = mmap (NULL, len,
- PROT_WRITE|PROT_READ, MAP_SHARED,
- storage->shared.shm_fd, 0);
-
- if (msg->body_buf.str == MAP_FAILED) {
- return FALSE;
- }
-
- msg->body_buf.begin = msg->body_buf.str;
- msg->body_buf.allocated_len = len;
-
- if (data != NULL) {
- memcpy (msg->body_buf.str, data, len);
- msg->body_buf.len = len;
- }
- }
- else {
- msg->body_buf.len = 0;
- msg->body_buf.begin = NULL;
- msg->body_buf.str = NULL;
- msg->body_buf.allocated_len = 0;
- }
- }
- else {
- if (len != 0 && len != ULLONG_MAX) {
- if (data == NULL) {
- storage->normal = rspamd_fstring_sized_new (len);
- msg->body_buf.len = 0;
- }
- else {
- storage->normal = rspamd_fstring_new_init (data, len);
- msg->body_buf.len = len;
- }
- }
- else {
- storage->normal = rspamd_fstring_new ();
- }
-
- msg->body_buf.begin = storage->normal->str;
- msg->body_buf.str = storage->normal->str;
- msg->body_buf.allocated_len = storage->normal->allocated;
- }
-
- msg->flags |= RSPAMD_HTTP_FLAG_HAS_BODY;
-
- return TRUE;
-}
-
-void
-rspamd_http_message_set_method (struct rspamd_http_message *msg,
- const gchar *method)
-{
- gint i;
-
- /* Linear search: not very efficient method */
- for (i = 0; i < HTTP_METHOD_MAX; i ++) {
- if (g_ascii_strcasecmp (method, http_method_str (i)) == 0) {
- msg->method = i;
- }
- }
-}
-
-gboolean
-rspamd_http_message_set_body_from_fd (struct rspamd_http_message *msg,
- gint fd)
-{
- union _rspamd_storage_u *storage;
- struct stat st;
-
- rspamd_http_message_storage_cleanup (msg);
-
- storage = &msg->body_buf.c;
- msg->flags |= RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE;
-
- storage->shared.shm_fd = dup (fd);
- msg->body_buf.str = MAP_FAILED;
-
- if (storage->shared.shm_fd == -1) {
- return FALSE;
- }
-
- if (fstat (storage->shared.shm_fd, &st) == -1) {
- return FALSE;
- }
-
- msg->body_buf.str = mmap (NULL, st.st_size,
- PROT_READ, MAP_SHARED,
- storage->shared.shm_fd, 0);
-
- if (msg->body_buf.str == MAP_FAILED) {
- return FALSE;
- }
-
- msg->body_buf.begin = msg->body_buf.str;
- msg->body_buf.len = st.st_size;
- msg->body_buf.allocated_len = st.st_size;
-
- return TRUE;
-}
-
-gboolean
-rspamd_http_message_set_body_from_fstring_steal (struct rspamd_http_message *msg,
- rspamd_fstring_t *fstr)
-{
- union _rspamd_storage_u *storage;
-
- rspamd_http_message_storage_cleanup (msg);
-
- storage = &msg->body_buf.c;
- msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE);
-
- storage->normal = fstr;
- msg->body_buf.str = fstr->str;
- msg->body_buf.begin = msg->body_buf.str;
- msg->body_buf.len = fstr->len;
- msg->body_buf.allocated_len = fstr->allocated;
-
- return TRUE;
-}
-
-gboolean
-rspamd_http_message_set_body_from_fstring_copy (struct rspamd_http_message *msg,
- const rspamd_fstring_t *fstr)
-{
- union _rspamd_storage_u *storage;
-
- rspamd_http_message_storage_cleanup (msg);
-
- storage = &msg->body_buf.c;
- msg->flags &= ~(RSPAMD_HTTP_FLAG_SHMEM|RSPAMD_HTTP_FLAG_SHMEM_IMMUTABLE);
-
- storage->normal = rspamd_fstring_new_init (fstr->str, fstr->len);
- msg->body_buf.str = storage->normal->str;
- msg->body_buf.begin = msg->body_buf.str;
- msg->body_buf.len = storage->normal->len;
- msg->body_buf.allocated_len = storage->normal->allocated;
-
- return TRUE;
-}
-
-
-static gboolean
-rspamd_http_message_grow_body (struct rspamd_http_message *msg, gsize len)
-{
- struct stat st;
- union _rspamd_storage_u *storage;
- gsize newlen;
-
- storage = &msg->body_buf.c;
-
- if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
- if (storage->shared.shm_fd == -1) {
- return FALSE;
- }
-
- if (fstat (storage->shared.shm_fd, &st) == -1) {
- return FALSE;
- }
-
- /* Check if we need to grow */
- if ((gsize)st.st_size < msg->body_buf.len + len) {
- /* Need to grow */
- newlen = rspamd_fstring_suggest_size (msg->body_buf.len, st.st_size,
- len);
- /* Unmap as we need another size of segment */
- if (msg->body_buf.str != MAP_FAILED) {
- munmap (msg->body_buf.str, st.st_size);
- }
-
- if (ftruncate (storage->shared.shm_fd, newlen) == -1) {
- return FALSE;
- }
-
- msg->body_buf.str = mmap (NULL, newlen,
- PROT_WRITE|PROT_READ, MAP_SHARED,
- storage->shared.shm_fd, 0);
- if (msg->body_buf.str == MAP_FAILED) {
- return FALSE;
- }
-
- msg->body_buf.begin = msg->body_buf.str;
- msg->body_buf.allocated_len = newlen;
- }
- }
- else {
- storage->normal = rspamd_fstring_grow (storage->normal, len);
-
- /* Append might cause realloc */
- msg->body_buf.begin = storage->normal->str;
- msg->body_buf.len = storage->normal->len;
- msg->body_buf.str = storage->normal->str;
- msg->body_buf.allocated_len = storage->normal->allocated;
- }
-
- return TRUE;
-}
-
-gboolean
-rspamd_http_message_append_body (struct rspamd_http_message *msg,
- const gchar *data, gsize len)
-{
- union _rspamd_storage_u *storage;
-
- storage = &msg->body_buf.c;
-
- if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
- if (!rspamd_http_message_grow_body (msg, len)) {
- return FALSE;
- }
-
- memcpy (msg->body_buf.str + msg->body_buf.len, data, len);
- msg->body_buf.len += len;
- }
- else {
- storage->normal = rspamd_fstring_append (storage->normal, data, len);
-
- /* Append might cause realloc */
- msg->body_buf.begin = storage->normal->str;
- msg->body_buf.len = storage->normal->len;
- msg->body_buf.str = storage->normal->str;
- msg->body_buf.allocated_len = storage->normal->allocated;
- }
-
- return TRUE;
-}
-
-static void
-rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg)
-{
- union _rspamd_storage_u *storage;
- struct stat st;
-
- if (msg->flags & RSPAMD_HTTP_FLAG_SHMEM) {
- storage = &msg->body_buf.c;
-
- if (storage->shared.shm_fd > 0) {
- g_assert (fstat (storage->shared.shm_fd, &st) != -1);
-
- if (msg->body_buf.str != MAP_FAILED) {
- munmap (msg->body_buf.str, st.st_size);
- }
-
- close (storage->shared.shm_fd);
- }
-
- if (storage->shared.name != NULL) {
- REF_RELEASE (storage->shared.name);
- }
-
- storage->shared.shm_fd = -1;
- msg->body_buf.str = MAP_FAILED;
- }
- else {
- if (msg->body_buf.c.normal) {
- rspamd_fstring_free (msg->body_buf.c.normal);
- }
-
- msg->body_buf.c.normal = NULL;
- }
-
- msg->body_buf.len = 0;
-}
void
rspamd_http_connection_set_max_size (struct rspamd_http_connection *conn,
@@ -3027,557 +2557,38 @@ rspamd_http_message_remove_header (struct rspamd_http_message *msg,
return res;
}
-/*
- * HTTP router functions
- */
-
-static void
-rspamd_http_entry_free (struct rspamd_http_connection_entry *entry)
+void
+rspamd_http_connection_set_key (struct rspamd_http_connection *conn,
+ struct rspamd_cryptobox_keypair *key)
{
- if (entry != NULL) {
- close (entry->conn->fd);
- rspamd_http_connection_unref (entry->conn);
- if (entry->rt->finish_handler) {
*** OUTPUT TRUNCATED, 2956 LINES SKIPPED ***
More information about the Commits
mailing list