commit 81bc945: [Rework] Start rework of the HTTP library
Vsevolod Stakhov
vsevolod at highsecure.ru
Fri Feb 15 18:28:04 UTC 2019
Author: Vsevolod Stakhov
Date: 2019-02-15 16:41:24 +0000
URL: https://github.com/rspamd/rspamd/commit/81bc945a76963cfbc36ce286fee5540f3eb134b3
[Rework] Start rework of the HTTP library
---
src/client/rspamc.c | 2 +-
src/client/rspamdclient.c | 2 +-
src/libserver/milter.c | 2 +-
src/libserver/protocol.h | 2 +-
src/libserver/rspamd_control.c | 2 +-
src/libserver/task.h | 2 +-
src/libserver/worker_util.h | 2 +-
src/libutil/CMakeLists.txt | 48 +-
src/libutil/http.c | 3971 ----------------------------------------
src/libutil/http.h | 578 ------
src/libutil/http_private.h | 2 +-
src/libutil/map.c | 2 +-
src/rspamadm/control.c | 2 +-
src/rspamadm/lua_repl.c | 2 +-
src/rspamd.h | 2 +-
src/rspamd_proxy.c | 2 +-
test/rspamd_http_test.c | 2 +-
17 files changed, 38 insertions(+), 4587 deletions(-)
diff --git a/src/client/rspamc.c b/src/client/rspamc.c
index 3433ef7d6..588496d46 100644
--- a/src/client/rspamc.c
+++ b/src/client/rspamc.c
@@ -15,7 +15,7 @@
*/
#include "config.h"
#include "libutil/util.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
#include "rspamdclient.h"
#include "utlist.h"
diff --git a/src/client/rspamdclient.c b/src/client/rspamdclient.c
index a4a1fb95e..b1b550024 100644
--- a/src/client/rspamdclient.c
+++ b/src/client/rspamdclient.c
@@ -15,7 +15,7 @@
*/
#include "rspamdclient.h"
#include "libutil/util.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
#include "unix-std.h"
#include "contrib/zstd/zstd.h"
diff --git a/src/libserver/milter.c b/src/libserver/milter.c
index b3cd46226..6e4a03e4d 100644
--- a/src/libserver/milter.c
+++ b/src/libserver/milter.c
@@ -22,7 +22,7 @@
#include "unix-std.h"
#include "logger.h"
#include "ottery.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
#include "libserver/protocol_internal.h"
#include "libserver/cfg_file_private.h"
diff --git a/src/libserver/protocol.h b/src/libserver/protocol.h
index 254c5fcd1..08372d765 100644
--- a/src/libserver/protocol.h
+++ b/src/libserver/protocol.h
@@ -8,7 +8,7 @@
#include "config.h"
#include "filter.h"
-#include "http.h"
+#include "http_connection.h"
#include "task.h"
#define RSPAMD_BASE_ERROR 500
diff --git a/src/libserver/rspamd_control.c b/src/libserver/rspamd_control.c
index 84c53700e..2fd1d983f 100644
--- a/src/libserver/rspamd_control.c
+++ b/src/libserver/rspamd_control.c
@@ -16,7 +16,7 @@
#include "config.h"
#include "rspamd.h"
#include "rspamd_control.h"
-#include "libutil/http.h"
+#include "libutil/http_connection.h"
#include "libutil/http_private.h"
#include "unix-std.h"
#include "utlist.h"
diff --git a/src/libserver/task.h b/src/libserver/task.h
index 93e0ae0e8..684f5c2c0 100644
--- a/src/libserver/task.h
+++ b/src/libserver/task.h
@@ -17,7 +17,7 @@
#define TASK_H_
#include "config.h"
-#include "http.h"
+#include "http_connection.h"
#include "events.h"
#include "util.h"
#include "mem_pool.h"
diff --git a/src/libserver/worker_util.h b/src/libserver/worker_util.h
index dbcc8f8a2..3186025b3 100644
--- a/src/libserver/worker_util.h
+++ b/src/libserver/worker_util.h
@@ -18,7 +18,7 @@
#include "config.h"
#include "util.h"
-#include "http.h"
+#include "http_connection.h"
#include "rspamd.h"
#ifndef HAVE_SA_SIGINFO
diff --git a/src/libutil/CMakeLists.txt b/src/libutil/CMakeLists.txt
index aab754195..aef2ed268 100644
--- a/src/libutil/CMakeLists.txt
+++ b/src/libutil/CMakeLists.txt
@@ -1,27 +1,27 @@
# Librspamd-util
-SET(LIBRSPAMDUTILSRC
- ${CMAKE_CURRENT_SOURCE_DIR}/addr.c
- ${CMAKE_CURRENT_SOURCE_DIR}/aio_event.c
- ${CMAKE_CURRENT_SOURCE_DIR}/bloom.c
- ${CMAKE_CURRENT_SOURCE_DIR}/expression.c
- ${CMAKE_CURRENT_SOURCE_DIR}/fstring.c
- ${CMAKE_CURRENT_SOURCE_DIR}/hash.c
- ${CMAKE_CURRENT_SOURCE_DIR}/http.c
- ${CMAKE_CURRENT_SOURCE_DIR}/logger.c
- ${CMAKE_CURRENT_SOURCE_DIR}/map.c
- ${CMAKE_CURRENT_SOURCE_DIR}/map_helpers.c
- ${CMAKE_CURRENT_SOURCE_DIR}/mem_pool.c
- ${CMAKE_CURRENT_SOURCE_DIR}/printf.c
- ${CMAKE_CURRENT_SOURCE_DIR}/radix.c
- ${CMAKE_CURRENT_SOURCE_DIR}/regexp.c
- ${CMAKE_CURRENT_SOURCE_DIR}/rrd.c
- ${CMAKE_CURRENT_SOURCE_DIR}/shingles.c
- ${CMAKE_CURRENT_SOURCE_DIR}/sqlite_utils.c
- ${CMAKE_CURRENT_SOURCE_DIR}/str_util.c
- ${CMAKE_CURRENT_SOURCE_DIR}/upstream.c
- ${CMAKE_CURRENT_SOURCE_DIR}/util.c
- ${CMAKE_CURRENT_SOURCE_DIR}/heap.c
- ${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c
- ${CMAKE_CURRENT_SOURCE_DIR}/ssl_util.c)
+SET(LIBRSPAMDUTILSRC
+ ${CMAKE_CURRENT_SOURCE_DIR}/addr.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/aio_event.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/bloom.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/expression.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/fstring.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/hash.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/http_connection.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/logger.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/map.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/map_helpers.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/mem_pool.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/printf.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/radix.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/regexp.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/rrd.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/shingles.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/sqlite_utils.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/str_util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/upstream.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/util.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/heap.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/multipattern.c
+ ${CMAKE_CURRENT_SOURCE_DIR}/ssl_util.c)
# Rspamdutil
SET(RSPAMD_UTIL ${LIBRSPAMDUTILSRC} PARENT_SCOPE)
\ No newline at end of file
diff --git a/src/libutil/http.c b/src/libutil/http.c
deleted file mode 100644
index a82fc24f7..000000000
--- a/src/libutil/http.c
+++ /dev/null
@@ -1,3971 +0,0 @@
-/*-
- * Copyright 2016 Vsevolod Stakhov
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "config.h"
-#include "../../contrib/mumhash/mum.h"
-#include "http_private.h"
-#include "utlist.h"
-#include "util.h"
-#include "printf.h"
-#include "logger.h"
-#include "ref.h"
-#include "ottery.h"
-#include "keypair_private.h"
-#include "cryptobox.h"
-#include "unix-std.h"
-#include "libutil/ssl_util.h"
-#include "libutil/regexp.h"
-#include "libserver/url.h"
-
-#include <openssl/err.h>
-
-#define ENCRYPTED_VERSION " HTTP/1.0"
-
-struct _rspamd_http_privbuf {
- rspamd_fstring_t *data;
- const gchar *zc_buf;
- gsize zc_remain;
- ref_entry_t ref;
-};
-
-enum rspamd_http_priv_flags {
- RSPAMD_HTTP_CONN_FLAG_ENCRYPTED = 1 << 0,
- RSPAMD_HTTP_CONN_FLAG_NEW_HEADER = 1 << 1,
- RSPAMD_HTTP_CONN_FLAG_RESETED = 1 << 2,
- RSPAMD_HTTP_CONN_FLAG_TOO_LARGE = 1 << 3,
- RSPAMD_HTTP_CONN_FLAG_ENCRYPTION_NEEDED = 1 << 4,
-};
-
-#define IS_CONN_ENCRYPTED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_ENCRYPTED)
-#define IS_CONN_RESETED(c) ((c)->flags & RSPAMD_HTTP_CONN_FLAG_RESETED)
-
-struct rspamd_http_connection_private {
- gpointer ssl_ctx;
- struct rspamd_ssl_connection *ssl;
- struct _rspamd_http_privbuf *buf;
- struct rspamd_cryptobox_pubkey *peer_key;
- struct rspamd_cryptobox_keypair *local_key;
- struct rspamd_http_header *header;
- struct http_parser parser;
- struct http_parser_settings parser_cb;
- struct event ev;
- struct timeval tv;
- struct timeval *ptv;
- struct rspamd_http_message *msg;
- struct iovec *out;
- guint outlen;
- enum rspamd_http_priv_flags flags;
- gsize wr_pos;
- gsize wr_total;
-};
-
-enum http_magic_type {
- HTTP_MAGIC_PLAIN = 0,
- HTTP_MAGIC_HTML,
- HTTP_MAGIC_CSS,
- HTTP_MAGIC_JS,
- HTTP_MAGIC_PNG,
- HTTP_MAGIC_JPG
-};
-
-static const struct _rspamd_http_magic {
- const gchar *ext;
- const gchar *ct;
-} http_file_types[] = {
- [HTTP_MAGIC_PLAIN] = { "txt", "text/plain" },
- [HTTP_MAGIC_HTML] = { "html", "text/html" },
- [HTTP_MAGIC_CSS] = { "css", "text/css" },
- [HTTP_MAGIC_JS] = { "js", "application/javascript" },
- [HTTP_MAGIC_PNG] = { "png", "image/png" },
- [HTTP_MAGIC_JPG] = { "jpg", "image/jpeg" },
-};
-
-static const gchar *http_week[] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" };
-static const gchar *http_month[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun",
- "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" };
-static const rspamd_ftok_t key_header = {
- .begin = "Key",
- .len = 3
-};
-static const rspamd_ftok_t date_header = {
- .begin = "Date",
- .len = 4
-};
-static const rspamd_ftok_t last_modified_header = {
- .begin = "Last-Modified",
- .len = 13
-};
-
-static void rspamd_http_message_storage_cleanup (struct rspamd_http_message *msg);
-static gboolean rspamd_http_message_grow_body (struct rspamd_http_message *msg,
- gsize len);
-
-#define HTTP_ERROR http_error_quark ()
-GQuark
-http_error_quark (void)
-{
- return g_quark_from_static_string ("http-error-quark");
-}
-
-static void
-rspamd_http_privbuf_dtor (gpointer ud)
-{
- struct _rspamd_http_privbuf *p = (struct _rspamd_http_privbuf *)ud;
-
- if (p->data) {
- rspamd_fstring_free (p->data);
- }
-
- g_free (p);
-}
-
-static const gchar *
-rspamd_http_code_to_str (gint code)
-{
- if (code == 200) {
- return "OK";
- }
- else if (code == 404) {
- return "Not found";
- }
- else if (code == 403 || code == 401) {
- return "Not authorized";
- }
- else if (code >= 400 && code < 500) {
- return "Bad request";
- }
- else if (code >= 300 && code < 400) {
- return "See Other";
- }
- else if (code >= 500 && code < 600) {
- return "Internal server error";
- }
-
- return "Unknown error";
-}
-
-/*
- * Obtained from nginx
- * Copyright (C) Igor Sysoev
- */
-static guint mday[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 };
-
-time_t
-rspamd_http_parse_date (const gchar *header, gsize len)
-{
- const gchar *p, *end;
- gint month;
- guint day, year, hour, min, sec;
- guint64 time;
- enum {
- no = 0, rfc822, /* Tue, 10 Nov 2002 23:50:13 */
- rfc850, /* Tuesday, 10-Dec-02 23:50:13 */
- isoc /* Tue Dec 10 23:50:13 2002 */
- } fmt;
-
- fmt = 0;
- if (len > 0) {
- end = header + len;
- }
- else {
- end = header + strlen (header);
- }
-
- day = 32;
- year = 2038;
-
- for (p = header; p < end; p++) {
- if (*p == ',') {
- break;
- }
-
- if (*p == ' ') {
- fmt = isoc;
- break;
- }
- }
-
- for (p++; p < end; p++)
- if (*p != ' ') {
- break;
- }
-
- if (end - p < 18) {
- return (time_t)-1;
- }
-
- if (fmt != isoc) {
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
- return (time_t)-1;
- }
-
- day = (*p - '0') * 10 + *(p + 1) - '0';
- p += 2;
-
- if (*p == ' ') {
- if (end - p < 18) {
- return (time_t)-1;
- }
- fmt = rfc822;
-
- }
- else if (*p == '-') {
- fmt = rfc850;
-
- }
- else {
- return (time_t)-1;
- }
-
- p++;
- }
-
- switch (*p) {
-
- case 'J':
- month = *(p + 1) == 'a' ? 0 : *(p + 2) == 'n' ? 5 : 6;
- break;
-
- case 'F':
- month = 1;
- break;
-
- case 'M':
- month = *(p + 2) == 'r' ? 2 : 4;
- break;
-
- case 'A':
- month = *(p + 1) == 'p' ? 3 : 7;
- break;
-
- case 'S':
- month = 8;
- break;
-
- case 'O':
- month = 9;
- break;
-
- case 'N':
- month = 10;
- break;
-
- case 'D':
- month = 11;
- break;
-
- default:
- return (time_t)-1;
- }
-
- p += 3;
-
- if ((fmt == rfc822 && *p != ' ') || (fmt == rfc850 && *p != '-')) {
- return (time_t)-1;
- }
-
- p++;
-
- if (fmt == rfc822) {
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9'
- || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0'
- || *(p + 3) > '9') {
- return (time_t)-1;
- }
-
- year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100
- + (*(p + 2) - '0') * 10 + *(p + 3) - '0';
- p += 4;
-
- }
- else if (fmt == rfc850) {
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
- return (time_t)-1;
- }
-
- year = (*p - '0') * 10 + *(p + 1) - '0';
- year += (year < 70) ? 2000 : 1900;
- p += 2;
- }
-
- if (fmt == isoc) {
- if (*p == ' ') {
- p++;
- }
-
- if (*p < '0' || *p > '9') {
- return (time_t)-1;
- }
-
- day = *p++ - '0';
-
- if (*p != ' ') {
- if (*p < '0' || *p > '9') {
- return (time_t)-1;
- }
-
- day = day * 10 + *p++ - '0';
- }
-
- if (end - p < 14) {
- return (time_t)-1;
- }
- }
-
- if (*p++ != ' ') {
- return (time_t)-1;
- }
-
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
- return (time_t)-1;
- }
-
- hour = (*p - '0') * 10 + *(p + 1) - '0';
- p += 2;
-
- if (*p++ != ':') {
- return (time_t)-1;
- }
-
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
- return (time_t)-1;
- }
-
- min = (*p - '0') * 10 + *(p + 1) - '0';
- p += 2;
-
- if (*p++ != ':') {
- return (time_t)-1;
- }
-
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9') {
- return (time_t)-1;
- }
-
- sec = (*p - '0') * 10 + *(p + 1) - '0';
-
- if (fmt == isoc) {
- p += 2;
-
- if (*p++ != ' ') {
- return (time_t)-1;
- }
-
- if (*p < '0' || *p > '9' || *(p + 1) < '0' || *(p + 1) > '9'
- || *(p + 2) < '0' || *(p + 2) > '9' || *(p + 3) < '0'
- || *(p + 3) > '9') {
- return (time_t)-1;
- }
-
- year = (*p - '0') * 1000 + (*(p + 1) - '0') * 100
- + (*(p + 2) - '0') * 10 + *(p + 3) - '0';
- }
-
- if (hour > 23 || min > 59 || sec > 59) {
- return (time_t)-1;
- }
-
- if (day == 29 && month == 1) {
- if ((year & 3) || ((year % 100 == 0) && (year % 400) != 0)) {
- return (time_t)-1;
- }
-
- }
- else if (day > mday[month]) {
- return (time_t)-1;
- }
-
- /*
- * shift new year to March 1 and start months from 1 (not 0),
- * it is needed for Gauss' formula
- */
-
- if (--month <= 0) {
- month += 12;
- year -= 1;
- }
-
- /* Gauss' formula for Gregorian days since March 1, 1 BC */
-
- time = (guint64) (
- /* days in years including leap years since March 1, 1 BC */
-
- 365 * year + year / 4 - year / 100 + year / 400
-
- /* days before the month */
-
- + 367 * month / 12 - 30
-
- /* days before the day */
-
- + day - 1
-
- /*
- * 719527 days were between March 1, 1 BC and March 1, 1970,
- * 31 and 28 days were in January and February 1970
- */
-
- - 719527 + 31 + 28) * 86400 + hour * 3600 + min * 60 + sec;
-
- return (time_t) time;
-}
-
-static void
-rspamd_http_parse_key (rspamd_ftok_t *data, struct rspamd_http_connection *conn,
- struct rspamd_http_connection_private *priv)
-{
- guchar *decoded_id;
- const gchar *eq_pos;
- gsize id_len;
- struct rspamd_cryptobox_pubkey *pk;
-
- if (priv->local_key == NULL) {
- /* In this case we cannot do anything, e.g. we cannot decrypt payload */
- priv->flags &= ~RSPAMD_HTTP_CONN_FLAG_ENCRYPTED;
- }
- else {
- /* Check sanity of what we have */
- eq_pos = memchr (data->begin, '=', data->len);
- if (eq_pos != NULL) {
- decoded_id = rspamd_decode_base32 (data->begin, eq_pos - data->begin,
- &id_len);
-
- if (decoded_id != NULL && id_len >= RSPAMD_KEYPAIR_SHORT_ID_LEN) {
- pk = rspamd_pubkey_from_base32 (eq_pos + 1,
- data->begin + data->len - eq_pos - 1,
- RSPAMD_KEYPAIR_KEX,
- RSPAMD_CRYPTOBOX_MODE_25519);
- if (pk != NULL) {
- if (memcmp (rspamd_keypair_get_id (priv->local_key),
- decoded_id,
- RSPAMD_KEYPAIR_SHORT_ID_LEN) == 0) {
- priv->msg->peer_key = pk;
-
- if (conn->cache && priv->msg->peer_key) {
- rspamd_keypair_cache_process (conn->cache,
- priv->local_key, priv->msg->peer_key);
- }
- }
- else {
- rspamd_pubkey_unref (pk);
- }
- }
- }
-
- priv->flags |= RSPAMD_HTTP_CONN_FLAG_ENCRYPTED;
- g_free (decoded_id);
- }
- }
-}
-
-static inline void
-rspamd_http_check_special_header (struct rspamd_http_connection *conn,
- struct rspamd_http_connection_private *priv)
-{
- if (rspamd_ftok_casecmp (&priv->header->name, &date_header) == 0) {
- priv->msg->date = rspamd_http_parse_date (priv->header->value.begin,
- priv->header->value.len);
- }
- else if (rspamd_ftok_casecmp (&priv->header->name, &key_header) == 0) {
- rspamd_http_parse_key (&priv->header->value, conn, priv);
- }
- else if (rspamd_ftok_casecmp (&priv->header->name, &last_modified_header) == 0) {
- priv->msg->last_modified = rspamd_http_parse_date (
- priv->header->value.begin,
- priv->header->value.len);
- }
-}
-
-static gint
-rspamd_http_on_url (http_parser * parser, const gchar *at, size_t length)
-{
- struct rspamd_http_connection *conn =
- (struct rspamd_http_connection *)parser->data;
- struct rspamd_http_connection_private *priv;
-
- priv = conn->priv;
-
- priv->msg->url = rspamd_fstring_append (priv->msg->url, at, length);
-
- return 0;
-}
-
-static gint
-rspamd_http_on_status (http_parser * parser, const gchar *at, size_t length)
-{
- struct rspamd_http_connection *conn =
- (struct rspamd_http_connection *)parser->data;
- struct rspamd_http_connection_private *priv;
-
- priv = conn->priv;
-
- if (parser->status_code != 200) {
- if (priv->msg->status == NULL) {
- priv->msg->status = rspamd_fstring_new ();
- }
-
- priv->msg->status = rspamd_fstring_append (priv->msg->status, at, length);
- }
*** OUTPUT TRUNCATED, 4127 LINES SKIPPED ***
More information about the Commits
mailing list