commit 4fe53c7: [Feature] Store etag in cached HTTP maps + better logging

[Vsevolod Stakhov]

Author: Vsevolod Stakhov
Date: 2019-10-21 13:31:09 +0100
URL: https://github.com/rspamd/rspamd/commit/4fe53c718fe00ed1964a7ec099ff579168eb0cbb (HEAD -> master)

[Feature] Store etag in cached HTTP maps + better logging

---
 src/libutil/map.c         | 79 +++++++++++++++++++++++++++++++++++++++++------
 src/libutil/map_private.h |  3 +-
 2 files changed, 71 insertions(+), 11 deletions(-)

diff --git a/src/libutil/map.c b/src/libutil/map.c
index 9a9da740a..7d8443d08 100644
--- a/src/libutil/map.c
+++ b/src/libutil/map.c
@@ -1290,17 +1290,36 @@ rspamd_map_save_http_cached_file (struct rspamd_map *map,
 	header.next_check = map->next_check;
 	header.data_off = sizeof (header);
 
+	if (htdata->etag) {
+		header.data_off += RSPAMD_FSTRING_LEN (htdata->etag);
+		header.etag_len = RSPAMD_FSTRING_LEN (htdata->etag);
+	}
+	else {
+		header.etag_len = 0;
+	}
+
 	if (write (fd, &header, sizeof (header)) != sizeof (header)) {
-		msg_err_map ("cannot write file %s: %s", path, strerror (errno));
+		msg_err_map ("cannot write file %s (header stage): %s", path, strerror (errno));
 		rspamd_file_unlock (fd, FALSE);
 		close (fd);
 
 		return FALSE;
 	}
 
+	if (header.etag_len > 0) {
+		if (write (fd, RSPAMD_FSTRING_DATA (htdata->etag), header.etag_len) !=
+				header.etag_len) {
+			msg_err_map ("cannot write file %s (etag stage): %s", path, strerror (errno));
+			rspamd_file_unlock (fd, FALSE);
+			close (fd);
+
+			return FALSE;
+		}
+	}
+
 	/* Now write the rest */
 	if (write (fd, data, len) != len) {
-		msg_err_map ("cannot write file %s: %s", path, strerror (errno));
+		msg_err_map ("cannot write file %s (data stage): %s", path, strerror (errno));
 		rspamd_file_unlock (fd, FALSE);
 		close (fd);
 
@@ -1310,7 +1329,8 @@ rspamd_map_save_http_cached_file (struct rspamd_map *map,
 	rspamd_file_unlock (fd, FALSE);
 	close (fd);
 
-	msg_info_map ("saved data from %s in %s, %uz bytes", bk->uri, path, len);
+	msg_info_map ("saved data from %s in %s, %uz bytes", bk->uri, path, len +
+			sizeof (header) + header.etag_len);
 
 	return TRUE;
 }
@@ -1352,7 +1372,7 @@ rspamd_map_read_http_cached_file (struct rspamd_map *map,
 	(void)fstat (fd, &st);
 
 	if (read (fd, &header, sizeof (header)) != sizeof (header)) {
-		msg_err_map ("cannot read file %s: %s", path, strerror (errno));
+		msg_err_map ("cannot read file %s (header stage): %s", path, strerror (errno));
 		rspamd_file_unlock (fd, FALSE);
 		close (fd);
 
@@ -1361,19 +1381,45 @@ rspamd_map_read_http_cached_file (struct rspamd_map *map,
 
 	if (memcmp (header.magic, rspamd_http_file_magic,
 			sizeof (rspamd_http_file_magic)) != 0) {
-		msg_err_map ("invalid magic in file %s: %s", path, strerror (errno));
+		msg_warn_map ("invalid or old version magic in file %s; ignore it", path);
 		rspamd_file_unlock (fd, FALSE);
 		close (fd);
 
 		return FALSE;
 	}
 
-	rspamd_file_unlock (fd, FALSE);
-	close (fd);
-
 	map->next_check = header.next_check;
 	htdata->last_modified = header.mtime;
 
+	if (header.etag_len > 0) {
+		rspamd_fstring_t *etag = rspamd_fstring_sized_new (header.etag_len);
+
+		if (read (fd, RSPAMD_FSTRING_DATA (etag), header.etag_len) != header.etag_len) {
+			msg_err_map ("cannot read file %s (etag stage): %s", path,
+					strerror (errno));
+			rspamd_file_unlock (fd, FALSE);
+			rspamd_fstring_free (etag);
+			close (fd);
+
+			return FALSE;
+		}
+
+		etag->len = header.etag_len;
+
+		if (htdata->etag) {
+			/* FIXME: should be dealt somehow better */
+			msg_warn_map ("etag is already defined as %V; cached is %V; ignore cached",
+					htdata->etag, etag);
+			rspamd_fstring_free (etag);
+		}
+		else {
+			htdata->etag = etag;
+		}
+	}
+
+	rspamd_file_unlock (fd, FALSE);
+	close (fd);
+
 	/* Now read file data */
 	/* Perform buffered read: fail-safe */
 	if (!read_map_file_chunks (map, cbdata, path,
@@ -1381,8 +1427,21 @@ rspamd_map_read_http_cached_file (struct rspamd_map *map,
 		return FALSE;
 	}
 
-	msg_info_map ("read cached data for %s from %s, %uz bytes", bk->uri, path,
-			st.st_size - header.data_off);
+	struct tm tm;
+	gchar ncheck_buf[32], lm_buf[32];
+
+	rspamd_localtime (map->next_check, &tm);
+	strftime (ncheck_buf, sizeof (ncheck_buf) - 1, "%Y-%m-%d %H:%M:%S", &tm);
+	rspamd_localtime (htdata->last_modified, &tm);
+	strftime (lm_buf, sizeof (lm_buf) - 1, "%Y-%m-%d %H:%M:%S", &tm);
+
+	msg_info_map ("read cached data for %s from %s, %uz bytes; next check at: %s;"
+				  " last modified on: %s; etag: %V",
+			bk->uri, path,
+			st.st_size - header.data_off,
+			ncheck_buf,
+			lm_buf,
+			htdata->etag);
 
 	return TRUE;
 }
diff --git a/src/libutil/map_private.h b/src/libutil/map_private.h
index e285c8498..adcdd9d98 100644
--- a/src/libutil/map_private.h
+++ b/src/libutil/map_private.h
@@ -180,13 +180,14 @@ struct map_periodic_cbdata {
 };
 
 static const gchar rspamd_http_file_magic[] =
-		{'r', 'm', 'c', 'd', '1', '0', '0', '0'};
+		{'r', 'm', 'c', 'd', '2', '0', '0', '0'};
 
 struct rspamd_http_file_data {
 	guchar magic[sizeof (rspamd_http_file_magic)];
 	goffset data_off;
 	gulong mtime;
 	gulong next_check;
+	gulong etag_len;
 };
 
 struct http_callback_data {