commit 3af0224: [Project] Css: More meat to the lexer

[Vsevolod Stakhov]

Author: Vsevolod Stakhov
Date: 2021-01-26 15:48:40 +0000
URL: https://github.com/rspamd/rspamd/commit/3af022430973c47a0f1be61923c27292bc24baae (HEAD -> master)

[Project] Css: More meat to the lexer

---
 src/libserver/css/css_tokeniser.cxx | 72 +++++++++++++++++++++++++++++++++++++
 src/libserver/css/css_tokeniser.hxx |  2 --
 2 files changed, 72 insertions(+), 2 deletions(-)

diff --git a/src/libserver/css/css_tokeniser.cxx b/src/libserver/css/css_tokeniser.cxx
index f545af47a..49032c64f 100644
--- a/src/libserver/css/css_tokeniser.cxx
+++ b/src/libserver/css/css_tokeniser.cxx
@@ -546,11 +546,83 @@ auto css_tokeniser::next_token(void) -> struct css_parser_token
 					/* Numeric token */
 					return consume_number();
 				}
+				else {
+					offset = i + 1;
+					return make_token<css_parser_token::token_type::delim_token>(c);
+				}
 			}
 			/* No other options, a delimiter - */
 			offset = i + 1;
 			return make_token<css_parser_token::token_type::delim_token>(c);
 
+			break;
+		case '\\':
+			if (i + 1 < input.size()) {
+				if (input[i + 1] == '\n' || input[i + 1] == '\r') {
+					offset = i + 1;
+					return make_token<css_parser_token::token_type::delim_token>(c);
+				}
+				else {
+					/* Valid escape, assume ident */
+					return consume_ident();
+				}
+			}
+			else {
+				offset = i + 1;
+				return make_token<css_parser_token::token_type::delim_token>(c);
+			}
+			break;
+		case '@':
+			if (i + 3 < input.size()) {
+				if (is_plain_ident(input[i + 1]) &&
+					is_plain_ident(input[i + 2]) && is_plain_ident(input[i + 3])) {
+					offset = i + 1;
+					auto ident_token = consume_ident();
+
+					if (ident_token.type == css_parser_token::token_type::ident_token) {
+						/* Update type */
+						ident_token.type = css_parser_token::token_type::at_keyword_token;
+					}
+
+					return ident_token;
+				}
+				else {
+					offset = i + 1;
+					return make_token<css_parser_token::token_type::delim_token>(c);
+				}
+			}
+			else {
+				offset = i + 1;
+				return make_token<css_parser_token::token_type::delim_token>(c);
+			}
+			break;
+		case '#':
+			/* TODO: make it more conformant */
+			if (i + 2 < input.size()) {
+				if (is_plain_ident(input[i + 1]) &&
+					is_plain_ident(input[i + 2])) {
+					offset = i + 1;
+					auto ident_token = consume_ident();
+
+					if (ident_token.type == css_parser_token::token_type::ident_token) {
+						/* Update type */
+						ident_token.type = css_parser_token::token_type::hash_token;
+					}
+
+					return ident_token;
+				}
+				else {
+					offset = i + 1;
+					return make_token<css_parser_token::token_type::delim_token>(c);
+				}
+			}
+			else {
+				offset = i + 1;
+				return make_token<css_parser_token::token_type::delim_token>(c);
+			}
+			break;
+		default:
+			/* Generic parsing code */
 			break;
 		}
 
diff --git a/src/libserver/css/css_tokeniser.hxx b/src/libserver/css/css_tokeniser.hxx
index 5880241c1..cd9c47cfd 100644
--- a/src/libserver/css/css_tokeniser.hxx
+++ b/src/libserver/css/css_tokeniser.hxx
@@ -38,8 +38,6 @@ struct css_parser_token {
 		string_token,
 		number_token,
 		url_token,
-		dimension_token,
-		percentage_token,
 		cdo_token, /* xml open comment */
 		cdc_token, /* xml close comment */
 		delim_token,