From 9bcd25f54708c2d6bde79f266f1b74d432f4571f Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Fri, 6 Dec 2024 09:13:31 +0100 Subject: lexer: Preserve keyword, regexp flags until processing non-comment tokens When the lexer has either the `no_keyword` or `no_regexp` flag set, it should retain these flags until it encounters a non-comment token. Only then should it stop preventing the interpretation of tokens beginning with a letter or forward slash as keywords or regular expression literals, respectively. Previously, these flags were being reset too early when processing comments, which could cause incorrect parsing when comments appeared between relevant tokens. The flags should only be reset after consuming an actual non-comment token to ensure consistent parsing behavior. Fixes: #250 Fixes: 855854f ("lexer: emit comment and template statement block tokens") Signed-off-by: Jo-Philipp Wich --- lexer.c | 6 ++++-- tests/custom/99_bugs/51_preserve_lexer_flags | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/custom/99_bugs/51_preserve_lexer_flags diff --git a/lexer.c b/lexer.c index 875e5dd..c9359b4 100644 --- a/lexer.c +++ b/lexer.c @@ -1176,8 +1176,10 @@ uc_lexer_next_token(uc_lexer_t *lex) rv = lex_step(lex); - lex->no_keyword = false; - lex->no_regexp = false; + if (rv && rv->type != TK_COMMENT) { + lex->no_keyword = false; + lex->no_regexp = false; + } return rv; } diff --git a/tests/custom/99_bugs/51_preserve_lexer_flags b/tests/custom/99_bugs/51_preserve_lexer_flags new file mode 100644 index 0000000..aba646c --- /dev/null +++ b/tests/custom/99_bugs/51_preserve_lexer_flags @@ -0,0 +1,20 @@ +Ensure keyword and regexp flags are preserved across comments when lexing +object literals and division operators. + +-- Testcase -- +{% + printf("%.J\n", [ + { /* comment */ default: true }, + 4 /* comment */ /2/1 + ]); +%} +-- End -- + +-- Expect stdout -- +[ + { + "default": true + }, + 2 +] +-- End -- -- cgit v1.2.3