From 855854f6c2ae2e667f6bbfd5f67caab32b4ebf86 Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Mon, 23 Sep 2024 23:33:47 +0200 Subject: lexer: emit comment and template statement block tokens Tweak the token stream reported by the lexer in order to make it more useful for alternative, non-compilation downstream parse processes such as code intelligence gathering within a language server implementation. - Instead of silently discarding source code comments in the lexing phase, emit TK_COMMENT tokens which is useful to e.g. parse type annotations and other structured information. - Do not silently discard TK_LSTM tokens but report them to downstream parsers instead. - Do not silently emit TK_RSTM tokens as TK_SCOL but report them as-is to downstrem parsers. - Adjust the byte code compiler to properly deal with the changed token reporting by discarding incoming TK_COMMENT and TK_LSTM tokens and by remapping read TK_RSTM tokens to the TK_SCOL type. Signed-off-by: Jo-Philipp Wich --- compiler.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'compiler.c') diff --git a/compiler.c b/compiler.c index 53e61e7..b64537a 100644 --- a/compiler.c +++ b/compiler.c @@ -242,7 +242,17 @@ uc_compiler_parse_advance(uc_compiler_t *compiler) compiler->parser->prev = compiler->parser->curr; while (true) { - compiler->parser->curr = *uc_lexer_next_token(&compiler->parser->lex); + uc_token_t *tok = uc_lexer_next_token(&compiler->parser->lex); + + if (tok->type == TK_COMMENT || tok->type == TK_LSTM) { + ucv_put(tok->uv); + continue; + } + else if (tok->type == TK_RSTM) { + tok->type = TK_SCOL; + } + + compiler->parser->curr = *tok; if (compiler->parser->curr.type != TK_ERROR) break; -- cgit v1.2.3