From 80a2dd2cc27e2fc0c55f03f01a1d460a91954ab6 Mon Sep 17 00:00:00 2001 From: Jo-Philipp Wich Date: Thu, 23 Jul 2009 00:41:44 +0000 Subject: libs/web: add template parser reimplemented in C --- libs/web/Makefile | 26 +++ libs/web/src/template_lualib.c | 62 ++++++ libs/web/src/template_lualib.h | 28 +++ libs/web/src/template_parser.c | 463 +++++++++++++++++++++++++++++++++++++++++ libs/web/src/template_parser.h | 81 +++++++ 5 files changed, 660 insertions(+) create mode 100644 libs/web/src/template_lualib.c create mode 100644 libs/web/src/template_lualib.h create mode 100644 libs/web/src/template_parser.c create mode 100644 libs/web/src/template_parser.h (limited to 'libs') diff --git a/libs/web/Makefile b/libs/web/Makefile index f7fac7740..cc3bc96fc 100644 --- a/libs/web/Makefile +++ b/libs/web/Makefile @@ -1,2 +1,28 @@ include ../../build/config.mk include ../../build/module.mk +include ../../build/gccconfig.mk + +TPL_LDFLAGS = +TPL_CFLAGS = +TPL_SO = parser.so +TPL_COMMON_OBJ = src/template_parser.o +TPL_LUALIB_OBJ = src/template_lualib.o + +%.o: %.c + $(COMPILE) $(TPL_CFLAGS) $(LUA_CFLAGS) $(FPIC) -c -o $@ $< + +compile: build-clean $(TPL_COMMON_OBJ) $(TPL_LUALIB_OBJ) + $(LINK) $(SHLIB_FLAGS) $(TPL_LDFLAGS) -o src/$(TPL_SO) \ + $(TPL_COMMON_OBJ) $(TPL_LUALIB_OBJ) + mkdir -p dist$(LUCI_LIBRARYDIR)/template + cp src/$(TPL_SO) dist$(LUCI_LIBRARYDIR)/template/$(TPL_SO) + +install: build + cp -pR dist$(LUA_LIBRARYDIR)/* $(LUA_LIBRARYDIR) + +clean: build-clean + +build-clean: + rm -f src/*.o src/$(TPL_SO) + + diff --git a/libs/web/src/template_lualib.c b/libs/web/src/template_lualib.c new file mode 100644 index 000000000..685613fbd --- /dev/null +++ b/libs/web/src/template_lualib.c @@ -0,0 +1,62 @@ +/* + * LuCI Template - Lua binding + * + * Copyright (C) 2009 Jo-Philipp Wich + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "template_lualib.h" + +int template_L_parse(lua_State *L) +{ + const char *file = luaL_checkstring(L, 1); + struct template_parser parser; + int lua_status; + + if( (parser.fd = open(file, O_RDONLY)) > 0 ) + { + parser.flags = 0; + parser.bufsize = 0; + parser.state = T_STATE_TEXT_NEXT; + + if( !(lua_status = lua_load(L, template_reader, &parser, file)) ) + { + return 1; + } + else + { + lua_pushnil(L); + lua_pushinteger(L, lua_status); + lua_pushlstring(L, parser.out, parser.outsize); + return 3; + } + } + + lua_pushnil(L); + lua_pushinteger(L, 255); + lua_pushstring(L, "No such file or directory"); + return 3; +} + +/* module table */ +static const luaL_reg R[] = { + {"parse", template_L_parse}, + {NULL, NULL} +}; + +LUALIB_API int luaopen_luci_template_parser(lua_State *L) { + luaL_register(L, TEMPLATE_LUALIB_META, R); + return 1; +} + diff --git a/libs/web/src/template_lualib.h b/libs/web/src/template_lualib.h new file mode 100644 index 000000000..de915e1a2 --- /dev/null +++ b/libs/web/src/template_lualib.h @@ -0,0 +1,28 @@ +/* + * LuCI Template - Lua library header + * + * Copyright (C) 2009 Jo-Philipp Wich + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _TEMPLATE_LUALIB_H_ +#define _TEMPLATE_LUALIB_H_ + +#include "template_parser.h" + +#define TEMPLATE_LUALIB_META "template.parser" + +LUALIB_API int luaopen_luci_template_parser(lua_State *L); + +#endif diff --git a/libs/web/src/template_parser.c b/libs/web/src/template_parser.c new file mode 100644 index 000000000..58de5bb77 --- /dev/null +++ b/libs/web/src/template_parser.c @@ -0,0 +1,463 @@ +/* + * LuCI Template - Parser implementation + * + * Copyright (C) 2009 Jo-Philipp Wich + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "template_parser.h" + + +/* leading and trailing code for different types */ +const char * gen_code[6][2] = { + { "write(\"", "\")" }, + { NULL, NULL }, + { "write(tostring(", "))" }, + { "include(\"", "\")" }, + { "write(translate(\"", "\"))" }, + { NULL, " " } +}; + +/* Simple strstr() like function that takes len arguments for both haystack and needle. */ +static char *strfind(char *haystack, int hslen, const char *needle, int ndlen) +{ + int match = 0; + int i, j; + + for( i = 0; i < hslen; i++ ) + { + if( haystack[i] == needle[0] ) + { + match = ((ndlen == 1) || ((i + ndlen) <= hslen)); + + for( j = 1; (j < ndlen) && ((i + j) < hslen); j++ ) + { + if( haystack[i+j] != needle[j] ) + { + match = 0; + break; + } + } + + if( match ) + return &haystack[i]; + } + } + + return NULL; +} + +/* + * Inspect current read buffer and find the number of "vague" characters at the end + * which could indicate an opening token. Returns the number of "vague" chars. + * The last continuous sequence of whitespace, optionally followed by a "<" is + * treated as "vague" because whitespace may be discarded if the upcoming opening + * token indicates pre-whitespace-removal ("<%-"). A single remaining "<" char + * can't be differentiated from an opening token ("<%"), so it's kept to be processed + * in the next cycle. + */ +static int stokscan(struct template_parser *data, int off, int no_whitespace) +{ + int i; + int skip = 0; + int tokoff = data->bufsize - 1; + + for( i = tokoff; i >= off; i-- ) + { + if( data->buf[i] == T_TOK_START[0] ) + { + skip = tokoff - i + 1; + tokoff = i - 1; + break; + } + } + + if( !no_whitespace ) + { + for( i = tokoff; i >= off; i-- ) + { + if( isspace(data->buf[i]) ) + skip++; + else + break; + } + } + + return skip; +} + +/* + * Similar to stokscan() but looking for closing token indicators. + * Matches "-", optionally followed by a "%" char. + */ +static int etokscan(struct template_parser *data) +{ + int skip = 0; + + if( (data->bufsize > 0) && (data->buf[data->bufsize-1] == T_TOK_END[0]) ) + skip++; + + if( (data->bufsize > skip) && (data->buf[data->bufsize-skip-1] == T_TOK_SKIPWS[0]) ) + skip++; + + return skip; +} + +/* + * Generate Lua expressions from the given raw code, write it into the + * output buffer and set the lua_Reader specific size pointer. + * Takes parser-state, lua_Reader's size pointer and generator flags + * as parameter. The given flags indicate whether leading or trailing + * code should be added. Returns a pointer to the output buffer. + */ +static const char * generate_expression(struct template_parser *data, size_t *sz, int what) +{ + char tmp[T_OUTBUFSZ]; + int i; + int size = 0; + int start = 0; + int i18n_hasdef = 0; + + memset(tmp, 0, T_OUTBUFSZ); + + /* Inject leading expression code (if any) */ + if( (what & T_GEN_START) && (gen_code[data->type][0] != NULL) ) + { + memcpy(tmp, gen_code[data->type][0], strlen(gen_code[data->type][0])); + size += strlen(gen_code[data->type][0]); + } + + /* Parse source buffer */ + for( i = 0; i < data->outsize; i++ ) + { + /* Skip leading whitespace for non-raw and non-expr chunks */ + if( !start && isspace(data->out[i]) && (data->type == T_TYPE_I18N || data->type == T_TYPE_INCLUDE) ) + continue; + else if( !start ) + start = 1; + + /* Found whitespace after i18n key */ + if( (data->type == T_TYPE_I18N) && (i18n_hasdef == 1) ) + { + /* At non-whitespace char, inject seperator token */ + if( !isspace(data->out[i]) ) + { + memcpy(&tmp[size], T_TOK_I18NSEP, strlen(T_TOK_I18NSEP)); + size += strlen(T_TOK_I18NSEP); + i18n_hasdef = 2; + } + + /* At further whitespace, skip */ + else + { + continue; + } + } + + /* Escape quotes, backslashes and newlines for plain, i18n and include expressions */ + if( (data->type == T_TYPE_TEXT || data->type == T_TYPE_I18N || data->type == T_TYPE_INCLUDE) && + (data->out[i] == '\\' || data->out[i] == '"' || data->out[i] == '\n' || data->out[i] == '\t') ) + { + tmp[size++] = '\\'; + + switch(data->out[i]) + { + case '\n': + tmp[size++] = 'n'; + break; + + case '\t': + tmp[size++] = 't'; + break; + + default: + tmp[size++] = data->out[i]; + } + } + + /* Found whitespace in i18n expression, raise flag */ + else if( isspace(data->out[i]) && (data->type == T_TYPE_I18N) ) + { + i18n_hasdef = 1; + } + + /* Normal char */ + else + { + tmp[size++] = data->out[i]; + } + } + + /* Processed i18n expression without default text, inject separator */ + if( (data->type == T_TYPE_I18N) && (i18n_hasdef < 2) ) + { + memcpy(&tmp[size], T_TOK_I18NSEP, strlen(T_TOK_I18NSEP)); + size += strlen(T_TOK_I18NSEP); + } + + /* Inject trailing expression code (if any) */ + if( (what & T_GEN_END) && (gen_code[data->type][1] != NULL) ) + { + memcpy(&tmp[size], gen_code[data->type][1], strlen(gen_code[data->type][1])); + size += strlen(gen_code[data->type][1]); + } + + *sz = data->outsize = size; + memset(data->out, 0, T_OUTBUFSZ); + memcpy(data->out, tmp, size); + + //printf("<<<%i|%i|%i|%s>>>\n", what, data->type, *sz, data->out); + + return data->out; +} + +/* + * Move the number of bytes specified in data->bufsize from the + * given source pointer to the beginning of the read buffer. + */ +static void bufmove(struct template_parser *data, const char *src) +{ + if( data->bufsize > 0 ) + memmove(data->buf, src, data->bufsize); + else if( data->bufsize < 0 ) + data->bufsize = 0; + + data->buf[data->bufsize] = 0; +} + +/* + * Move the given amount of bytes from the given source pointer + * to the output buffer and set data->outputsize. + */ +static void bufout(struct template_parser *data, const char *src, int len) +{ + if( len >= 0 ) + { + memset(data->out, 0, T_OUTBUFSZ); + memcpy(data->out, src, len); + data->outsize = len; + } + else + { + data->outsize = 0; + } +} + +/* + * lua_Reader compatible function that parses template code on demand from + * the given file handle. + */ +const char *template_reader(lua_State *L, void *ud, size_t *sz) +{ + struct template_parser *data = ud; + char *match = NULL; + int off = 0; + int ignore = 0; + int genflags = 0; + int readlen = 0; + int vague = 0; + + while( !(data->flags & T_FLAG_EOF) || (data->bufsize > 0) ) + { + /* Fill buffer */ + if( !(data->flags & T_FLAG_EOF) && (data->bufsize < T_READBUFSZ) ) + { + if( (readlen = read(data->fd, &data->buf[data->bufsize], T_READBUFSZ - data->bufsize)) > 0 ) + data->bufsize += readlen; + else if( readlen == 0 ) + data->flags |= T_FLAG_EOF; + else + return NULL; + } + + /* Evaluate state */ + switch(data->state) + { + /* Plain text chunk (before "<%") */ + case T_STATE_TEXT_INIT: + case T_STATE_TEXT_NEXT: + off = 0; ignore = 0; *sz = 0; + data->type = T_TYPE_TEXT; + + /* Skip leading whitespace if requested */ + if( data->flags & T_FLAG_SKIPWS ) + { + data->flags &= ~T_FLAG_SKIPWS; + while( (off < data->bufsize) && isspace(data->buf[off]) ) + off++; + } + + /* Found "<%" */ + if( (match = strfind(&data->buf[off], data->bufsize - off - 1, T_TOK_START, strlen(T_TOK_START))) != NULL ) + { + readlen = (int)(match - &data->buf[off]); + data->bufsize -= (readlen + strlen(T_TOK_START) + off); + match += strlen(T_TOK_START); + + /* Check for leading '-' */ + if( match[0] == T_TOK_SKIPWS[0] ) + { + data->bufsize--; + match++; + + while( (readlen > 1) && isspace(data->buf[off+readlen-1]) ) + { + readlen--; + } + } + + bufout(data, &data->buf[off], readlen); + bufmove(data, match); + data->state = T_STATE_CODE_INIT; + } + + /* Maybe plain chunk */ + else + { + /* Preserve trailing "<" or white space, maybe a start token */ + vague = stokscan(data, off, 0); + + /* We can process some bytes ... */ + if( vague < data->bufsize ) + { + readlen = data->bufsize - vague - off; + } + + /* No bytes to process, so try to remove at least whitespace ... */ + else + { + /* ... but try to preserve trailing "<" ... */ + vague = stokscan(data, off, 1); + + if( vague < data->bufsize ) + { + readlen = data->bufsize - vague - off; + } + + /* ... no chance, push out buffer */ + else + { + readlen = vague - off; + vague = 0; + } + } + + bufout(data, &data->buf[off], readlen); + + data->state = T_STATE_TEXT_NEXT; + data->bufsize = vague; + bufmove(data, &data->buf[off+readlen]); + } + + if( ignore || data->outsize == 0 ) + continue; + else + return generate_expression(data, sz, T_GEN_START | T_GEN_END); + + break; + + /* Ignored chunk (inside "<%# ... %>") */ + case T_STATE_SKIP: + ignore = 1; + + /* Initial code chunk ("<% ...") */ + case T_STATE_CODE_INIT: + off = 0; + + /* Check for leading '-' */ + if( data->buf[off] == T_TOK_SKIPWS[0] ) + off++; + + /* Determine code type */ + switch(data->buf[off]) + { + case '#': + ignore = 1; + off++; + data->type = T_TYPE_COMMENT; + break; + + case '=': + off++; + data->type = T_TYPE_EXPR; + break; + + case '+': + off++; + data->type = T_TYPE_INCLUDE; + break; + + case ':': + off++; + data->type = T_TYPE_I18N; + break; + + default: + data->type = T_TYPE_CODE; + break; + } + + /* Subsequent code chunk ("..." or "... %>") */ + case T_STATE_CODE_NEXT: + /* Found "%>" */ + if( (match = strfind(&data->buf[off], data->bufsize - off, T_TOK_END, strlen(T_TOK_END))) != NULL ) + { + genflags = ( data->state == T_STATE_CODE_INIT ) + ? (T_GEN_START | T_GEN_END) : T_GEN_END; + + readlen = (int)(match - &data->buf[off]); + + /* Check for trailing '-' */ + if( (match > data->buf) && (*(match-1) == T_TOK_SKIPWS[0]) ) + { + readlen--; + data->flags |= T_FLAG_SKIPWS; + } + + bufout(data, &data->buf[off], readlen); + + data->state = T_STATE_TEXT_INIT; + data->bufsize -= ((int)(match - &data->buf[off]) + strlen(T_TOK_END) + off); + bufmove(data, &match[strlen(T_TOK_END)]); + } + + /* Code chunk */ + else + { + genflags = ( data->state == T_STATE_CODE_INIT ) ? T_GEN_START : 0; + + /* Preserve trailing "%" and "-", maybe an end token */ + vague = etokscan(data); + readlen = data->bufsize - off - vague; + bufout(data, &data->buf[off], readlen); + + data->state = T_STATE_CODE_NEXT; + data->bufsize = vague; + bufmove(data, &data->buf[readlen+off]); + } + + if( ignore || (data->outsize == 0 && !genflags) ) + continue; + else + return generate_expression(data, sz, genflags); + + break; + } + } + + *sz = 0; + return NULL; +} + + diff --git a/libs/web/src/template_parser.h b/libs/web/src/template_parser.h new file mode 100644 index 000000000..42ebdff69 --- /dev/null +++ b/libs/web/src/template_parser.h @@ -0,0 +1,81 @@ +/* + * LuCI Template - Parser header + * + * Copyright (C) 2009 Jo-Philipp Wich + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _TEMPLATE_PARSER_H_ +#define _TEMPLATE_PARSER_H_ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + + +#define T_READBUFSZ 1024 +#define T_OUTBUFSZ T_READBUFSZ * 3 + +/* parser states */ +#define T_STATE_TEXT_INIT 0 +#define T_STATE_TEXT_NEXT 1 +#define T_STATE_CODE_INIT 2 +#define T_STATE_CODE_NEXT 3 +#define T_STATE_SKIP 4 + +/* parser flags */ +#define T_FLAG_EOF 0x01 +#define T_FLAG_SKIPWS 0x02 + +/* tokens used in matching and expression generation */ +#define T_TOK_START "<%" +#define T_TOK_END "%>" +#define T_TOK_SKIPWS "-" +#define T_TOK_I18NSEP "\", \"" + +/* generator flags */ +#define T_GEN_START 0x01 +#define T_GEN_END 0x02 + +/* code types */ +#define T_TYPE_TEXT 0 +#define T_TYPE_COMMENT 1 +#define T_TYPE_EXPR 2 +#define T_TYPE_INCLUDE 3 +#define T_TYPE_I18N 4 +#define T_TYPE_CODE 5 + +/* parser state */ +struct template_parser { + int fd; + int bufsize; + int outsize; + int state; + int flags; + int type; + char buf[T_READBUFSZ]; + char out[T_OUTBUFSZ]; +}; + + +const char *template_reader(lua_State *L, void *ud, size_t *sz); + +#endif -- cgit v1.2.3