diff options
Diffstat (limited to 'contrib/luasrcdiet/lua/optlex.lua')
| -rw-r--r-- | contrib/luasrcdiet/lua/optlex.lua | 832 |
1 files changed, 832 insertions, 0 deletions
diff --git a/contrib/luasrcdiet/lua/optlex.lua b/contrib/luasrcdiet/lua/optlex.lua new file mode 100644 index 0000000000..4c46b918bf --- /dev/null +++ b/contrib/luasrcdiet/lua/optlex.lua @@ -0,0 +1,832 @@ +--[[-------------------------------------------------------------------- + + optlex.lua: does lexer-based optimizations + This file is part of LuaSrcDiet. + + Copyright (c) 2008 Kein-Hong Man <khman@users.sf.net> + The COPYRIGHT file describes the conditions + under which this software may be distributed. + + See the ChangeLog for more information. + +----------------------------------------------------------------------]] + +--[[-------------------------------------------------------------------- +-- NOTES: +-- * For more lexer-based optimization ideas, see the TODO items or +-- look at technotes.txt. +-- * TODO: general string delimiter conversion optimizer +-- * TODO: (numbers) warn if overly significant digit +----------------------------------------------------------------------]] + +local base = _G +local string = require "string" +module "optlex" +local match = string.match +local sub = string.sub +local find = string.find +local rep = string.rep +local print + +------------------------------------------------------------------------ +-- variables and data structures +------------------------------------------------------------------------ + +-- error function, can override by setting own function into module +error = base.error + +warn = {} -- table for warning flags + +local stoks, sinfos, stoklns -- source lists + +local is_realtoken = { -- significant (grammar) tokens + TK_KEYWORD = true, + TK_NAME = true, + TK_NUMBER = true, + TK_STRING = true, + TK_LSTRING = true, + TK_OP = true, + TK_EOS = true, +} +local is_faketoken = { -- whitespace (non-grammar) tokens + TK_COMMENT = true, + TK_LCOMMENT = true, + TK_EOL = true, + TK_SPACE = true, +} + +local opt_details -- for extra information + +------------------------------------------------------------------------ +-- true if current token is at the start of a line +-- * skips over deleted tokens via recursion +------------------------------------------------------------------------ + +local function atlinestart(i) + local tok = stoks[i - 1] + if i <= 1 or tok == "TK_EOL" then + return true + elseif tok == "" then + return atlinestart(i - 1) + end + return false +end + +------------------------------------------------------------------------ +-- true if current token is at the end of a line +-- * skips over deleted tokens via recursion +------------------------------------------------------------------------ + +local function atlineend(i) + local tok = stoks[i + 1] + if i >= #stoks or tok == "TK_EOL" or tok == "TK_EOS" then + return true + elseif tok == "" then + return atlineend(i + 1) + end + return false +end + +------------------------------------------------------------------------ +-- counts comment EOLs inside a long comment +-- * in order to keep line numbering, EOLs need to be reinserted +------------------------------------------------------------------------ + +local function commenteols(lcomment) + local sep = #match(lcomment, "^%-%-%[=*%[") + local z = sub(lcomment, sep + 1, -(sep - 1)) -- remove delims + local i, c = 1, 0 + while true do + local p, q, r, s = find(z, "([\r\n])([\r\n]?)", i) + if not p then break end -- if no matches, done + i = p + 1 + c = c + 1 + if #s > 0 and r ~= s then -- skip CRLF or LFCR + i = i + 1 + end + end + return c +end + +------------------------------------------------------------------------ +-- compares two tokens (i, j) and returns the whitespace required +-- * important! see technotes.txt for more information +-- * only two grammar/real tokens are being considered +-- * if "", no separation is needed +-- * if " ", then at least one whitespace (or EOL) is required +------------------------------------------------------------------------ + +local function checkpair(i, j) + local match = match + local t1, t2 = stoks[i], stoks[j] + -------------------------------------------------------------------- + if t1 == "TK_STRING" or t1 == "TK_LSTRING" or + t2 == "TK_STRING" or t2 == "TK_LSTRING" then + return "" + -------------------------------------------------------------------- + elseif t1 == "TK_OP" or t2 == "TK_OP" then + if (t1 == "TK_OP" and (t2 == "TK_KEYWORD" or t2 == "TK_NAME")) or + (t2 == "TK_OP" and (t1 == "TK_KEYWORD" or t1 == "TK_NAME")) then + return "" + end + if t1 == "TK_OP" and t2 == "TK_OP" then + -- for TK_OP/TK_OP pairs, see notes in technotes.txt + local op, op2 = sinfos[i], sinfos[j] + if (match(op, "^%.%.?$") and match(op2, "^%.")) or + (match(op, "^[~=<>]$") and op2 == "=") or + (op == "[" and (op2 == "[" or op2 == "=")) then + return " " + end + return "" + end + -- "TK_OP" + "TK_NUMBER" case + local op = sinfos[i] + if t2 == "TK_OP" then op = sinfos[j] end + if match(op, "^%.%.?%.?$") then + return " " + end + return "" + -------------------------------------------------------------------- + else-- "TK_KEYWORD" | "TK_NAME" | "TK_NUMBER" then + return " " + -------------------------------------------------------------------- + end +end + +------------------------------------------------------------------------ +-- repack tokens, removing deletions caused by optimization process +------------------------------------------------------------------------ + +local function repack_tokens() + local dtoks, dinfos, dtoklns = {}, {}, {} + local j = 1 + for i = 1, #stoks do + local tok = stoks[i] + if tok ~= "" then + dtoks[j], dinfos[j], dtoklns[j] = tok, sinfos[i], stoklns[i] + j = j + 1 + end + end + stoks, sinfos, stoklns = dtoks, dinfos, dtoklns +end + +------------------------------------------------------------------------ +-- number optimization +-- * optimization using string formatting functions is one way of doing +-- this, but here, we consider all cases and handle them separately +-- (possibly an idiotic approach...) +-- * scientific notation being generated is not in canonical form, this +-- may or may not be a bad thing, feedback welcome +-- * note: intermediate portions need to fit into a normal number range +-- * optimizations can be divided based on number patterns: +-- * hexadecimal: +-- (1) no need to remove leading zeros, just skip to (2) +-- (2) convert to integer if size equal or smaller +-- * change if equal size -> lose the 'x' to reduce entropy +-- (3) number is then processed as an integer +-- (4) note: does not make 0[xX] consistent +-- * integer: +-- (1) note: includes anything with trailing ".", ".0", ... +-- (2) remove useless fractional part, if present, e.g. 123.000 +-- (3) remove leading zeros, e.g. 000123 +-- (4) switch to scientific if shorter, e.g. 123000 -> 123e3 +-- * with fraction: +-- (1) split into digits dot digits +-- (2) if no integer portion, take as zero (can omit later) +-- (3) handle degenerate .000 case, after which the fractional part +-- must be non-zero (if zero, it's matched as an integer) +-- (4) remove trailing zeros for fractional portion +-- (5) p.q where p > 0 and q > 0 cannot be shortened any more +-- (6) otherwise p == 0 and the form is .q, e.g. .000123 +-- (7) if scientific shorter, convert, e.g. .000123 -> 123e-6 +-- * scientific: +-- (1) split into (digits dot digits) [eE] ([+-] digits) +-- (2) if significand has ".", shift it out so it becomes an integer +-- (3) if significand is zero, just use zero +-- (4) remove leading zeros for significand +-- (5) shift out trailing zeros for significand +-- (6) examine exponent and determine which format is best: +-- integer, with fraction, scientific +------------------------------------------------------------------------ + +local function do_number(i) + local before = sinfos[i] -- 'before' + local z = before -- working representation + local y -- 'after', if better + -------------------------------------------------------------------- + if match(z, "^0[xX]") then -- hexadecimal number + local v = base.tostring(base.tonumber(z)) + if #v <= #z then + z = v -- change to integer, AND continue + else + return -- no change; stick to hex + end + end + -------------------------------------------------------------------- + if match(z, "^%d+%.?0*$") then -- integer or has useless frac + z = match(z, "^(%d+)%.?0*$") -- int portion only + if z + 0 > 0 then + z = match(z, "^0*([1-9]%d*)$") -- remove leading zeros + local v = #match(z, "0*$") + local nv = base.tostring(v) + if v > #nv + 1 then -- scientific is shorter + z = sub(z, 1, #z - v).."e"..nv + end + y = z + else + y = "0" -- basic zero + end + -------------------------------------------------------------------- + elseif not match(z, "[eE]") then -- number with fraction part + local p, q = match(z, "^(%d*)%.(%d+)$") -- split + if p == "" then p = 0 end -- int part zero + if q + 0 == 0 and p == 0 then + y = "0" -- degenerate .000 case + else + -- now, q > 0 holds and p is a number + local v = #match(q, "0*$") -- remove trailing zeros + if v > 0 then + q = sub(q, 1, #q - v) + end + -- if p > 0, nothing else we can do to simplify p.q case + if p + 0 > 0 then + y = p.."."..q + else + y = "."..q -- tentative, e.g. .000123 + local v = #match(q, "^0*") -- # leading spaces + local w = #q - v -- # significant digits + local nv = base.tostring(#q) + -- e.g. compare 123e-6 versus .000123 + if w + 2 + #nv < 1 + #q then + y = sub(q, -w).."e-"..nv + end + end + end + -------------------------------------------------------------------- + else -- scientific number + local sig, ex = match(z, "^([^eE]+)[eE]([%+%-]?%d+)$") + ex = base.tonumber(ex) + -- if got ".", shift out fractional portion of significand + local p, q = match(sig, "^(%d*)%.(%d*)$") + if p then + ex = ex - #q + sig = p..q + end + if sig + 0 == 0 then + y = "0" -- basic zero + else + local v = #match(sig, "^0*") -- remove leading zeros + sig = sub(sig, v + 1) + v = #match(sig, "0*$") -- shift out trailing zeros + if v > 0 then + sig = sub(sig, 1, #sig - v) + ex = ex + v + end + -- examine exponent and determine which format is best + local nex = base.tostring(ex) + if ex == 0 then -- it's just an integer + y = sig + elseif ex > 0 and (ex <= 1 + #nex) then -- a number + y = sig..rep("0", ex) + elseif ex < 0 and (ex >= -#sig) then -- fraction, e.g. .123 + v = #sig + ex + y = sub(sig, 1, v).."."..sub(sig, v + 1) + elseif ex < 0 and (#nex >= -ex - #sig) then + -- e.g. compare 1234e-5 versus .01234 + -- gives: #sig + 1 + #nex >= 1 + (-ex - #sig) + #sig + -- -> #nex >= -ex - #sig + v = -ex - #sig + y = "."..rep("0", v)..sig + else -- non-canonical scientific representation + y = sig.."e"..ex + end + end--if sig + end + -------------------------------------------------------------------- + if y and y ~= sinfos[i] then + if opt_details then + print("<number> (line "..stoklns[i]..") "..sinfos[i].." -> "..y) + opt_details = opt_details + 1 + end + sinfos[i] = y + end +end + +------------------------------------------------------------------------ +-- string optimization +-- * note: works on well-formed strings only! +-- * optimizations on characters can be summarized as follows: +-- \a\b\f\n\r\t\v -- no change +-- \\ -- no change +-- \"\' -- depends on delim, other can remove \ +-- \[\] -- remove \ +-- \<char> -- general escape, remove \ +-- \<eol> -- normalize the EOL only +-- \ddd -- if \a\b\f\n\r\t\v, change to latter +-- if other < ascii 32, keep ddd but zap leading zeros +-- if >= ascii 32, translate it into the literal, then also +-- do escapes for \\,\",\' cases +-- <other> -- no change +-- * switch delimiters if string becomes shorter +------------------------------------------------------------------------ + +local function do_string(I) + local info = sinfos[I] + local delim = sub(info, 1, 1) -- delimiter used + local ndelim = (delim == "'") and '"' or "'" -- opposite " <-> ' + local z = sub(info, 2, -2) -- actual string + local i = 1 + local c_delim, c_ndelim = 0, 0 -- "/' counts + -------------------------------------------------------------------- + while i <= #z do + local c = sub(z, i, i) + ---------------------------------------------------------------- + if c == "\\" then -- escaped stuff + local j = i + 1 + local d = sub(z, j, j) + local p = find("abfnrtv\\\n\r\"\'0123456789", d, 1, true) + ------------------------------------------------------------ + if not p then -- \<char> -- remove \ + z = sub(z, 1, i - 1)..sub(z, j) + i = i + 1 + ------------------------------------------------------------ + elseif p <= 8 then -- \a\b\f\n\r\t\v\\ + i = i + 2 -- no change + ------------------------------------------------------------ + elseif p <= 10 then -- \<eol> -- normalize EOL + local eol = sub(z, j, j + 1) + if eol == "\r\n" or eol == "\n\r" then + z = sub(z, 1, i).."\n"..sub(z, j + 2) + elseif p == 10 then -- \r case + z = sub(z, 1, i).."\n"..sub(z, j + 1) + end + i = i + 2 + ------------------------------------------------------------ + elseif p <= 12 then -- \"\' -- remove \ for ndelim + if d == delim then + c_delim = c_delim + 1 + i = i + 2 + else + c_ndelim = c_ndelim + 1 + z = sub(z, 1, i - 1)..sub(z, j) + i = i + 1 + end + ------------------------------------------------------------ + else -- \ddd -- various steps + local s = match(z, "^(%d%d?%d?)", j) + j = i + 1 + #s -- skip to location + local cv = s + 0 + local cc = string.char(cv) + local p = find("\a\b\f\n\r\t\v", cc, 1, true) + if p then -- special escapes + s = "\\"..sub("abfnrtv", p, p) + elseif cv < 32 then -- normalized \ddd + s = "\\"..cv + elseif cc == delim then -- \<delim> + s = "\\"..cc + c_delim = c_delim + 1 + elseif cc == "\\" then -- \\ + s = "\\\\" + else -- literal character + s = cc + if cc == ndelim then + c_ndelim = c_ndelim + 1 + end + end + z = sub(z, 1, i - 1)..s..sub(z, j) + i = i + #s + ------------------------------------------------------------ + end--if p + ---------------------------------------------------------------- + else-- c ~= "\\" -- <other> -- no change + i = i + 1 + if c == ndelim then -- count ndelim, for switching delimiters + c_ndelim = c_ndelim + 1 + end + ---------------------------------------------------------------- + end--if c + end--while + -------------------------------------------------------------------- + -- switching delimiters, a long-winded derivation: + -- (1) delim takes 2+2*c_delim bytes, ndelim takes c_ndelim bytes + -- (2) delim becomes c_delim bytes, ndelim becomes 2+2*c_ndelim bytes + -- simplifying the condition (1)>(2) --> c_delim > c_ndelim + if c_delim > c_ndelim then + i = 1 + while i <= #z do + local p, q, r = find(z, "([\'\"])", i) + if not p then break end + if r == delim then -- \<delim> -> <delim> + z = sub(z, 1, p - 2)..sub(z, p) + i = p + else-- r == ndelim -- <ndelim> -> \<ndelim> + z = sub(z, 1, p - 1).."\\"..sub(z, p) + i = p + 2 + end + end--while + delim = ndelim -- actually change delimiters + end + -------------------------------------------------------------------- + z = delim..z..delim + if z ~= sinfos[I] then + if opt_details then + print("<string> (line "..stoklns[I]..") "..sinfos[I].." -> "..z) + opt_details = opt_details + 1 + end + sinfos[I] = z + end +end + +------------------------------------------------------------------------ +-- long string optimization +-- * note: warning flagged if trailing whitespace found, not trimmed +-- * remove first optional newline +-- * normalize embedded newlines +-- * reduce '=' separators in delimiters if possible +------------------------------------------------------------------------ + +local function do_lstring(I) + local info = sinfos[I] + local delim1 = match(info, "^%[=*%[") -- cut out delimiters + local sep = #delim1 + local delim2 = sub(info, -sep, -1) + local z = sub(info, sep + 1, -(sep + 1)) -- lstring without delims + local y = "" + local i = 1 + -------------------------------------------------------------------- + while true do + local p, q, r, s = find(z, "([\r\n])([\r\n]?)", i) + -- deal with a single line + local ln + if not p then + ln = sub(z, i) + elseif p >= i then + ln = sub(z, i, p - 1) + end + if ln ~= "" then + -- flag a warning if there are trailing spaces, won't optimize! + if match(ln, "%s+$") then + warn.lstring = "trailing whitespace in long string near line "..stoklns[I] + end + y = y..ln + end + if not p then -- done if no more EOLs + break + end + -- deal with line endings, normalize them + i = p + 1 + if p then + if #s > 0 and r ~= s then -- skip CRLF or LFCR + i = i + 1 + end + -- skip first newline, which can be safely deleted + if not(i == 1 and i == p) then + y = y.."\n" + end + end + end--while + -------------------------------------------------------------------- + -- handle possible deletion of one or more '=' separators + if sep >= 3 then + local chk, okay = sep - 1 + -- loop to test ending delimiter with less of '=' down to zero + while chk >= 2 do + local delim = "%]"..rep("=", chk - 2).."%]" + if not match(y, delim) then okay = chk end + chk = chk - 1 + end + if okay then -- change delimiters + sep = rep("=", okay - 2) + delim1, delim2 = "["..sep.."[", "]"..sep.."]" + end + end + -------------------------------------------------------------------- + sinfos[I] = delim1..y..delim2 +end + +------------------------------------------------------------------------ +-- long comment optimization +-- * note: does not remove first optional newline +-- * trim trailing whitespace +-- * normalize embedded newlines +-- * reduce '=' separators in delimiters if possible +------------------------------------------------------------------------ + +local function do_lcomment(I) + local info = sinfos[I] + local delim1 = match(info, "^%-%-%[=*%[") -- cut out delimiters + local sep = #delim1 + local delim2 = sub(info, -sep, -1) + local z = sub(info, sep + 1, -(sep - 1)) -- comment without delims + local y = "" + local i = 1 + -------------------------------------------------------------------- + while true do + local p, q, r, s = find(z, "([\r\n])([\r\n]?)", i) + -- deal with a single line, extract and check trailing whitespace + local ln + if not p then + ln = sub(z, i) + elseif p >= i then + ln = sub(z, i, p - 1) + end + if ln ~= "" then + -- trim trailing whitespace if non-empty line + local ws = match(ln, "%s*$") + if #ws > 0 then ln = sub(ln, 1, -(ws + 1)) end + y = y..ln + end + if not p then -- done if no more EOLs + break + end + -- deal with line endings, normalize them + i = p + 1 + if p then + if #s > 0 and r ~= s then -- skip CRLF or LFCR + i = i + 1 + end + y = y.."\n" + end + end--while + -------------------------------------------------------------------- + -- handle possible deletion of one or more '=' separators + sep = sep - 2 + if sep >= 3 then + local chk, okay = sep - 1 + -- loop to test ending delimiter with less of '=' down to zero + while chk >= 2 do + local delim = "%]"..rep("=", chk - 2).."%]" + if not match(y, delim) then okay = chk end + chk = chk - 1 + end + if okay then -- change delimiters + sep = rep("=", okay - 2) + delim1, delim2 = "--["..sep.."[", "]"..sep.."]" + end + end + -------------------------------------------------------------------- + sinfos[I] = delim1..y..delim2 +end + +------------------------------------------------------------------------ +-- short comment optimization +-- * trim trailing whitespace +------------------------------------------------------------------------ + +local function do_comment(i) + local info = sinfos[i] + local ws = match(info, "%s*$") -- just look from end of string + if #ws > 0 then + info = sub(info, 1, -(ws + 1)) -- trim trailing whitespace + end + sinfos[i] = info +end + +------------------------------------------------------------------------ +-- returns true if string found in long comment +-- * this is a feature to keep copyright or license texts +------------------------------------------------------------------------ + +local function keep_lcomment(opt_keep, info) + if not opt_keep then return false end -- option not set + local delim1 = match(info, "^%-%-%[=*%[") -- cut out delimiters + local sep = #delim1 + local delim2 = sub(info, -sep, -1) + local z = sub(info, sep + 1, -(sep - 1)) -- comment without delims + if find(z, opt_keep, 1, true) then -- try to match + return true + end +end + +------------------------------------------------------------------------ +-- main entry point +-- * currently, lexer processing has 2 passes +-- * processing is done on a line-oriented basis, which is easier to +-- grok due to the next point... +-- * since there are various options that can be enabled or disabled, +-- processing is a little messy or convoluted +------------------------------------------------------------------------ + +function optimize(option, toklist, semlist, toklnlist) + -------------------------------------------------------------------- + -- set option flags + -------------------------------------------------------------------- + local opt_comments = option["opt-comments"] + local opt_whitespace = option["opt-whitespace"] + local opt_emptylines = option["opt-emptylines"] + local opt_eols = option["opt-eols"] + local opt_strings = option["opt-strings"] + local opt_numbers = option["opt-numbers"] + local opt_keep = option.KEEP + opt_details = option.DETAILS and 0 -- upvalues for details display + print = print or base.print + if opt_eols then -- forced settings, otherwise won't work properly + opt_comments = true + opt_whitespace = true + opt_emptylines = true + end + -------------------------------------------------------------------- + -- variable initialization + -------------------------------------------------------------------- + stoks, sinfos, stoklns -- set source lists + = toklist, semlist, toklnlist + local i = 1 -- token position + local tok, info -- current token + local prev -- position of last grammar token + -- on same line (for TK_SPACE stuff) + -------------------------------------------------------------------- + -- changes a token, info pair + -------------------------------------------------------------------- + local function settoken(tok, info, I) + I = I or i + stoks[I] = tok or "" + sinfos[I] = info or "" + end + -------------------------------------------------------------------- + -- processing loop (PASS 1) + -------------------------------------------------------------------- + while true do + tok, info = stoks[i], sinfos[i] + ---------------------------------------------------------------- + local atstart = atlinestart(i) -- set line begin flag + if atstart then prev = nil end + ---------------------------------------------------------------- + if tok == "TK_EOS" then -- end of stream/pass + break + ---------------------------------------------------------------- + elseif tok == "TK_KEYWORD" or -- keywords, identifiers, + tok == "TK_NAME" or -- operators + tok == "TK_OP" then + -- TK_KEYWORD and TK_OP can't be optimized without a big + -- optimization framework; it would be more of an optimizing + -- compiler, not a source code compressor + -- TK_NAME that are locals needs parser to analyze/optimize + prev = i + ---------------------------------------------------------------- + elseif tok == "TK_NUMBER" then -- numbers + if opt_numbers then + do_number(i) -- optimize + end + prev = i + ---------------------------------------------------------------- + elseif tok == "TK_STRING" or -- strings, long strings + tok == "TK_LSTRING" then + if opt_strings then + if tok == "TK_STRING" then + do_string(i) -- optimize + else + do_lstring(i) -- optimize + end + end + prev = i + ---------------------------------------------------------------- + elseif tok == "TK_COMMENT" then -- short comments + if opt_comments then + if i == 1 and sub(info, 1, 1) == "#" then + -- keep shbang comment, trim whitespace + do_comment(i) + else + -- safe to delete, as a TK_EOL (or TK_EOS) always follows + settoken() -- remove entirely + end + elseif opt_whitespace then -- trim whitespace only + do_comment(i) + end + ---------------------------------------------------------------- + elseif tok == "TK_LCOMMENT" then -- long comments + if keep_lcomment(opt_keep, info) then + ------------------------------------------------------------ + -- if --keep, we keep a long comment if <msg> is found; + -- this is a feature to keep copyright or license texts + if opt_whitespace then -- trim whitespace only + do_lcomment(i) + end + prev = i + elseif opt_comments then + local eols = commenteols(info) + ------------------------------------------------------------ + -- prepare opt_emptylines case first, if a disposable token + -- follows, current one is safe to dump, else keep a space; + -- it is implied that the operation is safe for '-', because + -- current is a TK_LCOMMENT, and must be separate from a '-' + if is_faketoken[stoks[i + 1]] then + settoken() -- remove entirely + tok = "" + else + settoken("TK_SPACE", " ") + end + ------------------------------------------------------------ + -- if there are embedded EOLs to keep and opt_emptylines is + -- disabled, then switch the token into one or more EOLs + if not opt_emptylines and eols > 0 then + settoken("TK_EOL", rep("\n", eols)) + end + ------------------------------------------------------------ + -- if optimizing whitespaces, force reinterpretation of the + -- token to give a chance for the space to be optimized away + if opt_whitespace and tok ~= "" then + i = i - 1 -- to reinterpret + end + ------------------------------------------------------------ + else -- disabled case + if opt_whitespace then -- trim whitespace only + do_lcomment(i) + end + prev = i + end + ---------------------------------------------------------------- + elseif tok == "TK_EOL" then -- line endings + if atstart and opt_emptylines then + settoken() -- remove entirely + elseif info == "\r\n" or info == "\n\r" then + -- normalize the rest of the EOLs for CRLF/LFCR only + -- (note that TK_LCOMMENT can change into several EOLs) + settoken("TK_EOL", "\n") + end + ---------------------------------------------------------------- + elseif tok == "TK_SPACE" then -- whitespace + if opt_whitespace then + if atstart or atlineend(i) then + -- delete leading and trailing whitespace + settoken() -- remove entirely + else + ------------------------------------------------------------ + -- at this point, since leading whitespace have been removed, + -- there should be a either a real token or a TK_LCOMMENT + -- prior to hitting this whitespace; the TK_LCOMMENT case + -- only happens if opt_comments is disabled; so prev ~= nil + local ptok = stoks[prev] + if ptok == "TK_LCOMMENT" then + -- previous TK_LCOMMENT can abut with anything + settoken() -- remove entirely + else + -- prev must be a grammar token; consecutive TK_SPACE + -- tokens is impossible when optimizing whitespace + local ntok = stoks[i + 1] + if is_faketoken[ntok] then + -- handle special case where a '-' cannot abut with + -- either a short comment or a long comment + if (ntok == "TK_COMMENT" or ntok == "TK_LCOMMENT") and + ptok == "TK_OP" and sinfos[prev] == "-" then + -- keep token + else + settoken() -- remove entirely + end + else--is_realtoken + -- check a pair of grammar tokens, if can abut, then + -- delete space token entirely, otherwise keep one space + local s = checkpair(prev, i + 1) + if s == "" then + settoken() -- remove entirely + else + settoken("TK_SPACE", " ") + end + end + end + ------------------------------------------------------------ + end + end + ---------------------------------------------------------------- + else + error("unidentified token encountered") + end + ---------------------------------------------------------------- + i = i + 1 + end--while + repack_tokens() + -------------------------------------------------------------------- + -- processing loop (PASS 2) + -------------------------------------------------------------------- + if opt_eols then + i = 1 + -- aggressive EOL removal only works with most non-grammar tokens + -- optimized away because it is a rather simple scheme -- basically + -- it just checks 'real' token pairs around EOLs + if stoks[1] == "TK_COMMENT" then + -- first comment still existing must be shbang, skip whole line + i = 3 + end + while true do + tok, info = stoks[i], sinfos[i] + -------------------------------------------------------------- + if tok == "TK_EOS" then -- end of stream/pass + break + -------------------------------------------------------------- + elseif tok == "TK_EOL" then -- consider each TK_EOL + local t1, t2 = stoks[i - 1], stoks[i + 1] + if is_realtoken[t1] and is_realtoken[t2] then -- sanity check + local s = checkpair(i - 1, i + 1) + if s == "" then + settoken() -- remove entirely + end + end + end--if tok + -------------------------------------------------------------- + i = i + 1 + end--while + repack_tokens() + end + -------------------------------------------------------------------- + if opt_details and opt_details > 0 then print() end -- spacing + return stoks, sinfos, stoklns +end |