contrib, build: bundle LuaSrcDiet and make it available in build targets

1 files changed, 670 insertions, 0 deletions

diff --git a/contrib/luasrcdiet/lua/LuaSrcDiet.lua b/contrib/luasrcdiet/lua/LuaSrcDiet.lua
new file mode 100755
index 0000000000..d9a41f24bc
--- /dev/null
+++ b/contrib/luasrcdiet/lua/LuaSrcDiet.lua
@@ -0,0 +1,670 @@
+#!/usr/bin/env lua
+--[[--------------------------------------------------------------------
+
+  LuaSrcDiet
+  Compresses Lua source code by removing unnecessary characters.
+  For Lua 5.1.x source code.
+
+  Copyright (c) 2008 Kein-Hong Man <khman@users.sf.net>
+  The COPYRIGHT file describes the conditions
+  under which this software may be distributed.
+
+  See the ChangeLog for more information.
+
+----------------------------------------------------------------------]]
+
+--[[--------------------------------------------------------------------
+-- NOTES:
+-- * Remember to update version and date information below (MSG_TITLE)
+-- * TODO: to implement pcall() to properly handle lexer etc. errors
+-- * TODO: verify token stream or double-check binary chunk?
+-- * TODO: need some automatic testing for a semblance of sanity
+-- * TODO: the plugin module is highly experimental and unstable
+----------------------------------------------------------------------]]
+
+-- standard libraries, functions
+local string = string
+local math = math
+local table = table
+local require = require
+local print = print
+local sub = string.sub
+local gmatch = string.gmatch
+
+-- support modules
+local llex = require "llex"
+local lparser = require "lparser"
+local optlex = require "optlex"
+local optparser = require "optparser"
+local plugin
+
+--[[--------------------------------------------------------------------
+-- messages and textual data
+----------------------------------------------------------------------]]
+
+local MSG_TITLE = [[
+LuaSrcDiet: Puts your Lua 5.1 source code on a diet
+Version 0.11.2 (20080608)  Copyright (c) 2005-2008 Kein-Hong Man
+The COPYRIGHT file describes the conditions under which this
+software may be distributed.
+]]
+
+local MSG_USAGE = [[
+usage: LuaSrcDiet [options] [filenames]
+
+example:
+  >LuaSrcDiet myscript.lua -o myscript_.lua
+
+options:
+  -v, --version       prints version information
+  -h, --help          prints usage information
+  -o <file>           specify file name to write output
+  -s <suffix>         suffix for output files (default '_')
+  --keep <msg>        keep block comment with <msg> inside
+  --plugin <module>   run <module> in plugin/ directory
+  -                   stop handling arguments
+
+  (optimization levels)
+  --none              all optimizations off (normalizes EOLs only)
+  --basic             lexer-based optimizations only
+  --maximum           maximize reduction of source
+
+  (informational)
+  --quiet             process files quietly
+  --read-only         read file and print token stats only
+  --dump-lexer        dump raw tokens from lexer to stdout
+  --dump-parser       dump variable tracking tables from parser
+  --details           extra info (strings, numbers, locals)
+
+features (to disable, insert 'no' prefix like --noopt-comments):
+%s
+default settings:
+%s]]
+
+------------------------------------------------------------------------
+-- optimization options, for ease of switching on and off
+-- * positive to enable optimization, negative (no) to disable
+-- * these options should follow --opt-* and --noopt-* style for now
+------------------------------------------------------------------------
+
+local OPTION = [[
+--opt-comments,'remove comments and block comments'
+--opt-whitespace,'remove whitespace excluding EOLs'
+--opt-emptylines,'remove empty lines'
+--opt-eols,'all above, plus remove unnecessary EOLs'
+--opt-strings,'optimize strings and long strings'
+--opt-numbers,'optimize numbers'
+--opt-locals,'optimize local variable names'
+--opt-entropy,'tries to reduce symbol entropy of locals'
+]]
+
+-- preset configuration
+local DEFAULT_CONFIG = [[
+  --opt-comments --opt-whitespace --opt-emptylines
+  --opt-numbers --opt-locals
+]]
+-- override configurations: MUST explicitly enable/disable everything
+local BASIC_CONFIG = [[
+  --opt-comments --opt-whitespace --opt-emptylines
+  --noopt-eols --noopt-strings --noopt-numbers
+  --noopt-locals
+]]
+local MAXIMUM_CONFIG = [[
+  --opt-comments --opt-whitespace --opt-emptylines
+  --opt-eols --opt-strings --opt-numbers
+  --opt-locals --opt-entropy
+]]
+local NONE_CONFIG = [[
+  --noopt-comments --noopt-whitespace --noopt-emptylines
+  --noopt-eols --noopt-strings --noopt-numbers
+  --noopt-locals
+]]
+
+local DEFAULT_SUFFIX = "_"      -- default suffix for file renaming
+local PLUGIN_SUFFIX = "plugin/" -- relative location of plugins
+
+--[[--------------------------------------------------------------------
+-- startup and initialize option list handling
+----------------------------------------------------------------------]]
+
+-- simple error message handler; change to error if traceback wanted
+local function die(msg)
+  print("LuaSrcDiet: "..msg); os.exit()
+end
+--die = error--DEBUG
+
+if not string.match(_VERSION, "5.1", 1, 1) then  -- sanity check
+  die("requires Lua 5.1 to run")
+end
+
+------------------------------------------------------------------------
+-- prepares text for list of optimizations, prepare lookup table
+------------------------------------------------------------------------
+
+local MSG_OPTIONS = ""
+do
+  local WIDTH = 24
+  local o = {}
+  for op, desc in gmatch(OPTION, "%s*([^,]+),'([^']+)'") do
+    local msg = "  "..op
+    msg = msg..string.rep(" ", WIDTH - #msg)..desc.."\n"
+    MSG_OPTIONS = MSG_OPTIONS..msg
+    o[op] = true
+    o["--no"..sub(op, 3)] = true
+  end
+  OPTION = o  -- replace OPTION with lookup table
+end
+
+MSG_USAGE = string.format(MSG_USAGE, MSG_OPTIONS, DEFAULT_CONFIG)
+
+------------------------------------------------------------------------
+-- global variable initialization, option set handling
+------------------------------------------------------------------------
+
+local suffix = DEFAULT_SUFFIX           -- file suffix
+local option = {}                       -- program options
+local stat_c, stat_l                    -- statistics tables
+
+-- function to set option lookup table based on a text list of options
+-- note: additional forced settings for --opt-eols is done in optlex.lua
+local function set_options(CONFIG)
+  for op in gmatch(CONFIG, "(%-%-%S+)") do
+    if sub(op, 3, 4) == "no" and        -- handle negative options
+       OPTION["--"..sub(op, 5)] then
+      option[sub(op, 5)] = false
+    else
+      option[sub(op, 3)] = true
+    end
+  end
+end
+
+--[[--------------------------------------------------------------------
+-- support functions
+----------------------------------------------------------------------]]
+
+-- list of token types, parser-significant types are up to TTYPE_GRAMMAR
+-- while the rest are not used by parsers; arranged for stats display
+local TTYPES = {
+  "TK_KEYWORD", "TK_NAME", "TK_NUMBER",         -- grammar
+  "TK_STRING", "TK_LSTRING", "TK_OP",
+  "TK_EOS",
+  "TK_COMMENT", "TK_LCOMMENT",                  -- non-grammar
+  "TK_EOL", "TK_SPACE",
+}
+local TTYPE_GRAMMAR = 7
+
+local EOLTYPES = {                      -- EOL names for token dump
+  ["\n"] = "LF", ["\r"] = "CR",
+  ["\n\r"] = "LFCR", ["\r\n"] = "CRLF",
+}
+
+------------------------------------------------------------------------
+-- read source code from file
+------------------------------------------------------------------------
+
+local function load_file(fname)
+  local INF = io.open(fname, "rb")
+  if not INF then die("cannot open \""..fname.."\" for reading") end
+  local dat = INF:read("*a")
+  if not dat then die("cannot read from \""..fname.."\"") end
+  INF:close()
+  return dat
+end
+
+------------------------------------------------------------------------
+-- save source code to file
+------------------------------------------------------------------------
+
+local function save_file(fname, dat)
+  local OUTF = io.open(fname, "wb")
+  if not OUTF then die("cannot open \""..fname.."\" for writing") end
+  local status = OUTF:write(dat)
+  if not status then die("cannot write to \""..fname.."\"") end
+  OUTF:close()
+end
+
+------------------------------------------------------------------------
+-- functions to deal with statistics
+------------------------------------------------------------------------
+
+-- initialize statistics table
+local function stat_init()
+  stat_c, stat_l = {}, {}
+  for i = 1, #TTYPES do
+    local ttype = TTYPES[i]
+    stat_c[ttype], stat_l[ttype] = 0, 0
+  end
+end
+
+-- add a token to statistics table
+local function stat_add(tok, seminfo)
+  stat_c[tok] = stat_c[tok] + 1
+  stat_l[tok] = stat_l[tok] + #seminfo
+end
+
+-- do totals for statistics table, return average table
+local function stat_calc()
+  local function avg(c, l)                      -- safe average function
+    if c == 0 then return 0 end
+    return l / c
+  end
+  local stat_a = {}
+  local c, l = 0, 0
+  for i = 1, TTYPE_GRAMMAR do                   -- total grammar tokens
+    local ttype = TTYPES[i]
+    c = c + stat_c[ttype]; l = l + stat_l[ttype]
+  end
+  stat_c.TOTAL_TOK, stat_l.TOTAL_TOK = c, l
+  stat_a.TOTAL_TOK = avg(c, l)
+  c, l = 0, 0
+  for i = 1, #TTYPES do                         -- total all tokens
+    local ttype = TTYPES[i]
+    c = c + stat_c[ttype]; l = l + stat_l[ttype]
+    stat_a[ttype] = avg(stat_c[ttype], stat_l[ttype])
+  end
+  stat_c.TOTAL_ALL, stat_l.TOTAL_ALL = c, l
+  stat_a.TOTAL_ALL = avg(c, l)
+  return stat_a
+end
+
+--[[--------------------------------------------------------------------
+-- main tasks
+----------------------------------------------------------------------]]
+
+------------------------------------------------------------------------
+-- a simple token dumper, minimal translation of seminfo data
+------------------------------------------------------------------------
+
+local function dump_tokens(srcfl)
+  --------------------------------------------------------------------
+  -- load file and process source input into tokens
+  --------------------------------------------------------------------
+  local z = load_file(srcfl)
+  llex.init(z)
+  llex.llex()
+  local toklist, seminfolist = llex.tok, llex.seminfo
+  --------------------------------------------------------------------
+  -- display output
+  --------------------------------------------------------------------
+  for i = 1, #toklist do
+    local tok, seminfo = toklist[i], seminfolist[i]
+    if tok == "TK_OP" and string.byte(seminfo) < 32 then
+      seminfo = "(".. string.byte(seminfo)..")"
+    elseif tok == "TK_EOL" then
+      seminfo = EOLTYPES[seminfo]
+    else
+      seminfo = "'"..seminfo.."'"
+    end
+    print(tok.." "..seminfo)
+  end--for
+end
+
+----------------------------------------------------------------------
+-- parser dump; dump globalinfo and localinfo tables
+----------------------------------------------------------------------
+
+local function dump_parser(srcfl)
+  local print = print
+  --------------------------------------------------------------------
+  -- load file and process source input into tokens
+  --------------------------------------------------------------------
+  local z = load_file(srcfl)
+  llex.init(z)
+  llex.llex()
+  local toklist, seminfolist, toklnlist
+    = llex.tok, llex.seminfo, llex.tokln
+  --------------------------------------------------------------------
+  -- do parser optimization here
+  --------------------------------------------------------------------
+  lparser.init(toklist, seminfolist, toklnlist)
+  local globalinfo, localinfo = lparser.parser()
+  --------------------------------------------------------------------
+  -- display output
+  --------------------------------------------------------------------
+  local hl = string.rep("-", 72)
+  print("*** Local/Global Variable Tracker Tables ***")
+  print(hl.."\n GLOBALS\n"..hl)
+  -- global tables have a list of xref numbers only
+  for i = 1, #globalinfo do
+    local obj = globalinfo[i]
+    local msg = "("..i..") '"..obj.name.."' -> "
+    local xref = obj.xref
+    for j = 1, #xref do msg = msg..xref[j].." " end
+    print(msg)
+  end
+  -- local tables have xref numbers and a few other special
+  -- numbers that are specially named: decl (declaration xref),
+  -- act (activation xref), rem (removal xref)
+  print(hl.."\n LOCALS (decl=declared act=activated rem=removed)\n"..hl)
+  for i = 1, #localinfo do
+    local obj = localinfo[i]
+    local msg = "("..i..") '"..obj.name.."' decl:"..obj.decl..
+                " act:"..obj.act.." rem:"..obj.rem
+    if obj.isself then
+      msg = msg.." isself"
+    end
+    msg = msg.." -> "
+    local xref = obj.xref
+    for j = 1, #xref do msg = msg..xref[j].." " end
+    print(msg)
+  end
+  print(hl.."\n")
+end
+
+------------------------------------------------------------------------
+-- reads source file(s) and reports some statistics
+------------------------------------------------------------------------
+
+local function read_only(srcfl)
+  local print = print
+  --------------------------------------------------------------------
+  -- load file and process source input into tokens
+  --------------------------------------------------------------------
+  local z = load_file(srcfl)
+  llex.init(z)
+  llex.llex()
+  local toklist, seminfolist = llex.tok, llex.seminfo
+  print(MSG_TITLE)
+  print("Statistics for: "..srcfl.."\n")
+  --------------------------------------------------------------------
+  -- collect statistics
+  --------------------------------------------------------------------
+  stat_init()
+  for i = 1, #toklist do
+    local tok, seminfo = toklist[i], seminfolist[i]
+    stat_add(tok, seminfo)
+  end--for
+  local stat_a = stat_calc()
+  --------------------------------------------------------------------
+  -- display output
+  --------------------------------------------------------------------
+  local fmt = string.format
+  local function figures(tt)
+    return stat_c[tt], stat_l[tt], stat_a[tt]
+  end
+  local tabf1, tabf2 = "%-16s%8s%8s%10s", "%-16s%8d%8d%10.2f"
+  local hl = string.rep("-", 42)
+  print(fmt(tabf1, "Lexical",  "Input", "Input", "Input"))
+  print(fmt(tabf1, "Elements", "Count", "Bytes", "Average"))
+  print(hl)
+  for i = 1, #TTYPES do
+    local ttype = TTYPES[i]
+    print(fmt(tabf2, ttype, figures(ttype)))
+    if ttype == "TK_EOS" then print(hl) end
+  end
+  print(hl)
+  print(fmt(tabf2, "Total Elements", figures("TOTAL_ALL")))
+  print(hl)
+  print(fmt(tabf2, "Total Tokens", figures("TOTAL_TOK")))
+  print(hl.."\n")
+end
+
+------------------------------------------------------------------------
+-- process source file(s), write output and reports some statistics
+------------------------------------------------------------------------
+
+local function process_file(srcfl, destfl)
+  local function print(...)             -- handle quiet option
+    if option.QUIET then return end
+    _G.print(...)
+  end
+  if plugin and plugin.init then        -- plugin init
+    option.EXIT = false
+    plugin.init(option, srcfl, destfl)
+    if option.EXIT then return end
+  end
+  print(MSG_TITLE)                      -- title message
+  --------------------------------------------------------------------
+  -- load file and process source input into tokens
+  --------------------------------------------------------------------
+  local z = load_file(srcfl)
+  if plugin and plugin.post_load then   -- plugin post-load
+    z = plugin.post_load(z) or z
+    if option.EXIT then return end
+  end
+  llex.init(z)
+  llex.llex()
+  local toklist, seminfolist, toklnlist
+    = llex.tok, llex.seminfo, llex.tokln
+  if plugin and plugin.post_lex then    -- plugin post-lex
+    plugin.post_lex(toklist, seminfolist, toklnlist)
+    if option.EXIT then return end
+  end
+  --------------------------------------------------------------------
+  -- collect 'before' statistics
+  --------------------------------------------------------------------
+  stat_init()
+  for i = 1, #toklist do
+    local tok, seminfo = toklist[i], seminfolist[i]
+    stat_add(tok, seminfo)
+  end--for
+  local stat1_a = stat_calc()
+  local stat1_c, stat1_l = stat_c, stat_l
+  --------------------------------------------------------------------
+  -- do parser optimization here
+  --------------------------------------------------------------------
+  if option["opt-locals"] then
+    optparser.print = print  -- hack
+    lparser.init(toklist, seminfolist, toklnlist)
+    local globalinfo, localinfo = lparser.parser()
+    if plugin and plugin.post_parse then        -- plugin post-parse
+      plugin.post_parse(globalinfo, localinfo)
+      if option.EXIT then return end
+    end
+    optparser.optimize(option, toklist, seminfolist, globalinfo, localinfo)
+    if plugin and plugin.post_optparse then     -- plugin post-optparse
+      plugin.post_optparse()
+      if option.EXIT then return end
+    end
+  end
+  --------------------------------------------------------------------
+  -- do lexer optimization here, save output file
+  --------------------------------------------------------------------
+  optlex.print = print  -- hack
+  toklist, seminfolist, toklnlist
+    = optlex.optimize(option, toklist, seminfolist, toklnlist)
+  if plugin and plugin.post_optlex then         -- plugin post-optlex
+    plugin.post_optlex(toklist, seminfolist, toklnlist)
+    if option.EXIT then return end
+  end
+  local dat = table.concat(seminfolist)
+  -- depending on options selected, embedded EOLs in long strings and
+  -- long comments may not have been translated to \n, tack a warning
+  if string.find(dat, "\r\n", 1, 1) or
+     string.find(dat, "\n\r", 1, 1) then
+    optlex.warn.mixedeol = true
+  end
+  -- save optimized source stream to output file
+  save_file(destfl, dat)
+  --------------------------------------------------------------------
+  -- collect 'after' statistics
+  --------------------------------------------------------------------
+  stat_init()
+  for i = 1, #toklist do
+    local tok, seminfo = toklist[i], seminfolist[i]
+    stat_add(tok, seminfo)
+  end--for
+  local stat_a = stat_calc()
+  --------------------------------------------------------------------
+  -- display output
+  --------------------------------------------------------------------
+  print("Statistics for: "..srcfl.." -> "..destfl.."\n")
+  local fmt = string.format
+  local function figures(tt)
+    return stat1_c[tt], stat1_l[tt], stat1_a[tt],
+           stat_c[tt],  stat_l[tt],  stat_a[tt]
+  end
+  local tabf1, tabf2 = "%-16s%8s%8s%10s%8s%8s%10s",
+                       "%-16s%8d%8d%10.2f%8d%8d%10.2f"
+  local hl = string.rep("-", 68)
+  print("*** lexer-based optimizations summary ***\n"..hl)
+  print(fmt(tabf1, "Lexical",
+            "Input", "Input", "Input",
+            "Output", "Output", "Output"))
+  print(fmt(tabf1, "Elements",
+            "Count", "Bytes", "Average",
+            "Count", "Bytes", "Average"))
+  print(hl)
+  for i = 1, #TTYPES do
+    local ttype = TTYPES[i]
+    print(fmt(tabf2, ttype, figures(ttype)))
+    if ttype == "TK_EOS" then print(hl) end
+  end
+  print(hl)
+  print(fmt(tabf2, "Total Elements", figures("TOTAL_ALL")))
+  print(hl)
+  print(fmt(tabf2, "Total Tokens", figures("TOTAL_TOK")))
+  print(hl)
+  --------------------------------------------------------------------
+  -- report warning flags from optimizing process
+  --------------------------------------------------------------------
+  if optlex.warn.lstring then
+    print("* WARNING: "..optlex.warn.lstring)
+  elseif optlex.warn.mixedeol then
+    print("* WARNING: ".."output still contains some CRLF or LFCR line endings")
+  end
+  print()
+end
+
+--[[--------------------------------------------------------------------
+-- main functions
+----------------------------------------------------------------------]]
+
+local arg = {...}  -- program arguments
+local fspec = {}
+set_options(DEFAULT_CONFIG)     -- set to default options at beginning
+
+------------------------------------------------------------------------
+-- per-file handling, ship off to tasks
+------------------------------------------------------------------------
+
+local function do_files(fspec)
+  for _, srcfl in ipairs(fspec) do
+    local destfl
+    ------------------------------------------------------------------
+    -- find and replace extension for filenames
+    ------------------------------------------------------------------
+    local extb, exte = string.find(srcfl, "%.[^%.%\\%/]*$")
+    local basename, extension = srcfl, ""
+    if extb and extb > 1 then
+      basename = sub(srcfl, 1, extb - 1)
+      extension = sub(srcfl, extb, exte)
+    end
+    destfl = basename..suffix..extension
+    if #fspec == 1 and option.OUTPUT_FILE then
+      destfl = option.OUTPUT_FILE
+    end
+    if srcfl == destfl then
+      die("output filename identical to input filename")
+    end
+    ------------------------------------------------------------------
+    -- perform requested operations
+    ------------------------------------------------------------------
+    if option.DUMP_LEXER then
+      dump_tokens(srcfl)
+    elseif option.DUMP_PARSER then
+      dump_parser(srcfl)
+    elseif option.READ_ONLY then
+      read_only(srcfl)
+    else
+      process_file(srcfl, destfl)
+    end
+  end--for
+end
+
+------------------------------------------------------------------------
+-- main function (entry point is after this definition)
+------------------------------------------------------------------------
+
+local function main()
+  local argn, i = #arg, 1
+  if argn == 0 then
+    option.HELP = true
+  end
+  --------------------------------------------------------------------
+  -- handle arguments
+  --------------------------------------------------------------------
+  while i <= argn do
+    local o, p = arg[i], arg[i + 1]
+    local dash = string.match(o, "^%-%-?")
+    if dash == "-" then                 -- single-dash options
+      if o == "-h" then
+        option.HELP = true; break
+      elseif o == "-v" then
+        option.VERSION = true; break
+      elseif o == "-s" then
+        if not p then die("-s option needs suffix specification") end
+        suffix = p
+        i = i + 1
+      elseif o == "-o" then
+        if not p then die("-o option needs a file name") end
+        option.OUTPUT_FILE = p
+        i = i + 1
+      elseif o == "-" then
+        break -- ignore rest of args
+      else
+        die("unrecognized option "..o)
+      end
+    elseif dash == "--" then            -- double-dash options
+      if o == "--help" then
+        option.HELP = true; break
+      elseif o == "--version" then
+        option.VERSION = true; break
+      elseif o == "--keep" then
+        if not p then die("--keep option needs a string to match for") end
+        option.KEEP = p
+        i = i + 1
+      elseif o == "--plugin" then
+        if not p then die("--plugin option needs a module name") end
+        if option.PLUGIN then die("only one plugin can be specified") end
+        option.PLUGIN = p
+        plugin = require(PLUGIN_SUFFIX..p)
+        i = i + 1
+      elseif o == "--quiet" then
+        option.QUIET = true
+      elseif o == "--read-only" then
+        option.READ_ONLY = true
+      elseif o == "--basic" then
+        set_options(BASIC_CONFIG)
+      elseif o == "--maximum" then
+        set_options(MAXIMUM_CONFIG)
+      elseif o == "--none" then
+        set_options(NONE_CONFIG)
+      elseif o == "--dump-lexer" then
+        option.DUMP_LEXER = true
+      elseif o == "--dump-parser" then
+        option.DUMP_PARSER = true
+      elseif o == "--details" then
+        option.DETAILS = true
+      elseif OPTION[o] then  -- lookup optimization options
+        set_options(o)
+      else
+        die("unrecognized option "..o)
+      end
+    else
+      fspec[#fspec + 1] = o             -- potential filename
+    end
+    i = i + 1
+  end--while
+  if option.HELP then
+    print(MSG_TITLE..MSG_USAGE); return true
+  elseif option.VERSION then
+    print(MSG_TITLE); return true
+  end
+  if #fspec > 0 then
+    if #fspec > 1 and option.OUTPUT_FILE then
+      die("with -o, only one source file can be specified")
+    end
+    do_files(fspec)
+    return true
+  else
+    die("nothing to do!")
+  end
+end
+
+-- entry point -> main() -> do_files()
+if not main() then
+  die("Please run with option -h or --help for usage information")
+end
+
+-- end of script