Rework LuCI build system

 * Rename subdirectories to their repective OpenWrt package names
 * Make each LuCI module its own standalone package
 * Deploy a shared luci.mk which is used by each module Makefile

Signed-off-by: Jo-Philipp Wich <jow@openwrt.org>

3 files changed, 678 insertions, 0 deletions

diff --git a/libs/luci-lib-httpclient/Makefile b/libs/luci-lib-httpclient/Makefile
new file mode 100644
index 000000000..1e7fd1bc8
--- /dev/null
+++ b/libs/luci-lib-httpclient/Makefile
@@ -0,0 +1,14 @@
+#
+# Copyright (C) 2008-2014 The LuCI Team <luci@lists.subsignal.org>
+#
+# This is free software, licensed under the Apache License, Version 2.0 .
+#
+
+include $(TOPDIR)/rules.mk
+
+LUCI_TITLE:=HTTP(S) client library
+LUCI_DEPENDS:=+luci-base +luci-lib-nixio
+
+include ../../luci.mk
+
+# call BuildPackage - OpenWrt buildroot signature
diff --git a/libs/luci-lib-httpclient/luasrc/httpclient.lua b/libs/luci-lib-httpclient/luasrc/httpclient.lua
new file mode 100644
index 000000000..e9fec5dbb
--- /dev/null
+++ b/libs/luci-lib-httpclient/luasrc/httpclient.lua
@@ -0,0 +1,369 @@
+--[[
+LuCI - Lua Development Framework
+
+Copyright 2009 Steven Barth <steven@midlink.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+$Id$
+]]--
+
+require "nixio.util"
+local nixio = require "nixio"
+
+local ltn12 = require "luci.ltn12"
+local util = require "luci.util"
+local table = require "table"
+local http = require "luci.http.protocol"
+local date = require "luci.http.protocol.date"
+
+local type, pairs, ipairs, tonumber = type, pairs, ipairs, tonumber
+local unpack = unpack
+
+module "luci.httpclient"
+
+function chunksource(sock, buffer)
+	buffer = buffer or ""
+	return function()
+		local output
+		local _, endp, count = buffer:find("^([0-9a-fA-F]+);?.-\r\n")
+		while not count and #buffer <= 1024 do
+			local newblock, code = sock:recv(1024 - #buffer)
+			if not newblock then
+				return nil, code
+			end
+			buffer = buffer .. newblock  
+			_, endp, count = buffer:find("^([0-9a-fA-F]+);?.-\r\n")
+		end
+		count = tonumber(count, 16)
+		if not count then
+			return nil, -1, "invalid encoding"
+		elseif count == 0 then
+			return nil
+		elseif count + 2 <= #buffer - endp then
+			output = buffer:sub(endp+1, endp+count)
+			buffer = buffer:sub(endp+count+3)
+			return output
+		else
+			output = buffer:sub(endp+1, endp+count)
+			buffer = ""
+			if count - #output > 0 then
+				local remain, code = sock:recvall(count-#output)
+				if not remain then
+					return nil, code
+				end
+				output = output .. remain
+				count, code = sock:recvall(2)
+			else
+				count, code = sock:recvall(count+2-#buffer+endp)
+			end
+			if not count then
+				return nil, code
+			end
+			return output
+		end
+	end
+end
+
+
+function request_to_buffer(uri, options)
+	local source, code, msg = request_to_source(uri, options)
+	local output = {}
+	
+	if not source then
+		return nil, code, msg
+	end
+	
+	source, code = ltn12.pump.all(source, (ltn12.sink.table(output)))
+	
+	if not source then
+		return nil, code
+	end
+	
+	return table.concat(output)
+end
+
+function request_to_source(uri, options)
+	local status, response, buffer, sock = request_raw(uri, options)
+	if not status then
+		return status, response, buffer
+	elseif status ~= 200 and status ~= 206 then
+		return nil, status, buffer
+	end
+	
+	if response.headers["Transfer-Encoding"] == "chunked" then
+		return chunksource(sock, buffer)
+	else
+		return ltn12.source.cat(ltn12.source.string(buffer), sock:blocksource())
+	end
+end
+
+--
+-- GET HTTP-resource
+--
+function request_raw(uri, options)
+	options = options or {}
+	local pr, auth, host, port, path
+	
+	if uri:find("%[") then
+		if uri:find("@") then
+			pr, auth, host, port, path = uri:match("(%w+)://(.+)@(%b[]):?([0-9]*)(.*)")
+			host = host:sub(2,-2)
+		else
+			pr, host, port, path = uri:match("(%w+)://(%b[]):?([0-9]*)(.*)")
+			host = host:sub(2,-2)
+		end
+	else
+		if uri:find("@") then
+			pr, auth, host, port, path =
+				uri:match("(%w+)://(.+)@([%w-.]+):?([0-9]*)(.*)")
+		else
+			pr, host, port, path = uri:match("(%w+)://([%w-.]+):?([0-9]*)(.*)")
+		end
+	end
+
+	if not host then
+		return nil, -1, "unable to parse URI"
+	end
+	
+	if pr ~= "http" and pr ~= "https" then
+		return nil, -2, "protocol not supported"
+	end
+	
+	port = #port > 0 and port or (pr == "https" and 443 or 80)
+	path = #path > 0 and path or "/"
+	
+	options.depth = options.depth or 10
+	local headers = options.headers or {}
+	local protocol = options.protocol or "HTTP/1.1"
+	headers["User-Agent"] = headers["User-Agent"] or "LuCI httpclient 0.1"
+	
+	if headers.Connection == nil then
+		headers.Connection = "close"
+	end
+	
+	if auth and not headers.Authorization then
+		headers.Authorization = "Basic " .. nixio.bin.b64encode(auth)
+	end
+
+	local sock, code, msg = nixio.connect(host, port)
+	if not sock then
+		return nil, code, msg
+	end
+	
+	sock:setsockopt("socket", "sndtimeo", options.sndtimeo or 15)
+	sock:setsockopt("socket", "rcvtimeo", options.rcvtimeo or 15)
+	
+	if pr == "https" then
+		local tls = options.tls_context or nixio.tls()
+		sock = tls:create(sock)
+		local stat, code, error = sock:connect()
+		if not stat then
+			return stat, code, error
+		end
+	end
+
+	-- Pre assemble fixes	
+	if protocol == "HTTP/1.1" then
+		headers.Host = headers.Host or host
+	end
+	
+	if type(options.body) == "table" then
+		options.body = http.urlencode_params(options.body)
+	end
+
+	if type(options.body) == "string" then
+		headers["Content-Length"] = headers["Content-Length"] or #options.body
+		headers["Content-Type"] = headers["Content-Type"] or
+			"application/x-www-form-urlencoded"
+		options.method = options.method or "POST"
+	end
+	
+	if type(options.body) == "function" then
+		options.method = options.method or "POST"
+	end
+
+	-- Assemble message
+	local message = {(options.method or "GET") .. " " .. path .. " " .. protocol}
+	
+	for k, v in pairs(headers) do
+		if type(v) == "string" or type(v) == "number" then
+			message[#message+1] = k .. ": " .. v
+		elseif type(v) == "table" then
+			for i, j in ipairs(v) do
+				message[#message+1] = k .. ": " .. j
+			end
+		end
+	end
+	
+	if options.cookies then
+		for _, c in ipairs(options.cookies) do
+			local cdo = c.flags.domain
+			local cpa = c.flags.path
+			if   (cdo == host or cdo == "."..host or host:sub(-#cdo) == cdo) 
+			 and (cpa == path or cpa == "/" or cpa .. "/" == path:sub(#cpa+1))
+			 and (not c.flags.secure or pr == "https")
+			then
+				message[#message+1] = "Cookie: " .. c.key .. "=" .. c.value
+			end 
+		end
+	end
+	
+	message[#message+1] = ""
+	message[#message+1] = ""
+	
+	-- Send request
+	sock:sendall(table.concat(message, "\r\n"))
+	
+	if type(options.body) == "string" then
+		sock:sendall(options.body)
+	elseif type(options.body) == "function" then
+		local res = {options.body(sock)}
+		if not res[1] then
+			sock:close()
+			return unpack(res)
+		end
+	end
+	
+	-- Create source and fetch response
+	local linesrc = sock:linesource()
+	local line, code, error = linesrc()
+	
+	if not line then
+		sock:close()
+		return nil, code, error
+	end
+	
+	local protocol, status, msg = line:match("^([%w./]+) ([0-9]+) (.*)")
+	
+	if not protocol then
+		sock:close()
+		return nil, -3, "invalid response magic: " .. line
+	end
+	
+	local response = {
+		status = line, headers = {}, code = 0, cookies = {}, uri = uri
+	}
+	
+	line = linesrc()
+	while line and line ~= "" do
+		local key, val = line:match("^([%w-]+)%s?:%s?(.*)")
+		if key and key ~= "Status" then
+			if type(response.headers[key]) == "string" then
+				response.headers[key] = {response.headers[key], val}
+			elseif type(response.headers[key]) == "table" then
+				response.headers[key][#response.headers[key]+1] = val
+			else
+				response.headers[key] = val
+			end
+		end
+		line = linesrc()
+	end
+	
+	if not line then
+		sock:close()
+		return nil, -4, "protocol error"
+	end
+	
+	-- Parse cookies
+	if response.headers["Set-Cookie"] then
+		local cookies = response.headers["Set-Cookie"]
+		for _, c in ipairs(type(cookies) == "table" and cookies or {cookies}) do
+			local cobj = cookie_parse(c)
+			cobj.flags.path = cobj.flags.path or path:match("(/.*)/?[^/]*")
+			if not cobj.flags.domain or cobj.flags.domain == "" then
+				cobj.flags.domain = host
+				response.cookies[#response.cookies+1] = cobj
+			else
+				local hprt, cprt = {}, {}
+				
+				-- Split hostnames and save them in reverse order
+				for part in host:gmatch("[^.]*") do
+					table.insert(hprt, 1, part)
+				end
+				for part in cobj.flags.domain:gmatch("[^.]*") do
+					table.insert(cprt, 1, part)
+				end
+				
+				local valid = true
+				for i, part in ipairs(cprt) do
+					-- If parts are different and no wildcard
+					if hprt[i] ~= part and #part ~= 0 then
+						valid = false
+						break
+					-- Wildcard on invalid position
+					elseif hprt[i] ~= part and #part == 0 then
+						if i ~= #cprt or (#hprt ~= i and #hprt+1 ~= i) then
+							valid = false
+							break
+						end
+					end
+				end
+				-- No TLD cookies
+				if valid and #cprt > 1 and #cprt[2] > 0 then
+					response.cookies[#response.cookies+1] = cobj
+				end
+			end
+		end
+	end
+	
+	-- Follow 
+	response.code = tonumber(status)
+	if response.code and options.depth > 0 then
+		if (response.code == 301 or response.code == 302 or response.code == 307)
+		 and response.headers.Location then
+			local nuri = response.headers.Location or response.headers.location
+			if not nuri then
+				return nil, -5, "invalid reference"
+			end
+			if not nuri:find("https?://") then
+				nuri = pr .. "://" .. host .. ":" .. port .. nuri
+			end
+			
+			options.depth = options.depth - 1
+			if options.headers then
+				options.headers.Host = nil
+			end
+			sock:close()
+			
+			return request_raw(nuri, options)
+		end
+	end
+	
+	return response.code, response, linesrc(true), sock
+end
+
+function cookie_parse(cookiestr)
+	local key, val, flags = cookiestr:match("%s?([^=;]+)=?([^;]*)(.*)")
+	if not key then
+		return nil
+	end
+
+	local cookie = {key = key, value = val, flags = {}}
+	for fkey, fval in flags:gmatch(";%s?([^=;]+)=?([^;]*)") do
+		fkey = fkey:lower()
+		if fkey == "expires" then
+			fval = date.to_unix(fval:gsub("%-", " "))
+		end
+		cookie.flags[fkey] = fval
+	end
+
+	return cookie
+end
+
+function cookie_create(cookie)
+	local cookiedata = {cookie.key .. "=" .. cookie.value}
+
+	for k, v in pairs(cookie.flags) do
+		if k == "expires" then
+			v = date.to_http(v):gsub(", (%w+) (%w+) (%w+) ", ", %1-%2-%3 ")
+		end
+		cookiedata[#cookiedata+1] = k .. ((#v > 0) and ("=" .. v) or "")
+	end
+
+	return table.concat(cookiedata, "; ")
+end
diff --git a/libs/luci-lib-httpclient/luasrc/httpclient/receiver.lua b/libs/luci-lib-httpclient/luasrc/httpclient/receiver.lua
new file mode 100644
index 000000000..4f08e93fe
--- /dev/null
+++ b/libs/luci-lib-httpclient/luasrc/httpclient/receiver.lua
@@ -0,0 +1,295 @@
+--[[
+LuCI - Lua Development Framework 
+
+Copyright 2009 Steven Barth <steven@midlink.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+$Id$
+]]--
+
+require "nixio.util"
+local nixio = require "nixio"
+local httpc = require "luci.httpclient"
+local ltn12 = require "luci.ltn12"
+
+local print, tonumber, require, unpack = print, tonumber, require, unpack
+
+module "luci.httpclient.receiver"
+
+local function prepare_fd(target)
+	-- Open fd for appending
+	local oflags = nixio.open_flags("wronly", "creat")
+	local file, code, msg = nixio.open(target, oflags)
+	if not file then
+		return file, code, msg
+	end
+	
+	-- Acquire lock
+	local stat, code, msg = file:lock("tlock")
+	if not stat then
+		return stat, code, msg
+	end
+	
+	file:seek(0, "end") 
+	
+	return file
+end
+
+local function splice_async(sock, pipeout, pipein, file, cb)
+	local ssize = 65536
+	local smode = nixio.splice_flags("move", "more", "nonblock")
+
+	-- Set pipe non-blocking otherwise we might end in a deadlock
+	local stat, code, msg = pipein:setblocking(false)
+	if stat then
+		stat, code, msg = pipeout:setblocking(false)
+	end
+	if not stat then
+		return stat, code, msg
+	end
+	
+	
+	local pollsock = {
+		{fd=sock, events=nixio.poll_flags("in")}
+	}
+	
+	local pollfile = {
+		{fd=file, events=nixio.poll_flags("out")}
+	}
+	
+	local done
+	local active -- Older splice implementations sometimes don't detect EOS
+	
+	repeat
+		active = false
+		
+		-- Socket -> Pipe
+		repeat
+			nixio.poll(pollsock, 15000)
+		
+			stat, code, msg = nixio.splice(sock, pipeout, ssize, smode)
+			if stat == nil then
+				return stat, code, msg
+			elseif stat == 0 then
+				done = true
+				break
+			elseif stat then
+				active = true
+			end
+		until stat == false
+		
+		-- Pipe -> File
+		repeat
+			nixio.poll(pollfile, 15000)
+		
+			stat, code, msg = nixio.splice(pipein, file, ssize, smode)
+			if stat == nil then
+				return stat, code, msg
+			elseif stat then
+				active = true
+			end
+		until stat == false
+		
+		if cb then
+			cb(file)
+		end
+		
+		if not active then
+			-- We did not splice any data, maybe EOS, fallback to default
+			return false
+		end
+	until done
+	
+	pipein:close()
+	pipeout:close()
+	sock:close()
+	file:close()
+	return true
+end
+
+local function splice_sync(sock, pipeout, pipein, file, cb)
+	local os = require "os"
+	local ssize = 65536
+	local smode = nixio.splice_flags("move", "more")
+	local stat
+	
+	-- This is probably the only forking http-client ;-)
+	local pid, code, msg = nixio.fork()
+	if not pid then
+		return pid, code, msg
+	elseif pid == 0 then
+		pipein:close()
+		file:close()
+
+		repeat
+			stat, code = nixio.splice(sock, pipeout, ssize, smode)
+		until not stat or stat == 0
+	
+		pipeout:close()
+		sock:close()
+		os.exit(stat or code)
+	else
+		pipeout:close()
+		sock:close()
+		
+		repeat
+			stat, code, msg = nixio.splice(pipein, file, ssize, smode)
+			if cb then
+				cb(file)
+			end
+		until not stat or stat == 0
+		
+		pipein:close()
+		file:close()
+		
+		if not stat then
+			nixio.kill(pid, 15)
+			nixio.wait(pid)
+			return stat, code, msg
+		else
+			pid, msg, code = nixio.wait(pid)
+			if msg == "exited" then
+				if code == 0 then
+					return true
+				else
+					return nil, code, nixio.strerror(code)
+				end
+			else
+				return nil, -0x11, "broken pump"
+			end
+		end
+	end
+end
+
+function request_to_file(uri, target, options, cbs)
+	options = options or {}
+	cbs = cbs or {}
+	options.headers = options.headers or {}
+	local hdr = options.headers
+	local file, code, msg
+	
+	if target then
+		file, code, msg = prepare_fd(target)
+		if not file then
+			return file, code, msg
+		end
+	
+		local off = file:tell()
+		
+		-- Set content range
+		if off > 0 then
+			hdr.Range = hdr.Range or ("bytes=" .. off .. "-")  
+		end
+	end
+	
+	local code, resp, buffer, sock = httpc.request_raw(uri, options)
+	if not code then
+		-- No success
+		if file then
+			file:close()
+		end
+		return code, resp, buffer
+	elseif hdr.Range and code ~= 206 then
+		-- We wanted a part but we got the while file
+		sock:close()
+		if file then
+			file:close()
+		end
+		return nil, -4, code, resp
+	elseif not hdr.Range and code ~= 200 then
+		-- We encountered an error
+		sock:close()
+		if file then
+			file:close()
+		end
+		return nil, -4, code, resp
+	end
+	
+	if cbs.on_header then
+		local stat = {cbs.on_header(file, code, resp)}
+		if stat[1] == false then
+			if file then
+				file:close()
+			end
+			sock:close()
+			return unpack(stat)
+		elseif stat[2] then
+			file = file and stat[2]
+		end
+	end
+	
+	if not file then
+		return nil, -5, "no target given"
+	end
+
+	local chunked = resp.headers["Transfer-Encoding"] == "chunked"
+	local stat
+
+	-- Write the buffer to file
+	file:writeall(buffer)
+	
+	repeat
+		if not options.splice or not sock:is_socket() or chunked then
+			break
+		end
+		
+		-- This is a plain TCP socket and there is no encoding so we can splice
+	
+		local pipein, pipeout, msg = nixio.pipe()
+		if not pipein then
+			sock:close()
+			file:close()
+			return pipein, pipeout, msg
+		end
+		
+		
+		-- Adjust splice values
+		local ssize = 65536
+		local smode = nixio.splice_flags("move", "more")
+		
+		-- Splicing 512 bytes should never block on a fresh pipe
+		local stat, code, msg = nixio.splice(sock, pipeout, 512, smode)
+		if stat == nil then
+			break
+		end
+		
+		-- Now do the real splicing
+		local cb = cbs.on_write
+		if options.splice == "asynchronous" then
+			stat, code, msg = splice_async(sock, pipeout, pipein, file, cb)
+		elseif options.splice == "synchronous" then
+			stat, code, msg = splice_sync(sock, pipeout, pipein, file, cb)
+		else
+			break
+		end
+		
+		if stat == false then
+			break
+		end
+
+		return stat, code, msg
+	until true
+	
+	local src = chunked and httpc.chunksource(sock) or sock:blocksource()
+	local snk = file:sink()
+	
+	if cbs.on_write then
+		src = ltn12.source.chain(src, function(chunk)
+			cbs.on_write(file)
+			return chunk
+		end)
+	end
+	
+	-- Fallback to read/write
+	stat, code, msg = ltn12.pump.all(src, snk)
+
+	file:close()
+	sock:close()
+	return stat and true, code, msg
+end
+