diff options
Diffstat (limited to 'libs/luci-lib-httpclient')
| -rw-r--r-- | libs/luci-lib-httpclient/Makefile | 14 | ||||
| -rw-r--r-- | libs/luci-lib-httpclient/luasrc/httpclient.lua | 369 | ||||
| -rw-r--r-- | libs/luci-lib-httpclient/luasrc/httpclient/receiver.lua | 295 |
3 files changed, 678 insertions, 0 deletions
diff --git a/libs/luci-lib-httpclient/Makefile b/libs/luci-lib-httpclient/Makefile new file mode 100644 index 000000000..1e7fd1bc8 --- /dev/null +++ b/libs/luci-lib-httpclient/Makefile @@ -0,0 +1,14 @@ +# +# Copyright (C) 2008-2014 The LuCI Team <luci@lists.subsignal.org> +# +# This is free software, licensed under the Apache License, Version 2.0 . +# + +include $(TOPDIR)/rules.mk + +LUCI_TITLE:=HTTP(S) client library +LUCI_DEPENDS:=+luci-base +luci-lib-nixio + +include ../../luci.mk + +# call BuildPackage - OpenWrt buildroot signature diff --git a/libs/luci-lib-httpclient/luasrc/httpclient.lua b/libs/luci-lib-httpclient/luasrc/httpclient.lua new file mode 100644 index 000000000..e9fec5dbb --- /dev/null +++ b/libs/luci-lib-httpclient/luasrc/httpclient.lua @@ -0,0 +1,369 @@ +--[[ +LuCI - Lua Development Framework + +Copyright 2009 Steven Barth <steven@midlink.org> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +$Id$ +]]-- + +require "nixio.util" +local nixio = require "nixio" + +local ltn12 = require "luci.ltn12" +local util = require "luci.util" +local table = require "table" +local http = require "luci.http.protocol" +local date = require "luci.http.protocol.date" + +local type, pairs, ipairs, tonumber = type, pairs, ipairs, tonumber +local unpack = unpack + +module "luci.httpclient" + +function chunksource(sock, buffer) + buffer = buffer or "" + return function() + local output + local _, endp, count = buffer:find("^([0-9a-fA-F]+);?.-\r\n") + while not count and #buffer <= 1024 do + local newblock, code = sock:recv(1024 - #buffer) + if not newblock then + return nil, code + end + buffer = buffer .. newblock + _, endp, count = buffer:find("^([0-9a-fA-F]+);?.-\r\n") + end + count = tonumber(count, 16) + if not count then + return nil, -1, "invalid encoding" + elseif count == 0 then + return nil + elseif count + 2 <= #buffer - endp then + output = buffer:sub(endp+1, endp+count) + buffer = buffer:sub(endp+count+3) + return output + else + output = buffer:sub(endp+1, endp+count) + buffer = "" + if count - #output > 0 then + local remain, code = sock:recvall(count-#output) + if not remain then + return nil, code + end + output = output .. remain + count, code = sock:recvall(2) + else + count, code = sock:recvall(count+2-#buffer+endp) + end + if not count then + return nil, code + end + return output + end + end +end + + +function request_to_buffer(uri, options) + local source, code, msg = request_to_source(uri, options) + local output = {} + + if not source then + return nil, code, msg + end + + source, code = ltn12.pump.all(source, (ltn12.sink.table(output))) + + if not source then + return nil, code + end + + return table.concat(output) +end + +function request_to_source(uri, options) + local status, response, buffer, sock = request_raw(uri, options) + if not status then + return status, response, buffer + elseif status ~= 200 and status ~= 206 then + return nil, status, buffer + end + + if response.headers["Transfer-Encoding"] == "chunked" then + return chunksource(sock, buffer) + else + return ltn12.source.cat(ltn12.source.string(buffer), sock:blocksource()) + end +end + +-- +-- GET HTTP-resource +-- +function request_raw(uri, options) + options = options or {} + local pr, auth, host, port, path + + if uri:find("%[") then + if uri:find("@") then + pr, auth, host, port, path = uri:match("(%w+)://(.+)@(%b[]):?([0-9]*)(.*)") + host = host:sub(2,-2) + else + pr, host, port, path = uri:match("(%w+)://(%b[]):?([0-9]*)(.*)") + host = host:sub(2,-2) + end + else + if uri:find("@") then + pr, auth, host, port, path = + uri:match("(%w+)://(.+)@([%w-.]+):?([0-9]*)(.*)") + else + pr, host, port, path = uri:match("(%w+)://([%w-.]+):?([0-9]*)(.*)") + end + end + + if not host then + return nil, -1, "unable to parse URI" + end + + if pr ~= "http" and pr ~= "https" then + return nil, -2, "protocol not supported" + end + + port = #port > 0 and port or (pr == "https" and 443 or 80) + path = #path > 0 and path or "/" + + options.depth = options.depth or 10 + local headers = options.headers or {} + local protocol = options.protocol or "HTTP/1.1" + headers["User-Agent"] = headers["User-Agent"] or "LuCI httpclient 0.1" + + if headers.Connection == nil then + headers.Connection = "close" + end + + if auth and not headers.Authorization then + headers.Authorization = "Basic " .. nixio.bin.b64encode(auth) + end + + local sock, code, msg = nixio.connect(host, port) + if not sock then + return nil, code, msg + end + + sock:setsockopt("socket", "sndtimeo", options.sndtimeo or 15) + sock:setsockopt("socket", "rcvtimeo", options.rcvtimeo or 15) + + if pr == "https" then + local tls = options.tls_context or nixio.tls() + sock = tls:create(sock) + local stat, code, error = sock:connect() + if not stat then + return stat, code, error + end + end + + -- Pre assemble fixes + if protocol == "HTTP/1.1" then + headers.Host = headers.Host or host + end + + if type(options.body) == "table" then + options.body = http.urlencode_params(options.body) + end + + if type(options.body) == "string" then + headers["Content-Length"] = headers["Content-Length"] or #options.body + headers["Content-Type"] = headers["Content-Type"] or + "application/x-www-form-urlencoded" + options.method = options.method or "POST" + end + + if type(options.body) == "function" then + options.method = options.method or "POST" + end + + -- Assemble message + local message = {(options.method or "GET") .. " " .. path .. " " .. protocol} + + for k, v in pairs(headers) do + if type(v) == "string" or type(v) == "number" then + message[#message+1] = k .. ": " .. v + elseif type(v) == "table" then + for i, j in ipairs(v) do + message[#message+1] = k .. ": " .. j + end + end + end + + if options.cookies then + for _, c in ipairs(options.cookies) do + local cdo = c.flags.domain + local cpa = c.flags.path + if (cdo == host or cdo == "."..host or host:sub(-#cdo) == cdo) + and (cpa == path or cpa == "/" or cpa .. "/" == path:sub(#cpa+1)) + and (not c.flags.secure or pr == "https") + then + message[#message+1] = "Cookie: " .. c.key .. "=" .. c.value + end + end + end + + message[#message+1] = "" + message[#message+1] = "" + + -- Send request + sock:sendall(table.concat(message, "\r\n")) + + if type(options.body) == "string" then + sock:sendall(options.body) + elseif type(options.body) == "function" then + local res = {options.body(sock)} + if not res[1] then + sock:close() + return unpack(res) + end + end + + -- Create source and fetch response + local linesrc = sock:linesource() + local line, code, error = linesrc() + + if not line then + sock:close() + return nil, code, error + end + + local protocol, status, msg = line:match("^([%w./]+) ([0-9]+) (.*)") + + if not protocol then + sock:close() + return nil, -3, "invalid response magic: " .. line + end + + local response = { + status = line, headers = {}, code = 0, cookies = {}, uri = uri + } + + line = linesrc() + while line and line ~= "" do + local key, val = line:match("^([%w-]+)%s?:%s?(.*)") + if key and key ~= "Status" then + if type(response.headers[key]) == "string" then + response.headers[key] = {response.headers[key], val} + elseif type(response.headers[key]) == "table" then + response.headers[key][#response.headers[key]+1] = val + else + response.headers[key] = val + end + end + line = linesrc() + end + + if not line then + sock:close() + return nil, -4, "protocol error" + end + + -- Parse cookies + if response.headers["Set-Cookie"] then + local cookies = response.headers["Set-Cookie"] + for _, c in ipairs(type(cookies) == "table" and cookies or {cookies}) do + local cobj = cookie_parse(c) + cobj.flags.path = cobj.flags.path or path:match("(/.*)/?[^/]*") + if not cobj.flags.domain or cobj.flags.domain == "" then + cobj.flags.domain = host + response.cookies[#response.cookies+1] = cobj + else + local hprt, cprt = {}, {} + + -- Split hostnames and save them in reverse order + for part in host:gmatch("[^.]*") do + table.insert(hprt, 1, part) + end + for part in cobj.flags.domain:gmatch("[^.]*") do + table.insert(cprt, 1, part) + end + + local valid = true + for i, part in ipairs(cprt) do + -- If parts are different and no wildcard + if hprt[i] ~= part and #part ~= 0 then + valid = false + break + -- Wildcard on invalid position + elseif hprt[i] ~= part and #part == 0 then + if i ~= #cprt or (#hprt ~= i and #hprt+1 ~= i) then + valid = false + break + end + end + end + -- No TLD cookies + if valid and #cprt > 1 and #cprt[2] > 0 then + response.cookies[#response.cookies+1] = cobj + end + end + end + end + + -- Follow + response.code = tonumber(status) + if response.code and options.depth > 0 then + if (response.code == 301 or response.code == 302 or response.code == 307) + and response.headers.Location then + local nuri = response.headers.Location or response.headers.location + if not nuri then + return nil, -5, "invalid reference" + end + if not nuri:find("https?://") then + nuri = pr .. "://" .. host .. ":" .. port .. nuri + end + + options.depth = options.depth - 1 + if options.headers then + options.headers.Host = nil + end + sock:close() + + return request_raw(nuri, options) + end + end + + return response.code, response, linesrc(true), sock +end + +function cookie_parse(cookiestr) + local key, val, flags = cookiestr:match("%s?([^=;]+)=?([^;]*)(.*)") + if not key then + return nil + end + + local cookie = {key = key, value = val, flags = {}} + for fkey, fval in flags:gmatch(";%s?([^=;]+)=?([^;]*)") do + fkey = fkey:lower() + if fkey == "expires" then + fval = date.to_unix(fval:gsub("%-", " ")) + end + cookie.flags[fkey] = fval + end + + return cookie +end + +function cookie_create(cookie) + local cookiedata = {cookie.key .. "=" .. cookie.value} + + for k, v in pairs(cookie.flags) do + if k == "expires" then + v = date.to_http(v):gsub(", (%w+) (%w+) (%w+) ", ", %1-%2-%3 ") + end + cookiedata[#cookiedata+1] = k .. ((#v > 0) and ("=" .. v) or "") + end + + return table.concat(cookiedata, "; ") +end diff --git a/libs/luci-lib-httpclient/luasrc/httpclient/receiver.lua b/libs/luci-lib-httpclient/luasrc/httpclient/receiver.lua new file mode 100644 index 000000000..4f08e93fe --- /dev/null +++ b/libs/luci-lib-httpclient/luasrc/httpclient/receiver.lua @@ -0,0 +1,295 @@ +--[[ +LuCI - Lua Development Framework + +Copyright 2009 Steven Barth <steven@midlink.org> + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +$Id$ +]]-- + +require "nixio.util" +local nixio = require "nixio" +local httpc = require "luci.httpclient" +local ltn12 = require "luci.ltn12" + +local print, tonumber, require, unpack = print, tonumber, require, unpack + +module "luci.httpclient.receiver" + +local function prepare_fd(target) + -- Open fd for appending + local oflags = nixio.open_flags("wronly", "creat") + local file, code, msg = nixio.open(target, oflags) + if not file then + return file, code, msg + end + + -- Acquire lock + local stat, code, msg = file:lock("tlock") + if not stat then + return stat, code, msg + end + + file:seek(0, "end") + + return file +end + +local function splice_async(sock, pipeout, pipein, file, cb) + local ssize = 65536 + local smode = nixio.splice_flags("move", "more", "nonblock") + + -- Set pipe non-blocking otherwise we might end in a deadlock + local stat, code, msg = pipein:setblocking(false) + if stat then + stat, code, msg = pipeout:setblocking(false) + end + if not stat then + return stat, code, msg + end + + + local pollsock = { + {fd=sock, events=nixio.poll_flags("in")} + } + + local pollfile = { + {fd=file, events=nixio.poll_flags("out")} + } + + local done + local active -- Older splice implementations sometimes don't detect EOS + + repeat + active = false + + -- Socket -> Pipe + repeat + nixio.poll(pollsock, 15000) + + stat, code, msg = nixio.splice(sock, pipeout, ssize, smode) + if stat == nil then + return stat, code, msg + elseif stat == 0 then + done = true + break + elseif stat then + active = true + end + until stat == false + + -- Pipe -> File + repeat + nixio.poll(pollfile, 15000) + + stat, code, msg = nixio.splice(pipein, file, ssize, smode) + if stat == nil then + return stat, code, msg + elseif stat then + active = true + end + until stat == false + + if cb then + cb(file) + end + + if not active then + -- We did not splice any data, maybe EOS, fallback to default + return false + end + until done + + pipein:close() + pipeout:close() + sock:close() + file:close() + return true +end + +local function splice_sync(sock, pipeout, pipein, file, cb) + local os = require "os" + local ssize = 65536 + local smode = nixio.splice_flags("move", "more") + local stat + + -- This is probably the only forking http-client ;-) + local pid, code, msg = nixio.fork() + if not pid then + return pid, code, msg + elseif pid == 0 then + pipein:close() + file:close() + + repeat + stat, code = nixio.splice(sock, pipeout, ssize, smode) + until not stat or stat == 0 + + pipeout:close() + sock:close() + os.exit(stat or code) + else + pipeout:close() + sock:close() + + repeat + stat, code, msg = nixio.splice(pipein, file, ssize, smode) + if cb then + cb(file) + end + until not stat or stat == 0 + + pipein:close() + file:close() + + if not stat then + nixio.kill(pid, 15) + nixio.wait(pid) + return stat, code, msg + else + pid, msg, code = nixio.wait(pid) + if msg == "exited" then + if code == 0 then + return true + else + return nil, code, nixio.strerror(code) + end + else + return nil, -0x11, "broken pump" + end + end + end +end + +function request_to_file(uri, target, options, cbs) + options = options or {} + cbs = cbs or {} + options.headers = options.headers or {} + local hdr = options.headers + local file, code, msg + + if target then + file, code, msg = prepare_fd(target) + if not file then + return file, code, msg + end + + local off = file:tell() + + -- Set content range + if off > 0 then + hdr.Range = hdr.Range or ("bytes=" .. off .. "-") + end + end + + local code, resp, buffer, sock = httpc.request_raw(uri, options) + if not code then + -- No success + if file then + file:close() + end + return code, resp, buffer + elseif hdr.Range and code ~= 206 then + -- We wanted a part but we got the while file + sock:close() + if file then + file:close() + end + return nil, -4, code, resp + elseif not hdr.Range and code ~= 200 then + -- We encountered an error + sock:close() + if file then + file:close() + end + return nil, -4, code, resp + end + + if cbs.on_header then + local stat = {cbs.on_header(file, code, resp)} + if stat[1] == false then + if file then + file:close() + end + sock:close() + return unpack(stat) + elseif stat[2] then + file = file and stat[2] + end + end + + if not file then + return nil, -5, "no target given" + end + + local chunked = resp.headers["Transfer-Encoding"] == "chunked" + local stat + + -- Write the buffer to file + file:writeall(buffer) + + repeat + if not options.splice or not sock:is_socket() or chunked then + break + end + + -- This is a plain TCP socket and there is no encoding so we can splice + + local pipein, pipeout, msg = nixio.pipe() + if not pipein then + sock:close() + file:close() + return pipein, pipeout, msg + end + + + -- Adjust splice values + local ssize = 65536 + local smode = nixio.splice_flags("move", "more") + + -- Splicing 512 bytes should never block on a fresh pipe + local stat, code, msg = nixio.splice(sock, pipeout, 512, smode) + if stat == nil then + break + end + + -- Now do the real splicing + local cb = cbs.on_write + if options.splice == "asynchronous" then + stat, code, msg = splice_async(sock, pipeout, pipein, file, cb) + elseif options.splice == "synchronous" then + stat, code, msg = splice_sync(sock, pipeout, pipein, file, cb) + else + break + end + + if stat == false then + break + end + + return stat, code, msg + until true + + local src = chunked and httpc.chunksource(sock) or sock:blocksource() + local snk = file:sink() + + if cbs.on_write then + src = ltn12.source.chain(src, function(chunk) + cbs.on_write(file) + return chunk + end) + end + + -- Fallback to read/write + stat, code, msg = ltn12.pump.all(src, snk) + + file:close() + sock:close() + return stat and true, code, msg +end + |