From 7196b2cd848577f640d9268a0a34ab3ad8706c26 Mon Sep 17 00:00:00 2001 From: Steven Barth Date: Fri, 27 Feb 2009 14:51:37 +0000 Subject: nixio: Fixes, use POSIX calls for file i/o httpclient: resume support, splice() support, cookie support --- libs/httpclient/luasrc/httpclient.lua | 123 +++++++++++++-- libs/httpclient/luasrc/httpclient/receiver.lua | 197 +++++++++++++++++++++++++ 2 files changed, 308 insertions(+), 12 deletions(-) create mode 100644 libs/httpclient/luasrc/httpclient/receiver.lua (limited to 'libs/httpclient') diff --git a/libs/httpclient/luasrc/httpclient.lua b/libs/httpclient/luasrc/httpclient.lua index 542e6b6cd..767a02ea2 100644 --- a/libs/httpclient/luasrc/httpclient.lua +++ b/libs/httpclient/luasrc/httpclient.lua @@ -1,5 +1,5 @@ --[[ -LuCI - Lua Configuration Interface +LuCI - Lua Development Framework Copyright 2009 Steven Barth @@ -19,8 +19,9 @@ local ltn12 = require "luci.ltn12" local util = require "luci.util" local table = require "table" local http = require "luci.http.protocol" +local date = require "luci.http.protocol.date" -local type, pairs, tonumber, print = type, pairs, tonumber, print +local type, pairs, ipairs, tonumber = type, pairs, ipairs, tonumber module "luci.httpclient" @@ -93,7 +94,7 @@ function request_to_source(uri, options) return nil, status, response end - if response["Transfer-Encoding"] == "chunked" then + if response.headers["Transfer-Encoding"] == "chunked" then return chunksource(sock, buffer) else return ltn12.source.cat(ltn12.source.string(buffer), sock:blocksource()) @@ -114,7 +115,7 @@ function request_raw(uri, options) return nil, -2, "protocol not supported" end - port = #port > 0 and port or (pr == "https" and "443" or "80") + port = #port > 0 and port or (pr == "https" and 443 or 80) path = #path > 0 and path or "/" options.depth = options.depth or 10 @@ -163,10 +164,28 @@ function request_raw(uri, options) local message = {method .. " " .. path .. " " .. protocol} for k, v in pairs(headers) do - if v then + if type(v) == "string" then message[#message+1] = k .. ": " .. v + elseif type(v) == "table" then + for i, j in ipairs(v) do + message[#message+1] = k .. ": " .. j + end + end + end + + if options.cookies then + for _, c in ipairs(options.cookies) do + local cdo = c.flags.domain + local cpa = c.flags.path + if (cdo == host or cdo == "."..host or host:sub(-#cdo) == cdo) + and (cpa == "/" or cpa .. "/" == path:sub(#cpa+1)) + and (not c.flags.secure or pr == "https") + then + message[#message+1] = "Cookie: " .. c.key .. "=" .. c.value + end end end + message[#message+1] = "" message[#message+1] = "" @@ -191,13 +210,19 @@ function request_raw(uri, options) return nil, -3, "invalid response magic: " .. line end - local response = {Status=line} + local response = {status = line, headers = {}, code = 0, cookies = {}} line = linesrc() while line and line ~= "" do local key, val = line:match("^([%w-]+)%s?:%s?(.*)") if key and key ~= "Status" then - response[key] = val + if type(response[key]) == "string" then + response.headers[key] = {response.headers[key], val} + elseif type(response[key]) == "table" then + response.headers[key][#response.headers[key]+1] = val + else + response.headers[key] = val + end end line = linesrc() end @@ -206,11 +231,54 @@ function request_raw(uri, options) return nil, -4, "protocol error" end + -- Parse cookies + if response.headers["Set-Cookie"] then + local cookies = response.headers["Set-Cookie"] + for _, c in ipairs(type(cookies) == "table" and cookies or {cookies}) do + local cobj = cookie_parse(c) + cobj.flags.path = cobj.flags.path or path:match("(/.*)/?[^/]*") + if not cobj.flags.domain or cobj.flags.domain == "" then + cobj.flags.domain = host + response.cookies[#response.cookies+1] = cobj + else + local hprt, cprt = {}, {} + + -- Split hostnames and save them in reverse order + for part in host:gmatch("[^.]*") do + table.insert(hprt, 1, part) + end + for part in cobj.flags.domain:gmatch("[^.]*") do + table.insert(cprt, 1, part) + end + + local valid = true + for i, part in ipairs(cprt) do + -- If parts are different and no wildcard + if hprt[i] ~= part and #part ~= 0 then + valid = false + break + -- Wildcard on invalid position + elseif hprt[i] ~= part and #part == 0 then + if i ~= #cprt or (#hprt ~= i and #hprt+1 ~= i) then + valid = false + break + end + end + end + -- No TLD cookies + if valid and #cprt > 1 and #cprt[2] > 0 then + response.cookies[#response.cookies+1] = cobj + end + end + end + end + -- Follow - local code = tonumber(status) - if code and options.depth > 0 then - if code == 301 or code == 302 or code == 307 and response.Location then - local nexturi = response.Location + response.code = tonumber(status) + if response.code and options.depth > 0 then + if response.code == 301 or response.code == 302 or response.code == 307 + and response.headers.Location then + local nexturi = response.headers.Location if not nexturi:find("https?://") then nexturi = pr .. "://" .. host .. ":" .. port .. nexturi end @@ -222,5 +290,36 @@ function request_raw(uri, options) end end - return code, response, linesrc(true), sock + return response.code, response, linesrc(true), sock +end + +function cookie_parse(cookiestr) + local key, val, flags = cookiestr:match("%s?([^=;]+)=?([^;]*)(.*)") + if not key then + return nil + end + + local cookie = {key = key, value = val, flags = {}} + for fkey, fval in flags:gmatch(";%s?([^=;]+)=?([^;]*)") do + fkey = fkey:lower() + if fkey == "expires" then + fval = date.to_unix(fval:gsub("%-", " ")) + end + cookie.flags[fkey] = fval + end + + return cookie +end + +function cookie_create(cookie) + local cookiedata = {cookie.key .. "=" .. cookie.value} + + for k, v in pairs(cookie.flags) do + if k == "expires" then + v = date.to_http(v):gsub(", (%w+) (%w+) (%w+) ", ", %1-%2-%3 ") + end + cookiedata[#cookiedata+1] = k .. ((#v > 0) and ("=" .. v) or "") + end + + return table.concat(cookiedata, "; ") end \ No newline at end of file diff --git a/libs/httpclient/luasrc/httpclient/receiver.lua b/libs/httpclient/luasrc/httpclient/receiver.lua new file mode 100644 index 000000000..f478fe850 --- /dev/null +++ b/libs/httpclient/luasrc/httpclient/receiver.lua @@ -0,0 +1,197 @@ +--[[ +LuCI - Lua Development Framework + +Copyright 2009 Steven Barth + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +$Id$ +]]-- + +require "nixio.util" +local nixio = require "nixio" +local httpclient = require "luci.httpclient" +local ltn12 = require "luci.ltn12" + +local print = print + +module "luci.httpclient.receiver" + +local function prepare_fd(target) + -- Open fd for appending + local file, code, msg = nixio.open(target, "r+") + if not file and code == nixio.const.ENOENT then + file, code, msg = nixio.open(target, "w") + if file then + file:flush() + end + end + if not file then + return file, code, msg + end + + -- Acquire lock + local stat, code, msg = file:lock("ex", "nb") + if not stat then + return stat, code, msg + end + + file:seek(0, "end") + + return file +end + + +function request_to_file(uri, target, options, cbs) + options = options or {} + cbs = cbs or {} + options.headers = options.headers or {} + local hdr = options.headers + + local file, code, msg = prepare_fd(target) + if not file then + return file, code, msg + end + + local off = file:tell() + + -- Set content range + if off > 0 then + hdr.Range = hdr.Range or ("bytes=" .. off .. "-") + end + + local code, resp, buffer, sock = httpclient.request_raw(uri, options) + if not code then + -- No success + file:close() + return code, resp, buffer + elseif hdr.Range and code ~= 206 then + -- We wanted a part but we got the while file + sock:close() + file:close() + return nil, -4, code, resp + elseif not hdr.Range and code ~= 200 then + -- We encountered an error + sock:close() + file:close() + return nil, -4, code, resp + end + + if cbs.on_header then + cbs.on_header(file, code, resp) + end + + local chunked = resp.headers["Transfer-Encoding"] == "chunked" + + -- Write the buffer to file + file:writeall(buffer) + print ("Buffered data: " .. #buffer .. " Byte") + + repeat + if not sock:is_socket() or chunked then + break + end + + -- This is a plain TCP socket and there is no encoding so we can splice + + local pipein, pipeout, msg = nixio.pipe() + if not pipein then + sock:close() + file:close() + return pipein, pipeout, msg + end + + + -- Disable blocking for the pipe otherwise we might end in a deadlock + local stat, code, msg = pipein:setblocking(false) + if stat then + stat, code, msg = pipeout:setblocking(false) + end + if not stat then + sock:close() + file:close() + return stat, code, msg + end + + + -- Adjust splice values + local ssize = 65536 + local smode = nixio.splice_flags("move", "more", "nonblock") + + local stat, code, msg = nixio.splice(sock, pipeout, ssize, smode) + if stat == nil then + break + end + + local pollsock = { + {fd=sock, events=nixio.poll_flags("in")} + } + + local pollfile = { + {fd=file, events=nixio.poll_flags("out")} + } + + local done + + repeat + -- Socket -> Pipe + repeat + nixio.poll(pollsock, 15000) + + stat, code, msg = nixio.splice(sock, pipeout, ssize, smode) + if stat == nil then + sock:close() + file:close() + return stat, code, msg + elseif stat == 0 then + done = true + break + end + until stat == false + + -- Pipe -> File + repeat + nixio.poll(pollfile, 15000) + + stat, code, msg = nixio.splice(pipein, file, ssize, smode) + if stat == nil then + sock:close() + file:close() + return stat, code, msg + end + until stat == false + + if cbs.on_write then + cbs.on_write(file) + end + until done + + file:close() + sock:close() + return true + until true + + print "Warning: splice() failed, falling back to read/write mode" + + local src = chunked and httpclient.chunksource(sock) or sock:blocksource() + local snk = file:sink() + + if cbs.on_write then + src = ltn12.source.chain(src, function(chunk) + cbs.on_write(file) + return chunk + end) + end + + -- Fallback to read/write + local stat, code, msg = ltn12.pump.all(src, snk) + if stat then + file:close() + sock:close() + end + return stat, code, msg +end \ No newline at end of file -- cgit v1.2.3