diff --git a/tests/archive_spec.lua b/tests/archive_spec.lua index 221c0e2..88101d6 100644 --- a/tests/archive_spec.lua +++ b/tests/archive_spec.lua @@ -1,5 +1,6 @@ local t = require('tests.harness') local archive = require('arti-pack.archive') +local deflate = require('tests.deflate') local zzlib = require('arti-pack.vendor.zzlib') local function tmpdir() @@ -18,45 +19,56 @@ local function le(n, bytes) end -- Fixtures are built here rather than shelled out to zip/tar so the hostile --- entry names below are exactly the bytes under test. +-- entry names below are exactly the bytes under test. `deflate = true` runs the +-- payload through a real compressor, so DEFLATE entries are not approximated +-- and the tests cannot skip themselves when no zip binary is around. local function zipf(entries) local locals, central, offset = {}, {}, 0 for _, e in ipairs(entries) do local data = e.data or '' - local crc = zzlib.crc32(data) + local method, payload = 0, data + if e.deflate then + method, payload = 8, deflate.compress(data, e.mode) + end + method = e.method or method + if e.mangle then + payload = e.mangle(payload) + end + local crc = e.crc or zzlib.crc32(data) + local csize, usize = #payload, e.usize or #data local head = 'PK\3\4' .. le(20, 2) .. le(e.flags or 0, 2) - .. le(0, 2) + .. le(method, 2) .. le(0, 4) - .. le(e.crc or crc, 4) - .. le(#data, 4) - .. le(#data, 4) + .. le(crc, 4) + .. le(csize, 4) + .. le(usize, 4) .. le(#e.name, 2) .. le(0, 2) .. e.name - table.insert(locals, head .. data) + table.insert(locals, head .. payload) table.insert( central, 'PK\1\2' .. le(e.made_by or (3 * 256 + 20), 2) .. le(20, 2) .. le(e.flags or 0, 2) - .. le(0, 2) + .. le(method, 2) .. le(0, 4) - .. le(e.crc or crc, 4) - .. le(#data, 4) - .. le(#data, 4) + .. le(crc, 4) + .. le(csize, 4) + .. le(usize, 4) .. le(#e.name, 2) .. le(0, 2) .. le(0, 2) .. le(0, 2) .. le(0, 2) .. le(e.attr or 0, 4) - .. le(offset, 4) + .. le(e.local_offset or offset, 4) .. e.name ) - offset = offset + #head + #data + offset = offset + #head + #payload end local body = table.concat(locals) local cd = table.concat(central) @@ -73,9 +85,7 @@ local function zipf(entries) return body .. cd .. eocd end --- A gzip member whose DEFLATE payload is all stored blocks: valid, and buildable --- without a compressor. -local function gz(data) +local function stored_blocks(data) local blocks, pos = {}, 1 repeat local chunk = data:sub(pos, pos + 65534) @@ -83,7 +93,14 @@ local function gz(data) local final = pos > #data and 1 or 0 table.insert(blocks, string.char(final) .. le(#chunk, 2) .. le(65535 - #chunk, 2) .. chunk) until pos > #data - return '\31\139\8\0\0\0\0\0\0\3' .. table.concat(blocks) .. le(zzlib.crc32(data), 4) .. le(#data, 4) + return table.concat(blocks) +end + +-- A gzip member. Without a mode the DEFLATE payload is all stored blocks; with +-- one it is genuinely compressed, which is what drives the huffman decoder. +local function gz(data, mode) + local payload = mode and deflate.compress(data, mode) or stored_blocks(data) + return '\31\139\8\0\0\0\0\0\0\3' .. payload .. le(zzlib.crc32(data), 4) .. le(#data, 4) end local function tar_header(name, size, kind, mode) @@ -140,6 +157,42 @@ local function extract(bytes, ext) return ok, err, dest end +-- Deterministic pseudo-random bytes, so a compressed fixture is the same on +-- every run but still spans the whole literal alphabet. +local function noise(n, seed) + local out, state = {}, seed + for _ = 1, n do + state = (state * 1103515245 + 12345) % 2147483648 + table.insert(out, string.char(math.floor(state / 65536) % 256)) + end + return table.concat(out) +end + +-- Every byte value, then repeated text: literal codes of both widths plus real +-- back-references. +local function sample() + local parts = {} + for i = 0, 255 do + table.insert(parts, string.char(i)) + end + for i = 1, 40 do + table.insert(parts, string.format('local module_%d = require("arti-pack.mod")\n', i)) + end + return table.concat(parts) +end + +local function patch32(bytes, at, value) + return bytes:sub(1, at - 1) .. le(value, 4) .. bytes:sub(at + 4) +end + +local function flip(bytes, at, count) + local out = bytes:sub(1, at - 1) + for i = at, at + count - 1 do + out = out .. string.char((bytes:byte(i) + 137) % 256) + end + return out .. bytes:sub(at + count) +end + t.describe('archive.safe_path', function() t.it('accepts an ordinary nested path', function() t.eq('plugin/lua/init.lua', archive.safe_path('plugin/lua/init.lua')) @@ -261,6 +314,154 @@ t.describe('archive.extract zip', function() local ok = extract('PK\3\4' .. string.rep('\200', 400)) t.falsy(ok) end) + t.it('rejects an unsupported compression method', function() + local ok, err = extract(zipf({ { name = 'root/a.lua', data = 'x', method = 12 } })) + t.falsy(ok) + t.matches('unsupported compression method 12', err) + end) +end) + +t.describe('archive.extract zip malformed directory', function() + -- A central directory record that stops before its extra-field and comment + -- lengths used to raise instead of reporting. + local function truncate_cd(keep) + local bytes = zipf({ { name = 'root/a.lua', data = 'hi' } }) + local cd = bytes:find('PK\1\2', 1, true) + local eocd = bytes:find('PK\5\6', cd, true) + return bytes:sub(1, cd + keep - 1) .. bytes:sub(eocd) + end + + t.it('reports a record cut off before its length fields', function() + local ok, err = extract(truncate_cd(30)) + t.falsy(ok) + t.matches('truncated central directory', err) + end) + + t.it('reports a record cut off just after its signature', function() + local ok, err = extract(truncate_cd(4)) + t.falsy(ok) + t.matches('truncated central directory', err) + end) + + t.it('reports a local header that runs off the end of the archive', function() + -- The trailing comment plants a local-header signature in the last four + -- bytes; the entry is pointed at it. + local bytes = zipf({ { name = 'root/a.lua', data = 'hi' }, comment = 'PK\3\4' }) + local cd = bytes:find('PK\1\2', 1, true) + local ok, err = extract(patch32(bytes, cd + 42, #bytes - 4)) + t.falsy(ok) + t.matches('truncated local header', err) + end) +end) + +t.describe('archive.extract zip deflate', function() + local payload = sample() + + t.it('round-trips a dynamic-huffman entry byte for byte', function() + local ok, err, dest = extract(zipf({ { name = 'root/a.lua', data = payload, deflate = true } })) + t.truthy(ok, err) + t.eq(payload, read(dest .. '/root/a.lua')) + end) + + t.it('round-trips a fixed-huffman entry byte for byte', function() + local bytes = zipf({ { name = 'root/a.lua', data = payload, deflate = true, mode = 'fixed' } }) + local ok, err, dest = extract(bytes) + t.truthy(ok, err) + t.eq(payload, read(dest .. '/root/a.lua')) + end) + + t.it('round-trips deflated entries alongside stored ones', function() + local overlapping = string.rep('ab', 3000) + local ok, err, dest = extract(zipf({ + { name = 'root/' }, + { name = 'root/stored.lua', data = 'return {}' }, + { name = 'root/packed.lua', data = payload, deflate = true }, + { name = 'root/nested/overlap.txt', data = overlapping, deflate = true }, + })) + t.truthy(ok, err) + t.eq('return {}', read(dest .. '/root/stored.lua')) + t.eq(payload, read(dest .. '/root/packed.lua')) + t.eq(overlapping, read(dest .. '/root/nested/overlap.txt')) + end) + + t.it('round-trips a deflated entry that decompresses to nothing', function() + local ok, err, dest = extract(zipf({ { name = 'root/empty.lua', data = '', deflate = true } })) + t.truthy(ok, err) + t.eq('', read(dest .. '/root/empty.lua')) + end) + + t.it('round-trips a large deflated entry', function() + local big = string.rep(noise(1024, 11), 400) + local bytes = zipf({ { name = 'root/big.bin', data = big, deflate = true } }) + t.truthy(#bytes < #big / 4, 'the fixture is not actually compressed') + local ok, err, dest = extract(bytes) + t.truthy(ok, err) + local out = read(dest .. '/root/big.bin') + t.eq(#big, #out) + t.truthy(out == big, 'the large entry did not round-trip exactly') + end) + + t.it('fails loudly when a deflated entry carries the wrong CRC', function() + local ok, err, dest = extract(zipf({ { name = 'root/a.lua', data = payload, deflate = true, crc = 12345 } })) + t.falsy(ok) + t.matches('checksum', err) + t.falsy(read(dest .. '/root/a.lua'), 'nothing should be written') + end) + + t.it('fails loudly when a deflated entry disagrees with the stated size', function() + local bytes = zipf({ { name = 'root/a.lua', data = payload, deflate = true, usize = #payload + 1 } }) + local ok, err, dest = extract(bytes) + t.falsy(ok) + t.matches('does not match the directory', err) + t.falsy(read(dest .. '/root/a.lua'), 'nothing should be written') + end) + + t.it('fails loudly on a truncated deflate stream', function() + local ok, err, dest = extract(zipf({ + { + name = 'root/a.lua', + data = payload, + deflate = true, + mangle = function(s) + return s:sub(1, math.floor(#s / 2)) + end, + }, + })) + t.falsy(ok) + t.matches('root/a%.lua', err) + t.falsy(read(dest .. '/root/a.lua'), 'a partial inflate must not reach disk') + end) + + t.it('fails loudly on a deflate stream corrupted in the middle', function() + local ok, err, dest = extract(zipf({ + { + name = 'root/a.lua', + data = payload, + deflate = true, + mangle = function(s) + return flip(s, math.floor(#s / 2), 4) + end, + }, + })) + t.falsy(ok) + t.matches('root/a%.lua', err) + t.falsy(read(dest .. '/root/a.lua'), 'a corrupt inflate must not reach disk') + end) + + t.it('refuses a deflated entry that walks out of the destination', function() + local ok, err, dest = extract(zipf({ { name = '../evil.lua', data = payload, deflate = true } })) + t.falsy(ok) + t.matches('escapes', err) + t.falsy(read(dest .. '/../evil.lua'), 'nothing should be written') + end) + + t.it('refuses a deflated symlink entry', function() + local ok, err = extract(zipf({ + { name = 'root/link', data = string.rep('/etc/passwd', 40), deflate = true, attr = (0xa000 + 0x1ff) * 65536 }, + })) + t.falsy(ok) + t.matches('not a regular file', err) + end) end) t.describe('archive.extract tar.gz', function() @@ -338,3 +539,86 @@ t.describe('archive.extract tar.gz', function() t.matches('checksum mismatch', err) end) end) + +-- The stored-block fixtures above never reach the huffman decoder, so the +-- compressed cases are covered separately. +t.describe('archive.extract tar.gz deflate', function() + local payload = sample() + + t.it('inflates a dynamic-huffman member', function() + local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic') + local ok, err, dest = extract(bytes, 'tar.gz') + t.truthy(ok, err) + t.eq(payload, read(dest .. '/root/a.lua')) + end) + + t.it('inflates a fixed-huffman member', function() + local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'fixed') + local ok, err, dest = extract(bytes, 'tar.gz') + t.truthy(ok, err) + t.eq(payload, read(dest .. '/root/a.lua')) + end) + + t.it('inflates an empty member', function() + local bytes = gz(tarf({ { name = 'root/empty.lua', data = '' } }), 'dynamic') + local ok, err, dest = extract(bytes, 'tar.gz') + t.truthy(ok, err) + t.eq('', read(dest .. '/root/empty.lua')) + end) + + t.it('inflates a large member', function() + local big = string.rep(noise(1024, 29), 300) + local bytes = gz(tarf({ { name = 'root/big.bin', data = big } }), 'dynamic') + t.truthy(#bytes < #big / 4, 'the fixture is not actually compressed') + local ok, err, dest = extract(bytes, 'tar.gz') + t.truthy(ok, err) + local out = read(dest .. '/root/big.bin') + t.eq(#big, #out) + t.truthy(out == big, 'the large member did not round-trip exactly') + end) + + t.it('fails loudly when the gzip checksum disagrees', function() + local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic') + local ok, err, dest = extract(patch32(bytes, #bytes - 7, 12345), 'tar.gz') + t.falsy(ok) + t.matches('gunzip failed', err) + t.falsy(read(dest .. '/root/a.lua'), 'nothing should be written') + end) + + t.it('fails loudly on a truncated compressed member', function() + local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic') + local ok, err, dest = extract(bytes:sub(1, 10 + math.floor((#bytes - 18) / 2)), 'tar.gz') + t.falsy(ok) + t.matches('gunzip failed', err) + t.falsy(read(dest .. '/root/a.lua'), 'a partial inflate must not reach disk') + end) + + t.it('fails loudly on a member corrupted in the middle', function() + local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic') + local ok, err, dest = extract(flip(bytes, math.floor(#bytes / 2), 4), 'tar.gz') + t.falsy(ok) + t.matches('gunzip failed', err) + t.falsy(read(dest .. '/root/a.lua'), 'a corrupt inflate must not reach disk') + end) + + t.it('refuses a hostile path inside a compressed member', function() + local bytes = gz(tarf({ { name = '../evil.lua', data = payload } }), 'dynamic') + local ok, err, dest = extract(bytes, 'tar.gz') + t.falsy(ok) + t.matches('escapes', err) + t.falsy(read(dest .. '/../evil.lua'), 'nothing should be written') + end) + + t.it('stops a pax record that claims a zero length', function() + local bytes = gz( + tarf({ + { name = 'root/pax', kind = 'x', data = '0 path=root/a.lua\n' }, + { name = 'root/a.lua', data = 'ok' }, + }), + 'dynamic' + ) + local ok, err, dest = extract(bytes, 'tar.gz') + t.truthy(ok, err) + t.eq('ok', read(dest .. '/root/a.lua')) + end) +end) diff --git a/tests/deflate.lua b/tests/deflate.lua new file mode 100644 index 0000000..c967bc3 --- /dev/null +++ b/tests/deflate.lua @@ -0,0 +1,315 @@ +-- A minimal DEFLATE encoder, so compressed fixtures are built here rather than +-- shelled out to a compressor that may be absent and silently skip the tests. +local M = {} + +local MIN_MATCH, MAX_MATCH, WINDOW = 3, 258, 32768 +local END_OF_BLOCK = 256 +local MAX_LIT_SYMBOLS, MAX_DIST_SYMBOLS, MAX_CL_SYMBOLS = 286, 30, 19 + +local function nums(spec) + local out = {} + for n in spec:gmatch('%d+') do + table.insert(out, tonumber(n)) + end + return out +end + +-- RFC 1951 section 3.2.5: length codes 257..285 and distance codes 0..29. +local LENGTH_BASE = nums('3 4 5 6 7 8 9 10 11 13 15 17 19 23 27 31 35 43 51 59 67 83 99 115 131 163 195 227 258') +local LENGTH_EXTRA = nums('0 0 0 0 0 0 0 0 1 1 1 1 2 2 2 2 3 3 3 3 4 4 4 4 5 5 5 5 0') +local DIST_BASE = nums( + '1 2 3 4 5 7 9 13 17 25 33 49 65 97 129 193 257 385 513 769 1025 1537 2049 3073 4097 6145 8193 12289 16385 24577' +) +local DIST_EXTRA = nums('0 0 0 0 1 1 2 2 3 3 4 4 5 5 6 6 7 7 8 8 9 9 10 10 11 11 12 12 13 13') +local CL_ORDER = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 } + +local Writer = {} +Writer.__index = Writer + +local function writer() + return setmetatable({ out = {}, acc = 0, n = 0 }, Writer) +end + +-- Plain values go out least significant bit first. +function Writer:bits(value, count) + for _ = 1, count do + self.acc = self.acc + (value % 2) * 2 ^ self.n + value = math.floor(value / 2) + self.n = self.n + 1 + if self.n == 8 then + table.insert(self.out, string.char(self.acc)) + self.acc, self.n = 0, 0 + end + end +end + +-- Huffman codes go out most significant bit first. +function Writer:code(code, len) + for i = len - 1, 0, -1 do + self:bits(math.floor(code / 2 ^ i) % 2, 1) + end +end + +function Writer:finish() + if self.n > 0 then + table.insert(self.out, string.char(self.acc)) + self.acc, self.n = 0, 0 + end + return table.concat(self.out) +end + +local function length_code(len) + for i = #LENGTH_BASE, 1, -1 do + if len >= LENGTH_BASE[i] then + return 256 + i, len - LENGTH_BASE[i], LENGTH_EXTRA[i] + end + end + error('length out of range: ' .. len) +end + +local function dist_code(dist) + for i = #DIST_BASE, 1, -1 do + if dist >= DIST_BASE[i] then + return i - 1, dist - DIST_BASE[i], DIST_EXTRA[i] + end + end + error('distance out of range: ' .. dist) +end + +-- Greedy LZ77 over a single-slot hash of the next three bytes. Good enough to +-- put real matches, extra bits and overlapping copies into the stream. +local function tokenize(data) + local tokens, head, pos, n = {}, {}, 1, #data + while pos <= n do + local best, dist = 0, 0 + if pos + MIN_MATCH - 1 <= n then + local key = data:sub(pos, pos + MIN_MATCH - 1) + local prev = head[key] + if prev and pos - prev <= WINDOW then + local len = 0 + while len < MAX_MATCH and pos + len <= n and data:byte(prev + len) == data:byte(pos + len) do + len = len + 1 + end + if len >= MIN_MATCH then + best, dist = len, pos - prev + end + end + head[key] = pos + end + if best > 0 then + table.insert(tokens, { len = best, dist = dist }) + for i = pos + 1, math.min(pos + best - 1, n - MIN_MATCH + 1) do + head[data:sub(i, i + MIN_MATCH - 1)] = i + end + pos = pos + best + else + table.insert(tokens, data:byte(pos)) + pos = pos + 1 + end + end + return tokens +end + +local function fixed_lit(sym) + if sym <= 143 then + return 0x30 + sym, 8 + elseif sym <= 255 then + return 0x190 + sym - 144, 9 + elseif sym <= 279 then + return sym - 256, 7 + end + return 0xc0 + sym - 280, 8 +end + +local function emit(w, tokens, litcode, distcode) + for _, tk in ipairs(tokens) do + if type(tk) == 'number' then + w:code(litcode(tk)) + else + local lc, lextra, lbits = length_code(tk.len) + w:code(litcode(lc)) + if lbits > 0 then + w:bits(lextra, lbits) + end + local dc, dextra, dbits = dist_code(tk.dist) + w:code(distcode(dc)) + if dbits > 0 then + w:bits(dextra, dbits) + end + end + end + w:code(litcode(END_OF_BLOCK)) +end + +-- A one-leaf alphabet cannot produce a complete prefix code, so pad it. +local function ensure_two(freq, limit) + local used = 0 + for _ in pairs(freq) do + used = used + 1 + end + for sym = 0, limit - 1 do + if used >= 2 then + return + end + if not freq[sym] then + freq[sym] = 1 + used = used + 1 + end + end +end + +local function build_lengths(weights) + local nodes, lens = {}, {} + for sym, weight in pairs(weights) do + table.insert(nodes, { w = weight, syms = { sym } }) + lens[sym] = 0 + end + assert(#nodes >= 2, 'huffman needs at least two symbols') + while #nodes > 1 do + table.sort(nodes, function(a, b) + if a.w ~= b.w then + return a.w < b.w + end + return a.syms[1] < b.syms[1] + end) + local a, b = table.remove(nodes, 1), table.remove(nodes, 1) + local syms = {} + for _, list in ipairs({ a.syms, b.syms }) do + for _, sym in ipairs(list) do + lens[sym] = lens[sym] + 1 + table.insert(syms, sym) + end + end + table.insert(nodes, { w = a.w + b.w, syms = syms }) + end + return lens +end + +-- Flattening the weights shortens the deepest codes; a flat alphabet of at most +-- 286 symbols never needs more than nine bits, so this always settles. +local function huffman(freq, limit) + local weights = {} + for sym, weight in pairs(freq) do + weights[sym] = weight + end + while true do + local lens, worst = build_lengths(weights), 0 + for _, len in pairs(lens) do + worst = math.max(worst, len) + end + if worst <= limit then + return lens + end + for sym, weight in pairs(weights) do + weights[sym] = math.ceil(weight / 2) + end + end +end + +local function canonical(lens) + local counts, maxlen, syms = {}, 0, {} + for sym, len in pairs(lens) do + counts[len] = (counts[len] or 0) + 1 + maxlen = math.max(maxlen, len) + table.insert(syms, sym) + end + local next_code, code = {}, 0 + for bits = 1, maxlen do + code = (code + (counts[bits - 1] or 0)) * 2 + next_code[bits] = code + end + table.sort(syms) + local codes = {} + for _, sym in ipairs(syms) do + codes[sym] = next_code[lens[sym]] + next_code[lens[sym]] = next_code[lens[sym]] + 1 + end + return codes +end + +local function highest(lens) + local top = 0 + for sym in pairs(lens) do + top = math.max(top, sym) + end + return top +end + +local function fixed_block(w, tokens, final) + w:bits(final, 1) + w:bits(1, 2) + emit(w, tokens, fixed_lit, function(sym) + return sym, 5 + end) +end + +local function dynamic_block(w, tokens, final) + local litfreq, distfreq = { [END_OF_BLOCK] = 1 }, {} + for _, tk in ipairs(tokens) do + if type(tk) == 'number' then + litfreq[tk] = (litfreq[tk] or 0) + 1 + else + local lc = length_code(tk.len) + litfreq[lc] = (litfreq[lc] or 0) + 1 + local dc = dist_code(tk.dist) + distfreq[dc] = (distfreq[dc] or 0) + 1 + end + end + ensure_two(litfreq, MAX_LIT_SYMBOLS) + ensure_two(distfreq, MAX_DIST_SYMBOLS) + + local litlens, distlens = huffman(litfreq, 15), huffman(distfreq, 15) + local hlit = math.max(257, highest(litlens) + 1) + local hdist = highest(distlens) + 1 + + local seq = {} + for sym = 0, hlit - 1 do + table.insert(seq, litlens[sym] or 0) + end + for sym = 0, hdist - 1 do + table.insert(seq, distlens[sym] or 0) + end + + -- Code lengths are written one per symbol; the run-length codes 16..18 are + -- left unused so the encoder stays small. + local clfreq = {} + for _, len in ipairs(seq) do + clfreq[len] = (clfreq[len] or 0) + 1 + end + ensure_two(clfreq, 16) + local cllens = huffman(clfreq, 7) + local clcodes = canonical(cllens) + + w:bits(final, 1) + w:bits(2, 2) + w:bits(hlit - 257, 5) + w:bits(hdist - 1, 5) + w:bits(MAX_CL_SYMBOLS - 4, 4) + for _, sym in ipairs(CL_ORDER) do + w:bits(cllens[sym] or 0, 3) + end + for _, len in ipairs(seq) do + w:code(clcodes[len], cllens[len]) + end + + local litcodes, distcodes = canonical(litlens), canonical(distlens) + emit(w, tokens, function(sym) + return litcodes[sym], litlens[sym] + end, function(sym) + return distcodes[sym], distlens[sym] + end) +end + +-- Returns a raw DEFLATE stream. mode is 'dynamic' (default) or 'fixed'. +function M.compress(data, mode) + local w = writer() + local tokens = tokenize(data) + if mode == 'fixed' then + fixed_block(w, tokens, 1) + else + dynamic_block(w, tokens, 1) + end + return w:finish() +end + +return M