unkin/arti-pack
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Test the DEFLATE path with real compressed fixtures

Browse Source 
The fixture builder only produced stored entries, so the vendored inflate
never ran under test. Add a small DEFLATE encoder and drive zip and
tar.gz entries through it: round-trips for fixed and dynamic huffman,
empty and large payloads, plus wrong CRC, wrong stated size, truncated
and corrupted streams, and a traversal name on a compressed entry.

Also cover the central directory and local header truncations that used
to raise.
This commit is contained in:
Unkin Agent
2026-09-10 23:18:29 +10:00
parent 407ad8093f
commit 78a5df6cf3
2 changed files with 616 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/archive_spec.lua
+301 -17
View File
@@ -1,5 +1,6 @@
local t = require('tests.harness')
local archive = require('arti-pack.archive')
local deflate = require('tests.deflate')
local zzlib = require('arti-pack.vendor.zzlib')
local function tmpdir()
@@ -18,45 +19,56 @@ local function le(n, bytes)
end
-- Fixtures are built here rather than shelled out to zip/tar so the hostile
-- entry names below are exactly the bytes under test.
-- entry names below are exactly the bytes under test. `deflate = true` runs the
-- payload through a real compressor, so DEFLATE entries are not approximated
-- and the tests cannot skip themselves when no zip binary is around.
local function zipf(entries)
local locals, central, offset = {}, {}, 0
for _, e in ipairs(entries) do
local data = e.data or ''
local crc = zzlib.crc32(data)
local method, payload = 0, data
if e.deflate then
method, payload = 8, deflate.compress(data, e.mode)
end
method = e.method or method
if e.mangle then
payload = e.mangle(payload)
end
local crc = e.crc or zzlib.crc32(data)
local csize, usize = #payload, e.usize or #data
local head = 'PK\3\4'
.. le(20, 2)
.. le(e.flags or 0, 2)
.. le(0, 2)
.. le(method, 2)
.. le(0, 4)
.. le(e.crc or crc, 4)
.. le(#data, 4)
.. le(#data, 4)
.. le(crc, 4)
.. le(csize, 4)
.. le(usize, 4)
.. le(#e.name, 2)
.. le(0, 2)
.. e.name
table.insert(locals, head .. data)
table.insert(locals, head .. payload)
table.insert(
central,
'PK\1\2'
.. le(e.made_by or (3 * 256 + 20), 2)
.. le(20, 2)
.. le(e.flags or 0, 2)
.. le(0, 2)
.. le(method, 2)
.. le(0, 4)
.. le(e.crc or crc, 4)
.. le(#data, 4)
.. le(#data, 4)
.. le(crc, 4)
.. le(csize, 4)
.. le(usize, 4)
.. le(#e.name, 2)
.. le(0, 2)
.. le(0, 2)
.. le(0, 2)
.. le(0, 2)
.. le(e.attr or 0, 4)
.. le(offset, 4)
.. le(e.local_offset or offset, 4)
.. e.name
)
offset = offset + #head + #data
offset = offset + #head + #payload
end
local body = table.concat(locals)
local cd = table.concat(central)
@@ -73,9 +85,7 @@ local function zipf(entries)
return body .. cd .. eocd
end
-- A gzip member whose DEFLATE payload is all stored blocks: valid, and buildable
-- without a compressor.
local function gz(data)
local function stored_blocks(data)
local blocks, pos = {}, 1
repeat
local chunk = data:sub(pos, pos + 65534)
@@ -83,7 +93,14 @@ local function gz(data)
local final = pos > #data and 1 or 0
table.insert(blocks, string.char(final) .. le(#chunk, 2) .. le(65535 - #chunk, 2) .. chunk)
until pos > #data
return '\31\139\8\0\0\0\0\0\0\3' .. table.concat(blocks) .. le(zzlib.crc32(data), 4) .. le(#data, 4)
return table.concat(blocks)
end
-- A gzip member. Without a mode the DEFLATE payload is all stored blocks; with
-- one it is genuinely compressed, which is what drives the huffman decoder.
local function gz(data, mode)
local payload = mode and deflate.compress(data, mode) or stored_blocks(data)
return '\31\139\8\0\0\0\0\0\0\3' .. payload .. le(zzlib.crc32(data), 4) .. le(#data, 4)
end
local function tar_header(name, size, kind, mode)
@@ -140,6 +157,42 @@ local function extract(bytes, ext)
return ok, err, dest
end
-- Deterministic pseudo-random bytes, so a compressed fixture is the same on
-- every run but still spans the whole literal alphabet.
local function noise(n, seed)
local out, state = {}, seed
for _ = 1, n do
state = (state * 1103515245 + 12345) % 2147483648
table.insert(out, string.char(math.floor(state / 65536) % 256))
end
return table.concat(out)
end
-- Every byte value, then repeated text: literal codes of both widths plus real
-- back-references.
local function sample()
local parts = {}
for i = 0, 255 do
table.insert(parts, string.char(i))
end
for i = 1, 40 do
table.insert(parts, string.format('local module_%d = require("arti-pack.mod")\n', i))
end
return table.concat(parts)
end
local function patch32(bytes, at, value)
return bytes:sub(1, at - 1) .. le(value, 4) .. bytes:sub(at + 4)
end
local function flip(bytes, at, count)
local out = bytes:sub(1, at - 1)
for i = at, at + count - 1 do
out = out .. string.char((bytes:byte(i) + 137) % 256)
end
return out .. bytes:sub(at + count)
end
t.describe('archive.safe_path', function()
t.it('accepts an ordinary nested path', function()
t.eq('plugin/lua/init.lua', archive.safe_path('plugin/lua/init.lua'))
@@ -261,6 +314,154 @@ t.describe('archive.extract zip', function()
local ok = extract('PK\3\4' .. string.rep('\200', 400))
t.falsy(ok)
end)
t.it('rejects an unsupported compression method', function()
local ok, err = extract(zipf({ { name = 'root/a.lua', data = 'x', method = 12 } }))
t.falsy(ok)
t.matches('unsupported compression method 12', err)
end)
end)
t.describe('archive.extract zip malformed directory', function()
-- A central directory record that stops before its extra-field and comment
-- lengths used to raise instead of reporting.
local function truncate_cd(keep)
local bytes = zipf({ { name = 'root/a.lua', data = 'hi' } })
local cd = bytes:find('PK\1\2', 1, true)
local eocd = bytes:find('PK\5\6', cd, true)
return bytes:sub(1, cd + keep - 1) .. bytes:sub(eocd)
end
t.it('reports a record cut off before its length fields', function()
local ok, err = extract(truncate_cd(30))
t.falsy(ok)
t.matches('truncated central directory', err)
end)
t.it('reports a record cut off just after its signature', function()
local ok, err = extract(truncate_cd(4))
t.falsy(ok)
t.matches('truncated central directory', err)
end)
t.it('reports a local header that runs off the end of the archive', function()
-- The trailing comment plants a local-header signature in the last four
-- bytes; the entry is pointed at it.
local bytes = zipf({ { name = 'root/a.lua', data = 'hi' }, comment = 'PK\3\4' })
local cd = bytes:find('PK\1\2', 1, true)
local ok, err = extract(patch32(bytes, cd + 42, #bytes - 4))
t.falsy(ok)
t.matches('truncated local header', err)
end)
end)
t.describe('archive.extract zip deflate', function()
local payload = sample()
t.it('round-trips a dynamic-huffman entry byte for byte', function()
local ok, err, dest = extract(zipf({ { name = 'root/a.lua', data = payload, deflate = true } }))
t.truthy(ok, err)
t.eq(payload, read(dest .. '/root/a.lua'))
end)
t.it('round-trips a fixed-huffman entry byte for byte', function()
local bytes = zipf({ { name = 'root/a.lua', data = payload, deflate = true, mode = 'fixed' } })
local ok, err, dest = extract(bytes)
t.truthy(ok, err)
t.eq(payload, read(dest .. '/root/a.lua'))
end)
t.it('round-trips deflated entries alongside stored ones', function()
local overlapping = string.rep('ab', 3000)
local ok, err, dest = extract(zipf({
{ name = 'root/' },
{ name = 'root/stored.lua', data = 'return {}' },
{ name = 'root/packed.lua', data = payload, deflate = true },
{ name = 'root/nested/overlap.txt', data = overlapping, deflate = true },
}))
t.truthy(ok, err)
t.eq('return {}', read(dest .. '/root/stored.lua'))
t.eq(payload, read(dest .. '/root/packed.lua'))
t.eq(overlapping, read(dest .. '/root/nested/overlap.txt'))
end)
t.it('round-trips a deflated entry that decompresses to nothing', function()
local ok, err, dest = extract(zipf({ { name = 'root/empty.lua', data = '', deflate = true } }))
t.truthy(ok, err)
t.eq('', read(dest .. '/root/empty.lua'))
end)
t.it('round-trips a large deflated entry', function()
local big = string.rep(noise(1024, 11), 400)
local bytes = zipf({ { name = 'root/big.bin', data = big, deflate = true } })
t.truthy(#bytes < #big / 4, 'the fixture is not actually compressed')
local ok, err, dest = extract(bytes)
t.truthy(ok, err)
local out = read(dest .. '/root/big.bin')
t.eq(#big, #out)
t.truthy(out == big, 'the large entry did not round-trip exactly')
end)
t.it('fails loudly when a deflated entry carries the wrong CRC', function()
local ok, err, dest = extract(zipf({ { name = 'root/a.lua', data = payload, deflate = true, crc = 12345 } }))
t.falsy(ok)
t.matches('checksum', err)
t.falsy(read(dest .. '/root/a.lua'), 'nothing should be written')
end)
t.it('fails loudly when a deflated entry disagrees with the stated size', function()
local bytes = zipf({ { name = 'root/a.lua', data = payload, deflate = true, usize = #payload + 1 } })
local ok, err, dest = extract(bytes)
t.falsy(ok)
t.matches('does not match the directory', err)
t.falsy(read(dest .. '/root/a.lua'), 'nothing should be written')
end)
t.it('fails loudly on a truncated deflate stream', function()
local ok, err, dest = extract(zipf({
{
name = 'root/a.lua',
data = payload,
deflate = true,
mangle = function(s)
return s:sub(1, math.floor(#s / 2))
end,
},
}))
t.falsy(ok)
t.matches('root/a%.lua', err)
t.falsy(read(dest .. '/root/a.lua'), 'a partial inflate must not reach disk')
end)
t.it('fails loudly on a deflate stream corrupted in the middle', function()
local ok, err, dest = extract(zipf({
{
name = 'root/a.lua',
data = payload,
deflate = true,
mangle = function(s)
return flip(s, math.floor(#s / 2), 4)
end,
},
}))
t.falsy(ok)
t.matches('root/a%.lua', err)
t.falsy(read(dest .. '/root/a.lua'), 'a corrupt inflate must not reach disk')
end)
t.it('refuses a deflated entry that walks out of the destination', function()
local ok, err, dest = extract(zipf({ { name = '../evil.lua', data = payload, deflate = true } }))
t.falsy(ok)
t.matches('escapes', err)
t.falsy(read(dest .. '/../evil.lua'), 'nothing should be written')
end)
t.it('refuses a deflated symlink entry', function()
local ok, err = extract(zipf({
{ name = 'root/link', data = string.rep('/etc/passwd', 40), deflate = true, attr = (0xa000 + 0x1ff) * 65536 },
}))
t.falsy(ok)
t.matches('not a regular file', err)
end)
end)
t.describe('archive.extract tar.gz', function()
@@ -338,3 +539,86 @@ t.describe('archive.extract tar.gz', function()
t.matches('checksum mismatch', err)
end)
end)
-- The stored-block fixtures above never reach the huffman decoder, so the
-- compressed cases are covered separately.
t.describe('archive.extract tar.gz deflate', function()
local payload = sample()
t.it('inflates a dynamic-huffman member', function()
local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic')
local ok, err, dest = extract(bytes, 'tar.gz')
t.truthy(ok, err)
t.eq(payload, read(dest .. '/root/a.lua'))
end)
t.it('inflates a fixed-huffman member', function()
local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'fixed')
local ok, err, dest = extract(bytes, 'tar.gz')
t.truthy(ok, err)
t.eq(payload, read(dest .. '/root/a.lua'))
end)
t.it('inflates an empty member', function()
local bytes = gz(tarf({ { name = 'root/empty.lua', data = '' } }), 'dynamic')
local ok, err, dest = extract(bytes, 'tar.gz')
t.truthy(ok, err)
t.eq('', read(dest .. '/root/empty.lua'))
end)
t.it('inflates a large member', function()
local big = string.rep(noise(1024, 29), 300)
local bytes = gz(tarf({ { name = 'root/big.bin', data = big } }), 'dynamic')
t.truthy(#bytes < #big / 4, 'the fixture is not actually compressed')
local ok, err, dest = extract(bytes, 'tar.gz')
t.truthy(ok, err)
local out = read(dest .. '/root/big.bin')
t.eq(#big, #out)
t.truthy(out == big, 'the large member did not round-trip exactly')
end)
t.it('fails loudly when the gzip checksum disagrees', function()
local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic')
local ok, err, dest = extract(patch32(bytes, #bytes - 7, 12345), 'tar.gz')
t.falsy(ok)
t.matches('gunzip failed', err)
t.falsy(read(dest .. '/root/a.lua'), 'nothing should be written')
end)
t.it('fails loudly on a truncated compressed member', function()
local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic')
local ok, err, dest = extract(bytes:sub(1, 10 + math.floor((#bytes - 18) / 2)), 'tar.gz')
t.falsy(ok)
t.matches('gunzip failed', err)
t.falsy(read(dest .. '/root/a.lua'), 'a partial inflate must not reach disk')
end)
t.it('fails loudly on a member corrupted in the middle', function()
local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic')
local ok, err, dest = extract(flip(bytes, math.floor(#bytes / 2), 4), 'tar.gz')
t.falsy(ok)
t.matches('gunzip failed', err)
t.falsy(read(dest .. '/root/a.lua'), 'a corrupt inflate must not reach disk')
end)
t.it('refuses a hostile path inside a compressed member', function()
local bytes = gz(tarf({ { name = '../evil.lua', data = payload } }), 'dynamic')
local ok, err, dest = extract(bytes, 'tar.gz')
t.falsy(ok)
t.matches('escapes', err)
t.falsy(read(dest .. '/../evil.lua'), 'nothing should be written')
end)
t.it('stops a pax record that claims a zero length', function()
local bytes = gz(
tarf({
{ name = 'root/pax', kind = 'x', data = '0 path=root/a.lua\n' },
{ name = 'root/a.lua', data = 'ok' },
}),
'dynamic'
)
local ok, err, dest = extract(bytes, 'tar.gz')
t.truthy(ok, err)
t.eq('ok', read(dest .. '/root/a.lua'))
end)
end)
tests/deflate.lua
+315
View File
@@ -0,0 +1,315 @@
-- A minimal DEFLATE encoder, so compressed fixtures are built here rather than
-- shelled out to a compressor that may be absent and silently skip the tests.
local M = {}
local MIN_MATCH, MAX_MATCH, WINDOW = 3, 258, 32768
local END_OF_BLOCK = 256
local MAX_LIT_SYMBOLS, MAX_DIST_SYMBOLS, MAX_CL_SYMBOLS = 286, 30, 19
local function nums(spec)
local out = {}
for n in spec:gmatch('%d+') do
table.insert(out, tonumber(n))
end
return out
end
-- RFC 1951 section 3.2.5: length codes 257..285 and distance codes 0..29.
local LENGTH_BASE = nums('3 4 5 6 7 8 9 10 11 13 15 17 19 23 27 31 35 43 51 59 67 83 99 115 131 163 195 227 258')
local LENGTH_EXTRA = nums('0 0 0 0 0 0 0 0 1 1 1 1 2 2 2 2 3 3 3 3 4 4 4 4 5 5 5 5 0')
local DIST_BASE = nums(
'1 2 3 4 5 7 9 13 17 25 33 49 65 97 129 193 257 385 513 769 1025 1537 2049 3073 4097 6145 8193 12289 16385 24577'
)
local DIST_EXTRA = nums('0 0 0 0 1 1 2 2 3 3 4 4 5 5 6 6 7 7 8 8 9 9 10 10 11 11 12 12 13 13')
local CL_ORDER = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }
local Writer = {}
Writer.__index = Writer
local function writer()
return setmetatable({ out = {}, acc = 0, n = 0 }, Writer)
end
-- Plain values go out least significant bit first.
function Writer:bits(value, count)
for _ = 1, count do
self.acc = self.acc + (value % 2) * 2 ^ self.n
value = math.floor(value / 2)
self.n = self.n + 1
if self.n == 8 then
table.insert(self.out, string.char(self.acc))
self.acc, self.n = 0, 0
end
end
end
-- Huffman codes go out most significant bit first.
function Writer:code(code, len)
for i = len - 1, 0, -1 do
self:bits(math.floor(code / 2 ^ i) % 2, 1)
end
end
function Writer:finish()
if self.n > 0 then
table.insert(self.out, string.char(self.acc))
self.acc, self.n = 0, 0
end
return table.concat(self.out)
end
local function length_code(len)
for i = #LENGTH_BASE, 1, -1 do
if len >= LENGTH_BASE[i] then
return 256 + i, len - LENGTH_BASE[i], LENGTH_EXTRA[i]
end
end
error('length out of range: ' .. len)
end
local function dist_code(dist)
for i = #DIST_BASE, 1, -1 do
if dist >= DIST_BASE[i] then
return i - 1, dist - DIST_BASE[i], DIST_EXTRA[i]
end
end
error('distance out of range: ' .. dist)
end
-- Greedy LZ77 over a single-slot hash of the next three bytes. Good enough to
-- put real matches, extra bits and overlapping copies into the stream.
local function tokenize(data)
local tokens, head, pos, n = {}, {}, 1, #data
while pos <= n do
local best, dist = 0, 0
if pos + MIN_MATCH - 1 <= n then
local key = data:sub(pos, pos + MIN_MATCH - 1)
local prev = head[key]
if prev and pos - prev <= WINDOW then
local len = 0
while len < MAX_MATCH and pos + len <= n and data:byte(prev + len) == data:byte(pos + len) do
len = len + 1
end
if len >= MIN_MATCH then
best, dist = len, pos - prev
end
end
head[key] = pos
end
if best > 0 then
table.insert(tokens, { len = best, dist = dist })
for i = pos + 1, math.min(pos + best - 1, n - MIN_MATCH + 1) do
head[data:sub(i, i + MIN_MATCH - 1)] = i
end
pos = pos + best
else
table.insert(tokens, data:byte(pos))
pos = pos + 1
end
end
return tokens
end
local function fixed_lit(sym)
if sym <= 143 then
return 0x30 + sym, 8
elseif sym <= 255 then
return 0x190 + sym - 144, 9
elseif sym <= 279 then
return sym - 256, 7
end
return 0xc0 + sym - 280, 8
end
local function emit(w, tokens, litcode, distcode)
for _, tk in ipairs(tokens) do
if type(tk) == 'number' then
w:code(litcode(tk))
else
local lc, lextra, lbits = length_code(tk.len)
w:code(litcode(lc))
if lbits > 0 then
w:bits(lextra, lbits)
end
local dc, dextra, dbits = dist_code(tk.dist)
w:code(distcode(dc))
if dbits > 0 then
w:bits(dextra, dbits)
end
end
end
w:code(litcode(END_OF_BLOCK))
end
-- A one-leaf alphabet cannot produce a complete prefix code, so pad it.
local function ensure_two(freq, limit)
local used = 0
for _ in pairs(freq) do
used = used + 1
end
for sym = 0, limit - 1 do
if used >= 2 then
return
end
if not freq[sym] then
freq[sym] = 1
used = used + 1
end
end
end
local function build_lengths(weights)
local nodes, lens = {}, {}
for sym, weight in pairs(weights) do
table.insert(nodes, { w = weight, syms = { sym } })
lens[sym] = 0
end
assert(#nodes >= 2, 'huffman needs at least two symbols')
while #nodes > 1 do
table.sort(nodes, function(a, b)
if a.w ~= b.w then
return a.w < b.w
end
return a.syms[1] < b.syms[1]
end)
local a, b = table.remove(nodes, 1), table.remove(nodes, 1)
local syms = {}
for _, list in ipairs({ a.syms, b.syms }) do
for _, sym in ipairs(list) do
lens[sym] = lens[sym] + 1
table.insert(syms, sym)
end
end
table.insert(nodes, { w = a.w + b.w, syms = syms })
end
return lens
end
-- Flattening the weights shortens the deepest codes; a flat alphabet of at most
-- 286 symbols never needs more than nine bits, so this always settles.
local function huffman(freq, limit)
local weights = {}
for sym, weight in pairs(freq) do
weights[sym] = weight
end
while true do
local lens, worst = build_lengths(weights), 0
for _, len in pairs(lens) do
worst = math.max(worst, len)
end
if worst <= limit then
return lens
end
for sym, weight in pairs(weights) do
weights[sym] = math.ceil(weight / 2)
end
end
end
local function canonical(lens)
local counts, maxlen, syms = {}, 0, {}
for sym, len in pairs(lens) do
counts[len] = (counts[len] or 0) + 1
maxlen = math.max(maxlen, len)
table.insert(syms, sym)
end
local next_code, code = {}, 0
for bits = 1, maxlen do
code = (code + (counts[bits - 1] or 0)) * 2
next_code[bits] = code
end
table.sort(syms)
local codes = {}
for _, sym in ipairs(syms) do
codes[sym] = next_code[lens[sym]]
next_code[lens[sym]] = next_code[lens[sym]] + 1
end
return codes
end
local function highest(lens)
local top = 0
for sym in pairs(lens) do
top = math.max(top, sym)
end
return top
end
local function fixed_block(w, tokens, final)
w:bits(final, 1)
w:bits(1, 2)
emit(w, tokens, fixed_lit, function(sym)
return sym, 5
end)
end
local function dynamic_block(w, tokens, final)
local litfreq, distfreq = { [END_OF_BLOCK] = 1 }, {}
for _, tk in ipairs(tokens) do
if type(tk) == 'number' then
litfreq[tk] = (litfreq[tk] or 0) + 1
else
local lc = length_code(tk.len)
litfreq[lc] = (litfreq[lc] or 0) + 1
local dc = dist_code(tk.dist)
distfreq[dc] = (distfreq[dc] or 0) + 1
end
end
ensure_two(litfreq, MAX_LIT_SYMBOLS)
ensure_two(distfreq, MAX_DIST_SYMBOLS)
local litlens, distlens = huffman(litfreq, 15), huffman(distfreq, 15)
local hlit = math.max(257, highest(litlens) + 1)
local hdist = highest(distlens) + 1
local seq = {}
for sym = 0, hlit - 1 do
table.insert(seq, litlens[sym] or 0)
end
for sym = 0, hdist - 1 do
table.insert(seq, distlens[sym] or 0)
end
-- Code lengths are written one per symbol; the run-length codes 16..18 are
-- left unused so the encoder stays small.
local clfreq = {}
for _, len in ipairs(seq) do
clfreq[len] = (clfreq[len] or 0) + 1
end
ensure_two(clfreq, 16)
local cllens = huffman(clfreq, 7)
local clcodes = canonical(cllens)
w:bits(final, 1)
w:bits(2, 2)
w:bits(hlit - 257, 5)
w:bits(hdist - 1, 5)
w:bits(MAX_CL_SYMBOLS - 4, 4)
for _, sym in ipairs(CL_ORDER) do
w:bits(cllens[sym] or 0, 3)
end
for _, len in ipairs(seq) do
w:code(clcodes[len], cllens[len])
end
local litcodes, distcodes = canonical(litlens), canonical(distlens)
emit(w, tokens, function(sym)
return litcodes[sym], litlens[sym]
end, function(sym)
return distcodes[sym], distlens[sym]
end)
end
-- Returns a raw DEFLATE stream. mode is 'dynamic' (default) or 'fixed'.
function M.compress(data, mode)
local w = writer()
local tokens = tokenize(data)
if mode == 'fixed' then
fixed_block(w, tokens, 1)
else
dynamic_block(w, tokens, 1)
end
return w:finish()
end
return M