lua/arti-pack/archive.lua

@@ -70,15 +70,24 @@ local function reject(name, why)
   return nil, string.format('%s: %s', name, why)
 end
 
+-- vim.fn.mkdir raises when a path component is an existing file, which a
+-- hostile archive can arrange by shipping a file and then a path under it.
 local function mkdirp(path)
-  vim.fn.mkdir(path, 'p')
+  local ok, err = pcall(vim.fn.mkdir, path, 'p')
+  if not ok then
+    return nil, (tostring(err):gsub('^Vim:', ''))
+  end
+  return true
 end
 
 local function write_file(dest, rel, data, executable)
   local path = dest .. '/' .. rel
   local parent = path:match('^(.*)/[^/]+$')
   if parent then
-    mkdirp(parent)
+    local made, merr = mkdirp(parent)
+    if not made then
+      return nil, string.format('%s: %s', rel, merr)
+    end
   end
   local fd, err = io.open(path, 'wb')
   if not fd then
@@ -140,6 +149,11 @@ local function zip_entries(buf)
     if u32(buf, pos) ~= CD_SIG then
       return nil, 'malformed central directory'
     end
+    -- The fixed 46-byte header has to be there in full, otherwise the fields
+    -- read below are nil and the parse blows up instead of reporting.
+    if pos + 45 > #buf then
+      return nil, 'truncated central directory'
+    end
     local namelen = u16(buf, pos + 28)
     local entry = {
       version_made_by = u16(buf, pos + 4),
@@ -166,6 +180,9 @@ local function zip_data(buf, entry)
   if u32(buf, lho) ~= LOCAL_SIG then
     return reject(entry.name, 'invalid local header signature')
   end
+  if lho + 29 > #buf then
+    return reject(entry.name, 'truncated local header')
+  end
   local start = lho + 30 + u16(buf, lho + 26) + u16(buf, lho + 28)
 
   local data
@@ -220,7 +237,10 @@ function M.unzip(buf, dest)
     end
 
     if entry.name:sub(-1) == '/' then
-      mkdirp(dest .. '/' .. rel)
+      local made, merr = mkdirp(dest .. '/' .. rel)
+      if not made then
+        return reject(entry.name, merr)
+      end
     else
       local data, derr = zip_data(buf, entry)
       if not data then
@@ -262,6 +282,10 @@ local function pax_records(block)
       break
     end
     len = tonumber(len)
+    -- A zero length would leave pos where it is and spin forever.
+    if len < 1 then
+      break
+    end
     local record = block:sub(rest, pos + len - 2)
     local key, value = record:match('^([^=]+)=(.*)$')
     if key then
@@ -327,7 +351,10 @@ function M.untar(buf, dest)
           return reject(name, perr)
         end
         if kind == '5' then
-          mkdirp(dest .. '/' .. rel)
+          local made, merr = mkdirp(dest .. '/' .. rel)
+          if not made then
+            return reject(name, merr)
+          end
         else
           local mode = octal(header:sub(101, 108))
           local ok, werr = write_file(dest, rel, body, mode % (MODE_EXEC * 2) >= MODE_EXEC)
@@ -357,7 +384,10 @@ function M.extract(path, dest, ext)
     return nil, 'archive is empty'
   end
 
-  mkdirp(dest)
+  local made, merr = mkdirp(dest)
+  if not made then
+    return nil, merr
+  end
 
   if ext == 'zip' then
     return M.unzip(buf, dest)

tests/archive_spec.lua

@@ -1,5 +1,6 @@
 local t = require('tests.harness')
 local archive = require('arti-pack.archive')
+local deflate = require('tests.deflate')
 local zzlib = require('arti-pack.vendor.zzlib')
 
 local function tmpdir()
@@ -18,45 +19,56 @@ local function le(n, bytes)
 end
 
 -- Fixtures are built here rather than shelled out to zip/tar so the hostile
--- entry names below are exactly the bytes under test.
+-- entry names below are exactly the bytes under test. `deflate = true` runs the
+-- payload through a real compressor, so DEFLATE entries are not approximated
+-- and the tests cannot skip themselves when no zip binary is around.
 local function zipf(entries)
   local locals, central, offset = {}, {}, 0
   for _, e in ipairs(entries) do
     local data = e.data or ''
-    local crc = zzlib.crc32(data)
+    local method, payload = 0, data
+    if e.deflate then
+      method, payload = 8, deflate.compress(data, e.mode)
+    end
+    method = e.method or method
+    if e.mangle then
+      payload = e.mangle(payload)
+    end
+    local crc = e.crc or zzlib.crc32(data)
+    local csize, usize = #payload, e.usize or #data
     local head = 'PK\3\4'
       .. le(20, 2)
       .. le(e.flags or 0, 2)
-      .. le(0, 2)
+      .. le(method, 2)
       .. le(0, 4)
-      .. le(e.crc or crc, 4)
-      .. le(#data, 4)
-      .. le(#data, 4)
+      .. le(crc, 4)
+      .. le(csize, 4)
+      .. le(usize, 4)
       .. le(#e.name, 2)
       .. le(0, 2)
       .. e.name
-    table.insert(locals, head .. data)
+    table.insert(locals, head .. payload)
     table.insert(
       central,
       'PK\1\2'
         .. le(e.made_by or (3 * 256 + 20), 2)
         .. le(20, 2)
         .. le(e.flags or 0, 2)
-        .. le(0, 2)
+        .. le(method, 2)
         .. le(0, 4)
-        .. le(e.crc or crc, 4)
-        .. le(#data, 4)
-        .. le(#data, 4)
+        .. le(crc, 4)
+        .. le(csize, 4)
+        .. le(usize, 4)
         .. le(#e.name, 2)
         .. le(0, 2)
         .. le(0, 2)
         .. le(0, 2)
         .. le(0, 2)
         .. le(e.attr or 0, 4)
-        .. le(offset, 4)
+        .. le(e.local_offset or offset, 4)
         .. e.name
     )
-    offset = offset + #head + #data
+    offset = offset + #head + #payload
   end
   local body = table.concat(locals)
   local cd = table.concat(central)
@@ -73,9 +85,7 @@ local function zipf(entries)
   return body .. cd .. eocd
 end
 
--- A gzip member whose DEFLATE payload is all stored blocks: valid, and buildable
--- without a compressor.
-local function gz(data)
+local function stored_blocks(data)
   local blocks, pos = {}, 1
   repeat
     local chunk = data:sub(pos, pos + 65534)
@@ -83,7 +93,14 @@ local function gz(data)
     local final = pos > #data and 1 or 0
     table.insert(blocks, string.char(final) .. le(#chunk, 2) .. le(65535 - #chunk, 2) .. chunk)
   until pos > #data
-  return '\31\139\8\0\0\0\0\0\0\3' .. table.concat(blocks) .. le(zzlib.crc32(data), 4) .. le(#data, 4)
+  return table.concat(blocks)
+end
+
+-- A gzip member. Without a mode the DEFLATE payload is all stored blocks; with
+-- one it is genuinely compressed, which is what drives the huffman decoder.
+local function gz(data, mode)
+  local payload = mode and deflate.compress(data, mode) or stored_blocks(data)
+  return '\31\139\8\0\0\0\0\0\0\3' .. payload .. le(zzlib.crc32(data), 4) .. le(#data, 4)
 end
 
 local function tar_header(name, size, kind, mode)
@@ -140,6 +157,42 @@ local function extract(bytes, ext)
   return ok, err, dest
 end
 
+-- Deterministic pseudo-random bytes, so a compressed fixture is the same on
+-- every run but still spans the whole literal alphabet.
+local function noise(n, seed)
+  local out, state = {}, seed
+  for _ = 1, n do
+    state = (state * 1103515245 + 12345) % 2147483648
+    table.insert(out, string.char(math.floor(state / 65536) % 256))
+  end
+  return table.concat(out)
+end
+
+-- Every byte value, then repeated text: literal codes of both widths plus real
+-- back-references.
+local function sample()
+  local parts = {}
+  for i = 0, 255 do
+    table.insert(parts, string.char(i))
+  end
+  for i = 1, 40 do
+    table.insert(parts, string.format('local module_%d = require("arti-pack.mod")\n', i))
+  end
+  return table.concat(parts)
+end
+
+local function patch32(bytes, at, value)
+  return bytes:sub(1, at - 1) .. le(value, 4) .. bytes:sub(at + 4)
+end
+
+local function flip(bytes, at, count)
+  local out = bytes:sub(1, at - 1)
+  for i = at, at + count - 1 do
+    out = out .. string.char((bytes:byte(i) + 137) % 256)
+  end
+  return out .. bytes:sub(at + count)
+end
+
 t.describe('archive.safe_path', function()
   t.it('accepts an ordinary nested path', function()
     t.eq('plugin/lua/init.lua', archive.safe_path('plugin/lua/init.lua'))
@@ -261,6 +314,175 @@ t.describe('archive.extract zip', function()
     local ok = extract('PK\3\4' .. string.rep('\200', 400))
     t.falsy(ok)
   end)
+  t.it('rejects an unsupported compression method', function()
+    local ok, err = extract(zipf({ { name = 'root/a.lua', data = 'x', method = 12 } }))
+    t.falsy(ok)
+    t.matches('unsupported compression method 12', err)
+  end)
+end)
+
+t.describe('archive.extract zip malformed directory', function()
+  -- A central directory record that stops short of its own header used to
+  -- raise instead of reporting.
+  local function truncate_cd(keep)
+    local bytes = zipf({ { name = 'root/a.lua', data = 'hi' } })
+    local cd = bytes:find('PK\1\2', 1, true)
+    local eocd = bytes:find('PK\5\6', cd, true)
+    return bytes:sub(1, cd + keep - 1) .. bytes:sub(eocd)
+  end
+
+  t.it('reports a record cut off before its extra-field lengths', function()
+    -- The directory offset is attacker-controlled, so point it at a record
+    -- planted in the trailing comment: a zero name length lets the parse run
+    -- cleanly up to the extra-field and comment lengths, which is where it
+    -- used to raise.
+    local comment = 'PK\1\2' .. string.rep('\0', 26)
+    local bytes = zipf({ { name = 'root/a.lua', data = 'hi' }, comment = comment })
+    local ok, err = extract(patch32(bytes, #bytes - #comment - 21 + 16, #bytes - #comment))
+    t.falsy(ok)
+    t.matches('truncated central directory', err)
+  end)
+
+  t.it('reports a record cut off just after its signature', function()
+    local ok, err = extract(truncate_cd(4))
+    t.falsy(ok)
+    t.matches('truncated central directory', err)
+  end)
+
+  t.it('reports a file that another entry then treats as a directory', function()
+    local ok, err = extract(zipf({
+      { name = 'root/a', data = 'x' },
+      { name = 'root/a/b', data = 'y' },
+    }))
+    t.falsy(ok)
+    t.matches('Cannot create directory', err)
+  end)
+
+  t.it('reports a directory entry that collides with an earlier file', function()
+    local ok, err = extract(zipf({ { name = 'root/a', data = 'x' }, { name = 'root/a/' } }))
+    t.falsy(ok)
+    t.matches('Cannot create directory', err)
+  end)
+
+  t.it('reports a local header that runs off the end of the archive', function()
+    -- The trailing comment plants a local-header signature in the last four
+    -- bytes; the entry is pointed at it.
+    local bytes = zipf({ { name = 'root/a.lua', data = 'hi' }, comment = 'PK\3\4' })
+    local cd = bytes:find('PK\1\2', 1, true)
+    local ok, err = extract(patch32(bytes, cd + 42, #bytes - 4))
+    t.falsy(ok)
+    t.matches('truncated local header', err)
+  end)
+end)
+
+t.describe('archive.extract zip deflate', function()
+  local payload = sample()
+
+  t.it('round-trips a dynamic-huffman entry byte for byte', function()
+    local ok, err, dest = extract(zipf({ { name = 'root/a.lua', data = payload, deflate = true } }))
+    t.truthy(ok, err)
+    t.eq(payload, read(dest .. '/root/a.lua'))
+  end)
+
+  t.it('round-trips a fixed-huffman entry byte for byte', function()
+    local bytes = zipf({ { name = 'root/a.lua', data = payload, deflate = true, mode = 'fixed' } })
+    local ok, err, dest = extract(bytes)
+    t.truthy(ok, err)
+    t.eq(payload, read(dest .. '/root/a.lua'))
+  end)
+
+  t.it('round-trips deflated entries alongside stored ones', function()
+    local overlapping = string.rep('ab', 3000)
+    local ok, err, dest = extract(zipf({
+      { name = 'root/' },
+      { name = 'root/stored.lua', data = 'return {}' },
+      { name = 'root/packed.lua', data = payload, deflate = true },
+      { name = 'root/nested/overlap.txt', data = overlapping, deflate = true },
+    }))
+    t.truthy(ok, err)
+    t.eq('return {}', read(dest .. '/root/stored.lua'))
+    t.eq(payload, read(dest .. '/root/packed.lua'))
+    t.eq(overlapping, read(dest .. '/root/nested/overlap.txt'))
+  end)
+
+  t.it('round-trips a deflated entry that decompresses to nothing', function()
+    local ok, err, dest = extract(zipf({ { name = 'root/empty.lua', data = '', deflate = true } }))
+    t.truthy(ok, err)
+    t.eq('', read(dest .. '/root/empty.lua'))
+  end)
+
+  t.it('round-trips a large deflated entry', function()
+    local big = string.rep(noise(1024, 11), 400)
+    local bytes = zipf({ { name = 'root/big.bin', data = big, deflate = true } })
+    t.truthy(#bytes < #big / 4, 'the fixture is not actually compressed')
+    local ok, err, dest = extract(bytes)
+    t.truthy(ok, err)
+    local out = read(dest .. '/root/big.bin')
+    t.eq(#big, #out)
+    t.truthy(out == big, 'the large entry did not round-trip exactly')
+  end)
+
+  t.it('fails loudly when a deflated entry carries the wrong CRC', function()
+    local ok, err, dest = extract(zipf({ { name = 'root/a.lua', data = payload, deflate = true, crc = 12345 } }))
+    t.falsy(ok)
+    t.matches('checksum', err)
+    t.falsy(read(dest .. '/root/a.lua'), 'nothing should be written')
+  end)
+
+  t.it('fails loudly when a deflated entry disagrees with the stated size', function()
+    local bytes = zipf({ { name = 'root/a.lua', data = payload, deflate = true, usize = #payload + 1 } })
+    local ok, err, dest = extract(bytes)
+    t.falsy(ok)
+    t.matches('does not match the directory', err)
+    t.falsy(read(dest .. '/root/a.lua'), 'nothing should be written')
+  end)
+
+  t.it('fails loudly on a truncated deflate stream', function()
+    local ok, err, dest = extract(zipf({
+      {
+        name = 'root/a.lua',
+        data = payload,
+        deflate = true,
+        mangle = function(s)
+          return s:sub(1, math.floor(#s / 2))
+        end,
+      },
+    }))
+    t.falsy(ok)
+    t.matches('root/a%.lua', err)
+    t.falsy(read(dest .. '/root/a.lua'), 'a partial inflate must not reach disk')
+  end)
+
+  t.it('fails loudly on a deflate stream corrupted in the middle', function()
+    local ok, err, dest = extract(zipf({
+      {
+        name = 'root/a.lua',
+        data = payload,
+        deflate = true,
+        mangle = function(s)
+          return flip(s, math.floor(#s / 2), 4)
+        end,
+      },
+    }))
+    t.falsy(ok)
+    t.matches('root/a%.lua', err)
+    t.falsy(read(dest .. '/root/a.lua'), 'a corrupt inflate must not reach disk')
+  end)
+
+  t.it('refuses a deflated entry that walks out of the destination', function()
+    local ok, err, dest = extract(zipf({ { name = '../evil.lua', data = payload, deflate = true } }))
+    t.falsy(ok)
+    t.matches('escapes', err)
+    t.falsy(read(dest .. '/../evil.lua'), 'nothing should be written')
+  end)
+
+  t.it('refuses a deflated symlink entry', function()
+    local ok, err = extract(zipf({
+      { name = 'root/link', data = string.rep('/etc/passwd', 40), deflate = true, attr = (0xa000 + 0x1ff) * 65536 },
+    }))
+    t.falsy(ok)
+    t.matches('not a regular file', err)
+  end)
 end)
 
 t.describe('archive.extract tar.gz', function()
@@ -331,6 +553,13 @@ t.describe('archive.extract tar.gz', function()
     t.matches('gunzip failed', err)
   end)
 
+  t.it('reports a tar directory that collides with an earlier file', function()
+    local bytes = gz(tarf({ { name = 'root/a', data = 'x' }, { name = 'root/a/', kind = '5' } }))
+    local ok, err = extract(bytes, 'tar.gz')
+    t.falsy(ok)
+    t.matches('Cannot create directory', err)
+  end)
+
   t.it('fails on a corrupt tar header', function()
     local tar = tarf({ { name = 'root/a.lua', data = 'hello' } })
     local ok, err = extract(gz('X' .. tar:sub(2)), 'tar.gz')
@@ -338,3 +567,86 @@ t.describe('archive.extract tar.gz', function()
     t.matches('checksum mismatch', err)
   end)
 end)
+
+-- The stored-block fixtures above never reach the huffman decoder, so the
+-- compressed cases are covered separately.
+t.describe('archive.extract tar.gz deflate', function()
+  local payload = sample()
+
+  t.it('inflates a dynamic-huffman member', function()
+    local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic')
+    local ok, err, dest = extract(bytes, 'tar.gz')
+    t.truthy(ok, err)
+    t.eq(payload, read(dest .. '/root/a.lua'))
+  end)
+
+  t.it('inflates a fixed-huffman member', function()
+    local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'fixed')
+    local ok, err, dest = extract(bytes, 'tar.gz')
+    t.truthy(ok, err)
+    t.eq(payload, read(dest .. '/root/a.lua'))
+  end)
+
+  t.it('inflates an empty member', function()
+    local bytes = gz(tarf({ { name = 'root/empty.lua', data = '' } }), 'dynamic')
+    local ok, err, dest = extract(bytes, 'tar.gz')
+    t.truthy(ok, err)
+    t.eq('', read(dest .. '/root/empty.lua'))
+  end)
+
+  t.it('inflates a large member', function()
+    local big = string.rep(noise(1024, 29), 300)
+    local bytes = gz(tarf({ { name = 'root/big.bin', data = big } }), 'dynamic')
+    t.truthy(#bytes < #big / 4, 'the fixture is not actually compressed')
+    local ok, err, dest = extract(bytes, 'tar.gz')
+    t.truthy(ok, err)
+    local out = read(dest .. '/root/big.bin')
+    t.eq(#big, #out)
+    t.truthy(out == big, 'the large member did not round-trip exactly')
+  end)
+
+  t.it('fails loudly when the gzip checksum disagrees', function()
+    local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic')
+    local ok, err, dest = extract(patch32(bytes, #bytes - 7, 12345), 'tar.gz')
+    t.falsy(ok)
+    t.matches('gunzip failed', err)
+    t.falsy(read(dest .. '/root/a.lua'), 'nothing should be written')
+  end)
+
+  t.it('fails loudly on a truncated compressed member', function()
+    local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic')
+    local ok, err, dest = extract(bytes:sub(1, 10 + math.floor((#bytes - 18) / 2)), 'tar.gz')
+    t.falsy(ok)
+    t.matches('gunzip failed', err)
+    t.falsy(read(dest .. '/root/a.lua'), 'a partial inflate must not reach disk')
+  end)
+
+  t.it('fails loudly on a member corrupted in the middle', function()
+    local bytes = gz(tarf({ { name = 'root/a.lua', data = payload } }), 'dynamic')
+    local ok, err, dest = extract(flip(bytes, math.floor(#bytes / 2), 4), 'tar.gz')
+    t.falsy(ok)
+    t.matches('gunzip failed', err)
+    t.falsy(read(dest .. '/root/a.lua'), 'a corrupt inflate must not reach disk')
+  end)
+
+  t.it('refuses a hostile path inside a compressed member', function()
+    local bytes = gz(tarf({ { name = '../evil.lua', data = payload } }), 'dynamic')
+    local ok, err, dest = extract(bytes, 'tar.gz')
+    t.falsy(ok)
+    t.matches('escapes', err)
+    t.falsy(read(dest .. '/../evil.lua'), 'nothing should be written')
+  end)
+
+  t.it('stops a pax record that claims a zero length', function()
+    local bytes = gz(
+      tarf({
+        { name = 'root/pax', kind = 'x', data = '0 path=root/a.lua\n' },
+        { name = 'root/a.lua', data = 'ok' },
+      }),
+      'dynamic'
+    )
+    local ok, err, dest = extract(bytes, 'tar.gz')
+    t.truthy(ok, err)
+    t.eq('ok', read(dest .. '/root/a.lua'))
+  end)
+end)

tests/deflate.lua

@@ -0,0 +1,315 @@
+-- A minimal DEFLATE encoder, so compressed fixtures are built here rather than
+-- shelled out to a compressor that may be absent and silently skip the tests.
+local M = {}
+
+local MIN_MATCH, MAX_MATCH, WINDOW = 3, 258, 32768
+local END_OF_BLOCK = 256
+local MAX_LIT_SYMBOLS, MAX_DIST_SYMBOLS, MAX_CL_SYMBOLS = 286, 30, 19
+
+local function nums(spec)
+  local out = {}
+  for n in spec:gmatch('%d+') do
+    table.insert(out, tonumber(n))
+  end
+  return out
+end
+
+-- RFC 1951 section 3.2.5: length codes 257..285 and distance codes 0..29.
+local LENGTH_BASE = nums('3 4 5 6 7 8 9 10 11 13 15 17 19 23 27 31 35 43 51 59 67 83 99 115 131 163 195 227 258')
+local LENGTH_EXTRA = nums('0 0 0 0 0 0 0 0 1 1 1 1 2 2 2 2 3 3 3 3 4 4 4 4 5 5 5 5 0')
+local DIST_BASE = nums(
+  '1 2 3 4 5 7 9 13 17 25 33 49 65 97 129 193 257 385 513 769 1025 1537 2049 3073 4097 6145 8193 12289 16385 24577'
+)
+local DIST_EXTRA = nums('0 0 0 0 1 1 2 2 3 3 4 4 5 5 6 6 7 7 8 8 9 9 10 10 11 11 12 12 13 13')
+local CL_ORDER = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }
+
+local Writer = {}
+Writer.__index = Writer
+
+local function writer()
+  return setmetatable({ out = {}, acc = 0, n = 0 }, Writer)
+end
+
+-- Plain values go out least significant bit first.
+function Writer:bits(value, count)
+  for _ = 1, count do
+    self.acc = self.acc + (value % 2) * 2 ^ self.n
+    value = math.floor(value / 2)
+    self.n = self.n + 1
+    if self.n == 8 then
+      table.insert(self.out, string.char(self.acc))
+      self.acc, self.n = 0, 0
+    end
+  end
+end
+
+-- Huffman codes go out most significant bit first.
+function Writer:code(code, len)
+  for i = len - 1, 0, -1 do
+    self:bits(math.floor(code / 2 ^ i) % 2, 1)
+  end
+end
+
+function Writer:finish()
+  if self.n > 0 then
+    table.insert(self.out, string.char(self.acc))
+    self.acc, self.n = 0, 0
+  end
+  return table.concat(self.out)
+end
+
+local function length_code(len)
+  for i = #LENGTH_BASE, 1, -1 do
+    if len >= LENGTH_BASE[i] then
+      return 256 + i, len - LENGTH_BASE[i], LENGTH_EXTRA[i]
+    end
+  end
+  error('length out of range: ' .. len)
+end
+
+local function dist_code(dist)
+  for i = #DIST_BASE, 1, -1 do
+    if dist >= DIST_BASE[i] then
+      return i - 1, dist - DIST_BASE[i], DIST_EXTRA[i]
+    end
+  end
+  error('distance out of range: ' .. dist)
+end
+
+-- Greedy LZ77 over a single-slot hash of the next three bytes. Good enough to
+-- put real matches, extra bits and overlapping copies into the stream.
+local function tokenize(data)
+  local tokens, head, pos, n = {}, {}, 1, #data
+  while pos <= n do
+    local best, dist = 0, 0
+    if pos + MIN_MATCH - 1 <= n then
+      local key = data:sub(pos, pos + MIN_MATCH - 1)
+      local prev = head[key]
+      if prev and pos - prev <= WINDOW then
+        local len = 0
+        while len < MAX_MATCH and pos + len <= n and data:byte(prev + len) == data:byte(pos + len) do
+          len = len + 1
+        end
+        if len >= MIN_MATCH then
+          best, dist = len, pos - prev
+        end
+      end
+      head[key] = pos
+    end
+    if best > 0 then
+      table.insert(tokens, { len = best, dist = dist })
+      for i = pos + 1, math.min(pos + best - 1, n - MIN_MATCH + 1) do
+        head[data:sub(i, i + MIN_MATCH - 1)] = i
+      end
+      pos = pos + best
+    else
+      table.insert(tokens, data:byte(pos))
+      pos = pos + 1
+    end
+  end
+  return tokens
+end
+
+local function fixed_lit(sym)
+  if sym <= 143 then
+    return 0x30 + sym, 8
+  elseif sym <= 255 then
+    return 0x190 + sym - 144, 9
+  elseif sym <= 279 then
+    return sym - 256, 7
+  end
+  return 0xc0 + sym - 280, 8
+end
+
+local function emit(w, tokens, litcode, distcode)
+  for _, tk in ipairs(tokens) do
+    if type(tk) == 'number' then
+      w:code(litcode(tk))
+    else
+      local lc, lextra, lbits = length_code(tk.len)
+      w:code(litcode(lc))
+      if lbits > 0 then
+        w:bits(lextra, lbits)
+      end
+      local dc, dextra, dbits = dist_code(tk.dist)
+      w:code(distcode(dc))
+      if dbits > 0 then
+        w:bits(dextra, dbits)
+      end
+    end
+  end
+  w:code(litcode(END_OF_BLOCK))
+end
+
+-- A one-leaf alphabet cannot produce a complete prefix code, so pad it.
+local function ensure_two(freq, limit)
+  local used = 0
+  for _ in pairs(freq) do
+    used = used + 1
+  end
+  for sym = 0, limit - 1 do
+    if used >= 2 then
+      return
+    end
+    if not freq[sym] then
+      freq[sym] = 1
+      used = used + 1
+    end
+  end
+end
+
+local function build_lengths(weights)
+  local nodes, lens = {}, {}
+  for sym, weight in pairs(weights) do
+    table.insert(nodes, { w = weight, syms = { sym } })
+    lens[sym] = 0
+  end
+  assert(#nodes >= 2, 'huffman needs at least two symbols')
+  while #nodes > 1 do
+    table.sort(nodes, function(a, b)
+      if a.w ~= b.w then
+        return a.w < b.w
+      end
+      return a.syms[1] < b.syms[1]
+    end)
+    local a, b = table.remove(nodes, 1), table.remove(nodes, 1)
+    local syms = {}
+    for _, list in ipairs({ a.syms, b.syms }) do
+      for _, sym in ipairs(list) do
+        lens[sym] = lens[sym] + 1
+        table.insert(syms, sym)
+      end
+    end
+    table.insert(nodes, { w = a.w + b.w, syms = syms })
+  end
+  return lens
+end
+
+-- Flattening the weights shortens the deepest codes; a flat alphabet of at most
+-- 286 symbols never needs more than nine bits, so this always settles.
+local function huffman(freq, limit)
+  local weights = {}
+  for sym, weight in pairs(freq) do
+    weights[sym] = weight
+  end
+  while true do
+    local lens, worst = build_lengths(weights), 0
+    for _, len in pairs(lens) do
+      worst = math.max(worst, len)
+    end
+    if worst <= limit then
+      return lens
+    end
+    for sym, weight in pairs(weights) do
+      weights[sym] = math.ceil(weight / 2)
+    end
+  end
+end
+
+local function canonical(lens)
+  local counts, maxlen, syms = {}, 0, {}
+  for sym, len in pairs(lens) do
+    counts[len] = (counts[len] or 0) + 1
+    maxlen = math.max(maxlen, len)
+    table.insert(syms, sym)
+  end
+  local next_code, code = {}, 0
+  for bits = 1, maxlen do
+    code = (code + (counts[bits - 1] or 0)) * 2
+    next_code[bits] = code
+  end
+  table.sort(syms)
+  local codes = {}
+  for _, sym in ipairs(syms) do
+    codes[sym] = next_code[lens[sym]]
+    next_code[lens[sym]] = next_code[lens[sym]] + 1
+  end
+  return codes
+end
+
+local function highest(lens)
+  local top = 0
+  for sym in pairs(lens) do
+    top = math.max(top, sym)
+  end
+  return top
+end
+
+local function fixed_block(w, tokens, final)
+  w:bits(final, 1)
+  w:bits(1, 2)
+  emit(w, tokens, fixed_lit, function(sym)
+    return sym, 5
+  end)
+end
+
+local function dynamic_block(w, tokens, final)
+  local litfreq, distfreq = { [END_OF_BLOCK] = 1 }, {}
+  for _, tk in ipairs(tokens) do
+    if type(tk) == 'number' then
+      litfreq[tk] = (litfreq[tk] or 0) + 1
+    else
+      local lc = length_code(tk.len)
+      litfreq[lc] = (litfreq[lc] or 0) + 1
+      local dc = dist_code(tk.dist)
+      distfreq[dc] = (distfreq[dc] or 0) + 1
+    end
+  end
+  ensure_two(litfreq, MAX_LIT_SYMBOLS)
+  ensure_two(distfreq, MAX_DIST_SYMBOLS)
+
+  local litlens, distlens = huffman(litfreq, 15), huffman(distfreq, 15)
+  local hlit = math.max(257, highest(litlens) + 1)
+  local hdist = highest(distlens) + 1
+
+  local seq = {}
+  for sym = 0, hlit - 1 do
+    table.insert(seq, litlens[sym] or 0)
+  end
+  for sym = 0, hdist - 1 do
+    table.insert(seq, distlens[sym] or 0)
+  end
+
+  -- Code lengths are written one per symbol; the run-length codes 16..18 are
+  -- left unused so the encoder stays small.
+  local clfreq = {}
+  for _, len in ipairs(seq) do
+    clfreq[len] = (clfreq[len] or 0) + 1
+  end
+  ensure_two(clfreq, 16)
+  local cllens = huffman(clfreq, 7)
+  local clcodes = canonical(cllens)
+
+  w:bits(final, 1)
+  w:bits(2, 2)
+  w:bits(hlit - 257, 5)
+  w:bits(hdist - 1, 5)
+  w:bits(MAX_CL_SYMBOLS - 4, 4)
+  for _, sym in ipairs(CL_ORDER) do
+    w:bits(cllens[sym] or 0, 3)
+  end
+  for _, len in ipairs(seq) do
+    w:code(clcodes[len], cllens[len])
+  end
+
+  local litcodes, distcodes = canonical(litlens), canonical(distlens)
+  emit(w, tokens, function(sym)
+    return litcodes[sym], litlens[sym]
+  end, function(sym)
+    return distcodes[sym], distlens[sym]
+  end)
+end
+
+-- Returns a raw DEFLATE stream. mode is 'dynamic' (default) or 'fixed'.
+function M.compress(data, mode)
+  local w = writer()
+  local tokens = tokenize(data)
+  if mode == 'fixed' then
+    fixed_block(w, tokens, 1)
+  else
+    dynamic_block(w, tokens, 1)
+  end
+  return w:finish()
+end
+
+return M

tests/harness.lua

@@ -1,14 +1,39 @@
 local M = { passed = 0, failed = 0, failures = {} }
 
+local uv = vim.uv or vim.loop
 local current = '?'
 
+M.timeout_ms = tonumber(vim.env.ARTI_PACK_TEST_TIMEOUT_MS) or 5000
+
+-- LuaJIT never reaches a debug hook from inside a compiled trace, so the
+-- interpreter has to stay in charge for the budget below to be enforceable.
+if _G.jit then
+  _G.jit.off()
+  _G.jit.flush()
+end
+
+local HOOK_INTERVAL = 20000
+
+local function bounded(fn)
+  local deadline = uv.hrtime() + M.timeout_ms * 1e6
+  debug.sethook(function()
+    if uv.hrtime() > deadline then
+      debug.sethook()
+      error(string.format('timed out after %dms', M.timeout_ms), 0)
+    end
+  end, '', HOOK_INTERVAL)
+  local ok, err = pcall(fn)
+  debug.sethook()
+  return ok, err
+end
+
 function M.describe(name, fn)
   current = name
   fn()
 end
 
 function M.it(name, fn)
-  local ok, err = pcall(fn)
+  local ok, err = bounded(fn)
   if ok then
     M.passed = M.passed + 1
   else