if not modules then modules = { } end modules ['util-zip'] = {
version = 1.001,
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files"
}
-- This module is mostly meant for relative simple zip and unzip tasks. We can read
-- and write zip files but with limitations. Performance is quite good and it makes
-- us independent of zip tools, which (for some reason) are not always installed.
--
-- This is an lmtx module and at some point will be lmtx only but for a while we
-- keep some hybrid functionality.
local type, tostring, tonumber = type, tostring, tonumber
local sort, concat = table.sort, table.concat
local find, format, sub, gsub = string.find, string.format, string.sub, string.gsub
local osdate, ostime, osclock = os.date, os.time, os.clock
local ioopen = io.open
local loaddata, savedata = io.loaddata, io.savedata
local filejoin, isdir, dirname, mkdirs = file.join, lfs.isdir, file.dirname, dir.mkdirs
local suffix, suffixes = file.suffix, file.suffixes
local openfile = io.open
gzip = gzip or { } -- so in luatex we keep the old ones too
if not zlib then
zlib = xzip -- in luametatex we shadow the old one
elseif not xzip then
xzip = zlib
end
local files = utilities.files
local openfile = files.open
local closefile = files.close
local getsize = files.size
local readstring = files.readstring
local readcardinal2 = files.readcardinal2le
local readcardinal4 = files.readcardinal4le
local setposition = files.setposition
local getposition = files.getposition
local skipbytes = files.skip
local band = bit32.band
local rshift = bit32.rshift
local lshift = bit32.lshift
local zlibdecompress = zlib.decompress
local zlibdecompresssize = zlib.decompresssize
local zlibchecksum = zlib.crc32
if not CONTEXTLMTXMODE or CONTEXTLMTXMODE == 0 then
local cs = zlibchecksum
zlibchecksum = function(str,n) return cs(n or 0, str) end
end
local decompress = function(source) return zlibdecompress (source,-15) end -- auto
local decompresssize = function(source,targetsize) return zlibdecompresssize(source,targetsize,-15) end -- auto
local calculatecrc = function(buffer,initial) return zlibchecksum (initial or 0,buffer) end
local zipfiles = { }
utilities.zipfiles = zipfiles
local openzipfile, closezipfile, unzipfile, foundzipfile, getziphash, getziplist do
function openzipfile(name)
return {
name = name,
handle = openfile(name,0),
}
end
-- https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT
-- local function collect(z)
-- if not z.list then
-- local list = { }
-- local hash = { }
-- local position = 0
-- local index = 0
-- local handle = z.handle
-- while true do
-- setposition(handle,position)
-- local signature = readstring(handle,4)
-- if signature == "PK\3\4" then
-- -- [local file header 1]
-- -- [encryption header 1]
-- -- [file data 1]
-- -- [data descriptor 1]
-- local version = readcardinal2(handle)
-- local flag = readcardinal2(handle)
-- local method = readcardinal2(handle)
-- local filetime = readcardinal2(handle)
-- local filedate = readcardinal2(handle)
-- local crc32 = readcardinal4(handle)
-- local compressed = readcardinal4(handle)
-- local uncompressed = readcardinal4(handle)
-- local namelength = readcardinal2(handle)
-- local extralength = readcardinal2(handle)
-- local filename = readstring(handle,namelength)
-- local descriptor = band(flag,8) ~= 0
-- local encrypted = band(flag,1) ~= 0
-- local acceptable = method == 0 or method == 8
-- -- 30 bytes of header including the signature
-- local skipped = 0
-- local size = 0
-- if encrypted then
-- size = readcardinal2(handle)
-- skipbytes(handle,size)
-- skipped = skipped + size + 2
-- skipbytes(8)
-- skipped = skipped + 8
-- size = readcardinal2(handle)
-- skipbytes(handle,size)
-- skipped = skipped + size + 2
-- size = readcardinal4(handle)
-- skipbytes(handle,size)
-- skipped = skipped + size + 4
-- size = readcardinal2(handle)
-- skipbytes(handle,size)
-- skipped = skipped + size + 2
-- end
-- position = position + 30 + namelength + extralength + skipped
-- -- if descriptor then
-- -- -- where is this one located
-- -- setposition(handle,position + compressed)
-- -- crc32 = readcardinal4(handle)
-- -- compressed = readcardinal4(handle)
-- -- uncompressed = readcardinal4(handle)
-- -- end
-- if acceptable then
-- index = index + 1
-- local data = {
-- filename = filename,
-- index = index,
-- position = position,
-- method = method,
-- compressed = compressed,
-- uncompressed = uncompressed,
-- crc32 = crc32,
-- encrypted = encrypted,
-- }
-- hash[filename] = data
-- list[index] = data
-- else
-- -- maybe a warning when encrypted
-- end
-- position = position + compressed
-- else
-- break
-- end
-- z.list = list
-- z.hash = hash
-- end
-- end
-- end
-- end
-- end
-- end
local function update(handle,data)
position = data.offset
setposition(handle,position)
local signature = readstring(handle,4)
if signature == "PK\3\4" then -- 0x04034B50
-- [local file header 1]
-- [encryption header 1]
-- [file data 1]
-- [data descriptor 1]
local version = readcardinal2(handle)
local flag = readcardinal2(handle)
local method = readcardinal2(handle)
skipbytes(handle,4)
----- filetime = readcardinal2(handle)
----- filedate = readcardinal2(handle)
local crc32 = readcardinal4(handle)
local compressed = readcardinal4(handle)
local uncompressed = readcardinal4(handle)
local namelength = readcardinal2(handle)
local extralength = readcardinal2(handle)
local filename = readstring(handle,namelength)
local descriptor = band(flag,8) ~= 0
local encrypted = band(flag,1) ~= 0
local acceptable = method == 0 or method == 8
-- 30 bytes of header including the signature
local skipped = 0
local size = 0
if encrypted then
size = readcardinal2(handle)
skipbytes(handle,size)
skipped = skipped + size + 2
skipbytes(8)
skipped = skipped + 8
size = readcardinal2(handle)
skipbytes(handle,size)
skipped = skipped + size + 2
size = readcardinal4(handle)
skipbytes(handle,size)
skipped = skipped + size + 4
size = readcardinal2(handle)
skipbytes(handle,size)
skipped = skipped + size + 2
end
if acceptable then
if filename ~= data.filename then
-- elseif method ~= data.method then
-- elseif encrypted ~= data.encrypted then
-- elseif crc32 ~= 0 and crc32 ~= data.crc32 then
-- elseif uncompressed ~= 0 and uncompressed ~= data.uncompressed then
-- elseif compressed ~= 0 and compressed ~= data.compressed then
else
position = position + 30 + namelength + extralength + skipped
data.position = position
return position
end
else
-- maybe a warning when encrypted
end
end
data.position = false
return false
end
local function collect(z)
if not z.list then
local list = { }
local hash = { }
local position = 0
local index = 0
local handle = z.handle
local size = getsize(handle)
--
-- Not all files have the compressed into set so we need to get the directory
-- first. We only handle single disk zip files.
--
for i=size-4,size-64*1024,-1 do
setposition(handle,i)
local enddirsignature = readcardinal4(handle)
if enddirsignature == 0x06054B50 then
local thisdisknumber = readcardinal2(handle)
local centraldisknumber = readcardinal2(handle)
local thisnofentries = readcardinal2(handle)
local totalnofentries = readcardinal2(handle)
local centralsize = readcardinal4(handle)
local centraloffset = readcardinal4(handle)
local commentlength = readcardinal2(handle)
local comment = readstring(handle,length)
if size - i >= 22 then
if thisdisknumber == centraldisknumber then
setposition(handle,centraloffset)
while true do
if readcardinal4(handle) == 0x02014B50 then
skipbytes(handle,4)
----- versionmadeby = readcardinal2(handle)
----- versionneeded = readcardinal2(handle)
local flag = readcardinal2(handle)
local method = readcardinal2(handle)
skipbytes(handle,4)
----- filetime = readcardinal2(handle)
----- filedate = readcardinal2(handle)
local crc32 = readcardinal4(handle)
local compressed = readcardinal4(handle)
local uncompressed = readcardinal4(handle)
local namelength = readcardinal2(handle)
local extralength = readcardinal2(handle)
local commentlength = readcardinal2(handle)
skipbytes(handle,8)
----- disknumber = readcardinal2(handle)
----- intattributes = readcardinal2(handle)
----- extattributes = readcardinal4(handle)
local headeroffset = readcardinal4(handle)
local filename = readstring(handle,namelength)
skipbytes(handle,extralength+commentlength)
----- extradata = readstring(handle,extralength)
----- comment = readstring(handle,commentlength)
--
local descriptor = band(flag,8) ~= 0
local encrypted = band(flag,1) ~= 0
local acceptable = method == 0 or method == 8
if acceptable then
index = index + 1
local data = {
filename = filename,
index = index,
position = nil,
method = method,
compressed = compressed,
uncompressed = uncompressed,
crc32 = crc32,
encrypted = encrypted,
offset = headeroffset,
}
hash[filename] = data
list[index] = data
end
else
break
end
end
end
break
end
end
end
-- for i=1,index do -- delayed
-- local data = list[i]
-- if not data.position then
-- update(handle,list[i])
-- end
-- end
z.list = list
z.hash = hash
end
end
function getziplist(z)
local list = z.list
if not list then
collect(z)
end
-- inspect(z.list)
return z.list
end
function getziphash(z)
local hash = z.hash
if not hash then
collect(z)
end
return z.hash
end
function foundzipfile(z,name)
return getziphash(z)[name]
end
function closezipfile(z)
local f = z.handle
if f then
closefile(f)
z.handle = nil
end
end
function unzipfile(z,filename,check)
local hash = z.hash
if not hash then
hash = zipfiles.hash(z)
end
local data = hash[filename] -- normalize
if not data then
-- lower and cleanup
-- only name
end
if data then
local handle = z.handle
local position = data.position
local compressed = data.compressed
if position == nil then
position = update(handle,data)
end
if position and compressed > 0 then
setposition(handle,position)
local result = readstring(handle,compressed)
if data.method == 8 then
if decompresssize then
result = decompresssize(result,data.uncompressed)
else
result = decompress(result)
end
end
if check and data.crc32 ~= calculatecrc(result) then
print("checksum mismatch")
return ""
end
return result
else
return ""
end
end
end
zipfiles.open = openzipfile
zipfiles.close = closezipfile
zipfiles.unzip = unzipfile
zipfiles.hash = getziphash
zipfiles.list = getziplist
zipfiles.found = foundzipfile
end
if xzip then -- flate then do
local writecardinal1 = files.writebyte
local writecardinal2 = files.writecardinal2le
local writecardinal4 = files.writecardinal4le
local logwriter = logs.writer
local globpattern = dir.globpattern
-- local compress = flate.flate_compress
-- local checksum = flate.update_crc32
local compress = xzip.compress
local checksum = xzip.crc32
-- local function fromdostime(dostime,dosdate)
-- return ostime {
-- year = (dosdate >> 9) + 1980, -- 25 .. 31
-- month = (dosdate >> 5) & 0x0F, -- 21 .. 24
-- day = (dosdate ) & 0x1F, -- 16 .. 20
-- hour = (dostime >> 11) , -- 11 .. 15
-- min = (dostime >> 5) & 0x3F, -- 5 .. 10
-- sec = (dostime ) & 0x1F, -- 0 .. 4
-- }
-- end
--
-- local function todostime(time)
-- local t = osdate("*t",time)
-- return
-- ((t.year - 1980) << 9) + (t.month << 5) + t.day,
-- (t.hour << 11) + (t.min << 5) + (t.sec >> 1)
-- end
local function fromdostime(dostime,dosdate)
return ostime {
year = rshift(dosdate, 9) + 1980, -- 25 .. 31
month = band(rshift(dosdate, 5), 0x0F), -- 21 .. 24
day = band( (dosdate ), 0x1F), -- 16 .. 20
hour = band(rshift(dostime,11) ), -- 11 .. 15
min = band(rshift(dostime, 5), 0x3F), -- 5 .. 10
sec = band( (dostime ), 0x1F), -- 0 .. 4
}
end
local function todostime(time)
local t = osdate("*t",time)
return
lshift(t.year - 1980, 9) + lshift(t.month,5) + t.day,
lshift(t.hour ,11) + lshift(t.min ,5) + rshift(t.sec,1)
end
local function openzip(filename,level,comment,verbose)
local f = ioopen(filename,"wb")
if f then
return {
filename = filename,
handle = f,
list = { },
level = tonumber(level) or 3,
comment = tostring(comment),
verbose = verbose,
uncompressed = 0,
compressed = 0,
}
end
end
local function writezip(z,name,data,level,time)
local f = z.handle
local list = z.list
local level = tonumber(level) or z.level or 3
local method = 8
local zipped = compress(data,level)
local checksum = checksum(data)
local verbose = z.verbose
--
if not zipped then
method = 0
zipped = data
end
--
local start = f:seek()
local compressed = #zipped
local uncompressed = #data
--
z.compressed = z.compressed + compressed
z.uncompressed = z.uncompressed + uncompressed
--
if verbose then
local pct = 100 * compressed/uncompressed
if pct >= 100 then
logwriter(format("%10i %s",uncompressed,name))
else
logwriter(format("%10i %02.1f %s",uncompressed,pct,name))
end
end
--
f:write("\x50\x4b\x03\x04") -- PK.. 0x04034b50
--
writecardinal2(f,0) -- minimum version
writecardinal2(f,0) -- flag
writecardinal2(f,method) -- method
writecardinal2(f,0) -- time
writecardinal2(f,0) -- date
writecardinal4(f,checksum) -- crc32
writecardinal4(f,compressed) -- compressed
writecardinal4(f,uncompressed) -- uncompressed
writecardinal2(f,#name) -- namelength
writecardinal2(f,0) -- extralength
--
f:write(name) -- name
f:write(zipped)
--
list[#list+1] = { #zipped, #data, name, checksum, start, time or 0 }
end
local function closezip(z)
local f = z.handle
local list = z.list
local comment = z.comment
local verbose = z.verbose
local count = #list
local start = f:seek()
--
for i=1,count do
local l = list[i]
local compressed = l[1]
local uncompressed = l[2]
local name = l[3]
local checksum = l[4]
local start = l[5]
local time = l[6]
local date, time = todostime(time)
f:write('\x50\x4b\x01\x02')
writecardinal2(f,0) -- version made by
writecardinal2(f,0) -- version needed to extract
writecardinal2(f,0) -- flags
writecardinal2(f,8) -- method
writecardinal2(f,time) -- time
writecardinal2(f,date) -- date
writecardinal4(f,checksum) -- crc32
writecardinal4(f,compressed) -- compressed
writecardinal4(f,uncompressed) -- uncompressed
writecardinal2(f,#name) -- namelength
writecardinal2(f,0) -- extralength
writecardinal2(f,0) -- commentlength
writecardinal2(f,0) -- nofdisks -- ?
writecardinal2(f,0) -- internal attr (type)
writecardinal4(f,0) -- external attr (mode)
writecardinal4(f,start) -- local offset
f:write(name) -- name
end
--
local stop = f:seek()
local size = stop - start
--
f:write('\x50\x4b\x05\x06')
writecardinal2(f,0) -- disk
writecardinal2(f,0) -- disks
writecardinal2(f,count) -- entries
writecardinal2(f,count) -- entries
writecardinal4(f,size) -- dir size
writecardinal4(f,start) -- dir offset
if type(comment) == "string" and comment ~= "" then
writecardinal2(f,#comment) -- comment length
f:write(comment) -- comemnt
else
writecardinal2(f,0)
end
--
if verbose then
local compressed = z.compressed
local uncompressed = z.uncompressed
local filename = z.filename
--
local pct = 100 * compressed/uncompressed
logwriter("")
if pct >= 100 then
logwriter(format("%10i %s",uncompressed,filename))
else
logwriter(format("%10i %02.1f %s",uncompressed,pct,filename))
end
end
--
f:close()
end
local function zipdir(zipname,path,level,verbose)
if type(zipname) == "table" then
verbose = zipname.verbose
level = zipname.level
path = zipname.path
zipname = zipname.zipname
end
if not zipname or zipname == "" then
return
end
if not path or path == "" then
path = "."
end
if not isdir(path) then
return
end
path = gsub(path,"\\+","/")
path = gsub(path,"/+","/")
local list = { }
local count = 0
globpattern(path,"",true,function(name,size,time)
count = count + 1
list[count] = { name, time }
end)
sort(list,function(a,b)
return a[1] < b[1]
end)
local zipf = openzip(zipname,level,comment,verbose)
if zipf then
local p = #path + 2
for i=1,count do
local li = list[i]
local name = li[1]
local time = li[2]
local data = loaddata(name)
local name = sub(name,p,#name)
writezip(zipf,name,data,level,time,verbose)
end
closezip(zipf)
end
end
local function unzipdir(zipname,path,verbose,collect,validate)
if type(zipname) == "table" then
validate = zipname.validate
collect = zipname.collect
verbose = zipname.verbose
path = zipname.path
zipname = zipname.zipname
end
if not zipname or zipname == "" then
return
end
if not path or path == "" then
path = "."
end
local z = openzipfile(zipname)
if z then
local list = getziplist(z)
if list then
local total = 0
local count = #list
local step = number.idiv(count,10)
local done = 0
local steps = verbose == "steps"
local time = steps and osclock()
-- local skip = 0
if collect then
collect = { }
else
collect = false
end
for i=1,count do
local l = list[i]
local n = l.filename
if not validate or validate(n) then
local d = unzipfile(z,n) -- true for check
if d then
local p = filejoin(path,n)
if mkdirs(dirname(p)) then
if steps then
total = total + #d
done = done + 1
if done >= step then
done = 0
logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",i,count,total,osclock()-time))
end
elseif verbose then
logwriter(n)
end
savedata(p,d)
if collect then
collect[#collect+1] = p
end
end
else
logwriter(format("problem with file %s",n))
end
else
-- skip = skip + 1
end
end
if steps then
logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",count,count,total,osclock()-time))
end
closezipfile(z)
if collect then
return collect
end
else
closezipfile(z)
end
end
end
zipfiles.zipdir = zipdir
zipfiles.unzipdir = unzipdir
end
-- todo: compress/decompress that work with offset in string
-- We only have a few official methods here:
--
-- local decompressed = gzip.load (filename)
-- local resultsize = gzip.save (filename,compresslevel)
-- local compressed = gzip.compress (str,compresslevel)
-- local decompressed = gzip.decompress (str)
-- local iscompressed = gzip.compressed (str)
-- local suffix, okay = gzip.suffix (filename)
--
-- In LuaMetaTeX we have only xzip which implements a very few methods:
--
-- compress (str,level,method,window,memory,strategy)
-- decompress (str,window)
-- adler32 (str,checksum)
-- crc32 (str,checksum)
local pattern = "^\x1F\x8B\x08"
local gziplevel = 3
function gzip.suffix(filename)
local suffix, extra = suffixes(filename)
local gzipped = extra == "gz"
return suffix, gzipped
end
function gzip.compressed(s)
return s and find(s,pattern)
end
local getdecompressed
local putcompressed
if gzip.compress then
local gzipwindow = 15 + 16 -- +16: gzip, +32: gzip|zlib
local compress = zlib.compress
local decompress = zlib.decompress
getdecompressed = function(str)
return decompress(str,gzipwindow) -- pass offset
end
putcompressed = function(str,level)
return compress(str,level or gziplevel,nil,gzipwindow)
end
else
-- Special window values are: flate: -15, zlib: 15, gzip : -15
local gzipwindow = -15 -- miniz needs this
local identifier = "\x1F\x8B"
local compress = zlib.compress
local decompress = zlib.decompress
local zlibchecksum = zlib.crc32
if not CONTEXTLMTXMODE or CONTEXTLMTXMODE == 0 then
local cs = zlibchecksum
zlibchecksum = function(str,n) return cs(n or 0, str) end
end
local streams = utilities.streams
local openstream = streams.openstring
local closestream = streams.close
local getposition = streams.getposition
local readbyte = streams.readbyte
local readcardinal4 = streams.readcardinal4le
local readcardinal2 = streams.readcardinal2le
local readstring = streams.readstring
local readcstring = streams.readcstring
local skipbytes = streams.skip
local tocardinal1 = streams.tocardinal1
local tocardinal4 = streams.tocardinal4le
getdecompressed = function(str)
local s = openstream(str)
local identifier = readstring(s,2)
local method = readbyte(s,1)
local flags = readbyte(s,1)
local timestamp = readcardinal4(s)
local compression = readbyte(s,1)
local operating = readbyte(s,1)
-- local isjusttext = (flags & 0x01 ~= 0) and true or false
-- local extrasize = (flags & 0x04 ~= 0) and readcardinal2(s) or 0
-- local filename = (flags & 0x08 ~= 0) and readcstring(s) or ""
-- local comment = (flags & 0x10 ~= 0) and readcstring(s) or ""
-- local checksum = (flags & 0x02 ~= 0) and readcardinal2(s) or 0
local isjusttext = band(flags,0x01) ~= 0 and true or false
local extrasize = band(flags,0x04) ~= 0 and readcardinal2(s) or 0
local filename = band(flags,0x08) ~= 0 and readcstring(s) or ""
local comment = band(flags,0x10) ~= 0 and readcstring(s) or ""
local checksum = band(flags,0x02) ~= 0 and readcardinal2(s) or 0
local compressed = readstring(s,#str)
local data = decompress(compressed,gzipwindow) -- pass offset
return data
end
putcompressed = function(str,level,originalname)
return concat {
identifier, -- 2 identifier
tocardinal1(0x08), -- 1 method
tocardinal1(0x08), -- 1 flags
tocardinal4(os.time()), -- 4 mtime
tocardinal1(0x02), -- 1 compression (2 or 4)
tocardinal1(0xFF), -- 1 operating
(originalname or "unknownname") .. "\0",
compress(str,level,nil,gzipwindow),
tocardinal4(zlibchecksum(str)), -- 4
tocardinal4(#str), -- 4
}
end
end
function gzip.load(filename)
local f = openfile(filename,"rb")
if not f then
-- invalid file
else
local data = f:read("*all")
f:close()
if data and data ~= "" then
if suffix(filename) == "gz" then
data = getdecompressed(data)
end
return data
end
end
end
function gzip.save(filename,data,level,originalname)
if suffix(filename) ~= "gz" then
filename = filename .. ".gz"
end
local f = openfile(filename,"wb")
if f then
data = putcompressed(data or "",level or gziplevel,originalname)
f:write(data)
f:close()
return #data
end
end
function gzip.compress(s,level)
if s and not find(s,pattern) then
if not level then
level = gziplevel
elseif level <= 0 then
return s
elseif level > 9 then
level = 9
end
return putcompressed(s,level or gziplevel) or s
end
end
function gzip.decompress(s)
if s and find(s,pattern) then
return getdecompressed(s)
else
return s
end
end
-- return zipfiles