zlib.nim 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426
  1. # Converted from Pascal
  2. ## Interface to the zlib http://www.zlib.net/ compression library.
  3. when defined(windows):
  4. const libz = "zlib1.dll"
  5. elif defined(macosx):
  6. const libz = "libz.dylib"
  7. else:
  8. const libz = "libz.so.1"
  9. type
  10. Uint* = int32
  11. Ulong* = int
  12. Ulongf* = int
  13. Pulongf* = ptr Ulongf
  14. ZOffT* = int32
  15. Pbyte* = cstring
  16. Pbytef* = cstring
  17. Allocfunc* = proc (p: pointer, items: Uint, size: Uint): pointer{.cdecl.}
  18. FreeFunc* = proc (p: pointer, address: pointer){.cdecl.}
  19. InternalState*{.final, pure.} = object
  20. PInternalState* = ptr InternalState
  21. ZStream*{.final, pure.} = object
  22. nextIn*: Pbytef
  23. availIn*: Uint
  24. totalIn*: Ulong
  25. nextOut*: Pbytef
  26. availOut*: Uint
  27. totalOut*: Ulong
  28. msg*: Pbytef
  29. state*: PInternalState
  30. zalloc*: Allocfunc
  31. zfree*: FreeFunc
  32. opaque*: pointer
  33. dataType*: int32
  34. adler*: Ulong
  35. reserved*: Ulong
  36. ZStreamRec* = ZStream
  37. PZstream* = ptr ZStream
  38. GzFile* = pointer
  39. ZStreamHeader* = enum
  40. DETECT_STREAM,
  41. RAW_DEFLATE,
  42. ZLIB_STREAM,
  43. GZIP_STREAM
  44. ZlibStreamError* = object of Exception
  45. {.deprecated: [TInternalState: InternalState, TAllocfunc: Allocfunc,
  46. TFreeFunc: FreeFunc, TZStream: ZStream, TZStreamRec: ZStreamRec].}
  47. const
  48. Z_NO_FLUSH* = 0
  49. Z_PARTIAL_FLUSH* = 1
  50. Z_SYNC_FLUSH* = 2
  51. Z_FULL_FLUSH* = 3
  52. Z_FINISH* = 4
  53. Z_OK* = 0
  54. Z_STREAM_END* = 1
  55. Z_NEED_DICT* = 2
  56. Z_ERRNO* = -1
  57. Z_STREAM_ERROR* = -2
  58. Z_DATA_ERROR* = -3
  59. Z_MEM_ERROR* = -4
  60. Z_BUF_ERROR* = -5
  61. Z_VERSION_ERROR* = -6
  62. Z_NO_COMPRESSION* = 0
  63. Z_BEST_SPEED* = 1
  64. Z_BEST_COMPRESSION* = 9
  65. Z_DEFAULT_COMPRESSION* = -1
  66. Z_FILTERED* = 1
  67. Z_HUFFMAN_ONLY* = 2
  68. Z_DEFAULT_STRATEGY* = 0
  69. Z_BINARY* = 0
  70. Z_ASCII* = 1
  71. Z_UNKNOWN* = 2
  72. Z_DEFLATED* = 8
  73. Z_NULL* = 0
  74. Z_MEM_LEVEL* = 8
  75. MAX_WBITS* = 15
  76. proc zlibVersion*(): cstring{.cdecl, dynlib: libz, importc: "zlibVersion".}
  77. proc deflate*(strm: var ZStream, flush: int32): int32{.cdecl, dynlib: libz,
  78. importc: "deflate".}
  79. proc deflateEnd*(strm: var ZStream): int32{.cdecl, dynlib: libz,
  80. importc: "deflateEnd".}
  81. proc inflate*(strm: var ZStream, flush: int32): int32{.cdecl, dynlib: libz,
  82. importc: "inflate".}
  83. proc inflateEnd*(strm: var ZStream): int32{.cdecl, dynlib: libz,
  84. importc: "inflateEnd".}
  85. proc deflateSetDictionary*(strm: var ZStream, dictionary: Pbytef,
  86. dictLength: Uint): int32{.cdecl, dynlib: libz,
  87. importc: "deflateSetDictionary".}
  88. proc deflateCopy*(dest, source: var ZStream): int32{.cdecl, dynlib: libz,
  89. importc: "deflateCopy".}
  90. proc deflateReset*(strm: var ZStream): int32{.cdecl, dynlib: libz,
  91. importc: "deflateReset".}
  92. proc deflateParams*(strm: var ZStream, level: int32, strategy: int32): int32{.
  93. cdecl, dynlib: libz, importc: "deflateParams".}
  94. proc inflateSetDictionary*(strm: var ZStream, dictionary: Pbytef,
  95. dictLength: Uint): int32{.cdecl, dynlib: libz,
  96. importc: "inflateSetDictionary".}
  97. proc inflateSync*(strm: var ZStream): int32{.cdecl, dynlib: libz,
  98. importc: "inflateSync".}
  99. proc inflateReset*(strm: var ZStream): int32{.cdecl, dynlib: libz,
  100. importc: "inflateReset".}
  101. proc compress*(dest: Pbytef, destLen: Pulongf, source: Pbytef, sourceLen: Ulong): cint{.
  102. cdecl, dynlib: libz, importc: "compress".}
  103. proc compress2*(dest: Pbytef, destLen: Pulongf, source: Pbytef,
  104. sourceLen: Ulong, level: cint): cint{.cdecl, dynlib: libz,
  105. importc: "compress2".}
  106. proc uncompress*(dest: Pbytef, destLen: Pulongf, source: Pbytef,
  107. sourceLen: Ulong): cint{.cdecl, dynlib: libz,
  108. importc: "uncompress".}
  109. proc compressBound*(sourceLen: Ulong): Ulong {.cdecl, dynlib: libz, importc.}
  110. proc gzopen*(path: cstring, mode: cstring): GzFile{.cdecl, dynlib: libz,
  111. importc: "gzopen".}
  112. proc gzdopen*(fd: int32, mode: cstring): GzFile{.cdecl, dynlib: libz,
  113. importc: "gzdopen".}
  114. proc gzsetparams*(thefile: GzFile, level: int32, strategy: int32): int32{.cdecl,
  115. dynlib: libz, importc: "gzsetparams".}
  116. proc gzread*(thefile: GzFile, buf: pointer, length: int): int32{.cdecl,
  117. dynlib: libz, importc: "gzread".}
  118. proc gzwrite*(thefile: GzFile, buf: pointer, length: int): int32{.cdecl,
  119. dynlib: libz, importc: "gzwrite".}
  120. proc gzprintf*(thefile: GzFile, format: Pbytef): int32{.varargs, cdecl,
  121. dynlib: libz, importc: "gzprintf".}
  122. proc gzputs*(thefile: GzFile, s: Pbytef): int32{.cdecl, dynlib: libz,
  123. importc: "gzputs".}
  124. proc gzgets*(thefile: GzFile, buf: Pbytef, length: int32): Pbytef{.cdecl,
  125. dynlib: libz, importc: "gzgets".}
  126. proc gzputc*(thefile: GzFile, c: char): char{.cdecl, dynlib: libz,
  127. importc: "gzputc".}
  128. proc gzgetc*(thefile: GzFile): char{.cdecl, dynlib: libz, importc: "gzgetc".}
  129. proc gzflush*(thefile: GzFile, flush: int32): int32{.cdecl, dynlib: libz,
  130. importc: "gzflush".}
  131. proc gzseek*(thefile: GzFile, offset: ZOffT, whence: int32): ZOffT{.cdecl,
  132. dynlib: libz, importc: "gzseek".}
  133. proc gzrewind*(thefile: GzFile): int32{.cdecl, dynlib: libz, importc: "gzrewind".}
  134. proc gztell*(thefile: GzFile): ZOffT{.cdecl, dynlib: libz, importc: "gztell".}
  135. proc gzeof*(thefile: GzFile): int {.cdecl, dynlib: libz, importc: "gzeof".}
  136. proc gzclose*(thefile: GzFile): int32{.cdecl, dynlib: libz, importc: "gzclose".}
  137. proc gzerror*(thefile: GzFile, errnum: var int32): Pbytef{.cdecl, dynlib: libz,
  138. importc: "gzerror".}
  139. proc adler32*(adler: Ulong, buf: Pbytef, length: Uint): Ulong{.cdecl,
  140. dynlib: libz, importc: "adler32".}
  141. ## **Warning**: Adler-32 requires at least a few hundred bytes to get rolling.
  142. proc crc32*(crc: Ulong, buf: Pbytef, length: Uint): Ulong{.cdecl, dynlib: libz,
  143. importc: "crc32".}
  144. proc deflateInitu*(strm: var ZStream, level: int32, version: cstring,
  145. streamSize: int32): int32{.cdecl, dynlib: libz,
  146. importc: "deflateInit_".}
  147. proc inflateInitu*(strm: var ZStream, version: cstring,
  148. streamSize: int32): int32 {.
  149. cdecl, dynlib: libz, importc: "inflateInit_".}
  150. proc deflateInit*(strm: var ZStream, level: int32): int32
  151. proc inflateInit*(strm: var ZStream): int32
  152. proc deflateInit2u*(strm: var ZStream, level: int32, `method`: int32,
  153. windowBits: int32, memLevel: int32, strategy: int32,
  154. version: cstring, streamSize: int32): int32 {.cdecl,
  155. dynlib: libz, importc: "deflateInit2_".}
  156. proc inflateInit2u*(strm: var ZStream, windowBits: int32, version: cstring,
  157. streamSize: int32): int32{.cdecl, dynlib: libz,
  158. importc: "inflateInit2_".}
  159. proc deflateInit2*(strm: var ZStream,
  160. level, `method`, windowBits, memLevel,
  161. strategy: int32): int32
  162. proc inflateInit2*(strm: var ZStream, windowBits: int32): int32
  163. proc zError*(err: int32): cstring{.cdecl, dynlib: libz, importc: "zError".}
  164. proc inflateSyncPoint*(z: PZstream): int32{.cdecl, dynlib: libz,
  165. importc: "inflateSyncPoint".}
  166. proc getCrcTable*(): pointer{.cdecl, dynlib: libz, importc: "get_crc_table".}
  167. proc deflateBound*(strm: var ZStream, sourceLen: ULong): ULong {.cdecl,
  168. dynlib: libz, importc: "deflateBound".}
  169. proc deflateInit(strm: var ZStream, level: int32): int32 =
  170. result = deflateInitu(strm, level, zlibVersion(), sizeof(ZStream).cint)
  171. proc inflateInit(strm: var ZStream): int32 =
  172. result = inflateInitu(strm, zlibVersion(), sizeof(ZStream).cint)
  173. proc deflateInit2(strm: var ZStream,
  174. level, `method`, windowBits, memLevel,
  175. strategy: int32): int32 =
  176. result = deflateInit2u(strm, level, `method`, windowBits, memLevel,
  177. strategy, zlibVersion(), sizeof(ZStream).cint)
  178. proc inflateInit2(strm: var ZStream, windowBits: int32): int32 =
  179. result = inflateInit2u(strm, windowBits, zlibVersion(),
  180. sizeof(ZStream).cint)
  181. proc zlibAllocMem*(appData: pointer, items, size: int): pointer {.cdecl.} =
  182. result = alloc(items * size)
  183. proc zlibFreeMem*(appData, `block`: pointer) {.cdecl.} =
  184. dealloc(`block`)
  185. proc compress*(sourceBuf: cstring; sourceLen: int; level=Z_DEFAULT_COMPRESSION; stream=GZIP_STREAM): string =
  186. ## Given a cstring, returns its deflated version with an optional header.
  187. ##
  188. ## Valid argument for ``stream`` are
  189. ## - ``ZLIB_STREAM`` - add a zlib header and footer.
  190. ## - ``GZIP_STREAM`` - add a basic gzip header and footer.
  191. ## - ``RAW_DEFLATE`` - no header is generated.
  192. ##
  193. ## Passing a nil cstring will crash this proc in release mode and assert in
  194. ## debug mode.
  195. ##
  196. ## Compression level can be set with ``level`` argument. Currently
  197. ## ``Z_DEFAULT_COMPRESSION`` is 6.
  198. ##
  199. ## Returns "" on failure.
  200. assert(not sourceBuf.isNil)
  201. assert(sourceLen >= 0)
  202. var z: ZStream
  203. var windowBits = MAX_WBITS
  204. case (stream)
  205. of RAW_DEFLATE: windowBits = -MAX_WBITS
  206. of GZIP_STREAM: windowBits = MAX_WBITS + 16
  207. of ZLIB_STREAM, DETECT_STREAM:
  208. discard # DETECT_STREAM defaults to ZLIB_STREAM
  209. var status = deflateInit2(z, level.int32, Z_DEFLATED.int32,
  210. windowBits.int32, Z_MEM_LEVEL.int32,
  211. Z_DEFAULT_STRATEGY.int32)
  212. case status
  213. of Z_OK: discard
  214. of Z_MEM_ERROR: raise newException(OutOfMemError, "")
  215. of Z_STREAM_ERROR: raise newException(ZlibStreamError, "invalid zlib stream parameter!")
  216. of Z_VERSION_ERROR: raise newException(ZlibStreamError, "zlib version mismatch!")
  217. else: raise newException(ZlibStreamError, "Unkown error(" & $status & ") : " & $z.msg)
  218. let space = deflateBound(z, sourceLen)
  219. var compressed = newStringOfCap(space)
  220. z.next_in = sourceBuf
  221. z.avail_in = sourceLen.Uint
  222. z.next_out = addr(compressed[0])
  223. z.avail_out = space.Uint
  224. status = deflate(z, Z_FINISH)
  225. if status != Z_STREAM_END:
  226. discard deflateEnd(z) # cleanup allocated ressources
  227. raise newException(ZlibStreamError, "Invalid stream state(" & $status & ") : " & $z.msg)
  228. status = deflateEnd(z)
  229. if status != Z_OK: # cleanup allocated ressources
  230. raise newException(ZlibStreamError, "Invalid stream state(" & $status & ") : " & $z.msg)
  231. compressed.setLen(z.total_out)
  232. swap(result, compressed)
  233. proc compress*(input: string; level=Z_DEFAULT_COMPRESSION; stream=GZIP_STREAM): string =
  234. ## Given a string, returns its deflated version with an optional header.
  235. ##
  236. ## Valid arguments for ``stream`` are
  237. ## - ``ZLIB_STREAM`` - add a zlib header and footer.
  238. ## - ``GZIP_STREAM`` - add a basic gzip header and footer.
  239. ## - ``RAW_DEFLATE`` - no header is generated.
  240. ##
  241. ## Compression level can be set with ``level`` argument. Currently
  242. ## ``Z_DEFAULT_COMPRESSION`` is 6.
  243. ##
  244. ## Returns "" on failure.
  245. result = compress(input, input.len, level, stream)
  246. proc uncompress*(sourceBuf: cstring, sourceLen: Natural; stream=DETECT_STREAM): string =
  247. ## Given a deflated buffer returns its inflated content as a string.
  248. ##
  249. ## Valid arguments for ``stream`` are
  250. ## - ``DETECT_STREAM`` - detect if zlib or gzip header is present
  251. ## and decompress stream. Fail on raw deflate stream.
  252. ## - ``ZLIB_STREAM`` - decompress a zlib stream.
  253. ## - ``GZIP_STREAM`` - decompress a gzip stream.
  254. ## - ``RAW_DEFLATE`` - decompress a raw deflate stream.
  255. ##
  256. ## Passing a nil cstring will crash this proc in release mode and assert in
  257. ## debug mode.
  258. ##
  259. ## Returns "" on problems. Failure is a very loose concept, it could be you
  260. ## passing a non deflated string, or it could mean not having enough memory
  261. ## for the inflated version.
  262. ##
  263. ## The uncompression algorithm is based on http://zlib.net/zpipe.c.
  264. assert(not sourceBuf.isNil)
  265. assert(sourceLen >= 0)
  266. var z: ZStream
  267. var decompressed: string = ""
  268. var sbytes = 0
  269. var wbytes = 0
  270. ## allocate inflate state
  271. z.availIn = 0
  272. var wbits = case (stream)
  273. of RAW_DEFLATE: -MAX_WBITS
  274. of ZLIB_STREAM: MAX_WBITS
  275. of GZIP_STREAM: MAX_WBITS + 16
  276. of DETECT_STREAM: MAX_WBITS + 32
  277. var status = inflateInit2(z, wbits.int32)
  278. case status
  279. of Z_OK: discard
  280. of Z_MEM_ERROR: raise newException(OutOfMemError, "")
  281. of Z_STREAM_ERROR: raise newException(ZlibStreamError, "invalid zlib stream parameter!")
  282. of Z_VERSION_ERROR: raise newException(ZlibStreamError, "zlib version mismatch!")
  283. else: raise newException(ZlibStreamError, "Unkown error(" & $status & ") : " & $z.msg)
  284. # run loop until all input is consumed.
  285. # handle concatenated deflated stream with header.
  286. while true:
  287. z.availIn = (sourceLen - sbytes).int32
  288. # no more input available
  289. if (sourceLen - sbytes) <= 0: break
  290. z.nextIn = sourceBuf[sbytes].unsafeaddr
  291. # run inflate() on available input until output buffer is full
  292. while true:
  293. # if written bytes >= output size : resize output
  294. if wbytes >= decompressed.len:
  295. let cur_outlen = decompressed.len
  296. let new_outlen = if decompressed.len == 0: sourceLen*2 else: decompressed.len*2
  297. if new_outlen < cur_outlen: # unsigned integer overflow, buffer too large
  298. discard inflateEnd(z);
  299. raise newException(OverflowError, "zlib stream decompressed size is too large! (size > " & $int.high & ")")
  300. decompressed.setLen(new_outlen)
  301. # available space for decompression
  302. let space = decompressed.len - wbytes
  303. z.availOut = space.Uint
  304. z.nextOut = decompressed[wbytes].addr
  305. status = inflate(z, Z_NO_FLUSH)
  306. if status.int8 notin {Z_OK.int8, Z_STREAM_END.int8, Z_BUF_ERROR.int8}:
  307. discard inflateEnd(z)
  308. case status
  309. of Z_MEM_ERROR: raise newException(OutOfMemError, "")
  310. of Z_DATA_ERROR: raise newException(ZlibStreamError, "invalid zlib stream parameter!")
  311. else: raise newException(ZlibStreamError, "Unkown error(" & $status & ") : " & $z.msg)
  312. # add written bytes, if any.
  313. wbytes += space - z.availOut.int
  314. # may need more input
  315. if not (z.availOut == 0): break
  316. # inflate() says stream is done
  317. if (status == Z_STREAM_END):
  318. # may have another stream concatenated
  319. if z.availIn != 0:
  320. sbytes = sourceLen - z.availIn # add consumed bytes
  321. if inflateReset(z) != Z_OK: # reset zlib struct and try again
  322. raise newException(ZlibStreamError, "Invalid stream state(" & $status & ") : " & $z.msg)
  323. else:
  324. break # end of decompression
  325. # clean up and don't care about any error
  326. discard inflateEnd(z)
  327. if status != Z_STREAM_END:
  328. raise newException(ZlibStreamError, "Invalid stream state(" & $status & ") : " & $z.msg)
  329. decompressed.setLen(wbytes)
  330. swap(result, decompressed)
  331. proc uncompress*(sourceBuf: string; stream=DETECT_STREAM): string =
  332. ## Given a GZIP-ed string return its inflated content.
  333. ##
  334. ## Valid arguments for ``stream`` are
  335. ## - ``DETECT_STREAM`` - detect if zlib or gzip header is present
  336. ## and decompress stream. Fail on raw deflate stream.
  337. ## - ``ZLIB_STREAM`` - decompress a zlib stream.
  338. ## - ``GZIP_STREAM`` - decompress a gzip stream.
  339. ## - ``RAW_DEFLATE`` - decompress a raw deflate stream.
  340. ##
  341. ## Returns "" on failure.
  342. result = uncompress(sourceBuf, sourceBuf.len, stream)
  343. proc deflate*(buffer: var string; level=Z_DEFAULT_COMPRESSION; stream=GZIP_STREAM): bool {.discardable.} =
  344. ## Convenience proc which deflates a string and insert an optional header/footer.
  345. ##
  346. ## Valid arguments for ``stream`` are
  347. ## - ``ZLIB_STREAM`` - add a zlib header and footer.
  348. ## - ``GZIP_STREAM`` - add a basic gzip header and footer.
  349. ## - ``RAW_DEFLATE`` - no header is generated.
  350. ##
  351. ## Compression level can be set with ``level`` argument. Currently
  352. ## ``Z_DEFAULT_COMPRESSION`` is 6.
  353. ##
  354. ## Returns true if `buffer` was successfully deflated otherwise the buffer is untouched.
  355. var temp = compress(addr(buffer[0]), buffer.len, level, stream)
  356. if temp.len != 0:
  357. swap(buffer, temp)
  358. result = true
  359. proc inflate*(buffer: var string; stream=DETECT_STREAM): bool {.discardable.} =
  360. ## Convenience proc which inflates a string containing compressed data
  361. ## with an optional header.
  362. ##
  363. ## Valid argument for ``stream`` are:
  364. ## - ``DETECT_STREAM`` - detect if zlib or gzip header is present
  365. ## and decompress stream. Fail on raw deflate stream.
  366. ## - ``ZLIB_STREAM`` - decompress a zlib stream.
  367. ## - ``GZIP_STREAM`` - decompress a gzip stream.
  368. ## - ``RAW_DEFLATE`` - decompress a raw deflate stream.
  369. ##
  370. ## It is ok to pass a buffer which doesn't contain deflated data,
  371. ## in this case the proc won't modify the buffer.
  372. ##
  373. ## Returns true if `buffer` was successfully inflated.
  374. var temp = uncompress(addr(buffer[0]), buffer.len, stream)
  375. if temp.len != 0:
  376. swap(buffer, temp)
  377. result = true