jchuff-sse2-64.asm 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361
  1. ;
  2. ; jchuff-sse2-64.asm - Huffman entropy encoding (64-bit SSE2)
  3. ;
  4. ; Copyright (C) 2009-2011, 2014-2016, D. R. Commander.
  5. ; Copyright (C) 2015, Matthieu Darbois.
  6. ;
  7. ; Based on the x86 SIMD extension for IJG JPEG library
  8. ; Copyright (C) 1999-2006, MIYASAKA Masaru.
  9. ; For conditions of distribution and use, see copyright notice in jsimdext.inc
  10. ;
  11. ; This file should be assembled with NASM (Netwide Assembler),
  12. ; can *not* be assembled with Microsoft's MASM or any compatible
  13. ; assembler (including Borland's Turbo Assembler).
  14. ; NASM is available from http://nasm.sourceforge.net/ or
  15. ; http://sourceforge.net/project/showfiles.php?group_id=6208
  16. ;
  17. ; This file contains an SSE2 implementation for Huffman coding of one block.
  18. ; The following code is based directly on jchuff.c; see jchuff.c for more
  19. ; details.
  20. ;
  21. ; [TAB8]
  22. %include "jsimdext.inc"
  23. ; --------------------------------------------------------------------------
  24. SECTION SEG_CONST
  25. alignz 16
  26. global EXTN(jconst_huff_encode_one_block)
  27. EXTN(jconst_huff_encode_one_block):
  28. %include "jpeg_nbits_table.inc"
  29. alignz 16
  30. ; --------------------------------------------------------------------------
  31. SECTION SEG_TEXT
  32. BITS 64
  33. ; These macros perform the same task as the emit_bits() function in the
  34. ; original libjpeg code. In addition to reducing overhead by explicitly
  35. ; inlining the code, additional performance is achieved by taking into
  36. ; account the size of the bit buffer and waiting until it is almost full
  37. ; before emptying it. This mostly benefits 64-bit platforms, since 6
  38. ; bytes can be stored in a 64-bit bit buffer before it has to be emptied.
  39. %macro EMIT_BYTE 0
  40. sub put_bits, 8 ; put_bits -= 8;
  41. mov rdx, put_buffer
  42. mov ecx, put_bits
  43. shr rdx, cl ; c = (JOCTET)GETJOCTET(put_buffer >> put_bits);
  44. mov byte [buffer], dl ; *buffer++ = c;
  45. add buffer, 1
  46. cmp dl, 0xFF ; need to stuff a zero byte?
  47. jne %%.EMIT_BYTE_END
  48. mov byte [buffer], 0 ; *buffer++ = 0;
  49. add buffer, 1
  50. %%.EMIT_BYTE_END:
  51. %endmacro
  52. %macro PUT_BITS 1
  53. add put_bits, ecx ; put_bits += size;
  54. shl put_buffer, cl ; put_buffer = (put_buffer << size);
  55. or put_buffer, %1
  56. %endmacro
  57. %macro CHECKBUF31 0
  58. cmp put_bits, 32 ; if (put_bits > 31) {
  59. jl %%.CHECKBUF31_END
  60. EMIT_BYTE
  61. EMIT_BYTE
  62. EMIT_BYTE
  63. EMIT_BYTE
  64. %%.CHECKBUF31_END:
  65. %endmacro
  66. %macro CHECKBUF47 0
  67. cmp put_bits, 48 ; if (put_bits > 47) {
  68. jl %%.CHECKBUF47_END
  69. EMIT_BYTE
  70. EMIT_BYTE
  71. EMIT_BYTE
  72. EMIT_BYTE
  73. EMIT_BYTE
  74. EMIT_BYTE
  75. %%.CHECKBUF47_END:
  76. %endmacro
  77. %macro EMIT_BITS 2
  78. CHECKBUF47
  79. mov ecx, %2
  80. PUT_BITS %1
  81. %endmacro
  82. %macro kloop_prepare 37 ;(ko, jno0, ..., jno31, xmm0, xmm1, xmm2, xmm3)
  83. pxor xmm8, xmm8 ; __m128i neg = _mm_setzero_si128();
  84. pxor xmm9, xmm9 ; __m128i neg = _mm_setzero_si128();
  85. pxor xmm10, xmm10 ; __m128i neg = _mm_setzero_si128();
  86. pxor xmm11, xmm11 ; __m128i neg = _mm_setzero_si128();
  87. pinsrw %34, word [r12 + %2 * SIZEOF_WORD], 0 ; xmm_shadow[0] = block[jno0];
  88. pinsrw %35, word [r12 + %10 * SIZEOF_WORD], 0 ; xmm_shadow[8] = block[jno8];
  89. pinsrw %36, word [r12 + %18 * SIZEOF_WORD], 0 ; xmm_shadow[16] = block[jno16];
  90. pinsrw %37, word [r12 + %26 * SIZEOF_WORD], 0 ; xmm_shadow[24] = block[jno24];
  91. pinsrw %34, word [r12 + %3 * SIZEOF_WORD], 1 ; xmm_shadow[1] = block[jno1];
  92. pinsrw %35, word [r12 + %11 * SIZEOF_WORD], 1 ; xmm_shadow[9] = block[jno9];
  93. pinsrw %36, word [r12 + %19 * SIZEOF_WORD], 1 ; xmm_shadow[17] = block[jno17];
  94. pinsrw %37, word [r12 + %27 * SIZEOF_WORD], 1 ; xmm_shadow[25] = block[jno25];
  95. pinsrw %34, word [r12 + %4 * SIZEOF_WORD], 2 ; xmm_shadow[2] = block[jno2];
  96. pinsrw %35, word [r12 + %12 * SIZEOF_WORD], 2 ; xmm_shadow[10] = block[jno10];
  97. pinsrw %36, word [r12 + %20 * SIZEOF_WORD], 2 ; xmm_shadow[18] = block[jno18];
  98. pinsrw %37, word [r12 + %28 * SIZEOF_WORD], 2 ; xmm_shadow[26] = block[jno26];
  99. pinsrw %34, word [r12 + %5 * SIZEOF_WORD], 3 ; xmm_shadow[3] = block[jno3];
  100. pinsrw %35, word [r12 + %13 * SIZEOF_WORD], 3 ; xmm_shadow[11] = block[jno11];
  101. pinsrw %36, word [r12 + %21 * SIZEOF_WORD], 3 ; xmm_shadow[19] = block[jno19];
  102. pinsrw %37, word [r12 + %29 * SIZEOF_WORD], 3 ; xmm_shadow[27] = block[jno27];
  103. pinsrw %34, word [r12 + %6 * SIZEOF_WORD], 4 ; xmm_shadow[4] = block[jno4];
  104. pinsrw %35, word [r12 + %14 * SIZEOF_WORD], 4 ; xmm_shadow[12] = block[jno12];
  105. pinsrw %36, word [r12 + %22 * SIZEOF_WORD], 4 ; xmm_shadow[20] = block[jno20];
  106. pinsrw %37, word [r12 + %30 * SIZEOF_WORD], 4 ; xmm_shadow[28] = block[jno28];
  107. pinsrw %34, word [r12 + %7 * SIZEOF_WORD], 5 ; xmm_shadow[5] = block[jno5];
  108. pinsrw %35, word [r12 + %15 * SIZEOF_WORD], 5 ; xmm_shadow[13] = block[jno13];
  109. pinsrw %36, word [r12 + %23 * SIZEOF_WORD], 5 ; xmm_shadow[21] = block[jno21];
  110. pinsrw %37, word [r12 + %31 * SIZEOF_WORD], 5 ; xmm_shadow[29] = block[jno29];
  111. pinsrw %34, word [r12 + %8 * SIZEOF_WORD], 6 ; xmm_shadow[6] = block[jno6];
  112. pinsrw %35, word [r12 + %16 * SIZEOF_WORD], 6 ; xmm_shadow[14] = block[jno14];
  113. pinsrw %36, word [r12 + %24 * SIZEOF_WORD], 6 ; xmm_shadow[22] = block[jno22];
  114. pinsrw %37, word [r12 + %32 * SIZEOF_WORD], 6 ; xmm_shadow[30] = block[jno30];
  115. pinsrw %34, word [r12 + %9 * SIZEOF_WORD], 7 ; xmm_shadow[7] = block[jno7];
  116. pinsrw %35, word [r12 + %17 * SIZEOF_WORD], 7 ; xmm_shadow[15] = block[jno15];
  117. pinsrw %36, word [r12 + %25 * SIZEOF_WORD], 7 ; xmm_shadow[23] = block[jno23];
  118. %if %1 != 32
  119. pinsrw %37, word [r12 + %33 * SIZEOF_WORD], 7 ; xmm_shadow[31] = block[jno31];
  120. %else
  121. pinsrw %37, ebx, 7 ; xmm_shadow[31] = block[jno31];
  122. %endif
  123. pcmpgtw xmm8, %34 ; neg = _mm_cmpgt_epi16(neg, x1);
  124. pcmpgtw xmm9, %35 ; neg = _mm_cmpgt_epi16(neg, x1);
  125. pcmpgtw xmm10, %36 ; neg = _mm_cmpgt_epi16(neg, x1);
  126. pcmpgtw xmm11, %37 ; neg = _mm_cmpgt_epi16(neg, x1);
  127. paddw %34, xmm8 ; x1 = _mm_add_epi16(x1, neg);
  128. paddw %35, xmm9 ; x1 = _mm_add_epi16(x1, neg);
  129. paddw %36, xmm10 ; x1 = _mm_add_epi16(x1, neg);
  130. paddw %37, xmm11 ; x1 = _mm_add_epi16(x1, neg);
  131. pxor %34, xmm8 ; x1 = _mm_xor_si128(x1, neg);
  132. pxor %35, xmm9 ; x1 = _mm_xor_si128(x1, neg);
  133. pxor %36, xmm10 ; x1 = _mm_xor_si128(x1, neg);
  134. pxor %37, xmm11 ; x1 = _mm_xor_si128(x1, neg);
  135. pxor xmm8, %34 ; neg = _mm_xor_si128(neg, x1);
  136. pxor xmm9, %35 ; neg = _mm_xor_si128(neg, x1);
  137. pxor xmm10, %36 ; neg = _mm_xor_si128(neg, x1);
  138. pxor xmm11, %37 ; neg = _mm_xor_si128(neg, x1);
  139. movdqa XMMWORD [t1 + %1 * SIZEOF_WORD], %34 ; _mm_storeu_si128((__m128i *)(t1 + ko), x1);
  140. movdqa XMMWORD [t1 + (%1 + 8) * SIZEOF_WORD], %35 ; _mm_storeu_si128((__m128i *)(t1 + ko + 8), x1);
  141. movdqa XMMWORD [t1 + (%1 + 16) * SIZEOF_WORD], %36 ; _mm_storeu_si128((__m128i *)(t1 + ko + 16), x1);
  142. movdqa XMMWORD [t1 + (%1 + 24) * SIZEOF_WORD], %37 ; _mm_storeu_si128((__m128i *)(t1 + ko + 24), x1);
  143. movdqa XMMWORD [t2 + %1 * SIZEOF_WORD], xmm8 ; _mm_storeu_si128((__m128i *)(t2 + ko), neg);
  144. movdqa XMMWORD [t2 + (%1 + 8) * SIZEOF_WORD], xmm9 ; _mm_storeu_si128((__m128i *)(t2 + ko + 8), neg);
  145. movdqa XMMWORD [t2 + (%1 + 16) * SIZEOF_WORD], xmm10 ; _mm_storeu_si128((__m128i *)(t2 + ko + 16), neg);
  146. movdqa XMMWORD [t2 + (%1 + 24) * SIZEOF_WORD], xmm11 ; _mm_storeu_si128((__m128i *)(t2 + ko + 24), neg);
  147. %endmacro
  148. ;
  149. ; Encode a single block's worth of coefficients.
  150. ;
  151. ; GLOBAL(JOCTET*)
  152. ; jsimd_huff_encode_one_block_sse2 (working_state *state, JOCTET *buffer,
  153. ; JCOEFPTR block, int last_dc_val,
  154. ; c_derived_tbl *dctbl, c_derived_tbl *actbl)
  155. ;
  156. ; r10 = working_state *state
  157. ; r11 = JOCTET *buffer
  158. ; r12 = JCOEFPTR block
  159. ; r13 = int last_dc_val
  160. ; r14 = c_derived_tbl *dctbl
  161. ; r15 = c_derived_tbl *actbl
  162. %define t1 rbp-(DCTSIZE2*SIZEOF_WORD)
  163. %define t2 t1-(DCTSIZE2*SIZEOF_WORD)
  164. %define put_buffer r8
  165. %define put_bits r9d
  166. %define buffer rax
  167. align 16
  168. global EXTN(jsimd_huff_encode_one_block_sse2)
  169. EXTN(jsimd_huff_encode_one_block_sse2):
  170. push rbp
  171. mov rax,rsp ; rax = original rbp
  172. sub rsp, byte 4
  173. and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
  174. mov [rsp],rax
  175. mov rbp,rsp ; rbp = aligned rbp
  176. lea rsp, [t2]
  177. collect_args
  178. %ifdef WIN64
  179. movaps XMMWORD [rsp-1*SIZEOF_XMMWORD], xmm8
  180. movaps XMMWORD [rsp-2*SIZEOF_XMMWORD], xmm9
  181. movaps XMMWORD [rsp-3*SIZEOF_XMMWORD], xmm10
  182. movaps XMMWORD [rsp-4*SIZEOF_XMMWORD], xmm11
  183. sub rsp, 4*SIZEOF_XMMWORD
  184. %endif
  185. push rbx
  186. mov buffer, r11 ; r11 is now sratch
  187. mov put_buffer, MMWORD [r10+16] ; put_buffer = state->cur.put_buffer;
  188. mov put_bits, DWORD [r10+24] ; put_bits = state->cur.put_bits;
  189. push r10 ; r10 is now scratch
  190. ; Encode the DC coefficient difference per section F.1.2.1
  191. movsx edi, word [r12] ; temp = temp2 = block[0] - last_dc_val;
  192. sub edi, r13d ; r13 is not used anymore
  193. mov ebx, edi
  194. ; This is a well-known technique for obtaining the absolute value
  195. ; without a branch. It is derived from an assembly language technique
  196. ; presented in "How to Optimize for the Pentium Processors",
  197. ; Copyright (c) 1996, 1997 by Agner Fog.
  198. mov esi, edi
  199. sar esi, 31 ; temp3 = temp >> (CHAR_BIT * sizeof(int) - 1);
  200. xor edi, esi ; temp ^= temp3;
  201. sub edi, esi ; temp -= temp3;
  202. ; For a negative input, want temp2 = bitwise complement of abs(input)
  203. ; This code assumes we are on a two's complement machine
  204. add ebx, esi ; temp2 += temp3;
  205. ; Find the number of bits needed for the magnitude of the coefficient
  206. lea r11, [rel jpeg_nbits_table]
  207. movzx rdi, byte [r11 + rdi] ; nbits = JPEG_NBITS(temp);
  208. ; Emit the Huffman-coded symbol for the number of bits
  209. mov r11d, INT [r14 + rdi * 4] ; code = dctbl->ehufco[nbits];
  210. movzx esi, byte [r14 + rdi + 1024] ; size = dctbl->ehufsi[nbits];
  211. EMIT_BITS r11, esi ; EMIT_BITS(code, size)
  212. ; Mask off any extra bits in code
  213. mov esi, 1
  214. mov ecx, edi
  215. shl esi, cl
  216. dec esi
  217. and ebx, esi ; temp2 &= (((JLONG) 1)<<nbits) - 1;
  218. ; Emit that number of bits of the value, if positive,
  219. ; or the complement of its magnitude, if negative.
  220. EMIT_BITS rbx, edi ; EMIT_BITS(temp2, nbits)
  221. ; Prepare data
  222. xor ebx, ebx
  223. kloop_prepare 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, \
  224. 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, \
  225. 27, 20, 13, 6, 7, 14, 21, 28, 35, \
  226. xmm0, xmm1, xmm2, xmm3
  227. kloop_prepare 32, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, \
  228. 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, \
  229. 53, 60, 61, 54, 47, 55, 62, 63, 63, \
  230. xmm4, xmm5, xmm6, xmm7
  231. pxor xmm8, xmm8
  232. pcmpeqw xmm0, xmm8 ; tmp0 = _mm_cmpeq_epi16(tmp0, zero);
  233. pcmpeqw xmm1, xmm8 ; tmp1 = _mm_cmpeq_epi16(tmp1, zero);
  234. pcmpeqw xmm2, xmm8 ; tmp2 = _mm_cmpeq_epi16(tmp2, zero);
  235. pcmpeqw xmm3, xmm8 ; tmp3 = _mm_cmpeq_epi16(tmp3, zero);
  236. pcmpeqw xmm4, xmm8 ; tmp4 = _mm_cmpeq_epi16(tmp4, zero);
  237. pcmpeqw xmm5, xmm8 ; tmp5 = _mm_cmpeq_epi16(tmp5, zero);
  238. pcmpeqw xmm6, xmm8 ; tmp6 = _mm_cmpeq_epi16(tmp6, zero);
  239. pcmpeqw xmm7, xmm8 ; tmp7 = _mm_cmpeq_epi16(tmp7, zero);
  240. packsswb xmm0, xmm1 ; tmp0 = _mm_packs_epi16(tmp0, tmp1);
  241. packsswb xmm2, xmm3 ; tmp2 = _mm_packs_epi16(tmp2, tmp3);
  242. packsswb xmm4, xmm5 ; tmp4 = _mm_packs_epi16(tmp4, tmp5);
  243. packsswb xmm6, xmm7 ; tmp6 = _mm_packs_epi16(tmp6, tmp7);
  244. pmovmskb r11d, xmm0 ; index = ((uint64_t)_mm_movemask_epi8(tmp0)) << 0;
  245. pmovmskb r12d, xmm2 ; index = ((uint64_t)_mm_movemask_epi8(tmp2)) << 16;
  246. pmovmskb r13d, xmm4 ; index = ((uint64_t)_mm_movemask_epi8(tmp4)) << 32;
  247. pmovmskb r14d, xmm6 ; index = ((uint64_t)_mm_movemask_epi8(tmp6)) << 48;
  248. shl r12, 16
  249. shl r14, 16
  250. or r11, r12
  251. or r13, r14
  252. shl r13, 32
  253. or r11, r13
  254. not r11 ; index = ~index;
  255. ;mov MMWORD [ t1 + DCTSIZE2 * SIZEOF_WORD ], r11
  256. ;jmp .EFN
  257. mov r13d, INT [r15 + 240 * 4] ; code_0xf0 = actbl->ehufco[0xf0];
  258. movzx r14d, byte [r15 + 1024 + 240] ; size_0xf0 = actbl->ehufsi[0xf0];
  259. lea rsi, [t1]
  260. .BLOOP:
  261. bsf r12, r11 ; r = __builtin_ctzl(index);
  262. jz .ELOOP
  263. mov rcx, r12
  264. lea rsi, [rsi+r12*2] ; k += r;
  265. shr r11, cl ; index >>= r;
  266. movzx rdi, word [rsi] ; temp = t1[k];
  267. lea rbx, [rel jpeg_nbits_table]
  268. movzx rdi, byte [rbx + rdi] ; nbits = JPEG_NBITS(temp);
  269. .BRLOOP:
  270. cmp r12, 16 ; while (r > 15) {
  271. jl .ERLOOP
  272. EMIT_BITS r13, r14d ; EMIT_BITS(code_0xf0, size_0xf0)
  273. sub r12, 16 ; r -= 16;
  274. jmp .BRLOOP
  275. .ERLOOP:
  276. ; Emit Huffman symbol for run length / number of bits
  277. CHECKBUF31 ; uses rcx, rdx
  278. shl r12, 4 ; temp3 = (r << 4) + nbits;
  279. add r12, rdi
  280. mov ebx, INT [r15 + r12 * 4] ; code = actbl->ehufco[temp3];
  281. movzx ecx, byte [r15 + r12 + 1024] ; size = actbl->ehufsi[temp3];
  282. PUT_BITS rbx
  283. ;EMIT_CODE(code, size)
  284. movsx ebx, word [rsi-DCTSIZE2*2] ; temp2 = t2[k];
  285. ; Mask off any extra bits in code
  286. mov rcx, rdi
  287. mov rdx, 1
  288. shl rdx, cl
  289. dec rdx
  290. and rbx, rdx ; temp2 &= (((JLONG) 1)<<nbits) - 1;
  291. PUT_BITS rbx ; PUT_BITS(temp2, nbits)
  292. shr r11, 1 ; index >>= 1;
  293. add rsi, 2 ; ++k;
  294. jmp .BLOOP
  295. .ELOOP:
  296. ; If the last coef(s) were zero, emit an end-of-block code
  297. lea rdi, [t1 + (DCTSIZE2-1) * 2] ; r = DCTSIZE2-1-k;
  298. cmp rdi, rsi ; if (r > 0) {
  299. je .EFN
  300. mov ebx, INT [r15] ; code = actbl->ehufco[0];
  301. movzx r12d, byte [r15 + 1024] ; size = actbl->ehufsi[0];
  302. EMIT_BITS rbx, r12d
  303. .EFN:
  304. pop r10
  305. ; Save put_buffer & put_bits
  306. mov MMWORD [r10+16], put_buffer ; state->cur.put_buffer = put_buffer;
  307. mov DWORD [r10+24], put_bits ; state->cur.put_bits = put_bits;
  308. pop rbx
  309. %ifdef WIN64
  310. movaps xmm11, XMMWORD [rsp+0*SIZEOF_XMMWORD]
  311. movaps xmm10, XMMWORD [rsp+1*SIZEOF_XMMWORD]
  312. movaps xmm9, XMMWORD [rsp+2*SIZEOF_XMMWORD]
  313. movaps xmm8, XMMWORD [rsp+3*SIZEOF_XMMWORD]
  314. add rsp, 4*SIZEOF_XMMWORD
  315. %endif
  316. uncollect_args
  317. mov rsp,rbp ; rsp <- aligned rbp
  318. pop rsp ; rsp <- original rbp
  319. pop rbp
  320. ret
  321. ; For some reason, the OS X linker does not honor the request to align the
  322. ; segment unless we do this.
  323. align 16