Home Explore Help
Sign In
khronosschoty
/
palemoon-dev
Watch 2
Star 1
Fork 0
Files Issues 0 Pull Requests 0 Wiki
Branch: master
Branches Tags
palemoon-dev
/
platform
/
media
/
libjpeg
/
simd
/
jfdctfst-sse2-64.asm

jfdctfst-sse2-64.asm 17 KB
Permalink History Raw

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392 
  1. ;
    2. 

  2. ; jfdctfst.asm - fast integer FDCT (64-bit SSE2)
    3. 

  3. ;
    4. 

  4. ; Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
    5. 

  5. ; Copyright (C) 2009, D. R. Commander.
    6. 

  6. ;
    7. 

  7. ; Based on the x86 SIMD extension for IJG JPEG library
    8. 

  8. ; Copyright (C) 1999-2006, MIYASAKA Masaru.
    9. 

  9. ; For conditions of distribution and use, see copyright notice in jsimdext.inc
    10. 

  10. ;
    11. 

  11. ; This file should be assembled with NASM (Netwide Assembler),
    12. 

  12. ; can *not* be assembled with Microsoft's MASM or any compatible
    13. 

  13. ; assembler (including Borland's Turbo Assembler).
    14. 

  14. ; NASM is available from http://nasm.sourceforge.net/ or
    15. 

  15. ; http://sourceforge.net/project/showfiles.php?group_id=6208
    16. 

  16. ;
    17. 

  17. ; This file contains a fast, not so accurate integer implementation of
    18. 

  18. ; the forward DCT (Discrete Cosine Transform). The following code is
    19. 

  19. ; based directly on the IJG's original jfdctfst.c; see the jfdctfst.c
    20. 

  20. ; for more details.
    21. 

  21. ;
    22. 

  22. ; [TAB8]
    23. 

  23. 

  24. %include "jsimdext.inc"
    25. 

  25. %include "jdct.inc"
    26. 

  26. 

  27. ; --------------------------------------------------------------------------
    28. 

  28. 

  29. %define CONST_BITS      8       ; 14 is also OK.
    30. 

  30. 

  31. %if CONST_BITS == 8
    32. 

  32. F_0_382 equ      98             ; FIX(0.382683433)
    33. 

  33. F_0_541 equ     139             ; FIX(0.541196100)
    34. 

  34. F_0_707 equ     181             ; FIX(0.707106781)
    35. 

  35. F_1_306 equ     334             ; FIX(1.306562965)
    36. 

  36. %else
    37. 

  37. ; NASM cannot do compile-time arithmetic on floating-point constants.
    38. 

  38. %define DESCALE(x,n)  (((x)+(1<<((n)-1)))>>(n))
    39. 

  39. F_0_382 equ     DESCALE( 410903207,30-CONST_BITS)       ; FIX(0.382683433)
    40. 

  40. F_0_541 equ     DESCALE( 581104887,30-CONST_BITS)       ; FIX(0.541196100)
    41. 

  41. F_0_707 equ     DESCALE( 759250124,30-CONST_BITS)       ; FIX(0.707106781)
    42. 

  42. F_1_306 equ     DESCALE(1402911301,30-CONST_BITS)       ; FIX(1.306562965)
    43. 

  43. %endif
    44. 

  44. 

  45. ; --------------------------------------------------------------------------
    46. 

  46.         SECTION SEG_CONST
    47. 

  47. 

  48. ; PRE_MULTIPLY_SCALE_BITS <= 2 (to avoid overflow)
    49. 

  49. ; CONST_BITS + CONST_SHIFT + PRE_MULTIPLY_SCALE_BITS == 16 (for pmulhw)
    50. 

  50. 

  51. %define PRE_MULTIPLY_SCALE_BITS   2
    52. 

  52. %define CONST_SHIFT     (16 - PRE_MULTIPLY_SCALE_BITS - CONST_BITS)
    53. 

  53. 

  54.         alignz  16
    55. 

  55.         global  EXTN(jconst_fdct_ifast_sse2)
    56. 

  56. 

  57. EXTN(jconst_fdct_ifast_sse2):
    58. 

  58. 

  59. PW_F0707        times 8 dw  F_0_707 << CONST_SHIFT
    60. 

  60. PW_F0382        times 8 dw  F_0_382 << CONST_SHIFT
    61. 

  61. PW_F0541        times 8 dw  F_0_541 << CONST_SHIFT
    62. 

  62. PW_F1306        times 8 dw  F_1_306 << CONST_SHIFT
    63. 

  63. 

  64.         alignz  16
    65. 

  65. 

  66. ; --------------------------------------------------------------------------
    67. 

  67.         SECTION SEG_TEXT
    68. 

  68.         BITS    64
    69. 

  69. ;
    70. 

  70. ; Perform the forward DCT on one block of samples.
    71. 

  71. ;
    72. 

  72. ; GLOBAL(void)
    73. 

  73. ; jsimd_fdct_ifast_sse2 (DCTELEM *data)
    74. 

  74. ;
    75. 

  75. 

  76. ; r10 = DCTELEM *data
    77. 

  77. 

  78. %define wk(i)           rbp-(WK_NUM-(i))*SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
    79. 

  79. %define WK_NUM          2
    80. 

  80. 

  81.         align   16
    82. 

  82.         global  EXTN(jsimd_fdct_ifast_sse2)
    83. 

  83. 

  84. EXTN(jsimd_fdct_ifast_sse2):
    85. 

  85.         push    rbp
    86. 

  86.         mov     rax,rsp                         ; rax = original rbp
    87. 

  87.         sub     rsp, byte 4
    88. 

  88.         and     rsp, byte (-SIZEOF_XMMWORD)     ; align to 128 bits
    89. 

  89.         mov     [rsp],rax
    90. 

  90.         mov     rbp,rsp                         ; rbp = aligned rbp
    91. 

  91.         lea     rsp, [wk(0)]
    92. 

  92.         collect_args
    93. 

  93. 

  94.         ; ---- Pass 1: process rows.
    95. 

  95. 

  96.         mov     rdx, r10        ; (DCTELEM *)
    97. 

  97. 

  98.         movdqa  xmm0, XMMWORD [XMMBLOCK(0,0,rdx,SIZEOF_DCTELEM)]
    99. 

  99.         movdqa  xmm1, XMMWORD [XMMBLOCK(1,0,rdx,SIZEOF_DCTELEM)]
    100. 

  100.         movdqa  xmm2, XMMWORD [XMMBLOCK(2,0,rdx,SIZEOF_DCTELEM)]
    101. 

  101.         movdqa  xmm3, XMMWORD [XMMBLOCK(3,0,rdx,SIZEOF_DCTELEM)]
    102. 

  102. 

  103.         ; xmm0=(00 01 02 03 04 05 06 07), xmm2=(20 21 22 23 24 25 26 27)
    104. 

  104.         ; xmm1=(10 11 12 13 14 15 16 17), xmm3=(30 31 32 33 34 35 36 37)
    105. 

  105. 

  106.         movdqa    xmm4,xmm0             ; transpose coefficients(phase 1)
    107. 

  107.         punpcklwd xmm0,xmm1             ; xmm0=(00 10 01 11 02 12 03 13)
    108. 

  108.         punpckhwd xmm4,xmm1             ; xmm4=(04 14 05 15 06 16 07 17)
    109. 

  109.         movdqa    xmm5,xmm2             ; transpose coefficients(phase 1)
    110. 

  110.         punpcklwd xmm2,xmm3             ; xmm2=(20 30 21 31 22 32 23 33)
    111. 

  111.         punpckhwd xmm5,xmm3             ; xmm5=(24 34 25 35 26 36 27 37)
    112. 

  112. 

  113.         movdqa  xmm6, XMMWORD [XMMBLOCK(4,0,rdx,SIZEOF_DCTELEM)]
    114. 

  114.         movdqa  xmm7, XMMWORD [XMMBLOCK(5,0,rdx,SIZEOF_DCTELEM)]
    115. 

  115.         movdqa  xmm1, XMMWORD [XMMBLOCK(6,0,rdx,SIZEOF_DCTELEM)]
    116. 

  116.         movdqa  xmm3, XMMWORD [XMMBLOCK(7,0,rdx,SIZEOF_DCTELEM)]
    117. 

  117. 

  118.         ; xmm6=( 4 12 20 28 36 44 52 60), xmm1=( 6 14 22 30 38 46 54 62)
    119. 

  119.         ; xmm7=( 5 13 21 29 37 45 53 61), xmm3=( 7 15 23 31 39 47 55 63)
    120. 

  120. 

  121.         movdqa  XMMWORD [wk(0)], xmm2   ; wk(0)=(20 30 21 31 22 32 23 33)
    122. 

  122.         movdqa  XMMWORD [wk(1)], xmm5   ; wk(1)=(24 34 25 35 26 36 27 37)
    123. 

  123. 

  124.         movdqa    xmm2,xmm6             ; transpose coefficients(phase 1)
    125. 

  125.         punpcklwd xmm6,xmm7             ; xmm6=(40 50 41 51 42 52 43 53)
    126. 

  126.         punpckhwd xmm2,xmm7             ; xmm2=(44 54 45 55 46 56 47 57)
    127. 

  127.         movdqa    xmm5,xmm1             ; transpose coefficients(phase 1)
    128. 

  128.         punpcklwd xmm1,xmm3             ; xmm1=(60 70 61 71 62 72 63 73)
    129. 

  129.         punpckhwd xmm5,xmm3             ; xmm5=(64 74 65 75 66 76 67 77)
    130. 

  130. 

  131.         movdqa    xmm7,xmm6             ; transpose coefficients(phase 2)
    132. 

  132.         punpckldq xmm6,xmm1             ; xmm6=(40 50 60 70 41 51 61 71)
    133. 

  133.         punpckhdq xmm7,xmm1             ; xmm7=(42 52 62 72 43 53 63 73)
    134. 

  134.         movdqa    xmm3,xmm2             ; transpose coefficients(phase 2)
    135. 

  135.         punpckldq xmm2,xmm5             ; xmm2=(44 54 64 74 45 55 65 75)
    136. 

  136.         punpckhdq xmm3,xmm5             ; xmm3=(46 56 66 76 47 57 67 77)
    137. 

  137. 

  138.         movdqa  xmm1, XMMWORD [wk(0)]   ; xmm1=(20 30 21 31 22 32 23 33)
    139. 

  139.         movdqa  xmm5, XMMWORD [wk(1)]   ; xmm5=(24 34 25 35 26 36 27 37)
    140. 

  140.         movdqa  XMMWORD [wk(0)], xmm7   ; wk(0)=(42 52 62 72 43 53 63 73)
    141. 

  141.         movdqa  XMMWORD [wk(1)], xmm2   ; wk(1)=(44 54 64 74 45 55 65 75)
    142. 

  142. 

  143.         movdqa    xmm7,xmm0             ; transpose coefficients(phase 2)
    144. 

  144.         punpckldq xmm0,xmm1             ; xmm0=(00 10 20 30 01 11 21 31)
    145. 

  145.         punpckhdq xmm7,xmm1             ; xmm7=(02 12 22 32 03 13 23 33)
    146. 

  146.         movdqa    xmm2,xmm4             ; transpose coefficients(phase 2)
    147. 

  147.         punpckldq xmm4,xmm5             ; xmm4=(04 14 24 34 05 15 25 35)
    148. 

  148.         punpckhdq xmm2,xmm5             ; xmm2=(06 16 26 36 07 17 27 37)
    149. 

  149. 

  150.         movdqa     xmm1,xmm0            ; transpose coefficients(phase 3)
    151. 

  151.         punpcklqdq xmm0,xmm6            ; xmm0=(00 10 20 30 40 50 60 70)=data0
    152. 

  152.         punpckhqdq xmm1,xmm6            ; xmm1=(01 11 21 31 41 51 61 71)=data1
    153. 

  153.         movdqa     xmm5,xmm2            ; transpose coefficients(phase 3)
    154. 

  154.         punpcklqdq xmm2,xmm3            ; xmm2=(06 16 26 36 46 56 66 76)=data6
    155. 

  155.         punpckhqdq xmm5,xmm3            ; xmm5=(07 17 27 37 47 57 67 77)=data7
    156. 

  156. 

  157.         movdqa  xmm6,xmm1
    158. 

  158.         movdqa  xmm3,xmm0
    159. 

  159.         psubw   xmm1,xmm2               ; xmm1=data1-data6=tmp6
    160. 

  160.         psubw   xmm0,xmm5               ; xmm0=data0-data7=tmp7
    161. 

  161.         paddw   xmm6,xmm2               ; xmm6=data1+data6=tmp1
    162. 

  162.         paddw   xmm3,xmm5               ; xmm3=data0+data7=tmp0
    163. 

  163. 

  164.         movdqa  xmm2, XMMWORD [wk(0)]   ; xmm2=(42 52 62 72 43 53 63 73)
    165. 

  165.         movdqa  xmm5, XMMWORD [wk(1)]   ; xmm5=(44 54 64 74 45 55 65 75)
    166. 

  166.         movdqa  XMMWORD [wk(0)], xmm1   ; wk(0)=tmp6
    167. 

  167.         movdqa  XMMWORD [wk(1)], xmm0   ; wk(1)=tmp7
    168. 

  168. 

  169.         movdqa     xmm1,xmm7            ; transpose coefficients(phase 3)
    170. 

  170.         punpcklqdq xmm7,xmm2            ; xmm7=(02 12 22 32 42 52 62 72)=data2
    171. 

  171.         punpckhqdq xmm1,xmm2            ; xmm1=(03 13 23 33 43 53 63 73)=data3
    172. 

  172.         movdqa     xmm0,xmm4            ; transpose coefficients(phase 3)
    173. 

  173.         punpcklqdq xmm4,xmm5            ; xmm4=(04 14 24 34 44 54 64 74)=data4
    174. 

  174.         punpckhqdq xmm0,xmm5            ; xmm0=(05 15 25 35 45 55 65 75)=data5
    175. 

  175. 

  176.         movdqa  xmm2,xmm1
    177. 

  177.         movdqa  xmm5,xmm7
    178. 

  178.         paddw   xmm1,xmm4               ; xmm1=data3+data4=tmp3
    179. 

  179.         paddw   xmm7,xmm0               ; xmm7=data2+data5=tmp2
    180. 

  180.         psubw   xmm2,xmm4               ; xmm2=data3-data4=tmp4
    181. 

  181.         psubw   xmm5,xmm0               ; xmm5=data2-data5=tmp5
    182. 

  182. 

  183.         ; -- Even part
    184. 

  184. 

  185.         movdqa  xmm4,xmm3
    186. 

  186.         movdqa  xmm0,xmm6
    187. 

  187.         psubw   xmm3,xmm1               ; xmm3=tmp13
    188. 

  188.         psubw   xmm6,xmm7               ; xmm6=tmp12
    189. 

  189.         paddw   xmm4,xmm1               ; xmm4=tmp10
    190. 

  190.         paddw   xmm0,xmm7               ; xmm0=tmp11
    191. 

  191. 

  192.         paddw   xmm6,xmm3
    193. 

  193.         psllw   xmm6,PRE_MULTIPLY_SCALE_BITS
    194. 

  194.         pmulhw  xmm6,[rel PW_F0707] ; xmm6=z1
    195. 

  195. 

  196.         movdqa  xmm1,xmm4
    197. 

  197.         movdqa  xmm7,xmm3
    198. 

  198.         psubw   xmm4,xmm0               ; xmm4=data4
    199. 

  199.         psubw   xmm3,xmm6               ; xmm3=data6
    200. 

  200.         paddw   xmm1,xmm0               ; xmm1=data0
    201. 

  201.         paddw   xmm7,xmm6               ; xmm7=data2
    202. 

  202. 

  203.         movdqa  xmm0, XMMWORD [wk(0)]   ; xmm0=tmp6
    204. 

  204.         movdqa  xmm6, XMMWORD [wk(1)]   ; xmm6=tmp7
    205. 

  205.         movdqa  XMMWORD [wk(0)], xmm4   ; wk(0)=data4
    206. 

  206.         movdqa  XMMWORD [wk(1)], xmm3   ; wk(1)=data6
    207. 

  207. 

  208.         ; -- Odd part
    209. 

  209. 

  210.         paddw   xmm2,xmm5               ; xmm2=tmp10
    211. 

  211.         paddw   xmm5,xmm0               ; xmm5=tmp11
    212. 

  212.         paddw   xmm0,xmm6               ; xmm0=tmp12, xmm6=tmp7
    213. 

  213. 

  214.         psllw   xmm2,PRE_MULTIPLY_SCALE_BITS
    215. 

  215.         psllw   xmm0,PRE_MULTIPLY_SCALE_BITS
    216. 

  216. 

  217.         psllw   xmm5,PRE_MULTIPLY_SCALE_BITS
    218. 

  218.         pmulhw  xmm5,[rel PW_F0707] ; xmm5=z3
    219. 

  219. 

  220.         movdqa  xmm4,xmm2               ; xmm4=tmp10
    221. 

  221.         psubw   xmm2,xmm0
    222. 

  222.         pmulhw  xmm2,[rel PW_F0382] ; xmm2=z5
    223. 

  223.         pmulhw  xmm4,[rel PW_F0541] ; xmm4=MULTIPLY(tmp10,FIX_0_541196)
    224. 

  224.         pmulhw  xmm0,[rel PW_F1306] ; xmm0=MULTIPLY(tmp12,FIX_1_306562)
    225. 

  225.         paddw   xmm4,xmm2               ; xmm4=z2
    226. 

  226.         paddw   xmm0,xmm2               ; xmm0=z4
    227. 

  227. 

  228.         movdqa  xmm3,xmm6
    229. 

  229.         psubw   xmm6,xmm5               ; xmm6=z13
    230. 

  230.         paddw   xmm3,xmm5               ; xmm3=z11
    231. 

  231. 

  232.         movdqa  xmm2,xmm6
    233. 

  233.         movdqa  xmm5,xmm3
    234. 

  234.         psubw   xmm6,xmm4               ; xmm6=data3
    235. 

  235.         psubw   xmm3,xmm0               ; xmm3=data7
    236. 

  236.         paddw   xmm2,xmm4               ; xmm2=data5
    237. 

  237.         paddw   xmm5,xmm0               ; xmm5=data1
    238. 

  238. 

  239.         ; ---- Pass 2: process columns.
    240. 

  240. 

  241.         ; xmm1=(00 10 20 30 40 50 60 70), xmm7=(02 12 22 32 42 52 62 72)
    242. 

  242.         ; xmm5=(01 11 21 31 41 51 61 71), xmm6=(03 13 23 33 43 53 63 73)
    243. 

  243. 

  244.         movdqa    xmm4,xmm1             ; transpose coefficients(phase 1)
    245. 

  245.         punpcklwd xmm1,xmm5             ; xmm1=(00 01 10 11 20 21 30 31)
    246. 

  246.         punpckhwd xmm4,xmm5             ; xmm4=(40 41 50 51 60 61 70 71)
    247. 

  247.         movdqa    xmm0,xmm7             ; transpose coefficients(phase 1)
    248. 

  248.         punpcklwd xmm7,xmm6             ; xmm7=(02 03 12 13 22 23 32 33)
    249. 

  249.         punpckhwd xmm0,xmm6             ; xmm0=(42 43 52 53 62 63 72 73)
    250. 

  250. 

  251.         movdqa  xmm5, XMMWORD [wk(0)]   ; xmm5=col4
    252. 

  252.         movdqa  xmm6, XMMWORD [wk(1)]   ; xmm6=col6
    253. 

  253. 

  254.         ; xmm5=(04 14 24 34 44 54 64 74), xmm6=(06 16 26 36 46 56 66 76)
    255. 

  255.         ; xmm2=(05 15 25 35 45 55 65 75), xmm3=(07 17 27 37 47 57 67 77)
    256. 

  256. 

  257.         movdqa  XMMWORD [wk(0)], xmm7   ; wk(0)=(02 03 12 13 22 23 32 33)
    258. 

  258.         movdqa  XMMWORD [wk(1)], xmm0   ; wk(1)=(42 43 52 53 62 63 72 73)
    259. 

  259. 

  260.         movdqa    xmm7,xmm5             ; transpose coefficients(phase 1)
    261. 

  261.         punpcklwd xmm5,xmm2             ; xmm5=(04 05 14 15 24 25 34 35)
    262. 

  262.         punpckhwd xmm7,xmm2             ; xmm7=(44 45 54 55 64 65 74 75)
    263. 

  263.         movdqa    xmm0,xmm6             ; transpose coefficients(phase 1)
    264. 

  264.         punpcklwd xmm6,xmm3             ; xmm6=(06 07 16 17 26 27 36 37)
    265. 

  265.         punpckhwd xmm0,xmm3             ; xmm0=(46 47 56 57 66 67 76 77)
    266. 

  266. 

  267.         movdqa    xmm2,xmm5             ; transpose coefficients(phase 2)
    268. 

  268.         punpckldq xmm5,xmm6             ; xmm5=(04 05 06 07 14 15 16 17)
    269. 

  269.         punpckhdq xmm2,xmm6             ; xmm2=(24 25 26 27 34 35 36 37)
    270. 

  270.         movdqa    xmm3,xmm7             ; transpose coefficients(phase 2)
    271. 

  271.         punpckldq xmm7,xmm0             ; xmm7=(44 45 46 47 54 55 56 57)
    272. 

  272.         punpckhdq xmm3,xmm0             ; xmm3=(64 65 66 67 74 75 76 77)
    273. 

  273. 

  274.         movdqa  xmm6, XMMWORD [wk(0)]   ; xmm6=(02 03 12 13 22 23 32 33)
    275. 

  275.         movdqa  xmm0, XMMWORD [wk(1)]   ; xmm0=(42 43 52 53 62 63 72 73)
    276. 

  276.         movdqa  XMMWORD [wk(0)], xmm2   ; wk(0)=(24 25 26 27 34 35 36 37)
    277. 

  277.         movdqa  XMMWORD [wk(1)], xmm7   ; wk(1)=(44 45 46 47 54 55 56 57)
    278. 

  278. 

  279.         movdqa    xmm2,xmm1             ; transpose coefficients(phase 2)
    280. 

  280.         punpckldq xmm1,xmm6             ; xmm1=(00 01 02 03 10 11 12 13)
    281. 

  281.         punpckhdq xmm2,xmm6             ; xmm2=(20 21 22 23 30 31 32 33)
    282. 

  282.         movdqa    xmm7,xmm4             ; transpose coefficients(phase 2)
    283. 

  283.         punpckldq xmm4,xmm0             ; xmm4=(40 41 42 43 50 51 52 53)
    284. 

  284.         punpckhdq xmm7,xmm0             ; xmm7=(60 61 62 63 70 71 72 73)
    285. 

  285. 

  286.         movdqa     xmm6,xmm1            ; transpose coefficients(phase 3)
    287. 

  287.         punpcklqdq xmm1,xmm5            ; xmm1=(00 01 02 03 04 05 06 07)=data0
    288. 

  288.         punpckhqdq xmm6,xmm5            ; xmm6=(10 11 12 13 14 15 16 17)=data1
    289. 

  289.         movdqa     xmm0,xmm7            ; transpose coefficients(phase 3)
    290. 

  290.         punpcklqdq xmm7,xmm3            ; xmm7=(60 61 62 63 64 65 66 67)=data6
    291. 

  291.         punpckhqdq xmm0,xmm3            ; xmm0=(70 71 72 73 74 75 76 77)=data7
    292. 

  292. 

  293.         movdqa  xmm5,xmm6
    294. 

  294.         movdqa  xmm3,xmm1
    295. 

  295.         psubw   xmm6,xmm7               ; xmm6=data1-data6=tmp6
    296. 

  296.         psubw   xmm1,xmm0               ; xmm1=data0-data7=tmp7
    297. 

  297.         paddw   xmm5,xmm7               ; xmm5=data1+data6=tmp1
    298. 

  298.         paddw   xmm3,xmm0               ; xmm3=data0+data7=tmp0
    299. 

  299. 

  300.         movdqa  xmm7, XMMWORD [wk(0)]   ; xmm7=(24 25 26 27 34 35 36 37)
    301. 

  301.         movdqa  xmm0, XMMWORD [wk(1)]   ; xmm0=(44 45 46 47 54 55 56 57)
    302. 

  302.         movdqa  XMMWORD [wk(0)], xmm6   ; wk(0)=tmp6
    303. 

  303.         movdqa  XMMWORD [wk(1)], xmm1   ; wk(1)=tmp7
    304. 

  304. 

  305.         movdqa     xmm6,xmm2            ; transpose coefficients(phase 3)
    306. 

  306.         punpcklqdq xmm2,xmm7            ; xmm2=(20 21 22 23 24 25 26 27)=data2
    307. 

  307.         punpckhqdq xmm6,xmm7            ; xmm6=(30 31 32 33 34 35 36 37)=data3
    308. 

  308.         movdqa     xmm1,xmm4            ; transpose coefficients(phase 3)
    309. 

  309.         punpcklqdq xmm4,xmm0            ; xmm4=(40 41 42 43 44 45 46 47)=data4
    310. 

  310.         punpckhqdq xmm1,xmm0            ; xmm1=(50 51 52 53 54 55 56 57)=data5
    311. 

  311. 

  312.         movdqa  xmm7,xmm6
    313. 

  313.         movdqa  xmm0,xmm2
    314. 

  314.         paddw   xmm6,xmm4               ; xmm6=data3+data4=tmp3
    315. 

  315.         paddw   xmm2,xmm1               ; xmm2=data2+data5=tmp2
    316. 

  316.         psubw   xmm7,xmm4               ; xmm7=data3-data4=tmp4
    317. 

  317.         psubw   xmm0,xmm1               ; xmm0=data2-data5=tmp5
    318. 

  318. 

  319.         ; -- Even part
    320. 

  320. 

  321.         movdqa  xmm4,xmm3
    322. 

  322.         movdqa  xmm1,xmm5
    323. 

  323.         psubw   xmm3,xmm6               ; xmm3=tmp13
    324. 

  324.         psubw   xmm5,xmm2               ; xmm5=tmp12
    325. 

  325.         paddw   xmm4,xmm6               ; xmm4=tmp10
    326. 

  326.         paddw   xmm1,xmm2               ; xmm1=tmp11
    327. 

  327. 

  328.         paddw   xmm5,xmm3
    329. 

  329.         psllw   xmm5,PRE_MULTIPLY_SCALE_BITS
    330. 

  330.         pmulhw  xmm5,[rel PW_F0707] ; xmm5=z1
    331. 

  331. 

  332.         movdqa  xmm6,xmm4
    333. 

  333.         movdqa  xmm2,xmm3
    334. 

  334.         psubw   xmm4,xmm1               ; xmm4=data4
    335. 

  335.         psubw   xmm3,xmm5               ; xmm3=data6
    336. 

  336.         paddw   xmm6,xmm1               ; xmm6=data0
    337. 

  337.         paddw   xmm2,xmm5               ; xmm2=data2
    338. 

  338. 

  339.         movdqa  XMMWORD [XMMBLOCK(4,0,rdx,SIZEOF_DCTELEM)], xmm4
    340. 

  340.         movdqa  XMMWORD [XMMBLOCK(6,0,rdx,SIZEOF_DCTELEM)], xmm3
    341. 

  341.         movdqa  XMMWORD [XMMBLOCK(0,0,rdx,SIZEOF_DCTELEM)], xmm6
    342. 

  342.         movdqa  XMMWORD [XMMBLOCK(2,0,rdx,SIZEOF_DCTELEM)], xmm2
    343. 

  343. 

  344.         ; -- Odd part
    345. 

  345. 

  346.         movdqa  xmm1, XMMWORD [wk(0)]   ; xmm1=tmp6
    347. 

  347.         movdqa  xmm5, XMMWORD [wk(1)]   ; xmm5=tmp7
    348. 

  348. 

  349.         paddw   xmm7,xmm0               ; xmm7=tmp10
    350. 

  350.         paddw   xmm0,xmm1               ; xmm0=tmp11
    351. 

  351.         paddw   xmm1,xmm5               ; xmm1=tmp12, xmm5=tmp7
    352. 

  352. 

  353.         psllw   xmm7,PRE_MULTIPLY_SCALE_BITS
    354. 

  354.         psllw   xmm1,PRE_MULTIPLY_SCALE_BITS
    355. 

  355. 

  356.         psllw   xmm0,PRE_MULTIPLY_SCALE_BITS
    357. 

  357.         pmulhw  xmm0,[rel PW_F0707] ; xmm0=z3
    358. 

  358. 

  359.         movdqa  xmm4,xmm7               ; xmm4=tmp10
    360. 

  360.         psubw   xmm7,xmm1
    361. 

  361.         pmulhw  xmm7,[rel PW_F0382] ; xmm7=z5
    362. 

  362.         pmulhw  xmm4,[rel PW_F0541] ; xmm4=MULTIPLY(tmp10,FIX_0_541196)
    363. 

  363.         pmulhw  xmm1,[rel PW_F1306] ; xmm1=MULTIPLY(tmp12,FIX_1_306562)
    364. 

  364.         paddw   xmm4,xmm7               ; xmm4=z2
    365. 

  365.         paddw   xmm1,xmm7               ; xmm1=z4
    366. 

  366. 

  367.         movdqa  xmm3,xmm5
    368. 

  368.         psubw   xmm5,xmm0               ; xmm5=z13
    369. 

  369.         paddw   xmm3,xmm0               ; xmm3=z11
    370. 

  370. 

  371.         movdqa  xmm6,xmm5
    372. 

  372.         movdqa  xmm2,xmm3
    373. 

  373.         psubw   xmm5,xmm4               ; xmm5=data3
    374. 

  374.         psubw   xmm3,xmm1               ; xmm3=data7
    375. 

  375.         paddw   xmm6,xmm4               ; xmm6=data5
    376. 

  376.         paddw   xmm2,xmm1               ; xmm2=data1
    377. 

  377. 

  378.         movdqa  XMMWORD [XMMBLOCK(3,0,rdx,SIZEOF_DCTELEM)], xmm5
    379. 

  379.         movdqa  XMMWORD [XMMBLOCK(7,0,rdx,SIZEOF_DCTELEM)], xmm3
    380. 

  380.         movdqa  XMMWORD [XMMBLOCK(5,0,rdx,SIZEOF_DCTELEM)], xmm6
    381. 

  381.         movdqa  XMMWORD [XMMBLOCK(1,0,rdx,SIZEOF_DCTELEM)], xmm2
    382. 

  382. 

  383.         uncollect_args
    384. 

  384.         mov     rsp,rbp         ; rsp <- aligned rbp
    385. 

  385.         pop     rsp             ; rsp <- original rbp
    386. 

  386.         pop     rbp
    387. 

  387.         ret
    388. 

  388. 

  389. ; For some reason, the OS X linker does not honor the request to align the
    390. 

  390. ; segment unless we do this.
    391. 

  391.         align   16
    392. 

  392.