output.asm 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416
  1. ;******************************************************************************
  2. ;* x86-optimized vertical line scaling functions
  3. ;* Copyright (c) 2011 Ronald S. Bultje <rsbultje@gmail.com>
  4. ;* Kieran Kunhya <kieran@kunhya.com>
  5. ;*
  6. ;* This file is part of Libav.
  7. ;*
  8. ;* Libav is free software; you can redistribute it and/or
  9. ;* modify it under the terms of the GNU Lesser General Public
  10. ;* License as published by the Free Software Foundation; either
  11. ;* version 2.1 of the License, or (at your option) any later version.
  12. ;*
  13. ;* Libav is distributed in the hope that it will be useful,
  14. ;* but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16. ;* Lesser General Public License for more details.
  17. ;*
  18. ;* You should have received a copy of the GNU Lesser General Public
  19. ;* License along with Libav; if not, write to the Free Software
  20. ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21. ;******************************************************************************
  22. %include "x86inc.asm"
  23. %include "x86util.asm"
  24. SECTION_RODATA
  25. minshort: times 8 dw 0x8000
  26. yuv2yuvX_16_start: times 4 dd 0x4000 - 0x40000000
  27. yuv2yuvX_10_start: times 4 dd 0x10000
  28. yuv2yuvX_9_start: times 4 dd 0x20000
  29. yuv2yuvX_10_upper: times 8 dw 0x3ff
  30. yuv2yuvX_9_upper: times 8 dw 0x1ff
  31. pd_4: times 4 dd 4
  32. pd_4min0x40000:times 4 dd 4 - (0x40000)
  33. pw_16: times 8 dw 16
  34. pw_32: times 8 dw 32
  35. pw_512: times 8 dw 512
  36. pw_1024: times 8 dw 1024
  37. SECTION .text
  38. ;-----------------------------------------------------------------------------
  39. ; vertical line scaling
  40. ;
  41. ; void yuv2plane1_<output_size>_<opt>(const int16_t *src, uint8_t *dst, int dstW,
  42. ; const uint8_t *dither, int offset)
  43. ; and
  44. ; void yuv2planeX_<output_size>_<opt>(const int16_t *filter, int filterSize,
  45. ; const int16_t **src, uint8_t *dst, int dstW,
  46. ; const uint8_t *dither, int offset)
  47. ;
  48. ; Scale one or $filterSize lines of source data to generate one line of output
  49. ; data. The input is 15-bit in int16_t if $output_size is [8,10] and 19-bit in
  50. ; int32_t if $output_size is 16. $filter is 12-bits. $filterSize is a multiple
  51. ; of 2. $offset is either 0 or 3. $dither holds 8 values.
  52. ;-----------------------------------------------------------------------------
  53. %macro yuv2planeX_fn 3
  54. %if ARCH_X86_32
  55. %define cntr_reg fltsizeq
  56. %define movsx mov
  57. %else
  58. %define cntr_reg r7
  59. %define movsx movsxd
  60. %endif
  61. cglobal yuv2planeX_%1, %3, 8, %2, filter, fltsize, src, dst, w, dither, offset
  62. %if %1 == 8 || %1 == 9 || %1 == 10
  63. pxor m6, m6
  64. %endif ; %1 == 8/9/10
  65. %if %1 == 8
  66. %if ARCH_X86_32
  67. %assign pad 0x2c - (stack_offset & 15)
  68. SUB rsp, pad
  69. %define m_dith m7
  70. %else ; x86-64
  71. %define m_dith m9
  72. %endif ; x86-32
  73. ; create registers holding dither
  74. movq m_dith, [ditherq] ; dither
  75. test offsetd, offsetd
  76. jz .no_rot
  77. %if mmsize == 16
  78. punpcklqdq m_dith, m_dith
  79. %endif ; mmsize == 16
  80. PALIGNR m_dith, m_dith, 3, m0
  81. .no_rot:
  82. %if mmsize == 16
  83. punpcklbw m_dith, m6
  84. %if ARCH_X86_64
  85. punpcklwd m8, m_dith, m6
  86. pslld m8, 12
  87. %else ; x86-32
  88. punpcklwd m5, m_dith, m6
  89. pslld m5, 12
  90. %endif ; x86-32/64
  91. punpckhwd m_dith, m6
  92. pslld m_dith, 12
  93. %if ARCH_X86_32
  94. mova [rsp+ 0], m5
  95. mova [rsp+16], m_dith
  96. %endif
  97. %else ; mmsize == 8
  98. punpcklbw m5, m_dith, m6
  99. punpckhbw m_dith, m6
  100. punpcklwd m4, m5, m6
  101. punpckhwd m5, m6
  102. punpcklwd m3, m_dith, m6
  103. punpckhwd m_dith, m6
  104. pslld m4, 12
  105. pslld m5, 12
  106. pslld m3, 12
  107. pslld m_dith, 12
  108. mova [rsp+ 0], m4
  109. mova [rsp+ 8], m5
  110. mova [rsp+16], m3
  111. mova [rsp+24], m_dith
  112. %endif ; mmsize == 8/16
  113. %endif ; %1 == 8
  114. xor r5, r5
  115. .pixelloop:
  116. %assign %%i 0
  117. ; the rep here is for the 8bit output mmx case, where dither covers
  118. ; 8 pixels but we can only handle 2 pixels per register, and thus 4
  119. ; pixels per iteration. In order to not have to keep track of where
  120. ; we are w.r.t. dithering, we unroll the mmx/8bit loop x2.
  121. %if %1 == 8
  122. %assign %%repcnt 16/mmsize
  123. %else
  124. %assign %%repcnt 1
  125. %endif
  126. %rep %%repcnt
  127. %if %1 == 8
  128. %if ARCH_X86_32
  129. mova m2, [rsp+mmsize*(0+%%i)]
  130. mova m1, [rsp+mmsize*(1+%%i)]
  131. %else ; x86-64
  132. mova m2, m8
  133. mova m1, m_dith
  134. %endif ; x86-32/64
  135. %else ; %1 == 9/10/16
  136. mova m1, [yuv2yuvX_%1_start]
  137. mova m2, m1
  138. %endif ; %1 == 8/9/10/16
  139. movsx cntr_reg, fltsizem
  140. .filterloop_ %+ %%i:
  141. ; input pixels
  142. mov r6, [srcq+gprsize*cntr_reg-2*gprsize]
  143. %if %1 == 16
  144. mova m3, [r6+r5*4]
  145. mova m5, [r6+r5*4+mmsize]
  146. %else ; %1 == 8/9/10
  147. mova m3, [r6+r5*2]
  148. %endif ; %1 == 8/9/10/16
  149. mov r6, [srcq+gprsize*cntr_reg-gprsize]
  150. %if %1 == 16
  151. mova m4, [r6+r5*4]
  152. mova m6, [r6+r5*4+mmsize]
  153. %else ; %1 == 8/9/10
  154. mova m4, [r6+r5*2]
  155. %endif ; %1 == 8/9/10/16
  156. ; coefficients
  157. movd m0, [filterq+2*cntr_reg-4] ; coeff[0], coeff[1]
  158. %if %1 == 16
  159. pshuflw m7, m0, 0 ; coeff[0]
  160. pshuflw m0, m0, 0x55 ; coeff[1]
  161. pmovsxwd m7, m7 ; word -> dword
  162. pmovsxwd m0, m0 ; word -> dword
  163. pmulld m3, m7
  164. pmulld m5, m7
  165. pmulld m4, m0
  166. pmulld m6, m0
  167. paddd m2, m3
  168. paddd m1, m5
  169. paddd m2, m4
  170. paddd m1, m6
  171. %else ; %1 == 10/9/8
  172. punpcklwd m5, m3, m4
  173. punpckhwd m3, m4
  174. SPLATD m0, m0
  175. pmaddwd m5, m0
  176. pmaddwd m3, m0
  177. paddd m2, m5
  178. paddd m1, m3
  179. %endif ; %1 == 8/9/10/16
  180. sub cntr_reg, 2
  181. jg .filterloop_ %+ %%i
  182. %if %1 == 16
  183. psrad m2, 31 - %1
  184. psrad m1, 31 - %1
  185. %else ; %1 == 10/9/8
  186. psrad m2, 27 - %1
  187. psrad m1, 27 - %1
  188. %endif ; %1 == 8/9/10/16
  189. %if %1 == 8
  190. packssdw m2, m1
  191. packuswb m2, m2
  192. movh [dstq+r5*1], m2
  193. %else ; %1 == 9/10/16
  194. %if %1 == 16
  195. packssdw m2, m1
  196. paddw m2, [minshort]
  197. %else ; %1 == 9/10
  198. %if cpuflag(sse4)
  199. packusdw m2, m1
  200. %else ; mmx2/sse2
  201. packssdw m2, m1
  202. pmaxsw m2, m6
  203. %endif ; mmx2/sse2/sse4/avx
  204. pminsw m2, [yuv2yuvX_%1_upper]
  205. %endif ; %1 == 9/10/16
  206. mova [dstq+r5*2], m2
  207. %endif ; %1 == 8/9/10/16
  208. add r5, mmsize/2
  209. sub wd, mmsize/2
  210. %assign %%i %%i+2
  211. %endrep
  212. jg .pixelloop
  213. %if %1 == 8
  214. %if ARCH_X86_32
  215. ADD rsp, pad
  216. RET
  217. %else ; x86-64
  218. REP_RET
  219. %endif ; x86-32/64
  220. %else ; %1 == 9/10/16
  221. REP_RET
  222. %endif ; %1 == 8/9/10/16
  223. %endmacro
  224. %define PALIGNR PALIGNR_MMX
  225. %if ARCH_X86_32
  226. INIT_MMX mmx2
  227. yuv2planeX_fn 8, 0, 7
  228. yuv2planeX_fn 9, 0, 5
  229. yuv2planeX_fn 10, 0, 5
  230. %endif
  231. INIT_XMM sse2
  232. yuv2planeX_fn 8, 10, 7
  233. yuv2planeX_fn 9, 7, 5
  234. yuv2planeX_fn 10, 7, 5
  235. %define PALIGNR PALIGNR_SSSE3
  236. INIT_XMM sse4
  237. yuv2planeX_fn 8, 10, 7
  238. yuv2planeX_fn 9, 7, 5
  239. yuv2planeX_fn 10, 7, 5
  240. yuv2planeX_fn 16, 8, 5
  241. %if HAVE_AVX
  242. INIT_XMM avx
  243. yuv2planeX_fn 8, 10, 7
  244. yuv2planeX_fn 9, 7, 5
  245. yuv2planeX_fn 10, 7, 5
  246. %endif
  247. ; %1=outout-bpc, %2=alignment (u/a)
  248. %macro yuv2plane1_mainloop 2
  249. .loop_%2:
  250. %if %1 == 8
  251. paddsw m0, m2, [srcq+wq*2+mmsize*0]
  252. paddsw m1, m3, [srcq+wq*2+mmsize*1]
  253. psraw m0, 7
  254. psraw m1, 7
  255. packuswb m0, m1
  256. mov%2 [dstq+wq], m0
  257. %elif %1 == 16
  258. paddd m0, m4, [srcq+wq*4+mmsize*0]
  259. paddd m1, m4, [srcq+wq*4+mmsize*1]
  260. paddd m2, m4, [srcq+wq*4+mmsize*2]
  261. paddd m3, m4, [srcq+wq*4+mmsize*3]
  262. psrad m0, 3
  263. psrad m1, 3
  264. psrad m2, 3
  265. psrad m3, 3
  266. %if cpuflag(sse4) ; avx/sse4
  267. packusdw m0, m1
  268. packusdw m2, m3
  269. %else ; mmx/sse2
  270. packssdw m0, m1
  271. packssdw m2, m3
  272. paddw m0, m5
  273. paddw m2, m5
  274. %endif ; mmx/sse2/sse4/avx
  275. mov%2 [dstq+wq*2+mmsize*0], m0
  276. mov%2 [dstq+wq*2+mmsize*1], m2
  277. %else ; %1 == 9/10
  278. paddsw m0, m2, [srcq+wq*2+mmsize*0]
  279. paddsw m1, m2, [srcq+wq*2+mmsize*1]
  280. psraw m0, 15 - %1
  281. psraw m1, 15 - %1
  282. pmaxsw m0, m4
  283. pmaxsw m1, m4
  284. pminsw m0, m3
  285. pminsw m1, m3
  286. mov%2 [dstq+wq*2+mmsize*0], m0
  287. mov%2 [dstq+wq*2+mmsize*1], m1
  288. %endif
  289. add wq, mmsize
  290. jl .loop_%2
  291. %endmacro
  292. %macro yuv2plane1_fn 3
  293. cglobal yuv2plane1_%1, %3, %3, %2, src, dst, w, dither, offset
  294. movsxdifnidn wq, wd
  295. add wq, mmsize - 1
  296. and wq, ~(mmsize - 1)
  297. %if %1 == 8
  298. add dstq, wq
  299. %else ; %1 != 8
  300. lea dstq, [dstq+wq*2]
  301. %endif ; %1 == 8
  302. %if %1 == 16
  303. lea srcq, [srcq+wq*4]
  304. %else ; %1 != 16
  305. lea srcq, [srcq+wq*2]
  306. %endif ; %1 == 16
  307. neg wq
  308. %if %1 == 8
  309. pxor m4, m4 ; zero
  310. ; create registers holding dither
  311. movq m3, [ditherq] ; dither
  312. test offsetd, offsetd
  313. jz .no_rot
  314. %if mmsize == 16
  315. punpcklqdq m3, m3
  316. %endif ; mmsize == 16
  317. PALIGNR_MMX m3, m3, 3, m2
  318. .no_rot:
  319. %if mmsize == 8
  320. mova m2, m3
  321. punpckhbw m3, m4 ; byte->word
  322. punpcklbw m2, m4 ; byte->word
  323. %else
  324. punpcklbw m3, m4
  325. mova m2, m3
  326. %endif
  327. %elif %1 == 9
  328. pxor m4, m4
  329. mova m3, [pw_512]
  330. mova m2, [pw_32]
  331. %elif %1 == 10
  332. pxor m4, m4
  333. mova m3, [pw_1024]
  334. mova m2, [pw_16]
  335. %else ; %1 == 16
  336. %if cpuflag(sse4) ; sse4/avx
  337. mova m4, [pd_4]
  338. %else ; mmx/sse2
  339. mova m4, [pd_4min0x40000]
  340. mova m5, [minshort]
  341. %endif ; mmx/sse2/sse4/avx
  342. %endif ; %1 == ..
  343. ; actual pixel scaling
  344. %if mmsize == 8
  345. yuv2plane1_mainloop %1, a
  346. %else ; mmsize == 16
  347. test dstq, 15
  348. jnz .unaligned
  349. yuv2plane1_mainloop %1, a
  350. REP_RET
  351. .unaligned:
  352. yuv2plane1_mainloop %1, u
  353. %endif ; mmsize == 8/16
  354. REP_RET
  355. %endmacro
  356. %if ARCH_X86_32
  357. INIT_MMX mmx
  358. yuv2plane1_fn 8, 0, 5
  359. yuv2plane1_fn 16, 0, 3
  360. INIT_MMX mmx2
  361. yuv2plane1_fn 9, 0, 3
  362. yuv2plane1_fn 10, 0, 3
  363. %endif
  364. INIT_XMM sse2
  365. yuv2plane1_fn 8, 5, 5
  366. yuv2plane1_fn 9, 5, 3
  367. yuv2plane1_fn 10, 5, 3
  368. yuv2plane1_fn 16, 6, 3
  369. INIT_XMM sse4
  370. yuv2plane1_fn 16, 5, 3
  371. %if HAVE_AVX
  372. INIT_XMM avx
  373. yuv2plane1_fn 8, 5, 5
  374. yuv2plane1_fn 9, 5, 3
  375. yuv2plane1_fn 10, 5, 3
  376. yuv2plane1_fn 16, 5, 3
  377. %endif