gf_5vect_dot_prod_avx2.asm 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320
  1. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  2. ; Copyright(c) 2011-2015 Intel Corporation All rights reserved.
  3. ;
  4. ; Redistribution and use in source and binary forms, with or without
  5. ; modification, are permitted provided that the following conditions
  6. ; are met:
  7. ; * Redistributions of source code must retain the above copyright
  8. ; notice, this list of conditions and the following disclaimer.
  9. ; * Redistributions in binary form must reproduce the above copyright
  10. ; notice, this list of conditions and the following disclaimer in
  11. ; the documentation and/or other materials provided with the
  12. ; distribution.
  13. ; * Neither the name of Intel Corporation nor the names of its
  14. ; contributors may be used to endorse or promote products derived
  15. ; from this software without specific prior written permission.
  16. ;
  17. ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18. ; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19. ; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20. ; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21. ; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22. ; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23. ; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24. ; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25. ; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26. ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27. ; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28. ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29. ;;;
  30. ;;; gf_5vect_dot_prod_avx2(len, vec, *g_tbls, **buffs, **dests);
  31. ;;;
  32. %include "reg_sizes.asm"
  33. %ifidn __OUTPUT_FORMAT__, elf64
  34. %define arg0 rdi
  35. %define arg1 rsi
  36. %define arg2 rdx
  37. %define arg3 rcx
  38. %define arg4 r8
  39. %define arg5 r9
  40. %define tmp r11
  41. %define tmp.w r11d
  42. %define tmp.b r11b
  43. %define tmp2 r10
  44. %define tmp3 r13 ; must be saved and restored
  45. %define tmp4 r12 ; must be saved and restored
  46. %define tmp5 r14 ; must be saved and restored
  47. %define tmp6 r15 ; must be saved and restored
  48. %define return rax
  49. %define PS 8
  50. %define LOG_PS 3
  51. %define func(x) x:
  52. %macro FUNC_SAVE 0
  53. push r12
  54. push r13
  55. push r14
  56. push r15
  57. %endmacro
  58. %macro FUNC_RESTORE 0
  59. pop r15
  60. pop r14
  61. pop r13
  62. pop r12
  63. %endmacro
  64. %endif
  65. %ifidn __OUTPUT_FORMAT__, win64
  66. %define arg0 rcx
  67. %define arg1 rdx
  68. %define arg2 r8
  69. %define arg3 r9
  70. %define arg4 r12 ; must be saved, loaded and restored
  71. %define arg5 r15 ; must be saved and restored
  72. %define tmp r11
  73. %define tmp.w r11d
  74. %define tmp.b r11b
  75. %define tmp2 r10
  76. %define tmp3 r13 ; must be saved and restored
  77. %define tmp4 r14 ; must be saved and restored
  78. %define tmp5 rdi ; must be saved and restored
  79. %define tmp6 rsi ; must be saved and restored
  80. %define return rax
  81. %define PS 8
  82. %define LOG_PS 3
  83. %define stack_size 10*16 + 7*8 ; must be an odd multiple of 8
  84. %define arg(x) [rsp + stack_size + PS + PS*x]
  85. %define func(x) proc_frame x
  86. %macro FUNC_SAVE 0
  87. alloc_stack stack_size
  88. vmovdqa [rsp + 0*16], xmm6
  89. vmovdqa [rsp + 1*16], xmm7
  90. vmovdqa [rsp + 2*16], xmm8
  91. vmovdqa [rsp + 3*16], xmm9
  92. vmovdqa [rsp + 4*16], xmm10
  93. vmovdqa [rsp + 5*16], xmm11
  94. vmovdqa [rsp + 6*16], xmm12
  95. vmovdqa [rsp + 7*16], xmm13
  96. vmovdqa [rsp + 8*16], xmm14
  97. vmovdqa [rsp + 9*16], xmm15
  98. save_reg r12, 10*16 + 0*8
  99. save_reg r13, 10*16 + 1*8
  100. save_reg r14, 10*16 + 2*8
  101. save_reg r15, 10*16 + 3*8
  102. save_reg rdi, 10*16 + 4*8
  103. save_reg rsi, 10*16 + 5*8
  104. end_prolog
  105. mov arg4, arg(4)
  106. %endmacro
  107. %macro FUNC_RESTORE 0
  108. vmovdqa xmm6, [rsp + 0*16]
  109. vmovdqa xmm7, [rsp + 1*16]
  110. vmovdqa xmm8, [rsp + 2*16]
  111. vmovdqa xmm9, [rsp + 3*16]
  112. vmovdqa xmm10, [rsp + 4*16]
  113. vmovdqa xmm11, [rsp + 5*16]
  114. vmovdqa xmm12, [rsp + 6*16]
  115. vmovdqa xmm13, [rsp + 7*16]
  116. vmovdqa xmm14, [rsp + 8*16]
  117. vmovdqa xmm15, [rsp + 9*16]
  118. mov r12, [rsp + 10*16 + 0*8]
  119. mov r13, [rsp + 10*16 + 1*8]
  120. mov r14, [rsp + 10*16 + 2*8]
  121. mov r15, [rsp + 10*16 + 3*8]
  122. mov rdi, [rsp + 10*16 + 4*8]
  123. mov rsi, [rsp + 10*16 + 5*8]
  124. add rsp, stack_size
  125. %endmacro
  126. %endif
  127. %define len arg0
  128. %define vec arg1
  129. %define mul_array arg2
  130. %define src arg3
  131. %define dest arg4
  132. %define ptr arg5
  133. %define vec_i tmp2
  134. %define dest1 tmp3
  135. %define dest2 tmp4
  136. %define vskip1 tmp5
  137. %define vskip3 tmp6
  138. %define pos return
  139. %ifndef EC_ALIGNED_ADDR
  140. ;;; Use Un-aligned load/store
  141. %define XLDR vmovdqu
  142. %define XSTR vmovdqu
  143. %else
  144. ;;; Use Non-temporal load/stor
  145. %ifdef NO_NT_LDST
  146. %define XLDR vmovdqa
  147. %define XSTR vmovdqa
  148. %else
  149. %define XLDR vmovntdqa
  150. %define XSTR vmovntdq
  151. %endif
  152. %endif
  153. default rel
  154. [bits 64]
  155. section .text
  156. %define xmask0f ymm15
  157. %define xmask0fx xmm15
  158. %define xgft1_lo ymm14
  159. %define xgft1_hi ymm13
  160. %define xgft2_lo ymm12
  161. %define xgft2_hi ymm11
  162. %define xgft3_lo ymm10
  163. %define xgft3_hi ymm9
  164. %define xgft4_lo ymm8
  165. %define xgft4_hi ymm7
  166. %define x0 ymm0
  167. %define xtmpa ymm1
  168. %define xp1 ymm2
  169. %define xp2 ymm3
  170. %define xp3 ymm4
  171. %define xp4 ymm5
  172. %define xp5 ymm6
  173. align 16
  174. global gf_5vect_dot_prod_avx2:ISAL_SYM_TYPE_FUNCTION
  175. func(gf_5vect_dot_prod_avx2)
  176. %ifidn __OUTPUT_FORMAT__, macho64
  177. global _gf_5vect_dot_prod_avx2:ISAL_SYM_TYPE_FUNCTION
  178. func(_gf_5vect_dot_prod_avx2)
  179. %endif
  180. FUNC_SAVE
  181. sub len, 32
  182. jl .return_fail
  183. xor pos, pos
  184. mov tmp.b, 0x0f
  185. vpinsrb xmask0fx, xmask0fx, tmp.w, 0
  186. vpbroadcastb xmask0f, xmask0fx ;Construct mask 0x0f0f0f...
  187. mov vskip1, vec
  188. imul vskip1, 32
  189. mov vskip3, vec
  190. imul vskip3, 96
  191. sal vec, LOG_PS ;vec *= PS. Make vec_i count by PS
  192. mov dest1, [dest]
  193. mov dest2, [dest+PS]
  194. .loop32:
  195. mov tmp, mul_array
  196. xor vec_i, vec_i
  197. vpxor xp1, xp1
  198. vpxor xp2, xp2
  199. vpxor xp3, xp3
  200. vpxor xp4, xp4
  201. vpxor xp5, xp5
  202. .next_vect:
  203. mov ptr, [src+vec_i]
  204. XLDR x0, [ptr+pos] ;Get next source vector
  205. add vec_i, PS
  206. vpand xgft4_lo, x0, xmask0f ;Mask low src nibble in bits 4-0
  207. vpsraw x0, x0, 4 ;Shift to put high nibble into bits 4-0
  208. vpand x0, x0, xmask0f ;Mask high src nibble in bits 4-0
  209. vperm2i128 xtmpa, xgft4_lo, x0, 0x30 ;swap xtmpa from 1lo|2lo to 1lo|2hi
  210. vperm2i128 x0, xgft4_lo, x0, 0x12 ;swap x0 from 1hi|2hi to 1hi|2lo
  211. vmovdqu xgft1_lo, [tmp] ;Load array Ax{00}, Ax{01}, ..., Ax{0f}
  212. ; " Ax{00}, Ax{10}, ..., Ax{f0}
  213. vmovdqu xgft2_lo, [tmp+vskip1*1] ;Load array Bx{00}, Bx{01}, ..., Bx{0f}
  214. ; " Bx{00}, Bx{10}, ..., Bx{f0}
  215. vmovdqu xgft3_lo, [tmp+vskip1*2] ;Load array Cx{00}, Cx{01}, ..., Cx{0f}
  216. ; " Cx{00}, Cx{10}, ..., Cx{f0}
  217. vmovdqu xgft4_lo, [tmp+vskip3] ;Load array Dx{00}, Dx{01}, ..., Dx{0f}
  218. ; " Dx{00}, Dx{10}, ..., Dx{f0}
  219. vperm2i128 xgft1_hi, xgft1_lo, xgft1_lo, 0x01 ; swapped to hi | lo
  220. vperm2i128 xgft2_hi, xgft2_lo, xgft2_lo, 0x01 ; swapped to hi | lo
  221. vperm2i128 xgft3_hi, xgft3_lo, xgft3_lo, 0x01 ; swapped to hi | lo
  222. vperm2i128 xgft4_hi, xgft4_lo, xgft4_lo, 0x01 ; swapped to hi | lo
  223. vpshufb xgft1_hi, x0 ;Lookup mul table of high nibble
  224. vpshufb xgft1_lo, xtmpa ;Lookup mul table of low nibble
  225. vpxor xgft1_hi, xgft1_lo ;GF add high and low partials
  226. vpxor xp1, xgft1_hi ;xp1 += partial
  227. vpshufb xgft2_hi, x0 ;Lookup mul table of high nibble
  228. vpshufb xgft2_lo, xtmpa ;Lookup mul table of low nibble
  229. vpxor xgft2_hi, xgft2_lo ;GF add high and low partials
  230. vpxor xp2, xgft2_hi ;xp2 += partial
  231. vmovdqu xgft1_lo, [tmp+vskip1*4] ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
  232. ; " Ex{00}, Ex{10}, ..., Ex{f0}
  233. vperm2i128 xgft1_hi, xgft1_lo, xgft1_lo, 0x01 ; swapped to hi | lo
  234. add tmp, 32
  235. vpshufb xgft3_hi, x0 ;Lookup mul table of high nibble
  236. vpshufb xgft3_lo, xtmpa ;Lookup mul table of low nibble
  237. vpxor xgft3_hi, xgft3_lo ;GF add high and low partials
  238. vpxor xp3, xgft3_hi ;xp3 += partial
  239. vpshufb xgft4_hi, x0 ;Lookup mul table of high nibble
  240. vpshufb xgft4_lo, xtmpa ;Lookup mul table of low nibble
  241. vpxor xgft4_hi, xgft4_lo ;GF add high and low partials
  242. vpxor xp4, xgft4_hi ;xp4 += partial
  243. vpshufb xgft1_hi, x0 ;Lookup mul table of high nibble
  244. vpshufb xgft1_lo, xtmpa ;Lookup mul table of low nibble
  245. vpxor xgft1_hi, xgft1_lo ;GF add high and low partials
  246. vpxor xp5, xgft1_hi ;xp5 += partial
  247. cmp vec_i, vec
  248. jl .next_vect
  249. mov tmp, [dest+2*PS]
  250. mov ptr, [dest+3*PS]
  251. mov vec_i, [dest+4*PS]
  252. XSTR [dest1+pos], xp1
  253. XSTR [dest2+pos], xp2
  254. XSTR [tmp+pos], xp3
  255. XSTR [ptr+pos], xp4
  256. XSTR [vec_i+pos], xp5
  257. add pos, 32 ;Loop on 32 bytes at a time
  258. cmp pos, len
  259. jle .loop32
  260. lea tmp, [len + 32]
  261. cmp pos, tmp
  262. je .return_pass
  263. ;; Tail len
  264. mov pos, len ;Overlapped offset length-16
  265. jmp .loop32 ;Do one more overlap pass
  266. .return_pass:
  267. FUNC_RESTORE
  268. mov return, 0
  269. ret
  270. .return_fail:
  271. FUNC_RESTORE
  272. mov return, 1
  273. ret
  274. endproc_frame
  275. section .data
  276. ;;; func core, ver, snum
  277. slversion gf_5vect_dot_prod_avx2, 04, 04, 0199