ghashv8-armx.S 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552
  1. #include "arm_arch.h"
  2. #if __ARM_MAX_ARCH__>=7
  3. .text
  4. .globl _gcm_init_v8
  5. .align 4
  6. _gcm_init_v8:
  7. ld1 {v17.2d},[x1] //load input H
  8. movi v19.16b,#0xe1
  9. shl v19.2d,v19.2d,#57 //0xc2.0
  10. ext v3.16b,v17.16b,v17.16b,#8
  11. ushr v18.2d,v19.2d,#63
  12. dup v17.4s,v17.s[1]
  13. ext v16.16b,v18.16b,v19.16b,#8 //t0=0xc2....01
  14. ushr v18.2d,v3.2d,#63
  15. sshr v17.4s,v17.4s,#31 //broadcast carry bit
  16. and v18.16b,v18.16b,v16.16b
  17. shl v3.2d,v3.2d,#1
  18. ext v18.16b,v18.16b,v18.16b,#8
  19. and v16.16b,v16.16b,v17.16b
  20. orr v3.16b,v3.16b,v18.16b //H<<<=1
  21. eor v20.16b,v3.16b,v16.16b //twisted H
  22. st1 {v20.2d},[x0],#16 //store Htable[0]
  23. //calculate H^2
  24. ext v16.16b,v20.16b,v20.16b,#8 //Karatsuba pre-processing
  25. pmull v0.1q,v20.1d,v20.1d
  26. eor v16.16b,v16.16b,v20.16b
  27. pmull2 v2.1q,v20.2d,v20.2d
  28. pmull v1.1q,v16.1d,v16.1d
  29. ext v17.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing
  30. eor v18.16b,v0.16b,v2.16b
  31. eor v1.16b,v1.16b,v17.16b
  32. eor v1.16b,v1.16b,v18.16b
  33. pmull v18.1q,v0.1d,v19.1d //1st phase
  34. ins v2.d[0],v1.d[1]
  35. ins v1.d[1],v0.d[0]
  36. eor v0.16b,v1.16b,v18.16b
  37. ext v18.16b,v0.16b,v0.16b,#8 //2nd phase
  38. pmull v0.1q,v0.1d,v19.1d
  39. eor v18.16b,v18.16b,v2.16b
  40. eor v22.16b,v0.16b,v18.16b
  41. ext v17.16b,v22.16b,v22.16b,#8 //Karatsuba pre-processing
  42. eor v17.16b,v17.16b,v22.16b
  43. ext v21.16b,v16.16b,v17.16b,#8 //pack Karatsuba pre-processed
  44. st1 {v21.2d,v22.2d},[x0],#32 //store Htable[1..2]
  45. //calculate H^3 and H^4
  46. pmull v0.1q,v20.1d, v22.1d
  47. pmull v5.1q,v22.1d,v22.1d
  48. pmull2 v2.1q,v20.2d, v22.2d
  49. pmull2 v7.1q,v22.2d,v22.2d
  50. pmull v1.1q,v16.1d,v17.1d
  51. pmull v6.1q,v17.1d,v17.1d
  52. ext v16.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing
  53. ext v17.16b,v5.16b,v7.16b,#8
  54. eor v18.16b,v0.16b,v2.16b
  55. eor v1.16b,v1.16b,v16.16b
  56. eor v4.16b,v5.16b,v7.16b
  57. eor v6.16b,v6.16b,v17.16b
  58. eor v1.16b,v1.16b,v18.16b
  59. pmull v18.1q,v0.1d,v19.1d //1st phase
  60. eor v6.16b,v6.16b,v4.16b
  61. pmull v4.1q,v5.1d,v19.1d
  62. ins v2.d[0],v1.d[1]
  63. ins v7.d[0],v6.d[1]
  64. ins v1.d[1],v0.d[0]
  65. ins v6.d[1],v5.d[0]
  66. eor v0.16b,v1.16b,v18.16b
  67. eor v5.16b,v6.16b,v4.16b
  68. ext v18.16b,v0.16b,v0.16b,#8 //2nd phase
  69. ext v4.16b,v5.16b,v5.16b,#8
  70. pmull v0.1q,v0.1d,v19.1d
  71. pmull v5.1q,v5.1d,v19.1d
  72. eor v18.16b,v18.16b,v2.16b
  73. eor v4.16b,v4.16b,v7.16b
  74. eor v20.16b, v0.16b,v18.16b //H^3
  75. eor v22.16b,v5.16b,v4.16b //H^4
  76. ext v16.16b,v20.16b, v20.16b,#8 //Karatsuba pre-processing
  77. ext v17.16b,v22.16b,v22.16b,#8
  78. eor v16.16b,v16.16b,v20.16b
  79. eor v17.16b,v17.16b,v22.16b
  80. ext v21.16b,v16.16b,v17.16b,#8 //pack Karatsuba pre-processed
  81. st1 {v20.2d,v21.2d,v22.2d},[x0] //store Htable[3..5]
  82. ret
  83. .globl _gcm_gmult_v8
  84. .align 4
  85. _gcm_gmult_v8:
  86. ld1 {v17.2d},[x0] //load Xi
  87. movi v19.16b,#0xe1
  88. ld1 {v20.2d,v21.2d},[x1] //load twisted H, ...
  89. shl v19.2d,v19.2d,#57
  90. #ifndef __ARMEB__
  91. rev64 v17.16b,v17.16b
  92. #endif
  93. ext v3.16b,v17.16b,v17.16b,#8
  94. pmull v0.1q,v20.1d,v3.1d //H.lo·Xi.lo
  95. eor v17.16b,v17.16b,v3.16b //Karatsuba pre-processing
  96. pmull2 v2.1q,v20.2d,v3.2d //H.hi·Xi.hi
  97. pmull v1.1q,v21.1d,v17.1d //(H.lo+H.hi)·(Xi.lo+Xi.hi)
  98. ext v17.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing
  99. eor v18.16b,v0.16b,v2.16b
  100. eor v1.16b,v1.16b,v17.16b
  101. eor v1.16b,v1.16b,v18.16b
  102. pmull v18.1q,v0.1d,v19.1d //1st phase of reduction
  103. ins v2.d[0],v1.d[1]
  104. ins v1.d[1],v0.d[0]
  105. eor v0.16b,v1.16b,v18.16b
  106. ext v18.16b,v0.16b,v0.16b,#8 //2nd phase of reduction
  107. pmull v0.1q,v0.1d,v19.1d
  108. eor v18.16b,v18.16b,v2.16b
  109. eor v0.16b,v0.16b,v18.16b
  110. #ifndef __ARMEB__
  111. rev64 v0.16b,v0.16b
  112. #endif
  113. ext v0.16b,v0.16b,v0.16b,#8
  114. st1 {v0.2d},[x0] //write out Xi
  115. ret
  116. .globl _gcm_ghash_v8
  117. .align 4
  118. _gcm_ghash_v8:
  119. cmp x3,#64
  120. b.hs Lgcm_ghash_v8_4x
  121. ld1 {v0.2d},[x0] //load [rotated] Xi
  122. //"[rotated]" means that
  123. //loaded value would have
  124. //to be rotated in order to
  125. //make it appear as in
  126. //algorithm specification
  127. subs x3,x3,#32 //see if x3 is 32 or larger
  128. mov x12,#16 //x12 is used as post-
  129. //increment for input pointer;
  130. //as loop is modulo-scheduled
  131. //x12 is zeroed just in time
  132. //to preclude overstepping
  133. //inp[len], which means that
  134. //last block[s] are actually
  135. //loaded twice, but last
  136. //copy is not processed
  137. ld1 {v20.2d,v21.2d},[x1],#32 //load twisted H, ..., H^2
  138. movi v19.16b,#0xe1
  139. ld1 {v22.2d},[x1]
  140. csel x12,xzr,x12,eq //is it time to zero x12?
  141. ext v0.16b,v0.16b,v0.16b,#8 //rotate Xi
  142. ld1 {v16.2d},[x2],#16 //load [rotated] I[0]
  143. shl v19.2d,v19.2d,#57 //compose 0xc2.0 constant
  144. #ifndef __ARMEB__
  145. rev64 v16.16b,v16.16b
  146. rev64 v0.16b,v0.16b
  147. #endif
  148. ext v3.16b,v16.16b,v16.16b,#8 //rotate I[0]
  149. b.lo Lodd_tail_v8 //x3 was less than 32
  150. ld1 {v17.2d},[x2],x12 //load [rotated] I[1]
  151. #ifndef __ARMEB__
  152. rev64 v17.16b,v17.16b
  153. #endif
  154. ext v7.16b,v17.16b,v17.16b,#8
  155. eor v3.16b,v3.16b,v0.16b //I[i]^=Xi
  156. pmull v4.1q,v20.1d,v7.1d //H·Ii+1
  157. eor v17.16b,v17.16b,v7.16b //Karatsuba pre-processing
  158. pmull2 v6.1q,v20.2d,v7.2d
  159. b Loop_mod2x_v8
  160. .align 4
  161. Loop_mod2x_v8:
  162. ext v18.16b,v3.16b,v3.16b,#8
  163. subs x3,x3,#32 //is there more data?
  164. pmull v0.1q,v22.1d,v3.1d //H^2.lo·Xi.lo
  165. csel x12,xzr,x12,lo //is it time to zero x12?
  166. pmull v5.1q,v21.1d,v17.1d
  167. eor v18.16b,v18.16b,v3.16b //Karatsuba pre-processing
  168. pmull2 v2.1q,v22.2d,v3.2d //H^2.hi·Xi.hi
  169. eor v0.16b,v0.16b,v4.16b //accumulate
  170. pmull2 v1.1q,v21.2d,v18.2d //(H^2.lo+H^2.hi)·(Xi.lo+Xi.hi)
  171. ld1 {v16.2d},[x2],x12 //load [rotated] I[i+2]
  172. eor v2.16b,v2.16b,v6.16b
  173. csel x12,xzr,x12,eq //is it time to zero x12?
  174. eor v1.16b,v1.16b,v5.16b
  175. ext v17.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing
  176. eor v18.16b,v0.16b,v2.16b
  177. eor v1.16b,v1.16b,v17.16b
  178. ld1 {v17.2d},[x2],x12 //load [rotated] I[i+3]
  179. #ifndef __ARMEB__
  180. rev64 v16.16b,v16.16b
  181. #endif
  182. eor v1.16b,v1.16b,v18.16b
  183. pmull v18.1q,v0.1d,v19.1d //1st phase of reduction
  184. #ifndef __ARMEB__
  185. rev64 v17.16b,v17.16b
  186. #endif
  187. ins v2.d[0],v1.d[1]
  188. ins v1.d[1],v0.d[0]
  189. ext v7.16b,v17.16b,v17.16b,#8
  190. ext v3.16b,v16.16b,v16.16b,#8
  191. eor v0.16b,v1.16b,v18.16b
  192. pmull v4.1q,v20.1d,v7.1d //H·Ii+1
  193. eor v3.16b,v3.16b,v2.16b //accumulate v3.16b early
  194. ext v18.16b,v0.16b,v0.16b,#8 //2nd phase of reduction
  195. pmull v0.1q,v0.1d,v19.1d
  196. eor v3.16b,v3.16b,v18.16b
  197. eor v17.16b,v17.16b,v7.16b //Karatsuba pre-processing
  198. eor v3.16b,v3.16b,v0.16b
  199. pmull2 v6.1q,v20.2d,v7.2d
  200. b.hs Loop_mod2x_v8 //there was at least 32 more bytes
  201. eor v2.16b,v2.16b,v18.16b
  202. ext v3.16b,v16.16b,v16.16b,#8 //re-construct v3.16b
  203. adds x3,x3,#32 //re-construct x3
  204. eor v0.16b,v0.16b,v2.16b //re-construct v0.16b
  205. b.eq Ldone_v8 //is x3 zero?
  206. Lodd_tail_v8:
  207. ext v18.16b,v0.16b,v0.16b,#8
  208. eor v3.16b,v3.16b,v0.16b //inp^=Xi
  209. eor v17.16b,v16.16b,v18.16b //v17.16b is rotated inp^Xi
  210. pmull v0.1q,v20.1d,v3.1d //H.lo·Xi.lo
  211. eor v17.16b,v17.16b,v3.16b //Karatsuba pre-processing
  212. pmull2 v2.1q,v20.2d,v3.2d //H.hi·Xi.hi
  213. pmull v1.1q,v21.1d,v17.1d //(H.lo+H.hi)·(Xi.lo+Xi.hi)
  214. ext v17.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing
  215. eor v18.16b,v0.16b,v2.16b
  216. eor v1.16b,v1.16b,v17.16b
  217. eor v1.16b,v1.16b,v18.16b
  218. pmull v18.1q,v0.1d,v19.1d //1st phase of reduction
  219. ins v2.d[0],v1.d[1]
  220. ins v1.d[1],v0.d[0]
  221. eor v0.16b,v1.16b,v18.16b
  222. ext v18.16b,v0.16b,v0.16b,#8 //2nd phase of reduction
  223. pmull v0.1q,v0.1d,v19.1d
  224. eor v18.16b,v18.16b,v2.16b
  225. eor v0.16b,v0.16b,v18.16b
  226. Ldone_v8:
  227. #ifndef __ARMEB__
  228. rev64 v0.16b,v0.16b
  229. #endif
  230. ext v0.16b,v0.16b,v0.16b,#8
  231. st1 {v0.2d},[x0] //write out Xi
  232. ret
  233. .align 4
  234. gcm_ghash_v8_4x:
  235. Lgcm_ghash_v8_4x:
  236. ld1 {v0.2d},[x0] //load [rotated] Xi
  237. ld1 {v20.2d,v21.2d,v22.2d},[x1],#48 //load twisted H, ..., H^2
  238. movi v19.16b,#0xe1
  239. ld1 {v26.2d,v27.2d,v28.2d},[x1] //load twisted H^3, ..., H^4
  240. shl v19.2d,v19.2d,#57 //compose 0xc2.0 constant
  241. ld1 {v4.2d,v5.2d,v6.2d,v7.2d},[x2],#64
  242. #ifndef __ARMEB__
  243. rev64 v0.16b,v0.16b
  244. rev64 v5.16b,v5.16b
  245. rev64 v6.16b,v6.16b
  246. rev64 v7.16b,v7.16b
  247. rev64 v4.16b,v4.16b
  248. #endif
  249. ext v25.16b,v7.16b,v7.16b,#8
  250. ext v24.16b,v6.16b,v6.16b,#8
  251. ext v23.16b,v5.16b,v5.16b,#8
  252. pmull v29.1q,v20.1d,v25.1d //H·Ii+3
  253. eor v7.16b,v7.16b,v25.16b
  254. pmull2 v31.1q,v20.2d,v25.2d
  255. pmull v30.1q,v21.1d,v7.1d
  256. pmull v16.1q,v22.1d,v24.1d //H^2·Ii+2
  257. eor v6.16b,v6.16b,v24.16b
  258. pmull2 v24.1q,v22.2d,v24.2d
  259. pmull2 v6.1q,v21.2d,v6.2d
  260. eor v29.16b,v29.16b,v16.16b
  261. eor v31.16b,v31.16b,v24.16b
  262. eor v30.16b,v30.16b,v6.16b
  263. pmull v7.1q,v26.1d,v23.1d //H^3·Ii+1
  264. eor v5.16b,v5.16b,v23.16b
  265. pmull2 v23.1q,v26.2d,v23.2d
  266. pmull v5.1q,v27.1d,v5.1d
  267. eor v29.16b,v29.16b,v7.16b
  268. eor v31.16b,v31.16b,v23.16b
  269. eor v30.16b,v30.16b,v5.16b
  270. subs x3,x3,#128
  271. b.lo Ltail4x
  272. b Loop4x
  273. .align 4
  274. Loop4x:
  275. eor v16.16b,v4.16b,v0.16b
  276. ld1 {v4.2d,v5.2d,v6.2d,v7.2d},[x2],#64
  277. ext v3.16b,v16.16b,v16.16b,#8
  278. #ifndef __ARMEB__
  279. rev64 v5.16b,v5.16b
  280. rev64 v6.16b,v6.16b
  281. rev64 v7.16b,v7.16b
  282. rev64 v4.16b,v4.16b
  283. #endif
  284. pmull v0.1q,v28.1d,v3.1d //H^4·(Xi+Ii)
  285. eor v16.16b,v16.16b,v3.16b
  286. pmull2 v2.1q,v28.2d,v3.2d
  287. ext v25.16b,v7.16b,v7.16b,#8
  288. pmull2 v1.1q,v27.2d,v16.2d
  289. eor v0.16b,v0.16b,v29.16b
  290. eor v2.16b,v2.16b,v31.16b
  291. ext v24.16b,v6.16b,v6.16b,#8
  292. eor v1.16b,v1.16b,v30.16b
  293. ext v23.16b,v5.16b,v5.16b,#8
  294. ext v17.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing
  295. eor v18.16b,v0.16b,v2.16b
  296. pmull v29.1q,v20.1d,v25.1d //H·Ii+3
  297. eor v7.16b,v7.16b,v25.16b
  298. eor v1.16b,v1.16b,v17.16b
  299. pmull2 v31.1q,v20.2d,v25.2d
  300. eor v1.16b,v1.16b,v18.16b
  301. pmull v30.1q,v21.1d,v7.1d
  302. pmull v18.1q,v0.1d,v19.1d //1st phase of reduction
  303. ins v2.d[0],v1.d[1]
  304. ins v1.d[1],v0.d[0]
  305. pmull v16.1q,v22.1d,v24.1d //H^2·Ii+2
  306. eor v6.16b,v6.16b,v24.16b
  307. pmull2 v24.1q,v22.2d,v24.2d
  308. eor v0.16b,v1.16b,v18.16b
  309. pmull2 v6.1q,v21.2d,v6.2d
  310. eor v29.16b,v29.16b,v16.16b
  311. eor v31.16b,v31.16b,v24.16b
  312. eor v30.16b,v30.16b,v6.16b
  313. ext v18.16b,v0.16b,v0.16b,#8 //2nd phase of reduction
  314. pmull v0.1q,v0.1d,v19.1d
  315. pmull v7.1q,v26.1d,v23.1d //H^3·Ii+1
  316. eor v5.16b,v5.16b,v23.16b
  317. eor v18.16b,v18.16b,v2.16b
  318. pmull2 v23.1q,v26.2d,v23.2d
  319. pmull v5.1q,v27.1d,v5.1d
  320. eor v0.16b,v0.16b,v18.16b
  321. eor v29.16b,v29.16b,v7.16b
  322. eor v31.16b,v31.16b,v23.16b
  323. ext v0.16b,v0.16b,v0.16b,#8
  324. eor v30.16b,v30.16b,v5.16b
  325. subs x3,x3,#64
  326. b.hs Loop4x
  327. Ltail4x:
  328. eor v16.16b,v4.16b,v0.16b
  329. ext v3.16b,v16.16b,v16.16b,#8
  330. pmull v0.1q,v28.1d,v3.1d //H^4·(Xi+Ii)
  331. eor v16.16b,v16.16b,v3.16b
  332. pmull2 v2.1q,v28.2d,v3.2d
  333. pmull2 v1.1q,v27.2d,v16.2d
  334. eor v0.16b,v0.16b,v29.16b
  335. eor v2.16b,v2.16b,v31.16b
  336. eor v1.16b,v1.16b,v30.16b
  337. adds x3,x3,#64
  338. b.eq Ldone4x
  339. cmp x3,#32
  340. b.lo Lone
  341. b.eq Ltwo
  342. Lthree:
  343. ext v17.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing
  344. eor v18.16b,v0.16b,v2.16b
  345. eor v1.16b,v1.16b,v17.16b
  346. ld1 {v4.2d,v5.2d,v6.2d},[x2]
  347. eor v1.16b,v1.16b,v18.16b
  348. #ifndef __ARMEB__
  349. rev64 v5.16b,v5.16b
  350. rev64 v6.16b,v6.16b
  351. rev64 v4.16b,v4.16b
  352. #endif
  353. pmull v18.1q,v0.1d,v19.1d //1st phase of reduction
  354. ins v2.d[0],v1.d[1]
  355. ins v1.d[1],v0.d[0]
  356. ext v24.16b,v6.16b,v6.16b,#8
  357. ext v23.16b,v5.16b,v5.16b,#8
  358. eor v0.16b,v1.16b,v18.16b
  359. pmull v29.1q,v20.1d,v24.1d //H·Ii+2
  360. eor v6.16b,v6.16b,v24.16b
  361. ext v18.16b,v0.16b,v0.16b,#8 //2nd phase of reduction
  362. pmull v0.1q,v0.1d,v19.1d
  363. eor v18.16b,v18.16b,v2.16b
  364. pmull2 v31.1q,v20.2d,v24.2d
  365. pmull v30.1q,v21.1d,v6.1d
  366. eor v0.16b,v0.16b,v18.16b
  367. pmull v7.1q,v22.1d,v23.1d //H^2·Ii+1
  368. eor v5.16b,v5.16b,v23.16b
  369. ext v0.16b,v0.16b,v0.16b,#8
  370. pmull2 v23.1q,v22.2d,v23.2d
  371. eor v16.16b,v4.16b,v0.16b
  372. pmull2 v5.1q,v21.2d,v5.2d
  373. ext v3.16b,v16.16b,v16.16b,#8
  374. eor v29.16b,v29.16b,v7.16b
  375. eor v31.16b,v31.16b,v23.16b
  376. eor v30.16b,v30.16b,v5.16b
  377. pmull v0.1q,v26.1d,v3.1d //H^3·(Xi+Ii)
  378. eor v16.16b,v16.16b,v3.16b
  379. pmull2 v2.1q,v26.2d,v3.2d
  380. pmull v1.1q,v27.1d,v16.1d
  381. eor v0.16b,v0.16b,v29.16b
  382. eor v2.16b,v2.16b,v31.16b
  383. eor v1.16b,v1.16b,v30.16b
  384. b Ldone4x
  385. .align 4
  386. Ltwo:
  387. ext v17.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing
  388. eor v18.16b,v0.16b,v2.16b
  389. eor v1.16b,v1.16b,v17.16b
  390. ld1 {v4.2d,v5.2d},[x2]
  391. eor v1.16b,v1.16b,v18.16b
  392. #ifndef __ARMEB__
  393. rev64 v5.16b,v5.16b
  394. rev64 v4.16b,v4.16b
  395. #endif
  396. pmull v18.1q,v0.1d,v19.1d //1st phase of reduction
  397. ins v2.d[0],v1.d[1]
  398. ins v1.d[1],v0.d[0]
  399. ext v23.16b,v5.16b,v5.16b,#8
  400. eor v0.16b,v1.16b,v18.16b
  401. ext v18.16b,v0.16b,v0.16b,#8 //2nd phase of reduction
  402. pmull v0.1q,v0.1d,v19.1d
  403. eor v18.16b,v18.16b,v2.16b
  404. eor v0.16b,v0.16b,v18.16b
  405. ext v0.16b,v0.16b,v0.16b,#8
  406. pmull v29.1q,v20.1d,v23.1d //H·Ii+1
  407. eor v5.16b,v5.16b,v23.16b
  408. eor v16.16b,v4.16b,v0.16b
  409. ext v3.16b,v16.16b,v16.16b,#8
  410. pmull2 v31.1q,v20.2d,v23.2d
  411. pmull v30.1q,v21.1d,v5.1d
  412. pmull v0.1q,v22.1d,v3.1d //H^2·(Xi+Ii)
  413. eor v16.16b,v16.16b,v3.16b
  414. pmull2 v2.1q,v22.2d,v3.2d
  415. pmull2 v1.1q,v21.2d,v16.2d
  416. eor v0.16b,v0.16b,v29.16b
  417. eor v2.16b,v2.16b,v31.16b
  418. eor v1.16b,v1.16b,v30.16b
  419. b Ldone4x
  420. .align 4
  421. Lone:
  422. ext v17.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing
  423. eor v18.16b,v0.16b,v2.16b
  424. eor v1.16b,v1.16b,v17.16b
  425. ld1 {v4.2d},[x2]
  426. eor v1.16b,v1.16b,v18.16b
  427. #ifndef __ARMEB__
  428. rev64 v4.16b,v4.16b
  429. #endif
  430. pmull v18.1q,v0.1d,v19.1d //1st phase of reduction
  431. ins v2.d[0],v1.d[1]
  432. ins v1.d[1],v0.d[0]
  433. eor v0.16b,v1.16b,v18.16b
  434. ext v18.16b,v0.16b,v0.16b,#8 //2nd phase of reduction
  435. pmull v0.1q,v0.1d,v19.1d
  436. eor v18.16b,v18.16b,v2.16b
  437. eor v0.16b,v0.16b,v18.16b
  438. ext v0.16b,v0.16b,v0.16b,#8
  439. eor v16.16b,v4.16b,v0.16b
  440. ext v3.16b,v16.16b,v16.16b,#8
  441. pmull v0.1q,v20.1d,v3.1d
  442. eor v16.16b,v16.16b,v3.16b
  443. pmull2 v2.1q,v20.2d,v3.2d
  444. pmull v1.1q,v21.1d,v16.1d
  445. Ldone4x:
  446. ext v17.16b,v0.16b,v2.16b,#8 //Karatsuba post-processing
  447. eor v18.16b,v0.16b,v2.16b
  448. eor v1.16b,v1.16b,v17.16b
  449. eor v1.16b,v1.16b,v18.16b
  450. pmull v18.1q,v0.1d,v19.1d //1st phase of reduction
  451. ins v2.d[0],v1.d[1]
  452. ins v1.d[1],v0.d[0]
  453. eor v0.16b,v1.16b,v18.16b
  454. ext v18.16b,v0.16b,v0.16b,#8 //2nd phase of reduction
  455. pmull v0.1q,v0.1d,v19.1d
  456. eor v18.16b,v18.16b,v2.16b
  457. eor v0.16b,v0.16b,v18.16b
  458. ext v0.16b,v0.16b,v0.16b,#8
  459. #ifndef __ARMEB__
  460. rev64 v0.16b,v0.16b
  461. #endif
  462. st1 {v0.2d},[x0] //write out Xi
  463. ret
  464. .byte 71,72,65,83,72,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
  465. .align 2
  466. .align 2
  467. #endif