e_padlock-x86_64.s 18 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037
  1. .text
  2. .globl padlock_capability
  3. .type padlock_capability,@function
  4. .align 16
  5. padlock_capability:
  6. movq %rbx,%r8
  7. xorl %eax,%eax
  8. cpuid
  9. xorl %eax,%eax
  10. cmpl $0x746e6543,%ebx
  11. jne .Lzhaoxin
  12. cmpl $0x48727561,%edx
  13. jne .Lnoluck
  14. cmpl $0x736c7561,%ecx
  15. jne .Lnoluck
  16. jmp .LzhaoxinEnd
  17. .Lzhaoxin:
  18. cmpl $0x68532020,%ebx
  19. jne .Lnoluck
  20. cmpl $0x68676e61,%edx
  21. jne .Lnoluck
  22. cmpl $0x20206961,%ecx
  23. jne .Lnoluck
  24. .LzhaoxinEnd:
  25. movl $0xC0000000,%eax
  26. cpuid
  27. movl %eax,%edx
  28. xorl %eax,%eax
  29. cmpl $0xC0000001,%edx
  30. jb .Lnoluck
  31. movl $0xC0000001,%eax
  32. cpuid
  33. movl %edx,%eax
  34. andl $0xffffffef,%eax
  35. orl $0x10,%eax
  36. .Lnoluck:
  37. movq %r8,%rbx
  38. .byte 0xf3,0xc3
  39. .size padlock_capability,.-padlock_capability
  40. .globl padlock_key_bswap
  41. .type padlock_key_bswap,@function
  42. .align 16
  43. padlock_key_bswap:
  44. movl 240(%rdi),%edx
  45. incl %edx
  46. shll $2,%edx
  47. .Lbswap_loop:
  48. movl (%rdi),%eax
  49. bswapl %eax
  50. movl %eax,(%rdi)
  51. leaq 4(%rdi),%rdi
  52. subl $1,%edx
  53. jnz .Lbswap_loop
  54. .byte 0xf3,0xc3
  55. .size padlock_key_bswap,.-padlock_key_bswap
  56. .globl padlock_verify_context
  57. .type padlock_verify_context,@function
  58. .align 16
  59. padlock_verify_context:
  60. movq %rdi,%rdx
  61. pushf
  62. leaq .Lpadlock_saved_context(%rip),%rax
  63. call _padlock_verify_ctx
  64. leaq 8(%rsp),%rsp
  65. .byte 0xf3,0xc3
  66. .size padlock_verify_context,.-padlock_verify_context
  67. .type _padlock_verify_ctx,@function
  68. .align 16
  69. _padlock_verify_ctx:
  70. movq 8(%rsp),%r8
  71. btq $30,%r8
  72. jnc .Lverified
  73. cmpq (%rax),%rdx
  74. je .Lverified
  75. pushf
  76. popf
  77. .Lverified:
  78. movq %rdx,(%rax)
  79. .byte 0xf3,0xc3
  80. .size _padlock_verify_ctx,.-_padlock_verify_ctx
  81. .globl padlock_reload_key
  82. .type padlock_reload_key,@function
  83. .align 16
  84. padlock_reload_key:
  85. pushf
  86. popf
  87. .byte 0xf3,0xc3
  88. .size padlock_reload_key,.-padlock_reload_key
  89. .globl padlock_aes_block
  90. .type padlock_aes_block,@function
  91. .align 16
  92. padlock_aes_block:
  93. movq %rbx,%r8
  94. movq $1,%rcx
  95. leaq 32(%rdx),%rbx
  96. leaq 16(%rdx),%rdx
  97. .byte 0xf3,0x0f,0xa7,0xc8
  98. movq %r8,%rbx
  99. .byte 0xf3,0xc3
  100. .size padlock_aes_block,.-padlock_aes_block
  101. .globl padlock_xstore
  102. .type padlock_xstore,@function
  103. .align 16
  104. padlock_xstore:
  105. movl %esi,%edx
  106. .byte 0x0f,0xa7,0xc0
  107. .byte 0xf3,0xc3
  108. .size padlock_xstore,.-padlock_xstore
  109. .globl padlock_sha1_oneshot
  110. .type padlock_sha1_oneshot,@function
  111. .align 16
  112. padlock_sha1_oneshot:
  113. movq %rdx,%rcx
  114. movq %rdi,%rdx
  115. movups (%rdi),%xmm0
  116. subq $128+8,%rsp
  117. movl 16(%rdi),%eax
  118. movaps %xmm0,(%rsp)
  119. movq %rsp,%rdi
  120. movl %eax,16(%rsp)
  121. xorq %rax,%rax
  122. .byte 0xf3,0x0f,0xa6,0xc8
  123. movaps (%rsp),%xmm0
  124. movl 16(%rsp),%eax
  125. addq $128+8,%rsp
  126. movups %xmm0,(%rdx)
  127. movl %eax,16(%rdx)
  128. .byte 0xf3,0xc3
  129. .size padlock_sha1_oneshot,.-padlock_sha1_oneshot
  130. .globl padlock_sha1_blocks
  131. .type padlock_sha1_blocks,@function
  132. .align 16
  133. padlock_sha1_blocks:
  134. movq %rdx,%rcx
  135. movq %rdi,%rdx
  136. movups (%rdi),%xmm0
  137. subq $128+8,%rsp
  138. movl 16(%rdi),%eax
  139. movaps %xmm0,(%rsp)
  140. movq %rsp,%rdi
  141. movl %eax,16(%rsp)
  142. movq $-1,%rax
  143. .byte 0xf3,0x0f,0xa6,0xc8
  144. movaps (%rsp),%xmm0
  145. movl 16(%rsp),%eax
  146. addq $128+8,%rsp
  147. movups %xmm0,(%rdx)
  148. movl %eax,16(%rdx)
  149. .byte 0xf3,0xc3
  150. .size padlock_sha1_blocks,.-padlock_sha1_blocks
  151. .globl padlock_sha256_oneshot
  152. .type padlock_sha256_oneshot,@function
  153. .align 16
  154. padlock_sha256_oneshot:
  155. movq %rdx,%rcx
  156. movq %rdi,%rdx
  157. movups (%rdi),%xmm0
  158. subq $128+8,%rsp
  159. movups 16(%rdi),%xmm1
  160. movaps %xmm0,(%rsp)
  161. movq %rsp,%rdi
  162. movaps %xmm1,16(%rsp)
  163. xorq %rax,%rax
  164. .byte 0xf3,0x0f,0xa6,0xd0
  165. movaps (%rsp),%xmm0
  166. movaps 16(%rsp),%xmm1
  167. addq $128+8,%rsp
  168. movups %xmm0,(%rdx)
  169. movups %xmm1,16(%rdx)
  170. .byte 0xf3,0xc3
  171. .size padlock_sha256_oneshot,.-padlock_sha256_oneshot
  172. .globl padlock_sha256_blocks
  173. .type padlock_sha256_blocks,@function
  174. .align 16
  175. padlock_sha256_blocks:
  176. movq %rdx,%rcx
  177. movq %rdi,%rdx
  178. movups (%rdi),%xmm0
  179. subq $128+8,%rsp
  180. movups 16(%rdi),%xmm1
  181. movaps %xmm0,(%rsp)
  182. movq %rsp,%rdi
  183. movaps %xmm1,16(%rsp)
  184. movq $-1,%rax
  185. .byte 0xf3,0x0f,0xa6,0xd0
  186. movaps (%rsp),%xmm0
  187. movaps 16(%rsp),%xmm1
  188. addq $128+8,%rsp
  189. movups %xmm0,(%rdx)
  190. movups %xmm1,16(%rdx)
  191. .byte 0xf3,0xc3
  192. .size padlock_sha256_blocks,.-padlock_sha256_blocks
  193. .globl padlock_sha512_blocks
  194. .type padlock_sha512_blocks,@function
  195. .align 16
  196. padlock_sha512_blocks:
  197. movq %rdx,%rcx
  198. movq %rdi,%rdx
  199. movups (%rdi),%xmm0
  200. subq $128+8,%rsp
  201. movups 16(%rdi),%xmm1
  202. movups 32(%rdi),%xmm2
  203. movups 48(%rdi),%xmm3
  204. movaps %xmm0,(%rsp)
  205. movq %rsp,%rdi
  206. movaps %xmm1,16(%rsp)
  207. movaps %xmm2,32(%rsp)
  208. movaps %xmm3,48(%rsp)
  209. .byte 0xf3,0x0f,0xa6,0xe0
  210. movaps (%rsp),%xmm0
  211. movaps 16(%rsp),%xmm1
  212. movaps 32(%rsp),%xmm2
  213. movaps 48(%rsp),%xmm3
  214. addq $128+8,%rsp
  215. movups %xmm0,(%rdx)
  216. movups %xmm1,16(%rdx)
  217. movups %xmm2,32(%rdx)
  218. movups %xmm3,48(%rdx)
  219. .byte 0xf3,0xc3
  220. .size padlock_sha512_blocks,.-padlock_sha512_blocks
  221. .globl padlock_ecb_encrypt
  222. .type padlock_ecb_encrypt,@function
  223. .align 16
  224. padlock_ecb_encrypt:
  225. pushq %rbp
  226. pushq %rbx
  227. xorl %eax,%eax
  228. testq $15,%rdx
  229. jnz .Lecb_abort
  230. testq $15,%rcx
  231. jnz .Lecb_abort
  232. leaq .Lpadlock_saved_context(%rip),%rax
  233. pushf
  234. cld
  235. call _padlock_verify_ctx
  236. leaq 16(%rdx),%rdx
  237. xorl %eax,%eax
  238. xorl %ebx,%ebx
  239. testl $32,(%rdx)
  240. jnz .Lecb_aligned
  241. testq $0x0f,%rdi
  242. setz %al
  243. testq $0x0f,%rsi
  244. setz %bl
  245. testl %ebx,%eax
  246. jnz .Lecb_aligned
  247. negq %rax
  248. movq $512,%rbx
  249. notq %rax
  250. leaq (%rsp),%rbp
  251. cmpq %rbx,%rcx
  252. cmovcq %rcx,%rbx
  253. andq %rbx,%rax
  254. movq %rcx,%rbx
  255. negq %rax
  256. andq $512-1,%rbx
  257. leaq (%rax,%rbp,1),%rsp
  258. movq $512,%rax
  259. cmovzq %rax,%rbx
  260. cmpq %rbx,%rcx
  261. ja .Lecb_loop
  262. movq %rsi,%rax
  263. cmpq %rsp,%rbp
  264. cmoveq %rdi,%rax
  265. addq %rcx,%rax
  266. negq %rax
  267. andq $0xfff,%rax
  268. cmpq $128,%rax
  269. movq $-128,%rax
  270. cmovaeq %rbx,%rax
  271. andq %rax,%rbx
  272. jz .Lecb_unaligned_tail
  273. jmp .Lecb_loop
  274. .align 16
  275. .Lecb_loop:
  276. cmpq %rcx,%rbx
  277. cmovaq %rcx,%rbx
  278. movq %rdi,%r8
  279. movq %rsi,%r9
  280. movq %rcx,%r10
  281. movq %rbx,%rcx
  282. movq %rbx,%r11
  283. testq $0x0f,%rdi
  284. cmovnzq %rsp,%rdi
  285. testq $0x0f,%rsi
  286. jz .Lecb_inp_aligned
  287. shrq $3,%rcx
  288. .byte 0xf3,0x48,0xa5
  289. subq %rbx,%rdi
  290. movq %rbx,%rcx
  291. movq %rdi,%rsi
  292. .Lecb_inp_aligned:
  293. leaq -16(%rdx),%rax
  294. leaq 16(%rdx),%rbx
  295. shrq $4,%rcx
  296. .byte 0xf3,0x0f,0xa7,200
  297. movq %r8,%rdi
  298. movq %r11,%rbx
  299. testq $0x0f,%rdi
  300. jz .Lecb_out_aligned
  301. movq %rbx,%rcx
  302. leaq (%rsp),%rsi
  303. shrq $3,%rcx
  304. .byte 0xf3,0x48,0xa5
  305. subq %rbx,%rdi
  306. .Lecb_out_aligned:
  307. movq %r9,%rsi
  308. movq %r10,%rcx
  309. addq %rbx,%rdi
  310. addq %rbx,%rsi
  311. subq %rbx,%rcx
  312. movq $512,%rbx
  313. jz .Lecb_break
  314. cmpq %rbx,%rcx
  315. jae .Lecb_loop
  316. .Lecb_unaligned_tail:
  317. xorl %eax,%eax
  318. cmpq %rsp,%rbp
  319. cmoveq %rcx,%rax
  320. movq %rdi,%r8
  321. movq %rcx,%rbx
  322. subq %rax,%rsp
  323. shrq $3,%rcx
  324. leaq (%rsp),%rdi
  325. .byte 0xf3,0x48,0xa5
  326. movq %rsp,%rsi
  327. movq %r8,%rdi
  328. movq %rbx,%rcx
  329. jmp .Lecb_loop
  330. .align 16
  331. .Lecb_break:
  332. cmpq %rbp,%rsp
  333. je .Lecb_done
  334. pxor %xmm0,%xmm0
  335. leaq (%rsp),%rax
  336. .Lecb_bzero:
  337. movaps %xmm0,(%rax)
  338. leaq 16(%rax),%rax
  339. cmpq %rax,%rbp
  340. ja .Lecb_bzero
  341. .Lecb_done:
  342. leaq (%rbp),%rsp
  343. jmp .Lecb_exit
  344. .align 16
  345. .Lecb_aligned:
  346. leaq (%rsi,%rcx,1),%rbp
  347. negq %rbp
  348. andq $0xfff,%rbp
  349. xorl %eax,%eax
  350. cmpq $128,%rbp
  351. movq $128-1,%rbp
  352. cmovaeq %rax,%rbp
  353. andq %rcx,%rbp
  354. subq %rbp,%rcx
  355. jz .Lecb_aligned_tail
  356. leaq -16(%rdx),%rax
  357. leaq 16(%rdx),%rbx
  358. shrq $4,%rcx
  359. .byte 0xf3,0x0f,0xa7,200
  360. testq %rbp,%rbp
  361. jz .Lecb_exit
  362. .Lecb_aligned_tail:
  363. movq %rdi,%r8
  364. movq %rbp,%rbx
  365. movq %rbp,%rcx
  366. leaq (%rsp),%rbp
  367. subq %rcx,%rsp
  368. shrq $3,%rcx
  369. leaq (%rsp),%rdi
  370. .byte 0xf3,0x48,0xa5
  371. leaq (%r8),%rdi
  372. leaq (%rsp),%rsi
  373. movq %rbx,%rcx
  374. jmp .Lecb_loop
  375. .Lecb_exit:
  376. movl $1,%eax
  377. leaq 8(%rsp),%rsp
  378. .Lecb_abort:
  379. popq %rbx
  380. popq %rbp
  381. .byte 0xf3,0xc3
  382. .size padlock_ecb_encrypt,.-padlock_ecb_encrypt
  383. .globl padlock_cbc_encrypt
  384. .type padlock_cbc_encrypt,@function
  385. .align 16
  386. padlock_cbc_encrypt:
  387. pushq %rbp
  388. pushq %rbx
  389. xorl %eax,%eax
  390. testq $15,%rdx
  391. jnz .Lcbc_abort
  392. testq $15,%rcx
  393. jnz .Lcbc_abort
  394. leaq .Lpadlock_saved_context(%rip),%rax
  395. pushf
  396. cld
  397. call _padlock_verify_ctx
  398. leaq 16(%rdx),%rdx
  399. xorl %eax,%eax
  400. xorl %ebx,%ebx
  401. testl $32,(%rdx)
  402. jnz .Lcbc_aligned
  403. testq $0x0f,%rdi
  404. setz %al
  405. testq $0x0f,%rsi
  406. setz %bl
  407. testl %ebx,%eax
  408. jnz .Lcbc_aligned
  409. negq %rax
  410. movq $512,%rbx
  411. notq %rax
  412. leaq (%rsp),%rbp
  413. cmpq %rbx,%rcx
  414. cmovcq %rcx,%rbx
  415. andq %rbx,%rax
  416. movq %rcx,%rbx
  417. negq %rax
  418. andq $512-1,%rbx
  419. leaq (%rax,%rbp,1),%rsp
  420. movq $512,%rax
  421. cmovzq %rax,%rbx
  422. cmpq %rbx,%rcx
  423. ja .Lcbc_loop
  424. movq %rsi,%rax
  425. cmpq %rsp,%rbp
  426. cmoveq %rdi,%rax
  427. addq %rcx,%rax
  428. negq %rax
  429. andq $0xfff,%rax
  430. cmpq $64,%rax
  431. movq $-64,%rax
  432. cmovaeq %rbx,%rax
  433. andq %rax,%rbx
  434. jz .Lcbc_unaligned_tail
  435. jmp .Lcbc_loop
  436. .align 16
  437. .Lcbc_loop:
  438. cmpq %rcx,%rbx
  439. cmovaq %rcx,%rbx
  440. movq %rdi,%r8
  441. movq %rsi,%r9
  442. movq %rcx,%r10
  443. movq %rbx,%rcx
  444. movq %rbx,%r11
  445. testq $0x0f,%rdi
  446. cmovnzq %rsp,%rdi
  447. testq $0x0f,%rsi
  448. jz .Lcbc_inp_aligned
  449. shrq $3,%rcx
  450. .byte 0xf3,0x48,0xa5
  451. subq %rbx,%rdi
  452. movq %rbx,%rcx
  453. movq %rdi,%rsi
  454. .Lcbc_inp_aligned:
  455. leaq -16(%rdx),%rax
  456. leaq 16(%rdx),%rbx
  457. shrq $4,%rcx
  458. .byte 0xf3,0x0f,0xa7,208
  459. movdqa (%rax),%xmm0
  460. movdqa %xmm0,-16(%rdx)
  461. movq %r8,%rdi
  462. movq %r11,%rbx
  463. testq $0x0f,%rdi
  464. jz .Lcbc_out_aligned
  465. movq %rbx,%rcx
  466. leaq (%rsp),%rsi
  467. shrq $3,%rcx
  468. .byte 0xf3,0x48,0xa5
  469. subq %rbx,%rdi
  470. .Lcbc_out_aligned:
  471. movq %r9,%rsi
  472. movq %r10,%rcx
  473. addq %rbx,%rdi
  474. addq %rbx,%rsi
  475. subq %rbx,%rcx
  476. movq $512,%rbx
  477. jz .Lcbc_break
  478. cmpq %rbx,%rcx
  479. jae .Lcbc_loop
  480. .Lcbc_unaligned_tail:
  481. xorl %eax,%eax
  482. cmpq %rsp,%rbp
  483. cmoveq %rcx,%rax
  484. movq %rdi,%r8
  485. movq %rcx,%rbx
  486. subq %rax,%rsp
  487. shrq $3,%rcx
  488. leaq (%rsp),%rdi
  489. .byte 0xf3,0x48,0xa5
  490. movq %rsp,%rsi
  491. movq %r8,%rdi
  492. movq %rbx,%rcx
  493. jmp .Lcbc_loop
  494. .align 16
  495. .Lcbc_break:
  496. cmpq %rbp,%rsp
  497. je .Lcbc_done
  498. pxor %xmm0,%xmm0
  499. leaq (%rsp),%rax
  500. .Lcbc_bzero:
  501. movaps %xmm0,(%rax)
  502. leaq 16(%rax),%rax
  503. cmpq %rax,%rbp
  504. ja .Lcbc_bzero
  505. .Lcbc_done:
  506. leaq (%rbp),%rsp
  507. jmp .Lcbc_exit
  508. .align 16
  509. .Lcbc_aligned:
  510. leaq (%rsi,%rcx,1),%rbp
  511. negq %rbp
  512. andq $0xfff,%rbp
  513. xorl %eax,%eax
  514. cmpq $64,%rbp
  515. movq $64-1,%rbp
  516. cmovaeq %rax,%rbp
  517. andq %rcx,%rbp
  518. subq %rbp,%rcx
  519. jz .Lcbc_aligned_tail
  520. leaq -16(%rdx),%rax
  521. leaq 16(%rdx),%rbx
  522. shrq $4,%rcx
  523. .byte 0xf3,0x0f,0xa7,208
  524. movdqa (%rax),%xmm0
  525. movdqa %xmm0,-16(%rdx)
  526. testq %rbp,%rbp
  527. jz .Lcbc_exit
  528. .Lcbc_aligned_tail:
  529. movq %rdi,%r8
  530. movq %rbp,%rbx
  531. movq %rbp,%rcx
  532. leaq (%rsp),%rbp
  533. subq %rcx,%rsp
  534. shrq $3,%rcx
  535. leaq (%rsp),%rdi
  536. .byte 0xf3,0x48,0xa5
  537. leaq (%r8),%rdi
  538. leaq (%rsp),%rsi
  539. movq %rbx,%rcx
  540. jmp .Lcbc_loop
  541. .Lcbc_exit:
  542. movl $1,%eax
  543. leaq 8(%rsp),%rsp
  544. .Lcbc_abort:
  545. popq %rbx
  546. popq %rbp
  547. .byte 0xf3,0xc3
  548. .size padlock_cbc_encrypt,.-padlock_cbc_encrypt
  549. .globl padlock_cfb_encrypt
  550. .type padlock_cfb_encrypt,@function
  551. .align 16
  552. padlock_cfb_encrypt:
  553. pushq %rbp
  554. pushq %rbx
  555. xorl %eax,%eax
  556. testq $15,%rdx
  557. jnz .Lcfb_abort
  558. testq $15,%rcx
  559. jnz .Lcfb_abort
  560. leaq .Lpadlock_saved_context(%rip),%rax
  561. pushf
  562. cld
  563. call _padlock_verify_ctx
  564. leaq 16(%rdx),%rdx
  565. xorl %eax,%eax
  566. xorl %ebx,%ebx
  567. testl $32,(%rdx)
  568. jnz .Lcfb_aligned
  569. testq $0x0f,%rdi
  570. setz %al
  571. testq $0x0f,%rsi
  572. setz %bl
  573. testl %ebx,%eax
  574. jnz .Lcfb_aligned
  575. negq %rax
  576. movq $512,%rbx
  577. notq %rax
  578. leaq (%rsp),%rbp
  579. cmpq %rbx,%rcx
  580. cmovcq %rcx,%rbx
  581. andq %rbx,%rax
  582. movq %rcx,%rbx
  583. negq %rax
  584. andq $512-1,%rbx
  585. leaq (%rax,%rbp,1),%rsp
  586. movq $512,%rax
  587. cmovzq %rax,%rbx
  588. jmp .Lcfb_loop
  589. .align 16
  590. .Lcfb_loop:
  591. cmpq %rcx,%rbx
  592. cmovaq %rcx,%rbx
  593. movq %rdi,%r8
  594. movq %rsi,%r9
  595. movq %rcx,%r10
  596. movq %rbx,%rcx
  597. movq %rbx,%r11
  598. testq $0x0f,%rdi
  599. cmovnzq %rsp,%rdi
  600. testq $0x0f,%rsi
  601. jz .Lcfb_inp_aligned
  602. shrq $3,%rcx
  603. .byte 0xf3,0x48,0xa5
  604. subq %rbx,%rdi
  605. movq %rbx,%rcx
  606. movq %rdi,%rsi
  607. .Lcfb_inp_aligned:
  608. leaq -16(%rdx),%rax
  609. leaq 16(%rdx),%rbx
  610. shrq $4,%rcx
  611. .byte 0xf3,0x0f,0xa7,224
  612. movdqa (%rax),%xmm0
  613. movdqa %xmm0,-16(%rdx)
  614. movq %r8,%rdi
  615. movq %r11,%rbx
  616. testq $0x0f,%rdi
  617. jz .Lcfb_out_aligned
  618. movq %rbx,%rcx
  619. leaq (%rsp),%rsi
  620. shrq $3,%rcx
  621. .byte 0xf3,0x48,0xa5
  622. subq %rbx,%rdi
  623. .Lcfb_out_aligned:
  624. movq %r9,%rsi
  625. movq %r10,%rcx
  626. addq %rbx,%rdi
  627. addq %rbx,%rsi
  628. subq %rbx,%rcx
  629. movq $512,%rbx
  630. jnz .Lcfb_loop
  631. cmpq %rbp,%rsp
  632. je .Lcfb_done
  633. pxor %xmm0,%xmm0
  634. leaq (%rsp),%rax
  635. .Lcfb_bzero:
  636. movaps %xmm0,(%rax)
  637. leaq 16(%rax),%rax
  638. cmpq %rax,%rbp
  639. ja .Lcfb_bzero
  640. .Lcfb_done:
  641. leaq (%rbp),%rsp
  642. jmp .Lcfb_exit
  643. .align 16
  644. .Lcfb_aligned:
  645. leaq -16(%rdx),%rax
  646. leaq 16(%rdx),%rbx
  647. shrq $4,%rcx
  648. .byte 0xf3,0x0f,0xa7,224
  649. movdqa (%rax),%xmm0
  650. movdqa %xmm0,-16(%rdx)
  651. .Lcfb_exit:
  652. movl $1,%eax
  653. leaq 8(%rsp),%rsp
  654. .Lcfb_abort:
  655. popq %rbx
  656. popq %rbp
  657. .byte 0xf3,0xc3
  658. .size padlock_cfb_encrypt,.-padlock_cfb_encrypt
  659. .globl padlock_ofb_encrypt
  660. .type padlock_ofb_encrypt,@function
  661. .align 16
  662. padlock_ofb_encrypt:
  663. pushq %rbp
  664. pushq %rbx
  665. xorl %eax,%eax
  666. testq $15,%rdx
  667. jnz .Lofb_abort
  668. testq $15,%rcx
  669. jnz .Lofb_abort
  670. leaq .Lpadlock_saved_context(%rip),%rax
  671. pushf
  672. cld
  673. call _padlock_verify_ctx
  674. leaq 16(%rdx),%rdx
  675. xorl %eax,%eax
  676. xorl %ebx,%ebx
  677. testl $32,(%rdx)
  678. jnz .Lofb_aligned
  679. testq $0x0f,%rdi
  680. setz %al
  681. testq $0x0f,%rsi
  682. setz %bl
  683. testl %ebx,%eax
  684. jnz .Lofb_aligned
  685. negq %rax
  686. movq $512,%rbx
  687. notq %rax
  688. leaq (%rsp),%rbp
  689. cmpq %rbx,%rcx
  690. cmovcq %rcx,%rbx
  691. andq %rbx,%rax
  692. movq %rcx,%rbx
  693. negq %rax
  694. andq $512-1,%rbx
  695. leaq (%rax,%rbp,1),%rsp
  696. movq $512,%rax
  697. cmovzq %rax,%rbx
  698. jmp .Lofb_loop
  699. .align 16
  700. .Lofb_loop:
  701. cmpq %rcx,%rbx
  702. cmovaq %rcx,%rbx
  703. movq %rdi,%r8
  704. movq %rsi,%r9
  705. movq %rcx,%r10
  706. movq %rbx,%rcx
  707. movq %rbx,%r11
  708. testq $0x0f,%rdi
  709. cmovnzq %rsp,%rdi
  710. testq $0x0f,%rsi
  711. jz .Lofb_inp_aligned
  712. shrq $3,%rcx
  713. .byte 0xf3,0x48,0xa5
  714. subq %rbx,%rdi
  715. movq %rbx,%rcx
  716. movq %rdi,%rsi
  717. .Lofb_inp_aligned:
  718. leaq -16(%rdx),%rax
  719. leaq 16(%rdx),%rbx
  720. shrq $4,%rcx
  721. .byte 0xf3,0x0f,0xa7,232
  722. movdqa (%rax),%xmm0
  723. movdqa %xmm0,-16(%rdx)
  724. movq %r8,%rdi
  725. movq %r11,%rbx
  726. testq $0x0f,%rdi
  727. jz .Lofb_out_aligned
  728. movq %rbx,%rcx
  729. leaq (%rsp),%rsi
  730. shrq $3,%rcx
  731. .byte 0xf3,0x48,0xa5
  732. subq %rbx,%rdi
  733. .Lofb_out_aligned:
  734. movq %r9,%rsi
  735. movq %r10,%rcx
  736. addq %rbx,%rdi
  737. addq %rbx,%rsi
  738. subq %rbx,%rcx
  739. movq $512,%rbx
  740. jnz .Lofb_loop
  741. cmpq %rbp,%rsp
  742. je .Lofb_done
  743. pxor %xmm0,%xmm0
  744. leaq (%rsp),%rax
  745. .Lofb_bzero:
  746. movaps %xmm0,(%rax)
  747. leaq 16(%rax),%rax
  748. cmpq %rax,%rbp
  749. ja .Lofb_bzero
  750. .Lofb_done:
  751. leaq (%rbp),%rsp
  752. jmp .Lofb_exit
  753. .align 16
  754. .Lofb_aligned:
  755. leaq -16(%rdx),%rax
  756. leaq 16(%rdx),%rbx
  757. shrq $4,%rcx
  758. .byte 0xf3,0x0f,0xa7,232
  759. movdqa (%rax),%xmm0
  760. movdqa %xmm0,-16(%rdx)
  761. .Lofb_exit:
  762. movl $1,%eax
  763. leaq 8(%rsp),%rsp
  764. .Lofb_abort:
  765. popq %rbx
  766. popq %rbp
  767. .byte 0xf3,0xc3
  768. .size padlock_ofb_encrypt,.-padlock_ofb_encrypt
  769. .globl padlock_ctr32_encrypt
  770. .type padlock_ctr32_encrypt,@function
  771. .align 16
  772. padlock_ctr32_encrypt:
  773. pushq %rbp
  774. pushq %rbx
  775. xorl %eax,%eax
  776. testq $15,%rdx
  777. jnz .Lctr32_abort
  778. testq $15,%rcx
  779. jnz .Lctr32_abort
  780. leaq .Lpadlock_saved_context(%rip),%rax
  781. pushf
  782. cld
  783. call _padlock_verify_ctx
  784. leaq 16(%rdx),%rdx
  785. xorl %eax,%eax
  786. xorl %ebx,%ebx
  787. testl $32,(%rdx)
  788. jnz .Lctr32_aligned
  789. testq $0x0f,%rdi
  790. setz %al
  791. testq $0x0f,%rsi
  792. setz %bl
  793. testl %ebx,%eax
  794. jnz .Lctr32_aligned
  795. negq %rax
  796. movq $512,%rbx
  797. notq %rax
  798. leaq (%rsp),%rbp
  799. cmpq %rbx,%rcx
  800. cmovcq %rcx,%rbx
  801. andq %rbx,%rax
  802. movq %rcx,%rbx
  803. negq %rax
  804. andq $512-1,%rbx
  805. leaq (%rax,%rbp,1),%rsp
  806. movq $512,%rax
  807. cmovzq %rax,%rbx
  808. .Lctr32_reenter:
  809. movl -4(%rdx),%eax
  810. bswapl %eax
  811. negl %eax
  812. andl $31,%eax
  813. movq $512,%rbx
  814. shll $4,%eax
  815. cmovzq %rbx,%rax
  816. cmpq %rax,%rcx
  817. cmovaq %rax,%rbx
  818. cmovbeq %rcx,%rbx
  819. cmpq %rbx,%rcx
  820. ja .Lctr32_loop
  821. movq %rsi,%rax
  822. cmpq %rsp,%rbp
  823. cmoveq %rdi,%rax
  824. addq %rcx,%rax
  825. negq %rax
  826. andq $0xfff,%rax
  827. cmpq $32,%rax
  828. movq $-32,%rax
  829. cmovaeq %rbx,%rax
  830. andq %rax,%rbx
  831. jz .Lctr32_unaligned_tail
  832. jmp .Lctr32_loop
  833. .align 16
  834. .Lctr32_loop:
  835. cmpq %rcx,%rbx
  836. cmovaq %rcx,%rbx
  837. movq %rdi,%r8
  838. movq %rsi,%r9
  839. movq %rcx,%r10
  840. movq %rbx,%rcx
  841. movq %rbx,%r11
  842. testq $0x0f,%rdi
  843. cmovnzq %rsp,%rdi
  844. testq $0x0f,%rsi
  845. jz .Lctr32_inp_aligned
  846. shrq $3,%rcx
  847. .byte 0xf3,0x48,0xa5
  848. subq %rbx,%rdi
  849. movq %rbx,%rcx
  850. movq %rdi,%rsi
  851. .Lctr32_inp_aligned:
  852. leaq -16(%rdx),%rax
  853. leaq 16(%rdx),%rbx
  854. shrq $4,%rcx
  855. .byte 0xf3,0x0f,0xa7,216
  856. movl -4(%rdx),%eax
  857. testl $0xffff0000,%eax
  858. jnz .Lctr32_no_carry
  859. bswapl %eax
  860. addl $0x10000,%eax
  861. bswapl %eax
  862. movl %eax,-4(%rdx)
  863. .Lctr32_no_carry:
  864. movq %r8,%rdi
  865. movq %r11,%rbx
  866. testq $0x0f,%rdi
  867. jz .Lctr32_out_aligned
  868. movq %rbx,%rcx
  869. leaq (%rsp),%rsi
  870. shrq $3,%rcx
  871. .byte 0xf3,0x48,0xa5
  872. subq %rbx,%rdi
  873. .Lctr32_out_aligned:
  874. movq %r9,%rsi
  875. movq %r10,%rcx
  876. addq %rbx,%rdi
  877. addq %rbx,%rsi
  878. subq %rbx,%rcx
  879. movq $512,%rbx
  880. jz .Lctr32_break
  881. cmpq %rbx,%rcx
  882. jae .Lctr32_loop
  883. movq %rcx,%rbx
  884. movq %rsi,%rax
  885. cmpq %rsp,%rbp
  886. cmoveq %rdi,%rax
  887. addq %rcx,%rax
  888. negq %rax
  889. andq $0xfff,%rax
  890. cmpq $32,%rax
  891. movq $-32,%rax
  892. cmovaeq %rbx,%rax
  893. andq %rax,%rbx
  894. jnz .Lctr32_loop
  895. .Lctr32_unaligned_tail:
  896. xorl %eax,%eax
  897. cmpq %rsp,%rbp
  898. cmoveq %rcx,%rax
  899. movq %rdi,%r8
  900. movq %rcx,%rbx
  901. subq %rax,%rsp
  902. shrq $3,%rcx
  903. leaq (%rsp),%rdi
  904. .byte 0xf3,0x48,0xa5
  905. movq %rsp,%rsi
  906. movq %r8,%rdi
  907. movq %rbx,%rcx
  908. jmp .Lctr32_loop
  909. .align 16
  910. .Lctr32_break:
  911. cmpq %rbp,%rsp
  912. je .Lctr32_done
  913. pxor %xmm0,%xmm0
  914. leaq (%rsp),%rax
  915. .Lctr32_bzero:
  916. movaps %xmm0,(%rax)
  917. leaq 16(%rax),%rax
  918. cmpq %rax,%rbp
  919. ja .Lctr32_bzero
  920. .Lctr32_done:
  921. leaq (%rbp),%rsp
  922. jmp .Lctr32_exit
  923. .align 16
  924. .Lctr32_aligned:
  925. movl -4(%rdx),%eax
  926. bswapl %eax
  927. negl %eax
  928. andl $0xffff,%eax
  929. movq $1048576,%rbx
  930. shll $4,%eax
  931. cmovzq %rbx,%rax
  932. cmpq %rax,%rcx
  933. cmovaq %rax,%rbx
  934. cmovbeq %rcx,%rbx
  935. jbe .Lctr32_aligned_skip
  936. .Lctr32_aligned_loop:
  937. movq %rcx,%r10
  938. movq %rbx,%rcx
  939. movq %rbx,%r11
  940. leaq -16(%rdx),%rax
  941. leaq 16(%rdx),%rbx
  942. shrq $4,%rcx
  943. .byte 0xf3,0x0f,0xa7,216
  944. movl -4(%rdx),%eax
  945. bswapl %eax
  946. addl $0x10000,%eax
  947. bswapl %eax
  948. movl %eax,-4(%rdx)
  949. movq %r10,%rcx
  950. subq %r11,%rcx
  951. movq $1048576,%rbx
  952. jz .Lctr32_exit
  953. cmpq %rbx,%rcx
  954. jae .Lctr32_aligned_loop
  955. .Lctr32_aligned_skip:
  956. leaq (%rsi,%rcx,1),%rbp
  957. negq %rbp
  958. andq $0xfff,%rbp
  959. xorl %eax,%eax
  960. cmpq $32,%rbp
  961. movq $32-1,%rbp
  962. cmovaeq %rax,%rbp
  963. andq %rcx,%rbp
  964. subq %rbp,%rcx
  965. jz .Lctr32_aligned_tail
  966. leaq -16(%rdx),%rax
  967. leaq 16(%rdx),%rbx
  968. shrq $4,%rcx
  969. .byte 0xf3,0x0f,0xa7,216
  970. testq %rbp,%rbp
  971. jz .Lctr32_exit
  972. .Lctr32_aligned_tail:
  973. movq %rdi,%r8
  974. movq %rbp,%rbx
  975. movq %rbp,%rcx
  976. leaq (%rsp),%rbp
  977. subq %rcx,%rsp
  978. shrq $3,%rcx
  979. leaq (%rsp),%rdi
  980. .byte 0xf3,0x48,0xa5
  981. leaq (%r8),%rdi
  982. leaq (%rsp),%rsi
  983. movq %rbx,%rcx
  984. jmp .Lctr32_loop
  985. .Lctr32_exit:
  986. movl $1,%eax
  987. leaq 8(%rsp),%rsp
  988. .Lctr32_abort:
  989. popq %rbx
  990. popq %rbp
  991. .byte 0xf3,0xc3
  992. .size padlock_ctr32_encrypt,.-padlock_ctr32_encrypt
  993. .byte 86,73,65,32,80,97,100,108,111,99,107,32,120,56,54,95,54,52,32,109,111,100,117,108,101,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
  994. .align 16
  995. .data
  996. .align 8
  997. .Lpadlock_saved_context:
  998. .quad 0