Lines Matching refs:b
33 ld1 {v11.16b}, [x3]
34 ext v11.16b, v11.16b, v11.16b, #8
35 rev64 v11.16b, v11.16b
46 ext v15.16b, v15.16b, v15.16b, #8
60 …ld1 { v0.16b}, [x16] //special case vector load initial counter so we …
80 ext v14.16b, v14.16b, v14.16b, #8
82 aese v1.16b, v18.16b
83 aesmc v1.16b, v1.16b //AES block 1 - round 0
86 aese v2.16b, v18.16b
87 aesmc v2.16b, v2.16b //AES block 2 - round 0
90 ext v12.16b, v12.16b, v12.16b, #8
93 aese v0.16b, v18.16b
94 aesmc v0.16b, v0.16b //AES block 0 - round 0
97 aese v3.16b, v18.16b
98 aesmc v3.16b, v3.16b //AES block 3 - round 0
101 aese v2.16b, v19.16b
102 aesmc v2.16b, v2.16b //AES block 2 - round 1
105 aese v0.16b, v19.16b
106 aesmc v0.16b, v0.16b //AES block 0 - round 1
109 aese v1.16b, v19.16b
110 aesmc v1.16b, v1.16b //AES block 1 - round 1
113 aese v3.16b, v19.16b
114 aesmc v3.16b, v3.16b //AES block 3 - round 1
117 aese v0.16b, v20.16b
118 aesmc v0.16b, v0.16b //AES block 0 - round 2
121 aese v1.16b, v20.16b
122 aesmc v1.16b, v1.16b //AES block 1 - round 2
125 ext v13.16b, v13.16b, v13.16b, #8
128 aese v3.16b, v20.16b
129 aesmc v3.16b, v3.16b //AES block 3 - round 2
131 aese v2.16b, v20.16b
132 aesmc v2.16b, v2.16b //AES block 2 - round 2
133 eor v17.16b, v17.16b, v9.16b //h4k | h3k
135 aese v0.16b, v21.16b
136 aesmc v0.16b, v0.16b //AES block 0 - round 3
138 aese v1.16b, v21.16b
139 aesmc v1.16b, v1.16b //AES block 1 - round 3
141 aese v2.16b, v21.16b
142 aesmc v2.16b, v2.16b //AES block 2 - round 3
145 aese v3.16b, v21.16b
146 aesmc v3.16b, v3.16b //AES block 3 - round 3
151 aese v3.16b, v22.16b
152 aesmc v3.16b, v3.16b //AES block 3 - round 4
155 aese v2.16b, v22.16b
156 aesmc v2.16b, v2.16b //AES block 2 - round 4
159 aese v0.16b, v22.16b
160 aesmc v0.16b, v0.16b //AES block 0 - round 4
162 aese v3.16b, v23.16b
163 aesmc v3.16b, v3.16b //AES block 3 - round 5
165 aese v2.16b, v23.16b
166 aesmc v2.16b, v2.16b //AES block 2 - round 5
168 aese v0.16b, v23.16b
169 aesmc v0.16b, v0.16b //AES block 0 - round 5
171 aese v3.16b, v24.16b
172 aesmc v3.16b, v3.16b //AES block 3 - round 6
174 aese v1.16b, v22.16b
175 aesmc v1.16b, v1.16b //AES block 1 - round 4
177 aese v2.16b, v24.16b
178 aesmc v2.16b, v2.16b //AES block 2 - round 6
181 aese v0.16b, v24.16b
182 aesmc v0.16b, v0.16b //AES block 0 - round 6
184 aese v1.16b, v23.16b
185 aesmc v1.16b, v1.16b //AES block 1 - round 5
187 aese v3.16b, v25.16b
188 aesmc v3.16b, v3.16b //AES block 3 - round 7
190 aese v0.16b, v25.16b
191 aesmc v0.16b, v0.16b //AES block 0 - round 7
193 aese v1.16b, v24.16b
194 aesmc v1.16b, v1.16b //AES block 1 - round 6
196 aese v2.16b, v25.16b
197 aesmc v2.16b, v2.16b //AES block 2 - round 7
199 aese v0.16b, v26.16b
200 aesmc v0.16b, v0.16b //AES block 0 - round 8
202 aese v1.16b, v25.16b
203 aesmc v1.16b, v1.16b //AES block 1 - round 7
205 aese v2.16b, v26.16b
206 aesmc v2.16b, v2.16b //AES block 2 - round 8
208 aese v3.16b, v26.16b
209 aesmc v3.16b, v3.16b //AES block 3 - round 8
211 aese v1.16b, v26.16b
212 aesmc v1.16b, v1.16b //AES block 1 - round 8
214 aese v2.16b, v27.16b //AES block 2 - round 9
216 aese v0.16b, v27.16b //AES block 0 - round 9
218 eor v16.16b, v16.16b, v8.16b //h2k | h1k
220 aese v1.16b, v27.16b //AES block 1 - round 9
222 aese v3.16b, v27.16b //AES block 3 - round 9
223 b.ge .L128_enc_tail //handle tail
268 eor v4.16b, v4.16b, v0.16b //AES block 0 - result
275 eor v5.16b, v5.16b, v1.16b //AES block 1 - result
285 st1 { v4.16b}, [x2], #16 //AES block 0 - store result
291 eor v6.16b, v6.16b, v2.16b //AES block 2 - result
292 st1 { v5.16b}, [x2], #16 //AES block 1 - store result
299 st1 { v6.16b}, [x2], #16 //AES block 2 - store result
303 eor v7.16b, v7.16b, v3.16b //AES block 3 - result
304 st1 { v7.16b}, [x2], #16 //AES block 3 - store result
305 b.ge .L128_enc_prepretail //do prepretail
313 rev64 v4.16b, v4.16b //GHASH block 4k (only t0 is free)
314 rev64 v6.16b, v6.16b //GHASH block 4k+2 (t0, t1, and t2 free)
316 aese v2.16b, v18.16b
317 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
320 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
321 rev64 v5.16b, v5.16b //GHASH block 4k+1 (t0 and t1 free)
323 aese v1.16b, v18.16b
324 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
328 aese v0.16b, v18.16b
329 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
332 aese v2.16b, v19.16b
333 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
336 aese v1.16b, v19.16b
337 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
338 eor v4.16b, v4.16b, v11.16b //PRE 1
340 aese v3.16b, v18.16b
341 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
345 eor v31.8b, v31.8b, v6.8b //GHASH block 4k+2 - mid
351 aese v0.16b, v19.16b
352 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
355 eor v30.8b, v30.8b, v5.8b //GHASH block 4k+1 - mid
363 aese v0.16b, v20.16b
364 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
367 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
369 aese v1.16b, v20.16b
370 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
372 aese v0.16b, v21.16b
373 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
374 eor v9.16b, v9.16b, v28.16b //GHASH block 4k+1 - high
379 rev64 v7.16b, v7.16b //GHASH block 4k+3 (t0, t1, t2 and t3 free)
389 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+1 - mid
392 aese v3.16b, v19.16b
393 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
394 eor v11.16b, v11.16b, v29.16b //GHASH block 4k+1 - low
396 aese v2.16b, v20.16b
397 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
400 aese v1.16b, v21.16b
401 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
402 eor v30.8b, v30.8b, v7.8b //GHASH block 4k+3 - mid
406 aese v2.16b, v21.16b
407 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
408 eor v9.16b, v9.16b, v8.16b //GHASH block 4k+2 - high
413 movi v8.8b, #0xc2
416 eor v11.16b, v11.16b, v28.16b //GHASH block 4k+2 - low
418 aese v1.16b, v22.16b
419 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
421 aese v3.16b, v20.16b
422 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
425 aese v0.16b, v22.16b
426 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
427 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+3 - high
429 aese v1.16b, v23.16b
430 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
436 aese v3.16b, v21.16b
437 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
438 eor v10.16b, v10.16b, v31.16b //GHASH block 4k+2 - mid
440 aese v0.16b, v23.16b
441 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
448 eor v11.16b, v11.16b, v29.16b //GHASH block 4k+3 - low
450 aese v2.16b, v22.16b
451 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
454 aese v3.16b, v22.16b
455 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
456 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+3 - mid
458 aese v1.16b, v24.16b
459 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
462 aese v2.16b, v23.16b
463 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
464 eor v30.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
467 aese v0.16b, v24.16b
468 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
473 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
475 aese v3.16b, v23.16b
476 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
479 aese v0.16b, v25.16b
480 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
481 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up
483 aese v2.16b, v24.16b
484 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
487 aese v1.16b, v25.16b
488 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
491 aese v0.16b, v26.16b
492 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
495 aese v3.16b, v24.16b
496 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
499 aese v1.16b, v26.16b
500 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
501 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid
503 aese v0.16b, v27.16b //AES block 4k+4 - round 9
507 aese v3.16b, v25.16b
508 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
511 aese v1.16b, v27.16b //AES block 4k+5 - round 9
514 aese v2.16b, v25.16b
515 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
516 eor v4.16b, v4.16b, v0.16b //AES block 4k+4 - result
519 aese v3.16b, v26.16b
520 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
524 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
526 aese v2.16b, v26.16b
527 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
528 eor v5.16b, v5.16b, v1.16b //AES block 4k+5 - result
538 aese v2.16b, v27.16b //AES block 4k+6 - round 9
539 st1 { v4.16b}, [x2], #16 //AES block 4k+4 - store result
540 eor v6.16b, v6.16b, v2.16b //AES block 4k+6 - result
543 aese v3.16b, v27.16b //AES block 4k+7 - round 9
545 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
548 eor v11.16b, v11.16b, v9.16b //MODULO - fold into low
549 st1 { v5.16b}, [x2], #16 //AES block 4k+5 - store result
552 st1 { v6.16b}, [x2], #16 //AES block 4k+6 - store result
556 eor v7.16b, v7.16b, v3.16b //AES block 4k+3 - result
558 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
559 st1 { v7.16b}, [x2], #16 //AES block 4k+3 - store result
560 b.lt .L128_enc_main_loop
563 rev64 v4.16b, v4.16b //GHASH block 4k (only t0 is free)
565 rev64 v5.16b, v5.16b //GHASH block 4k+1 (t0 and t1 free)
567 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
571 aese v1.16b, v18.16b
572 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
573 rev64 v6.16b, v6.16b //GHASH block 4k+2 (t0, t1, and t2 free)
577 rev64 v7.16b, v7.16b //GHASH block 4k+3 (t0, t1, t2 and t3 free)
578 eor v4.16b, v4.16b, v11.16b //PRE 1
582 aese v3.16b, v18.16b
583 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
592 aese v1.16b, v19.16b
593 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
594 eor v30.8b, v30.8b, v5.8b //GHASH block 4k+1 - mid
596 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
599 eor v31.8b, v31.8b, v6.8b //GHASH block 4k+2 - mid
601 aese v3.16b, v19.16b
602 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
605 eor v11.16b, v11.16b, v29.16b //GHASH block 4k+1 - low
609 aese v0.16b, v18.16b
610 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
613 aese v2.16b, v18.16b
614 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
616 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+1 - mid
619 aese v0.16b, v19.16b
620 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
621 eor v9.16b, v9.16b, v28.16b //GHASH block 4k+1 - high
626 eor v30.8b, v30.8b, v7.8b //GHASH block 4k+3 - mid
632 aese v2.16b, v19.16b
633 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
634 eor v9.16b, v9.16b, v8.16b //GHASH block 4k+2 - high
636 aese v0.16b, v20.16b
637 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
640 movi v8.8b, #0xc2
642 aese v2.16b, v20.16b
643 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
644 eor v11.16b, v11.16b, v28.16b //GHASH block 4k+2 - low
646 aese v3.16b, v20.16b
647 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
650 eor v10.16b, v10.16b, v31.16b //GHASH block 4k+2 - mid
652 aese v2.16b, v21.16b
653 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
655 aese v1.16b, v20.16b
656 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
657 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+3 - high
659 aese v0.16b, v21.16b
660 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
662 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+3 - mid
665 aese v1.16b, v21.16b
666 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
667 eor v11.16b, v11.16b, v29.16b //GHASH block 4k+3 - low
669 aese v0.16b, v22.16b
670 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
673 eor v10.16b, v10.16b, v9.16b //karatsuba tidy up
675 aese v1.16b, v22.16b
676 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
678 aese v0.16b, v23.16b
679 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
680 ext v9.16b, v9.16b, v9.16b, #8
682 aese v3.16b, v21.16b
683 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
685 aese v2.16b, v22.16b
686 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
687 eor v10.16b, v10.16b, v11.16b
689 aese v0.16b, v24.16b
690 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
692 aese v3.16b, v22.16b
693 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
695 aese v1.16b, v23.16b
696 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
698 aese v2.16b, v23.16b
699 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
700 eor v10.16b, v10.16b, v28.16b
702 aese v3.16b, v23.16b
703 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
705 aese v1.16b, v24.16b
706 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
708 aese v2.16b, v24.16b
709 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
711 aese v3.16b, v24.16b
712 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
713 eor v10.16b, v10.16b, v9.16b
715 aese v0.16b, v25.16b
716 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
718 aese v2.16b, v25.16b
719 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
721 aese v3.16b, v25.16b
722 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
726 aese v1.16b, v25.16b
727 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
728 ext v10.16b, v10.16b, v10.16b, #8
730 aese v3.16b, v26.16b
731 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
733 aese v0.16b, v26.16b
734 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
735 eor v11.16b, v11.16b, v28.16b
737 aese v1.16b, v26.16b
738 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
740 aese v3.16b, v27.16b //AES block 4k+7 - round 9
742 aese v2.16b, v26.16b
743 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
745 aese v0.16b, v27.16b //AES block 4k+4 - round 9
747 aese v1.16b, v27.16b //AES block 4k+5 - round 9
748 eor v11.16b, v11.16b, v10.16b
750 aese v2.16b, v27.16b //AES block 4k+6 - round 9
761 ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag
769 eor v5.16b, v4.16b, v0.16b //AES block 4k+4 - result
771 b.gt .L128_enc_blocks_more_than_3
774 movi v11.8b, #0
775 mov v3.16b, v2.16b
778 mov v2.16b, v1.16b
779 movi v9.8b, #0
781 movi v10.8b, #0
782 b.gt .L128_enc_blocks_more_than_2
784 mov v3.16b, v1.16b
788 b.gt .L128_enc_blocks_more_than_1
791 b .L128_enc_blocks_less_than_1
793 st1 { v5.16b}, [x2], #16 //AES final-3 block - store result
800 rev64 v4.16b, v5.16b //GHASH final-3 block
802 eor v4.16b, v4.16b, v8.16b //feed in partial tag
808 movi v8.8b, #0 //suppress further partial tag feed in
818 eor v5.16b, v5.16b, v1.16b //AES final-2 block - result
819 eor v22.8b, v22.8b, v4.8b //GHASH final-3 block - mid
824 st1 { v5.16b}, [x2], #16 //AES final-2 block - store result
826 rev64 v4.16b, v5.16b //GHASH final-2 block
832 eor v4.16b, v4.16b, v8.16b //feed in partial tag
846 eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high
848 eor v22.8b, v22.8b, v4.8b //GHASH final-2 block - mid
850 eor v5.16b, v5.16b, v2.16b //AES final-1 block - result
852 eor v11.16b, v11.16b, v21.16b //GHASH final-2 block - low
856 movi v8.8b, #0 //suppress further partial tag feed in
858 eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid
861 st1 { v5.16b}, [x2], #16 //AES final-1 block - store result
863 rev64 v4.16b, v5.16b //GHASH final-1 block
869 eor v4.16b, v4.16b, v8.16b //feed in partial tag
883 eor v22.8b, v22.8b, v4.8b //GHASH final-1 block - mid
885 eor v5.16b, v5.16b, v3.16b //AES final block - result
891 eor v11.16b, v11.16b, v21.16b //GHASH final-1 block - low
893 eor v9.16b, v9.16b, v20.16b //GHASH final-1 block - high
895 eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid
896 movi v8.8b, #0 //suppress further partial tag feed in
919 …and v5.16b, v5.16b, v0.16b //possibly partial last block has zeroes in …
921 rev64 v4.16b, v5.16b //GHASH final block
923 eor v4.16b, v4.16b, v8.16b //feed in partial tag
928 …ld1 { v18.16b}, [x2] //load existing bytes where the possibly partial l…
930 eor v8.8b, v8.8b, v4.8b //GHASH final block - mid
940 eor v11.16b, v11.16b, v21.16b //GHASH final block - low
942 eor v9.16b, v9.16b, v20.16b //GHASH final block - high
944 eor v10.16b, v10.16b, v8.16b //GHASH final block - mid
945 movi v8.8b, #0xc2
947 eor v30.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
951 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up
955 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
957 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid
959 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
963 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
965 …bif v5.16b, v18.16b, v0.16b //insert existing bytes in top end of res…
967 eor v11.16b, v11.16b, v9.16b //MODULO - fold into low
968 st1 { v5.16b}, [x2] //store all 16B
972 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
973 ext v11.16b, v11.16b, v11.16b, #8
974 rev64 v11.16b, v11.16b
976 st1 { v11.16b }, [x3]
1022 …ld1 { v0.16b}, [x16] //special case vector load initial counter so we …
1026 ext v13.16b, v13.16b, v13.16b, #8
1038 aese v0.16b, v18.16b
1039 aesmc v0.16b, v0.16b //AES block 0 - round 0
1050 aese v0.16b, v19.16b
1051 aesmc v0.16b, v0.16b //AES block 0 - round 1
1064 aese v1.16b, v18.16b
1065 aesmc v1.16b, v1.16b //AES block 1 - round 0
1068 aese v0.16b, v20.16b
1069 aesmc v0.16b, v0.16b //AES block 0 - round 2
1072 aese v2.16b, v18.16b
1073 aesmc v2.16b, v2.16b //AES block 2 - round 0
1076 aese v1.16b, v19.16b
1077 aesmc v1.16b, v1.16b //AES block 1 - round 1
1080 aese v3.16b, v18.16b
1081 aesmc v3.16b, v3.16b //AES block 3 - round 0
1083 aese v2.16b, v19.16b
1084 aesmc v2.16b, v2.16b //AES block 2 - round 1
1086 aese v1.16b, v20.16b
1087 aesmc v1.16b, v1.16b //AES block 1 - round 2
1089 aese v3.16b, v19.16b
1090 aesmc v3.16b, v3.16b //AES block 3 - round 1
1091 ld1 { v11.16b}, [x3]
1092 ext v11.16b, v11.16b, v11.16b, #8
1093 rev64 v11.16b, v11.16b
1095 aese v0.16b, v21.16b
1096 aesmc v0.16b, v0.16b //AES block 0 - round 3
1099 aese v1.16b, v21.16b
1100 aesmc v1.16b, v1.16b //AES block 1 - round 3
1102 aese v3.16b, v20.16b
1103 aesmc v3.16b, v3.16b //AES block 3 - round 2
1105 aese v2.16b, v20.16b
1106 aesmc v2.16b, v2.16b //AES block 2 - round 2
1109 aese v1.16b, v22.16b
1110 aesmc v1.16b, v1.16b //AES block 1 - round 4
1112 aese v3.16b, v21.16b
1113 aesmc v3.16b, v3.16b //AES block 3 - round 3
1115 aese v2.16b, v21.16b
1116 aesmc v2.16b, v2.16b //AES block 2 - round 3
1119 ext v14.16b, v14.16b, v14.16b, #8
1121 aese v0.16b, v22.16b
1122 aesmc v0.16b, v0.16b //AES block 0 - round 4
1125 aese v1.16b, v23.16b
1126 aesmc v1.16b, v1.16b //AES block 1 - round 5
1128 aese v2.16b, v22.16b
1129 aesmc v2.16b, v2.16b //AES block 2 - round 4
1131 aese v3.16b, v22.16b
1132 aesmc v3.16b, v3.16b //AES block 3 - round 4
1134 aese v0.16b, v23.16b
1135 aesmc v0.16b, v0.16b //AES block 0 - round 5
1137 aese v2.16b, v23.16b
1138 aesmc v2.16b, v2.16b //AES block 2 - round 5
1141 ext v12.16b, v12.16b, v12.16b, #8
1143 aese v3.16b, v23.16b
1144 aesmc v3.16b, v3.16b //AES block 3 - round 5
1146 aese v0.16b, v24.16b
1147 aesmc v0.16b, v0.16b //AES block 0 - round 6
1149 aese v1.16b, v24.16b
1150 aesmc v1.16b, v1.16b //AES block 1 - round 6
1152 aese v3.16b, v24.16b
1153 aesmc v3.16b, v3.16b //AES block 3 - round 6
1155 aese v2.16b, v24.16b
1156 aesmc v2.16b, v2.16b //AES block 2 - round 6
1161 ext v15.16b, v15.16b, v15.16b, #8
1166 aese v1.16b, v25.16b
1167 aesmc v1.16b, v1.16b //AES block 1 - round 7
1169 aese v2.16b, v25.16b
1170 aesmc v2.16b, v2.16b //AES block 2 - round 7
1172 aese v0.16b, v25.16b
1173 aesmc v0.16b, v0.16b //AES block 0 - round 7
1174 eor v16.16b, v16.16b, v8.16b //h2k | h1k
1176 aese v3.16b, v25.16b
1177 aesmc v3.16b, v3.16b //AES block 3 - round 7
1179 aese v1.16b, v26.16b
1180 aesmc v1.16b, v1.16b //AES block 1 - round 8
1183 aese v2.16b, v26.16b
1184 aesmc v2.16b, v2.16b //AES block 2 - round 8
1186 aese v3.16b, v26.16b
1187 aesmc v3.16b, v3.16b //AES block 3 - round 8
1189 aese v0.16b, v26.16b
1190 aesmc v0.16b, v0.16b //AES block 0 - round 8
1193 aese v2.16b, v27.16b //AES block 2 - round 9
1195 aese v3.16b, v27.16b //AES block 3 - round 9
1197 aese v0.16b, v27.16b //AES block 0 - round 9
1200 aese v1.16b, v27.16b //AES block 1 - round 9
1201 eor v17.16b, v17.16b, v9.16b //h4k | h3k
1202 b.ge .L128_dec_tail //handle tail
1204 …ld1 {v4.16b, v5.16b}, [x0], #32 //AES block 0 - load ciphertext; AES block 1 - load …
1206 eor v1.16b, v5.16b, v1.16b //AES block 1 - result
1207 ld1 {v6.16b}, [x0], #16 //AES block 2 - load ciphertext
1209 eor v0.16b, v4.16b, v0.16b //AES block 0 - result
1210 rev64 v4.16b, v4.16b //GHASH block 0
1215 ld1 {v7.16b}, [x0], #16 //AES block 3 - load ciphertext
1217 rev64 v5.16b, v5.16b //GHASH block 1
1253 eor v2.16b, v6.16b, v2.16b //AES block 2 - result
1262 b.ge .L128_dec_prepretail //do prepretail
1265 eor v3.16b, v7.16b, v3.16b //AES block 4k+3 - result
1266 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
1272 aese v1.16b, v18.16b
1273 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
1276 rev64 v6.16b, v6.16b //GHASH block 4k+2
1281 eor v4.16b, v4.16b, v11.16b //PRE 1
1284 aese v1.16b, v19.16b
1285 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
1286 rev64 v7.16b, v7.16b //GHASH block 4k+3
1294 eor v30.8b, v30.8b, v5.8b //GHASH block 4k+1 - mid
1296 aese v1.16b, v20.16b
1297 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
1300 aese v2.16b, v18.16b
1301 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
1305 eor v11.16b, v11.16b, v29.16b //GHASH block 4k+1 - low
1309 aese v1.16b, v21.16b
1310 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
1313 aese v3.16b, v18.16b
1314 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
1315 eor v9.16b, v9.16b, v28.16b //GHASH block 4k+1 - high
1317 aese v0.16b, v18.16b
1318 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
1321 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
1323 aese v3.16b, v19.16b
1324 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
1336 aese v0.16b, v19.16b
1337 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
1338 eor v11.16b, v11.16b, v28.16b //GHASH block 4k+2 - low
1342 aese v3.16b, v20.16b
1343 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
1344 eor v31.8b, v31.8b, v6.8b //GHASH block 4k+2 - mid
1346 aese v0.16b, v20.16b
1347 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
1349 aese v1.16b, v22.16b
1350 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
1351 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+1 - mid
1355 aese v0.16b, v21.16b
1356 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
1361 aese v2.16b, v19.16b
1362 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
1365 aese v0.16b, v22.16b
1366 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
1367 eor v9.16b, v9.16b, v8.16b //GHASH block 4k+2 - high
1374 aese v2.16b, v20.16b
1375 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
1376 eor v30.8b, v30.8b, v7.8b //GHASH block 4k+3 - mid
1378 aese v1.16b, v23.16b
1379 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
1384 aese v0.16b, v23.16b
1385 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
1386 movi v8.8b, #0xc2
1388 aese v2.16b, v21.16b
1389 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
1390 eor v11.16b, v11.16b, v29.16b //GHASH block 4k+3 - low
1392 aese v1.16b, v24.16b
1393 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
1395 aese v0.16b, v24.16b
1396 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
1397 eor v10.16b, v10.16b, v31.16b //GHASH block 4k+2 - mid
1399 aese v2.16b, v22.16b
1400 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
1404 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+3 - high
1405 ld1 {v4.16b}, [x0], #16 //AES block 4k+3 - load ciphertext
1407 aese v1.16b, v25.16b
1408 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
1411 aese v0.16b, v25.16b
1412 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
1415 aese v2.16b, v23.16b
1416 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
1417 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+3 - mid
1419 aese v1.16b, v26.16b
1420 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
1423 aese v0.16b, v26.16b
1424 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
1425 eor v30.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
1427 aese v3.16b, v21.16b
1428 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
1432 ld1 {v5.16b}, [x0], #16 //AES block 4k+4 - load ciphertext
1433 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
1435 aese v0.16b, v27.16b //AES block 4k+4 - round 9
1438 aese v3.16b, v22.16b
1439 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
1440 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up
1442 aese v1.16b, v27.16b //AES block 4k+5 - round 9
1444 aese v2.16b, v24.16b
1445 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
1446 eor v0.16b, v4.16b, v0.16b //AES block 4k+4 - result
1448 aese v3.16b, v23.16b
1449 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
1450 ld1 {v6.16b}, [x0], #16 //AES block 4k+5 - load ciphertext
1453 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid
1454 eor v1.16b, v5.16b, v1.16b //AES block 4k+5 - result
1456 aese v2.16b, v25.16b
1457 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
1458 ld1 {v7.16b}, [x0], #16 //AES block 4k+6 - load ciphertext
1460 aese v3.16b, v24.16b
1461 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
1463 rev64 v5.16b, v5.16b //GHASH block 4k+5
1464 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
1467 aese v2.16b, v26.16b
1468 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
1471 aese v3.16b, v25.16b
1472 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
1479 aese v2.16b, v27.16b //AES block 4k+6 - round 9
1481 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
1483 aese v3.16b, v26.16b
1484 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
1489 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low
1495 eor v2.16b, v6.16b, v2.16b //AES block 4k+6 - result
1499 aese v3.16b, v27.16b //AES block 4k+7 - round 9
1503 rev64 v4.16b, v4.16b //GHASH block 4k+4
1504 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
1523 b.lt .L128_dec_main_loop
1526 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
1530 aese v0.16b, v18.16b
1531 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
1532 eor v3.16b, v7.16b, v3.16b //AES block 4k+3 - result
1534 aese v1.16b, v18.16b
1535 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
1538 eor v4.16b, v4.16b, v11.16b //PRE 1
1540 rev64 v6.16b, v6.16b //GHASH block 4k+2
1542 aese v0.16b, v19.16b
1543 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
1548 eor v30.8b, v30.8b, v5.8b //GHASH block 4k+1 - mid
1554 aese v1.16b, v19.16b
1555 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
1558 aese v0.16b, v20.16b
1559 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
1566 aese v2.16b, v18.16b
1567 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
1571 eor v31.8b, v31.8b, v6.8b //GHASH block 4k+2 - mid
1573 rev64 v7.16b, v7.16b //GHASH block 4k+3
1575 aese v2.16b, v19.16b
1576 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
1577 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
1581 aese v3.16b, v18.16b
1582 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
1588 eor v11.16b, v11.16b, v29.16b //GHASH block 4k+1 - low
1593 eor v9.16b, v9.16b, v28.16b //GHASH block 4k+1 - high
1595 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+1 - mid
1602 aese v1.16b, v20.16b
1603 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
1604 eor v10.16b, v10.16b, v31.16b //GHASH block 4k+2 - mid
1608 eor v9.16b, v9.16b, v8.16b //GHASH block 4k+2 - high
1609 movi v8.8b, #0xc2
1611 aese v3.16b, v19.16b
1612 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
1613 eor v30.8b, v30.8b, v7.8b //GHASH block 4k+3 - mid
1615 eor v11.16b, v11.16b, v28.16b //GHASH block 4k+2 - low
1617 aese v2.16b, v20.16b
1618 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
1619 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+3 - high
1621 aese v3.16b, v20.16b
1622 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
1632 eor v11.16b, v11.16b, v29.16b //GHASH block 4k+3 - low
1634 aese v2.16b, v21.16b
1635 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
1637 aese v1.16b, v21.16b
1638 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
1641 aese v0.16b, v21.16b
1642 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
1644 aese v2.16b, v22.16b
1645 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
1646 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+3 - mid
1648 aese v1.16b, v22.16b
1649 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
1651 aese v3.16b, v21.16b
1652 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
1653 eor v30.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
1655 aese v2.16b, v23.16b
1656 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
1658 aese v1.16b, v23.16b
1659 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
1661 aese v3.16b, v22.16b
1662 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
1664 aese v0.16b, v22.16b
1665 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
1666 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up
1670 aese v1.16b, v24.16b
1671 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
1672 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
1674 aese v3.16b, v23.16b
1675 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
1677 aese v0.16b, v23.16b
1678 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
1679 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid
1681 aese v1.16b, v25.16b
1682 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
1684 aese v2.16b, v24.16b
1685 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
1687 aese v0.16b, v24.16b
1688 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
1690 aese v1.16b, v26.16b
1691 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
1692 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
1694 aese v3.16b, v24.16b
1695 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
1697 aese v0.16b, v25.16b
1698 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
1700 aese v1.16b, v27.16b //AES block 4k+5 - round 9
1707 aese v2.16b, v25.16b
1708 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
1709 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
1711 aese v3.16b, v25.16b
1712 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
1714 aese v0.16b, v26.16b
1715 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
1716 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low
1718 aese v2.16b, v26.16b
1719 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
1721 aese v3.16b, v26.16b
1722 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
1727 aese v0.16b, v27.16b //AES block 4k+4 - round 9
1730 aese v2.16b, v27.16b //AES block 4k+6 - round 9
1734 aese v3.16b, v27.16b //AES block 4k+7 - round 9
1735 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
1739 ld1 { v5.16b}, [x0], #16 //AES block 4k+4 - load ciphertext
1741 eor v0.16b, v5.16b, v0.16b //AES block 4k+4 - result
1753 ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag
1758 b.gt .L128_dec_blocks_more_than_3
1760 mov v3.16b, v2.16b
1762 movi v11.8b, #0
1764 movi v9.8b, #0
1765 mov v2.16b, v1.16b
1767 movi v10.8b, #0
1769 b.gt .L128_dec_blocks_more_than_2
1773 mov v3.16b, v1.16b
1775 b.gt .L128_dec_blocks_more_than_1
1778 b .L128_dec_blocks_less_than_1
1780 rev64 v4.16b, v5.16b //GHASH final-3 block
1781 ld1 { v5.16b}, [x0], #16 //AES final-2 block - load ciphertext
1783 eor v4.16b, v4.16b, v8.16b //feed in partial tag
1787 eor v0.16b, v5.16b, v1.16b //AES final-2 block - result
1797 eor v22.8b, v22.8b, v4.8b //GHASH final-3 block - mid
1799 movi v8.8b, #0 //suppress further partial tag feed in
1811 rev64 v4.16b, v5.16b //GHASH final-2 block
1812 ld1 { v5.16b}, [x0], #16 //AES final-1 block - load ciphertext
1814 eor v4.16b, v4.16b, v8.16b //feed in partial tag
1816 eor v0.16b, v5.16b, v2.16b //AES final-1 block - result
1827 eor v22.8b, v22.8b, v4.8b //GHASH final-2 block - mid
1829 movi v8.8b, #0 //suppress further partial tag feed in
1837 eor v11.16b, v11.16b, v21.16b //GHASH final-2 block - low
1839 eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high
1841 eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid
1848 rev64 v4.16b, v5.16b //GHASH final-1 block
1850 ld1 { v5.16b}, [x0], #16 //AES final block - load ciphertext
1851 eor v4.16b, v4.16b, v8.16b //feed in partial tag
1855 eor v0.16b, v5.16b, v3.16b //AES final block - result
1857 eor v22.8b, v22.8b, v4.8b //GHASH final-1 block - mid
1870 movi v8.8b, #0 //suppress further partial tag feed in
1872 eor v11.16b, v11.16b, v21.16b //GHASH final-1 block - low
1874 eor v9.16b, v9.16b, v20.16b //GHASH final-1 block - high
1883 eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid
1906 …and v5.16b, v5.16b, v0.16b //possibly partial last block has zeroes in …
1908 rev64 v4.16b, v5.16b //GHASH final block
1910 eor v4.16b, v4.16b, v8.16b //feed in partial tag
1919 eor v8.8b, v8.8b, v4.8b //GHASH final block - mid
1920 eor v9.16b, v9.16b, v20.16b //GHASH final block - high
1934 eor v10.16b, v10.16b, v8.16b //GHASH final block - mid
1935 movi v8.8b, #0xc2
1937 eor v11.16b, v11.16b, v21.16b //GHASH final block - low
1942 eor v30.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
1946 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up
1953 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
1955 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid
1957 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
1960 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
1962 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low
1964 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
1965 ext v11.16b, v11.16b, v11.16b, #8
1966 rev64 v11.16b, v11.16b
1968 st1 { v11.16b }, [x3]
2030 …ld1 { v0.16b}, [x16] //special case vector load initial counter so we c…
2051 aese v0.16b, v18.16b
2052 aesmc v0.16b, v0.16b //AES block 0 - round 0
2053 ld1 { v11.16b}, [x3]
2054 ext v11.16b, v11.16b, v11.16b, #8
2055 rev64 v11.16b, v11.16b
2057 aese v3.16b, v18.16b
2058 aesmc v3.16b, v3.16b //AES block 3 - round 0
2061 aese v1.16b, v18.16b
2062 aesmc v1.16b, v1.16b //AES block 1 - round 0
2065 ext v15.16b, v15.16b, v15.16b, #8
2067 aese v2.16b, v18.16b
2068 aesmc v2.16b, v2.16b //AES block 2 - round 0
2071 aese v0.16b, v19.16b
2072 aesmc v0.16b, v0.16b //AES block 0 - round 1
2075 aese v1.16b, v19.16b
2076 aesmc v1.16b, v1.16b //AES block 1 - round 1
2079 ext v12.16b, v12.16b, v12.16b, #8
2081 aese v2.16b, v19.16b
2082 aesmc v2.16b, v2.16b //AES block 2 - round 1
2085 aese v3.16b, v19.16b
2086 aesmc v3.16b, v3.16b //AES block 3 - round 1
2089 ext v14.16b, v14.16b, v14.16b, #8
2091 aese v0.16b, v20.16b
2092 aesmc v0.16b, v0.16b //AES block 0 - round 2
2094 aese v2.16b, v20.16b
2095 aesmc v2.16b, v2.16b //AES block 2 - round 2
2097 aese v3.16b, v20.16b
2098 aesmc v3.16b, v3.16b //AES block 3 - round 2
2100 aese v0.16b, v21.16b
2101 aesmc v0.16b, v0.16b //AES block 0 - round 3
2104 aese v2.16b, v21.16b
2105 aesmc v2.16b, v2.16b //AES block 2 - round 3
2107 aese v1.16b, v20.16b
2108 aesmc v1.16b, v1.16b //AES block 1 - round 2
2111 aese v0.16b, v22.16b
2112 aesmc v0.16b, v0.16b //AES block 0 - round 4
2114 aese v3.16b, v21.16b
2115 aesmc v3.16b, v3.16b //AES block 3 - round 3
2117 aese v1.16b, v21.16b
2118 aesmc v1.16b, v1.16b //AES block 1 - round 3
2120 aese v0.16b, v23.16b
2121 aesmc v0.16b, v0.16b //AES block 0 - round 5
2123 aese v2.16b, v22.16b
2124 aesmc v2.16b, v2.16b //AES block 2 - round 4
2126 aese v1.16b, v22.16b
2127 aesmc v1.16b, v1.16b //AES block 1 - round 4
2129 aese v0.16b, v24.16b
2130 aesmc v0.16b, v0.16b //AES block 0 - round 6
2132 aese v3.16b, v22.16b
2133 aesmc v3.16b, v3.16b //AES block 3 - round 4
2135 aese v2.16b, v23.16b
2136 aesmc v2.16b, v2.16b //AES block 2 - round 5
2138 aese v1.16b, v23.16b
2139 aesmc v1.16b, v1.16b //AES block 1 - round 5
2141 aese v3.16b, v23.16b
2142 aesmc v3.16b, v3.16b //AES block 3 - round 5
2144 aese v2.16b, v24.16b
2145 aesmc v2.16b, v2.16b //AES block 2 - round 6
2148 ext v13.16b, v13.16b, v13.16b, #8
2150 aese v1.16b, v24.16b
2151 aesmc v1.16b, v1.16b //AES block 1 - round 6
2153 aese v3.16b, v24.16b
2154 aesmc v3.16b, v3.16b //AES block 3 - round 6
2156 aese v0.16b, v25.16b
2157 aesmc v0.16b, v0.16b //AES block 0 - round 7
2159 aese v1.16b, v25.16b
2160 aesmc v1.16b, v1.16b //AES block 1 - round 7
2163 aese v3.16b, v25.16b
2164 aesmc v3.16b, v3.16b //AES block 3 - round 7
2166 aese v0.16b, v26.16b
2167 aesmc v0.16b, v0.16b //AES block 0 - round 8
2169 aese v2.16b, v25.16b
2170 aesmc v2.16b, v2.16b //AES block 2 - round 7
2173 aese v1.16b, v26.16b
2174 aesmc v1.16b, v1.16b //AES block 1 - round 8
2176 aese v3.16b, v26.16b
2177 aesmc v3.16b, v3.16b //AES block 3 - round 8
2179 aese v2.16b, v26.16b
2180 aesmc v2.16b, v2.16b //AES block 2 - round 8
2182 aese v0.16b, v27.16b
2183 aesmc v0.16b, v0.16b //AES block 0 - round 9
2185 aese v3.16b, v27.16b
2186 aesmc v3.16b, v3.16b //AES block 3 - round 9
2188 aese v2.16b, v27.16b
2189 aesmc v2.16b, v2.16b //AES block 2 - round 9
2191 aese v1.16b, v27.16b
2192 aesmc v1.16b, v1.16b //AES block 1 - round 9
2194 aese v0.16b, v28.16b
2195 aesmc v0.16b, v0.16b //AES block 0 - round 10
2197 aese v2.16b, v28.16b
2198 aesmc v2.16b, v2.16b //AES block 2 - round 10
2200 aese v1.16b, v28.16b
2201 aesmc v1.16b, v1.16b //AES block 1 - round 10
2205 aese v3.16b, v28.16b
2206 aesmc v3.16b, v3.16b //AES block 3 - round 10
2209 eor v16.16b, v16.16b, v8.16b //h2k | h1k
2212 eor v17.16b, v17.16b, v9.16b //h4k | h3k
2214 aese v2.16b, v29.16b //AES block 2 - round 11
2218 aese v1.16b, v29.16b //AES block 1 - round 11
2221 aese v0.16b, v29.16b //AES block 0 - round 11
2224 aese v3.16b, v29.16b //AES block 3 - round 11
2225 b.ge .L192_enc_tail //handle tail
2273 eor v4.16b, v4.16b, v0.16b //AES block 0 - result
2283 st1 { v4.16b}, [x2], #16 //AES block 0 - store result
2287 eor v5.16b, v5.16b, v1.16b //AES block 1 - result
2289 st1 { v5.16b}, [x2], #16 //AES block 1 - store result
2299 eor v6.16b, v6.16b, v2.16b //AES block 2 - result
2306 st1 { v6.16b}, [x2], #16 //AES block 2 - store result
2308 eor v7.16b, v7.16b, v3.16b //AES block 3 - result
2309 st1 { v7.16b}, [x2], #16 //AES block 3 - store result
2310 b.ge .L192_enc_prepretail //do prepretail
2313 aese v2.16b, v18.16b
2314 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
2315 rev64 v5.16b, v5.16b //GHASH block 4k+1 (t0 and t1 free)
2317 aese v1.16b, v18.16b
2318 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
2324 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
2326 rev64 v4.16b, v4.16b //GHASH block 4k (only t0 is free)
2328 aese v2.16b, v19.16b
2329 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
2333 rev64 v7.16b, v7.16b //GHASH block 4k+3 (t0, t1, t2 and t3 free)
2339 aese v0.16b, v18.16b
2340 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
2347 eor v4.16b, v4.16b, v11.16b //PRE 1
2349 aese v1.16b, v19.16b
2350 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
2352 aese v0.16b, v19.16b
2353 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
2354 rev64 v6.16b, v6.16b //GHASH block 4k+2 (t0, t1, and t2 free)
2356 aese v3.16b, v18.16b
2357 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
2363 aese v0.16b, v20.16b
2364 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
2366 aese v3.16b, v19.16b
2367 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
2370 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
2371 eor v11.16b, v11.16b, v31.16b //GHASH block 4k+1 - low
2373 aese v0.16b, v21.16b
2374 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
2377 aese v1.16b, v20.16b
2378 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
2384 aese v2.16b, v20.16b
2385 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
2387 aese v1.16b, v21.16b
2388 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
2391 eor v9.16b, v9.16b, v30.16b //GHASH block 4k+1 - high
2393 aese v3.16b, v20.16b
2394 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
2395 eor v31.8b, v31.8b, v6.8b //GHASH block 4k+2 - mid
2399 aese v0.16b, v22.16b
2400 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
2401 eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid
2403 aese v3.16b, v21.16b
2404 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
2410 aese v0.16b, v23.16b
2411 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
2414 aese v3.16b, v22.16b
2415 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
2416 eor v9.16b, v9.16b, v30.16b //GHASH block 4k+2 - high
2431 aese v2.16b, v21.16b
2432 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
2433 eor v30.8b, v30.8b, v7.8b //GHASH block 4k+3 - mid
2435 aese v1.16b, v22.16b
2436 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
2442 aese v0.16b, v24.16b
2443 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
2444 eor v11.16b, v11.16b, v8.16b //GHASH block 4k+2 - low
2446 aese v2.16b, v22.16b
2447 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
2450 aese v1.16b, v23.16b
2451 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
2452 movi v8.8b, #0xc2
2456 eor v10.16b, v10.16b, v4.16b //GHASH block 4k+1 - mid
2458 aese v2.16b, v23.16b
2459 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
2462 aese v1.16b, v24.16b
2463 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
2466 aese v3.16b, v23.16b
2467 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
2468 eor v9.16b, v9.16b, v5.16b //GHASH block 4k+3 - high
2470 aese v0.16b, v25.16b
2471 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
2474 aese v1.16b, v25.16b
2475 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
2476 eor v10.16b, v10.16b, v31.16b //GHASH block 4k+2 - mid
2478 aese v3.16b, v24.16b
2479 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
2482 aese v0.16b, v26.16b
2483 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
2484 eor v11.16b, v11.16b, v6.16b //GHASH block 4k+3 - low
2490 aese v2.16b, v24.16b
2491 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
2494 aese v1.16b, v26.16b
2495 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
2498 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+3 - mid
2499 eor v30.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
2502 aese v2.16b, v25.16b
2503 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
2507 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
2510 aese v3.16b, v25.16b
2511 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
2513 aese v0.16b, v27.16b
2514 aesmc v0.16b, v0.16b //AES block 4k+4 - round 9
2515 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up
2517 aese v2.16b, v26.16b
2518 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
2520 aese v3.16b, v26.16b
2521 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
2523 aese v1.16b, v27.16b
2524 aesmc v1.16b, v1.16b //AES block 4k+5 - round 9
2526 aese v0.16b, v28.16b
2527 aesmc v0.16b, v0.16b //AES block 4k+4 - round 10
2528 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid
2530 aese v3.16b, v27.16b
2531 aesmc v3.16b, v3.16b //AES block 4k+7 - round 9
2533 aese v2.16b, v27.16b
2534 aesmc v2.16b, v2.16b //AES block 4k+6 - round 9
2536 aese v0.16b, v29.16b //AES block 4k+4 - round 11
2538 aese v1.16b, v28.16b
2539 aesmc v1.16b, v1.16b //AES block 4k+5 - round 10
2540 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
2542 aese v2.16b, v28.16b
2543 aesmc v2.16b, v2.16b //AES block 4k+6 - round 10
2545 eor v4.16b, v4.16b, v0.16b //AES block 4k+4 - result
2548 aese v1.16b, v29.16b //AES block 4k+5 - round 11
2554 st1 { v4.16b}, [x2], #16 //AES block 4k+4 - store result
2556 aese v3.16b, v28.16b
2557 aesmc v3.16b, v3.16b //AES block 4k+7 - round 10
2560 eor v5.16b, v5.16b, v1.16b //AES block 4k+5 - result
2564 aese v2.16b, v29.16b //AES block 4k+6 - round 11
2569 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
2572 st1 { v5.16b}, [x2], #16 //AES block 4k+5 - store result
2573 eor v11.16b, v11.16b, v9.16b //MODULO - fold into low
2575 aese v3.16b, v29.16b //AES block 4k+7 - round 11
2576 eor v6.16b, v6.16b, v2.16b //AES block 4k+6 - result
2579 st1 { v6.16b}, [x2], #16 //AES block 4k+6 - store result
2583 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
2586 eor v7.16b, v7.16b, v3.16b //AES block 4k+3 - result
2587 st1 { v7.16b}, [x2], #16 //AES block 4k+3 - store result
2588 b.lt .L192_enc_main_loop
2591 aese v0.16b, v18.16b
2592 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
2593 rev64 v4.16b, v4.16b //GHASH block 4k (only t0 is free)
2596 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
2599 aese v1.16b, v18.16b
2600 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
2601 rev64 v5.16b, v5.16b //GHASH block 4k+1 (t0 and t1 free)
2603 aese v2.16b, v18.16b
2604 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
2607 eor v4.16b, v4.16b, v11.16b //PRE 1
2610 aese v1.16b, v19.16b
2611 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
2612 rev64 v6.16b, v6.16b //GHASH block 4k+2 (t0, t1, and t2 free)
2620 rev64 v7.16b, v7.16b //GHASH block 4k+3 (t0, t1, t2 and t3 free)
2624 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
2627 eor v11.16b, v11.16b, v31.16b //GHASH block 4k+1 - low
2630 aese v3.16b, v18.16b
2631 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
2632 eor v9.16b, v9.16b, v30.16b //GHASH block 4k+1 - high
2636 eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid
2637 eor v31.8b, v31.8b, v6.8b //GHASH block 4k+2 - mid
2639 aese v3.16b, v19.16b
2640 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
2642 aese v2.16b, v19.16b
2643 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
2644 eor v9.16b, v9.16b, v30.16b //GHASH block 4k+2 - high
2646 aese v0.16b, v19.16b
2647 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
2649 aese v1.16b, v20.16b
2650 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
2656 aese v0.16b, v20.16b
2657 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
2660 eor v30.8b, v30.8b, v7.8b //GHASH block 4k+3 - mid
2662 aese v1.16b, v21.16b
2663 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
2670 eor v9.16b, v9.16b, v5.16b //GHASH block 4k+3 - high
2674 aese v0.16b, v21.16b
2675 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
2676 eor v10.16b, v10.16b, v4.16b //GHASH block 4k+1 - mid
2678 aese v3.16b, v20.16b
2679 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
2681 aese v2.16b, v20.16b
2682 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
2683 eor v11.16b, v11.16b, v8.16b //GHASH block 4k+2 - low
2685 aese v0.16b, v22.16b
2686 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
2688 aese v3.16b, v21.16b
2689 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
2690 eor v10.16b, v10.16b, v31.16b //GHASH block 4k+2 - mid
2692 aese v2.16b, v21.16b
2693 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
2696 movi v8.8b, #0xc2
2698 aese v3.16b, v22.16b
2699 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
2701 aese v2.16b, v22.16b
2702 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
2704 aese v1.16b, v22.16b
2705 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
2706 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+3 - mid
2708 aese v3.16b, v23.16b
2709 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
2711 aese v2.16b, v23.16b
2712 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
2714 aese v1.16b, v23.16b
2715 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
2716 eor v11.16b, v11.16b, v6.16b //GHASH block 4k+3 - low
2718 aese v0.16b, v23.16b
2719 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
2721 aese v3.16b, v24.16b
2722 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
2723 eor v10.16b, v10.16b, v9.16b //karatsuba tidy up
2725 aese v1.16b, v24.16b
2726 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
2728 aese v0.16b, v24.16b
2729 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
2732 aese v3.16b, v25.16b
2733 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
2735 aese v1.16b, v25.16b
2736 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
2737 eor v10.16b, v10.16b, v11.16b
2739 aese v0.16b, v25.16b
2740 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
2744 aese v2.16b, v24.16b
2745 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
2746 ext v9.16b, v9.16b, v9.16b, #8
2748 aese v0.16b, v26.16b
2749 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
2751 aese v1.16b, v26.16b
2752 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
2753 eor v10.16b, v10.16b, v30.16b
2755 aese v2.16b, v25.16b
2756 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
2758 aese v3.16b, v26.16b
2759 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
2761 aese v0.16b, v27.16b
2762 aesmc v0.16b, v0.16b //AES block 4k+4 - round 9
2764 aese v2.16b, v26.16b
2765 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
2766 eor v10.16b, v10.16b, v9.16b
2768 aese v3.16b, v27.16b
2769 aesmc v3.16b, v3.16b //AES block 4k+7 - round 9
2771 aese v1.16b, v27.16b
2772 aesmc v1.16b, v1.16b //AES block 4k+5 - round 9
2774 aese v2.16b, v27.16b
2775 aesmc v2.16b, v2.16b //AES block 4k+6 - round 9
2779 ext v10.16b, v10.16b, v10.16b, #8
2781 aese v3.16b, v28.16b
2782 aesmc v3.16b, v3.16b //AES block 4k+7 - round 10
2784 aese v0.16b, v28.16b
2785 aesmc v0.16b, v0.16b //AES block 4k+4 - round 10
2787 aese v2.16b, v28.16b
2788 aesmc v2.16b, v2.16b //AES block 4k+6 - round 10
2790 aese v1.16b, v28.16b
2791 aesmc v1.16b, v1.16b //AES block 4k+5 - round 10
2792 eor v11.16b, v11.16b, v30.16b
2794 aese v0.16b, v29.16b //AES block 4k+4 - round 11
2796 aese v3.16b, v29.16b //AES block 4k+7 - round 11
2798 aese v2.16b, v29.16b //AES block 4k+6 - round 11
2800 aese v1.16b, v29.16b //AES block 4k+5 - round 11
2801 eor v11.16b, v11.16b, v10.16b
2818 eor v5.16b, v4.16b, v0.16b //AES block 4k+4 - result
2820 ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag
2821 b.gt .L192_enc_blocks_more_than_3
2824 movi v10.8b, #0
2826 mov v3.16b, v2.16b
2827 movi v9.8b, #0
2830 mov v2.16b, v1.16b
2831 movi v11.8b, #0
2832 b.gt .L192_enc_blocks_more_than_2
2836 mov v3.16b, v1.16b
2838 b.gt .L192_enc_blocks_more_than_1
2841 b .L192_enc_blocks_less_than_1
2843 st1 { v5.16b}, [x2], #16 //AES final-3 block - store result
2850 rev64 v4.16b, v5.16b //GHASH final-3 block
2853 eor v4.16b, v4.16b, v8.16b //feed in partial tag
2866 eor v22.8b, v22.8b, v4.8b //GHASH final-3 block - mid
2868 movi v8.8b, #0 //suppress further partial tag feed in
2873 eor v5.16b, v5.16b, v1.16b //AES final-2 block - result
2876 st1 { v5.16b}, [x2], #16 //AES final-2 block - store result
2878 rev64 v4.16b, v5.16b //GHASH final-2 block
2884 eor v4.16b, v4.16b, v8.16b //feed in partial tag
2897 eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high
2898 eor v22.8b, v22.8b, v4.8b //GHASH final-2 block - mid
2900 eor v11.16b, v11.16b, v21.16b //GHASH final-2 block - low
2904 movi v8.8b, #0 //suppress further partial tag feed in
2906 eor v5.16b, v5.16b, v2.16b //AES final-1 block - result
2908 eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid
2911 st1 { v5.16b}, [x2], #16 //AES final-1 block - store result
2918 rev64 v4.16b, v5.16b //GHASH final-1 block
2921 eor v4.16b, v4.16b, v8.16b //feed in partial tag
2922 movi v8.8b, #0 //suppress further partial tag feed in
2926 eor v22.8b, v22.8b, v4.8b //GHASH final-1 block - mid
2935 eor v9.16b, v9.16b, v20.16b //GHASH final-1 block - high
2941 eor v5.16b, v5.16b, v3.16b //AES final block - result
2943 eor v11.16b, v11.16b, v21.16b //GHASH final-1 block - low
2945 eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid
2948 …ld1 { v18.16b}, [x2] //load existing bytes where the possibly partial la…
2974 …and v5.16b, v5.16b, v0.16b //possibly partial last block has zeroes in h…
2976 rev64 v4.16b, v5.16b //GHASH final block
2978 eor v4.16b, v4.16b, v8.16b //feed in partial tag
2986 eor v8.8b, v8.8b, v4.8b //GHASH final block - mid
2988 eor v11.16b, v11.16b, v21.16b //GHASH final block - low
2990 eor v9.16b, v9.16b, v20.16b //GHASH final block - high
2994 eor v10.16b, v10.16b, v8.16b //GHASH final block - mid
2995 movi v8.8b, #0xc2
2997 eor v30.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
3001 …bif v5.16b, v18.16b, v0.16b //insert existing bytes in top end of resu…
3003 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up
3007 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
3009 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid
3011 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
3015 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
3017 eor v11.16b, v11.16b, v9.16b //MODULO - fold into low
3020 st1 { v5.16b}, [x2] //store all 16B
3022 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
3023 ext v11.16b, v11.16b, v11.16b, #8
3024 rev64 v11.16b, v11.16b
3026 st1 { v11.16b }, [x3]
3068 …ld1 { v0.16b}, [x16] //special case vector load initial counter so we …
3086 aese v0.16b, v18.16b
3087 aesmc v0.16b, v0.16b //AES block 0 - round 0
3104 aese v0.16b, v19.16b
3105 aesmc v0.16b, v0.16b //AES block 0 - round 1
3112 aese v0.16b, v20.16b
3113 aesmc v0.16b, v0.16b //AES block 0 - round 2
3115 aese v2.16b, v18.16b
3116 aesmc v2.16b, v2.16b //AES block 2 - round 0
3119 aese v1.16b, v18.16b
3120 aesmc v1.16b, v1.16b //AES block 1 - round 0
3123 ext v15.16b, v15.16b, v15.16b, #8
3125 aese v3.16b, v18.16b
3126 aesmc v3.16b, v3.16b //AES block 3 - round 0
3129 ext v13.16b, v13.16b, v13.16b, #8
3131 aese v2.16b, v19.16b
3132 aesmc v2.16b, v2.16b //AES block 2 - round 1
3135 ext v14.16b, v14.16b, v14.16b, #8
3137 aese v1.16b, v19.16b
3138 aesmc v1.16b, v1.16b //AES block 1 - round 1
3140 aese v3.16b, v19.16b
3141 aesmc v3.16b, v3.16b //AES block 3 - round 1
3144 ext v12.16b, v12.16b, v12.16b, #8
3146 aese v2.16b, v20.16b
3147 aesmc v2.16b, v2.16b //AES block 2 - round 2
3150 aese v0.16b, v21.16b
3151 aesmc v0.16b, v0.16b //AES block 0 - round 3
3154 aese v1.16b, v20.16b
3155 aesmc v1.16b, v1.16b //AES block 1 - round 2
3158 aese v3.16b, v20.16b
3159 aesmc v3.16b, v3.16b //AES block 3 - round 2
3162 aese v2.16b, v21.16b
3163 aesmc v2.16b, v2.16b //AES block 2 - round 3
3164 ld1 { v11.16b}, [x3]
3165 ext v11.16b, v11.16b, v11.16b, #8
3166 rev64 v11.16b, v11.16b
3168 aese v1.16b, v21.16b
3169 aesmc v1.16b, v1.16b //AES block 1 - round 3
3172 aese v3.16b, v21.16b
3173 aesmc v3.16b, v3.16b //AES block 3 - round 3
3176 aese v0.16b, v22.16b
3177 aesmc v0.16b, v0.16b //AES block 0 - round 4
3180 aese v1.16b, v22.16b
3181 aesmc v1.16b, v1.16b //AES block 1 - round 4
3184 aese v2.16b, v22.16b
3185 aesmc v2.16b, v2.16b //AES block 2 - round 4
3187 aese v3.16b, v22.16b
3188 aesmc v3.16b, v3.16b //AES block 3 - round 4
3191 aese v0.16b, v23.16b
3192 aesmc v0.16b, v0.16b //AES block 0 - round 5
3195 aese v1.16b, v23.16b
3196 aesmc v1.16b, v1.16b //AES block 1 - round 5
3198 aese v2.16b, v23.16b
3199 aesmc v2.16b, v2.16b //AES block 2 - round 5
3201 aese v3.16b, v23.16b
3202 aesmc v3.16b, v3.16b //AES block 3 - round 5
3204 aese v0.16b, v24.16b
3205 aesmc v0.16b, v0.16b //AES block 0 - round 6
3207 aese v2.16b, v24.16b
3208 aesmc v2.16b, v2.16b //AES block 2 - round 6
3210 aese v3.16b, v24.16b
3211 aesmc v3.16b, v3.16b //AES block 3 - round 6
3213 aese v0.16b, v25.16b
3214 aesmc v0.16b, v0.16b //AES block 0 - round 7
3216 aese v2.16b, v25.16b
3217 aesmc v2.16b, v2.16b //AES block 2 - round 7
3219 aese v3.16b, v25.16b
3220 aesmc v3.16b, v3.16b //AES block 3 - round 7
3222 aese v1.16b, v24.16b
3223 aesmc v1.16b, v1.16b //AES block 1 - round 6
3225 aese v2.16b, v26.16b
3226 aesmc v2.16b, v2.16b //AES block 2 - round 8
3228 aese v3.16b, v26.16b
3229 aesmc v3.16b, v3.16b //AES block 3 - round 8
3231 aese v1.16b, v25.16b
3232 aesmc v1.16b, v1.16b //AES block 1 - round 7
3234 aese v2.16b, v27.16b
3235 aesmc v2.16b, v2.16b //AES block 2 - round 9
3237 aese v3.16b, v27.16b
3238 aesmc v3.16b, v3.16b //AES block 3 - round 9
3240 aese v1.16b, v26.16b
3241 aesmc v1.16b, v1.16b //AES block 1 - round 8
3244 aese v0.16b, v26.16b
3245 aesmc v0.16b, v0.16b //AES block 0 - round 8
3248 aese v3.16b, v28.16b
3249 aesmc v3.16b, v3.16b //AES block 3 - round 10
3252 aese v1.16b, v27.16b
3253 aesmc v1.16b, v1.16b //AES block 1 - round 9
3256 aese v0.16b, v27.16b
3257 aesmc v0.16b, v0.16b //AES block 0 - round 9
3260 aese v3.16b, v29.16b //AES block 3 - round 11
3262 aese v2.16b, v28.16b
3263 aesmc v2.16b, v2.16b //AES block 2 - round 10
3265 aese v1.16b, v28.16b
3266 aesmc v1.16b, v1.16b //AES block 1 - round 10
3268 aese v0.16b, v28.16b
3269 aesmc v0.16b, v0.16b //AES block 0 - round 10
3270 eor v16.16b, v16.16b, v8.16b //h2k | h1k
3272 aese v2.16b, v29.16b //AES block 2 - round 11
3274 aese v1.16b, v29.16b //AES block 1 - round 11
3275 eor v17.16b, v17.16b, v9.16b //h4k | h3k
3277 aese v0.16b, v29.16b //AES block 0 - round 11
3278 b.ge .L192_dec_tail //handle tail
3280 ld1 {v4.16b, v5.16b}, [x0], #32 //AES block 0,1 - load ciphertext
3282 eor v1.16b, v5.16b, v1.16b //AES block 1 - result
3284 eor v0.16b, v4.16b, v0.16b //AES block 0 - result
3286 ld1 {v6.16b, v7.16b}, [x0], #32 //AES block 2,3 - load ciphertext
3297 rev64 v4.16b, v4.16b //GHASH block 0
3300 rev64 v5.16b, v5.16b //GHASH block 1
3333 eor v2.16b, v6.16b, v2.16b //AES block 2 - result
3334 b.ge .L192_dec_prepretail //do prepretail
3337 aese v1.16b, v18.16b
3338 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
3339 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
3345 eor v3.16b, v7.16b, v3.16b //AES block 4k+3 - result
3346 rev64 v7.16b, v7.16b //GHASH block 4k+3
3348 aese v1.16b, v19.16b
3349 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
3352 aese v0.16b, v18.16b
3353 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
3354 eor v4.16b, v4.16b, v11.16b //PRE 1
3359 aese v1.16b, v20.16b
3360 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
3363 aese v0.16b, v19.16b
3364 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
3375 aese v2.16b, v18.16b
3376 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
3380 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
3383 aese v1.16b, v21.16b
3384 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
3386 aese v0.16b, v20.16b
3387 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
3392 aese v2.16b, v19.16b
3393 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
3394 eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid
3398 aese v3.16b, v18.16b
3399 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
3400 rev64 v6.16b, v6.16b //GHASH block 4k+2
3402 aese v2.16b, v20.16b
3403 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
3406 eor v11.16b, v11.16b, v31.16b //GHASH block 4k+1 - low
3411 aese v1.16b, v22.16b
3412 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
3414 aese v0.16b, v21.16b
3415 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
3417 eor v10.16b, v10.16b, v4.16b //GHASH block 4k+1 - mid
3420 aese v3.16b, v19.16b
3421 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
3422 eor v9.16b, v9.16b, v30.16b //GHASH block 4k+1 - high
3424 aese v0.16b, v22.16b
3425 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
3428 eor v31.8b, v31.8b, v6.8b //GHASH block 4k+2 - mid
3432 aese v0.16b, v23.16b
3433 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
3435 eor v9.16b, v9.16b, v30.16b //GHASH block 4k+2 - high
3438 aese v1.16b, v23.16b
3439 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
3443 aese v3.16b, v20.16b
3444 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
3445 eor v30.8b, v30.8b, v7.8b //GHASH block 4k+3 - mid
3447 aese v1.16b, v24.16b
3448 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
3450 aese v0.16b, v24.16b
3451 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
3454 aese v3.16b, v21.16b
3455 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
3458 eor v11.16b, v11.16b, v8.16b //GHASH block 4k+2 - low
3460 aese v0.16b, v25.16b
3461 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
3464 eor v9.16b, v9.16b, v5.16b //GHASH block 4k+3 - high
3466 aese v1.16b, v25.16b
3467 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
3469 aese v0.16b, v26.16b
3470 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
3471 movi v8.8b, #0xc2
3475 aese v1.16b, v26.16b
3476 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
3477 eor v10.16b, v10.16b, v31.16b //GHASH block 4k+2 - mid
3479 aese v2.16b, v21.16b
3480 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
3482 aese v0.16b, v27.16b
3483 aesmc v0.16b, v0.16b //AES block 4k+4 - round 9
3484 eor v11.16b, v11.16b, v6.16b //GHASH block 4k+3 - low
3486 aese v3.16b, v22.16b
3487 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
3489 aese v2.16b, v22.16b
3490 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
3491 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+3 - mid
3493 aese v0.16b, v28.16b
3494 aesmc v0.16b, v0.16b //AES block 4k+4 - round 10
3496 aese v1.16b, v27.16b
3497 aesmc v1.16b, v1.16b //AES block 4k+5 - round 9
3498 eor v30.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
3500 aese v2.16b, v23.16b
3501 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
3503 aese v3.16b, v23.16b
3504 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
3507 aese v1.16b, v28.16b
3508 aesmc v1.16b, v1.16b //AES block 4k+5 - round 10
3510 aese v2.16b, v24.16b
3511 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
3512 ld1 {v4.16b}, [x0], #16 //AES block 4k+4 - load ciphertext
3514 aese v3.16b, v24.16b
3515 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
3516 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up
3519 ld1 {v5.16b}, [x0], #16 //AES block 4k+5 - load ciphertext
3524 aese v2.16b, v25.16b
3525 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
3526 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
3528 aese v0.16b, v29.16b //AES block 4k+4 - round 11
3531 aese v3.16b, v25.16b
3532 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
3533 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid
3535 aese v2.16b, v26.16b
3536 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
3537 ld1 {v6.16b}, [x0], #16 //AES block 4k+6 - load ciphertext
3539 aese v1.16b, v29.16b //AES block 4k+5 - round 11
3540 ld1 {v7.16b}, [x0], #16 //AES block 4k+7 - load ciphertext
3543 aese v3.16b, v26.16b
3544 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
3547 aese v2.16b, v27.16b
3548 aesmc v2.16b, v2.16b //AES block 4k+6 - round 9
3549 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
3553 eor v0.16b, v4.16b, v0.16b //AES block 4k+4 - result
3558 eor v1.16b, v5.16b, v1.16b //AES block 4k+5 - result
3560 aese v2.16b, v28.16b
3561 aesmc v2.16b, v2.16b //AES block 4k+6 - round 10
3564 aese v3.16b, v27.16b
3565 aesmc v3.16b, v3.16b //AES block 4k+7 - round 9
3572 rev64 v5.16b, v5.16b //GHASH block 4k+5
3574 aese v2.16b, v29.16b //AES block 4k+6 - round 11
3577 aese v3.16b, v28.16b
3578 aesmc v3.16b, v3.16b //AES block 4k+7 - round 10
3583 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
3585 eor v2.16b, v6.16b, v2.16b //AES block 4k+6 - result
3594 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low
3613 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
3616 rev64 v4.16b, v4.16b //GHASH block 4k+4
3619 aese v3.16b, v29.16b //AES block 4k+7 - round 11
3621 b.lt .L192_dec_main_loop
3625 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
3626 eor v3.16b, v7.16b, v3.16b //AES block 4k+3 - result
3628 aese v1.16b, v18.16b
3629 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
3632 aese v0.16b, v18.16b
3633 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
3636 eor v4.16b, v4.16b, v11.16b //PRE 1
3639 aese v1.16b, v19.16b
3640 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
3643 aese v0.16b, v19.16b
3644 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
3651 aese v1.16b, v20.16b
3652 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
3653 rev64 v6.16b, v6.16b //GHASH block 4k+2
3660 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
3670 aese v0.16b, v20.16b
3671 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
3681 eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid
3690 rev64 v7.16b, v7.16b //GHASH block 4k+3
3693 aese v3.16b, v18.16b
3694 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
3695 eor v9.16b, v9.16b, v30.16b //GHASH block 4k+1 - high
3701 eor v11.16b, v11.16b, v31.16b //GHASH block 4k+1 - low
3703 aese v2.16b, v18.16b
3704 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
3706 eor v10.16b, v10.16b, v4.16b //GHASH block 4k+1 - mid
3709 aese v3.16b, v19.16b
3710 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
3712 aese v2.16b, v19.16b
3713 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
3714 eor v9.16b, v9.16b, v30.16b //GHASH block 4k+2 - high
3716 eor v31.8b, v31.8b, v6.8b //GHASH block 4k+2 - mid
3720 aese v2.16b, v20.16b
3721 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
3724 aese v3.16b, v20.16b
3725 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
3730 aese v0.16b, v21.16b
3731 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
3732 eor v30.8b, v30.8b, v7.8b //GHASH block 4k+3 - mid
3734 aese v1.16b, v21.16b
3735 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
3738 eor v11.16b, v11.16b, v8.16b //GHASH block 4k+2 - low
3740 aese v0.16b, v22.16b
3741 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
3744 movi v8.8b, #0xc2
3748 aese v2.16b, v21.16b
3749 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
3752 eor v9.16b, v9.16b, v5.16b //GHASH block 4k+3 - high
3754 aese v0.16b, v23.16b
3755 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
3756 eor v10.16b, v10.16b, v31.16b //GHASH block 4k+2 - mid
3758 aese v2.16b, v22.16b
3759 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
3762 eor v11.16b, v11.16b, v6.16b //GHASH block 4k+3 - low
3764 aese v0.16b, v24.16b
3765 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
3767 aese v3.16b, v21.16b
3768 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
3769 eor v10.16b, v10.16b, v30.16b //GHASH block 4k+3 - mid
3771 aese v2.16b, v23.16b
3772 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
3774 aese v0.16b, v25.16b
3775 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
3776 eor v30.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
3778 aese v3.16b, v22.16b
3779 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
3781 aese v2.16b, v24.16b
3782 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
3783 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
3785 aese v0.16b, v26.16b
3786 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
3788 aese v3.16b, v23.16b
3789 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
3790 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up
3792 aese v1.16b, v22.16b
3793 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
3795 aese v2.16b, v25.16b
3796 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
3798 aese v0.16b, v27.16b
3799 aesmc v0.16b, v0.16b //AES block 4k+4 - round 9
3801 aese v1.16b, v23.16b
3802 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
3804 aese v3.16b, v24.16b
3805 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
3806 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid
3808 aese v0.16b, v28.16b
3809 aesmc v0.16b, v0.16b //AES block 4k+4 - round 10
3811 aese v1.16b, v24.16b
3812 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
3814 aese v3.16b, v25.16b
3815 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
3817 aese v2.16b, v26.16b
3818 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
3819 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
3821 aese v1.16b, v25.16b
3822 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
3824 aese v3.16b, v26.16b
3825 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
3827 aese v2.16b, v27.16b
3828 aesmc v2.16b, v2.16b //AES block 4k+6 - round 9
3830 aese v1.16b, v26.16b
3831 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
3833 aese v3.16b, v27.16b
3834 aesmc v3.16b, v3.16b //AES block 4k+7 - round 9
3838 aese v1.16b, v27.16b
3839 aesmc v1.16b, v1.16b //AES block 4k+5 - round 9
3841 aese v2.16b, v28.16b
3842 aesmc v2.16b, v2.16b //AES block 4k+6 - round 10
3844 aese v3.16b, v28.16b
3845 aesmc v3.16b, v3.16b //AES block 4k+7 - round 10
3846 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
3848 aese v1.16b, v28.16b
3849 aesmc v1.16b, v1.16b //AES block 4k+5 - round 10
3851 aese v0.16b, v29.16b
3852 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low
3854 aese v2.16b, v29.16b
3856 aese v1.16b, v29.16b
3858 aese v3.16b, v29.16b
3860 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
3864 ld1 { v5.16b}, [x0], #16 //AES block 4k+4 - load ciphertext
3866 eor v0.16b, v5.16b, v0.16b //AES block 4k+4 - result
3872 ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag
3884 b.gt .L192_dec_blocks_more_than_3
3886 movi v11.8b, #0
3887 movi v9.8b, #0
3889 mov v3.16b, v2.16b
3890 mov v2.16b, v1.16b
3893 movi v10.8b, #0
3895 b.gt .L192_dec_blocks_more_than_2
3897 mov v3.16b, v1.16b
3901 b.gt .L192_dec_blocks_more_than_1
3904 b .L192_dec_blocks_less_than_1
3906 rev64 v4.16b, v5.16b //GHASH final-3 block
3907 ld1 { v5.16b}, [x0], #16 //AES final-2 block - load ciphertext
3911 eor v4.16b, v4.16b, v8.16b //feed in partial tag
3913 eor v0.16b, v5.16b, v1.16b //AES final-2 block - result
3922 eor v22.8b, v22.8b, v4.8b //GHASH final-3 block - mid
3930 movi v8.8b, #0 //suppress further partial tag feed in
3939 rev64 v4.16b, v5.16b //GHASH final-2 block
3940 ld1 { v5.16b}, [x0], #16 //AES final-1 block - load ciphertext
3942 eor v4.16b, v4.16b, v8.16b //feed in partial tag
3944 movi v8.8b, #0 //suppress further partial tag feed in
3946 eor v0.16b, v5.16b, v2.16b //AES final-1 block - result
3954 eor v22.8b, v22.8b, v4.8b //GHASH final-2 block - mid
3957 eor v11.16b, v11.16b, v21.16b //GHASH final-2 block - low
3964 eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high
3973 eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid
3976 rev64 v4.16b, v5.16b //GHASH final-1 block
3978 eor v4.16b, v4.16b, v8.16b //feed in partial tag
3979 ld1 { v5.16b}, [x0], #16 //AES final block - load ciphertext
3985 eor v0.16b, v5.16b, v3.16b //AES final block - result
3988 eor v22.8b, v22.8b, v4.8b //GHASH final-1 block - mid
3990 eor v9.16b, v9.16b, v20.16b //GHASH final-1 block - high
4000 movi v8.8b, #0 //suppress further partial tag feed in
4001 eor v11.16b, v11.16b, v21.16b //GHASH final-1 block - low
4010 eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid
4042 …and v5.16b, v5.16b, v0.16b //possibly partial last block has zeroes in …
4045 rev64 v4.16b, v5.16b //GHASH final block
4047 eor v4.16b, v4.16b, v8.16b //feed in partial tag
4057 eor v8.8b, v8.8b, v4.8b //GHASH final block - mid
4059 eor v9.16b, v9.16b, v20.16b //GHASH final block - high
4063 eor v11.16b, v11.16b, v21.16b //GHASH final block - low
4065 eor v10.16b, v10.16b, v8.16b //GHASH final block - mid
4066 movi v8.8b, #0xc2
4068 eor v30.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
4072 eor v10.16b, v10.16b, v30.16b //MODULO - karatsuba tidy up
4078 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
4080 eor v10.16b, v10.16b, v31.16b //MODULO - fold into mid
4082 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
4086 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low
4088 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
4090 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
4091 ext v11.16b, v11.16b, v11.16b, #8
4092 rev64 v11.16b, v11.16b
4094 st1 { v11.16b }, [x3]
4138 …ld1 { v0.16b}, [x16] //special case vector load initial counter so we …
4155 aese v0.16b, v18.16b
4156 aesmc v0.16b, v0.16b //AES block 0 - round 0
4176 aese v0.16b, v19.16b
4177 aesmc v0.16b, v0.16b //AES block 0 - round 1
4182 aese v1.16b, v18.16b
4183 aesmc v1.16b, v1.16b //AES block 1 - round 0
4186 aese v0.16b, v20.16b
4187 aesmc v0.16b, v0.16b //AES block 0 - round 2
4190 aese v2.16b, v18.16b
4191 aesmc v2.16b, v2.16b //AES block 2 - round 0
4194 aese v1.16b, v19.16b
4195 aesmc v1.16b, v1.16b //AES block 1 - round 1
4198 ext v14.16b, v14.16b, v14.16b, #8
4200 aese v3.16b, v18.16b
4201 aesmc v3.16b, v3.16b //AES block 3 - round 0
4204 aese v2.16b, v19.16b
4205 aesmc v2.16b, v2.16b //AES block 2 - round 1
4208 aese v1.16b, v20.16b
4209 aesmc v1.16b, v1.16b //AES block 1 - round 2
4212 ext v13.16b, v13.16b, v13.16b, #8
4214 aese v3.16b, v19.16b
4215 aesmc v3.16b, v3.16b //AES block 3 - round 1
4218 aese v2.16b, v20.16b
4219 aesmc v2.16b, v2.16b //AES block 2 - round 2
4222 ext v15.16b, v15.16b, v15.16b, #8
4224 aese v1.16b, v21.16b
4225 aesmc v1.16b, v1.16b //AES block 1 - round 3
4228 aese v3.16b, v20.16b
4229 aesmc v3.16b, v3.16b //AES block 3 - round 2
4232 aese v2.16b, v21.16b
4233 aesmc v2.16b, v2.16b //AES block 2 - round 3
4236 aese v0.16b, v21.16b
4237 aesmc v0.16b, v0.16b //AES block 0 - round 3
4239 aese v3.16b, v21.16b
4240 aesmc v3.16b, v3.16b //AES block 3 - round 3
4241 ld1 { v11.16b}, [x3]
4242 ext v11.16b, v11.16b, v11.16b, #8
4243 rev64 v11.16b, v11.16b
4245 aese v2.16b, v22.16b
4246 aesmc v2.16b, v2.16b //AES block 2 - round 4
4248 aese v0.16b, v22.16b
4249 aesmc v0.16b, v0.16b //AES block 0 - round 4
4251 aese v1.16b, v22.16b
4252 aesmc v1.16b, v1.16b //AES block 1 - round 4
4254 aese v3.16b, v22.16b
4255 aesmc v3.16b, v3.16b //AES block 3 - round 4
4257 aese v0.16b, v23.16b
4258 aesmc v0.16b, v0.16b //AES block 0 - round 5
4260 aese v1.16b, v23.16b
4261 aesmc v1.16b, v1.16b //AES block 1 - round 5
4263 aese v3.16b, v23.16b
4264 aesmc v3.16b, v3.16b //AES block 3 - round 5
4266 aese v2.16b, v23.16b
4267 aesmc v2.16b, v2.16b //AES block 2 - round 5
4269 aese v1.16b, v24.16b
4270 aesmc v1.16b, v1.16b //AES block 1 - round 6
4273 aese v3.16b, v24.16b
4274 aesmc v3.16b, v3.16b //AES block 3 - round 6
4277 aese v0.16b, v24.16b
4278 aesmc v0.16b, v0.16b //AES block 0 - round 6
4281 ext v12.16b, v12.16b, v12.16b, #8
4283 aese v2.16b, v24.16b
4284 aesmc v2.16b, v2.16b //AES block 2 - round 6
4287 aese v1.16b, v25.16b
4288 aesmc v1.16b, v1.16b //AES block 1 - round 7
4291 aese v0.16b, v25.16b
4292 aesmc v0.16b, v0.16b //AES block 0 - round 7
4294 aese v2.16b, v25.16b
4295 aesmc v2.16b, v2.16b //AES block 2 - round 7
4297 aese v3.16b, v25.16b
4298 aesmc v3.16b, v3.16b //AES block 3 - round 7
4301 aese v1.16b, v26.16b
4302 aesmc v1.16b, v1.16b //AES block 1 - round 8
4304 aese v2.16b, v26.16b
4305 aesmc v2.16b, v2.16b //AES block 2 - round 8
4307 aese v3.16b, v26.16b
4308 aesmc v3.16b, v3.16b //AES block 3 - round 8
4310 aese v1.16b, v27.16b
4311 aesmc v1.16b, v1.16b //AES block 1 - round 9
4313 aese v2.16b, v27.16b
4314 aesmc v2.16b, v2.16b //AES block 2 - round 9
4316 aese v0.16b, v26.16b
4317 aesmc v0.16b, v0.16b //AES block 0 - round 8
4319 aese v1.16b, v28.16b
4320 aesmc v1.16b, v1.16b //AES block 1 - round 10
4322 aese v3.16b, v27.16b
4323 aesmc v3.16b, v3.16b //AES block 3 - round 9
4325 aese v0.16b, v27.16b
4326 aesmc v0.16b, v0.16b //AES block 0 - round 9
4328 aese v2.16b, v28.16b
4329 aesmc v2.16b, v2.16b //AES block 2 - round 10
4331 aese v3.16b, v28.16b
4332 aesmc v3.16b, v3.16b //AES block 3 - round 10
4334 aese v1.16b, v29.16b
4335 aesmc v1.16b, v1.16b //AES block 1 - round 11
4337 aese v2.16b, v29.16b
4338 aesmc v2.16b, v2.16b //AES block 2 - round 11
4340 aese v0.16b, v28.16b
4341 aesmc v0.16b, v0.16b //AES block 0 - round 10
4343 aese v1.16b, v30.16b
4344 aesmc v1.16b, v1.16b //AES block 1 - round 12
4346 aese v2.16b, v30.16b
4347 aesmc v2.16b, v2.16b //AES block 2 - round 12
4349 aese v0.16b, v29.16b
4350 aesmc v0.16b, v0.16b //AES block 0 - round 11
4351 eor v17.16b, v17.16b, v9.16b //h4k | h3k
4353 aese v3.16b, v29.16b
4354 aesmc v3.16b, v3.16b //AES block 3 - round 11
4356 aese v2.16b, v31.16b //AES block 2 - round 13
4359 aese v0.16b, v30.16b
4360 aesmc v0.16b, v0.16b //AES block 0 - round 12
4362 aese v3.16b, v30.16b
4363 aesmc v3.16b, v3.16b //AES block 3 - round 12
4365 aese v1.16b, v31.16b //AES block 1 - round 13
4367 aese v0.16b, v31.16b //AES block 0 - round 13
4369 aese v3.16b, v31.16b //AES block 3 - round 13
4370 eor v16.16b, v16.16b, v8.16b //h2k | h1k
4371 b.ge .L256_enc_tail //handle tail
4422 eor v4.16b, v4.16b, v0.16b //AES block 0 - result
4429 eor v5.16b, v5.16b, v1.16b //AES block 1 - result
4435 st1 { v4.16b}, [x2], #16 //AES block 0 - store result
4439 eor v6.16b, v6.16b, v2.16b //AES block 2 - result
4441 st1 { v5.16b}, [x2], #16 //AES block 1 - store result
4447 st1 { v6.16b}, [x2], #16 //AES block 2 - store result
4452 eor v7.16b, v7.16b, v3.16b //AES block 3 - result
4453 st1 { v7.16b}, [x2], #16 //AES block 3 - store result
4454 b.ge .L256_enc_prepretail //do prepretail
4457 aese v0.16b, v18.16b
4458 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
4459 rev64 v4.16b, v4.16b //GHASH block 4k (only t0 is free)
4461 aese v1.16b, v18.16b
4462 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
4465 aese v2.16b, v18.16b
4466 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
4467 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
4469 aese v0.16b, v19.16b
4470 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
4473 aese v1.16b, v19.16b
4474 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
4480 aese v2.16b, v19.16b
4481 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
4487 aese v0.16b, v20.16b
4488 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
4489 eor v4.16b, v4.16b, v11.16b //PRE 1
4491 aese v1.16b, v20.16b
4492 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
4494 aese v3.16b, v18.16b
4495 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
4498 aese v0.16b, v21.16b
4499 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
4506 aese v3.16b, v19.16b
4507 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
4508 rev64 v5.16b, v5.16b //GHASH block 4k+1 (t0 and t1 free)
4510 aese v0.16b, v22.16b
4511 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
4514 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
4516 aese v2.16b, v20.16b
4517 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
4519 aese v0.16b, v23.16b
4520 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
4521 rev64 v7.16b, v7.16b //GHASH block 4k+3 (t0, t1, t2 and t3 free)
4526 rev64 v6.16b, v6.16b //GHASH block 4k+2 (t0, t1, and t2 free)
4530 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+1 - high
4533 aese v1.16b, v21.16b
4534 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
4536 aese v3.16b, v20.16b
4537 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
4538 eor v11.16b, v11.16b, v8.16b //GHASH block 4k+1 - low
4540 aese v2.16b, v21.16b
4541 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
4543 aese v1.16b, v22.16b
4544 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
4547 aese v3.16b, v21.16b
4548 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
4549 eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid
4551 aese v2.16b, v22.16b
4552 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
4554 aese v0.16b, v24.16b
4555 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
4556 eor v8.8b, v8.8b, v6.8b //GHASH block 4k+2 - mid
4558 aese v3.16b, v22.16b
4559 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
4563 aese v0.16b, v25.16b
4564 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
4566 aese v3.16b, v23.16b
4567 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
4570 aese v1.16b, v23.16b
4571 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
4573 aese v0.16b, v26.16b
4574 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
4576 aese v2.16b, v23.16b
4577 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
4579 aese v1.16b, v24.16b
4580 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
4581 eor v10.16b, v10.16b, v4.16b //GHASH block 4k+1 - mid
4587 aese v1.16b, v25.16b
4588 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
4591 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+2 - high
4593 aese v3.16b, v24.16b
4594 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
4600 aese v1.16b, v26.16b
4601 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
4604 aese v2.16b, v24.16b
4605 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
4606 eor v11.16b, v11.16b, v5.16b //GHASH block 4k+2 - low
4611 eor v4.8b, v4.8b, v7.8b //GHASH block 4k+3 - mid
4613 aese v2.16b, v25.16b
4614 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
4617 aese v1.16b, v27.16b
4618 aesmc v1.16b, v1.16b //AES block 4k+5 - round 9
4619 eor v10.16b, v10.16b, v8.16b //GHASH block 4k+2 - mid
4621 aese v3.16b, v25.16b
4622 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
4625 aese v0.16b, v27.16b
4626 aesmc v0.16b, v0.16b //AES block 4k+4 - round 9
4627 movi v8.8b, #0xc2
4630 eor v9.16b, v9.16b, v5.16b //GHASH block 4k+3 - high
4633 aese v2.16b, v26.16b
4634 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
4640 aese v0.16b, v28.16b
4641 aesmc v0.16b, v0.16b //AES block 4k+4 - round 10
4644 aese v3.16b, v26.16b
4645 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
4646 eor v11.16b, v11.16b, v6.16b //GHASH block 4k+3 - low
4648 aese v2.16b, v27.16b
4649 aesmc v2.16b, v2.16b //AES block 4k+6 - round 9
4651 aese v1.16b, v28.16b
4652 aesmc v1.16b, v1.16b //AES block 4k+5 - round 10
4653 eor v10.16b, v10.16b, v4.16b //GHASH block 4k+3 - mid
4655 aese v3.16b, v27.16b
4656 aesmc v3.16b, v3.16b //AES block 4k+7 - round 9
4659 aese v0.16b, v29.16b
4660 aesmc v0.16b, v0.16b //AES block 4k+4 - round 11
4661 eor v4.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
4663 aese v1.16b, v29.16b
4664 aesmc v1.16b, v1.16b //AES block 4k+5 - round 11
4669 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
4671 aese v2.16b, v28.16b
4672 aesmc v2.16b, v2.16b //AES block 4k+6 - round 10
4675 aese v1.16b, v30.16b
4676 aesmc v1.16b, v1.16b //AES block 4k+5 - round 12
4677 eor v10.16b, v10.16b, v4.16b //MODULO - karatsuba tidy up
4679 aese v3.16b, v28.16b
4680 aesmc v3.16b, v3.16b //AES block 4k+7 - round 10
4685 eor v7.16b, v9.16b, v7.16b //MODULO - fold into mid
4687 aese v0.16b, v30.16b
4688 aesmc v0.16b, v0.16b //AES block 4k+4 - round 12
4691 aese v2.16b, v29.16b
4692 aesmc v2.16b, v2.16b //AES block 4k+6 - round 11
4695 aese v3.16b, v29.16b
4696 aesmc v3.16b, v3.16b //AES block 4k+7 - round 11
4699 aese v0.16b, v31.16b //AES block 4k+4 - round 13
4701 eor v10.16b, v10.16b, v7.16b //MODULO - fold into mid
4703 aese v2.16b, v30.16b
4704 aesmc v2.16b, v2.16b //AES block 4k+6 - round 12
4707 aese v1.16b, v31.16b //AES block 4k+5 - round 13
4716 eor v4.16b, v4.16b, v0.16b //AES block 4k+4 - result
4723 eor v5.16b, v5.16b, v1.16b //AES block 4k+5 - result
4727 aese v3.16b, v30.16b
4728 aesmc v3.16b, v3.16b //AES block 4k+7 - round 12
4731 aese v2.16b, v31.16b //AES block 4k+6 - round 13
4733 st1 { v4.16b}, [x2], #16 //AES block 4k+4 - store result
4736 eor v11.16b, v11.16b, v9.16b //MODULO - fold into low
4739 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
4740 st1 { v5.16b}, [x2], #16 //AES block 4k+5 - store result
4743 aese v3.16b, v31.16b //AES block 4k+7 - round 13
4744 eor v6.16b, v6.16b, v2.16b //AES block 4k+6 - result
4747 st1 { v6.16b}, [x2], #16 //AES block 4k+6 - store result
4751 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
4754 eor v7.16b, v7.16b, v3.16b //AES block 4k+7 - result
4755 st1 { v7.16b}, [x2], #16 //AES block 4k+7 - store result
4756 b.lt .L256_enc_main_loop
4759 aese v1.16b, v18.16b
4760 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
4761 rev64 v6.16b, v6.16b //GHASH block 4k+2 (t0, t1, and t2 free)
4763 aese v2.16b, v18.16b
4764 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
4767 aese v0.16b, v18.16b
4768 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
4769 rev64 v4.16b, v4.16b //GHASH block 4k (only t0 is free)
4772 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
4774 aese v2.16b, v19.16b
4775 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
4777 aese v0.16b, v19.16b
4778 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
4780 eor v4.16b, v4.16b, v11.16b //PRE 1
4781 rev64 v5.16b, v5.16b //GHASH block 4k+1 (t0 and t1 free)
4783 aese v2.16b, v20.16b
4784 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
4786 aese v3.16b, v18.16b
4787 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
4790 aese v1.16b, v19.16b
4791 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
4798 aese v2.16b, v21.16b
4799 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
4801 aese v1.16b, v20.16b
4802 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
4803 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
4805 aese v0.16b, v20.16b
4806 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
4808 aese v3.16b, v19.16b
4809 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
4811 aese v1.16b, v21.16b
4812 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
4820 aese v3.16b, v20.16b
4821 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
4823 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+1 - high
4826 aese v0.16b, v21.16b
4827 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
4828 eor v11.16b, v11.16b, v8.16b //GHASH block 4k+1 - low
4830 aese v3.16b, v21.16b
4831 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
4833 eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid
4836 aese v0.16b, v22.16b
4837 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
4838 rev64 v7.16b, v7.16b //GHASH block 4k+3 (t0, t1, t2 and t3 free)
4840 aese v3.16b, v22.16b
4841 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
4844 eor v8.8b, v8.8b, v6.8b //GHASH block 4k+2 - mid
4849 aese v3.16b, v23.16b
4850 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
4852 aese v2.16b, v22.16b
4853 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
4854 eor v10.16b, v10.16b, v4.16b //GHASH block 4k+1 - mid
4858 eor v11.16b, v11.16b, v5.16b //GHASH block 4k+2 - low
4861 aese v2.16b, v23.16b
4862 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
4864 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+2 - high
4867 aese v1.16b, v22.16b
4868 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
4872 eor v4.8b, v4.8b, v7.8b //GHASH block 4k+3 - mid
4876 aese v1.16b, v23.16b
4877 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
4880 eor v10.16b, v10.16b, v8.16b //GHASH block 4k+2 - mid
4882 aese v0.16b, v23.16b
4883 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
4885 aese v1.16b, v24.16b
4886 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
4888 aese v2.16b, v24.16b
4889 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
4891 aese v0.16b, v24.16b
4892 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
4893 movi v8.8b, #0xc2
4895 aese v3.16b, v24.16b
4896 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
4898 aese v1.16b, v25.16b
4899 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
4900 eor v9.16b, v9.16b, v5.16b //GHASH block 4k+3 - high
4902 aese v0.16b, v25.16b
4903 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
4905 aese v3.16b, v25.16b
4906 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
4909 aese v1.16b, v26.16b
4910 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
4911 eor v10.16b, v10.16b, v4.16b //GHASH block 4k+3 - mid
4915 aese v3.16b, v26.16b
4916 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
4918 aese v1.16b, v27.16b
4919 aesmc v1.16b, v1.16b //AES block 4k+5 - round 9
4921 aese v0.16b, v26.16b
4922 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
4923 eor v11.16b, v11.16b, v6.16b //GHASH block 4k+3 - low
4925 aese v3.16b, v27.16b
4926 aesmc v3.16b, v3.16b //AES block 4k+7 - round 9
4928 eor v10.16b, v10.16b, v9.16b //karatsuba tidy up
4931 ext v9.16b, v9.16b, v9.16b, #8
4933 aese v3.16b, v28.16b
4934 aesmc v3.16b, v3.16b //AES block 4k+7 - round 10
4936 aese v2.16b, v25.16b
4937 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
4938 eor v10.16b, v10.16b, v11.16b
4940 aese v1.16b, v28.16b
4941 aesmc v1.16b, v1.16b //AES block 4k+5 - round 10
4943 aese v0.16b, v27.16b
4944 aesmc v0.16b, v0.16b //AES block 4k+4 - round 9
4946 aese v2.16b, v26.16b
4947 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
4949 aese v1.16b, v29.16b
4950 aesmc v1.16b, v1.16b //AES block 4k+5 - round 11
4951 eor v10.16b, v10.16b, v4.16b
4953 aese v0.16b, v28.16b
4954 aesmc v0.16b, v0.16b //AES block 4k+4 - round 10
4956 aese v2.16b, v27.16b
4957 aesmc v2.16b, v2.16b //AES block 4k+6 - round 9
4959 aese v1.16b, v30.16b
4960 aesmc v1.16b, v1.16b //AES block 4k+5 - round 12
4962 aese v0.16b, v29.16b
4963 aesmc v0.16b, v0.16b //AES block 4k+4 - round 11
4964 eor v10.16b, v10.16b, v9.16b
4966 aese v3.16b, v29.16b
4967 aesmc v3.16b, v3.16b //AES block 4k+7 - round 11
4969 aese v2.16b, v28.16b
4970 aesmc v2.16b, v2.16b //AES block 4k+6 - round 10
4972 aese v0.16b, v30.16b
4973 aesmc v0.16b, v0.16b //AES block 4k+4 - round 12
4977 aese v2.16b, v29.16b
4978 aesmc v2.16b, v2.16b //AES block 4k+6 - round 11
4979 ext v10.16b, v10.16b, v10.16b, #8
4981 aese v3.16b, v30.16b
4982 aesmc v3.16b, v3.16b //AES block 4k+7 - round 12
4984 aese v1.16b, v31.16b //AES block 4k+5 - round 13
4985 eor v11.16b, v11.16b, v4.16b
4987 aese v2.16b, v30.16b
4988 aesmc v2.16b, v2.16b //AES block 4k+6 - round 12
4990 aese v3.16b, v31.16b //AES block 4k+7 - round 13
4992 aese v0.16b, v31.16b //AES block 4k+4 - round 13
4994 aese v2.16b, v31.16b //AES block 4k+6 - round 13
4995 eor v11.16b, v11.16b, v10.16b
4998 ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag
5013 eor v5.16b, v4.16b, v0.16b //AES block 4k+4 - result
5014 b.gt .L256_enc_blocks_more_than_3
5017 mov v3.16b, v2.16b
5018 movi v11.8b, #0
5020 movi v9.8b, #0
5023 mov v2.16b, v1.16b
5024 movi v10.8b, #0
5025 b.gt .L256_enc_blocks_more_than_2
5027 mov v3.16b, v1.16b
5031 b.gt .L256_enc_blocks_more_than_1
5034 b .L256_enc_blocks_less_than_1
5036 st1 { v5.16b}, [x2], #16 //AES final-3 block - store result
5043 rev64 v4.16b, v5.16b //GHASH final-3 block
5046 eor v4.16b, v4.16b, v8.16b //feed in partial tag
5055 eor v22.8b, v22.8b, v4.8b //GHASH final-3 block - mid
5056 movi v8.8b, #0 //suppress further partial tag feed in
5065 eor v5.16b, v5.16b, v1.16b //AES final-2 block - result
5068 st1 { v5.16b}, [x2], #16 //AES final-2 block - store result
5075 rev64 v4.16b, v5.16b //GHASH final-2 block
5078 eor v4.16b, v4.16b, v8.16b //feed in partial tag
5085 movi v8.8b, #0 //suppress further partial tag feed in
5092 eor v22.8b, v22.8b, v4.8b //GHASH final-2 block - mid
5094 eor v5.16b, v5.16b, v2.16b //AES final-1 block - result
5096 eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high
5100 eor v11.16b, v11.16b, v21.16b //GHASH final-2 block - low
5102 eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid
5105 st1 { v5.16b}, [x2], #16 //AES final-1 block - store result
5107 rev64 v4.16b, v5.16b //GHASH final-1 block
5114 eor v4.16b, v4.16b, v8.16b //feed in partial tag
5116 movi v8.8b, #0 //suppress further partial tag feed in
5124 eor v22.8b, v22.8b, v4.8b //GHASH final-1 block - mid
5126 eor v9.16b, v9.16b, v20.16b //GHASH final-1 block - high
5137 eor v5.16b, v5.16b, v3.16b //AES final block - result
5138 eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid
5140 eor v11.16b, v11.16b, v21.16b //GHASH final-1 block - low
5149 …ld1 { v18.16b}, [x2] //load existing bytes where the possibly partial la…
5164 …and v5.16b, v5.16b, v0.16b //possibly partial last block has zeroes in h…
5166 rev64 v4.16b, v5.16b //GHASH final block
5168 eor v4.16b, v4.16b, v8.16b //feed in partial tag
5170 …bif v5.16b, v18.16b, v0.16b //insert existing bytes in top end of resu…
5182 eor v9.16b, v9.16b, v20.16b //GHASH final block - high
5183 eor v8.8b, v8.8b, v4.8b //GHASH final block - mid
5187 eor v11.16b, v11.16b, v21.16b //GHASH final block - low
5189 eor v10.16b, v10.16b, v8.16b //GHASH final block - mid
5190 movi v8.8b, #0xc2
5192 eor v4.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
5196 eor v10.16b, v10.16b, v4.16b //MODULO - karatsuba tidy up
5200 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
5202 eor v10.16b, v10.16b, v7.16b //MODULO - fold into mid
5204 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
5208 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
5212 st1 { v5.16b}, [x2] //store all 16B
5213 eor v11.16b, v11.16b, v9.16b //MODULO - fold into low
5215 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
5216 ext v11.16b, v11.16b, v11.16b, #8
5217 rev64 v11.16b, v11.16b
5219 st1 { v11.16b }, [x3]
5287 …ld1 { v0.16b}, [x16] //special case vector load initial counter so we …
5311 aese v0.16b, v18.16b
5312 aesmc v0.16b, v0.16b //AES block 0 - round 0
5315 ext v14.16b, v14.16b, v14.16b, #8
5318 aese v3.16b, v18.16b
5319 aesmc v3.16b, v3.16b //AES block 3 - round 0
5322 ext v15.16b, v15.16b, v15.16b, #8
5325 aese v1.16b, v18.16b
5326 aesmc v1.16b, v1.16b //AES block 1 - round 0
5329 ext v13.16b, v13.16b, v13.16b, #8
5332 aese v2.16b, v18.16b
5333 aesmc v2.16b, v2.16b //AES block 2 - round 0
5336 aese v0.16b, v19.16b
5337 aesmc v0.16b, v0.16b //AES block 0 - round 1
5339 aese v1.16b, v19.16b
5340 aesmc v1.16b, v1.16b //AES block 1 - round 1
5341 ld1 { v11.16b}, [x3]
5342 ext v11.16b, v11.16b, v11.16b, #8
5343 rev64 v11.16b, v11.16b
5345 aese v2.16b, v19.16b
5346 aesmc v2.16b, v2.16b //AES block 2 - round 1
5349 aese v3.16b, v19.16b
5350 aesmc v3.16b, v3.16b //AES block 3 - round 1
5353 aese v0.16b, v20.16b
5354 aesmc v0.16b, v0.16b //AES block 0 - round 2
5357 ext v12.16b, v12.16b, v12.16b, #8
5359 aese v2.16b, v20.16b
5360 aesmc v2.16b, v2.16b //AES block 2 - round 2
5363 aese v3.16b, v20.16b
5364 aesmc v3.16b, v3.16b //AES block 3 - round 2
5366 aese v0.16b, v21.16b
5367 aesmc v0.16b, v0.16b //AES block 0 - round 3
5369 aese v1.16b, v20.16b
5370 aesmc v1.16b, v1.16b //AES block 1 - round 2
5372 aese v3.16b, v21.16b
5373 aesmc v3.16b, v3.16b //AES block 3 - round 3
5375 aese v0.16b, v22.16b
5376 aesmc v0.16b, v0.16b //AES block 0 - round 4
5379 aese v2.16b, v21.16b
5380 aesmc v2.16b, v2.16b //AES block 2 - round 3
5382 aese v1.16b, v21.16b
5383 aesmc v1.16b, v1.16b //AES block 1 - round 3
5385 aese v3.16b, v22.16b
5386 aesmc v3.16b, v3.16b //AES block 3 - round 4
5388 aese v2.16b, v22.16b
5389 aesmc v2.16b, v2.16b //AES block 2 - round 4
5391 aese v1.16b, v22.16b
5392 aesmc v1.16b, v1.16b //AES block 1 - round 4
5394 aese v3.16b, v23.16b
5395 aesmc v3.16b, v3.16b //AES block 3 - round 5
5397 aese v0.16b, v23.16b
5398 aesmc v0.16b, v0.16b //AES block 0 - round 5
5400 aese v1.16b, v23.16b
5401 aesmc v1.16b, v1.16b //AES block 1 - round 5
5403 aese v2.16b, v23.16b
5404 aesmc v2.16b, v2.16b //AES block 2 - round 5
5406 aese v0.16b, v24.16b
5407 aesmc v0.16b, v0.16b //AES block 0 - round 6
5409 aese v3.16b, v24.16b
5410 aesmc v3.16b, v3.16b //AES block 3 - round 6
5412 aese v1.16b, v24.16b
5413 aesmc v1.16b, v1.16b //AES block 1 - round 6
5415 aese v2.16b, v24.16b
5416 aesmc v2.16b, v2.16b //AES block 2 - round 6
5418 aese v0.16b, v25.16b
5419 aesmc v0.16b, v0.16b //AES block 0 - round 7
5421 aese v1.16b, v25.16b
5422 aesmc v1.16b, v1.16b //AES block 1 - round 7
5424 aese v3.16b, v25.16b
5425 aesmc v3.16b, v3.16b //AES block 3 - round 7
5427 aese v0.16b, v26.16b
5428 aesmc v0.16b, v0.16b //AES block 0 - round 8
5430 aese v2.16b, v25.16b
5431 aesmc v2.16b, v2.16b //AES block 2 - round 7
5433 aese v3.16b, v26.16b
5434 aesmc v3.16b, v3.16b //AES block 3 - round 8
5436 aese v1.16b, v26.16b
5437 aesmc v1.16b, v1.16b //AES block 1 - round 8
5439 aese v0.16b, v27.16b
5440 aesmc v0.16b, v0.16b //AES block 0 - round 9
5442 aese v2.16b, v26.16b
5443 aesmc v2.16b, v2.16b //AES block 2 - round 8
5446 aese v1.16b, v27.16b
5447 aesmc v1.16b, v1.16b //AES block 1 - round 9
5449 aese v0.16b, v28.16b
5450 aesmc v0.16b, v0.16b //AES block 0 - round 10
5452 aese v3.16b, v27.16b
5453 aesmc v3.16b, v3.16b //AES block 3 - round 9
5455 aese v1.16b, v28.16b
5456 aesmc v1.16b, v1.16b //AES block 1 - round 10
5458 aese v2.16b, v27.16b
5459 aesmc v2.16b, v2.16b //AES block 2 - round 9
5461 aese v3.16b, v28.16b
5462 aesmc v3.16b, v3.16b //AES block 3 - round 10
5464 aese v0.16b, v29.16b
5465 aesmc v0.16b, v0.16b //AES block 0 - round 11
5467 aese v2.16b, v28.16b
5468 aesmc v2.16b, v2.16b //AES block 2 - round 10
5470 aese v3.16b, v29.16b
5471 aesmc v3.16b, v3.16b //AES block 3 - round 11
5473 aese v1.16b, v29.16b
5474 aesmc v1.16b, v1.16b //AES block 1 - round 11
5476 aese v2.16b, v29.16b
5477 aesmc v2.16b, v2.16b //AES block 2 - round 11
5486 aese v1.16b, v30.16b
5487 aesmc v1.16b, v1.16b //AES block 1 - round 12
5489 aese v0.16b, v30.16b
5490 aesmc v0.16b, v0.16b //AES block 0 - round 12
5492 aese v2.16b, v30.16b
5493 aesmc v2.16b, v2.16b //AES block 2 - round 12
5495 aese v3.16b, v30.16b
5496 aesmc v3.16b, v3.16b //AES block 3 - round 12
5497 eor v17.16b, v17.16b, v9.16b //h4k | h3k
5499 aese v1.16b, v31.16b //AES block 1 - round 13
5501 aese v2.16b, v31.16b //AES block 2 - round 13
5502 eor v16.16b, v16.16b, v8.16b //h2k | h1k
5504 aese v3.16b, v31.16b //AES block 3 - round 13
5506 aese v0.16b, v31.16b //AES block 0 - round 13
5507 b.ge .L256_dec_tail //handle tail
5509 ld1 {v4.16b, v5.16b}, [x0], #32 //AES block 0,1 - load ciphertext
5513 eor v0.16b, v4.16b, v0.16b //AES block 0 - result
5515 eor v1.16b, v5.16b, v1.16b //AES block 1 - result
5516 rev64 v5.16b, v5.16b //GHASH block 1
5517 ld1 {v6.16b}, [x0], #16 //AES block 2 - load ciphertext
5522 rev64 v4.16b, v4.16b //GHASH block 0
5547 ld1 {v7.16b}, [x0], #16 //AES block 3 - load ciphertext
5565 eor v2.16b, v6.16b, v2.16b //AES block 2 - result
5567 b.ge .L256_dec_prepretail //do prepretail
5571 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
5572 eor v3.16b, v7.16b, v3.16b //AES block 4k+3 - result
5574 aese v0.16b, v18.16b
5575 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
5578 aese v1.16b, v18.16b
5579 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
5583 eor v4.16b, v4.16b, v11.16b //PRE 1
5586 aese v0.16b, v19.16b
5587 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
5590 aese v1.16b, v19.16b
5591 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
5598 aese v0.16b, v20.16b
5599 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
5602 aese v2.16b, v18.16b
5603 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
5606 aese v1.16b, v20.16b
5607 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
5608 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
5610 aese v0.16b, v21.16b
5611 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
5616 aese v2.16b, v19.16b
5617 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
5620 aese v1.16b, v21.16b
5621 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
5622 rev64 v6.16b, v6.16b //GHASH block 4k+2
5624 aese v3.16b, v18.16b
5625 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
5630 aese v2.16b, v20.16b
5631 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
5638 aese v2.16b, v21.16b
5639 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
5640 rev64 v7.16b, v7.16b //GHASH block 4k+3
5652 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+1 - high
5654 aese v2.16b, v22.16b
5655 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
5657 aese v3.16b, v19.16b
5658 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
5661 aese v0.16b, v22.16b
5662 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
5663 eor v11.16b, v11.16b, v8.16b //GHASH block 4k+1 - low
5665 aese v2.16b, v23.16b
5666 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
5669 aese v3.16b, v20.16b
5670 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
5673 aese v1.16b, v22.16b
5674 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
5675 eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid
5679 aese v3.16b, v21.16b
5680 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
5681 eor v8.8b, v8.8b, v6.8b //GHASH block 4k+2 - mid
5683 aese v1.16b, v23.16b
5684 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
5686 aese v0.16b, v23.16b
5687 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
5688 eor v11.16b, v11.16b, v5.16b //GHASH block 4k+2 - low
5693 aese v1.16b, v24.16b
5694 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
5697 aese v0.16b, v24.16b
5698 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
5701 aese v3.16b, v22.16b
5702 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
5704 aese v1.16b, v25.16b
5705 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
5706 eor v10.16b, v10.16b, v4.16b //GHASH block 4k+1 - mid
5708 aese v0.16b, v25.16b
5709 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
5714 aese v3.16b, v23.16b
5715 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
5719 aese v0.16b, v26.16b
5720 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
5721 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+2 - high
5723 aese v3.16b, v24.16b
5724 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
5728 eor v10.16b, v10.16b, v8.16b //GHASH block 4k+2 - mid
5732 aese v0.16b, v27.16b
5733 aesmc v0.16b, v0.16b //AES block 4k+4 - round 9
5734 eor v6.8b, v6.8b, v7.8b //GHASH block 4k+3 - mid
5736 aese v1.16b, v26.16b
5737 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
5739 aese v2.16b, v24.16b
5740 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
5741 eor v9.16b, v9.16b, v5.16b //GHASH block 4k+3 - high
5743 aese v0.16b, v28.16b
5744 aesmc v0.16b, v0.16b //AES block 4k+4 - round 10
5747 movi v8.8b, #0xc2
5749 aese v2.16b, v25.16b
5750 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
5751 eor v11.16b, v11.16b, v4.16b //GHASH block 4k+3 - low
5753 aese v0.16b, v29.16b
5754 aesmc v0.16b, v0.16b //AES block 4k+4 - round 11
5756 aese v3.16b, v25.16b
5757 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
5760 aese v2.16b, v26.16b
5761 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
5762 eor v10.16b, v10.16b, v6.16b //GHASH block 4k+3 - mid
5764 aese v0.16b, v30.16b
5765 aesmc v0.16b, v0.16b //AES block 4k+4 - round 12
5768 eor v6.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
5770 aese v1.16b, v27.16b
5771 aesmc v1.16b, v1.16b //AES block 4k+5 - round 9
5772 ld1 {v4.16b}, [x0], #16 //AES block 4k+4 - load ciphertext
5774 aese v0.16b, v31.16b //AES block 4k+4 - round 13
5775 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
5777 aese v1.16b, v28.16b
5778 aesmc v1.16b, v1.16b //AES block 4k+5 - round 10
5779 eor v10.16b, v10.16b, v6.16b //MODULO - karatsuba tidy up
5781 aese v2.16b, v27.16b
5782 aesmc v2.16b, v2.16b //AES block 4k+6 - round 9
5783 ld1 {v5.16b}, [x0], #16 //AES block 4k+5 - load ciphertext
5785 aese v3.16b, v26.16b
5786 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
5787 eor v0.16b, v4.16b, v0.16b //AES block 4k+4 - result
5789 aese v1.16b, v29.16b
5790 aesmc v1.16b, v1.16b //AES block 4k+5 - round 11
5793 aese v2.16b, v28.16b
5794 aesmc v2.16b, v2.16b //AES block 4k+6 - round 10
5795 eor v10.16b, v10.16b, v7.16b //MODULO - fold into mid
5797 aese v3.16b, v27.16b
5798 aesmc v3.16b, v3.16b //AES block 4k+7 - round 9
5799 ld1 {v6.16b}, [x0], #16 //AES block 4k+6 - load ciphertext
5801 aese v1.16b, v30.16b
5802 aesmc v1.16b, v1.16b //AES block 4k+5 - round 12
5803 ld1 {v7.16b}, [x0], #16 //AES block 4k+7 - load ciphertext
5805 aese v2.16b, v29.16b
5806 aesmc v2.16b, v2.16b //AES block 4k+6 - round 11
5809 aese v3.16b, v28.16b
5810 aesmc v3.16b, v3.16b //AES block 4k+7 - round 10
5811 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
5813 aese v1.16b, v31.16b //AES block 4k+5 - round 13
5816 aese v2.16b, v30.16b
5817 aesmc v2.16b, v2.16b //AES block 4k+6 - round 12
5820 aese v3.16b, v29.16b
5821 aesmc v3.16b, v3.16b //AES block 4k+7 - round 11
5825 eor v1.16b, v5.16b, v1.16b //AES block 4k+5 - result
5828 aese v2.16b, v31.16b //AES block 4k+6 - round 13
5843 eor v2.16b, v6.16b, v2.16b //AES block 4k+6 - result
5844 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low
5846 aese v3.16b, v30.16b
5847 aesmc v3.16b, v3.16b //AES block 4k+7 - round 12
5851 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
5857 aese v3.16b, v31.16b //AES block 4k+7 - round 13
5860 rev64 v5.16b, v5.16b //GHASH block 4k+5
5873 rev64 v4.16b, v4.16b //GHASH block 4k+4
5874 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
5875 b.lt .L256_dec_main_loop
5879 ext v11.16b, v11.16b, v11.16b, #8 //PRE 0
5881 eor v3.16b, v7.16b, v3.16b //AES block 4k+3 - result
5883 aese v0.16b, v18.16b
5884 aesmc v0.16b, v0.16b //AES block 4k+4 - round 0
5887 aese v1.16b, v18.16b
5888 aesmc v1.16b, v1.16b //AES block 4k+5 - round 0
5893 eor v4.16b, v4.16b, v11.16b //PRE 1
5895 rev64 v6.16b, v6.16b //GHASH block 4k+2
5899 aese v1.16b, v19.16b
5900 aesmc v1.16b, v1.16b //AES block 4k+5 - round 1
5910 aese v2.16b, v18.16b
5911 aesmc v2.16b, v2.16b //AES block 4k+6 - round 0
5914 aese v0.16b, v19.16b
5915 aesmc v0.16b, v0.16b //AES block 4k+4 - round 1
5916 eor v8.8b, v8.8b, v4.8b //GHASH block 4k - mid
5920 aese v2.16b, v19.16b
5921 aesmc v2.16b, v2.16b //AES block 4k+6 - round 1
5922 rev64 v7.16b, v7.16b //GHASH block 4k+3
5924 aese v3.16b, v18.16b
5925 aesmc v3.16b, v3.16b //AES block 4k+7 - round 0
5928 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+1 - high
5932 aese v3.16b, v19.16b
5933 aesmc v3.16b, v3.16b //AES block 4k+7 - round 1
5936 aese v0.16b, v20.16b
5937 aesmc v0.16b, v0.16b //AES block 4k+4 - round 2
5939 aese v1.16b, v20.16b
5940 aesmc v1.16b, v1.16b //AES block 4k+5 - round 2
5941 eor v11.16b, v11.16b, v8.16b //GHASH block 4k+1 - low
5943 aese v2.16b, v20.16b
5944 aesmc v2.16b, v2.16b //AES block 4k+6 - round 2
5946 aese v0.16b, v21.16b
5947 aesmc v0.16b, v0.16b //AES block 4k+4 - round 3
5950 aese v3.16b, v20.16b
5951 aesmc v3.16b, v3.16b //AES block 4k+7 - round 2
5952 eor v4.8b, v4.8b, v5.8b //GHASH block 4k+1 - mid
5956 aese v0.16b, v22.16b
5957 aesmc v0.16b, v0.16b //AES block 4k+4 - round 4
5959 aese v3.16b, v21.16b
5960 aesmc v3.16b, v3.16b //AES block 4k+7 - round 3
5961 eor v8.8b, v8.8b, v6.8b //GHASH block 4k+2 - mid
5965 aese v0.16b, v23.16b
5966 aesmc v0.16b, v0.16b //AES block 4k+4 - round 5
5967 eor v11.16b, v11.16b, v5.16b //GHASH block 4k+2 - low
5969 aese v3.16b, v22.16b
5970 aesmc v3.16b, v3.16b //AES block 4k+7 - round 4
5973 eor v10.16b, v10.16b, v4.16b //GHASH block 4k+1 - mid
5977 aese v3.16b, v23.16b
5978 aesmc v3.16b, v3.16b //AES block 4k+7 - round 5
5981 aese v2.16b, v21.16b
5982 aesmc v2.16b, v2.16b //AES block 4k+6 - round 3
5984 aese v1.16b, v21.16b
5985 aesmc v1.16b, v1.16b //AES block 4k+5 - round 3
5986 eor v9.16b, v9.16b, v4.16b //GHASH block 4k+2 - high
5990 aese v2.16b, v22.16b
5991 aesmc v2.16b, v2.16b //AES block 4k+6 - round 4
5994 aese v1.16b, v22.16b
5995 aesmc v1.16b, v1.16b //AES block 4k+5 - round 4
5999 aese v2.16b, v23.16b
6000 aesmc v2.16b, v2.16b //AES block 4k+6 - round 5
6001 eor v6.8b, v6.8b, v7.8b //GHASH block 4k+3 - mid
6003 aese v1.16b, v23.16b
6004 aesmc v1.16b, v1.16b //AES block 4k+5 - round 5
6006 aese v3.16b, v24.16b
6007 aesmc v3.16b, v3.16b //AES block 4k+7 - round 6
6008 eor v10.16b, v10.16b, v8.16b //GHASH block 4k+2 - mid
6010 aese v2.16b, v24.16b
6011 aesmc v2.16b, v2.16b //AES block 4k+6 - round 6
6013 aese v0.16b, v24.16b
6014 aesmc v0.16b, v0.16b //AES block 4k+4 - round 6
6015 movi v8.8b, #0xc2
6017 aese v1.16b, v24.16b
6018 aesmc v1.16b, v1.16b //AES block 4k+5 - round 6
6019 eor v11.16b, v11.16b, v4.16b //GHASH block 4k+3 - low
6023 aese v3.16b, v25.16b
6024 aesmc v3.16b, v3.16b //AES block 4k+7 - round 7
6025 eor v9.16b, v9.16b, v5.16b //GHASH block 4k+3 - high
6027 aese v1.16b, v25.16b
6028 aesmc v1.16b, v1.16b //AES block 4k+5 - round 7
6030 aese v0.16b, v25.16b
6031 aesmc v0.16b, v0.16b //AES block 4k+4 - round 7
6032 eor v10.16b, v10.16b, v6.16b //GHASH block 4k+3 - mid
6034 aese v3.16b, v26.16b
6035 aesmc v3.16b, v3.16b //AES block 4k+7 - round 8
6037 aese v2.16b, v25.16b
6038 aesmc v2.16b, v2.16b //AES block 4k+6 - round 7
6039 eor v6.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
6041 aese v1.16b, v26.16b
6042 aesmc v1.16b, v1.16b //AES block 4k+5 - round 8
6044 aese v0.16b, v26.16b
6045 aesmc v0.16b, v0.16b //AES block 4k+4 - round 8
6048 aese v2.16b, v26.16b
6049 aesmc v2.16b, v2.16b //AES block 4k+6 - round 8
6051 aese v1.16b, v27.16b
6052 aesmc v1.16b, v1.16b //AES block 4k+5 - round 9
6053 eor v10.16b, v10.16b, v6.16b //MODULO - karatsuba tidy up
6057 aese v2.16b, v27.16b
6058 aesmc v2.16b, v2.16b //AES block 4k+6 - round 9
6059 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
6061 aese v3.16b, v27.16b
6062 aesmc v3.16b, v3.16b //AES block 4k+7 - round 9
6064 aese v0.16b, v27.16b
6065 aesmc v0.16b, v0.16b //AES block 4k+4 - round 9
6066 eor v10.16b, v10.16b, v7.16b //MODULO - fold into mid
6068 aese v2.16b, v28.16b
6069 aesmc v2.16b, v2.16b //AES block 4k+6 - round 10
6071 aese v3.16b, v28.16b
6072 aesmc v3.16b, v3.16b //AES block 4k+7 - round 10
6074 aese v0.16b, v28.16b
6075 aesmc v0.16b, v0.16b //AES block 4k+4 - round 10
6080 aese v1.16b, v28.16b
6081 aesmc v1.16b, v1.16b //AES block 4k+5 - round 10
6086 aese v2.16b, v29.16b
6087 aesmc v2.16b, v2.16b //AES block 4k+6 - round 11
6088 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
6090 aese v0.16b, v29.16b
6091 aesmc v0.16b, v0.16b //AES block 4k+4 - round 11
6094 aese v1.16b, v29.16b
6095 aesmc v1.16b, v1.16b //AES block 4k+5 - round 11
6101 aese v2.16b, v30.16b
6102 aesmc v2.16b, v2.16b //AES block 4k+6 - round 12
6110 aese v3.16b, v29.16b
6111 aesmc v3.16b, v3.16b //AES block 4k+7 - round 11
6114 aese v1.16b, v30.16b
6115 aesmc v1.16b, v1.16b //AES block 4k+5 - round 12
6116 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
6118 aese v0.16b, v30.16b
6119 aesmc v0.16b, v0.16b //AES block 4k+4 - round 12
6122 aese v3.16b, v30.16b
6123 aesmc v3.16b, v3.16b //AES block 4k+7 - round 12
6124 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low
6126 aese v1.16b, v31.16b //AES block 4k+5 - round 13
6128 aese v0.16b, v31.16b //AES block 4k+4 - round 13
6130 aese v3.16b, v31.16b //AES block 4k+7 - round 13
6132 aese v2.16b, v31.16b //AES block 4k+6 - round 13
6133 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
6137 ld1 { v5.16b}, [x0], #16 //AES block 4k+4 - load ciphertext
6139 eor v0.16b, v5.16b, v0.16b //AES block 4k+4 - result
6144 ext v8.16b, v11.16b, v11.16b, #8 //prepare final partial tag
6157 b.gt .L256_dec_blocks_more_than_3
6160 mov v3.16b, v2.16b
6161 movi v10.8b, #0
6163 movi v11.8b, #0
6166 movi v9.8b, #0
6167 mov v2.16b, v1.16b
6168 b.gt .L256_dec_blocks_more_than_2
6172 mov v3.16b, v1.16b
6174 b.gt .L256_dec_blocks_more_than_1
6177 b .L256_dec_blocks_less_than_1
6179 rev64 v4.16b, v5.16b //GHASH final-3 block
6180 ld1 { v5.16b}, [x0], #16 //AES final-2 block - load ciphertext
6186 eor v4.16b, v4.16b, v8.16b //feed in partial tag
6188 eor v0.16b, v5.16b, v1.16b //AES final-2 block - result
6196 eor v22.8b, v22.8b, v4.8b //GHASH final-3 block - mid
6198 movi v8.8b, #0 //suppress further partial tag feed in
6215 rev64 v4.16b, v5.16b //GHASH final-2 block
6216 ld1 { v5.16b}, [x0], #16 //AES final-1 block - load ciphertext
6218 eor v4.16b, v4.16b, v8.16b //feed in partial tag
6221 eor v0.16b, v5.16b, v2.16b //AES final-1 block - result
6229 eor v22.8b, v22.8b, v4.8b //GHASH final-2 block - mid
6233 eor v11.16b, v11.16b, v21.16b //GHASH final-2 block - low
6234 movi v8.8b, #0 //suppress further partial tag feed in
6238 eor v9.16b, v9.16b, v20.16b //GHASH final-2 block - high
6244 eor v10.16b, v10.16b, v22.16b //GHASH final-2 block - mid
6252 rev64 v4.16b, v5.16b //GHASH final-1 block
6254 ld1 { v5.16b}, [x0], #16 //AES final block - load ciphertext
6256 eor v4.16b, v4.16b, v8.16b //feed in partial tag
6257 movi v8.8b, #0 //suppress further partial tag feed in
6261 eor v0.16b, v5.16b, v3.16b //AES final block - result
6265 eor v22.8b, v22.8b, v4.8b //GHASH final-1 block - mid
6279 eor v11.16b, v11.16b, v21.16b //GHASH final-1 block - low
6281 eor v9.16b, v9.16b, v20.16b //GHASH final-1 block - high
6283 eor v10.16b, v10.16b, v22.16b //GHASH final-1 block - mid
6327 …and v5.16b, v5.16b, v0.16b //possibly partial last block has zeroes in …
6329 rev64 v4.16b, v5.16b //GHASH final block
6331 eor v4.16b, v4.16b, v8.16b //feed in partial tag
6337 eor v8.8b, v8.8b, v4.8b //GHASH final block - mid
6343 eor v9.16b, v9.16b, v20.16b //GHASH final block - high
6345 eor v11.16b, v11.16b, v21.16b //GHASH final block - low
6347 eor v10.16b, v10.16b, v8.16b //GHASH final block - mid
6348 movi v8.8b, #0xc2
6350 eor v6.16b, v11.16b, v9.16b //MODULO - karatsuba tidy up
6354 eor v10.16b, v10.16b, v6.16b //MODULO - karatsuba tidy up
6358 ext v9.16b, v9.16b, v9.16b, #8 //MODULO - other top alignment
6360 eor v10.16b, v10.16b, v7.16b //MODULO - fold into mid
6362 eor v10.16b, v10.16b, v9.16b //MODULO - fold into mid
6366 ext v10.16b, v10.16b, v10.16b, #8 //MODULO - other mid alignment
6368 eor v11.16b, v11.16b, v8.16b //MODULO - fold into low
6374 eor v11.16b, v11.16b, v10.16b //MODULO - fold into low
6375 ext v11.16b, v11.16b, v11.16b, #8
6376 rev64 v11.16b, v11.16b
6378 st1 { v11.16b }, [x3]