1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * ARIA Cipher 32-way parallel algorithm (AVX2) 4 * 5 * Copyright (c) 2022 Taehee Yoo <ap420073@gmail.com> 6 * 7 */ 8 9#include <linux/linkage.h> 10#include <asm/frame.h> 11#include <asm/asm-offsets.h> 12#include <linux/cfi_types.h> 13 14/* register macros */ 15#define CTX %rdi 16 17#define ymm0_x xmm0 18#define ymm1_x xmm1 19#define ymm2_x xmm2 20#define ymm3_x xmm3 21#define ymm4_x xmm4 22#define ymm5_x xmm5 23#define ymm6_x xmm6 24#define ymm7_x xmm7 25#define ymm8_x xmm8 26#define ymm9_x xmm9 27#define ymm10_x xmm10 28#define ymm11_x xmm11 29#define ymm12_x xmm12 30#define ymm13_x xmm13 31#define ymm14_x xmm14 32#define ymm15_x xmm15 33 34#define BV8(a0, a1, a2, a3, a4, a5, a6, a7) \ 35 ( (((a0) & 1) << 0) | \ 36 (((a1) & 1) << 1) | \ 37 (((a2) & 1) << 2) | \ 38 (((a3) & 1) << 3) | \ 39 (((a4) & 1) << 4) | \ 40 (((a5) & 1) << 5) | \ 41 (((a6) & 1) << 6) | \ 42 (((a7) & 1) << 7) ) 43 44#define BM8X8(l0, l1, l2, l3, l4, l5, l6, l7) \ 45 ( ((l7) << (0 * 8)) | \ 46 ((l6) << (1 * 8)) | \ 47 ((l5) << (2 * 8)) | \ 48 ((l4) << (3 * 8)) | \ 49 ((l3) << (4 * 8)) | \ 50 ((l2) << (5 * 8)) | \ 51 ((l1) << (6 * 8)) | \ 52 ((l0) << (7 * 8)) ) 53 54#define inc_le128(x, minus_one, tmp) \ 55 vpcmpeqq minus_one, x, tmp; \ 56 vpsubq minus_one, x, x; \ 57 vpslldq $8, tmp, tmp; \ 58 vpsubq tmp, x, x; 59 60#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \ 61 vpand x, mask4bit, tmp0; \ 62 vpandn x, mask4bit, x; \ 63 vpsrld $4, x, x; \ 64 \ 65 vpshufb tmp0, lo_t, tmp0; \ 66 vpshufb x, hi_t, x; \ 67 vpxor tmp0, x, x; 68 69#define transpose_4x4(x0, x1, x2, x3, t1, t2) \ 70 vpunpckhdq x1, x0, t2; \ 71 vpunpckldq x1, x0, x0; \ 72 \ 73 vpunpckldq x3, x2, t1; \ 74 vpunpckhdq x3, x2, x2; \ 75 \ 76 vpunpckhqdq t1, x0, x1; \ 77 vpunpcklqdq t1, x0, x0; \ 78 \ 79 vpunpckhqdq x2, t2, x3; \ 80 vpunpcklqdq x2, t2, x2; 81 82#define byteslice_16x16b(a0, b0, c0, d0, \ 83 a1, b1, c1, d1, \ 84 a2, b2, c2, d2, \ 85 a3, b3, c3, d3, \ 86 st0, st1) \ 87 vmovdqu d2, st0; \ 88 vmovdqu d3, st1; \ 89 transpose_4x4(a0, a1, a2, a3, d2, d3); \ 90 transpose_4x4(b0, b1, b2, b3, d2, d3); \ 91 vmovdqu st0, d2; \ 92 vmovdqu st1, d3; \ 93 \ 94 vmovdqu a0, st0; \ 95 vmovdqu a1, st1; \ 96 transpose_4x4(c0, c1, c2, c3, a0, a1); \ 97 transpose_4x4(d0, d1, d2, d3, a0, a1); \ 98 \ 99 vbroadcasti128 .Lshufb_16x16b(%rip), a0; \ 100 vmovdqu st1, a1; \ 101 vpshufb a0, a2, a2; \ 102 vpshufb a0, a3, a3; \ 103 vpshufb a0, b0, b0; \ 104 vpshufb a0, b1, b1; \ 105 vpshufb a0, b2, b2; \ 106 vpshufb a0, b3, b3; \ 107 vpshufb a0, a1, a1; \ 108 vpshufb a0, c0, c0; \ 109 vpshufb a0, c1, c1; \ 110 vpshufb a0, c2, c2; \ 111 vpshufb a0, c3, c3; \ 112 vpshufb a0, d0, d0; \ 113 vpshufb a0, d1, d1; \ 114 vpshufb a0, d2, d2; \ 115 vpshufb a0, d3, d3; \ 116 vmovdqu d3, st1; \ 117 vmovdqu st0, d3; \ 118 vpshufb a0, d3, a0; \ 119 vmovdqu d2, st0; \ 120 \ 121 transpose_4x4(a0, b0, c0, d0, d2, d3); \ 122 transpose_4x4(a1, b1, c1, d1, d2, d3); \ 123 vmovdqu st0, d2; \ 124 vmovdqu st1, d3; \ 125 \ 126 vmovdqu b0, st0; \ 127 vmovdqu b1, st1; \ 128 transpose_4x4(a2, b2, c2, d2, b0, b1); \ 129 transpose_4x4(a3, b3, c3, d3, b0, b1); \ 130 vmovdqu st0, b0; \ 131 vmovdqu st1, b1; \ 132 /* does not adjust output bytes inside vectors */ 133 134#define debyteslice_16x16b(a0, b0, c0, d0, \ 135 a1, b1, c1, d1, \ 136 a2, b2, c2, d2, \ 137 a3, b3, c3, d3, \ 138 st0, st1) \ 139 vmovdqu d2, st0; \ 140 vmovdqu d3, st1; \ 141 transpose_4x4(a0, a1, a2, a3, d2, d3); \ 142 transpose_4x4(b0, b1, b2, b3, d2, d3); \ 143 vmovdqu st0, d2; \ 144 vmovdqu st1, d3; \ 145 \ 146 vmovdqu a0, st0; \ 147 vmovdqu a1, st1; \ 148 transpose_4x4(c0, c1, c2, c3, a0, a1); \ 149 transpose_4x4(d0, d1, d2, d3, a0, a1); \ 150 \ 151 vbroadcasti128 .Lshufb_16x16b(%rip), a0; \ 152 vmovdqu st1, a1; \ 153 vpshufb a0, a2, a2; \ 154 vpshufb a0, a3, a3; \ 155 vpshufb a0, b0, b0; \ 156 vpshufb a0, b1, b1; \ 157 vpshufb a0, b2, b2; \ 158 vpshufb a0, b3, b3; \ 159 vpshufb a0, a1, a1; \ 160 vpshufb a0, c0, c0; \ 161 vpshufb a0, c1, c1; \ 162 vpshufb a0, c2, c2; \ 163 vpshufb a0, c3, c3; \ 164 vpshufb a0, d0, d0; \ 165 vpshufb a0, d1, d1; \ 166 vpshufb a0, d2, d2; \ 167 vpshufb a0, d3, d3; \ 168 vmovdqu d3, st1; \ 169 vmovdqu st0, d3; \ 170 vpshufb a0, d3, a0; \ 171 vmovdqu d2, st0; \ 172 \ 173 transpose_4x4(c0, d0, a0, b0, d2, d3); \ 174 transpose_4x4(c1, d1, a1, b1, d2, d3); \ 175 vmovdqu st0, d2; \ 176 vmovdqu st1, d3; \ 177 \ 178 vmovdqu b0, st0; \ 179 vmovdqu b1, st1; \ 180 transpose_4x4(c2, d2, a2, b2, b0, b1); \ 181 transpose_4x4(c3, d3, a3, b3, b0, b1); \ 182 vmovdqu st0, b0; \ 183 vmovdqu st1, b1; \ 184 /* does not adjust output bytes inside vectors */ 185 186/* load blocks to registers and apply pre-whitening */ 187#define inpack16_pre(x0, x1, x2, x3, \ 188 x4, x5, x6, x7, \ 189 y0, y1, y2, y3, \ 190 y4, y5, y6, y7, \ 191 rio) \ 192 vmovdqu (0 * 32)(rio), x0; \ 193 vmovdqu (1 * 32)(rio), x1; \ 194 vmovdqu (2 * 32)(rio), x2; \ 195 vmovdqu (3 * 32)(rio), x3; \ 196 vmovdqu (4 * 32)(rio), x4; \ 197 vmovdqu (5 * 32)(rio), x5; \ 198 vmovdqu (6 * 32)(rio), x6; \ 199 vmovdqu (7 * 32)(rio), x7; \ 200 vmovdqu (8 * 32)(rio), y0; \ 201 vmovdqu (9 * 32)(rio), y1; \ 202 vmovdqu (10 * 32)(rio), y2; \ 203 vmovdqu (11 * 32)(rio), y3; \ 204 vmovdqu (12 * 32)(rio), y4; \ 205 vmovdqu (13 * 32)(rio), y5; \ 206 vmovdqu (14 * 32)(rio), y6; \ 207 vmovdqu (15 * 32)(rio), y7; 208 209/* byteslice pre-whitened blocks and store to temporary memory */ 210#define inpack16_post(x0, x1, x2, x3, \ 211 x4, x5, x6, x7, \ 212 y0, y1, y2, y3, \ 213 y4, y5, y6, y7, \ 214 mem_ab, mem_cd) \ 215 byteslice_16x16b(x0, x1, x2, x3, \ 216 x4, x5, x6, x7, \ 217 y0, y1, y2, y3, \ 218 y4, y5, y6, y7, \ 219 (mem_ab), (mem_cd)); \ 220 \ 221 vmovdqu x0, 0 * 32(mem_ab); \ 222 vmovdqu x1, 1 * 32(mem_ab); \ 223 vmovdqu x2, 2 * 32(mem_ab); \ 224 vmovdqu x3, 3 * 32(mem_ab); \ 225 vmovdqu x4, 4 * 32(mem_ab); \ 226 vmovdqu x5, 5 * 32(mem_ab); \ 227 vmovdqu x6, 6 * 32(mem_ab); \ 228 vmovdqu x7, 7 * 32(mem_ab); \ 229 vmovdqu y0, 0 * 32(mem_cd); \ 230 vmovdqu y1, 1 * 32(mem_cd); \ 231 vmovdqu y2, 2 * 32(mem_cd); \ 232 vmovdqu y3, 3 * 32(mem_cd); \ 233 vmovdqu y4, 4 * 32(mem_cd); \ 234 vmovdqu y5, 5 * 32(mem_cd); \ 235 vmovdqu y6, 6 * 32(mem_cd); \ 236 vmovdqu y7, 7 * 32(mem_cd); 237 238#define write_output(x0, x1, x2, x3, \ 239 x4, x5, x6, x7, \ 240 y0, y1, y2, y3, \ 241 y4, y5, y6, y7, \ 242 mem) \ 243 vmovdqu x0, 0 * 32(mem); \ 244 vmovdqu x1, 1 * 32(mem); \ 245 vmovdqu x2, 2 * 32(mem); \ 246 vmovdqu x3, 3 * 32(mem); \ 247 vmovdqu x4, 4 * 32(mem); \ 248 vmovdqu x5, 5 * 32(mem); \ 249 vmovdqu x6, 6 * 32(mem); \ 250 vmovdqu x7, 7 * 32(mem); \ 251 vmovdqu y0, 8 * 32(mem); \ 252 vmovdqu y1, 9 * 32(mem); \ 253 vmovdqu y2, 10 * 32(mem); \ 254 vmovdqu y3, 11 * 32(mem); \ 255 vmovdqu y4, 12 * 32(mem); \ 256 vmovdqu y5, 13 * 32(mem); \ 257 vmovdqu y6, 14 * 32(mem); \ 258 vmovdqu y7, 15 * 32(mem); \ 259 260#define aria_store_state_8way(x0, x1, x2, x3, \ 261 x4, x5, x6, x7, \ 262 mem_tmp, idx) \ 263 vmovdqu x0, ((idx + 0) * 32)(mem_tmp); \ 264 vmovdqu x1, ((idx + 1) * 32)(mem_tmp); \ 265 vmovdqu x2, ((idx + 2) * 32)(mem_tmp); \ 266 vmovdqu x3, ((idx + 3) * 32)(mem_tmp); \ 267 vmovdqu x4, ((idx + 4) * 32)(mem_tmp); \ 268 vmovdqu x5, ((idx + 5) * 32)(mem_tmp); \ 269 vmovdqu x6, ((idx + 6) * 32)(mem_tmp); \ 270 vmovdqu x7, ((idx + 7) * 32)(mem_tmp); 271 272#define aria_load_state_8way(x0, x1, x2, x3, \ 273 x4, x5, x6, x7, \ 274 mem_tmp, idx) \ 275 vmovdqu ((idx + 0) * 32)(mem_tmp), x0; \ 276 vmovdqu ((idx + 1) * 32)(mem_tmp), x1; \ 277 vmovdqu ((idx + 2) * 32)(mem_tmp), x2; \ 278 vmovdqu ((idx + 3) * 32)(mem_tmp), x3; \ 279 vmovdqu ((idx + 4) * 32)(mem_tmp), x4; \ 280 vmovdqu ((idx + 5) * 32)(mem_tmp), x5; \ 281 vmovdqu ((idx + 6) * 32)(mem_tmp), x6; \ 282 vmovdqu ((idx + 7) * 32)(mem_tmp), x7; 283 284#define aria_ark_8way(x0, x1, x2, x3, \ 285 x4, x5, x6, x7, \ 286 t0, rk, idx, round) \ 287 /* AddRoundKey */ \ 288 vpbroadcastb ((round * 16) + idx + 3)(rk), t0; \ 289 vpxor t0, x0, x0; \ 290 vpbroadcastb ((round * 16) + idx + 2)(rk), t0; \ 291 vpxor t0, x1, x1; \ 292 vpbroadcastb ((round * 16) + idx + 1)(rk), t0; \ 293 vpxor t0, x2, x2; \ 294 vpbroadcastb ((round * 16) + idx + 0)(rk), t0; \ 295 vpxor t0, x3, x3; \ 296 vpbroadcastb ((round * 16) + idx + 7)(rk), t0; \ 297 vpxor t0, x4, x4; \ 298 vpbroadcastb ((round * 16) + idx + 6)(rk), t0; \ 299 vpxor t0, x5, x5; \ 300 vpbroadcastb ((round * 16) + idx + 5)(rk), t0; \ 301 vpxor t0, x6, x6; \ 302 vpbroadcastb ((round * 16) + idx + 4)(rk), t0; \ 303 vpxor t0, x7, x7; 304 305#define aria_sbox_8way_gfni(x0, x1, x2, x3, \ 306 x4, x5, x6, x7, \ 307 t0, t1, t2, t3, \ 308 t4, t5, t6, t7) \ 309 vpbroadcastq .Ltf_s2_bitmatrix(%rip), t0; \ 310 vpbroadcastq .Ltf_inv_bitmatrix(%rip), t1; \ 311 vpbroadcastq .Ltf_id_bitmatrix(%rip), t2; \ 312 vpbroadcastq .Ltf_aff_bitmatrix(%rip), t3; \ 313 vpbroadcastq .Ltf_x2_bitmatrix(%rip), t4; \ 314 vgf2p8affineinvqb $(tf_s2_const), t0, x1, x1; \ 315 vgf2p8affineinvqb $(tf_s2_const), t0, x5, x5; \ 316 vgf2p8affineqb $(tf_inv_const), t1, x2, x2; \ 317 vgf2p8affineqb $(tf_inv_const), t1, x6, x6; \ 318 vgf2p8affineinvqb $0, t2, x2, x2; \ 319 vgf2p8affineinvqb $0, t2, x6, x6; \ 320 vgf2p8affineinvqb $(tf_aff_const), t3, x0, x0; \ 321 vgf2p8affineinvqb $(tf_aff_const), t3, x4, x4; \ 322 vgf2p8affineqb $(tf_x2_const), t4, x3, x3; \ 323 vgf2p8affineqb $(tf_x2_const), t4, x7, x7; \ 324 vgf2p8affineinvqb $0, t2, x3, x3; \ 325 vgf2p8affineinvqb $0, t2, x7, x7 326 327#define aria_sbox_8way(x0, x1, x2, x3, \ 328 x4, x5, x6, x7, \ 329 t0, t1, t2, t3, \ 330 t4, t5, t6, t7) \ 331 vpxor t7, t7, t7; \ 332 vpxor t6, t6, t6; \ 333 vbroadcasti128 .Linv_shift_row(%rip), t0; \ 334 vbroadcasti128 .Lshift_row(%rip), t1; \ 335 vbroadcasti128 .Ltf_lo__inv_aff__and__s2(%rip), t2; \ 336 vbroadcasti128 .Ltf_hi__inv_aff__and__s2(%rip), t3; \ 337 vbroadcasti128 .Ltf_lo__x2__and__fwd_aff(%rip), t4; \ 338 vbroadcasti128 .Ltf_hi__x2__and__fwd_aff(%rip), t5; \ 339 \ 340 vextracti128 $1, x0, t6##_x; \ 341 vaesenclast t7##_x, x0##_x, x0##_x; \ 342 vaesenclast t7##_x, t6##_x, t6##_x; \ 343 vinserti128 $1, t6##_x, x0, x0; \ 344 \ 345 vextracti128 $1, x4, t6##_x; \ 346 vaesenclast t7##_x, x4##_x, x4##_x; \ 347 vaesenclast t7##_x, t6##_x, t6##_x; \ 348 vinserti128 $1, t6##_x, x4, x4; \ 349 \ 350 vextracti128 $1, x1, t6##_x; \ 351 vaesenclast t7##_x, x1##_x, x1##_x; \ 352 vaesenclast t7##_x, t6##_x, t6##_x; \ 353 vinserti128 $1, t6##_x, x1, x1; \ 354 \ 355 vextracti128 $1, x5, t6##_x; \ 356 vaesenclast t7##_x, x5##_x, x5##_x; \ 357 vaesenclast t7##_x, t6##_x, t6##_x; \ 358 vinserti128 $1, t6##_x, x5, x5; \ 359 \ 360 vextracti128 $1, x2, t6##_x; \ 361 vaesdeclast t7##_x, x2##_x, x2##_x; \ 362 vaesdeclast t7##_x, t6##_x, t6##_x; \ 363 vinserti128 $1, t6##_x, x2, x2; \ 364 \ 365 vextracti128 $1, x6, t6##_x; \ 366 vaesdeclast t7##_x, x6##_x, x6##_x; \ 367 vaesdeclast t7##_x, t6##_x, t6##_x; \ 368 vinserti128 $1, t6##_x, x6, x6; \ 369 \ 370 vpbroadcastd .L0f0f0f0f(%rip), t6; \ 371 \ 372 /* AES inverse shift rows */ \ 373 vpshufb t0, x0, x0; \ 374 vpshufb t0, x4, x4; \ 375 vpshufb t0, x1, x1; \ 376 vpshufb t0, x5, x5; \ 377 vpshufb t1, x3, x3; \ 378 vpshufb t1, x7, x7; \ 379 vpshufb t1, x2, x2; \ 380 vpshufb t1, x6, x6; \ 381 \ 382 /* affine transformation for S2 */ \ 383 filter_8bit(x1, t2, t3, t6, t0); \ 384 /* affine transformation for S2 */ \ 385 filter_8bit(x5, t2, t3, t6, t0); \ 386 \ 387 /* affine transformation for X2 */ \ 388 filter_8bit(x3, t4, t5, t6, t0); \ 389 /* affine transformation for X2 */ \ 390 filter_8bit(x7, t4, t5, t6, t0); \ 391 \ 392 vpxor t6, t6, t6; \ 393 vextracti128 $1, x3, t6##_x; \ 394 vaesdeclast t7##_x, x3##_x, x3##_x; \ 395 vaesdeclast t7##_x, t6##_x, t6##_x; \ 396 vinserti128 $1, t6##_x, x3, x3; \ 397 \ 398 vextracti128 $1, x7, t6##_x; \ 399 vaesdeclast t7##_x, x7##_x, x7##_x; \ 400 vaesdeclast t7##_x, t6##_x, t6##_x; \ 401 vinserti128 $1, t6##_x, x7, x7; \ 402 403#define aria_diff_m(x0, x1, x2, x3, \ 404 t0, t1, t2, t3) \ 405 /* T = rotr32(X, 8); */ \ 406 /* X ^= T */ \ 407 vpxor x0, x3, t0; \ 408 vpxor x1, x0, t1; \ 409 vpxor x2, x1, t2; \ 410 vpxor x3, x2, t3; \ 411 /* X = T ^ rotr(X, 16); */ \ 412 vpxor t2, x0, x0; \ 413 vpxor x1, t3, t3; \ 414 vpxor t0, x2, x2; \ 415 vpxor t1, x3, x1; \ 416 vmovdqu t3, x3; 417 418#define aria_diff_word(x0, x1, x2, x3, \ 419 x4, x5, x6, x7, \ 420 y0, y1, y2, y3, \ 421 y4, y5, y6, y7) \ 422 /* t1 ^= t2; */ \ 423 vpxor y0, x4, x4; \ 424 vpxor y1, x5, x5; \ 425 vpxor y2, x6, x6; \ 426 vpxor y3, x7, x7; \ 427 \ 428 /* t2 ^= t3; */ \ 429 vpxor y4, y0, y0; \ 430 vpxor y5, y1, y1; \ 431 vpxor y6, y2, y2; \ 432 vpxor y7, y3, y3; \ 433 \ 434 /* t0 ^= t1; */ \ 435 vpxor x4, x0, x0; \ 436 vpxor x5, x1, x1; \ 437 vpxor x6, x2, x2; \ 438 vpxor x7, x3, x3; \ 439 \ 440 /* t3 ^= t1; */ \ 441 vpxor x4, y4, y4; \ 442 vpxor x5, y5, y5; \ 443 vpxor x6, y6, y6; \ 444 vpxor x7, y7, y7; \ 445 \ 446 /* t2 ^= t0; */ \ 447 vpxor x0, y0, y0; \ 448 vpxor x1, y1, y1; \ 449 vpxor x2, y2, y2; \ 450 vpxor x3, y3, y3; \ 451 \ 452 /* t1 ^= t2; */ \ 453 vpxor y0, x4, x4; \ 454 vpxor y1, x5, x5; \ 455 vpxor y2, x6, x6; \ 456 vpxor y3, x7, x7; 457 458#define aria_fe(x0, x1, x2, x3, \ 459 x4, x5, x6, x7, \ 460 y0, y1, y2, y3, \ 461 y4, y5, y6, y7, \ 462 mem_tmp, rk, round) \ 463 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 464 y0, rk, 8, round); \ 465 \ 466 aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ 467 y0, y1, y2, y3, y4, y5, y6, y7); \ 468 \ 469 aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ 470 aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ 471 aria_store_state_8way(x0, x1, x2, x3, \ 472 x4, x5, x6, x7, \ 473 mem_tmp, 8); \ 474 \ 475 aria_load_state_8way(x0, x1, x2, x3, \ 476 x4, x5, x6, x7, \ 477 mem_tmp, 0); \ 478 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 479 y0, rk, 0, round); \ 480 \ 481 aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ 482 y0, y1, y2, y3, y4, y5, y6, y7); \ 483 \ 484 aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ 485 aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ 486 aria_store_state_8way(x0, x1, x2, x3, \ 487 x4, x5, x6, x7, \ 488 mem_tmp, 0); \ 489 aria_load_state_8way(y0, y1, y2, y3, \ 490 y4, y5, y6, y7, \ 491 mem_tmp, 8); \ 492 aria_diff_word(x0, x1, x2, x3, \ 493 x4, x5, x6, x7, \ 494 y0, y1, y2, y3, \ 495 y4, y5, y6, y7); \ 496 /* aria_diff_byte() \ 497 * T3 = ABCD -> BADC \ 498 * T3 = y4, y5, y6, y7 -> y5, y4, y7, y6 \ 499 * T0 = ABCD -> CDAB \ 500 * T0 = x0, x1, x2, x3 -> x2, x3, x0, x1 \ 501 * T1 = ABCD -> DCBA \ 502 * T1 = x4, x5, x6, x7 -> x7, x6, x5, x4 \ 503 */ \ 504 aria_diff_word(x2, x3, x0, x1, \ 505 x7, x6, x5, x4, \ 506 y0, y1, y2, y3, \ 507 y5, y4, y7, y6); \ 508 aria_store_state_8way(x3, x2, x1, x0, \ 509 x6, x7, x4, x5, \ 510 mem_tmp, 0); 511 512#define aria_fo(x0, x1, x2, x3, \ 513 x4, x5, x6, x7, \ 514 y0, y1, y2, y3, \ 515 y4, y5, y6, y7, \ 516 mem_tmp, rk, round) \ 517 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 518 y0, rk, 8, round); \ 519 \ 520 aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 521 y0, y1, y2, y3, y4, y5, y6, y7); \ 522 \ 523 aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ 524 aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ 525 aria_store_state_8way(x0, x1, x2, x3, \ 526 x4, x5, x6, x7, \ 527 mem_tmp, 8); \ 528 \ 529 aria_load_state_8way(x0, x1, x2, x3, \ 530 x4, x5, x6, x7, \ 531 mem_tmp, 0); \ 532 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 533 y0, rk, 0, round); \ 534 \ 535 aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 536 y0, y1, y2, y3, y4, y5, y6, y7); \ 537 \ 538 aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ 539 aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ 540 aria_store_state_8way(x0, x1, x2, x3, \ 541 x4, x5, x6, x7, \ 542 mem_tmp, 0); \ 543 aria_load_state_8way(y0, y1, y2, y3, \ 544 y4, y5, y6, y7, \ 545 mem_tmp, 8); \ 546 aria_diff_word(x0, x1, x2, x3, \ 547 x4, x5, x6, x7, \ 548 y0, y1, y2, y3, \ 549 y4, y5, y6, y7); \ 550 /* aria_diff_byte() \ 551 * T1 = ABCD -> BADC \ 552 * T1 = x4, x5, x6, x7 -> x5, x4, x7, x6 \ 553 * T2 = ABCD -> CDAB \ 554 * T2 = y0, y1, y2, y3, -> y2, y3, y0, y1 \ 555 * T3 = ABCD -> DCBA \ 556 * T3 = y4, y5, y6, y7 -> y7, y6, y5, y4 \ 557 */ \ 558 aria_diff_word(x0, x1, x2, x3, \ 559 x5, x4, x7, x6, \ 560 y2, y3, y0, y1, \ 561 y7, y6, y5, y4); \ 562 aria_store_state_8way(x3, x2, x1, x0, \ 563 x6, x7, x4, x5, \ 564 mem_tmp, 0); 565 566#define aria_ff(x0, x1, x2, x3, \ 567 x4, x5, x6, x7, \ 568 y0, y1, y2, y3, \ 569 y4, y5, y6, y7, \ 570 mem_tmp, rk, round, last_round) \ 571 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 572 y0, rk, 8, round); \ 573 \ 574 aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ 575 y0, y1, y2, y3, y4, y5, y6, y7); \ 576 \ 577 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 578 y0, rk, 8, last_round); \ 579 \ 580 aria_store_state_8way(x0, x1, x2, x3, \ 581 x4, x5, x6, x7, \ 582 mem_tmp, 8); \ 583 \ 584 aria_load_state_8way(x0, x1, x2, x3, \ 585 x4, x5, x6, x7, \ 586 mem_tmp, 0); \ 587 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 588 y0, rk, 0, round); \ 589 \ 590 aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \ 591 y0, y1, y2, y3, y4, y5, y6, y7); \ 592 \ 593 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 594 y0, rk, 0, last_round); \ 595 \ 596 aria_load_state_8way(y0, y1, y2, y3, \ 597 y4, y5, y6, y7, \ 598 mem_tmp, 8); 599 600#define aria_fe_gfni(x0, x1, x2, x3, \ 601 x4, x5, x6, x7, \ 602 y0, y1, y2, y3, \ 603 y4, y5, y6, y7, \ 604 mem_tmp, rk, round) \ 605 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 606 y0, rk, 8, round); \ 607 \ 608 aria_sbox_8way_gfni(x2, x3, x0, x1, \ 609 x6, x7, x4, x5, \ 610 y0, y1, y2, y3, \ 611 y4, y5, y6, y7); \ 612 \ 613 aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ 614 aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ 615 aria_store_state_8way(x0, x1, x2, x3, \ 616 x4, x5, x6, x7, \ 617 mem_tmp, 8); \ 618 \ 619 aria_load_state_8way(x0, x1, x2, x3, \ 620 x4, x5, x6, x7, \ 621 mem_tmp, 0); \ 622 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 623 y0, rk, 0, round); \ 624 \ 625 aria_sbox_8way_gfni(x2, x3, x0, x1, \ 626 x6, x7, x4, x5, \ 627 y0, y1, y2, y3, \ 628 y4, y5, y6, y7); \ 629 \ 630 aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ 631 aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ 632 aria_store_state_8way(x0, x1, x2, x3, \ 633 x4, x5, x6, x7, \ 634 mem_tmp, 0); \ 635 aria_load_state_8way(y0, y1, y2, y3, \ 636 y4, y5, y6, y7, \ 637 mem_tmp, 8); \ 638 aria_diff_word(x0, x1, x2, x3, \ 639 x4, x5, x6, x7, \ 640 y0, y1, y2, y3, \ 641 y4, y5, y6, y7); \ 642 /* aria_diff_byte() \ 643 * T3 = ABCD -> BADC \ 644 * T3 = y4, y5, y6, y7 -> y5, y4, y7, y6 \ 645 * T0 = ABCD -> CDAB \ 646 * T0 = x0, x1, x2, x3 -> x2, x3, x0, x1 \ 647 * T1 = ABCD -> DCBA \ 648 * T1 = x4, x5, x6, x7 -> x7, x6, x5, x4 \ 649 */ \ 650 aria_diff_word(x2, x3, x0, x1, \ 651 x7, x6, x5, x4, \ 652 y0, y1, y2, y3, \ 653 y5, y4, y7, y6); \ 654 aria_store_state_8way(x3, x2, x1, x0, \ 655 x6, x7, x4, x5, \ 656 mem_tmp, 0); 657 658#define aria_fo_gfni(x0, x1, x2, x3, \ 659 x4, x5, x6, x7, \ 660 y0, y1, y2, y3, \ 661 y4, y5, y6, y7, \ 662 mem_tmp, rk, round) \ 663 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 664 y0, rk, 8, round); \ 665 \ 666 aria_sbox_8way_gfni(x0, x1, x2, x3, \ 667 x4, x5, x6, x7, \ 668 y0, y1, y2, y3, \ 669 y4, y5, y6, y7); \ 670 \ 671 aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ 672 aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ 673 aria_store_state_8way(x0, x1, x2, x3, \ 674 x4, x5, x6, x7, \ 675 mem_tmp, 8); \ 676 \ 677 aria_load_state_8way(x0, x1, x2, x3, \ 678 x4, x5, x6, x7, \ 679 mem_tmp, 0); \ 680 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 681 y0, rk, 0, round); \ 682 \ 683 aria_sbox_8way_gfni(x0, x1, x2, x3, \ 684 x4, x5, x6, x7, \ 685 y0, y1, y2, y3, \ 686 y4, y5, y6, y7); \ 687 \ 688 aria_diff_m(x0, x1, x2, x3, y0, y1, y2, y3); \ 689 aria_diff_m(x4, x5, x6, x7, y0, y1, y2, y3); \ 690 aria_store_state_8way(x0, x1, x2, x3, \ 691 x4, x5, x6, x7, \ 692 mem_tmp, 0); \ 693 aria_load_state_8way(y0, y1, y2, y3, \ 694 y4, y5, y6, y7, \ 695 mem_tmp, 8); \ 696 aria_diff_word(x0, x1, x2, x3, \ 697 x4, x5, x6, x7, \ 698 y0, y1, y2, y3, \ 699 y4, y5, y6, y7); \ 700 /* aria_diff_byte() \ 701 * T1 = ABCD -> BADC \ 702 * T1 = x4, x5, x6, x7 -> x5, x4, x7, x6 \ 703 * T2 = ABCD -> CDAB \ 704 * T2 = y0, y1, y2, y3, -> y2, y3, y0, y1 \ 705 * T3 = ABCD -> DCBA \ 706 * T3 = y4, y5, y6, y7 -> y7, y6, y5, y4 \ 707 */ \ 708 aria_diff_word(x0, x1, x2, x3, \ 709 x5, x4, x7, x6, \ 710 y2, y3, y0, y1, \ 711 y7, y6, y5, y4); \ 712 aria_store_state_8way(x3, x2, x1, x0, \ 713 x6, x7, x4, x5, \ 714 mem_tmp, 0); 715 716#define aria_ff_gfni(x0, x1, x2, x3, \ 717 x4, x5, x6, x7, \ 718 y0, y1, y2, y3, \ 719 y4, y5, y6, y7, \ 720 mem_tmp, rk, round, last_round) \ 721 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 722 y0, rk, 8, round); \ 723 \ 724 aria_sbox_8way_gfni(x2, x3, x0, x1, \ 725 x6, x7, x4, x5, \ 726 y0, y1, y2, y3, \ 727 y4, y5, y6, y7); \ 728 \ 729 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 730 y0, rk, 8, last_round); \ 731 \ 732 aria_store_state_8way(x0, x1, x2, x3, \ 733 x4, x5, x6, x7, \ 734 mem_tmp, 8); \ 735 \ 736 aria_load_state_8way(x0, x1, x2, x3, \ 737 x4, x5, x6, x7, \ 738 mem_tmp, 0); \ 739 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 740 y0, rk, 0, round); \ 741 \ 742 aria_sbox_8way_gfni(x2, x3, x0, x1, \ 743 x6, x7, x4, x5, \ 744 y0, y1, y2, y3, \ 745 y4, y5, y6, y7); \ 746 \ 747 aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \ 748 y0, rk, 0, last_round); \ 749 \ 750 aria_load_state_8way(y0, y1, y2, y3, \ 751 y4, y5, y6, y7, \ 752 mem_tmp, 8); 753 754.section .rodata.cst32.shufb_16x16b, "aM", @progbits, 32 755.align 32 756#define SHUFB_BYTES(idx) \ 757 0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx) 758.Lshufb_16x16b: 759 .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3) 760 .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3) 761 762.section .rodata.cst16, "aM", @progbits, 16 763.align 16 764/* For isolating SubBytes from AESENCLAST, inverse shift row */ 765.Linv_shift_row: 766 .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b 767 .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03 768.Lshift_row: 769 .byte 0x00, 0x05, 0x0a, 0x0f, 0x04, 0x09, 0x0e, 0x03 770 .byte 0x08, 0x0d, 0x02, 0x07, 0x0c, 0x01, 0x06, 0x0b 771/* For CTR-mode IV byteswap */ 772.Lbswap128_mask: 773 .byte 0x0f, 0x0e, 0x0d, 0x0c, 0x0b, 0x0a, 0x09, 0x08 774 .byte 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00 775 776/* AES inverse affine and S2 combined: 777 * 1 1 0 0 0 0 0 1 x0 0 778 * 0 1 0 0 1 0 0 0 x1 0 779 * 1 1 0 0 1 1 1 1 x2 0 780 * 0 1 1 0 1 0 0 1 x3 1 781 * 0 1 0 0 1 1 0 0 * x4 + 0 782 * 0 1 0 1 1 0 0 0 x5 0 783 * 0 0 0 0 0 1 0 1 x6 0 784 * 1 1 1 0 0 1 1 1 x7 1 785 */ 786.Ltf_lo__inv_aff__and__s2: 787 .octa 0x92172DA81A9FA520B2370D883ABF8500 788.Ltf_hi__inv_aff__and__s2: 789 .octa 0x2B15FFC1AF917B45E6D8320C625CB688 790 791/* X2 and AES forward affine combined: 792 * 1 0 1 1 0 0 0 1 x0 0 793 * 0 1 1 1 1 0 1 1 x1 0 794 * 0 0 0 1 1 0 1 0 x2 1 795 * 0 1 0 0 0 1 0 0 x3 0 796 * 0 0 1 1 1 0 1 1 * x4 + 0 797 * 0 1 0 0 1 0 0 0 x5 0 798 * 1 1 0 1 0 0 1 1 x6 0 799 * 0 1 0 0 1 0 1 0 x7 0 800 */ 801.Ltf_lo__x2__and__fwd_aff: 802 .octa 0xEFAE0544FCBD1657B8F95213ABEA4100 803.Ltf_hi__x2__and__fwd_aff: 804 .octa 0x3F893781E95FE1576CDA64D2BA0CB204 805 806.section .rodata.cst8, "aM", @progbits, 8 807.align 8 808/* AES affine: */ 809#define tf_aff_const BV8(1, 1, 0, 0, 0, 1, 1, 0) 810.Ltf_aff_bitmatrix: 811 .quad BM8X8(BV8(1, 0, 0, 0, 1, 1, 1, 1), 812 BV8(1, 1, 0, 0, 0, 1, 1, 1), 813 BV8(1, 1, 1, 0, 0, 0, 1, 1), 814 BV8(1, 1, 1, 1, 0, 0, 0, 1), 815 BV8(1, 1, 1, 1, 1, 0, 0, 0), 816 BV8(0, 1, 1, 1, 1, 1, 0, 0), 817 BV8(0, 0, 1, 1, 1, 1, 1, 0), 818 BV8(0, 0, 0, 1, 1, 1, 1, 1)) 819 820/* AES inverse affine: */ 821#define tf_inv_const BV8(1, 0, 1, 0, 0, 0, 0, 0) 822.Ltf_inv_bitmatrix: 823 .quad BM8X8(BV8(0, 0, 1, 0, 0, 1, 0, 1), 824 BV8(1, 0, 0, 1, 0, 0, 1, 0), 825 BV8(0, 1, 0, 0, 1, 0, 0, 1), 826 BV8(1, 0, 1, 0, 0, 1, 0, 0), 827 BV8(0, 1, 0, 1, 0, 0, 1, 0), 828 BV8(0, 0, 1, 0, 1, 0, 0, 1), 829 BV8(1, 0, 0, 1, 0, 1, 0, 0), 830 BV8(0, 1, 0, 0, 1, 0, 1, 0)) 831 832/* S2: */ 833#define tf_s2_const BV8(0, 1, 0, 0, 0, 1, 1, 1) 834.Ltf_s2_bitmatrix: 835 .quad BM8X8(BV8(0, 1, 0, 1, 0, 1, 1, 1), 836 BV8(0, 0, 1, 1, 1, 1, 1, 1), 837 BV8(1, 1, 1, 0, 1, 1, 0, 1), 838 BV8(1, 1, 0, 0, 0, 0, 1, 1), 839 BV8(0, 1, 0, 0, 0, 0, 1, 1), 840 BV8(1, 1, 0, 0, 1, 1, 1, 0), 841 BV8(0, 1, 1, 0, 0, 0, 1, 1), 842 BV8(1, 1, 1, 1, 0, 1, 1, 0)) 843 844/* X2: */ 845#define tf_x2_const BV8(0, 0, 1, 1, 0, 1, 0, 0) 846.Ltf_x2_bitmatrix: 847 .quad BM8X8(BV8(0, 0, 0, 1, 1, 0, 0, 0), 848 BV8(0, 0, 1, 0, 0, 1, 1, 0), 849 BV8(0, 0, 0, 0, 1, 0, 1, 0), 850 BV8(1, 1, 1, 0, 0, 0, 1, 1), 851 BV8(1, 1, 1, 0, 1, 1, 0, 0), 852 BV8(0, 1, 1, 0, 1, 0, 1, 1), 853 BV8(1, 0, 1, 1, 1, 1, 0, 1), 854 BV8(1, 0, 0, 1, 0, 0, 1, 1)) 855 856/* Identity matrix: */ 857.Ltf_id_bitmatrix: 858 .quad BM8X8(BV8(1, 0, 0, 0, 0, 0, 0, 0), 859 BV8(0, 1, 0, 0, 0, 0, 0, 0), 860 BV8(0, 0, 1, 0, 0, 0, 0, 0), 861 BV8(0, 0, 0, 1, 0, 0, 0, 0), 862 BV8(0, 0, 0, 0, 1, 0, 0, 0), 863 BV8(0, 0, 0, 0, 0, 1, 0, 0), 864 BV8(0, 0, 0, 0, 0, 0, 1, 0), 865 BV8(0, 0, 0, 0, 0, 0, 0, 1)) 866 867/* 4-bit mask */ 868.section .rodata.cst4.L0f0f0f0f, "aM", @progbits, 4 869.align 4 870.L0f0f0f0f: 871 .long 0x0f0f0f0f 872 873.text 874 875SYM_FUNC_START_LOCAL(__aria_aesni_avx2_crypt_32way) 876 /* input: 877 * %r9: rk 878 * %rsi: dst 879 * %rdx: src 880 * %ymm0..%ymm15: byte-sliced blocks 881 */ 882 883 FRAME_BEGIN 884 885 movq %rsi, %rax; 886 leaq 8 * 32(%rax), %r8; 887 888 inpack16_post(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 889 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 890 %ymm15, %rax, %r8); 891 aria_fo(%ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, 892 %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 893 %rax, %r9, 0); 894 aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 895 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 896 %ymm15, %rax, %r9, 1); 897 aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15, 898 %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 899 %rax, %r9, 2); 900 aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 901 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 902 %ymm15, %rax, %r9, 3); 903 aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15, 904 %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 905 %rax, %r9, 4); 906 aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 907 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 908 %ymm15, %rax, %r9, 5); 909 aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15, 910 %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 911 %rax, %r9, 6); 912 aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 913 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 914 %ymm15, %rax, %r9, 7); 915 aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15, 916 %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 917 %rax, %r9, 8); 918 aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 919 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 920 %ymm15, %rax, %r9, 9); 921 aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15, 922 %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 923 %rax, %r9, 10); 924 cmpl $12, ARIA_CTX_rounds(CTX); 925 jne .Laria_192; 926 aria_ff(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 927 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 928 %ymm15, %rax, %r9, 11, 12); 929 jmp .Laria_end; 930.Laria_192: 931 aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 932 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 933 %ymm15, %rax, %r9, 11); 934 aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15, 935 %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 936 %rax, %r9, 12); 937 cmpl $14, ARIA_CTX_rounds(CTX); 938 jne .Laria_256; 939 aria_ff(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 940 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 941 %ymm15, %rax, %r9, 13, 14); 942 jmp .Laria_end; 943.Laria_256: 944 aria_fe(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 945 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 946 %ymm15, %rax, %r9, 13); 947 aria_fo(%ymm9, %ymm8, %ymm11, %ymm10, %ymm12, %ymm13, %ymm14, %ymm15, 948 %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 949 %rax, %r9, 14); 950 aria_ff(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 951 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 952 %ymm15, %rax, %r9, 15, 16); 953.Laria_end: 954 debyteslice_16x16b(%ymm8, %ymm12, %ymm1, %ymm4, 955 %ymm9, %ymm13, %ymm0, %ymm5, 956 %ymm10, %ymm14, %ymm3, %ymm6, 957 %ymm11, %ymm15, %ymm2, %ymm7, 958 (%rax), (%r8)); 959 960 FRAME_END 961 RET; 962SYM_FUNC_END(__aria_aesni_avx2_crypt_32way) 963 964SYM_TYPED_FUNC_START(aria_aesni_avx2_encrypt_32way) 965 /* input: 966 * %rdi: ctx, CTX 967 * %rsi: dst 968 * %rdx: src 969 */ 970 971 FRAME_BEGIN 972 973 leaq ARIA_CTX_enc_key(CTX), %r9; 974 975 inpack16_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 976 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 977 %ymm15, %rdx); 978 979 call __aria_aesni_avx2_crypt_32way; 980 981 write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 982 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 983 %ymm15, %rax); 984 985 vzeroupper; 986 FRAME_END 987 RET; 988SYM_FUNC_END(aria_aesni_avx2_encrypt_32way) 989 990SYM_TYPED_FUNC_START(aria_aesni_avx2_decrypt_32way) 991 /* input: 992 * %rdi: ctx, CTX 993 * %rsi: dst 994 * %rdx: src 995 */ 996 997 FRAME_BEGIN 998 999 leaq ARIA_CTX_dec_key(CTX), %r9; 1000 1001 inpack16_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 1002 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 1003 %ymm15, %rdx); 1004 1005 call __aria_aesni_avx2_crypt_32way; 1006 1007 write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 1008 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 1009 %ymm15, %rax); 1010 1011 vzeroupper; 1012 FRAME_END 1013 RET; 1014SYM_FUNC_END(aria_aesni_avx2_decrypt_32way) 1015 1016SYM_FUNC_START_LOCAL(__aria_aesni_avx2_ctr_gen_keystream_32way) 1017 /* input: 1018 * %rdi: ctx 1019 * %rsi: dst 1020 * %rdx: src 1021 * %rcx: keystream 1022 * %r8: iv (big endian, 128bit) 1023 */ 1024 1025 FRAME_BEGIN 1026 movq 8(%r8), %r11; 1027 bswapq %r11; 1028 1029 vbroadcasti128 .Lbswap128_mask (%rip), %ymm6; 1030 vpcmpeqd %ymm0, %ymm0, %ymm0; 1031 vpsrldq $8, %ymm0, %ymm0; /* ab: -1:0 ; cd: -1:0 */ 1032 vpaddq %ymm0, %ymm0, %ymm5; /* ab: -2:0 ; cd: -2:0 */ 1033 1034 /* load IV and byteswap */ 1035 vmovdqu (%r8), %xmm7; 1036 vpshufb %xmm6, %xmm7, %xmm7; 1037 vmovdqa %xmm7, %xmm3; 1038 inc_le128(%xmm7, %xmm0, %xmm4); 1039 vinserti128 $1, %xmm7, %ymm3, %ymm3; 1040 vpshufb %ymm6, %ymm3, %ymm8; /* +1 ; +0 */ 1041 1042 /* check need for handling 64-bit overflow and carry */ 1043 cmpq $(0xffffffffffffffff - 32), %r11; 1044 ja .Lhandle_ctr_carry; 1045 1046 /* construct IVs */ 1047 vpsubq %ymm5, %ymm3, %ymm3; /* +3 ; +2 */ 1048 vpshufb %ymm6, %ymm3, %ymm9; 1049 vpsubq %ymm5, %ymm3, %ymm3; /* +5 ; +4 */ 1050 vpshufb %ymm6, %ymm3, %ymm10; 1051 vpsubq %ymm5, %ymm3, %ymm3; /* +7 ; +6 */ 1052 vpshufb %ymm6, %ymm3, %ymm11; 1053 vpsubq %ymm5, %ymm3, %ymm3; /* +9 ; +8 */ 1054 vpshufb %ymm6, %ymm3, %ymm12; 1055 vpsubq %ymm5, %ymm3, %ymm3; /* +11 ; +10 */ 1056 vpshufb %ymm6, %ymm3, %ymm13; 1057 vpsubq %ymm5, %ymm3, %ymm3; /* +13 ; +12 */ 1058 vpshufb %ymm6, %ymm3, %ymm14; 1059 vpsubq %ymm5, %ymm3, %ymm3; /* +15 ; +14 */ 1060 vpshufb %ymm6, %ymm3, %ymm15; 1061 vmovdqu %ymm8, (0 * 32)(%rcx); 1062 vmovdqu %ymm9, (1 * 32)(%rcx); 1063 vmovdqu %ymm10, (2 * 32)(%rcx); 1064 vmovdqu %ymm11, (3 * 32)(%rcx); 1065 vmovdqu %ymm12, (4 * 32)(%rcx); 1066 vmovdqu %ymm13, (5 * 32)(%rcx); 1067 vmovdqu %ymm14, (6 * 32)(%rcx); 1068 vmovdqu %ymm15, (7 * 32)(%rcx); 1069 1070 vpsubq %ymm5, %ymm3, %ymm3; /* +17 ; +16 */ 1071 vpshufb %ymm6, %ymm3, %ymm8; 1072 vpsubq %ymm5, %ymm3, %ymm3; /* +19 ; +18 */ 1073 vpshufb %ymm6, %ymm3, %ymm9; 1074 vpsubq %ymm5, %ymm3, %ymm3; /* +21 ; +20 */ 1075 vpshufb %ymm6, %ymm3, %ymm10; 1076 vpsubq %ymm5, %ymm3, %ymm3; /* +23 ; +22 */ 1077 vpshufb %ymm6, %ymm3, %ymm11; 1078 vpsubq %ymm5, %ymm3, %ymm3; /* +25 ; +24 */ 1079 vpshufb %ymm6, %ymm3, %ymm12; 1080 vpsubq %ymm5, %ymm3, %ymm3; /* +27 ; +26 */ 1081 vpshufb %ymm6, %ymm3, %ymm13; 1082 vpsubq %ymm5, %ymm3, %ymm3; /* +29 ; +28 */ 1083 vpshufb %ymm6, %ymm3, %ymm14; 1084 vpsubq %ymm5, %ymm3, %ymm3; /* +31 ; +30 */ 1085 vpshufb %ymm6, %ymm3, %ymm15; 1086 vpsubq %ymm5, %ymm3, %ymm3; /* +32 */ 1087 vpshufb %xmm6, %xmm3, %xmm3; 1088 vmovdqu %xmm3, (%r8); 1089 vmovdqu (0 * 32)(%rcx), %ymm0; 1090 vmovdqu (1 * 32)(%rcx), %ymm1; 1091 vmovdqu (2 * 32)(%rcx), %ymm2; 1092 vmovdqu (3 * 32)(%rcx), %ymm3; 1093 vmovdqu (4 * 32)(%rcx), %ymm4; 1094 vmovdqu (5 * 32)(%rcx), %ymm5; 1095 vmovdqu (6 * 32)(%rcx), %ymm6; 1096 vmovdqu (7 * 32)(%rcx), %ymm7; 1097 jmp .Lctr_carry_done; 1098 1099 .Lhandle_ctr_carry: 1100 /* construct IVs */ 1101 inc_le128(%ymm3, %ymm0, %ymm4); 1102 inc_le128(%ymm3, %ymm0, %ymm4); 1103 vpshufb %ymm6, %ymm3, %ymm9; /* +3 ; +2 */ 1104 inc_le128(%ymm3, %ymm0, %ymm4); 1105 inc_le128(%ymm3, %ymm0, %ymm4); 1106 vpshufb %ymm6, %ymm3, %ymm10; /* +5 ; +4 */ 1107 inc_le128(%ymm3, %ymm0, %ymm4); 1108 inc_le128(%ymm3, %ymm0, %ymm4); 1109 vpshufb %ymm6, %ymm3, %ymm11; /* +7 ; +6 */ 1110 inc_le128(%ymm3, %ymm0, %ymm4); 1111 inc_le128(%ymm3, %ymm0, %ymm4); 1112 vpshufb %ymm6, %ymm3, %ymm12; /* +9 ; +8 */ 1113 inc_le128(%ymm3, %ymm0, %ymm4); 1114 inc_le128(%ymm3, %ymm0, %ymm4); 1115 vpshufb %ymm6, %ymm3, %ymm13; /* +11 ; +10 */ 1116 inc_le128(%ymm3, %ymm0, %ymm4); 1117 inc_le128(%ymm3, %ymm0, %ymm4); 1118 vpshufb %ymm6, %ymm3, %ymm14; /* +13 ; +12 */ 1119 inc_le128(%ymm3, %ymm0, %ymm4); 1120 inc_le128(%ymm3, %ymm0, %ymm4); 1121 vpshufb %ymm6, %ymm3, %ymm15; /* +15 ; +14 */ 1122 vmovdqu %ymm8, (0 * 32)(%rcx); 1123 vmovdqu %ymm9, (1 * 32)(%rcx); 1124 vmovdqu %ymm10, (2 * 32)(%rcx); 1125 vmovdqu %ymm11, (3 * 32)(%rcx); 1126 vmovdqu %ymm12, (4 * 32)(%rcx); 1127 vmovdqu %ymm13, (5 * 32)(%rcx); 1128 vmovdqu %ymm14, (6 * 32)(%rcx); 1129 vmovdqu %ymm15, (7 * 32)(%rcx); 1130 1131 inc_le128(%ymm3, %ymm0, %ymm4); 1132 inc_le128(%ymm3, %ymm0, %ymm4); 1133 vpshufb %ymm6, %ymm3, %ymm8; /* +17 ; +16 */ 1134 inc_le128(%ymm3, %ymm0, %ymm4); 1135 inc_le128(%ymm3, %ymm0, %ymm4); 1136 vpshufb %ymm6, %ymm3, %ymm9; /* +19 ; +18 */ 1137 inc_le128(%ymm3, %ymm0, %ymm4); 1138 inc_le128(%ymm3, %ymm0, %ymm4); 1139 vpshufb %ymm6, %ymm3, %ymm10; /* +21 ; +20 */ 1140 inc_le128(%ymm3, %ymm0, %ymm4); 1141 inc_le128(%ymm3, %ymm0, %ymm4); 1142 vpshufb %ymm6, %ymm3, %ymm11; /* +23 ; +22 */ 1143 inc_le128(%ymm3, %ymm0, %ymm4); 1144 inc_le128(%ymm3, %ymm0, %ymm4); 1145 vpshufb %ymm6, %ymm3, %ymm12; /* +25 ; +24 */ 1146 inc_le128(%ymm3, %ymm0, %ymm4); 1147 inc_le128(%ymm3, %ymm0, %ymm4); 1148 vpshufb %ymm6, %ymm3, %ymm13; /* +27 ; +26 */ 1149 inc_le128(%ymm3, %ymm0, %ymm4); 1150 inc_le128(%ymm3, %ymm0, %ymm4); 1151 vpshufb %ymm6, %ymm3, %ymm14; /* +29 ; +28 */ 1152 inc_le128(%ymm3, %ymm0, %ymm4); 1153 inc_le128(%ymm3, %ymm0, %ymm4); 1154 vpshufb %ymm6, %ymm3, %ymm15; /* +31 ; +30 */ 1155 inc_le128(%ymm3, %ymm0, %ymm4); 1156 vextracti128 $1, %ymm3, %xmm3; 1157 vpshufb %xmm6, %xmm3, %xmm3; /* +32 */ 1158 vmovdqu %xmm3, (%r8); 1159 vmovdqu (0 * 32)(%rcx), %ymm0; 1160 vmovdqu (1 * 32)(%rcx), %ymm1; 1161 vmovdqu (2 * 32)(%rcx), %ymm2; 1162 vmovdqu (3 * 32)(%rcx), %ymm3; 1163 vmovdqu (4 * 32)(%rcx), %ymm4; 1164 vmovdqu (5 * 32)(%rcx), %ymm5; 1165 vmovdqu (6 * 32)(%rcx), %ymm6; 1166 vmovdqu (7 * 32)(%rcx), %ymm7; 1167 1168 .Lctr_carry_done: 1169 1170 FRAME_END 1171 RET; 1172SYM_FUNC_END(__aria_aesni_avx2_ctr_gen_keystream_32way) 1173 1174SYM_TYPED_FUNC_START(aria_aesni_avx2_ctr_crypt_32way) 1175 /* input: 1176 * %rdi: ctx 1177 * %rsi: dst 1178 * %rdx: src 1179 * %rcx: keystream 1180 * %r8: iv (big endian, 128bit) 1181 */ 1182 FRAME_BEGIN 1183 1184 call __aria_aesni_avx2_ctr_gen_keystream_32way; 1185 1186 leaq (%rsi), %r10; 1187 leaq (%rdx), %r11; 1188 leaq (%rcx), %rsi; 1189 leaq (%rcx), %rdx; 1190 leaq ARIA_CTX_enc_key(CTX), %r9; 1191 1192 call __aria_aesni_avx2_crypt_32way; 1193 1194 vpxor (0 * 32)(%r11), %ymm1, %ymm1; 1195 vpxor (1 * 32)(%r11), %ymm0, %ymm0; 1196 vpxor (2 * 32)(%r11), %ymm3, %ymm3; 1197 vpxor (3 * 32)(%r11), %ymm2, %ymm2; 1198 vpxor (4 * 32)(%r11), %ymm4, %ymm4; 1199 vpxor (5 * 32)(%r11), %ymm5, %ymm5; 1200 vpxor (6 * 32)(%r11), %ymm6, %ymm6; 1201 vpxor (7 * 32)(%r11), %ymm7, %ymm7; 1202 vpxor (8 * 32)(%r11), %ymm8, %ymm8; 1203 vpxor (9 * 32)(%r11), %ymm9, %ymm9; 1204 vpxor (10 * 32)(%r11), %ymm10, %ymm10; 1205 vpxor (11 * 32)(%r11), %ymm11, %ymm11; 1206 vpxor (12 * 32)(%r11), %ymm12, %ymm12; 1207 vpxor (13 * 32)(%r11), %ymm13, %ymm13; 1208 vpxor (14 * 32)(%r11), %ymm14, %ymm14; 1209 vpxor (15 * 32)(%r11), %ymm15, %ymm15; 1210 write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 1211 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 1212 %ymm15, %r10); 1213 1214 vzeroupper; 1215 FRAME_END 1216 RET; 1217SYM_FUNC_END(aria_aesni_avx2_ctr_crypt_32way) 1218 1219SYM_FUNC_START_LOCAL(__aria_aesni_avx2_gfni_crypt_32way) 1220 /* input: 1221 * %r9: rk 1222 * %rsi: dst 1223 * %rdx: src 1224 * %ymm0..%ymm15: 16 byte-sliced blocks 1225 */ 1226 1227 FRAME_BEGIN 1228 1229 movq %rsi, %rax; 1230 leaq 8 * 32(%rax), %r8; 1231 1232 inpack16_post(%ymm0, %ymm1, %ymm2, %ymm3, 1233 %ymm4, %ymm5, %ymm6, %ymm7, 1234 %ymm8, %ymm9, %ymm10, %ymm11, 1235 %ymm12, %ymm13, %ymm14, 1236 %ymm15, %rax, %r8); 1237 aria_fo_gfni(%ymm8, %ymm9, %ymm10, %ymm11, 1238 %ymm12, %ymm13, %ymm14, %ymm15, 1239 %ymm0, %ymm1, %ymm2, %ymm3, 1240 %ymm4, %ymm5, %ymm6, %ymm7, 1241 %rax, %r9, 0); 1242 aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2, 1243 %ymm4, %ymm5, %ymm6, %ymm7, 1244 %ymm8, %ymm9, %ymm10, %ymm11, 1245 %ymm12, %ymm13, %ymm14, 1246 %ymm15, %rax, %r9, 1); 1247 aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10, 1248 %ymm12, %ymm13, %ymm14, %ymm15, 1249 %ymm0, %ymm1, %ymm2, %ymm3, 1250 %ymm4, %ymm5, %ymm6, %ymm7, 1251 %rax, %r9, 2); 1252 aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2, 1253 %ymm4, %ymm5, %ymm6, %ymm7, 1254 %ymm8, %ymm9, %ymm10, %ymm11, 1255 %ymm12, %ymm13, %ymm14, 1256 %ymm15, %rax, %r9, 3); 1257 aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10, 1258 %ymm12, %ymm13, %ymm14, %ymm15, 1259 %ymm0, %ymm1, %ymm2, %ymm3, 1260 %ymm4, %ymm5, %ymm6, %ymm7, 1261 %rax, %r9, 4); 1262 aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2, 1263 %ymm4, %ymm5, %ymm6, %ymm7, 1264 %ymm8, %ymm9, %ymm10, %ymm11, 1265 %ymm12, %ymm13, %ymm14, 1266 %ymm15, %rax, %r9, 5); 1267 aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10, 1268 %ymm12, %ymm13, %ymm14, %ymm15, 1269 %ymm0, %ymm1, %ymm2, %ymm3, 1270 %ymm4, %ymm5, %ymm6, %ymm7, 1271 %rax, %r9, 6); 1272 aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2, 1273 %ymm4, %ymm5, %ymm6, %ymm7, 1274 %ymm8, %ymm9, %ymm10, %ymm11, 1275 %ymm12, %ymm13, %ymm14, 1276 %ymm15, %rax, %r9, 7); 1277 aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10, 1278 %ymm12, %ymm13, %ymm14, %ymm15, 1279 %ymm0, %ymm1, %ymm2, %ymm3, 1280 %ymm4, %ymm5, %ymm6, %ymm7, 1281 %rax, %r9, 8); 1282 aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2, 1283 %ymm4, %ymm5, %ymm6, %ymm7, 1284 %ymm8, %ymm9, %ymm10, %ymm11, 1285 %ymm12, %ymm13, %ymm14, 1286 %ymm15, %rax, %r9, 9); 1287 aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10, 1288 %ymm12, %ymm13, %ymm14, %ymm15, 1289 %ymm0, %ymm1, %ymm2, %ymm3, 1290 %ymm4, %ymm5, %ymm6, %ymm7, 1291 %rax, %r9, 10); 1292 cmpl $12, ARIA_CTX_rounds(CTX); 1293 jne .Laria_gfni_192; 1294 aria_ff_gfni(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 1295 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 1296 %ymm15, %rax, %r9, 11, 12); 1297 jmp .Laria_gfni_end; 1298.Laria_gfni_192: 1299 aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2, 1300 %ymm4, %ymm5, %ymm6, %ymm7, 1301 %ymm8, %ymm9, %ymm10, %ymm11, 1302 %ymm12, %ymm13, %ymm14, 1303 %ymm15, %rax, %r9, 11); 1304 aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10, 1305 %ymm12, %ymm13, %ymm14, %ymm15, 1306 %ymm0, %ymm1, %ymm2, %ymm3, 1307 %ymm4, %ymm5, %ymm6, %ymm7, 1308 %rax, %r9, 12); 1309 cmpl $14, ARIA_CTX_rounds(CTX); 1310 jne .Laria_gfni_256; 1311 aria_ff_gfni(%ymm1, %ymm0, %ymm3, %ymm2, 1312 %ymm4, %ymm5, %ymm6, %ymm7, 1313 %ymm8, %ymm9, %ymm10, %ymm11, 1314 %ymm12, %ymm13, %ymm14, 1315 %ymm15, %rax, %r9, 13, 14); 1316 jmp .Laria_gfni_end; 1317.Laria_gfni_256: 1318 aria_fe_gfni(%ymm1, %ymm0, %ymm3, %ymm2, 1319 %ymm4, %ymm5, %ymm6, %ymm7, 1320 %ymm8, %ymm9, %ymm10, %ymm11, 1321 %ymm12, %ymm13, %ymm14, 1322 %ymm15, %rax, %r9, 13); 1323 aria_fo_gfni(%ymm9, %ymm8, %ymm11, %ymm10, 1324 %ymm12, %ymm13, %ymm14, %ymm15, 1325 %ymm0, %ymm1, %ymm2, %ymm3, 1326 %ymm4, %ymm5, %ymm6, %ymm7, 1327 %rax, %r9, 14); 1328 aria_ff_gfni(%ymm1, %ymm0, %ymm3, %ymm2, 1329 %ymm4, %ymm5, %ymm6, %ymm7, 1330 %ymm8, %ymm9, %ymm10, %ymm11, 1331 %ymm12, %ymm13, %ymm14, 1332 %ymm15, %rax, %r9, 15, 16); 1333.Laria_gfni_end: 1334 debyteslice_16x16b(%ymm8, %ymm12, %ymm1, %ymm4, 1335 %ymm9, %ymm13, %ymm0, %ymm5, 1336 %ymm10, %ymm14, %ymm3, %ymm6, 1337 %ymm11, %ymm15, %ymm2, %ymm7, 1338 (%rax), (%r8)); 1339 1340 FRAME_END 1341 RET; 1342SYM_FUNC_END(__aria_aesni_avx2_gfni_crypt_32way) 1343 1344SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_encrypt_32way) 1345 /* input: 1346 * %rdi: ctx, CTX 1347 * %rsi: dst 1348 * %rdx: src 1349 */ 1350 1351 FRAME_BEGIN 1352 1353 leaq ARIA_CTX_enc_key(CTX), %r9; 1354 1355 inpack16_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 1356 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 1357 %ymm15, %rdx); 1358 1359 call __aria_aesni_avx2_gfni_crypt_32way; 1360 1361 write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 1362 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 1363 %ymm15, %rax); 1364 1365 vzeroupper; 1366 FRAME_END 1367 RET; 1368SYM_FUNC_END(aria_aesni_avx2_gfni_encrypt_32way) 1369 1370SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_decrypt_32way) 1371 /* input: 1372 * %rdi: ctx, CTX 1373 * %rsi: dst 1374 * %rdx: src 1375 */ 1376 1377 FRAME_BEGIN 1378 1379 leaq ARIA_CTX_dec_key(CTX), %r9; 1380 1381 inpack16_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 1382 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 1383 %ymm15, %rdx); 1384 1385 call __aria_aesni_avx2_gfni_crypt_32way; 1386 1387 write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 1388 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 1389 %ymm15, %rax); 1390 1391 vzeroupper; 1392 FRAME_END 1393 RET; 1394SYM_FUNC_END(aria_aesni_avx2_gfni_decrypt_32way) 1395 1396SYM_TYPED_FUNC_START(aria_aesni_avx2_gfni_ctr_crypt_32way) 1397 /* input: 1398 * %rdi: ctx 1399 * %rsi: dst 1400 * %rdx: src 1401 * %rcx: keystream 1402 * %r8: iv (big endian, 128bit) 1403 */ 1404 FRAME_BEGIN 1405 1406 call __aria_aesni_avx2_ctr_gen_keystream_32way 1407 1408 leaq (%rsi), %r10; 1409 leaq (%rdx), %r11; 1410 leaq (%rcx), %rsi; 1411 leaq (%rcx), %rdx; 1412 leaq ARIA_CTX_enc_key(CTX), %r9; 1413 1414 call __aria_aesni_avx2_gfni_crypt_32way; 1415 1416 vpxor (0 * 32)(%r11), %ymm1, %ymm1; 1417 vpxor (1 * 32)(%r11), %ymm0, %ymm0; 1418 vpxor (2 * 32)(%r11), %ymm3, %ymm3; 1419 vpxor (3 * 32)(%r11), %ymm2, %ymm2; 1420 vpxor (4 * 32)(%r11), %ymm4, %ymm4; 1421 vpxor (5 * 32)(%r11), %ymm5, %ymm5; 1422 vpxor (6 * 32)(%r11), %ymm6, %ymm6; 1423 vpxor (7 * 32)(%r11), %ymm7, %ymm7; 1424 vpxor (8 * 32)(%r11), %ymm8, %ymm8; 1425 vpxor (9 * 32)(%r11), %ymm9, %ymm9; 1426 vpxor (10 * 32)(%r11), %ymm10, %ymm10; 1427 vpxor (11 * 32)(%r11), %ymm11, %ymm11; 1428 vpxor (12 * 32)(%r11), %ymm12, %ymm12; 1429 vpxor (13 * 32)(%r11), %ymm13, %ymm13; 1430 vpxor (14 * 32)(%r11), %ymm14, %ymm14; 1431 vpxor (15 * 32)(%r11), %ymm15, %ymm15; 1432 write_output(%ymm1, %ymm0, %ymm3, %ymm2, %ymm4, %ymm5, %ymm6, %ymm7, 1433 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 1434 %ymm15, %r10); 1435 1436 vzeroupper; 1437 FRAME_END 1438 RET; 1439SYM_FUNC_END(aria_aesni_avx2_gfni_ctr_crypt_32way) 1440