1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * x86_64/AVX2/AES-NI assembler implementation of Camellia 4 * 5 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@iki.fi> 6 */ 7 8#include <linux/linkage.h> 9#include <linux/cfi_types.h> 10#include <asm/frame.h> 11 12#define CAMELLIA_TABLE_BYTE_LEN 272 13 14/* struct camellia_ctx: */ 15#define key_table 0 16#define key_length CAMELLIA_TABLE_BYTE_LEN 17 18/* register macros */ 19#define CTX %rdi 20#define RIO %r8 21 22/********************************************************************** 23 helper macros 24 **********************************************************************/ 25#define filter_8bit(x, lo_t, hi_t, mask4bit, tmp0) \ 26 vpand x, mask4bit, tmp0; \ 27 vpandn x, mask4bit, x; \ 28 vpsrld $4, x, x; \ 29 \ 30 vpshufb tmp0, lo_t, tmp0; \ 31 vpshufb x, hi_t, x; \ 32 vpxor tmp0, x, x; 33 34#define ymm0_x xmm0 35#define ymm1_x xmm1 36#define ymm2_x xmm2 37#define ymm3_x xmm3 38#define ymm4_x xmm4 39#define ymm5_x xmm5 40#define ymm6_x xmm6 41#define ymm7_x xmm7 42#define ymm8_x xmm8 43#define ymm9_x xmm9 44#define ymm10_x xmm10 45#define ymm11_x xmm11 46#define ymm12_x xmm12 47#define ymm13_x xmm13 48#define ymm14_x xmm14 49#define ymm15_x xmm15 50 51/********************************************************************** 52 32-way camellia 53 **********************************************************************/ 54 55/* 56 * IN: 57 * x0..x7: byte-sliced AB state 58 * mem_cd: register pointer storing CD state 59 * key: index for key material 60 * OUT: 61 * x0..x7: new byte-sliced CD state 62 */ 63#define roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, t0, t1, t2, t3, t4, t5, t6, \ 64 t7, mem_cd, key) \ 65 /* \ 66 * S-function with AES subbytes \ 67 */ \ 68 vbroadcasti128 .Linv_shift_row(%rip), t4; \ 69 vpbroadcastd .L0f0f0f0f(%rip), t7; \ 70 vbroadcasti128 .Lpre_tf_lo_s1(%rip), t5; \ 71 vbroadcasti128 .Lpre_tf_hi_s1(%rip), t6; \ 72 vbroadcasti128 .Lpre_tf_lo_s4(%rip), t2; \ 73 vbroadcasti128 .Lpre_tf_hi_s4(%rip), t3; \ 74 \ 75 /* AES inverse shift rows */ \ 76 vpshufb t4, x0, x0; \ 77 vpshufb t4, x7, x7; \ 78 vpshufb t4, x3, x3; \ 79 vpshufb t4, x6, x6; \ 80 vpshufb t4, x2, x2; \ 81 vpshufb t4, x5, x5; \ 82 vpshufb t4, x1, x1; \ 83 vpshufb t4, x4, x4; \ 84 \ 85 /* prefilter sboxes 1, 2 and 3 */ \ 86 /* prefilter sbox 4 */ \ 87 filter_8bit(x0, t5, t6, t7, t4); \ 88 filter_8bit(x7, t5, t6, t7, t4); \ 89 vextracti128 $1, x0, t0##_x; \ 90 vextracti128 $1, x7, t1##_x; \ 91 filter_8bit(x3, t2, t3, t7, t4); \ 92 filter_8bit(x6, t2, t3, t7, t4); \ 93 vextracti128 $1, x3, t3##_x; \ 94 vextracti128 $1, x6, t2##_x; \ 95 filter_8bit(x2, t5, t6, t7, t4); \ 96 filter_8bit(x5, t5, t6, t7, t4); \ 97 filter_8bit(x1, t5, t6, t7, t4); \ 98 filter_8bit(x4, t5, t6, t7, t4); \ 99 \ 100 vpxor t4##_x, t4##_x, t4##_x; \ 101 \ 102 /* AES subbytes + AES shift rows */ \ 103 vextracti128 $1, x2, t6##_x; \ 104 vextracti128 $1, x5, t5##_x; \ 105 vaesenclast t4##_x, x0##_x, x0##_x; \ 106 vaesenclast t4##_x, t0##_x, t0##_x; \ 107 vinserti128 $1, t0##_x, x0, x0; \ 108 vaesenclast t4##_x, x7##_x, x7##_x; \ 109 vaesenclast t4##_x, t1##_x, t1##_x; \ 110 vinserti128 $1, t1##_x, x7, x7; \ 111 vaesenclast t4##_x, x3##_x, x3##_x; \ 112 vaesenclast t4##_x, t3##_x, t3##_x; \ 113 vinserti128 $1, t3##_x, x3, x3; \ 114 vaesenclast t4##_x, x6##_x, x6##_x; \ 115 vaesenclast t4##_x, t2##_x, t2##_x; \ 116 vinserti128 $1, t2##_x, x6, x6; \ 117 vextracti128 $1, x1, t3##_x; \ 118 vextracti128 $1, x4, t2##_x; \ 119 vbroadcasti128 .Lpost_tf_lo_s1(%rip), t0; \ 120 vbroadcasti128 .Lpost_tf_hi_s1(%rip), t1; \ 121 vaesenclast t4##_x, x2##_x, x2##_x; \ 122 vaesenclast t4##_x, t6##_x, t6##_x; \ 123 vinserti128 $1, t6##_x, x2, x2; \ 124 vaesenclast t4##_x, x5##_x, x5##_x; \ 125 vaesenclast t4##_x, t5##_x, t5##_x; \ 126 vinserti128 $1, t5##_x, x5, x5; \ 127 vaesenclast t4##_x, x1##_x, x1##_x; \ 128 vaesenclast t4##_x, t3##_x, t3##_x; \ 129 vinserti128 $1, t3##_x, x1, x1; \ 130 vaesenclast t4##_x, x4##_x, x4##_x; \ 131 vaesenclast t4##_x, t2##_x, t2##_x; \ 132 vinserti128 $1, t2##_x, x4, x4; \ 133 \ 134 /* postfilter sboxes 1 and 4 */ \ 135 vbroadcasti128 .Lpost_tf_lo_s3(%rip), t2; \ 136 vbroadcasti128 .Lpost_tf_hi_s3(%rip), t3; \ 137 filter_8bit(x0, t0, t1, t7, t6); \ 138 filter_8bit(x7, t0, t1, t7, t6); \ 139 filter_8bit(x3, t0, t1, t7, t6); \ 140 filter_8bit(x6, t0, t1, t7, t6); \ 141 \ 142 /* postfilter sbox 3 */ \ 143 vbroadcasti128 .Lpost_tf_lo_s2(%rip), t4; \ 144 vbroadcasti128 .Lpost_tf_hi_s2(%rip), t5; \ 145 filter_8bit(x2, t2, t3, t7, t6); \ 146 filter_8bit(x5, t2, t3, t7, t6); \ 147 \ 148 vpbroadcastq key, t0; /* higher 64-bit duplicate ignored */ \ 149 \ 150 /* postfilter sbox 2 */ \ 151 filter_8bit(x1, t4, t5, t7, t2); \ 152 filter_8bit(x4, t4, t5, t7, t2); \ 153 vpxor t7, t7, t7; \ 154 \ 155 vpsrldq $1, t0, t1; \ 156 vpsrldq $2, t0, t2; \ 157 vpshufb t7, t1, t1; \ 158 vpsrldq $3, t0, t3; \ 159 \ 160 /* P-function */ \ 161 vpxor x5, x0, x0; \ 162 vpxor x6, x1, x1; \ 163 vpxor x7, x2, x2; \ 164 vpxor x4, x3, x3; \ 165 \ 166 vpshufb t7, t2, t2; \ 167 vpsrldq $4, t0, t4; \ 168 vpshufb t7, t3, t3; \ 169 vpsrldq $5, t0, t5; \ 170 vpshufb t7, t4, t4; \ 171 \ 172 vpxor x2, x4, x4; \ 173 vpxor x3, x5, x5; \ 174 vpxor x0, x6, x6; \ 175 vpxor x1, x7, x7; \ 176 \ 177 vpsrldq $6, t0, t6; \ 178 vpshufb t7, t5, t5; \ 179 vpshufb t7, t6, t6; \ 180 \ 181 vpxor x7, x0, x0; \ 182 vpxor x4, x1, x1; \ 183 vpxor x5, x2, x2; \ 184 vpxor x6, x3, x3; \ 185 \ 186 vpxor x3, x4, x4; \ 187 vpxor x0, x5, x5; \ 188 vpxor x1, x6, x6; \ 189 vpxor x2, x7, x7; /* note: high and low parts swapped */ \ 190 \ 191 /* Add key material and result to CD (x becomes new CD) */ \ 192 \ 193 vpxor t6, x1, x1; \ 194 vpxor 5 * 32(mem_cd), x1, x1; \ 195 \ 196 vpsrldq $7, t0, t6; \ 197 vpshufb t7, t0, t0; \ 198 vpshufb t7, t6, t7; \ 199 \ 200 vpxor t7, x0, x0; \ 201 vpxor 4 * 32(mem_cd), x0, x0; \ 202 \ 203 vpxor t5, x2, x2; \ 204 vpxor 6 * 32(mem_cd), x2, x2; \ 205 \ 206 vpxor t4, x3, x3; \ 207 vpxor 7 * 32(mem_cd), x3, x3; \ 208 \ 209 vpxor t3, x4, x4; \ 210 vpxor 0 * 32(mem_cd), x4, x4; \ 211 \ 212 vpxor t2, x5, x5; \ 213 vpxor 1 * 32(mem_cd), x5, x5; \ 214 \ 215 vpxor t1, x6, x6; \ 216 vpxor 2 * 32(mem_cd), x6, x6; \ 217 \ 218 vpxor t0, x7, x7; \ 219 vpxor 3 * 32(mem_cd), x7, x7; 220 221/* 222 * Size optimization... with inlined roundsm32 binary would be over 5 times 223 * larger and would only marginally faster. 224 */ 225SYM_FUNC_START_LOCAL(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) 226 roundsm32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 227 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, %ymm15, 228 %rcx, (%r9)); 229 RET; 230SYM_FUNC_END(roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd) 231 232SYM_FUNC_START_LOCAL(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) 233 roundsm32(%ymm4, %ymm5, %ymm6, %ymm7, %ymm0, %ymm1, %ymm2, %ymm3, 234 %ymm12, %ymm13, %ymm14, %ymm15, %ymm8, %ymm9, %ymm10, %ymm11, 235 %rax, (%r9)); 236 RET; 237SYM_FUNC_END(roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab) 238 239/* 240 * IN/OUT: 241 * x0..x7: byte-sliced AB state preloaded 242 * mem_ab: byte-sliced AB state in memory 243 * mem_cb: byte-sliced CD state in memory 244 */ 245#define two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 246 y6, y7, mem_ab, mem_cd, i, dir, store_ab) \ 247 leaq (key_table + (i) * 8)(CTX), %r9; \ 248 call roundsm32_x0_x1_x2_x3_x4_x5_x6_x7_y0_y1_y2_y3_y4_y5_y6_y7_cd; \ 249 \ 250 vmovdqu x0, 4 * 32(mem_cd); \ 251 vmovdqu x1, 5 * 32(mem_cd); \ 252 vmovdqu x2, 6 * 32(mem_cd); \ 253 vmovdqu x3, 7 * 32(mem_cd); \ 254 vmovdqu x4, 0 * 32(mem_cd); \ 255 vmovdqu x5, 1 * 32(mem_cd); \ 256 vmovdqu x6, 2 * 32(mem_cd); \ 257 vmovdqu x7, 3 * 32(mem_cd); \ 258 \ 259 leaq (key_table + ((i) + (dir)) * 8)(CTX), %r9; \ 260 call roundsm32_x4_x5_x6_x7_x0_x1_x2_x3_y4_y5_y6_y7_y0_y1_y2_y3_ab; \ 261 \ 262 store_ab(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab); 263 264#define dummy_store(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) /* do nothing */ 265 266#define store_ab_state(x0, x1, x2, x3, x4, x5, x6, x7, mem_ab) \ 267 /* Store new AB state */ \ 268 vmovdqu x4, 4 * 32(mem_ab); \ 269 vmovdqu x5, 5 * 32(mem_ab); \ 270 vmovdqu x6, 6 * 32(mem_ab); \ 271 vmovdqu x7, 7 * 32(mem_ab); \ 272 vmovdqu x0, 0 * 32(mem_ab); \ 273 vmovdqu x1, 1 * 32(mem_ab); \ 274 vmovdqu x2, 2 * 32(mem_ab); \ 275 vmovdqu x3, 3 * 32(mem_ab); 276 277#define enc_rounds32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 278 y6, y7, mem_ab, mem_cd, i) \ 279 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 280 y6, y7, mem_ab, mem_cd, (i) + 2, 1, store_ab_state); \ 281 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 282 y6, y7, mem_ab, mem_cd, (i) + 4, 1, store_ab_state); \ 283 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 284 y6, y7, mem_ab, mem_cd, (i) + 6, 1, dummy_store); 285 286#define dec_rounds32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 287 y6, y7, mem_ab, mem_cd, i) \ 288 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 289 y6, y7, mem_ab, mem_cd, (i) + 7, -1, store_ab_state); \ 290 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 291 y6, y7, mem_ab, mem_cd, (i) + 5, -1, store_ab_state); \ 292 two_roundsm32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 293 y6, y7, mem_ab, mem_cd, (i) + 3, -1, dummy_store); 294 295/* 296 * IN: 297 * v0..3: byte-sliced 32-bit integers 298 * OUT: 299 * v0..3: (IN <<< 1) 300 */ 301#define rol32_1_32(v0, v1, v2, v3, t0, t1, t2, zero) \ 302 vpcmpgtb v0, zero, t0; \ 303 vpaddb v0, v0, v0; \ 304 vpabsb t0, t0; \ 305 \ 306 vpcmpgtb v1, zero, t1; \ 307 vpaddb v1, v1, v1; \ 308 vpabsb t1, t1; \ 309 \ 310 vpcmpgtb v2, zero, t2; \ 311 vpaddb v2, v2, v2; \ 312 vpabsb t2, t2; \ 313 \ 314 vpor t0, v1, v1; \ 315 \ 316 vpcmpgtb v3, zero, t0; \ 317 vpaddb v3, v3, v3; \ 318 vpabsb t0, t0; \ 319 \ 320 vpor t1, v2, v2; \ 321 vpor t2, v3, v3; \ 322 vpor t0, v0, v0; 323 324/* 325 * IN: 326 * r: byte-sliced AB state in memory 327 * l: byte-sliced CD state in memory 328 * OUT: 329 * x0..x7: new byte-sliced CD state 330 */ 331#define fls32(l, l0, l1, l2, l3, l4, l5, l6, l7, r, t0, t1, t2, t3, tt0, \ 332 tt1, tt2, tt3, kll, klr, krl, krr) \ 333 /* \ 334 * t0 = kll; \ 335 * t0 &= ll; \ 336 * lr ^= rol32(t0, 1); \ 337 */ \ 338 vpbroadcastd kll, t0; /* only lowest 32-bit used */ \ 339 vpxor tt0, tt0, tt0; \ 340 vpshufb tt0, t0, t3; \ 341 vpsrldq $1, t0, t0; \ 342 vpshufb tt0, t0, t2; \ 343 vpsrldq $1, t0, t0; \ 344 vpshufb tt0, t0, t1; \ 345 vpsrldq $1, t0, t0; \ 346 vpshufb tt0, t0, t0; \ 347 \ 348 vpand l0, t0, t0; \ 349 vpand l1, t1, t1; \ 350 vpand l2, t2, t2; \ 351 vpand l3, t3, t3; \ 352 \ 353 rol32_1_32(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \ 354 \ 355 vpxor l4, t0, l4; \ 356 vpbroadcastd krr, t0; /* only lowest 32-bit used */ \ 357 vmovdqu l4, 4 * 32(l); \ 358 vpxor l5, t1, l5; \ 359 vmovdqu l5, 5 * 32(l); \ 360 vpxor l6, t2, l6; \ 361 vmovdqu l6, 6 * 32(l); \ 362 vpxor l7, t3, l7; \ 363 vmovdqu l7, 7 * 32(l); \ 364 \ 365 /* \ 366 * t2 = krr; \ 367 * t2 |= rr; \ 368 * rl ^= t2; \ 369 */ \ 370 \ 371 vpshufb tt0, t0, t3; \ 372 vpsrldq $1, t0, t0; \ 373 vpshufb tt0, t0, t2; \ 374 vpsrldq $1, t0, t0; \ 375 vpshufb tt0, t0, t1; \ 376 vpsrldq $1, t0, t0; \ 377 vpshufb tt0, t0, t0; \ 378 \ 379 vpor 4 * 32(r), t0, t0; \ 380 vpor 5 * 32(r), t1, t1; \ 381 vpor 6 * 32(r), t2, t2; \ 382 vpor 7 * 32(r), t3, t3; \ 383 \ 384 vpxor 0 * 32(r), t0, t0; \ 385 vpxor 1 * 32(r), t1, t1; \ 386 vpxor 2 * 32(r), t2, t2; \ 387 vpxor 3 * 32(r), t3, t3; \ 388 vmovdqu t0, 0 * 32(r); \ 389 vpbroadcastd krl, t0; /* only lowest 32-bit used */ \ 390 vmovdqu t1, 1 * 32(r); \ 391 vmovdqu t2, 2 * 32(r); \ 392 vmovdqu t3, 3 * 32(r); \ 393 \ 394 /* \ 395 * t2 = krl; \ 396 * t2 &= rl; \ 397 * rr ^= rol32(t2, 1); \ 398 */ \ 399 vpshufb tt0, t0, t3; \ 400 vpsrldq $1, t0, t0; \ 401 vpshufb tt0, t0, t2; \ 402 vpsrldq $1, t0, t0; \ 403 vpshufb tt0, t0, t1; \ 404 vpsrldq $1, t0, t0; \ 405 vpshufb tt0, t0, t0; \ 406 \ 407 vpand 0 * 32(r), t0, t0; \ 408 vpand 1 * 32(r), t1, t1; \ 409 vpand 2 * 32(r), t2, t2; \ 410 vpand 3 * 32(r), t3, t3; \ 411 \ 412 rol32_1_32(t3, t2, t1, t0, tt1, tt2, tt3, tt0); \ 413 \ 414 vpxor 4 * 32(r), t0, t0; \ 415 vpxor 5 * 32(r), t1, t1; \ 416 vpxor 6 * 32(r), t2, t2; \ 417 vpxor 7 * 32(r), t3, t3; \ 418 vmovdqu t0, 4 * 32(r); \ 419 vpbroadcastd klr, t0; /* only lowest 32-bit used */ \ 420 vmovdqu t1, 5 * 32(r); \ 421 vmovdqu t2, 6 * 32(r); \ 422 vmovdqu t3, 7 * 32(r); \ 423 \ 424 /* \ 425 * t0 = klr; \ 426 * t0 |= lr; \ 427 * ll ^= t0; \ 428 */ \ 429 \ 430 vpshufb tt0, t0, t3; \ 431 vpsrldq $1, t0, t0; \ 432 vpshufb tt0, t0, t2; \ 433 vpsrldq $1, t0, t0; \ 434 vpshufb tt0, t0, t1; \ 435 vpsrldq $1, t0, t0; \ 436 vpshufb tt0, t0, t0; \ 437 \ 438 vpor l4, t0, t0; \ 439 vpor l5, t1, t1; \ 440 vpor l6, t2, t2; \ 441 vpor l7, t3, t3; \ 442 \ 443 vpxor l0, t0, l0; \ 444 vmovdqu l0, 0 * 32(l); \ 445 vpxor l1, t1, l1; \ 446 vmovdqu l1, 1 * 32(l); \ 447 vpxor l2, t2, l2; \ 448 vmovdqu l2, 2 * 32(l); \ 449 vpxor l3, t3, l3; \ 450 vmovdqu l3, 3 * 32(l); 451 452#define transpose_4x4(x0, x1, x2, x3, t1, t2) \ 453 vpunpckhdq x1, x0, t2; \ 454 vpunpckldq x1, x0, x0; \ 455 \ 456 vpunpckldq x3, x2, t1; \ 457 vpunpckhdq x3, x2, x2; \ 458 \ 459 vpunpckhqdq t1, x0, x1; \ 460 vpunpcklqdq t1, x0, x0; \ 461 \ 462 vpunpckhqdq x2, t2, x3; \ 463 vpunpcklqdq x2, t2, x2; 464 465#define byteslice_16x16b_fast(a0, b0, c0, d0, a1, b1, c1, d1, a2, b2, c2, d2, \ 466 a3, b3, c3, d3, st0, st1) \ 467 vmovdqu d2, st0; \ 468 vmovdqu d3, st1; \ 469 transpose_4x4(a0, a1, a2, a3, d2, d3); \ 470 transpose_4x4(b0, b1, b2, b3, d2, d3); \ 471 vmovdqu st0, d2; \ 472 vmovdqu st1, d3; \ 473 \ 474 vmovdqu a0, st0; \ 475 vmovdqu a1, st1; \ 476 transpose_4x4(c0, c1, c2, c3, a0, a1); \ 477 transpose_4x4(d0, d1, d2, d3, a0, a1); \ 478 \ 479 vbroadcasti128 .Lshufb_16x16b(%rip), a0; \ 480 vmovdqu st1, a1; \ 481 vpshufb a0, a2, a2; \ 482 vpshufb a0, a3, a3; \ 483 vpshufb a0, b0, b0; \ 484 vpshufb a0, b1, b1; \ 485 vpshufb a0, b2, b2; \ 486 vpshufb a0, b3, b3; \ 487 vpshufb a0, a1, a1; \ 488 vpshufb a0, c0, c0; \ 489 vpshufb a0, c1, c1; \ 490 vpshufb a0, c2, c2; \ 491 vpshufb a0, c3, c3; \ 492 vpshufb a0, d0, d0; \ 493 vpshufb a0, d1, d1; \ 494 vpshufb a0, d2, d2; \ 495 vpshufb a0, d3, d3; \ 496 vmovdqu d3, st1; \ 497 vmovdqu st0, d3; \ 498 vpshufb a0, d3, a0; \ 499 vmovdqu d2, st0; \ 500 \ 501 transpose_4x4(a0, b0, c0, d0, d2, d3); \ 502 transpose_4x4(a1, b1, c1, d1, d2, d3); \ 503 vmovdqu st0, d2; \ 504 vmovdqu st1, d3; \ 505 \ 506 vmovdqu b0, st0; \ 507 vmovdqu b1, st1; \ 508 transpose_4x4(a2, b2, c2, d2, b0, b1); \ 509 transpose_4x4(a3, b3, c3, d3, b0, b1); \ 510 vmovdqu st0, b0; \ 511 vmovdqu st1, b1; \ 512 /* does not adjust output bytes inside vectors */ 513 514/* load blocks to registers and apply pre-whitening */ 515#define inpack32_pre(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 516 y6, y7, rio, key) \ 517 vpbroadcastq key, x0; \ 518 vpshufb .Lpack_bswap(%rip), x0, x0; \ 519 \ 520 vpxor 0 * 32(rio), x0, y7; \ 521 vpxor 1 * 32(rio), x0, y6; \ 522 vpxor 2 * 32(rio), x0, y5; \ 523 vpxor 3 * 32(rio), x0, y4; \ 524 vpxor 4 * 32(rio), x0, y3; \ 525 vpxor 5 * 32(rio), x0, y2; \ 526 vpxor 6 * 32(rio), x0, y1; \ 527 vpxor 7 * 32(rio), x0, y0; \ 528 vpxor 8 * 32(rio), x0, x7; \ 529 vpxor 9 * 32(rio), x0, x6; \ 530 vpxor 10 * 32(rio), x0, x5; \ 531 vpxor 11 * 32(rio), x0, x4; \ 532 vpxor 12 * 32(rio), x0, x3; \ 533 vpxor 13 * 32(rio), x0, x2; \ 534 vpxor 14 * 32(rio), x0, x1; \ 535 vpxor 15 * 32(rio), x0, x0; 536 537/* byteslice pre-whitened blocks and store to temporary memory */ 538#define inpack32_post(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 539 y6, y7, mem_ab, mem_cd) \ 540 byteslice_16x16b_fast(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, \ 541 y4, y5, y6, y7, (mem_ab), (mem_cd)); \ 542 \ 543 vmovdqu x0, 0 * 32(mem_ab); \ 544 vmovdqu x1, 1 * 32(mem_ab); \ 545 vmovdqu x2, 2 * 32(mem_ab); \ 546 vmovdqu x3, 3 * 32(mem_ab); \ 547 vmovdqu x4, 4 * 32(mem_ab); \ 548 vmovdqu x5, 5 * 32(mem_ab); \ 549 vmovdqu x6, 6 * 32(mem_ab); \ 550 vmovdqu x7, 7 * 32(mem_ab); \ 551 vmovdqu y0, 0 * 32(mem_cd); \ 552 vmovdqu y1, 1 * 32(mem_cd); \ 553 vmovdqu y2, 2 * 32(mem_cd); \ 554 vmovdqu y3, 3 * 32(mem_cd); \ 555 vmovdqu y4, 4 * 32(mem_cd); \ 556 vmovdqu y5, 5 * 32(mem_cd); \ 557 vmovdqu y6, 6 * 32(mem_cd); \ 558 vmovdqu y7, 7 * 32(mem_cd); 559 560/* de-byteslice, apply post-whitening and store blocks */ 561#define outunpack32(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, \ 562 y5, y6, y7, key, stack_tmp0, stack_tmp1) \ 563 byteslice_16x16b_fast(y0, y4, x0, x4, y1, y5, x1, x5, y2, y6, x2, x6, \ 564 y3, y7, x3, x7, stack_tmp0, stack_tmp1); \ 565 \ 566 vmovdqu x0, stack_tmp0; \ 567 \ 568 vpbroadcastq key, x0; \ 569 vpshufb .Lpack_bswap(%rip), x0, x0; \ 570 \ 571 vpxor x0, y7, y7; \ 572 vpxor x0, y6, y6; \ 573 vpxor x0, y5, y5; \ 574 vpxor x0, y4, y4; \ 575 vpxor x0, y3, y3; \ 576 vpxor x0, y2, y2; \ 577 vpxor x0, y1, y1; \ 578 vpxor x0, y0, y0; \ 579 vpxor x0, x7, x7; \ 580 vpxor x0, x6, x6; \ 581 vpxor x0, x5, x5; \ 582 vpxor x0, x4, x4; \ 583 vpxor x0, x3, x3; \ 584 vpxor x0, x2, x2; \ 585 vpxor x0, x1, x1; \ 586 vpxor stack_tmp0, x0, x0; 587 588#define write_output(x0, x1, x2, x3, x4, x5, x6, x7, y0, y1, y2, y3, y4, y5, \ 589 y6, y7, rio) \ 590 vmovdqu x0, 0 * 32(rio); \ 591 vmovdqu x1, 1 * 32(rio); \ 592 vmovdqu x2, 2 * 32(rio); \ 593 vmovdqu x3, 3 * 32(rio); \ 594 vmovdqu x4, 4 * 32(rio); \ 595 vmovdqu x5, 5 * 32(rio); \ 596 vmovdqu x6, 6 * 32(rio); \ 597 vmovdqu x7, 7 * 32(rio); \ 598 vmovdqu y0, 8 * 32(rio); \ 599 vmovdqu y1, 9 * 32(rio); \ 600 vmovdqu y2, 10 * 32(rio); \ 601 vmovdqu y3, 11 * 32(rio); \ 602 vmovdqu y4, 12 * 32(rio); \ 603 vmovdqu y5, 13 * 32(rio); \ 604 vmovdqu y6, 14 * 32(rio); \ 605 vmovdqu y7, 15 * 32(rio); 606 607 608.section .rodata.cst32.shufb_16x16b, "aM", @progbits, 32 609.align 32 610#define SHUFB_BYTES(idx) \ 611 0 + (idx), 4 + (idx), 8 + (idx), 12 + (idx) 612.Lshufb_16x16b: 613 .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3) 614 .byte SHUFB_BYTES(0), SHUFB_BYTES(1), SHUFB_BYTES(2), SHUFB_BYTES(3) 615 616.section .rodata.cst32.pack_bswap, "aM", @progbits, 32 617.align 32 618.Lpack_bswap: 619 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080 620 .long 0x00010203, 0x04050607, 0x80808080, 0x80808080 621 622/* NB: section is mergeable, all elements must be aligned 16-byte blocks */ 623.section .rodata.cst16, "aM", @progbits, 16 624.align 16 625 626/* 627 * pre-SubByte transform 628 * 629 * pre-lookup for sbox1, sbox2, sbox3: 630 * swap_bitendianness( 631 * isom_map_camellia_to_aes( 632 * camellia_f( 633 * swap_bitendianess(in) 634 * ) 635 * ) 636 * ) 637 * 638 * (note: '⊕ 0xc5' inside camellia_f()) 639 */ 640.Lpre_tf_lo_s1: 641 .byte 0x45, 0xe8, 0x40, 0xed, 0x2e, 0x83, 0x2b, 0x86 642 .byte 0x4b, 0xe6, 0x4e, 0xe3, 0x20, 0x8d, 0x25, 0x88 643.Lpre_tf_hi_s1: 644 .byte 0x00, 0x51, 0xf1, 0xa0, 0x8a, 0xdb, 0x7b, 0x2a 645 .byte 0x09, 0x58, 0xf8, 0xa9, 0x83, 0xd2, 0x72, 0x23 646 647/* 648 * pre-SubByte transform 649 * 650 * pre-lookup for sbox4: 651 * swap_bitendianness( 652 * isom_map_camellia_to_aes( 653 * camellia_f( 654 * swap_bitendianess(in <<< 1) 655 * ) 656 * ) 657 * ) 658 * 659 * (note: '⊕ 0xc5' inside camellia_f()) 660 */ 661.Lpre_tf_lo_s4: 662 .byte 0x45, 0x40, 0x2e, 0x2b, 0x4b, 0x4e, 0x20, 0x25 663 .byte 0x14, 0x11, 0x7f, 0x7a, 0x1a, 0x1f, 0x71, 0x74 664.Lpre_tf_hi_s4: 665 .byte 0x00, 0xf1, 0x8a, 0x7b, 0x09, 0xf8, 0x83, 0x72 666 .byte 0xad, 0x5c, 0x27, 0xd6, 0xa4, 0x55, 0x2e, 0xdf 667 668/* 669 * post-SubByte transform 670 * 671 * post-lookup for sbox1, sbox4: 672 * swap_bitendianness( 673 * camellia_h( 674 * isom_map_aes_to_camellia( 675 * swap_bitendianness( 676 * aes_inverse_affine_transform(in) 677 * ) 678 * ) 679 * ) 680 * ) 681 * 682 * (note: '⊕ 0x6e' inside camellia_h()) 683 */ 684.Lpost_tf_lo_s1: 685 .byte 0x3c, 0xcc, 0xcf, 0x3f, 0x32, 0xc2, 0xc1, 0x31 686 .byte 0xdc, 0x2c, 0x2f, 0xdf, 0xd2, 0x22, 0x21, 0xd1 687.Lpost_tf_hi_s1: 688 .byte 0x00, 0xf9, 0x86, 0x7f, 0xd7, 0x2e, 0x51, 0xa8 689 .byte 0xa4, 0x5d, 0x22, 0xdb, 0x73, 0x8a, 0xf5, 0x0c 690 691/* 692 * post-SubByte transform 693 * 694 * post-lookup for sbox2: 695 * swap_bitendianness( 696 * camellia_h( 697 * isom_map_aes_to_camellia( 698 * swap_bitendianness( 699 * aes_inverse_affine_transform(in) 700 * ) 701 * ) 702 * ) 703 * ) <<< 1 704 * 705 * (note: '⊕ 0x6e' inside camellia_h()) 706 */ 707.Lpost_tf_lo_s2: 708 .byte 0x78, 0x99, 0x9f, 0x7e, 0x64, 0x85, 0x83, 0x62 709 .byte 0xb9, 0x58, 0x5e, 0xbf, 0xa5, 0x44, 0x42, 0xa3 710.Lpost_tf_hi_s2: 711 .byte 0x00, 0xf3, 0x0d, 0xfe, 0xaf, 0x5c, 0xa2, 0x51 712 .byte 0x49, 0xba, 0x44, 0xb7, 0xe6, 0x15, 0xeb, 0x18 713 714/* 715 * post-SubByte transform 716 * 717 * post-lookup for sbox3: 718 * swap_bitendianness( 719 * camellia_h( 720 * isom_map_aes_to_camellia( 721 * swap_bitendianness( 722 * aes_inverse_affine_transform(in) 723 * ) 724 * ) 725 * ) 726 * ) >>> 1 727 * 728 * (note: '⊕ 0x6e' inside camellia_h()) 729 */ 730.Lpost_tf_lo_s3: 731 .byte 0x1e, 0x66, 0xe7, 0x9f, 0x19, 0x61, 0xe0, 0x98 732 .byte 0x6e, 0x16, 0x97, 0xef, 0x69, 0x11, 0x90, 0xe8 733.Lpost_tf_hi_s3: 734 .byte 0x00, 0xfc, 0x43, 0xbf, 0xeb, 0x17, 0xa8, 0x54 735 .byte 0x52, 0xae, 0x11, 0xed, 0xb9, 0x45, 0xfa, 0x06 736 737/* For isolating SubBytes from AESENCLAST, inverse shift row */ 738.Linv_shift_row: 739 .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b 740 .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03 741 742.section .rodata.cst4.L0f0f0f0f, "aM", @progbits, 4 743.align 4 744/* 4-bit mask */ 745.L0f0f0f0f: 746 .long 0x0f0f0f0f 747 748.text 749 750SYM_FUNC_START_LOCAL(__camellia_enc_blk32) 751 /* input: 752 * %rdi: ctx, CTX 753 * %rax: temporary storage, 512 bytes 754 * %ymm0..%ymm15: 32 plaintext blocks 755 * output: 756 * %ymm0..%ymm15: 32 encrypted blocks, order swapped: 757 * 7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 758 */ 759 FRAME_BEGIN 760 761 leaq 8 * 32(%rax), %rcx; 762 763 inpack32_post(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 764 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 765 %ymm15, %rax, %rcx); 766 767 enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 768 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 769 %ymm15, %rax, %rcx, 0); 770 771 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 772 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 773 %ymm15, 774 ((key_table + (8) * 8) + 0)(CTX), 775 ((key_table + (8) * 8) + 4)(CTX), 776 ((key_table + (8) * 8) + 8)(CTX), 777 ((key_table + (8) * 8) + 12)(CTX)); 778 779 enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 780 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 781 %ymm15, %rax, %rcx, 8); 782 783 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 784 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 785 %ymm15, 786 ((key_table + (16) * 8) + 0)(CTX), 787 ((key_table + (16) * 8) + 4)(CTX), 788 ((key_table + (16) * 8) + 8)(CTX), 789 ((key_table + (16) * 8) + 12)(CTX)); 790 791 enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 792 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 793 %ymm15, %rax, %rcx, 16); 794 795 movl $24, %r8d; 796 cmpl $16, key_length(CTX); 797 jne .Lenc_max32; 798 799.Lenc_done: 800 /* load CD for output */ 801 vmovdqu 0 * 32(%rcx), %ymm8; 802 vmovdqu 1 * 32(%rcx), %ymm9; 803 vmovdqu 2 * 32(%rcx), %ymm10; 804 vmovdqu 3 * 32(%rcx), %ymm11; 805 vmovdqu 4 * 32(%rcx), %ymm12; 806 vmovdqu 5 * 32(%rcx), %ymm13; 807 vmovdqu 6 * 32(%rcx), %ymm14; 808 vmovdqu 7 * 32(%rcx), %ymm15; 809 810 outunpack32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 811 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 812 %ymm15, (key_table)(CTX, %r8, 8), (%rax), 1 * 32(%rax)); 813 814 FRAME_END 815 RET; 816 817.align 8 818.Lenc_max32: 819 movl $32, %r8d; 820 821 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 822 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 823 %ymm15, 824 ((key_table + (24) * 8) + 0)(CTX), 825 ((key_table + (24) * 8) + 4)(CTX), 826 ((key_table + (24) * 8) + 8)(CTX), 827 ((key_table + (24) * 8) + 12)(CTX)); 828 829 enc_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 830 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 831 %ymm15, %rax, %rcx, 24); 832 833 jmp .Lenc_done; 834SYM_FUNC_END(__camellia_enc_blk32) 835 836SYM_FUNC_START_LOCAL(__camellia_dec_blk32) 837 /* input: 838 * %rdi: ctx, CTX 839 * %rax: temporary storage, 512 bytes 840 * %r8d: 24 for 16 byte key, 32 for larger 841 * %ymm0..%ymm15: 16 encrypted blocks 842 * output: 843 * %ymm0..%ymm15: 16 plaintext blocks, order swapped: 844 * 7, 8, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 845 */ 846 FRAME_BEGIN 847 848 leaq 8 * 32(%rax), %rcx; 849 850 inpack32_post(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 851 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 852 %ymm15, %rax, %rcx); 853 854 cmpl $32, %r8d; 855 je .Ldec_max32; 856 857.Ldec_max24: 858 dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 859 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 860 %ymm15, %rax, %rcx, 16); 861 862 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 863 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 864 %ymm15, 865 ((key_table + (16) * 8) + 8)(CTX), 866 ((key_table + (16) * 8) + 12)(CTX), 867 ((key_table + (16) * 8) + 0)(CTX), 868 ((key_table + (16) * 8) + 4)(CTX)); 869 870 dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 871 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 872 %ymm15, %rax, %rcx, 8); 873 874 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 875 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 876 %ymm15, 877 ((key_table + (8) * 8) + 8)(CTX), 878 ((key_table + (8) * 8) + 12)(CTX), 879 ((key_table + (8) * 8) + 0)(CTX), 880 ((key_table + (8) * 8) + 4)(CTX)); 881 882 dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 883 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 884 %ymm15, %rax, %rcx, 0); 885 886 /* load CD for output */ 887 vmovdqu 0 * 32(%rcx), %ymm8; 888 vmovdqu 1 * 32(%rcx), %ymm9; 889 vmovdqu 2 * 32(%rcx), %ymm10; 890 vmovdqu 3 * 32(%rcx), %ymm11; 891 vmovdqu 4 * 32(%rcx), %ymm12; 892 vmovdqu 5 * 32(%rcx), %ymm13; 893 vmovdqu 6 * 32(%rcx), %ymm14; 894 vmovdqu 7 * 32(%rcx), %ymm15; 895 896 outunpack32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 897 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 898 %ymm15, (key_table)(CTX), (%rax), 1 * 32(%rax)); 899 900 FRAME_END 901 RET; 902 903.align 8 904.Ldec_max32: 905 dec_rounds32(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 906 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 907 %ymm15, %rax, %rcx, 24); 908 909 fls32(%rax, %ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 910 %rcx, %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 911 %ymm15, 912 ((key_table + (24) * 8) + 8)(CTX), 913 ((key_table + (24) * 8) + 12)(CTX), 914 ((key_table + (24) * 8) + 0)(CTX), 915 ((key_table + (24) * 8) + 4)(CTX)); 916 917 jmp .Ldec_max24; 918SYM_FUNC_END(__camellia_dec_blk32) 919 920SYM_FUNC_START(camellia_ecb_enc_32way) 921 /* input: 922 * %rdi: ctx, CTX 923 * %rsi: dst (32 blocks) 924 * %rdx: src (32 blocks) 925 */ 926 FRAME_BEGIN 927 928 vzeroupper; 929 930 inpack32_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 931 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 932 %ymm15, %rdx, (key_table)(CTX)); 933 934 /* now dst can be used as temporary buffer (even in src == dst case) */ 935 movq %rsi, %rax; 936 937 call __camellia_enc_blk32; 938 939 write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, 940 %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, 941 %ymm8, %rsi); 942 943 vzeroupper; 944 945 FRAME_END 946 RET; 947SYM_FUNC_END(camellia_ecb_enc_32way) 948 949SYM_FUNC_START(camellia_ecb_dec_32way) 950 /* input: 951 * %rdi: ctx, CTX 952 * %rsi: dst (32 blocks) 953 * %rdx: src (32 blocks) 954 */ 955 FRAME_BEGIN 956 957 vzeroupper; 958 959 cmpl $16, key_length(CTX); 960 movl $32, %r8d; 961 movl $24, %eax; 962 cmovel %eax, %r8d; /* max */ 963 964 inpack32_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 965 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 966 %ymm15, %rdx, (key_table)(CTX, %r8, 8)); 967 968 /* now dst can be used as temporary buffer (even in src == dst case) */ 969 movq %rsi, %rax; 970 971 call __camellia_dec_blk32; 972 973 write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, 974 %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, 975 %ymm8, %rsi); 976 977 vzeroupper; 978 979 FRAME_END 980 RET; 981SYM_FUNC_END(camellia_ecb_dec_32way) 982 983SYM_FUNC_START(camellia_cbc_dec_32way) 984 /* input: 985 * %rdi: ctx, CTX 986 * %rsi: dst (32 blocks) 987 * %rdx: src (32 blocks) 988 */ 989 FRAME_BEGIN 990 subq $(16 * 32), %rsp; 991 992 vzeroupper; 993 994 cmpl $16, key_length(CTX); 995 movl $32, %r8d; 996 movl $24, %eax; 997 cmovel %eax, %r8d; /* max */ 998 999 inpack32_pre(%ymm0, %ymm1, %ymm2, %ymm3, %ymm4, %ymm5, %ymm6, %ymm7, 1000 %ymm8, %ymm9, %ymm10, %ymm11, %ymm12, %ymm13, %ymm14, 1001 %ymm15, %rdx, (key_table)(CTX, %r8, 8)); 1002 1003 cmpq %rsi, %rdx; 1004 je .Lcbc_dec_use_stack; 1005 1006 /* dst can be used as temporary storage, src is not overwritten. */ 1007 movq %rsi, %rax; 1008 jmp .Lcbc_dec_continue; 1009 1010.Lcbc_dec_use_stack: 1011 /* 1012 * dst still in-use (because dst == src), so use stack for temporary 1013 * storage. 1014 */ 1015 movq %rsp, %rax; 1016 1017.Lcbc_dec_continue: 1018 call __camellia_dec_blk32; 1019 1020 vmovdqu %ymm7, (%rax); 1021 vpxor %ymm7, %ymm7, %ymm7; 1022 vinserti128 $1, (%rdx), %ymm7, %ymm7; 1023 vpxor (%rax), %ymm7, %ymm7; 1024 vpxor (0 * 32 + 16)(%rdx), %ymm6, %ymm6; 1025 vpxor (1 * 32 + 16)(%rdx), %ymm5, %ymm5; 1026 vpxor (2 * 32 + 16)(%rdx), %ymm4, %ymm4; 1027 vpxor (3 * 32 + 16)(%rdx), %ymm3, %ymm3; 1028 vpxor (4 * 32 + 16)(%rdx), %ymm2, %ymm2; 1029 vpxor (5 * 32 + 16)(%rdx), %ymm1, %ymm1; 1030 vpxor (6 * 32 + 16)(%rdx), %ymm0, %ymm0; 1031 vpxor (7 * 32 + 16)(%rdx), %ymm15, %ymm15; 1032 vpxor (8 * 32 + 16)(%rdx), %ymm14, %ymm14; 1033 vpxor (9 * 32 + 16)(%rdx), %ymm13, %ymm13; 1034 vpxor (10 * 32 + 16)(%rdx), %ymm12, %ymm12; 1035 vpxor (11 * 32 + 16)(%rdx), %ymm11, %ymm11; 1036 vpxor (12 * 32 + 16)(%rdx), %ymm10, %ymm10; 1037 vpxor (13 * 32 + 16)(%rdx), %ymm9, %ymm9; 1038 vpxor (14 * 32 + 16)(%rdx), %ymm8, %ymm8; 1039 write_output(%ymm7, %ymm6, %ymm5, %ymm4, %ymm3, %ymm2, %ymm1, %ymm0, 1040 %ymm15, %ymm14, %ymm13, %ymm12, %ymm11, %ymm10, %ymm9, 1041 %ymm8, %rsi); 1042 1043 vzeroupper; 1044 1045 addq $(16 * 32), %rsp; 1046 FRAME_END 1047 RET; 1048SYM_FUNC_END(camellia_cbc_dec_32way) 1049