1/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ 2// 3// AES-GCM implementation for x86_64 CPUs that support the following CPU 4// features: VAES && VPCLMULQDQ && AVX2 5// 6// Copyright 2025 Google LLC 7// 8// Author: Eric Biggers <ebiggers@google.com> 9// 10//------------------------------------------------------------------------------ 11// 12// This file is dual-licensed, meaning that you can use it under your choice of 13// either of the following two licenses: 14// 15// Licensed under the Apache License 2.0 (the "License"). You may obtain a copy 16// of the License at 17// 18// http://www.apache.org/licenses/LICENSE-2.0 19// 20// Unless required by applicable law or agreed to in writing, software 21// distributed under the License is distributed on an "AS IS" BASIS, 22// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23// See the License for the specific language governing permissions and 24// limitations under the License. 25// 26// or 27// 28// Redistribution and use in source and binary forms, with or without 29// modification, are permitted provided that the following conditions are met: 30// 31// 1. Redistributions of source code must retain the above copyright notice, 32// this list of conditions and the following disclaimer. 33// 34// 2. Redistributions in binary form must reproduce the above copyright 35// notice, this list of conditions and the following disclaimer in the 36// documentation and/or other materials provided with the distribution. 37// 38// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 39// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 40// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 41// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 42// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 43// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 44// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 45// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 46// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 47// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 48// POSSIBILITY OF SUCH DAMAGE. 49// 50// ----------------------------------------------------------------------------- 51// 52// This is similar to aes-gcm-vaes-avx512.S, but it uses AVX2 instead of AVX512. 53// This means it can only use 16 vector registers instead of 32, the maximum 54// vector length is 32 bytes, and some instructions such as vpternlogd and 55// masked loads/stores are unavailable. However, it is able to run on CPUs that 56// have VAES without AVX512, namely AMD Zen 3 (including "Milan" server CPUs), 57// various Intel client CPUs such as Alder Lake, and Intel Sierra Forest. 58// 59// This implementation also uses Karatsuba multiplication instead of schoolbook 60// multiplication for GHASH in its main loop. This does not help much on Intel, 61// but it improves performance by ~5% on AMD Zen 3. Other factors weighing 62// slightly in favor of Karatsuba multiplication in this implementation are the 63// lower maximum vector length (which means there are fewer key powers, so we 64// can cache the halves of each key power XOR'd together and still use less 65// memory than the AVX512 implementation), and the unavailability of the 66// vpternlogd instruction (which helped schoolbook a bit more than Karatsuba). 67 68#include <linux/linkage.h> 69 70.section .rodata 71.p2align 4 72 73 // The below three 16-byte values must be in the order that they are, as 74 // they are really two 32-byte tables and a 16-byte value that overlap: 75 // 76 // - The first 32-byte table begins at .Lselect_high_bytes_table. 77 // For 0 <= len <= 16, the 16-byte value at 78 // '.Lselect_high_bytes_table + len' selects the high 'len' bytes of 79 // another 16-byte value when AND'ed with it. 80 // 81 // - The second 32-byte table begins at .Lrshift_and_bswap_table. 82 // For 0 <= len <= 16, the 16-byte value at 83 // '.Lrshift_and_bswap_table + len' is a vpshufb mask that does the 84 // following operation: right-shift by '16 - len' bytes (shifting in 85 // zeroes), then reflect all 16 bytes. 86 // 87 // - The 16-byte value at .Lbswap_mask is a vpshufb mask that reflects 88 // all 16 bytes. 89.Lselect_high_bytes_table: 90 .octa 0 91.Lrshift_and_bswap_table: 92 .octa 0xffffffffffffffffffffffffffffffff 93.Lbswap_mask: 94 .octa 0x000102030405060708090a0b0c0d0e0f 95 96 // Sixteen 0x0f bytes. By XOR'ing an entry of .Lrshift_and_bswap_table 97 // with this, we get a mask that left-shifts by '16 - len' bytes. 98.Lfifteens: 99 .octa 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f 100 101 // This is the GHASH reducing polynomial without its constant term, i.e. 102 // x^128 + x^7 + x^2 + x, represented using the backwards mapping 103 // between bits and polynomial coefficients. 104 // 105 // Alternatively, it can be interpreted as the naturally-ordered 106 // representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the 107 // "reversed" GHASH reducing polynomial without its x^128 term. 108.Lgfpoly: 109 .octa 0xc2000000000000000000000000000001 110 111 // Same as above, but with the (1 << 64) bit set. 112.Lgfpoly_and_internal_carrybit: 113 .octa 0xc2000000000000010000000000000001 114 115 // Values needed to prepare the initial vector of counter blocks. 116.Lctr_pattern: 117 .octa 0 118 .octa 1 119 120 // The number of AES blocks per vector, as a 128-bit value. 121.Linc_2blocks: 122 .octa 2 123 124// Offsets in struct aes_gcm_key_vaes_avx2 125#define OFFSETOF_AESKEYLEN 480 126#define OFFSETOF_H_POWERS 512 127#define NUM_H_POWERS 8 128#define OFFSETOFEND_H_POWERS (OFFSETOF_H_POWERS + (NUM_H_POWERS * 16)) 129#define OFFSETOF_H_POWERS_XORED OFFSETOFEND_H_POWERS 130 131.text 132 133// Do one step of GHASH-multiplying the 128-bit lanes of \a by the 128-bit lanes 134// of \b and storing the reduced products in \dst. Uses schoolbook 135// multiplication. 136.macro _ghash_mul_step i, a, b, dst, gfpoly, t0, t1, t2 137.if \i == 0 138 vpclmulqdq $0x00, \a, \b, \t0 // LO = a_L * b_L 139 vpclmulqdq $0x01, \a, \b, \t1 // MI_0 = a_L * b_H 140.elseif \i == 1 141 vpclmulqdq $0x10, \a, \b, \t2 // MI_1 = a_H * b_L 142.elseif \i == 2 143 vpxor \t2, \t1, \t1 // MI = MI_0 + MI_1 144.elseif \i == 3 145 vpclmulqdq $0x01, \t0, \gfpoly, \t2 // LO_L*(x^63 + x^62 + x^57) 146.elseif \i == 4 147 vpshufd $0x4e, \t0, \t0 // Swap halves of LO 148.elseif \i == 5 149 vpxor \t0, \t1, \t1 // Fold LO into MI (part 1) 150 vpxor \t2, \t1, \t1 // Fold LO into MI (part 2) 151.elseif \i == 6 152 vpclmulqdq $0x11, \a, \b, \dst // HI = a_H * b_H 153.elseif \i == 7 154 vpclmulqdq $0x01, \t1, \gfpoly, \t0 // MI_L*(x^63 + x^62 + x^57) 155.elseif \i == 8 156 vpshufd $0x4e, \t1, \t1 // Swap halves of MI 157.elseif \i == 9 158 vpxor \t1, \dst, \dst // Fold MI into HI (part 1) 159 vpxor \t0, \dst, \dst // Fold MI into HI (part 2) 160.endif 161.endm 162 163// GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and store 164// the reduced products in \dst. See _ghash_mul_step for full explanation. 165.macro _ghash_mul a, b, dst, gfpoly, t0, t1, t2 166.irp i, 0,1,2,3,4,5,6,7,8,9 167 _ghash_mul_step \i, \a, \b, \dst, \gfpoly, \t0, \t1, \t2 168.endr 169.endm 170 171// GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and add the 172// *unreduced* products to \lo, \mi, and \hi. 173.macro _ghash_mul_noreduce a, b, lo, mi, hi, t0 174 vpclmulqdq $0x00, \a, \b, \t0 // a_L * b_L 175 vpxor \t0, \lo, \lo 176 vpclmulqdq $0x01, \a, \b, \t0 // a_L * b_H 177 vpxor \t0, \mi, \mi 178 vpclmulqdq $0x10, \a, \b, \t0 // a_H * b_L 179 vpxor \t0, \mi, \mi 180 vpclmulqdq $0x11, \a, \b, \t0 // a_H * b_H 181 vpxor \t0, \hi, \hi 182.endm 183 184// Reduce the unreduced products from \lo, \mi, and \hi and store the 128-bit 185// reduced products in \hi. See _ghash_mul_step for explanation of reduction. 186.macro _ghash_reduce lo, mi, hi, gfpoly, t0 187 vpclmulqdq $0x01, \lo, \gfpoly, \t0 188 vpshufd $0x4e, \lo, \lo 189 vpxor \lo, \mi, \mi 190 vpxor \t0, \mi, \mi 191 vpclmulqdq $0x01, \mi, \gfpoly, \t0 192 vpshufd $0x4e, \mi, \mi 193 vpxor \mi, \hi, \hi 194 vpxor \t0, \hi, \hi 195.endm 196 197// This is a specialized version of _ghash_mul that computes \a * \a, i.e. it 198// squares \a. It skips computing MI = (a_L * a_H) + (a_H * a_L) = 0. 199.macro _ghash_square a, dst, gfpoly, t0, t1 200 vpclmulqdq $0x00, \a, \a, \t0 // LO = a_L * a_L 201 vpclmulqdq $0x11, \a, \a, \dst // HI = a_H * a_H 202 vpclmulqdq $0x01, \t0, \gfpoly, \t1 // LO_L*(x^63 + x^62 + x^57) 203 vpshufd $0x4e, \t0, \t0 // Swap halves of LO 204 vpxor \t0, \t1, \t1 // Fold LO into MI 205 vpclmulqdq $0x01, \t1, \gfpoly, \t0 // MI_L*(x^63 + x^62 + x^57) 206 vpshufd $0x4e, \t1, \t1 // Swap halves of MI 207 vpxor \t1, \dst, \dst // Fold MI into HI (part 1) 208 vpxor \t0, \dst, \dst // Fold MI into HI (part 2) 209.endm 210 211// void aes_gcm_precompute_vaes_avx2(struct aes_gcm_key_vaes_avx2 *key); 212// 213// Given the expanded AES key |key->base.aes_key|, derive the GHASH subkey and 214// initialize |key->h_powers| and |key->h_powers_xored|. 215// 216// We use h_powers[0..7] to store H^8 through H^1, and h_powers_xored[0..7] to 217// store the 64-bit halves of the key powers XOR'd together (for Karatsuba 218// multiplication) in the order 8,6,7,5,4,2,3,1. 219SYM_FUNC_START(aes_gcm_precompute_vaes_avx2) 220 221 // Function arguments 222 .set KEY, %rdi 223 224 // Additional local variables 225 .set POWERS_PTR, %rsi 226 .set RNDKEYLAST_PTR, %rdx 227 .set TMP0, %ymm0 228 .set TMP0_XMM, %xmm0 229 .set TMP1, %ymm1 230 .set TMP1_XMM, %xmm1 231 .set TMP2, %ymm2 232 .set TMP2_XMM, %xmm2 233 .set H_CUR, %ymm3 234 .set H_CUR_XMM, %xmm3 235 .set H_CUR2, %ymm4 236 .set H_INC, %ymm5 237 .set H_INC_XMM, %xmm5 238 .set GFPOLY, %ymm6 239 .set GFPOLY_XMM, %xmm6 240 241 // Encrypt an all-zeroes block to get the raw hash subkey. 242 movl OFFSETOF_AESKEYLEN(KEY), %eax 243 lea 6*16(KEY,%rax,4), RNDKEYLAST_PTR 244 vmovdqu (KEY), H_CUR_XMM // Zero-th round key XOR all-zeroes block 245 lea 16(KEY), %rax 2461: 247 vaesenc (%rax), H_CUR_XMM, H_CUR_XMM 248 add $16, %rax 249 cmp %rax, RNDKEYLAST_PTR 250 jne 1b 251 vaesenclast (RNDKEYLAST_PTR), H_CUR_XMM, H_CUR_XMM 252 253 // Reflect the bytes of the raw hash subkey. 254 vpshufb .Lbswap_mask(%rip), H_CUR_XMM, H_CUR_XMM 255 256 // Finish preprocessing the byte-reflected hash subkey by multiplying it 257 // by x^-1 ("standard" interpretation of polynomial coefficients) or 258 // equivalently x^1 (natural interpretation). This gets the key into a 259 // format that avoids having to bit-reflect the data blocks later. 260 vpshufd $0xd3, H_CUR_XMM, TMP0_XMM 261 vpsrad $31, TMP0_XMM, TMP0_XMM 262 vpaddq H_CUR_XMM, H_CUR_XMM, H_CUR_XMM 263 vpand .Lgfpoly_and_internal_carrybit(%rip), TMP0_XMM, TMP0_XMM 264 vpxor TMP0_XMM, H_CUR_XMM, H_CUR_XMM 265 266 // Load the gfpoly constant. 267 vbroadcasti128 .Lgfpoly(%rip), GFPOLY 268 269 // Square H^1 to get H^2. 270 _ghash_square H_CUR_XMM, H_INC_XMM, GFPOLY_XMM, TMP0_XMM, TMP1_XMM 271 272 // Create H_CUR = [H^2, H^1] and H_INC = [H^2, H^2]. 273 vinserti128 $1, H_CUR_XMM, H_INC, H_CUR 274 vinserti128 $1, H_INC_XMM, H_INC, H_INC 275 276 // Compute H_CUR2 = [H^4, H^3]. 277 _ghash_mul H_INC, H_CUR, H_CUR2, GFPOLY, TMP0, TMP1, TMP2 278 279 // Store [H^2, H^1] and [H^4, H^3]. 280 vmovdqu H_CUR, OFFSETOF_H_POWERS+3*32(KEY) 281 vmovdqu H_CUR2, OFFSETOF_H_POWERS+2*32(KEY) 282 283 // For Karatsuba multiplication: compute and store the two 64-bit halves 284 // of each key power XOR'd together. Order is 4,2,3,1. 285 vpunpcklqdq H_CUR, H_CUR2, TMP0 286 vpunpckhqdq H_CUR, H_CUR2, TMP1 287 vpxor TMP1, TMP0, TMP0 288 vmovdqu TMP0, OFFSETOF_H_POWERS_XORED+32(KEY) 289 290 // Compute and store H_CUR = [H^6, H^5] and H_CUR2 = [H^8, H^7]. 291 _ghash_mul H_INC, H_CUR2, H_CUR, GFPOLY, TMP0, TMP1, TMP2 292 _ghash_mul H_INC, H_CUR, H_CUR2, GFPOLY, TMP0, TMP1, TMP2 293 vmovdqu H_CUR, OFFSETOF_H_POWERS+1*32(KEY) 294 vmovdqu H_CUR2, OFFSETOF_H_POWERS+0*32(KEY) 295 296 // Again, compute and store the two 64-bit halves of each key power 297 // XOR'd together. Order is 8,6,7,5. 298 vpunpcklqdq H_CUR, H_CUR2, TMP0 299 vpunpckhqdq H_CUR, H_CUR2, TMP1 300 vpxor TMP1, TMP0, TMP0 301 vmovdqu TMP0, OFFSETOF_H_POWERS_XORED(KEY) 302 303 vzeroupper 304 RET 305SYM_FUNC_END(aes_gcm_precompute_vaes_avx2) 306 307// Do one step of the GHASH update of four vectors of data blocks. 308// \i: the step to do, 0 through 9 309// \ghashdata_ptr: pointer to the data blocks (ciphertext or AAD) 310// KEY: pointer to struct aes_gcm_key_vaes_avx2 311// BSWAP_MASK: mask for reflecting the bytes of blocks 312// H_POW[2-1]_XORED: cached values from KEY->h_powers_xored 313// TMP[0-2]: temporary registers. TMP[1-2] must be preserved across steps. 314// LO, MI: working state for this macro that must be preserved across steps 315// GHASH_ACC: the GHASH accumulator (input/output) 316.macro _ghash_step_4x i, ghashdata_ptr 317 .set HI, GHASH_ACC # alias 318 .set HI_XMM, GHASH_ACC_XMM 319.if \i == 0 320 // First vector 321 vmovdqu 0*32(\ghashdata_ptr), TMP1 322 vpshufb BSWAP_MASK, TMP1, TMP1 323 vmovdqu OFFSETOF_H_POWERS+0*32(KEY), TMP2 324 vpxor GHASH_ACC, TMP1, TMP1 325 vpclmulqdq $0x00, TMP2, TMP1, LO 326 vpclmulqdq $0x11, TMP2, TMP1, HI 327 vpunpckhqdq TMP1, TMP1, TMP0 328 vpxor TMP1, TMP0, TMP0 329 vpclmulqdq $0x00, H_POW2_XORED, TMP0, MI 330.elseif \i == 1 331.elseif \i == 2 332 // Second vector 333 vmovdqu 1*32(\ghashdata_ptr), TMP1 334 vpshufb BSWAP_MASK, TMP1, TMP1 335 vmovdqu OFFSETOF_H_POWERS+1*32(KEY), TMP2 336 vpclmulqdq $0x00, TMP2, TMP1, TMP0 337 vpxor TMP0, LO, LO 338 vpclmulqdq $0x11, TMP2, TMP1, TMP0 339 vpxor TMP0, HI, HI 340 vpunpckhqdq TMP1, TMP1, TMP0 341 vpxor TMP1, TMP0, TMP0 342 vpclmulqdq $0x10, H_POW2_XORED, TMP0, TMP0 343 vpxor TMP0, MI, MI 344.elseif \i == 3 345 // Third vector 346 vmovdqu 2*32(\ghashdata_ptr), TMP1 347 vpshufb BSWAP_MASK, TMP1, TMP1 348 vmovdqu OFFSETOF_H_POWERS+2*32(KEY), TMP2 349.elseif \i == 4 350 vpclmulqdq $0x00, TMP2, TMP1, TMP0 351 vpxor TMP0, LO, LO 352 vpclmulqdq $0x11, TMP2, TMP1, TMP0 353 vpxor TMP0, HI, HI 354.elseif \i == 5 355 vpunpckhqdq TMP1, TMP1, TMP0 356 vpxor TMP1, TMP0, TMP0 357 vpclmulqdq $0x00, H_POW1_XORED, TMP0, TMP0 358 vpxor TMP0, MI, MI 359 360 // Fourth vector 361 vmovdqu 3*32(\ghashdata_ptr), TMP1 362 vpshufb BSWAP_MASK, TMP1, TMP1 363.elseif \i == 6 364 vmovdqu OFFSETOF_H_POWERS+3*32(KEY), TMP2 365 vpclmulqdq $0x00, TMP2, TMP1, TMP0 366 vpxor TMP0, LO, LO 367 vpclmulqdq $0x11, TMP2, TMP1, TMP0 368 vpxor TMP0, HI, HI 369 vpunpckhqdq TMP1, TMP1, TMP0 370 vpxor TMP1, TMP0, TMP0 371 vpclmulqdq $0x10, H_POW1_XORED, TMP0, TMP0 372 vpxor TMP0, MI, MI 373.elseif \i == 7 374 // Finalize 'mi' following Karatsuba multiplication. 375 vpxor LO, MI, MI 376 vpxor HI, MI, MI 377 378 // Fold lo into mi. 379 vbroadcasti128 .Lgfpoly(%rip), TMP2 380 vpclmulqdq $0x01, LO, TMP2, TMP0 381 vpshufd $0x4e, LO, LO 382 vpxor LO, MI, MI 383 vpxor TMP0, MI, MI 384.elseif \i == 8 385 // Fold mi into hi. 386 vpclmulqdq $0x01, MI, TMP2, TMP0 387 vpshufd $0x4e, MI, MI 388 vpxor MI, HI, HI 389 vpxor TMP0, HI, HI 390.elseif \i == 9 391 vextracti128 $1, HI, TMP0_XMM 392 vpxor TMP0_XMM, HI_XMM, GHASH_ACC_XMM 393.endif 394.endm 395 396// Update GHASH with four vectors of data blocks. See _ghash_step_4x for full 397// explanation. 398.macro _ghash_4x ghashdata_ptr 399.irp i, 0,1,2,3,4,5,6,7,8,9 400 _ghash_step_4x \i, \ghashdata_ptr 401.endr 402.endm 403 404// Load 1 <= %ecx <= 16 bytes from the pointer \src into the xmm register \dst 405// and zeroize any remaining bytes. Clobbers %rax, %rcx, and \tmp{64,32}. 406.macro _load_partial_block src, dst, tmp64, tmp32 407 sub $8, %ecx // LEN - 8 408 jle .Lle8\@ 409 410 // Load 9 <= LEN <= 16 bytes. 411 vmovq (\src), \dst // Load first 8 bytes 412 mov (\src, %rcx), %rax // Load last 8 bytes 413 neg %ecx 414 shl $3, %ecx 415 shr %cl, %rax // Discard overlapping bytes 416 vpinsrq $1, %rax, \dst, \dst 417 jmp .Ldone\@ 418 419.Lle8\@: 420 add $4, %ecx // LEN - 4 421 jl .Llt4\@ 422 423 // Load 4 <= LEN <= 8 bytes. 424 mov (\src), %eax // Load first 4 bytes 425 mov (\src, %rcx), \tmp32 // Load last 4 bytes 426 jmp .Lcombine\@ 427 428.Llt4\@: 429 // Load 1 <= LEN <= 3 bytes. 430 add $2, %ecx // LEN - 2 431 movzbl (\src), %eax // Load first byte 432 jl .Lmovq\@ 433 movzwl (\src, %rcx), \tmp32 // Load last 2 bytes 434.Lcombine\@: 435 shl $3, %ecx 436 shl %cl, \tmp64 437 or \tmp64, %rax // Combine the two parts 438.Lmovq\@: 439 vmovq %rax, \dst 440.Ldone\@: 441.endm 442 443// Store 1 <= %ecx <= 16 bytes from the xmm register \src to the pointer \dst. 444// Clobbers %rax, %rcx, and \tmp{64,32}. 445.macro _store_partial_block src, dst, tmp64, tmp32 446 sub $8, %ecx // LEN - 8 447 jl .Llt8\@ 448 449 // Store 8 <= LEN <= 16 bytes. 450 vpextrq $1, \src, %rax 451 mov %ecx, \tmp32 452 shl $3, %ecx 453 ror %cl, %rax 454 mov %rax, (\dst, \tmp64) // Store last LEN - 8 bytes 455 vmovq \src, (\dst) // Store first 8 bytes 456 jmp .Ldone\@ 457 458.Llt8\@: 459 add $4, %ecx // LEN - 4 460 jl .Llt4\@ 461 462 // Store 4 <= LEN <= 7 bytes. 463 vpextrd $1, \src, %eax 464 mov %ecx, \tmp32 465 shl $3, %ecx 466 ror %cl, %eax 467 mov %eax, (\dst, \tmp64) // Store last LEN - 4 bytes 468 vmovd \src, (\dst) // Store first 4 bytes 469 jmp .Ldone\@ 470 471.Llt4\@: 472 // Store 1 <= LEN <= 3 bytes. 473 vpextrb $0, \src, 0(\dst) 474 cmp $-2, %ecx // LEN - 4 == -2, i.e. LEN == 2? 475 jl .Ldone\@ 476 vpextrb $1, \src, 1(\dst) 477 je .Ldone\@ 478 vpextrb $2, \src, 2(\dst) 479.Ldone\@: 480.endm 481 482// void aes_gcm_aad_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key, 483// u8 ghash_acc[16], 484// const u8 *aad, int aadlen); 485// 486// This function processes the AAD (Additional Authenticated Data) in GCM. 487// Using the key |key|, it updates the GHASH accumulator |ghash_acc| with the 488// data given by |aad| and |aadlen|. On the first call, |ghash_acc| must be all 489// zeroes. |aadlen| must be a multiple of 16, except on the last call where it 490// can be any length. The caller must do any buffering needed to ensure this. 491// 492// This handles large amounts of AAD efficiently, while also keeping overhead 493// low for small amounts which is the common case. TLS and IPsec use less than 494// one block of AAD, but (uncommonly) other use cases may use much more. 495SYM_FUNC_START(aes_gcm_aad_update_vaes_avx2) 496 497 // Function arguments 498 .set KEY, %rdi 499 .set GHASH_ACC_PTR, %rsi 500 .set AAD, %rdx 501 .set AADLEN, %ecx // Must be %ecx for _load_partial_block 502 .set AADLEN64, %rcx // Zero-extend AADLEN before using! 503 504 // Additional local variables. 505 // %rax and %r8 are used as temporary registers. 506 .set TMP0, %ymm0 507 .set TMP0_XMM, %xmm0 508 .set TMP1, %ymm1 509 .set TMP1_XMM, %xmm1 510 .set TMP2, %ymm2 511 .set TMP2_XMM, %xmm2 512 .set LO, %ymm3 513 .set LO_XMM, %xmm3 514 .set MI, %ymm4 515 .set MI_XMM, %xmm4 516 .set GHASH_ACC, %ymm5 517 .set GHASH_ACC_XMM, %xmm5 518 .set BSWAP_MASK, %ymm6 519 .set BSWAP_MASK_XMM, %xmm6 520 .set GFPOLY, %ymm7 521 .set GFPOLY_XMM, %xmm7 522 .set H_POW2_XORED, %ymm8 523 .set H_POW1_XORED, %ymm9 524 525 // Load the bswap_mask and gfpoly constants. Since AADLEN is usually 526 // small, usually only 128-bit vectors will be used. So as an 527 // optimization, don't broadcast these constants to both 128-bit lanes 528 // quite yet. 529 vmovdqu .Lbswap_mask(%rip), BSWAP_MASK_XMM 530 vmovdqu .Lgfpoly(%rip), GFPOLY_XMM 531 532 // Load the GHASH accumulator. 533 vmovdqu (GHASH_ACC_PTR), GHASH_ACC_XMM 534 535 // Check for the common case of AADLEN <= 16, as well as AADLEN == 0. 536 test AADLEN, AADLEN 537 jz .Laad_done 538 cmp $16, AADLEN 539 jle .Laad_lastblock 540 541 // AADLEN > 16, so we'll operate on full vectors. Broadcast bswap_mask 542 // and gfpoly to both 128-bit lanes. 543 vinserti128 $1, BSWAP_MASK_XMM, BSWAP_MASK, BSWAP_MASK 544 vinserti128 $1, GFPOLY_XMM, GFPOLY, GFPOLY 545 546 // If AADLEN >= 128, update GHASH with 128 bytes of AAD at a time. 547 add $-128, AADLEN // 128 is 4 bytes, -128 is 1 byte 548 jl .Laad_loop_4x_done 549 vmovdqu OFFSETOF_H_POWERS_XORED(KEY), H_POW2_XORED 550 vmovdqu OFFSETOF_H_POWERS_XORED+32(KEY), H_POW1_XORED 551.Laad_loop_4x: 552 _ghash_4x AAD 553 sub $-128, AAD 554 add $-128, AADLEN 555 jge .Laad_loop_4x 556.Laad_loop_4x_done: 557 558 // If AADLEN >= 32, update GHASH with 32 bytes of AAD at a time. 559 add $96, AADLEN 560 jl .Laad_loop_1x_done 561.Laad_loop_1x: 562 vmovdqu (AAD), TMP0 563 vpshufb BSWAP_MASK, TMP0, TMP0 564 vpxor TMP0, GHASH_ACC, GHASH_ACC 565 vmovdqu OFFSETOFEND_H_POWERS-32(KEY), TMP0 566 _ghash_mul TMP0, GHASH_ACC, GHASH_ACC, GFPOLY, TMP1, TMP2, LO 567 vextracti128 $1, GHASH_ACC, TMP0_XMM 568 vpxor TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM 569 add $32, AAD 570 sub $32, AADLEN 571 jge .Laad_loop_1x 572.Laad_loop_1x_done: 573 add $32, AADLEN 574 // Now 0 <= AADLEN < 32. 575 576 jz .Laad_done 577 cmp $16, AADLEN 578 jle .Laad_lastblock 579 580 // Update GHASH with the remaining 17 <= AADLEN <= 31 bytes of AAD. 581 mov AADLEN, AADLEN // Zero-extend AADLEN to AADLEN64. 582 vmovdqu (AAD), TMP0_XMM 583 vmovdqu -16(AAD, AADLEN64), TMP1_XMM 584 vpshufb BSWAP_MASK_XMM, TMP0_XMM, TMP0_XMM 585 vpxor TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM 586 lea .Lrshift_and_bswap_table(%rip), %rax 587 vpshufb -16(%rax, AADLEN64), TMP1_XMM, TMP1_XMM 588 vinserti128 $1, TMP1_XMM, GHASH_ACC, GHASH_ACC 589 vmovdqu OFFSETOFEND_H_POWERS-32(KEY), TMP0 590 _ghash_mul TMP0, GHASH_ACC, GHASH_ACC, GFPOLY, TMP1, TMP2, LO 591 vextracti128 $1, GHASH_ACC, TMP0_XMM 592 vpxor TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM 593 jmp .Laad_done 594 595.Laad_lastblock: 596 // Update GHASH with the remaining 1 <= AADLEN <= 16 bytes of AAD. 597 _load_partial_block AAD, TMP0_XMM, %r8, %r8d 598 vpshufb BSWAP_MASK_XMM, TMP0_XMM, TMP0_XMM 599 vpxor TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM 600 vmovdqu OFFSETOFEND_H_POWERS-16(KEY), TMP0_XMM 601 _ghash_mul TMP0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM, GFPOLY_XMM, \ 602 TMP1_XMM, TMP2_XMM, LO_XMM 603 604.Laad_done: 605 // Store the updated GHASH accumulator back to memory. 606 vmovdqu GHASH_ACC_XMM, (GHASH_ACC_PTR) 607 608 vzeroupper 609 RET 610SYM_FUNC_END(aes_gcm_aad_update_vaes_avx2) 611 612// Do one non-last round of AES encryption on the blocks in the given AESDATA 613// vectors using the round key that has been broadcast to all 128-bit lanes of 614// \round_key. 615.macro _vaesenc round_key, vecs:vararg 616.irp i, \vecs 617 vaesenc \round_key, AESDATA\i, AESDATA\i 618.endr 619.endm 620 621// Generate counter blocks in the given AESDATA vectors, then do the zero-th AES 622// round on them. Clobbers TMP0. 623.macro _ctr_begin vecs:vararg 624 vbroadcasti128 .Linc_2blocks(%rip), TMP0 625.irp i, \vecs 626 vpshufb BSWAP_MASK, LE_CTR, AESDATA\i 627 vpaddd TMP0, LE_CTR, LE_CTR 628.endr 629.irp i, \vecs 630 vpxor RNDKEY0, AESDATA\i, AESDATA\i 631.endr 632.endm 633 634// Generate and encrypt counter blocks in the given AESDATA vectors, excluding 635// the last AES round. Clobbers %rax and TMP0. 636.macro _aesenc_loop vecs:vararg 637 _ctr_begin \vecs 638 lea 16(KEY), %rax 639.Laesenc_loop\@: 640 vbroadcasti128 (%rax), TMP0 641 _vaesenc TMP0, \vecs 642 add $16, %rax 643 cmp %rax, RNDKEYLAST_PTR 644 jne .Laesenc_loop\@ 645.endm 646 647// Finalize the keystream blocks in the given AESDATA vectors by doing the last 648// AES round, then XOR those keystream blocks with the corresponding data. 649// Reduce latency by doing the XOR before the vaesenclast, utilizing the 650// property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a). Clobbers TMP0. 651.macro _aesenclast_and_xor vecs:vararg 652.irp i, \vecs 653 vpxor \i*32(SRC), RNDKEYLAST, TMP0 654 vaesenclast TMP0, AESDATA\i, AESDATA\i 655.endr 656.irp i, \vecs 657 vmovdqu AESDATA\i, \i*32(DST) 658.endr 659.endm 660 661// void aes_gcm_{enc,dec}_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key, 662// const u32 le_ctr[4], u8 ghash_acc[16], 663// const u8 *src, u8 *dst, int datalen); 664// 665// This macro generates a GCM encryption or decryption update function with the 666// above prototype (with \enc selecting which one). The function computes the 667// next portion of the CTR keystream, XOR's it with |datalen| bytes from |src|, 668// and writes the resulting encrypted or decrypted data to |dst|. It also 669// updates the GHASH accumulator |ghash_acc| using the next |datalen| ciphertext 670// bytes. 671// 672// |datalen| must be a multiple of 16, except on the last call where it can be 673// any length. The caller must do any buffering needed to ensure this. Both 674// in-place and out-of-place en/decryption are supported. 675// 676// |le_ctr| must give the current counter in little-endian format. This 677// function loads the counter from |le_ctr| and increments the loaded counter as 678// needed, but it does *not* store the updated counter back to |le_ctr|. The 679// caller must update |le_ctr| if any more data segments follow. Internally, 680// only the low 32-bit word of the counter is incremented, following the GCM 681// standard. 682.macro _aes_gcm_update enc 683 684 // Function arguments 685 .set KEY, %rdi 686 .set LE_CTR_PTR, %rsi 687 .set LE_CTR_PTR32, %esi 688 .set GHASH_ACC_PTR, %rdx 689 .set SRC, %rcx // Assumed to be %rcx. 690 // See .Ltail_xor_and_ghash_1to16bytes 691 .set DST, %r8 692 .set DATALEN, %r9d 693 .set DATALEN64, %r9 // Zero-extend DATALEN before using! 694 695 // Additional local variables 696 697 // %rax is used as a temporary register. LE_CTR_PTR is also available 698 // as a temporary register after the counter is loaded. 699 700 // AES key length in bytes 701 .set AESKEYLEN, %r10d 702 .set AESKEYLEN64, %r10 703 704 // Pointer to the last AES round key for the chosen AES variant 705 .set RNDKEYLAST_PTR, %r11 706 707 // BSWAP_MASK is the shuffle mask for byte-reflecting 128-bit values 708 // using vpshufb, copied to all 128-bit lanes. 709 .set BSWAP_MASK, %ymm0 710 .set BSWAP_MASK_XMM, %xmm0 711 712 // GHASH_ACC is the accumulator variable for GHASH. When fully reduced, 713 // only the lowest 128-bit lane can be nonzero. When not fully reduced, 714 // more than one lane may be used, and they need to be XOR'd together. 715 .set GHASH_ACC, %ymm1 716 .set GHASH_ACC_XMM, %xmm1 717 718 // TMP[0-2] are temporary registers. 719 .set TMP0, %ymm2 720 .set TMP0_XMM, %xmm2 721 .set TMP1, %ymm3 722 .set TMP1_XMM, %xmm3 723 .set TMP2, %ymm4 724 .set TMP2_XMM, %xmm4 725 726 // LO and MI are used to accumulate unreduced GHASH products. 727 .set LO, %ymm5 728 .set LO_XMM, %xmm5 729 .set MI, %ymm6 730 .set MI_XMM, %xmm6 731 732 // H_POW[2-1]_XORED contain cached values from KEY->h_powers_xored. The 733 // descending numbering reflects the order of the key powers. 734 .set H_POW2_XORED, %ymm7 735 .set H_POW2_XORED_XMM, %xmm7 736 .set H_POW1_XORED, %ymm8 737 738 // RNDKEY0 caches the zero-th round key, and RNDKEYLAST the last one. 739 .set RNDKEY0, %ymm9 740 .set RNDKEYLAST, %ymm10 741 742 // LE_CTR contains the next set of little-endian counter blocks. 743 .set LE_CTR, %ymm11 744 745 // AESDATA[0-3] hold the counter blocks that are being encrypted by AES. 746 .set AESDATA0, %ymm12 747 .set AESDATA0_XMM, %xmm12 748 .set AESDATA1, %ymm13 749 .set AESDATA1_XMM, %xmm13 750 .set AESDATA2, %ymm14 751 .set AESDATA3, %ymm15 752 753.if \enc 754 .set GHASHDATA_PTR, DST 755.else 756 .set GHASHDATA_PTR, SRC 757.endif 758 759 vbroadcasti128 .Lbswap_mask(%rip), BSWAP_MASK 760 761 // Load the GHASH accumulator and the starting counter. 762 vmovdqu (GHASH_ACC_PTR), GHASH_ACC_XMM 763 vbroadcasti128 (LE_CTR_PTR), LE_CTR 764 765 // Load the AES key length in bytes. 766 movl OFFSETOF_AESKEYLEN(KEY), AESKEYLEN 767 768 // Make RNDKEYLAST_PTR point to the last AES round key. This is the 769 // round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 770 // respectively. Then load the zero-th and last round keys. 771 lea 6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR 772 vbroadcasti128 (KEY), RNDKEY0 773 vbroadcasti128 (RNDKEYLAST_PTR), RNDKEYLAST 774 775 // Finish initializing LE_CTR by adding 1 to the second block. 776 vpaddd .Lctr_pattern(%rip), LE_CTR, LE_CTR 777 778 // If there are at least 128 bytes of data, then continue into the loop 779 // that processes 128 bytes of data at a time. Otherwise skip it. 780 add $-128, DATALEN // 128 is 4 bytes, -128 is 1 byte 781 jl .Lcrypt_loop_4x_done\@ 782 783 vmovdqu OFFSETOF_H_POWERS_XORED(KEY), H_POW2_XORED 784 vmovdqu OFFSETOF_H_POWERS_XORED+32(KEY), H_POW1_XORED 785 786 // Main loop: en/decrypt and hash 4 vectors (128 bytes) at a time. 787 788.if \enc 789 // Encrypt the first 4 vectors of plaintext blocks. 790 _aesenc_loop 0,1,2,3 791 _aesenclast_and_xor 0,1,2,3 792 sub $-128, SRC // 128 is 4 bytes, -128 is 1 byte 793 add $-128, DATALEN 794 jl .Lghash_last_ciphertext_4x\@ 795.endif 796 797.align 16 798.Lcrypt_loop_4x\@: 799 800 // Start the AES encryption of the counter blocks. 801 _ctr_begin 0,1,2,3 802 cmp $24, AESKEYLEN 803 jl 128f // AES-128? 804 je 192f // AES-192? 805 // AES-256 806 vbroadcasti128 -13*16(RNDKEYLAST_PTR), TMP0 807 _vaesenc TMP0, 0,1,2,3 808 vbroadcasti128 -12*16(RNDKEYLAST_PTR), TMP0 809 _vaesenc TMP0, 0,1,2,3 810192: 811 vbroadcasti128 -11*16(RNDKEYLAST_PTR), TMP0 812 _vaesenc TMP0, 0,1,2,3 813 vbroadcasti128 -10*16(RNDKEYLAST_PTR), TMP0 814 _vaesenc TMP0, 0,1,2,3 815128: 816 817 // Finish the AES encryption of the counter blocks in AESDATA[0-3], 818 // interleaved with the GHASH update of the ciphertext blocks. 819.irp i, 9,8,7,6,5,4,3,2,1 820 _ghash_step_4x (9 - \i), GHASHDATA_PTR 821 vbroadcasti128 -\i*16(RNDKEYLAST_PTR), TMP0 822 _vaesenc TMP0, 0,1,2,3 823.endr 824 _ghash_step_4x 9, GHASHDATA_PTR 825.if \enc 826 sub $-128, DST // 128 is 4 bytes, -128 is 1 byte 827.endif 828 _aesenclast_and_xor 0,1,2,3 829 sub $-128, SRC 830.if !\enc 831 sub $-128, DST 832.endif 833 add $-128, DATALEN 834 jge .Lcrypt_loop_4x\@ 835 836.if \enc 837.Lghash_last_ciphertext_4x\@: 838 // Update GHASH with the last set of ciphertext blocks. 839 _ghash_4x DST 840 sub $-128, DST 841.endif 842 843.Lcrypt_loop_4x_done\@: 844 845 // Undo the extra subtraction by 128 and check whether data remains. 846 sub $-128, DATALEN // 128 is 4 bytes, -128 is 1 byte 847 jz .Ldone\@ 848 849 // The data length isn't a multiple of 128 bytes. Process the remaining 850 // data of length 1 <= DATALEN < 128. 851 // 852 // Since there are enough key powers available for all remaining data, 853 // there is no need to do a GHASH reduction after each iteration. 854 // Instead, multiply each remaining block by its own key power, and only 855 // do a GHASH reduction at the very end. 856 857 // Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N 858 // is the number of blocks that remain. 859 .set POWERS_PTR, LE_CTR_PTR // LE_CTR_PTR is free to be reused. 860 .set POWERS_PTR32, LE_CTR_PTR32 861 mov DATALEN, %eax 862 neg %rax 863 and $~15, %rax // -round_up(DATALEN, 16) 864 lea OFFSETOFEND_H_POWERS(KEY,%rax), POWERS_PTR 865 866 // Start collecting the unreduced GHASH intermediate value LO, MI, HI. 867 .set HI, H_POW2_XORED // H_POW2_XORED is free to be reused. 868 .set HI_XMM, H_POW2_XORED_XMM 869 vpxor LO_XMM, LO_XMM, LO_XMM 870 vpxor MI_XMM, MI_XMM, MI_XMM 871 vpxor HI_XMM, HI_XMM, HI_XMM 872 873 // 1 <= DATALEN < 128. Generate 2 or 4 more vectors of keystream blocks 874 // excluding the last AES round, depending on the remaining DATALEN. 875 cmp $64, DATALEN 876 jg .Ltail_gen_4_keystream_vecs\@ 877 _aesenc_loop 0,1 878 cmp $32, DATALEN 879 jge .Ltail_xor_and_ghash_full_vec_loop\@ 880 jmp .Ltail_xor_and_ghash_partial_vec\@ 881.Ltail_gen_4_keystream_vecs\@: 882 _aesenc_loop 0,1,2,3 883 884 // XOR the remaining data and accumulate the unreduced GHASH products 885 // for DATALEN >= 32, starting with one full 32-byte vector at a time. 886.Ltail_xor_and_ghash_full_vec_loop\@: 887.if \enc 888 _aesenclast_and_xor 0 889 vpshufb BSWAP_MASK, AESDATA0, AESDATA0 890.else 891 vmovdqu (SRC), TMP1 892 vpxor TMP1, RNDKEYLAST, TMP0 893 vaesenclast TMP0, AESDATA0, AESDATA0 894 vmovdqu AESDATA0, (DST) 895 vpshufb BSWAP_MASK, TMP1, AESDATA0 896.endif 897 // The ciphertext blocks (i.e. GHASH input data) are now in AESDATA0. 898 vpxor GHASH_ACC, AESDATA0, AESDATA0 899 vmovdqu (POWERS_PTR), TMP2 900 _ghash_mul_noreduce TMP2, AESDATA0, LO, MI, HI, TMP0 901 vmovdqa AESDATA1, AESDATA0 902 vmovdqa AESDATA2, AESDATA1 903 vmovdqa AESDATA3, AESDATA2 904 vpxor GHASH_ACC_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM 905 add $32, SRC 906 add $32, DST 907 add $32, POWERS_PTR 908 sub $32, DATALEN 909 cmp $32, DATALEN 910 jge .Ltail_xor_and_ghash_full_vec_loop\@ 911 test DATALEN, DATALEN 912 jz .Ltail_ghash_reduce\@ 913 914.Ltail_xor_and_ghash_partial_vec\@: 915 // XOR the remaining data and accumulate the unreduced GHASH products, 916 // for 1 <= DATALEN < 32. 917 vaesenclast RNDKEYLAST, AESDATA0, AESDATA0 918 cmp $16, DATALEN 919 jle .Ltail_xor_and_ghash_1to16bytes\@ 920 921 // Handle 17 <= DATALEN < 32. 922 923 // Load a vpshufb mask that will right-shift by '32 - DATALEN' bytes 924 // (shifting in zeroes), then reflect all 16 bytes. 925 lea .Lrshift_and_bswap_table(%rip), %rax 926 vmovdqu -16(%rax, DATALEN64), TMP2_XMM 927 928 // Move the second keystream block to its own register and left-align it 929 vextracti128 $1, AESDATA0, AESDATA1_XMM 930 vpxor .Lfifteens(%rip), TMP2_XMM, TMP0_XMM 931 vpshufb TMP0_XMM, AESDATA1_XMM, AESDATA1_XMM 932 933 // Using overlapping loads and stores, XOR the source data with the 934 // keystream and write the destination data. Then prepare the GHASH 935 // input data: the full ciphertext block and the zero-padded partial 936 // ciphertext block, both byte-reflected, in AESDATA0. 937.if \enc 938 vpxor -16(SRC, DATALEN64), AESDATA1_XMM, AESDATA1_XMM 939 vpxor (SRC), AESDATA0_XMM, AESDATA0_XMM 940 vmovdqu AESDATA1_XMM, -16(DST, DATALEN64) 941 vmovdqu AESDATA0_XMM, (DST) 942 vpshufb TMP2_XMM, AESDATA1_XMM, AESDATA1_XMM 943 vpshufb BSWAP_MASK_XMM, AESDATA0_XMM, AESDATA0_XMM 944.else 945 vmovdqu -16(SRC, DATALEN64), TMP1_XMM 946 vmovdqu (SRC), TMP0_XMM 947 vpxor TMP1_XMM, AESDATA1_XMM, AESDATA1_XMM 948 vpxor TMP0_XMM, AESDATA0_XMM, AESDATA0_XMM 949 vmovdqu AESDATA1_XMM, -16(DST, DATALEN64) 950 vmovdqu AESDATA0_XMM, (DST) 951 vpshufb TMP2_XMM, TMP1_XMM, AESDATA1_XMM 952 vpshufb BSWAP_MASK_XMM, TMP0_XMM, AESDATA0_XMM 953.endif 954 vpxor GHASH_ACC_XMM, AESDATA0_XMM, AESDATA0_XMM 955 vinserti128 $1, AESDATA1_XMM, AESDATA0, AESDATA0 956 vmovdqu (POWERS_PTR), TMP2 957 jmp .Ltail_ghash_last_vec\@ 958 959.Ltail_xor_and_ghash_1to16bytes\@: 960 // Handle 1 <= DATALEN <= 16. Carefully load and store the 961 // possibly-partial block, which we mustn't access out of bounds. 962 vmovdqu (POWERS_PTR), TMP2_XMM 963 mov SRC, KEY // Free up %rcx, assuming SRC == %rcx 964 mov DATALEN, %ecx 965 _load_partial_block KEY, TMP0_XMM, POWERS_PTR, POWERS_PTR32 966 vpxor TMP0_XMM, AESDATA0_XMM, AESDATA0_XMM 967 mov DATALEN, %ecx 968 _store_partial_block AESDATA0_XMM, DST, POWERS_PTR, POWERS_PTR32 969.if \enc 970 lea .Lselect_high_bytes_table(%rip), %rax 971 vpshufb BSWAP_MASK_XMM, AESDATA0_XMM, AESDATA0_XMM 972 vpand (%rax, DATALEN64), AESDATA0_XMM, AESDATA0_XMM 973.else 974 vpshufb BSWAP_MASK_XMM, TMP0_XMM, AESDATA0_XMM 975.endif 976 vpxor GHASH_ACC_XMM, AESDATA0_XMM, AESDATA0_XMM 977 978.Ltail_ghash_last_vec\@: 979 // Accumulate the unreduced GHASH products for the last 1-2 blocks. The 980 // GHASH input data is in AESDATA0. If only one block remains, then the 981 // second block in AESDATA0 is zero and does not affect the result. 982 _ghash_mul_noreduce TMP2, AESDATA0, LO, MI, HI, TMP0 983 984.Ltail_ghash_reduce\@: 985 // Finally, do the GHASH reduction. 986 vbroadcasti128 .Lgfpoly(%rip), TMP0 987 _ghash_reduce LO, MI, HI, TMP0, TMP1 988 vextracti128 $1, HI, GHASH_ACC_XMM 989 vpxor HI_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM 990 991.Ldone\@: 992 // Store the updated GHASH accumulator back to memory. 993 vmovdqu GHASH_ACC_XMM, (GHASH_ACC_PTR) 994 995 vzeroupper 996 RET 997.endm 998 999// void aes_gcm_enc_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key, 1000// const u32 le_ctr[4], u8 ghash_acc[16], 1001// u64 total_aadlen, u64 total_datalen); 1002// bool aes_gcm_dec_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key, 1003// const u32 le_ctr[4], const u8 ghash_acc[16], 1004// u64 total_aadlen, u64 total_datalen, 1005// const u8 tag[16], int taglen); 1006// 1007// This macro generates one of the above two functions (with \enc selecting 1008// which one). Both functions finish computing the GCM authentication tag by 1009// updating GHASH with the lengths block and encrypting the GHASH accumulator. 1010// |total_aadlen| and |total_datalen| must be the total length of the additional 1011// authenticated data and the en/decrypted data in bytes, respectively. 1012// 1013// The encryption function then stores the full-length (16-byte) computed 1014// authentication tag to |ghash_acc|. The decryption function instead loads the 1015// expected authentication tag (the one that was transmitted) from the 16-byte 1016// buffer |tag|, compares the first 4 <= |taglen| <= 16 bytes of it to the 1017// computed tag in constant time, and returns true if and only if they match. 1018.macro _aes_gcm_final enc 1019 1020 // Function arguments 1021 .set KEY, %rdi 1022 .set LE_CTR_PTR, %rsi 1023 .set GHASH_ACC_PTR, %rdx 1024 .set TOTAL_AADLEN, %rcx 1025 .set TOTAL_DATALEN, %r8 1026 .set TAG, %r9 1027 .set TAGLEN, %r10d // Originally at 8(%rsp) 1028 .set TAGLEN64, %r10 1029 1030 // Additional local variables. 1031 // %rax and %xmm0-%xmm3 are used as temporary registers. 1032 .set AESKEYLEN, %r11d 1033 .set AESKEYLEN64, %r11 1034 .set GFPOLY, %xmm4 1035 .set BSWAP_MASK, %xmm5 1036 .set LE_CTR, %xmm6 1037 .set GHASH_ACC, %xmm7 1038 .set H_POW1, %xmm8 1039 1040 // Load some constants. 1041 vmovdqa .Lgfpoly(%rip), GFPOLY 1042 vmovdqa .Lbswap_mask(%rip), BSWAP_MASK 1043 1044 // Load the AES key length in bytes. 1045 movl OFFSETOF_AESKEYLEN(KEY), AESKEYLEN 1046 1047 // Set up a counter block with 1 in the low 32-bit word. This is the 1048 // counter that produces the ciphertext needed to encrypt the auth tag. 1049 // GFPOLY has 1 in the low word, so grab the 1 from there using a blend. 1050 vpblendd $0xe, (LE_CTR_PTR), GFPOLY, LE_CTR 1051 1052 // Build the lengths block and XOR it with the GHASH accumulator. 1053 // Although the lengths block is defined as the AAD length followed by 1054 // the en/decrypted data length, both in big-endian byte order, a byte 1055 // reflection of the full block is needed because of the way we compute 1056 // GHASH (see _ghash_mul_step). By using little-endian values in the 1057 // opposite order, we avoid having to reflect any bytes here. 1058 vmovq TOTAL_DATALEN, %xmm0 1059 vpinsrq $1, TOTAL_AADLEN, %xmm0, %xmm0 1060 vpsllq $3, %xmm0, %xmm0 // Bytes to bits 1061 vpxor (GHASH_ACC_PTR), %xmm0, GHASH_ACC 1062 1063 // Load the first hash key power (H^1), which is stored last. 1064 vmovdqu OFFSETOFEND_H_POWERS-16(KEY), H_POW1 1065 1066 // Load TAGLEN if decrypting. 1067.if !\enc 1068 movl 8(%rsp), TAGLEN 1069.endif 1070 1071 // Make %rax point to the last AES round key for the chosen AES variant. 1072 lea 6*16(KEY,AESKEYLEN64,4), %rax 1073 1074 // Start the AES encryption of the counter block by swapping the counter 1075 // block to big-endian and XOR-ing it with the zero-th AES round key. 1076 vpshufb BSWAP_MASK, LE_CTR, %xmm0 1077 vpxor (KEY), %xmm0, %xmm0 1078 1079 // Complete the AES encryption and multiply GHASH_ACC by H^1. 1080 // Interleave the AES and GHASH instructions to improve performance. 1081 cmp $24, AESKEYLEN 1082 jl 128f // AES-128? 1083 je 192f // AES-192? 1084 // AES-256 1085 vaesenc -13*16(%rax), %xmm0, %xmm0 1086 vaesenc -12*16(%rax), %xmm0, %xmm0 1087192: 1088 vaesenc -11*16(%rax), %xmm0, %xmm0 1089 vaesenc -10*16(%rax), %xmm0, %xmm0 1090128: 1091.irp i, 0,1,2,3,4,5,6,7,8 1092 _ghash_mul_step \i, H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \ 1093 %xmm1, %xmm2, %xmm3 1094 vaesenc (\i-9)*16(%rax), %xmm0, %xmm0 1095.endr 1096 _ghash_mul_step 9, H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \ 1097 %xmm1, %xmm2, %xmm3 1098 1099 // Undo the byte reflection of the GHASH accumulator. 1100 vpshufb BSWAP_MASK, GHASH_ACC, GHASH_ACC 1101 1102 // Do the last AES round and XOR the resulting keystream block with the 1103 // GHASH accumulator to produce the full computed authentication tag. 1104 // 1105 // Reduce latency by taking advantage of the property vaesenclast(key, 1106 // a) ^ b == vaesenclast(key ^ b, a). I.e., XOR GHASH_ACC into the last 1107 // round key, instead of XOR'ing the final AES output with GHASH_ACC. 1108 // 1109 // enc_final then returns the computed auth tag, while dec_final 1110 // compares it with the transmitted one and returns a bool. To compare 1111 // the tags, dec_final XORs them together and uses vptest to check 1112 // whether the result is all-zeroes. This should be constant-time. 1113 // dec_final applies the vaesenclast optimization to this additional 1114 // value XOR'd too. 1115.if \enc 1116 vpxor (%rax), GHASH_ACC, %xmm1 1117 vaesenclast %xmm1, %xmm0, GHASH_ACC 1118 vmovdqu GHASH_ACC, (GHASH_ACC_PTR) 1119.else 1120 vpxor (TAG), GHASH_ACC, GHASH_ACC 1121 vpxor (%rax), GHASH_ACC, GHASH_ACC 1122 vaesenclast GHASH_ACC, %xmm0, %xmm0 1123 lea .Lselect_high_bytes_table(%rip), %rax 1124 vmovdqu (%rax, TAGLEN64), %xmm1 1125 vpshufb BSWAP_MASK, %xmm1, %xmm1 // select low bytes, not high 1126 xor %eax, %eax 1127 vptest %xmm1, %xmm0 1128 sete %al 1129.endif 1130 // No need for vzeroupper here, since only used xmm registers were used. 1131 RET 1132.endm 1133 1134SYM_FUNC_START(aes_gcm_enc_update_vaes_avx2) 1135 _aes_gcm_update 1 1136SYM_FUNC_END(aes_gcm_enc_update_vaes_avx2) 1137SYM_FUNC_START(aes_gcm_dec_update_vaes_avx2) 1138 _aes_gcm_update 0 1139SYM_FUNC_END(aes_gcm_dec_update_vaes_avx2) 1140 1141SYM_FUNC_START(aes_gcm_enc_final_vaes_avx2) 1142 _aes_gcm_final 1 1143SYM_FUNC_END(aes_gcm_enc_final_vaes_avx2) 1144SYM_FUNC_START(aes_gcm_dec_final_vaes_avx2) 1145 _aes_gcm_final 0 1146SYM_FUNC_END(aes_gcm_dec_final_vaes_avx2) 1147