1/* SPDX-License-Identifier: Apache-2.0 OR BSD-2-Clause */ 2// 3// AES-GCM implementation for x86_64 CPUs that support the following CPU 4// features: VAES && VPCLMULQDQ && AVX512BW && AVX512VL && BMI2 5// 6// Copyright 2024 Google LLC 7// 8// Author: Eric Biggers <ebiggers@google.com> 9// 10//------------------------------------------------------------------------------ 11// 12// This file is dual-licensed, meaning that you can use it under your choice of 13// either of the following two licenses: 14// 15// Licensed under the Apache License 2.0 (the "License"). You may obtain a copy 16// of the License at 17// 18// http://www.apache.org/licenses/LICENSE-2.0 19// 20// Unless required by applicable law or agreed to in writing, software 21// distributed under the License is distributed on an "AS IS" BASIS, 22// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23// See the License for the specific language governing permissions and 24// limitations under the License. 25// 26// or 27// 28// Redistribution and use in source and binary forms, with or without 29// modification, are permitted provided that the following conditions are met: 30// 31// 1. Redistributions of source code must retain the above copyright notice, 32// this list of conditions and the following disclaimer. 33// 34// 2. Redistributions in binary form must reproduce the above copyright 35// notice, this list of conditions and the following disclaimer in the 36// documentation and/or other materials provided with the distribution. 37// 38// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 39// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 40// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 41// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 42// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 43// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 44// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 45// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 46// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 47// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 48// POSSIBILITY OF SUCH DAMAGE. 49 50#include <linux/linkage.h> 51 52.section .rodata 53.p2align 6 54 55 // A shuffle mask that reflects the bytes of 16-byte blocks 56.Lbswap_mask: 57 .octa 0x000102030405060708090a0b0c0d0e0f 58 59 // This is the GHASH reducing polynomial without its constant term, i.e. 60 // x^128 + x^7 + x^2 + x, represented using the backwards mapping 61 // between bits and polynomial coefficients. 62 // 63 // Alternatively, it can be interpreted as the naturally-ordered 64 // representation of the polynomial x^127 + x^126 + x^121 + 1, i.e. the 65 // "reversed" GHASH reducing polynomial without its x^128 term. 66.Lgfpoly: 67 .octa 0xc2000000000000000000000000000001 68 69 // Same as above, but with the (1 << 64) bit set. 70.Lgfpoly_and_internal_carrybit: 71 .octa 0xc2000000000000010000000000000001 72 73 // Values needed to prepare the initial vector of counter blocks. 74.Lctr_pattern: 75 .octa 0 76 .octa 1 77 .octa 2 78 .octa 3 79 80 // The number of AES blocks per vector, as a 128-bit value. 81.Linc_4blocks: 82 .octa 4 83 84// Number of powers of the hash key stored in the key struct. The powers are 85// stored from highest (H^NUM_H_POWERS) to lowest (H^1). 86#define NUM_H_POWERS 16 87 88// Offset to AES key length (in bytes) in the key struct 89#define OFFSETOF_AESKEYLEN 480 90 91// Offset to start of hash key powers array in the key struct 92#define OFFSETOF_H_POWERS 512 93 94// Offset to end of hash key powers array in the key struct. 95// 96// This is immediately followed by three zeroized padding blocks, which are 97// included so that partial vectors can be handled more easily. E.g. if two 98// blocks remain, we load the 4 values [H^2, H^1, 0, 0]. The most padding 99// blocks needed is 3, which occurs if [H^1, 0, 0, 0] is loaded. 100#define OFFSETOFEND_H_POWERS (OFFSETOF_H_POWERS + (NUM_H_POWERS * 16)) 101 102.text 103 104// The _ghash_mul_step macro does one step of GHASH multiplication of the 105// 128-bit lanes of \a by the corresponding 128-bit lanes of \b and storing the 106// reduced products in \dst. \t0, \t1, and \t2 are temporary registers of the 107// same size as \a and \b. To complete all steps, this must invoked with \i=0 108// through \i=9. The division into steps allows users of this macro to 109// optionally interleave the computation with other instructions. Users of this 110// macro must preserve the parameter registers across steps. 111// 112// The multiplications are done in GHASH's representation of the finite field 113// GF(2^128). Elements of GF(2^128) are represented as binary polynomials 114// (i.e. polynomials whose coefficients are bits) modulo a reducing polynomial 115// G. The GCM specification uses G = x^128 + x^7 + x^2 + x + 1. Addition is 116// just XOR, while multiplication is more complex and has two parts: (a) do 117// carryless multiplication of two 128-bit input polynomials to get a 256-bit 118// intermediate product polynomial, and (b) reduce the intermediate product to 119// 128 bits by adding multiples of G that cancel out terms in it. (Adding 120// multiples of G doesn't change which field element the polynomial represents.) 121// 122// Unfortunately, the GCM specification maps bits to/from polynomial 123// coefficients backwards from the natural order. In each byte it specifies the 124// highest bit to be the lowest order polynomial coefficient, *not* the highest! 125// This makes it nontrivial to work with the GHASH polynomials. We could 126// reflect the bits, but x86 doesn't have an instruction that does that. 127// 128// Instead, we operate on the values without bit-reflecting them. This *mostly* 129// just works, since XOR and carryless multiplication are symmetric with respect 130// to bit order, but it has some consequences. First, due to GHASH's byte 131// order, by skipping bit reflection, *byte* reflection becomes necessary to 132// give the polynomial terms a consistent order. E.g., considering an N-bit 133// value interpreted using the G = x^128 + x^7 + x^2 + x + 1 convention, bits 0 134// through N-1 of the byte-reflected value represent the coefficients of x^(N-1) 135// through x^0, whereas bits 0 through N-1 of the non-byte-reflected value 136// represent x^7...x^0, x^15...x^8, ..., x^(N-1)...x^(N-8) which can't be worked 137// with. Fortunately, x86's vpshufb instruction can do byte reflection. 138// 139// Second, forgoing the bit reflection causes an extra multiple of x (still 140// using the G = x^128 + x^7 + x^2 + x + 1 convention) to be introduced by each 141// multiplication. This is because an M-bit by N-bit carryless multiplication 142// really produces a (M+N-1)-bit product, but in practice it's zero-extended to 143// M+N bits. In the G = x^128 + x^7 + x^2 + x + 1 convention, which maps bits 144// to polynomial coefficients backwards, this zero-extension actually changes 145// the product by introducing an extra factor of x. Therefore, users of this 146// macro must ensure that one of the inputs has an extra factor of x^-1, i.e. 147// the multiplicative inverse of x, to cancel out the extra x. 148// 149// Third, the backwards coefficients convention is just confusing to work with, 150// since it makes "low" and "high" in the polynomial math mean the opposite of 151// their normal meaning in computer programming. This can be solved by using an 152// alternative interpretation: the polynomial coefficients are understood to be 153// in the natural order, and the multiplication is actually \a * \b * x^-128 mod 154// x^128 + x^127 + x^126 + x^121 + 1. This doesn't change the inputs, outputs, 155// or the implementation at all; it just changes the mathematical interpretation 156// of what each instruction is doing. Starting from here, we'll use this 157// alternative interpretation, as it's easier to understand the code that way. 158// 159// Moving onto the implementation, the vpclmulqdq instruction does 64 x 64 => 160// 128-bit carryless multiplication, so we break the 128 x 128 multiplication 161// into parts as follows (the _L and _H suffixes denote low and high 64 bits): 162// 163// LO = a_L * b_L 164// MI = (a_L * b_H) + (a_H * b_L) 165// HI = a_H * b_H 166// 167// The 256-bit product is x^128*HI + x^64*MI + LO. LO, MI, and HI are 128-bit. 168// Note that MI "overlaps" with LO and HI. We don't consolidate MI into LO and 169// HI right away, since the way the reduction works makes that unnecessary. 170// 171// For the reduction, we cancel out the low 128 bits by adding multiples of G = 172// x^128 + x^127 + x^126 + x^121 + 1. This is done by two iterations, each of 173// which cancels out the next lowest 64 bits. Consider a value x^64*A + B, 174// where A and B are 128-bit. Adding B_L*G to that value gives: 175// 176// x^64*A + B + B_L*G 177// = x^64*A + x^64*B_H + B_L + B_L*(x^128 + x^127 + x^126 + x^121 + 1) 178// = x^64*A + x^64*B_H + B_L + x^128*B_L + x^64*B_L*(x^63 + x^62 + x^57) + B_L 179// = x^64*A + x^64*B_H + x^128*B_L + x^64*B_L*(x^63 + x^62 + x^57) + B_L + B_L 180// = x^64*(A + B_H + x^64*B_L + B_L*(x^63 + x^62 + x^57)) 181// 182// So: if we sum A, B with its halves swapped, and the low half of B times x^63 183// + x^62 + x^57, we get a 128-bit value C where x^64*C is congruent to the 184// original value x^64*A + B. I.e., the low 64 bits got canceled out. 185// 186// We just need to apply this twice: first to fold LO into MI, and second to 187// fold the updated MI into HI. 188// 189// The needed three-argument XORs are done using the vpternlogd instruction with 190// immediate 0x96, since this is faster than two vpxord instructions. 191// 192// A potential optimization, assuming that b is fixed per-key (if a is fixed 193// per-key it would work the other way around), is to use one iteration of the 194// reduction described above to precompute a value c such that x^64*c = b mod G, 195// and then multiply a_L by c (and implicitly by x^64) instead of by b: 196// 197// MI = (a_L * c_L) + (a_H * b_L) 198// HI = (a_L * c_H) + (a_H * b_H) 199// 200// This would eliminate the LO part of the intermediate product, which would 201// eliminate the need to fold LO into MI. This would save two instructions, 202// including a vpclmulqdq. However, we currently don't use this optimization 203// because it would require twice as many per-key precomputed values. 204// 205// Using Karatsuba multiplication instead of "schoolbook" multiplication 206// similarly would save a vpclmulqdq but does not seem to be worth it. 207.macro _ghash_mul_step i, a, b, dst, gfpoly, t0, t1, t2 208.if \i == 0 209 vpclmulqdq $0x00, \a, \b, \t0 // LO = a_L * b_L 210 vpclmulqdq $0x01, \a, \b, \t1 // MI_0 = a_L * b_H 211.elseif \i == 1 212 vpclmulqdq $0x10, \a, \b, \t2 // MI_1 = a_H * b_L 213.elseif \i == 2 214 vpxord \t2, \t1, \t1 // MI = MI_0 + MI_1 215.elseif \i == 3 216 vpclmulqdq $0x01, \t0, \gfpoly, \t2 // LO_L*(x^63 + x^62 + x^57) 217.elseif \i == 4 218 vpshufd $0x4e, \t0, \t0 // Swap halves of LO 219.elseif \i == 5 220 vpternlogd $0x96, \t2, \t0, \t1 // Fold LO into MI 221.elseif \i == 6 222 vpclmulqdq $0x11, \a, \b, \dst // HI = a_H * b_H 223.elseif \i == 7 224 vpclmulqdq $0x01, \t1, \gfpoly, \t0 // MI_L*(x^63 + x^62 + x^57) 225.elseif \i == 8 226 vpshufd $0x4e, \t1, \t1 // Swap halves of MI 227.elseif \i == 9 228 vpternlogd $0x96, \t0, \t1, \dst // Fold MI into HI 229.endif 230.endm 231 232// GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and store 233// the reduced products in \dst. See _ghash_mul_step for full explanation. 234.macro _ghash_mul a, b, dst, gfpoly, t0, t1, t2 235.irp i, 0,1,2,3,4,5,6,7,8,9 236 _ghash_mul_step \i, \a, \b, \dst, \gfpoly, \t0, \t1, \t2 237.endr 238.endm 239 240// GHASH-multiply the 128-bit lanes of \a by the 128-bit lanes of \b and add the 241// *unreduced* products to \lo, \mi, and \hi. 242.macro _ghash_mul_noreduce a, b, lo, mi, hi, t0, t1, t2, t3 243 vpclmulqdq $0x00, \a, \b, \t0 // a_L * b_L 244 vpclmulqdq $0x01, \a, \b, \t1 // a_L * b_H 245 vpclmulqdq $0x10, \a, \b, \t2 // a_H * b_L 246 vpclmulqdq $0x11, \a, \b, \t3 // a_H * b_H 247 vpxord \t0, \lo, \lo 248 vpternlogd $0x96, \t2, \t1, \mi 249 vpxord \t3, \hi, \hi 250.endm 251 252// Reduce the unreduced products from \lo, \mi, and \hi and store the 128-bit 253// reduced products in \hi. See _ghash_mul_step for explanation of reduction. 254.macro _ghash_reduce lo, mi, hi, gfpoly, t0 255 vpclmulqdq $0x01, \lo, \gfpoly, \t0 256 vpshufd $0x4e, \lo, \lo 257 vpternlogd $0x96, \t0, \lo, \mi 258 vpclmulqdq $0x01, \mi, \gfpoly, \t0 259 vpshufd $0x4e, \mi, \mi 260 vpternlogd $0x96, \t0, \mi, \hi 261.endm 262 263// This is a specialized version of _ghash_mul that computes \a * \a, i.e. it 264// squares \a. It skips computing MI = (a_L * a_H) + (a_H * a_L) = 0. 265.macro _ghash_square a, dst, gfpoly, t0, t1 266 vpclmulqdq $0x00, \a, \a, \t0 // LO = a_L * a_L 267 vpclmulqdq $0x11, \a, \a, \dst // HI = a_H * a_H 268 vpclmulqdq $0x01, \t0, \gfpoly, \t1 // LO_L*(x^63 + x^62 + x^57) 269 vpshufd $0x4e, \t0, \t0 // Swap halves of LO 270 vpxord \t0, \t1, \t1 // Fold LO into MI 271 vpclmulqdq $0x01, \t1, \gfpoly, \t0 // MI_L*(x^63 + x^62 + x^57) 272 vpshufd $0x4e, \t1, \t1 // Swap halves of MI 273 vpternlogd $0x96, \t0, \t1, \dst // Fold MI into HI 274.endm 275 276// void aes_gcm_precompute_vaes_avx512(struct aes_gcm_key_vaes_avx512 *key); 277// 278// Given the expanded AES key |key->base.aes_key|, derive the GHASH subkey and 279// initialize |key->h_powers| and |key->padding|. 280SYM_FUNC_START(aes_gcm_precompute_vaes_avx512) 281 282 // Function arguments 283 .set KEY, %rdi 284 285 // Additional local variables. 286 // %zmm[0-2] and %rax are used as temporaries. 287 .set POWERS_PTR, %rsi 288 .set RNDKEYLAST_PTR, %rdx 289 .set H_CUR, %zmm3 290 .set H_CUR_YMM, %ymm3 291 .set H_CUR_XMM, %xmm3 292 .set H_INC, %zmm4 293 .set H_INC_YMM, %ymm4 294 .set H_INC_XMM, %xmm4 295 .set GFPOLY, %zmm5 296 .set GFPOLY_YMM, %ymm5 297 .set GFPOLY_XMM, %xmm5 298 299 // Get pointer to lowest set of key powers (located at end of array). 300 lea OFFSETOFEND_H_POWERS-64(KEY), POWERS_PTR 301 302 // Encrypt an all-zeroes block to get the raw hash subkey. 303 movl OFFSETOF_AESKEYLEN(KEY), %eax 304 lea 6*16(KEY,%rax,4), RNDKEYLAST_PTR 305 vmovdqu (KEY), %xmm0 // Zero-th round key XOR all-zeroes block 306 add $16, KEY 3071: 308 vaesenc (KEY), %xmm0, %xmm0 309 add $16, KEY 310 cmp KEY, RNDKEYLAST_PTR 311 jne 1b 312 vaesenclast (RNDKEYLAST_PTR), %xmm0, %xmm0 313 314 // Reflect the bytes of the raw hash subkey. 315 vpshufb .Lbswap_mask(%rip), %xmm0, H_CUR_XMM 316 317 // Zeroize the padding blocks. 318 vpxor %xmm0, %xmm0, %xmm0 319 vmovdqu %ymm0, 64(POWERS_PTR) 320 vmovdqu %xmm0, 64+2*16(POWERS_PTR) 321 322 // Finish preprocessing the first key power, H^1. Since this GHASH 323 // implementation operates directly on values with the backwards bit 324 // order specified by the GCM standard, it's necessary to preprocess the 325 // raw key as follows. First, reflect its bytes. Second, multiply it 326 // by x^-1 mod x^128 + x^7 + x^2 + x + 1 (if using the backwards 327 // interpretation of polynomial coefficients), which can also be 328 // interpreted as multiplication by x mod x^128 + x^127 + x^126 + x^121 329 // + 1 using the alternative, natural interpretation of polynomial 330 // coefficients. For details, see the comment above _ghash_mul_step. 331 // 332 // Either way, for the multiplication the concrete operation performed 333 // is a left shift of the 128-bit value by 1 bit, then an XOR with (0xc2 334 // << 120) | 1 if a 1 bit was carried out. However, there's no 128-bit 335 // wide shift instruction, so instead double each of the two 64-bit 336 // halves and incorporate the internal carry bit into the value XOR'd. 337 vpshufd $0xd3, H_CUR_XMM, %xmm0 338 vpsrad $31, %xmm0, %xmm0 339 vpaddq H_CUR_XMM, H_CUR_XMM, H_CUR_XMM 340 // H_CUR_XMM ^= xmm0 & gfpoly_and_internal_carrybit 341 vpternlogd $0x78, .Lgfpoly_and_internal_carrybit(%rip), %xmm0, H_CUR_XMM 342 343 // Load the gfpoly constant. 344 vbroadcasti32x4 .Lgfpoly(%rip), GFPOLY 345 346 // Square H^1 to get H^2. 347 // 348 // Note that as with H^1, all higher key powers also need an extra 349 // factor of x^-1 (or x using the natural interpretation). Nothing 350 // special needs to be done to make this happen, though: H^1 * H^1 would 351 // end up with two factors of x^-1, but the multiplication consumes one. 352 // So the product H^2 ends up with the desired one factor of x^-1. 353 _ghash_square H_CUR_XMM, H_INC_XMM, GFPOLY_XMM, %xmm0, %xmm1 354 355 // Create H_CUR_YMM = [H^2, H^1] and H_INC_YMM = [H^2, H^2]. 356 vinserti128 $1, H_CUR_XMM, H_INC_YMM, H_CUR_YMM 357 vinserti128 $1, H_INC_XMM, H_INC_YMM, H_INC_YMM 358 359 // Create H_CUR = [H^4, H^3, H^2, H^1] and H_INC = [H^4, H^4, H^4, H^4]. 360 _ghash_mul H_INC_YMM, H_CUR_YMM, H_INC_YMM, GFPOLY_YMM, \ 361 %ymm0, %ymm1, %ymm2 362 vinserti64x4 $1, H_CUR_YMM, H_INC, H_CUR 363 vshufi64x2 $0, H_INC, H_INC, H_INC 364 365 // Store the lowest set of key powers. 366 vmovdqu8 H_CUR, (POWERS_PTR) 367 368 // Compute and store the remaining key powers. 369 // Repeatedly multiply [H^(i+3), H^(i+2), H^(i+1), H^i] by 370 // [H^4, H^4, H^4, H^4] to get [H^(i+7), H^(i+6), H^(i+5), H^(i+4)]. 371 mov $3, %eax 372.Lprecompute_next: 373 sub $64, POWERS_PTR 374 _ghash_mul H_INC, H_CUR, H_CUR, GFPOLY, %zmm0, %zmm1, %zmm2 375 vmovdqu8 H_CUR, (POWERS_PTR) 376 dec %eax 377 jnz .Lprecompute_next 378 379 vzeroupper // This is needed after using ymm or zmm registers. 380 RET 381SYM_FUNC_END(aes_gcm_precompute_vaes_avx512) 382 383// XOR together the 128-bit lanes of \src (whose low lane is \src_xmm) and store 384// the result in \dst_xmm. This implicitly zeroizes the other lanes of dst. 385.macro _horizontal_xor src, src_xmm, dst_xmm, t0_xmm, t1_xmm, t2_xmm 386 vextracti32x4 $1, \src, \t0_xmm 387 vextracti32x4 $2, \src, \t1_xmm 388 vextracti32x4 $3, \src, \t2_xmm 389 vpxord \t0_xmm, \src_xmm, \dst_xmm 390 vpternlogd $0x96, \t1_xmm, \t2_xmm, \dst_xmm 391.endm 392 393// Do one step of the GHASH update of the data blocks given in the vector 394// registers GHASHDATA[0-3]. \i specifies the step to do, 0 through 9. The 395// division into steps allows users of this macro to optionally interleave the 396// computation with other instructions. This macro uses the vector register 397// GHASH_ACC as input/output; GHASHDATA[0-3] as inputs that are clobbered; 398// H_POW[4-1], GFPOLY, and BSWAP_MASK as inputs that aren't clobbered; and 399// GHASHTMP[0-2] as temporaries. This macro handles the byte-reflection of the 400// data blocks. The parameter registers must be preserved across steps. 401// 402// The GHASH update does: GHASH_ACC = H_POW4*(GHASHDATA0 + GHASH_ACC) + 403// H_POW3*GHASHDATA1 + H_POW2*GHASHDATA2 + H_POW1*GHASHDATA3, where the 404// operations are vectorized operations on 512-bit vectors of 128-bit blocks. 405// The vectorized terms correspond to the following non-vectorized terms: 406// 407// H_POW4*(GHASHDATA0 + GHASH_ACC) => H^16*(blk0 + GHASH_ACC_XMM), 408// H^15*(blk1 + 0), H^14*(blk2 + 0), and H^13*(blk3 + 0) 409// H_POW3*GHASHDATA1 => H^12*blk4, H^11*blk5, H^10*blk6, and H^9*blk7 410// H_POW2*GHASHDATA2 => H^8*blk8, H^7*blk9, H^6*blk10, and H^5*blk11 411// H_POW1*GHASHDATA3 => H^4*blk12, H^3*blk13, H^2*blk14, and H^1*blk15 412// 413// More concretely, this code does: 414// - Do vectorized "schoolbook" multiplications to compute the intermediate 415// 256-bit product of each block and its corresponding hash key power. 416// - Sum (XOR) the intermediate 256-bit products across vectors. 417// - Do a vectorized reduction of these 256-bit intermediate values to 418// 128-bits each. 419// - Sum (XOR) these values and store the 128-bit result in GHASH_ACC_XMM. 420// 421// See _ghash_mul_step for the full explanation of the operations performed for 422// each individual finite field multiplication and reduction. 423.macro _ghash_step_4x i 424.if \i == 0 425 vpshufb BSWAP_MASK, GHASHDATA0, GHASHDATA0 426 vpxord GHASH_ACC, GHASHDATA0, GHASHDATA0 427 vpshufb BSWAP_MASK, GHASHDATA1, GHASHDATA1 428 vpshufb BSWAP_MASK, GHASHDATA2, GHASHDATA2 429.elseif \i == 1 430 vpshufb BSWAP_MASK, GHASHDATA3, GHASHDATA3 431 vpclmulqdq $0x00, H_POW4, GHASHDATA0, GHASH_ACC // LO_0 432 vpclmulqdq $0x00, H_POW3, GHASHDATA1, GHASHTMP0 // LO_1 433 vpclmulqdq $0x00, H_POW2, GHASHDATA2, GHASHTMP1 // LO_2 434.elseif \i == 2 435 vpxord GHASHTMP0, GHASH_ACC, GHASH_ACC // sum(LO_{1,0}) 436 vpclmulqdq $0x00, H_POW1, GHASHDATA3, GHASHTMP2 // LO_3 437 vpternlogd $0x96, GHASHTMP2, GHASHTMP1, GHASH_ACC // LO = sum(LO_{3,2,1,0}) 438 vpclmulqdq $0x01, H_POW4, GHASHDATA0, GHASHTMP0 // MI_0 439.elseif \i == 3 440 vpclmulqdq $0x01, H_POW3, GHASHDATA1, GHASHTMP1 // MI_1 441 vpclmulqdq $0x01, H_POW2, GHASHDATA2, GHASHTMP2 // MI_2 442 vpternlogd $0x96, GHASHTMP2, GHASHTMP1, GHASHTMP0 // sum(MI_{2,1,0}) 443 vpclmulqdq $0x01, H_POW1, GHASHDATA3, GHASHTMP1 // MI_3 444.elseif \i == 4 445 vpclmulqdq $0x10, H_POW4, GHASHDATA0, GHASHTMP2 // MI_4 446 vpternlogd $0x96, GHASHTMP2, GHASHTMP1, GHASHTMP0 // sum(MI_{4,3,2,1,0}) 447 vpclmulqdq $0x10, H_POW3, GHASHDATA1, GHASHTMP1 // MI_5 448 vpclmulqdq $0x10, H_POW2, GHASHDATA2, GHASHTMP2 // MI_6 449.elseif \i == 5 450 vpternlogd $0x96, GHASHTMP2, GHASHTMP1, GHASHTMP0 // sum(MI_{6,5,4,3,2,1,0}) 451 vpclmulqdq $0x01, GHASH_ACC, GFPOLY, GHASHTMP2 // LO_L*(x^63 + x^62 + x^57) 452 vpclmulqdq $0x10, H_POW1, GHASHDATA3, GHASHTMP1 // MI_7 453 vpxord GHASHTMP1, GHASHTMP0, GHASHTMP0 // MI = sum(MI_{7,6,5,4,3,2,1,0}) 454.elseif \i == 6 455 vpshufd $0x4e, GHASH_ACC, GHASH_ACC // Swap halves of LO 456 vpclmulqdq $0x11, H_POW4, GHASHDATA0, GHASHDATA0 // HI_0 457 vpclmulqdq $0x11, H_POW3, GHASHDATA1, GHASHDATA1 // HI_1 458 vpclmulqdq $0x11, H_POW2, GHASHDATA2, GHASHDATA2 // HI_2 459.elseif \i == 7 460 vpternlogd $0x96, GHASHTMP2, GHASH_ACC, GHASHTMP0 // Fold LO into MI 461 vpclmulqdq $0x11, H_POW1, GHASHDATA3, GHASHDATA3 // HI_3 462 vpternlogd $0x96, GHASHDATA2, GHASHDATA1, GHASHDATA0 // sum(HI_{2,1,0}) 463 vpclmulqdq $0x01, GHASHTMP0, GFPOLY, GHASHTMP1 // MI_L*(x^63 + x^62 + x^57) 464.elseif \i == 8 465 vpxord GHASHDATA3, GHASHDATA0, GHASH_ACC // HI = sum(HI_{3,2,1,0}) 466 vpshufd $0x4e, GHASHTMP0, GHASHTMP0 // Swap halves of MI 467 vpternlogd $0x96, GHASHTMP1, GHASHTMP0, GHASH_ACC // Fold MI into HI 468.elseif \i == 9 469 _horizontal_xor GHASH_ACC, GHASH_ACC_XMM, GHASH_ACC_XMM, \ 470 GHASHDATA0_XMM, GHASHDATA1_XMM, GHASHDATA2_XMM 471.endif 472.endm 473 474// Update GHASH with four vectors of data blocks. See _ghash_step_4x for full 475// explanation. 476.macro _ghash_4x 477.irp i, 0,1,2,3,4,5,6,7,8,9 478 _ghash_step_4x \i 479.endr 480.endm 481 482// void aes_gcm_aad_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key, 483// u8 ghash_acc[16], 484// const u8 *aad, int aadlen); 485// 486// This function processes the AAD (Additional Authenticated Data) in GCM. 487// Using the key |key|, it updates the GHASH accumulator |ghash_acc| with the 488// data given by |aad| and |aadlen|. On the first call, |ghash_acc| must be all 489// zeroes. |aadlen| must be a multiple of 16, except on the last call where it 490// can be any length. The caller must do any buffering needed to ensure this. 491// 492// This handles large amounts of AAD efficiently, while also keeping overhead 493// low for small amounts which is the common case. TLS and IPsec use less than 494// one block of AAD, but (uncommonly) other use cases may use much more. 495SYM_FUNC_START(aes_gcm_aad_update_vaes_avx512) 496 497 // Function arguments 498 .set KEY, %rdi 499 .set GHASH_ACC_PTR, %rsi 500 .set AAD, %rdx 501 .set AADLEN, %ecx 502 .set AADLEN64, %rcx // Zero-extend AADLEN before using! 503 504 // Additional local variables. 505 // %rax and %k1 are used as temporary registers. 506 .set GHASHDATA0, %zmm0 507 .set GHASHDATA0_XMM, %xmm0 508 .set GHASHDATA1, %zmm1 509 .set GHASHDATA1_XMM, %xmm1 510 .set GHASHDATA2, %zmm2 511 .set GHASHDATA2_XMM, %xmm2 512 .set GHASHDATA3, %zmm3 513 .set BSWAP_MASK, %zmm4 514 .set BSWAP_MASK_XMM, %xmm4 515 .set GHASH_ACC, %zmm5 516 .set GHASH_ACC_XMM, %xmm5 517 .set H_POW4, %zmm6 518 .set H_POW3, %zmm7 519 .set H_POW2, %zmm8 520 .set H_POW1, %zmm9 521 .set H_POW1_XMM, %xmm9 522 .set GFPOLY, %zmm10 523 .set GFPOLY_XMM, %xmm10 524 .set GHASHTMP0, %zmm11 525 .set GHASHTMP1, %zmm12 526 .set GHASHTMP2, %zmm13 527 528 // Load the GHASH accumulator. 529 vmovdqu (GHASH_ACC_PTR), GHASH_ACC_XMM 530 531 // Check for the common case of AADLEN <= 16, as well as AADLEN == 0. 532 cmp $16, AADLEN 533 jg .Laad_more_than_16bytes 534 test AADLEN, AADLEN 535 jz .Laad_done 536 537 // Fast path: update GHASH with 1 <= AADLEN <= 16 bytes of AAD. 538 vmovdqu .Lbswap_mask(%rip), BSWAP_MASK_XMM 539 vmovdqu .Lgfpoly(%rip), GFPOLY_XMM 540 mov $-1, %eax 541 bzhi AADLEN, %eax, %eax 542 kmovd %eax, %k1 543 vmovdqu8 (AAD), GHASHDATA0_XMM{%k1}{z} 544 vmovdqu OFFSETOFEND_H_POWERS-16(KEY), H_POW1_XMM 545 vpshufb BSWAP_MASK_XMM, GHASHDATA0_XMM, GHASHDATA0_XMM 546 vpxor GHASHDATA0_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM 547 _ghash_mul H_POW1_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM, GFPOLY_XMM, \ 548 GHASHDATA0_XMM, GHASHDATA1_XMM, GHASHDATA2_XMM 549 jmp .Laad_done 550 551.Laad_more_than_16bytes: 552 vbroadcasti32x4 .Lbswap_mask(%rip), BSWAP_MASK 553 vbroadcasti32x4 .Lgfpoly(%rip), GFPOLY 554 555 // If AADLEN >= 256, update GHASH with 256 bytes of AAD at a time. 556 sub $256, AADLEN 557 jl .Laad_loop_4x_done 558 vmovdqu8 OFFSETOFEND_H_POWERS-4*64(KEY), H_POW4 559 vmovdqu8 OFFSETOFEND_H_POWERS-3*64(KEY), H_POW3 560 vmovdqu8 OFFSETOFEND_H_POWERS-2*64(KEY), H_POW2 561 vmovdqu8 OFFSETOFEND_H_POWERS-1*64(KEY), H_POW1 562.Laad_loop_4x: 563 vmovdqu8 0*64(AAD), GHASHDATA0 564 vmovdqu8 1*64(AAD), GHASHDATA1 565 vmovdqu8 2*64(AAD), GHASHDATA2 566 vmovdqu8 3*64(AAD), GHASHDATA3 567 _ghash_4x 568 add $256, AAD 569 sub $256, AADLEN 570 jge .Laad_loop_4x 571.Laad_loop_4x_done: 572 573 // If AADLEN >= 64, update GHASH with 64 bytes of AAD at a time. 574 add $192, AADLEN 575 jl .Laad_loop_1x_done 576 vmovdqu8 OFFSETOFEND_H_POWERS-1*64(KEY), H_POW1 577.Laad_loop_1x: 578 vmovdqu8 (AAD), GHASHDATA0 579 vpshufb BSWAP_MASK, GHASHDATA0, GHASHDATA0 580 vpxord GHASHDATA0, GHASH_ACC, GHASH_ACC 581 _ghash_mul H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \ 582 GHASHDATA0, GHASHDATA1, GHASHDATA2 583 _horizontal_xor GHASH_ACC, GHASH_ACC_XMM, GHASH_ACC_XMM, \ 584 GHASHDATA0_XMM, GHASHDATA1_XMM, GHASHDATA2_XMM 585 add $64, AAD 586 sub $64, AADLEN 587 jge .Laad_loop_1x 588.Laad_loop_1x_done: 589 590 // Update GHASH with the remaining 0 <= AADLEN < 64 bytes of AAD. 591 add $64, AADLEN 592 jz .Laad_done 593 mov $-1, %rax 594 bzhi AADLEN64, %rax, %rax 595 kmovq %rax, %k1 596 vmovdqu8 (AAD), GHASHDATA0{%k1}{z} 597 neg AADLEN64 598 and $~15, AADLEN64 // -round_up(AADLEN, 16) 599 vmovdqu8 OFFSETOFEND_H_POWERS(KEY,AADLEN64), H_POW1 600 vpshufb BSWAP_MASK, GHASHDATA0, GHASHDATA0 601 vpxord GHASHDATA0, GHASH_ACC, GHASH_ACC 602 _ghash_mul H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \ 603 GHASHDATA0, GHASHDATA1, GHASHDATA2 604 _horizontal_xor GHASH_ACC, GHASH_ACC_XMM, GHASH_ACC_XMM, \ 605 GHASHDATA0_XMM, GHASHDATA1_XMM, GHASHDATA2_XMM 606 607.Laad_done: 608 // Store the updated GHASH accumulator back to memory. 609 vmovdqu GHASH_ACC_XMM, (GHASH_ACC_PTR) 610 611 vzeroupper // This is needed after using ymm or zmm registers. 612 RET 613SYM_FUNC_END(aes_gcm_aad_update_vaes_avx512) 614 615// Do one non-last round of AES encryption on the blocks in %zmm[0-3] using the 616// round key that has been broadcast to all 128-bit lanes of \round_key. 617.macro _vaesenc_4x round_key 618 vaesenc \round_key, %zmm0, %zmm0 619 vaesenc \round_key, %zmm1, %zmm1 620 vaesenc \round_key, %zmm2, %zmm2 621 vaesenc \round_key, %zmm3, %zmm3 622.endm 623 624// Start the AES encryption of four vectors of counter blocks. 625.macro _ctr_begin_4x 626 627 // Increment LE_CTR four times to generate four vectors of little-endian 628 // counter blocks, swap each to big-endian, and store them in %zmm[0-3]. 629 vpshufb BSWAP_MASK, LE_CTR, %zmm0 630 vpaddd LE_CTR_INC, LE_CTR, LE_CTR 631 vpshufb BSWAP_MASK, LE_CTR, %zmm1 632 vpaddd LE_CTR_INC, LE_CTR, LE_CTR 633 vpshufb BSWAP_MASK, LE_CTR, %zmm2 634 vpaddd LE_CTR_INC, LE_CTR, LE_CTR 635 vpshufb BSWAP_MASK, LE_CTR, %zmm3 636 vpaddd LE_CTR_INC, LE_CTR, LE_CTR 637 638 // AES "round zero": XOR in the zero-th round key. 639 vpxord RNDKEY0, %zmm0, %zmm0 640 vpxord RNDKEY0, %zmm1, %zmm1 641 vpxord RNDKEY0, %zmm2, %zmm2 642 vpxord RNDKEY0, %zmm3, %zmm3 643.endm 644 645// Do the last AES round for four vectors of counter blocks %zmm[0-3], XOR 646// source data with the resulting keystream, and write the result to DST and 647// GHASHDATA[0-3]. (Implementation differs slightly, but has the same effect.) 648.macro _aesenclast_and_xor_4x 649 // XOR the source data with the last round key, saving the result in 650 // GHASHDATA[0-3]. This reduces latency by taking advantage of the 651 // property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a). 652 vpxord 0*64(SRC), RNDKEYLAST, GHASHDATA0 653 vpxord 1*64(SRC), RNDKEYLAST, GHASHDATA1 654 vpxord 2*64(SRC), RNDKEYLAST, GHASHDATA2 655 vpxord 3*64(SRC), RNDKEYLAST, GHASHDATA3 656 657 // Do the last AES round. This handles the XOR with the source data 658 // too, as per the optimization described above. 659 vaesenclast GHASHDATA0, %zmm0, GHASHDATA0 660 vaesenclast GHASHDATA1, %zmm1, GHASHDATA1 661 vaesenclast GHASHDATA2, %zmm2, GHASHDATA2 662 vaesenclast GHASHDATA3, %zmm3, GHASHDATA3 663 664 // Store the en/decrypted data to DST. 665 vmovdqu8 GHASHDATA0, 0*64(DST) 666 vmovdqu8 GHASHDATA1, 1*64(DST) 667 vmovdqu8 GHASHDATA2, 2*64(DST) 668 vmovdqu8 GHASHDATA3, 3*64(DST) 669.endm 670 671// void aes_gcm_{enc,dec}_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key, 672// const u32 le_ctr[4], u8 ghash_acc[16], 673// const u8 *src, u8 *dst, int datalen); 674// 675// This macro generates a GCM encryption or decryption update function with the 676// above prototype (with \enc selecting which one). The function computes the 677// next portion of the CTR keystream, XOR's it with |datalen| bytes from |src|, 678// and writes the resulting encrypted or decrypted data to |dst|. It also 679// updates the GHASH accumulator |ghash_acc| using the next |datalen| ciphertext 680// bytes. 681// 682// |datalen| must be a multiple of 16, except on the last call where it can be 683// any length. The caller must do any buffering needed to ensure this. Both 684// in-place and out-of-place en/decryption are supported. 685// 686// |le_ctr| must give the current counter in little-endian format. This 687// function loads the counter from |le_ctr| and increments the loaded counter as 688// needed, but it does *not* store the updated counter back to |le_ctr|. The 689// caller must update |le_ctr| if any more data segments follow. Internally, 690// only the low 32-bit word of the counter is incremented, following the GCM 691// standard. 692.macro _aes_gcm_update enc 693 694 // Function arguments 695 .set KEY, %rdi 696 .set LE_CTR_PTR, %rsi 697 .set GHASH_ACC_PTR, %rdx 698 .set SRC, %rcx 699 .set DST, %r8 700 .set DATALEN, %r9d 701 .set DATALEN64, %r9 // Zero-extend DATALEN before using! 702 703 // Additional local variables 704 705 // %rax and %k1 are used as temporary registers. LE_CTR_PTR is also 706 // available as a temporary register after the counter is loaded. 707 708 // AES key length in bytes 709 .set AESKEYLEN, %r10d 710 .set AESKEYLEN64, %r10 711 712 // Pointer to the last AES round key for the chosen AES variant 713 .set RNDKEYLAST_PTR, %r11 714 715 // In the main loop, %zmm[0-3] are used as AES input and output. 716 // Elsewhere they are used as temporary registers. 717 718 // GHASHDATA[0-3] hold the ciphertext blocks and GHASH input data. 719 .set GHASHDATA0, %zmm4 720 .set GHASHDATA0_XMM, %xmm4 721 .set GHASHDATA1, %zmm5 722 .set GHASHDATA1_XMM, %xmm5 723 .set GHASHDATA2, %zmm6 724 .set GHASHDATA2_XMM, %xmm6 725 .set GHASHDATA3, %zmm7 726 727 // BSWAP_MASK is the shuffle mask for byte-reflecting 128-bit values 728 // using vpshufb, copied to all 128-bit lanes. 729 .set BSWAP_MASK, %zmm8 730 731 // RNDKEY temporarily holds the next AES round key. 732 .set RNDKEY, %zmm9 733 734 // GHASH_ACC is the accumulator variable for GHASH. When fully reduced, 735 // only the lowest 128-bit lane can be nonzero. When not fully reduced, 736 // more than one lane may be used, and they need to be XOR'd together. 737 .set GHASH_ACC, %zmm10 738 .set GHASH_ACC_XMM, %xmm10 739 740 // LE_CTR_INC is the vector of 32-bit words that need to be added to a 741 // vector of little-endian counter blocks to advance it forwards. 742 .set LE_CTR_INC, %zmm11 743 744 // LE_CTR contains the next set of little-endian counter blocks. 745 .set LE_CTR, %zmm12 746 747 // RNDKEY0, RNDKEYLAST, and RNDKEY_M[9-1] contain cached AES round keys, 748 // copied to all 128-bit lanes. RNDKEY0 is the zero-th round key, 749 // RNDKEYLAST the last, and RNDKEY_M\i the one \i-th from the last. 750 .set RNDKEY0, %zmm13 751 .set RNDKEYLAST, %zmm14 752 .set RNDKEY_M9, %zmm15 753 .set RNDKEY_M8, %zmm16 754 .set RNDKEY_M7, %zmm17 755 .set RNDKEY_M6, %zmm18 756 .set RNDKEY_M5, %zmm19 757 .set RNDKEY_M4, %zmm20 758 .set RNDKEY_M3, %zmm21 759 .set RNDKEY_M2, %zmm22 760 .set RNDKEY_M1, %zmm23 761 762 // GHASHTMP[0-2] are temporary variables used by _ghash_step_4x. These 763 // cannot coincide with anything used for AES encryption, since for 764 // performance reasons GHASH and AES encryption are interleaved. 765 .set GHASHTMP0, %zmm24 766 .set GHASHTMP1, %zmm25 767 .set GHASHTMP2, %zmm26 768 769 // H_POW[4-1] contain the powers of the hash key H^16...H^1. The 770 // descending numbering reflects the order of the key powers. 771 .set H_POW4, %zmm27 772 .set H_POW3, %zmm28 773 .set H_POW2, %zmm29 774 .set H_POW1, %zmm30 775 776 // GFPOLY contains the .Lgfpoly constant, copied to all 128-bit lanes. 777 .set GFPOLY, %zmm31 778 779 // Load some constants. 780 vbroadcasti32x4 .Lbswap_mask(%rip), BSWAP_MASK 781 vbroadcasti32x4 .Lgfpoly(%rip), GFPOLY 782 783 // Load the GHASH accumulator and the starting counter. 784 vmovdqu (GHASH_ACC_PTR), GHASH_ACC_XMM 785 vbroadcasti32x4 (LE_CTR_PTR), LE_CTR 786 787 // Load the AES key length in bytes. 788 movl OFFSETOF_AESKEYLEN(KEY), AESKEYLEN 789 790 // Make RNDKEYLAST_PTR point to the last AES round key. This is the 791 // round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 792 // respectively. Then load the zero-th and last round keys. 793 lea 6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR 794 vbroadcasti32x4 (KEY), RNDKEY0 795 vbroadcasti32x4 (RNDKEYLAST_PTR), RNDKEYLAST 796 797 // Finish initializing LE_CTR by adding [0, 1, ...] to its low words. 798 vpaddd .Lctr_pattern(%rip), LE_CTR, LE_CTR 799 800 // Load 4 into all 128-bit lanes of LE_CTR_INC. 801 vbroadcasti32x4 .Linc_4blocks(%rip), LE_CTR_INC 802 803 // If there are at least 256 bytes of data, then continue into the loop 804 // that processes 256 bytes of data at a time. Otherwise skip it. 805 // 806 // Pre-subtracting 256 from DATALEN saves an instruction from the main 807 // loop and also ensures that at least one write always occurs to 808 // DATALEN, zero-extending it and allowing DATALEN64 to be used later. 809 sub $256, DATALEN 810 jl .Lcrypt_loop_4x_done\@ 811 812 // Load powers of the hash key. 813 vmovdqu8 OFFSETOFEND_H_POWERS-4*64(KEY), H_POW4 814 vmovdqu8 OFFSETOFEND_H_POWERS-3*64(KEY), H_POW3 815 vmovdqu8 OFFSETOFEND_H_POWERS-2*64(KEY), H_POW2 816 vmovdqu8 OFFSETOFEND_H_POWERS-1*64(KEY), H_POW1 817 818 // Main loop: en/decrypt and hash 4 vectors at a time. 819 // 820 // When possible, interleave the AES encryption of the counter blocks 821 // with the GHASH update of the ciphertext blocks. This improves 822 // performance on many CPUs because the execution ports used by the VAES 823 // instructions often differ from those used by vpclmulqdq and other 824 // instructions used in GHASH. For example, many Intel CPUs dispatch 825 // vaesenc to ports 0 and 1 and vpclmulqdq to port 5. 826 // 827 // The interleaving is easiest to do during decryption, since during 828 // decryption the ciphertext blocks are immediately available. For 829 // encryption, instead encrypt the first set of blocks, then hash those 830 // blocks while encrypting the next set of blocks, repeat that as 831 // needed, and finally hash the last set of blocks. 832 833.if \enc 834 // Encrypt the first 4 vectors of plaintext blocks. Leave the resulting 835 // ciphertext in GHASHDATA[0-3] for GHASH. 836 _ctr_begin_4x 837 lea 16(KEY), %rax 8381: 839 vbroadcasti32x4 (%rax), RNDKEY 840 _vaesenc_4x RNDKEY 841 add $16, %rax 842 cmp %rax, RNDKEYLAST_PTR 843 jne 1b 844 _aesenclast_and_xor_4x 845 add $256, SRC 846 add $256, DST 847 sub $256, DATALEN 848 jl .Lghash_last_ciphertext_4x\@ 849.endif 850 851 // Cache as many additional AES round keys as possible. 852.irp i, 9,8,7,6,5,4,3,2,1 853 vbroadcasti32x4 -\i*16(RNDKEYLAST_PTR), RNDKEY_M\i 854.endr 855 856.Lcrypt_loop_4x\@: 857 858 // If decrypting, load more ciphertext blocks into GHASHDATA[0-3]. If 859 // encrypting, GHASHDATA[0-3] already contain the previous ciphertext. 860.if !\enc 861 vmovdqu8 0*64(SRC), GHASHDATA0 862 vmovdqu8 1*64(SRC), GHASHDATA1 863 vmovdqu8 2*64(SRC), GHASHDATA2 864 vmovdqu8 3*64(SRC), GHASHDATA3 865.endif 866 867 // Start the AES encryption of the counter blocks. 868 _ctr_begin_4x 869 cmp $24, AESKEYLEN 870 jl 128f // AES-128? 871 je 192f // AES-192? 872 // AES-256 873 vbroadcasti32x4 -13*16(RNDKEYLAST_PTR), RNDKEY 874 _vaesenc_4x RNDKEY 875 vbroadcasti32x4 -12*16(RNDKEYLAST_PTR), RNDKEY 876 _vaesenc_4x RNDKEY 877192: 878 vbroadcasti32x4 -11*16(RNDKEYLAST_PTR), RNDKEY 879 _vaesenc_4x RNDKEY 880 vbroadcasti32x4 -10*16(RNDKEYLAST_PTR), RNDKEY 881 _vaesenc_4x RNDKEY 882128: 883 884 // Finish the AES encryption of the counter blocks in %zmm[0-3], 885 // interleaved with the GHASH update of the ciphertext blocks in 886 // GHASHDATA[0-3]. 887.irp i, 9,8,7,6,5,4,3,2,1 888 _ghash_step_4x (9 - \i) 889 _vaesenc_4x RNDKEY_M\i 890.endr 891 _ghash_step_4x 9 892 _aesenclast_and_xor_4x 893 add $256, SRC 894 add $256, DST 895 sub $256, DATALEN 896 jge .Lcrypt_loop_4x\@ 897 898.if \enc 899.Lghash_last_ciphertext_4x\@: 900 // Update GHASH with the last set of ciphertext blocks. 901 _ghash_4x 902.endif 903 904.Lcrypt_loop_4x_done\@: 905 906 // Undo the extra subtraction by 256 and check whether data remains. 907 add $256, DATALEN 908 jz .Ldone\@ 909 910 // The data length isn't a multiple of 256 bytes. Process the remaining 911 // data of length 1 <= DATALEN < 256, up to one 64-byte vector at a 912 // time. Going one vector at a time may seem inefficient compared to 913 // having separate code paths for each possible number of vectors 914 // remaining. However, using a loop keeps the code size down, and it 915 // performs surprising well; modern CPUs will start executing the next 916 // iteration before the previous one finishes and also predict the 917 // number of loop iterations. For a similar reason, we roll up the AES 918 // rounds. 919 // 920 // On the last iteration, the remaining length may be less than 64 921 // bytes. Handle this using masking. 922 // 923 // Since there are enough key powers available for all remaining data, 924 // there is no need to do a GHASH reduction after each iteration. 925 // Instead, multiply each remaining block by its own key power, and only 926 // do a GHASH reduction at the very end. 927 928 // Make POWERS_PTR point to the key powers [H^N, H^(N-1), ...] where N 929 // is the number of blocks that remain. 930 .set POWERS_PTR, LE_CTR_PTR // LE_CTR_PTR is free to be reused. 931 mov DATALEN, %eax 932 neg %rax 933 and $~15, %rax // -round_up(DATALEN, 16) 934 lea OFFSETOFEND_H_POWERS(KEY,%rax), POWERS_PTR 935 936 // Start collecting the unreduced GHASH intermediate value LO, MI, HI. 937 .set LO, GHASHDATA0 938 .set LO_XMM, GHASHDATA0_XMM 939 .set MI, GHASHDATA1 940 .set MI_XMM, GHASHDATA1_XMM 941 .set HI, GHASHDATA2 942 .set HI_XMM, GHASHDATA2_XMM 943 vpxor LO_XMM, LO_XMM, LO_XMM 944 vpxor MI_XMM, MI_XMM, MI_XMM 945 vpxor HI_XMM, HI_XMM, HI_XMM 946 947.Lcrypt_loop_1x\@: 948 949 // Select the appropriate mask for this iteration: all 1's if 950 // DATALEN >= 64, otherwise DATALEN 1's. Do this branchlessly using the 951 // bzhi instruction from BMI2. (This relies on DATALEN <= 255.) 952 mov $-1, %rax 953 bzhi DATALEN64, %rax, %rax 954 kmovq %rax, %k1 955 956 // Encrypt a vector of counter blocks. This does not need to be masked. 957 vpshufb BSWAP_MASK, LE_CTR, %zmm0 958 vpaddd LE_CTR_INC, LE_CTR, LE_CTR 959 vpxord RNDKEY0, %zmm0, %zmm0 960 lea 16(KEY), %rax 9611: 962 vbroadcasti32x4 (%rax), RNDKEY 963 vaesenc RNDKEY, %zmm0, %zmm0 964 add $16, %rax 965 cmp %rax, RNDKEYLAST_PTR 966 jne 1b 967 vaesenclast RNDKEYLAST, %zmm0, %zmm0 968 969 // XOR the data with the appropriate number of keystream bytes. 970 vmovdqu8 (SRC), %zmm1{%k1}{z} 971 vpxord %zmm1, %zmm0, %zmm0 972 vmovdqu8 %zmm0, (DST){%k1} 973 974 // Update GHASH with the ciphertext block(s), without reducing. 975 // 976 // In the case of DATALEN < 64, the ciphertext is zero-padded to 64 977 // bytes. (If decrypting, it's done by the above masked load. If 978 // encrypting, it's done by the below masked register-to-register move.) 979 // Note that if DATALEN <= 48, there will be additional padding beyond 980 // the padding of the last block specified by GHASH itself; i.e., there 981 // may be whole block(s) that get processed by the GHASH multiplication 982 // and reduction instructions but should not actually be included in the 983 // GHASH. However, any such blocks are all-zeroes, and the values that 984 // they're multiplied with are also all-zeroes. Therefore they just add 985 // 0 * 0 = 0 to the final GHASH result, which makes no difference. 986 vmovdqu8 (POWERS_PTR), H_POW1 987.if \enc 988 vmovdqu8 %zmm0, %zmm1{%k1}{z} 989.endif 990 vpshufb BSWAP_MASK, %zmm1, %zmm0 991 vpxord GHASH_ACC, %zmm0, %zmm0 992 _ghash_mul_noreduce H_POW1, %zmm0, LO, MI, HI, \ 993 GHASHDATA3, %zmm1, %zmm2, %zmm3 994 vpxor GHASH_ACC_XMM, GHASH_ACC_XMM, GHASH_ACC_XMM 995 996 add $64, POWERS_PTR 997 add $64, SRC 998 add $64, DST 999 sub $64, DATALEN 1000 jg .Lcrypt_loop_1x\@ 1001 1002 // Finally, do the GHASH reduction. 1003 _ghash_reduce LO, MI, HI, GFPOLY, %zmm0 1004 _horizontal_xor HI, HI_XMM, GHASH_ACC_XMM, %xmm0, %xmm1, %xmm2 1005 1006.Ldone\@: 1007 // Store the updated GHASH accumulator back to memory. 1008 vmovdqu GHASH_ACC_XMM, (GHASH_ACC_PTR) 1009 1010 vzeroupper // This is needed after using ymm or zmm registers. 1011 RET 1012.endm 1013 1014// void aes_gcm_enc_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key, 1015// const u32 le_ctr[4], u8 ghash_acc[16], 1016// u64 total_aadlen, u64 total_datalen); 1017// bool aes_gcm_dec_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key, 1018// const u32 le_ctr[4], 1019// const u8 ghash_acc[16], 1020// u64 total_aadlen, u64 total_datalen, 1021// const u8 tag[16], int taglen); 1022// 1023// This macro generates one of the above two functions (with \enc selecting 1024// which one). Both functions finish computing the GCM authentication tag by 1025// updating GHASH with the lengths block and encrypting the GHASH accumulator. 1026// |total_aadlen| and |total_datalen| must be the total length of the additional 1027// authenticated data and the en/decrypted data in bytes, respectively. 1028// 1029// The encryption function then stores the full-length (16-byte) computed 1030// authentication tag to |ghash_acc|. The decryption function instead loads the 1031// expected authentication tag (the one that was transmitted) from the 16-byte 1032// buffer |tag|, compares the first 4 <= |taglen| <= 16 bytes of it to the 1033// computed tag in constant time, and returns true if and only if they match. 1034.macro _aes_gcm_final enc 1035 1036 // Function arguments 1037 .set KEY, %rdi 1038 .set LE_CTR_PTR, %rsi 1039 .set GHASH_ACC_PTR, %rdx 1040 .set TOTAL_AADLEN, %rcx 1041 .set TOTAL_DATALEN, %r8 1042 .set TAG, %r9 1043 .set TAGLEN, %r10d // Originally at 8(%rsp) 1044 1045 // Additional local variables. 1046 // %rax, %xmm0-%xmm3, and %k1 are used as temporary registers. 1047 .set AESKEYLEN, %r11d 1048 .set AESKEYLEN64, %r11 1049 .set GFPOLY, %xmm4 1050 .set BSWAP_MASK, %xmm5 1051 .set LE_CTR, %xmm6 1052 .set GHASH_ACC, %xmm7 1053 .set H_POW1, %xmm8 1054 1055 // Load some constants. 1056 vmovdqa .Lgfpoly(%rip), GFPOLY 1057 vmovdqa .Lbswap_mask(%rip), BSWAP_MASK 1058 1059 // Load the AES key length in bytes. 1060 movl OFFSETOF_AESKEYLEN(KEY), AESKEYLEN 1061 1062 // Set up a counter block with 1 in the low 32-bit word. This is the 1063 // counter that produces the ciphertext needed to encrypt the auth tag. 1064 // GFPOLY has 1 in the low word, so grab the 1 from there using a blend. 1065 vpblendd $0xe, (LE_CTR_PTR), GFPOLY, LE_CTR 1066 1067 // Build the lengths block and XOR it with the GHASH accumulator. 1068 // Although the lengths block is defined as the AAD length followed by 1069 // the en/decrypted data length, both in big-endian byte order, a byte 1070 // reflection of the full block is needed because of the way we compute 1071 // GHASH (see _ghash_mul_step). By using little-endian values in the 1072 // opposite order, we avoid having to reflect any bytes here. 1073 vmovq TOTAL_DATALEN, %xmm0 1074 vpinsrq $1, TOTAL_AADLEN, %xmm0, %xmm0 1075 vpsllq $3, %xmm0, %xmm0 // Bytes to bits 1076 vpxor (GHASH_ACC_PTR), %xmm0, GHASH_ACC 1077 1078 // Load the first hash key power (H^1), which is stored last. 1079 vmovdqu8 OFFSETOFEND_H_POWERS-16(KEY), H_POW1 1080 1081.if !\enc 1082 // Prepare a mask of TAGLEN one bits. 1083 movl 8(%rsp), TAGLEN 1084 mov $-1, %eax 1085 bzhi TAGLEN, %eax, %eax 1086 kmovd %eax, %k1 1087.endif 1088 1089 // Make %rax point to the last AES round key for the chosen AES variant. 1090 lea 6*16(KEY,AESKEYLEN64,4), %rax 1091 1092 // Start the AES encryption of the counter block by swapping the counter 1093 // block to big-endian and XOR-ing it with the zero-th AES round key. 1094 vpshufb BSWAP_MASK, LE_CTR, %xmm0 1095 vpxor (KEY), %xmm0, %xmm0 1096 1097 // Complete the AES encryption and multiply GHASH_ACC by H^1. 1098 // Interleave the AES and GHASH instructions to improve performance. 1099 cmp $24, AESKEYLEN 1100 jl 128f // AES-128? 1101 je 192f // AES-192? 1102 // AES-256 1103 vaesenc -13*16(%rax), %xmm0, %xmm0 1104 vaesenc -12*16(%rax), %xmm0, %xmm0 1105192: 1106 vaesenc -11*16(%rax), %xmm0, %xmm0 1107 vaesenc -10*16(%rax), %xmm0, %xmm0 1108128: 1109.irp i, 0,1,2,3,4,5,6,7,8 1110 _ghash_mul_step \i, H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \ 1111 %xmm1, %xmm2, %xmm3 1112 vaesenc (\i-9)*16(%rax), %xmm0, %xmm0 1113.endr 1114 _ghash_mul_step 9, H_POW1, GHASH_ACC, GHASH_ACC, GFPOLY, \ 1115 %xmm1, %xmm2, %xmm3 1116 1117 // Undo the byte reflection of the GHASH accumulator. 1118 vpshufb BSWAP_MASK, GHASH_ACC, GHASH_ACC 1119 1120 // Do the last AES round and XOR the resulting keystream block with the 1121 // GHASH accumulator to produce the full computed authentication tag. 1122 // 1123 // Reduce latency by taking advantage of the property vaesenclast(key, 1124 // a) ^ b == vaesenclast(key ^ b, a). I.e., XOR GHASH_ACC into the last 1125 // round key, instead of XOR'ing the final AES output with GHASH_ACC. 1126 // 1127 // enc_final then returns the computed auth tag, while dec_final 1128 // compares it with the transmitted one and returns a bool. To compare 1129 // the tags, dec_final XORs them together and uses vptest to check 1130 // whether the result is all-zeroes. This should be constant-time. 1131 // dec_final applies the vaesenclast optimization to this additional 1132 // value XOR'd too, using vpternlogd to XOR the last round key, GHASH 1133 // accumulator, and transmitted auth tag together in one instruction. 1134.if \enc 1135 vpxor (%rax), GHASH_ACC, %xmm1 1136 vaesenclast %xmm1, %xmm0, GHASH_ACC 1137 vmovdqu GHASH_ACC, (GHASH_ACC_PTR) 1138.else 1139 vmovdqu (TAG), %xmm1 1140 vpternlogd $0x96, (%rax), GHASH_ACC, %xmm1 1141 vaesenclast %xmm1, %xmm0, %xmm0 1142 xor %eax, %eax 1143 vmovdqu8 %xmm0, %xmm0{%k1}{z} // Truncate to TAGLEN bytes 1144 vptest %xmm0, %xmm0 1145 sete %al 1146.endif 1147 // No need for vzeroupper here, since only used xmm registers were used. 1148 RET 1149.endm 1150 1151SYM_FUNC_START(aes_gcm_enc_update_vaes_avx512) 1152 _aes_gcm_update 1 1153SYM_FUNC_END(aes_gcm_enc_update_vaes_avx512) 1154SYM_FUNC_START(aes_gcm_dec_update_vaes_avx512) 1155 _aes_gcm_update 0 1156SYM_FUNC_END(aes_gcm_dec_update_vaes_avx512) 1157 1158SYM_FUNC_START(aes_gcm_enc_final_vaes_avx512) 1159 _aes_gcm_final 1 1160SYM_FUNC_END(aes_gcm_enc_final_vaes_avx512) 1161SYM_FUNC_START(aes_gcm_dec_final_vaes_avx512) 1162 _aes_gcm_final 0 1163SYM_FUNC_END(aes_gcm_dec_final_vaes_avx512) 1164