1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Poly1305 authenticator algorithm, RFC7539. 4 * 5 * Copyright 2023- IBM Corp. All rights reserved. 6 */ 7 #include <asm/switch_to.h> 8 #include <linux/cpufeature.h> 9 #include <linux/jump_label.h> 10 #include <linux/kernel.h> 11 #include <linux/unaligned.h> 12 13 asmlinkage void poly1305_p10le_4blocks(struct poly1305_block_state *state, const u8 *m, u32 mlen); 14 asmlinkage void poly1305_64s(struct poly1305_block_state *state, const u8 *m, u32 mlen, int highbit); 15 asmlinkage void poly1305_emit_64(const struct poly1305_state *state, const u32 nonce[4], u8 digest[POLY1305_DIGEST_SIZE]); 16 17 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); 18 19 static void vsx_begin(void) 20 { 21 preempt_disable(); 22 enable_kernel_vsx(); 23 } 24 25 static void vsx_end(void) 26 { 27 disable_kernel_vsx(); 28 preempt_enable(); 29 } 30 31 static void poly1305_block_init(struct poly1305_block_state *dctx, 32 const u8 raw_key[POLY1305_BLOCK_SIZE]) 33 { 34 if (!static_key_enabled(&have_p10)) 35 return poly1305_block_init_generic(dctx, raw_key); 36 37 dctx->h = (struct poly1305_state){}; 38 dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0); 39 dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8); 40 } 41 42 static void poly1305_blocks(struct poly1305_block_state *state, const u8 *src, 43 unsigned int len, u32 padbit) 44 { 45 if (!static_key_enabled(&have_p10)) 46 return poly1305_blocks_generic(state, src, len, padbit); 47 vsx_begin(); 48 if (len >= POLY1305_BLOCK_SIZE * 4) { 49 poly1305_p10le_4blocks(state, src, len); 50 src += len - (len % (POLY1305_BLOCK_SIZE * 4)); 51 len %= POLY1305_BLOCK_SIZE * 4; 52 } 53 while (len >= POLY1305_BLOCK_SIZE) { 54 poly1305_64s(state, src, POLY1305_BLOCK_SIZE, padbit); 55 len -= POLY1305_BLOCK_SIZE; 56 src += POLY1305_BLOCK_SIZE; 57 } 58 vsx_end(); 59 } 60 61 static void poly1305_emit(const struct poly1305_state *state, 62 u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4]) 63 { 64 if (!static_key_enabled(&have_p10)) 65 return poly1305_emit_generic(state, digest, nonce); 66 poly1305_emit_64(state, nonce, digest); 67 } 68 69 #define poly1305_mod_init_arch poly1305_mod_init_arch 70 static void poly1305_mod_init_arch(void) 71 { 72 if (cpu_has_feature(CPU_FTR_ARCH_31)) 73 static_branch_enable(&have_p10); 74 } 75