1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * ChaCha stream cipher (P10 accelerated) 4 * 5 * Copyright 2023- IBM Corp. All rights reserved. 6 */ 7 8 #include <crypto/chacha.h> 9 #include <crypto/internal/simd.h> 10 #include <linux/kernel.h> 11 #include <linux/module.h> 12 #include <linux/cpufeature.h> 13 #include <linux/sizes.h> 14 #include <asm/simd.h> 15 #include <asm/switch_to.h> 16 17 asmlinkage void chacha_p10le_8x(const struct chacha_state *state, u8 *dst, 18 const u8 *src, unsigned int len, int nrounds); 19 20 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); 21 22 static void vsx_begin(void) 23 { 24 preempt_disable(); 25 enable_kernel_vsx(); 26 } 27 28 static void vsx_end(void) 29 { 30 disable_kernel_vsx(); 31 preempt_enable(); 32 } 33 34 static void chacha_p10_do_8x(struct chacha_state *state, u8 *dst, const u8 *src, 35 unsigned int bytes, int nrounds) 36 { 37 unsigned int l = bytes & ~0x0FF; 38 39 if (l > 0) { 40 chacha_p10le_8x(state, dst, src, l, nrounds); 41 bytes -= l; 42 src += l; 43 dst += l; 44 state->x[12] += l / CHACHA_BLOCK_SIZE; 45 } 46 47 if (bytes > 0) 48 chacha_crypt_generic(state, dst, src, bytes, nrounds); 49 } 50 51 void hchacha_block_arch(const struct chacha_state *state, 52 u32 out[HCHACHA_OUT_WORDS], int nrounds) 53 { 54 hchacha_block_generic(state, out, nrounds); 55 } 56 EXPORT_SYMBOL(hchacha_block_arch); 57 58 void chacha_crypt_arch(struct chacha_state *state, u8 *dst, const u8 *src, 59 unsigned int bytes, int nrounds) 60 { 61 if (!static_branch_likely(&have_p10) || bytes <= CHACHA_BLOCK_SIZE || 62 !crypto_simd_usable()) 63 return chacha_crypt_generic(state, dst, src, bytes, nrounds); 64 65 do { 66 unsigned int todo = min_t(unsigned int, bytes, SZ_4K); 67 68 vsx_begin(); 69 chacha_p10_do_8x(state, dst, src, todo, nrounds); 70 vsx_end(); 71 72 bytes -= todo; 73 src += todo; 74 dst += todo; 75 } while (bytes); 76 } 77 EXPORT_SYMBOL(chacha_crypt_arch); 78 79 bool chacha_is_arch_optimized(void) 80 { 81 return static_key_enabled(&have_p10); 82 } 83 EXPORT_SYMBOL(chacha_is_arch_optimized); 84 85 static int __init chacha_p10_init(void) 86 { 87 if (cpu_has_feature(CPU_FTR_ARCH_31)) 88 static_branch_enable(&have_p10); 89 return 0; 90 } 91 subsys_initcall(chacha_p10_init); 92 93 static void __exit chacha_p10_exit(void) 94 { 95 } 96 module_exit(chacha_p10_exit); 97 98 MODULE_DESCRIPTION("ChaCha stream cipher (P10 accelerated)"); 99 MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com>"); 100 MODULE_LICENSE("GPL v2"); 101