1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * GHASH routines supporting VMX instructions on the Power 8 4 * 5 * Copyright (C) 2015, 2019 International Business Machines Inc. 6 * Copyright (C) 2014 - 2018 Linaro Ltd. 7 * Copyright 2026 Google LLC 8 */ 9 10 #include <asm/simd.h> 11 #include <asm/switch_to.h> 12 #include <linux/cpufeature.h> 13 #include <linux/jump_label.h> 14 #include <linux/preempt.h> 15 #include <linux/uaccess.h> 16 17 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); 18 19 void gcm_init_p8(u64 htable[4][2], const u8 h[16]); 20 void gcm_gmult_p8(u8 Xi[16], const u64 htable[4][2]); 21 void gcm_ghash_p8(u8 Xi[16], const u64 htable[4][2], const u8 *in, size_t len); 22 23 #define ghash_preparekey_arch ghash_preparekey_arch 24 static void ghash_preparekey_arch(struct ghash_key *key, 25 const u8 raw_key[GHASH_BLOCK_SIZE]) 26 { 27 ghash_key_to_polyval(raw_key, &key->h); 28 29 if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { 30 preempt_disable(); 31 pagefault_disable(); 32 enable_kernel_vsx(); 33 gcm_init_p8(key->htable, raw_key); 34 disable_kernel_vsx(); 35 pagefault_enable(); 36 preempt_enable(); 37 } else { 38 /* This reproduces gcm_init_p8() on both LE and BE systems. */ 39 key->htable[0][0] = 0; 40 key->htable[0][1] = 0xc200000000000000; 41 42 key->htable[1][0] = 0; 43 key->htable[1][1] = le64_to_cpu(key->h.lo); 44 45 key->htable[2][0] = le64_to_cpu(key->h.lo); 46 key->htable[2][1] = le64_to_cpu(key->h.hi); 47 48 key->htable[3][0] = le64_to_cpu(key->h.hi); 49 key->htable[3][1] = 0; 50 } 51 } 52 53 #define ghash_mul_arch ghash_mul_arch 54 static void ghash_mul_arch(struct polyval_elem *acc, 55 const struct ghash_key *key) 56 { 57 if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { 58 u8 ghash_acc[GHASH_BLOCK_SIZE]; 59 60 polyval_acc_to_ghash(acc, ghash_acc); 61 62 preempt_disable(); 63 pagefault_disable(); 64 enable_kernel_vsx(); 65 gcm_gmult_p8(ghash_acc, key->htable); 66 disable_kernel_vsx(); 67 pagefault_enable(); 68 preempt_enable(); 69 70 ghash_acc_to_polyval(ghash_acc, acc); 71 memzero_explicit(ghash_acc, sizeof(ghash_acc)); 72 } else { 73 polyval_mul_generic(acc, &key->h); 74 } 75 } 76 77 #define ghash_blocks_arch ghash_blocks_arch 78 static void ghash_blocks_arch(struct polyval_elem *acc, 79 const struct ghash_key *key, 80 const u8 *data, size_t nblocks) 81 { 82 if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) { 83 u8 ghash_acc[GHASH_BLOCK_SIZE]; 84 85 polyval_acc_to_ghash(acc, ghash_acc); 86 87 preempt_disable(); 88 pagefault_disable(); 89 enable_kernel_vsx(); 90 gcm_ghash_p8(ghash_acc, key->htable, data, 91 nblocks * GHASH_BLOCK_SIZE); 92 disable_kernel_vsx(); 93 pagefault_enable(); 94 preempt_enable(); 95 96 ghash_acc_to_polyval(ghash_acc, acc); 97 memzero_explicit(ghash_acc, sizeof(ghash_acc)); 98 } else { 99 ghash_blocks_generic(acc, &key->h, data, nblocks); 100 } 101 } 102 103 #define gf128hash_mod_init_arch gf128hash_mod_init_arch 104 static void gf128hash_mod_init_arch(void) 105 { 106 if (cpu_has_feature(CPU_FTR_ARCH_207S) && 107 (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) 108 static_branch_enable(&have_vec_crypto); 109 } 110