1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Calculate a CRC T10-DIF with vpmsum acceleration 4 * 5 * Copyright 2017, Daniel Axtens, IBM Corporation. 6 * [based on crc32c-vpmsum_glue.c] 7 */ 8 9 #include <asm/simd.h> 10 #include <asm/switch_to.h> 11 #include <linux/cpufeature.h> 12 #include <linux/jump_label.h> 13 #include <linux/preempt.h> 14 #include <linux/uaccess.h> 15 16 #define VMX_ALIGN 16 17 #define VMX_ALIGN_MASK (VMX_ALIGN-1) 18 19 #define VECTOR_BREAKPOINT 64 20 21 static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto); 22 23 u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len); 24 25 static inline u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len) 26 { 27 unsigned int prealign; 28 unsigned int tail; 29 u32 crc = crci; 30 31 if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || 32 !static_branch_likely(&have_vec_crypto) || 33 unlikely(!may_use_simd())) 34 return crc_t10dif_generic(crc, p, len); 35 36 if ((unsigned long)p & VMX_ALIGN_MASK) { 37 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); 38 crc = crc_t10dif_generic(crc, p, prealign); 39 len -= prealign; 40 p += prealign; 41 } 42 43 if (len & ~VMX_ALIGN_MASK) { 44 crc <<= 16; 45 preempt_disable(); 46 pagefault_disable(); 47 enable_kernel_altivec(); 48 crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); 49 disable_kernel_altivec(); 50 pagefault_enable(); 51 preempt_enable(); 52 crc >>= 16; 53 } 54 55 tail = len & VMX_ALIGN_MASK; 56 if (tail) { 57 p += len & ~VMX_ALIGN_MASK; 58 crc = crc_t10dif_generic(crc, p, tail); 59 } 60 61 return crc & 0xffff; 62 } 63 64 #define crc_t10dif_mod_init_arch crc_t10dif_mod_init_arch 65 static void crc_t10dif_mod_init_arch(void) 66 { 67 if (cpu_has_feature(CPU_FTR_ARCH_207S) && 68 (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) 69 static_branch_enable(&have_vec_crypto); 70 } 71