1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Calculate a CRC T10-DIF with vpmsum acceleration 4 * 5 * Copyright 2017, Daniel Axtens, IBM Corporation. 6 * [based on crc32c-vpmsum_glue.c] 7 */ 8 9 #include <linux/crc-t10dif.h> 10 #include <crypto/internal/simd.h> 11 #include <linux/init.h> 12 #include <linux/module.h> 13 #include <linux/string.h> 14 #include <linux/kernel.h> 15 #include <linux/cpufeature.h> 16 #include <asm/simd.h> 17 #include <asm/switch_to.h> 18 19 #define VMX_ALIGN 16 20 #define VMX_ALIGN_MASK (VMX_ALIGN-1) 21 22 #define VECTOR_BREAKPOINT 64 23 24 static DEFINE_STATIC_KEY_FALSE(have_vec_crypto); 25 26 u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len); 27 28 u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len) 29 { 30 unsigned int prealign; 31 unsigned int tail; 32 u32 crc = crci; 33 34 if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || 35 !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable()) 36 return crc_t10dif_generic(crc, p, len); 37 38 if ((unsigned long)p & VMX_ALIGN_MASK) { 39 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); 40 crc = crc_t10dif_generic(crc, p, prealign); 41 len -= prealign; 42 p += prealign; 43 } 44 45 if (len & ~VMX_ALIGN_MASK) { 46 crc <<= 16; 47 preempt_disable(); 48 pagefault_disable(); 49 enable_kernel_altivec(); 50 crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); 51 disable_kernel_altivec(); 52 pagefault_enable(); 53 preempt_enable(); 54 crc >>= 16; 55 } 56 57 tail = len & VMX_ALIGN_MASK; 58 if (tail) { 59 p += len & ~VMX_ALIGN_MASK; 60 crc = crc_t10dif_generic(crc, p, tail); 61 } 62 63 return crc & 0xffff; 64 } 65 EXPORT_SYMBOL(crc_t10dif_arch); 66 67 static int __init crc_t10dif_powerpc_init(void) 68 { 69 if (cpu_has_feature(CPU_FTR_ARCH_207S) && 70 (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO)) 71 static_branch_enable(&have_vec_crypto); 72 return 0; 73 } 74 arch_initcall(crc_t10dif_powerpc_init); 75 76 static void __exit crc_t10dif_powerpc_exit(void) 77 { 78 } 79 module_exit(crc_t10dif_powerpc_exit); 80 81 bool crc_t10dif_is_optimized(void) 82 { 83 return static_key_enabled(&have_vec_crypto); 84 } 85 EXPORT_SYMBOL(crc_t10dif_is_optimized); 86 87 MODULE_AUTHOR("Daniel Axtens <dja@axtens.net>"); 88 MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions"); 89 MODULE_LICENSE("GPL"); 90