1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * CRC-32 implemented with the z/Architecture Vector Extension Facility. 4 * 5 * Copyright IBM Corp. 2015 6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> 7 */ 8 #define KMSG_COMPONENT "crc32-vx" 9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 10 11 #include <linux/module.h> 12 #include <linux/cpufeature.h> 13 #include <linux/crc32.h> 14 #include <asm/fpu.h> 15 #include "crc32-vx.h" 16 17 #define VX_MIN_LEN 64 18 #define VX_ALIGNMENT 16L 19 #define VX_ALIGN_MASK (VX_ALIGNMENT - 1) 20 21 /* 22 * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension 23 * 24 * Creates a function to perform a particular CRC-32 computation. Depending 25 * on the message buffer, the hardware-accelerated or software implementation 26 * is used. Note that the message buffer is aligned to improve fetch 27 * operations of VECTOR LOAD MULTIPLE instructions. 28 */ 29 #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \ 30 u32 ___fname(u32 crc, const u8 *data, size_t datalen) \ 31 { \ 32 unsigned long prealign, aligned, remaining; \ 33 DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \ 34 \ 35 if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || !cpu_has_vx()) \ 36 return ___crc32_sw(crc, data, datalen); \ 37 \ 38 if ((unsigned long)data & VX_ALIGN_MASK) { \ 39 prealign = VX_ALIGNMENT - \ 40 ((unsigned long)data & VX_ALIGN_MASK); \ 41 datalen -= prealign; \ 42 crc = ___crc32_sw(crc, data, prealign); \ 43 data = (void *)((unsigned long)data + prealign); \ 44 } \ 45 \ 46 aligned = datalen & ~VX_ALIGN_MASK; \ 47 remaining = datalen & VX_ALIGN_MASK; \ 48 \ 49 kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \ 50 crc = ___crc32_vx(crc, data, aligned); \ 51 kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \ 52 \ 53 if (remaining) \ 54 crc = ___crc32_sw(crc, data + aligned, remaining); \ 55 \ 56 return crc; \ 57 } \ 58 EXPORT_SYMBOL(___fname); 59 60 DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base) 61 DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base) 62 DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base) 63 64 u32 crc32_optimizations(void) 65 { 66 if (cpu_has_vx()) { 67 return CRC32_LE_OPTIMIZATION | 68 CRC32_BE_OPTIMIZATION | 69 CRC32C_OPTIMIZATION; 70 } 71 return 0; 72 } 73 EXPORT_SYMBOL(crc32_optimizations); 74 75 MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>"); 76 MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility"); 77 MODULE_LICENSE("GPL"); 78