1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * CRC-32 implemented with the z/Architecture Vector Extension Facility. 4 * 5 * Copyright IBM Corp. 2015 6 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> 7 */ 8 9 #include <linux/cpufeature.h> 10 #include <asm/fpu.h> 11 #include "crc32-vx.h" 12 13 #define VX_MIN_LEN 64 14 #define VX_ALIGNMENT 16L 15 #define VX_ALIGN_MASK (VX_ALIGNMENT - 1) 16 17 /* 18 * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension 19 * 20 * Creates a function to perform a particular CRC-32 computation. Depending 21 * on the message buffer, the hardware-accelerated or software implementation 22 * is used. Note that the message buffer is aligned to improve fetch 23 * operations of VECTOR LOAD MULTIPLE instructions. 24 */ 25 #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \ 26 static inline u32 ___fname(u32 crc, const u8 *data, size_t datalen) \ 27 { \ 28 unsigned long prealign, aligned, remaining; \ 29 DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \ 30 \ 31 if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || !cpu_has_vx()) \ 32 return ___crc32_sw(crc, data, datalen); \ 33 \ 34 if ((unsigned long)data & VX_ALIGN_MASK) { \ 35 prealign = VX_ALIGNMENT - \ 36 ((unsigned long)data & VX_ALIGN_MASK); \ 37 datalen -= prealign; \ 38 crc = ___crc32_sw(crc, data, prealign); \ 39 data = (void *)((unsigned long)data + prealign); \ 40 } \ 41 \ 42 aligned = datalen & ~VX_ALIGN_MASK; \ 43 remaining = datalen & VX_ALIGN_MASK; \ 44 \ 45 kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \ 46 crc = ___crc32_vx(crc, data, aligned); \ 47 kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \ 48 \ 49 if (remaining) \ 50 crc = ___crc32_sw(crc, data + aligned, remaining); \ 51 \ 52 return crc; \ 53 } 54 55 DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base) 56 DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base) 57 DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base) 58 59 static inline u32 crc32_optimizations_arch(void) 60 { 61 if (cpu_has_vx()) { 62 return CRC32_LE_OPTIMIZATION | 63 CRC32_BE_OPTIMIZATION | 64 CRC32C_OPTIMIZATION; 65 } 66 return 0; 67 } 68