1*fa7ed85cSEric Biggers // SPDX-License-Identifier: GPL-2.0 2*fa7ed85cSEric Biggers /* 3*fa7ed85cSEric Biggers * CRC-32 implemented with the z/Architecture Vector Extension Facility. 4*fa7ed85cSEric Biggers * 5*fa7ed85cSEric Biggers * Copyright IBM Corp. 2015 6*fa7ed85cSEric Biggers * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> 7*fa7ed85cSEric Biggers */ 8*fa7ed85cSEric Biggers #define KMSG_COMPONENT "crc32-vx" 9*fa7ed85cSEric Biggers #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 10*fa7ed85cSEric Biggers 11*fa7ed85cSEric Biggers #include <linux/module.h> 12*fa7ed85cSEric Biggers #include <linux/cpufeature.h> 13*fa7ed85cSEric Biggers #include <linux/crc32.h> 14*fa7ed85cSEric Biggers #include <asm/fpu.h> 15*fa7ed85cSEric Biggers #include "crc32-vx.h" 16*fa7ed85cSEric Biggers 17*fa7ed85cSEric Biggers #define VX_MIN_LEN 64 18*fa7ed85cSEric Biggers #define VX_ALIGNMENT 16L 19*fa7ed85cSEric Biggers #define VX_ALIGN_MASK (VX_ALIGNMENT - 1) 20*fa7ed85cSEric Biggers 21*fa7ed85cSEric Biggers /* 22*fa7ed85cSEric Biggers * DEFINE_CRC32_VX() - Define a CRC-32 function using the vector extension 23*fa7ed85cSEric Biggers * 24*fa7ed85cSEric Biggers * Creates a function to perform a particular CRC-32 computation. Depending 25*fa7ed85cSEric Biggers * on the message buffer, the hardware-accelerated or software implementation 26*fa7ed85cSEric Biggers * is used. Note that the message buffer is aligned to improve fetch 27*fa7ed85cSEric Biggers * operations of VECTOR LOAD MULTIPLE instructions. 28*fa7ed85cSEric Biggers */ 29*fa7ed85cSEric Biggers #define DEFINE_CRC32_VX(___fname, ___crc32_vx, ___crc32_sw) \ 30*fa7ed85cSEric Biggers u32 ___fname(u32 crc, const u8 *data, size_t datalen) \ 31*fa7ed85cSEric Biggers { \ 32*fa7ed85cSEric Biggers unsigned long prealign, aligned, remaining; \ 33*fa7ed85cSEric Biggers DECLARE_KERNEL_FPU_ONSTACK16(vxstate); \ 34*fa7ed85cSEric Biggers \ 35*fa7ed85cSEric Biggers if (datalen < VX_MIN_LEN + VX_ALIGN_MASK || !cpu_has_vx()) \ 36*fa7ed85cSEric Biggers return ___crc32_sw(crc, data, datalen); \ 37*fa7ed85cSEric Biggers \ 38*fa7ed85cSEric Biggers if ((unsigned long)data & VX_ALIGN_MASK) { \ 39*fa7ed85cSEric Biggers prealign = VX_ALIGNMENT - \ 40*fa7ed85cSEric Biggers ((unsigned long)data & VX_ALIGN_MASK); \ 41*fa7ed85cSEric Biggers datalen -= prealign; \ 42*fa7ed85cSEric Biggers crc = ___crc32_sw(crc, data, prealign); \ 43*fa7ed85cSEric Biggers data = (void *)((unsigned long)data + prealign); \ 44*fa7ed85cSEric Biggers } \ 45*fa7ed85cSEric Biggers \ 46*fa7ed85cSEric Biggers aligned = datalen & ~VX_ALIGN_MASK; \ 47*fa7ed85cSEric Biggers remaining = datalen & VX_ALIGN_MASK; \ 48*fa7ed85cSEric Biggers \ 49*fa7ed85cSEric Biggers kernel_fpu_begin(&vxstate, KERNEL_VXR_LOW); \ 50*fa7ed85cSEric Biggers crc = ___crc32_vx(crc, data, aligned); \ 51*fa7ed85cSEric Biggers kernel_fpu_end(&vxstate, KERNEL_VXR_LOW); \ 52*fa7ed85cSEric Biggers \ 53*fa7ed85cSEric Biggers if (remaining) \ 54*fa7ed85cSEric Biggers crc = ___crc32_sw(crc, data + aligned, remaining); \ 55*fa7ed85cSEric Biggers \ 56*fa7ed85cSEric Biggers return crc; \ 57*fa7ed85cSEric Biggers } \ 58*fa7ed85cSEric Biggers EXPORT_SYMBOL(___fname); 59*fa7ed85cSEric Biggers 60*fa7ed85cSEric Biggers DEFINE_CRC32_VX(crc32_le_arch, crc32_le_vgfm_16, crc32_le_base) 61*fa7ed85cSEric Biggers DEFINE_CRC32_VX(crc32_be_arch, crc32_be_vgfm_16, crc32_be_base) 62*fa7ed85cSEric Biggers DEFINE_CRC32_VX(crc32c_arch, crc32c_le_vgfm_16, crc32c_base) 63*fa7ed85cSEric Biggers 64*fa7ed85cSEric Biggers u32 crc32_optimizations(void) 65*fa7ed85cSEric Biggers { 66*fa7ed85cSEric Biggers if (cpu_has_vx()) { 67*fa7ed85cSEric Biggers return CRC32_LE_OPTIMIZATION | 68*fa7ed85cSEric Biggers CRC32_BE_OPTIMIZATION | 69*fa7ed85cSEric Biggers CRC32C_OPTIMIZATION; 70*fa7ed85cSEric Biggers } 71*fa7ed85cSEric Biggers return 0; 72*fa7ed85cSEric Biggers } 73*fa7ed85cSEric Biggers EXPORT_SYMBOL(crc32_optimizations); 74*fa7ed85cSEric Biggers 75*fa7ed85cSEric Biggers MODULE_AUTHOR("Hendrik Brueckner <brueckner@linux.vnet.ibm.com>"); 76*fa7ed85cSEric Biggers MODULE_DESCRIPTION("CRC-32 algorithms using z/Architecture Vector Extension Facility"); 77*fa7ed85cSEric Biggers MODULE_LICENSE("GPL"); 78