1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/crc32.h> 3 #include <crypto/internal/hash.h> 4 #include <crypto/internal/simd.h> 5 #include <linux/init.h> 6 #include <linux/module.h> 7 #include <linux/string.h> 8 #include <linux/kernel.h> 9 #include <linux/cpufeature.h> 10 #include <asm/simd.h> 11 #include <asm/switch_to.h> 12 13 #define CHKSUM_BLOCK_SIZE 1 14 #define CHKSUM_DIGEST_SIZE 4 15 16 #define VMX_ALIGN 16 17 #define VMX_ALIGN_MASK (VMX_ALIGN-1) 18 19 #define VECTOR_BREAKPOINT 512 20 21 u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len); 22 23 static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len) 24 { 25 unsigned int prealign; 26 unsigned int tail; 27 28 if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || !crypto_simd_usable()) 29 return __crc32c_le(crc, p, len); 30 31 if ((unsigned long)p & VMX_ALIGN_MASK) { 32 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); 33 crc = __crc32c_le(crc, p, prealign); 34 len -= prealign; 35 p += prealign; 36 } 37 38 if (len & ~VMX_ALIGN_MASK) { 39 preempt_disable(); 40 pagefault_disable(); 41 enable_kernel_altivec(); 42 crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); 43 disable_kernel_altivec(); 44 pagefault_enable(); 45 preempt_enable(); 46 } 47 48 tail = len & VMX_ALIGN_MASK; 49 if (tail) { 50 p += len & ~VMX_ALIGN_MASK; 51 crc = __crc32c_le(crc, p, tail); 52 } 53 54 return crc; 55 } 56 57 static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm) 58 { 59 u32 *key = crypto_tfm_ctx(tfm); 60 61 *key = ~0; 62 63 return 0; 64 } 65 66 /* 67 * Setting the seed allows arbitrary accumulators and flexible XOR policy 68 * If your algorithm starts with ~0, then XOR with ~0 before you set 69 * the seed. 70 */ 71 static int crc32c_vpmsum_setkey(struct crypto_shash *hash, const u8 *key, 72 unsigned int keylen) 73 { 74 u32 *mctx = crypto_shash_ctx(hash); 75 76 if (keylen != sizeof(u32)) 77 return -EINVAL; 78 *mctx = le32_to_cpup((__le32 *)key); 79 return 0; 80 } 81 82 static int crc32c_vpmsum_init(struct shash_desc *desc) 83 { 84 u32 *mctx = crypto_shash_ctx(desc->tfm); 85 u32 *crcp = shash_desc_ctx(desc); 86 87 *crcp = *mctx; 88 89 return 0; 90 } 91 92 static int crc32c_vpmsum_update(struct shash_desc *desc, const u8 *data, 93 unsigned int len) 94 { 95 u32 *crcp = shash_desc_ctx(desc); 96 97 *crcp = crc32c_vpmsum(*crcp, data, len); 98 99 return 0; 100 } 101 102 static int __crc32c_vpmsum_finup(u32 *crcp, const u8 *data, unsigned int len, 103 u8 *out) 104 { 105 *(__le32 *)out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len)); 106 107 return 0; 108 } 109 110 static int crc32c_vpmsum_finup(struct shash_desc *desc, const u8 *data, 111 unsigned int len, u8 *out) 112 { 113 return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out); 114 } 115 116 static int crc32c_vpmsum_final(struct shash_desc *desc, u8 *out) 117 { 118 u32 *crcp = shash_desc_ctx(desc); 119 120 *(__le32 *)out = ~cpu_to_le32p(crcp); 121 122 return 0; 123 } 124 125 static int crc32c_vpmsum_digest(struct shash_desc *desc, const u8 *data, 126 unsigned int len, u8 *out) 127 { 128 return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len, 129 out); 130 } 131 132 static struct shash_alg alg = { 133 .setkey = crc32c_vpmsum_setkey, 134 .init = crc32c_vpmsum_init, 135 .update = crc32c_vpmsum_update, 136 .final = crc32c_vpmsum_final, 137 .finup = crc32c_vpmsum_finup, 138 .digest = crc32c_vpmsum_digest, 139 .descsize = sizeof(u32), 140 .digestsize = CHKSUM_DIGEST_SIZE, 141 .base = { 142 .cra_name = "crc32c", 143 .cra_driver_name = "crc32c-vpmsum", 144 .cra_priority = 200, 145 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, 146 .cra_blocksize = CHKSUM_BLOCK_SIZE, 147 .cra_ctxsize = sizeof(u32), 148 .cra_module = THIS_MODULE, 149 .cra_init = crc32c_vpmsum_cra_init, 150 } 151 }; 152 153 static int __init crc32c_vpmsum_mod_init(void) 154 { 155 if (!cpu_has_feature(CPU_FTR_ARCH_207S)) 156 return -ENODEV; 157 158 return crypto_register_shash(&alg); 159 } 160 161 static void __exit crc32c_vpmsum_mod_fini(void) 162 { 163 crypto_unregister_shash(&alg); 164 } 165 166 module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crc32c_vpmsum_mod_init); 167 module_exit(crc32c_vpmsum_mod_fini); 168 169 MODULE_AUTHOR("Anton Blanchard <anton@samba.org>"); 170 MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions"); 171 MODULE_LICENSE("GPL"); 172 MODULE_ALIAS_CRYPTO("crc32c"); 173 MODULE_ALIAS_CRYPTO("crc32c-vpmsum"); 174