1 #include <linux/crc32.h> 2 #include <crypto/internal/hash.h> 3 #include <linux/init.h> 4 #include <linux/module.h> 5 #include <linux/string.h> 6 #include <linux/kernel.h> 7 #include <linux/cpufeature.h> 8 #include <asm/switch_to.h> 9 10 #define CHKSUM_BLOCK_SIZE 1 11 #define CHKSUM_DIGEST_SIZE 4 12 13 #define VMX_ALIGN 16 14 #define VMX_ALIGN_MASK (VMX_ALIGN-1) 15 16 #define VECTOR_BREAKPOINT 512 17 18 u32 __crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len); 19 20 static u32 crc32c_vpmsum(u32 crc, unsigned char const *p, size_t len) 21 { 22 unsigned int prealign; 23 unsigned int tail; 24 25 if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) || in_interrupt()) 26 return __crc32c_le(crc, p, len); 27 28 if ((unsigned long)p & VMX_ALIGN_MASK) { 29 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK); 30 crc = __crc32c_le(crc, p, prealign); 31 len -= prealign; 32 p += prealign; 33 } 34 35 if (len & ~VMX_ALIGN_MASK) { 36 preempt_disable(); 37 pagefault_disable(); 38 enable_kernel_altivec(); 39 crc = __crc32c_vpmsum(crc, p, len & ~VMX_ALIGN_MASK); 40 disable_kernel_altivec(); 41 pagefault_enable(); 42 preempt_enable(); 43 } 44 45 tail = len & VMX_ALIGN_MASK; 46 if (tail) { 47 p += len & ~VMX_ALIGN_MASK; 48 crc = __crc32c_le(crc, p, tail); 49 } 50 51 return crc; 52 } 53 54 static int crc32c_vpmsum_cra_init(struct crypto_tfm *tfm) 55 { 56 u32 *key = crypto_tfm_ctx(tfm); 57 58 *key = ~0; 59 60 return 0; 61 } 62 63 /* 64 * Setting the seed allows arbitrary accumulators and flexible XOR policy 65 * If your algorithm starts with ~0, then XOR with ~0 before you set 66 * the seed. 67 */ 68 static int crc32c_vpmsum_setkey(struct crypto_shash *hash, const u8 *key, 69 unsigned int keylen) 70 { 71 u32 *mctx = crypto_shash_ctx(hash); 72 73 if (keylen != sizeof(u32)) { 74 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN); 75 return -EINVAL; 76 } 77 *mctx = le32_to_cpup((__le32 *)key); 78 return 0; 79 } 80 81 static int crc32c_vpmsum_init(struct shash_desc *desc) 82 { 83 u32 *mctx = crypto_shash_ctx(desc->tfm); 84 u32 *crcp = shash_desc_ctx(desc); 85 86 *crcp = *mctx; 87 88 return 0; 89 } 90 91 static int crc32c_vpmsum_update(struct shash_desc *desc, const u8 *data, 92 unsigned int len) 93 { 94 u32 *crcp = shash_desc_ctx(desc); 95 96 *crcp = crc32c_vpmsum(*crcp, data, len); 97 98 return 0; 99 } 100 101 static int __crc32c_vpmsum_finup(u32 *crcp, const u8 *data, unsigned int len, 102 u8 *out) 103 { 104 *(__le32 *)out = ~cpu_to_le32(crc32c_vpmsum(*crcp, data, len)); 105 106 return 0; 107 } 108 109 static int crc32c_vpmsum_finup(struct shash_desc *desc, const u8 *data, 110 unsigned int len, u8 *out) 111 { 112 return __crc32c_vpmsum_finup(shash_desc_ctx(desc), data, len, out); 113 } 114 115 static int crc32c_vpmsum_final(struct shash_desc *desc, u8 *out) 116 { 117 u32 *crcp = shash_desc_ctx(desc); 118 119 *(__le32 *)out = ~cpu_to_le32p(crcp); 120 121 return 0; 122 } 123 124 static int crc32c_vpmsum_digest(struct shash_desc *desc, const u8 *data, 125 unsigned int len, u8 *out) 126 { 127 return __crc32c_vpmsum_finup(crypto_shash_ctx(desc->tfm), data, len, 128 out); 129 } 130 131 static struct shash_alg alg = { 132 .setkey = crc32c_vpmsum_setkey, 133 .init = crc32c_vpmsum_init, 134 .update = crc32c_vpmsum_update, 135 .final = crc32c_vpmsum_final, 136 .finup = crc32c_vpmsum_finup, 137 .digest = crc32c_vpmsum_digest, 138 .descsize = sizeof(u32), 139 .digestsize = CHKSUM_DIGEST_SIZE, 140 .base = { 141 .cra_name = "crc32c", 142 .cra_driver_name = "crc32c-vpmsum", 143 .cra_priority = 200, 144 .cra_blocksize = CHKSUM_BLOCK_SIZE, 145 .cra_ctxsize = sizeof(u32), 146 .cra_module = THIS_MODULE, 147 .cra_init = crc32c_vpmsum_cra_init, 148 } 149 }; 150 151 static int __init crc32c_vpmsum_mod_init(void) 152 { 153 if (!cpu_has_feature(CPU_FTR_ARCH_207S)) 154 return -ENODEV; 155 156 return crypto_register_shash(&alg); 157 } 158 159 static void __exit crc32c_vpmsum_mod_fini(void) 160 { 161 crypto_unregister_shash(&alg); 162 } 163 164 module_cpu_feature_match(PPC_MODULE_FEATURE_VEC_CRYPTO, crc32c_vpmsum_mod_init); 165 module_exit(crc32c_vpmsum_mod_fini); 166 167 MODULE_AUTHOR("Anton Blanchard <anton@samba.org>"); 168 MODULE_DESCRIPTION("CRC32C using vector polynomial multiply-sum instructions"); 169 MODULE_LICENSE("GPL"); 170 MODULE_ALIAS_CRYPTO("crc32c"); 171 MODULE_ALIAS_CRYPTO("crc32c-vpmsum"); 172