1 /* 2 * Cryptographic API. 3 * 4 * Glue code for the SHA512 Secure Hash Algorithm assembler 5 * implementation using supplemental SSE3 / AVX / AVX2 instructions. 6 * 7 * This file is based on sha512_generic.c 8 * 9 * Copyright (C) 2013 Intel Corporation 10 * Author: Tim Chen <tim.c.chen@linux.intel.com> 11 * 12 * This program is free software; you can redistribute it and/or modify it 13 * under the terms of the GNU General Public License as published by the Free 14 * Software Foundation; either version 2 of the License, or (at your option) 15 * any later version. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 21 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 23 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 * SOFTWARE. 25 * 26 */ 27 28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30 #include <crypto/internal/hash.h> 31 #include <crypto/internal/simd.h> 32 #include <linux/init.h> 33 #include <linux/module.h> 34 #include <linux/mm.h> 35 #include <linux/string.h> 36 #include <linux/types.h> 37 #include <crypto/sha2.h> 38 #include <crypto/sha512_base.h> 39 #include <asm/cpu_device_id.h> 40 #include <asm/simd.h> 41 42 asmlinkage void sha512_transform_ssse3(struct sha512_state *state, 43 const u8 *data, int blocks); 44 45 static int sha512_update(struct shash_desc *desc, const u8 *data, 46 unsigned int len, sha512_block_fn *sha512_xform) 47 { 48 struct sha512_state *sctx = shash_desc_ctx(desc); 49 50 if (!crypto_simd_usable() || 51 (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) 52 return crypto_sha512_update(desc, data, len); 53 54 /* 55 * Make sure struct sha512_state begins directly with the SHA512 56 * 512-bit internal state, as this is what the asm functions expect. 57 */ 58 BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); 59 60 kernel_fpu_begin(); 61 sha512_base_do_update(desc, data, len, sha512_xform); 62 kernel_fpu_end(); 63 64 return 0; 65 } 66 67 static int sha512_finup(struct shash_desc *desc, const u8 *data, 68 unsigned int len, u8 *out, sha512_block_fn *sha512_xform) 69 { 70 if (!crypto_simd_usable()) 71 return crypto_sha512_finup(desc, data, len, out); 72 73 kernel_fpu_begin(); 74 if (len) 75 sha512_base_do_update(desc, data, len, sha512_xform); 76 sha512_base_do_finalize(desc, sha512_xform); 77 kernel_fpu_end(); 78 79 return sha512_base_finish(desc, out); 80 } 81 82 static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, 83 unsigned int len) 84 { 85 return sha512_update(desc, data, len, sha512_transform_ssse3); 86 } 87 88 static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, 89 unsigned int len, u8 *out) 90 { 91 return sha512_finup(desc, data, len, out, sha512_transform_ssse3); 92 } 93 94 /* Add padding and return the message digest. */ 95 static int sha512_ssse3_final(struct shash_desc *desc, u8 *out) 96 { 97 return sha512_ssse3_finup(desc, NULL, 0, out); 98 } 99 100 static struct shash_alg sha512_ssse3_algs[] = { { 101 .digestsize = SHA512_DIGEST_SIZE, 102 .init = sha512_base_init, 103 .update = sha512_ssse3_update, 104 .final = sha512_ssse3_final, 105 .finup = sha512_ssse3_finup, 106 .descsize = sizeof(struct sha512_state), 107 .base = { 108 .cra_name = "sha512", 109 .cra_driver_name = "sha512-ssse3", 110 .cra_priority = 150, 111 .cra_blocksize = SHA512_BLOCK_SIZE, 112 .cra_module = THIS_MODULE, 113 } 114 }, { 115 .digestsize = SHA384_DIGEST_SIZE, 116 .init = sha384_base_init, 117 .update = sha512_ssse3_update, 118 .final = sha512_ssse3_final, 119 .finup = sha512_ssse3_finup, 120 .descsize = sizeof(struct sha512_state), 121 .base = { 122 .cra_name = "sha384", 123 .cra_driver_name = "sha384-ssse3", 124 .cra_priority = 150, 125 .cra_blocksize = SHA384_BLOCK_SIZE, 126 .cra_module = THIS_MODULE, 127 } 128 } }; 129 130 static int register_sha512_ssse3(void) 131 { 132 if (boot_cpu_has(X86_FEATURE_SSSE3)) 133 return crypto_register_shashes(sha512_ssse3_algs, 134 ARRAY_SIZE(sha512_ssse3_algs)); 135 return 0; 136 } 137 138 static void unregister_sha512_ssse3(void) 139 { 140 if (boot_cpu_has(X86_FEATURE_SSSE3)) 141 crypto_unregister_shashes(sha512_ssse3_algs, 142 ARRAY_SIZE(sha512_ssse3_algs)); 143 } 144 145 asmlinkage void sha512_transform_avx(struct sha512_state *state, 146 const u8 *data, int blocks); 147 static bool avx_usable(void) 148 { 149 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { 150 if (boot_cpu_has(X86_FEATURE_AVX)) 151 pr_info("AVX detected but unusable.\n"); 152 return false; 153 } 154 155 return true; 156 } 157 158 static int sha512_avx_update(struct shash_desc *desc, const u8 *data, 159 unsigned int len) 160 { 161 return sha512_update(desc, data, len, sha512_transform_avx); 162 } 163 164 static int sha512_avx_finup(struct shash_desc *desc, const u8 *data, 165 unsigned int len, u8 *out) 166 { 167 return sha512_finup(desc, data, len, out, sha512_transform_avx); 168 } 169 170 /* Add padding and return the message digest. */ 171 static int sha512_avx_final(struct shash_desc *desc, u8 *out) 172 { 173 return sha512_avx_finup(desc, NULL, 0, out); 174 } 175 176 static struct shash_alg sha512_avx_algs[] = { { 177 .digestsize = SHA512_DIGEST_SIZE, 178 .init = sha512_base_init, 179 .update = sha512_avx_update, 180 .final = sha512_avx_final, 181 .finup = sha512_avx_finup, 182 .descsize = sizeof(struct sha512_state), 183 .base = { 184 .cra_name = "sha512", 185 .cra_driver_name = "sha512-avx", 186 .cra_priority = 160, 187 .cra_blocksize = SHA512_BLOCK_SIZE, 188 .cra_module = THIS_MODULE, 189 } 190 }, { 191 .digestsize = SHA384_DIGEST_SIZE, 192 .init = sha384_base_init, 193 .update = sha512_avx_update, 194 .final = sha512_avx_final, 195 .finup = sha512_avx_finup, 196 .descsize = sizeof(struct sha512_state), 197 .base = { 198 .cra_name = "sha384", 199 .cra_driver_name = "sha384-avx", 200 .cra_priority = 160, 201 .cra_blocksize = SHA384_BLOCK_SIZE, 202 .cra_module = THIS_MODULE, 203 } 204 } }; 205 206 static int register_sha512_avx(void) 207 { 208 if (avx_usable()) 209 return crypto_register_shashes(sha512_avx_algs, 210 ARRAY_SIZE(sha512_avx_algs)); 211 return 0; 212 } 213 214 static void unregister_sha512_avx(void) 215 { 216 if (avx_usable()) 217 crypto_unregister_shashes(sha512_avx_algs, 218 ARRAY_SIZE(sha512_avx_algs)); 219 } 220 221 asmlinkage void sha512_transform_rorx(struct sha512_state *state, 222 const u8 *data, int blocks); 223 224 static int sha512_avx2_update(struct shash_desc *desc, const u8 *data, 225 unsigned int len) 226 { 227 return sha512_update(desc, data, len, sha512_transform_rorx); 228 } 229 230 static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data, 231 unsigned int len, u8 *out) 232 { 233 return sha512_finup(desc, data, len, out, sha512_transform_rorx); 234 } 235 236 /* Add padding and return the message digest. */ 237 static int sha512_avx2_final(struct shash_desc *desc, u8 *out) 238 { 239 return sha512_avx2_finup(desc, NULL, 0, out); 240 } 241 242 static struct shash_alg sha512_avx2_algs[] = { { 243 .digestsize = SHA512_DIGEST_SIZE, 244 .init = sha512_base_init, 245 .update = sha512_avx2_update, 246 .final = sha512_avx2_final, 247 .finup = sha512_avx2_finup, 248 .descsize = sizeof(struct sha512_state), 249 .base = { 250 .cra_name = "sha512", 251 .cra_driver_name = "sha512-avx2", 252 .cra_priority = 170, 253 .cra_blocksize = SHA512_BLOCK_SIZE, 254 .cra_module = THIS_MODULE, 255 } 256 }, { 257 .digestsize = SHA384_DIGEST_SIZE, 258 .init = sha384_base_init, 259 .update = sha512_avx2_update, 260 .final = sha512_avx2_final, 261 .finup = sha512_avx2_finup, 262 .descsize = sizeof(struct sha512_state), 263 .base = { 264 .cra_name = "sha384", 265 .cra_driver_name = "sha384-avx2", 266 .cra_priority = 170, 267 .cra_blocksize = SHA384_BLOCK_SIZE, 268 .cra_module = THIS_MODULE, 269 } 270 } }; 271 272 static bool avx2_usable(void) 273 { 274 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && 275 boot_cpu_has(X86_FEATURE_BMI2)) 276 return true; 277 278 return false; 279 } 280 281 static int register_sha512_avx2(void) 282 { 283 if (avx2_usable()) 284 return crypto_register_shashes(sha512_avx2_algs, 285 ARRAY_SIZE(sha512_avx2_algs)); 286 return 0; 287 } 288 static const struct x86_cpu_id module_cpu_ids[] = { 289 X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), 290 X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), 291 X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), 292 {} 293 }; 294 MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); 295 296 static void unregister_sha512_avx2(void) 297 { 298 if (avx2_usable()) 299 crypto_unregister_shashes(sha512_avx2_algs, 300 ARRAY_SIZE(sha512_avx2_algs)); 301 } 302 303 static int __init sha512_ssse3_mod_init(void) 304 { 305 if (!x86_match_cpu(module_cpu_ids)) 306 return -ENODEV; 307 308 if (register_sha512_ssse3()) 309 goto fail; 310 311 if (register_sha512_avx()) { 312 unregister_sha512_ssse3(); 313 goto fail; 314 } 315 316 if (register_sha512_avx2()) { 317 unregister_sha512_avx(); 318 unregister_sha512_ssse3(); 319 goto fail; 320 } 321 322 return 0; 323 fail: 324 return -ENODEV; 325 } 326 327 static void __exit sha512_ssse3_mod_fini(void) 328 { 329 unregister_sha512_avx2(); 330 unregister_sha512_avx(); 331 unregister_sha512_ssse3(); 332 } 333 334 module_init(sha512_ssse3_mod_init); 335 module_exit(sha512_ssse3_mod_fini); 336 337 MODULE_LICENSE("GPL"); 338 MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated"); 339 340 MODULE_ALIAS_CRYPTO("sha512"); 341 MODULE_ALIAS_CRYPTO("sha512-ssse3"); 342 MODULE_ALIAS_CRYPTO("sha512-avx"); 343 MODULE_ALIAS_CRYPTO("sha512-avx2"); 344 MODULE_ALIAS_CRYPTO("sha384"); 345 MODULE_ALIAS_CRYPTO("sha384-ssse3"); 346 MODULE_ALIAS_CRYPTO("sha384-avx"); 347 MODULE_ALIAS_CRYPTO("sha384-avx2"); 348