1 /* 2 * Cryptographic API. 3 * 4 * Glue code for the SHA256 Secure Hash Algorithm assembler 5 * implementation using supplemental SSE3 / AVX / AVX2 instructions. 6 * 7 * This file is based on sha256_generic.c 8 * 9 * Copyright (C) 2013 Intel Corporation. 10 * 11 * Author: 12 * Tim Chen <tim.c.chen@linux.intel.com> 13 * 14 * This program is free software; you can redistribute it and/or modify it 15 * under the terms of the GNU General Public License as published by the Free 16 * Software Foundation; either version 2 of the License, or (at your option) 17 * any later version. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 23 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 24 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 25 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 * SOFTWARE. 27 */ 28 29 30 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 31 32 #include <crypto/internal/hash.h> 33 #include <crypto/internal/simd.h> 34 #include <linux/init.h> 35 #include <linux/module.h> 36 #include <linux/mm.h> 37 #include <linux/types.h> 38 #include <crypto/sha2.h> 39 #include <crypto/sha256_base.h> 40 #include <linux/string.h> 41 #include <asm/cpu_device_id.h> 42 #include <asm/simd.h> 43 44 asmlinkage void sha256_transform_ssse3(struct sha256_state *state, 45 const u8 *data, int blocks); 46 47 static const struct x86_cpu_id module_cpu_ids[] = { 48 X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), 49 X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), 50 X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), 51 {} 52 }; 53 MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); 54 55 static int _sha256_update(struct shash_desc *desc, const u8 *data, 56 unsigned int len, sha256_block_fn *sha256_xform) 57 { 58 struct sha256_state *sctx = shash_desc_ctx(desc); 59 60 if (!crypto_simd_usable() || 61 (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) 62 return crypto_sha256_update(desc, data, len); 63 64 /* 65 * Make sure struct sha256_state begins directly with the SHA256 66 * 256-bit internal state, as this is what the asm functions expect. 67 */ 68 BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); 69 70 kernel_fpu_begin(); 71 sha256_base_do_update(desc, data, len, sha256_xform); 72 kernel_fpu_end(); 73 74 return 0; 75 } 76 77 static int sha256_finup(struct shash_desc *desc, const u8 *data, 78 unsigned int len, u8 *out, sha256_block_fn *sha256_xform) 79 { 80 if (!crypto_simd_usable()) 81 return crypto_sha256_finup(desc, data, len, out); 82 83 kernel_fpu_begin(); 84 if (len) 85 sha256_base_do_update(desc, data, len, sha256_xform); 86 sha256_base_do_finalize(desc, sha256_xform); 87 kernel_fpu_end(); 88 89 return sha256_base_finish(desc, out); 90 } 91 92 static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data, 93 unsigned int len) 94 { 95 return _sha256_update(desc, data, len, sha256_transform_ssse3); 96 } 97 98 static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, 99 unsigned int len, u8 *out) 100 { 101 return sha256_finup(desc, data, len, out, sha256_transform_ssse3); 102 } 103 104 /* Add padding and return the message digest. */ 105 static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) 106 { 107 return sha256_ssse3_finup(desc, NULL, 0, out); 108 } 109 110 static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data, 111 unsigned int len, u8 *out) 112 { 113 return sha256_base_init(desc) ?: 114 sha256_ssse3_finup(desc, data, len, out); 115 } 116 117 static struct shash_alg sha256_ssse3_algs[] = { { 118 .digestsize = SHA256_DIGEST_SIZE, 119 .init = sha256_base_init, 120 .update = sha256_ssse3_update, 121 .final = sha256_ssse3_final, 122 .finup = sha256_ssse3_finup, 123 .digest = sha256_ssse3_digest, 124 .descsize = sizeof(struct sha256_state), 125 .base = { 126 .cra_name = "sha256", 127 .cra_driver_name = "sha256-ssse3", 128 .cra_priority = 150, 129 .cra_blocksize = SHA256_BLOCK_SIZE, 130 .cra_module = THIS_MODULE, 131 } 132 }, { 133 .digestsize = SHA224_DIGEST_SIZE, 134 .init = sha224_base_init, 135 .update = sha256_ssse3_update, 136 .final = sha256_ssse3_final, 137 .finup = sha256_ssse3_finup, 138 .descsize = sizeof(struct sha256_state), 139 .base = { 140 .cra_name = "sha224", 141 .cra_driver_name = "sha224-ssse3", 142 .cra_priority = 150, 143 .cra_blocksize = SHA224_BLOCK_SIZE, 144 .cra_module = THIS_MODULE, 145 } 146 } }; 147 148 static int register_sha256_ssse3(void) 149 { 150 if (boot_cpu_has(X86_FEATURE_SSSE3)) 151 return crypto_register_shashes(sha256_ssse3_algs, 152 ARRAY_SIZE(sha256_ssse3_algs)); 153 return 0; 154 } 155 156 static void unregister_sha256_ssse3(void) 157 { 158 if (boot_cpu_has(X86_FEATURE_SSSE3)) 159 crypto_unregister_shashes(sha256_ssse3_algs, 160 ARRAY_SIZE(sha256_ssse3_algs)); 161 } 162 163 asmlinkage void sha256_transform_avx(struct sha256_state *state, 164 const u8 *data, int blocks); 165 166 static int sha256_avx_update(struct shash_desc *desc, const u8 *data, 167 unsigned int len) 168 { 169 return _sha256_update(desc, data, len, sha256_transform_avx); 170 } 171 172 static int sha256_avx_finup(struct shash_desc *desc, const u8 *data, 173 unsigned int len, u8 *out) 174 { 175 return sha256_finup(desc, data, len, out, sha256_transform_avx); 176 } 177 178 static int sha256_avx_final(struct shash_desc *desc, u8 *out) 179 { 180 return sha256_avx_finup(desc, NULL, 0, out); 181 } 182 183 static int sha256_avx_digest(struct shash_desc *desc, const u8 *data, 184 unsigned int len, u8 *out) 185 { 186 return sha256_base_init(desc) ?: 187 sha256_avx_finup(desc, data, len, out); 188 } 189 190 static struct shash_alg sha256_avx_algs[] = { { 191 .digestsize = SHA256_DIGEST_SIZE, 192 .init = sha256_base_init, 193 .update = sha256_avx_update, 194 .final = sha256_avx_final, 195 .finup = sha256_avx_finup, 196 .digest = sha256_avx_digest, 197 .descsize = sizeof(struct sha256_state), 198 .base = { 199 .cra_name = "sha256", 200 .cra_driver_name = "sha256-avx", 201 .cra_priority = 160, 202 .cra_blocksize = SHA256_BLOCK_SIZE, 203 .cra_module = THIS_MODULE, 204 } 205 }, { 206 .digestsize = SHA224_DIGEST_SIZE, 207 .init = sha224_base_init, 208 .update = sha256_avx_update, 209 .final = sha256_avx_final, 210 .finup = sha256_avx_finup, 211 .descsize = sizeof(struct sha256_state), 212 .base = { 213 .cra_name = "sha224", 214 .cra_driver_name = "sha224-avx", 215 .cra_priority = 160, 216 .cra_blocksize = SHA224_BLOCK_SIZE, 217 .cra_module = THIS_MODULE, 218 } 219 } }; 220 221 static bool avx_usable(void) 222 { 223 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { 224 if (boot_cpu_has(X86_FEATURE_AVX)) 225 pr_info("AVX detected but unusable.\n"); 226 return false; 227 } 228 229 return true; 230 } 231 232 static int register_sha256_avx(void) 233 { 234 if (avx_usable()) 235 return crypto_register_shashes(sha256_avx_algs, 236 ARRAY_SIZE(sha256_avx_algs)); 237 return 0; 238 } 239 240 static void unregister_sha256_avx(void) 241 { 242 if (avx_usable()) 243 crypto_unregister_shashes(sha256_avx_algs, 244 ARRAY_SIZE(sha256_avx_algs)); 245 } 246 247 asmlinkage void sha256_transform_rorx(struct sha256_state *state, 248 const u8 *data, int blocks); 249 250 static int sha256_avx2_update(struct shash_desc *desc, const u8 *data, 251 unsigned int len) 252 { 253 return _sha256_update(desc, data, len, sha256_transform_rorx); 254 } 255 256 static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data, 257 unsigned int len, u8 *out) 258 { 259 return sha256_finup(desc, data, len, out, sha256_transform_rorx); 260 } 261 262 static int sha256_avx2_final(struct shash_desc *desc, u8 *out) 263 { 264 return sha256_avx2_finup(desc, NULL, 0, out); 265 } 266 267 static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data, 268 unsigned int len, u8 *out) 269 { 270 return sha256_base_init(desc) ?: 271 sha256_avx2_finup(desc, data, len, out); 272 } 273 274 static struct shash_alg sha256_avx2_algs[] = { { 275 .digestsize = SHA256_DIGEST_SIZE, 276 .init = sha256_base_init, 277 .update = sha256_avx2_update, 278 .final = sha256_avx2_final, 279 .finup = sha256_avx2_finup, 280 .digest = sha256_avx2_digest, 281 .descsize = sizeof(struct sha256_state), 282 .base = { 283 .cra_name = "sha256", 284 .cra_driver_name = "sha256-avx2", 285 .cra_priority = 170, 286 .cra_blocksize = SHA256_BLOCK_SIZE, 287 .cra_module = THIS_MODULE, 288 } 289 }, { 290 .digestsize = SHA224_DIGEST_SIZE, 291 .init = sha224_base_init, 292 .update = sha256_avx2_update, 293 .final = sha256_avx2_final, 294 .finup = sha256_avx2_finup, 295 .descsize = sizeof(struct sha256_state), 296 .base = { 297 .cra_name = "sha224", 298 .cra_driver_name = "sha224-avx2", 299 .cra_priority = 170, 300 .cra_blocksize = SHA224_BLOCK_SIZE, 301 .cra_module = THIS_MODULE, 302 } 303 } }; 304 305 static bool avx2_usable(void) 306 { 307 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && 308 boot_cpu_has(X86_FEATURE_BMI2)) 309 return true; 310 311 return false; 312 } 313 314 static int register_sha256_avx2(void) 315 { 316 if (avx2_usable()) 317 return crypto_register_shashes(sha256_avx2_algs, 318 ARRAY_SIZE(sha256_avx2_algs)); 319 return 0; 320 } 321 322 static void unregister_sha256_avx2(void) 323 { 324 if (avx2_usable()) 325 crypto_unregister_shashes(sha256_avx2_algs, 326 ARRAY_SIZE(sha256_avx2_algs)); 327 } 328 329 #ifdef CONFIG_AS_SHA256_NI 330 asmlinkage void sha256_ni_transform(struct sha256_state *digest, 331 const u8 *data, int rounds); 332 333 static int sha256_ni_update(struct shash_desc *desc, const u8 *data, 334 unsigned int len) 335 { 336 return _sha256_update(desc, data, len, sha256_ni_transform); 337 } 338 339 static int sha256_ni_finup(struct shash_desc *desc, const u8 *data, 340 unsigned int len, u8 *out) 341 { 342 return sha256_finup(desc, data, len, out, sha256_ni_transform); 343 } 344 345 static int sha256_ni_final(struct shash_desc *desc, u8 *out) 346 { 347 return sha256_ni_finup(desc, NULL, 0, out); 348 } 349 350 static int sha256_ni_digest(struct shash_desc *desc, const u8 *data, 351 unsigned int len, u8 *out) 352 { 353 return sha256_base_init(desc) ?: 354 sha256_ni_finup(desc, data, len, out); 355 } 356 357 static struct shash_alg sha256_ni_algs[] = { { 358 .digestsize = SHA256_DIGEST_SIZE, 359 .init = sha256_base_init, 360 .update = sha256_ni_update, 361 .final = sha256_ni_final, 362 .finup = sha256_ni_finup, 363 .digest = sha256_ni_digest, 364 .descsize = sizeof(struct sha256_state), 365 .base = { 366 .cra_name = "sha256", 367 .cra_driver_name = "sha256-ni", 368 .cra_priority = 250, 369 .cra_blocksize = SHA256_BLOCK_SIZE, 370 .cra_module = THIS_MODULE, 371 } 372 }, { 373 .digestsize = SHA224_DIGEST_SIZE, 374 .init = sha224_base_init, 375 .update = sha256_ni_update, 376 .final = sha256_ni_final, 377 .finup = sha256_ni_finup, 378 .descsize = sizeof(struct sha256_state), 379 .base = { 380 .cra_name = "sha224", 381 .cra_driver_name = "sha224-ni", 382 .cra_priority = 250, 383 .cra_blocksize = SHA224_BLOCK_SIZE, 384 .cra_module = THIS_MODULE, 385 } 386 } }; 387 388 static int register_sha256_ni(void) 389 { 390 if (boot_cpu_has(X86_FEATURE_SHA_NI)) 391 return crypto_register_shashes(sha256_ni_algs, 392 ARRAY_SIZE(sha256_ni_algs)); 393 return 0; 394 } 395 396 static void unregister_sha256_ni(void) 397 { 398 if (boot_cpu_has(X86_FEATURE_SHA_NI)) 399 crypto_unregister_shashes(sha256_ni_algs, 400 ARRAY_SIZE(sha256_ni_algs)); 401 } 402 403 #else 404 static inline int register_sha256_ni(void) { return 0; } 405 static inline void unregister_sha256_ni(void) { } 406 #endif 407 408 static int __init sha256_ssse3_mod_init(void) 409 { 410 if (!x86_match_cpu(module_cpu_ids)) 411 return -ENODEV; 412 413 if (register_sha256_ssse3()) 414 goto fail; 415 416 if (register_sha256_avx()) { 417 unregister_sha256_ssse3(); 418 goto fail; 419 } 420 421 if (register_sha256_avx2()) { 422 unregister_sha256_avx(); 423 unregister_sha256_ssse3(); 424 goto fail; 425 } 426 427 if (register_sha256_ni()) { 428 unregister_sha256_avx2(); 429 unregister_sha256_avx(); 430 unregister_sha256_ssse3(); 431 goto fail; 432 } 433 434 return 0; 435 fail: 436 return -ENODEV; 437 } 438 439 static void __exit sha256_ssse3_mod_fini(void) 440 { 441 unregister_sha256_ni(); 442 unregister_sha256_avx2(); 443 unregister_sha256_avx(); 444 unregister_sha256_ssse3(); 445 } 446 447 module_init(sha256_ssse3_mod_init); 448 module_exit(sha256_ssse3_mod_fini); 449 450 MODULE_LICENSE("GPL"); 451 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated"); 452 453 MODULE_ALIAS_CRYPTO("sha256"); 454 MODULE_ALIAS_CRYPTO("sha256-ssse3"); 455 MODULE_ALIAS_CRYPTO("sha256-avx"); 456 MODULE_ALIAS_CRYPTO("sha256-avx2"); 457 MODULE_ALIAS_CRYPTO("sha224"); 458 MODULE_ALIAS_CRYPTO("sha224-ssse3"); 459 MODULE_ALIAS_CRYPTO("sha224-avx"); 460 MODULE_ALIAS_CRYPTO("sha224-avx2"); 461 #ifdef CONFIG_AS_SHA256_NI 462 MODULE_ALIAS_CRYPTO("sha256-ni"); 463 MODULE_ALIAS_CRYPTO("sha224-ni"); 464 #endif 465