1 /* 2 * Cryptographic API. 3 * 4 * Glue code for the SHA256 Secure Hash Algorithm assembler 5 * implementation using supplemental SSE3 / AVX / AVX2 instructions. 6 * 7 * This file is based on sha256_generic.c 8 * 9 * Copyright (C) 2013 Intel Corporation. 10 * 11 * Author: 12 * Tim Chen <tim.c.chen@linux.intel.com> 13 * 14 * This program is free software; you can redistribute it and/or modify it 15 * under the terms of the GNU General Public License as published by the Free 16 * Software Foundation; either version 2 of the License, or (at your option) 17 * any later version. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 23 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 24 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 25 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 * SOFTWARE. 27 */ 28 29 30 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 31 32 #include <crypto/internal/hash.h> 33 #include <linux/init.h> 34 #include <linux/module.h> 35 #include <linux/mm.h> 36 #include <linux/cryptohash.h> 37 #include <linux/types.h> 38 #include <crypto/sha.h> 39 #include <crypto/sha256_base.h> 40 #include <asm/fpu/api.h> 41 #include <linux/string.h> 42 43 asmlinkage void sha256_transform_ssse3(u32 *digest, const char *data, 44 u64 rounds); 45 #ifdef CONFIG_AS_AVX 46 asmlinkage void sha256_transform_avx(u32 *digest, const char *data, 47 u64 rounds); 48 #endif 49 #ifdef CONFIG_AS_AVX2 50 asmlinkage void sha256_transform_rorx(u32 *digest, const char *data, 51 u64 rounds); 52 #endif 53 54 static void (*sha256_transform_asm)(u32 *, const char *, u64); 55 56 static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data, 57 unsigned int len) 58 { 59 struct sha256_state *sctx = shash_desc_ctx(desc); 60 61 if (!irq_fpu_usable() || 62 (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) 63 return crypto_sha256_update(desc, data, len); 64 65 /* make sure casting to sha256_block_fn() is safe */ 66 BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); 67 68 kernel_fpu_begin(); 69 sha256_base_do_update(desc, data, len, 70 (sha256_block_fn *)sha256_transform_asm); 71 kernel_fpu_end(); 72 73 return 0; 74 } 75 76 static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, 77 unsigned int len, u8 *out) 78 { 79 if (!irq_fpu_usable()) 80 return crypto_sha256_finup(desc, data, len, out); 81 82 kernel_fpu_begin(); 83 if (len) 84 sha256_base_do_update(desc, data, len, 85 (sha256_block_fn *)sha256_transform_asm); 86 sha256_base_do_finalize(desc, (sha256_block_fn *)sha256_transform_asm); 87 kernel_fpu_end(); 88 89 return sha256_base_finish(desc, out); 90 } 91 92 /* Add padding and return the message digest. */ 93 static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) 94 { 95 return sha256_ssse3_finup(desc, NULL, 0, out); 96 } 97 98 static struct shash_alg algs[] = { { 99 .digestsize = SHA256_DIGEST_SIZE, 100 .init = sha256_base_init, 101 .update = sha256_ssse3_update, 102 .final = sha256_ssse3_final, 103 .finup = sha256_ssse3_finup, 104 .descsize = sizeof(struct sha256_state), 105 .base = { 106 .cra_name = "sha256", 107 .cra_driver_name = "sha256-ssse3", 108 .cra_priority = 150, 109 .cra_flags = CRYPTO_ALG_TYPE_SHASH, 110 .cra_blocksize = SHA256_BLOCK_SIZE, 111 .cra_module = THIS_MODULE, 112 } 113 }, { 114 .digestsize = SHA224_DIGEST_SIZE, 115 .init = sha224_base_init, 116 .update = sha256_ssse3_update, 117 .final = sha256_ssse3_final, 118 .finup = sha256_ssse3_finup, 119 .descsize = sizeof(struct sha256_state), 120 .base = { 121 .cra_name = "sha224", 122 .cra_driver_name = "sha224-ssse3", 123 .cra_priority = 150, 124 .cra_flags = CRYPTO_ALG_TYPE_SHASH, 125 .cra_blocksize = SHA224_BLOCK_SIZE, 126 .cra_module = THIS_MODULE, 127 } 128 } }; 129 130 #ifdef CONFIG_AS_AVX 131 static bool __init avx_usable(void) 132 { 133 if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL)) { 134 if (cpu_has_avx) 135 pr_info("AVX detected but unusable.\n"); 136 return false; 137 } 138 139 return true; 140 } 141 #endif 142 143 static int __init sha256_ssse3_mod_init(void) 144 { 145 /* test for SSSE3 first */ 146 if (cpu_has_ssse3) 147 sha256_transform_asm = sha256_transform_ssse3; 148 149 #ifdef CONFIG_AS_AVX 150 /* allow AVX to override SSSE3, it's a little faster */ 151 if (avx_usable()) { 152 #ifdef CONFIG_AS_AVX2 153 if (boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_BMI2)) 154 sha256_transform_asm = sha256_transform_rorx; 155 else 156 #endif 157 sha256_transform_asm = sha256_transform_avx; 158 } 159 #endif 160 161 if (sha256_transform_asm) { 162 #ifdef CONFIG_AS_AVX 163 if (sha256_transform_asm == sha256_transform_avx) 164 pr_info("Using AVX optimized SHA-256 implementation\n"); 165 #ifdef CONFIG_AS_AVX2 166 else if (sha256_transform_asm == sha256_transform_rorx) 167 pr_info("Using AVX2 optimized SHA-256 implementation\n"); 168 #endif 169 else 170 #endif 171 pr_info("Using SSSE3 optimized SHA-256 implementation\n"); 172 return crypto_register_shashes(algs, ARRAY_SIZE(algs)); 173 } 174 pr_info("Neither AVX nor SSSE3 is available/usable.\n"); 175 176 return -ENODEV; 177 } 178 179 static void __exit sha256_ssse3_mod_fini(void) 180 { 181 crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 182 } 183 184 module_init(sha256_ssse3_mod_init); 185 module_exit(sha256_ssse3_mod_fini); 186 187 MODULE_LICENSE("GPL"); 188 MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated"); 189 190 MODULE_ALIAS_CRYPTO("sha256"); 191 MODULE_ALIAS_CRYPTO("sha224"); 192