1 /* 2 * Cryptographic API. 3 * 4 * Glue code for the SHA512 Secure Hash Algorithm assembler 5 * implementation using supplemental SSE3 / AVX / AVX2 instructions. 6 * 7 * This file is based on sha512_generic.c 8 * 9 * Copyright (C) 2013 Intel Corporation 10 * Author: Tim Chen <tim.c.chen@linux.intel.com> 11 * 12 * This program is free software; you can redistribute it and/or modify it 13 * under the terms of the GNU General Public License as published by the Free 14 * Software Foundation; either version 2 of the License, or (at your option) 15 * any later version. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 21 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 23 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 * SOFTWARE. 25 * 26 */ 27 28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30 #include <crypto/internal/hash.h> 31 #include <linux/init.h> 32 #include <linux/module.h> 33 #include <linux/mm.h> 34 #include <linux/cryptohash.h> 35 #include <linux/types.h> 36 #include <crypto/sha.h> 37 #include <crypto/sha512_base.h> 38 #include <asm/fpu/api.h> 39 40 #include <linux/string.h> 41 42 asmlinkage void sha512_transform_ssse3(u64 *digest, const char *data, 43 u64 rounds); 44 #ifdef CONFIG_AS_AVX 45 asmlinkage void sha512_transform_avx(u64 *digest, const char *data, 46 u64 rounds); 47 #endif 48 #ifdef CONFIG_AS_AVX2 49 asmlinkage void sha512_transform_rorx(u64 *digest, const char *data, 50 u64 rounds); 51 #endif 52 53 static void (*sha512_transform_asm)(u64 *, const char *, u64); 54 55 static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, 56 unsigned int len) 57 { 58 struct sha512_state *sctx = shash_desc_ctx(desc); 59 60 if (!irq_fpu_usable() || 61 (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) 62 return crypto_sha512_update(desc, data, len); 63 64 /* make sure casting to sha512_block_fn() is safe */ 65 BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); 66 67 kernel_fpu_begin(); 68 sha512_base_do_update(desc, data, len, 69 (sha512_block_fn *)sha512_transform_asm); 70 kernel_fpu_end(); 71 72 return 0; 73 } 74 75 static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, 76 unsigned int len, u8 *out) 77 { 78 if (!irq_fpu_usable()) 79 return crypto_sha512_finup(desc, data, len, out); 80 81 kernel_fpu_begin(); 82 if (len) 83 sha512_base_do_update(desc, data, len, 84 (sha512_block_fn *)sha512_transform_asm); 85 sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_transform_asm); 86 kernel_fpu_end(); 87 88 return sha512_base_finish(desc, out); 89 } 90 91 /* Add padding and return the message digest. */ 92 static int sha512_ssse3_final(struct shash_desc *desc, u8 *out) 93 { 94 return sha512_ssse3_finup(desc, NULL, 0, out); 95 } 96 97 static struct shash_alg algs[] = { { 98 .digestsize = SHA512_DIGEST_SIZE, 99 .init = sha512_base_init, 100 .update = sha512_ssse3_update, 101 .final = sha512_ssse3_final, 102 .finup = sha512_ssse3_finup, 103 .descsize = sizeof(struct sha512_state), 104 .base = { 105 .cra_name = "sha512", 106 .cra_driver_name = "sha512-ssse3", 107 .cra_priority = 150, 108 .cra_flags = CRYPTO_ALG_TYPE_SHASH, 109 .cra_blocksize = SHA512_BLOCK_SIZE, 110 .cra_module = THIS_MODULE, 111 } 112 }, { 113 .digestsize = SHA384_DIGEST_SIZE, 114 .init = sha384_base_init, 115 .update = sha512_ssse3_update, 116 .final = sha512_ssse3_final, 117 .finup = sha512_ssse3_finup, 118 .descsize = sizeof(struct sha512_state), 119 .base = { 120 .cra_name = "sha384", 121 .cra_driver_name = "sha384-ssse3", 122 .cra_priority = 150, 123 .cra_flags = CRYPTO_ALG_TYPE_SHASH, 124 .cra_blocksize = SHA384_BLOCK_SIZE, 125 .cra_module = THIS_MODULE, 126 } 127 } }; 128 129 #ifdef CONFIG_AS_AVX 130 static bool __init avx_usable(void) 131 { 132 if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL)) { 133 if (cpu_has_avx) 134 pr_info("AVX detected but unusable.\n"); 135 return false; 136 } 137 138 return true; 139 } 140 #endif 141 142 static int __init sha512_ssse3_mod_init(void) 143 { 144 /* test for SSSE3 first */ 145 if (cpu_has_ssse3) 146 sha512_transform_asm = sha512_transform_ssse3; 147 148 #ifdef CONFIG_AS_AVX 149 /* allow AVX to override SSSE3, it's a little faster */ 150 if (avx_usable()) { 151 #ifdef CONFIG_AS_AVX2 152 if (boot_cpu_has(X86_FEATURE_AVX2)) 153 sha512_transform_asm = sha512_transform_rorx; 154 else 155 #endif 156 sha512_transform_asm = sha512_transform_avx; 157 } 158 #endif 159 160 if (sha512_transform_asm) { 161 #ifdef CONFIG_AS_AVX 162 if (sha512_transform_asm == sha512_transform_avx) 163 pr_info("Using AVX optimized SHA-512 implementation\n"); 164 #ifdef CONFIG_AS_AVX2 165 else if (sha512_transform_asm == sha512_transform_rorx) 166 pr_info("Using AVX2 optimized SHA-512 implementation\n"); 167 #endif 168 else 169 #endif 170 pr_info("Using SSSE3 optimized SHA-512 implementation\n"); 171 return crypto_register_shashes(algs, ARRAY_SIZE(algs)); 172 } 173 pr_info("Neither AVX nor SSSE3 is available/usable.\n"); 174 175 return -ENODEV; 176 } 177 178 static void __exit sha512_ssse3_mod_fini(void) 179 { 180 crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); 181 } 182 183 module_init(sha512_ssse3_mod_init); 184 module_exit(sha512_ssse3_mod_fini); 185 186 MODULE_LICENSE("GPL"); 187 MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated"); 188 189 MODULE_ALIAS_CRYPTO("sha512"); 190 MODULE_ALIAS_CRYPTO("sha384"); 191