1 /* 2 * Cryptographic API. 3 * 4 * Glue code for the SHA512 Secure Hash Algorithm assembler 5 * implementation using supplemental SSE3 / AVX / AVX2 instructions. 6 * 7 * This file is based on sha512_generic.c 8 * 9 * Copyright (C) 2013 Intel Corporation 10 * Author: Tim Chen <tim.c.chen@linux.intel.com> 11 * 12 * This program is free software; you can redistribute it and/or modify it 13 * under the terms of the GNU General Public License as published by the Free 14 * Software Foundation; either version 2 of the License, or (at your option) 15 * any later version. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 21 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 23 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 * SOFTWARE. 25 * 26 */ 27 28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30 #include <crypto/internal/hash.h> 31 #include <crypto/internal/simd.h> 32 #include <linux/init.h> 33 #include <linux/module.h> 34 #include <linux/mm.h> 35 #include <linux/string.h> 36 #include <linux/types.h> 37 #include <crypto/sha2.h> 38 #include <crypto/sha512_base.h> 39 #include <asm/simd.h> 40 41 asmlinkage void sha512_transform_ssse3(struct sha512_state *state, 42 const u8 *data, int blocks); 43 44 static int sha512_update(struct shash_desc *desc, const u8 *data, 45 unsigned int len, sha512_block_fn *sha512_xform) 46 { 47 struct sha512_state *sctx = shash_desc_ctx(desc); 48 49 if (!crypto_simd_usable() || 50 (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) 51 return crypto_sha512_update(desc, data, len); 52 53 /* 54 * Make sure struct sha512_state begins directly with the SHA512 55 * 512-bit internal state, as this is what the asm functions expect. 56 */ 57 BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); 58 59 kernel_fpu_begin(); 60 sha512_base_do_update(desc, data, len, sha512_xform); 61 kernel_fpu_end(); 62 63 return 0; 64 } 65 66 static int sha512_finup(struct shash_desc *desc, const u8 *data, 67 unsigned int len, u8 *out, sha512_block_fn *sha512_xform) 68 { 69 if (!crypto_simd_usable()) 70 return crypto_sha512_finup(desc, data, len, out); 71 72 kernel_fpu_begin(); 73 if (len) 74 sha512_base_do_update(desc, data, len, sha512_xform); 75 sha512_base_do_finalize(desc, sha512_xform); 76 kernel_fpu_end(); 77 78 return sha512_base_finish(desc, out); 79 } 80 81 static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, 82 unsigned int len) 83 { 84 return sha512_update(desc, data, len, sha512_transform_ssse3); 85 } 86 87 static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, 88 unsigned int len, u8 *out) 89 { 90 return sha512_finup(desc, data, len, out, sha512_transform_ssse3); 91 } 92 93 /* Add padding and return the message digest. */ 94 static int sha512_ssse3_final(struct shash_desc *desc, u8 *out) 95 { 96 return sha512_ssse3_finup(desc, NULL, 0, out); 97 } 98 99 static struct shash_alg sha512_ssse3_algs[] = { { 100 .digestsize = SHA512_DIGEST_SIZE, 101 .init = sha512_base_init, 102 .update = sha512_ssse3_update, 103 .final = sha512_ssse3_final, 104 .finup = sha512_ssse3_finup, 105 .descsize = sizeof(struct sha512_state), 106 .base = { 107 .cra_name = "sha512", 108 .cra_driver_name = "sha512-ssse3", 109 .cra_priority = 150, 110 .cra_blocksize = SHA512_BLOCK_SIZE, 111 .cra_module = THIS_MODULE, 112 } 113 }, { 114 .digestsize = SHA384_DIGEST_SIZE, 115 .init = sha384_base_init, 116 .update = sha512_ssse3_update, 117 .final = sha512_ssse3_final, 118 .finup = sha512_ssse3_finup, 119 .descsize = sizeof(struct sha512_state), 120 .base = { 121 .cra_name = "sha384", 122 .cra_driver_name = "sha384-ssse3", 123 .cra_priority = 150, 124 .cra_blocksize = SHA384_BLOCK_SIZE, 125 .cra_module = THIS_MODULE, 126 } 127 } }; 128 129 static int register_sha512_ssse3(void) 130 { 131 if (boot_cpu_has(X86_FEATURE_SSSE3)) 132 return crypto_register_shashes(sha512_ssse3_algs, 133 ARRAY_SIZE(sha512_ssse3_algs)); 134 return 0; 135 } 136 137 static void unregister_sha512_ssse3(void) 138 { 139 if (boot_cpu_has(X86_FEATURE_SSSE3)) 140 crypto_unregister_shashes(sha512_ssse3_algs, 141 ARRAY_SIZE(sha512_ssse3_algs)); 142 } 143 144 asmlinkage void sha512_transform_avx(struct sha512_state *state, 145 const u8 *data, int blocks); 146 static bool avx_usable(void) 147 { 148 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { 149 if (boot_cpu_has(X86_FEATURE_AVX)) 150 pr_info("AVX detected but unusable.\n"); 151 return false; 152 } 153 154 return true; 155 } 156 157 static int sha512_avx_update(struct shash_desc *desc, const u8 *data, 158 unsigned int len) 159 { 160 return sha512_update(desc, data, len, sha512_transform_avx); 161 } 162 163 static int sha512_avx_finup(struct shash_desc *desc, const u8 *data, 164 unsigned int len, u8 *out) 165 { 166 return sha512_finup(desc, data, len, out, sha512_transform_avx); 167 } 168 169 /* Add padding and return the message digest. */ 170 static int sha512_avx_final(struct shash_desc *desc, u8 *out) 171 { 172 return sha512_avx_finup(desc, NULL, 0, out); 173 } 174 175 static struct shash_alg sha512_avx_algs[] = { { 176 .digestsize = SHA512_DIGEST_SIZE, 177 .init = sha512_base_init, 178 .update = sha512_avx_update, 179 .final = sha512_avx_final, 180 .finup = sha512_avx_finup, 181 .descsize = sizeof(struct sha512_state), 182 .base = { 183 .cra_name = "sha512", 184 .cra_driver_name = "sha512-avx", 185 .cra_priority = 160, 186 .cra_blocksize = SHA512_BLOCK_SIZE, 187 .cra_module = THIS_MODULE, 188 } 189 }, { 190 .digestsize = SHA384_DIGEST_SIZE, 191 .init = sha384_base_init, 192 .update = sha512_avx_update, 193 .final = sha512_avx_final, 194 .finup = sha512_avx_finup, 195 .descsize = sizeof(struct sha512_state), 196 .base = { 197 .cra_name = "sha384", 198 .cra_driver_name = "sha384-avx", 199 .cra_priority = 160, 200 .cra_blocksize = SHA384_BLOCK_SIZE, 201 .cra_module = THIS_MODULE, 202 } 203 } }; 204 205 static int register_sha512_avx(void) 206 { 207 if (avx_usable()) 208 return crypto_register_shashes(sha512_avx_algs, 209 ARRAY_SIZE(sha512_avx_algs)); 210 return 0; 211 } 212 213 static void unregister_sha512_avx(void) 214 { 215 if (avx_usable()) 216 crypto_unregister_shashes(sha512_avx_algs, 217 ARRAY_SIZE(sha512_avx_algs)); 218 } 219 220 asmlinkage void sha512_transform_rorx(struct sha512_state *state, 221 const u8 *data, int blocks); 222 223 static int sha512_avx2_update(struct shash_desc *desc, const u8 *data, 224 unsigned int len) 225 { 226 return sha512_update(desc, data, len, sha512_transform_rorx); 227 } 228 229 static int sha512_avx2_finup(struct shash_desc *desc, const u8 *data, 230 unsigned int len, u8 *out) 231 { 232 return sha512_finup(desc, data, len, out, sha512_transform_rorx); 233 } 234 235 /* Add padding and return the message digest. */ 236 static int sha512_avx2_final(struct shash_desc *desc, u8 *out) 237 { 238 return sha512_avx2_finup(desc, NULL, 0, out); 239 } 240 241 static struct shash_alg sha512_avx2_algs[] = { { 242 .digestsize = SHA512_DIGEST_SIZE, 243 .init = sha512_base_init, 244 .update = sha512_avx2_update, 245 .final = sha512_avx2_final, 246 .finup = sha512_avx2_finup, 247 .descsize = sizeof(struct sha512_state), 248 .base = { 249 .cra_name = "sha512", 250 .cra_driver_name = "sha512-avx2", 251 .cra_priority = 170, 252 .cra_blocksize = SHA512_BLOCK_SIZE, 253 .cra_module = THIS_MODULE, 254 } 255 }, { 256 .digestsize = SHA384_DIGEST_SIZE, 257 .init = sha384_base_init, 258 .update = sha512_avx2_update, 259 .final = sha512_avx2_final, 260 .finup = sha512_avx2_finup, 261 .descsize = sizeof(struct sha512_state), 262 .base = { 263 .cra_name = "sha384", 264 .cra_driver_name = "sha384-avx2", 265 .cra_priority = 170, 266 .cra_blocksize = SHA384_BLOCK_SIZE, 267 .cra_module = THIS_MODULE, 268 } 269 } }; 270 271 static bool avx2_usable(void) 272 { 273 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && 274 boot_cpu_has(X86_FEATURE_BMI2)) 275 return true; 276 277 return false; 278 } 279 280 static int register_sha512_avx2(void) 281 { 282 if (avx2_usable()) 283 return crypto_register_shashes(sha512_avx2_algs, 284 ARRAY_SIZE(sha512_avx2_algs)); 285 return 0; 286 } 287 288 static void unregister_sha512_avx2(void) 289 { 290 if (avx2_usable()) 291 crypto_unregister_shashes(sha512_avx2_algs, 292 ARRAY_SIZE(sha512_avx2_algs)); 293 } 294 295 static int __init sha512_ssse3_mod_init(void) 296 { 297 298 if (register_sha512_ssse3()) 299 goto fail; 300 301 if (register_sha512_avx()) { 302 unregister_sha512_ssse3(); 303 goto fail; 304 } 305 306 if (register_sha512_avx2()) { 307 unregister_sha512_avx(); 308 unregister_sha512_ssse3(); 309 goto fail; 310 } 311 312 return 0; 313 fail: 314 return -ENODEV; 315 } 316 317 static void __exit sha512_ssse3_mod_fini(void) 318 { 319 unregister_sha512_avx2(); 320 unregister_sha512_avx(); 321 unregister_sha512_ssse3(); 322 } 323 324 module_init(sha512_ssse3_mod_init); 325 module_exit(sha512_ssse3_mod_fini); 326 327 MODULE_LICENSE("GPL"); 328 MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated"); 329 330 MODULE_ALIAS_CRYPTO("sha512"); 331 MODULE_ALIAS_CRYPTO("sha512-ssse3"); 332 MODULE_ALIAS_CRYPTO("sha512-avx"); 333 MODULE_ALIAS_CRYPTO("sha512-avx2"); 334 MODULE_ALIAS_CRYPTO("sha384"); 335 MODULE_ALIAS_CRYPTO("sha384-ssse3"); 336 MODULE_ALIAS_CRYPTO("sha384-avx"); 337 MODULE_ALIAS_CRYPTO("sha384-avx2"); 338