1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * GHASH routines supporting VMX instructions on the Power 8 4 * 5 * Copyright (C) 2015, 2019 International Business Machines Inc. 6 * 7 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> 8 * 9 * Extended by Daniel Axtens <dja@axtens.net> to replace the fallback 10 * mechanism. The new approach is based on arm64 code, which is: 11 * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> 12 */ 13 14 #include "aesp8-ppc.h" 15 #include <asm/switch_to.h> 16 #include <crypto/aes.h> 17 #include <crypto/gf128mul.h> 18 #include <crypto/ghash.h> 19 #include <crypto/internal/hash.h> 20 #include <crypto/internal/simd.h> 21 #include <linux/err.h> 22 #include <linux/kernel.h> 23 #include <linux/module.h> 24 #include <linux/string.h> 25 #include <linux/uaccess.h> 26 27 void gcm_init_p8(u128 htable[16], const u64 Xi[2]); 28 void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]); 29 void gcm_ghash_p8(u64 Xi[2], const u128 htable[16], 30 const u8 *in, size_t len); 31 32 struct p8_ghash_ctx { 33 /* key used by vector asm */ 34 u128 htable[16]; 35 /* key used by software fallback */ 36 be128 key; 37 }; 38 39 struct p8_ghash_desc_ctx { 40 u64 shash[2]; 41 }; 42 43 static int p8_ghash_init(struct shash_desc *desc) 44 { 45 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); 46 47 memset(dctx->shash, 0, GHASH_DIGEST_SIZE); 48 return 0; 49 } 50 51 static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key, 52 unsigned int keylen) 53 { 54 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm)); 55 56 if (keylen != GHASH_BLOCK_SIZE) 57 return -EINVAL; 58 59 preempt_disable(); 60 pagefault_disable(); 61 enable_kernel_vsx(); 62 gcm_init_p8(ctx->htable, (const u64 *) key); 63 disable_kernel_vsx(); 64 pagefault_enable(); 65 preempt_enable(); 66 67 memcpy(&ctx->key, key, GHASH_BLOCK_SIZE); 68 69 return 0; 70 } 71 72 static inline void __ghash_block(struct p8_ghash_ctx *ctx, 73 struct p8_ghash_desc_ctx *dctx, 74 const u8 *src) 75 { 76 if (crypto_simd_usable()) { 77 preempt_disable(); 78 pagefault_disable(); 79 enable_kernel_vsx(); 80 gcm_ghash_p8(dctx->shash, ctx->htable, src, GHASH_BLOCK_SIZE); 81 disable_kernel_vsx(); 82 pagefault_enable(); 83 preempt_enable(); 84 } else { 85 crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); 86 gf128mul_lle((be128 *)dctx->shash, &ctx->key); 87 } 88 } 89 90 static inline int __ghash_blocks(struct p8_ghash_ctx *ctx, 91 struct p8_ghash_desc_ctx *dctx, 92 const u8 *src, unsigned int srclen) 93 { 94 int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE); 95 96 srclen -= remain; 97 if (crypto_simd_usable()) { 98 preempt_disable(); 99 pagefault_disable(); 100 enable_kernel_vsx(); 101 gcm_ghash_p8(dctx->shash, ctx->htable, 102 src, srclen); 103 disable_kernel_vsx(); 104 pagefault_enable(); 105 preempt_enable(); 106 } else { 107 do { 108 crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE); 109 gf128mul_lle((be128 *)dctx->shash, &ctx->key); 110 srclen -= GHASH_BLOCK_SIZE; 111 src += GHASH_BLOCK_SIZE; 112 } while (srclen); 113 } 114 115 return remain; 116 } 117 118 static int p8_ghash_update(struct shash_desc *desc, 119 const u8 *src, unsigned int srclen) 120 { 121 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); 122 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); 123 124 return __ghash_blocks(ctx, dctx, src, srclen); 125 } 126 127 static int p8_ghash_finup(struct shash_desc *desc, const u8 *src, 128 unsigned int len, u8 *out) 129 { 130 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm)); 131 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc); 132 133 if (len) { 134 u8 buf[GHASH_BLOCK_SIZE] = {}; 135 136 memcpy(buf, src, len); 137 __ghash_block(ctx, dctx, buf); 138 memzero_explicit(buf, sizeof(buf)); 139 } 140 memcpy(out, dctx->shash, GHASH_DIGEST_SIZE); 141 return 0; 142 } 143 144 struct shash_alg p8_ghash_alg = { 145 .digestsize = GHASH_DIGEST_SIZE, 146 .init = p8_ghash_init, 147 .update = p8_ghash_update, 148 .finup = p8_ghash_finup, 149 .setkey = p8_ghash_setkey, 150 .descsize = sizeof(struct p8_ghash_desc_ctx), 151 .base = { 152 .cra_name = "ghash", 153 .cra_driver_name = "p8_ghash", 154 .cra_priority = 1000, 155 .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 156 .cra_blocksize = GHASH_BLOCK_SIZE, 157 .cra_ctxsize = sizeof(struct p8_ghash_ctx), 158 .cra_module = THIS_MODULE, 159 }, 160 }; 161