1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * GHASH using the RISC-V vector crypto extensions 4 * 5 * Copyright (C) 2023 VRULL GmbH 6 * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> 7 * 8 * Copyright (C) 2023 SiFive, Inc. 9 * Author: Jerry Shih <jerry.shih@sifive.com> 10 */ 11 12 #include <asm/simd.h> 13 #include <asm/vector.h> 14 #include <crypto/b128ops.h> 15 #include <crypto/gf128mul.h> 16 #include <crypto/ghash.h> 17 #include <crypto/internal/hash.h> 18 #include <crypto/internal/simd.h> 19 #include <crypto/utils.h> 20 #include <linux/errno.h> 21 #include <linux/kernel.h> 22 #include <linux/module.h> 23 #include <linux/string.h> 24 25 asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, 26 size_t len); 27 28 struct riscv64_ghash_tfm_ctx { 29 be128 key; 30 }; 31 32 struct riscv64_ghash_desc_ctx { 33 be128 accumulator; 34 }; 35 36 static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key, 37 unsigned int keylen) 38 { 39 struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm); 40 41 if (keylen != GHASH_BLOCK_SIZE) 42 return -EINVAL; 43 44 memcpy(&tctx->key, key, GHASH_BLOCK_SIZE); 45 46 return 0; 47 } 48 49 static int riscv64_ghash_init(struct shash_desc *desc) 50 { 51 struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); 52 53 *dctx = (struct riscv64_ghash_desc_ctx){}; 54 55 return 0; 56 } 57 58 static inline void 59 riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx, 60 struct riscv64_ghash_desc_ctx *dctx, 61 const u8 *src, size_t srclen) 62 { 63 /* The srclen is nonzero and a multiple of 16. */ 64 if (crypto_simd_usable()) { 65 kernel_vector_begin(); 66 ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen); 67 kernel_vector_end(); 68 } else { 69 do { 70 crypto_xor((u8 *)&dctx->accumulator, src, 71 GHASH_BLOCK_SIZE); 72 gf128mul_lle(&dctx->accumulator, &tctx->key); 73 src += GHASH_BLOCK_SIZE; 74 srclen -= GHASH_BLOCK_SIZE; 75 } while (srclen); 76 } 77 } 78 79 static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src, 80 unsigned int srclen) 81 { 82 const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 83 struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); 84 85 riscv64_ghash_blocks(tctx, dctx, src, 86 round_down(srclen, GHASH_BLOCK_SIZE)); 87 return srclen - round_down(srclen, GHASH_BLOCK_SIZE); 88 } 89 90 static int riscv64_ghash_finup(struct shash_desc *desc, const u8 *src, 91 unsigned int len, u8 *out) 92 { 93 const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 94 struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); 95 96 if (len) { 97 u8 buf[GHASH_BLOCK_SIZE] = {}; 98 99 memcpy(buf, src, len); 100 riscv64_ghash_blocks(tctx, dctx, buf, GHASH_BLOCK_SIZE); 101 memzero_explicit(buf, sizeof(buf)); 102 } 103 104 memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE); 105 return 0; 106 } 107 108 static struct shash_alg riscv64_ghash_alg = { 109 .init = riscv64_ghash_init, 110 .update = riscv64_ghash_update, 111 .finup = riscv64_ghash_finup, 112 .setkey = riscv64_ghash_setkey, 113 .descsize = sizeof(struct riscv64_ghash_desc_ctx), 114 .digestsize = GHASH_DIGEST_SIZE, 115 .base = { 116 .cra_blocksize = GHASH_BLOCK_SIZE, 117 .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx), 118 .cra_priority = 300, 119 .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 120 .cra_name = "ghash", 121 .cra_driver_name = "ghash-riscv64-zvkg", 122 .cra_module = THIS_MODULE, 123 }, 124 }; 125 126 static int __init riscv64_ghash_mod_init(void) 127 { 128 if (riscv_isa_extension_available(NULL, ZVKG) && 129 riscv_vector_vlen() >= 128) 130 return crypto_register_shash(&riscv64_ghash_alg); 131 132 return -ENODEV; 133 } 134 135 static void __exit riscv64_ghash_mod_exit(void) 136 { 137 crypto_unregister_shash(&riscv64_ghash_alg); 138 } 139 140 module_init(riscv64_ghash_mod_init); 141 module_exit(riscv64_ghash_mod_exit); 142 143 MODULE_DESCRIPTION("GHASH (RISC-V accelerated)"); 144 MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>"); 145 MODULE_LICENSE("GPL"); 146 MODULE_ALIAS_CRYPTO("ghash"); 147