1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * GHASH using the RISC-V vector crypto extensions 4 * 5 * Copyright (C) 2023 VRULL GmbH 6 * Author: Heiko Stuebner <heiko.stuebner@vrull.eu> 7 * 8 * Copyright (C) 2023 SiFive, Inc. 9 * Author: Jerry Shih <jerry.shih@sifive.com> 10 */ 11 12 #include <asm/simd.h> 13 #include <asm/vector.h> 14 #include <crypto/ghash.h> 15 #include <crypto/internal/hash.h> 16 #include <crypto/internal/simd.h> 17 #include <linux/linkage.h> 18 #include <linux/module.h> 19 20 asmlinkage void ghash_zvkg(be128 *accumulator, const be128 *key, const u8 *data, 21 size_t len); 22 23 struct riscv64_ghash_tfm_ctx { 24 be128 key; 25 }; 26 27 struct riscv64_ghash_desc_ctx { 28 be128 accumulator; 29 u8 buffer[GHASH_BLOCK_SIZE]; 30 u32 bytes; 31 }; 32 33 static int riscv64_ghash_setkey(struct crypto_shash *tfm, const u8 *key, 34 unsigned int keylen) 35 { 36 struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(tfm); 37 38 if (keylen != GHASH_BLOCK_SIZE) 39 return -EINVAL; 40 41 memcpy(&tctx->key, key, GHASH_BLOCK_SIZE); 42 43 return 0; 44 } 45 46 static int riscv64_ghash_init(struct shash_desc *desc) 47 { 48 struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); 49 50 *dctx = (struct riscv64_ghash_desc_ctx){}; 51 52 return 0; 53 } 54 55 static inline void 56 riscv64_ghash_blocks(const struct riscv64_ghash_tfm_ctx *tctx, 57 struct riscv64_ghash_desc_ctx *dctx, 58 const u8 *src, size_t srclen) 59 { 60 /* The srclen is nonzero and a multiple of 16. */ 61 if (crypto_simd_usable()) { 62 kernel_vector_begin(); 63 ghash_zvkg(&dctx->accumulator, &tctx->key, src, srclen); 64 kernel_vector_end(); 65 } else { 66 do { 67 crypto_xor((u8 *)&dctx->accumulator, src, 68 GHASH_BLOCK_SIZE); 69 gf128mul_lle(&dctx->accumulator, &tctx->key); 70 src += GHASH_BLOCK_SIZE; 71 srclen -= GHASH_BLOCK_SIZE; 72 } while (srclen); 73 } 74 } 75 76 static int riscv64_ghash_update(struct shash_desc *desc, const u8 *src, 77 unsigned int srclen) 78 { 79 const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 80 struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); 81 unsigned int len; 82 83 if (dctx->bytes) { 84 if (dctx->bytes + srclen < GHASH_BLOCK_SIZE) { 85 memcpy(dctx->buffer + dctx->bytes, src, srclen); 86 dctx->bytes += srclen; 87 return 0; 88 } 89 memcpy(dctx->buffer + dctx->bytes, src, 90 GHASH_BLOCK_SIZE - dctx->bytes); 91 riscv64_ghash_blocks(tctx, dctx, dctx->buffer, 92 GHASH_BLOCK_SIZE); 93 src += GHASH_BLOCK_SIZE - dctx->bytes; 94 srclen -= GHASH_BLOCK_SIZE - dctx->bytes; 95 dctx->bytes = 0; 96 } 97 98 len = round_down(srclen, GHASH_BLOCK_SIZE); 99 if (len) { 100 riscv64_ghash_blocks(tctx, dctx, src, len); 101 src += len; 102 srclen -= len; 103 } 104 105 if (srclen) { 106 memcpy(dctx->buffer, src, srclen); 107 dctx->bytes = srclen; 108 } 109 110 return 0; 111 } 112 113 static int riscv64_ghash_final(struct shash_desc *desc, u8 *out) 114 { 115 const struct riscv64_ghash_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 116 struct riscv64_ghash_desc_ctx *dctx = shash_desc_ctx(desc); 117 int i; 118 119 if (dctx->bytes) { 120 for (i = dctx->bytes; i < GHASH_BLOCK_SIZE; i++) 121 dctx->buffer[i] = 0; 122 123 riscv64_ghash_blocks(tctx, dctx, dctx->buffer, 124 GHASH_BLOCK_SIZE); 125 } 126 127 memcpy(out, &dctx->accumulator, GHASH_DIGEST_SIZE); 128 return 0; 129 } 130 131 static struct shash_alg riscv64_ghash_alg = { 132 .init = riscv64_ghash_init, 133 .update = riscv64_ghash_update, 134 .final = riscv64_ghash_final, 135 .setkey = riscv64_ghash_setkey, 136 .descsize = sizeof(struct riscv64_ghash_desc_ctx), 137 .digestsize = GHASH_DIGEST_SIZE, 138 .base = { 139 .cra_blocksize = GHASH_BLOCK_SIZE, 140 .cra_ctxsize = sizeof(struct riscv64_ghash_tfm_ctx), 141 .cra_priority = 300, 142 .cra_name = "ghash", 143 .cra_driver_name = "ghash-riscv64-zvkg", 144 .cra_module = THIS_MODULE, 145 }, 146 }; 147 148 static int __init riscv64_ghash_mod_init(void) 149 { 150 if (riscv_isa_extension_available(NULL, ZVKG) && 151 riscv_vector_vlen() >= 128) 152 return crypto_register_shash(&riscv64_ghash_alg); 153 154 return -ENODEV; 155 } 156 157 static void __exit riscv64_ghash_mod_exit(void) 158 { 159 crypto_unregister_shash(&riscv64_ghash_alg); 160 } 161 162 module_init(riscv64_ghash_mod_init); 163 module_exit(riscv64_ghash_mod_exit); 164 165 MODULE_DESCRIPTION("GHASH (RISC-V accelerated)"); 166 MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>"); 167 MODULE_LICENSE("GPL"); 168 MODULE_ALIAS_CRYPTO("ghash"); 169