1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Glue code for POLYVAL using ARMv8 Crypto Extensions 4 * 5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> 6 * Copyright (c) 2009 Intel Corp. 7 * Author: Huang Ying <ying.huang@intel.com> 8 * Copyright 2021 Google LLC 9 */ 10 11 /* 12 * Glue code based on ghash-clmulni-intel_glue.c. 13 * 14 * This implementation of POLYVAL uses montgomery multiplication accelerated by 15 * ARMv8 Crypto Extensions instructions to implement the finite field operations. 16 */ 17 18 #include <asm/neon.h> 19 #include <crypto/internal/hash.h> 20 #include <crypto/polyval.h> 21 #include <crypto/utils.h> 22 #include <linux/cpufeature.h> 23 #include <linux/errno.h> 24 #include <linux/kernel.h> 25 #include <linux/module.h> 26 #include <linux/string.h> 27 28 #define NUM_KEY_POWERS 8 29 30 struct polyval_tfm_ctx { 31 /* 32 * These powers must be in the order h^8, ..., h^1. 33 */ 34 u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE]; 35 }; 36 37 struct polyval_desc_ctx { 38 u8 buffer[POLYVAL_BLOCK_SIZE]; 39 }; 40 41 asmlinkage void pmull_polyval_update(const struct polyval_tfm_ctx *keys, 42 const u8 *in, size_t nblocks, u8 *accumulator); 43 asmlinkage void pmull_polyval_mul(u8 *op1, const u8 *op2); 44 45 static void internal_polyval_update(const struct polyval_tfm_ctx *keys, 46 const u8 *in, size_t nblocks, u8 *accumulator) 47 { 48 kernel_neon_begin(); 49 pmull_polyval_update(keys, in, nblocks, accumulator); 50 kernel_neon_end(); 51 } 52 53 static void internal_polyval_mul(u8 *op1, const u8 *op2) 54 { 55 kernel_neon_begin(); 56 pmull_polyval_mul(op1, op2); 57 kernel_neon_end(); 58 } 59 60 static int polyval_arm64_setkey(struct crypto_shash *tfm, 61 const u8 *key, unsigned int keylen) 62 { 63 struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm); 64 int i; 65 66 if (keylen != POLYVAL_BLOCK_SIZE) 67 return -EINVAL; 68 69 memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE); 70 71 for (i = NUM_KEY_POWERS-2; i >= 0; i--) { 72 memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE); 73 internal_polyval_mul(tctx->key_powers[i], 74 tctx->key_powers[i+1]); 75 } 76 77 return 0; 78 } 79 80 static int polyval_arm64_init(struct shash_desc *desc) 81 { 82 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); 83 84 memset(dctx, 0, sizeof(*dctx)); 85 86 return 0; 87 } 88 89 static int polyval_arm64_update(struct shash_desc *desc, 90 const u8 *src, unsigned int srclen) 91 { 92 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); 93 const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 94 unsigned int nblocks; 95 96 do { 97 /* allow rescheduling every 4K bytes */ 98 nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE; 99 internal_polyval_update(tctx, src, nblocks, dctx->buffer); 100 srclen -= nblocks * POLYVAL_BLOCK_SIZE; 101 src += nblocks * POLYVAL_BLOCK_SIZE; 102 } while (srclen >= POLYVAL_BLOCK_SIZE); 103 104 return srclen; 105 } 106 107 static int polyval_arm64_finup(struct shash_desc *desc, const u8 *src, 108 unsigned int len, u8 *dst) 109 { 110 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); 111 const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 112 113 if (len) { 114 crypto_xor(dctx->buffer, src, len); 115 internal_polyval_mul(dctx->buffer, 116 tctx->key_powers[NUM_KEY_POWERS-1]); 117 } 118 119 memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE); 120 121 return 0; 122 } 123 124 static struct shash_alg polyval_alg = { 125 .digestsize = POLYVAL_DIGEST_SIZE, 126 .init = polyval_arm64_init, 127 .update = polyval_arm64_update, 128 .finup = polyval_arm64_finup, 129 .setkey = polyval_arm64_setkey, 130 .descsize = sizeof(struct polyval_desc_ctx), 131 .base = { 132 .cra_name = "polyval", 133 .cra_driver_name = "polyval-ce", 134 .cra_priority = 200, 135 .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 136 .cra_blocksize = POLYVAL_BLOCK_SIZE, 137 .cra_ctxsize = sizeof(struct polyval_tfm_ctx), 138 .cra_module = THIS_MODULE, 139 }, 140 }; 141 142 static int __init polyval_ce_mod_init(void) 143 { 144 return crypto_register_shash(&polyval_alg); 145 } 146 147 static void __exit polyval_ce_mod_exit(void) 148 { 149 crypto_unregister_shash(&polyval_alg); 150 } 151 152 module_cpu_feature_match(PMULL, polyval_ce_mod_init) 153 module_exit(polyval_ce_mod_exit); 154 155 MODULE_LICENSE("GPL"); 156 MODULE_DESCRIPTION("POLYVAL hash function accelerated by ARMv8 Crypto Extensions"); 157 MODULE_ALIAS_CRYPTO("polyval"); 158 MODULE_ALIAS_CRYPTO("polyval-ce"); 159