1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
11 #include <crypto/aes.h>
12 #include <crypto/gcm.h>
25 MODULE_DESCRIPTION("GHASH and AES-GCM using ARMv8 Crypto Extensions");
91 		gf128mul_lle(&dst, &key->k);  in ghash_do_update()
92 	} while (--blocks);  in ghash_do_update()
108 		simd_update(blocks, dg, src, key->h, head);  in ghash_do_simd_update()
122 	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;  in ghash_update()
124 	ctx->count += len;  in ghash_update()
127 		struct ghash_key *key = crypto_shash_ctx(desc->tfm);  in ghash_update()
131 			int p = GHASH_BLOCK_SIZE - partial;  in ghash_update()
133 			memcpy(ctx->buf + partial, src, p);  in ghash_update()
135 			len -= p;  in ghash_update()
144 			ghash_do_simd_update(chunk, ctx->digest, src, key,  in ghash_update()
145 					     partial ? ctx->buf : NULL,  in ghash_update()
148 			blocks -= chunk;  in ghash_update()
154 		memcpy(ctx->buf + partial, src, len);  in ghash_update()
161 	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;  in ghash_final()
164 		struct ghash_key *key = crypto_shash_ctx(desc->tfm);  in ghash_final()
166 		memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);  in ghash_final()
168 		ghash_do_simd_update(1, ctx->digest, ctx->buf, key, NULL,  in ghash_final()
171 	put_unaligned_be64(ctx->digest[1], dst);  in ghash_final()
172 	put_unaligned_be64(ctx->digest[0], dst + 8);  in ghash_final()
180 	u64 carry = be64_to_cpu(k->a) & BIT(63) ? 1 : 0;  in ghash_reflect()
182 	h[0] = (be64_to_cpu(k->b) << 1) | carry;  in ghash_reflect()
183 	h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);  in ghash_reflect()
195 		return -EINVAL;  in ghash_setkey()
198 	memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);  in ghash_setkey()
200 	ghash_reflect(key->h[0], &key->k);  in ghash_setkey()
206 	.base.cra_driver_name	= "ghash-neon",
223 	 * # of rounds specified by AES:  in num_rounds()
229 	return 6 + ctx->key_length / 4;  in num_rounds()
240 	ret = aes_expandkey(&ctx->aes_key, inkey, keylen);  in gcm_aes_setkey()
242 		return -EINVAL;  in gcm_aes_setkey()
244 	aes_encrypt(&ctx->aes_key, key, (u8[AES_BLOCK_SIZE]){});  in gcm_aes_setkey()
247 	memcpy(&ctx->ghash_key.k, key, GHASH_BLOCK_SIZE);  in gcm_aes_setkey()
249 	ghash_reflect(ctx->ghash_key.h[0], &ctx->ghash_key.k);  in gcm_aes_setkey()
251 	h = ctx->ghash_key.k;  in gcm_aes_setkey()
252 	gf128mul_lle(&h, &ctx->ghash_key.k);  in gcm_aes_setkey()
253 	ghash_reflect(ctx->ghash_key.h[1], &h);  in gcm_aes_setkey()
255 	gf128mul_lle(&h, &ctx->ghash_key.k);  in gcm_aes_setkey()
256 	ghash_reflect(ctx->ghash_key.h[2], &h);  in gcm_aes_setkey()
258 	gf128mul_lle(&h, &ctx->ghash_key.k);  in gcm_aes_setkey()
259 	ghash_reflect(ctx->ghash_key.h[3], &h);  in gcm_aes_setkey()
273 		int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count);  in gcm_update_mac()
279 		count -= buf_added;  in gcm_update_mac()
285 		ghash_do_simd_update(blocks, dg, src, &ctx->ghash_key,  in gcm_update_mac()
308 	scatterwalk_start(&walk, req->src);  in gcm_calculate_auth_mac()
321 		len -= n;  in gcm_calculate_auth_mac()
329 		memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);  in gcm_calculate_auth_mac()
330 		ghash_do_simd_update(1, dg, buf, &ctx->ghash_key, NULL,  in gcm_calculate_auth_mac()
339 	int nrounds = num_rounds(&ctx->aes_key);  in gcm_encrypt()
348 	lengths.b = cpu_to_be64(req->cryptlen * 8);  in gcm_encrypt()
365 			src = dst = memcpy(buf + sizeof(buf) - nbytes,  in gcm_encrypt()
368 			nbytes &= ~(AES_BLOCK_SIZE - 1);  in gcm_encrypt()
373 		pmull_gcm_encrypt(nbytes, dst, src, ctx->ghash_key.h,  in gcm_encrypt()
374 				  dg, iv, ctx->aes_key.key_enc, nrounds,  in gcm_encrypt()
383 			       buf + sizeof(buf) - nbytes, nbytes);  in gcm_encrypt()
385 		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);  in gcm_encrypt()
392 	scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen,  in gcm_encrypt()
403 	int nrounds = num_rounds(&ctx->aes_key);  in gcm_decrypt()
414 	lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8);  in gcm_decrypt()
421 	scatterwalk_map_and_copy(otag, req->src,  in gcm_decrypt()
422 				 req->assoclen + req->cryptlen - authsize,  in gcm_decrypt()
435 			src = dst = memcpy(buf + sizeof(buf) - nbytes,  in gcm_decrypt()
438 			nbytes &= ~(AES_BLOCK_SIZE - 1);  in gcm_decrypt()
443 		ret = pmull_gcm_decrypt(nbytes, dst, src, ctx->ghash_key.h,  in gcm_decrypt()
444 					dg, iv, ctx->aes_key.key_enc,  in gcm_decrypt()
453 			       buf + sizeof(buf) - nbytes, nbytes);  in gcm_decrypt()
455 		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);  in gcm_decrypt()
461 	return ret ? -EBADMSG : 0;  in gcm_decrypt()
468 	memcpy(iv, req->iv, GCM_AES_IV_SIZE);  in gcm_aes_encrypt()
469 	return gcm_encrypt(req, iv, req->assoclen);  in gcm_aes_encrypt()
476 	memcpy(iv, req->iv, GCM_AES_IV_SIZE);  in gcm_aes_decrypt()
477 	return gcm_decrypt(req, iv, req->assoclen);  in gcm_aes_decrypt()
486 	keylen -= RFC4106_NONCE_SIZE;  in rfc4106_setkey()
491 	memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE);  in rfc4106_setkey()
506 	memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);  in rfc4106_encrypt()
507 	memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);  in rfc4106_encrypt()
509 	return crypto_ipsec_check_assoclen(req->assoclen) ?:  in rfc4106_encrypt()
510 	       gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);  in rfc4106_encrypt()
519 	memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);  in rfc4106_decrypt()
520 	memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);  in rfc4106_decrypt()
522 	return crypto_ipsec_check_assoclen(req->assoclen) ?:  in rfc4106_decrypt()
523 	       gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);  in rfc4106_decrypt()
535 	.base.cra_name		= "gcm(aes)",
536 	.base.cra_driver_name	= "gcm-aes-ce",
551 	.base.cra_name		= "rfc4106(gcm(aes))",
552 	.base.cra_driver_name	= "rfc4106-gcm-aes-ce",
563 		return -ENODEV;  in ghash_ce_mod_init()