1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2f1e866b1SArd Biesheuvel /*
3f1e866b1SArd Biesheuvel * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
4f1e866b1SArd Biesheuvel *
5*b575b5a1SArd Biesheuvel * Copyright (C) 2015 - 2018 Linaro Ltd.
6*b575b5a1SArd Biesheuvel * Copyright (C) 2023 Google LLC.
7f1e866b1SArd Biesheuvel */
8f1e866b1SArd Biesheuvel
9f1e866b1SArd Biesheuvel #include <asm/hwcap.h>
10f1e866b1SArd Biesheuvel #include <asm/neon.h>
11f1e866b1SArd Biesheuvel #include <asm/simd.h>
12f1e866b1SArd Biesheuvel #include <asm/unaligned.h>
13*b575b5a1SArd Biesheuvel #include <crypto/aes.h>
14*b575b5a1SArd Biesheuvel #include <crypto/gcm.h>
150a5dff98SArd Biesheuvel #include <crypto/b128ops.h>
16f1e866b1SArd Biesheuvel #include <crypto/cryptd.h>
17*b575b5a1SArd Biesheuvel #include <crypto/internal/aead.h>
18f1e866b1SArd Biesheuvel #include <crypto/internal/hash.h>
1999680c5eSEric Biggers #include <crypto/internal/simd.h>
20*b575b5a1SArd Biesheuvel #include <crypto/internal/skcipher.h>
21f1e866b1SArd Biesheuvel #include <crypto/gf128mul.h>
22*b575b5a1SArd Biesheuvel #include <crypto/scatterwalk.h>
23c9d9f608SArd Biesheuvel #include <linux/cpufeature.h>
24f1e866b1SArd Biesheuvel #include <linux/crypto.h>
253d2df845SArd Biesheuvel #include <linux/jump_label.h>
26f1e866b1SArd Biesheuvel #include <linux/module.h>
27f1e866b1SArd Biesheuvel
288dfa20fcSEric Biggers MODULE_DESCRIPTION("GHASH hash function using ARMv8 Crypto Extensions");
29*b575b5a1SArd Biesheuvel MODULE_AUTHOR("Ard Biesheuvel <ardb@kernel.org>");
30*b575b5a1SArd Biesheuvel MODULE_LICENSE("GPL");
313759ee05SArd Biesheuvel MODULE_ALIAS_CRYPTO("ghash");
32*b575b5a1SArd Biesheuvel MODULE_ALIAS_CRYPTO("gcm(aes)");
33*b575b5a1SArd Biesheuvel MODULE_ALIAS_CRYPTO("rfc4106(gcm(aes))");
34f1e866b1SArd Biesheuvel
35f1e866b1SArd Biesheuvel #define GHASH_BLOCK_SIZE 16
36f1e866b1SArd Biesheuvel #define GHASH_DIGEST_SIZE 16
37f1e866b1SArd Biesheuvel
38*b575b5a1SArd Biesheuvel #define RFC4106_NONCE_SIZE 4
39*b575b5a1SArd Biesheuvel
40f1e866b1SArd Biesheuvel struct ghash_key {
410a5dff98SArd Biesheuvel be128 k;
423d2df845SArd Biesheuvel u64 h[][2];
43f1e866b1SArd Biesheuvel };
44f1e866b1SArd Biesheuvel
45*b575b5a1SArd Biesheuvel struct gcm_key {
46*b575b5a1SArd Biesheuvel u64 h[4][2];
47*b575b5a1SArd Biesheuvel u32 rk[AES_MAX_KEYLENGTH_U32];
48*b575b5a1SArd Biesheuvel int rounds;
49*b575b5a1SArd Biesheuvel u8 nonce[]; // for RFC4106 nonce
50*b575b5a1SArd Biesheuvel };
51*b575b5a1SArd Biesheuvel
52f1e866b1SArd Biesheuvel struct ghash_desc_ctx {
53f1e866b1SArd Biesheuvel u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
54f1e866b1SArd Biesheuvel u8 buf[GHASH_BLOCK_SIZE];
55f1e866b1SArd Biesheuvel u32 count;
56f1e866b1SArd Biesheuvel };
57f1e866b1SArd Biesheuvel
58f1e866b1SArd Biesheuvel struct ghash_async_ctx {
59f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm;
60f1e866b1SArd Biesheuvel };
61f1e866b1SArd Biesheuvel
623759ee05SArd Biesheuvel asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src,
633d2df845SArd Biesheuvel u64 const h[][2], const char *head);
643759ee05SArd Biesheuvel
653759ee05SArd Biesheuvel asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src,
663d2df845SArd Biesheuvel u64 const h[][2], const char *head);
673759ee05SArd Biesheuvel
683d2df845SArd Biesheuvel static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_p64);
69f1e866b1SArd Biesheuvel
ghash_init(struct shash_desc * desc)70f1e866b1SArd Biesheuvel static int ghash_init(struct shash_desc *desc)
71f1e866b1SArd Biesheuvel {
72f1e866b1SArd Biesheuvel struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
73f1e866b1SArd Biesheuvel
74f1e866b1SArd Biesheuvel *ctx = (struct ghash_desc_ctx){};
75f1e866b1SArd Biesheuvel return 0;
76f1e866b1SArd Biesheuvel }
77f1e866b1SArd Biesheuvel
ghash_do_update(int blocks,u64 dg[],const char * src,struct ghash_key * key,const char * head)780a5dff98SArd Biesheuvel static void ghash_do_update(int blocks, u64 dg[], const char *src,
790a5dff98SArd Biesheuvel struct ghash_key *key, const char *head)
800a5dff98SArd Biesheuvel {
810a5dff98SArd Biesheuvel if (likely(crypto_simd_usable())) {
820a5dff98SArd Biesheuvel kernel_neon_begin();
833d2df845SArd Biesheuvel if (static_branch_likely(&use_p64))
843d2df845SArd Biesheuvel pmull_ghash_update_p64(blocks, dg, src, key->h, head);
853d2df845SArd Biesheuvel else
863d2df845SArd Biesheuvel pmull_ghash_update_p8(blocks, dg, src, key->h, head);
870a5dff98SArd Biesheuvel kernel_neon_end();
880a5dff98SArd Biesheuvel } else {
890a5dff98SArd Biesheuvel be128 dst = { cpu_to_be64(dg[1]), cpu_to_be64(dg[0]) };
900a5dff98SArd Biesheuvel
910a5dff98SArd Biesheuvel do {
920a5dff98SArd Biesheuvel const u8 *in = src;
930a5dff98SArd Biesheuvel
940a5dff98SArd Biesheuvel if (head) {
950a5dff98SArd Biesheuvel in = head;
960a5dff98SArd Biesheuvel blocks++;
970a5dff98SArd Biesheuvel head = NULL;
980a5dff98SArd Biesheuvel } else {
990a5dff98SArd Biesheuvel src += GHASH_BLOCK_SIZE;
1000a5dff98SArd Biesheuvel }
1010a5dff98SArd Biesheuvel
1020a5dff98SArd Biesheuvel crypto_xor((u8 *)&dst, in, GHASH_BLOCK_SIZE);
1030a5dff98SArd Biesheuvel gf128mul_lle(&dst, &key->k);
1040a5dff98SArd Biesheuvel } while (--blocks);
1050a5dff98SArd Biesheuvel
1060a5dff98SArd Biesheuvel dg[0] = be64_to_cpu(dst.b);
1070a5dff98SArd Biesheuvel dg[1] = be64_to_cpu(dst.a);
1080a5dff98SArd Biesheuvel }
1090a5dff98SArd Biesheuvel }
1100a5dff98SArd Biesheuvel
ghash_update(struct shash_desc * desc,const u8 * src,unsigned int len)111f1e866b1SArd Biesheuvel static int ghash_update(struct shash_desc *desc, const u8 *src,
112f1e866b1SArd Biesheuvel unsigned int len)
113f1e866b1SArd Biesheuvel {
114f1e866b1SArd Biesheuvel struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
115f1e866b1SArd Biesheuvel unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
116f1e866b1SArd Biesheuvel
117f1e866b1SArd Biesheuvel ctx->count += len;
118f1e866b1SArd Biesheuvel
119f1e866b1SArd Biesheuvel if ((partial + len) >= GHASH_BLOCK_SIZE) {
120f1e866b1SArd Biesheuvel struct ghash_key *key = crypto_shash_ctx(desc->tfm);
121f1e866b1SArd Biesheuvel int blocks;
122f1e866b1SArd Biesheuvel
123f1e866b1SArd Biesheuvel if (partial) {
124f1e866b1SArd Biesheuvel int p = GHASH_BLOCK_SIZE - partial;
125f1e866b1SArd Biesheuvel
126f1e866b1SArd Biesheuvel memcpy(ctx->buf + partial, src, p);
127f1e866b1SArd Biesheuvel src += p;
128f1e866b1SArd Biesheuvel len -= p;
129f1e866b1SArd Biesheuvel }
130f1e866b1SArd Biesheuvel
131f1e866b1SArd Biesheuvel blocks = len / GHASH_BLOCK_SIZE;
132f1e866b1SArd Biesheuvel len %= GHASH_BLOCK_SIZE;
133f1e866b1SArd Biesheuvel
1340a5dff98SArd Biesheuvel ghash_do_update(blocks, ctx->digest, src, key,
135f1e866b1SArd Biesheuvel partial ? ctx->buf : NULL);
136f1e866b1SArd Biesheuvel src += blocks * GHASH_BLOCK_SIZE;
137f1e866b1SArd Biesheuvel partial = 0;
138f1e866b1SArd Biesheuvel }
139f1e866b1SArd Biesheuvel if (len)
140f1e866b1SArd Biesheuvel memcpy(ctx->buf + partial, src, len);
141f1e866b1SArd Biesheuvel return 0;
142f1e866b1SArd Biesheuvel }
143f1e866b1SArd Biesheuvel
ghash_final(struct shash_desc * desc,u8 * dst)144f1e866b1SArd Biesheuvel static int ghash_final(struct shash_desc *desc, u8 *dst)
145f1e866b1SArd Biesheuvel {
146f1e866b1SArd Biesheuvel struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
147f1e866b1SArd Biesheuvel unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
148f1e866b1SArd Biesheuvel
149f1e866b1SArd Biesheuvel if (partial) {
150f1e866b1SArd Biesheuvel struct ghash_key *key = crypto_shash_ctx(desc->tfm);
151f1e866b1SArd Biesheuvel
152f1e866b1SArd Biesheuvel memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
1530a5dff98SArd Biesheuvel ghash_do_update(1, ctx->digest, ctx->buf, key, NULL);
154f1e866b1SArd Biesheuvel }
155f1e866b1SArd Biesheuvel put_unaligned_be64(ctx->digest[1], dst);
156f1e866b1SArd Biesheuvel put_unaligned_be64(ctx->digest[0], dst + 8);
157f1e866b1SArd Biesheuvel
158f1e866b1SArd Biesheuvel *ctx = (struct ghash_desc_ctx){};
159f1e866b1SArd Biesheuvel return 0;
160f1e866b1SArd Biesheuvel }
161f1e866b1SArd Biesheuvel
ghash_reflect(u64 h[],const be128 * k)16200227e3aSArd Biesheuvel static void ghash_reflect(u64 h[], const be128 *k)
16300227e3aSArd Biesheuvel {
16400227e3aSArd Biesheuvel u64 carry = be64_to_cpu(k->a) >> 63;
16500227e3aSArd Biesheuvel
16600227e3aSArd Biesheuvel h[0] = (be64_to_cpu(k->b) << 1) | carry;
16700227e3aSArd Biesheuvel h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63);
16800227e3aSArd Biesheuvel
16900227e3aSArd Biesheuvel if (carry)
17000227e3aSArd Biesheuvel h[1] ^= 0xc200000000000000UL;
17100227e3aSArd Biesheuvel }
17200227e3aSArd Biesheuvel
ghash_setkey(struct crypto_shash * tfm,const u8 * inkey,unsigned int keylen)173f1e866b1SArd Biesheuvel static int ghash_setkey(struct crypto_shash *tfm,
174f1e866b1SArd Biesheuvel const u8 *inkey, unsigned int keylen)
175f1e866b1SArd Biesheuvel {
176f1e866b1SArd Biesheuvel struct ghash_key *key = crypto_shash_ctx(tfm);
177f1e866b1SArd Biesheuvel
178674f368aSEric Biggers if (keylen != GHASH_BLOCK_SIZE)
179f1e866b1SArd Biesheuvel return -EINVAL;
180f1e866b1SArd Biesheuvel
1810a5dff98SArd Biesheuvel /* needed for the fallback */
1820a5dff98SArd Biesheuvel memcpy(&key->k, inkey, GHASH_BLOCK_SIZE);
1833d2df845SArd Biesheuvel ghash_reflect(key->h[0], &key->k);
184f1e866b1SArd Biesheuvel
1853d2df845SArd Biesheuvel if (static_branch_likely(&use_p64)) {
1863d2df845SArd Biesheuvel be128 h = key->k;
187f1e866b1SArd Biesheuvel
1880a5dff98SArd Biesheuvel gf128mul_lle(&h, &key->k);
1893d2df845SArd Biesheuvel ghash_reflect(key->h[1], &h);
19000227e3aSArd Biesheuvel
1910a5dff98SArd Biesheuvel gf128mul_lle(&h, &key->k);
1923d2df845SArd Biesheuvel ghash_reflect(key->h[2], &h);
193f1e866b1SArd Biesheuvel
1943d2df845SArd Biesheuvel gf128mul_lle(&h, &key->k);
1953d2df845SArd Biesheuvel ghash_reflect(key->h[3], &h);
1963d2df845SArd Biesheuvel }
197f1e866b1SArd Biesheuvel return 0;
198f1e866b1SArd Biesheuvel }
199f1e866b1SArd Biesheuvel
200f1e866b1SArd Biesheuvel static struct shash_alg ghash_alg = {
201f1e866b1SArd Biesheuvel .digestsize = GHASH_DIGEST_SIZE,
202f1e866b1SArd Biesheuvel .init = ghash_init,
203f1e866b1SArd Biesheuvel .update = ghash_update,
204f1e866b1SArd Biesheuvel .final = ghash_final,
205f1e866b1SArd Biesheuvel .setkey = ghash_setkey,
206f1e866b1SArd Biesheuvel .descsize = sizeof(struct ghash_desc_ctx),
2070a5dff98SArd Biesheuvel
2080a5dff98SArd Biesheuvel .base.cra_name = "ghash",
2090a5dff98SArd Biesheuvel .base.cra_driver_name = "ghash-ce-sync",
2100a5dff98SArd Biesheuvel .base.cra_priority = 300 - 1,
2110a5dff98SArd Biesheuvel .base.cra_blocksize = GHASH_BLOCK_SIZE,
2123d2df845SArd Biesheuvel .base.cra_ctxsize = sizeof(struct ghash_key) + sizeof(u64[2]),
2130a5dff98SArd Biesheuvel .base.cra_module = THIS_MODULE,
214f1e866b1SArd Biesheuvel };
215f1e866b1SArd Biesheuvel
ghash_async_init(struct ahash_request * req)216f1e866b1SArd Biesheuvel static int ghash_async_init(struct ahash_request *req)
217f1e866b1SArd Biesheuvel {
218f1e866b1SArd Biesheuvel struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
219f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
220f1e866b1SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req);
221f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
222f1e866b1SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
223f1e866b1SArd Biesheuvel struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
224f1e866b1SArd Biesheuvel
225f1e866b1SArd Biesheuvel desc->tfm = child;
226f1e866b1SArd Biesheuvel return crypto_shash_init(desc);
227f1e866b1SArd Biesheuvel }
228f1e866b1SArd Biesheuvel
ghash_async_update(struct ahash_request * req)229f1e866b1SArd Biesheuvel static int ghash_async_update(struct ahash_request *req)
230f1e866b1SArd Biesheuvel {
231f1e866b1SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req);
232f1e866b1SArd Biesheuvel struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
233f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
234f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
235f1e866b1SArd Biesheuvel
23699680c5eSEric Biggers if (!crypto_simd_usable() ||
237820573ebSHerbert Xu (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
238f1e866b1SArd Biesheuvel memcpy(cryptd_req, req, sizeof(*req));
239f1e866b1SArd Biesheuvel ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
240f1e866b1SArd Biesheuvel return crypto_ahash_update(cryptd_req);
241f1e866b1SArd Biesheuvel } else {
242f1e866b1SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
243f1e866b1SArd Biesheuvel return shash_ahash_update(req, desc);
244f1e866b1SArd Biesheuvel }
245f1e866b1SArd Biesheuvel }
246f1e866b1SArd Biesheuvel
ghash_async_final(struct ahash_request * req)247f1e866b1SArd Biesheuvel static int ghash_async_final(struct ahash_request *req)
248f1e866b1SArd Biesheuvel {
249f1e866b1SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req);
250f1e866b1SArd Biesheuvel struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
251f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
252f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
253f1e866b1SArd Biesheuvel
25499680c5eSEric Biggers if (!crypto_simd_usable() ||
255820573ebSHerbert Xu (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
256f1e866b1SArd Biesheuvel memcpy(cryptd_req, req, sizeof(*req));
257f1e866b1SArd Biesheuvel ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
258f1e866b1SArd Biesheuvel return crypto_ahash_final(cryptd_req);
259f1e866b1SArd Biesheuvel } else {
260f1e866b1SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
261f1e866b1SArd Biesheuvel return crypto_shash_final(desc, req->result);
262f1e866b1SArd Biesheuvel }
263f1e866b1SArd Biesheuvel }
264f1e866b1SArd Biesheuvel
ghash_async_digest(struct ahash_request * req)265f1e866b1SArd Biesheuvel static int ghash_async_digest(struct ahash_request *req)
266f1e866b1SArd Biesheuvel {
267f1e866b1SArd Biesheuvel struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
268f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
269f1e866b1SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req);
270f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
271f1e866b1SArd Biesheuvel
27299680c5eSEric Biggers if (!crypto_simd_usable() ||
273820573ebSHerbert Xu (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
274f1e866b1SArd Biesheuvel memcpy(cryptd_req, req, sizeof(*req));
275f1e866b1SArd Biesheuvel ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
276f1e866b1SArd Biesheuvel return crypto_ahash_digest(cryptd_req);
277f1e866b1SArd Biesheuvel } else {
278f1e866b1SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
279f1e866b1SArd Biesheuvel struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
280f1e866b1SArd Biesheuvel
281f1e866b1SArd Biesheuvel desc->tfm = child;
282f1e866b1SArd Biesheuvel return shash_ahash_digest(req, desc);
283f1e866b1SArd Biesheuvel }
284f1e866b1SArd Biesheuvel }
285f1e866b1SArd Biesheuvel
ghash_async_import(struct ahash_request * req,const void * in)286ed4767d6SArd Biesheuvel static int ghash_async_import(struct ahash_request *req, const void *in)
287ed4767d6SArd Biesheuvel {
288ed4767d6SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req);
289ed4767d6SArd Biesheuvel struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
290ed4767d6SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
291ed4767d6SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
292ed4767d6SArd Biesheuvel
293ed4767d6SArd Biesheuvel desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm);
294ed4767d6SArd Biesheuvel
295ed4767d6SArd Biesheuvel return crypto_shash_import(desc, in);
296ed4767d6SArd Biesheuvel }
297ed4767d6SArd Biesheuvel
ghash_async_export(struct ahash_request * req,void * out)298ed4767d6SArd Biesheuvel static int ghash_async_export(struct ahash_request *req, void *out)
299ed4767d6SArd Biesheuvel {
300ed4767d6SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req);
301ed4767d6SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
302ed4767d6SArd Biesheuvel
303ed4767d6SArd Biesheuvel return crypto_shash_export(desc, out);
304ed4767d6SArd Biesheuvel }
305ed4767d6SArd Biesheuvel
ghash_async_setkey(struct crypto_ahash * tfm,const u8 * key,unsigned int keylen)306f1e866b1SArd Biesheuvel static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
307f1e866b1SArd Biesheuvel unsigned int keylen)
308f1e866b1SArd Biesheuvel {
309f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
310f1e866b1SArd Biesheuvel struct crypto_ahash *child = &ctx->cryptd_tfm->base;
311f1e866b1SArd Biesheuvel
312f1e866b1SArd Biesheuvel crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
313f1e866b1SArd Biesheuvel crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
314f1e866b1SArd Biesheuvel & CRYPTO_TFM_REQ_MASK);
315af5034e8SEric Biggers return crypto_ahash_setkey(child, key, keylen);
316f1e866b1SArd Biesheuvel }
317f1e866b1SArd Biesheuvel
ghash_async_init_tfm(struct crypto_tfm * tfm)318f1e866b1SArd Biesheuvel static int ghash_async_init_tfm(struct crypto_tfm *tfm)
319f1e866b1SArd Biesheuvel {
320f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm;
321f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
322f1e866b1SArd Biesheuvel
3230a5dff98SArd Biesheuvel cryptd_tfm = cryptd_alloc_ahash("ghash-ce-sync", 0, 0);
324f1e866b1SArd Biesheuvel if (IS_ERR(cryptd_tfm))
325f1e866b1SArd Biesheuvel return PTR_ERR(cryptd_tfm);
326f1e866b1SArd Biesheuvel ctx->cryptd_tfm = cryptd_tfm;
327f1e866b1SArd Biesheuvel crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
328f1e866b1SArd Biesheuvel sizeof(struct ahash_request) +
329f1e866b1SArd Biesheuvel crypto_ahash_reqsize(&cryptd_tfm->base));
330f1e866b1SArd Biesheuvel
331f1e866b1SArd Biesheuvel return 0;
332f1e866b1SArd Biesheuvel }
333f1e866b1SArd Biesheuvel
ghash_async_exit_tfm(struct crypto_tfm * tfm)334f1e866b1SArd Biesheuvel static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
335f1e866b1SArd Biesheuvel {
336f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
337f1e866b1SArd Biesheuvel
338f1e866b1SArd Biesheuvel cryptd_free_ahash(ctx->cryptd_tfm);
339f1e866b1SArd Biesheuvel }
340f1e866b1SArd Biesheuvel
341f1e866b1SArd Biesheuvel static struct ahash_alg ghash_async_alg = {
342f1e866b1SArd Biesheuvel .init = ghash_async_init,
343f1e866b1SArd Biesheuvel .update = ghash_async_update,
344f1e866b1SArd Biesheuvel .final = ghash_async_final,
345f1e866b1SArd Biesheuvel .setkey = ghash_async_setkey,
346f1e866b1SArd Biesheuvel .digest = ghash_async_digest,
347ed4767d6SArd Biesheuvel .import = ghash_async_import,
348ed4767d6SArd Biesheuvel .export = ghash_async_export,
349f1e866b1SArd Biesheuvel .halg.digestsize = GHASH_DIGEST_SIZE,
350ed4767d6SArd Biesheuvel .halg.statesize = sizeof(struct ghash_desc_ctx),
351f1e866b1SArd Biesheuvel .halg.base = {
352f1e866b1SArd Biesheuvel .cra_name = "ghash",
353f1e866b1SArd Biesheuvel .cra_driver_name = "ghash-ce",
354f1e866b1SArd Biesheuvel .cra_priority = 300,
3556a38f622SEric Biggers .cra_flags = CRYPTO_ALG_ASYNC,
356f1e866b1SArd Biesheuvel .cra_blocksize = GHASH_BLOCK_SIZE,
357f1e866b1SArd Biesheuvel .cra_ctxsize = sizeof(struct ghash_async_ctx),
358f1e866b1SArd Biesheuvel .cra_module = THIS_MODULE,
359f1e866b1SArd Biesheuvel .cra_init = ghash_async_init_tfm,
360f1e866b1SArd Biesheuvel .cra_exit = ghash_async_exit_tfm,
361f1e866b1SArd Biesheuvel },
362f1e866b1SArd Biesheuvel };
363f1e866b1SArd Biesheuvel
364*b575b5a1SArd Biesheuvel
365*b575b5a1SArd Biesheuvel void pmull_gcm_encrypt(int blocks, u64 dg[], const char *src,
366*b575b5a1SArd Biesheuvel struct gcm_key const *k, char *dst,
367*b575b5a1SArd Biesheuvel const char *iv, int rounds, u32 counter);
368*b575b5a1SArd Biesheuvel
369*b575b5a1SArd Biesheuvel void pmull_gcm_enc_final(int blocks, u64 dg[], char *tag,
370*b575b5a1SArd Biesheuvel struct gcm_key const *k, char *head,
371*b575b5a1SArd Biesheuvel const char *iv, int rounds, u32 counter);
372*b575b5a1SArd Biesheuvel
373*b575b5a1SArd Biesheuvel void pmull_gcm_decrypt(int bytes, u64 dg[], const char *src,
374*b575b5a1SArd Biesheuvel struct gcm_key const *k, char *dst,
375*b575b5a1SArd Biesheuvel const char *iv, int rounds, u32 counter);
376*b575b5a1SArd Biesheuvel
377*b575b5a1SArd Biesheuvel int pmull_gcm_dec_final(int bytes, u64 dg[], char *tag,
378*b575b5a1SArd Biesheuvel struct gcm_key const *k, char *head,
379*b575b5a1SArd Biesheuvel const char *iv, int rounds, u32 counter,
380*b575b5a1SArd Biesheuvel const char *otag, int authsize);
381*b575b5a1SArd Biesheuvel
gcm_aes_setkey(struct crypto_aead * tfm,const u8 * inkey,unsigned int keylen)382*b575b5a1SArd Biesheuvel static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey,
383*b575b5a1SArd Biesheuvel unsigned int keylen)
384*b575b5a1SArd Biesheuvel {
385*b575b5a1SArd Biesheuvel struct gcm_key *ctx = crypto_aead_ctx(tfm);
386*b575b5a1SArd Biesheuvel struct crypto_aes_ctx aes_ctx;
387*b575b5a1SArd Biesheuvel be128 h, k;
388*b575b5a1SArd Biesheuvel int ret;
389*b575b5a1SArd Biesheuvel
390*b575b5a1SArd Biesheuvel ret = aes_expandkey(&aes_ctx, inkey, keylen);
391*b575b5a1SArd Biesheuvel if (ret)
392*b575b5a1SArd Biesheuvel return -EINVAL;
393*b575b5a1SArd Biesheuvel
394*b575b5a1SArd Biesheuvel aes_encrypt(&aes_ctx, (u8 *)&k, (u8[AES_BLOCK_SIZE]){});
395*b575b5a1SArd Biesheuvel
396*b575b5a1SArd Biesheuvel memcpy(ctx->rk, aes_ctx.key_enc, sizeof(ctx->rk));
397*b575b5a1SArd Biesheuvel ctx->rounds = 6 + keylen / 4;
398*b575b5a1SArd Biesheuvel
399*b575b5a1SArd Biesheuvel memzero_explicit(&aes_ctx, sizeof(aes_ctx));
400*b575b5a1SArd Biesheuvel
401*b575b5a1SArd Biesheuvel ghash_reflect(ctx->h[0], &k);
402*b575b5a1SArd Biesheuvel
403*b575b5a1SArd Biesheuvel h = k;
404*b575b5a1SArd Biesheuvel gf128mul_lle(&h, &k);
405*b575b5a1SArd Biesheuvel ghash_reflect(ctx->h[1], &h);
406*b575b5a1SArd Biesheuvel
407*b575b5a1SArd Biesheuvel gf128mul_lle(&h, &k);
408*b575b5a1SArd Biesheuvel ghash_reflect(ctx->h[2], &h);
409*b575b5a1SArd Biesheuvel
410*b575b5a1SArd Biesheuvel gf128mul_lle(&h, &k);
411*b575b5a1SArd Biesheuvel ghash_reflect(ctx->h[3], &h);
412*b575b5a1SArd Biesheuvel
413*b575b5a1SArd Biesheuvel return 0;
414*b575b5a1SArd Biesheuvel }
415*b575b5a1SArd Biesheuvel
gcm_aes_setauthsize(struct crypto_aead * tfm,unsigned int authsize)416*b575b5a1SArd Biesheuvel static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
417*b575b5a1SArd Biesheuvel {
418*b575b5a1SArd Biesheuvel return crypto_gcm_check_authsize(authsize);
419*b575b5a1SArd Biesheuvel }
420*b575b5a1SArd Biesheuvel
gcm_update_mac(u64 dg[],const u8 * src,int count,u8 buf[],int * buf_count,struct gcm_key * ctx)421*b575b5a1SArd Biesheuvel static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[],
422*b575b5a1SArd Biesheuvel int *buf_count, struct gcm_key *ctx)
423*b575b5a1SArd Biesheuvel {
424*b575b5a1SArd Biesheuvel if (*buf_count > 0) {
425*b575b5a1SArd Biesheuvel int buf_added = min(count, GHASH_BLOCK_SIZE - *buf_count);
426*b575b5a1SArd Biesheuvel
427*b575b5a1SArd Biesheuvel memcpy(&buf[*buf_count], src, buf_added);
428*b575b5a1SArd Biesheuvel
429*b575b5a1SArd Biesheuvel *buf_count += buf_added;
430*b575b5a1SArd Biesheuvel src += buf_added;
431*b575b5a1SArd Biesheuvel count -= buf_added;
432*b575b5a1SArd Biesheuvel }
433*b575b5a1SArd Biesheuvel
434*b575b5a1SArd Biesheuvel if (count >= GHASH_BLOCK_SIZE || *buf_count == GHASH_BLOCK_SIZE) {
435*b575b5a1SArd Biesheuvel int blocks = count / GHASH_BLOCK_SIZE;
436*b575b5a1SArd Biesheuvel
437*b575b5a1SArd Biesheuvel pmull_ghash_update_p64(blocks, dg, src, ctx->h,
438*b575b5a1SArd Biesheuvel *buf_count ? buf : NULL);
439*b575b5a1SArd Biesheuvel
440*b575b5a1SArd Biesheuvel src += blocks * GHASH_BLOCK_SIZE;
441*b575b5a1SArd Biesheuvel count %= GHASH_BLOCK_SIZE;
442*b575b5a1SArd Biesheuvel *buf_count = 0;
443*b575b5a1SArd Biesheuvel }
444*b575b5a1SArd Biesheuvel
445*b575b5a1SArd Biesheuvel if (count > 0) {
446*b575b5a1SArd Biesheuvel memcpy(buf, src, count);
447*b575b5a1SArd Biesheuvel *buf_count = count;
448*b575b5a1SArd Biesheuvel }
449*b575b5a1SArd Biesheuvel }
450*b575b5a1SArd Biesheuvel
gcm_calculate_auth_mac(struct aead_request * req,u64 dg[],u32 len)451*b575b5a1SArd Biesheuvel static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len)
452*b575b5a1SArd Biesheuvel {
453*b575b5a1SArd Biesheuvel struct crypto_aead *aead = crypto_aead_reqtfm(req);
454*b575b5a1SArd Biesheuvel struct gcm_key *ctx = crypto_aead_ctx(aead);
455*b575b5a1SArd Biesheuvel u8 buf[GHASH_BLOCK_SIZE];
456*b575b5a1SArd Biesheuvel struct scatter_walk walk;
457*b575b5a1SArd Biesheuvel int buf_count = 0;
458*b575b5a1SArd Biesheuvel
459*b575b5a1SArd Biesheuvel scatterwalk_start(&walk, req->src);
460*b575b5a1SArd Biesheuvel
461*b575b5a1SArd Biesheuvel do {
462*b575b5a1SArd Biesheuvel u32 n = scatterwalk_clamp(&walk, len);
463*b575b5a1SArd Biesheuvel u8 *p;
464*b575b5a1SArd Biesheuvel
465*b575b5a1SArd Biesheuvel if (!n) {
466*b575b5a1SArd Biesheuvel scatterwalk_start(&walk, sg_next(walk.sg));
467*b575b5a1SArd Biesheuvel n = scatterwalk_clamp(&walk, len);
468*b575b5a1SArd Biesheuvel }
469*b575b5a1SArd Biesheuvel
470*b575b5a1SArd Biesheuvel p = scatterwalk_map(&walk);
471*b575b5a1SArd Biesheuvel gcm_update_mac(dg, p, n, buf, &buf_count, ctx);
472*b575b5a1SArd Biesheuvel scatterwalk_unmap(p);
473*b575b5a1SArd Biesheuvel
474*b575b5a1SArd Biesheuvel if (unlikely(len / SZ_4K > (len - n) / SZ_4K)) {
475*b575b5a1SArd Biesheuvel kernel_neon_end();
476*b575b5a1SArd Biesheuvel kernel_neon_begin();
477*b575b5a1SArd Biesheuvel }
478*b575b5a1SArd Biesheuvel
479*b575b5a1SArd Biesheuvel len -= n;
480*b575b5a1SArd Biesheuvel scatterwalk_advance(&walk, n);
481*b575b5a1SArd Biesheuvel scatterwalk_done(&walk, 0, len);
482*b575b5a1SArd Biesheuvel } while (len);
483*b575b5a1SArd Biesheuvel
484*b575b5a1SArd Biesheuvel if (buf_count) {
485*b575b5a1SArd Biesheuvel memset(&buf[buf_count], 0, GHASH_BLOCK_SIZE - buf_count);
486*b575b5a1SArd Biesheuvel pmull_ghash_update_p64(1, dg, buf, ctx->h, NULL);
487*b575b5a1SArd Biesheuvel }
488*b575b5a1SArd Biesheuvel }
489*b575b5a1SArd Biesheuvel
gcm_encrypt(struct aead_request * req,const u8 * iv,u32 assoclen)490*b575b5a1SArd Biesheuvel static int gcm_encrypt(struct aead_request *req, const u8 *iv, u32 assoclen)
491*b575b5a1SArd Biesheuvel {
492*b575b5a1SArd Biesheuvel struct crypto_aead *aead = crypto_aead_reqtfm(req);
493*b575b5a1SArd Biesheuvel struct gcm_key *ctx = crypto_aead_ctx(aead);
494*b575b5a1SArd Biesheuvel struct skcipher_walk walk;
495*b575b5a1SArd Biesheuvel u8 buf[AES_BLOCK_SIZE];
496*b575b5a1SArd Biesheuvel u32 counter = 2;
497*b575b5a1SArd Biesheuvel u64 dg[2] = {};
498*b575b5a1SArd Biesheuvel be128 lengths;
499*b575b5a1SArd Biesheuvel const u8 *src;
500*b575b5a1SArd Biesheuvel u8 *tag, *dst;
501*b575b5a1SArd Biesheuvel int tail, err;
502*b575b5a1SArd Biesheuvel
503*b575b5a1SArd Biesheuvel if (WARN_ON_ONCE(!may_use_simd()))
504*b575b5a1SArd Biesheuvel return -EBUSY;
505*b575b5a1SArd Biesheuvel
506*b575b5a1SArd Biesheuvel err = skcipher_walk_aead_encrypt(&walk, req, false);
507*b575b5a1SArd Biesheuvel
508*b575b5a1SArd Biesheuvel kernel_neon_begin();
509*b575b5a1SArd Biesheuvel
510*b575b5a1SArd Biesheuvel if (assoclen)
511*b575b5a1SArd Biesheuvel gcm_calculate_auth_mac(req, dg, assoclen);
512*b575b5a1SArd Biesheuvel
513*b575b5a1SArd Biesheuvel src = walk.src.virt.addr;
514*b575b5a1SArd Biesheuvel dst = walk.dst.virt.addr;
515*b575b5a1SArd Biesheuvel
516*b575b5a1SArd Biesheuvel while (walk.nbytes >= AES_BLOCK_SIZE) {
517*b575b5a1SArd Biesheuvel int nblocks = walk.nbytes / AES_BLOCK_SIZE;
518*b575b5a1SArd Biesheuvel
519*b575b5a1SArd Biesheuvel pmull_gcm_encrypt(nblocks, dg, src, ctx, dst, iv,
520*b575b5a1SArd Biesheuvel ctx->rounds, counter);
521*b575b5a1SArd Biesheuvel counter += nblocks;
522*b575b5a1SArd Biesheuvel
523*b575b5a1SArd Biesheuvel if (walk.nbytes == walk.total) {
524*b575b5a1SArd Biesheuvel src += nblocks * AES_BLOCK_SIZE;
525*b575b5a1SArd Biesheuvel dst += nblocks * AES_BLOCK_SIZE;
526*b575b5a1SArd Biesheuvel break;
527*b575b5a1SArd Biesheuvel }
528*b575b5a1SArd Biesheuvel
529*b575b5a1SArd Biesheuvel kernel_neon_end();
530*b575b5a1SArd Biesheuvel
531*b575b5a1SArd Biesheuvel err = skcipher_walk_done(&walk,
532*b575b5a1SArd Biesheuvel walk.nbytes % AES_BLOCK_SIZE);
533*b575b5a1SArd Biesheuvel if (err)
534*b575b5a1SArd Biesheuvel return err;
535*b575b5a1SArd Biesheuvel
536*b575b5a1SArd Biesheuvel src = walk.src.virt.addr;
537*b575b5a1SArd Biesheuvel dst = walk.dst.virt.addr;
538*b575b5a1SArd Biesheuvel
539*b575b5a1SArd Biesheuvel kernel_neon_begin();
540*b575b5a1SArd Biesheuvel }
541*b575b5a1SArd Biesheuvel
542*b575b5a1SArd Biesheuvel
543*b575b5a1SArd Biesheuvel lengths.a = cpu_to_be64(assoclen * 8);
544*b575b5a1SArd Biesheuvel lengths.b = cpu_to_be64(req->cryptlen * 8);
545*b575b5a1SArd Biesheuvel
546*b575b5a1SArd Biesheuvel tag = (u8 *)&lengths;
547*b575b5a1SArd Biesheuvel tail = walk.nbytes % AES_BLOCK_SIZE;
548*b575b5a1SArd Biesheuvel
549*b575b5a1SArd Biesheuvel /*
550*b575b5a1SArd Biesheuvel * Bounce via a buffer unless we are encrypting in place and src/dst
551*b575b5a1SArd Biesheuvel * are not pointing to the start of the walk buffer. In that case, we
552*b575b5a1SArd Biesheuvel * can do a NEON load/xor/store sequence in place as long as we move
553*b575b5a1SArd Biesheuvel * the plain/ciphertext and keystream to the start of the register. If
554*b575b5a1SArd Biesheuvel * not, do a memcpy() to the end of the buffer so we can reuse the same
555*b575b5a1SArd Biesheuvel * logic.
556*b575b5a1SArd Biesheuvel */
557*b575b5a1SArd Biesheuvel if (unlikely(tail && (tail == walk.nbytes || src != dst)))
558*b575b5a1SArd Biesheuvel src = memcpy(buf + sizeof(buf) - tail, src, tail);
559*b575b5a1SArd Biesheuvel
560*b575b5a1SArd Biesheuvel pmull_gcm_enc_final(tail, dg, tag, ctx, (u8 *)src, iv,
561*b575b5a1SArd Biesheuvel ctx->rounds, counter);
562*b575b5a1SArd Biesheuvel kernel_neon_end();
563*b575b5a1SArd Biesheuvel
564*b575b5a1SArd Biesheuvel if (unlikely(tail && src != dst))
565*b575b5a1SArd Biesheuvel memcpy(dst, src, tail);
566*b575b5a1SArd Biesheuvel
567*b575b5a1SArd Biesheuvel if (walk.nbytes) {
568*b575b5a1SArd Biesheuvel err = skcipher_walk_done(&walk, 0);
569*b575b5a1SArd Biesheuvel if (err)
570*b575b5a1SArd Biesheuvel return err;
571*b575b5a1SArd Biesheuvel }
572*b575b5a1SArd Biesheuvel
573*b575b5a1SArd Biesheuvel /* copy authtag to end of dst */
574*b575b5a1SArd Biesheuvel scatterwalk_map_and_copy(tag, req->dst, req->assoclen + req->cryptlen,
575*b575b5a1SArd Biesheuvel crypto_aead_authsize(aead), 1);
576*b575b5a1SArd Biesheuvel
577*b575b5a1SArd Biesheuvel return 0;
578*b575b5a1SArd Biesheuvel }
579*b575b5a1SArd Biesheuvel
gcm_decrypt(struct aead_request * req,const u8 * iv,u32 assoclen)580*b575b5a1SArd Biesheuvel static int gcm_decrypt(struct aead_request *req, const u8 *iv, u32 assoclen)
581*b575b5a1SArd Biesheuvel {
582*b575b5a1SArd Biesheuvel struct crypto_aead *aead = crypto_aead_reqtfm(req);
583*b575b5a1SArd Biesheuvel struct gcm_key *ctx = crypto_aead_ctx(aead);
584*b575b5a1SArd Biesheuvel int authsize = crypto_aead_authsize(aead);
585*b575b5a1SArd Biesheuvel struct skcipher_walk walk;
586*b575b5a1SArd Biesheuvel u8 otag[AES_BLOCK_SIZE];
587*b575b5a1SArd Biesheuvel u8 buf[AES_BLOCK_SIZE];
588*b575b5a1SArd Biesheuvel u32 counter = 2;
589*b575b5a1SArd Biesheuvel u64 dg[2] = {};
590*b575b5a1SArd Biesheuvel be128 lengths;
591*b575b5a1SArd Biesheuvel const u8 *src;
592*b575b5a1SArd Biesheuvel u8 *tag, *dst;
593*b575b5a1SArd Biesheuvel int tail, err, ret;
594*b575b5a1SArd Biesheuvel
595*b575b5a1SArd Biesheuvel if (WARN_ON_ONCE(!may_use_simd()))
596*b575b5a1SArd Biesheuvel return -EBUSY;
597*b575b5a1SArd Biesheuvel
598*b575b5a1SArd Biesheuvel scatterwalk_map_and_copy(otag, req->src,
599*b575b5a1SArd Biesheuvel req->assoclen + req->cryptlen - authsize,
600*b575b5a1SArd Biesheuvel authsize, 0);
601*b575b5a1SArd Biesheuvel
602*b575b5a1SArd Biesheuvel err = skcipher_walk_aead_decrypt(&walk, req, false);
603*b575b5a1SArd Biesheuvel
604*b575b5a1SArd Biesheuvel kernel_neon_begin();
605*b575b5a1SArd Biesheuvel
606*b575b5a1SArd Biesheuvel if (assoclen)
607*b575b5a1SArd Biesheuvel gcm_calculate_auth_mac(req, dg, assoclen);
608*b575b5a1SArd Biesheuvel
609*b575b5a1SArd Biesheuvel src = walk.src.virt.addr;
610*b575b5a1SArd Biesheuvel dst = walk.dst.virt.addr;
611*b575b5a1SArd Biesheuvel
612*b575b5a1SArd Biesheuvel while (walk.nbytes >= AES_BLOCK_SIZE) {
613*b575b5a1SArd Biesheuvel int nblocks = walk.nbytes / AES_BLOCK_SIZE;
614*b575b5a1SArd Biesheuvel
615*b575b5a1SArd Biesheuvel pmull_gcm_decrypt(nblocks, dg, src, ctx, dst, iv,
616*b575b5a1SArd Biesheuvel ctx->rounds, counter);
617*b575b5a1SArd Biesheuvel counter += nblocks;
618*b575b5a1SArd Biesheuvel
619*b575b5a1SArd Biesheuvel if (walk.nbytes == walk.total) {
620*b575b5a1SArd Biesheuvel src += nblocks * AES_BLOCK_SIZE;
621*b575b5a1SArd Biesheuvel dst += nblocks * AES_BLOCK_SIZE;
622*b575b5a1SArd Biesheuvel break;
623*b575b5a1SArd Biesheuvel }
624*b575b5a1SArd Biesheuvel
625*b575b5a1SArd Biesheuvel kernel_neon_end();
626*b575b5a1SArd Biesheuvel
627*b575b5a1SArd Biesheuvel err = skcipher_walk_done(&walk,
628*b575b5a1SArd Biesheuvel walk.nbytes % AES_BLOCK_SIZE);
629*b575b5a1SArd Biesheuvel if (err)
630*b575b5a1SArd Biesheuvel return err;
631*b575b5a1SArd Biesheuvel
632*b575b5a1SArd Biesheuvel src = walk.src.virt.addr;
633*b575b5a1SArd Biesheuvel dst = walk.dst.virt.addr;
634*b575b5a1SArd Biesheuvel
635*b575b5a1SArd Biesheuvel kernel_neon_begin();
636*b575b5a1SArd Biesheuvel }
637*b575b5a1SArd Biesheuvel
638*b575b5a1SArd Biesheuvel lengths.a = cpu_to_be64(assoclen * 8);
639*b575b5a1SArd Biesheuvel lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8);
640*b575b5a1SArd Biesheuvel
641*b575b5a1SArd Biesheuvel tag = (u8 *)&lengths;
642*b575b5a1SArd Biesheuvel tail = walk.nbytes % AES_BLOCK_SIZE;
643*b575b5a1SArd Biesheuvel
644*b575b5a1SArd Biesheuvel if (unlikely(tail && (tail == walk.nbytes || src != dst)))
645*b575b5a1SArd Biesheuvel src = memcpy(buf + sizeof(buf) - tail, src, tail);
646*b575b5a1SArd Biesheuvel
647*b575b5a1SArd Biesheuvel ret = pmull_gcm_dec_final(tail, dg, tag, ctx, (u8 *)src, iv,
648*b575b5a1SArd Biesheuvel ctx->rounds, counter, otag, authsize);
649*b575b5a1SArd Biesheuvel kernel_neon_end();
650*b575b5a1SArd Biesheuvel
651*b575b5a1SArd Biesheuvel if (unlikely(tail && src != dst))
652*b575b5a1SArd Biesheuvel memcpy(dst, src, tail);
653*b575b5a1SArd Biesheuvel
654*b575b5a1SArd Biesheuvel if (walk.nbytes) {
655*b575b5a1SArd Biesheuvel err = skcipher_walk_done(&walk, 0);
656*b575b5a1SArd Biesheuvel if (err)
657*b575b5a1SArd Biesheuvel return err;
658*b575b5a1SArd Biesheuvel }
659*b575b5a1SArd Biesheuvel
660*b575b5a1SArd Biesheuvel return ret ? -EBADMSG : 0;
661*b575b5a1SArd Biesheuvel }
662*b575b5a1SArd Biesheuvel
gcm_aes_encrypt(struct aead_request * req)663*b575b5a1SArd Biesheuvel static int gcm_aes_encrypt(struct aead_request *req)
664*b575b5a1SArd Biesheuvel {
665*b575b5a1SArd Biesheuvel return gcm_encrypt(req, req->iv, req->assoclen);
666*b575b5a1SArd Biesheuvel }
667*b575b5a1SArd Biesheuvel
gcm_aes_decrypt(struct aead_request * req)668*b575b5a1SArd Biesheuvel static int gcm_aes_decrypt(struct aead_request *req)
669*b575b5a1SArd Biesheuvel {
670*b575b5a1SArd Biesheuvel return gcm_decrypt(req, req->iv, req->assoclen);
671*b575b5a1SArd Biesheuvel }
672*b575b5a1SArd Biesheuvel
rfc4106_setkey(struct crypto_aead * tfm,const u8 * inkey,unsigned int keylen)673*b575b5a1SArd Biesheuvel static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey,
674*b575b5a1SArd Biesheuvel unsigned int keylen)
675*b575b5a1SArd Biesheuvel {
676*b575b5a1SArd Biesheuvel struct gcm_key *ctx = crypto_aead_ctx(tfm);
677*b575b5a1SArd Biesheuvel int err;
678*b575b5a1SArd Biesheuvel
679*b575b5a1SArd Biesheuvel keylen -= RFC4106_NONCE_SIZE;
680*b575b5a1SArd Biesheuvel err = gcm_aes_setkey(tfm, inkey, keylen);
681*b575b5a1SArd Biesheuvel if (err)
682*b575b5a1SArd Biesheuvel return err;
683*b575b5a1SArd Biesheuvel
684*b575b5a1SArd Biesheuvel memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE);
685*b575b5a1SArd Biesheuvel return 0;
686*b575b5a1SArd Biesheuvel }
687*b575b5a1SArd Biesheuvel
rfc4106_setauthsize(struct crypto_aead * tfm,unsigned int authsize)688*b575b5a1SArd Biesheuvel static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
689*b575b5a1SArd Biesheuvel {
690*b575b5a1SArd Biesheuvel return crypto_rfc4106_check_authsize(authsize);
691*b575b5a1SArd Biesheuvel }
692*b575b5a1SArd Biesheuvel
rfc4106_encrypt(struct aead_request * req)693*b575b5a1SArd Biesheuvel static int rfc4106_encrypt(struct aead_request *req)
694*b575b5a1SArd Biesheuvel {
695*b575b5a1SArd Biesheuvel struct crypto_aead *aead = crypto_aead_reqtfm(req);
696*b575b5a1SArd Biesheuvel struct gcm_key *ctx = crypto_aead_ctx(aead);
697*b575b5a1SArd Biesheuvel u8 iv[GCM_AES_IV_SIZE];
698*b575b5a1SArd Biesheuvel
699*b575b5a1SArd Biesheuvel memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);
700*b575b5a1SArd Biesheuvel memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);
701*b575b5a1SArd Biesheuvel
702*b575b5a1SArd Biesheuvel return crypto_ipsec_check_assoclen(req->assoclen) ?:
703*b575b5a1SArd Biesheuvel gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);
704*b575b5a1SArd Biesheuvel }
705*b575b5a1SArd Biesheuvel
rfc4106_decrypt(struct aead_request * req)706*b575b5a1SArd Biesheuvel static int rfc4106_decrypt(struct aead_request *req)
707*b575b5a1SArd Biesheuvel {
708*b575b5a1SArd Biesheuvel struct crypto_aead *aead = crypto_aead_reqtfm(req);
709*b575b5a1SArd Biesheuvel struct gcm_key *ctx = crypto_aead_ctx(aead);
710*b575b5a1SArd Biesheuvel u8 iv[GCM_AES_IV_SIZE];
711*b575b5a1SArd Biesheuvel
712*b575b5a1SArd Biesheuvel memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);
713*b575b5a1SArd Biesheuvel memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);
714*b575b5a1SArd Biesheuvel
715*b575b5a1SArd Biesheuvel return crypto_ipsec_check_assoclen(req->assoclen) ?:
716*b575b5a1SArd Biesheuvel gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);
717*b575b5a1SArd Biesheuvel }
718*b575b5a1SArd Biesheuvel
719*b575b5a1SArd Biesheuvel static struct aead_alg gcm_aes_algs[] = {{
720*b575b5a1SArd Biesheuvel .ivsize = GCM_AES_IV_SIZE,
721*b575b5a1SArd Biesheuvel .chunksize = AES_BLOCK_SIZE,
722*b575b5a1SArd Biesheuvel .maxauthsize = AES_BLOCK_SIZE,
723*b575b5a1SArd Biesheuvel .setkey = gcm_aes_setkey,
724*b575b5a1SArd Biesheuvel .setauthsize = gcm_aes_setauthsize,
725*b575b5a1SArd Biesheuvel .encrypt = gcm_aes_encrypt,
726*b575b5a1SArd Biesheuvel .decrypt = gcm_aes_decrypt,
727*b575b5a1SArd Biesheuvel
728*b575b5a1SArd Biesheuvel .base.cra_name = "gcm(aes)",
729*b575b5a1SArd Biesheuvel .base.cra_driver_name = "gcm-aes-ce",
730*b575b5a1SArd Biesheuvel .base.cra_priority = 400,
731*b575b5a1SArd Biesheuvel .base.cra_blocksize = 1,
732*b575b5a1SArd Biesheuvel .base.cra_ctxsize = sizeof(struct gcm_key),
733*b575b5a1SArd Biesheuvel .base.cra_module = THIS_MODULE,
734*b575b5a1SArd Biesheuvel }, {
735*b575b5a1SArd Biesheuvel .ivsize = GCM_RFC4106_IV_SIZE,
736*b575b5a1SArd Biesheuvel .chunksize = AES_BLOCK_SIZE,
737*b575b5a1SArd Biesheuvel .maxauthsize = AES_BLOCK_SIZE,
738*b575b5a1SArd Biesheuvel .setkey = rfc4106_setkey,
739*b575b5a1SArd Biesheuvel .setauthsize = rfc4106_setauthsize,
740*b575b5a1SArd Biesheuvel .encrypt = rfc4106_encrypt,
741*b575b5a1SArd Biesheuvel .decrypt = rfc4106_decrypt,
742*b575b5a1SArd Biesheuvel
743*b575b5a1SArd Biesheuvel .base.cra_name = "rfc4106(gcm(aes))",
744*b575b5a1SArd Biesheuvel .base.cra_driver_name = "rfc4106-gcm-aes-ce",
745*b575b5a1SArd Biesheuvel .base.cra_priority = 400,
746*b575b5a1SArd Biesheuvel .base.cra_blocksize = 1,
747*b575b5a1SArd Biesheuvel .base.cra_ctxsize = sizeof(struct gcm_key) + RFC4106_NONCE_SIZE,
748*b575b5a1SArd Biesheuvel .base.cra_module = THIS_MODULE,
749*b575b5a1SArd Biesheuvel }};
750*b575b5a1SArd Biesheuvel
ghash_ce_mod_init(void)751f1e866b1SArd Biesheuvel static int __init ghash_ce_mod_init(void)
752f1e866b1SArd Biesheuvel {
753f1e866b1SArd Biesheuvel int err;
754f1e866b1SArd Biesheuvel
7553759ee05SArd Biesheuvel if (!(elf_hwcap & HWCAP_NEON))
7563759ee05SArd Biesheuvel return -ENODEV;
7573759ee05SArd Biesheuvel
7583d2df845SArd Biesheuvel if (elf_hwcap2 & HWCAP2_PMULL) {
759*b575b5a1SArd Biesheuvel err = crypto_register_aeads(gcm_aes_algs,
760*b575b5a1SArd Biesheuvel ARRAY_SIZE(gcm_aes_algs));
761*b575b5a1SArd Biesheuvel if (err)
762*b575b5a1SArd Biesheuvel return err;
7633d2df845SArd Biesheuvel ghash_alg.base.cra_ctxsize += 3 * sizeof(u64[2]);
7643d2df845SArd Biesheuvel static_branch_enable(&use_p64);
7653d2df845SArd Biesheuvel }
7663759ee05SArd Biesheuvel
767f1e866b1SArd Biesheuvel err = crypto_register_shash(&ghash_alg);
768f1e866b1SArd Biesheuvel if (err)
769*b575b5a1SArd Biesheuvel goto err_aead;
770f1e866b1SArd Biesheuvel err = crypto_register_ahash(&ghash_async_alg);
771f1e866b1SArd Biesheuvel if (err)
772f1e866b1SArd Biesheuvel goto err_shash;
773f1e866b1SArd Biesheuvel
774f1e866b1SArd Biesheuvel return 0;
775f1e866b1SArd Biesheuvel
776f1e866b1SArd Biesheuvel err_shash:
777f1e866b1SArd Biesheuvel crypto_unregister_shash(&ghash_alg);
778*b575b5a1SArd Biesheuvel err_aead:
779*b575b5a1SArd Biesheuvel if (elf_hwcap2 & HWCAP2_PMULL)
780*b575b5a1SArd Biesheuvel crypto_unregister_aeads(gcm_aes_algs,
781*b575b5a1SArd Biesheuvel ARRAY_SIZE(gcm_aes_algs));
782f1e866b1SArd Biesheuvel return err;
783f1e866b1SArd Biesheuvel }
784f1e866b1SArd Biesheuvel
ghash_ce_mod_exit(void)785f1e866b1SArd Biesheuvel static void __exit ghash_ce_mod_exit(void)
786f1e866b1SArd Biesheuvel {
787f1e866b1SArd Biesheuvel crypto_unregister_ahash(&ghash_async_alg);
788f1e866b1SArd Biesheuvel crypto_unregister_shash(&ghash_alg);
789*b575b5a1SArd Biesheuvel if (elf_hwcap2 & HWCAP2_PMULL)
790*b575b5a1SArd Biesheuvel crypto_unregister_aeads(gcm_aes_algs,
791*b575b5a1SArd Biesheuvel ARRAY_SIZE(gcm_aes_algs));
792f1e866b1SArd Biesheuvel }
793f1e866b1SArd Biesheuvel
7943759ee05SArd Biesheuvel module_init(ghash_ce_mod_init);
795f1e866b1SArd Biesheuvel module_exit(ghash_ce_mod_exit);
796