1*d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2f1e866b1SArd Biesheuvel /* 3f1e866b1SArd Biesheuvel * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions. 4f1e866b1SArd Biesheuvel * 500227e3aSArd Biesheuvel * Copyright (C) 2015 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org> 6f1e866b1SArd Biesheuvel */ 7f1e866b1SArd Biesheuvel 8f1e866b1SArd Biesheuvel #include <asm/hwcap.h> 9f1e866b1SArd Biesheuvel #include <asm/neon.h> 10f1e866b1SArd Biesheuvel #include <asm/simd.h> 11f1e866b1SArd Biesheuvel #include <asm/unaligned.h> 12f1e866b1SArd Biesheuvel #include <crypto/cryptd.h> 13f1e866b1SArd Biesheuvel #include <crypto/internal/hash.h> 1499680c5eSEric Biggers #include <crypto/internal/simd.h> 15f1e866b1SArd Biesheuvel #include <crypto/gf128mul.h> 16c9d9f608SArd Biesheuvel #include <linux/cpufeature.h> 17f1e866b1SArd Biesheuvel #include <linux/crypto.h> 18f1e866b1SArd Biesheuvel #include <linux/module.h> 19f1e866b1SArd Biesheuvel 20f1e866b1SArd Biesheuvel MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions"); 21f1e866b1SArd Biesheuvel MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 22f1e866b1SArd Biesheuvel MODULE_LICENSE("GPL v2"); 233759ee05SArd Biesheuvel MODULE_ALIAS_CRYPTO("ghash"); 24f1e866b1SArd Biesheuvel 25f1e866b1SArd Biesheuvel #define GHASH_BLOCK_SIZE 16 26f1e866b1SArd Biesheuvel #define GHASH_DIGEST_SIZE 16 27f1e866b1SArd Biesheuvel 28f1e866b1SArd Biesheuvel struct ghash_key { 2900227e3aSArd Biesheuvel u64 h[2]; 3000227e3aSArd Biesheuvel u64 h2[2]; 3100227e3aSArd Biesheuvel u64 h3[2]; 3200227e3aSArd Biesheuvel u64 h4[2]; 33f1e866b1SArd Biesheuvel }; 34f1e866b1SArd Biesheuvel 35f1e866b1SArd Biesheuvel struct ghash_desc_ctx { 36f1e866b1SArd Biesheuvel u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)]; 37f1e866b1SArd Biesheuvel u8 buf[GHASH_BLOCK_SIZE]; 38f1e866b1SArd Biesheuvel u32 count; 39f1e866b1SArd Biesheuvel }; 40f1e866b1SArd Biesheuvel 41f1e866b1SArd Biesheuvel struct ghash_async_ctx { 42f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm; 43f1e866b1SArd Biesheuvel }; 44f1e866b1SArd Biesheuvel 453759ee05SArd Biesheuvel asmlinkage void pmull_ghash_update_p64(int blocks, u64 dg[], const char *src, 463759ee05SArd Biesheuvel struct ghash_key const *k, 473759ee05SArd Biesheuvel const char *head); 483759ee05SArd Biesheuvel 493759ee05SArd Biesheuvel asmlinkage void pmull_ghash_update_p8(int blocks, u64 dg[], const char *src, 503759ee05SArd Biesheuvel struct ghash_key const *k, 513759ee05SArd Biesheuvel const char *head); 523759ee05SArd Biesheuvel 533759ee05SArd Biesheuvel static void (*pmull_ghash_update)(int blocks, u64 dg[], const char *src, 543759ee05SArd Biesheuvel struct ghash_key const *k, 553759ee05SArd Biesheuvel const char *head); 56f1e866b1SArd Biesheuvel 57f1e866b1SArd Biesheuvel static int ghash_init(struct shash_desc *desc) 58f1e866b1SArd Biesheuvel { 59f1e866b1SArd Biesheuvel struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 60f1e866b1SArd Biesheuvel 61f1e866b1SArd Biesheuvel *ctx = (struct ghash_desc_ctx){}; 62f1e866b1SArd Biesheuvel return 0; 63f1e866b1SArd Biesheuvel } 64f1e866b1SArd Biesheuvel 65f1e866b1SArd Biesheuvel static int ghash_update(struct shash_desc *desc, const u8 *src, 66f1e866b1SArd Biesheuvel unsigned int len) 67f1e866b1SArd Biesheuvel { 68f1e866b1SArd Biesheuvel struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 69f1e866b1SArd Biesheuvel unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 70f1e866b1SArd Biesheuvel 71f1e866b1SArd Biesheuvel ctx->count += len; 72f1e866b1SArd Biesheuvel 73f1e866b1SArd Biesheuvel if ((partial + len) >= GHASH_BLOCK_SIZE) { 74f1e866b1SArd Biesheuvel struct ghash_key *key = crypto_shash_ctx(desc->tfm); 75f1e866b1SArd Biesheuvel int blocks; 76f1e866b1SArd Biesheuvel 77f1e866b1SArd Biesheuvel if (partial) { 78f1e866b1SArd Biesheuvel int p = GHASH_BLOCK_SIZE - partial; 79f1e866b1SArd Biesheuvel 80f1e866b1SArd Biesheuvel memcpy(ctx->buf + partial, src, p); 81f1e866b1SArd Biesheuvel src += p; 82f1e866b1SArd Biesheuvel len -= p; 83f1e866b1SArd Biesheuvel } 84f1e866b1SArd Biesheuvel 85f1e866b1SArd Biesheuvel blocks = len / GHASH_BLOCK_SIZE; 86f1e866b1SArd Biesheuvel len %= GHASH_BLOCK_SIZE; 87f1e866b1SArd Biesheuvel 88f1e866b1SArd Biesheuvel kernel_neon_begin(); 89f1e866b1SArd Biesheuvel pmull_ghash_update(blocks, ctx->digest, src, key, 90f1e866b1SArd Biesheuvel partial ? ctx->buf : NULL); 91f1e866b1SArd Biesheuvel kernel_neon_end(); 92f1e866b1SArd Biesheuvel src += blocks * GHASH_BLOCK_SIZE; 93f1e866b1SArd Biesheuvel partial = 0; 94f1e866b1SArd Biesheuvel } 95f1e866b1SArd Biesheuvel if (len) 96f1e866b1SArd Biesheuvel memcpy(ctx->buf + partial, src, len); 97f1e866b1SArd Biesheuvel return 0; 98f1e866b1SArd Biesheuvel } 99f1e866b1SArd Biesheuvel 100f1e866b1SArd Biesheuvel static int ghash_final(struct shash_desc *desc, u8 *dst) 101f1e866b1SArd Biesheuvel { 102f1e866b1SArd Biesheuvel struct ghash_desc_ctx *ctx = shash_desc_ctx(desc); 103f1e866b1SArd Biesheuvel unsigned int partial = ctx->count % GHASH_BLOCK_SIZE; 104f1e866b1SArd Biesheuvel 105f1e866b1SArd Biesheuvel if (partial) { 106f1e866b1SArd Biesheuvel struct ghash_key *key = crypto_shash_ctx(desc->tfm); 107f1e866b1SArd Biesheuvel 108f1e866b1SArd Biesheuvel memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial); 109f1e866b1SArd Biesheuvel kernel_neon_begin(); 110f1e866b1SArd Biesheuvel pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL); 111f1e866b1SArd Biesheuvel kernel_neon_end(); 112f1e866b1SArd Biesheuvel } 113f1e866b1SArd Biesheuvel put_unaligned_be64(ctx->digest[1], dst); 114f1e866b1SArd Biesheuvel put_unaligned_be64(ctx->digest[0], dst + 8); 115f1e866b1SArd Biesheuvel 116f1e866b1SArd Biesheuvel *ctx = (struct ghash_desc_ctx){}; 117f1e866b1SArd Biesheuvel return 0; 118f1e866b1SArd Biesheuvel } 119f1e866b1SArd Biesheuvel 12000227e3aSArd Biesheuvel static void ghash_reflect(u64 h[], const be128 *k) 12100227e3aSArd Biesheuvel { 12200227e3aSArd Biesheuvel u64 carry = be64_to_cpu(k->a) >> 63; 12300227e3aSArd Biesheuvel 12400227e3aSArd Biesheuvel h[0] = (be64_to_cpu(k->b) << 1) | carry; 12500227e3aSArd Biesheuvel h[1] = (be64_to_cpu(k->a) << 1) | (be64_to_cpu(k->b) >> 63); 12600227e3aSArd Biesheuvel 12700227e3aSArd Biesheuvel if (carry) 12800227e3aSArd Biesheuvel h[1] ^= 0xc200000000000000UL; 12900227e3aSArd Biesheuvel } 13000227e3aSArd Biesheuvel 131f1e866b1SArd Biesheuvel static int ghash_setkey(struct crypto_shash *tfm, 132f1e866b1SArd Biesheuvel const u8 *inkey, unsigned int keylen) 133f1e866b1SArd Biesheuvel { 134f1e866b1SArd Biesheuvel struct ghash_key *key = crypto_shash_ctx(tfm); 13500227e3aSArd Biesheuvel be128 h, k; 136f1e866b1SArd Biesheuvel 137f1e866b1SArd Biesheuvel if (keylen != GHASH_BLOCK_SIZE) { 138f1e866b1SArd Biesheuvel crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 139f1e866b1SArd Biesheuvel return -EINVAL; 140f1e866b1SArd Biesheuvel } 141f1e866b1SArd Biesheuvel 14200227e3aSArd Biesheuvel memcpy(&k, inkey, GHASH_BLOCK_SIZE); 14300227e3aSArd Biesheuvel ghash_reflect(key->h, &k); 144f1e866b1SArd Biesheuvel 14500227e3aSArd Biesheuvel h = k; 14600227e3aSArd Biesheuvel gf128mul_lle(&h, &k); 14700227e3aSArd Biesheuvel ghash_reflect(key->h2, &h); 148f1e866b1SArd Biesheuvel 14900227e3aSArd Biesheuvel gf128mul_lle(&h, &k); 15000227e3aSArd Biesheuvel ghash_reflect(key->h3, &h); 15100227e3aSArd Biesheuvel 15200227e3aSArd Biesheuvel gf128mul_lle(&h, &k); 15300227e3aSArd Biesheuvel ghash_reflect(key->h4, &h); 154f1e866b1SArd Biesheuvel 155f1e866b1SArd Biesheuvel return 0; 156f1e866b1SArd Biesheuvel } 157f1e866b1SArd Biesheuvel 158f1e866b1SArd Biesheuvel static struct shash_alg ghash_alg = { 159f1e866b1SArd Biesheuvel .digestsize = GHASH_DIGEST_SIZE, 160f1e866b1SArd Biesheuvel .init = ghash_init, 161f1e866b1SArd Biesheuvel .update = ghash_update, 162f1e866b1SArd Biesheuvel .final = ghash_final, 163f1e866b1SArd Biesheuvel .setkey = ghash_setkey, 164f1e866b1SArd Biesheuvel .descsize = sizeof(struct ghash_desc_ctx), 165f1e866b1SArd Biesheuvel .base = { 16671f89917SArd Biesheuvel .cra_name = "__ghash", 167f1e866b1SArd Biesheuvel .cra_driver_name = "__driver-ghash-ce", 168f1e866b1SArd Biesheuvel .cra_priority = 0, 169e50944e2SEric Biggers .cra_flags = CRYPTO_ALG_INTERNAL, 170f1e866b1SArd Biesheuvel .cra_blocksize = GHASH_BLOCK_SIZE, 171f1e866b1SArd Biesheuvel .cra_ctxsize = sizeof(struct ghash_key), 172f1e866b1SArd Biesheuvel .cra_module = THIS_MODULE, 173f1e866b1SArd Biesheuvel }, 174f1e866b1SArd Biesheuvel }; 175f1e866b1SArd Biesheuvel 176f1e866b1SArd Biesheuvel static int ghash_async_init(struct ahash_request *req) 177f1e866b1SArd Biesheuvel { 178f1e866b1SArd Biesheuvel struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 179f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 180f1e866b1SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req); 181f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 182f1e866b1SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 183f1e866b1SArd Biesheuvel struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); 184f1e866b1SArd Biesheuvel 185f1e866b1SArd Biesheuvel desc->tfm = child; 186f1e866b1SArd Biesheuvel return crypto_shash_init(desc); 187f1e866b1SArd Biesheuvel } 188f1e866b1SArd Biesheuvel 189f1e866b1SArd Biesheuvel static int ghash_async_update(struct ahash_request *req) 190f1e866b1SArd Biesheuvel { 191f1e866b1SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req); 192f1e866b1SArd Biesheuvel struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 193f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 194f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 195f1e866b1SArd Biesheuvel 19699680c5eSEric Biggers if (!crypto_simd_usable() || 197820573ebSHerbert Xu (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 198f1e866b1SArd Biesheuvel memcpy(cryptd_req, req, sizeof(*req)); 199f1e866b1SArd Biesheuvel ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 200f1e866b1SArd Biesheuvel return crypto_ahash_update(cryptd_req); 201f1e866b1SArd Biesheuvel } else { 202f1e866b1SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 203f1e866b1SArd Biesheuvel return shash_ahash_update(req, desc); 204f1e866b1SArd Biesheuvel } 205f1e866b1SArd Biesheuvel } 206f1e866b1SArd Biesheuvel 207f1e866b1SArd Biesheuvel static int ghash_async_final(struct ahash_request *req) 208f1e866b1SArd Biesheuvel { 209f1e866b1SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req); 210f1e866b1SArd Biesheuvel struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 211f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 212f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 213f1e866b1SArd Biesheuvel 21499680c5eSEric Biggers if (!crypto_simd_usable() || 215820573ebSHerbert Xu (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 216f1e866b1SArd Biesheuvel memcpy(cryptd_req, req, sizeof(*req)); 217f1e866b1SArd Biesheuvel ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 218f1e866b1SArd Biesheuvel return crypto_ahash_final(cryptd_req); 219f1e866b1SArd Biesheuvel } else { 220f1e866b1SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 221f1e866b1SArd Biesheuvel return crypto_shash_final(desc, req->result); 222f1e866b1SArd Biesheuvel } 223f1e866b1SArd Biesheuvel } 224f1e866b1SArd Biesheuvel 225f1e866b1SArd Biesheuvel static int ghash_async_digest(struct ahash_request *req) 226f1e866b1SArd Biesheuvel { 227f1e866b1SArd Biesheuvel struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 228f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 229f1e866b1SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req); 230f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm; 231f1e866b1SArd Biesheuvel 23299680c5eSEric Biggers if (!crypto_simd_usable() || 233820573ebSHerbert Xu (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) { 234f1e866b1SArd Biesheuvel memcpy(cryptd_req, req, sizeof(*req)); 235f1e866b1SArd Biesheuvel ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base); 236f1e866b1SArd Biesheuvel return crypto_ahash_digest(cryptd_req); 237f1e866b1SArd Biesheuvel } else { 238f1e866b1SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 239f1e866b1SArd Biesheuvel struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm); 240f1e866b1SArd Biesheuvel 241f1e866b1SArd Biesheuvel desc->tfm = child; 242f1e866b1SArd Biesheuvel return shash_ahash_digest(req, desc); 243f1e866b1SArd Biesheuvel } 244f1e866b1SArd Biesheuvel } 245f1e866b1SArd Biesheuvel 246ed4767d6SArd Biesheuvel static int ghash_async_import(struct ahash_request *req, const void *in) 247ed4767d6SArd Biesheuvel { 248ed4767d6SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req); 249ed4767d6SArd Biesheuvel struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 250ed4767d6SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 251ed4767d6SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 252ed4767d6SArd Biesheuvel 253ed4767d6SArd Biesheuvel desc->tfm = cryptd_ahash_child(ctx->cryptd_tfm); 254ed4767d6SArd Biesheuvel 255ed4767d6SArd Biesheuvel return crypto_shash_import(desc, in); 256ed4767d6SArd Biesheuvel } 257ed4767d6SArd Biesheuvel 258ed4767d6SArd Biesheuvel static int ghash_async_export(struct ahash_request *req, void *out) 259ed4767d6SArd Biesheuvel { 260ed4767d6SArd Biesheuvel struct ahash_request *cryptd_req = ahash_request_ctx(req); 261ed4767d6SArd Biesheuvel struct shash_desc *desc = cryptd_shash_desc(cryptd_req); 262ed4767d6SArd Biesheuvel 263ed4767d6SArd Biesheuvel return crypto_shash_export(desc, out); 264ed4767d6SArd Biesheuvel } 265ed4767d6SArd Biesheuvel 266f1e866b1SArd Biesheuvel static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key, 267f1e866b1SArd Biesheuvel unsigned int keylen) 268f1e866b1SArd Biesheuvel { 269f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm); 270f1e866b1SArd Biesheuvel struct crypto_ahash *child = &ctx->cryptd_tfm->base; 271f1e866b1SArd Biesheuvel int err; 272f1e866b1SArd Biesheuvel 273f1e866b1SArd Biesheuvel crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); 274f1e866b1SArd Biesheuvel crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm) 275f1e866b1SArd Biesheuvel & CRYPTO_TFM_REQ_MASK); 276f1e866b1SArd Biesheuvel err = crypto_ahash_setkey(child, key, keylen); 277f1e866b1SArd Biesheuvel crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child) 278f1e866b1SArd Biesheuvel & CRYPTO_TFM_RES_MASK); 279f1e866b1SArd Biesheuvel 280f1e866b1SArd Biesheuvel return err; 281f1e866b1SArd Biesheuvel } 282f1e866b1SArd Biesheuvel 283f1e866b1SArd Biesheuvel static int ghash_async_init_tfm(struct crypto_tfm *tfm) 284f1e866b1SArd Biesheuvel { 285f1e866b1SArd Biesheuvel struct cryptd_ahash *cryptd_tfm; 286f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); 287f1e866b1SArd Biesheuvel 2884b3f4e37SStephan Mueller cryptd_tfm = cryptd_alloc_ahash("__driver-ghash-ce", 2894b3f4e37SStephan Mueller CRYPTO_ALG_INTERNAL, 2904b3f4e37SStephan Mueller CRYPTO_ALG_INTERNAL); 291f1e866b1SArd Biesheuvel if (IS_ERR(cryptd_tfm)) 292f1e866b1SArd Biesheuvel return PTR_ERR(cryptd_tfm); 293f1e866b1SArd Biesheuvel ctx->cryptd_tfm = cryptd_tfm; 294f1e866b1SArd Biesheuvel crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 295f1e866b1SArd Biesheuvel sizeof(struct ahash_request) + 296f1e866b1SArd Biesheuvel crypto_ahash_reqsize(&cryptd_tfm->base)); 297f1e866b1SArd Biesheuvel 298f1e866b1SArd Biesheuvel return 0; 299f1e866b1SArd Biesheuvel } 300f1e866b1SArd Biesheuvel 301f1e866b1SArd Biesheuvel static void ghash_async_exit_tfm(struct crypto_tfm *tfm) 302f1e866b1SArd Biesheuvel { 303f1e866b1SArd Biesheuvel struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm); 304f1e866b1SArd Biesheuvel 305f1e866b1SArd Biesheuvel cryptd_free_ahash(ctx->cryptd_tfm); 306f1e866b1SArd Biesheuvel } 307f1e866b1SArd Biesheuvel 308f1e866b1SArd Biesheuvel static struct ahash_alg ghash_async_alg = { 309f1e866b1SArd Biesheuvel .init = ghash_async_init, 310f1e866b1SArd Biesheuvel .update = ghash_async_update, 311f1e866b1SArd Biesheuvel .final = ghash_async_final, 312f1e866b1SArd Biesheuvel .setkey = ghash_async_setkey, 313f1e866b1SArd Biesheuvel .digest = ghash_async_digest, 314ed4767d6SArd Biesheuvel .import = ghash_async_import, 315ed4767d6SArd Biesheuvel .export = ghash_async_export, 316f1e866b1SArd Biesheuvel .halg.digestsize = GHASH_DIGEST_SIZE, 317ed4767d6SArd Biesheuvel .halg.statesize = sizeof(struct ghash_desc_ctx), 318f1e866b1SArd Biesheuvel .halg.base = { 319f1e866b1SArd Biesheuvel .cra_name = "ghash", 320f1e866b1SArd Biesheuvel .cra_driver_name = "ghash-ce", 321f1e866b1SArd Biesheuvel .cra_priority = 300, 3226a38f622SEric Biggers .cra_flags = CRYPTO_ALG_ASYNC, 323f1e866b1SArd Biesheuvel .cra_blocksize = GHASH_BLOCK_SIZE, 324f1e866b1SArd Biesheuvel .cra_ctxsize = sizeof(struct ghash_async_ctx), 325f1e866b1SArd Biesheuvel .cra_module = THIS_MODULE, 326f1e866b1SArd Biesheuvel .cra_init = ghash_async_init_tfm, 327f1e866b1SArd Biesheuvel .cra_exit = ghash_async_exit_tfm, 328f1e866b1SArd Biesheuvel }, 329f1e866b1SArd Biesheuvel }; 330f1e866b1SArd Biesheuvel 331f1e866b1SArd Biesheuvel static int __init ghash_ce_mod_init(void) 332f1e866b1SArd Biesheuvel { 333f1e866b1SArd Biesheuvel int err; 334f1e866b1SArd Biesheuvel 3353759ee05SArd Biesheuvel if (!(elf_hwcap & HWCAP_NEON)) 3363759ee05SArd Biesheuvel return -ENODEV; 3373759ee05SArd Biesheuvel 3383759ee05SArd Biesheuvel if (elf_hwcap2 & HWCAP2_PMULL) 3393759ee05SArd Biesheuvel pmull_ghash_update = pmull_ghash_update_p64; 3403759ee05SArd Biesheuvel else 3413759ee05SArd Biesheuvel pmull_ghash_update = pmull_ghash_update_p8; 3423759ee05SArd Biesheuvel 343f1e866b1SArd Biesheuvel err = crypto_register_shash(&ghash_alg); 344f1e866b1SArd Biesheuvel if (err) 345f1e866b1SArd Biesheuvel return err; 346f1e866b1SArd Biesheuvel err = crypto_register_ahash(&ghash_async_alg); 347f1e866b1SArd Biesheuvel if (err) 348f1e866b1SArd Biesheuvel goto err_shash; 349f1e866b1SArd Biesheuvel 350f1e866b1SArd Biesheuvel return 0; 351f1e866b1SArd Biesheuvel 352f1e866b1SArd Biesheuvel err_shash: 353f1e866b1SArd Biesheuvel crypto_unregister_shash(&ghash_alg); 354f1e866b1SArd Biesheuvel return err; 355f1e866b1SArd Biesheuvel } 356f1e866b1SArd Biesheuvel 357f1e866b1SArd Biesheuvel static void __exit ghash_ce_mod_exit(void) 358f1e866b1SArd Biesheuvel { 359f1e866b1SArd Biesheuvel crypto_unregister_ahash(&ghash_async_alg); 360f1e866b1SArd Biesheuvel crypto_unregister_shash(&ghash_alg); 361f1e866b1SArd Biesheuvel } 362f1e866b1SArd Biesheuvel 3633759ee05SArd Biesheuvel module_init(ghash_ce_mod_init); 364f1e866b1SArd Biesheuvel module_exit(ghash_ce_mod_exit); 365