1*1ccea77eSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 2266d0516SHerbert Xu /* 3266d0516SHerbert Xu * Shared crypto simd helpers 4266d0516SHerbert Xu * 5266d0516SHerbert Xu * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 6266d0516SHerbert Xu * Copyright (c) 2016 Herbert Xu <herbert@gondor.apana.org.au> 71661131aSEric Biggers * Copyright (c) 2019 Google LLC 8266d0516SHerbert Xu * 9266d0516SHerbert Xu * Based on aesni-intel_glue.c by: 10266d0516SHerbert Xu * Copyright (C) 2008, Intel Corp. 11266d0516SHerbert Xu * Author: Huang Ying <ying.huang@intel.com> 121661131aSEric Biggers */ 131661131aSEric Biggers 141661131aSEric Biggers /* 151661131aSEric Biggers * Shared crypto SIMD helpers. These functions dynamically create and register 161661131aSEric Biggers * an skcipher or AEAD algorithm that wraps another, internal algorithm. The 171661131aSEric Biggers * wrapper ensures that the internal algorithm is only executed in a context 181661131aSEric Biggers * where SIMD instructions are usable, i.e. where may_use_simd() returns true. 191661131aSEric Biggers * If SIMD is already usable, the wrapper directly calls the internal algorithm. 201661131aSEric Biggers * Otherwise it defers execution to a workqueue via cryptd. 21266d0516SHerbert Xu * 221661131aSEric Biggers * This is an alternative to the internal algorithm implementing a fallback for 231661131aSEric Biggers * the !may_use_simd() case itself. 241661131aSEric Biggers * 251661131aSEric Biggers * Note that the wrapper algorithm is asynchronous, i.e. it has the 261661131aSEric Biggers * CRYPTO_ALG_ASYNC flag set. Therefore it won't be found by users who 271661131aSEric Biggers * explicitly allocate a synchronous algorithm. 28266d0516SHerbert Xu */ 29266d0516SHerbert Xu 30266d0516SHerbert Xu #include <crypto/cryptd.h> 311661131aSEric Biggers #include <crypto/internal/aead.h> 32266d0516SHerbert Xu #include <crypto/internal/simd.h> 33266d0516SHerbert Xu #include <crypto/internal/skcipher.h> 34266d0516SHerbert Xu #include <linux/kernel.h> 35266d0516SHerbert Xu #include <linux/module.h> 36266d0516SHerbert Xu #include <linux/preempt.h> 37266d0516SHerbert Xu #include <asm/simd.h> 38266d0516SHerbert Xu 391661131aSEric Biggers /* skcipher support */ 401661131aSEric Biggers 41266d0516SHerbert Xu struct simd_skcipher_alg { 42266d0516SHerbert Xu const char *ialg_name; 43266d0516SHerbert Xu struct skcipher_alg alg; 44266d0516SHerbert Xu }; 45266d0516SHerbert Xu 46266d0516SHerbert Xu struct simd_skcipher_ctx { 47266d0516SHerbert Xu struct cryptd_skcipher *cryptd_tfm; 48266d0516SHerbert Xu }; 49266d0516SHerbert Xu 50266d0516SHerbert Xu static int simd_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 51266d0516SHerbert Xu unsigned int key_len) 52266d0516SHerbert Xu { 53266d0516SHerbert Xu struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 54266d0516SHerbert Xu struct crypto_skcipher *child = &ctx->cryptd_tfm->base; 55266d0516SHerbert Xu int err; 56266d0516SHerbert Xu 57266d0516SHerbert Xu crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 58266d0516SHerbert Xu crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(tfm) & 59266d0516SHerbert Xu CRYPTO_TFM_REQ_MASK); 60266d0516SHerbert Xu err = crypto_skcipher_setkey(child, key, key_len); 61266d0516SHerbert Xu crypto_skcipher_set_flags(tfm, crypto_skcipher_get_flags(child) & 62266d0516SHerbert Xu CRYPTO_TFM_RES_MASK); 63266d0516SHerbert Xu return err; 64266d0516SHerbert Xu } 65266d0516SHerbert Xu 66266d0516SHerbert Xu static int simd_skcipher_encrypt(struct skcipher_request *req) 67266d0516SHerbert Xu { 68266d0516SHerbert Xu struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 69266d0516SHerbert Xu struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 70266d0516SHerbert Xu struct skcipher_request *subreq; 71266d0516SHerbert Xu struct crypto_skcipher *child; 72266d0516SHerbert Xu 73266d0516SHerbert Xu subreq = skcipher_request_ctx(req); 74266d0516SHerbert Xu *subreq = *req; 75266d0516SHerbert Xu 768b8d91d4SEric Biggers if (!crypto_simd_usable() || 77266d0516SHerbert Xu (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm))) 78266d0516SHerbert Xu child = &ctx->cryptd_tfm->base; 79266d0516SHerbert Xu else 80266d0516SHerbert Xu child = cryptd_skcipher_child(ctx->cryptd_tfm); 81266d0516SHerbert Xu 82266d0516SHerbert Xu skcipher_request_set_tfm(subreq, child); 83266d0516SHerbert Xu 84266d0516SHerbert Xu return crypto_skcipher_encrypt(subreq); 85266d0516SHerbert Xu } 86266d0516SHerbert Xu 87266d0516SHerbert Xu static int simd_skcipher_decrypt(struct skcipher_request *req) 88266d0516SHerbert Xu { 89266d0516SHerbert Xu struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 90266d0516SHerbert Xu struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 91266d0516SHerbert Xu struct skcipher_request *subreq; 92266d0516SHerbert Xu struct crypto_skcipher *child; 93266d0516SHerbert Xu 94266d0516SHerbert Xu subreq = skcipher_request_ctx(req); 95266d0516SHerbert Xu *subreq = *req; 96266d0516SHerbert Xu 978b8d91d4SEric Biggers if (!crypto_simd_usable() || 98266d0516SHerbert Xu (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm))) 99266d0516SHerbert Xu child = &ctx->cryptd_tfm->base; 100266d0516SHerbert Xu else 101266d0516SHerbert Xu child = cryptd_skcipher_child(ctx->cryptd_tfm); 102266d0516SHerbert Xu 103266d0516SHerbert Xu skcipher_request_set_tfm(subreq, child); 104266d0516SHerbert Xu 105266d0516SHerbert Xu return crypto_skcipher_decrypt(subreq); 106266d0516SHerbert Xu } 107266d0516SHerbert Xu 108266d0516SHerbert Xu static void simd_skcipher_exit(struct crypto_skcipher *tfm) 109266d0516SHerbert Xu { 110266d0516SHerbert Xu struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 111266d0516SHerbert Xu 112266d0516SHerbert Xu cryptd_free_skcipher(ctx->cryptd_tfm); 113266d0516SHerbert Xu } 114266d0516SHerbert Xu 115266d0516SHerbert Xu static int simd_skcipher_init(struct crypto_skcipher *tfm) 116266d0516SHerbert Xu { 117266d0516SHerbert Xu struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 118266d0516SHerbert Xu struct cryptd_skcipher *cryptd_tfm; 119266d0516SHerbert Xu struct simd_skcipher_alg *salg; 120266d0516SHerbert Xu struct skcipher_alg *alg; 121266d0516SHerbert Xu unsigned reqsize; 122266d0516SHerbert Xu 123266d0516SHerbert Xu alg = crypto_skcipher_alg(tfm); 124266d0516SHerbert Xu salg = container_of(alg, struct simd_skcipher_alg, alg); 125266d0516SHerbert Xu 126266d0516SHerbert Xu cryptd_tfm = cryptd_alloc_skcipher(salg->ialg_name, 127266d0516SHerbert Xu CRYPTO_ALG_INTERNAL, 128266d0516SHerbert Xu CRYPTO_ALG_INTERNAL); 129266d0516SHerbert Xu if (IS_ERR(cryptd_tfm)) 130266d0516SHerbert Xu return PTR_ERR(cryptd_tfm); 131266d0516SHerbert Xu 132266d0516SHerbert Xu ctx->cryptd_tfm = cryptd_tfm; 133266d0516SHerbert Xu 134508a1c4dSArd Biesheuvel reqsize = crypto_skcipher_reqsize(cryptd_skcipher_child(cryptd_tfm)); 135508a1c4dSArd Biesheuvel reqsize = max(reqsize, crypto_skcipher_reqsize(&cryptd_tfm->base)); 136508a1c4dSArd Biesheuvel reqsize += sizeof(struct skcipher_request); 137266d0516SHerbert Xu 138266d0516SHerbert Xu crypto_skcipher_set_reqsize(tfm, reqsize); 139266d0516SHerbert Xu 140266d0516SHerbert Xu return 0; 141266d0516SHerbert Xu } 142266d0516SHerbert Xu 143266d0516SHerbert Xu struct simd_skcipher_alg *simd_skcipher_create_compat(const char *algname, 144266d0516SHerbert Xu const char *drvname, 145266d0516SHerbert Xu const char *basename) 146266d0516SHerbert Xu { 147266d0516SHerbert Xu struct simd_skcipher_alg *salg; 148266d0516SHerbert Xu struct crypto_skcipher *tfm; 149266d0516SHerbert Xu struct skcipher_alg *ialg; 150266d0516SHerbert Xu struct skcipher_alg *alg; 151266d0516SHerbert Xu int err; 152266d0516SHerbert Xu 153266d0516SHerbert Xu tfm = crypto_alloc_skcipher(basename, CRYPTO_ALG_INTERNAL, 154266d0516SHerbert Xu CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC); 155266d0516SHerbert Xu if (IS_ERR(tfm)) 156266d0516SHerbert Xu return ERR_CAST(tfm); 157266d0516SHerbert Xu 158266d0516SHerbert Xu ialg = crypto_skcipher_alg(tfm); 159266d0516SHerbert Xu 160266d0516SHerbert Xu salg = kzalloc(sizeof(*salg), GFP_KERNEL); 161266d0516SHerbert Xu if (!salg) { 162266d0516SHerbert Xu salg = ERR_PTR(-ENOMEM); 163266d0516SHerbert Xu goto out_put_tfm; 164266d0516SHerbert Xu } 165266d0516SHerbert Xu 166266d0516SHerbert Xu salg->ialg_name = basename; 167266d0516SHerbert Xu alg = &salg->alg; 168266d0516SHerbert Xu 169266d0516SHerbert Xu err = -ENAMETOOLONG; 170266d0516SHerbert Xu if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >= 171266d0516SHerbert Xu CRYPTO_MAX_ALG_NAME) 172266d0516SHerbert Xu goto out_free_salg; 173266d0516SHerbert Xu 174266d0516SHerbert Xu if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 175266d0516SHerbert Xu drvname) >= CRYPTO_MAX_ALG_NAME) 176266d0516SHerbert Xu goto out_free_salg; 177266d0516SHerbert Xu 178266d0516SHerbert Xu alg->base.cra_flags = CRYPTO_ALG_ASYNC; 179266d0516SHerbert Xu alg->base.cra_priority = ialg->base.cra_priority; 180266d0516SHerbert Xu alg->base.cra_blocksize = ialg->base.cra_blocksize; 181266d0516SHerbert Xu alg->base.cra_alignmask = ialg->base.cra_alignmask; 182266d0516SHerbert Xu alg->base.cra_module = ialg->base.cra_module; 183266d0516SHerbert Xu alg->base.cra_ctxsize = sizeof(struct simd_skcipher_ctx); 184266d0516SHerbert Xu 185266d0516SHerbert Xu alg->ivsize = ialg->ivsize; 186266d0516SHerbert Xu alg->chunksize = ialg->chunksize; 187266d0516SHerbert Xu alg->min_keysize = ialg->min_keysize; 188266d0516SHerbert Xu alg->max_keysize = ialg->max_keysize; 189266d0516SHerbert Xu 190266d0516SHerbert Xu alg->init = simd_skcipher_init; 191266d0516SHerbert Xu alg->exit = simd_skcipher_exit; 192266d0516SHerbert Xu 193266d0516SHerbert Xu alg->setkey = simd_skcipher_setkey; 194266d0516SHerbert Xu alg->encrypt = simd_skcipher_encrypt; 195266d0516SHerbert Xu alg->decrypt = simd_skcipher_decrypt; 196266d0516SHerbert Xu 197266d0516SHerbert Xu err = crypto_register_skcipher(alg); 198266d0516SHerbert Xu if (err) 199266d0516SHerbert Xu goto out_free_salg; 200266d0516SHerbert Xu 201266d0516SHerbert Xu out_put_tfm: 202266d0516SHerbert Xu crypto_free_skcipher(tfm); 203266d0516SHerbert Xu return salg; 204266d0516SHerbert Xu 205266d0516SHerbert Xu out_free_salg: 206266d0516SHerbert Xu kfree(salg); 207266d0516SHerbert Xu salg = ERR_PTR(err); 208266d0516SHerbert Xu goto out_put_tfm; 209266d0516SHerbert Xu } 210266d0516SHerbert Xu EXPORT_SYMBOL_GPL(simd_skcipher_create_compat); 211266d0516SHerbert Xu 212266d0516SHerbert Xu struct simd_skcipher_alg *simd_skcipher_create(const char *algname, 213266d0516SHerbert Xu const char *basename) 214266d0516SHerbert Xu { 215266d0516SHerbert Xu char drvname[CRYPTO_MAX_ALG_NAME]; 216266d0516SHerbert Xu 217266d0516SHerbert Xu if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >= 218266d0516SHerbert Xu CRYPTO_MAX_ALG_NAME) 219266d0516SHerbert Xu return ERR_PTR(-ENAMETOOLONG); 220266d0516SHerbert Xu 221266d0516SHerbert Xu return simd_skcipher_create_compat(algname, drvname, basename); 222266d0516SHerbert Xu } 223266d0516SHerbert Xu EXPORT_SYMBOL_GPL(simd_skcipher_create); 224266d0516SHerbert Xu 225266d0516SHerbert Xu void simd_skcipher_free(struct simd_skcipher_alg *salg) 226266d0516SHerbert Xu { 227266d0516SHerbert Xu crypto_unregister_skcipher(&salg->alg); 228266d0516SHerbert Xu kfree(salg); 229266d0516SHerbert Xu } 230266d0516SHerbert Xu EXPORT_SYMBOL_GPL(simd_skcipher_free); 231266d0516SHerbert Xu 232d14f0a1fSEric Biggers int simd_register_skciphers_compat(struct skcipher_alg *algs, int count, 233d14f0a1fSEric Biggers struct simd_skcipher_alg **simd_algs) 234d14f0a1fSEric Biggers { 235d14f0a1fSEric Biggers int err; 236d14f0a1fSEric Biggers int i; 237d14f0a1fSEric Biggers const char *algname; 238d14f0a1fSEric Biggers const char *drvname; 239d14f0a1fSEric Biggers const char *basename; 240d14f0a1fSEric Biggers struct simd_skcipher_alg *simd; 241d14f0a1fSEric Biggers 242d14f0a1fSEric Biggers err = crypto_register_skciphers(algs, count); 243d14f0a1fSEric Biggers if (err) 244d14f0a1fSEric Biggers return err; 245d14f0a1fSEric Biggers 246d14f0a1fSEric Biggers for (i = 0; i < count; i++) { 247d14f0a1fSEric Biggers WARN_ON(strncmp(algs[i].base.cra_name, "__", 2)); 248d14f0a1fSEric Biggers WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2)); 249d14f0a1fSEric Biggers algname = algs[i].base.cra_name + 2; 250d14f0a1fSEric Biggers drvname = algs[i].base.cra_driver_name + 2; 251d14f0a1fSEric Biggers basename = algs[i].base.cra_driver_name; 252d14f0a1fSEric Biggers simd = simd_skcipher_create_compat(algname, drvname, basename); 253d14f0a1fSEric Biggers err = PTR_ERR(simd); 254d14f0a1fSEric Biggers if (IS_ERR(simd)) 255d14f0a1fSEric Biggers goto err_unregister; 256d14f0a1fSEric Biggers simd_algs[i] = simd; 257d14f0a1fSEric Biggers } 258d14f0a1fSEric Biggers return 0; 259d14f0a1fSEric Biggers 260d14f0a1fSEric Biggers err_unregister: 261d14f0a1fSEric Biggers simd_unregister_skciphers(algs, count, simd_algs); 262d14f0a1fSEric Biggers return err; 263d14f0a1fSEric Biggers } 264d14f0a1fSEric Biggers EXPORT_SYMBOL_GPL(simd_register_skciphers_compat); 265d14f0a1fSEric Biggers 266d14f0a1fSEric Biggers void simd_unregister_skciphers(struct skcipher_alg *algs, int count, 267d14f0a1fSEric Biggers struct simd_skcipher_alg **simd_algs) 268d14f0a1fSEric Biggers { 269d14f0a1fSEric Biggers int i; 270d14f0a1fSEric Biggers 271d14f0a1fSEric Biggers crypto_unregister_skciphers(algs, count); 272d14f0a1fSEric Biggers 273d14f0a1fSEric Biggers for (i = 0; i < count; i++) { 274d14f0a1fSEric Biggers if (simd_algs[i]) { 275d14f0a1fSEric Biggers simd_skcipher_free(simd_algs[i]); 276d14f0a1fSEric Biggers simd_algs[i] = NULL; 277d14f0a1fSEric Biggers } 278d14f0a1fSEric Biggers } 279d14f0a1fSEric Biggers } 280d14f0a1fSEric Biggers EXPORT_SYMBOL_GPL(simd_unregister_skciphers); 281d14f0a1fSEric Biggers 2821661131aSEric Biggers /* AEAD support */ 2831661131aSEric Biggers 2841661131aSEric Biggers struct simd_aead_alg { 2851661131aSEric Biggers const char *ialg_name; 2861661131aSEric Biggers struct aead_alg alg; 2871661131aSEric Biggers }; 2881661131aSEric Biggers 2891661131aSEric Biggers struct simd_aead_ctx { 2901661131aSEric Biggers struct cryptd_aead *cryptd_tfm; 2911661131aSEric Biggers }; 2921661131aSEric Biggers 2931661131aSEric Biggers static int simd_aead_setkey(struct crypto_aead *tfm, const u8 *key, 2941661131aSEric Biggers unsigned int key_len) 2951661131aSEric Biggers { 2961661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 2971661131aSEric Biggers struct crypto_aead *child = &ctx->cryptd_tfm->base; 2981661131aSEric Biggers int err; 2991661131aSEric Biggers 3001661131aSEric Biggers crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 3011661131aSEric Biggers crypto_aead_set_flags(child, crypto_aead_get_flags(tfm) & 3021661131aSEric Biggers CRYPTO_TFM_REQ_MASK); 3031661131aSEric Biggers err = crypto_aead_setkey(child, key, key_len); 3041661131aSEric Biggers crypto_aead_set_flags(tfm, crypto_aead_get_flags(child) & 3051661131aSEric Biggers CRYPTO_TFM_RES_MASK); 3061661131aSEric Biggers return err; 3071661131aSEric Biggers } 3081661131aSEric Biggers 3091661131aSEric Biggers static int simd_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 3101661131aSEric Biggers { 3111661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 3121661131aSEric Biggers struct crypto_aead *child = &ctx->cryptd_tfm->base; 3131661131aSEric Biggers 3141661131aSEric Biggers return crypto_aead_setauthsize(child, authsize); 3151661131aSEric Biggers } 3161661131aSEric Biggers 3171661131aSEric Biggers static int simd_aead_encrypt(struct aead_request *req) 3181661131aSEric Biggers { 3191661131aSEric Biggers struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3201661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 3211661131aSEric Biggers struct aead_request *subreq; 3221661131aSEric Biggers struct crypto_aead *child; 3231661131aSEric Biggers 3241661131aSEric Biggers subreq = aead_request_ctx(req); 3251661131aSEric Biggers *subreq = *req; 3261661131aSEric Biggers 3278b8d91d4SEric Biggers if (!crypto_simd_usable() || 3281661131aSEric Biggers (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm))) 3291661131aSEric Biggers child = &ctx->cryptd_tfm->base; 3301661131aSEric Biggers else 3311661131aSEric Biggers child = cryptd_aead_child(ctx->cryptd_tfm); 3321661131aSEric Biggers 3331661131aSEric Biggers aead_request_set_tfm(subreq, child); 3341661131aSEric Biggers 3351661131aSEric Biggers return crypto_aead_encrypt(subreq); 3361661131aSEric Biggers } 3371661131aSEric Biggers 3381661131aSEric Biggers static int simd_aead_decrypt(struct aead_request *req) 3391661131aSEric Biggers { 3401661131aSEric Biggers struct crypto_aead *tfm = crypto_aead_reqtfm(req); 3411661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 3421661131aSEric Biggers struct aead_request *subreq; 3431661131aSEric Biggers struct crypto_aead *child; 3441661131aSEric Biggers 3451661131aSEric Biggers subreq = aead_request_ctx(req); 3461661131aSEric Biggers *subreq = *req; 3471661131aSEric Biggers 3488b8d91d4SEric Biggers if (!crypto_simd_usable() || 3491661131aSEric Biggers (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm))) 3501661131aSEric Biggers child = &ctx->cryptd_tfm->base; 3511661131aSEric Biggers else 3521661131aSEric Biggers child = cryptd_aead_child(ctx->cryptd_tfm); 3531661131aSEric Biggers 3541661131aSEric Biggers aead_request_set_tfm(subreq, child); 3551661131aSEric Biggers 3561661131aSEric Biggers return crypto_aead_decrypt(subreq); 3571661131aSEric Biggers } 3581661131aSEric Biggers 3591661131aSEric Biggers static void simd_aead_exit(struct crypto_aead *tfm) 3601661131aSEric Biggers { 3611661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 3621661131aSEric Biggers 3631661131aSEric Biggers cryptd_free_aead(ctx->cryptd_tfm); 3641661131aSEric Biggers } 3651661131aSEric Biggers 3661661131aSEric Biggers static int simd_aead_init(struct crypto_aead *tfm) 3671661131aSEric Biggers { 3681661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 3691661131aSEric Biggers struct cryptd_aead *cryptd_tfm; 3701661131aSEric Biggers struct simd_aead_alg *salg; 3711661131aSEric Biggers struct aead_alg *alg; 3721661131aSEric Biggers unsigned reqsize; 3731661131aSEric Biggers 3741661131aSEric Biggers alg = crypto_aead_alg(tfm); 3751661131aSEric Biggers salg = container_of(alg, struct simd_aead_alg, alg); 3761661131aSEric Biggers 3771661131aSEric Biggers cryptd_tfm = cryptd_alloc_aead(salg->ialg_name, CRYPTO_ALG_INTERNAL, 3781661131aSEric Biggers CRYPTO_ALG_INTERNAL); 3791661131aSEric Biggers if (IS_ERR(cryptd_tfm)) 3801661131aSEric Biggers return PTR_ERR(cryptd_tfm); 3811661131aSEric Biggers 3821661131aSEric Biggers ctx->cryptd_tfm = cryptd_tfm; 3831661131aSEric Biggers 3841661131aSEric Biggers reqsize = crypto_aead_reqsize(cryptd_aead_child(cryptd_tfm)); 3851661131aSEric Biggers reqsize = max(reqsize, crypto_aead_reqsize(&cryptd_tfm->base)); 3861661131aSEric Biggers reqsize += sizeof(struct aead_request); 3871661131aSEric Biggers 3881661131aSEric Biggers crypto_aead_set_reqsize(tfm, reqsize); 3891661131aSEric Biggers 3901661131aSEric Biggers return 0; 3911661131aSEric Biggers } 3921661131aSEric Biggers 3931661131aSEric Biggers struct simd_aead_alg *simd_aead_create_compat(const char *algname, 3941661131aSEric Biggers const char *drvname, 3951661131aSEric Biggers const char *basename) 3961661131aSEric Biggers { 3971661131aSEric Biggers struct simd_aead_alg *salg; 3981661131aSEric Biggers struct crypto_aead *tfm; 3991661131aSEric Biggers struct aead_alg *ialg; 4001661131aSEric Biggers struct aead_alg *alg; 4011661131aSEric Biggers int err; 4021661131aSEric Biggers 4031661131aSEric Biggers tfm = crypto_alloc_aead(basename, CRYPTO_ALG_INTERNAL, 4041661131aSEric Biggers CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC); 4051661131aSEric Biggers if (IS_ERR(tfm)) 4061661131aSEric Biggers return ERR_CAST(tfm); 4071661131aSEric Biggers 4081661131aSEric Biggers ialg = crypto_aead_alg(tfm); 4091661131aSEric Biggers 4101661131aSEric Biggers salg = kzalloc(sizeof(*salg), GFP_KERNEL); 4111661131aSEric Biggers if (!salg) { 4121661131aSEric Biggers salg = ERR_PTR(-ENOMEM); 4131661131aSEric Biggers goto out_put_tfm; 4141661131aSEric Biggers } 4151661131aSEric Biggers 4161661131aSEric Biggers salg->ialg_name = basename; 4171661131aSEric Biggers alg = &salg->alg; 4181661131aSEric Biggers 4191661131aSEric Biggers err = -ENAMETOOLONG; 4201661131aSEric Biggers if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >= 4211661131aSEric Biggers CRYPTO_MAX_ALG_NAME) 4221661131aSEric Biggers goto out_free_salg; 4231661131aSEric Biggers 4241661131aSEric Biggers if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 4251661131aSEric Biggers drvname) >= CRYPTO_MAX_ALG_NAME) 4261661131aSEric Biggers goto out_free_salg; 4271661131aSEric Biggers 4281661131aSEric Biggers alg->base.cra_flags = CRYPTO_ALG_ASYNC; 4291661131aSEric Biggers alg->base.cra_priority = ialg->base.cra_priority; 4301661131aSEric Biggers alg->base.cra_blocksize = ialg->base.cra_blocksize; 4311661131aSEric Biggers alg->base.cra_alignmask = ialg->base.cra_alignmask; 4321661131aSEric Biggers alg->base.cra_module = ialg->base.cra_module; 4331661131aSEric Biggers alg->base.cra_ctxsize = sizeof(struct simd_aead_ctx); 4341661131aSEric Biggers 4351661131aSEric Biggers alg->ivsize = ialg->ivsize; 4361661131aSEric Biggers alg->maxauthsize = ialg->maxauthsize; 4371661131aSEric Biggers alg->chunksize = ialg->chunksize; 4381661131aSEric Biggers 4391661131aSEric Biggers alg->init = simd_aead_init; 4401661131aSEric Biggers alg->exit = simd_aead_exit; 4411661131aSEric Biggers 4421661131aSEric Biggers alg->setkey = simd_aead_setkey; 4431661131aSEric Biggers alg->setauthsize = simd_aead_setauthsize; 4441661131aSEric Biggers alg->encrypt = simd_aead_encrypt; 4451661131aSEric Biggers alg->decrypt = simd_aead_decrypt; 4461661131aSEric Biggers 4471661131aSEric Biggers err = crypto_register_aead(alg); 4481661131aSEric Biggers if (err) 4491661131aSEric Biggers goto out_free_salg; 4501661131aSEric Biggers 4511661131aSEric Biggers out_put_tfm: 4521661131aSEric Biggers crypto_free_aead(tfm); 4531661131aSEric Biggers return salg; 4541661131aSEric Biggers 4551661131aSEric Biggers out_free_salg: 4561661131aSEric Biggers kfree(salg); 4571661131aSEric Biggers salg = ERR_PTR(err); 4581661131aSEric Biggers goto out_put_tfm; 4591661131aSEric Biggers } 4601661131aSEric Biggers EXPORT_SYMBOL_GPL(simd_aead_create_compat); 4611661131aSEric Biggers 4621661131aSEric Biggers struct simd_aead_alg *simd_aead_create(const char *algname, 4631661131aSEric Biggers const char *basename) 4641661131aSEric Biggers { 4651661131aSEric Biggers char drvname[CRYPTO_MAX_ALG_NAME]; 4661661131aSEric Biggers 4671661131aSEric Biggers if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >= 4681661131aSEric Biggers CRYPTO_MAX_ALG_NAME) 4691661131aSEric Biggers return ERR_PTR(-ENAMETOOLONG); 4701661131aSEric Biggers 4711661131aSEric Biggers return simd_aead_create_compat(algname, drvname, basename); 4721661131aSEric Biggers } 4731661131aSEric Biggers EXPORT_SYMBOL_GPL(simd_aead_create); 4741661131aSEric Biggers 4751661131aSEric Biggers void simd_aead_free(struct simd_aead_alg *salg) 4761661131aSEric Biggers { 4771661131aSEric Biggers crypto_unregister_aead(&salg->alg); 4781661131aSEric Biggers kfree(salg); 4791661131aSEric Biggers } 4801661131aSEric Biggers EXPORT_SYMBOL_GPL(simd_aead_free); 4811661131aSEric Biggers 4821661131aSEric Biggers int simd_register_aeads_compat(struct aead_alg *algs, int count, 4831661131aSEric Biggers struct simd_aead_alg **simd_algs) 4841661131aSEric Biggers { 4851661131aSEric Biggers int err; 4861661131aSEric Biggers int i; 4871661131aSEric Biggers const char *algname; 4881661131aSEric Biggers const char *drvname; 4891661131aSEric Biggers const char *basename; 4901661131aSEric Biggers struct simd_aead_alg *simd; 4911661131aSEric Biggers 4921661131aSEric Biggers err = crypto_register_aeads(algs, count); 4931661131aSEric Biggers if (err) 4941661131aSEric Biggers return err; 4951661131aSEric Biggers 4961661131aSEric Biggers for (i = 0; i < count; i++) { 4971661131aSEric Biggers WARN_ON(strncmp(algs[i].base.cra_name, "__", 2)); 4981661131aSEric Biggers WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2)); 4991661131aSEric Biggers algname = algs[i].base.cra_name + 2; 5001661131aSEric Biggers drvname = algs[i].base.cra_driver_name + 2; 5011661131aSEric Biggers basename = algs[i].base.cra_driver_name; 5021661131aSEric Biggers simd = simd_aead_create_compat(algname, drvname, basename); 5031661131aSEric Biggers err = PTR_ERR(simd); 5041661131aSEric Biggers if (IS_ERR(simd)) 5051661131aSEric Biggers goto err_unregister; 5061661131aSEric Biggers simd_algs[i] = simd; 5071661131aSEric Biggers } 5081661131aSEric Biggers return 0; 5091661131aSEric Biggers 5101661131aSEric Biggers err_unregister: 5111661131aSEric Biggers simd_unregister_aeads(algs, count, simd_algs); 5121661131aSEric Biggers return err; 5131661131aSEric Biggers } 5141661131aSEric Biggers EXPORT_SYMBOL_GPL(simd_register_aeads_compat); 5151661131aSEric Biggers 5161661131aSEric Biggers void simd_unregister_aeads(struct aead_alg *algs, int count, 5171661131aSEric Biggers struct simd_aead_alg **simd_algs) 5181661131aSEric Biggers { 5191661131aSEric Biggers int i; 5201661131aSEric Biggers 5211661131aSEric Biggers crypto_unregister_aeads(algs, count); 5221661131aSEric Biggers 5231661131aSEric Biggers for (i = 0; i < count; i++) { 5241661131aSEric Biggers if (simd_algs[i]) { 5251661131aSEric Biggers simd_aead_free(simd_algs[i]); 5261661131aSEric Biggers simd_algs[i] = NULL; 5271661131aSEric Biggers } 5281661131aSEric Biggers } 5291661131aSEric Biggers } 5301661131aSEric Biggers EXPORT_SYMBOL_GPL(simd_unregister_aeads); 5311661131aSEric Biggers 532266d0516SHerbert Xu MODULE_LICENSE("GPL"); 533