1266d0516SHerbert Xu /* 2266d0516SHerbert Xu * Shared crypto simd helpers 3266d0516SHerbert Xu * 4266d0516SHerbert Xu * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 5266d0516SHerbert Xu * Copyright (c) 2016 Herbert Xu <herbert@gondor.apana.org.au> 6*1661131aSEric Biggers * Copyright (c) 2019 Google LLC 7266d0516SHerbert Xu * 8266d0516SHerbert Xu * Based on aesni-intel_glue.c by: 9266d0516SHerbert Xu * Copyright (C) 2008, Intel Corp. 10266d0516SHerbert Xu * Author: Huang Ying <ying.huang@intel.com> 11266d0516SHerbert Xu * 12266d0516SHerbert Xu * This program is free software; you can redistribute it and/or modify 13266d0516SHerbert Xu * it under the terms of the GNU General Public License as published by 14266d0516SHerbert Xu * the Free Software Foundation; either version 2 of the License, or 15266d0516SHerbert Xu * (at your option) any later version. 16266d0516SHerbert Xu * 17266d0516SHerbert Xu * This program is distributed in the hope that it will be useful, 18266d0516SHerbert Xu * but WITHOUT ANY WARRANTY; without even the implied warranty of 19266d0516SHerbert Xu * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20266d0516SHerbert Xu * GNU General Public License for more details. 21266d0516SHerbert Xu * 22266d0516SHerbert Xu * You should have received a copy of the GNU General Public License 231af39daaSMartin Kepplinger * along with this program. If not, see <http://www.gnu.org/licenses/>. 24*1661131aSEric Biggers */ 25*1661131aSEric Biggers 26*1661131aSEric Biggers /* 27*1661131aSEric Biggers * Shared crypto SIMD helpers. These functions dynamically create and register 28*1661131aSEric Biggers * an skcipher or AEAD algorithm that wraps another, internal algorithm. The 29*1661131aSEric Biggers * wrapper ensures that the internal algorithm is only executed in a context 30*1661131aSEric Biggers * where SIMD instructions are usable, i.e. where may_use_simd() returns true. 31*1661131aSEric Biggers * If SIMD is already usable, the wrapper directly calls the internal algorithm. 32*1661131aSEric Biggers * Otherwise it defers execution to a workqueue via cryptd. 33266d0516SHerbert Xu * 34*1661131aSEric Biggers * This is an alternative to the internal algorithm implementing a fallback for 35*1661131aSEric Biggers * the !may_use_simd() case itself. 36*1661131aSEric Biggers * 37*1661131aSEric Biggers * Note that the wrapper algorithm is asynchronous, i.e. it has the 38*1661131aSEric Biggers * CRYPTO_ALG_ASYNC flag set. Therefore it won't be found by users who 39*1661131aSEric Biggers * explicitly allocate a synchronous algorithm. 40266d0516SHerbert Xu */ 41266d0516SHerbert Xu 42266d0516SHerbert Xu #include <crypto/cryptd.h> 43*1661131aSEric Biggers #include <crypto/internal/aead.h> 44266d0516SHerbert Xu #include <crypto/internal/simd.h> 45266d0516SHerbert Xu #include <crypto/internal/skcipher.h> 46266d0516SHerbert Xu #include <linux/kernel.h> 47266d0516SHerbert Xu #include <linux/module.h> 48266d0516SHerbert Xu #include <linux/preempt.h> 49266d0516SHerbert Xu #include <asm/simd.h> 50266d0516SHerbert Xu 51*1661131aSEric Biggers /* skcipher support */ 52*1661131aSEric Biggers 53266d0516SHerbert Xu struct simd_skcipher_alg { 54266d0516SHerbert Xu const char *ialg_name; 55266d0516SHerbert Xu struct skcipher_alg alg; 56266d0516SHerbert Xu }; 57266d0516SHerbert Xu 58266d0516SHerbert Xu struct simd_skcipher_ctx { 59266d0516SHerbert Xu struct cryptd_skcipher *cryptd_tfm; 60266d0516SHerbert Xu }; 61266d0516SHerbert Xu 62266d0516SHerbert Xu static int simd_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 63266d0516SHerbert Xu unsigned int key_len) 64266d0516SHerbert Xu { 65266d0516SHerbert Xu struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 66266d0516SHerbert Xu struct crypto_skcipher *child = &ctx->cryptd_tfm->base; 67266d0516SHerbert Xu int err; 68266d0516SHerbert Xu 69266d0516SHerbert Xu crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 70266d0516SHerbert Xu crypto_skcipher_set_flags(child, crypto_skcipher_get_flags(tfm) & 71266d0516SHerbert Xu CRYPTO_TFM_REQ_MASK); 72266d0516SHerbert Xu err = crypto_skcipher_setkey(child, key, key_len); 73266d0516SHerbert Xu crypto_skcipher_set_flags(tfm, crypto_skcipher_get_flags(child) & 74266d0516SHerbert Xu CRYPTO_TFM_RES_MASK); 75266d0516SHerbert Xu return err; 76266d0516SHerbert Xu } 77266d0516SHerbert Xu 78266d0516SHerbert Xu static int simd_skcipher_encrypt(struct skcipher_request *req) 79266d0516SHerbert Xu { 80266d0516SHerbert Xu struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 81266d0516SHerbert Xu struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 82266d0516SHerbert Xu struct skcipher_request *subreq; 83266d0516SHerbert Xu struct crypto_skcipher *child; 84266d0516SHerbert Xu 85266d0516SHerbert Xu subreq = skcipher_request_ctx(req); 86266d0516SHerbert Xu *subreq = *req; 87266d0516SHerbert Xu 88266d0516SHerbert Xu if (!may_use_simd() || 89266d0516SHerbert Xu (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm))) 90266d0516SHerbert Xu child = &ctx->cryptd_tfm->base; 91266d0516SHerbert Xu else 92266d0516SHerbert Xu child = cryptd_skcipher_child(ctx->cryptd_tfm); 93266d0516SHerbert Xu 94266d0516SHerbert Xu skcipher_request_set_tfm(subreq, child); 95266d0516SHerbert Xu 96266d0516SHerbert Xu return crypto_skcipher_encrypt(subreq); 97266d0516SHerbert Xu } 98266d0516SHerbert Xu 99266d0516SHerbert Xu static int simd_skcipher_decrypt(struct skcipher_request *req) 100266d0516SHerbert Xu { 101266d0516SHerbert Xu struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 102266d0516SHerbert Xu struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 103266d0516SHerbert Xu struct skcipher_request *subreq; 104266d0516SHerbert Xu struct crypto_skcipher *child; 105266d0516SHerbert Xu 106266d0516SHerbert Xu subreq = skcipher_request_ctx(req); 107266d0516SHerbert Xu *subreq = *req; 108266d0516SHerbert Xu 109266d0516SHerbert Xu if (!may_use_simd() || 110266d0516SHerbert Xu (in_atomic() && cryptd_skcipher_queued(ctx->cryptd_tfm))) 111266d0516SHerbert Xu child = &ctx->cryptd_tfm->base; 112266d0516SHerbert Xu else 113266d0516SHerbert Xu child = cryptd_skcipher_child(ctx->cryptd_tfm); 114266d0516SHerbert Xu 115266d0516SHerbert Xu skcipher_request_set_tfm(subreq, child); 116266d0516SHerbert Xu 117266d0516SHerbert Xu return crypto_skcipher_decrypt(subreq); 118266d0516SHerbert Xu } 119266d0516SHerbert Xu 120266d0516SHerbert Xu static void simd_skcipher_exit(struct crypto_skcipher *tfm) 121266d0516SHerbert Xu { 122266d0516SHerbert Xu struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 123266d0516SHerbert Xu 124266d0516SHerbert Xu cryptd_free_skcipher(ctx->cryptd_tfm); 125266d0516SHerbert Xu } 126266d0516SHerbert Xu 127266d0516SHerbert Xu static int simd_skcipher_init(struct crypto_skcipher *tfm) 128266d0516SHerbert Xu { 129266d0516SHerbert Xu struct simd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); 130266d0516SHerbert Xu struct cryptd_skcipher *cryptd_tfm; 131266d0516SHerbert Xu struct simd_skcipher_alg *salg; 132266d0516SHerbert Xu struct skcipher_alg *alg; 133266d0516SHerbert Xu unsigned reqsize; 134266d0516SHerbert Xu 135266d0516SHerbert Xu alg = crypto_skcipher_alg(tfm); 136266d0516SHerbert Xu salg = container_of(alg, struct simd_skcipher_alg, alg); 137266d0516SHerbert Xu 138266d0516SHerbert Xu cryptd_tfm = cryptd_alloc_skcipher(salg->ialg_name, 139266d0516SHerbert Xu CRYPTO_ALG_INTERNAL, 140266d0516SHerbert Xu CRYPTO_ALG_INTERNAL); 141266d0516SHerbert Xu if (IS_ERR(cryptd_tfm)) 142266d0516SHerbert Xu return PTR_ERR(cryptd_tfm); 143266d0516SHerbert Xu 144266d0516SHerbert Xu ctx->cryptd_tfm = cryptd_tfm; 145266d0516SHerbert Xu 146508a1c4dSArd Biesheuvel reqsize = crypto_skcipher_reqsize(cryptd_skcipher_child(cryptd_tfm)); 147508a1c4dSArd Biesheuvel reqsize = max(reqsize, crypto_skcipher_reqsize(&cryptd_tfm->base)); 148508a1c4dSArd Biesheuvel reqsize += sizeof(struct skcipher_request); 149266d0516SHerbert Xu 150266d0516SHerbert Xu crypto_skcipher_set_reqsize(tfm, reqsize); 151266d0516SHerbert Xu 152266d0516SHerbert Xu return 0; 153266d0516SHerbert Xu } 154266d0516SHerbert Xu 155266d0516SHerbert Xu struct simd_skcipher_alg *simd_skcipher_create_compat(const char *algname, 156266d0516SHerbert Xu const char *drvname, 157266d0516SHerbert Xu const char *basename) 158266d0516SHerbert Xu { 159266d0516SHerbert Xu struct simd_skcipher_alg *salg; 160266d0516SHerbert Xu struct crypto_skcipher *tfm; 161266d0516SHerbert Xu struct skcipher_alg *ialg; 162266d0516SHerbert Xu struct skcipher_alg *alg; 163266d0516SHerbert Xu int err; 164266d0516SHerbert Xu 165266d0516SHerbert Xu tfm = crypto_alloc_skcipher(basename, CRYPTO_ALG_INTERNAL, 166266d0516SHerbert Xu CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC); 167266d0516SHerbert Xu if (IS_ERR(tfm)) 168266d0516SHerbert Xu return ERR_CAST(tfm); 169266d0516SHerbert Xu 170266d0516SHerbert Xu ialg = crypto_skcipher_alg(tfm); 171266d0516SHerbert Xu 172266d0516SHerbert Xu salg = kzalloc(sizeof(*salg), GFP_KERNEL); 173266d0516SHerbert Xu if (!salg) { 174266d0516SHerbert Xu salg = ERR_PTR(-ENOMEM); 175266d0516SHerbert Xu goto out_put_tfm; 176266d0516SHerbert Xu } 177266d0516SHerbert Xu 178266d0516SHerbert Xu salg->ialg_name = basename; 179266d0516SHerbert Xu alg = &salg->alg; 180266d0516SHerbert Xu 181266d0516SHerbert Xu err = -ENAMETOOLONG; 182266d0516SHerbert Xu if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >= 183266d0516SHerbert Xu CRYPTO_MAX_ALG_NAME) 184266d0516SHerbert Xu goto out_free_salg; 185266d0516SHerbert Xu 186266d0516SHerbert Xu if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 187266d0516SHerbert Xu drvname) >= CRYPTO_MAX_ALG_NAME) 188266d0516SHerbert Xu goto out_free_salg; 189266d0516SHerbert Xu 190266d0516SHerbert Xu alg->base.cra_flags = CRYPTO_ALG_ASYNC; 191266d0516SHerbert Xu alg->base.cra_priority = ialg->base.cra_priority; 192266d0516SHerbert Xu alg->base.cra_blocksize = ialg->base.cra_blocksize; 193266d0516SHerbert Xu alg->base.cra_alignmask = ialg->base.cra_alignmask; 194266d0516SHerbert Xu alg->base.cra_module = ialg->base.cra_module; 195266d0516SHerbert Xu alg->base.cra_ctxsize = sizeof(struct simd_skcipher_ctx); 196266d0516SHerbert Xu 197266d0516SHerbert Xu alg->ivsize = ialg->ivsize; 198266d0516SHerbert Xu alg->chunksize = ialg->chunksize; 199266d0516SHerbert Xu alg->min_keysize = ialg->min_keysize; 200266d0516SHerbert Xu alg->max_keysize = ialg->max_keysize; 201266d0516SHerbert Xu 202266d0516SHerbert Xu alg->init = simd_skcipher_init; 203266d0516SHerbert Xu alg->exit = simd_skcipher_exit; 204266d0516SHerbert Xu 205266d0516SHerbert Xu alg->setkey = simd_skcipher_setkey; 206266d0516SHerbert Xu alg->encrypt = simd_skcipher_encrypt; 207266d0516SHerbert Xu alg->decrypt = simd_skcipher_decrypt; 208266d0516SHerbert Xu 209266d0516SHerbert Xu err = crypto_register_skcipher(alg); 210266d0516SHerbert Xu if (err) 211266d0516SHerbert Xu goto out_free_salg; 212266d0516SHerbert Xu 213266d0516SHerbert Xu out_put_tfm: 214266d0516SHerbert Xu crypto_free_skcipher(tfm); 215266d0516SHerbert Xu return salg; 216266d0516SHerbert Xu 217266d0516SHerbert Xu out_free_salg: 218266d0516SHerbert Xu kfree(salg); 219266d0516SHerbert Xu salg = ERR_PTR(err); 220266d0516SHerbert Xu goto out_put_tfm; 221266d0516SHerbert Xu } 222266d0516SHerbert Xu EXPORT_SYMBOL_GPL(simd_skcipher_create_compat); 223266d0516SHerbert Xu 224266d0516SHerbert Xu struct simd_skcipher_alg *simd_skcipher_create(const char *algname, 225266d0516SHerbert Xu const char *basename) 226266d0516SHerbert Xu { 227266d0516SHerbert Xu char drvname[CRYPTO_MAX_ALG_NAME]; 228266d0516SHerbert Xu 229266d0516SHerbert Xu if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >= 230266d0516SHerbert Xu CRYPTO_MAX_ALG_NAME) 231266d0516SHerbert Xu return ERR_PTR(-ENAMETOOLONG); 232266d0516SHerbert Xu 233266d0516SHerbert Xu return simd_skcipher_create_compat(algname, drvname, basename); 234266d0516SHerbert Xu } 235266d0516SHerbert Xu EXPORT_SYMBOL_GPL(simd_skcipher_create); 236266d0516SHerbert Xu 237266d0516SHerbert Xu void simd_skcipher_free(struct simd_skcipher_alg *salg) 238266d0516SHerbert Xu { 239266d0516SHerbert Xu crypto_unregister_skcipher(&salg->alg); 240266d0516SHerbert Xu kfree(salg); 241266d0516SHerbert Xu } 242266d0516SHerbert Xu EXPORT_SYMBOL_GPL(simd_skcipher_free); 243266d0516SHerbert Xu 244d14f0a1fSEric Biggers int simd_register_skciphers_compat(struct skcipher_alg *algs, int count, 245d14f0a1fSEric Biggers struct simd_skcipher_alg **simd_algs) 246d14f0a1fSEric Biggers { 247d14f0a1fSEric Biggers int err; 248d14f0a1fSEric Biggers int i; 249d14f0a1fSEric Biggers const char *algname; 250d14f0a1fSEric Biggers const char *drvname; 251d14f0a1fSEric Biggers const char *basename; 252d14f0a1fSEric Biggers struct simd_skcipher_alg *simd; 253d14f0a1fSEric Biggers 254d14f0a1fSEric Biggers err = crypto_register_skciphers(algs, count); 255d14f0a1fSEric Biggers if (err) 256d14f0a1fSEric Biggers return err; 257d14f0a1fSEric Biggers 258d14f0a1fSEric Biggers for (i = 0; i < count; i++) { 259d14f0a1fSEric Biggers WARN_ON(strncmp(algs[i].base.cra_name, "__", 2)); 260d14f0a1fSEric Biggers WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2)); 261d14f0a1fSEric Biggers algname = algs[i].base.cra_name + 2; 262d14f0a1fSEric Biggers drvname = algs[i].base.cra_driver_name + 2; 263d14f0a1fSEric Biggers basename = algs[i].base.cra_driver_name; 264d14f0a1fSEric Biggers simd = simd_skcipher_create_compat(algname, drvname, basename); 265d14f0a1fSEric Biggers err = PTR_ERR(simd); 266d14f0a1fSEric Biggers if (IS_ERR(simd)) 267d14f0a1fSEric Biggers goto err_unregister; 268d14f0a1fSEric Biggers simd_algs[i] = simd; 269d14f0a1fSEric Biggers } 270d14f0a1fSEric Biggers return 0; 271d14f0a1fSEric Biggers 272d14f0a1fSEric Biggers err_unregister: 273d14f0a1fSEric Biggers simd_unregister_skciphers(algs, count, simd_algs); 274d14f0a1fSEric Biggers return err; 275d14f0a1fSEric Biggers } 276d14f0a1fSEric Biggers EXPORT_SYMBOL_GPL(simd_register_skciphers_compat); 277d14f0a1fSEric Biggers 278d14f0a1fSEric Biggers void simd_unregister_skciphers(struct skcipher_alg *algs, int count, 279d14f0a1fSEric Biggers struct simd_skcipher_alg **simd_algs) 280d14f0a1fSEric Biggers { 281d14f0a1fSEric Biggers int i; 282d14f0a1fSEric Biggers 283d14f0a1fSEric Biggers crypto_unregister_skciphers(algs, count); 284d14f0a1fSEric Biggers 285d14f0a1fSEric Biggers for (i = 0; i < count; i++) { 286d14f0a1fSEric Biggers if (simd_algs[i]) { 287d14f0a1fSEric Biggers simd_skcipher_free(simd_algs[i]); 288d14f0a1fSEric Biggers simd_algs[i] = NULL; 289d14f0a1fSEric Biggers } 290d14f0a1fSEric Biggers } 291d14f0a1fSEric Biggers } 292d14f0a1fSEric Biggers EXPORT_SYMBOL_GPL(simd_unregister_skciphers); 293d14f0a1fSEric Biggers 294*1661131aSEric Biggers /* AEAD support */ 295*1661131aSEric Biggers 296*1661131aSEric Biggers struct simd_aead_alg { 297*1661131aSEric Biggers const char *ialg_name; 298*1661131aSEric Biggers struct aead_alg alg; 299*1661131aSEric Biggers }; 300*1661131aSEric Biggers 301*1661131aSEric Biggers struct simd_aead_ctx { 302*1661131aSEric Biggers struct cryptd_aead *cryptd_tfm; 303*1661131aSEric Biggers }; 304*1661131aSEric Biggers 305*1661131aSEric Biggers static int simd_aead_setkey(struct crypto_aead *tfm, const u8 *key, 306*1661131aSEric Biggers unsigned int key_len) 307*1661131aSEric Biggers { 308*1661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 309*1661131aSEric Biggers struct crypto_aead *child = &ctx->cryptd_tfm->base; 310*1661131aSEric Biggers int err; 311*1661131aSEric Biggers 312*1661131aSEric Biggers crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); 313*1661131aSEric Biggers crypto_aead_set_flags(child, crypto_aead_get_flags(tfm) & 314*1661131aSEric Biggers CRYPTO_TFM_REQ_MASK); 315*1661131aSEric Biggers err = crypto_aead_setkey(child, key, key_len); 316*1661131aSEric Biggers crypto_aead_set_flags(tfm, crypto_aead_get_flags(child) & 317*1661131aSEric Biggers CRYPTO_TFM_RES_MASK); 318*1661131aSEric Biggers return err; 319*1661131aSEric Biggers } 320*1661131aSEric Biggers 321*1661131aSEric Biggers static int simd_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 322*1661131aSEric Biggers { 323*1661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 324*1661131aSEric Biggers struct crypto_aead *child = &ctx->cryptd_tfm->base; 325*1661131aSEric Biggers 326*1661131aSEric Biggers return crypto_aead_setauthsize(child, authsize); 327*1661131aSEric Biggers } 328*1661131aSEric Biggers 329*1661131aSEric Biggers static int simd_aead_encrypt(struct aead_request *req) 330*1661131aSEric Biggers { 331*1661131aSEric Biggers struct crypto_aead *tfm = crypto_aead_reqtfm(req); 332*1661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 333*1661131aSEric Biggers struct aead_request *subreq; 334*1661131aSEric Biggers struct crypto_aead *child; 335*1661131aSEric Biggers 336*1661131aSEric Biggers subreq = aead_request_ctx(req); 337*1661131aSEric Biggers *subreq = *req; 338*1661131aSEric Biggers 339*1661131aSEric Biggers if (!may_use_simd() || 340*1661131aSEric Biggers (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm))) 341*1661131aSEric Biggers child = &ctx->cryptd_tfm->base; 342*1661131aSEric Biggers else 343*1661131aSEric Biggers child = cryptd_aead_child(ctx->cryptd_tfm); 344*1661131aSEric Biggers 345*1661131aSEric Biggers aead_request_set_tfm(subreq, child); 346*1661131aSEric Biggers 347*1661131aSEric Biggers return crypto_aead_encrypt(subreq); 348*1661131aSEric Biggers } 349*1661131aSEric Biggers 350*1661131aSEric Biggers static int simd_aead_decrypt(struct aead_request *req) 351*1661131aSEric Biggers { 352*1661131aSEric Biggers struct crypto_aead *tfm = crypto_aead_reqtfm(req); 353*1661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 354*1661131aSEric Biggers struct aead_request *subreq; 355*1661131aSEric Biggers struct crypto_aead *child; 356*1661131aSEric Biggers 357*1661131aSEric Biggers subreq = aead_request_ctx(req); 358*1661131aSEric Biggers *subreq = *req; 359*1661131aSEric Biggers 360*1661131aSEric Biggers if (!may_use_simd() || 361*1661131aSEric Biggers (in_atomic() && cryptd_aead_queued(ctx->cryptd_tfm))) 362*1661131aSEric Biggers child = &ctx->cryptd_tfm->base; 363*1661131aSEric Biggers else 364*1661131aSEric Biggers child = cryptd_aead_child(ctx->cryptd_tfm); 365*1661131aSEric Biggers 366*1661131aSEric Biggers aead_request_set_tfm(subreq, child); 367*1661131aSEric Biggers 368*1661131aSEric Biggers return crypto_aead_decrypt(subreq); 369*1661131aSEric Biggers } 370*1661131aSEric Biggers 371*1661131aSEric Biggers static void simd_aead_exit(struct crypto_aead *tfm) 372*1661131aSEric Biggers { 373*1661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 374*1661131aSEric Biggers 375*1661131aSEric Biggers cryptd_free_aead(ctx->cryptd_tfm); 376*1661131aSEric Biggers } 377*1661131aSEric Biggers 378*1661131aSEric Biggers static int simd_aead_init(struct crypto_aead *tfm) 379*1661131aSEric Biggers { 380*1661131aSEric Biggers struct simd_aead_ctx *ctx = crypto_aead_ctx(tfm); 381*1661131aSEric Biggers struct cryptd_aead *cryptd_tfm; 382*1661131aSEric Biggers struct simd_aead_alg *salg; 383*1661131aSEric Biggers struct aead_alg *alg; 384*1661131aSEric Biggers unsigned reqsize; 385*1661131aSEric Biggers 386*1661131aSEric Biggers alg = crypto_aead_alg(tfm); 387*1661131aSEric Biggers salg = container_of(alg, struct simd_aead_alg, alg); 388*1661131aSEric Biggers 389*1661131aSEric Biggers cryptd_tfm = cryptd_alloc_aead(salg->ialg_name, CRYPTO_ALG_INTERNAL, 390*1661131aSEric Biggers CRYPTO_ALG_INTERNAL); 391*1661131aSEric Biggers if (IS_ERR(cryptd_tfm)) 392*1661131aSEric Biggers return PTR_ERR(cryptd_tfm); 393*1661131aSEric Biggers 394*1661131aSEric Biggers ctx->cryptd_tfm = cryptd_tfm; 395*1661131aSEric Biggers 396*1661131aSEric Biggers reqsize = crypto_aead_reqsize(cryptd_aead_child(cryptd_tfm)); 397*1661131aSEric Biggers reqsize = max(reqsize, crypto_aead_reqsize(&cryptd_tfm->base)); 398*1661131aSEric Biggers reqsize += sizeof(struct aead_request); 399*1661131aSEric Biggers 400*1661131aSEric Biggers crypto_aead_set_reqsize(tfm, reqsize); 401*1661131aSEric Biggers 402*1661131aSEric Biggers return 0; 403*1661131aSEric Biggers } 404*1661131aSEric Biggers 405*1661131aSEric Biggers struct simd_aead_alg *simd_aead_create_compat(const char *algname, 406*1661131aSEric Biggers const char *drvname, 407*1661131aSEric Biggers const char *basename) 408*1661131aSEric Biggers { 409*1661131aSEric Biggers struct simd_aead_alg *salg; 410*1661131aSEric Biggers struct crypto_aead *tfm; 411*1661131aSEric Biggers struct aead_alg *ialg; 412*1661131aSEric Biggers struct aead_alg *alg; 413*1661131aSEric Biggers int err; 414*1661131aSEric Biggers 415*1661131aSEric Biggers tfm = crypto_alloc_aead(basename, CRYPTO_ALG_INTERNAL, 416*1661131aSEric Biggers CRYPTO_ALG_INTERNAL | CRYPTO_ALG_ASYNC); 417*1661131aSEric Biggers if (IS_ERR(tfm)) 418*1661131aSEric Biggers return ERR_CAST(tfm); 419*1661131aSEric Biggers 420*1661131aSEric Biggers ialg = crypto_aead_alg(tfm); 421*1661131aSEric Biggers 422*1661131aSEric Biggers salg = kzalloc(sizeof(*salg), GFP_KERNEL); 423*1661131aSEric Biggers if (!salg) { 424*1661131aSEric Biggers salg = ERR_PTR(-ENOMEM); 425*1661131aSEric Biggers goto out_put_tfm; 426*1661131aSEric Biggers } 427*1661131aSEric Biggers 428*1661131aSEric Biggers salg->ialg_name = basename; 429*1661131aSEric Biggers alg = &salg->alg; 430*1661131aSEric Biggers 431*1661131aSEric Biggers err = -ENAMETOOLONG; 432*1661131aSEric Biggers if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", algname) >= 433*1661131aSEric Biggers CRYPTO_MAX_ALG_NAME) 434*1661131aSEric Biggers goto out_free_salg; 435*1661131aSEric Biggers 436*1661131aSEric Biggers if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", 437*1661131aSEric Biggers drvname) >= CRYPTO_MAX_ALG_NAME) 438*1661131aSEric Biggers goto out_free_salg; 439*1661131aSEric Biggers 440*1661131aSEric Biggers alg->base.cra_flags = CRYPTO_ALG_ASYNC; 441*1661131aSEric Biggers alg->base.cra_priority = ialg->base.cra_priority; 442*1661131aSEric Biggers alg->base.cra_blocksize = ialg->base.cra_blocksize; 443*1661131aSEric Biggers alg->base.cra_alignmask = ialg->base.cra_alignmask; 444*1661131aSEric Biggers alg->base.cra_module = ialg->base.cra_module; 445*1661131aSEric Biggers alg->base.cra_ctxsize = sizeof(struct simd_aead_ctx); 446*1661131aSEric Biggers 447*1661131aSEric Biggers alg->ivsize = ialg->ivsize; 448*1661131aSEric Biggers alg->maxauthsize = ialg->maxauthsize; 449*1661131aSEric Biggers alg->chunksize = ialg->chunksize; 450*1661131aSEric Biggers 451*1661131aSEric Biggers alg->init = simd_aead_init; 452*1661131aSEric Biggers alg->exit = simd_aead_exit; 453*1661131aSEric Biggers 454*1661131aSEric Biggers alg->setkey = simd_aead_setkey; 455*1661131aSEric Biggers alg->setauthsize = simd_aead_setauthsize; 456*1661131aSEric Biggers alg->encrypt = simd_aead_encrypt; 457*1661131aSEric Biggers alg->decrypt = simd_aead_decrypt; 458*1661131aSEric Biggers 459*1661131aSEric Biggers err = crypto_register_aead(alg); 460*1661131aSEric Biggers if (err) 461*1661131aSEric Biggers goto out_free_salg; 462*1661131aSEric Biggers 463*1661131aSEric Biggers out_put_tfm: 464*1661131aSEric Biggers crypto_free_aead(tfm); 465*1661131aSEric Biggers return salg; 466*1661131aSEric Biggers 467*1661131aSEric Biggers out_free_salg: 468*1661131aSEric Biggers kfree(salg); 469*1661131aSEric Biggers salg = ERR_PTR(err); 470*1661131aSEric Biggers goto out_put_tfm; 471*1661131aSEric Biggers } 472*1661131aSEric Biggers EXPORT_SYMBOL_GPL(simd_aead_create_compat); 473*1661131aSEric Biggers 474*1661131aSEric Biggers struct simd_aead_alg *simd_aead_create(const char *algname, 475*1661131aSEric Biggers const char *basename) 476*1661131aSEric Biggers { 477*1661131aSEric Biggers char drvname[CRYPTO_MAX_ALG_NAME]; 478*1661131aSEric Biggers 479*1661131aSEric Biggers if (snprintf(drvname, CRYPTO_MAX_ALG_NAME, "simd-%s", basename) >= 480*1661131aSEric Biggers CRYPTO_MAX_ALG_NAME) 481*1661131aSEric Biggers return ERR_PTR(-ENAMETOOLONG); 482*1661131aSEric Biggers 483*1661131aSEric Biggers return simd_aead_create_compat(algname, drvname, basename); 484*1661131aSEric Biggers } 485*1661131aSEric Biggers EXPORT_SYMBOL_GPL(simd_aead_create); 486*1661131aSEric Biggers 487*1661131aSEric Biggers void simd_aead_free(struct simd_aead_alg *salg) 488*1661131aSEric Biggers { 489*1661131aSEric Biggers crypto_unregister_aead(&salg->alg); 490*1661131aSEric Biggers kfree(salg); 491*1661131aSEric Biggers } 492*1661131aSEric Biggers EXPORT_SYMBOL_GPL(simd_aead_free); 493*1661131aSEric Biggers 494*1661131aSEric Biggers int simd_register_aeads_compat(struct aead_alg *algs, int count, 495*1661131aSEric Biggers struct simd_aead_alg **simd_algs) 496*1661131aSEric Biggers { 497*1661131aSEric Biggers int err; 498*1661131aSEric Biggers int i; 499*1661131aSEric Biggers const char *algname; 500*1661131aSEric Biggers const char *drvname; 501*1661131aSEric Biggers const char *basename; 502*1661131aSEric Biggers struct simd_aead_alg *simd; 503*1661131aSEric Biggers 504*1661131aSEric Biggers err = crypto_register_aeads(algs, count); 505*1661131aSEric Biggers if (err) 506*1661131aSEric Biggers return err; 507*1661131aSEric Biggers 508*1661131aSEric Biggers for (i = 0; i < count; i++) { 509*1661131aSEric Biggers WARN_ON(strncmp(algs[i].base.cra_name, "__", 2)); 510*1661131aSEric Biggers WARN_ON(strncmp(algs[i].base.cra_driver_name, "__", 2)); 511*1661131aSEric Biggers algname = algs[i].base.cra_name + 2; 512*1661131aSEric Biggers drvname = algs[i].base.cra_driver_name + 2; 513*1661131aSEric Biggers basename = algs[i].base.cra_driver_name; 514*1661131aSEric Biggers simd = simd_aead_create_compat(algname, drvname, basename); 515*1661131aSEric Biggers err = PTR_ERR(simd); 516*1661131aSEric Biggers if (IS_ERR(simd)) 517*1661131aSEric Biggers goto err_unregister; 518*1661131aSEric Biggers simd_algs[i] = simd; 519*1661131aSEric Biggers } 520*1661131aSEric Biggers return 0; 521*1661131aSEric Biggers 522*1661131aSEric Biggers err_unregister: 523*1661131aSEric Biggers simd_unregister_aeads(algs, count, simd_algs); 524*1661131aSEric Biggers return err; 525*1661131aSEric Biggers } 526*1661131aSEric Biggers EXPORT_SYMBOL_GPL(simd_register_aeads_compat); 527*1661131aSEric Biggers 528*1661131aSEric Biggers void simd_unregister_aeads(struct aead_alg *algs, int count, 529*1661131aSEric Biggers struct simd_aead_alg **simd_algs) 530*1661131aSEric Biggers { 531*1661131aSEric Biggers int i; 532*1661131aSEric Biggers 533*1661131aSEric Biggers crypto_unregister_aeads(algs, count); 534*1661131aSEric Biggers 535*1661131aSEric Biggers for (i = 0; i < count; i++) { 536*1661131aSEric Biggers if (simd_algs[i]) { 537*1661131aSEric Biggers simd_aead_free(simd_algs[i]); 538*1661131aSEric Biggers simd_algs[i] = NULL; 539*1661131aSEric Biggers } 540*1661131aSEric Biggers } 541*1661131aSEric Biggers } 542*1661131aSEric Biggers EXPORT_SYMBOL_GPL(simd_unregister_aeads); 543*1661131aSEric Biggers 544266d0516SHerbert Xu MODULE_LICENSE("GPL"); 545