1 /* 2 * pcrypt - Parallel crypto wrapper. 3 * 4 * Copyright (C) 2009 secunet Security Networks AG 5 * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms and conditions of the GNU General Public License, 9 * version 2, as published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 14 * more details. 15 * 16 * You should have received a copy of the GNU General Public License along with 17 * this program; if not, write to the Free Software Foundation, Inc., 18 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. 19 */ 20 21 #include <crypto/algapi.h> 22 #include <crypto/internal/aead.h> 23 #include <linux/err.h> 24 #include <linux/init.h> 25 #include <linux/module.h> 26 #include <linux/slab.h> 27 #include <linux/notifier.h> 28 #include <linux/kobject.h> 29 #include <linux/cpu.h> 30 #include <crypto/pcrypt.h> 31 32 struct padata_pcrypt { 33 struct padata_instance *pinst; 34 struct workqueue_struct *wq; 35 36 /* 37 * Cpumask for callback CPUs. It should be 38 * equal to serial cpumask of corresponding padata instance, 39 * so it is updated when padata notifies us about serial 40 * cpumask change. 41 * 42 * cb_cpumask is protected by RCU. This fact prevents us from 43 * using cpumask_var_t directly because the actual type of 44 * cpumsak_var_t depends on kernel configuration(particularly on 45 * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration 46 * cpumask_var_t may be either a pointer to the struct cpumask 47 * or a variable allocated on the stack. Thus we can not safely use 48 * cpumask_var_t with RCU operations such as rcu_assign_pointer or 49 * rcu_dereference. So cpumask_var_t is wrapped with struct 50 * pcrypt_cpumask which makes possible to use it with RCU. 51 */ 52 struct pcrypt_cpumask { 53 cpumask_var_t mask; 54 } *cb_cpumask; 55 struct notifier_block nblock; 56 }; 57 58 static struct padata_pcrypt pencrypt; 59 static struct padata_pcrypt pdecrypt; 60 static struct kset *pcrypt_kset; 61 62 struct pcrypt_instance_ctx { 63 struct crypto_spawn spawn; 64 unsigned int tfm_count; 65 }; 66 67 struct pcrypt_aead_ctx { 68 struct crypto_aead *child; 69 unsigned int cb_cpu; 70 }; 71 72 static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, 73 struct padata_pcrypt *pcrypt) 74 { 75 unsigned int cpu_index, cpu, i; 76 struct pcrypt_cpumask *cpumask; 77 78 cpu = *cb_cpu; 79 80 rcu_read_lock_bh(); 81 cpumask = rcu_dereference_bh(pcrypt->cb_cpumask); 82 if (cpumask_test_cpu(cpu, cpumask->mask)) 83 goto out; 84 85 if (!cpumask_weight(cpumask->mask)) 86 goto out; 87 88 cpu_index = cpu % cpumask_weight(cpumask->mask); 89 90 cpu = cpumask_first(cpumask->mask); 91 for (i = 0; i < cpu_index; i++) 92 cpu = cpumask_next(cpu, cpumask->mask); 93 94 *cb_cpu = cpu; 95 96 out: 97 rcu_read_unlock_bh(); 98 return padata_do_parallel(pcrypt->pinst, padata, cpu); 99 } 100 101 static int pcrypt_aead_setkey(struct crypto_aead *parent, 102 const u8 *key, unsigned int keylen) 103 { 104 struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent); 105 106 return crypto_aead_setkey(ctx->child, key, keylen); 107 } 108 109 static int pcrypt_aead_setauthsize(struct crypto_aead *parent, 110 unsigned int authsize) 111 { 112 struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent); 113 114 return crypto_aead_setauthsize(ctx->child, authsize); 115 } 116 117 static void pcrypt_aead_serial(struct padata_priv *padata) 118 { 119 struct pcrypt_request *preq = pcrypt_padata_request(padata); 120 struct aead_request *req = pcrypt_request_ctx(preq); 121 122 aead_request_complete(req->base.data, padata->info); 123 } 124 125 static void pcrypt_aead_giv_serial(struct padata_priv *padata) 126 { 127 struct pcrypt_request *preq = pcrypt_padata_request(padata); 128 struct aead_givcrypt_request *req = pcrypt_request_ctx(preq); 129 130 aead_request_complete(req->areq.base.data, padata->info); 131 } 132 133 static void pcrypt_aead_done(struct crypto_async_request *areq, int err) 134 { 135 struct aead_request *req = areq->data; 136 struct pcrypt_request *preq = aead_request_ctx(req); 137 struct padata_priv *padata = pcrypt_request_padata(preq); 138 139 padata->info = err; 140 req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; 141 142 padata_do_serial(padata); 143 } 144 145 static void pcrypt_aead_enc(struct padata_priv *padata) 146 { 147 struct pcrypt_request *preq = pcrypt_padata_request(padata); 148 struct aead_request *req = pcrypt_request_ctx(preq); 149 150 padata->info = crypto_aead_encrypt(req); 151 152 if (padata->info == -EINPROGRESS) 153 return; 154 155 padata_do_serial(padata); 156 } 157 158 static int pcrypt_aead_encrypt(struct aead_request *req) 159 { 160 int err; 161 struct pcrypt_request *preq = aead_request_ctx(req); 162 struct aead_request *creq = pcrypt_request_ctx(preq); 163 struct padata_priv *padata = pcrypt_request_padata(preq); 164 struct crypto_aead *aead = crypto_aead_reqtfm(req); 165 struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); 166 u32 flags = aead_request_flags(req); 167 168 memset(padata, 0, sizeof(struct padata_priv)); 169 170 padata->parallel = pcrypt_aead_enc; 171 padata->serial = pcrypt_aead_serial; 172 173 aead_request_set_tfm(creq, ctx->child); 174 aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, 175 pcrypt_aead_done, req); 176 aead_request_set_crypt(creq, req->src, req->dst, 177 req->cryptlen, req->iv); 178 aead_request_set_assoc(creq, req->assoc, req->assoclen); 179 180 err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); 181 if (!err) 182 return -EINPROGRESS; 183 184 return err; 185 } 186 187 static void pcrypt_aead_dec(struct padata_priv *padata) 188 { 189 struct pcrypt_request *preq = pcrypt_padata_request(padata); 190 struct aead_request *req = pcrypt_request_ctx(preq); 191 192 padata->info = crypto_aead_decrypt(req); 193 194 if (padata->info == -EINPROGRESS) 195 return; 196 197 padata_do_serial(padata); 198 } 199 200 static int pcrypt_aead_decrypt(struct aead_request *req) 201 { 202 int err; 203 struct pcrypt_request *preq = aead_request_ctx(req); 204 struct aead_request *creq = pcrypt_request_ctx(preq); 205 struct padata_priv *padata = pcrypt_request_padata(preq); 206 struct crypto_aead *aead = crypto_aead_reqtfm(req); 207 struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); 208 u32 flags = aead_request_flags(req); 209 210 memset(padata, 0, sizeof(struct padata_priv)); 211 212 padata->parallel = pcrypt_aead_dec; 213 padata->serial = pcrypt_aead_serial; 214 215 aead_request_set_tfm(creq, ctx->child); 216 aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, 217 pcrypt_aead_done, req); 218 aead_request_set_crypt(creq, req->src, req->dst, 219 req->cryptlen, req->iv); 220 aead_request_set_assoc(creq, req->assoc, req->assoclen); 221 222 err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt); 223 if (!err) 224 return -EINPROGRESS; 225 226 return err; 227 } 228 229 static void pcrypt_aead_givenc(struct padata_priv *padata) 230 { 231 struct pcrypt_request *preq = pcrypt_padata_request(padata); 232 struct aead_givcrypt_request *req = pcrypt_request_ctx(preq); 233 234 padata->info = crypto_aead_givencrypt(req); 235 236 if (padata->info == -EINPROGRESS) 237 return; 238 239 padata_do_serial(padata); 240 } 241 242 static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req) 243 { 244 int err; 245 struct aead_request *areq = &req->areq; 246 struct pcrypt_request *preq = aead_request_ctx(areq); 247 struct aead_givcrypt_request *creq = pcrypt_request_ctx(preq); 248 struct padata_priv *padata = pcrypt_request_padata(preq); 249 struct crypto_aead *aead = aead_givcrypt_reqtfm(req); 250 struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); 251 u32 flags = aead_request_flags(areq); 252 253 memset(padata, 0, sizeof(struct padata_priv)); 254 255 padata->parallel = pcrypt_aead_givenc; 256 padata->serial = pcrypt_aead_giv_serial; 257 258 aead_givcrypt_set_tfm(creq, ctx->child); 259 aead_givcrypt_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, 260 pcrypt_aead_done, areq); 261 aead_givcrypt_set_crypt(creq, areq->src, areq->dst, 262 areq->cryptlen, areq->iv); 263 aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen); 264 aead_givcrypt_set_giv(creq, req->giv, req->seq); 265 266 err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); 267 if (!err) 268 return -EINPROGRESS; 269 270 return err; 271 } 272 273 static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm) 274 { 275 int cpu, cpu_index; 276 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); 277 struct pcrypt_instance_ctx *ictx = crypto_instance_ctx(inst); 278 struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm); 279 struct crypto_aead *cipher; 280 281 ictx->tfm_count++; 282 283 cpu_index = ictx->tfm_count % cpumask_weight(cpu_online_mask); 284 285 ctx->cb_cpu = cpumask_first(cpu_online_mask); 286 for (cpu = 0; cpu < cpu_index; cpu++) 287 ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask); 288 289 cipher = crypto_spawn_aead(crypto_instance_ctx(inst)); 290 291 if (IS_ERR(cipher)) 292 return PTR_ERR(cipher); 293 294 ctx->child = cipher; 295 crypto_aead_set_reqsize(__crypto_aead_cast(tfm), 296 sizeof(struct pcrypt_request) + 297 sizeof(struct aead_givcrypt_request) + 298 crypto_aead_reqsize(cipher)); 299 300 return 0; 301 } 302 303 static void pcrypt_aead_exit_tfm(struct crypto_tfm *tfm) 304 { 305 struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm); 306 307 crypto_free_aead(ctx->child); 308 } 309 310 static struct crypto_instance *pcrypt_alloc_instance(struct crypto_alg *alg) 311 { 312 struct crypto_instance *inst; 313 struct pcrypt_instance_ctx *ctx; 314 int err; 315 316 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); 317 if (!inst) { 318 inst = ERR_PTR(-ENOMEM); 319 goto out; 320 } 321 322 err = -ENAMETOOLONG; 323 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 324 "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 325 goto out_free_inst; 326 327 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); 328 329 ctx = crypto_instance_ctx(inst); 330 err = crypto_init_spawn(&ctx->spawn, alg, inst, 331 CRYPTO_ALG_TYPE_MASK); 332 if (err) 333 goto out_free_inst; 334 335 inst->alg.cra_priority = alg->cra_priority + 100; 336 inst->alg.cra_blocksize = alg->cra_blocksize; 337 inst->alg.cra_alignmask = alg->cra_alignmask; 338 339 out: 340 return inst; 341 342 out_free_inst: 343 kfree(inst); 344 inst = ERR_PTR(err); 345 goto out; 346 } 347 348 static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb, 349 u32 type, u32 mask) 350 { 351 struct crypto_instance *inst; 352 struct crypto_alg *alg; 353 354 alg = crypto_get_attr_alg(tb, type, (mask & CRYPTO_ALG_TYPE_MASK)); 355 if (IS_ERR(alg)) 356 return ERR_CAST(alg); 357 358 inst = pcrypt_alloc_instance(alg); 359 if (IS_ERR(inst)) 360 goto out_put_alg; 361 362 inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; 363 inst->alg.cra_type = &crypto_aead_type; 364 365 inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize; 366 inst->alg.cra_aead.geniv = alg->cra_aead.geniv; 367 inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize; 368 369 inst->alg.cra_ctxsize = sizeof(struct pcrypt_aead_ctx); 370 371 inst->alg.cra_init = pcrypt_aead_init_tfm; 372 inst->alg.cra_exit = pcrypt_aead_exit_tfm; 373 374 inst->alg.cra_aead.setkey = pcrypt_aead_setkey; 375 inst->alg.cra_aead.setauthsize = pcrypt_aead_setauthsize; 376 inst->alg.cra_aead.encrypt = pcrypt_aead_encrypt; 377 inst->alg.cra_aead.decrypt = pcrypt_aead_decrypt; 378 inst->alg.cra_aead.givencrypt = pcrypt_aead_givencrypt; 379 380 out_put_alg: 381 crypto_mod_put(alg); 382 return inst; 383 } 384 385 static struct crypto_instance *pcrypt_alloc(struct rtattr **tb) 386 { 387 struct crypto_attr_type *algt; 388 389 algt = crypto_get_attr_type(tb); 390 if (IS_ERR(algt)) 391 return ERR_CAST(algt); 392 393 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { 394 case CRYPTO_ALG_TYPE_AEAD: 395 return pcrypt_alloc_aead(tb, algt->type, algt->mask); 396 } 397 398 return ERR_PTR(-EINVAL); 399 } 400 401 static void pcrypt_free(struct crypto_instance *inst) 402 { 403 struct pcrypt_instance_ctx *ctx = crypto_instance_ctx(inst); 404 405 crypto_drop_spawn(&ctx->spawn); 406 kfree(inst); 407 } 408 409 static int pcrypt_cpumask_change_notify(struct notifier_block *self, 410 unsigned long val, void *data) 411 { 412 struct padata_pcrypt *pcrypt; 413 struct pcrypt_cpumask *new_mask, *old_mask; 414 struct padata_cpumask *cpumask = (struct padata_cpumask *)data; 415 416 if (!(val & PADATA_CPU_SERIAL)) 417 return 0; 418 419 pcrypt = container_of(self, struct padata_pcrypt, nblock); 420 new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL); 421 if (!new_mask) 422 return -ENOMEM; 423 if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) { 424 kfree(new_mask); 425 return -ENOMEM; 426 } 427 428 old_mask = pcrypt->cb_cpumask; 429 430 cpumask_copy(new_mask->mask, cpumask->cbcpu); 431 rcu_assign_pointer(pcrypt->cb_cpumask, new_mask); 432 synchronize_rcu_bh(); 433 434 free_cpumask_var(old_mask->mask); 435 kfree(old_mask); 436 return 0; 437 } 438 439 static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name) 440 { 441 int ret; 442 443 pinst->kobj.kset = pcrypt_kset; 444 ret = kobject_add(&pinst->kobj, NULL, name); 445 if (!ret) 446 kobject_uevent(&pinst->kobj, KOBJ_ADD); 447 448 return ret; 449 } 450 451 static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, 452 const char *name) 453 { 454 int ret = -ENOMEM; 455 struct pcrypt_cpumask *mask; 456 457 get_online_cpus(); 458 459 pcrypt->wq = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 460 1, name); 461 if (!pcrypt->wq) 462 goto err; 463 464 pcrypt->pinst = padata_alloc_possible(pcrypt->wq); 465 if (!pcrypt->pinst) 466 goto err_destroy_workqueue; 467 468 mask = kmalloc(sizeof(*mask), GFP_KERNEL); 469 if (!mask) 470 goto err_free_padata; 471 if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) { 472 kfree(mask); 473 goto err_free_padata; 474 } 475 476 cpumask_and(mask->mask, cpu_possible_mask, cpu_online_mask); 477 rcu_assign_pointer(pcrypt->cb_cpumask, mask); 478 479 pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify; 480 ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); 481 if (ret) 482 goto err_free_cpumask; 483 484 ret = pcrypt_sysfs_add(pcrypt->pinst, name); 485 if (ret) 486 goto err_unregister_notifier; 487 488 put_online_cpus(); 489 490 return ret; 491 492 err_unregister_notifier: 493 padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); 494 err_free_cpumask: 495 free_cpumask_var(mask->mask); 496 kfree(mask); 497 err_free_padata: 498 padata_free(pcrypt->pinst); 499 err_destroy_workqueue: 500 destroy_workqueue(pcrypt->wq); 501 err: 502 put_online_cpus(); 503 504 return ret; 505 } 506 507 static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt) 508 { 509 free_cpumask_var(pcrypt->cb_cpumask->mask); 510 kfree(pcrypt->cb_cpumask); 511 512 padata_stop(pcrypt->pinst); 513 padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); 514 destroy_workqueue(pcrypt->wq); 515 padata_free(pcrypt->pinst); 516 } 517 518 static struct crypto_template pcrypt_tmpl = { 519 .name = "pcrypt", 520 .alloc = pcrypt_alloc, 521 .free = pcrypt_free, 522 .module = THIS_MODULE, 523 }; 524 525 static int __init pcrypt_init(void) 526 { 527 int err = -ENOMEM; 528 529 pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj); 530 if (!pcrypt_kset) 531 goto err; 532 533 err = pcrypt_init_padata(&pencrypt, "pencrypt"); 534 if (err) 535 goto err_unreg_kset; 536 537 err = pcrypt_init_padata(&pdecrypt, "pdecrypt"); 538 if (err) 539 goto err_deinit_pencrypt; 540 541 padata_start(pencrypt.pinst); 542 padata_start(pdecrypt.pinst); 543 544 return crypto_register_template(&pcrypt_tmpl); 545 546 err_deinit_pencrypt: 547 pcrypt_fini_padata(&pencrypt); 548 err_unreg_kset: 549 kset_unregister(pcrypt_kset); 550 err: 551 return err; 552 } 553 554 static void __exit pcrypt_exit(void) 555 { 556 pcrypt_fini_padata(&pencrypt); 557 pcrypt_fini_padata(&pdecrypt); 558 559 kset_unregister(pcrypt_kset); 560 crypto_unregister_template(&pcrypt_tmpl); 561 } 562 563 module_init(pcrypt_init); 564 module_exit(pcrypt_exit); 565 566 MODULE_LICENSE("GPL"); 567 MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>"); 568 MODULE_DESCRIPTION("Parallel crypto wrapper"); 569 MODULE_ALIAS_CRYPTO("pcrypt"); 570