1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Support for Marvell's Cryptographic Engine and Security Accelerator (CESA) 4 * that can be found on the following platform: Orion, Kirkwood, Armada. This 5 * driver supports the TDMA engine on platforms on which it is available. 6 * 7 * Author: Boris Brezillon <boris.brezillon@free-electrons.com> 8 * Author: Arnaud Ebalard <arno@natisbad.org> 9 * 10 * This work is based on an initial version written by 11 * Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > 12 */ 13 14 #include <linux/delay.h> 15 #include <linux/dma-mapping.h> 16 #include <linux/genalloc.h> 17 #include <linux/interrupt.h> 18 #include <linux/io.h> 19 #include <linux/kthread.h> 20 #include <linux/mbus.h> 21 #include <linux/minmax.h> 22 #include <linux/platform_device.h> 23 #include <linux/scatterlist.h> 24 #include <linux/slab.h> 25 #include <linux/module.h> 26 #include <linux/clk.h> 27 #include <linux/of.h> 28 #include <linux/of_platform.h> 29 #include <linux/of_irq.h> 30 31 #include "cesa.h" 32 33 /* Limit of the crypto queue before reaching the backlog */ 34 #define CESA_CRYPTO_DEFAULT_MAX_QLEN 128 35 36 struct mv_cesa_dev *cesa_dev; 37 38 struct crypto_async_request * 39 mv_cesa_dequeue_req_locked(struct mv_cesa_engine *engine, 40 struct crypto_async_request **backlog) 41 { 42 *backlog = crypto_get_backlog(&engine->queue); 43 44 return crypto_dequeue_request(&engine->queue); 45 } 46 47 static void mv_cesa_rearm_engine(struct mv_cesa_engine *engine) 48 { 49 struct crypto_async_request *req = NULL, *backlog = NULL; 50 struct mv_cesa_ctx *ctx; 51 52 53 spin_lock_bh(&engine->lock); 54 if (!engine->req) { 55 req = mv_cesa_dequeue_req_locked(engine, &backlog); 56 engine->req = req; 57 } 58 spin_unlock_bh(&engine->lock); 59 60 if (!req) 61 return; 62 63 if (backlog) 64 crypto_request_complete(backlog, -EINPROGRESS); 65 66 ctx = crypto_tfm_ctx(req->tfm); 67 ctx->ops->step(req); 68 } 69 70 static int mv_cesa_std_process(struct mv_cesa_engine *engine, u32 status) 71 { 72 struct crypto_async_request *req; 73 struct mv_cesa_ctx *ctx; 74 int res; 75 76 req = engine->req; 77 ctx = crypto_tfm_ctx(req->tfm); 78 res = ctx->ops->process(req, status); 79 80 if (res == 0) { 81 ctx->ops->complete(req); 82 mv_cesa_engine_enqueue_complete_request(engine, req); 83 } else if (res == -EINPROGRESS) { 84 ctx->ops->step(req); 85 } 86 87 return res; 88 } 89 90 static int mv_cesa_int_process(struct mv_cesa_engine *engine, u32 status) 91 { 92 if (engine->chain_hw.first && engine->chain_hw.last) 93 return mv_cesa_tdma_process(engine, status); 94 95 return mv_cesa_std_process(engine, status); 96 } 97 98 static inline void 99 mv_cesa_complete_req(struct mv_cesa_ctx *ctx, struct crypto_async_request *req, 100 int res) 101 { 102 ctx->ops->cleanup(req); 103 local_bh_disable(); 104 crypto_request_complete(req, res); 105 local_bh_enable(); 106 } 107 108 static irqreturn_t mv_cesa_int(int irq, void *priv) 109 { 110 struct mv_cesa_engine *engine = priv; 111 struct crypto_async_request *req; 112 struct mv_cesa_ctx *ctx; 113 u32 status, mask; 114 irqreturn_t ret = IRQ_NONE; 115 116 while (true) { 117 int res; 118 119 mask = mv_cesa_get_int_mask(engine); 120 status = readl(engine->regs + CESA_SA_INT_STATUS); 121 122 if (!(status & mask)) 123 break; 124 125 /* 126 * TODO: avoid clearing the FPGA_INT_STATUS if this not 127 * relevant on some platforms. 128 */ 129 writel(~status, engine->regs + CESA_SA_FPGA_INT_STATUS); 130 writel(~status, engine->regs + CESA_SA_INT_STATUS); 131 132 /* Process fetched requests */ 133 res = mv_cesa_int_process(engine, status & mask); 134 ret = IRQ_HANDLED; 135 136 spin_lock_bh(&engine->lock); 137 req = engine->req; 138 if (res != -EINPROGRESS) 139 engine->req = NULL; 140 spin_unlock_bh(&engine->lock); 141 142 ctx = crypto_tfm_ctx(req->tfm); 143 144 if (res && res != -EINPROGRESS) 145 mv_cesa_complete_req(ctx, req, res); 146 147 /* Launch the next pending request */ 148 mv_cesa_rearm_engine(engine); 149 150 /* Iterate over the complete queue */ 151 while (true) { 152 req = mv_cesa_engine_dequeue_complete_request(engine); 153 if (!req) 154 break; 155 156 ctx = crypto_tfm_ctx(req->tfm); 157 mv_cesa_complete_req(ctx, req, 0); 158 } 159 } 160 161 return ret; 162 } 163 164 int mv_cesa_queue_req(struct crypto_async_request *req, 165 struct mv_cesa_req *creq) 166 { 167 int ret; 168 struct mv_cesa_engine *engine = creq->engine; 169 170 spin_lock_bh(&engine->lock); 171 ret = crypto_enqueue_request(&engine->queue, req); 172 if ((mv_cesa_req_get_type(creq) == CESA_DMA_REQ) && 173 (ret == -EINPROGRESS || ret == -EBUSY)) 174 mv_cesa_tdma_chain(engine, creq); 175 spin_unlock_bh(&engine->lock); 176 177 if (ret != -EINPROGRESS) 178 return ret; 179 180 mv_cesa_rearm_engine(engine); 181 182 return -EINPROGRESS; 183 } 184 185 static int mv_cesa_add_algs(struct mv_cesa_dev *cesa) 186 { 187 int ret; 188 int i, j; 189 190 for (i = 0; i < cesa->caps->ncipher_algs; i++) { 191 ret = crypto_register_skcipher(cesa->caps->cipher_algs[i]); 192 if (ret) 193 goto err_unregister_crypto; 194 } 195 196 for (i = 0; i < cesa->caps->nahash_algs; i++) { 197 ret = crypto_register_ahash(cesa->caps->ahash_algs[i]); 198 if (ret) 199 goto err_unregister_ahash; 200 } 201 202 return 0; 203 204 err_unregister_ahash: 205 for (j = 0; j < i; j++) 206 crypto_unregister_ahash(cesa->caps->ahash_algs[j]); 207 i = cesa->caps->ncipher_algs; 208 209 err_unregister_crypto: 210 for (j = 0; j < i; j++) 211 crypto_unregister_skcipher(cesa->caps->cipher_algs[j]); 212 213 return ret; 214 } 215 216 static void mv_cesa_remove_algs(struct mv_cesa_dev *cesa) 217 { 218 int i; 219 220 for (i = 0; i < cesa->caps->nahash_algs; i++) 221 crypto_unregister_ahash(cesa->caps->ahash_algs[i]); 222 223 for (i = 0; i < cesa->caps->ncipher_algs; i++) 224 crypto_unregister_skcipher(cesa->caps->cipher_algs[i]); 225 } 226 227 static struct skcipher_alg *orion_cipher_algs[] = { 228 &mv_cesa_ecb_des_alg, 229 &mv_cesa_cbc_des_alg, 230 &mv_cesa_ecb_des3_ede_alg, 231 &mv_cesa_cbc_des3_ede_alg, 232 &mv_cesa_ecb_aes_alg, 233 &mv_cesa_cbc_aes_alg, 234 }; 235 236 static struct ahash_alg *orion_ahash_algs[] = { 237 &mv_md5_alg, 238 &mv_sha1_alg, 239 &mv_ahmac_md5_alg, 240 &mv_ahmac_sha1_alg, 241 }; 242 243 static struct skcipher_alg *armada_370_cipher_algs[] = { 244 &mv_cesa_ecb_des_alg, 245 &mv_cesa_cbc_des_alg, 246 &mv_cesa_ecb_des3_ede_alg, 247 &mv_cesa_cbc_des3_ede_alg, 248 &mv_cesa_ecb_aes_alg, 249 &mv_cesa_cbc_aes_alg, 250 }; 251 252 static struct ahash_alg *armada_370_ahash_algs[] = { 253 &mv_md5_alg, 254 &mv_sha1_alg, 255 &mv_sha256_alg, 256 &mv_ahmac_md5_alg, 257 &mv_ahmac_sha1_alg, 258 &mv_ahmac_sha256_alg, 259 }; 260 261 static const struct mv_cesa_caps orion_caps = { 262 .nengines = 1, 263 .cipher_algs = orion_cipher_algs, 264 .ncipher_algs = ARRAY_SIZE(orion_cipher_algs), 265 .ahash_algs = orion_ahash_algs, 266 .nahash_algs = ARRAY_SIZE(orion_ahash_algs), 267 .has_tdma = false, 268 }; 269 270 static const struct mv_cesa_caps kirkwood_caps = { 271 .nengines = 1, 272 .cipher_algs = orion_cipher_algs, 273 .ncipher_algs = ARRAY_SIZE(orion_cipher_algs), 274 .ahash_algs = orion_ahash_algs, 275 .nahash_algs = ARRAY_SIZE(orion_ahash_algs), 276 .has_tdma = true, 277 }; 278 279 static const struct mv_cesa_caps armada_370_caps = { 280 .nengines = 1, 281 .cipher_algs = armada_370_cipher_algs, 282 .ncipher_algs = ARRAY_SIZE(armada_370_cipher_algs), 283 .ahash_algs = armada_370_ahash_algs, 284 .nahash_algs = ARRAY_SIZE(armada_370_ahash_algs), 285 .has_tdma = true, 286 }; 287 288 static const struct mv_cesa_caps armada_xp_caps = { 289 .nengines = 2, 290 .cipher_algs = armada_370_cipher_algs, 291 .ncipher_algs = ARRAY_SIZE(armada_370_cipher_algs), 292 .ahash_algs = armada_370_ahash_algs, 293 .nahash_algs = ARRAY_SIZE(armada_370_ahash_algs), 294 .has_tdma = true, 295 }; 296 297 static const struct of_device_id mv_cesa_of_match_table[] = { 298 { .compatible = "marvell,orion-crypto", .data = &orion_caps }, 299 { .compatible = "marvell,kirkwood-crypto", .data = &kirkwood_caps }, 300 { .compatible = "marvell,dove-crypto", .data = &kirkwood_caps }, 301 { .compatible = "marvell,armada-370-crypto", .data = &armada_370_caps }, 302 { .compatible = "marvell,armada-xp-crypto", .data = &armada_xp_caps }, 303 { .compatible = "marvell,armada-375-crypto", .data = &armada_xp_caps }, 304 { .compatible = "marvell,armada-38x-crypto", .data = &armada_xp_caps }, 305 {} 306 }; 307 MODULE_DEVICE_TABLE(of, mv_cesa_of_match_table); 308 309 static void 310 mv_cesa_conf_mbus_windows(struct mv_cesa_engine *engine, 311 const struct mbus_dram_target_info *dram) 312 { 313 void __iomem *iobase = engine->regs; 314 int i; 315 316 for (i = 0; i < 4; i++) { 317 writel(0, iobase + CESA_TDMA_WINDOW_CTRL(i)); 318 writel(0, iobase + CESA_TDMA_WINDOW_BASE(i)); 319 } 320 321 for (i = 0; i < dram->num_cs; i++) { 322 const struct mbus_dram_window *cs = dram->cs + i; 323 324 writel(((cs->size - 1) & 0xffff0000) | 325 (cs->mbus_attr << 8) | 326 (dram->mbus_dram_target_id << 4) | 1, 327 iobase + CESA_TDMA_WINDOW_CTRL(i)); 328 writel(cs->base, iobase + CESA_TDMA_WINDOW_BASE(i)); 329 } 330 } 331 332 static int mv_cesa_dev_dma_init(struct mv_cesa_dev *cesa) 333 { 334 struct device *dev = cesa->dev; 335 struct mv_cesa_dev_dma *dma; 336 337 if (!cesa->caps->has_tdma) 338 return 0; 339 340 dma = devm_kzalloc(dev, sizeof(*dma), GFP_KERNEL); 341 if (!dma) 342 return -ENOMEM; 343 344 dma->tdma_desc_pool = dmam_pool_create("tdma_desc", dev, 345 sizeof(struct mv_cesa_tdma_desc), 346 16, 0); 347 if (!dma->tdma_desc_pool) 348 return -ENOMEM; 349 350 dma->op_pool = dmam_pool_create("cesa_op", dev, 351 sizeof(struct mv_cesa_op_ctx), 16, 0); 352 if (!dma->op_pool) 353 return -ENOMEM; 354 355 dma->cache_pool = dmam_pool_create("cesa_cache", dev, 356 CESA_MAX_HASH_BLOCK_SIZE, 1, 0); 357 if (!dma->cache_pool) 358 return -ENOMEM; 359 360 dma->padding_pool = dmam_pool_create("cesa_padding", dev, 72, 1, 0); 361 if (!dma->padding_pool) 362 return -ENOMEM; 363 364 cesa->dma = dma; 365 366 return 0; 367 } 368 369 static int mv_cesa_get_sram(struct platform_device *pdev, int idx) 370 { 371 struct mv_cesa_dev *cesa = platform_get_drvdata(pdev); 372 struct mv_cesa_engine *engine = &cesa->engines[idx]; 373 struct resource *res; 374 375 engine->pool = of_gen_pool_get(cesa->dev->of_node, 376 "marvell,crypto-srams", idx); 377 if (engine->pool) { 378 engine->sram_pool = gen_pool_dma_alloc(engine->pool, 379 cesa->sram_size, 380 &engine->sram_dma); 381 if (engine->sram_pool) 382 return 0; 383 384 engine->pool = NULL; 385 return -ENOMEM; 386 } 387 388 engine->sram = devm_platform_get_and_ioremap_resource(pdev, idx, &res); 389 if (IS_ERR(engine->sram)) 390 return PTR_ERR(engine->sram); 391 392 engine->sram_dma = dma_map_resource(cesa->dev, res->start, 393 cesa->sram_size, 394 DMA_BIDIRECTIONAL, 0); 395 if (dma_mapping_error(cesa->dev, engine->sram_dma)) 396 return -ENOMEM; 397 398 return 0; 399 } 400 401 static void mv_cesa_put_sram(struct platform_device *pdev, int idx) 402 { 403 struct mv_cesa_dev *cesa = platform_get_drvdata(pdev); 404 struct mv_cesa_engine *engine = &cesa->engines[idx]; 405 406 if (engine->pool) 407 gen_pool_free(engine->pool, (unsigned long)engine->sram_pool, 408 cesa->sram_size); 409 else 410 dma_unmap_resource(cesa->dev, engine->sram_dma, 411 cesa->sram_size, DMA_BIDIRECTIONAL, 0); 412 } 413 414 static int mv_cesa_probe(struct platform_device *pdev) 415 { 416 const struct mv_cesa_caps *caps = &orion_caps; 417 const struct mbus_dram_target_info *dram; 418 struct device *dev = &pdev->dev; 419 struct mv_cesa_dev *cesa; 420 struct mv_cesa_engine *engine; 421 int irq, ret, i, cpu; 422 u32 sram_size; 423 424 if (cesa_dev) { 425 dev_err(&pdev->dev, "Only one CESA device authorized\n"); 426 return -EEXIST; 427 } 428 429 if (dev->of_node) { 430 caps = of_device_get_match_data(dev); 431 if (!caps) 432 return -ENOTSUPP; 433 } 434 435 cesa = devm_kzalloc(dev, struct_size(cesa, engines, caps->nengines), 436 GFP_KERNEL); 437 if (!cesa) 438 return -ENOMEM; 439 440 cesa->caps = caps; 441 cesa->dev = dev; 442 443 sram_size = CESA_SA_DEFAULT_SRAM_SIZE; 444 of_property_read_u32(cesa->dev->of_node, "marvell,crypto-sram-size", 445 &sram_size); 446 447 cesa->sram_size = max(sram_size, CESA_SA_MIN_SRAM_SIZE); 448 449 spin_lock_init(&cesa->lock); 450 451 cesa->regs = devm_platform_ioremap_resource_byname(pdev, "regs"); 452 if (IS_ERR(cesa->regs)) 453 return PTR_ERR(cesa->regs); 454 455 ret = mv_cesa_dev_dma_init(cesa); 456 if (ret) 457 return ret; 458 459 dram = mv_mbus_dram_info_nooverlap(); 460 461 platform_set_drvdata(pdev, cesa); 462 463 for (i = 0; i < caps->nengines; i++) { 464 engine = &cesa->engines[i]; 465 char res_name[16]; 466 467 engine->id = i; 468 spin_lock_init(&engine->lock); 469 470 ret = mv_cesa_get_sram(pdev, i); 471 if (ret) 472 goto err_cleanup; 473 474 irq = platform_get_irq(pdev, i); 475 if (irq < 0) { 476 ret = irq; 477 goto err_cleanup; 478 } 479 480 engine->irq = irq; 481 482 /* 483 * Not all platforms can gate the CESA clocks: do not complain 484 * if the clock does not exist. 485 */ 486 snprintf(res_name, sizeof(res_name), "cesa%u", i); 487 engine->clk = devm_clk_get_optional_enabled(dev, res_name); 488 if (IS_ERR(engine->clk)) { 489 engine->clk = devm_clk_get_optional_enabled(dev, NULL); 490 if (IS_ERR(engine->clk)) { 491 ret = PTR_ERR(engine->clk); 492 goto err_cleanup; 493 } 494 } 495 496 snprintf(res_name, sizeof(res_name), "cesaz%u", i); 497 engine->zclk = devm_clk_get_optional_enabled(dev, res_name); 498 if (IS_ERR(engine->zclk)) { 499 ret = PTR_ERR(engine->zclk); 500 goto err_cleanup; 501 } 502 503 engine->regs = cesa->regs + CESA_ENGINE_OFF(i); 504 505 if (dram && cesa->caps->has_tdma) 506 mv_cesa_conf_mbus_windows(engine, dram); 507 508 writel(0, engine->regs + CESA_SA_INT_STATUS); 509 writel(CESA_SA_CFG_STOP_DIG_ERR, 510 engine->regs + CESA_SA_CFG); 511 writel(engine->sram_dma & CESA_SA_SRAM_MSK, 512 engine->regs + CESA_SA_DESC_P0); 513 514 ret = devm_request_threaded_irq(dev, irq, NULL, mv_cesa_int, 515 IRQF_ONESHOT, 516 dev_name(&pdev->dev), 517 engine); 518 if (ret) 519 goto err_cleanup; 520 521 /* Set affinity */ 522 cpu = cpumask_local_spread(engine->id, NUMA_NO_NODE); 523 irq_set_affinity_hint(irq, get_cpu_mask(cpu)); 524 525 crypto_init_queue(&engine->queue, CESA_CRYPTO_DEFAULT_MAX_QLEN); 526 atomic_set(&engine->load, 0); 527 INIT_LIST_HEAD(&engine->complete_queue); 528 } 529 530 cesa_dev = cesa; 531 532 ret = mv_cesa_add_algs(cesa); 533 if (ret) { 534 cesa_dev = NULL; 535 goto err_cleanup; 536 } 537 538 dev_info(dev, "CESA device successfully registered\n"); 539 540 return 0; 541 542 err_cleanup: 543 for (i = 0; i < caps->nengines; i++) 544 mv_cesa_put_sram(pdev, i); 545 546 return ret; 547 } 548 549 static void mv_cesa_remove(struct platform_device *pdev) 550 { 551 struct mv_cesa_dev *cesa = platform_get_drvdata(pdev); 552 int i; 553 554 mv_cesa_remove_algs(cesa); 555 556 for (i = 0; i < cesa->caps->nengines; i++) 557 mv_cesa_put_sram(pdev, i); 558 } 559 560 static const struct platform_device_id mv_cesa_plat_id_table[] = { 561 { .name = "mv_crypto" }, 562 { /* sentinel */ }, 563 }; 564 MODULE_DEVICE_TABLE(platform, mv_cesa_plat_id_table); 565 566 static struct platform_driver marvell_cesa = { 567 .probe = mv_cesa_probe, 568 .remove = mv_cesa_remove, 569 .id_table = mv_cesa_plat_id_table, 570 .driver = { 571 .name = "marvell-cesa", 572 .of_match_table = mv_cesa_of_match_table, 573 }, 574 }; 575 module_platform_driver(marvell_cesa); 576 577 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>"); 578 MODULE_AUTHOR("Arnaud Ebalard <arno@natisbad.org>"); 579 MODULE_DESCRIPTION("Support for Marvell's cryptographic engine"); 580 MODULE_LICENSE("GPL v2"); 581