1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * sl3516-ce-core.c - hardware cryptographic offloader for Storlink SL3516 SoC 4 * 5 * Copyright (C) 2021 Corentin Labbe <clabbe@baylibre.com> 6 * 7 * Core file which registers crypto algorithms supported by the CryptoEngine 8 */ 9 10 #include <crypto/engine.h> 11 #include <crypto/internal/rng.h> 12 #include <crypto/internal/skcipher.h> 13 #include <linux/clk.h> 14 #include <linux/debugfs.h> 15 #include <linux/dev_printk.h> 16 #include <linux/dma-mapping.h> 17 #include <linux/err.h> 18 #include <linux/interrupt.h> 19 #include <linux/io.h> 20 #include <linux/irq.h> 21 #include <linux/kernel.h> 22 #include <linux/module.h> 23 #include <linux/of.h> 24 #include <linux/platform_device.h> 25 #include <linux/pm_runtime.h> 26 #include <linux/reset.h> 27 28 #include "sl3516-ce.h" 29 30 static int sl3516_ce_desc_init(struct sl3516_ce_dev *ce) 31 { 32 const size_t sz = sizeof(struct descriptor) * MAXDESC; 33 int i; 34 35 ce->tx = dma_alloc_coherent(ce->dev, sz, &ce->dtx, GFP_KERNEL); 36 if (!ce->tx) 37 return -ENOMEM; 38 ce->rx = dma_alloc_coherent(ce->dev, sz, &ce->drx, GFP_KERNEL); 39 if (!ce->rx) 40 goto err_rx; 41 42 for (i = 0; i < MAXDESC; i++) { 43 ce->tx[i].frame_ctrl.bits.own = CE_CPU; 44 ce->tx[i].next_desc.next_descriptor = ce->dtx + (i + 1) * sizeof(struct descriptor); 45 } 46 ce->tx[MAXDESC - 1].next_desc.next_descriptor = ce->dtx; 47 48 for (i = 0; i < MAXDESC; i++) { 49 ce->rx[i].frame_ctrl.bits.own = CE_CPU; 50 ce->rx[i].next_desc.next_descriptor = ce->drx + (i + 1) * sizeof(struct descriptor); 51 } 52 ce->rx[MAXDESC - 1].next_desc.next_descriptor = ce->drx; 53 54 ce->pctrl = dma_alloc_coherent(ce->dev, sizeof(struct pkt_control_ecb), 55 &ce->dctrl, GFP_KERNEL); 56 if (!ce->pctrl) 57 goto err_pctrl; 58 59 return 0; 60 err_pctrl: 61 dma_free_coherent(ce->dev, sz, ce->rx, ce->drx); 62 err_rx: 63 dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx); 64 return -ENOMEM; 65 } 66 67 static void sl3516_ce_free_descs(struct sl3516_ce_dev *ce) 68 { 69 const size_t sz = sizeof(struct descriptor) * MAXDESC; 70 71 dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx); 72 dma_free_coherent(ce->dev, sz, ce->rx, ce->drx); 73 dma_free_coherent(ce->dev, sizeof(struct pkt_control_ecb), ce->pctrl, 74 ce->dctrl); 75 } 76 77 static void start_dma_tx(struct sl3516_ce_dev *ce) 78 { 79 u32 v; 80 81 v = TXDMA_CTRL_START | TXDMA_CTRL_CHAIN_MODE | TXDMA_CTRL_CONTINUE | \ 82 TXDMA_CTRL_INT_FAIL | TXDMA_CTRL_INT_PERR | TXDMA_CTRL_BURST_UNK; 83 84 writel(v, ce->base + IPSEC_TXDMA_CTRL); 85 } 86 87 static void start_dma_rx(struct sl3516_ce_dev *ce) 88 { 89 u32 v; 90 91 v = RXDMA_CTRL_START | RXDMA_CTRL_CHAIN_MODE | RXDMA_CTRL_CONTINUE | \ 92 RXDMA_CTRL_BURST_UNK | RXDMA_CTRL_INT_FINISH | \ 93 RXDMA_CTRL_INT_FAIL | RXDMA_CTRL_INT_PERR | \ 94 RXDMA_CTRL_INT_EOD | RXDMA_CTRL_INT_EOF; 95 96 writel(v, ce->base + IPSEC_RXDMA_CTRL); 97 } 98 99 static struct descriptor *get_desc_tx(struct sl3516_ce_dev *ce) 100 { 101 struct descriptor *dd; 102 103 dd = &ce->tx[ce->ctx]; 104 ce->ctx++; 105 if (ce->ctx >= MAXDESC) 106 ce->ctx = 0; 107 return dd; 108 } 109 110 static struct descriptor *get_desc_rx(struct sl3516_ce_dev *ce) 111 { 112 struct descriptor *rdd; 113 114 rdd = &ce->rx[ce->crx]; 115 ce->crx++; 116 if (ce->crx >= MAXDESC) 117 ce->crx = 0; 118 return rdd; 119 } 120 121 int sl3516_ce_run_task(struct sl3516_ce_dev *ce, struct sl3516_ce_cipher_req_ctx *rctx, 122 const char *name) 123 { 124 struct descriptor *dd, *rdd = NULL; 125 u32 v; 126 int i, err = 0; 127 128 ce->stat_req++; 129 130 reinit_completion(&ce->complete); 131 ce->status = 0; 132 133 for (i = 0; i < rctx->nr_sgd; i++) { 134 dev_dbg(ce->dev, "%s handle DST SG %d/%d len=%d\n", __func__, 135 i, rctx->nr_sgd, rctx->t_dst[i].len); 136 rdd = get_desc_rx(ce); 137 rdd->buf_adr = rctx->t_dst[i].addr; 138 rdd->frame_ctrl.bits.buffer_size = rctx->t_dst[i].len; 139 rdd->frame_ctrl.bits.own = CE_DMA; 140 } 141 rdd->next_desc.bits.eofie = 1; 142 143 for (i = 0; i < rctx->nr_sgs; i++) { 144 dev_dbg(ce->dev, "%s handle SRC SG %d/%d len=%d\n", __func__, 145 i, rctx->nr_sgs, rctx->t_src[i].len); 146 rctx->h->algorithm_len = rctx->t_src[i].len; 147 148 dd = get_desc_tx(ce); 149 dd->frame_ctrl.raw = 0; 150 dd->flag_status.raw = 0; 151 dd->frame_ctrl.bits.buffer_size = rctx->pctrllen; 152 dd->buf_adr = ce->dctrl; 153 dd->flag_status.tx_flag.tqflag = rctx->tqflag; 154 dd->next_desc.bits.eofie = 0; 155 dd->next_desc.bits.dec = 0; 156 dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST; 157 dd->frame_ctrl.bits.own = CE_DMA; 158 159 dd = get_desc_tx(ce); 160 dd->frame_ctrl.raw = 0; 161 dd->flag_status.raw = 0; 162 dd->frame_ctrl.bits.buffer_size = rctx->t_src[i].len; 163 dd->buf_adr = rctx->t_src[i].addr; 164 dd->flag_status.tx_flag.tqflag = 0; 165 dd->next_desc.bits.eofie = 0; 166 dd->next_desc.bits.dec = 0; 167 dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST; 168 dd->frame_ctrl.bits.own = CE_DMA; 169 start_dma_tx(ce); 170 start_dma_rx(ce); 171 } 172 wait_for_completion_interruptible_timeout(&ce->complete, 173 msecs_to_jiffies(5000)); 174 if (ce->status == 0) { 175 dev_err(ce->dev, "DMA timeout for %s\n", name); 176 err = -EFAULT; 177 } 178 v = readl(ce->base + IPSEC_STATUS_REG); 179 if (v & 0xFFF) { 180 dev_err(ce->dev, "IPSEC_STATUS_REG %x\n", v); 181 err = -EFAULT; 182 } 183 184 return err; 185 } 186 187 static irqreturn_t ce_irq_handler(int irq, void *data) 188 { 189 struct sl3516_ce_dev *ce = (struct sl3516_ce_dev *)data; 190 u32 v; 191 192 ce->stat_irq++; 193 194 v = readl(ce->base + IPSEC_DMA_STATUS); 195 writel(v, ce->base + IPSEC_DMA_STATUS); 196 197 if (v & DMA_STATUS_TS_DERR) 198 dev_err(ce->dev, "AHB bus Error While Tx !!!\n"); 199 if (v & DMA_STATUS_TS_PERR) 200 dev_err(ce->dev, "Tx Descriptor Protocol Error !!!\n"); 201 if (v & DMA_STATUS_RS_DERR) 202 dev_err(ce->dev, "AHB bus Error While Rx !!!\n"); 203 if (v & DMA_STATUS_RS_PERR) 204 dev_err(ce->dev, "Rx Descriptor Protocol Error !!!\n"); 205 206 if (v & DMA_STATUS_TS_EOFI) 207 ce->stat_irq_tx++; 208 if (v & DMA_STATUS_RS_EOFI) { 209 ce->status = 1; 210 complete(&ce->complete); 211 ce->stat_irq_rx++; 212 return IRQ_HANDLED; 213 } 214 215 return IRQ_HANDLED; 216 } 217 218 static struct sl3516_ce_alg_template ce_algs[] = { 219 { 220 .type = CRYPTO_ALG_TYPE_SKCIPHER, 221 .mode = ECB_AES, 222 .alg.skcipher.base = { 223 .base = { 224 .cra_name = "ecb(aes)", 225 .cra_driver_name = "ecb-aes-sl3516", 226 .cra_priority = 400, 227 .cra_blocksize = AES_BLOCK_SIZE, 228 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 229 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 230 .cra_ctxsize = sizeof(struct sl3516_ce_cipher_tfm_ctx), 231 .cra_module = THIS_MODULE, 232 .cra_alignmask = 0xf, 233 .cra_init = sl3516_ce_cipher_init, 234 .cra_exit = sl3516_ce_cipher_exit, 235 }, 236 .min_keysize = AES_MIN_KEY_SIZE, 237 .max_keysize = AES_MAX_KEY_SIZE, 238 .setkey = sl3516_ce_aes_setkey, 239 .encrypt = sl3516_ce_skencrypt, 240 .decrypt = sl3516_ce_skdecrypt, 241 }, 242 .alg.skcipher.op = { 243 .do_one_request = sl3516_ce_handle_cipher_request, 244 }, 245 }, 246 }; 247 248 static int sl3516_ce_debugfs_show(struct seq_file *seq, void *v) 249 { 250 struct sl3516_ce_dev *ce = seq->private; 251 unsigned int i; 252 253 seq_printf(seq, "HWRNG %lu %lu\n", 254 ce->hwrng_stat_req, ce->hwrng_stat_bytes); 255 seq_printf(seq, "IRQ %lu\n", ce->stat_irq); 256 seq_printf(seq, "IRQ TX %lu\n", ce->stat_irq_tx); 257 seq_printf(seq, "IRQ RX %lu\n", ce->stat_irq_rx); 258 seq_printf(seq, "nreq %lu\n", ce->stat_req); 259 seq_printf(seq, "fallback SG count TX %lu\n", ce->fallback_sg_count_tx); 260 seq_printf(seq, "fallback SG count RX %lu\n", ce->fallback_sg_count_rx); 261 seq_printf(seq, "fallback modulo16 %lu\n", ce->fallback_mod16); 262 seq_printf(seq, "fallback align16 %lu\n", ce->fallback_align16); 263 seq_printf(seq, "fallback not same len %lu\n", ce->fallback_not_same_len); 264 265 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 266 if (!ce_algs[i].ce) 267 continue; 268 switch (ce_algs[i].type) { 269 case CRYPTO_ALG_TYPE_SKCIPHER: 270 seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", 271 ce_algs[i].alg.skcipher.base.base.cra_driver_name, 272 ce_algs[i].alg.skcipher.base.base.cra_name, 273 ce_algs[i].stat_req, ce_algs[i].stat_fb); 274 break; 275 } 276 } 277 return 0; 278 } 279 280 DEFINE_SHOW_ATTRIBUTE(sl3516_ce_debugfs); 281 282 static int sl3516_ce_register_algs(struct sl3516_ce_dev *ce) 283 { 284 int err; 285 unsigned int i; 286 287 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 288 ce_algs[i].ce = ce; 289 switch (ce_algs[i].type) { 290 case CRYPTO_ALG_TYPE_SKCIPHER: 291 dev_info(ce->dev, "DEBUG: Register %s\n", 292 ce_algs[i].alg.skcipher.base.base.cra_name); 293 err = crypto_engine_register_skcipher(&ce_algs[i].alg.skcipher); 294 if (err) { 295 dev_err(ce->dev, "Fail to register %s\n", 296 ce_algs[i].alg.skcipher.base.base.cra_name); 297 ce_algs[i].ce = NULL; 298 return err; 299 } 300 break; 301 default: 302 ce_algs[i].ce = NULL; 303 dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); 304 } 305 } 306 return 0; 307 } 308 309 static void sl3516_ce_unregister_algs(struct sl3516_ce_dev *ce) 310 { 311 unsigned int i; 312 313 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 314 if (!ce_algs[i].ce) 315 continue; 316 switch (ce_algs[i].type) { 317 case CRYPTO_ALG_TYPE_SKCIPHER: 318 dev_info(ce->dev, "Unregister %d %s\n", i, 319 ce_algs[i].alg.skcipher.base.base.cra_name); 320 crypto_engine_unregister_skcipher(&ce_algs[i].alg.skcipher); 321 break; 322 } 323 } 324 } 325 326 static void sl3516_ce_start(struct sl3516_ce_dev *ce) 327 { 328 ce->ctx = 0; 329 ce->crx = 0; 330 writel(ce->dtx, ce->base + IPSEC_TXDMA_CURR_DESC); 331 writel(ce->drx, ce->base + IPSEC_RXDMA_CURR_DESC); 332 writel(0, ce->base + IPSEC_DMA_STATUS); 333 } 334 335 /* 336 * Power management strategy: The device is suspended unless a TFM exists for 337 * one of the algorithms proposed by this driver. 338 */ 339 static int sl3516_ce_pm_suspend(struct device *dev) 340 { 341 struct sl3516_ce_dev *ce = dev_get_drvdata(dev); 342 343 reset_control_assert(ce->reset); 344 clk_disable_unprepare(ce->clks); 345 return 0; 346 } 347 348 static int sl3516_ce_pm_resume(struct device *dev) 349 { 350 struct sl3516_ce_dev *ce = dev_get_drvdata(dev); 351 int err; 352 353 err = clk_prepare_enable(ce->clks); 354 if (err) { 355 dev_err(ce->dev, "Cannot prepare_enable\n"); 356 goto error; 357 } 358 err = reset_control_deassert(ce->reset); 359 if (err) { 360 dev_err(ce->dev, "Cannot deassert reset control\n"); 361 goto error; 362 } 363 364 sl3516_ce_start(ce); 365 366 return 0; 367 error: 368 sl3516_ce_pm_suspend(dev); 369 return err; 370 } 371 372 static const struct dev_pm_ops sl3516_ce_pm_ops = { 373 SET_RUNTIME_PM_OPS(sl3516_ce_pm_suspend, sl3516_ce_pm_resume, NULL) 374 }; 375 376 static int sl3516_ce_pm_init(struct sl3516_ce_dev *ce) 377 { 378 int err; 379 380 pm_runtime_use_autosuspend(ce->dev); 381 pm_runtime_set_autosuspend_delay(ce->dev, 2000); 382 383 err = pm_runtime_set_suspended(ce->dev); 384 if (err) 385 return err; 386 pm_runtime_enable(ce->dev); 387 return err; 388 } 389 390 static void sl3516_ce_pm_exit(struct sl3516_ce_dev *ce) 391 { 392 pm_runtime_disable(ce->dev); 393 } 394 395 static int sl3516_ce_probe(struct platform_device *pdev) 396 { 397 struct sl3516_ce_dev *ce; 398 int err, irq; 399 u32 v; 400 401 ce = devm_kzalloc(&pdev->dev, sizeof(*ce), GFP_KERNEL); 402 if (!ce) 403 return -ENOMEM; 404 405 ce->dev = &pdev->dev; 406 platform_set_drvdata(pdev, ce); 407 408 ce->base = devm_platform_ioremap_resource(pdev, 0); 409 if (IS_ERR(ce->base)) 410 return PTR_ERR(ce->base); 411 412 irq = platform_get_irq(pdev, 0); 413 if (irq < 0) 414 return irq; 415 416 err = devm_request_irq(&pdev->dev, irq, ce_irq_handler, 0, "crypto", ce); 417 if (err) { 418 dev_err(ce->dev, "Cannot request Crypto Engine IRQ (err=%d)\n", err); 419 return err; 420 } 421 422 ce->reset = devm_reset_control_get(&pdev->dev, NULL); 423 if (IS_ERR(ce->reset)) 424 return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset), 425 "No reset control found\n"); 426 ce->clks = devm_clk_get(ce->dev, NULL); 427 if (IS_ERR(ce->clks)) { 428 err = PTR_ERR(ce->clks); 429 dev_err(ce->dev, "Cannot get clock err=%d\n", err); 430 return err; 431 } 432 433 err = sl3516_ce_desc_init(ce); 434 if (err) 435 return err; 436 437 err = sl3516_ce_pm_init(ce); 438 if (err) 439 goto error_pm; 440 441 init_completion(&ce->complete); 442 443 ce->engine = crypto_engine_alloc_init(ce->dev, true); 444 if (!ce->engine) { 445 dev_err(ce->dev, "Cannot allocate engine\n"); 446 err = -ENOMEM; 447 goto error_engine; 448 } 449 450 err = crypto_engine_start(ce->engine); 451 if (err) { 452 dev_err(ce->dev, "Cannot start engine\n"); 453 goto error_engine; 454 } 455 456 err = sl3516_ce_register_algs(ce); 457 if (err) 458 goto error_alg; 459 460 err = sl3516_ce_rng_register(ce); 461 if (err) 462 goto error_rng; 463 464 err = pm_runtime_resume_and_get(ce->dev); 465 if (err < 0) 466 goto error_pmuse; 467 468 v = readl(ce->base + IPSEC_ID); 469 dev_info(ce->dev, "SL3516 dev %lx rev %lx\n", 470 v & GENMASK(31, 4), 471 v & GENMASK(3, 0)); 472 v = readl(ce->base + IPSEC_DMA_DEVICE_ID); 473 dev_info(ce->dev, "SL3516 DMA dev %lx rev %lx\n", 474 v & GENMASK(15, 4), 475 v & GENMASK(3, 0)); 476 477 pm_runtime_put_sync(ce->dev); 478 479 if (IS_ENABLED(CONFIG_CRYPTO_DEV_SL3516_DEBUG)) { 480 struct dentry *dbgfs_dir __maybe_unused; 481 struct dentry *dbgfs_stats __maybe_unused; 482 483 /* Ignore error of debugfs */ 484 dbgfs_dir = debugfs_create_dir("sl3516", NULL); 485 dbgfs_stats = debugfs_create_file("stats", 0444, 486 dbgfs_dir, ce, 487 &sl3516_ce_debugfs_fops); 488 #ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG 489 ce->dbgfs_dir = dbgfs_dir; 490 ce->dbgfs_stats = dbgfs_stats; 491 #endif 492 } 493 494 return 0; 495 error_pmuse: 496 sl3516_ce_rng_unregister(ce); 497 error_rng: 498 sl3516_ce_unregister_algs(ce); 499 error_alg: 500 crypto_engine_exit(ce->engine); 501 error_engine: 502 sl3516_ce_pm_exit(ce); 503 error_pm: 504 sl3516_ce_free_descs(ce); 505 return err; 506 } 507 508 static void sl3516_ce_remove(struct platform_device *pdev) 509 { 510 struct sl3516_ce_dev *ce = platform_get_drvdata(pdev); 511 512 sl3516_ce_rng_unregister(ce); 513 sl3516_ce_unregister_algs(ce); 514 crypto_engine_exit(ce->engine); 515 sl3516_ce_pm_exit(ce); 516 sl3516_ce_free_descs(ce); 517 518 #ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG 519 debugfs_remove_recursive(ce->dbgfs_dir); 520 #endif 521 } 522 523 static const struct of_device_id sl3516_ce_crypto_of_match_table[] = { 524 { .compatible = "cortina,sl3516-crypto"}, 525 {} 526 }; 527 MODULE_DEVICE_TABLE(of, sl3516_ce_crypto_of_match_table); 528 529 static struct platform_driver sl3516_ce_driver = { 530 .probe = sl3516_ce_probe, 531 .remove_new = sl3516_ce_remove, 532 .driver = { 533 .name = "sl3516-crypto", 534 .pm = &sl3516_ce_pm_ops, 535 .of_match_table = sl3516_ce_crypto_of_match_table, 536 }, 537 }; 538 539 module_platform_driver(sl3516_ce_driver); 540 541 MODULE_DESCRIPTION("SL3516 cryptographic offloader"); 542 MODULE_LICENSE("GPL"); 543 MODULE_AUTHOR("Corentin Labbe <clabbe@baylibre.com>"); 544