1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * sl3516-ce-core.c - hardware cryptographic offloader for Storlink SL3516 SoC 4 * 5 * Copyright (C) 2021 Corentin Labbe <clabbe@baylibre.com> 6 * 7 * Core file which registers crypto algorithms supported by the CryptoEngine 8 */ 9 #include <linux/clk.h> 10 #include <linux/crypto.h> 11 #include <linux/debugfs.h> 12 #include <linux/dev_printk.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/interrupt.h> 15 #include <linux/io.h> 16 #include <linux/irq.h> 17 #include <linux/module.h> 18 #include <linux/of.h> 19 #include <linux/of_device.h> 20 #include <linux/platform_device.h> 21 #include <linux/pm_runtime.h> 22 #include <linux/reset.h> 23 #include <crypto/internal/rng.h> 24 #include <crypto/internal/skcipher.h> 25 26 #include "sl3516-ce.h" 27 28 static int sl3516_ce_desc_init(struct sl3516_ce_dev *ce) 29 { 30 const size_t sz = sizeof(struct descriptor) * MAXDESC; 31 int i; 32 33 ce->tx = dma_alloc_coherent(ce->dev, sz, &ce->dtx, GFP_KERNEL); 34 if (!ce->tx) 35 return -ENOMEM; 36 ce->rx = dma_alloc_coherent(ce->dev, sz, &ce->drx, GFP_KERNEL); 37 if (!ce->rx) 38 goto err_rx; 39 40 for (i = 0; i < MAXDESC; i++) { 41 ce->tx[i].frame_ctrl.bits.own = CE_CPU; 42 ce->tx[i].next_desc.next_descriptor = ce->dtx + (i + 1) * sizeof(struct descriptor); 43 } 44 ce->tx[MAXDESC - 1].next_desc.next_descriptor = ce->dtx; 45 46 for (i = 0; i < MAXDESC; i++) { 47 ce->rx[i].frame_ctrl.bits.own = CE_CPU; 48 ce->rx[i].next_desc.next_descriptor = ce->drx + (i + 1) * sizeof(struct descriptor); 49 } 50 ce->rx[MAXDESC - 1].next_desc.next_descriptor = ce->drx; 51 52 ce->pctrl = dma_alloc_coherent(ce->dev, sizeof(struct pkt_control_ecb), 53 &ce->dctrl, GFP_KERNEL); 54 if (!ce->pctrl) 55 goto err_pctrl; 56 57 return 0; 58 err_pctrl: 59 dma_free_coherent(ce->dev, sz, ce->rx, ce->drx); 60 err_rx: 61 dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx); 62 return -ENOMEM; 63 } 64 65 static void sl3516_ce_free_descs(struct sl3516_ce_dev *ce) 66 { 67 const size_t sz = sizeof(struct descriptor) * MAXDESC; 68 69 dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx); 70 dma_free_coherent(ce->dev, sz, ce->rx, ce->drx); 71 dma_free_coherent(ce->dev, sizeof(struct pkt_control_ecb), ce->pctrl, 72 ce->dctrl); 73 } 74 75 static void start_dma_tx(struct sl3516_ce_dev *ce) 76 { 77 u32 v; 78 79 v = TXDMA_CTRL_START | TXDMA_CTRL_CHAIN_MODE | TXDMA_CTRL_CONTINUE | \ 80 TXDMA_CTRL_INT_FAIL | TXDMA_CTRL_INT_PERR | TXDMA_CTRL_BURST_UNK; 81 82 writel(v, ce->base + IPSEC_TXDMA_CTRL); 83 } 84 85 static void start_dma_rx(struct sl3516_ce_dev *ce) 86 { 87 u32 v; 88 89 v = RXDMA_CTRL_START | RXDMA_CTRL_CHAIN_MODE | RXDMA_CTRL_CONTINUE | \ 90 RXDMA_CTRL_BURST_UNK | RXDMA_CTRL_INT_FINISH | \ 91 RXDMA_CTRL_INT_FAIL | RXDMA_CTRL_INT_PERR | \ 92 RXDMA_CTRL_INT_EOD | RXDMA_CTRL_INT_EOF; 93 94 writel(v, ce->base + IPSEC_RXDMA_CTRL); 95 } 96 97 static struct descriptor *get_desc_tx(struct sl3516_ce_dev *ce) 98 { 99 struct descriptor *dd; 100 101 dd = &ce->tx[ce->ctx]; 102 ce->ctx++; 103 if (ce->ctx >= MAXDESC) 104 ce->ctx = 0; 105 return dd; 106 } 107 108 static struct descriptor *get_desc_rx(struct sl3516_ce_dev *ce) 109 { 110 struct descriptor *rdd; 111 112 rdd = &ce->rx[ce->crx]; 113 ce->crx++; 114 if (ce->crx >= MAXDESC) 115 ce->crx = 0; 116 return rdd; 117 } 118 119 int sl3516_ce_run_task(struct sl3516_ce_dev *ce, struct sl3516_ce_cipher_req_ctx *rctx, 120 const char *name) 121 { 122 struct descriptor *dd, *rdd = NULL; 123 u32 v; 124 int i, err = 0; 125 126 ce->stat_req++; 127 128 reinit_completion(&ce->complete); 129 ce->status = 0; 130 131 for (i = 0; i < rctx->nr_sgd; i++) { 132 dev_dbg(ce->dev, "%s handle DST SG %d/%d len=%d\n", __func__, 133 i, rctx->nr_sgd, rctx->t_dst[i].len); 134 rdd = get_desc_rx(ce); 135 rdd->buf_adr = rctx->t_dst[i].addr; 136 rdd->frame_ctrl.bits.buffer_size = rctx->t_dst[i].len; 137 rdd->frame_ctrl.bits.own = CE_DMA; 138 } 139 rdd->next_desc.bits.eofie = 1; 140 141 for (i = 0; i < rctx->nr_sgs; i++) { 142 dev_dbg(ce->dev, "%s handle SRC SG %d/%d len=%d\n", __func__, 143 i, rctx->nr_sgs, rctx->t_src[i].len); 144 rctx->h->algorithm_len = rctx->t_src[i].len; 145 146 dd = get_desc_tx(ce); 147 dd->frame_ctrl.raw = 0; 148 dd->flag_status.raw = 0; 149 dd->frame_ctrl.bits.buffer_size = rctx->pctrllen; 150 dd->buf_adr = ce->dctrl; 151 dd->flag_status.tx_flag.tqflag = rctx->tqflag; 152 dd->next_desc.bits.eofie = 0; 153 dd->next_desc.bits.dec = 0; 154 dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST; 155 dd->frame_ctrl.bits.own = CE_DMA; 156 157 dd = get_desc_tx(ce); 158 dd->frame_ctrl.raw = 0; 159 dd->flag_status.raw = 0; 160 dd->frame_ctrl.bits.buffer_size = rctx->t_src[i].len; 161 dd->buf_adr = rctx->t_src[i].addr; 162 dd->flag_status.tx_flag.tqflag = 0; 163 dd->next_desc.bits.eofie = 0; 164 dd->next_desc.bits.dec = 0; 165 dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST; 166 dd->frame_ctrl.bits.own = CE_DMA; 167 start_dma_tx(ce); 168 start_dma_rx(ce); 169 } 170 wait_for_completion_interruptible_timeout(&ce->complete, 171 msecs_to_jiffies(5000)); 172 if (ce->status == 0) { 173 dev_err(ce->dev, "DMA timeout for %s\n", name); 174 err = -EFAULT; 175 } 176 v = readl(ce->base + IPSEC_STATUS_REG); 177 if (v & 0xFFF) { 178 dev_err(ce->dev, "IPSEC_STATUS_REG %x\n", v); 179 err = -EFAULT; 180 } 181 182 return err; 183 } 184 185 static irqreturn_t ce_irq_handler(int irq, void *data) 186 { 187 struct sl3516_ce_dev *ce = (struct sl3516_ce_dev *)data; 188 u32 v; 189 190 ce->stat_irq++; 191 192 v = readl(ce->base + IPSEC_DMA_STATUS); 193 writel(v, ce->base + IPSEC_DMA_STATUS); 194 195 if (v & DMA_STATUS_TS_DERR) 196 dev_err(ce->dev, "AHB bus Error While Tx !!!\n"); 197 if (v & DMA_STATUS_TS_PERR) 198 dev_err(ce->dev, "Tx Descriptor Protocol Error !!!\n"); 199 if (v & DMA_STATUS_RS_DERR) 200 dev_err(ce->dev, "AHB bus Error While Rx !!!\n"); 201 if (v & DMA_STATUS_RS_PERR) 202 dev_err(ce->dev, "Rx Descriptor Protocol Error !!!\n"); 203 204 if (v & DMA_STATUS_TS_EOFI) 205 ce->stat_irq_tx++; 206 if (v & DMA_STATUS_RS_EOFI) { 207 ce->status = 1; 208 complete(&ce->complete); 209 ce->stat_irq_rx++; 210 return IRQ_HANDLED; 211 } 212 213 return IRQ_HANDLED; 214 } 215 216 static struct sl3516_ce_alg_template ce_algs[] = { 217 { 218 .type = CRYPTO_ALG_TYPE_SKCIPHER, 219 .mode = ECB_AES, 220 .alg.skcipher = { 221 .base = { 222 .cra_name = "ecb(aes)", 223 .cra_driver_name = "ecb-aes-sl3516", 224 .cra_priority = 400, 225 .cra_blocksize = AES_BLOCK_SIZE, 226 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 227 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, 228 .cra_ctxsize = sizeof(struct sl3516_ce_cipher_tfm_ctx), 229 .cra_module = THIS_MODULE, 230 .cra_alignmask = 0xf, 231 .cra_init = sl3516_ce_cipher_init, 232 .cra_exit = sl3516_ce_cipher_exit, 233 }, 234 .min_keysize = AES_MIN_KEY_SIZE, 235 .max_keysize = AES_MAX_KEY_SIZE, 236 .setkey = sl3516_ce_aes_setkey, 237 .encrypt = sl3516_ce_skencrypt, 238 .decrypt = sl3516_ce_skdecrypt, 239 } 240 }, 241 }; 242 243 #ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG 244 static int sl3516_ce_debugfs_show(struct seq_file *seq, void *v) 245 { 246 struct sl3516_ce_dev *ce = seq->private; 247 unsigned int i; 248 249 seq_printf(seq, "HWRNG %lu %lu\n", 250 ce->hwrng_stat_req, ce->hwrng_stat_bytes); 251 seq_printf(seq, "IRQ %lu\n", ce->stat_irq); 252 seq_printf(seq, "IRQ TX %lu\n", ce->stat_irq_tx); 253 seq_printf(seq, "IRQ RX %lu\n", ce->stat_irq_rx); 254 seq_printf(seq, "nreq %lu\n", ce->stat_req); 255 seq_printf(seq, "fallback SG count TX %lu\n", ce->fallback_sg_count_tx); 256 seq_printf(seq, "fallback SG count RX %lu\n", ce->fallback_sg_count_rx); 257 seq_printf(seq, "fallback modulo16 %lu\n", ce->fallback_mod16); 258 seq_printf(seq, "fallback align16 %lu\n", ce->fallback_align16); 259 seq_printf(seq, "fallback not same len %lu\n", ce->fallback_not_same_len); 260 261 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 262 if (!ce_algs[i].ce) 263 continue; 264 switch (ce_algs[i].type) { 265 case CRYPTO_ALG_TYPE_SKCIPHER: 266 seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", 267 ce_algs[i].alg.skcipher.base.cra_driver_name, 268 ce_algs[i].alg.skcipher.base.cra_name, 269 ce_algs[i].stat_req, ce_algs[i].stat_fb); 270 break; 271 } 272 } 273 return 0; 274 } 275 276 DEFINE_SHOW_ATTRIBUTE(sl3516_ce_debugfs); 277 #endif 278 279 static int sl3516_ce_register_algs(struct sl3516_ce_dev *ce) 280 { 281 int err; 282 unsigned int i; 283 284 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 285 ce_algs[i].ce = ce; 286 switch (ce_algs[i].type) { 287 case CRYPTO_ALG_TYPE_SKCIPHER: 288 dev_info(ce->dev, "DEBUG: Register %s\n", 289 ce_algs[i].alg.skcipher.base.cra_name); 290 err = crypto_register_skcipher(&ce_algs[i].alg.skcipher); 291 if (err) { 292 dev_err(ce->dev, "Fail to register %s\n", 293 ce_algs[i].alg.skcipher.base.cra_name); 294 ce_algs[i].ce = NULL; 295 return err; 296 } 297 break; 298 default: 299 ce_algs[i].ce = NULL; 300 dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); 301 } 302 } 303 return 0; 304 } 305 306 static void sl3516_ce_unregister_algs(struct sl3516_ce_dev *ce) 307 { 308 unsigned int i; 309 310 for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { 311 if (!ce_algs[i].ce) 312 continue; 313 switch (ce_algs[i].type) { 314 case CRYPTO_ALG_TYPE_SKCIPHER: 315 dev_info(ce->dev, "Unregister %d %s\n", i, 316 ce_algs[i].alg.skcipher.base.cra_name); 317 crypto_unregister_skcipher(&ce_algs[i].alg.skcipher); 318 break; 319 } 320 } 321 } 322 323 static void sl3516_ce_start(struct sl3516_ce_dev *ce) 324 { 325 ce->ctx = 0; 326 ce->crx = 0; 327 writel(ce->dtx, ce->base + IPSEC_TXDMA_CURR_DESC); 328 writel(ce->drx, ce->base + IPSEC_RXDMA_CURR_DESC); 329 writel(0, ce->base + IPSEC_DMA_STATUS); 330 } 331 332 /* 333 * Power management strategy: The device is suspended unless a TFM exists for 334 * one of the algorithms proposed by this driver. 335 */ 336 static int sl3516_ce_pm_suspend(struct device *dev) 337 { 338 struct sl3516_ce_dev *ce = dev_get_drvdata(dev); 339 340 reset_control_assert(ce->reset); 341 clk_disable_unprepare(ce->clks); 342 return 0; 343 } 344 345 static int sl3516_ce_pm_resume(struct device *dev) 346 { 347 struct sl3516_ce_dev *ce = dev_get_drvdata(dev); 348 int err; 349 350 err = clk_prepare_enable(ce->clks); 351 if (err) { 352 dev_err(ce->dev, "Cannot prepare_enable\n"); 353 goto error; 354 } 355 err = reset_control_deassert(ce->reset); 356 if (err) { 357 dev_err(ce->dev, "Cannot deassert reset control\n"); 358 goto error; 359 } 360 361 sl3516_ce_start(ce); 362 363 return 0; 364 error: 365 sl3516_ce_pm_suspend(dev); 366 return err; 367 } 368 369 static const struct dev_pm_ops sl3516_ce_pm_ops = { 370 SET_RUNTIME_PM_OPS(sl3516_ce_pm_suspend, sl3516_ce_pm_resume, NULL) 371 }; 372 373 static int sl3516_ce_pm_init(struct sl3516_ce_dev *ce) 374 { 375 int err; 376 377 pm_runtime_use_autosuspend(ce->dev); 378 pm_runtime_set_autosuspend_delay(ce->dev, 2000); 379 380 err = pm_runtime_set_suspended(ce->dev); 381 if (err) 382 return err; 383 pm_runtime_enable(ce->dev); 384 return err; 385 } 386 387 static void sl3516_ce_pm_exit(struct sl3516_ce_dev *ce) 388 { 389 pm_runtime_disable(ce->dev); 390 } 391 392 static int sl3516_ce_probe(struct platform_device *pdev) 393 { 394 struct sl3516_ce_dev *ce; 395 int err, irq; 396 u32 v; 397 398 ce = devm_kzalloc(&pdev->dev, sizeof(*ce), GFP_KERNEL); 399 if (!ce) 400 return -ENOMEM; 401 402 ce->dev = &pdev->dev; 403 platform_set_drvdata(pdev, ce); 404 405 ce->base = devm_platform_ioremap_resource(pdev, 0); 406 if (IS_ERR(ce->base)) 407 return PTR_ERR(ce->base); 408 409 irq = platform_get_irq(pdev, 0); 410 if (irq < 0) 411 return irq; 412 413 err = devm_request_irq(&pdev->dev, irq, ce_irq_handler, 0, "crypto", ce); 414 if (err) { 415 dev_err(ce->dev, "Cannot request Crypto Engine IRQ (err=%d)\n", err); 416 return err; 417 } 418 419 ce->reset = devm_reset_control_get(&pdev->dev, NULL); 420 if (IS_ERR(ce->reset)) 421 return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset), 422 "No reset control found\n"); 423 ce->clks = devm_clk_get(ce->dev, NULL); 424 if (IS_ERR(ce->clks)) { 425 err = PTR_ERR(ce->clks); 426 dev_err(ce->dev, "Cannot get clock err=%d\n", err); 427 return err; 428 } 429 430 err = sl3516_ce_desc_init(ce); 431 if (err) 432 return err; 433 434 err = sl3516_ce_pm_init(ce); 435 if (err) 436 goto error_pm; 437 438 init_completion(&ce->complete); 439 440 ce->engine = crypto_engine_alloc_init(ce->dev, true); 441 if (!ce->engine) { 442 dev_err(ce->dev, "Cannot allocate engine\n"); 443 err = -ENOMEM; 444 goto error_engine; 445 } 446 447 err = crypto_engine_start(ce->engine); 448 if (err) { 449 dev_err(ce->dev, "Cannot start engine\n"); 450 goto error_engine; 451 } 452 453 err = sl3516_ce_register_algs(ce); 454 if (err) 455 goto error_alg; 456 457 err = sl3516_ce_rng_register(ce); 458 if (err) 459 goto error_rng; 460 461 err = pm_runtime_resume_and_get(ce->dev); 462 if (err < 0) 463 goto error_pmuse; 464 465 v = readl(ce->base + IPSEC_ID); 466 dev_info(ce->dev, "SL3516 dev %lx rev %lx\n", 467 v & GENMASK(31, 4), 468 v & GENMASK(3, 0)); 469 v = readl(ce->base + IPSEC_DMA_DEVICE_ID); 470 dev_info(ce->dev, "SL3516 DMA dev %lx rev %lx\n", 471 v & GENMASK(15, 4), 472 v & GENMASK(3, 0)); 473 474 pm_runtime_put_sync(ce->dev); 475 476 #ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG 477 /* Ignore error of debugfs */ 478 ce->dbgfs_dir = debugfs_create_dir("sl3516", NULL); 479 ce->dbgfs_stats = debugfs_create_file("stats", 0444, 480 ce->dbgfs_dir, ce, 481 &sl3516_ce_debugfs_fops); 482 #endif 483 484 return 0; 485 error_pmuse: 486 sl3516_ce_rng_unregister(ce); 487 error_rng: 488 sl3516_ce_unregister_algs(ce); 489 error_alg: 490 crypto_engine_exit(ce->engine); 491 error_engine: 492 sl3516_ce_pm_exit(ce); 493 error_pm: 494 sl3516_ce_free_descs(ce); 495 return err; 496 } 497 498 static int sl3516_ce_remove(struct platform_device *pdev) 499 { 500 struct sl3516_ce_dev *ce = platform_get_drvdata(pdev); 501 502 sl3516_ce_rng_unregister(ce); 503 sl3516_ce_unregister_algs(ce); 504 crypto_engine_exit(ce->engine); 505 sl3516_ce_pm_exit(ce); 506 sl3516_ce_free_descs(ce); 507 508 #ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG 509 debugfs_remove_recursive(ce->dbgfs_dir); 510 #endif 511 512 return 0; 513 } 514 515 static const struct of_device_id sl3516_ce_crypto_of_match_table[] = { 516 { .compatible = "cortina,sl3516-crypto"}, 517 {} 518 }; 519 MODULE_DEVICE_TABLE(of, sl3516_ce_crypto_of_match_table); 520 521 static struct platform_driver sl3516_ce_driver = { 522 .probe = sl3516_ce_probe, 523 .remove = sl3516_ce_remove, 524 .driver = { 525 .name = "sl3516-crypto", 526 .pm = &sl3516_ce_pm_ops, 527 .of_match_table = sl3516_ce_crypto_of_match_table, 528 }, 529 }; 530 531 module_platform_driver(sl3516_ce_driver); 532 533 MODULE_DESCRIPTION("SL3516 cryptographic offloader"); 534 MODULE_LICENSE("GPL"); 535 MODULE_AUTHOR("Corentin Labbe <clabbe@baylibre.com>"); 536