1 // SPDX-License-Identifier: GPL-2.0 2 // 3 // Copyright (C) 2018 Socionext Inc. 4 // Author: Masahiro Yamada <yamada.masahiro@socionext.com> 5 6 #include <linux/bits.h> 7 #include <linux/clk.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/dmaengine.h> 10 #include <linux/interrupt.h> 11 #include <linux/iopoll.h> 12 #include <linux/list.h> 13 #include <linux/module.h> 14 #include <linux/of.h> 15 #include <linux/of_dma.h> 16 #include <linux/platform_device.h> 17 #include <linux/slab.h> 18 #include <linux/types.h> 19 20 #include "virt-dma.h" 21 22 /* registers common for all channels */ 23 #define UNIPHIER_MDMAC_CMD 0x000 /* issue DMA start/abort */ 24 #define UNIPHIER_MDMAC_CMD_ABORT BIT(31) /* 1: abort, 0: start */ 25 26 /* per-channel registers */ 27 #define UNIPHIER_MDMAC_CH_OFFSET 0x100 28 #define UNIPHIER_MDMAC_CH_STRIDE 0x040 29 30 #define UNIPHIER_MDMAC_CH_IRQ_STAT 0x010 /* current hw status (RO) */ 31 #define UNIPHIER_MDMAC_CH_IRQ_REQ 0x014 /* latched STAT (WOC) */ 32 #define UNIPHIER_MDMAC_CH_IRQ_EN 0x018 /* IRQ enable mask */ 33 #define UNIPHIER_MDMAC_CH_IRQ_DET 0x01c /* REQ & EN (RO) */ 34 #define UNIPHIER_MDMAC_CH_IRQ__ABORT BIT(13) 35 #define UNIPHIER_MDMAC_CH_IRQ__DONE BIT(1) 36 #define UNIPHIER_MDMAC_CH_SRC_MODE 0x020 /* mode of source */ 37 #define UNIPHIER_MDMAC_CH_DEST_MODE 0x024 /* mode of destination */ 38 #define UNIPHIER_MDMAC_CH_MODE__ADDR_INC (0 << 4) 39 #define UNIPHIER_MDMAC_CH_MODE__ADDR_DEC (1 << 4) 40 #define UNIPHIER_MDMAC_CH_MODE__ADDR_FIXED (2 << 4) 41 #define UNIPHIER_MDMAC_CH_SRC_ADDR 0x028 /* source address */ 42 #define UNIPHIER_MDMAC_CH_DEST_ADDR 0x02c /* destination address */ 43 #define UNIPHIER_MDMAC_CH_SIZE 0x030 /* transfer bytes */ 44 45 #define UNIPHIER_MDMAC_SLAVE_BUSWIDTHS \ 46 (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 47 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 48 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \ 49 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) 50 51 struct uniphier_mdmac_desc { 52 struct virt_dma_desc vd; 53 struct scatterlist *sgl; 54 unsigned int sg_len; 55 unsigned int sg_cur; 56 enum dma_transfer_direction dir; 57 }; 58 59 struct uniphier_mdmac_chan { 60 struct virt_dma_chan vc; 61 struct uniphier_mdmac_device *mdev; 62 struct uniphier_mdmac_desc *md; 63 void __iomem *reg_ch_base; 64 unsigned int chan_id; 65 }; 66 67 struct uniphier_mdmac_device { 68 struct dma_device ddev; 69 struct clk *clk; 70 void __iomem *reg_base; 71 struct uniphier_mdmac_chan channels[]; 72 }; 73 74 static struct uniphier_mdmac_chan * 75 to_uniphier_mdmac_chan(struct virt_dma_chan *vc) 76 { 77 return container_of(vc, struct uniphier_mdmac_chan, vc); 78 } 79 80 static struct uniphier_mdmac_desc * 81 to_uniphier_mdmac_desc(struct virt_dma_desc *vd) 82 { 83 return container_of(vd, struct uniphier_mdmac_desc, vd); 84 } 85 86 /* mc->vc.lock must be held by caller */ 87 static struct uniphier_mdmac_desc * 88 uniphier_mdmac_next_desc(struct uniphier_mdmac_chan *mc) 89 { 90 struct virt_dma_desc *vd; 91 92 vd = vchan_next_desc(&mc->vc); 93 if (!vd) { 94 mc->md = NULL; 95 return NULL; 96 } 97 98 list_del(&vd->node); 99 100 mc->md = to_uniphier_mdmac_desc(vd); 101 102 return mc->md; 103 } 104 105 /* mc->vc.lock must be held by caller */ 106 static void uniphier_mdmac_handle(struct uniphier_mdmac_chan *mc, 107 struct uniphier_mdmac_desc *md) 108 { 109 struct uniphier_mdmac_device *mdev = mc->mdev; 110 struct scatterlist *sg; 111 u32 irq_flag = UNIPHIER_MDMAC_CH_IRQ__DONE; 112 u32 src_mode, src_addr, dest_mode, dest_addr, chunk_size; 113 114 sg = &md->sgl[md->sg_cur]; 115 116 if (md->dir == DMA_MEM_TO_DEV) { 117 src_mode = UNIPHIER_MDMAC_CH_MODE__ADDR_INC; 118 src_addr = sg_dma_address(sg); 119 dest_mode = UNIPHIER_MDMAC_CH_MODE__ADDR_FIXED; 120 dest_addr = 0; 121 } else { 122 src_mode = UNIPHIER_MDMAC_CH_MODE__ADDR_FIXED; 123 src_addr = 0; 124 dest_mode = UNIPHIER_MDMAC_CH_MODE__ADDR_INC; 125 dest_addr = sg_dma_address(sg); 126 } 127 128 chunk_size = sg_dma_len(sg); 129 130 writel(src_mode, mc->reg_ch_base + UNIPHIER_MDMAC_CH_SRC_MODE); 131 writel(dest_mode, mc->reg_ch_base + UNIPHIER_MDMAC_CH_DEST_MODE); 132 writel(src_addr, mc->reg_ch_base + UNIPHIER_MDMAC_CH_SRC_ADDR); 133 writel(dest_addr, mc->reg_ch_base + UNIPHIER_MDMAC_CH_DEST_ADDR); 134 writel(chunk_size, mc->reg_ch_base + UNIPHIER_MDMAC_CH_SIZE); 135 136 /* write 1 to clear */ 137 writel(irq_flag, mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_REQ); 138 139 writel(irq_flag, mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_EN); 140 141 writel(BIT(mc->chan_id), mdev->reg_base + UNIPHIER_MDMAC_CMD); 142 } 143 144 /* mc->vc.lock must be held by caller */ 145 static void uniphier_mdmac_start(struct uniphier_mdmac_chan *mc) 146 { 147 struct uniphier_mdmac_desc *md; 148 149 md = uniphier_mdmac_next_desc(mc); 150 if (md) 151 uniphier_mdmac_handle(mc, md); 152 } 153 154 /* mc->vc.lock must be held by caller */ 155 static int uniphier_mdmac_abort(struct uniphier_mdmac_chan *mc) 156 { 157 struct uniphier_mdmac_device *mdev = mc->mdev; 158 u32 irq_flag = UNIPHIER_MDMAC_CH_IRQ__ABORT; 159 u32 val; 160 161 /* write 1 to clear */ 162 writel(irq_flag, mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_REQ); 163 164 writel(UNIPHIER_MDMAC_CMD_ABORT | BIT(mc->chan_id), 165 mdev->reg_base + UNIPHIER_MDMAC_CMD); 166 167 /* 168 * Abort should be accepted soon. We poll the bit here instead of 169 * waiting for the interrupt. 170 */ 171 return readl_poll_timeout(mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_REQ, 172 val, val & irq_flag, 0, 20); 173 } 174 175 static irqreturn_t uniphier_mdmac_interrupt(int irq, void *dev_id) 176 { 177 struct uniphier_mdmac_chan *mc = dev_id; 178 struct uniphier_mdmac_desc *md; 179 irqreturn_t ret = IRQ_HANDLED; 180 u32 irq_stat; 181 182 spin_lock(&mc->vc.lock); 183 184 irq_stat = readl(mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_DET); 185 186 /* 187 * Some channels share a single interrupt line. If the IRQ status is 0, 188 * this is probably triggered by a different channel. 189 */ 190 if (!irq_stat) { 191 ret = IRQ_NONE; 192 goto out; 193 } 194 195 /* write 1 to clear */ 196 writel(irq_stat, mc->reg_ch_base + UNIPHIER_MDMAC_CH_IRQ_REQ); 197 198 /* 199 * UNIPHIER_MDMAC_CH_IRQ__DONE interrupt is asserted even when the DMA 200 * is aborted. To distinguish the normal completion and the abort, 201 * check mc->md. If it is NULL, we are aborting. 202 */ 203 md = mc->md; 204 if (!md) 205 goto out; 206 207 md->sg_cur++; 208 209 if (md->sg_cur >= md->sg_len) { 210 vchan_cookie_complete(&md->vd); 211 md = uniphier_mdmac_next_desc(mc); 212 if (!md) 213 goto out; 214 } 215 216 uniphier_mdmac_handle(mc, md); 217 218 out: 219 spin_unlock(&mc->vc.lock); 220 221 return ret; 222 } 223 224 static void uniphier_mdmac_free_chan_resources(struct dma_chan *chan) 225 { 226 vchan_free_chan_resources(to_virt_chan(chan)); 227 } 228 229 static struct dma_async_tx_descriptor * 230 uniphier_mdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 231 unsigned int sg_len, 232 enum dma_transfer_direction direction, 233 unsigned long flags, void *context) 234 { 235 struct virt_dma_chan *vc = to_virt_chan(chan); 236 struct uniphier_mdmac_desc *md; 237 238 if (!is_slave_direction(direction)) 239 return NULL; 240 241 md = kzalloc(sizeof(*md), GFP_NOWAIT); 242 if (!md) 243 return NULL; 244 245 md->sgl = sgl; 246 md->sg_len = sg_len; 247 md->dir = direction; 248 249 return vchan_tx_prep(vc, &md->vd, flags); 250 } 251 252 static int uniphier_mdmac_terminate_all(struct dma_chan *chan) 253 { 254 struct virt_dma_chan *vc = to_virt_chan(chan); 255 struct uniphier_mdmac_chan *mc = to_uniphier_mdmac_chan(vc); 256 unsigned long flags; 257 int ret = 0; 258 LIST_HEAD(head); 259 260 spin_lock_irqsave(&vc->lock, flags); 261 262 if (mc->md) { 263 vchan_terminate_vdesc(&mc->md->vd); 264 mc->md = NULL; 265 ret = uniphier_mdmac_abort(mc); 266 } 267 vchan_get_all_descriptors(vc, &head); 268 269 spin_unlock_irqrestore(&vc->lock, flags); 270 271 vchan_dma_desc_free_list(vc, &head); 272 273 return ret; 274 } 275 276 static void uniphier_mdmac_synchronize(struct dma_chan *chan) 277 { 278 vchan_synchronize(to_virt_chan(chan)); 279 } 280 281 static enum dma_status uniphier_mdmac_tx_status(struct dma_chan *chan, 282 dma_cookie_t cookie, 283 struct dma_tx_state *txstate) 284 { 285 struct virt_dma_chan *vc; 286 struct virt_dma_desc *vd; 287 struct uniphier_mdmac_chan *mc; 288 struct uniphier_mdmac_desc *md = NULL; 289 enum dma_status stat; 290 unsigned long flags; 291 int i; 292 293 stat = dma_cookie_status(chan, cookie, txstate); 294 /* Return immediately if we do not need to compute the residue. */ 295 if (stat == DMA_COMPLETE || !txstate) 296 return stat; 297 298 vc = to_virt_chan(chan); 299 300 spin_lock_irqsave(&vc->lock, flags); 301 302 mc = to_uniphier_mdmac_chan(vc); 303 304 if (mc->md && mc->md->vd.tx.cookie == cookie) { 305 /* residue from the on-flight chunk */ 306 txstate->residue = readl(mc->reg_ch_base + 307 UNIPHIER_MDMAC_CH_SIZE); 308 md = mc->md; 309 } 310 311 if (!md) { 312 vd = vchan_find_desc(vc, cookie); 313 if (vd) 314 md = to_uniphier_mdmac_desc(vd); 315 } 316 317 if (md) { 318 /* residue from the queued chunks */ 319 for (i = md->sg_cur; i < md->sg_len; i++) 320 txstate->residue += sg_dma_len(&md->sgl[i]); 321 } 322 323 spin_unlock_irqrestore(&vc->lock, flags); 324 325 return stat; 326 } 327 328 static void uniphier_mdmac_issue_pending(struct dma_chan *chan) 329 { 330 struct virt_dma_chan *vc = to_virt_chan(chan); 331 struct uniphier_mdmac_chan *mc = to_uniphier_mdmac_chan(vc); 332 unsigned long flags; 333 334 spin_lock_irqsave(&vc->lock, flags); 335 336 if (vchan_issue_pending(vc) && !mc->md) 337 uniphier_mdmac_start(mc); 338 339 spin_unlock_irqrestore(&vc->lock, flags); 340 } 341 342 static void uniphier_mdmac_desc_free(struct virt_dma_desc *vd) 343 { 344 kfree(to_uniphier_mdmac_desc(vd)); 345 } 346 347 static int uniphier_mdmac_chan_init(struct platform_device *pdev, 348 struct uniphier_mdmac_device *mdev, 349 int chan_id) 350 { 351 struct device *dev = &pdev->dev; 352 struct uniphier_mdmac_chan *mc = &mdev->channels[chan_id]; 353 char *irq_name; 354 int irq, ret; 355 356 irq = platform_get_irq(pdev, chan_id); 357 if (irq < 0) 358 return irq; 359 360 irq_name = devm_kasprintf(dev, GFP_KERNEL, "uniphier-mio-dmac-ch%d", 361 chan_id); 362 if (!irq_name) 363 return -ENOMEM; 364 365 ret = devm_request_irq(dev, irq, uniphier_mdmac_interrupt, 366 IRQF_SHARED, irq_name, mc); 367 if (ret) 368 return ret; 369 370 mc->mdev = mdev; 371 mc->reg_ch_base = mdev->reg_base + UNIPHIER_MDMAC_CH_OFFSET + 372 UNIPHIER_MDMAC_CH_STRIDE * chan_id; 373 mc->chan_id = chan_id; 374 mc->vc.desc_free = uniphier_mdmac_desc_free; 375 vchan_init(&mc->vc, &mdev->ddev); 376 377 return 0; 378 } 379 380 static int uniphier_mdmac_probe(struct platform_device *pdev) 381 { 382 struct device *dev = &pdev->dev; 383 struct uniphier_mdmac_device *mdev; 384 struct dma_device *ddev; 385 int nr_chans, ret, i; 386 387 nr_chans = platform_irq_count(pdev); 388 if (nr_chans < 0) 389 return nr_chans; 390 391 ret = dma_set_mask(dev, DMA_BIT_MASK(32)); 392 if (ret) 393 return ret; 394 395 mdev = devm_kzalloc(dev, struct_size(mdev, channels, nr_chans), 396 GFP_KERNEL); 397 if (!mdev) 398 return -ENOMEM; 399 400 mdev->reg_base = devm_platform_ioremap_resource(pdev, 0); 401 if (IS_ERR(mdev->reg_base)) 402 return PTR_ERR(mdev->reg_base); 403 404 mdev->clk = devm_clk_get(dev, NULL); 405 if (IS_ERR(mdev->clk)) { 406 dev_err(dev, "failed to get clock\n"); 407 return PTR_ERR(mdev->clk); 408 } 409 410 ret = clk_prepare_enable(mdev->clk); 411 if (ret) 412 return ret; 413 414 ddev = &mdev->ddev; 415 ddev->dev = dev; 416 dma_cap_set(DMA_PRIVATE, ddev->cap_mask); 417 ddev->src_addr_widths = UNIPHIER_MDMAC_SLAVE_BUSWIDTHS; 418 ddev->dst_addr_widths = UNIPHIER_MDMAC_SLAVE_BUSWIDTHS; 419 ddev->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 420 ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 421 ddev->device_free_chan_resources = uniphier_mdmac_free_chan_resources; 422 ddev->device_prep_slave_sg = uniphier_mdmac_prep_slave_sg; 423 ddev->device_terminate_all = uniphier_mdmac_terminate_all; 424 ddev->device_synchronize = uniphier_mdmac_synchronize; 425 ddev->device_tx_status = uniphier_mdmac_tx_status; 426 ddev->device_issue_pending = uniphier_mdmac_issue_pending; 427 INIT_LIST_HEAD(&ddev->channels); 428 429 for (i = 0; i < nr_chans; i++) { 430 ret = uniphier_mdmac_chan_init(pdev, mdev, i); 431 if (ret) 432 goto disable_clk; 433 } 434 435 ret = dma_async_device_register(ddev); 436 if (ret) 437 goto disable_clk; 438 439 ret = of_dma_controller_register(dev->of_node, of_dma_xlate_by_chan_id, 440 ddev); 441 if (ret) 442 goto unregister_dmac; 443 444 platform_set_drvdata(pdev, mdev); 445 446 return 0; 447 448 unregister_dmac: 449 dma_async_device_unregister(ddev); 450 disable_clk: 451 clk_disable_unprepare(mdev->clk); 452 453 return ret; 454 } 455 456 static void uniphier_mdmac_remove(struct platform_device *pdev) 457 { 458 struct uniphier_mdmac_device *mdev = platform_get_drvdata(pdev); 459 struct dma_chan *chan; 460 int ret; 461 462 /* 463 * Before reaching here, almost all descriptors have been freed by the 464 * ->device_free_chan_resources() hook. However, each channel might 465 * be still holding one descriptor that was on-flight at that moment. 466 * Terminate it to make sure this hardware is no longer running. Then, 467 * free the channel resources once again to avoid memory leak. 468 */ 469 list_for_each_entry(chan, &mdev->ddev.channels, device_node) { 470 ret = dmaengine_terminate_sync(chan); 471 if (ret) { 472 /* 473 * This results in resource leakage and maybe also 474 * use-after-free errors as e.g. *mdev is kfreed. 475 */ 476 dev_alert(&pdev->dev, "Failed to terminate channel %d (%pe)\n", 477 chan->chan_id, ERR_PTR(ret)); 478 return; 479 } 480 uniphier_mdmac_free_chan_resources(chan); 481 } 482 483 of_dma_controller_free(pdev->dev.of_node); 484 dma_async_device_unregister(&mdev->ddev); 485 clk_disable_unprepare(mdev->clk); 486 } 487 488 static const struct of_device_id uniphier_mdmac_match[] = { 489 { .compatible = "socionext,uniphier-mio-dmac" }, 490 { /* sentinel */ } 491 }; 492 MODULE_DEVICE_TABLE(of, uniphier_mdmac_match); 493 494 static struct platform_driver uniphier_mdmac_driver = { 495 .probe = uniphier_mdmac_probe, 496 .remove_new = uniphier_mdmac_remove, 497 .driver = { 498 .name = "uniphier-mio-dmac", 499 .of_match_table = uniphier_mdmac_match, 500 }, 501 }; 502 module_platform_driver(uniphier_mdmac_driver); 503 504 MODULE_AUTHOR("Masahiro Yamada <yamada.masahiro@socionext.com>"); 505 MODULE_DESCRIPTION("UniPhier MIO DMAC driver"); 506 MODULE_LICENSE("GPL v2"); 507