1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * OMAP DMAengine support 4 */ 5 #include <linux/cpu_pm.h> 6 #include <linux/delay.h> 7 #include <linux/dmaengine.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/dmapool.h> 10 #include <linux/err.h> 11 #include <linux/init.h> 12 #include <linux/interrupt.h> 13 #include <linux/list.h> 14 #include <linux/module.h> 15 #include <linux/omap-dma.h> 16 #include <linux/platform_device.h> 17 #include <linux/slab.h> 18 #include <linux/spinlock.h> 19 #include <linux/of.h> 20 #include <linux/of_dma.h> 21 22 #include "../virt-dma.h" 23 24 #define OMAP_SDMA_REQUESTS 127 25 #define OMAP_SDMA_CHANNELS 32 26 27 struct omap_dma_config { 28 int lch_end; 29 unsigned int rw_priority:1; 30 unsigned int needs_busy_check:1; 31 unsigned int may_lose_context:1; 32 unsigned int needs_lch_clear:1; 33 }; 34 35 struct omap_dma_context { 36 u32 irqenable_l0; 37 u32 irqenable_l1; 38 u32 ocp_sysconfig; 39 u32 gcr; 40 }; 41 42 struct omap_dmadev { 43 struct dma_device ddev; 44 spinlock_t lock; 45 void __iomem *base; 46 const struct omap_dma_reg *reg_map; 47 struct omap_system_dma_plat_info *plat; 48 const struct omap_dma_config *cfg; 49 struct notifier_block nb; 50 struct omap_dma_context context; 51 int lch_count; 52 DECLARE_BITMAP(lch_bitmap, OMAP_SDMA_CHANNELS); 53 struct mutex lch_lock; /* for assigning logical channels */ 54 bool legacy; 55 bool ll123_supported; 56 struct dma_pool *desc_pool; 57 unsigned dma_requests; 58 spinlock_t irq_lock; 59 uint32_t irq_enable_mask; 60 struct omap_chan **lch_map; 61 }; 62 63 struct omap_chan { 64 struct virt_dma_chan vc; 65 void __iomem *channel_base; 66 const struct omap_dma_reg *reg_map; 67 uint32_t ccr; 68 69 struct dma_slave_config cfg; 70 unsigned dma_sig; 71 bool cyclic; 72 bool paused; 73 bool running; 74 75 int dma_ch; 76 struct omap_desc *desc; 77 unsigned sgidx; 78 }; 79 80 #define DESC_NXT_SV_REFRESH (0x1 << 24) 81 #define DESC_NXT_SV_REUSE (0x2 << 24) 82 #define DESC_NXT_DV_REFRESH (0x1 << 26) 83 #define DESC_NXT_DV_REUSE (0x2 << 26) 84 #define DESC_NTYPE_TYPE2 (0x2 << 29) 85 86 /* Type 2 descriptor with Source or Destination address update */ 87 struct omap_type2_desc { 88 uint32_t next_desc; 89 uint32_t en; 90 uint32_t addr; /* src or dst */ 91 uint16_t fn; 92 uint16_t cicr; 93 int16_t cdei; 94 int16_t csei; 95 int32_t cdfi; 96 int32_t csfi; 97 } __packed; 98 99 struct omap_sg { 100 dma_addr_t addr; 101 uint32_t en; /* number of elements (24-bit) */ 102 uint32_t fn; /* number of frames (16-bit) */ 103 int32_t fi; /* for double indexing */ 104 int16_t ei; /* for double indexing */ 105 106 /* Linked list */ 107 struct omap_type2_desc *t2_desc; 108 dma_addr_t t2_desc_paddr; 109 }; 110 111 struct omap_desc { 112 struct virt_dma_desc vd; 113 bool using_ll; 114 enum dma_transfer_direction dir; 115 dma_addr_t dev_addr; 116 bool polled; 117 118 int32_t fi; /* for OMAP_DMA_SYNC_PACKET / double indexing */ 119 int16_t ei; /* for double indexing */ 120 uint8_t es; /* CSDP_DATA_TYPE_xxx */ 121 uint32_t ccr; /* CCR value */ 122 uint16_t clnk_ctrl; /* CLNK_CTRL value */ 123 uint16_t cicr; /* CICR value */ 124 uint32_t csdp; /* CSDP value */ 125 126 unsigned sglen; 127 struct omap_sg sg[] __counted_by(sglen); 128 }; 129 130 enum { 131 CAPS_0_SUPPORT_LL123 = BIT(20), /* Linked List type1/2/3 */ 132 CAPS_0_SUPPORT_LL4 = BIT(21), /* Linked List type4 */ 133 134 CCR_FS = BIT(5), 135 CCR_READ_PRIORITY = BIT(6), 136 CCR_ENABLE = BIT(7), 137 CCR_AUTO_INIT = BIT(8), /* OMAP1 only */ 138 CCR_REPEAT = BIT(9), /* OMAP1 only */ 139 CCR_OMAP31_DISABLE = BIT(10), /* OMAP1 only */ 140 CCR_SUSPEND_SENSITIVE = BIT(8), /* OMAP2+ only */ 141 CCR_RD_ACTIVE = BIT(9), /* OMAP2+ only */ 142 CCR_WR_ACTIVE = BIT(10), /* OMAP2+ only */ 143 CCR_SRC_AMODE_CONSTANT = 0 << 12, 144 CCR_SRC_AMODE_POSTINC = 1 << 12, 145 CCR_SRC_AMODE_SGLIDX = 2 << 12, 146 CCR_SRC_AMODE_DBLIDX = 3 << 12, 147 CCR_DST_AMODE_CONSTANT = 0 << 14, 148 CCR_DST_AMODE_POSTINC = 1 << 14, 149 CCR_DST_AMODE_SGLIDX = 2 << 14, 150 CCR_DST_AMODE_DBLIDX = 3 << 14, 151 CCR_CONSTANT_FILL = BIT(16), 152 CCR_TRANSPARENT_COPY = BIT(17), 153 CCR_BS = BIT(18), 154 CCR_SUPERVISOR = BIT(22), 155 CCR_PREFETCH = BIT(23), 156 CCR_TRIGGER_SRC = BIT(24), 157 CCR_BUFFERING_DISABLE = BIT(25), 158 CCR_WRITE_PRIORITY = BIT(26), 159 CCR_SYNC_ELEMENT = 0, 160 CCR_SYNC_FRAME = CCR_FS, 161 CCR_SYNC_BLOCK = CCR_BS, 162 CCR_SYNC_PACKET = CCR_BS | CCR_FS, 163 164 CSDP_DATA_TYPE_8 = 0, 165 CSDP_DATA_TYPE_16 = 1, 166 CSDP_DATA_TYPE_32 = 2, 167 CSDP_SRC_PORT_EMIFF = 0 << 2, /* OMAP1 only */ 168 CSDP_SRC_PORT_EMIFS = 1 << 2, /* OMAP1 only */ 169 CSDP_SRC_PORT_OCP_T1 = 2 << 2, /* OMAP1 only */ 170 CSDP_SRC_PORT_TIPB = 3 << 2, /* OMAP1 only */ 171 CSDP_SRC_PORT_OCP_T2 = 4 << 2, /* OMAP1 only */ 172 CSDP_SRC_PORT_MPUI = 5 << 2, /* OMAP1 only */ 173 CSDP_SRC_PACKED = BIT(6), 174 CSDP_SRC_BURST_1 = 0 << 7, 175 CSDP_SRC_BURST_16 = 1 << 7, 176 CSDP_SRC_BURST_32 = 2 << 7, 177 CSDP_SRC_BURST_64 = 3 << 7, 178 CSDP_DST_PORT_EMIFF = 0 << 9, /* OMAP1 only */ 179 CSDP_DST_PORT_EMIFS = 1 << 9, /* OMAP1 only */ 180 CSDP_DST_PORT_OCP_T1 = 2 << 9, /* OMAP1 only */ 181 CSDP_DST_PORT_TIPB = 3 << 9, /* OMAP1 only */ 182 CSDP_DST_PORT_OCP_T2 = 4 << 9, /* OMAP1 only */ 183 CSDP_DST_PORT_MPUI = 5 << 9, /* OMAP1 only */ 184 CSDP_DST_PACKED = BIT(13), 185 CSDP_DST_BURST_1 = 0 << 14, 186 CSDP_DST_BURST_16 = 1 << 14, 187 CSDP_DST_BURST_32 = 2 << 14, 188 CSDP_DST_BURST_64 = 3 << 14, 189 CSDP_WRITE_NON_POSTED = 0 << 16, 190 CSDP_WRITE_POSTED = 1 << 16, 191 CSDP_WRITE_LAST_NON_POSTED = 2 << 16, 192 193 CICR_TOUT_IE = BIT(0), /* OMAP1 only */ 194 CICR_DROP_IE = BIT(1), 195 CICR_HALF_IE = BIT(2), 196 CICR_FRAME_IE = BIT(3), 197 CICR_LAST_IE = BIT(4), 198 CICR_BLOCK_IE = BIT(5), 199 CICR_PKT_IE = BIT(7), /* OMAP2+ only */ 200 CICR_TRANS_ERR_IE = BIT(8), /* OMAP2+ only */ 201 CICR_SUPERVISOR_ERR_IE = BIT(10), /* OMAP2+ only */ 202 CICR_MISALIGNED_ERR_IE = BIT(11), /* OMAP2+ only */ 203 CICR_DRAIN_IE = BIT(12), /* OMAP2+ only */ 204 CICR_SUPER_BLOCK_IE = BIT(14), /* OMAP2+ only */ 205 206 CLNK_CTRL_ENABLE_LNK = BIT(15), 207 208 CDP_DST_VALID_INC = 0 << 0, 209 CDP_DST_VALID_RELOAD = 1 << 0, 210 CDP_DST_VALID_REUSE = 2 << 0, 211 CDP_SRC_VALID_INC = 0 << 2, 212 CDP_SRC_VALID_RELOAD = 1 << 2, 213 CDP_SRC_VALID_REUSE = 2 << 2, 214 CDP_NTYPE_TYPE1 = 1 << 4, 215 CDP_NTYPE_TYPE2 = 2 << 4, 216 CDP_NTYPE_TYPE3 = 3 << 4, 217 CDP_TMODE_NORMAL = 0 << 8, 218 CDP_TMODE_LLIST = 1 << 8, 219 CDP_FAST = BIT(10), 220 }; 221 222 static const unsigned es_bytes[] = { 223 [CSDP_DATA_TYPE_8] = 1, 224 [CSDP_DATA_TYPE_16] = 2, 225 [CSDP_DATA_TYPE_32] = 4, 226 }; 227 228 static bool omap_dma_filter_fn(struct dma_chan *chan, void *param); 229 static struct of_dma_filter_info omap_dma_info = { 230 .filter_fn = omap_dma_filter_fn, 231 }; 232 233 static inline struct omap_dmadev *to_omap_dma_dev(struct dma_device *d) 234 { 235 return container_of(d, struct omap_dmadev, ddev); 236 } 237 238 static inline struct omap_chan *to_omap_dma_chan(struct dma_chan *c) 239 { 240 return container_of(c, struct omap_chan, vc.chan); 241 } 242 243 static inline struct omap_desc *to_omap_dma_desc(struct dma_async_tx_descriptor *t) 244 { 245 return container_of(t, struct omap_desc, vd.tx); 246 } 247 248 static void omap_dma_desc_free(struct virt_dma_desc *vd) 249 { 250 struct omap_desc *d = to_omap_dma_desc(&vd->tx); 251 252 if (d->using_ll) { 253 struct omap_dmadev *od = to_omap_dma_dev(vd->tx.chan->device); 254 int i; 255 256 for (i = 0; i < d->sglen; i++) { 257 if (d->sg[i].t2_desc) 258 dma_pool_free(od->desc_pool, d->sg[i].t2_desc, 259 d->sg[i].t2_desc_paddr); 260 } 261 } 262 263 kfree(d); 264 } 265 266 static void omap_dma_fill_type2_desc(struct omap_desc *d, int idx, 267 enum dma_transfer_direction dir, bool last) 268 { 269 struct omap_sg *sg = &d->sg[idx]; 270 struct omap_type2_desc *t2_desc = sg->t2_desc; 271 272 if (idx) 273 d->sg[idx - 1].t2_desc->next_desc = sg->t2_desc_paddr; 274 if (last) 275 t2_desc->next_desc = 0xfffffffc; 276 277 t2_desc->en = sg->en; 278 t2_desc->addr = sg->addr; 279 t2_desc->fn = sg->fn & 0xffff; 280 t2_desc->cicr = d->cicr; 281 if (!last) 282 t2_desc->cicr &= ~CICR_BLOCK_IE; 283 284 switch (dir) { 285 case DMA_DEV_TO_MEM: 286 t2_desc->cdei = sg->ei; 287 t2_desc->csei = d->ei; 288 t2_desc->cdfi = sg->fi; 289 t2_desc->csfi = d->fi; 290 291 t2_desc->en |= DESC_NXT_DV_REFRESH; 292 t2_desc->en |= DESC_NXT_SV_REUSE; 293 break; 294 case DMA_MEM_TO_DEV: 295 t2_desc->cdei = d->ei; 296 t2_desc->csei = sg->ei; 297 t2_desc->cdfi = d->fi; 298 t2_desc->csfi = sg->fi; 299 300 t2_desc->en |= DESC_NXT_SV_REFRESH; 301 t2_desc->en |= DESC_NXT_DV_REUSE; 302 break; 303 default: 304 return; 305 } 306 307 t2_desc->en |= DESC_NTYPE_TYPE2; 308 } 309 310 static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr) 311 { 312 switch (type) { 313 case OMAP_DMA_REG_16BIT: 314 writew_relaxed(val, addr); 315 break; 316 case OMAP_DMA_REG_2X16BIT: 317 writew_relaxed(val, addr); 318 writew_relaxed(val >> 16, addr + 2); 319 break; 320 case OMAP_DMA_REG_32BIT: 321 writel_relaxed(val, addr); 322 break; 323 default: 324 WARN_ON(1); 325 } 326 } 327 328 static unsigned omap_dma_read(unsigned type, void __iomem *addr) 329 { 330 unsigned val; 331 332 switch (type) { 333 case OMAP_DMA_REG_16BIT: 334 val = readw_relaxed(addr); 335 break; 336 case OMAP_DMA_REG_2X16BIT: 337 val = readw_relaxed(addr); 338 val |= readw_relaxed(addr + 2) << 16; 339 break; 340 case OMAP_DMA_REG_32BIT: 341 val = readl_relaxed(addr); 342 break; 343 default: 344 WARN_ON(1); 345 val = 0; 346 } 347 348 return val; 349 } 350 351 static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val) 352 { 353 const struct omap_dma_reg *r = od->reg_map + reg; 354 355 WARN_ON(r->stride); 356 357 omap_dma_write(val, r->type, od->base + r->offset); 358 } 359 360 static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg) 361 { 362 const struct omap_dma_reg *r = od->reg_map + reg; 363 364 WARN_ON(r->stride); 365 366 return omap_dma_read(r->type, od->base + r->offset); 367 } 368 369 static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val) 370 { 371 const struct omap_dma_reg *r = c->reg_map + reg; 372 373 omap_dma_write(val, r->type, c->channel_base + r->offset); 374 } 375 376 static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg) 377 { 378 const struct omap_dma_reg *r = c->reg_map + reg; 379 380 return omap_dma_read(r->type, c->channel_base + r->offset); 381 } 382 383 static void omap_dma_clear_csr(struct omap_chan *c) 384 { 385 if (dma_omap1()) 386 omap_dma_chan_read(c, CSR); 387 else 388 omap_dma_chan_write(c, CSR, ~0); 389 } 390 391 static unsigned omap_dma_get_csr(struct omap_chan *c) 392 { 393 unsigned val = omap_dma_chan_read(c, CSR); 394 395 if (!dma_omap1()) 396 omap_dma_chan_write(c, CSR, val); 397 398 return val; 399 } 400 401 static void omap_dma_clear_lch(struct omap_dmadev *od, int lch) 402 { 403 struct omap_chan *c; 404 int i; 405 406 c = od->lch_map[lch]; 407 if (!c) 408 return; 409 410 for (i = CSDP; i <= od->cfg->lch_end; i++) 411 omap_dma_chan_write(c, i, 0); 412 } 413 414 static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c, 415 unsigned lch) 416 { 417 c->channel_base = od->base + od->plat->channel_stride * lch; 418 419 od->lch_map[lch] = c; 420 } 421 422 static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) 423 { 424 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); 425 uint16_t cicr = d->cicr; 426 427 if (__dma_omap15xx(od->plat->dma_attr)) 428 omap_dma_chan_write(c, CPC, 0); 429 else 430 omap_dma_chan_write(c, CDAC, 0); 431 432 omap_dma_clear_csr(c); 433 434 if (d->using_ll) { 435 uint32_t cdp = CDP_TMODE_LLIST | CDP_NTYPE_TYPE2 | CDP_FAST; 436 437 if (d->dir == DMA_DEV_TO_MEM) 438 cdp |= (CDP_DST_VALID_RELOAD | CDP_SRC_VALID_REUSE); 439 else 440 cdp |= (CDP_DST_VALID_REUSE | CDP_SRC_VALID_RELOAD); 441 omap_dma_chan_write(c, CDP, cdp); 442 443 omap_dma_chan_write(c, CNDP, d->sg[0].t2_desc_paddr); 444 omap_dma_chan_write(c, CCDN, 0); 445 omap_dma_chan_write(c, CCFN, 0xffff); 446 omap_dma_chan_write(c, CCEN, 0xffffff); 447 448 cicr &= ~CICR_BLOCK_IE; 449 } else if (od->ll123_supported) { 450 omap_dma_chan_write(c, CDP, 0); 451 } 452 453 /* Enable interrupts */ 454 omap_dma_chan_write(c, CICR, cicr); 455 456 /* Enable channel */ 457 omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE); 458 459 c->running = true; 460 } 461 462 static void omap_dma_drain_chan(struct omap_chan *c) 463 { 464 int i; 465 u32 val; 466 467 /* Wait for sDMA FIFO to drain */ 468 for (i = 0; ; i++) { 469 val = omap_dma_chan_read(c, CCR); 470 if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))) 471 break; 472 473 if (i > 100) 474 break; 475 476 udelay(5); 477 } 478 479 if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)) 480 dev_err(c->vc.chan.device->dev, 481 "DMA drain did not complete on lch %d\n", 482 c->dma_ch); 483 } 484 485 static int omap_dma_stop(struct omap_chan *c) 486 { 487 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); 488 uint32_t val; 489 490 /* disable irq */ 491 omap_dma_chan_write(c, CICR, 0); 492 493 omap_dma_clear_csr(c); 494 495 val = omap_dma_chan_read(c, CCR); 496 if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) { 497 uint32_t sysconfig; 498 499 sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG); 500 val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK; 501 val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE); 502 omap_dma_glbl_write(od, OCP_SYSCONFIG, val); 503 504 val = omap_dma_chan_read(c, CCR); 505 val &= ~CCR_ENABLE; 506 omap_dma_chan_write(c, CCR, val); 507 508 if (!(c->ccr & CCR_BUFFERING_DISABLE)) 509 omap_dma_drain_chan(c); 510 511 omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig); 512 } else { 513 if (!(val & CCR_ENABLE)) 514 return -EINVAL; 515 516 val &= ~CCR_ENABLE; 517 omap_dma_chan_write(c, CCR, val); 518 519 if (!(c->ccr & CCR_BUFFERING_DISABLE)) 520 omap_dma_drain_chan(c); 521 } 522 523 mb(); 524 525 if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) { 526 val = omap_dma_chan_read(c, CLNK_CTRL); 527 528 if (dma_omap1()) 529 val |= 1 << 14; /* set the STOP_LNK bit */ 530 else 531 val &= ~CLNK_CTRL_ENABLE_LNK; 532 533 omap_dma_chan_write(c, CLNK_CTRL, val); 534 } 535 c->running = false; 536 return 0; 537 } 538 539 static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d) 540 { 541 struct omap_sg *sg = d->sg + c->sgidx; 542 unsigned cxsa, cxei, cxfi; 543 544 if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) { 545 cxsa = CDSA; 546 cxei = CDEI; 547 cxfi = CDFI; 548 } else { 549 cxsa = CSSA; 550 cxei = CSEI; 551 cxfi = CSFI; 552 } 553 554 omap_dma_chan_write(c, cxsa, sg->addr); 555 omap_dma_chan_write(c, cxei, sg->ei); 556 omap_dma_chan_write(c, cxfi, sg->fi); 557 omap_dma_chan_write(c, CEN, sg->en); 558 omap_dma_chan_write(c, CFN, sg->fn); 559 560 omap_dma_start(c, d); 561 c->sgidx++; 562 } 563 564 static void omap_dma_start_desc(struct omap_chan *c) 565 { 566 struct virt_dma_desc *vd = vchan_next_desc(&c->vc); 567 struct omap_desc *d; 568 unsigned cxsa, cxei, cxfi; 569 570 if (!vd) { 571 c->desc = NULL; 572 return; 573 } 574 575 list_del(&vd->node); 576 577 c->desc = d = to_omap_dma_desc(&vd->tx); 578 c->sgidx = 0; 579 580 /* 581 * This provides the necessary barrier to ensure data held in 582 * DMA coherent memory is visible to the DMA engine prior to 583 * the transfer starting. 584 */ 585 mb(); 586 587 omap_dma_chan_write(c, CCR, d->ccr); 588 if (dma_omap1()) 589 omap_dma_chan_write(c, CCR2, d->ccr >> 16); 590 591 if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) { 592 cxsa = CSSA; 593 cxei = CSEI; 594 cxfi = CSFI; 595 } else { 596 cxsa = CDSA; 597 cxei = CDEI; 598 cxfi = CDFI; 599 } 600 601 omap_dma_chan_write(c, cxsa, d->dev_addr); 602 omap_dma_chan_write(c, cxei, d->ei); 603 omap_dma_chan_write(c, cxfi, d->fi); 604 omap_dma_chan_write(c, CSDP, d->csdp); 605 omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl); 606 607 omap_dma_start_sg(c, d); 608 } 609 610 static void omap_dma_callback(int ch, u16 status, void *data) 611 { 612 struct omap_chan *c = data; 613 struct omap_desc *d; 614 unsigned long flags; 615 616 spin_lock_irqsave(&c->vc.lock, flags); 617 d = c->desc; 618 if (d) { 619 if (c->cyclic) { 620 vchan_cyclic_callback(&d->vd); 621 } else if (d->using_ll || c->sgidx == d->sglen) { 622 omap_dma_start_desc(c); 623 vchan_cookie_complete(&d->vd); 624 } else { 625 omap_dma_start_sg(c, d); 626 } 627 } 628 spin_unlock_irqrestore(&c->vc.lock, flags); 629 } 630 631 static irqreturn_t omap_dma_irq(int irq, void *devid) 632 { 633 struct omap_dmadev *od = devid; 634 unsigned status, channel; 635 636 spin_lock(&od->irq_lock); 637 638 status = omap_dma_glbl_read(od, IRQSTATUS_L1); 639 status &= od->irq_enable_mask; 640 if (status == 0) { 641 spin_unlock(&od->irq_lock); 642 return IRQ_NONE; 643 } 644 645 while ((channel = ffs(status)) != 0) { 646 unsigned mask, csr; 647 struct omap_chan *c; 648 649 channel -= 1; 650 mask = BIT(channel); 651 status &= ~mask; 652 653 c = od->lch_map[channel]; 654 if (c == NULL) { 655 /* This should never happen */ 656 dev_err(od->ddev.dev, "invalid channel %u\n", channel); 657 continue; 658 } 659 660 csr = omap_dma_get_csr(c); 661 omap_dma_glbl_write(od, IRQSTATUS_L1, mask); 662 663 omap_dma_callback(channel, csr, c); 664 } 665 666 spin_unlock(&od->irq_lock); 667 668 return IRQ_HANDLED; 669 } 670 671 static int omap_dma_get_lch(struct omap_dmadev *od, int *lch) 672 { 673 int channel; 674 675 mutex_lock(&od->lch_lock); 676 channel = find_first_zero_bit(od->lch_bitmap, od->lch_count); 677 if (channel >= od->lch_count) 678 goto out_busy; 679 set_bit(channel, od->lch_bitmap); 680 mutex_unlock(&od->lch_lock); 681 682 omap_dma_clear_lch(od, channel); 683 *lch = channel; 684 685 return 0; 686 687 out_busy: 688 mutex_unlock(&od->lch_lock); 689 *lch = -EINVAL; 690 691 return -EBUSY; 692 } 693 694 static void omap_dma_put_lch(struct omap_dmadev *od, int lch) 695 { 696 omap_dma_clear_lch(od, lch); 697 mutex_lock(&od->lch_lock); 698 clear_bit(lch, od->lch_bitmap); 699 mutex_unlock(&od->lch_lock); 700 } 701 702 static inline bool omap_dma_legacy(struct omap_dmadev *od) 703 { 704 return IS_ENABLED(CONFIG_ARCH_OMAP1) && od->legacy; 705 } 706 707 static int omap_dma_alloc_chan_resources(struct dma_chan *chan) 708 { 709 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 710 struct omap_chan *c = to_omap_dma_chan(chan); 711 struct device *dev = od->ddev.dev; 712 int ret; 713 714 if (omap_dma_legacy(od)) { 715 ret = omap_request_dma(c->dma_sig, "DMA engine", 716 omap_dma_callback, c, &c->dma_ch); 717 } else { 718 ret = omap_dma_get_lch(od, &c->dma_ch); 719 } 720 721 dev_dbg(dev, "allocating channel %u for %u\n", c->dma_ch, c->dma_sig); 722 723 if (ret >= 0) { 724 omap_dma_assign(od, c, c->dma_ch); 725 726 if (!omap_dma_legacy(od)) { 727 unsigned val; 728 729 spin_lock_irq(&od->irq_lock); 730 val = BIT(c->dma_ch); 731 omap_dma_glbl_write(od, IRQSTATUS_L1, val); 732 od->irq_enable_mask |= val; 733 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); 734 735 val = omap_dma_glbl_read(od, IRQENABLE_L0); 736 val &= ~BIT(c->dma_ch); 737 omap_dma_glbl_write(od, IRQENABLE_L0, val); 738 spin_unlock_irq(&od->irq_lock); 739 } 740 } 741 742 if (dma_omap1()) { 743 if (__dma_omap16xx(od->plat->dma_attr)) { 744 c->ccr = CCR_OMAP31_DISABLE; 745 /* Duplicate what plat-omap/dma.c does */ 746 c->ccr |= c->dma_ch + 1; 747 } else { 748 c->ccr = c->dma_sig & 0x1f; 749 } 750 } else { 751 c->ccr = c->dma_sig & 0x1f; 752 c->ccr |= (c->dma_sig & ~0x1f) << 14; 753 } 754 if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) 755 c->ccr |= CCR_BUFFERING_DISABLE; 756 757 return ret; 758 } 759 760 static void omap_dma_free_chan_resources(struct dma_chan *chan) 761 { 762 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 763 struct omap_chan *c = to_omap_dma_chan(chan); 764 765 if (!omap_dma_legacy(od)) { 766 spin_lock_irq(&od->irq_lock); 767 od->irq_enable_mask &= ~BIT(c->dma_ch); 768 omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); 769 spin_unlock_irq(&od->irq_lock); 770 } 771 772 c->channel_base = NULL; 773 od->lch_map[c->dma_ch] = NULL; 774 vchan_free_chan_resources(&c->vc); 775 776 if (omap_dma_legacy(od)) 777 omap_free_dma(c->dma_ch); 778 else 779 omap_dma_put_lch(od, c->dma_ch); 780 781 dev_dbg(od->ddev.dev, "freeing channel %u used for %u\n", c->dma_ch, 782 c->dma_sig); 783 c->dma_sig = 0; 784 } 785 786 static size_t omap_dma_sg_size(struct omap_sg *sg) 787 { 788 return sg->en * sg->fn; 789 } 790 791 static size_t omap_dma_desc_size(struct omap_desc *d) 792 { 793 unsigned i; 794 size_t size; 795 796 for (size = i = 0; i < d->sglen; i++) 797 size += omap_dma_sg_size(&d->sg[i]); 798 799 return size * es_bytes[d->es]; 800 } 801 802 static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr) 803 { 804 unsigned i; 805 size_t size, es_size = es_bytes[d->es]; 806 807 for (size = i = 0; i < d->sglen; i++) { 808 size_t this_size = omap_dma_sg_size(&d->sg[i]) * es_size; 809 810 if (size) 811 size += this_size; 812 else if (addr >= d->sg[i].addr && 813 addr < d->sg[i].addr + this_size) 814 size += d->sg[i].addr + this_size - addr; 815 } 816 return size; 817 } 818 819 /* 820 * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is 821 * read before the DMA controller finished disabling the channel. 822 */ 823 static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg) 824 { 825 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); 826 uint32_t val; 827 828 val = omap_dma_chan_read(c, reg); 829 if (val == 0 && od->plat->errata & DMA_ERRATA_3_3) 830 val = omap_dma_chan_read(c, reg); 831 832 return val; 833 } 834 835 static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c) 836 { 837 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); 838 dma_addr_t addr, cdac; 839 840 if (__dma_omap15xx(od->plat->dma_attr)) { 841 addr = omap_dma_chan_read(c, CPC); 842 } else { 843 addr = omap_dma_chan_read_3_3(c, CSAC); 844 cdac = omap_dma_chan_read_3_3(c, CDAC); 845 846 /* 847 * CDAC == 0 indicates that the DMA transfer on the channel has 848 * not been started (no data has been transferred so far). 849 * Return the programmed source start address in this case. 850 */ 851 if (cdac == 0) 852 addr = omap_dma_chan_read(c, CSSA); 853 } 854 855 if (dma_omap1()) 856 addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000; 857 858 return addr; 859 } 860 861 static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c) 862 { 863 struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); 864 dma_addr_t addr; 865 866 if (__dma_omap15xx(od->plat->dma_attr)) { 867 addr = omap_dma_chan_read(c, CPC); 868 } else { 869 addr = omap_dma_chan_read_3_3(c, CDAC); 870 871 /* 872 * CDAC == 0 indicates that the DMA transfer on the channel 873 * has not been started (no data has been transferred so 874 * far). Return the programmed destination start address in 875 * this case. 876 */ 877 if (addr == 0) 878 addr = omap_dma_chan_read(c, CDSA); 879 } 880 881 if (dma_omap1()) 882 addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000; 883 884 return addr; 885 } 886 887 static enum dma_status omap_dma_tx_status(struct dma_chan *chan, 888 dma_cookie_t cookie, struct dma_tx_state *txstate) 889 { 890 struct omap_chan *c = to_omap_dma_chan(chan); 891 enum dma_status ret; 892 unsigned long flags; 893 struct omap_desc *d = NULL; 894 895 ret = dma_cookie_status(chan, cookie, txstate); 896 if (ret == DMA_COMPLETE) 897 return ret; 898 899 spin_lock_irqsave(&c->vc.lock, flags); 900 if (c->desc && c->desc->vd.tx.cookie == cookie) 901 d = c->desc; 902 903 if (!txstate) 904 goto out; 905 906 if (d) { 907 dma_addr_t pos; 908 909 if (d->dir == DMA_MEM_TO_DEV) 910 pos = omap_dma_get_src_pos(c); 911 else if (d->dir == DMA_DEV_TO_MEM || d->dir == DMA_MEM_TO_MEM) 912 pos = omap_dma_get_dst_pos(c); 913 else 914 pos = 0; 915 916 txstate->residue = omap_dma_desc_size_pos(d, pos); 917 } else { 918 struct virt_dma_desc *vd = vchan_find_desc(&c->vc, cookie); 919 920 if (vd) 921 txstate->residue = omap_dma_desc_size( 922 to_omap_dma_desc(&vd->tx)); 923 else 924 txstate->residue = 0; 925 } 926 927 out: 928 if (ret == DMA_IN_PROGRESS && c->paused) { 929 ret = DMA_PAUSED; 930 } else if (d && d->polled && c->running) { 931 uint32_t ccr = omap_dma_chan_read(c, CCR); 932 /* 933 * The channel is no longer active, set the return value 934 * accordingly and mark it as completed 935 */ 936 if (!(ccr & CCR_ENABLE)) { 937 ret = DMA_COMPLETE; 938 omap_dma_start_desc(c); 939 vchan_cookie_complete(&d->vd); 940 } 941 } 942 943 spin_unlock_irqrestore(&c->vc.lock, flags); 944 945 return ret; 946 } 947 948 static void omap_dma_issue_pending(struct dma_chan *chan) 949 { 950 struct omap_chan *c = to_omap_dma_chan(chan); 951 unsigned long flags; 952 953 spin_lock_irqsave(&c->vc.lock, flags); 954 if (vchan_issue_pending(&c->vc) && !c->desc) 955 omap_dma_start_desc(c); 956 spin_unlock_irqrestore(&c->vc.lock, flags); 957 } 958 959 static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( 960 struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen, 961 enum dma_transfer_direction dir, unsigned long tx_flags, void *context) 962 { 963 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 964 struct omap_chan *c = to_omap_dma_chan(chan); 965 enum dma_slave_buswidth dev_width; 966 struct scatterlist *sgent; 967 struct omap_desc *d; 968 dma_addr_t dev_addr; 969 unsigned i, es, en, frame_bytes; 970 bool ll_failed = false; 971 u32 burst; 972 u32 port_window, port_window_bytes; 973 974 if (dir == DMA_DEV_TO_MEM) { 975 dev_addr = c->cfg.src_addr; 976 dev_width = c->cfg.src_addr_width; 977 burst = c->cfg.src_maxburst; 978 port_window = c->cfg.src_port_window_size; 979 } else if (dir == DMA_MEM_TO_DEV) { 980 dev_addr = c->cfg.dst_addr; 981 dev_width = c->cfg.dst_addr_width; 982 burst = c->cfg.dst_maxburst; 983 port_window = c->cfg.dst_port_window_size; 984 } else { 985 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 986 return NULL; 987 } 988 989 /* Bus width translates to the element size (ES) */ 990 switch (dev_width) { 991 case DMA_SLAVE_BUSWIDTH_1_BYTE: 992 es = CSDP_DATA_TYPE_8; 993 break; 994 case DMA_SLAVE_BUSWIDTH_2_BYTES: 995 es = CSDP_DATA_TYPE_16; 996 break; 997 case DMA_SLAVE_BUSWIDTH_4_BYTES: 998 es = CSDP_DATA_TYPE_32; 999 break; 1000 default: /* not reached */ 1001 return NULL; 1002 } 1003 1004 /* Now allocate and setup the descriptor. */ 1005 d = kzalloc(struct_size(d, sg, sglen), GFP_ATOMIC); 1006 if (!d) 1007 return NULL; 1008 d->sglen = sglen; 1009 1010 d->dir = dir; 1011 d->dev_addr = dev_addr; 1012 d->es = es; 1013 1014 /* When the port_window is used, one frame must cover the window */ 1015 if (port_window) { 1016 burst = port_window; 1017 port_window_bytes = port_window * es_bytes[es]; 1018 1019 d->ei = 1; 1020 /* 1021 * One frame covers the port_window and by configure 1022 * the source frame index to be -1 * (port_window - 1) 1023 * we instruct the sDMA that after a frame is processed 1024 * it should move back to the start of the window. 1025 */ 1026 d->fi = -(port_window_bytes - 1); 1027 } 1028 1029 d->ccr = c->ccr | CCR_SYNC_FRAME; 1030 if (dir == DMA_DEV_TO_MEM) { 1031 d->csdp = CSDP_DST_BURST_64 | CSDP_DST_PACKED; 1032 1033 d->ccr |= CCR_DST_AMODE_POSTINC; 1034 if (port_window) { 1035 d->ccr |= CCR_SRC_AMODE_DBLIDX; 1036 1037 if (port_window_bytes >= 64) 1038 d->csdp |= CSDP_SRC_BURST_64; 1039 else if (port_window_bytes >= 32) 1040 d->csdp |= CSDP_SRC_BURST_32; 1041 else if (port_window_bytes >= 16) 1042 d->csdp |= CSDP_SRC_BURST_16; 1043 1044 } else { 1045 d->ccr |= CCR_SRC_AMODE_CONSTANT; 1046 } 1047 } else { 1048 d->csdp = CSDP_SRC_BURST_64 | CSDP_SRC_PACKED; 1049 1050 d->ccr |= CCR_SRC_AMODE_POSTINC; 1051 if (port_window) { 1052 d->ccr |= CCR_DST_AMODE_DBLIDX; 1053 1054 if (port_window_bytes >= 64) 1055 d->csdp |= CSDP_DST_BURST_64; 1056 else if (port_window_bytes >= 32) 1057 d->csdp |= CSDP_DST_BURST_32; 1058 else if (port_window_bytes >= 16) 1059 d->csdp |= CSDP_DST_BURST_16; 1060 } else { 1061 d->ccr |= CCR_DST_AMODE_CONSTANT; 1062 } 1063 } 1064 1065 d->cicr = CICR_DROP_IE | CICR_BLOCK_IE; 1066 d->csdp |= es; 1067 1068 if (dma_omap1()) { 1069 d->cicr |= CICR_TOUT_IE; 1070 1071 if (dir == DMA_DEV_TO_MEM) 1072 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB; 1073 else 1074 d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF; 1075 } else { 1076 if (dir == DMA_DEV_TO_MEM) 1077 d->ccr |= CCR_TRIGGER_SRC; 1078 1079 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; 1080 1081 if (port_window) 1082 d->csdp |= CSDP_WRITE_LAST_NON_POSTED; 1083 } 1084 if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) 1085 d->clnk_ctrl = c->dma_ch; 1086 1087 /* 1088 * Build our scatterlist entries: each contains the address, 1089 * the number of elements (EN) in each frame, and the number of 1090 * frames (FN). Number of bytes for this entry = ES * EN * FN. 1091 * 1092 * Burst size translates to number of elements with frame sync. 1093 * Note: DMA engine defines burst to be the number of dev-width 1094 * transfers. 1095 */ 1096 en = burst; 1097 frame_bytes = es_bytes[es] * en; 1098 1099 if (sglen >= 2) 1100 d->using_ll = od->ll123_supported; 1101 1102 for_each_sg(sgl, sgent, sglen, i) { 1103 struct omap_sg *osg = &d->sg[i]; 1104 1105 osg->addr = sg_dma_address(sgent); 1106 osg->en = en; 1107 osg->fn = sg_dma_len(sgent) / frame_bytes; 1108 1109 if (d->using_ll) { 1110 osg->t2_desc = dma_pool_alloc(od->desc_pool, GFP_ATOMIC, 1111 &osg->t2_desc_paddr); 1112 if (!osg->t2_desc) { 1113 dev_err(chan->device->dev, 1114 "t2_desc[%d] allocation failed\n", i); 1115 ll_failed = true; 1116 d->using_ll = false; 1117 continue; 1118 } 1119 1120 omap_dma_fill_type2_desc(d, i, dir, (i == sglen - 1)); 1121 } 1122 } 1123 1124 /* Release the dma_pool entries if one allocation failed */ 1125 if (ll_failed) { 1126 for (i = 0; i < d->sglen; i++) { 1127 struct omap_sg *osg = &d->sg[i]; 1128 1129 if (osg->t2_desc) { 1130 dma_pool_free(od->desc_pool, osg->t2_desc, 1131 osg->t2_desc_paddr); 1132 osg->t2_desc = NULL; 1133 } 1134 } 1135 } 1136 1137 return vchan_tx_prep(&c->vc, &d->vd, tx_flags); 1138 } 1139 1140 static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( 1141 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 1142 size_t period_len, enum dma_transfer_direction dir, unsigned long flags) 1143 { 1144 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 1145 struct omap_chan *c = to_omap_dma_chan(chan); 1146 enum dma_slave_buswidth dev_width; 1147 struct omap_desc *d; 1148 dma_addr_t dev_addr; 1149 unsigned es; 1150 u32 burst; 1151 1152 if (dir == DMA_DEV_TO_MEM) { 1153 dev_addr = c->cfg.src_addr; 1154 dev_width = c->cfg.src_addr_width; 1155 burst = c->cfg.src_maxburst; 1156 } else if (dir == DMA_MEM_TO_DEV) { 1157 dev_addr = c->cfg.dst_addr; 1158 dev_width = c->cfg.dst_addr_width; 1159 burst = c->cfg.dst_maxburst; 1160 } else { 1161 dev_err(chan->device->dev, "%s: bad direction?\n", __func__); 1162 return NULL; 1163 } 1164 1165 /* Bus width translates to the element size (ES) */ 1166 switch (dev_width) { 1167 case DMA_SLAVE_BUSWIDTH_1_BYTE: 1168 es = CSDP_DATA_TYPE_8; 1169 break; 1170 case DMA_SLAVE_BUSWIDTH_2_BYTES: 1171 es = CSDP_DATA_TYPE_16; 1172 break; 1173 case DMA_SLAVE_BUSWIDTH_4_BYTES: 1174 es = CSDP_DATA_TYPE_32; 1175 break; 1176 default: /* not reached */ 1177 return NULL; 1178 } 1179 1180 /* Now allocate and setup the descriptor. */ 1181 d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC); 1182 if (!d) 1183 return NULL; 1184 1185 d->dir = dir; 1186 d->dev_addr = dev_addr; 1187 d->fi = burst; 1188 d->es = es; 1189 d->sg[0].addr = buf_addr; 1190 d->sg[0].en = period_len / es_bytes[es]; 1191 d->sg[0].fn = buf_len / period_len; 1192 d->sglen = 1; 1193 1194 d->ccr = c->ccr; 1195 if (dir == DMA_DEV_TO_MEM) 1196 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; 1197 else 1198 d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC; 1199 1200 d->cicr = CICR_DROP_IE; 1201 if (flags & DMA_PREP_INTERRUPT) 1202 d->cicr |= CICR_FRAME_IE; 1203 1204 d->csdp = es; 1205 1206 if (dma_omap1()) { 1207 d->cicr |= CICR_TOUT_IE; 1208 1209 if (dir == DMA_DEV_TO_MEM) 1210 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI; 1211 else 1212 d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF; 1213 } else { 1214 if (burst) 1215 d->ccr |= CCR_SYNC_PACKET; 1216 else 1217 d->ccr |= CCR_SYNC_ELEMENT; 1218 1219 if (dir == DMA_DEV_TO_MEM) { 1220 d->ccr |= CCR_TRIGGER_SRC; 1221 d->csdp |= CSDP_DST_PACKED; 1222 } else { 1223 d->csdp |= CSDP_SRC_PACKED; 1224 } 1225 1226 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; 1227 1228 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; 1229 } 1230 1231 if (__dma_omap15xx(od->plat->dma_attr)) 1232 d->ccr |= CCR_AUTO_INIT | CCR_REPEAT; 1233 else 1234 d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK; 1235 1236 c->cyclic = true; 1237 1238 return vchan_tx_prep(&c->vc, &d->vd, flags); 1239 } 1240 1241 static struct dma_async_tx_descriptor *omap_dma_prep_dma_memcpy( 1242 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 1243 size_t len, unsigned long tx_flags) 1244 { 1245 struct omap_chan *c = to_omap_dma_chan(chan); 1246 struct omap_desc *d; 1247 uint8_t data_type; 1248 1249 d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC); 1250 if (!d) 1251 return NULL; 1252 1253 data_type = __ffs((src | dest | len)); 1254 if (data_type > CSDP_DATA_TYPE_32) 1255 data_type = CSDP_DATA_TYPE_32; 1256 1257 d->dir = DMA_MEM_TO_MEM; 1258 d->dev_addr = src; 1259 d->fi = 0; 1260 d->es = data_type; 1261 d->sg[0].en = len / BIT(data_type); 1262 d->sg[0].fn = 1; 1263 d->sg[0].addr = dest; 1264 d->sglen = 1; 1265 d->ccr = c->ccr; 1266 d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_POSTINC; 1267 1268 if (tx_flags & DMA_PREP_INTERRUPT) 1269 d->cicr |= CICR_FRAME_IE; 1270 else 1271 d->polled = true; 1272 1273 d->csdp = data_type; 1274 1275 if (dma_omap1()) { 1276 d->cicr |= CICR_TOUT_IE; 1277 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF; 1278 } else { 1279 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED; 1280 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; 1281 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; 1282 } 1283 1284 return vchan_tx_prep(&c->vc, &d->vd, tx_flags); 1285 } 1286 1287 static struct dma_async_tx_descriptor *omap_dma_prep_dma_interleaved( 1288 struct dma_chan *chan, struct dma_interleaved_template *xt, 1289 unsigned long flags) 1290 { 1291 struct omap_chan *c = to_omap_dma_chan(chan); 1292 struct omap_desc *d; 1293 struct omap_sg *sg; 1294 uint8_t data_type; 1295 size_t src_icg, dst_icg; 1296 1297 /* Slave mode is not supported */ 1298 if (is_slave_direction(xt->dir)) 1299 return NULL; 1300 1301 if (xt->frame_size != 1 || xt->numf == 0) 1302 return NULL; 1303 1304 d = kzalloc(sizeof(*d) + sizeof(d->sg[0]), GFP_ATOMIC); 1305 if (!d) 1306 return NULL; 1307 1308 data_type = __ffs((xt->src_start | xt->dst_start | xt->sgl[0].size)); 1309 if (data_type > CSDP_DATA_TYPE_32) 1310 data_type = CSDP_DATA_TYPE_32; 1311 1312 sg = &d->sg[0]; 1313 d->dir = DMA_MEM_TO_MEM; 1314 d->dev_addr = xt->src_start; 1315 d->es = data_type; 1316 sg->en = xt->sgl[0].size / BIT(data_type); 1317 sg->fn = xt->numf; 1318 sg->addr = xt->dst_start; 1319 d->sglen = 1; 1320 d->ccr = c->ccr; 1321 1322 src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); 1323 dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); 1324 if (src_icg) { 1325 d->ccr |= CCR_SRC_AMODE_DBLIDX; 1326 d->ei = 1; 1327 d->fi = src_icg + 1; 1328 } else if (xt->src_inc) { 1329 d->ccr |= CCR_SRC_AMODE_POSTINC; 1330 d->fi = 0; 1331 } else { 1332 dev_err(chan->device->dev, 1333 "%s: SRC constant addressing is not supported\n", 1334 __func__); 1335 kfree(d); 1336 return NULL; 1337 } 1338 1339 if (dst_icg) { 1340 d->ccr |= CCR_DST_AMODE_DBLIDX; 1341 sg->ei = 1; 1342 sg->fi = dst_icg + 1; 1343 } else if (xt->dst_inc) { 1344 d->ccr |= CCR_DST_AMODE_POSTINC; 1345 sg->fi = 0; 1346 } else { 1347 dev_err(chan->device->dev, 1348 "%s: DST constant addressing is not supported\n", 1349 __func__); 1350 kfree(d); 1351 return NULL; 1352 } 1353 1354 d->cicr = CICR_DROP_IE | CICR_FRAME_IE; 1355 1356 d->csdp = data_type; 1357 1358 if (dma_omap1()) { 1359 d->cicr |= CICR_TOUT_IE; 1360 d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_EMIFF; 1361 } else { 1362 d->csdp |= CSDP_DST_PACKED | CSDP_SRC_PACKED; 1363 d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; 1364 d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; 1365 } 1366 1367 return vchan_tx_prep(&c->vc, &d->vd, flags); 1368 } 1369 1370 static int omap_dma_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) 1371 { 1372 struct omap_chan *c = to_omap_dma_chan(chan); 1373 1374 if (cfg->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || 1375 cfg->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 1376 return -EINVAL; 1377 1378 if (cfg->src_maxburst > chan->device->max_burst || 1379 cfg->dst_maxburst > chan->device->max_burst) 1380 return -EINVAL; 1381 1382 memcpy(&c->cfg, cfg, sizeof(c->cfg)); 1383 1384 return 0; 1385 } 1386 1387 static int omap_dma_terminate_all(struct dma_chan *chan) 1388 { 1389 struct omap_chan *c = to_omap_dma_chan(chan); 1390 unsigned long flags; 1391 LIST_HEAD(head); 1392 1393 spin_lock_irqsave(&c->vc.lock, flags); 1394 1395 /* 1396 * Stop DMA activity: we assume the callback will not be called 1397 * after omap_dma_stop() returns (even if it does, it will see 1398 * c->desc is NULL and exit.) 1399 */ 1400 if (c->desc) { 1401 vchan_terminate_vdesc(&c->desc->vd); 1402 c->desc = NULL; 1403 /* Avoid stopping the dma twice */ 1404 if (!c->paused) 1405 omap_dma_stop(c); 1406 } 1407 1408 c->cyclic = false; 1409 c->paused = false; 1410 1411 vchan_get_all_descriptors(&c->vc, &head); 1412 spin_unlock_irqrestore(&c->vc.lock, flags); 1413 vchan_dma_desc_free_list(&c->vc, &head); 1414 1415 return 0; 1416 } 1417 1418 static void omap_dma_synchronize(struct dma_chan *chan) 1419 { 1420 struct omap_chan *c = to_omap_dma_chan(chan); 1421 1422 vchan_synchronize(&c->vc); 1423 } 1424 1425 static int omap_dma_pause(struct dma_chan *chan) 1426 { 1427 struct omap_chan *c = to_omap_dma_chan(chan); 1428 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 1429 unsigned long flags; 1430 int ret = -EINVAL; 1431 bool can_pause = false; 1432 1433 spin_lock_irqsave(&od->irq_lock, flags); 1434 1435 if (!c->desc) 1436 goto out; 1437 1438 if (c->cyclic) 1439 can_pause = true; 1440 1441 /* 1442 * We do not allow DMA_MEM_TO_DEV transfers to be paused. 1443 * From the AM572x TRM, 16.1.4.18 Disabling a Channel During Transfer: 1444 * "When a channel is disabled during a transfer, the channel undergoes 1445 * an abort, unless it is hardware-source-synchronized …". 1446 * A source-synchronised channel is one where the fetching of data is 1447 * under control of the device. In other words, a device-to-memory 1448 * transfer. So, a destination-synchronised channel (which would be a 1449 * memory-to-device transfer) undergoes an abort if the CCR_ENABLE 1450 * bit is cleared. 1451 * From 16.1.4.20.4.6.2 Abort: "If an abort trigger occurs, the channel 1452 * aborts immediately after completion of current read/write 1453 * transactions and then the FIFO is cleaned up." The term "cleaned up" 1454 * is not defined. TI recommends to check that RD_ACTIVE and WR_ACTIVE 1455 * are both clear _before_ disabling the channel, otherwise data loss 1456 * will occur. 1457 * The problem is that if the channel is active, then device activity 1458 * can result in DMA activity starting between reading those as both 1459 * clear and the write to DMA_CCR to clear the enable bit hitting the 1460 * hardware. If the DMA hardware can't drain the data in its FIFO to the 1461 * destination, then data loss "might" occur (say if we write to an UART 1462 * and the UART is not accepting any further data). 1463 */ 1464 else if (c->desc->dir == DMA_DEV_TO_MEM) 1465 can_pause = true; 1466 1467 if (can_pause && !c->paused) { 1468 ret = omap_dma_stop(c); 1469 if (!ret) 1470 c->paused = true; 1471 } 1472 out: 1473 spin_unlock_irqrestore(&od->irq_lock, flags); 1474 1475 return ret; 1476 } 1477 1478 static int omap_dma_resume(struct dma_chan *chan) 1479 { 1480 struct omap_chan *c = to_omap_dma_chan(chan); 1481 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 1482 unsigned long flags; 1483 int ret = -EINVAL; 1484 1485 spin_lock_irqsave(&od->irq_lock, flags); 1486 1487 if (c->paused && c->desc) { 1488 mb(); 1489 1490 /* Restore channel link register */ 1491 omap_dma_chan_write(c, CLNK_CTRL, c->desc->clnk_ctrl); 1492 1493 omap_dma_start(c, c->desc); 1494 c->paused = false; 1495 ret = 0; 1496 } 1497 spin_unlock_irqrestore(&od->irq_lock, flags); 1498 1499 return ret; 1500 } 1501 1502 static int omap_dma_chan_init(struct omap_dmadev *od) 1503 { 1504 struct omap_chan *c; 1505 1506 c = kzalloc(sizeof(*c), GFP_KERNEL); 1507 if (!c) 1508 return -ENOMEM; 1509 1510 c->reg_map = od->reg_map; 1511 c->vc.desc_free = omap_dma_desc_free; 1512 vchan_init(&c->vc, &od->ddev); 1513 1514 return 0; 1515 } 1516 1517 static void omap_dma_free(struct omap_dmadev *od) 1518 { 1519 while (!list_empty(&od->ddev.channels)) { 1520 struct omap_chan *c = list_first_entry(&od->ddev.channels, 1521 struct omap_chan, vc.chan.device_node); 1522 1523 list_del(&c->vc.chan.device_node); 1524 tasklet_kill(&c->vc.task); 1525 kfree(c); 1526 } 1527 } 1528 1529 /* Currently used by omap2 & 3 to block deeper SoC idle states */ 1530 static bool omap_dma_busy(struct omap_dmadev *od) 1531 { 1532 struct omap_chan *c; 1533 int lch = -1; 1534 1535 while (1) { 1536 lch = find_next_bit(od->lch_bitmap, od->lch_count, lch + 1); 1537 if (lch >= od->lch_count) 1538 break; 1539 c = od->lch_map[lch]; 1540 if (!c) 1541 continue; 1542 if (omap_dma_chan_read(c, CCR) & CCR_ENABLE) 1543 return true; 1544 } 1545 1546 return false; 1547 } 1548 1549 /* Currently only used for omap2. For omap1, also a check for lcd_dma is needed */ 1550 static int omap_dma_busy_notifier(struct notifier_block *nb, 1551 unsigned long cmd, void *v) 1552 { 1553 struct omap_dmadev *od; 1554 1555 od = container_of(nb, struct omap_dmadev, nb); 1556 1557 switch (cmd) { 1558 case CPU_CLUSTER_PM_ENTER: 1559 if (omap_dma_busy(od)) 1560 return NOTIFY_BAD; 1561 break; 1562 case CPU_CLUSTER_PM_ENTER_FAILED: 1563 case CPU_CLUSTER_PM_EXIT: 1564 break; 1565 } 1566 1567 return NOTIFY_OK; 1568 } 1569 1570 /* 1571 * We are using IRQENABLE_L1, and legacy DMA code was using IRQENABLE_L0. 1572 * As the DSP may be using IRQENABLE_L2 and L3, let's not touch those for 1573 * now. Context save seems to be only currently needed on omap3. 1574 */ 1575 static void omap_dma_context_save(struct omap_dmadev *od) 1576 { 1577 od->context.irqenable_l0 = omap_dma_glbl_read(od, IRQENABLE_L0); 1578 od->context.irqenable_l1 = omap_dma_glbl_read(od, IRQENABLE_L1); 1579 od->context.ocp_sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG); 1580 od->context.gcr = omap_dma_glbl_read(od, GCR); 1581 } 1582 1583 static void omap_dma_context_restore(struct omap_dmadev *od) 1584 { 1585 int i; 1586 1587 omap_dma_glbl_write(od, GCR, od->context.gcr); 1588 omap_dma_glbl_write(od, OCP_SYSCONFIG, od->context.ocp_sysconfig); 1589 omap_dma_glbl_write(od, IRQENABLE_L0, od->context.irqenable_l0); 1590 omap_dma_glbl_write(od, IRQENABLE_L1, od->context.irqenable_l1); 1591 1592 /* Clear IRQSTATUS_L0 as legacy DMA code is no longer doing it */ 1593 if (od->plat->errata & DMA_ROMCODE_BUG) 1594 omap_dma_glbl_write(od, IRQSTATUS_L0, 0); 1595 1596 /* Clear dma channels */ 1597 for (i = 0; i < od->lch_count; i++) 1598 omap_dma_clear_lch(od, i); 1599 } 1600 1601 /* Currently only used for omap3 */ 1602 static int omap_dma_context_notifier(struct notifier_block *nb, 1603 unsigned long cmd, void *v) 1604 { 1605 struct omap_dmadev *od; 1606 1607 od = container_of(nb, struct omap_dmadev, nb); 1608 1609 switch (cmd) { 1610 case CPU_CLUSTER_PM_ENTER: 1611 if (omap_dma_busy(od)) 1612 return NOTIFY_BAD; 1613 omap_dma_context_save(od); 1614 break; 1615 case CPU_CLUSTER_PM_ENTER_FAILED: /* No need to restore context */ 1616 break; 1617 case CPU_CLUSTER_PM_EXIT: 1618 omap_dma_context_restore(od); 1619 break; 1620 } 1621 1622 return NOTIFY_OK; 1623 } 1624 1625 static void omap_dma_init_gcr(struct omap_dmadev *od, int arb_rate, 1626 int max_fifo_depth, int tparams) 1627 { 1628 u32 val; 1629 1630 /* Set only for omap2430 and later */ 1631 if (!od->cfg->rw_priority) 1632 return; 1633 1634 if (max_fifo_depth == 0) 1635 max_fifo_depth = 1; 1636 if (arb_rate == 0) 1637 arb_rate = 1; 1638 1639 val = 0xff & max_fifo_depth; 1640 val |= (0x3 & tparams) << 12; 1641 val |= (arb_rate & 0xff) << 16; 1642 1643 omap_dma_glbl_write(od, GCR, val); 1644 } 1645 1646 #define OMAP_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 1647 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 1648 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) 1649 1650 /* 1651 * No flags currently set for default configuration as omap1 is still 1652 * using platform data. 1653 */ 1654 static const struct omap_dma_config default_cfg; 1655 1656 static int omap_dma_probe(struct platform_device *pdev) 1657 { 1658 const struct omap_dma_config *conf; 1659 struct omap_dmadev *od; 1660 int rc, i, irq; 1661 u32 val; 1662 1663 od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); 1664 if (!od) 1665 return -ENOMEM; 1666 1667 od->base = devm_platform_ioremap_resource(pdev, 0); 1668 if (IS_ERR(od->base)) 1669 return PTR_ERR(od->base); 1670 1671 conf = of_device_get_match_data(&pdev->dev); 1672 if (conf) { 1673 od->cfg = conf; 1674 od->plat = dev_get_platdata(&pdev->dev); 1675 if (!od->plat) { 1676 dev_err(&pdev->dev, "omap_system_dma_plat_info is missing"); 1677 return -ENODEV; 1678 } 1679 } else if (IS_ENABLED(CONFIG_ARCH_OMAP1)) { 1680 od->cfg = &default_cfg; 1681 1682 od->plat = omap_get_plat_info(); 1683 if (!od->plat) 1684 return -EPROBE_DEFER; 1685 } else { 1686 return -ENODEV; 1687 } 1688 1689 od->reg_map = od->plat->reg_map; 1690 1691 dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); 1692 dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); 1693 dma_cap_set(DMA_MEMCPY, od->ddev.cap_mask); 1694 dma_cap_set(DMA_INTERLEAVE, od->ddev.cap_mask); 1695 od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources; 1696 od->ddev.device_free_chan_resources = omap_dma_free_chan_resources; 1697 od->ddev.device_tx_status = omap_dma_tx_status; 1698 od->ddev.device_issue_pending = omap_dma_issue_pending; 1699 od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg; 1700 od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic; 1701 od->ddev.device_prep_dma_memcpy = omap_dma_prep_dma_memcpy; 1702 od->ddev.device_prep_interleaved_dma = omap_dma_prep_dma_interleaved; 1703 od->ddev.device_config = omap_dma_slave_config; 1704 od->ddev.device_pause = omap_dma_pause; 1705 od->ddev.device_resume = omap_dma_resume; 1706 od->ddev.device_terminate_all = omap_dma_terminate_all; 1707 od->ddev.device_synchronize = omap_dma_synchronize; 1708 od->ddev.src_addr_widths = OMAP_DMA_BUSWIDTHS; 1709 od->ddev.dst_addr_widths = OMAP_DMA_BUSWIDTHS; 1710 od->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1711 if (__dma_omap15xx(od->plat->dma_attr)) 1712 od->ddev.residue_granularity = 1713 DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1714 else 1715 od->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1716 od->ddev.max_burst = SZ_16M - 1; /* CCEN: 24bit unsigned */ 1717 od->ddev.dev = &pdev->dev; 1718 INIT_LIST_HEAD(&od->ddev.channels); 1719 mutex_init(&od->lch_lock); 1720 spin_lock_init(&od->lock); 1721 spin_lock_init(&od->irq_lock); 1722 1723 /* Number of DMA requests */ 1724 od->dma_requests = OMAP_SDMA_REQUESTS; 1725 if (pdev->dev.of_node && of_property_read_u32(pdev->dev.of_node, 1726 "dma-requests", 1727 &od->dma_requests)) { 1728 dev_info(&pdev->dev, 1729 "Missing dma-requests property, using %u.\n", 1730 OMAP_SDMA_REQUESTS); 1731 } 1732 1733 /* Number of available logical channels */ 1734 if (!pdev->dev.of_node) { 1735 od->lch_count = od->plat->dma_attr->lch_count; 1736 if (unlikely(!od->lch_count)) 1737 od->lch_count = OMAP_SDMA_CHANNELS; 1738 } else if (of_property_read_u32(pdev->dev.of_node, "dma-channels", 1739 &od->lch_count)) { 1740 dev_info(&pdev->dev, 1741 "Missing dma-channels property, using %u.\n", 1742 OMAP_SDMA_CHANNELS); 1743 od->lch_count = OMAP_SDMA_CHANNELS; 1744 } 1745 1746 /* Mask of allowed logical channels */ 1747 if (pdev->dev.of_node && !of_property_read_u32(pdev->dev.of_node, 1748 "dma-channel-mask", 1749 &val)) { 1750 /* Tag channels not in mask as reserved */ 1751 val = ~val; 1752 bitmap_from_arr32(od->lch_bitmap, &val, od->lch_count); 1753 } 1754 if (od->plat->dma_attr->dev_caps & HS_CHANNELS_RESERVED) 1755 bitmap_set(od->lch_bitmap, 0, 2); 1756 1757 od->lch_map = devm_kcalloc(&pdev->dev, od->lch_count, 1758 sizeof(*od->lch_map), 1759 GFP_KERNEL); 1760 if (!od->lch_map) 1761 return -ENOMEM; 1762 1763 for (i = 0; i < od->dma_requests; i++) { 1764 rc = omap_dma_chan_init(od); 1765 if (rc) { 1766 omap_dma_free(od); 1767 return rc; 1768 } 1769 } 1770 1771 irq = platform_get_irq(pdev, 1); 1772 if (irq <= 0) { 1773 dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq); 1774 od->legacy = true; 1775 } else { 1776 /* Disable all interrupts */ 1777 od->irq_enable_mask = 0; 1778 omap_dma_glbl_write(od, IRQENABLE_L1, 0); 1779 1780 rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq, 1781 IRQF_SHARED, "omap-dma-engine", od); 1782 if (rc) { 1783 omap_dma_free(od); 1784 return rc; 1785 } 1786 } 1787 1788 if (omap_dma_glbl_read(od, CAPS_0) & CAPS_0_SUPPORT_LL123) 1789 od->ll123_supported = true; 1790 1791 od->ddev.filter.map = od->plat->slave_map; 1792 od->ddev.filter.mapcnt = od->plat->slavecnt; 1793 od->ddev.filter.fn = omap_dma_filter_fn; 1794 1795 if (od->ll123_supported) { 1796 od->desc_pool = dma_pool_create(dev_name(&pdev->dev), 1797 &pdev->dev, 1798 sizeof(struct omap_type2_desc), 1799 4, 0); 1800 if (!od->desc_pool) { 1801 dev_err(&pdev->dev, 1802 "unable to allocate descriptor pool\n"); 1803 od->ll123_supported = false; 1804 } 1805 } 1806 1807 rc = dma_async_device_register(&od->ddev); 1808 if (rc) { 1809 pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n", 1810 rc); 1811 omap_dma_free(od); 1812 return rc; 1813 } 1814 1815 platform_set_drvdata(pdev, od); 1816 1817 if (pdev->dev.of_node) { 1818 omap_dma_info.dma_cap = od->ddev.cap_mask; 1819 1820 /* Device-tree DMA controller registration */ 1821 rc = of_dma_controller_register(pdev->dev.of_node, 1822 of_dma_simple_xlate, &omap_dma_info); 1823 if (rc) { 1824 pr_warn("OMAP-DMA: failed to register DMA controller\n"); 1825 dma_async_device_unregister(&od->ddev); 1826 omap_dma_free(od); 1827 } 1828 } 1829 1830 omap_dma_init_gcr(od, DMA_DEFAULT_ARB_RATE, DMA_DEFAULT_FIFO_DEPTH, 0); 1831 1832 if (od->cfg->needs_busy_check) { 1833 od->nb.notifier_call = omap_dma_busy_notifier; 1834 cpu_pm_register_notifier(&od->nb); 1835 } else if (od->cfg->may_lose_context) { 1836 od->nb.notifier_call = omap_dma_context_notifier; 1837 cpu_pm_register_notifier(&od->nb); 1838 } 1839 1840 dev_info(&pdev->dev, "OMAP DMA engine driver%s\n", 1841 od->ll123_supported ? " (LinkedList1/2/3 supported)" : ""); 1842 1843 return rc; 1844 } 1845 1846 static void omap_dma_remove(struct platform_device *pdev) 1847 { 1848 struct omap_dmadev *od = platform_get_drvdata(pdev); 1849 int irq; 1850 1851 if (od->cfg->may_lose_context) 1852 cpu_pm_unregister_notifier(&od->nb); 1853 1854 if (pdev->dev.of_node) 1855 of_dma_controller_free(pdev->dev.of_node); 1856 1857 irq = platform_get_irq(pdev, 1); 1858 devm_free_irq(&pdev->dev, irq, od); 1859 1860 dma_async_device_unregister(&od->ddev); 1861 1862 if (!omap_dma_legacy(od)) { 1863 /* Disable all interrupts */ 1864 omap_dma_glbl_write(od, IRQENABLE_L0, 0); 1865 } 1866 1867 if (od->ll123_supported) 1868 dma_pool_destroy(od->desc_pool); 1869 1870 omap_dma_free(od); 1871 } 1872 1873 static const struct omap_dma_config omap2420_data = { 1874 .lch_end = CCFN, 1875 .rw_priority = true, 1876 .needs_lch_clear = true, 1877 .needs_busy_check = true, 1878 }; 1879 1880 static const struct omap_dma_config omap2430_data = { 1881 .lch_end = CCFN, 1882 .rw_priority = true, 1883 .needs_lch_clear = true, 1884 }; 1885 1886 static const struct omap_dma_config omap3430_data = { 1887 .lch_end = CCFN, 1888 .rw_priority = true, 1889 .needs_lch_clear = true, 1890 .may_lose_context = true, 1891 }; 1892 1893 static const struct omap_dma_config omap3630_data = { 1894 .lch_end = CCDN, 1895 .rw_priority = true, 1896 .needs_lch_clear = true, 1897 .may_lose_context = true, 1898 }; 1899 1900 static const struct omap_dma_config omap4_data = { 1901 .lch_end = CCDN, 1902 .rw_priority = true, 1903 .needs_lch_clear = true, 1904 }; 1905 1906 static const struct of_device_id omap_dma_match[] = { 1907 { .compatible = "ti,omap2420-sdma", .data = &omap2420_data, }, 1908 { .compatible = "ti,omap2430-sdma", .data = &omap2430_data, }, 1909 { .compatible = "ti,omap3430-sdma", .data = &omap3430_data, }, 1910 { .compatible = "ti,omap3630-sdma", .data = &omap3630_data, }, 1911 { .compatible = "ti,omap4430-sdma", .data = &omap4_data, }, 1912 {}, 1913 }; 1914 MODULE_DEVICE_TABLE(of, omap_dma_match); 1915 1916 static struct platform_driver omap_dma_driver = { 1917 .probe = omap_dma_probe, 1918 .remove_new = omap_dma_remove, 1919 .driver = { 1920 .name = "omap-dma-engine", 1921 .of_match_table = omap_dma_match, 1922 }, 1923 }; 1924 1925 static bool omap_dma_filter_fn(struct dma_chan *chan, void *param) 1926 { 1927 if (chan->device->dev->driver == &omap_dma_driver.driver) { 1928 struct omap_dmadev *od = to_omap_dma_dev(chan->device); 1929 struct omap_chan *c = to_omap_dma_chan(chan); 1930 unsigned req = *(unsigned *)param; 1931 1932 if (req <= od->dma_requests) { 1933 c->dma_sig = req; 1934 return true; 1935 } 1936 } 1937 return false; 1938 } 1939 1940 static int omap_dma_init(void) 1941 { 1942 return platform_driver_register(&omap_dma_driver); 1943 } 1944 subsys_initcall(omap_dma_init); 1945 1946 static void __exit omap_dma_exit(void) 1947 { 1948 platform_driver_unregister(&omap_dma_driver); 1949 } 1950 module_exit(omap_dma_exit); 1951 1952 MODULE_AUTHOR("Russell King"); 1953 MODULE_LICENSE("GPL"); 1954