1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2012 Marvell International Ltd. 4 */ 5 6 #include <linux/err.h> 7 #include <linux/module.h> 8 #include <linux/init.h> 9 #include <linux/types.h> 10 #include <linux/interrupt.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/slab.h> 13 #include <linux/dmaengine.h> 14 #include <linux/platform_device.h> 15 #include <linux/device.h> 16 #include <linux/platform_data/mmp_dma.h> 17 #include <linux/dmapool.h> 18 #include <linux/clk.h> 19 #include <linux/reset.h> 20 #include <linux/of_dma.h> 21 #include <linux/of.h> 22 23 #include "dmaengine.h" 24 25 #define DCSR 0x0000 26 #define DALGN 0x00a0 27 #define DINT 0x00f0 28 #define DDADR(n) (0x0200 + ((n) << 4)) 29 #define DSADR(n) (0x0204 + ((n) << 4)) 30 #define DTADR(n) (0x0208 + ((n) << 4)) 31 #define DDADRH(n) (0x0300 + ((n) << 4)) 32 #define DSADRH(n) (0x0304 + ((n) << 4)) 33 #define DTADRH(n) (0x0308 + ((n) << 4)) 34 #define DCMD 0x020c 35 36 #define DCSR_RUN BIT(31) /* Run Bit (read / write) */ 37 #define DCSR_NODESC BIT(30) /* No-Descriptor Fetch (read / write) */ 38 #define DCSR_STOPIRQEN BIT(29) /* Stop Interrupt Enable (read / write) */ 39 #define DCSR_REQPEND BIT(8) /* Request Pending (read-only) */ 40 #define DCSR_STOPSTATE BIT(3) /* Stop State (read-only) */ 41 #define DCSR_ENDINTR BIT(2) /* End Interrupt (read / write) */ 42 #define DCSR_STARTINTR BIT(1) /* Start Interrupt (read / write) */ 43 #define DCSR_BUSERR BIT(0) /* Bus Error Interrupt (read / write) */ 44 45 #define DCSR_EORIRQEN BIT(28) /* End of Receive Interrupt Enable (R/W) */ 46 #define DCSR_EORJMPEN BIT(27) /* Jump to next descriptor on EOR */ 47 #define DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */ 48 #define DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */ 49 #define DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */ 50 #define DCSR_LPAEEN BIT(21) /* Long Physical Address Extension Enable */ 51 #define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */ 52 #define DCSR_EORINTR BIT(9) /* The end of Receive */ 53 54 #define DRCMR_BASE 0x0100 55 #define DRCMR_EXT_BASE_DEFAULT 0x1100 56 #define DRCMR_REQ_LIMIT 64 57 #define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */ 58 #define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */ 59 60 #define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */ 61 #define DDADR_STOP BIT(0) /* Stop (read / write) */ 62 63 #define DCMD_INCSRCADDR BIT(31) /* Source Address Increment Setting. */ 64 #define DCMD_INCTRGADDR BIT(30) /* Target Address Increment Setting. */ 65 #define DCMD_FLOWSRC BIT(29) /* Flow Control by the source. */ 66 #define DCMD_FLOWTRG BIT(28) /* Flow Control by the target. */ 67 #define DCMD_STARTIRQEN BIT(22) /* Start Interrupt Enable */ 68 #define DCMD_ENDIRQEN BIT(21) /* End Interrupt Enable */ 69 #define DCMD_ENDIAN BIT(18) /* Device Endian-ness. */ 70 #define DCMD_BURST8 (1 << 16) /* 8 byte burst */ 71 #define DCMD_BURST16 (2 << 16) /* 16 byte burst */ 72 #define DCMD_BURST32 (3 << 16) /* 32 byte burst */ 73 #define DCMD_WIDTH1 (1 << 14) /* 1 byte width */ 74 #define DCMD_WIDTH2 (2 << 14) /* 2 byte width (HalfWord) */ 75 #define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */ 76 #define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */ 77 78 #define PDMA_MAX_DESC_BYTES DCMD_LENGTH 79 80 struct mmp_pdma_desc_hw { 81 u32 ddadr; /* Points to the next descriptor + flags */ 82 u32 dsadr; /* DSADR value for the current transfer */ 83 u32 dtadr; /* DTADR value for the current transfer */ 84 u32 dcmd; /* DCMD value for the current transfer */ 85 /* 86 * The following 32-bit words are only used in the 64-bit, ie. 87 * LPAE (Long Physical Address Extension) mode. 88 * They are used to specify the high 32 bits of the descriptor's 89 * addresses. 90 */ 91 u32 ddadrh; /* High 32-bit of DDADR */ 92 u32 dsadrh; /* High 32-bit of DSADR */ 93 u32 dtadrh; /* High 32-bit of DTADR */ 94 u32 rsvd; /* reserved */ 95 } __aligned(32); 96 97 struct mmp_pdma_desc_sw { 98 struct mmp_pdma_desc_hw desc; 99 struct list_head node; 100 struct list_head tx_list; 101 struct dma_async_tx_descriptor async_tx; 102 }; 103 104 struct mmp_pdma_phy; 105 106 struct mmp_pdma_chan { 107 struct device *dev; 108 struct dma_chan chan; 109 struct dma_async_tx_descriptor desc; 110 struct mmp_pdma_phy *phy; 111 enum dma_transfer_direction dir; 112 struct dma_slave_config slave_config; 113 114 struct mmp_pdma_desc_sw *cyclic_first; /* first desc_sw if channel 115 * is in cyclic mode */ 116 117 /* channel's basic info */ 118 struct tasklet_struct tasklet; 119 u32 dcmd; 120 u32 drcmr; 121 u32 dev_addr; 122 123 /* list for desc */ 124 spinlock_t desc_lock; /* Descriptor list lock */ 125 struct list_head chain_pending; /* Link descriptors queue for pending */ 126 struct list_head chain_running; /* Link descriptors queue for running */ 127 bool idle; /* channel statue machine */ 128 bool byte_align; 129 130 struct dma_pool *desc_pool; /* Descriptors pool */ 131 }; 132 133 struct mmp_pdma_phy { 134 int idx; 135 void __iomem *base; 136 struct mmp_pdma_chan *vchan; 137 }; 138 139 /** 140 * struct mmp_pdma_ops - Operations for the MMP PDMA controller 141 * 142 * Hardware Register Operations (read/write hardware registers): 143 * @write_next_addr: Function to program address of next descriptor into 144 * DDADR/DDADRH 145 * @read_src_addr: Function to read the source address from DSADR/DSADRH 146 * @read_dst_addr: Function to read the destination address from DTADR/DTADRH 147 * 148 * Descriptor Memory Operations (manipulate descriptor structs in memory): 149 * @set_desc_next_addr: Function to set next descriptor address in descriptor 150 * @set_desc_src_addr: Function to set the source address in descriptor 151 * @set_desc_dst_addr: Function to set the destination address in descriptor 152 * @get_desc_src_addr: Function to get the source address from descriptor 153 * @get_desc_dst_addr: Function to get the destination address from descriptor 154 * 155 * Controller Configuration: 156 * @run_bits: Control bits in DCSR register for channel start/stop 157 * @dma_width: DMA addressing width in bits (32 or 64). Determines the 158 * DMA mask capability of the controller hardware. 159 * @drcmr_ext_base: Base DRCMR address for extended requests 160 */ 161 struct mmp_pdma_ops { 162 /* Hardware Register Operations */ 163 void (*write_next_addr)(struct mmp_pdma_phy *phy, dma_addr_t addr); 164 u64 (*read_src_addr)(struct mmp_pdma_phy *phy); 165 u64 (*read_dst_addr)(struct mmp_pdma_phy *phy); 166 167 /* Descriptor Memory Operations */ 168 void (*set_desc_next_addr)(struct mmp_pdma_desc_hw *desc, 169 dma_addr_t addr); 170 void (*set_desc_src_addr)(struct mmp_pdma_desc_hw *desc, 171 dma_addr_t addr); 172 void (*set_desc_dst_addr)(struct mmp_pdma_desc_hw *desc, 173 dma_addr_t addr); 174 u64 (*get_desc_src_addr)(const struct mmp_pdma_desc_hw *desc); 175 u64 (*get_desc_dst_addr)(const struct mmp_pdma_desc_hw *desc); 176 177 /* Controller Configuration */ 178 u32 run_bits; 179 u32 dma_width; 180 u32 drcmr_ext_base; 181 }; 182 183 struct mmp_pdma_device { 184 int dma_channels; 185 void __iomem *base; 186 struct device *dev; 187 struct dma_device device; 188 struct mmp_pdma_phy *phy; 189 const struct mmp_pdma_ops *ops; 190 spinlock_t phy_lock; /* protect alloc/free phy channels */ 191 }; 192 193 #define tx_to_mmp_pdma_desc(tx) \ 194 container_of(tx, struct mmp_pdma_desc_sw, async_tx) 195 #define to_mmp_pdma_desc(lh) \ 196 container_of(lh, struct mmp_pdma_desc_sw, node) 197 #define to_mmp_pdma_chan(dchan) \ 198 container_of(dchan, struct mmp_pdma_chan, chan) 199 #define to_mmp_pdma_dev(dmadev) \ 200 container_of(dmadev, struct mmp_pdma_device, device) 201 202 static u32 mmp_pdma_get_drcmr(struct mmp_pdma_device *pdev, u32 drcmr) 203 { 204 if (drcmr < DRCMR_REQ_LIMIT) 205 return DRCMR_BASE + (drcmr << 2); 206 return pdev->ops->drcmr_ext_base + ((drcmr - DRCMR_REQ_LIMIT) << 2); 207 } 208 209 /* For 32-bit PDMA */ 210 static void write_next_addr_32(struct mmp_pdma_phy *phy, dma_addr_t addr) 211 { 212 writel(addr, phy->base + DDADR(phy->idx)); 213 } 214 215 static u64 read_src_addr_32(struct mmp_pdma_phy *phy) 216 { 217 return readl(phy->base + DSADR(phy->idx)); 218 } 219 220 static u64 read_dst_addr_32(struct mmp_pdma_phy *phy) 221 { 222 return readl(phy->base + DTADR(phy->idx)); 223 } 224 225 static void set_desc_next_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) 226 { 227 desc->ddadr = addr; 228 } 229 230 static void set_desc_src_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) 231 { 232 desc->dsadr = addr; 233 } 234 235 static void set_desc_dst_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) 236 { 237 desc->dtadr = addr; 238 } 239 240 static u64 get_desc_src_addr_32(const struct mmp_pdma_desc_hw *desc) 241 { 242 return desc->dsadr; 243 } 244 245 static u64 get_desc_dst_addr_32(const struct mmp_pdma_desc_hw *desc) 246 { 247 return desc->dtadr; 248 } 249 250 /* For 64-bit PDMA */ 251 static void write_next_addr_64(struct mmp_pdma_phy *phy, dma_addr_t addr) 252 { 253 writel(lower_32_bits(addr), phy->base + DDADR(phy->idx)); 254 writel(upper_32_bits(addr), phy->base + DDADRH(phy->idx)); 255 } 256 257 static u64 read_src_addr_64(struct mmp_pdma_phy *phy) 258 { 259 u32 low = readl(phy->base + DSADR(phy->idx)); 260 u32 high = readl(phy->base + DSADRH(phy->idx)); 261 262 return ((u64)high << 32) | low; 263 } 264 265 static u64 read_dst_addr_64(struct mmp_pdma_phy *phy) 266 { 267 u32 low = readl(phy->base + DTADR(phy->idx)); 268 u32 high = readl(phy->base + DTADRH(phy->idx)); 269 270 return ((u64)high << 32) | low; 271 } 272 273 static void set_desc_next_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) 274 { 275 desc->ddadr = lower_32_bits(addr); 276 desc->ddadrh = upper_32_bits(addr); 277 } 278 279 static void set_desc_src_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) 280 { 281 desc->dsadr = lower_32_bits(addr); 282 desc->dsadrh = upper_32_bits(addr); 283 } 284 285 static void set_desc_dst_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr) 286 { 287 desc->dtadr = lower_32_bits(addr); 288 desc->dtadrh = upper_32_bits(addr); 289 } 290 291 static u64 get_desc_src_addr_64(const struct mmp_pdma_desc_hw *desc) 292 { 293 return ((u64)desc->dsadrh << 32) | desc->dsadr; 294 } 295 296 static u64 get_desc_dst_addr_64(const struct mmp_pdma_desc_hw *desc) 297 { 298 return ((u64)desc->dtadrh << 32) | desc->dtadr; 299 } 300 301 static int mmp_pdma_config_write(struct dma_chan *dchan, 302 struct dma_slave_config *cfg, 303 enum dma_transfer_direction direction); 304 305 static void enable_chan(struct mmp_pdma_phy *phy) 306 { 307 u32 reg, dalgn; 308 struct mmp_pdma_device *pdev; 309 310 if (!phy->vchan) 311 return; 312 313 pdev = to_mmp_pdma_dev(phy->vchan->chan.device); 314 315 reg = mmp_pdma_get_drcmr(pdev, phy->vchan->drcmr); 316 writel(DRCMR_MAPVLD | phy->idx, phy->base + reg); 317 318 dalgn = readl(phy->base + DALGN); 319 if (phy->vchan->byte_align) 320 dalgn |= 1 << phy->idx; 321 else 322 dalgn &= ~(1 << phy->idx); 323 writel(dalgn, phy->base + DALGN); 324 325 reg = (phy->idx << 2) + DCSR; 326 writel(readl(phy->base + reg) | pdev->ops->run_bits, 327 phy->base + reg); 328 } 329 330 static void disable_chan(struct mmp_pdma_phy *phy) 331 { 332 u32 reg, dcsr; 333 334 if (!phy) 335 return; 336 337 reg = (phy->idx << 2) + DCSR; 338 dcsr = readl(phy->base + reg); 339 340 if (phy->vchan) { 341 struct mmp_pdma_device *pdev; 342 343 pdev = to_mmp_pdma_dev(phy->vchan->chan.device); 344 writel(dcsr & ~pdev->ops->run_bits, phy->base + reg); 345 } else { 346 /* If no vchan, just clear the RUN bit */ 347 writel(dcsr & ~DCSR_RUN, phy->base + reg); 348 } 349 } 350 351 static int clear_chan_irq(struct mmp_pdma_phy *phy) 352 { 353 u32 dcsr; 354 u32 dint = readl(phy->base + DINT); 355 u32 reg = (phy->idx << 2) + DCSR; 356 357 if (!(dint & BIT(phy->idx))) 358 return -EAGAIN; 359 360 /* clear irq */ 361 dcsr = readl(phy->base + reg); 362 writel(dcsr, phy->base + reg); 363 if ((dcsr & DCSR_BUSERR) && (phy->vchan)) 364 dev_warn(phy->vchan->dev, "DCSR_BUSERR\n"); 365 366 return 0; 367 } 368 369 static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id) 370 { 371 struct mmp_pdma_phy *phy = dev_id; 372 373 if (clear_chan_irq(phy) != 0) 374 return IRQ_NONE; 375 376 tasklet_schedule(&phy->vchan->tasklet); 377 return IRQ_HANDLED; 378 } 379 380 static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id) 381 { 382 struct mmp_pdma_device *pdev = dev_id; 383 struct mmp_pdma_phy *phy; 384 u32 dint = readl(pdev->base + DINT); 385 int i, ret; 386 int irq_num = 0; 387 388 while (dint) { 389 i = __ffs(dint); 390 /* only handle interrupts belonging to pdma driver*/ 391 if (i >= pdev->dma_channels) 392 break; 393 dint &= (dint - 1); 394 phy = &pdev->phy[i]; 395 ret = mmp_pdma_chan_handler(irq, phy); 396 if (ret == IRQ_HANDLED) 397 irq_num++; 398 } 399 400 if (irq_num) 401 return IRQ_HANDLED; 402 403 return IRQ_NONE; 404 } 405 406 /* lookup free phy channel as descending priority */ 407 static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan) 408 { 409 int prio, i; 410 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device); 411 struct mmp_pdma_phy *phy, *found = NULL; 412 unsigned long flags; 413 414 /* 415 * dma channel priorities 416 * ch 0 - 3, 16 - 19 <--> (0) 417 * ch 4 - 7, 20 - 23 <--> (1) 418 * ch 8 - 11, 24 - 27 <--> (2) 419 * ch 12 - 15, 28 - 31 <--> (3) 420 */ 421 422 spin_lock_irqsave(&pdev->phy_lock, flags); 423 for (prio = 0; prio <= ((pdev->dma_channels - 1) & 0xf) >> 2; prio++) { 424 for (i = 0; i < pdev->dma_channels; i++) { 425 if (prio != (i & 0xf) >> 2) 426 continue; 427 phy = &pdev->phy[i]; 428 if (!phy->vchan) { 429 phy->vchan = pchan; 430 found = phy; 431 goto out_unlock; 432 } 433 } 434 } 435 436 out_unlock: 437 spin_unlock_irqrestore(&pdev->phy_lock, flags); 438 return found; 439 } 440 441 static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan) 442 { 443 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device); 444 unsigned long flags; 445 u32 reg; 446 447 if (!pchan->phy) 448 return; 449 450 /* clear the channel mapping in DRCMR */ 451 reg = mmp_pdma_get_drcmr(pdev, pchan->drcmr); 452 writel(0, pchan->phy->base + reg); 453 454 spin_lock_irqsave(&pdev->phy_lock, flags); 455 pchan->phy->vchan = NULL; 456 pchan->phy = NULL; 457 spin_unlock_irqrestore(&pdev->phy_lock, flags); 458 } 459 460 /* 461 * start_pending_queue - transfer any pending transactions 462 * pending list ==> running list 463 */ 464 static void start_pending_queue(struct mmp_pdma_chan *chan) 465 { 466 struct mmp_pdma_desc_sw *desc; 467 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(chan->chan.device); 468 469 /* still in running, irq will start the pending list */ 470 if (!chan->idle) { 471 dev_dbg(chan->dev, "DMA controller still busy\n"); 472 return; 473 } 474 475 if (list_empty(&chan->chain_pending)) { 476 /* chance to re-fetch phy channel with higher prio */ 477 mmp_pdma_free_phy(chan); 478 dev_dbg(chan->dev, "no pending list\n"); 479 return; 480 } 481 482 if (!chan->phy) { 483 chan->phy = lookup_phy(chan); 484 if (!chan->phy) { 485 dev_dbg(chan->dev, "no free dma channel\n"); 486 return; 487 } 488 } 489 490 /* 491 * pending -> running 492 * reintilize pending list 493 */ 494 desc = list_first_entry(&chan->chain_pending, 495 struct mmp_pdma_desc_sw, node); 496 list_splice_tail_init(&chan->chain_pending, &chan->chain_running); 497 498 /* 499 * Program the descriptor's address into the DMA controller, 500 * then start the DMA transaction 501 */ 502 pdev->ops->write_next_addr(chan->phy, desc->async_tx.phys); 503 enable_chan(chan->phy); 504 chan->idle = false; 505 } 506 507 508 /* desc->tx_list ==> pending list */ 509 static dma_cookie_t mmp_pdma_tx_submit(struct dma_async_tx_descriptor *tx) 510 { 511 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(tx->chan); 512 struct mmp_pdma_desc_sw *desc = tx_to_mmp_pdma_desc(tx); 513 struct mmp_pdma_desc_sw *child; 514 unsigned long flags; 515 dma_cookie_t cookie = -EBUSY; 516 517 spin_lock_irqsave(&chan->desc_lock, flags); 518 519 list_for_each_entry(child, &desc->tx_list, node) { 520 cookie = dma_cookie_assign(&child->async_tx); 521 } 522 523 /* softly link to pending list - desc->tx_list ==> pending list */ 524 list_splice_tail_init(&desc->tx_list, &chan->chain_pending); 525 526 spin_unlock_irqrestore(&chan->desc_lock, flags); 527 528 return cookie; 529 } 530 531 static struct mmp_pdma_desc_sw * 532 mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan) 533 { 534 struct mmp_pdma_desc_sw *desc; 535 dma_addr_t pdesc; 536 537 desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc); 538 if (!desc) { 539 dev_err(chan->dev, "out of memory for link descriptor\n"); 540 return NULL; 541 } 542 543 INIT_LIST_HEAD(&desc->tx_list); 544 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan); 545 /* each desc has submit */ 546 desc->async_tx.tx_submit = mmp_pdma_tx_submit; 547 desc->async_tx.phys = pdesc; 548 549 return desc; 550 } 551 552 /* 553 * mmp_pdma_alloc_chan_resources - Allocate resources for DMA channel. 554 * 555 * This function will create a dma pool for descriptor allocation. 556 * Request irq only when channel is requested 557 * Return - The number of allocated descriptors. 558 */ 559 560 static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan) 561 { 562 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 563 564 if (chan->desc_pool) 565 return 1; 566 567 chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device), 568 chan->dev, 569 sizeof(struct mmp_pdma_desc_sw), 570 __alignof__(struct mmp_pdma_desc_sw), 571 0); 572 if (!chan->desc_pool) { 573 dev_err(chan->dev, "unable to allocate descriptor pool\n"); 574 return -ENOMEM; 575 } 576 577 mmp_pdma_free_phy(chan); 578 chan->idle = true; 579 chan->dev_addr = 0; 580 return 1; 581 } 582 583 static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan, 584 struct list_head *list) 585 { 586 struct mmp_pdma_desc_sw *desc, *_desc; 587 588 list_for_each_entry_safe(desc, _desc, list, node) { 589 list_del(&desc->node); 590 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); 591 } 592 } 593 594 static void mmp_pdma_free_chan_resources(struct dma_chan *dchan) 595 { 596 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 597 unsigned long flags; 598 599 spin_lock_irqsave(&chan->desc_lock, flags); 600 mmp_pdma_free_desc_list(chan, &chan->chain_pending); 601 mmp_pdma_free_desc_list(chan, &chan->chain_running); 602 spin_unlock_irqrestore(&chan->desc_lock, flags); 603 604 dma_pool_destroy(chan->desc_pool); 605 chan->desc_pool = NULL; 606 chan->idle = true; 607 chan->dev_addr = 0; 608 mmp_pdma_free_phy(chan); 609 return; 610 } 611 612 static struct dma_async_tx_descriptor * 613 mmp_pdma_prep_memcpy(struct dma_chan *dchan, 614 dma_addr_t dma_dst, dma_addr_t dma_src, 615 size_t len, unsigned long flags) 616 { 617 struct mmp_pdma_chan *chan; 618 struct mmp_pdma_device *pdev; 619 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; 620 size_t copy = 0; 621 622 if (!dchan || !len) 623 return NULL; 624 625 pdev = to_mmp_pdma_dev(dchan->device); 626 chan = to_mmp_pdma_chan(dchan); 627 chan->byte_align = false; 628 629 if (!chan->dir) { 630 chan->dir = DMA_MEM_TO_MEM; 631 chan->dcmd = DCMD_INCTRGADDR | DCMD_INCSRCADDR; 632 chan->dcmd |= DCMD_BURST32; 633 } 634 635 do { 636 /* Allocate the link descriptor from DMA pool */ 637 new = mmp_pdma_alloc_descriptor(chan); 638 if (!new) { 639 dev_err(chan->dev, "no memory for desc\n"); 640 goto fail; 641 } 642 643 copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES); 644 if (dma_src & 0x7 || dma_dst & 0x7) 645 chan->byte_align = true; 646 647 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy); 648 pdev->ops->set_desc_src_addr(&new->desc, dma_src); 649 pdev->ops->set_desc_dst_addr(&new->desc, dma_dst); 650 651 if (!first) 652 first = new; 653 else 654 pdev->ops->set_desc_next_addr(&prev->desc, 655 new->async_tx.phys); 656 657 new->async_tx.cookie = 0; 658 async_tx_ack(&new->async_tx); 659 660 prev = new; 661 len -= copy; 662 663 if (chan->dir == DMA_MEM_TO_DEV) { 664 dma_src += copy; 665 } else if (chan->dir == DMA_DEV_TO_MEM) { 666 dma_dst += copy; 667 } else if (chan->dir == DMA_MEM_TO_MEM) { 668 dma_src += copy; 669 dma_dst += copy; 670 } 671 672 /* Insert the link descriptor to the LD ring */ 673 list_add_tail(&new->node, &first->tx_list); 674 } while (len); 675 676 first->async_tx.flags = flags; /* client is in control of this ack */ 677 first->async_tx.cookie = -EBUSY; 678 679 /* last desc and fire IRQ */ 680 new->desc.ddadr = DDADR_STOP; 681 new->desc.dcmd |= DCMD_ENDIRQEN; 682 683 chan->cyclic_first = NULL; 684 685 return &first->async_tx; 686 687 fail: 688 if (first) 689 mmp_pdma_free_desc_list(chan, &first->tx_list); 690 return NULL; 691 } 692 693 static struct dma_async_tx_descriptor * 694 mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 695 unsigned int sg_len, enum dma_transfer_direction dir, 696 unsigned long flags, void *context) 697 { 698 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 699 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(dchan->device); 700 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL; 701 size_t len, avail; 702 struct scatterlist *sg; 703 dma_addr_t addr; 704 int i; 705 706 if ((sgl == NULL) || (sg_len == 0)) 707 return NULL; 708 709 chan->byte_align = false; 710 711 mmp_pdma_config_write(dchan, &chan->slave_config, dir); 712 713 for_each_sg(sgl, sg, sg_len, i) { 714 addr = sg_dma_address(sg); 715 avail = sg_dma_len(sg); 716 717 do { 718 len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES); 719 if (addr & 0x7) 720 chan->byte_align = true; 721 722 /* allocate and populate the descriptor */ 723 new = mmp_pdma_alloc_descriptor(chan); 724 if (!new) { 725 dev_err(chan->dev, "no memory for desc\n"); 726 goto fail; 727 } 728 729 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len); 730 if (dir == DMA_MEM_TO_DEV) { 731 pdev->ops->set_desc_src_addr(&new->desc, addr); 732 new->desc.dtadr = chan->dev_addr; 733 } else { 734 new->desc.dsadr = chan->dev_addr; 735 pdev->ops->set_desc_dst_addr(&new->desc, addr); 736 } 737 738 if (!first) 739 first = new; 740 else 741 pdev->ops->set_desc_next_addr(&prev->desc, 742 new->async_tx.phys); 743 744 new->async_tx.cookie = 0; 745 async_tx_ack(&new->async_tx); 746 prev = new; 747 748 /* Insert the link descriptor to the LD ring */ 749 list_add_tail(&new->node, &first->tx_list); 750 751 /* update metadata */ 752 addr += len; 753 avail -= len; 754 } while (avail); 755 } 756 757 first->async_tx.cookie = -EBUSY; 758 first->async_tx.flags = flags; 759 760 /* last desc and fire IRQ */ 761 new->desc.ddadr = DDADR_STOP; 762 new->desc.dcmd |= DCMD_ENDIRQEN; 763 764 chan->dir = dir; 765 chan->cyclic_first = NULL; 766 767 return &first->async_tx; 768 769 fail: 770 if (first) 771 mmp_pdma_free_desc_list(chan, &first->tx_list); 772 return NULL; 773 } 774 775 static struct dma_async_tx_descriptor * 776 mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, 777 dma_addr_t buf_addr, size_t len, size_t period_len, 778 enum dma_transfer_direction direction, 779 unsigned long flags) 780 { 781 struct mmp_pdma_chan *chan; 782 struct mmp_pdma_device *pdev; 783 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; 784 dma_addr_t dma_src, dma_dst; 785 786 if (!dchan || !len || !period_len) 787 return NULL; 788 789 pdev = to_mmp_pdma_dev(dchan->device); 790 791 /* the buffer length must be a multiple of period_len */ 792 if (len % period_len != 0) 793 return NULL; 794 795 if (period_len > PDMA_MAX_DESC_BYTES) 796 return NULL; 797 798 chan = to_mmp_pdma_chan(dchan); 799 mmp_pdma_config_write(dchan, &chan->slave_config, direction); 800 801 switch (direction) { 802 case DMA_MEM_TO_DEV: 803 dma_src = buf_addr; 804 dma_dst = chan->dev_addr; 805 break; 806 case DMA_DEV_TO_MEM: 807 dma_dst = buf_addr; 808 dma_src = chan->dev_addr; 809 break; 810 default: 811 dev_err(chan->dev, "Unsupported direction for cyclic DMA\n"); 812 return NULL; 813 } 814 815 chan->dir = direction; 816 817 do { 818 /* Allocate the link descriptor from DMA pool */ 819 new = mmp_pdma_alloc_descriptor(chan); 820 if (!new) { 821 dev_err(chan->dev, "no memory for desc\n"); 822 goto fail; 823 } 824 825 new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN | 826 (DCMD_LENGTH & period_len)); 827 pdev->ops->set_desc_src_addr(&new->desc, dma_src); 828 pdev->ops->set_desc_dst_addr(&new->desc, dma_dst); 829 830 if (!first) 831 first = new; 832 else 833 pdev->ops->set_desc_next_addr(&prev->desc, 834 new->async_tx.phys); 835 836 new->async_tx.cookie = 0; 837 async_tx_ack(&new->async_tx); 838 839 prev = new; 840 len -= period_len; 841 842 if (chan->dir == DMA_MEM_TO_DEV) 843 dma_src += period_len; 844 else 845 dma_dst += period_len; 846 847 /* Insert the link descriptor to the LD ring */ 848 list_add_tail(&new->node, &first->tx_list); 849 } while (len); 850 851 first->async_tx.flags = flags; /* client is in control of this ack */ 852 first->async_tx.cookie = -EBUSY; 853 854 /* make the cyclic link */ 855 pdev->ops->set_desc_next_addr(&new->desc, first->async_tx.phys); 856 chan->cyclic_first = first; 857 858 return &first->async_tx; 859 860 fail: 861 if (first) 862 mmp_pdma_free_desc_list(chan, &first->tx_list); 863 return NULL; 864 } 865 866 static int mmp_pdma_config_write(struct dma_chan *dchan, 867 struct dma_slave_config *cfg, 868 enum dma_transfer_direction direction) 869 { 870 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 871 u32 maxburst = 0, addr = 0; 872 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; 873 874 if (!dchan) 875 return -EINVAL; 876 877 if (direction == DMA_DEV_TO_MEM) { 878 chan->dcmd = DCMD_INCTRGADDR | DCMD_FLOWSRC; 879 maxburst = cfg->src_maxburst; 880 width = cfg->src_addr_width; 881 addr = cfg->src_addr; 882 } else if (direction == DMA_MEM_TO_DEV) { 883 chan->dcmd = DCMD_INCSRCADDR | DCMD_FLOWTRG; 884 maxburst = cfg->dst_maxburst; 885 width = cfg->dst_addr_width; 886 addr = cfg->dst_addr; 887 } 888 889 if (width == DMA_SLAVE_BUSWIDTH_1_BYTE) 890 chan->dcmd |= DCMD_WIDTH1; 891 else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES) 892 chan->dcmd |= DCMD_WIDTH2; 893 else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES) 894 chan->dcmd |= DCMD_WIDTH4; 895 896 if (maxburst == 8) 897 chan->dcmd |= DCMD_BURST8; 898 else if (maxburst == 16) 899 chan->dcmd |= DCMD_BURST16; 900 else if (maxburst == 32) 901 chan->dcmd |= DCMD_BURST32; 902 903 chan->dir = direction; 904 chan->dev_addr = addr; 905 906 return 0; 907 } 908 909 static int mmp_pdma_config(struct dma_chan *dchan, 910 struct dma_slave_config *cfg) 911 { 912 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 913 914 memcpy(&chan->slave_config, cfg, sizeof(*cfg)); 915 return 0; 916 } 917 918 static int mmp_pdma_terminate_all(struct dma_chan *dchan) 919 { 920 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 921 unsigned long flags; 922 923 if (!dchan) 924 return -EINVAL; 925 926 disable_chan(chan->phy); 927 mmp_pdma_free_phy(chan); 928 spin_lock_irqsave(&chan->desc_lock, flags); 929 mmp_pdma_free_desc_list(chan, &chan->chain_pending); 930 mmp_pdma_free_desc_list(chan, &chan->chain_running); 931 spin_unlock_irqrestore(&chan->desc_lock, flags); 932 chan->idle = true; 933 934 return 0; 935 } 936 937 static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, 938 dma_cookie_t cookie) 939 { 940 struct mmp_pdma_desc_sw *sw; 941 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(chan->chan.device); 942 unsigned long flags; 943 u64 curr; 944 u32 residue = 0; 945 bool passed = false; 946 bool cyclic = chan->cyclic_first != NULL; 947 948 /* 949 * If the channel does not have a phy pointer anymore, it has already 950 * been completed. Therefore, its residue is 0. 951 */ 952 if (!chan->phy) 953 return 0; 954 955 if (chan->dir == DMA_DEV_TO_MEM) 956 curr = pdev->ops->read_dst_addr(chan->phy); 957 else 958 curr = pdev->ops->read_src_addr(chan->phy); 959 960 spin_lock_irqsave(&chan->desc_lock, flags); 961 962 list_for_each_entry(sw, &chan->chain_running, node) { 963 u64 start, end; 964 u32 len; 965 966 if (chan->dir == DMA_DEV_TO_MEM) 967 start = pdev->ops->get_desc_dst_addr(&sw->desc); 968 else 969 start = pdev->ops->get_desc_src_addr(&sw->desc); 970 971 len = sw->desc.dcmd & DCMD_LENGTH; 972 end = start + len; 973 974 /* 975 * 'passed' will be latched once we found the descriptor which 976 * lies inside the boundaries of the curr pointer. All 977 * descriptors that occur in the list _after_ we found that 978 * partially handled descriptor are still to be processed and 979 * are hence added to the residual bytes counter. 980 */ 981 982 if (passed) { 983 residue += len; 984 } else if (curr >= start && curr <= end) { 985 residue += (u32)(end - curr); 986 passed = true; 987 } 988 989 /* 990 * Descriptors that have the ENDIRQEN bit set mark the end of a 991 * transaction chain, and the cookie assigned with it has been 992 * returned previously from mmp_pdma_tx_submit(). 993 * 994 * In case we have multiple transactions in the running chain, 995 * and the cookie does not match the one the user asked us 996 * about, reset the state variables and start over. 997 * 998 * This logic does not apply to cyclic transactions, where all 999 * descriptors have the ENDIRQEN bit set, and for which we 1000 * can't have multiple transactions on one channel anyway. 1001 */ 1002 if (cyclic || !(sw->desc.dcmd & DCMD_ENDIRQEN)) 1003 continue; 1004 1005 if (sw->async_tx.cookie == cookie) { 1006 spin_unlock_irqrestore(&chan->desc_lock, flags); 1007 return residue; 1008 } else { 1009 residue = 0; 1010 passed = false; 1011 } 1012 } 1013 1014 spin_unlock_irqrestore(&chan->desc_lock, flags); 1015 1016 /* We should only get here in case of cyclic transactions */ 1017 return residue; 1018 } 1019 1020 static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan, 1021 dma_cookie_t cookie, 1022 struct dma_tx_state *txstate) 1023 { 1024 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 1025 enum dma_status ret; 1026 1027 ret = dma_cookie_status(dchan, cookie, txstate); 1028 if (likely(ret != DMA_ERROR)) 1029 dma_set_residue(txstate, mmp_pdma_residue(chan, cookie)); 1030 1031 return ret; 1032 } 1033 1034 /* 1035 * mmp_pdma_issue_pending - Issue the DMA start command 1036 * pending list ==> running list 1037 */ 1038 static void mmp_pdma_issue_pending(struct dma_chan *dchan) 1039 { 1040 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); 1041 unsigned long flags; 1042 1043 spin_lock_irqsave(&chan->desc_lock, flags); 1044 start_pending_queue(chan); 1045 spin_unlock_irqrestore(&chan->desc_lock, flags); 1046 } 1047 1048 /* 1049 * dma_do_tasklet 1050 * Do call back 1051 * Start pending list 1052 */ 1053 static void dma_do_tasklet(struct tasklet_struct *t) 1054 { 1055 struct mmp_pdma_chan *chan = from_tasklet(chan, t, tasklet); 1056 struct mmp_pdma_desc_sw *desc, *_desc; 1057 LIST_HEAD(chain_cleanup); 1058 unsigned long flags; 1059 struct dmaengine_desc_callback cb; 1060 1061 if (chan->cyclic_first) { 1062 spin_lock_irqsave(&chan->desc_lock, flags); 1063 desc = chan->cyclic_first; 1064 dmaengine_desc_get_callback(&desc->async_tx, &cb); 1065 spin_unlock_irqrestore(&chan->desc_lock, flags); 1066 1067 dmaengine_desc_callback_invoke(&cb, NULL); 1068 1069 return; 1070 } 1071 1072 /* submit pending list; callback for each desc; free desc */ 1073 spin_lock_irqsave(&chan->desc_lock, flags); 1074 1075 list_for_each_entry_safe(desc, _desc, &chan->chain_running, node) { 1076 /* 1077 * move the descriptors to a temporary list so we can drop 1078 * the lock during the entire cleanup operation 1079 */ 1080 list_move(&desc->node, &chain_cleanup); 1081 1082 /* 1083 * Look for the first list entry which has the ENDIRQEN flag 1084 * set. That is the descriptor we got an interrupt for, so 1085 * complete that transaction and its cookie. 1086 */ 1087 if (desc->desc.dcmd & DCMD_ENDIRQEN) { 1088 dma_cookie_t cookie = desc->async_tx.cookie; 1089 dma_cookie_complete(&desc->async_tx); 1090 dev_dbg(chan->dev, "completed_cookie=%d\n", cookie); 1091 break; 1092 } 1093 } 1094 1095 /* 1096 * The hardware is idle and ready for more when the 1097 * chain_running list is empty. 1098 */ 1099 chan->idle = list_empty(&chan->chain_running); 1100 1101 /* Start any pending transactions automatically */ 1102 start_pending_queue(chan); 1103 spin_unlock_irqrestore(&chan->desc_lock, flags); 1104 1105 /* Run the callback for each descriptor, in order */ 1106 list_for_each_entry_safe(desc, _desc, &chain_cleanup, node) { 1107 struct dma_async_tx_descriptor *txd = &desc->async_tx; 1108 1109 /* Remove from the list of transactions */ 1110 list_del(&desc->node); 1111 /* Run the link descriptor callback function */ 1112 dmaengine_desc_get_callback(txd, &cb); 1113 dmaengine_desc_callback_invoke(&cb, NULL); 1114 1115 dma_pool_free(chan->desc_pool, desc, txd->phys); 1116 } 1117 } 1118 1119 static void mmp_pdma_remove(struct platform_device *op) 1120 { 1121 struct mmp_pdma_device *pdev = platform_get_drvdata(op); 1122 struct mmp_pdma_phy *phy; 1123 int i, irq = 0, irq_num = 0; 1124 1125 if (op->dev.of_node) 1126 of_dma_controller_free(op->dev.of_node); 1127 1128 for (i = 0; i < pdev->dma_channels; i++) { 1129 if (platform_get_irq(op, i) > 0) 1130 irq_num++; 1131 } 1132 1133 if (irq_num != pdev->dma_channels) { 1134 irq = platform_get_irq(op, 0); 1135 devm_free_irq(&op->dev, irq, pdev); 1136 } else { 1137 for (i = 0; i < pdev->dma_channels; i++) { 1138 phy = &pdev->phy[i]; 1139 irq = platform_get_irq(op, i); 1140 devm_free_irq(&op->dev, irq, phy); 1141 } 1142 } 1143 1144 dma_async_device_unregister(&pdev->device); 1145 } 1146 1147 static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) 1148 { 1149 struct mmp_pdma_phy *phy = &pdev->phy[idx]; 1150 struct mmp_pdma_chan *chan; 1151 int ret; 1152 1153 chan = devm_kzalloc(pdev->dev, sizeof(*chan), GFP_KERNEL); 1154 if (chan == NULL) 1155 return -ENOMEM; 1156 1157 phy->idx = idx; 1158 phy->base = pdev->base; 1159 1160 if (irq) { 1161 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_chan_handler, 1162 IRQF_SHARED, "pdma", phy); 1163 if (ret) { 1164 dev_err(pdev->dev, "channel request irq fail!\n"); 1165 return ret; 1166 } 1167 } 1168 1169 spin_lock_init(&chan->desc_lock); 1170 chan->dev = pdev->dev; 1171 chan->chan.device = &pdev->device; 1172 tasklet_setup(&chan->tasklet, dma_do_tasklet); 1173 INIT_LIST_HEAD(&chan->chain_pending); 1174 INIT_LIST_HEAD(&chan->chain_running); 1175 1176 /* register virt channel to dma engine */ 1177 list_add_tail(&chan->chan.device_node, &pdev->device.channels); 1178 1179 return 0; 1180 } 1181 1182 static const struct mmp_pdma_ops marvell_pdma_v1_ops = { 1183 .write_next_addr = write_next_addr_32, 1184 .read_src_addr = read_src_addr_32, 1185 .read_dst_addr = read_dst_addr_32, 1186 .set_desc_next_addr = set_desc_next_addr_32, 1187 .set_desc_src_addr = set_desc_src_addr_32, 1188 .set_desc_dst_addr = set_desc_dst_addr_32, 1189 .get_desc_src_addr = get_desc_src_addr_32, 1190 .get_desc_dst_addr = get_desc_dst_addr_32, 1191 .run_bits = (DCSR_RUN), 1192 .dma_width = 32, 1193 .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT, 1194 }; 1195 1196 static const struct mmp_pdma_ops spacemit_k1_pdma_ops = { 1197 .write_next_addr = write_next_addr_64, 1198 .read_src_addr = read_src_addr_64, 1199 .read_dst_addr = read_dst_addr_64, 1200 .set_desc_next_addr = set_desc_next_addr_64, 1201 .set_desc_src_addr = set_desc_src_addr_64, 1202 .set_desc_dst_addr = set_desc_dst_addr_64, 1203 .get_desc_src_addr = get_desc_src_addr_64, 1204 .get_desc_dst_addr = get_desc_dst_addr_64, 1205 .run_bits = (DCSR_RUN | DCSR_LPAEEN), 1206 .dma_width = 64, 1207 .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT, 1208 }; 1209 1210 static const struct mmp_pdma_ops spacemit_k3_pdma_ops = { 1211 .write_next_addr = write_next_addr_64, 1212 .read_src_addr = read_src_addr_64, 1213 .read_dst_addr = read_dst_addr_64, 1214 .set_desc_next_addr = set_desc_next_addr_64, 1215 .set_desc_src_addr = set_desc_src_addr_64, 1216 .set_desc_dst_addr = set_desc_dst_addr_64, 1217 .get_desc_src_addr = get_desc_src_addr_64, 1218 .get_desc_dst_addr = get_desc_dst_addr_64, 1219 .run_bits = (DCSR_RUN | DCSR_LPAEEN | DCSR_EORIRQEN | DCSR_EORSTOPEN), 1220 .dma_width = 64, 1221 .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT, 1222 }; 1223 1224 static const struct of_device_id mmp_pdma_dt_ids[] = { 1225 { 1226 .compatible = "marvell,pdma-1.0", 1227 .data = &marvell_pdma_v1_ops 1228 }, { 1229 .compatible = "spacemit,k1-pdma", 1230 .data = &spacemit_k1_pdma_ops 1231 }, { 1232 .compatible = "spacemit,k3-pdma", 1233 .data = &spacemit_k3_pdma_ops 1234 }, { 1235 /* sentinel */ 1236 } 1237 }; 1238 MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids); 1239 1240 static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec, 1241 struct of_dma *ofdma) 1242 { 1243 struct mmp_pdma_device *d = ofdma->of_dma_data; 1244 struct dma_chan *chan; 1245 1246 chan = dma_get_any_slave_channel(&d->device); 1247 if (!chan) 1248 return NULL; 1249 1250 to_mmp_pdma_chan(chan)->drcmr = dma_spec->args[0]; 1251 1252 return chan; 1253 } 1254 1255 static int mmp_pdma_probe(struct platform_device *op) 1256 { 1257 struct mmp_pdma_device *pdev; 1258 struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev); 1259 struct clk *clk; 1260 struct reset_control *rst; 1261 int i, ret, irq = 0; 1262 int dma_channels = 0, irq_num = 0; 1263 const enum dma_slave_buswidth widths = 1264 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES | 1265 DMA_SLAVE_BUSWIDTH_4_BYTES; 1266 1267 pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL); 1268 if (!pdev) 1269 return -ENOMEM; 1270 1271 pdev->dev = &op->dev; 1272 1273 spin_lock_init(&pdev->phy_lock); 1274 1275 pdev->base = devm_platform_ioremap_resource(op, 0); 1276 if (IS_ERR(pdev->base)) 1277 return PTR_ERR(pdev->base); 1278 1279 clk = devm_clk_get_optional_enabled(pdev->dev, NULL); 1280 if (IS_ERR(clk)) 1281 return PTR_ERR(clk); 1282 1283 rst = devm_reset_control_get_optional_exclusive_deasserted(pdev->dev, 1284 NULL); 1285 if (IS_ERR(rst)) 1286 return PTR_ERR(rst); 1287 1288 pdev->ops = of_device_get_match_data(&op->dev); 1289 if (!pdev->ops) 1290 return -ENODEV; 1291 1292 if (pdev->dev->of_node) { 1293 /* Parse new and deprecated dma-channels properties */ 1294 if (of_property_read_u32(pdev->dev->of_node, "dma-channels", 1295 &dma_channels)) 1296 of_property_read_u32(pdev->dev->of_node, "#dma-channels", 1297 &dma_channels); 1298 } else if (pdata && pdata->dma_channels) { 1299 dma_channels = pdata->dma_channels; 1300 } else { 1301 dma_channels = 32; /* default 32 channel */ 1302 } 1303 pdev->dma_channels = dma_channels; 1304 1305 for (i = 0; i < dma_channels; i++) { 1306 if (platform_get_irq_optional(op, i) > 0) 1307 irq_num++; 1308 } 1309 1310 pdev->phy = devm_kcalloc(pdev->dev, dma_channels, sizeof(*pdev->phy), 1311 GFP_KERNEL); 1312 if (pdev->phy == NULL) 1313 return -ENOMEM; 1314 1315 INIT_LIST_HEAD(&pdev->device.channels); 1316 1317 if (irq_num != dma_channels) { 1318 /* all chan share one irq, demux inside */ 1319 irq = platform_get_irq(op, 0); 1320 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_int_handler, 1321 IRQF_SHARED, "pdma", pdev); 1322 if (ret) 1323 return ret; 1324 } 1325 1326 for (i = 0; i < dma_channels; i++) { 1327 irq = (irq_num != dma_channels) ? 0 : platform_get_irq(op, i); 1328 ret = mmp_pdma_chan_init(pdev, i, irq); 1329 if (ret) 1330 return ret; 1331 } 1332 1333 dma_cap_set(DMA_SLAVE, pdev->device.cap_mask); 1334 dma_cap_set(DMA_MEMCPY, pdev->device.cap_mask); 1335 dma_cap_set(DMA_CYCLIC, pdev->device.cap_mask); 1336 dma_cap_set(DMA_PRIVATE, pdev->device.cap_mask); 1337 pdev->device.dev = &op->dev; 1338 pdev->device.device_alloc_chan_resources = mmp_pdma_alloc_chan_resources; 1339 pdev->device.device_free_chan_resources = mmp_pdma_free_chan_resources; 1340 pdev->device.device_tx_status = mmp_pdma_tx_status; 1341 pdev->device.device_prep_dma_memcpy = mmp_pdma_prep_memcpy; 1342 pdev->device.device_prep_slave_sg = mmp_pdma_prep_slave_sg; 1343 pdev->device.device_prep_dma_cyclic = mmp_pdma_prep_dma_cyclic; 1344 pdev->device.device_issue_pending = mmp_pdma_issue_pending; 1345 pdev->device.device_config = mmp_pdma_config; 1346 pdev->device.device_terminate_all = mmp_pdma_terminate_all; 1347 pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES; 1348 pdev->device.src_addr_widths = widths; 1349 pdev->device.dst_addr_widths = widths; 1350 pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); 1351 pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1352 1353 /* Set DMA mask based on controller hardware capabilities */ 1354 dma_set_mask_and_coherent(pdev->dev, 1355 DMA_BIT_MASK(pdev->ops->dma_width)); 1356 1357 ret = dma_async_device_register(&pdev->device); 1358 if (ret) { 1359 dev_err(pdev->device.dev, "unable to register\n"); 1360 return ret; 1361 } 1362 1363 if (op->dev.of_node) { 1364 /* Device-tree DMA controller registration */ 1365 ret = of_dma_controller_register(op->dev.of_node, 1366 mmp_pdma_dma_xlate, pdev); 1367 if (ret < 0) { 1368 dev_err(&op->dev, "of_dma_controller_register failed\n"); 1369 dma_async_device_unregister(&pdev->device); 1370 return ret; 1371 } 1372 } 1373 1374 platform_set_drvdata(op, pdev); 1375 dev_info(pdev->device.dev, "initialized %d channels\n", dma_channels); 1376 return 0; 1377 } 1378 1379 static const struct platform_device_id mmp_pdma_id_table[] = { 1380 { "mmp-pdma", }, 1381 { }, 1382 }; 1383 1384 static struct platform_driver mmp_pdma_driver = { 1385 .driver = { 1386 .name = "mmp-pdma", 1387 .of_match_table = mmp_pdma_dt_ids, 1388 }, 1389 .id_table = mmp_pdma_id_table, 1390 .probe = mmp_pdma_probe, 1391 .remove = mmp_pdma_remove, 1392 }; 1393 1394 module_platform_driver(mmp_pdma_driver); 1395 1396 MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver"); 1397 MODULE_AUTHOR("Marvell International Ltd."); 1398 MODULE_LICENSE("GPL v2"); 1399