1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * DMA driver for Altera mSGDMA IP core 4 * 5 * Copyright (C) 2017 Stefan Roese <sr@denx.de> 6 * 7 * Based on drivers/dma/xilinx/zynqmp_dma.c, which is: 8 * Copyright (C) 2016 Xilinx, Inc. All rights reserved. 9 */ 10 11 #include <linux/bitops.h> 12 #include <linux/delay.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/dmapool.h> 15 #include <linux/init.h> 16 #include <linux/interrupt.h> 17 #include <linux/io.h> 18 #include <linux/iopoll.h> 19 #include <linux/module.h> 20 #include <linux/platform_device.h> 21 #include <linux/slab.h> 22 #include <linux/of_dma.h> 23 24 #include "dmaengine.h" 25 26 #define MSGDMA_MAX_TRANS_LEN U32_MAX 27 #define MSGDMA_DESC_NUM 1024 28 29 /** 30 * struct msgdma_extended_desc - implements an extended descriptor 31 * @read_addr_lo: data buffer source address low bits 32 * @write_addr_lo: data buffer destination address low bits 33 * @len: the number of bytes to transfer per descriptor 34 * @burst_seq_num: bit 31:24 write burst 35 * bit 23:16 read burst 36 * bit 15:00 sequence number 37 * @stride: bit 31:16 write stride 38 * bit 15:00 read stride 39 * @read_addr_hi: data buffer source address high bits 40 * @write_addr_hi: data buffer destination address high bits 41 * @control: characteristics of the transfer 42 */ 43 struct msgdma_extended_desc { 44 u32 read_addr_lo; 45 u32 write_addr_lo; 46 u32 len; 47 u32 burst_seq_num; 48 u32 stride; 49 u32 read_addr_hi; 50 u32 write_addr_hi; 51 u32 control; 52 }; 53 54 /* mSGDMA descriptor control field bit definitions */ 55 #define MSGDMA_DESC_CTL_SET_CH(x) ((x) & 0xff) 56 #define MSGDMA_DESC_CTL_GEN_SOP BIT(8) 57 #define MSGDMA_DESC_CTL_GEN_EOP BIT(9) 58 #define MSGDMA_DESC_CTL_PARK_READS BIT(10) 59 #define MSGDMA_DESC_CTL_PARK_WRITES BIT(11) 60 #define MSGDMA_DESC_CTL_END_ON_EOP BIT(12) 61 #define MSGDMA_DESC_CTL_END_ON_LEN BIT(13) 62 #define MSGDMA_DESC_CTL_TR_COMP_IRQ BIT(14) 63 #define MSGDMA_DESC_CTL_EARLY_IRQ BIT(15) 64 #define MSGDMA_DESC_CTL_TR_ERR_IRQ GENMASK(23, 16) 65 #define MSGDMA_DESC_CTL_EARLY_DONE BIT(24) 66 67 /* 68 * Writing "1" the "go" bit commits the entire descriptor into the 69 * descriptor FIFO(s) 70 */ 71 #define MSGDMA_DESC_CTL_GO BIT(31) 72 73 /* Tx buffer control flags */ 74 #define MSGDMA_DESC_CTL_TX_FIRST (MSGDMA_DESC_CTL_GEN_SOP | \ 75 MSGDMA_DESC_CTL_TR_ERR_IRQ | \ 76 MSGDMA_DESC_CTL_GO) 77 78 #define MSGDMA_DESC_CTL_TX_MIDDLE (MSGDMA_DESC_CTL_TR_ERR_IRQ | \ 79 MSGDMA_DESC_CTL_GO) 80 81 #define MSGDMA_DESC_CTL_TX_LAST (MSGDMA_DESC_CTL_GEN_EOP | \ 82 MSGDMA_DESC_CTL_TR_COMP_IRQ | \ 83 MSGDMA_DESC_CTL_TR_ERR_IRQ | \ 84 MSGDMA_DESC_CTL_GO) 85 86 #define MSGDMA_DESC_CTL_TX_SINGLE (MSGDMA_DESC_CTL_GEN_SOP | \ 87 MSGDMA_DESC_CTL_GEN_EOP | \ 88 MSGDMA_DESC_CTL_TR_COMP_IRQ | \ 89 MSGDMA_DESC_CTL_TR_ERR_IRQ | \ 90 MSGDMA_DESC_CTL_GO) 91 92 #define MSGDMA_DESC_CTL_RX_SINGLE (MSGDMA_DESC_CTL_END_ON_EOP | \ 93 MSGDMA_DESC_CTL_END_ON_LEN | \ 94 MSGDMA_DESC_CTL_TR_COMP_IRQ | \ 95 MSGDMA_DESC_CTL_EARLY_IRQ | \ 96 MSGDMA_DESC_CTL_TR_ERR_IRQ | \ 97 MSGDMA_DESC_CTL_GO) 98 99 /* mSGDMA extended descriptor stride definitions */ 100 #define MSGDMA_DESC_STRIDE_RD 0x00000001 101 #define MSGDMA_DESC_STRIDE_WR 0x00010000 102 #define MSGDMA_DESC_STRIDE_RW 0x00010001 103 104 /* mSGDMA dispatcher control and status register map */ 105 #define MSGDMA_CSR_STATUS 0x00 /* Read / Clear */ 106 #define MSGDMA_CSR_CONTROL 0x04 /* Read / Write */ 107 #define MSGDMA_CSR_RW_FILL_LEVEL 0x08 /* 31:16 - write fill level */ 108 /* 15:00 - read fill level */ 109 #define MSGDMA_CSR_RESP_FILL_LEVEL 0x0c /* response FIFO fill level */ 110 #define MSGDMA_CSR_RW_SEQ_NUM 0x10 /* 31:16 - write seq number */ 111 /* 15:00 - read seq number */ 112 113 /* mSGDMA CSR status register bit definitions */ 114 #define MSGDMA_CSR_STAT_BUSY BIT(0) 115 #define MSGDMA_CSR_STAT_DESC_BUF_EMPTY BIT(1) 116 #define MSGDMA_CSR_STAT_DESC_BUF_FULL BIT(2) 117 #define MSGDMA_CSR_STAT_RESP_BUF_EMPTY BIT(3) 118 #define MSGDMA_CSR_STAT_RESP_BUF_FULL BIT(4) 119 #define MSGDMA_CSR_STAT_STOPPED BIT(5) 120 #define MSGDMA_CSR_STAT_RESETTING BIT(6) 121 #define MSGDMA_CSR_STAT_STOPPED_ON_ERR BIT(7) 122 #define MSGDMA_CSR_STAT_STOPPED_ON_EARLY BIT(8) 123 #define MSGDMA_CSR_STAT_IRQ BIT(9) 124 #define MSGDMA_CSR_STAT_MASK GENMASK(9, 0) 125 #define MSGDMA_CSR_STAT_MASK_WITHOUT_IRQ GENMASK(8, 0) 126 127 #define DESC_EMPTY (MSGDMA_CSR_STAT_DESC_BUF_EMPTY | \ 128 MSGDMA_CSR_STAT_RESP_BUF_EMPTY) 129 130 /* mSGDMA CSR control register bit definitions */ 131 #define MSGDMA_CSR_CTL_STOP BIT(0) 132 #define MSGDMA_CSR_CTL_RESET BIT(1) 133 #define MSGDMA_CSR_CTL_STOP_ON_ERR BIT(2) 134 #define MSGDMA_CSR_CTL_STOP_ON_EARLY BIT(3) 135 #define MSGDMA_CSR_CTL_GLOBAL_INTR BIT(4) 136 #define MSGDMA_CSR_CTL_STOP_DESCS BIT(5) 137 138 /* mSGDMA CSR fill level bits */ 139 #define MSGDMA_CSR_WR_FILL_LEVEL_GET(v) (((v) & 0xffff0000) >> 16) 140 #define MSGDMA_CSR_RD_FILL_LEVEL_GET(v) ((v) & 0x0000ffff) 141 #define MSGDMA_CSR_RESP_FILL_LEVEL_GET(v) ((v) & 0x0000ffff) 142 143 #define MSGDMA_CSR_SEQ_NUM_GET(v) (((v) & 0xffff0000) >> 16) 144 145 /* mSGDMA response register map */ 146 #define MSGDMA_RESP_BYTES_TRANSFERRED 0x00 147 #define MSGDMA_RESP_STATUS 0x04 148 149 /* mSGDMA response register bit definitions */ 150 #define MSGDMA_RESP_EARLY_TERM BIT(8) 151 #define MSGDMA_RESP_ERR_MASK 0xff 152 153 /** 154 * struct msgdma_sw_desc - implements a sw descriptor 155 * @async_tx: support for the async_tx api 156 * @hw_desc: associated HW descriptor 157 * @node: node to move from the free list to the tx list 158 * @tx_list: transmit list node 159 */ 160 struct msgdma_sw_desc { 161 struct dma_async_tx_descriptor async_tx; 162 struct msgdma_extended_desc hw_desc; 163 struct list_head node; 164 struct list_head tx_list; 165 }; 166 167 /* 168 * struct msgdma_device - DMA device structure 169 */ 170 struct msgdma_device { 171 spinlock_t lock; 172 struct device *dev; 173 struct tasklet_struct irq_tasklet; 174 struct list_head pending_list; 175 struct list_head free_list; 176 struct list_head active_list; 177 struct list_head done_list; 178 u32 desc_free_cnt; 179 bool idle; 180 181 struct dma_device dmadev; 182 struct dma_chan dmachan; 183 dma_addr_t hw_desq; 184 struct msgdma_sw_desc *sw_desq; 185 unsigned int npendings; 186 187 struct dma_slave_config slave_cfg; 188 189 int irq; 190 191 /* mSGDMA controller */ 192 void __iomem *csr; 193 194 /* mSGDMA descriptors */ 195 void __iomem *desc; 196 197 /* mSGDMA response */ 198 void __iomem *resp; 199 }; 200 201 #define to_mdev(chan) container_of(chan, struct msgdma_device, dmachan) 202 #define tx_to_desc(tx) container_of(tx, struct msgdma_sw_desc, async_tx) 203 204 /** 205 * msgdma_get_descriptor - Get the sw descriptor from the pool 206 * @mdev: Pointer to the Altera mSGDMA device structure 207 * 208 * Return: The sw descriptor 209 */ 210 static struct msgdma_sw_desc *msgdma_get_descriptor(struct msgdma_device *mdev) 211 { 212 struct msgdma_sw_desc *desc; 213 unsigned long flags; 214 215 spin_lock_irqsave(&mdev->lock, flags); 216 desc = list_first_entry(&mdev->free_list, struct msgdma_sw_desc, node); 217 list_del(&desc->node); 218 spin_unlock_irqrestore(&mdev->lock, flags); 219 220 INIT_LIST_HEAD(&desc->tx_list); 221 222 return desc; 223 } 224 225 /** 226 * msgdma_free_descriptor - Issue pending transactions 227 * @mdev: Pointer to the Altera mSGDMA device structure 228 * @desc: Transaction descriptor pointer 229 */ 230 static void msgdma_free_descriptor(struct msgdma_device *mdev, 231 struct msgdma_sw_desc *desc) 232 { 233 struct msgdma_sw_desc *child, *next; 234 235 mdev->desc_free_cnt++; 236 list_move_tail(&desc->node, &mdev->free_list); 237 list_for_each_entry_safe(child, next, &desc->tx_list, node) { 238 mdev->desc_free_cnt++; 239 list_move_tail(&child->node, &mdev->free_list); 240 } 241 } 242 243 /** 244 * msgdma_free_desc_list - Free descriptors list 245 * @mdev: Pointer to the Altera mSGDMA device structure 246 * @list: List to parse and delete the descriptor 247 */ 248 static void msgdma_free_desc_list(struct msgdma_device *mdev, 249 struct list_head *list) 250 { 251 struct msgdma_sw_desc *desc, *next; 252 253 list_for_each_entry_safe(desc, next, list, node) 254 msgdma_free_descriptor(mdev, desc); 255 } 256 257 /** 258 * msgdma_desc_config - Configure the descriptor 259 * @desc: Hw descriptor pointer 260 * @dst: Destination buffer address 261 * @src: Source buffer address 262 * @len: Transfer length 263 * @stride: Read/write stride value to set 264 */ 265 static void msgdma_desc_config(struct msgdma_extended_desc *desc, 266 dma_addr_t dst, dma_addr_t src, size_t len, 267 u32 stride) 268 { 269 /* Set lower 32bits of src & dst addresses in the descriptor */ 270 desc->read_addr_lo = lower_32_bits(src); 271 desc->write_addr_lo = lower_32_bits(dst); 272 273 /* Set upper 32bits of src & dst addresses in the descriptor */ 274 desc->read_addr_hi = upper_32_bits(src); 275 desc->write_addr_hi = upper_32_bits(dst); 276 277 desc->len = len; 278 desc->stride = stride; 279 desc->burst_seq_num = 0; /* 0 will result in max burst length */ 280 281 /* 282 * Don't set interrupt on xfer end yet, this will be done later 283 * for the "last" descriptor 284 */ 285 desc->control = MSGDMA_DESC_CTL_TR_ERR_IRQ | MSGDMA_DESC_CTL_GO | 286 MSGDMA_DESC_CTL_END_ON_LEN; 287 } 288 289 /** 290 * msgdma_desc_config_eod - Mark the descriptor as end descriptor 291 * @desc: Hw descriptor pointer 292 */ 293 static void msgdma_desc_config_eod(struct msgdma_extended_desc *desc) 294 { 295 desc->control |= MSGDMA_DESC_CTL_TR_COMP_IRQ; 296 } 297 298 /** 299 * msgdma_tx_submit - Submit DMA transaction 300 * @tx: Async transaction descriptor pointer 301 * 302 * Return: cookie value 303 */ 304 static dma_cookie_t msgdma_tx_submit(struct dma_async_tx_descriptor *tx) 305 { 306 struct msgdma_device *mdev = to_mdev(tx->chan); 307 struct msgdma_sw_desc *new; 308 dma_cookie_t cookie; 309 unsigned long flags; 310 311 new = tx_to_desc(tx); 312 spin_lock_irqsave(&mdev->lock, flags); 313 cookie = dma_cookie_assign(tx); 314 315 list_add_tail(&new->node, &mdev->pending_list); 316 spin_unlock_irqrestore(&mdev->lock, flags); 317 318 return cookie; 319 } 320 321 /** 322 * msgdma_prep_memcpy - prepare descriptors for memcpy transaction 323 * @dchan: DMA channel 324 * @dma_dst: Destination buffer address 325 * @dma_src: Source buffer address 326 * @len: Transfer length 327 * @flags: transfer ack flags 328 * 329 * Return: Async transaction descriptor on success and NULL on failure 330 */ 331 static struct dma_async_tx_descriptor * 332 msgdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst, 333 dma_addr_t dma_src, size_t len, ulong flags) 334 { 335 struct msgdma_device *mdev = to_mdev(dchan); 336 struct msgdma_sw_desc *new, *first = NULL; 337 struct msgdma_extended_desc *desc; 338 size_t copy; 339 u32 desc_cnt; 340 unsigned long irqflags; 341 342 desc_cnt = DIV_ROUND_UP(len, MSGDMA_MAX_TRANS_LEN); 343 344 spin_lock_irqsave(&mdev->lock, irqflags); 345 if (desc_cnt > mdev->desc_free_cnt) { 346 spin_unlock_irqrestore(&mdev->lock, irqflags); 347 dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev); 348 return NULL; 349 } 350 mdev->desc_free_cnt -= desc_cnt; 351 spin_unlock_irqrestore(&mdev->lock, irqflags); 352 353 do { 354 /* Allocate and populate the descriptor */ 355 new = msgdma_get_descriptor(mdev); 356 357 copy = min_t(size_t, len, MSGDMA_MAX_TRANS_LEN); 358 desc = &new->hw_desc; 359 msgdma_desc_config(desc, dma_dst, dma_src, copy, 360 MSGDMA_DESC_STRIDE_RW); 361 len -= copy; 362 dma_src += copy; 363 dma_dst += copy; 364 if (!first) 365 first = new; 366 else 367 list_add_tail(&new->node, &first->tx_list); 368 } while (len); 369 370 msgdma_desc_config_eod(desc); 371 async_tx_ack(&first->async_tx); 372 first->async_tx.flags = flags; 373 374 return &first->async_tx; 375 } 376 377 /** 378 * msgdma_prep_slave_sg - prepare descriptors for a slave sg transaction 379 * 380 * @dchan: DMA channel 381 * @sgl: Destination scatter list 382 * @sg_len: Number of entries in destination scatter list 383 * @dir: DMA transfer direction 384 * @flags: transfer ack flags 385 * @context: transfer context (unused) 386 */ 387 static struct dma_async_tx_descriptor * 388 msgdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, 389 unsigned int sg_len, enum dma_transfer_direction dir, 390 unsigned long flags, void *context) 391 392 { 393 struct msgdma_device *mdev = to_mdev(dchan); 394 struct dma_slave_config *cfg = &mdev->slave_cfg; 395 struct msgdma_sw_desc *new, *first = NULL; 396 void *desc = NULL; 397 size_t len, avail; 398 dma_addr_t dma_dst, dma_src; 399 u32 desc_cnt; 400 u32 stride; 401 unsigned long irqflags; 402 403 desc_cnt = sg_nents_for_dma(sgl, sg_len, MSGDMA_MAX_TRANS_LEN); 404 405 spin_lock_irqsave(&mdev->lock, irqflags); 406 if (desc_cnt > mdev->desc_free_cnt) { 407 spin_unlock_irqrestore(&mdev->lock, irqflags); 408 dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev); 409 return NULL; 410 } 411 mdev->desc_free_cnt -= desc_cnt; 412 spin_unlock_irqrestore(&mdev->lock, irqflags); 413 414 avail = sg_dma_len(sgl); 415 416 /* Run until we are out of scatterlist entries */ 417 while (true) { 418 /* Allocate and populate the descriptor */ 419 new = msgdma_get_descriptor(mdev); 420 421 desc = &new->hw_desc; 422 len = min_t(size_t, avail, MSGDMA_MAX_TRANS_LEN); 423 424 if (dir == DMA_MEM_TO_DEV) { 425 dma_src = sg_dma_address(sgl) + sg_dma_len(sgl) - avail; 426 dma_dst = cfg->dst_addr; 427 stride = MSGDMA_DESC_STRIDE_RD; 428 } else { 429 dma_src = cfg->src_addr; 430 dma_dst = sg_dma_address(sgl) + sg_dma_len(sgl) - avail; 431 stride = MSGDMA_DESC_STRIDE_WR; 432 } 433 msgdma_desc_config(desc, dma_dst, dma_src, len, stride); 434 avail -= len; 435 436 if (!first) 437 first = new; 438 else 439 list_add_tail(&new->node, &first->tx_list); 440 441 /* Fetch the next scatterlist entry */ 442 if (avail == 0) { 443 if (sg_len == 0) 444 break; 445 sgl = sg_next(sgl); 446 if (sgl == NULL) 447 break; 448 sg_len--; 449 avail = sg_dma_len(sgl); 450 } 451 } 452 453 msgdma_desc_config_eod(desc); 454 first->async_tx.flags = flags; 455 456 return &first->async_tx; 457 } 458 459 static int msgdma_dma_config(struct dma_chan *dchan, 460 struct dma_slave_config *config) 461 { 462 struct msgdma_device *mdev = to_mdev(dchan); 463 464 memcpy(&mdev->slave_cfg, config, sizeof(*config)); 465 466 return 0; 467 } 468 469 static void msgdma_reset(struct msgdma_device *mdev) 470 { 471 u32 val; 472 int ret; 473 474 /* Reset mSGDMA */ 475 iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS); 476 iowrite32(MSGDMA_CSR_CTL_RESET, mdev->csr + MSGDMA_CSR_CONTROL); 477 478 ret = readl_poll_timeout(mdev->csr + MSGDMA_CSR_STATUS, val, 479 (val & MSGDMA_CSR_STAT_RESETTING) == 0, 480 1, 10000); 481 if (ret) 482 dev_err(mdev->dev, "DMA channel did not reset\n"); 483 484 /* Clear all status bits */ 485 iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS); 486 487 /* Enable the DMA controller including interrupts */ 488 iowrite32(MSGDMA_CSR_CTL_STOP_ON_ERR | MSGDMA_CSR_CTL_STOP_ON_EARLY | 489 MSGDMA_CSR_CTL_GLOBAL_INTR, mdev->csr + MSGDMA_CSR_CONTROL); 490 491 mdev->idle = true; 492 }; 493 494 static void msgdma_copy_one(struct msgdma_device *mdev, 495 struct msgdma_sw_desc *desc) 496 { 497 void __iomem *hw_desc = mdev->desc; 498 499 /* Ensure control is the last field — required for correct FIFO flush ordering */ 500 static_assert(offsetof(struct msgdma_extended_desc, control) == 501 sizeof(struct msgdma_extended_desc) - sizeof(u32), 502 "control must be the last field in msgdma_extended_desc"); 503 504 /* 505 * Check if the DESC FIFO it not full. If its full, we need to wait 506 * for at least one entry to become free again 507 */ 508 while (ioread32(mdev->csr + MSGDMA_CSR_STATUS) & 509 MSGDMA_CSR_STAT_DESC_BUF_FULL) 510 mdelay(1); 511 512 /* Ensure control is the last field — required for correct FIFO flush ordering */ 513 static_assert(offsetof(struct msgdma_extended_desc, control) == 514 sizeof(struct msgdma_extended_desc) - sizeof(u32), 515 "control must be the last field in msgdma_extended_desc"); 516 517 /* 518 * Copy the descriptor into the descriptor FIFO of the DMA controller, 519 * excluding the control word. The FIFO is flushed and the descriptor 520 * becomes valid once the control word is written last. 521 */ 522 memcpy_toio(hw_desc, &desc->hw_desc, 523 offsetof(struct msgdma_extended_desc, control)); 524 525 /* Write control word last to flush this descriptor into the FIFO */ 526 mdev->idle = false; 527 wmb(); 528 iowrite32(desc->hw_desc.control, hw_desc + 529 offsetof(struct msgdma_extended_desc, control)); 530 wmb(); 531 } 532 533 /** 534 * msgdma_copy_desc_to_fifo - copy descriptor(s) into controller FIFO 535 * @mdev: Pointer to the Altera mSGDMA device structure 536 * @desc: Transaction descriptor pointer 537 */ 538 static void msgdma_copy_desc_to_fifo(struct msgdma_device *mdev, 539 struct msgdma_sw_desc *desc) 540 { 541 struct msgdma_sw_desc *sdesc, *next; 542 543 msgdma_copy_one(mdev, desc); 544 545 list_for_each_entry_safe(sdesc, next, &desc->tx_list, node) 546 msgdma_copy_one(mdev, sdesc); 547 } 548 549 /** 550 * msgdma_start_transfer - Initiate the new transfer 551 * @mdev: Pointer to the Altera mSGDMA device structure 552 */ 553 static void msgdma_start_transfer(struct msgdma_device *mdev) 554 { 555 struct msgdma_sw_desc *desc; 556 557 if (!mdev->idle) 558 return; 559 560 desc = list_first_entry_or_null(&mdev->pending_list, 561 struct msgdma_sw_desc, node); 562 if (!desc) 563 return; 564 565 list_splice_tail_init(&mdev->pending_list, &mdev->active_list); 566 msgdma_copy_desc_to_fifo(mdev, desc); 567 } 568 569 /** 570 * msgdma_issue_pending - Issue pending transactions 571 * @chan: DMA channel pointer 572 */ 573 static void msgdma_issue_pending(struct dma_chan *chan) 574 { 575 struct msgdma_device *mdev = to_mdev(chan); 576 unsigned long flags; 577 578 spin_lock_irqsave(&mdev->lock, flags); 579 msgdma_start_transfer(mdev); 580 spin_unlock_irqrestore(&mdev->lock, flags); 581 } 582 583 /** 584 * msgdma_chan_desc_cleanup - Cleanup the completed descriptors 585 * @mdev: Pointer to the Altera mSGDMA device structure 586 */ 587 static void msgdma_chan_desc_cleanup(struct msgdma_device *mdev) 588 { 589 struct msgdma_sw_desc *desc, *next; 590 unsigned long irqflags; 591 592 spin_lock_irqsave(&mdev->lock, irqflags); 593 594 list_for_each_entry_safe(desc, next, &mdev->done_list, node) { 595 struct dmaengine_desc_callback cb; 596 597 dmaengine_desc_get_callback(&desc->async_tx, &cb); 598 if (dmaengine_desc_callback_valid(&cb)) { 599 spin_unlock_irqrestore(&mdev->lock, irqflags); 600 dmaengine_desc_callback_invoke(&cb, NULL); 601 spin_lock_irqsave(&mdev->lock, irqflags); 602 } 603 604 /* Run any dependencies, then free the descriptor */ 605 msgdma_free_descriptor(mdev, desc); 606 } 607 608 spin_unlock_irqrestore(&mdev->lock, irqflags); 609 } 610 611 /** 612 * msgdma_complete_descriptor - Mark the active descriptor as complete 613 * @mdev: Pointer to the Altera mSGDMA device structure 614 */ 615 static void msgdma_complete_descriptor(struct msgdma_device *mdev) 616 { 617 struct msgdma_sw_desc *desc; 618 619 desc = list_first_entry_or_null(&mdev->active_list, 620 struct msgdma_sw_desc, node); 621 if (!desc) 622 return; 623 list_del(&desc->node); 624 dma_cookie_complete(&desc->async_tx); 625 list_add_tail(&desc->node, &mdev->done_list); 626 } 627 628 /** 629 * msgdma_free_descriptors - Free channel descriptors 630 * @mdev: Pointer to the Altera mSGDMA device structure 631 */ 632 static void msgdma_free_descriptors(struct msgdma_device *mdev) 633 { 634 msgdma_free_desc_list(mdev, &mdev->active_list); 635 msgdma_free_desc_list(mdev, &mdev->pending_list); 636 msgdma_free_desc_list(mdev, &mdev->done_list); 637 } 638 639 /** 640 * msgdma_free_chan_resources - Free channel resources 641 * @dchan: DMA channel pointer 642 */ 643 static void msgdma_free_chan_resources(struct dma_chan *dchan) 644 { 645 struct msgdma_device *mdev = to_mdev(dchan); 646 unsigned long flags; 647 648 spin_lock_irqsave(&mdev->lock, flags); 649 msgdma_free_descriptors(mdev); 650 spin_unlock_irqrestore(&mdev->lock, flags); 651 kfree(mdev->sw_desq); 652 } 653 654 /** 655 * msgdma_alloc_chan_resources - Allocate channel resources 656 * @dchan: DMA channel 657 * 658 * Return: Number of descriptors on success and failure value on error 659 */ 660 static int msgdma_alloc_chan_resources(struct dma_chan *dchan) 661 { 662 struct msgdma_device *mdev = to_mdev(dchan); 663 struct msgdma_sw_desc *desc; 664 int i; 665 666 mdev->sw_desq = kzalloc_objs(*desc, MSGDMA_DESC_NUM, GFP_NOWAIT); 667 if (!mdev->sw_desq) 668 return -ENOMEM; 669 670 mdev->idle = true; 671 mdev->desc_free_cnt = MSGDMA_DESC_NUM; 672 673 INIT_LIST_HEAD(&mdev->free_list); 674 675 for (i = 0; i < MSGDMA_DESC_NUM; i++) { 676 desc = mdev->sw_desq + i; 677 dma_async_tx_descriptor_init(&desc->async_tx, &mdev->dmachan); 678 desc->async_tx.tx_submit = msgdma_tx_submit; 679 list_add_tail(&desc->node, &mdev->free_list); 680 } 681 682 return MSGDMA_DESC_NUM; 683 } 684 685 /** 686 * msgdma_tasklet - Schedule completion tasklet 687 * @t: Pointer to the Altera sSGDMA channel structure 688 */ 689 static void msgdma_tasklet(struct tasklet_struct *t) 690 { 691 struct msgdma_device *mdev = from_tasklet(mdev, t, irq_tasklet); 692 u32 count; 693 u32 __maybe_unused size; 694 u32 __maybe_unused status; 695 unsigned long flags; 696 697 spin_lock_irqsave(&mdev->lock, flags); 698 699 if (mdev->resp) { 700 /* Read number of responses that are available */ 701 count = ioread32(mdev->csr + MSGDMA_CSR_RESP_FILL_LEVEL); 702 dev_dbg(mdev->dev, "%s (%d): response count=%d\n", 703 __func__, __LINE__, count); 704 } else { 705 count = 1; 706 } 707 708 while (count--) { 709 /* 710 * Read both longwords to purge this response from the FIFO 711 * On Avalon-MM implementations, size and status do not 712 * have any real values, like transferred bytes or error 713 * bits. So we need to just drop these values. 714 */ 715 if (mdev->resp) { 716 size = ioread32(mdev->resp + 717 MSGDMA_RESP_BYTES_TRANSFERRED); 718 status = ioread32(mdev->resp + 719 MSGDMA_RESP_STATUS); 720 } 721 722 msgdma_complete_descriptor(mdev); 723 } 724 725 spin_unlock_irqrestore(&mdev->lock, flags); 726 727 msgdma_chan_desc_cleanup(mdev); 728 } 729 730 /** 731 * msgdma_irq_handler - Altera mSGDMA Interrupt handler 732 * @irq: IRQ number 733 * @data: Pointer to the Altera mSGDMA device structure 734 * 735 * Return: IRQ_HANDLED/IRQ_NONE 736 */ 737 static irqreturn_t msgdma_irq_handler(int irq, void *data) 738 { 739 struct msgdma_device *mdev = data; 740 u32 status; 741 742 status = ioread32(mdev->csr + MSGDMA_CSR_STATUS); 743 if ((status & MSGDMA_CSR_STAT_BUSY) == 0) { 744 /* Start next transfer if the DMA controller is idle */ 745 spin_lock(&mdev->lock); 746 mdev->idle = true; 747 msgdma_start_transfer(mdev); 748 spin_unlock(&mdev->lock); 749 } 750 751 tasklet_schedule(&mdev->irq_tasklet); 752 753 /* Clear interrupt in mSGDMA controller */ 754 iowrite32(MSGDMA_CSR_STAT_IRQ, mdev->csr + MSGDMA_CSR_STATUS); 755 756 return IRQ_HANDLED; 757 } 758 759 /** 760 * msgdma_dev_remove() - Device remove function 761 * @mdev: Pointer to the Altera mSGDMA device structure 762 */ 763 static void msgdma_dev_remove(struct msgdma_device *mdev) 764 { 765 if (!mdev) 766 return; 767 768 devm_free_irq(mdev->dev, mdev->irq, mdev); 769 tasklet_kill(&mdev->irq_tasklet); 770 list_del(&mdev->dmachan.device_node); 771 } 772 773 static int request_and_map(struct platform_device *pdev, const char *name, 774 struct resource **res, void __iomem **ptr, 775 bool optional) 776 { 777 struct resource *region; 778 struct device *device = &pdev->dev; 779 780 *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); 781 if (*res == NULL) { 782 if (optional) { 783 *ptr = NULL; 784 dev_info(device, "optional resource %s not defined\n", 785 name); 786 return 0; 787 } 788 dev_err(device, "mandatory resource %s not defined\n", name); 789 return -ENODEV; 790 } 791 792 region = devm_request_mem_region(device, (*res)->start, 793 resource_size(*res), dev_name(device)); 794 if (region == NULL) { 795 dev_err(device, "unable to request %s\n", name); 796 return -EBUSY; 797 } 798 799 *ptr = devm_ioremap(device, region->start, 800 resource_size(region)); 801 if (*ptr == NULL) { 802 dev_err(device, "ioremap of %s failed!", name); 803 return -ENOMEM; 804 } 805 806 return 0; 807 } 808 809 /** 810 * msgdma_probe - Driver probe function 811 * @pdev: Pointer to the platform_device structure 812 * 813 * Return: '0' on success and failure value on error 814 */ 815 static int msgdma_probe(struct platform_device *pdev) 816 { 817 struct msgdma_device *mdev; 818 struct dma_device *dma_dev; 819 struct resource *dma_res; 820 int ret; 821 822 mdev = devm_kzalloc(&pdev->dev, sizeof(*mdev), GFP_NOWAIT); 823 if (!mdev) 824 return -ENOMEM; 825 826 mdev->dev = &pdev->dev; 827 828 /* Map CSR space */ 829 ret = request_and_map(pdev, "csr", &dma_res, &mdev->csr, false); 830 if (ret) 831 return ret; 832 833 /* Map (extended) descriptor space */ 834 ret = request_and_map(pdev, "desc", &dma_res, &mdev->desc, false); 835 if (ret) 836 return ret; 837 838 /* Map response space */ 839 ret = request_and_map(pdev, "resp", &dma_res, &mdev->resp, true); 840 if (ret) 841 return ret; 842 843 platform_set_drvdata(pdev, mdev); 844 845 /* Get interrupt nr from platform data */ 846 mdev->irq = platform_get_irq(pdev, 0); 847 if (mdev->irq < 0) 848 return -ENXIO; 849 850 ret = devm_request_irq(&pdev->dev, mdev->irq, msgdma_irq_handler, 851 0, dev_name(&pdev->dev), mdev); 852 if (ret) 853 return ret; 854 855 tasklet_setup(&mdev->irq_tasklet, msgdma_tasklet); 856 857 dma_cookie_init(&mdev->dmachan); 858 859 spin_lock_init(&mdev->lock); 860 861 INIT_LIST_HEAD(&mdev->active_list); 862 INIT_LIST_HEAD(&mdev->pending_list); 863 INIT_LIST_HEAD(&mdev->done_list); 864 INIT_LIST_HEAD(&mdev->free_list); 865 866 dma_dev = &mdev->dmadev; 867 868 /* Set DMA capabilities */ 869 dma_cap_zero(dma_dev->cap_mask); 870 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 871 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); 872 873 dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 874 dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 875 dma_dev->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM) | 876 BIT(DMA_MEM_TO_MEM); 877 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 878 879 /* Init DMA link list */ 880 INIT_LIST_HEAD(&dma_dev->channels); 881 882 /* Set base routines */ 883 dma_dev->device_tx_status = dma_cookie_status; 884 dma_dev->device_issue_pending = msgdma_issue_pending; 885 dma_dev->dev = &pdev->dev; 886 887 dma_dev->copy_align = DMAENGINE_ALIGN_4_BYTES; 888 dma_dev->device_prep_dma_memcpy = msgdma_prep_memcpy; 889 dma_dev->device_prep_slave_sg = msgdma_prep_slave_sg; 890 dma_dev->device_config = msgdma_dma_config; 891 892 dma_dev->device_alloc_chan_resources = msgdma_alloc_chan_resources; 893 dma_dev->device_free_chan_resources = msgdma_free_chan_resources; 894 895 mdev->dmachan.device = dma_dev; 896 list_add_tail(&mdev->dmachan.device_node, &dma_dev->channels); 897 898 /* Set DMA mask to 64 bits */ 899 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 900 if (ret) { 901 dev_warn(&pdev->dev, "unable to set coherent mask to 64"); 902 goto fail; 903 } 904 905 msgdma_reset(mdev); 906 907 ret = dma_async_device_register(dma_dev); 908 if (ret) 909 goto fail; 910 911 ret = of_dma_controller_register(pdev->dev.of_node, 912 of_dma_xlate_by_chan_id, dma_dev); 913 if (ret == -EINVAL) 914 dev_warn(&pdev->dev, "device was not probed from DT"); 915 else if (ret && ret != -ENODEV) 916 goto fail; 917 918 dev_notice(&pdev->dev, "Altera mSGDMA driver probe success\n"); 919 920 return 0; 921 922 fail: 923 msgdma_dev_remove(mdev); 924 925 return ret; 926 } 927 928 /** 929 * msgdma_remove() - Driver remove function 930 * @pdev: Pointer to the platform_device structure 931 * 932 * Return: Always '0' 933 */ 934 static void msgdma_remove(struct platform_device *pdev) 935 { 936 struct msgdma_device *mdev = platform_get_drvdata(pdev); 937 938 if (pdev->dev.of_node) 939 of_dma_controller_free(pdev->dev.of_node); 940 dma_async_device_unregister(&mdev->dmadev); 941 msgdma_dev_remove(mdev); 942 943 dev_notice(&pdev->dev, "Altera mSGDMA driver removed\n"); 944 } 945 946 #ifdef CONFIG_OF 947 static const struct of_device_id msgdma_match[] = { 948 { .compatible = "altr,socfpga-msgdma", }, 949 { } 950 }; 951 952 MODULE_DEVICE_TABLE(of, msgdma_match); 953 #endif 954 955 static struct platform_driver msgdma_driver = { 956 .driver = { 957 .name = "altera-msgdma", 958 .of_match_table = of_match_ptr(msgdma_match), 959 }, 960 .probe = msgdma_probe, 961 .remove = msgdma_remove, 962 }; 963 964 module_platform_driver(msgdma_driver); 965 966 MODULE_ALIAS("platform:altera-msgdma"); 967 MODULE_DESCRIPTION("Altera mSGDMA driver"); 968 MODULE_AUTHOR("Stefan Roese <sr@denx.de>"); 969 MODULE_LICENSE("GPL"); 970