1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Driver for the Analog Devices AXI-DMAC core 4 * 5 * Copyright 2013-2019 Analog Devices Inc. 6 * Author: Lars-Peter Clausen <lars@metafoo.de> 7 */ 8 9 #include <linux/adi-axi-common.h> 10 #include <linux/bitfield.h> 11 #include <linux/cleanup.h> 12 #include <linux/clk.h> 13 #include <linux/device.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/dmaengine.h> 16 #include <linux/dmapool.h> 17 #include <linux/err.h> 18 #include <linux/interrupt.h> 19 #include <linux/io.h> 20 #include <linux/kernel.h> 21 #include <linux/module.h> 22 #include <linux/of.h> 23 #include <linux/of_dma.h> 24 #include <linux/of_address.h> 25 #include <linux/platform_device.h> 26 #include <linux/regmap.h> 27 #include <linux/slab.h> 28 29 #include <dt-bindings/dma/axi-dmac.h> 30 31 #include "dmaengine.h" 32 #include "virt-dma.h" 33 34 /* 35 * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has 36 * various instantiation parameters which decided the exact feature set support 37 * by the core. 38 * 39 * Each channel of the core has a source interface and a destination interface. 40 * The number of channels and the type of the channel interfaces is selected at 41 * configuration time. A interface can either be a connected to a central memory 42 * interconnect, which allows access to system memory, or it can be connected to 43 * a dedicated bus which is directly connected to a data port on a peripheral. 44 * Given that those are configuration options of the core that are selected when 45 * it is instantiated this means that they can not be changed by software at 46 * runtime. By extension this means that each channel is uni-directional. It can 47 * either be device to memory or memory to device, but not both. Also since the 48 * device side is a dedicated data bus only connected to a single peripheral 49 * there is no address than can or needs to be configured for the device side. 50 */ 51 52 #define AXI_DMAC_REG_INTERFACE_DESC 0x10 53 #define AXI_DMAC_DMA_SRC_TYPE_MSK GENMASK(13, 12) 54 #define AXI_DMAC_DMA_SRC_TYPE_GET(x) FIELD_GET(AXI_DMAC_DMA_SRC_TYPE_MSK, x) 55 #define AXI_DMAC_DMA_SRC_WIDTH_MSK GENMASK(11, 8) 56 #define AXI_DMAC_DMA_SRC_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_SRC_WIDTH_MSK, x) 57 #define AXI_DMAC_DMA_DST_TYPE_MSK GENMASK(5, 4) 58 #define AXI_DMAC_DMA_DST_TYPE_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_TYPE_MSK, x) 59 #define AXI_DMAC_DMA_DST_WIDTH_MSK GENMASK(3, 0) 60 #define AXI_DMAC_DMA_DST_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_WIDTH_MSK, x) 61 #define AXI_DMAC_REG_COHERENCY_DESC 0x14 62 #define AXI_DMAC_DST_COHERENT_MSK BIT(0) 63 #define AXI_DMAC_DST_COHERENT_GET(x) FIELD_GET(AXI_DMAC_DST_COHERENT_MSK, x) 64 65 #define AXI_DMAC_REG_IRQ_MASK 0x80 66 #define AXI_DMAC_REG_IRQ_PENDING 0x84 67 #define AXI_DMAC_REG_IRQ_SOURCE 0x88 68 69 #define AXI_DMAC_REG_CTRL 0x400 70 #define AXI_DMAC_REG_TRANSFER_ID 0x404 71 #define AXI_DMAC_REG_START_TRANSFER 0x408 72 #define AXI_DMAC_REG_FLAGS 0x40c 73 #define AXI_DMAC_REG_DEST_ADDRESS 0x410 74 #define AXI_DMAC_REG_DEST_ADDRESS_HIGH 0x490 75 #define AXI_DMAC_REG_SRC_ADDRESS 0x414 76 #define AXI_DMAC_REG_SRC_ADDRESS_HIGH 0x494 77 #define AXI_DMAC_REG_X_LENGTH 0x418 78 #define AXI_DMAC_REG_Y_LENGTH 0x41c 79 #define AXI_DMAC_REG_DEST_STRIDE 0x420 80 #define AXI_DMAC_REG_SRC_STRIDE 0x424 81 #define AXI_DMAC_REG_TRANSFER_DONE 0x428 82 #define AXI_DMAC_REG_ACTIVE_TRANSFER_ID 0x42c 83 #define AXI_DMAC_REG_STATUS 0x430 84 #define AXI_DMAC_REG_CURRENT_SRC_ADDR 0x434 85 #define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438 86 #define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c 87 #define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450 88 #define AXI_DMAC_REG_CURRENT_SG_ID 0x454 89 #define AXI_DMAC_REG_SG_ADDRESS 0x47c 90 #define AXI_DMAC_REG_SG_ADDRESS_HIGH 0x4bc 91 92 #define AXI_DMAC_CTRL_ENABLE BIT(0) 93 #define AXI_DMAC_CTRL_PAUSE BIT(1) 94 #define AXI_DMAC_CTRL_ENABLE_SG BIT(2) 95 96 #define AXI_DMAC_IRQ_SOT BIT(0) 97 #define AXI_DMAC_IRQ_EOT BIT(1) 98 99 #define AXI_DMAC_FLAG_CYCLIC BIT(0) 100 #define AXI_DMAC_FLAG_LAST BIT(1) 101 #define AXI_DMAC_FLAG_PARTIAL_REPORT BIT(2) 102 103 #define AXI_DMAC_FLAG_PARTIAL_XFER_DONE BIT(31) 104 105 /* The maximum ID allocated by the hardware is 31 */ 106 #define AXI_DMAC_SG_UNUSED 32U 107 108 /* Flags for axi_dmac_hw_desc.flags */ 109 #define AXI_DMAC_HW_FLAG_LAST BIT(0) 110 #define AXI_DMAC_HW_FLAG_IRQ BIT(1) 111 112 struct axi_dmac_hw_desc { 113 u32 flags; 114 u32 id; 115 u64 dest_addr; 116 u64 src_addr; 117 u64 next_sg_addr; 118 u32 y_len; 119 u32 x_len; 120 u32 src_stride; 121 u32 dst_stride; 122 u64 __pad[2]; 123 }; 124 125 struct axi_dmac_sg { 126 unsigned int partial_len; 127 bool schedule_when_free; 128 129 struct axi_dmac_hw_desc *hw; 130 dma_addr_t hw_phys; 131 }; 132 133 struct axi_dmac_desc { 134 struct virt_dma_desc vdesc; 135 struct axi_dmac_chan *chan; 136 137 bool cyclic; 138 bool cyclic_eot; 139 bool have_partial_xfer; 140 141 unsigned int num_submitted; 142 unsigned int num_completed; 143 unsigned int num_sgs; 144 struct axi_dmac_sg sg[] __counted_by(num_sgs); 145 }; 146 147 struct axi_dmac_chan { 148 struct virt_dma_chan vchan; 149 150 struct axi_dmac_desc *next_desc; 151 void *pool; 152 struct list_head active_descs; 153 enum dma_transfer_direction direction; 154 155 unsigned int src_width; 156 unsigned int dest_width; 157 unsigned int src_type; 158 unsigned int dest_type; 159 160 unsigned int max_length; 161 unsigned int address_align_mask; 162 unsigned int length_align_mask; 163 164 bool hw_partial_xfer; 165 bool hw_cyclic; 166 bool hw_2d; 167 bool hw_sg; 168 bool hw_cyclic_hotfix; 169 }; 170 171 struct axi_dmac { 172 void __iomem *base; 173 int irq; 174 175 struct dma_device dma_dev; 176 struct axi_dmac_chan chan; 177 }; 178 179 static struct axi_dmac *chan_to_axi_dmac(struct axi_dmac_chan *chan) 180 { 181 return container_of(chan->vchan.chan.device, struct axi_dmac, 182 dma_dev); 183 } 184 185 static struct axi_dmac_chan *to_axi_dmac_chan(struct dma_chan *c) 186 { 187 return container_of(c, struct axi_dmac_chan, vchan.chan); 188 } 189 190 static struct axi_dmac_desc *to_axi_dmac_desc(struct virt_dma_desc *vdesc) 191 { 192 return container_of(vdesc, struct axi_dmac_desc, vdesc); 193 } 194 195 static void axi_dmac_write(struct axi_dmac *axi_dmac, unsigned int reg, 196 unsigned int val) 197 { 198 writel(val, axi_dmac->base + reg); 199 } 200 201 static int axi_dmac_read(struct axi_dmac *axi_dmac, unsigned int reg) 202 { 203 return readl(axi_dmac->base + reg); 204 } 205 206 static int axi_dmac_src_is_mem(struct axi_dmac_chan *chan) 207 { 208 return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM; 209 } 210 211 static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan) 212 { 213 return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM; 214 } 215 216 static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len) 217 { 218 if (len == 0) 219 return false; 220 if ((len & chan->length_align_mask) != 0) /* Not aligned */ 221 return false; 222 return true; 223 } 224 225 static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr) 226 { 227 if ((addr & chan->address_align_mask) != 0) /* Not aligned */ 228 return false; 229 return true; 230 } 231 232 static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) 233 { 234 return list_first_entry_or_null(&chan->active_descs, 235 struct axi_dmac_desc, vdesc.node); 236 } 237 238 static struct axi_dmac_desc *axi_dmac_get_next_desc(struct axi_dmac *dmac, 239 struct axi_dmac_chan *chan) 240 { 241 struct axi_dmac_desc *active = axi_dmac_active_desc(chan); 242 struct virt_dma_desc *vdesc; 243 struct axi_dmac_desc *desc; 244 unsigned int val; 245 246 /* 247 * Just play safe and ignore any SOF if we have an active cyclic transfer 248 * flagged to end. We'll start it as soon as the current cyclic one ends. 249 */ 250 if (active && active->cyclic_eot) 251 return NULL; 252 253 /* 254 * It means a SW cyclic transfer is in place so we should just return 255 * the same descriptor. SW cyclic transfer termination is handled 256 * in axi_dmac_transfer_done(). 257 */ 258 if (chan->next_desc) 259 return chan->next_desc; 260 261 vdesc = vchan_next_desc(&chan->vchan); 262 if (!vdesc) 263 return NULL; 264 265 if (active && active->cyclic && !(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) { 266 struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev; 267 268 dev_warn(dev, "Discarding non EOT transfer after cyclic\n"); 269 list_del(&vdesc->node); 270 return NULL; 271 } 272 273 list_move_tail(&vdesc->node, &chan->active_descs); 274 desc = to_axi_dmac_desc(vdesc); 275 chan->next_desc = desc; 276 277 if (!active || !active->cyclic) 278 return desc; 279 280 active->cyclic_eot = true; 281 282 if (chan->hw_sg) { 283 unsigned long flags = AXI_DMAC_HW_FLAG_IRQ | AXI_DMAC_HW_FLAG_LAST; 284 /* 285 * Let's then stop the current cyclic transfer by making sure we 286 * get an EOT interrupt and to open the cyclic loop by marking 287 * the last segment. 288 */ 289 active->sg[active->num_sgs - 1].hw->flags = flags; 290 return NULL; 291 } 292 293 /* 294 * Clear the cyclic bit if there's no Scatter-Gather HW so that we get 295 * at the end of the transfer. 296 */ 297 val = axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS); 298 val &= ~AXI_DMAC_FLAG_CYCLIC; 299 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, val); 300 301 return NULL; 302 } 303 304 static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) 305 { 306 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 307 struct axi_dmac_desc *desc; 308 struct axi_dmac_sg *sg; 309 unsigned int flags = 0; 310 unsigned int val; 311 312 desc = axi_dmac_get_next_desc(dmac, chan); 313 if (!desc) 314 return; 315 316 val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); 317 if (val) /* Queue is full, wait for the next SOT IRQ */ 318 return; 319 320 sg = &desc->sg[desc->num_submitted]; 321 322 /* Already queued in cyclic mode. Wait for it to finish */ 323 if (sg->hw->id != AXI_DMAC_SG_UNUSED) { 324 sg->schedule_when_free = true; 325 return; 326 } 327 328 if (chan->hw_sg) { 329 chan->next_desc = NULL; 330 } else if (++desc->num_submitted == desc->num_sgs || 331 desc->have_partial_xfer) { 332 if (desc->cyclic) 333 desc->num_submitted = 0; /* Start again */ 334 else 335 chan->next_desc = NULL; 336 flags |= AXI_DMAC_FLAG_LAST; 337 } 338 339 sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); 340 341 if (!chan->hw_sg) { 342 if (axi_dmac_dest_is_mem(chan)) { 343 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr); 344 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS_HIGH, 345 sg->hw->dest_addr >> 32); 346 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride); 347 } 348 349 if (axi_dmac_src_is_mem(chan)) { 350 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr); 351 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS_HIGH, sg->hw->src_addr >> 32); 352 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride); 353 } 354 } 355 356 /* 357 * If the hardware supports cyclic transfers and there is no callback to 358 * call, enable hw cyclic mode to avoid unnecessary interrupts. 359 */ 360 if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) { 361 if (chan->hw_sg) { 362 desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ; 363 } else if (desc->num_sgs == 1) { 364 chan->next_desc = NULL; 365 flags |= AXI_DMAC_FLAG_CYCLIC; 366 } 367 } 368 369 if (chan->hw_partial_xfer) 370 flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; 371 372 if (chan->hw_sg) { 373 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys); 374 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS_HIGH, 375 (u64)sg->hw_phys >> 32); 376 } else { 377 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len); 378 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len); 379 } 380 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); 381 axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); 382 } 383 384 static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan, 385 struct axi_dmac_sg *sg) 386 { 387 if (chan->hw_2d) 388 return (sg->hw->x_len + 1) * (sg->hw->y_len + 1); 389 else 390 return (sg->hw->x_len + 1); 391 } 392 393 static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) 394 { 395 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 396 struct axi_dmac_desc *desc; 397 struct axi_dmac_sg *sg; 398 u32 xfer_done, len, id, i; 399 bool found_sg; 400 401 do { 402 len = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_LEN); 403 id = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_ID); 404 405 found_sg = false; 406 list_for_each_entry(desc, &chan->active_descs, vdesc.node) { 407 for (i = 0; i < desc->num_sgs; i++) { 408 sg = &desc->sg[i]; 409 if (sg->hw->id == AXI_DMAC_SG_UNUSED) 410 continue; 411 if (sg->hw->id == id) { 412 desc->have_partial_xfer = true; 413 sg->partial_len = len; 414 found_sg = true; 415 break; 416 } 417 } 418 if (found_sg) 419 break; 420 } 421 422 if (found_sg) { 423 dev_dbg(dmac->dma_dev.dev, 424 "Found partial segment id=%u, len=%u\n", 425 id, len); 426 } else { 427 dev_warn(dmac->dma_dev.dev, 428 "Not found partial segment id=%u, len=%u\n", 429 id, len); 430 } 431 432 /* Check if we have any more partial transfers */ 433 xfer_done = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE); 434 xfer_done = !(xfer_done & AXI_DMAC_FLAG_PARTIAL_XFER_DONE); 435 436 } while (!xfer_done); 437 } 438 439 static void axi_dmac_compute_residue(struct axi_dmac_chan *chan, 440 struct axi_dmac_desc *active) 441 { 442 struct dmaengine_result *rslt = &active->vdesc.tx_result; 443 unsigned int start = active->num_completed - 1; 444 struct axi_dmac_sg *sg; 445 unsigned int i, total; 446 447 rslt->result = DMA_TRANS_NOERROR; 448 rslt->residue = 0; 449 450 if (chan->hw_sg) 451 return; 452 453 /* 454 * We get here if the last completed segment is partial, which 455 * means we can compute the residue from that segment onwards 456 */ 457 for (i = start; i < active->num_sgs; i++) { 458 sg = &active->sg[i]; 459 total = axi_dmac_total_sg_bytes(chan, sg); 460 rslt->residue += (total - sg->partial_len); 461 } 462 } 463 464 static bool axi_dmac_handle_cyclic_eot(struct axi_dmac_chan *chan, 465 struct axi_dmac_desc *active) 466 { 467 struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev; 468 struct virt_dma_desc *vdesc; 469 470 /* wrap around */ 471 active->num_completed = 0; 472 473 if (active->cyclic_eot) { 474 /* 475 * It means an HW cyclic transfer was marked to stop. And we 476 * know we have something to schedule, so start the next 477 * transfer now the cyclic one is done. 478 */ 479 list_del(&active->vdesc.node); 480 vchan_cookie_complete(&active->vdesc); 481 482 if (chan->hw_cyclic_hotfix) { 483 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 484 /* 485 * In older IP cores, ending a cyclic transfer by clearing 486 * the CYCLIC flag does not guarantee a graceful end. 487 * It can happen that some data (of the next frame) is 488 * already prefetched and will be wrongly visible in the 489 * next transfer. To workaround this, we need to reenable 490 * the core so everything is flushed. Newer cores handles 491 * this correctly and do not require this "hotfix". The 492 * SG IP also does not require this. 493 */ 494 dev_dbg(dev, "HW cyclic hotfix\n"); 495 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0); 496 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); 497 } 498 499 return true; 500 } 501 502 vdesc = vchan_next_desc(&chan->vchan); 503 if (!vdesc) 504 return false; 505 if (!(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) { 506 dev_warn(dev, "Discarding non EOT transfer after cyclic\n"); 507 list_del(&vdesc->node); 508 return false; 509 } 510 511 /* then let's end the cyclic transfer */ 512 chan->next_desc = NULL; 513 list_del(&active->vdesc.node); 514 vchan_cookie_complete(&active->vdesc); 515 516 return true; 517 } 518 519 static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, 520 unsigned int completed_transfers) 521 { 522 struct axi_dmac_desc *active; 523 struct axi_dmac_sg *sg; 524 bool start_next = false; 525 526 active = axi_dmac_active_desc(chan); 527 if (!active) 528 return false; 529 530 if (chan->hw_partial_xfer && 531 (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE)) 532 axi_dmac_dequeue_partial_xfers(chan); 533 534 if (chan->hw_sg) { 535 if (active->cyclic) { 536 vchan_cyclic_callback(&active->vdesc); 537 start_next = axi_dmac_handle_cyclic_eot(chan, active); 538 } else { 539 list_del(&active->vdesc.node); 540 vchan_cookie_complete(&active->vdesc); 541 active = axi_dmac_active_desc(chan); 542 start_next = !!active; 543 } 544 } else { 545 do { 546 sg = &active->sg[active->num_completed]; 547 if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ 548 break; 549 if (!(BIT(sg->hw->id) & completed_transfers)) 550 break; 551 active->num_completed++; 552 sg->hw->id = AXI_DMAC_SG_UNUSED; 553 if (sg->schedule_when_free) { 554 sg->schedule_when_free = false; 555 start_next = true; 556 } 557 558 if (sg->partial_len) 559 axi_dmac_compute_residue(chan, active); 560 561 if (active->cyclic) 562 vchan_cyclic_callback(&active->vdesc); 563 564 if (active->num_completed == active->num_sgs || 565 sg->partial_len) { 566 if (active->cyclic) { 567 /* keep start_next as is, if already true... */ 568 start_next |= axi_dmac_handle_cyclic_eot(chan, active); 569 } else { 570 list_del(&active->vdesc.node); 571 vchan_cookie_complete(&active->vdesc); 572 active = axi_dmac_active_desc(chan); 573 } 574 } 575 } while (active); 576 } 577 578 return start_next; 579 } 580 581 static irqreturn_t axi_dmac_interrupt_handler(int irq, void *devid) 582 { 583 struct axi_dmac *dmac = devid; 584 unsigned int pending; 585 bool start_next = false; 586 587 pending = axi_dmac_read(dmac, AXI_DMAC_REG_IRQ_PENDING); 588 if (!pending) 589 return IRQ_NONE; 590 591 axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_PENDING, pending); 592 593 spin_lock(&dmac->chan.vchan.lock); 594 /* One or more transfers have finished */ 595 if (pending & AXI_DMAC_IRQ_EOT) { 596 unsigned int completed; 597 598 completed = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE); 599 start_next = axi_dmac_transfer_done(&dmac->chan, completed); 600 } 601 /* Space has become available in the descriptor queue */ 602 if ((pending & AXI_DMAC_IRQ_SOT) || start_next) 603 axi_dmac_start_transfer(&dmac->chan); 604 spin_unlock(&dmac->chan.vchan.lock); 605 606 return IRQ_HANDLED; 607 } 608 609 static int axi_dmac_terminate_all(struct dma_chan *c) 610 { 611 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 612 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 613 unsigned long flags; 614 LIST_HEAD(head); 615 616 spin_lock_irqsave(&chan->vchan.lock, flags); 617 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0); 618 chan->next_desc = NULL; 619 vchan_get_all_descriptors(&chan->vchan, &head); 620 list_splice_tail_init(&chan->active_descs, &head); 621 spin_unlock_irqrestore(&chan->vchan.lock, flags); 622 623 vchan_dma_desc_free_list(&chan->vchan, &head); 624 625 return 0; 626 } 627 628 static void axi_dmac_synchronize(struct dma_chan *c) 629 { 630 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 631 632 vchan_synchronize(&chan->vchan); 633 } 634 635 static void axi_dmac_issue_pending(struct dma_chan *c) 636 { 637 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 638 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 639 unsigned long flags; 640 u32 ctrl = AXI_DMAC_CTRL_ENABLE; 641 642 if (chan->hw_sg) 643 ctrl |= AXI_DMAC_CTRL_ENABLE_SG; 644 645 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, ctrl); 646 647 spin_lock_irqsave(&chan->vchan.lock, flags); 648 if (vchan_issue_pending(&chan->vchan)) 649 axi_dmac_start_transfer(chan); 650 spin_unlock_irqrestore(&chan->vchan.lock, flags); 651 } 652 653 static void axi_dmac_free_desc(struct axi_dmac_desc *desc) 654 { 655 for (unsigned int i = 0; i < desc->num_sgs; i++) 656 dma_pool_free(desc->chan->pool, desc->sg[i].hw, desc->sg[i].hw_phys); 657 658 kfree(desc); 659 } 660 661 static struct axi_dmac_desc * 662 axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) 663 { 664 struct axi_dmac_hw_desc *hws; 665 struct axi_dmac_desc *desc; 666 dma_addr_t hw_phys; 667 unsigned int i; 668 669 desc = kzalloc_flex(*desc, sg, num_sgs, GFP_NOWAIT); 670 if (!desc) 671 return NULL; 672 desc->num_sgs = num_sgs; 673 desc->chan = chan; 674 675 for (i = 0; i < num_sgs; i++) { 676 hws = dma_pool_zalloc(chan->pool, GFP_NOWAIT, &hw_phys); 677 if (!hws) { 678 desc->num_sgs = i; 679 axi_dmac_free_desc(desc); 680 return NULL; 681 } 682 683 desc->sg[i].hw = hws; 684 desc->sg[i].hw_phys = hw_phys; 685 686 hws->id = AXI_DMAC_SG_UNUSED; 687 688 /* Link hardware descriptors */ 689 if (i) 690 desc->sg[i - 1].hw->next_sg_addr = hw_phys; 691 } 692 693 /* The last hardware descriptor will trigger an interrupt */ 694 desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ; 695 696 return desc; 697 } 698 699 static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, 700 enum dma_transfer_direction direction, dma_addr_t addr, 701 unsigned int num_periods, unsigned int period_len, 702 struct axi_dmac_sg *sg) 703 { 704 unsigned int num_segments, i; 705 unsigned int segment_size; 706 unsigned int len; 707 708 /* Split into multiple equally sized segments if necessary */ 709 num_segments = DIV_ROUND_UP(period_len, chan->max_length); 710 segment_size = DIV_ROUND_UP(period_len, num_segments); 711 /* Take care of alignment */ 712 segment_size = ((segment_size - 1) | chan->length_align_mask) + 1; 713 714 for (i = 0; i < num_periods; i++) { 715 for (len = period_len; len > segment_size; sg++) { 716 if (direction == DMA_DEV_TO_MEM) 717 sg->hw->dest_addr = addr; 718 else 719 sg->hw->src_addr = addr; 720 sg->hw->x_len = segment_size - 1; 721 sg->hw->y_len = 0; 722 sg->hw->flags = 0; 723 addr += segment_size; 724 len -= segment_size; 725 } 726 727 if (direction == DMA_DEV_TO_MEM) 728 sg->hw->dest_addr = addr; 729 else 730 sg->hw->src_addr = addr; 731 sg->hw->x_len = len - 1; 732 sg->hw->y_len = 0; 733 sg++; 734 addr += len; 735 } 736 737 return sg; 738 } 739 740 static struct dma_async_tx_descriptor * 741 axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs, 742 size_t nb, enum dma_transfer_direction direction, 743 unsigned long flags) 744 { 745 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 746 struct axi_dmac_desc *desc; 747 unsigned int num_sgs = 0; 748 struct axi_dmac_sg *dsg; 749 size_t i; 750 751 if (direction != chan->direction) 752 return NULL; 753 754 for (i = 0; i < nb; i++) 755 num_sgs += DIV_ROUND_UP(vecs[i].len, chan->max_length); 756 757 desc = axi_dmac_alloc_desc(chan, num_sgs); 758 if (!desc) 759 return NULL; 760 761 dsg = desc->sg; 762 763 for (i = 0; i < nb; i++) { 764 if (!axi_dmac_check_addr(chan, vecs[i].addr) || 765 !axi_dmac_check_len(chan, vecs[i].len)) { 766 axi_dmac_free_desc(desc); 767 return NULL; 768 } 769 770 dsg = axi_dmac_fill_linear_sg(chan, direction, vecs[i].addr, 1, 771 vecs[i].len, dsg); 772 } 773 774 desc->cyclic = flags & DMA_PREP_REPEAT; 775 if (desc->cyclic) { 776 /* Chain the last descriptor to the first, and remove its "last" flag */ 777 desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST; 778 desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys; 779 } 780 781 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 782 } 783 784 static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( 785 struct dma_chan *c, struct scatterlist *sgl, 786 unsigned int sg_len, enum dma_transfer_direction direction, 787 unsigned long flags, void *context) 788 { 789 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 790 struct axi_dmac_desc *desc; 791 struct axi_dmac_sg *dsg; 792 struct scatterlist *sg; 793 unsigned int num_sgs; 794 unsigned int i; 795 796 if (direction != chan->direction) 797 return NULL; 798 799 num_sgs = sg_nents_for_dma(sgl, sg_len, chan->max_length); 800 desc = axi_dmac_alloc_desc(chan, num_sgs); 801 if (!desc) 802 return NULL; 803 804 dsg = desc->sg; 805 806 for_each_sg(sgl, sg, sg_len, i) { 807 if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) || 808 !axi_dmac_check_len(chan, sg_dma_len(sg))) { 809 axi_dmac_free_desc(desc); 810 return NULL; 811 } 812 813 dsg = axi_dmac_fill_linear_sg(chan, direction, sg_dma_address(sg), 1, 814 sg_dma_len(sg), dsg); 815 } 816 817 desc->cyclic = false; 818 819 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 820 } 821 822 static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( 823 struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len, 824 size_t period_len, enum dma_transfer_direction direction, 825 unsigned long flags) 826 { 827 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 828 struct axi_dmac_desc *desc; 829 unsigned int num_periods, num_segments, num_sgs; 830 831 if (direction != chan->direction) 832 return NULL; 833 834 if (!axi_dmac_check_len(chan, buf_len) || 835 !axi_dmac_check_addr(chan, buf_addr)) 836 return NULL; 837 838 if (period_len == 0 || buf_len % period_len) 839 return NULL; 840 841 num_periods = buf_len / period_len; 842 num_segments = DIV_ROUND_UP(period_len, chan->max_length); 843 num_sgs = num_periods * num_segments; 844 845 desc = axi_dmac_alloc_desc(chan, num_sgs); 846 if (!desc) 847 return NULL; 848 849 /* Chain the last descriptor to the first, and remove its "last" flag */ 850 desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys; 851 desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST; 852 853 axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods, 854 period_len, desc->sg); 855 856 desc->cyclic = true; 857 858 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 859 } 860 861 static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved( 862 struct dma_chan *c, struct dma_interleaved_template *xt, 863 unsigned long flags) 864 { 865 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 866 struct axi_dmac_desc *desc; 867 size_t dst_icg, src_icg; 868 869 if (xt->frame_size != 1) 870 return NULL; 871 872 if (xt->dir != chan->direction) 873 return NULL; 874 875 if (axi_dmac_src_is_mem(chan)) { 876 if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start)) 877 return NULL; 878 } 879 880 if (axi_dmac_dest_is_mem(chan)) { 881 if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start)) 882 return NULL; 883 } 884 885 dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); 886 src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); 887 888 if (chan->hw_2d) { 889 if (!axi_dmac_check_len(chan, xt->sgl[0].size) || 890 xt->numf == 0) 891 return NULL; 892 if (xt->sgl[0].size + dst_icg > chan->max_length || 893 xt->sgl[0].size + src_icg > chan->max_length) 894 return NULL; 895 } else { 896 if (dst_icg != 0 || src_icg != 0) 897 return NULL; 898 if (chan->max_length / xt->sgl[0].size < xt->numf) 899 return NULL; 900 if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf)) 901 return NULL; 902 } 903 904 desc = axi_dmac_alloc_desc(chan, 1); 905 if (!desc) 906 return NULL; 907 908 if (axi_dmac_src_is_mem(chan)) { 909 desc->sg[0].hw->src_addr = xt->src_start; 910 desc->sg[0].hw->src_stride = xt->sgl[0].size + src_icg; 911 } 912 913 if (axi_dmac_dest_is_mem(chan)) { 914 desc->sg[0].hw->dest_addr = xt->dst_start; 915 desc->sg[0].hw->dst_stride = xt->sgl[0].size + dst_icg; 916 } 917 918 if (chan->hw_2d) { 919 desc->sg[0].hw->x_len = xt->sgl[0].size - 1; 920 desc->sg[0].hw->y_len = xt->numf - 1; 921 } else { 922 desc->sg[0].hw->x_len = xt->sgl[0].size * xt->numf - 1; 923 desc->sg[0].hw->y_len = 0; 924 } 925 926 if (flags & DMA_CYCLIC) 927 desc->cyclic = true; 928 929 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 930 } 931 932 static int axi_dmac_alloc_chan_resources(struct dma_chan *c) 933 { 934 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 935 struct device *dev = c->device->dev; 936 937 chan->pool = dma_pool_create(dev_name(dev), dev, 938 sizeof(struct axi_dmac_hw_desc), 939 __alignof__(struct axi_dmac_hw_desc), 0); 940 if (!chan->pool) 941 return -ENOMEM; 942 943 return 0; 944 } 945 946 static void axi_dmac_free_chan_resources(struct dma_chan *c) 947 { 948 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 949 950 vchan_free_chan_resources(to_virt_chan(c)); 951 dma_pool_destroy(chan->pool); 952 } 953 954 static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) 955 { 956 axi_dmac_free_desc(to_axi_dmac_desc(vdesc)); 957 } 958 959 static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) 960 { 961 switch (reg) { 962 case AXI_DMAC_REG_IRQ_MASK: 963 case AXI_DMAC_REG_IRQ_SOURCE: 964 case AXI_DMAC_REG_IRQ_PENDING: 965 case AXI_DMAC_REG_CTRL: 966 case AXI_DMAC_REG_TRANSFER_ID: 967 case AXI_DMAC_REG_START_TRANSFER: 968 case AXI_DMAC_REG_FLAGS: 969 case AXI_DMAC_REG_DEST_ADDRESS: 970 case AXI_DMAC_REG_SRC_ADDRESS: 971 case AXI_DMAC_REG_X_LENGTH: 972 case AXI_DMAC_REG_Y_LENGTH: 973 case AXI_DMAC_REG_DEST_STRIDE: 974 case AXI_DMAC_REG_SRC_STRIDE: 975 case AXI_DMAC_REG_TRANSFER_DONE: 976 case AXI_DMAC_REG_ACTIVE_TRANSFER_ID: 977 case AXI_DMAC_REG_STATUS: 978 case AXI_DMAC_REG_CURRENT_SRC_ADDR: 979 case AXI_DMAC_REG_CURRENT_DEST_ADDR: 980 case AXI_DMAC_REG_PARTIAL_XFER_LEN: 981 case AXI_DMAC_REG_PARTIAL_XFER_ID: 982 case AXI_DMAC_REG_CURRENT_SG_ID: 983 case AXI_DMAC_REG_SG_ADDRESS: 984 case AXI_DMAC_REG_SG_ADDRESS_HIGH: 985 return true; 986 default: 987 return false; 988 } 989 } 990 991 static const struct regmap_config axi_dmac_regmap_config = { 992 .reg_bits = 32, 993 .val_bits = 32, 994 .reg_stride = 4, 995 .max_register = AXI_DMAC_REG_PARTIAL_XFER_ID, 996 .readable_reg = axi_dmac_regmap_rdwr, 997 .writeable_reg = axi_dmac_regmap_rdwr, 998 }; 999 1000 static void axi_dmac_adjust_chan_params(struct axi_dmac_chan *chan) 1001 { 1002 chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1; 1003 1004 if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan)) 1005 chan->direction = DMA_MEM_TO_MEM; 1006 else if (!axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan)) 1007 chan->direction = DMA_MEM_TO_DEV; 1008 else if (axi_dmac_dest_is_mem(chan) && !axi_dmac_src_is_mem(chan)) 1009 chan->direction = DMA_DEV_TO_MEM; 1010 else 1011 chan->direction = DMA_DEV_TO_DEV; 1012 } 1013 1014 /* 1015 * The configuration stored in the devicetree matches the configuration 1016 * parameters of the peripheral instance and allows the driver to know which 1017 * features are implemented and how it should behave. 1018 */ 1019 static int axi_dmac_parse_chan_dt(struct device_node *of_chan, 1020 struct axi_dmac_chan *chan) 1021 { 1022 u32 val; 1023 int ret; 1024 1025 ret = of_property_read_u32(of_chan, "reg", &val); 1026 if (ret) 1027 return ret; 1028 1029 /* We only support 1 channel for now */ 1030 if (val != 0) 1031 return -EINVAL; 1032 1033 ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val); 1034 if (ret) 1035 return ret; 1036 if (val > AXI_DMAC_BUS_TYPE_FIFO) 1037 return -EINVAL; 1038 chan->src_type = val; 1039 1040 ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val); 1041 if (ret) 1042 return ret; 1043 if (val > AXI_DMAC_BUS_TYPE_FIFO) 1044 return -EINVAL; 1045 chan->dest_type = val; 1046 1047 ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val); 1048 if (ret) 1049 return ret; 1050 chan->src_width = val / 8; 1051 1052 ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val); 1053 if (ret) 1054 return ret; 1055 chan->dest_width = val / 8; 1056 1057 axi_dmac_adjust_chan_params(chan); 1058 1059 return 0; 1060 } 1061 1062 static int axi_dmac_parse_dt(struct device *dev, struct axi_dmac *dmac) 1063 { 1064 int ret; 1065 1066 struct device_node *of_channels __free(device_node) = of_get_child_by_name(dev->of_node, 1067 "adi,channels"); 1068 if (of_channels == NULL) 1069 return -ENODEV; 1070 1071 for_each_child_of_node_scoped(of_channels, of_chan) { 1072 ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan); 1073 if (ret) 1074 return -EINVAL; 1075 } 1076 1077 return 0; 1078 } 1079 1080 static int axi_dmac_read_chan_config(struct device *dev, struct axi_dmac *dmac) 1081 { 1082 struct axi_dmac_chan *chan = &dmac->chan; 1083 unsigned int val, desc; 1084 1085 desc = axi_dmac_read(dmac, AXI_DMAC_REG_INTERFACE_DESC); 1086 if (desc == 0) { 1087 dev_err(dev, "DMA interface register reads zero\n"); 1088 return -EFAULT; 1089 } 1090 1091 val = AXI_DMAC_DMA_SRC_TYPE_GET(desc); 1092 if (val > AXI_DMAC_BUS_TYPE_FIFO) { 1093 dev_err(dev, "Invalid source bus type read: %d\n", val); 1094 return -EINVAL; 1095 } 1096 chan->src_type = val; 1097 1098 val = AXI_DMAC_DMA_DST_TYPE_GET(desc); 1099 if (val > AXI_DMAC_BUS_TYPE_FIFO) { 1100 dev_err(dev, "Invalid destination bus type read: %d\n", val); 1101 return -EINVAL; 1102 } 1103 chan->dest_type = val; 1104 1105 val = AXI_DMAC_DMA_SRC_WIDTH_GET(desc); 1106 if (val == 0) { 1107 dev_err(dev, "Source bus width is zero\n"); 1108 return -EINVAL; 1109 } 1110 /* widths are stored in log2 */ 1111 chan->src_width = 1 << val; 1112 1113 val = AXI_DMAC_DMA_DST_WIDTH_GET(desc); 1114 if (val == 0) { 1115 dev_err(dev, "Destination bus width is zero\n"); 1116 return -EINVAL; 1117 } 1118 chan->dest_width = 1 << val; 1119 1120 axi_dmac_adjust_chan_params(chan); 1121 1122 return 0; 1123 } 1124 1125 static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) 1126 { 1127 struct axi_dmac_chan *chan = &dmac->chan; 1128 struct device *dev = dmac->dma_dev.dev; 1129 u32 mask; 1130 int ret; 1131 1132 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, AXI_DMAC_FLAG_CYCLIC); 1133 if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC) 1134 chan->hw_cyclic = true; 1135 1136 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, 0xffffffff); 1137 if (axi_dmac_read(dmac, AXI_DMAC_REG_SG_ADDRESS)) 1138 chan->hw_sg = true; 1139 1140 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1); 1141 if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1) 1142 chan->hw_2d = true; 1143 1144 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0xffffffff); 1145 chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH); 1146 if (chan->max_length != UINT_MAX) 1147 chan->max_length++; 1148 1149 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, 0xffffffff); 1150 if (axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS) == 0 && 1151 chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) { 1152 dev_err(dmac->dma_dev.dev, 1153 "Destination memory-mapped interface not supported."); 1154 return -ENODEV; 1155 } 1156 1157 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, 0xffffffff); 1158 if (axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS) == 0 && 1159 chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) { 1160 dev_err(dmac->dma_dev.dev, 1161 "Source memory-mapped interface not supported."); 1162 return -ENODEV; 1163 } 1164 1165 if (axi_dmac_dest_is_mem(chan)) { 1166 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS_HIGH, 0xffffffff); 1167 mask = axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS_HIGH); 1168 } else { 1169 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS_HIGH, 0xffffffff); 1170 mask = axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS_HIGH); 1171 } 1172 1173 mask = 32 + fls(mask); 1174 1175 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(mask)); 1176 if (ret) { 1177 dev_err(dev, "DMA mask set error %d\n", ret); 1178 return ret; 1179 } 1180 1181 if (version >= ADI_AXI_PCORE_VER(4, 2, 'a')) 1182 chan->hw_partial_xfer = true; 1183 1184 if (version >= ADI_AXI_PCORE_VER(4, 1, 'a')) { 1185 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0x00); 1186 chan->length_align_mask = 1187 axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH); 1188 } else { 1189 chan->length_align_mask = chan->address_align_mask; 1190 } 1191 1192 if (version < ADI_AXI_PCORE_VER(4, 6, 0) && !chan->hw_sg) 1193 chan->hw_cyclic_hotfix = true; 1194 1195 return 0; 1196 } 1197 1198 static void axi_dmac_tasklet_kill(void *task) 1199 { 1200 tasklet_kill(task); 1201 } 1202 1203 static void axi_dmac_free_dma_controller(void *of_node) 1204 { 1205 of_dma_controller_free(of_node); 1206 } 1207 1208 static int axi_dmac_probe(struct platform_device *pdev) 1209 { 1210 struct dma_device *dma_dev; 1211 struct axi_dmac *dmac; 1212 struct clk *clk; 1213 struct regmap *regmap; 1214 unsigned int version; 1215 u32 irq_mask = 0; 1216 int ret; 1217 1218 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 1219 if (!dmac) 1220 return -ENOMEM; 1221 1222 dmac->irq = platform_get_irq(pdev, 0); 1223 if (dmac->irq < 0) 1224 return dmac->irq; 1225 if (dmac->irq == 0) 1226 return -EINVAL; 1227 1228 dmac->base = devm_platform_ioremap_resource(pdev, 0); 1229 if (IS_ERR(dmac->base)) 1230 return PTR_ERR(dmac->base); 1231 1232 clk = devm_clk_get_enabled(&pdev->dev, NULL); 1233 if (IS_ERR(clk)) 1234 return PTR_ERR(clk); 1235 1236 version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION); 1237 1238 if (version >= ADI_AXI_PCORE_VER(4, 3, 'a')) 1239 ret = axi_dmac_read_chan_config(&pdev->dev, dmac); 1240 else 1241 ret = axi_dmac_parse_dt(&pdev->dev, dmac); 1242 1243 if (ret < 0) 1244 return ret; 1245 1246 INIT_LIST_HEAD(&dmac->chan.active_descs); 1247 1248 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 1249 1250 dma_dev = &dmac->dma_dev; 1251 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); 1252 dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); 1253 dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask); 1254 dma_dev->device_alloc_chan_resources = axi_dmac_alloc_chan_resources; 1255 dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources; 1256 dma_dev->device_tx_status = dma_cookie_status; 1257 dma_dev->device_issue_pending = axi_dmac_issue_pending; 1258 dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg; 1259 dma_dev->device_prep_peripheral_dma_vec = axi_dmac_prep_peripheral_dma_vec; 1260 dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic; 1261 dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved; 1262 dma_dev->device_terminate_all = axi_dmac_terminate_all; 1263 dma_dev->device_synchronize = axi_dmac_synchronize; 1264 dma_dev->dev = &pdev->dev; 1265 dma_dev->src_addr_widths = BIT(dmac->chan.src_width); 1266 dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width); 1267 dma_dev->directions = BIT(dmac->chan.direction); 1268 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1269 dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */ 1270 INIT_LIST_HEAD(&dma_dev->channels); 1271 1272 dmac->chan.vchan.desc_free = axi_dmac_desc_free; 1273 vchan_init(&dmac->chan.vchan, dma_dev); 1274 1275 ret = axi_dmac_detect_caps(dmac, version); 1276 if (ret) 1277 return ret; 1278 1279 dma_dev->copy_align = (dmac->chan.address_align_mask + 1); 1280 1281 if (dmac->chan.hw_sg) 1282 irq_mask |= AXI_DMAC_IRQ_SOT; 1283 1284 axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, irq_mask); 1285 1286 if (of_dma_is_coherent(pdev->dev.of_node)) { 1287 ret = axi_dmac_read(dmac, AXI_DMAC_REG_COHERENCY_DESC); 1288 1289 if (version < ADI_AXI_PCORE_VER(4, 4, 'a') || 1290 !AXI_DMAC_DST_COHERENT_GET(ret)) { 1291 dev_err(dmac->dma_dev.dev, 1292 "Coherent DMA not supported in hardware"); 1293 return -EINVAL; 1294 } 1295 } 1296 1297 ret = dmaenginem_async_device_register(dma_dev); 1298 if (ret) 1299 return ret; 1300 1301 /* 1302 * Put the action in here so it get's done before unregistering the DMA 1303 * device. 1304 */ 1305 ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_tasklet_kill, 1306 &dmac->chan.vchan.task); 1307 if (ret) 1308 return ret; 1309 1310 ret = of_dma_controller_register(pdev->dev.of_node, 1311 of_dma_xlate_by_chan_id, dma_dev); 1312 if (ret) 1313 return ret; 1314 1315 ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_free_dma_controller, 1316 pdev->dev.of_node); 1317 if (ret) 1318 return ret; 1319 1320 ret = devm_request_irq(&pdev->dev, dmac->irq, axi_dmac_interrupt_handler, 1321 IRQF_SHARED, dev_name(&pdev->dev), dmac); 1322 if (ret) 1323 return ret; 1324 1325 regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base, 1326 &axi_dmac_regmap_config); 1327 1328 return PTR_ERR_OR_ZERO(regmap); 1329 } 1330 1331 static const struct of_device_id axi_dmac_of_match_table[] = { 1332 { .compatible = "adi,axi-dmac-1.00.a" }, 1333 { }, 1334 }; 1335 MODULE_DEVICE_TABLE(of, axi_dmac_of_match_table); 1336 1337 static struct platform_driver axi_dmac_driver = { 1338 .driver = { 1339 .name = "dma-axi-dmac", 1340 .of_match_table = axi_dmac_of_match_table, 1341 }, 1342 .probe = axi_dmac_probe, 1343 }; 1344 module_platform_driver(axi_dmac_driver); 1345 1346 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>"); 1347 MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller"); 1348 MODULE_LICENSE("GPL v2"); 1349