1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Driver for the Analog Devices AXI-DMAC core 4 * 5 * Copyright 2013-2019 Analog Devices Inc. 6 * Author: Lars-Peter Clausen <lars@metafoo.de> 7 */ 8 9 #include <linux/adi-axi-common.h> 10 #include <linux/bitfield.h> 11 #include <linux/cleanup.h> 12 #include <linux/clk.h> 13 #include <linux/device.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/dmaengine.h> 16 #include <linux/err.h> 17 #include <linux/interrupt.h> 18 #include <linux/io.h> 19 #include <linux/kernel.h> 20 #include <linux/module.h> 21 #include <linux/of.h> 22 #include <linux/of_dma.h> 23 #include <linux/of_address.h> 24 #include <linux/platform_device.h> 25 #include <linux/regmap.h> 26 #include <linux/slab.h> 27 28 #include <dt-bindings/dma/axi-dmac.h> 29 30 #include "dmaengine.h" 31 #include "virt-dma.h" 32 33 /* 34 * The AXI-DMAC is a soft IP core that is used in FPGA designs. The core has 35 * various instantiation parameters which decided the exact feature set support 36 * by the core. 37 * 38 * Each channel of the core has a source interface and a destination interface. 39 * The number of channels and the type of the channel interfaces is selected at 40 * configuration time. A interface can either be a connected to a central memory 41 * interconnect, which allows access to system memory, or it can be connected to 42 * a dedicated bus which is directly connected to a data port on a peripheral. 43 * Given that those are configuration options of the core that are selected when 44 * it is instantiated this means that they can not be changed by software at 45 * runtime. By extension this means that each channel is uni-directional. It can 46 * either be device to memory or memory to device, but not both. Also since the 47 * device side is a dedicated data bus only connected to a single peripheral 48 * there is no address than can or needs to be configured for the device side. 49 */ 50 51 #define AXI_DMAC_REG_INTERFACE_DESC 0x10 52 #define AXI_DMAC_DMA_SRC_TYPE_MSK GENMASK(13, 12) 53 #define AXI_DMAC_DMA_SRC_TYPE_GET(x) FIELD_GET(AXI_DMAC_DMA_SRC_TYPE_MSK, x) 54 #define AXI_DMAC_DMA_SRC_WIDTH_MSK GENMASK(11, 8) 55 #define AXI_DMAC_DMA_SRC_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_SRC_WIDTH_MSK, x) 56 #define AXI_DMAC_DMA_DST_TYPE_MSK GENMASK(5, 4) 57 #define AXI_DMAC_DMA_DST_TYPE_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_TYPE_MSK, x) 58 #define AXI_DMAC_DMA_DST_WIDTH_MSK GENMASK(3, 0) 59 #define AXI_DMAC_DMA_DST_WIDTH_GET(x) FIELD_GET(AXI_DMAC_DMA_DST_WIDTH_MSK, x) 60 #define AXI_DMAC_REG_COHERENCY_DESC 0x14 61 #define AXI_DMAC_DST_COHERENT_MSK BIT(0) 62 #define AXI_DMAC_DST_COHERENT_GET(x) FIELD_GET(AXI_DMAC_DST_COHERENT_MSK, x) 63 64 #define AXI_DMAC_REG_IRQ_MASK 0x80 65 #define AXI_DMAC_REG_IRQ_PENDING 0x84 66 #define AXI_DMAC_REG_IRQ_SOURCE 0x88 67 68 #define AXI_DMAC_REG_CTRL 0x400 69 #define AXI_DMAC_REG_TRANSFER_ID 0x404 70 #define AXI_DMAC_REG_START_TRANSFER 0x408 71 #define AXI_DMAC_REG_FLAGS 0x40c 72 #define AXI_DMAC_REG_DEST_ADDRESS 0x410 73 #define AXI_DMAC_REG_DEST_ADDRESS_HIGH 0x490 74 #define AXI_DMAC_REG_SRC_ADDRESS 0x414 75 #define AXI_DMAC_REG_SRC_ADDRESS_HIGH 0x494 76 #define AXI_DMAC_REG_X_LENGTH 0x418 77 #define AXI_DMAC_REG_Y_LENGTH 0x41c 78 #define AXI_DMAC_REG_DEST_STRIDE 0x420 79 #define AXI_DMAC_REG_SRC_STRIDE 0x424 80 #define AXI_DMAC_REG_TRANSFER_DONE 0x428 81 #define AXI_DMAC_REG_ACTIVE_TRANSFER_ID 0x42c 82 #define AXI_DMAC_REG_STATUS 0x430 83 #define AXI_DMAC_REG_CURRENT_SRC_ADDR 0x434 84 #define AXI_DMAC_REG_CURRENT_DEST_ADDR 0x438 85 #define AXI_DMAC_REG_PARTIAL_XFER_LEN 0x44c 86 #define AXI_DMAC_REG_PARTIAL_XFER_ID 0x450 87 #define AXI_DMAC_REG_CURRENT_SG_ID 0x454 88 #define AXI_DMAC_REG_SG_ADDRESS 0x47c 89 #define AXI_DMAC_REG_SG_ADDRESS_HIGH 0x4bc 90 91 #define AXI_DMAC_CTRL_ENABLE BIT(0) 92 #define AXI_DMAC_CTRL_PAUSE BIT(1) 93 #define AXI_DMAC_CTRL_ENABLE_SG BIT(2) 94 95 #define AXI_DMAC_IRQ_SOT BIT(0) 96 #define AXI_DMAC_IRQ_EOT BIT(1) 97 98 #define AXI_DMAC_FLAG_CYCLIC BIT(0) 99 #define AXI_DMAC_FLAG_LAST BIT(1) 100 #define AXI_DMAC_FLAG_PARTIAL_REPORT BIT(2) 101 102 #define AXI_DMAC_FLAG_PARTIAL_XFER_DONE BIT(31) 103 104 /* The maximum ID allocated by the hardware is 31 */ 105 #define AXI_DMAC_SG_UNUSED 32U 106 107 /* Flags for axi_dmac_hw_desc.flags */ 108 #define AXI_DMAC_HW_FLAG_LAST BIT(0) 109 #define AXI_DMAC_HW_FLAG_IRQ BIT(1) 110 111 struct axi_dmac_hw_desc { 112 u32 flags; 113 u32 id; 114 u64 dest_addr; 115 u64 src_addr; 116 u64 next_sg_addr; 117 u32 y_len; 118 u32 x_len; 119 u32 src_stride; 120 u32 dst_stride; 121 u64 __pad[2]; 122 }; 123 124 struct axi_dmac_sg { 125 unsigned int partial_len; 126 bool schedule_when_free; 127 128 struct axi_dmac_hw_desc *hw; 129 dma_addr_t hw_phys; 130 }; 131 132 struct axi_dmac_desc { 133 struct virt_dma_desc vdesc; 134 struct axi_dmac_chan *chan; 135 136 bool cyclic; 137 bool cyclic_eot; 138 bool have_partial_xfer; 139 140 unsigned int num_submitted; 141 unsigned int num_completed; 142 unsigned int num_sgs; 143 struct axi_dmac_sg sg[] __counted_by(num_sgs); 144 }; 145 146 struct axi_dmac_chan { 147 struct virt_dma_chan vchan; 148 149 struct axi_dmac_desc *next_desc; 150 struct list_head active_descs; 151 enum dma_transfer_direction direction; 152 153 unsigned int src_width; 154 unsigned int dest_width; 155 unsigned int src_type; 156 unsigned int dest_type; 157 158 unsigned int max_length; 159 unsigned int address_align_mask; 160 unsigned int length_align_mask; 161 162 bool hw_partial_xfer; 163 bool hw_cyclic; 164 bool hw_2d; 165 bool hw_sg; 166 bool hw_cyclic_hotfix; 167 }; 168 169 struct axi_dmac { 170 void __iomem *base; 171 int irq; 172 173 struct clk *clk; 174 175 struct dma_device dma_dev; 176 struct axi_dmac_chan chan; 177 }; 178 179 static struct axi_dmac *chan_to_axi_dmac(struct axi_dmac_chan *chan) 180 { 181 return container_of(chan->vchan.chan.device, struct axi_dmac, 182 dma_dev); 183 } 184 185 static struct axi_dmac_chan *to_axi_dmac_chan(struct dma_chan *c) 186 { 187 return container_of(c, struct axi_dmac_chan, vchan.chan); 188 } 189 190 static struct axi_dmac_desc *to_axi_dmac_desc(struct virt_dma_desc *vdesc) 191 { 192 return container_of(vdesc, struct axi_dmac_desc, vdesc); 193 } 194 195 static void axi_dmac_write(struct axi_dmac *axi_dmac, unsigned int reg, 196 unsigned int val) 197 { 198 writel(val, axi_dmac->base + reg); 199 } 200 201 static int axi_dmac_read(struct axi_dmac *axi_dmac, unsigned int reg) 202 { 203 return readl(axi_dmac->base + reg); 204 } 205 206 static int axi_dmac_src_is_mem(struct axi_dmac_chan *chan) 207 { 208 return chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM; 209 } 210 211 static int axi_dmac_dest_is_mem(struct axi_dmac_chan *chan) 212 { 213 return chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM; 214 } 215 216 static bool axi_dmac_check_len(struct axi_dmac_chan *chan, unsigned int len) 217 { 218 if (len == 0) 219 return false; 220 if ((len & chan->length_align_mask) != 0) /* Not aligned */ 221 return false; 222 return true; 223 } 224 225 static bool axi_dmac_check_addr(struct axi_dmac_chan *chan, dma_addr_t addr) 226 { 227 if ((addr & chan->address_align_mask) != 0) /* Not aligned */ 228 return false; 229 return true; 230 } 231 232 static struct axi_dmac_desc *axi_dmac_active_desc(struct axi_dmac_chan *chan) 233 { 234 return list_first_entry_or_null(&chan->active_descs, 235 struct axi_dmac_desc, vdesc.node); 236 } 237 238 static struct axi_dmac_desc *axi_dmac_get_next_desc(struct axi_dmac *dmac, 239 struct axi_dmac_chan *chan) 240 { 241 struct axi_dmac_desc *active = axi_dmac_active_desc(chan); 242 struct virt_dma_desc *vdesc; 243 struct axi_dmac_desc *desc; 244 unsigned int val; 245 246 /* 247 * Just play safe and ignore any SOF if we have an active cyclic transfer 248 * flagged to end. We'll start it as soon as the current cyclic one ends. 249 */ 250 if (active && active->cyclic_eot) 251 return NULL; 252 253 /* 254 * It means a SW cyclic transfer is in place so we should just return 255 * the same descriptor. SW cyclic transfer termination is handled 256 * in axi_dmac_transfer_done(). 257 */ 258 if (chan->next_desc) 259 return chan->next_desc; 260 261 vdesc = vchan_next_desc(&chan->vchan); 262 if (!vdesc) 263 return NULL; 264 265 if (active && active->cyclic && !(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) { 266 struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev; 267 268 dev_warn(dev, "Discarding non EOT transfer after cyclic\n"); 269 list_del(&vdesc->node); 270 return NULL; 271 } 272 273 list_move_tail(&vdesc->node, &chan->active_descs); 274 desc = to_axi_dmac_desc(vdesc); 275 chan->next_desc = desc; 276 277 if (!active || !active->cyclic) 278 return desc; 279 280 active->cyclic_eot = true; 281 282 if (chan->hw_sg) { 283 unsigned long flags = AXI_DMAC_HW_FLAG_IRQ | AXI_DMAC_HW_FLAG_LAST; 284 /* 285 * Let's then stop the current cyclic transfer by making sure we 286 * get an EOT interrupt and to open the cyclic loop by marking 287 * the last segment. 288 */ 289 active->sg[active->num_sgs - 1].hw->flags = flags; 290 return NULL; 291 } 292 293 /* 294 * Clear the cyclic bit if there's no Scatter-Gather HW so that we get 295 * at the end of the transfer. 296 */ 297 val = axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS); 298 val &= ~AXI_DMAC_FLAG_CYCLIC; 299 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, val); 300 301 return NULL; 302 } 303 304 static void axi_dmac_start_transfer(struct axi_dmac_chan *chan) 305 { 306 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 307 struct axi_dmac_desc *desc; 308 struct axi_dmac_sg *sg; 309 unsigned int flags = 0; 310 unsigned int val; 311 312 desc = axi_dmac_get_next_desc(dmac, chan); 313 if (!desc) 314 return; 315 316 val = axi_dmac_read(dmac, AXI_DMAC_REG_START_TRANSFER); 317 if (val) /* Queue is full, wait for the next SOT IRQ */ 318 return; 319 320 sg = &desc->sg[desc->num_submitted]; 321 322 /* Already queued in cyclic mode. Wait for it to finish */ 323 if (sg->hw->id != AXI_DMAC_SG_UNUSED) { 324 sg->schedule_when_free = true; 325 return; 326 } 327 328 if (chan->hw_sg) { 329 chan->next_desc = NULL; 330 } else if (++desc->num_submitted == desc->num_sgs || 331 desc->have_partial_xfer) { 332 if (desc->cyclic) 333 desc->num_submitted = 0; /* Start again */ 334 else 335 chan->next_desc = NULL; 336 flags |= AXI_DMAC_FLAG_LAST; 337 } 338 339 sg->hw->id = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_ID); 340 341 if (!chan->hw_sg) { 342 if (axi_dmac_dest_is_mem(chan)) { 343 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, sg->hw->dest_addr); 344 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS_HIGH, 345 sg->hw->dest_addr >> 32); 346 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_STRIDE, sg->hw->dst_stride); 347 } 348 349 if (axi_dmac_src_is_mem(chan)) { 350 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, sg->hw->src_addr); 351 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS_HIGH, sg->hw->src_addr >> 32); 352 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_STRIDE, sg->hw->src_stride); 353 } 354 } 355 356 /* 357 * If the hardware supports cyclic transfers and there is no callback to 358 * call, enable hw cyclic mode to avoid unnecessary interrupts. 359 */ 360 if (chan->hw_cyclic && desc->cyclic && !desc->vdesc.tx.callback) { 361 if (chan->hw_sg) { 362 desc->sg[desc->num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_IRQ; 363 } else if (desc->num_sgs == 1) { 364 chan->next_desc = NULL; 365 flags |= AXI_DMAC_FLAG_CYCLIC; 366 } 367 } 368 369 if (chan->hw_partial_xfer) 370 flags |= AXI_DMAC_FLAG_PARTIAL_REPORT; 371 372 if (chan->hw_sg) { 373 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, (u32)sg->hw_phys); 374 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS_HIGH, 375 (u64)sg->hw_phys >> 32); 376 } else { 377 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, sg->hw->x_len); 378 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, sg->hw->y_len); 379 } 380 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, flags); 381 axi_dmac_write(dmac, AXI_DMAC_REG_START_TRANSFER, 1); 382 } 383 384 static inline unsigned int axi_dmac_total_sg_bytes(struct axi_dmac_chan *chan, 385 struct axi_dmac_sg *sg) 386 { 387 if (chan->hw_2d) 388 return (sg->hw->x_len + 1) * (sg->hw->y_len + 1); 389 else 390 return (sg->hw->x_len + 1); 391 } 392 393 static void axi_dmac_dequeue_partial_xfers(struct axi_dmac_chan *chan) 394 { 395 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 396 struct axi_dmac_desc *desc; 397 struct axi_dmac_sg *sg; 398 u32 xfer_done, len, id, i; 399 bool found_sg; 400 401 do { 402 len = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_LEN); 403 id = axi_dmac_read(dmac, AXI_DMAC_REG_PARTIAL_XFER_ID); 404 405 found_sg = false; 406 list_for_each_entry(desc, &chan->active_descs, vdesc.node) { 407 for (i = 0; i < desc->num_sgs; i++) { 408 sg = &desc->sg[i]; 409 if (sg->hw->id == AXI_DMAC_SG_UNUSED) 410 continue; 411 if (sg->hw->id == id) { 412 desc->have_partial_xfer = true; 413 sg->partial_len = len; 414 found_sg = true; 415 break; 416 } 417 } 418 if (found_sg) 419 break; 420 } 421 422 if (found_sg) { 423 dev_dbg(dmac->dma_dev.dev, 424 "Found partial segment id=%u, len=%u\n", 425 id, len); 426 } else { 427 dev_warn(dmac->dma_dev.dev, 428 "Not found partial segment id=%u, len=%u\n", 429 id, len); 430 } 431 432 /* Check if we have any more partial transfers */ 433 xfer_done = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE); 434 xfer_done = !(xfer_done & AXI_DMAC_FLAG_PARTIAL_XFER_DONE); 435 436 } while (!xfer_done); 437 } 438 439 static void axi_dmac_compute_residue(struct axi_dmac_chan *chan, 440 struct axi_dmac_desc *active) 441 { 442 struct dmaengine_result *rslt = &active->vdesc.tx_result; 443 unsigned int start = active->num_completed - 1; 444 struct axi_dmac_sg *sg; 445 unsigned int i, total; 446 447 rslt->result = DMA_TRANS_NOERROR; 448 rslt->residue = 0; 449 450 if (chan->hw_sg) 451 return; 452 453 /* 454 * We get here if the last completed segment is partial, which 455 * means we can compute the residue from that segment onwards 456 */ 457 for (i = start; i < active->num_sgs; i++) { 458 sg = &active->sg[i]; 459 total = axi_dmac_total_sg_bytes(chan, sg); 460 rslt->residue += (total - sg->partial_len); 461 } 462 } 463 464 static bool axi_dmac_handle_cyclic_eot(struct axi_dmac_chan *chan, 465 struct axi_dmac_desc *active) 466 { 467 struct device *dev = chan_to_axi_dmac(chan)->dma_dev.dev; 468 struct virt_dma_desc *vdesc; 469 470 /* wrap around */ 471 active->num_completed = 0; 472 473 if (active->cyclic_eot) { 474 /* 475 * It means an HW cyclic transfer was marked to stop. And we 476 * know we have something to schedule, so start the next 477 * transfer now the cyclic one is done. 478 */ 479 list_del(&active->vdesc.node); 480 vchan_cookie_complete(&active->vdesc); 481 482 if (chan->hw_cyclic_hotfix) { 483 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 484 /* 485 * In older IP cores, ending a cyclic transfer by clearing 486 * the CYCLIC flag does not guarantee a graceful end. 487 * It can happen that some data (of the next frame) is 488 * already prefetched and will be wrongly visible in the 489 * next transfer. To workaround this, we need to reenable 490 * the core so everything is flushed. Newer cores handles 491 * this correctly and do not require this "hotfix". The 492 * SG IP also does not require this. 493 */ 494 dev_dbg(dev, "HW cyclic hotfix\n"); 495 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0); 496 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, AXI_DMAC_CTRL_ENABLE); 497 } 498 499 return true; 500 } 501 502 vdesc = vchan_next_desc(&chan->vchan); 503 if (!vdesc) 504 return false; 505 if (!(vdesc->tx.flags & DMA_PREP_LOAD_EOT)) { 506 dev_warn(dev, "Discarding non EOT transfer after cyclic\n"); 507 list_del(&vdesc->node); 508 return false; 509 } 510 511 /* then let's end the cyclic transfer */ 512 chan->next_desc = NULL; 513 list_del(&active->vdesc.node); 514 vchan_cookie_complete(&active->vdesc); 515 516 return true; 517 } 518 519 static bool axi_dmac_transfer_done(struct axi_dmac_chan *chan, 520 unsigned int completed_transfers) 521 { 522 struct axi_dmac_desc *active; 523 struct axi_dmac_sg *sg; 524 bool start_next = false; 525 526 active = axi_dmac_active_desc(chan); 527 if (!active) 528 return false; 529 530 if (chan->hw_partial_xfer && 531 (completed_transfers & AXI_DMAC_FLAG_PARTIAL_XFER_DONE)) 532 axi_dmac_dequeue_partial_xfers(chan); 533 534 if (chan->hw_sg) { 535 if (active->cyclic) { 536 vchan_cyclic_callback(&active->vdesc); 537 start_next = axi_dmac_handle_cyclic_eot(chan, active); 538 } else { 539 list_del(&active->vdesc.node); 540 vchan_cookie_complete(&active->vdesc); 541 active = axi_dmac_active_desc(chan); 542 start_next = !!active; 543 } 544 } else { 545 do { 546 sg = &active->sg[active->num_completed]; 547 if (sg->hw->id == AXI_DMAC_SG_UNUSED) /* Not yet submitted */ 548 break; 549 if (!(BIT(sg->hw->id) & completed_transfers)) 550 break; 551 active->num_completed++; 552 sg->hw->id = AXI_DMAC_SG_UNUSED; 553 if (sg->schedule_when_free) { 554 sg->schedule_when_free = false; 555 start_next = true; 556 } 557 558 if (sg->partial_len) 559 axi_dmac_compute_residue(chan, active); 560 561 if (active->cyclic) 562 vchan_cyclic_callback(&active->vdesc); 563 564 if (active->num_completed == active->num_sgs || 565 sg->partial_len) { 566 if (active->cyclic) { 567 /* keep start_next as is, if already true... */ 568 start_next |= axi_dmac_handle_cyclic_eot(chan, active); 569 } else { 570 list_del(&active->vdesc.node); 571 vchan_cookie_complete(&active->vdesc); 572 active = axi_dmac_active_desc(chan); 573 } 574 } 575 } while (active); 576 } 577 578 return start_next; 579 } 580 581 static irqreturn_t axi_dmac_interrupt_handler(int irq, void *devid) 582 { 583 struct axi_dmac *dmac = devid; 584 unsigned int pending; 585 bool start_next = false; 586 587 pending = axi_dmac_read(dmac, AXI_DMAC_REG_IRQ_PENDING); 588 if (!pending) 589 return IRQ_NONE; 590 591 axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_PENDING, pending); 592 593 spin_lock(&dmac->chan.vchan.lock); 594 /* One or more transfers have finished */ 595 if (pending & AXI_DMAC_IRQ_EOT) { 596 unsigned int completed; 597 598 completed = axi_dmac_read(dmac, AXI_DMAC_REG_TRANSFER_DONE); 599 start_next = axi_dmac_transfer_done(&dmac->chan, completed); 600 } 601 /* Space has become available in the descriptor queue */ 602 if ((pending & AXI_DMAC_IRQ_SOT) || start_next) 603 axi_dmac_start_transfer(&dmac->chan); 604 spin_unlock(&dmac->chan.vchan.lock); 605 606 return IRQ_HANDLED; 607 } 608 609 static int axi_dmac_terminate_all(struct dma_chan *c) 610 { 611 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 612 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 613 unsigned long flags; 614 LIST_HEAD(head); 615 616 spin_lock_irqsave(&chan->vchan.lock, flags); 617 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, 0); 618 chan->next_desc = NULL; 619 vchan_get_all_descriptors(&chan->vchan, &head); 620 list_splice_tail_init(&chan->active_descs, &head); 621 spin_unlock_irqrestore(&chan->vchan.lock, flags); 622 623 vchan_dma_desc_free_list(&chan->vchan, &head); 624 625 return 0; 626 } 627 628 static void axi_dmac_synchronize(struct dma_chan *c) 629 { 630 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 631 632 vchan_synchronize(&chan->vchan); 633 } 634 635 static void axi_dmac_issue_pending(struct dma_chan *c) 636 { 637 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 638 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 639 unsigned long flags; 640 u32 ctrl = AXI_DMAC_CTRL_ENABLE; 641 642 if (chan->hw_sg) 643 ctrl |= AXI_DMAC_CTRL_ENABLE_SG; 644 645 axi_dmac_write(dmac, AXI_DMAC_REG_CTRL, ctrl); 646 647 spin_lock_irqsave(&chan->vchan.lock, flags); 648 if (vchan_issue_pending(&chan->vchan)) 649 axi_dmac_start_transfer(chan); 650 spin_unlock_irqrestore(&chan->vchan.lock, flags); 651 } 652 653 static struct axi_dmac_desc * 654 axi_dmac_alloc_desc(struct axi_dmac_chan *chan, unsigned int num_sgs) 655 { 656 struct axi_dmac *dmac = chan_to_axi_dmac(chan); 657 struct device *dev = dmac->dma_dev.dev; 658 struct axi_dmac_hw_desc *hws; 659 struct axi_dmac_desc *desc; 660 dma_addr_t hw_phys; 661 unsigned int i; 662 663 desc = kzalloc_flex(*desc, sg, num_sgs, GFP_NOWAIT); 664 if (!desc) 665 return NULL; 666 desc->num_sgs = num_sgs; 667 desc->chan = chan; 668 669 hws = dma_alloc_coherent(dev, PAGE_ALIGN(num_sgs * sizeof(*hws)), 670 &hw_phys, GFP_ATOMIC); 671 if (!hws) { 672 kfree(desc); 673 return NULL; 674 } 675 676 for (i = 0; i < num_sgs; i++) { 677 desc->sg[i].hw = &hws[i]; 678 desc->sg[i].hw_phys = hw_phys + i * sizeof(*hws); 679 680 hws[i].id = AXI_DMAC_SG_UNUSED; 681 hws[i].flags = 0; 682 683 /* Link hardware descriptors */ 684 hws[i].next_sg_addr = hw_phys + (i + 1) * sizeof(*hws); 685 } 686 687 /* The last hardware descriptor will trigger an interrupt */ 688 desc->sg[num_sgs - 1].hw->flags = AXI_DMAC_HW_FLAG_LAST | AXI_DMAC_HW_FLAG_IRQ; 689 690 return desc; 691 } 692 693 static void axi_dmac_free_desc(struct axi_dmac_desc *desc) 694 { 695 struct axi_dmac *dmac = chan_to_axi_dmac(desc->chan); 696 struct device *dev = dmac->dma_dev.dev; 697 struct axi_dmac_hw_desc *hw = desc->sg[0].hw; 698 dma_addr_t hw_phys = desc->sg[0].hw_phys; 699 700 dma_free_coherent(dev, PAGE_ALIGN(desc->num_sgs * sizeof(*hw)), 701 hw, hw_phys); 702 kfree(desc); 703 } 704 705 static struct axi_dmac_sg *axi_dmac_fill_linear_sg(struct axi_dmac_chan *chan, 706 enum dma_transfer_direction direction, dma_addr_t addr, 707 unsigned int num_periods, unsigned int period_len, 708 struct axi_dmac_sg *sg) 709 { 710 unsigned int num_segments, i; 711 unsigned int segment_size; 712 unsigned int len; 713 714 /* Split into multiple equally sized segments if necessary */ 715 num_segments = DIV_ROUND_UP(period_len, chan->max_length); 716 segment_size = DIV_ROUND_UP(period_len, num_segments); 717 /* Take care of alignment */ 718 segment_size = ((segment_size - 1) | chan->length_align_mask) + 1; 719 720 for (i = 0; i < num_periods; i++) { 721 for (len = period_len; len > segment_size; sg++) { 722 if (direction == DMA_DEV_TO_MEM) 723 sg->hw->dest_addr = addr; 724 else 725 sg->hw->src_addr = addr; 726 sg->hw->x_len = segment_size - 1; 727 sg->hw->y_len = 0; 728 sg->hw->flags = 0; 729 addr += segment_size; 730 len -= segment_size; 731 } 732 733 if (direction == DMA_DEV_TO_MEM) 734 sg->hw->dest_addr = addr; 735 else 736 sg->hw->src_addr = addr; 737 sg->hw->x_len = len - 1; 738 sg->hw->y_len = 0; 739 sg++; 740 addr += len; 741 } 742 743 return sg; 744 } 745 746 static struct dma_async_tx_descriptor * 747 axi_dmac_prep_peripheral_dma_vec(struct dma_chan *c, const struct dma_vec *vecs, 748 size_t nb, enum dma_transfer_direction direction, 749 unsigned long flags) 750 { 751 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 752 struct axi_dmac_desc *desc; 753 unsigned int num_sgs = 0; 754 struct axi_dmac_sg *dsg; 755 size_t i; 756 757 if (direction != chan->direction) 758 return NULL; 759 760 for (i = 0; i < nb; i++) 761 num_sgs += DIV_ROUND_UP(vecs[i].len, chan->max_length); 762 763 desc = axi_dmac_alloc_desc(chan, num_sgs); 764 if (!desc) 765 return NULL; 766 767 dsg = desc->sg; 768 769 for (i = 0; i < nb; i++) { 770 if (!axi_dmac_check_addr(chan, vecs[i].addr) || 771 !axi_dmac_check_len(chan, vecs[i].len)) { 772 kfree(desc); 773 return NULL; 774 } 775 776 dsg = axi_dmac_fill_linear_sg(chan, direction, vecs[i].addr, 1, 777 vecs[i].len, dsg); 778 } 779 780 desc->cyclic = flags & DMA_PREP_REPEAT; 781 if (desc->cyclic) { 782 /* Chain the last descriptor to the first, and remove its "last" flag */ 783 desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST; 784 desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys; 785 } 786 787 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 788 } 789 790 static struct dma_async_tx_descriptor *axi_dmac_prep_slave_sg( 791 struct dma_chan *c, struct scatterlist *sgl, 792 unsigned int sg_len, enum dma_transfer_direction direction, 793 unsigned long flags, void *context) 794 { 795 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 796 struct axi_dmac_desc *desc; 797 struct axi_dmac_sg *dsg; 798 struct scatterlist *sg; 799 unsigned int num_sgs; 800 unsigned int i; 801 802 if (direction != chan->direction) 803 return NULL; 804 805 num_sgs = sg_nents_for_dma(sgl, sg_len, chan->max_length); 806 desc = axi_dmac_alloc_desc(chan, num_sgs); 807 if (!desc) 808 return NULL; 809 810 dsg = desc->sg; 811 812 for_each_sg(sgl, sg, sg_len, i) { 813 if (!axi_dmac_check_addr(chan, sg_dma_address(sg)) || 814 !axi_dmac_check_len(chan, sg_dma_len(sg))) { 815 axi_dmac_free_desc(desc); 816 return NULL; 817 } 818 819 dsg = axi_dmac_fill_linear_sg(chan, direction, sg_dma_address(sg), 1, 820 sg_dma_len(sg), dsg); 821 } 822 823 desc->cyclic = false; 824 825 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 826 } 827 828 static struct dma_async_tx_descriptor *axi_dmac_prep_dma_cyclic( 829 struct dma_chan *c, dma_addr_t buf_addr, size_t buf_len, 830 size_t period_len, enum dma_transfer_direction direction, 831 unsigned long flags) 832 { 833 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 834 struct axi_dmac_desc *desc; 835 unsigned int num_periods, num_segments, num_sgs; 836 837 if (direction != chan->direction) 838 return NULL; 839 840 if (!axi_dmac_check_len(chan, buf_len) || 841 !axi_dmac_check_addr(chan, buf_addr)) 842 return NULL; 843 844 if (period_len == 0 || buf_len % period_len) 845 return NULL; 846 847 num_periods = buf_len / period_len; 848 num_segments = DIV_ROUND_UP(period_len, chan->max_length); 849 num_sgs = num_periods * num_segments; 850 851 desc = axi_dmac_alloc_desc(chan, num_sgs); 852 if (!desc) 853 return NULL; 854 855 /* Chain the last descriptor to the first, and remove its "last" flag */ 856 desc->sg[num_sgs - 1].hw->next_sg_addr = desc->sg[0].hw_phys; 857 desc->sg[num_sgs - 1].hw->flags &= ~AXI_DMAC_HW_FLAG_LAST; 858 859 axi_dmac_fill_linear_sg(chan, direction, buf_addr, num_periods, 860 period_len, desc->sg); 861 862 desc->cyclic = true; 863 864 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 865 } 866 867 static struct dma_async_tx_descriptor *axi_dmac_prep_interleaved( 868 struct dma_chan *c, struct dma_interleaved_template *xt, 869 unsigned long flags) 870 { 871 struct axi_dmac_chan *chan = to_axi_dmac_chan(c); 872 struct axi_dmac_desc *desc; 873 size_t dst_icg, src_icg; 874 875 if (xt->frame_size != 1) 876 return NULL; 877 878 if (xt->dir != chan->direction) 879 return NULL; 880 881 if (axi_dmac_src_is_mem(chan)) { 882 if (!xt->src_inc || !axi_dmac_check_addr(chan, xt->src_start)) 883 return NULL; 884 } 885 886 if (axi_dmac_dest_is_mem(chan)) { 887 if (!xt->dst_inc || !axi_dmac_check_addr(chan, xt->dst_start)) 888 return NULL; 889 } 890 891 dst_icg = dmaengine_get_dst_icg(xt, &xt->sgl[0]); 892 src_icg = dmaengine_get_src_icg(xt, &xt->sgl[0]); 893 894 if (chan->hw_2d) { 895 if (!axi_dmac_check_len(chan, xt->sgl[0].size) || 896 xt->numf == 0) 897 return NULL; 898 if (xt->sgl[0].size + dst_icg > chan->max_length || 899 xt->sgl[0].size + src_icg > chan->max_length) 900 return NULL; 901 } else { 902 if (dst_icg != 0 || src_icg != 0) 903 return NULL; 904 if (chan->max_length / xt->sgl[0].size < xt->numf) 905 return NULL; 906 if (!axi_dmac_check_len(chan, xt->sgl[0].size * xt->numf)) 907 return NULL; 908 } 909 910 desc = axi_dmac_alloc_desc(chan, 1); 911 if (!desc) 912 return NULL; 913 914 if (axi_dmac_src_is_mem(chan)) { 915 desc->sg[0].hw->src_addr = xt->src_start; 916 desc->sg[0].hw->src_stride = xt->sgl[0].size + src_icg; 917 } 918 919 if (axi_dmac_dest_is_mem(chan)) { 920 desc->sg[0].hw->dest_addr = xt->dst_start; 921 desc->sg[0].hw->dst_stride = xt->sgl[0].size + dst_icg; 922 } 923 924 if (chan->hw_2d) { 925 desc->sg[0].hw->x_len = xt->sgl[0].size - 1; 926 desc->sg[0].hw->y_len = xt->numf - 1; 927 } else { 928 desc->sg[0].hw->x_len = xt->sgl[0].size * xt->numf - 1; 929 desc->sg[0].hw->y_len = 0; 930 } 931 932 if (flags & DMA_CYCLIC) 933 desc->cyclic = true; 934 935 return vchan_tx_prep(&chan->vchan, &desc->vdesc, flags); 936 } 937 938 static void axi_dmac_free_chan_resources(struct dma_chan *c) 939 { 940 vchan_free_chan_resources(to_virt_chan(c)); 941 } 942 943 static void axi_dmac_desc_free(struct virt_dma_desc *vdesc) 944 { 945 axi_dmac_free_desc(to_axi_dmac_desc(vdesc)); 946 } 947 948 static bool axi_dmac_regmap_rdwr(struct device *dev, unsigned int reg) 949 { 950 switch (reg) { 951 case AXI_DMAC_REG_IRQ_MASK: 952 case AXI_DMAC_REG_IRQ_SOURCE: 953 case AXI_DMAC_REG_IRQ_PENDING: 954 case AXI_DMAC_REG_CTRL: 955 case AXI_DMAC_REG_TRANSFER_ID: 956 case AXI_DMAC_REG_START_TRANSFER: 957 case AXI_DMAC_REG_FLAGS: 958 case AXI_DMAC_REG_DEST_ADDRESS: 959 case AXI_DMAC_REG_SRC_ADDRESS: 960 case AXI_DMAC_REG_X_LENGTH: 961 case AXI_DMAC_REG_Y_LENGTH: 962 case AXI_DMAC_REG_DEST_STRIDE: 963 case AXI_DMAC_REG_SRC_STRIDE: 964 case AXI_DMAC_REG_TRANSFER_DONE: 965 case AXI_DMAC_REG_ACTIVE_TRANSFER_ID: 966 case AXI_DMAC_REG_STATUS: 967 case AXI_DMAC_REG_CURRENT_SRC_ADDR: 968 case AXI_DMAC_REG_CURRENT_DEST_ADDR: 969 case AXI_DMAC_REG_PARTIAL_XFER_LEN: 970 case AXI_DMAC_REG_PARTIAL_XFER_ID: 971 case AXI_DMAC_REG_CURRENT_SG_ID: 972 case AXI_DMAC_REG_SG_ADDRESS: 973 case AXI_DMAC_REG_SG_ADDRESS_HIGH: 974 return true; 975 default: 976 return false; 977 } 978 } 979 980 static const struct regmap_config axi_dmac_regmap_config = { 981 .reg_bits = 32, 982 .val_bits = 32, 983 .reg_stride = 4, 984 .max_register = AXI_DMAC_REG_PARTIAL_XFER_ID, 985 .readable_reg = axi_dmac_regmap_rdwr, 986 .writeable_reg = axi_dmac_regmap_rdwr, 987 }; 988 989 static void axi_dmac_adjust_chan_params(struct axi_dmac_chan *chan) 990 { 991 chan->address_align_mask = max(chan->dest_width, chan->src_width) - 1; 992 993 if (axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan)) 994 chan->direction = DMA_MEM_TO_MEM; 995 else if (!axi_dmac_dest_is_mem(chan) && axi_dmac_src_is_mem(chan)) 996 chan->direction = DMA_MEM_TO_DEV; 997 else if (axi_dmac_dest_is_mem(chan) && !axi_dmac_src_is_mem(chan)) 998 chan->direction = DMA_DEV_TO_MEM; 999 else 1000 chan->direction = DMA_DEV_TO_DEV; 1001 } 1002 1003 /* 1004 * The configuration stored in the devicetree matches the configuration 1005 * parameters of the peripheral instance and allows the driver to know which 1006 * features are implemented and how it should behave. 1007 */ 1008 static int axi_dmac_parse_chan_dt(struct device_node *of_chan, 1009 struct axi_dmac_chan *chan) 1010 { 1011 u32 val; 1012 int ret; 1013 1014 ret = of_property_read_u32(of_chan, "reg", &val); 1015 if (ret) 1016 return ret; 1017 1018 /* We only support 1 channel for now */ 1019 if (val != 0) 1020 return -EINVAL; 1021 1022 ret = of_property_read_u32(of_chan, "adi,source-bus-type", &val); 1023 if (ret) 1024 return ret; 1025 if (val > AXI_DMAC_BUS_TYPE_FIFO) 1026 return -EINVAL; 1027 chan->src_type = val; 1028 1029 ret = of_property_read_u32(of_chan, "adi,destination-bus-type", &val); 1030 if (ret) 1031 return ret; 1032 if (val > AXI_DMAC_BUS_TYPE_FIFO) 1033 return -EINVAL; 1034 chan->dest_type = val; 1035 1036 ret = of_property_read_u32(of_chan, "adi,source-bus-width", &val); 1037 if (ret) 1038 return ret; 1039 chan->src_width = val / 8; 1040 1041 ret = of_property_read_u32(of_chan, "adi,destination-bus-width", &val); 1042 if (ret) 1043 return ret; 1044 chan->dest_width = val / 8; 1045 1046 axi_dmac_adjust_chan_params(chan); 1047 1048 return 0; 1049 } 1050 1051 static int axi_dmac_parse_dt(struct device *dev, struct axi_dmac *dmac) 1052 { 1053 int ret; 1054 1055 struct device_node *of_channels __free(device_node) = of_get_child_by_name(dev->of_node, 1056 "adi,channels"); 1057 if (of_channels == NULL) 1058 return -ENODEV; 1059 1060 for_each_child_of_node_scoped(of_channels, of_chan) { 1061 ret = axi_dmac_parse_chan_dt(of_chan, &dmac->chan); 1062 if (ret) 1063 return -EINVAL; 1064 } 1065 1066 return 0; 1067 } 1068 1069 static int axi_dmac_read_chan_config(struct device *dev, struct axi_dmac *dmac) 1070 { 1071 struct axi_dmac_chan *chan = &dmac->chan; 1072 unsigned int val, desc; 1073 1074 desc = axi_dmac_read(dmac, AXI_DMAC_REG_INTERFACE_DESC); 1075 if (desc == 0) { 1076 dev_err(dev, "DMA interface register reads zero\n"); 1077 return -EFAULT; 1078 } 1079 1080 val = AXI_DMAC_DMA_SRC_TYPE_GET(desc); 1081 if (val > AXI_DMAC_BUS_TYPE_FIFO) { 1082 dev_err(dev, "Invalid source bus type read: %d\n", val); 1083 return -EINVAL; 1084 } 1085 chan->src_type = val; 1086 1087 val = AXI_DMAC_DMA_DST_TYPE_GET(desc); 1088 if (val > AXI_DMAC_BUS_TYPE_FIFO) { 1089 dev_err(dev, "Invalid destination bus type read: %d\n", val); 1090 return -EINVAL; 1091 } 1092 chan->dest_type = val; 1093 1094 val = AXI_DMAC_DMA_SRC_WIDTH_GET(desc); 1095 if (val == 0) { 1096 dev_err(dev, "Source bus width is zero\n"); 1097 return -EINVAL; 1098 } 1099 /* widths are stored in log2 */ 1100 chan->src_width = 1 << val; 1101 1102 val = AXI_DMAC_DMA_DST_WIDTH_GET(desc); 1103 if (val == 0) { 1104 dev_err(dev, "Destination bus width is zero\n"); 1105 return -EINVAL; 1106 } 1107 chan->dest_width = 1 << val; 1108 1109 axi_dmac_adjust_chan_params(chan); 1110 1111 return 0; 1112 } 1113 1114 static int axi_dmac_detect_caps(struct axi_dmac *dmac, unsigned int version) 1115 { 1116 struct axi_dmac_chan *chan = &dmac->chan; 1117 struct device *dev = dmac->dma_dev.dev; 1118 u32 mask; 1119 int ret; 1120 1121 axi_dmac_write(dmac, AXI_DMAC_REG_FLAGS, AXI_DMAC_FLAG_CYCLIC); 1122 if (axi_dmac_read(dmac, AXI_DMAC_REG_FLAGS) == AXI_DMAC_FLAG_CYCLIC) 1123 chan->hw_cyclic = true; 1124 1125 axi_dmac_write(dmac, AXI_DMAC_REG_SG_ADDRESS, 0xffffffff); 1126 if (axi_dmac_read(dmac, AXI_DMAC_REG_SG_ADDRESS)) 1127 chan->hw_sg = true; 1128 1129 axi_dmac_write(dmac, AXI_DMAC_REG_Y_LENGTH, 1); 1130 if (axi_dmac_read(dmac, AXI_DMAC_REG_Y_LENGTH) == 1) 1131 chan->hw_2d = true; 1132 1133 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0xffffffff); 1134 chan->max_length = axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH); 1135 if (chan->max_length != UINT_MAX) 1136 chan->max_length++; 1137 1138 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS, 0xffffffff); 1139 if (axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS) == 0 && 1140 chan->dest_type == AXI_DMAC_BUS_TYPE_AXI_MM) { 1141 dev_err(dmac->dma_dev.dev, 1142 "Destination memory-mapped interface not supported."); 1143 return -ENODEV; 1144 } 1145 1146 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS, 0xffffffff); 1147 if (axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS) == 0 && 1148 chan->src_type == AXI_DMAC_BUS_TYPE_AXI_MM) { 1149 dev_err(dmac->dma_dev.dev, 1150 "Source memory-mapped interface not supported."); 1151 return -ENODEV; 1152 } 1153 1154 if (axi_dmac_dest_is_mem(chan)) { 1155 axi_dmac_write(dmac, AXI_DMAC_REG_DEST_ADDRESS_HIGH, 0xffffffff); 1156 mask = axi_dmac_read(dmac, AXI_DMAC_REG_DEST_ADDRESS_HIGH); 1157 } else { 1158 axi_dmac_write(dmac, AXI_DMAC_REG_SRC_ADDRESS_HIGH, 0xffffffff); 1159 mask = axi_dmac_read(dmac, AXI_DMAC_REG_SRC_ADDRESS_HIGH); 1160 } 1161 1162 mask = 32 + fls(mask); 1163 1164 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(mask)); 1165 if (ret) { 1166 dev_err(dev, "DMA mask set error %d\n", ret); 1167 return ret; 1168 } 1169 1170 if (version >= ADI_AXI_PCORE_VER(4, 2, 'a')) 1171 chan->hw_partial_xfer = true; 1172 1173 if (version >= ADI_AXI_PCORE_VER(4, 1, 'a')) { 1174 axi_dmac_write(dmac, AXI_DMAC_REG_X_LENGTH, 0x00); 1175 chan->length_align_mask = 1176 axi_dmac_read(dmac, AXI_DMAC_REG_X_LENGTH); 1177 } else { 1178 chan->length_align_mask = chan->address_align_mask; 1179 } 1180 1181 if (version < ADI_AXI_PCORE_VER(4, 6, 0) && !chan->hw_sg) 1182 chan->hw_cyclic_hotfix = true; 1183 1184 return 0; 1185 } 1186 1187 static void axi_dmac_tasklet_kill(void *task) 1188 { 1189 tasklet_kill(task); 1190 } 1191 1192 static void axi_dmac_free_dma_controller(void *of_node) 1193 { 1194 of_dma_controller_free(of_node); 1195 } 1196 1197 static int axi_dmac_probe(struct platform_device *pdev) 1198 { 1199 struct dma_device *dma_dev; 1200 struct axi_dmac *dmac; 1201 struct regmap *regmap; 1202 unsigned int version; 1203 u32 irq_mask = 0; 1204 int ret; 1205 1206 dmac = devm_kzalloc(&pdev->dev, sizeof(*dmac), GFP_KERNEL); 1207 if (!dmac) 1208 return -ENOMEM; 1209 1210 dmac->irq = platform_get_irq(pdev, 0); 1211 if (dmac->irq < 0) 1212 return dmac->irq; 1213 if (dmac->irq == 0) 1214 return -EINVAL; 1215 1216 dmac->base = devm_platform_ioremap_resource(pdev, 0); 1217 if (IS_ERR(dmac->base)) 1218 return PTR_ERR(dmac->base); 1219 1220 dmac->clk = devm_clk_get_enabled(&pdev->dev, NULL); 1221 if (IS_ERR(dmac->clk)) 1222 return PTR_ERR(dmac->clk); 1223 1224 version = axi_dmac_read(dmac, ADI_AXI_REG_VERSION); 1225 1226 if (version >= ADI_AXI_PCORE_VER(4, 3, 'a')) 1227 ret = axi_dmac_read_chan_config(&pdev->dev, dmac); 1228 else 1229 ret = axi_dmac_parse_dt(&pdev->dev, dmac); 1230 1231 if (ret < 0) 1232 return ret; 1233 1234 INIT_LIST_HEAD(&dmac->chan.active_descs); 1235 1236 dma_set_max_seg_size(&pdev->dev, UINT_MAX); 1237 1238 dma_dev = &dmac->dma_dev; 1239 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); 1240 dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask); 1241 dma_cap_set(DMA_INTERLEAVE, dma_dev->cap_mask); 1242 dma_dev->device_free_chan_resources = axi_dmac_free_chan_resources; 1243 dma_dev->device_tx_status = dma_cookie_status; 1244 dma_dev->device_issue_pending = axi_dmac_issue_pending; 1245 dma_dev->device_prep_slave_sg = axi_dmac_prep_slave_sg; 1246 dma_dev->device_prep_peripheral_dma_vec = axi_dmac_prep_peripheral_dma_vec; 1247 dma_dev->device_prep_dma_cyclic = axi_dmac_prep_dma_cyclic; 1248 dma_dev->device_prep_interleaved_dma = axi_dmac_prep_interleaved; 1249 dma_dev->device_terminate_all = axi_dmac_terminate_all; 1250 dma_dev->device_synchronize = axi_dmac_synchronize; 1251 dma_dev->dev = &pdev->dev; 1252 dma_dev->src_addr_widths = BIT(dmac->chan.src_width); 1253 dma_dev->dst_addr_widths = BIT(dmac->chan.dest_width); 1254 dma_dev->directions = BIT(dmac->chan.direction); 1255 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 1256 dma_dev->max_sg_burst = 31; /* 31 SGs maximum in one burst */ 1257 INIT_LIST_HEAD(&dma_dev->channels); 1258 1259 dmac->chan.vchan.desc_free = axi_dmac_desc_free; 1260 vchan_init(&dmac->chan.vchan, dma_dev); 1261 1262 ret = axi_dmac_detect_caps(dmac, version); 1263 if (ret) 1264 return ret; 1265 1266 dma_dev->copy_align = (dmac->chan.address_align_mask + 1); 1267 1268 if (dmac->chan.hw_sg) 1269 irq_mask |= AXI_DMAC_IRQ_SOT; 1270 1271 axi_dmac_write(dmac, AXI_DMAC_REG_IRQ_MASK, irq_mask); 1272 1273 if (of_dma_is_coherent(pdev->dev.of_node)) { 1274 ret = axi_dmac_read(dmac, AXI_DMAC_REG_COHERENCY_DESC); 1275 1276 if (version < ADI_AXI_PCORE_VER(4, 4, 'a') || 1277 !AXI_DMAC_DST_COHERENT_GET(ret)) { 1278 dev_err(dmac->dma_dev.dev, 1279 "Coherent DMA not supported in hardware"); 1280 return -EINVAL; 1281 } 1282 } 1283 1284 ret = dmaenginem_async_device_register(dma_dev); 1285 if (ret) 1286 return ret; 1287 1288 /* 1289 * Put the action in here so it get's done before unregistering the DMA 1290 * device. 1291 */ 1292 ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_tasklet_kill, 1293 &dmac->chan.vchan.task); 1294 if (ret) 1295 return ret; 1296 1297 ret = of_dma_controller_register(pdev->dev.of_node, 1298 of_dma_xlate_by_chan_id, dma_dev); 1299 if (ret) 1300 return ret; 1301 1302 ret = devm_add_action_or_reset(&pdev->dev, axi_dmac_free_dma_controller, 1303 pdev->dev.of_node); 1304 if (ret) 1305 return ret; 1306 1307 ret = devm_request_irq(&pdev->dev, dmac->irq, axi_dmac_interrupt_handler, 1308 IRQF_SHARED, dev_name(&pdev->dev), dmac); 1309 if (ret) 1310 return ret; 1311 1312 regmap = devm_regmap_init_mmio(&pdev->dev, dmac->base, 1313 &axi_dmac_regmap_config); 1314 1315 return PTR_ERR_OR_ZERO(regmap); 1316 } 1317 1318 static const struct of_device_id axi_dmac_of_match_table[] = { 1319 { .compatible = "adi,axi-dmac-1.00.a" }, 1320 { }, 1321 }; 1322 MODULE_DEVICE_TABLE(of, axi_dmac_of_match_table); 1323 1324 static struct platform_driver axi_dmac_driver = { 1325 .driver = { 1326 .name = "dma-axi-dmac", 1327 .of_match_table = axi_dmac_of_match_table, 1328 }, 1329 .probe = axi_dmac_probe, 1330 }; 1331 module_platform_driver(axi_dmac_driver); 1332 1333 MODULE_AUTHOR("Lars-Peter Clausen <lars@metafoo.de>"); 1334 MODULE_DESCRIPTION("DMA controller driver for the AXI-DMAC controller"); 1335 MODULE_LICENSE("GPL v2"); 1336