1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2013-2015, The Linux Foundation. All rights reserved. 4 */ 5 6 #include <linux/clk.h> 7 #include <linux/delay.h> 8 #include <linux/device.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/dma/qcom_adm.h> 12 #include <linux/init.h> 13 #include <linux/interrupt.h> 14 #include <linux/io.h> 15 #include <linux/kernel.h> 16 #include <linux/module.h> 17 #include <linux/of.h> 18 #include <linux/of_address.h> 19 #include <linux/of_irq.h> 20 #include <linux/of_dma.h> 21 #include <linux/platform_device.h> 22 #include <linux/reset.h> 23 #include <linux/scatterlist.h> 24 #include <linux/slab.h> 25 26 #include "../dmaengine.h" 27 #include "../virt-dma.h" 28 29 /* ADM registers - calculated from channel number and security domain */ 30 #define ADM_CHAN_MULTI 0x4 31 #define ADM_CI_MULTI 0x4 32 #define ADM_CRCI_MULTI 0x4 33 #define ADM_EE_MULTI 0x800 34 #define ADM_CHAN_OFFS(chan) (ADM_CHAN_MULTI * (chan)) 35 #define ADM_EE_OFFS(ee) (ADM_EE_MULTI * (ee)) 36 #define ADM_CHAN_EE_OFFS(chan, ee) (ADM_CHAN_OFFS(chan) + ADM_EE_OFFS(ee)) 37 #define ADM_CHAN_OFFS(chan) (ADM_CHAN_MULTI * (chan)) 38 #define ADM_CI_OFFS(ci) (ADM_CHAN_OFF(ci)) 39 #define ADM_CH_CMD_PTR(chan, ee) (ADM_CHAN_EE_OFFS(chan, ee)) 40 #define ADM_CH_RSLT(chan, ee) (0x40 + ADM_CHAN_EE_OFFS(chan, ee)) 41 #define ADM_CH_FLUSH_STATE0(chan, ee) (0x80 + ADM_CHAN_EE_OFFS(chan, ee)) 42 #define ADM_CH_STATUS_SD(chan, ee) (0x200 + ADM_CHAN_EE_OFFS(chan, ee)) 43 #define ADM_CH_CONF(chan) (0x240 + ADM_CHAN_OFFS(chan)) 44 #define ADM_CH_RSLT_CONF(chan, ee) (0x300 + ADM_CHAN_EE_OFFS(chan, ee)) 45 #define ADM_SEC_DOMAIN_IRQ_STATUS(ee) (0x380 + ADM_EE_OFFS(ee)) 46 #define ADM_CI_CONF(ci) (0x390 + (ci) * ADM_CI_MULTI) 47 #define ADM_GP_CTL 0x3d8 48 #define ADM_CRCI_CTL(crci, ee) (0x400 + (crci) * ADM_CRCI_MULTI + \ 49 ADM_EE_OFFS(ee)) 50 51 /* channel status */ 52 #define ADM_CH_STATUS_VALID BIT(1) 53 54 /* channel result */ 55 #define ADM_CH_RSLT_VALID BIT(31) 56 #define ADM_CH_RSLT_ERR BIT(3) 57 #define ADM_CH_RSLT_FLUSH BIT(2) 58 #define ADM_CH_RSLT_TPD BIT(1) 59 60 /* channel conf */ 61 #define ADM_CH_CONF_SHADOW_EN BIT(12) 62 #define ADM_CH_CONF_MPU_DISABLE BIT(11) 63 #define ADM_CH_CONF_PERM_MPU_CONF BIT(9) 64 #define ADM_CH_CONF_FORCE_RSLT_EN BIT(7) 65 #define ADM_CH_CONF_SEC_DOMAIN(ee) ((((ee) & 0x3) << 4) | (((ee) & 0x4) << 11)) 66 67 /* channel result conf */ 68 #define ADM_CH_RSLT_CONF_FLUSH_EN BIT(1) 69 #define ADM_CH_RSLT_CONF_IRQ_EN BIT(0) 70 71 /* CRCI CTL */ 72 #define ADM_CRCI_CTL_MUX_SEL BIT(18) 73 #define ADM_CRCI_CTL_RST BIT(17) 74 75 /* CI configuration */ 76 #define ADM_CI_RANGE_END(x) ((x) << 24) 77 #define ADM_CI_RANGE_START(x) ((x) << 16) 78 #define ADM_CI_BURST_4_WORDS BIT(2) 79 #define ADM_CI_BURST_8_WORDS BIT(3) 80 81 /* GP CTL */ 82 #define ADM_GP_CTL_LP_EN BIT(12) 83 #define ADM_GP_CTL_LP_CNT(x) ((x) << 8) 84 85 /* Command pointer list entry */ 86 #define ADM_CPLE_LP BIT(31) 87 #define ADM_CPLE_CMD_PTR_LIST BIT(29) 88 89 /* Command list entry */ 90 #define ADM_CMD_LC BIT(31) 91 #define ADM_CMD_DST_CRCI(n) (((n) & 0xf) << 7) 92 #define ADM_CMD_SRC_CRCI(n) (((n) & 0xf) << 3) 93 94 #define ADM_CMD_TYPE_SINGLE 0x0 95 #define ADM_CMD_TYPE_BOX 0x3 96 97 #define ADM_CRCI_MUX_SEL BIT(4) 98 #define ADM_DESC_ALIGN 8 99 #define ADM_MAX_XFER (SZ_64K - 1) 100 #define ADM_MAX_ROWS (SZ_64K - 1) 101 #define ADM_MAX_CHANNELS 16 102 103 struct adm_desc_hw_box { 104 u32 cmd; 105 u32 src_addr; 106 u32 dst_addr; 107 u32 row_len; 108 u32 num_rows; 109 u32 row_offset; 110 }; 111 112 struct adm_desc_hw_single { 113 u32 cmd; 114 u32 src_addr; 115 u32 dst_addr; 116 u32 len; 117 }; 118 119 struct adm_async_desc { 120 struct virt_dma_desc vd; 121 struct adm_device *adev; 122 123 size_t length; 124 enum dma_transfer_direction dir; 125 dma_addr_t dma_addr; 126 size_t dma_len; 127 128 void *cpl; 129 dma_addr_t cp_addr; 130 u32 crci; 131 u32 mux; 132 u32 blk_size; 133 }; 134 135 struct adm_chan { 136 struct virt_dma_chan vc; 137 struct adm_device *adev; 138 139 /* parsed from DT */ 140 u32 id; /* channel id */ 141 142 struct adm_async_desc *curr_txd; 143 struct dma_slave_config slave; 144 u32 crci; 145 u32 mux; 146 struct list_head node; 147 148 int error; 149 int initialized; 150 }; 151 152 static inline struct adm_chan *to_adm_chan(struct dma_chan *common) 153 { 154 return container_of(common, struct adm_chan, vc.chan); 155 } 156 157 struct adm_device { 158 void __iomem *regs; 159 struct device *dev; 160 struct dma_device common; 161 struct device_dma_parameters dma_parms; 162 struct adm_chan *channels; 163 164 u32 ee; 165 166 struct clk *core_clk; 167 struct clk *iface_clk; 168 169 struct reset_control *clk_reset; 170 struct reset_control *c0_reset; 171 struct reset_control *c1_reset; 172 struct reset_control *c2_reset; 173 int irq; 174 }; 175 176 /** 177 * adm_free_chan - Frees dma resources associated with the specific channel 178 * 179 * @chan: dma channel 180 * 181 * Free all allocated descriptors associated with this channel 182 */ 183 static void adm_free_chan(struct dma_chan *chan) 184 { 185 /* free all queued descriptors */ 186 vchan_free_chan_resources(to_virt_chan(chan)); 187 } 188 189 /** 190 * adm_get_blksize - Get block size from burst value 191 * 192 * @burst: Burst size of transaction 193 */ 194 static int adm_get_blksize(unsigned int burst) 195 { 196 int ret; 197 198 switch (burst) { 199 case 16: 200 case 32: 201 case 64: 202 case 128: 203 ret = ffs(burst >> 4) - 1; 204 break; 205 case 192: 206 ret = 4; 207 break; 208 case 256: 209 ret = 5; 210 break; 211 default: 212 ret = -EINVAL; 213 break; 214 } 215 216 return ret; 217 } 218 219 /** 220 * adm_process_fc_descriptors - Process descriptors for flow controlled xfers 221 * 222 * @achan: ADM channel 223 * @desc: Descriptor memory pointer 224 * @sg: Scatterlist entry 225 * @crci: CRCI value 226 * @burst: Burst size of transaction 227 * @direction: DMA transfer direction 228 */ 229 static void *adm_process_fc_descriptors(struct adm_chan *achan, void *desc, 230 struct scatterlist *sg, u32 crci, 231 u32 burst, 232 enum dma_transfer_direction direction) 233 { 234 struct adm_desc_hw_box *box_desc = NULL; 235 struct adm_desc_hw_single *single_desc; 236 u32 remainder = sg_dma_len(sg); 237 u32 rows, row_offset, crci_cmd; 238 u32 mem_addr = sg_dma_address(sg); 239 u32 *incr_addr = &mem_addr; 240 u32 *src, *dst; 241 242 if (direction == DMA_DEV_TO_MEM) { 243 crci_cmd = ADM_CMD_SRC_CRCI(crci); 244 row_offset = burst; 245 src = &achan->slave.src_addr; 246 dst = &mem_addr; 247 } else { 248 crci_cmd = ADM_CMD_DST_CRCI(crci); 249 row_offset = burst << 16; 250 src = &mem_addr; 251 dst = &achan->slave.dst_addr; 252 } 253 254 while (remainder >= burst) { 255 box_desc = desc; 256 box_desc->cmd = ADM_CMD_TYPE_BOX | crci_cmd; 257 box_desc->row_offset = row_offset; 258 box_desc->src_addr = *src; 259 box_desc->dst_addr = *dst; 260 261 rows = remainder / burst; 262 rows = min_t(u32, rows, ADM_MAX_ROWS); 263 box_desc->num_rows = rows << 16 | rows; 264 box_desc->row_len = burst << 16 | burst; 265 266 *incr_addr += burst * rows; 267 remainder -= burst * rows; 268 desc += sizeof(*box_desc); 269 } 270 271 /* if leftover bytes, do one single descriptor */ 272 if (remainder) { 273 single_desc = desc; 274 single_desc->cmd = ADM_CMD_TYPE_SINGLE | crci_cmd; 275 single_desc->len = remainder; 276 single_desc->src_addr = *src; 277 single_desc->dst_addr = *dst; 278 desc += sizeof(*single_desc); 279 280 if (sg_is_last(sg)) 281 single_desc->cmd |= ADM_CMD_LC; 282 } else { 283 if (box_desc && sg_is_last(sg)) 284 box_desc->cmd |= ADM_CMD_LC; 285 } 286 287 return desc; 288 } 289 290 /** 291 * adm_process_non_fc_descriptors - Process descriptors for non-fc xfers 292 * 293 * @achan: ADM channel 294 * @desc: Descriptor memory pointer 295 * @sg: Scatterlist entry 296 * @direction: DMA transfer direction 297 */ 298 static void *adm_process_non_fc_descriptors(struct adm_chan *achan, void *desc, 299 struct scatterlist *sg, 300 enum dma_transfer_direction direction) 301 { 302 struct adm_desc_hw_single *single_desc; 303 u32 remainder = sg_dma_len(sg); 304 u32 mem_addr = sg_dma_address(sg); 305 u32 *incr_addr = &mem_addr; 306 u32 *src, *dst; 307 308 if (direction == DMA_DEV_TO_MEM) { 309 src = &achan->slave.src_addr; 310 dst = &mem_addr; 311 } else { 312 src = &mem_addr; 313 dst = &achan->slave.dst_addr; 314 } 315 316 do { 317 single_desc = desc; 318 single_desc->cmd = ADM_CMD_TYPE_SINGLE; 319 single_desc->src_addr = *src; 320 single_desc->dst_addr = *dst; 321 single_desc->len = (remainder > ADM_MAX_XFER) ? 322 ADM_MAX_XFER : remainder; 323 324 remainder -= single_desc->len; 325 *incr_addr += single_desc->len; 326 desc += sizeof(*single_desc); 327 } while (remainder); 328 329 /* set last command if this is the end of the whole transaction */ 330 if (sg_is_last(sg)) 331 single_desc->cmd |= ADM_CMD_LC; 332 333 return desc; 334 } 335 336 /** 337 * adm_prep_slave_sg - Prep slave sg transaction 338 * 339 * @chan: dma channel 340 * @sgl: scatter gather list 341 * @sg_len: length of sg 342 * @direction: DMA transfer direction 343 * @flags: DMA flags 344 * @context: transfer context (unused) 345 */ 346 static struct dma_async_tx_descriptor *adm_prep_slave_sg(struct dma_chan *chan, 347 struct scatterlist *sgl, 348 unsigned int sg_len, 349 enum dma_transfer_direction direction, 350 unsigned long flags, 351 void *context) 352 { 353 struct adm_chan *achan = to_adm_chan(chan); 354 struct adm_device *adev = achan->adev; 355 struct adm_async_desc *async_desc; 356 struct scatterlist *sg; 357 dma_addr_t cple_addr; 358 u32 i, burst; 359 u32 single_count = 0, box_count = 0, crci = 0; 360 void *desc; 361 u32 *cple; 362 int blk_size = 0; 363 364 if (!is_slave_direction(direction)) { 365 dev_err(adev->dev, "invalid dma direction\n"); 366 return NULL; 367 } 368 369 /* 370 * get burst value from slave configuration 371 */ 372 burst = (direction == DMA_MEM_TO_DEV) ? 373 achan->slave.dst_maxburst : 374 achan->slave.src_maxburst; 375 376 /* if using flow control, validate burst and crci values */ 377 if (achan->slave.device_fc) { 378 blk_size = adm_get_blksize(burst); 379 if (blk_size < 0) { 380 dev_err(adev->dev, "invalid burst value: %d\n", 381 burst); 382 return NULL; 383 } 384 385 crci = achan->crci & 0xf; 386 if (!crci || achan->crci > 0x1f) { 387 dev_err(adev->dev, "invalid crci value\n"); 388 return NULL; 389 } 390 } 391 392 /* iterate through sgs and compute allocation size of structures */ 393 for_each_sg(sgl, sg, sg_len, i) { 394 if (achan->slave.device_fc) { 395 box_count += DIV_ROUND_UP(sg_dma_len(sg) / burst, 396 ADM_MAX_ROWS); 397 if (sg_dma_len(sg) % burst) 398 single_count++; 399 } else { 400 single_count += DIV_ROUND_UP(sg_dma_len(sg), 401 ADM_MAX_XFER); 402 } 403 } 404 405 async_desc = kzalloc(sizeof(*async_desc), GFP_NOWAIT); 406 if (!async_desc) { 407 dev_err(adev->dev, "not enough memory for async_desc struct\n"); 408 return NULL; 409 } 410 411 async_desc->mux = achan->mux ? ADM_CRCI_CTL_MUX_SEL : 0; 412 async_desc->crci = crci; 413 async_desc->blk_size = blk_size; 414 async_desc->dma_len = single_count * sizeof(struct adm_desc_hw_single) + 415 box_count * sizeof(struct adm_desc_hw_box) + 416 sizeof(*cple) + 2 * ADM_DESC_ALIGN; 417 418 async_desc->cpl = kzalloc(async_desc->dma_len, GFP_NOWAIT); 419 if (!async_desc->cpl) { 420 dev_err(adev->dev, "not enough memory for cpl struct\n"); 421 goto free; 422 } 423 424 async_desc->adev = adev; 425 426 /* both command list entry and descriptors must be 8 byte aligned */ 427 cple = PTR_ALIGN(async_desc->cpl, ADM_DESC_ALIGN); 428 desc = PTR_ALIGN(cple + 1, ADM_DESC_ALIGN); 429 430 for_each_sg(sgl, sg, sg_len, i) { 431 async_desc->length += sg_dma_len(sg); 432 433 if (achan->slave.device_fc) 434 desc = adm_process_fc_descriptors(achan, desc, sg, crci, 435 burst, direction); 436 else 437 desc = adm_process_non_fc_descriptors(achan, desc, sg, 438 direction); 439 } 440 441 async_desc->dma_addr = dma_map_single(adev->dev, async_desc->cpl, 442 async_desc->dma_len, 443 DMA_TO_DEVICE); 444 if (dma_mapping_error(adev->dev, async_desc->dma_addr)) { 445 dev_err(adev->dev, "dma mapping error for cpl\n"); 446 goto free; 447 } 448 449 cple_addr = async_desc->dma_addr + ((void *)cple - async_desc->cpl); 450 451 /* init cmd list */ 452 dma_sync_single_for_cpu(adev->dev, cple_addr, sizeof(*cple), 453 DMA_TO_DEVICE); 454 *cple = ADM_CPLE_LP; 455 *cple |= (async_desc->dma_addr + ADM_DESC_ALIGN) >> 3; 456 dma_sync_single_for_device(adev->dev, cple_addr, sizeof(*cple), 457 DMA_TO_DEVICE); 458 459 return vchan_tx_prep(&achan->vc, &async_desc->vd, flags); 460 461 free: 462 kfree(async_desc); 463 return NULL; 464 } 465 466 /** 467 * adm_terminate_all - terminate all transactions on a channel 468 * @chan: dma channel 469 * 470 * Dequeues and frees all transactions, aborts current transaction 471 * No callbacks are done 472 * 473 */ 474 static int adm_terminate_all(struct dma_chan *chan) 475 { 476 struct adm_chan *achan = to_adm_chan(chan); 477 struct adm_device *adev = achan->adev; 478 unsigned long flags; 479 LIST_HEAD(head); 480 481 spin_lock_irqsave(&achan->vc.lock, flags); 482 vchan_get_all_descriptors(&achan->vc, &head); 483 484 /* send flush command to terminate current transaction */ 485 writel_relaxed(0x0, 486 adev->regs + ADM_CH_FLUSH_STATE0(achan->id, adev->ee)); 487 488 spin_unlock_irqrestore(&achan->vc.lock, flags); 489 490 vchan_dma_desc_free_list(&achan->vc, &head); 491 492 return 0; 493 } 494 495 static int adm_slave_config(struct dma_chan *chan, struct dma_slave_config *cfg) 496 { 497 struct adm_chan *achan = to_adm_chan(chan); 498 struct qcom_adm_peripheral_config *config = cfg->peripheral_config; 499 unsigned long flag; 500 501 spin_lock_irqsave(&achan->vc.lock, flag); 502 memcpy(&achan->slave, cfg, sizeof(struct dma_slave_config)); 503 if (cfg->peripheral_size == sizeof(*config)) 504 achan->crci = config->crci; 505 spin_unlock_irqrestore(&achan->vc.lock, flag); 506 507 return 0; 508 } 509 510 /** 511 * adm_start_dma - start next transaction 512 * @achan: ADM dma channel 513 */ 514 static void adm_start_dma(struct adm_chan *achan) 515 { 516 struct virt_dma_desc *vd = vchan_next_desc(&achan->vc); 517 struct adm_device *adev = achan->adev; 518 struct adm_async_desc *async_desc; 519 520 lockdep_assert_held(&achan->vc.lock); 521 522 if (!vd) 523 return; 524 525 list_del(&vd->node); 526 527 /* write next command list out to the CMD FIFO */ 528 async_desc = container_of(vd, struct adm_async_desc, vd); 529 achan->curr_txd = async_desc; 530 531 /* reset channel error */ 532 achan->error = 0; 533 534 if (!achan->initialized) { 535 /* enable interrupts */ 536 writel(ADM_CH_CONF_SHADOW_EN | 537 ADM_CH_CONF_PERM_MPU_CONF | 538 ADM_CH_CONF_MPU_DISABLE | 539 ADM_CH_CONF_SEC_DOMAIN(adev->ee), 540 adev->regs + ADM_CH_CONF(achan->id)); 541 542 writel(ADM_CH_RSLT_CONF_IRQ_EN | ADM_CH_RSLT_CONF_FLUSH_EN, 543 adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee)); 544 545 achan->initialized = 1; 546 } 547 548 /* set the crci block size if this transaction requires CRCI */ 549 if (async_desc->crci) { 550 writel(async_desc->mux | async_desc->blk_size, 551 adev->regs + ADM_CRCI_CTL(async_desc->crci, adev->ee)); 552 } 553 554 /* make sure IRQ enable doesn't get reordered */ 555 wmb(); 556 557 /* write next command list out to the CMD FIFO */ 558 writel(ALIGN(async_desc->dma_addr, ADM_DESC_ALIGN) >> 3, 559 adev->regs + ADM_CH_CMD_PTR(achan->id, adev->ee)); 560 } 561 562 /** 563 * adm_dma_irq - irq handler for ADM controller 564 * @irq: IRQ of interrupt 565 * @data: callback data 566 * 567 * IRQ handler for the bam controller 568 */ 569 static irqreturn_t adm_dma_irq(int irq, void *data) 570 { 571 struct adm_device *adev = data; 572 u32 srcs, i; 573 struct adm_async_desc *async_desc; 574 unsigned long flags; 575 576 srcs = readl_relaxed(adev->regs + 577 ADM_SEC_DOMAIN_IRQ_STATUS(adev->ee)); 578 579 for (i = 0; i < ADM_MAX_CHANNELS; i++) { 580 struct adm_chan *achan = &adev->channels[i]; 581 u32 status, result; 582 583 if (srcs & BIT(i)) { 584 status = readl_relaxed(adev->regs + 585 ADM_CH_STATUS_SD(i, adev->ee)); 586 587 /* if no result present, skip */ 588 if (!(status & ADM_CH_STATUS_VALID)) 589 continue; 590 591 result = readl_relaxed(adev->regs + 592 ADM_CH_RSLT(i, adev->ee)); 593 594 /* no valid results, skip */ 595 if (!(result & ADM_CH_RSLT_VALID)) 596 continue; 597 598 /* flag error if transaction was flushed or failed */ 599 if (result & (ADM_CH_RSLT_ERR | ADM_CH_RSLT_FLUSH)) 600 achan->error = 1; 601 602 spin_lock_irqsave(&achan->vc.lock, flags); 603 async_desc = achan->curr_txd; 604 605 achan->curr_txd = NULL; 606 607 if (async_desc) { 608 vchan_cookie_complete(&async_desc->vd); 609 610 /* kick off next DMA */ 611 adm_start_dma(achan); 612 } 613 614 spin_unlock_irqrestore(&achan->vc.lock, flags); 615 } 616 } 617 618 return IRQ_HANDLED; 619 } 620 621 /** 622 * adm_tx_status - returns status of transaction 623 * @chan: dma channel 624 * @cookie: transaction cookie 625 * @txstate: DMA transaction state 626 * 627 * Return status of dma transaction 628 */ 629 static enum dma_status adm_tx_status(struct dma_chan *chan, dma_cookie_t cookie, 630 struct dma_tx_state *txstate) 631 { 632 struct adm_chan *achan = to_adm_chan(chan); 633 struct virt_dma_desc *vd; 634 enum dma_status ret; 635 unsigned long flags; 636 size_t residue = 0; 637 638 ret = dma_cookie_status(chan, cookie, txstate); 639 if (ret == DMA_COMPLETE || !txstate) 640 return ret; 641 642 spin_lock_irqsave(&achan->vc.lock, flags); 643 644 vd = vchan_find_desc(&achan->vc, cookie); 645 if (vd) 646 residue = container_of(vd, struct adm_async_desc, vd)->length; 647 648 spin_unlock_irqrestore(&achan->vc.lock, flags); 649 650 /* 651 * residue is either the full length if it is in the issued list, or 0 652 * if it is in progress. We have no reliable way of determining 653 * anything inbetween 654 */ 655 dma_set_residue(txstate, residue); 656 657 if (achan->error) 658 return DMA_ERROR; 659 660 return ret; 661 } 662 663 /** 664 * adm_issue_pending - starts pending transactions 665 * @chan: dma channel 666 * 667 * Issues all pending transactions and starts DMA 668 */ 669 static void adm_issue_pending(struct dma_chan *chan) 670 { 671 struct adm_chan *achan = to_adm_chan(chan); 672 unsigned long flags; 673 674 spin_lock_irqsave(&achan->vc.lock, flags); 675 676 if (vchan_issue_pending(&achan->vc) && !achan->curr_txd) 677 adm_start_dma(achan); 678 spin_unlock_irqrestore(&achan->vc.lock, flags); 679 } 680 681 /** 682 * adm_dma_free_desc - free descriptor memory 683 * @vd: virtual descriptor 684 * 685 */ 686 static void adm_dma_free_desc(struct virt_dma_desc *vd) 687 { 688 struct adm_async_desc *async_desc = container_of(vd, 689 struct adm_async_desc, vd); 690 691 dma_unmap_single(async_desc->adev->dev, async_desc->dma_addr, 692 async_desc->dma_len, DMA_TO_DEVICE); 693 kfree(async_desc->cpl); 694 kfree(async_desc); 695 } 696 697 static void adm_channel_init(struct adm_device *adev, struct adm_chan *achan, 698 u32 index) 699 { 700 achan->id = index; 701 achan->adev = adev; 702 703 vchan_init(&achan->vc, &adev->common); 704 achan->vc.desc_free = adm_dma_free_desc; 705 } 706 707 /** 708 * adm_dma_xlate 709 * @dma_spec: pointer to DMA specifier as found in the device tree 710 * @ofdma: pointer to DMA controller data 711 * 712 * This can use either 1-cell or 2-cell formats, the first cell 713 * identifies the slave device, while the optional second cell 714 * contains the crci value. 715 * 716 * Returns pointer to appropriate dma channel on success or NULL on error. 717 */ 718 static struct dma_chan *adm_dma_xlate(struct of_phandle_args *dma_spec, 719 struct of_dma *ofdma) 720 { 721 struct dma_device *dev = ofdma->of_dma_data; 722 struct dma_chan *chan, *candidate = NULL; 723 struct adm_chan *achan; 724 725 if (!dev || dma_spec->args_count > 2) 726 return NULL; 727 728 list_for_each_entry(chan, &dev->channels, device_node) 729 if (chan->chan_id == dma_spec->args[0]) { 730 candidate = chan; 731 break; 732 } 733 734 if (!candidate) 735 return NULL; 736 737 achan = to_adm_chan(candidate); 738 if (dma_spec->args_count == 2) 739 achan->crci = dma_spec->args[1]; 740 else 741 achan->crci = 0; 742 743 return dma_get_slave_channel(candidate); 744 } 745 746 static int adm_dma_probe(struct platform_device *pdev) 747 { 748 struct adm_device *adev; 749 int ret; 750 u32 i; 751 752 adev = devm_kzalloc(&pdev->dev, sizeof(*adev), GFP_KERNEL); 753 if (!adev) 754 return -ENOMEM; 755 756 adev->dev = &pdev->dev; 757 758 adev->regs = devm_platform_ioremap_resource(pdev, 0); 759 if (IS_ERR(adev->regs)) 760 return PTR_ERR(adev->regs); 761 762 adev->irq = platform_get_irq(pdev, 0); 763 if (adev->irq < 0) 764 return adev->irq; 765 766 ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &adev->ee); 767 if (ret) { 768 dev_err(adev->dev, "Execution environment unspecified\n"); 769 return ret; 770 } 771 772 adev->core_clk = devm_clk_get(adev->dev, "core"); 773 if (IS_ERR(adev->core_clk)) 774 return PTR_ERR(adev->core_clk); 775 776 adev->iface_clk = devm_clk_get(adev->dev, "iface"); 777 if (IS_ERR(adev->iface_clk)) 778 return PTR_ERR(adev->iface_clk); 779 780 adev->clk_reset = devm_reset_control_get_exclusive(&pdev->dev, "clk"); 781 if (IS_ERR(adev->clk_reset)) { 782 dev_err(adev->dev, "failed to get ADM0 reset\n"); 783 return PTR_ERR(adev->clk_reset); 784 } 785 786 adev->c0_reset = devm_reset_control_get_exclusive(&pdev->dev, "c0"); 787 if (IS_ERR(adev->c0_reset)) { 788 dev_err(adev->dev, "failed to get ADM0 C0 reset\n"); 789 return PTR_ERR(adev->c0_reset); 790 } 791 792 adev->c1_reset = devm_reset_control_get_exclusive(&pdev->dev, "c1"); 793 if (IS_ERR(adev->c1_reset)) { 794 dev_err(adev->dev, "failed to get ADM0 C1 reset\n"); 795 return PTR_ERR(adev->c1_reset); 796 } 797 798 adev->c2_reset = devm_reset_control_get_exclusive(&pdev->dev, "c2"); 799 if (IS_ERR(adev->c2_reset)) { 800 dev_err(adev->dev, "failed to get ADM0 C2 reset\n"); 801 return PTR_ERR(adev->c2_reset); 802 } 803 804 ret = clk_prepare_enable(adev->core_clk); 805 if (ret) { 806 dev_err(adev->dev, "failed to prepare/enable core clock\n"); 807 return ret; 808 } 809 810 ret = clk_prepare_enable(adev->iface_clk); 811 if (ret) { 812 dev_err(adev->dev, "failed to prepare/enable iface clock\n"); 813 goto err_disable_core_clk; 814 } 815 816 reset_control_assert(adev->clk_reset); 817 reset_control_assert(adev->c0_reset); 818 reset_control_assert(adev->c1_reset); 819 reset_control_assert(adev->c2_reset); 820 821 udelay(2); 822 823 reset_control_deassert(adev->clk_reset); 824 reset_control_deassert(adev->c0_reset); 825 reset_control_deassert(adev->c1_reset); 826 reset_control_deassert(adev->c2_reset); 827 828 adev->channels = devm_kcalloc(adev->dev, ADM_MAX_CHANNELS, 829 sizeof(*adev->channels), GFP_KERNEL); 830 831 if (!adev->channels) { 832 ret = -ENOMEM; 833 goto err_disable_clks; 834 } 835 836 /* allocate and initialize channels */ 837 INIT_LIST_HEAD(&adev->common.channels); 838 839 for (i = 0; i < ADM_MAX_CHANNELS; i++) 840 adm_channel_init(adev, &adev->channels[i], i); 841 842 /* reset CRCIs */ 843 for (i = 0; i < 16; i++) 844 writel(ADM_CRCI_CTL_RST, adev->regs + 845 ADM_CRCI_CTL(i, adev->ee)); 846 847 /* configure client interfaces */ 848 writel(ADM_CI_RANGE_START(0x40) | ADM_CI_RANGE_END(0xb0) | 849 ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(0)); 850 writel(ADM_CI_RANGE_START(0x2a) | ADM_CI_RANGE_END(0x2c) | 851 ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(1)); 852 writel(ADM_CI_RANGE_START(0x12) | ADM_CI_RANGE_END(0x28) | 853 ADM_CI_BURST_8_WORDS, adev->regs + ADM_CI_CONF(2)); 854 writel(ADM_GP_CTL_LP_EN | ADM_GP_CTL_LP_CNT(0xf), 855 adev->regs + ADM_GP_CTL); 856 857 ret = devm_request_irq(adev->dev, adev->irq, adm_dma_irq, 858 0, "adm_dma", adev); 859 if (ret) 860 goto err_disable_clks; 861 862 platform_set_drvdata(pdev, adev); 863 864 adev->common.dev = adev->dev; 865 adev->common.dev->dma_parms = &adev->dma_parms; 866 867 /* set capabilities */ 868 dma_cap_zero(adev->common.cap_mask); 869 dma_cap_set(DMA_SLAVE, adev->common.cap_mask); 870 dma_cap_set(DMA_PRIVATE, adev->common.cap_mask); 871 872 /* initialize dmaengine apis */ 873 adev->common.directions = BIT(DMA_DEV_TO_MEM | DMA_MEM_TO_DEV); 874 adev->common.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; 875 adev->common.src_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; 876 adev->common.dst_addr_widths = DMA_SLAVE_BUSWIDTH_4_BYTES; 877 adev->common.device_free_chan_resources = adm_free_chan; 878 adev->common.device_prep_slave_sg = adm_prep_slave_sg; 879 adev->common.device_issue_pending = adm_issue_pending; 880 adev->common.device_tx_status = adm_tx_status; 881 adev->common.device_terminate_all = adm_terminate_all; 882 adev->common.device_config = adm_slave_config; 883 884 ret = dma_async_device_register(&adev->common); 885 if (ret) { 886 dev_err(adev->dev, "failed to register dma async device\n"); 887 goto err_disable_clks; 888 } 889 890 ret = of_dma_controller_register(pdev->dev.of_node, adm_dma_xlate, 891 &adev->common); 892 if (ret) 893 goto err_unregister_dma; 894 895 return 0; 896 897 err_unregister_dma: 898 dma_async_device_unregister(&adev->common); 899 err_disable_clks: 900 clk_disable_unprepare(adev->iface_clk); 901 err_disable_core_clk: 902 clk_disable_unprepare(adev->core_clk); 903 904 return ret; 905 } 906 907 static void adm_dma_remove(struct platform_device *pdev) 908 { 909 struct adm_device *adev = platform_get_drvdata(pdev); 910 struct adm_chan *achan; 911 u32 i; 912 913 of_dma_controller_free(pdev->dev.of_node); 914 dma_async_device_unregister(&adev->common); 915 916 for (i = 0; i < ADM_MAX_CHANNELS; i++) { 917 achan = &adev->channels[i]; 918 919 /* mask IRQs for this channel/EE pair */ 920 writel(0, adev->regs + ADM_CH_RSLT_CONF(achan->id, adev->ee)); 921 922 tasklet_kill(&adev->channels[i].vc.task); 923 adm_terminate_all(&adev->channels[i].vc.chan); 924 } 925 926 devm_free_irq(adev->dev, adev->irq, adev); 927 928 clk_disable_unprepare(adev->core_clk); 929 clk_disable_unprepare(adev->iface_clk); 930 } 931 932 static const struct of_device_id adm_of_match[] = { 933 { .compatible = "qcom,adm", }, 934 {} 935 }; 936 MODULE_DEVICE_TABLE(of, adm_of_match); 937 938 static struct platform_driver adm_dma_driver = { 939 .probe = adm_dma_probe, 940 .remove_new = adm_dma_remove, 941 .driver = { 942 .name = "adm-dma-engine", 943 .of_match_table = adm_of_match, 944 }, 945 }; 946 947 module_platform_driver(adm_dma_driver); 948 949 MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>"); 950 MODULE_DESCRIPTION("QCOM ADM DMA engine driver"); 951 MODULE_LICENSE("GPL v2"); 952