1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * ADMA driver for Nvidia's Tegra210 ADMA controller. 4 * 5 * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. 6 */ 7 8 #include <linux/clk.h> 9 #include <linux/iopoll.h> 10 #include <linux/module.h> 11 #include <linux/of.h> 12 #include <linux/of_dma.h> 13 #include <linux/of_irq.h> 14 #include <linux/platform_device.h> 15 #include <linux/pm_runtime.h> 16 #include <linux/slab.h> 17 18 #include "virt-dma.h" 19 20 #define ADMA_CH_CMD 0x00 21 #define ADMA_CH_STATUS 0x0c 22 #define ADMA_CH_STATUS_XFER_EN BIT(0) 23 #define ADMA_CH_STATUS_XFER_PAUSED BIT(1) 24 25 #define ADMA_CH_INT_STATUS 0x10 26 #define ADMA_CH_INT_STATUS_XFER_DONE BIT(0) 27 28 #define ADMA_CH_INT_CLEAR 0x1c 29 #define ADMA_CH_CTRL 0x24 30 #define ADMA_CH_CTRL_DIR(val, mask, shift) (((val) & (mask)) << (shift)) 31 #define ADMA_CH_CTRL_DIR_AHUB2MEM 2 32 #define ADMA_CH_CTRL_DIR_MEM2AHUB 4 33 #define ADMA_CH_CTRL_MODE_CONTINUOUS(shift) (2 << (shift)) 34 #define ADMA_CH_CTRL_FLOWCTRL_EN BIT(1) 35 #define ADMA_CH_CTRL_XFER_PAUSE_SHIFT 0 36 37 #define ADMA_CH_CONFIG 0x28 38 #define ADMA_CH_CONFIG_SRC_BUF(val) (((val) & 0x7) << 28) 39 #define ADMA_CH_CONFIG_TRG_BUF(val) (((val) & 0x7) << 24) 40 #define ADMA_CH_CONFIG_BURST_SIZE_SHIFT 20 41 #define ADMA_CH_CONFIG_MAX_BURST_SIZE 16 42 #define ADMA_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0xf) 43 #define ADMA_CH_CONFIG_MAX_BUFS 8 44 #define TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(reqs) ((reqs) << 4) 45 46 #define ADMA_GLOBAL_CH_CONFIG 0x400 47 #define ADMA_GLOBAL_CH_CONFIG_WEIGHT_FOR_WRR(val) ((val) & 0x7) 48 #define ADMA_GLOBAL_CH_CONFIG_OUTSTANDING_REQS(reqs) ((reqs) << 8) 49 50 #define TEGRA186_ADMA_GLOBAL_PAGE_CHGRP 0x30 51 #define TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ 0x70 52 #define TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ 0x84 53 #define TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_0 0x44 54 #define TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_1 0x48 55 #define TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_0 0x100 56 #define TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_1 0x104 57 #define TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_0 0x180 58 #define TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_1 0x184 59 #define TEGRA264_ADMA_GLOBAL_PAGE_OFFSET 0x8 60 61 #define ADMA_CH_FIFO_CTRL 0x2c 62 #define ADMA_CH_TX_FIFO_SIZE_SHIFT 8 63 #define ADMA_CH_RX_FIFO_SIZE_SHIFT 0 64 #define ADMA_GLOBAL_CH_FIFO_CTRL 0x300 65 66 #define ADMA_CH_LOWER_SRC_ADDR 0x34 67 #define ADMA_CH_LOWER_TRG_ADDR 0x3c 68 #define ADMA_CH_TC 0x44 69 #define ADMA_CH_TC_COUNT_MASK 0x3ffffffc 70 71 #define ADMA_CH_XFER_STATUS 0x54 72 #define ADMA_CH_XFER_STATUS_COUNT_MASK 0xffff 73 74 #define ADMA_GLOBAL_CMD 0x00 75 #define ADMA_GLOBAL_SOFT_RESET 0x04 76 77 #define TEGRA_ADMA_BURST_COMPLETE_TIME 20 78 79 #define ADMA_CH_REG_FIELD_VAL(val, mask, shift) (((val) & mask) << shift) 80 81 struct tegra_adma; 82 83 /* 84 * struct tegra_adma_chip_data - Tegra chip specific data 85 * @adma_get_burst_config: Function callback used to set DMA burst size. 86 * @global_reg_offset: Register offset of DMA global register. 87 * @global_int_clear: Register offset of DMA global interrupt clear. 88 * @global_ch_fifo_base: Global channel fifo ctrl base offset 89 * @global_ch_config_base: Global channel config base offset 90 * @ch_req_tx_shift: Register offset for AHUB transmit channel select. 91 * @ch_req_rx_shift: Register offset for AHUB receive channel select. 92 * @ch_dir_shift: Channel direction bit position. 93 * @ch_mode_shift: Channel mode bit position. 94 * @ch_base_offset: Register offset of DMA channel registers. 95 * @ch_tc_offset_diff: From TC register onwards offset differs for Tegra264 96 * @ch_fifo_ctrl: Default value for channel FIFO CTRL register. 97 * @ch_config: Outstanding and WRR config values 98 * @ch_req_mask: Mask for Tx or Rx channel select. 99 * @ch_dir_mask: Mask for channel direction. 100 * @ch_req_max: Maximum number of Tx or Rx channels available. 101 * @ch_reg_size: Size of DMA channel register space. 102 * @nr_channels: Number of DMA channels available. 103 * @ch_fifo_size_mask: Mask for FIFO size field. 104 * @sreq_index_offset: Slave channel index offset. 105 * @max_page: Maximum ADMA Channel Page. 106 * @set_global_pg_config: Global page programming. 107 */ 108 struct tegra_adma_chip_data { 109 unsigned int (*adma_get_burst_config)(unsigned int burst_size); 110 unsigned int global_reg_offset; 111 unsigned int global_int_clear; 112 unsigned int global_ch_fifo_base; 113 unsigned int global_ch_config_base; 114 unsigned int ch_req_tx_shift; 115 unsigned int ch_req_rx_shift; 116 unsigned int ch_dir_shift; 117 unsigned int ch_mode_shift; 118 unsigned int ch_base_offset; 119 unsigned int ch_tc_offset_diff; 120 unsigned int ch_fifo_ctrl; 121 unsigned int ch_config; 122 unsigned int ch_req_mask; 123 unsigned int ch_dir_mask; 124 unsigned int ch_req_max; 125 unsigned int ch_reg_size; 126 unsigned int nr_channels; 127 unsigned int ch_fifo_size_mask; 128 unsigned int sreq_index_offset; 129 unsigned int max_page; 130 void (*set_global_pg_config)(struct tegra_adma *tdma); 131 }; 132 133 /* 134 * struct tegra_adma_chan_regs - Tegra ADMA channel registers 135 */ 136 struct tegra_adma_chan_regs { 137 unsigned int ctrl; 138 unsigned int config; 139 unsigned int global_config; 140 unsigned int src_addr; 141 unsigned int trg_addr; 142 unsigned int fifo_ctrl; 143 unsigned int cmd; 144 unsigned int tc; 145 }; 146 147 /* 148 * struct tegra_adma_desc - Tegra ADMA descriptor to manage transfer requests. 149 */ 150 struct tegra_adma_desc { 151 struct virt_dma_desc vd; 152 struct tegra_adma_chan_regs ch_regs; 153 size_t buf_len; 154 size_t period_len; 155 size_t num_periods; 156 }; 157 158 /* 159 * struct tegra_adma_chan - Tegra ADMA channel information 160 */ 161 struct tegra_adma_chan { 162 struct virt_dma_chan vc; 163 struct tegra_adma_desc *desc; 164 struct tegra_adma *tdma; 165 int irq; 166 void __iomem *chan_addr; 167 168 /* Slave channel configuration info */ 169 struct dma_slave_config sconfig; 170 enum dma_transfer_direction sreq_dir; 171 unsigned int sreq_index; 172 bool sreq_reserved; 173 struct tegra_adma_chan_regs ch_regs; 174 175 /* Transfer count and position info */ 176 unsigned int tx_buf_count; 177 unsigned int tx_buf_pos; 178 179 unsigned int global_ch_fifo_offset; 180 unsigned int global_ch_config_offset; 181 }; 182 183 /* 184 * struct tegra_adma - Tegra ADMA controller information 185 */ 186 struct tegra_adma { 187 struct dma_device dma_dev; 188 struct device *dev; 189 void __iomem *base_addr; 190 void __iomem *ch_base_addr; 191 struct clk *ahub_clk; 192 unsigned int nr_channels; 193 unsigned long *dma_chan_mask; 194 unsigned long rx_requests_reserved; 195 unsigned long tx_requests_reserved; 196 197 /* Used to store global command register state when suspending */ 198 unsigned int global_cmd; 199 unsigned int ch_page_no; 200 201 const struct tegra_adma_chip_data *cdata; 202 203 /* Last member of the structure */ 204 struct tegra_adma_chan channels[] __counted_by(nr_channels); 205 }; 206 207 static inline void tdma_write(struct tegra_adma *tdma, u32 reg, u32 val) 208 { 209 writel(val, tdma->base_addr + tdma->cdata->global_reg_offset + reg); 210 } 211 212 static inline u32 tdma_read(struct tegra_adma *tdma, u32 reg) 213 { 214 return readl(tdma->base_addr + tdma->cdata->global_reg_offset + reg); 215 } 216 217 static inline void tdma_ch_global_write(struct tegra_adma *tdma, u32 reg, u32 val) 218 { 219 writel(val, tdma->ch_base_addr + tdma->cdata->global_reg_offset + reg); 220 } 221 222 static inline void tdma_ch_write(struct tegra_adma_chan *tdc, u32 reg, u32 val) 223 { 224 writel(val, tdc->chan_addr + reg); 225 } 226 227 static inline u32 tdma_ch_read(struct tegra_adma_chan *tdc, u32 reg) 228 { 229 return readl(tdc->chan_addr + reg); 230 } 231 232 static inline struct tegra_adma_chan *to_tegra_adma_chan(struct dma_chan *dc) 233 { 234 return container_of(dc, struct tegra_adma_chan, vc.chan); 235 } 236 237 static inline struct tegra_adma_desc *to_tegra_adma_desc( 238 struct dma_async_tx_descriptor *td) 239 { 240 return container_of(td, struct tegra_adma_desc, vd.tx); 241 } 242 243 static inline struct device *tdc2dev(struct tegra_adma_chan *tdc) 244 { 245 return tdc->tdma->dev; 246 } 247 248 static void tegra_adma_desc_free(struct virt_dma_desc *vd) 249 { 250 kfree(container_of(vd, struct tegra_adma_desc, vd)); 251 } 252 253 static int tegra_adma_slave_config(struct dma_chan *dc, 254 struct dma_slave_config *sconfig) 255 { 256 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 257 258 memcpy(&tdc->sconfig, sconfig, sizeof(*sconfig)); 259 260 return 0; 261 } 262 263 static void tegra186_adma_global_page_config(struct tegra_adma *tdma) 264 { 265 /* 266 * Clear the default page1 channel group configs and program 267 * the global registers based on the actual page usage 268 */ 269 tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_CHGRP, 0); 270 tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ, 0); 271 tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ, 0); 272 tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_CHGRP + (tdma->ch_page_no * 0x4), 0xff); 273 tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_RX_REQ + (tdma->ch_page_no * 0x4), 0x1ffffff); 274 tdma_write(tdma, TEGRA186_ADMA_GLOBAL_PAGE_TX_REQ + (tdma->ch_page_no * 0x4), 0xffffff); 275 } 276 277 static void tegra264_adma_global_page_config(struct tegra_adma *tdma) 278 { 279 u32 global_page_offset = tdma->ch_page_no * TEGRA264_ADMA_GLOBAL_PAGE_OFFSET; 280 281 /* If the default page (page1) is not used, then clear page1 registers */ 282 if (tdma->ch_page_no) { 283 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_0, 0); 284 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_1, 0); 285 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_0, 0); 286 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_1, 0); 287 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_0, 0); 288 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_1, 0); 289 } 290 291 /* Program global registers for selected page */ 292 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_0 + global_page_offset, 0xffffffff); 293 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_CHGRP_1 + global_page_offset, 0xffffffff); 294 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_0 + global_page_offset, 0xffffffff); 295 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_RX_REQ_1 + global_page_offset, 0x1); 296 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_0 + global_page_offset, 0xffffffff); 297 tdma_write(tdma, TEGRA264_ADMA_GLOBAL_PAGE_TX_REQ_1 + global_page_offset, 0x1); 298 } 299 300 static int tegra_adma_init(struct tegra_adma *tdma) 301 { 302 u32 status; 303 int ret; 304 305 /* Clear any channels group global interrupts */ 306 tdma_ch_global_write(tdma, tdma->cdata->global_int_clear, 0x1); 307 308 if (!tdma->base_addr) 309 return 0; 310 311 /* Assert soft reset */ 312 tdma_write(tdma, ADMA_GLOBAL_SOFT_RESET, 0x1); 313 314 /* Wait for reset to clear */ 315 ret = readx_poll_timeout(readl, 316 tdma->base_addr + 317 tdma->cdata->global_reg_offset + 318 ADMA_GLOBAL_SOFT_RESET, 319 status, status == 0, 20, 10000); 320 if (ret) 321 return ret; 322 323 if (tdma->cdata->set_global_pg_config) 324 tdma->cdata->set_global_pg_config(tdma); 325 326 /* Enable global ADMA registers */ 327 tdma_write(tdma, ADMA_GLOBAL_CMD, 1); 328 329 return 0; 330 } 331 332 static int tegra_adma_request_alloc(struct tegra_adma_chan *tdc, 333 enum dma_transfer_direction direction) 334 { 335 struct tegra_adma *tdma = tdc->tdma; 336 unsigned int sreq_index = tdc->sreq_index; 337 338 if (tdc->sreq_reserved) { 339 if (tdc->sreq_dir != direction) { 340 dev_err(tdma->dev, 341 "DMA request direction mismatch: reserved=%s, requested=%s\n", 342 dmaengine_get_direction_text(tdc->sreq_dir), 343 dmaengine_get_direction_text(direction)); 344 return -EINVAL; 345 } 346 return 0; 347 } 348 349 if (sreq_index > tdma->cdata->ch_req_max) { 350 dev_err(tdma->dev, "invalid DMA request\n"); 351 return -EINVAL; 352 } 353 354 switch (direction) { 355 case DMA_MEM_TO_DEV: 356 if (test_and_set_bit(sreq_index, &tdma->tx_requests_reserved)) { 357 dev_err(tdma->dev, "DMA request reserved\n"); 358 return -EINVAL; 359 } 360 break; 361 362 case DMA_DEV_TO_MEM: 363 if (test_and_set_bit(sreq_index, &tdma->rx_requests_reserved)) { 364 dev_err(tdma->dev, "DMA request reserved\n"); 365 return -EINVAL; 366 } 367 break; 368 369 default: 370 dev_WARN(tdma->dev, "channel %s has invalid transfer type\n", 371 dma_chan_name(&tdc->vc.chan)); 372 return -EINVAL; 373 } 374 375 tdc->sreq_dir = direction; 376 tdc->sreq_reserved = true; 377 378 return 0; 379 } 380 381 static void tegra_adma_request_free(struct tegra_adma_chan *tdc) 382 { 383 struct tegra_adma *tdma = tdc->tdma; 384 385 if (!tdc->sreq_reserved) 386 return; 387 388 switch (tdc->sreq_dir) { 389 case DMA_MEM_TO_DEV: 390 clear_bit(tdc->sreq_index, &tdma->tx_requests_reserved); 391 break; 392 393 case DMA_DEV_TO_MEM: 394 clear_bit(tdc->sreq_index, &tdma->rx_requests_reserved); 395 break; 396 397 default: 398 dev_WARN(tdma->dev, "channel %s has invalid transfer type\n", 399 dma_chan_name(&tdc->vc.chan)); 400 return; 401 } 402 403 tdc->sreq_reserved = false; 404 } 405 406 static u32 tegra_adma_irq_status(struct tegra_adma_chan *tdc) 407 { 408 u32 status = tdma_ch_read(tdc, ADMA_CH_INT_STATUS); 409 410 return status & ADMA_CH_INT_STATUS_XFER_DONE; 411 } 412 413 static u32 tegra_adma_irq_clear(struct tegra_adma_chan *tdc) 414 { 415 u32 status = tegra_adma_irq_status(tdc); 416 417 if (status) 418 tdma_ch_write(tdc, ADMA_CH_INT_CLEAR, status); 419 420 return status; 421 } 422 423 static void tegra_adma_stop(struct tegra_adma_chan *tdc) 424 { 425 unsigned int status; 426 427 /* Disable ADMA */ 428 tdma_ch_write(tdc, ADMA_CH_CMD, 0); 429 430 /* Clear interrupt status */ 431 tegra_adma_irq_clear(tdc); 432 433 if (readx_poll_timeout_atomic(readl, tdc->chan_addr + ADMA_CH_STATUS, 434 status, !(status & ADMA_CH_STATUS_XFER_EN), 435 20, 10000)) { 436 dev_err(tdc2dev(tdc), "unable to stop DMA channel\n"); 437 return; 438 } 439 440 vchan_terminate_vdesc(&tdc->desc->vd); 441 tdc->desc = NULL; 442 } 443 444 static void tegra_adma_synchronize(struct dma_chan *dc) 445 { 446 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 447 448 vchan_synchronize(&tdc->vc); 449 } 450 451 static void tegra_adma_start(struct tegra_adma_chan *tdc) 452 { 453 struct virt_dma_desc *vd = vchan_next_desc(&tdc->vc); 454 struct tegra_adma_chan_regs *ch_regs; 455 struct tegra_adma_desc *desc; 456 457 if (!vd) 458 return; 459 460 list_del(&vd->node); 461 462 desc = to_tegra_adma_desc(&vd->tx); 463 464 if (!desc) { 465 dev_warn(tdc2dev(tdc), "unable to start DMA, no descriptor\n"); 466 return; 467 } 468 469 ch_regs = &desc->ch_regs; 470 471 tdc->tx_buf_pos = 0; 472 tdc->tx_buf_count = 0; 473 tdma_ch_write(tdc, ADMA_CH_TC - tdc->tdma->cdata->ch_tc_offset_diff, ch_regs->tc); 474 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl); 475 tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR - tdc->tdma->cdata->ch_tc_offset_diff, 476 ch_regs->src_addr); 477 tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR - tdc->tdma->cdata->ch_tc_offset_diff, 478 ch_regs->trg_addr); 479 480 if (!tdc->tdma->cdata->global_ch_fifo_base) 481 tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_regs->fifo_ctrl); 482 else if (tdc->global_ch_fifo_offset) 483 tdma_write(tdc->tdma, tdc->global_ch_fifo_offset, ch_regs->fifo_ctrl); 484 485 if (tdc->global_ch_config_offset) 486 tdma_write(tdc->tdma, tdc->global_ch_config_offset, ch_regs->global_config); 487 488 tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_regs->config); 489 490 /* Start ADMA */ 491 tdma_ch_write(tdc, ADMA_CH_CMD, 1); 492 493 tdc->desc = desc; 494 } 495 496 static unsigned int tegra_adma_get_residue(struct tegra_adma_chan *tdc) 497 { 498 struct tegra_adma_desc *desc = tdc->desc; 499 unsigned int max = ADMA_CH_XFER_STATUS_COUNT_MASK + 1; 500 unsigned int pos = tdma_ch_read(tdc, ADMA_CH_XFER_STATUS - 501 tdc->tdma->cdata->ch_tc_offset_diff); 502 unsigned int periods_remaining; 503 504 /* 505 * Handle wrap around of buffer count register 506 */ 507 if (pos < tdc->tx_buf_pos) 508 tdc->tx_buf_count += pos + (max - tdc->tx_buf_pos); 509 else 510 tdc->tx_buf_count += pos - tdc->tx_buf_pos; 511 512 periods_remaining = tdc->tx_buf_count % desc->num_periods; 513 tdc->tx_buf_pos = pos; 514 515 return desc->buf_len - (periods_remaining * desc->period_len); 516 } 517 518 static irqreturn_t tegra_adma_isr(int irq, void *dev_id) 519 { 520 struct tegra_adma_chan *tdc = dev_id; 521 unsigned long status; 522 523 spin_lock(&tdc->vc.lock); 524 525 status = tegra_adma_irq_clear(tdc); 526 if (status == 0 || !tdc->desc) { 527 spin_unlock(&tdc->vc.lock); 528 return IRQ_NONE; 529 } 530 531 vchan_cyclic_callback(&tdc->desc->vd); 532 533 spin_unlock(&tdc->vc.lock); 534 535 return IRQ_HANDLED; 536 } 537 538 static void tegra_adma_issue_pending(struct dma_chan *dc) 539 { 540 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 541 unsigned long flags; 542 543 spin_lock_irqsave(&tdc->vc.lock, flags); 544 545 if (vchan_issue_pending(&tdc->vc)) { 546 if (!tdc->desc) 547 tegra_adma_start(tdc); 548 } 549 550 spin_unlock_irqrestore(&tdc->vc.lock, flags); 551 } 552 553 static bool tegra_adma_is_paused(struct tegra_adma_chan *tdc) 554 { 555 u32 csts; 556 557 csts = tdma_ch_read(tdc, ADMA_CH_STATUS); 558 csts &= ADMA_CH_STATUS_XFER_PAUSED; 559 560 return csts ? true : false; 561 } 562 563 static int tegra_adma_pause(struct dma_chan *dc) 564 { 565 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 566 struct tegra_adma_desc *desc = tdc->desc; 567 struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; 568 int dcnt = 10; 569 570 ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); 571 ch_regs->ctrl |= (1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT); 572 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl); 573 574 while (dcnt-- && !tegra_adma_is_paused(tdc)) 575 udelay(TEGRA_ADMA_BURST_COMPLETE_TIME); 576 577 if (dcnt < 0) { 578 dev_err(tdc2dev(tdc), "unable to pause DMA channel\n"); 579 return -EBUSY; 580 } 581 582 return 0; 583 } 584 585 static int tegra_adma_resume(struct dma_chan *dc) 586 { 587 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 588 struct tegra_adma_desc *desc = tdc->desc; 589 struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; 590 591 ch_regs->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); 592 ch_regs->ctrl &= ~(1 << ADMA_CH_CTRL_XFER_PAUSE_SHIFT); 593 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_regs->ctrl); 594 595 return 0; 596 } 597 598 static int tegra_adma_terminate_all(struct dma_chan *dc) 599 { 600 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 601 unsigned long flags; 602 LIST_HEAD(head); 603 604 spin_lock_irqsave(&tdc->vc.lock, flags); 605 606 if (tdc->desc) 607 tegra_adma_stop(tdc); 608 609 tegra_adma_request_free(tdc); 610 vchan_get_all_descriptors(&tdc->vc, &head); 611 spin_unlock_irqrestore(&tdc->vc.lock, flags); 612 vchan_dma_desc_free_list(&tdc->vc, &head); 613 614 return 0; 615 } 616 617 static enum dma_status tegra_adma_tx_status(struct dma_chan *dc, 618 dma_cookie_t cookie, 619 struct dma_tx_state *txstate) 620 { 621 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 622 struct tegra_adma_desc *desc; 623 struct virt_dma_desc *vd; 624 enum dma_status ret; 625 unsigned long flags; 626 unsigned int residual; 627 628 ret = dma_cookie_status(dc, cookie, txstate); 629 if (ret == DMA_COMPLETE || !txstate) 630 return ret; 631 632 spin_lock_irqsave(&tdc->vc.lock, flags); 633 634 vd = vchan_find_desc(&tdc->vc, cookie); 635 if (vd) { 636 desc = to_tegra_adma_desc(&vd->tx); 637 residual = desc->ch_regs.tc; 638 } else if (tdc->desc && tdc->desc->vd.tx.cookie == cookie) { 639 residual = tegra_adma_get_residue(tdc); 640 } else { 641 residual = 0; 642 } 643 644 spin_unlock_irqrestore(&tdc->vc.lock, flags); 645 646 dma_set_residue(txstate, residual); 647 648 return ret; 649 } 650 651 static unsigned int tegra210_adma_get_burst_config(unsigned int burst_size) 652 { 653 if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE) 654 burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE; 655 656 return fls(burst_size) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT; 657 } 658 659 static unsigned int tegra186_adma_get_burst_config(unsigned int burst_size) 660 { 661 if (!burst_size || burst_size > ADMA_CH_CONFIG_MAX_BURST_SIZE) 662 burst_size = ADMA_CH_CONFIG_MAX_BURST_SIZE; 663 664 return (burst_size - 1) << ADMA_CH_CONFIG_BURST_SIZE_SHIFT; 665 } 666 667 static int tegra_adma_set_xfer_params(struct tegra_adma_chan *tdc, 668 struct tegra_adma_desc *desc, 669 dma_addr_t buf_addr, 670 enum dma_transfer_direction direction) 671 { 672 struct tegra_adma_chan_regs *ch_regs = &desc->ch_regs; 673 const struct tegra_adma_chip_data *cdata = tdc->tdma->cdata; 674 unsigned int burst_size, adma_dir, fifo_size_shift; 675 676 if (desc->num_periods > ADMA_CH_CONFIG_MAX_BUFS) { 677 dev_err(tdc2dev(tdc), "invalid DMA periods %zu (max %u)\n", 678 desc->num_periods, ADMA_CH_CONFIG_MAX_BUFS); 679 return -EINVAL; 680 } 681 682 switch (direction) { 683 case DMA_MEM_TO_DEV: 684 fifo_size_shift = ADMA_CH_TX_FIFO_SIZE_SHIFT; 685 adma_dir = ADMA_CH_CTRL_DIR_MEM2AHUB; 686 burst_size = tdc->sconfig.dst_maxburst; 687 ch_regs->config = ADMA_CH_CONFIG_SRC_BUF(desc->num_periods - 1); 688 ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index, 689 cdata->ch_req_mask, 690 cdata->ch_req_tx_shift); 691 ch_regs->src_addr = buf_addr; 692 break; 693 694 case DMA_DEV_TO_MEM: 695 fifo_size_shift = ADMA_CH_RX_FIFO_SIZE_SHIFT; 696 adma_dir = ADMA_CH_CTRL_DIR_AHUB2MEM; 697 burst_size = tdc->sconfig.src_maxburst; 698 ch_regs->config = ADMA_CH_CONFIG_TRG_BUF(desc->num_periods - 1); 699 ch_regs->ctrl = ADMA_CH_REG_FIELD_VAL(tdc->sreq_index, 700 cdata->ch_req_mask, 701 cdata->ch_req_rx_shift); 702 ch_regs->trg_addr = buf_addr; 703 break; 704 705 default: 706 dev_err(tdc2dev(tdc), "DMA direction is not supported\n"); 707 return -EINVAL; 708 } 709 710 ch_regs->ctrl |= ADMA_CH_CTRL_DIR(adma_dir, cdata->ch_dir_mask, 711 cdata->ch_dir_shift) | 712 ADMA_CH_CTRL_MODE_CONTINUOUS(cdata->ch_mode_shift) | 713 ADMA_CH_CTRL_FLOWCTRL_EN; 714 ch_regs->config |= cdata->adma_get_burst_config(burst_size); 715 716 if (cdata->global_ch_config_base) 717 ch_regs->global_config |= cdata->ch_config; 718 else 719 ch_regs->config |= cdata->ch_config; 720 721 /* 722 * 'sreq_index' represents the current ADMAIF channel number and as per 723 * HW recommendation its FIFO size should match with the corresponding 724 * ADMA channel. 725 * 726 * ADMA FIFO size is set as per below (based on default ADMAIF channel 727 * FIFO sizes): 728 * fifo_size = 0x2 (sreq_index > sreq_index_offset) 729 * fifo_size = 0x3 (sreq_index <= sreq_index_offset) 730 * 731 */ 732 if (tdc->sreq_index > cdata->sreq_index_offset) 733 ch_regs->fifo_ctrl = 734 ADMA_CH_REG_FIELD_VAL(2, cdata->ch_fifo_size_mask, 735 fifo_size_shift); 736 else 737 ch_regs->fifo_ctrl = 738 ADMA_CH_REG_FIELD_VAL(3, cdata->ch_fifo_size_mask, 739 fifo_size_shift); 740 741 ch_regs->tc = desc->period_len & ADMA_CH_TC_COUNT_MASK; 742 743 return tegra_adma_request_alloc(tdc, direction); 744 } 745 746 static struct dma_async_tx_descriptor *tegra_adma_prep_dma_cyclic( 747 struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, 748 size_t period_len, enum dma_transfer_direction direction, 749 unsigned long flags) 750 { 751 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 752 struct tegra_adma_desc *desc = NULL; 753 754 if (!buf_len || !period_len || period_len > ADMA_CH_TC_COUNT_MASK) { 755 dev_err(tdc2dev(tdc), "invalid buffer/period len\n"); 756 return NULL; 757 } 758 759 if (buf_len % period_len) { 760 dev_err(tdc2dev(tdc), "buf_len not a multiple of period_len\n"); 761 return NULL; 762 } 763 764 if (!IS_ALIGNED(buf_addr, 4)) { 765 dev_err(tdc2dev(tdc), "invalid buffer alignment\n"); 766 return NULL; 767 } 768 769 desc = kzalloc_obj(*desc, GFP_NOWAIT); 770 if (!desc) 771 return NULL; 772 773 desc->buf_len = buf_len; 774 desc->period_len = period_len; 775 desc->num_periods = buf_len / period_len; 776 777 if (tegra_adma_set_xfer_params(tdc, desc, buf_addr, direction)) { 778 kfree(desc); 779 return NULL; 780 } 781 782 return vchan_tx_prep(&tdc->vc, &desc->vd, flags); 783 } 784 785 static int tegra_adma_alloc_chan_resources(struct dma_chan *dc) 786 { 787 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 788 int ret; 789 790 ret = request_irq(tdc->irq, tegra_adma_isr, 0, dma_chan_name(dc), tdc); 791 if (ret) { 792 dev_err(tdc2dev(tdc), "failed to get interrupt for %s\n", 793 dma_chan_name(dc)); 794 return ret; 795 } 796 797 ret = pm_runtime_resume_and_get(tdc2dev(tdc)); 798 if (ret < 0) { 799 free_irq(tdc->irq, tdc); 800 return ret; 801 } 802 803 dma_cookie_init(&tdc->vc.chan); 804 805 return 0; 806 } 807 808 static void tegra_adma_free_chan_resources(struct dma_chan *dc) 809 { 810 struct tegra_adma_chan *tdc = to_tegra_adma_chan(dc); 811 812 tegra_adma_terminate_all(dc); 813 vchan_free_chan_resources(&tdc->vc); 814 tasklet_kill(&tdc->vc.task); 815 free_irq(tdc->irq, tdc); 816 pm_runtime_put(tdc2dev(tdc)); 817 818 tdc->sreq_index = 0; 819 tdc->sreq_dir = DMA_TRANS_NONE; 820 } 821 822 static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, 823 struct of_dma *ofdma) 824 { 825 struct tegra_adma *tdma = ofdma->of_dma_data; 826 struct tegra_adma_chan *tdc; 827 struct dma_chan *chan; 828 unsigned int sreq_index; 829 830 if (dma_spec->args_count != 1) 831 return NULL; 832 833 sreq_index = dma_spec->args[0]; 834 835 if (sreq_index == 0) { 836 dev_err(tdma->dev, "DMA request must not be 0\n"); 837 return NULL; 838 } 839 840 chan = dma_get_any_slave_channel(&tdma->dma_dev); 841 if (!chan) 842 return NULL; 843 844 tdc = to_tegra_adma_chan(chan); 845 tdc->sreq_index = sreq_index; 846 847 return chan; 848 } 849 850 static int __maybe_unused tegra_adma_runtime_suspend(struct device *dev) 851 { 852 struct tegra_adma *tdma = dev_get_drvdata(dev); 853 struct tegra_adma_chan_regs *ch_reg; 854 struct tegra_adma_chan *tdc; 855 int i; 856 857 if (tdma->base_addr) 858 tdma->global_cmd = tdma_read(tdma, ADMA_GLOBAL_CMD); 859 860 if (!tdma->global_cmd) 861 goto clk_disable; 862 863 for (i = 0; i < tdma->nr_channels; i++) { 864 tdc = &tdma->channels[i]; 865 /* skip for reserved channels */ 866 if (!tdc->tdma) 867 continue; 868 869 ch_reg = &tdc->ch_regs; 870 ch_reg->cmd = tdma_ch_read(tdc, ADMA_CH_CMD); 871 /* skip if channel is not active */ 872 if (!ch_reg->cmd) 873 continue; 874 ch_reg->tc = tdma_ch_read(tdc, ADMA_CH_TC - tdma->cdata->ch_tc_offset_diff); 875 ch_reg->src_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_SRC_ADDR - 876 tdma->cdata->ch_tc_offset_diff); 877 ch_reg->trg_addr = tdma_ch_read(tdc, ADMA_CH_LOWER_TRG_ADDR - 878 tdma->cdata->ch_tc_offset_diff); 879 ch_reg->ctrl = tdma_ch_read(tdc, ADMA_CH_CTRL); 880 881 if (tdc->global_ch_config_offset) 882 ch_reg->global_config = tdma_read(tdc->tdma, tdc->global_ch_config_offset); 883 884 if (!tdc->tdma->cdata->global_ch_fifo_base) 885 ch_reg->fifo_ctrl = tdma_ch_read(tdc, ADMA_CH_FIFO_CTRL); 886 else if (tdc->global_ch_fifo_offset) 887 ch_reg->fifo_ctrl = tdma_read(tdc->tdma, tdc->global_ch_fifo_offset); 888 889 ch_reg->config = tdma_ch_read(tdc, ADMA_CH_CONFIG); 890 891 } 892 893 clk_disable: 894 clk_disable_unprepare(tdma->ahub_clk); 895 896 return 0; 897 } 898 899 static int __maybe_unused tegra_adma_runtime_resume(struct device *dev) 900 { 901 struct tegra_adma *tdma = dev_get_drvdata(dev); 902 struct tegra_adma_chan_regs *ch_reg; 903 struct tegra_adma_chan *tdc; 904 int ret, i; 905 906 ret = clk_prepare_enable(tdma->ahub_clk); 907 if (ret) { 908 dev_err(dev, "ahub clk_enable failed: %d\n", ret); 909 return ret; 910 } 911 if (tdma->base_addr) { 912 tdma_write(tdma, ADMA_GLOBAL_CMD, tdma->global_cmd); 913 if (tdma->cdata->set_global_pg_config) 914 tdma->cdata->set_global_pg_config(tdma); 915 } 916 917 if (!tdma->global_cmd) 918 return 0; 919 920 for (i = 0; i < tdma->nr_channels; i++) { 921 tdc = &tdma->channels[i]; 922 /* skip for reserved channels */ 923 if (!tdc->tdma) 924 continue; 925 ch_reg = &tdc->ch_regs; 926 /* skip if channel was not active earlier */ 927 if (!ch_reg->cmd) 928 continue; 929 tdma_ch_write(tdc, ADMA_CH_TC - tdma->cdata->ch_tc_offset_diff, ch_reg->tc); 930 tdma_ch_write(tdc, ADMA_CH_LOWER_SRC_ADDR - tdma->cdata->ch_tc_offset_diff, 931 ch_reg->src_addr); 932 tdma_ch_write(tdc, ADMA_CH_LOWER_TRG_ADDR - tdma->cdata->ch_tc_offset_diff, 933 ch_reg->trg_addr); 934 tdma_ch_write(tdc, ADMA_CH_CTRL, ch_reg->ctrl); 935 936 if (!tdc->tdma->cdata->global_ch_fifo_base) 937 tdma_ch_write(tdc, ADMA_CH_FIFO_CTRL, ch_reg->fifo_ctrl); 938 else if (tdc->global_ch_fifo_offset) 939 tdma_write(tdc->tdma, tdc->global_ch_fifo_offset, ch_reg->fifo_ctrl); 940 941 if (tdc->global_ch_config_offset) 942 tdma_write(tdc->tdma, tdc->global_ch_config_offset, ch_reg->global_config); 943 944 tdma_ch_write(tdc, ADMA_CH_CONFIG, ch_reg->config); 945 946 tdma_ch_write(tdc, ADMA_CH_CMD, ch_reg->cmd); 947 } 948 949 return 0; 950 } 951 952 static const struct tegra_adma_chip_data tegra210_chip_data = { 953 .adma_get_burst_config = tegra210_adma_get_burst_config, 954 .global_reg_offset = 0xc00, 955 .global_int_clear = 0x20, 956 .global_ch_fifo_base = 0, 957 .global_ch_config_base = 0, 958 .ch_req_tx_shift = 28, 959 .ch_req_rx_shift = 24, 960 .ch_dir_shift = 12, 961 .ch_mode_shift = 8, 962 .ch_base_offset = 0, 963 .ch_tc_offset_diff = 0, 964 .ch_config = ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1), 965 .ch_req_mask = 0xf, 966 .ch_dir_mask = 0xf, 967 .ch_req_max = 10, 968 .ch_reg_size = 0x80, 969 .nr_channels = 22, 970 .ch_fifo_size_mask = 0xf, 971 .sreq_index_offset = 2, 972 .max_page = 0, 973 .set_global_pg_config = NULL, 974 }; 975 976 static const struct tegra_adma_chip_data tegra186_chip_data = { 977 .adma_get_burst_config = tegra186_adma_get_burst_config, 978 .global_reg_offset = 0, 979 .global_int_clear = 0x402c, 980 .global_ch_fifo_base = 0, 981 .global_ch_config_base = 0, 982 .ch_req_tx_shift = 27, 983 .ch_req_rx_shift = 22, 984 .ch_dir_shift = 12, 985 .ch_mode_shift = 8, 986 .ch_base_offset = 0x10000, 987 .ch_tc_offset_diff = 0, 988 .ch_config = ADMA_CH_CONFIG_WEIGHT_FOR_WRR(1) | 989 TEGRA186_ADMA_CH_CONFIG_OUTSTANDING_REQS(8), 990 .ch_req_mask = 0x1f, 991 .ch_dir_mask = 0xf, 992 .ch_req_max = 20, 993 .ch_reg_size = 0x100, 994 .nr_channels = 32, 995 .ch_fifo_size_mask = 0x1f, 996 .sreq_index_offset = 4, 997 .max_page = 4, 998 .set_global_pg_config = tegra186_adma_global_page_config, 999 }; 1000 1001 static const struct tegra_adma_chip_data tegra264_chip_data = { 1002 .adma_get_burst_config = tegra186_adma_get_burst_config, 1003 .global_reg_offset = 0, 1004 .global_int_clear = 0x800c, 1005 .global_ch_fifo_base = ADMA_GLOBAL_CH_FIFO_CTRL, 1006 .global_ch_config_base = ADMA_GLOBAL_CH_CONFIG, 1007 .ch_req_tx_shift = 26, 1008 .ch_req_rx_shift = 20, 1009 .ch_dir_shift = 10, 1010 .ch_mode_shift = 7, 1011 .ch_base_offset = 0x10000, 1012 .ch_tc_offset_diff = 4, 1013 .ch_config = ADMA_GLOBAL_CH_CONFIG_WEIGHT_FOR_WRR(1) | 1014 ADMA_GLOBAL_CH_CONFIG_OUTSTANDING_REQS(8), 1015 .ch_req_mask = 0x3f, 1016 .ch_dir_mask = 7, 1017 .ch_req_max = 32, 1018 .ch_reg_size = 0x100, 1019 .nr_channels = 64, 1020 .ch_fifo_size_mask = 0x7f, 1021 .sreq_index_offset = 0, 1022 .max_page = 10, 1023 .set_global_pg_config = tegra264_adma_global_page_config, 1024 }; 1025 1026 static const struct of_device_id tegra_adma_of_match[] = { 1027 { .compatible = "nvidia,tegra210-adma", .data = &tegra210_chip_data }, 1028 { .compatible = "nvidia,tegra186-adma", .data = &tegra186_chip_data }, 1029 { .compatible = "nvidia,tegra264-adma", .data = &tegra264_chip_data }, 1030 { }, 1031 }; 1032 MODULE_DEVICE_TABLE(of, tegra_adma_of_match); 1033 1034 static int tegra_adma_probe(struct platform_device *pdev) 1035 { 1036 const struct tegra_adma_chip_data *cdata; 1037 struct tegra_adma *tdma; 1038 struct resource *res_page, *res_base; 1039 int ret, i; 1040 1041 cdata = of_device_get_match_data(&pdev->dev); 1042 if (!cdata) { 1043 return dev_err_probe(&pdev->dev, -ENODEV, 1044 "device match data not found\n"); 1045 } 1046 1047 tdma = devm_kzalloc(&pdev->dev, 1048 struct_size(tdma, channels, cdata->nr_channels), 1049 GFP_KERNEL); 1050 if (!tdma) 1051 return -ENOMEM; 1052 1053 tdma->dev = &pdev->dev; 1054 tdma->cdata = cdata; 1055 tdma->nr_channels = cdata->nr_channels; 1056 platform_set_drvdata(pdev, tdma); 1057 1058 res_page = platform_get_resource_byname(pdev, IORESOURCE_MEM, "page"); 1059 if (res_page) { 1060 tdma->ch_base_addr = devm_ioremap_resource(&pdev->dev, res_page); 1061 if (IS_ERR(tdma->ch_base_addr)) 1062 return PTR_ERR(tdma->ch_base_addr); 1063 1064 res_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "global"); 1065 if (res_base) { 1066 resource_size_t page_offset, page_no; 1067 unsigned int ch_base_offset; 1068 1069 if (res_page->start < res_base->start) 1070 return dev_err_probe(&pdev->dev, -EINVAL, 1071 "invalid page/global resource order\n"); 1072 page_offset = res_page->start - res_base->start; 1073 ch_base_offset = cdata->ch_base_offset; 1074 if (!ch_base_offset) 1075 return -EINVAL; 1076 1077 page_no = div_u64(page_offset, ch_base_offset); 1078 if (!page_no || page_no > INT_MAX) 1079 return dev_err_probe(&pdev->dev, -EINVAL, 1080 "invalid page number %llu\n", 1081 (unsigned long long)page_no); 1082 1083 tdma->ch_page_no = page_no - 1; 1084 tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); 1085 if (IS_ERR(tdma->base_addr)) 1086 return PTR_ERR(tdma->base_addr); 1087 } 1088 } else { 1089 /* If no 'page' property found, then reg DT binding would be legacy */ 1090 res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1091 if (res_base) { 1092 tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); 1093 if (IS_ERR(tdma->base_addr)) 1094 return PTR_ERR(tdma->base_addr); 1095 } else { 1096 return dev_err_probe(&pdev->dev, -ENODEV, 1097 "failed to get memory resource\n"); 1098 } 1099 1100 tdma->ch_base_addr = tdma->base_addr + cdata->ch_base_offset; 1101 } 1102 1103 tdma->ahub_clk = devm_clk_get(&pdev->dev, "d_audio"); 1104 if (IS_ERR(tdma->ahub_clk)) { 1105 return dev_err_probe(&pdev->dev, PTR_ERR(tdma->ahub_clk), 1106 "failed to get ahub clock\n"); 1107 } 1108 1109 tdma->dma_chan_mask = devm_kzalloc(&pdev->dev, 1110 BITS_TO_LONGS(tdma->nr_channels) * sizeof(unsigned long), 1111 GFP_KERNEL); 1112 if (!tdma->dma_chan_mask) 1113 return -ENOMEM; 1114 1115 /* Enable all channels by default */ 1116 bitmap_fill(tdma->dma_chan_mask, tdma->nr_channels); 1117 1118 ret = of_property_read_u32_array(pdev->dev.of_node, "dma-channel-mask", 1119 (u32 *)tdma->dma_chan_mask, 1120 BITS_TO_U32(tdma->nr_channels)); 1121 if (ret < 0 && (ret != -EINVAL)) { 1122 return dev_err_probe(&pdev->dev, ret, 1123 "dma-channel-mask is not complete.\n"); 1124 } 1125 1126 INIT_LIST_HEAD(&tdma->dma_dev.channels); 1127 for (i = 0; i < tdma->nr_channels; i++) { 1128 struct tegra_adma_chan *tdc = &tdma->channels[i]; 1129 1130 /* skip for reserved channels */ 1131 if (!test_bit(i, tdma->dma_chan_mask)) 1132 continue; 1133 1134 tdc->chan_addr = tdma->ch_base_addr + (cdata->ch_reg_size * i); 1135 1136 if (tdma->base_addr) { 1137 if (cdata->global_ch_fifo_base) 1138 tdc->global_ch_fifo_offset = cdata->global_ch_fifo_base + (4 * i); 1139 1140 if (cdata->global_ch_config_base) 1141 tdc->global_ch_config_offset = 1142 cdata->global_ch_config_base + (4 * i); 1143 } 1144 1145 ret = of_irq_get(pdev->dev.of_node, i); 1146 if (ret <= 0) { 1147 ret = dev_err_probe(&pdev->dev, ret ?: -ENXIO, 1148 "failed to get IRQ for channel %d\n", i); 1149 goto irq_dispose; 1150 } 1151 tdc->irq = ret; 1152 1153 vchan_init(&tdc->vc, &tdma->dma_dev); 1154 tdc->vc.desc_free = tegra_adma_desc_free; 1155 tdc->tdma = tdma; 1156 } 1157 1158 pm_runtime_enable(&pdev->dev); 1159 1160 ret = pm_runtime_resume_and_get(&pdev->dev); 1161 if (ret < 0) { 1162 ret = dev_err_probe(&pdev->dev, ret, 1163 "runtime PM resume failed\n"); 1164 goto rpm_disable; 1165 } 1166 1167 ret = tegra_adma_init(tdma); 1168 if (ret) { 1169 ret = dev_err_probe(&pdev->dev, ret, 1170 "failed to initialize ADMA\n"); 1171 goto rpm_put; 1172 } 1173 1174 dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); 1175 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); 1176 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); 1177 1178 tdma->dma_dev.dev = &pdev->dev; 1179 tdma->dma_dev.device_alloc_chan_resources = 1180 tegra_adma_alloc_chan_resources; 1181 tdma->dma_dev.device_free_chan_resources = 1182 tegra_adma_free_chan_resources; 1183 tdma->dma_dev.device_issue_pending = tegra_adma_issue_pending; 1184 tdma->dma_dev.device_prep_dma_cyclic = tegra_adma_prep_dma_cyclic; 1185 tdma->dma_dev.device_config = tegra_adma_slave_config; 1186 tdma->dma_dev.device_tx_status = tegra_adma_tx_status; 1187 tdma->dma_dev.device_terminate_all = tegra_adma_terminate_all; 1188 tdma->dma_dev.device_synchronize = tegra_adma_synchronize; 1189 tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 1190 tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); 1191 tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 1192 tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 1193 tdma->dma_dev.device_pause = tegra_adma_pause; 1194 tdma->dma_dev.device_resume = tegra_adma_resume; 1195 1196 ret = dma_async_device_register(&tdma->dma_dev); 1197 if (ret < 0) { 1198 ret = dev_err_probe(&pdev->dev, ret, 1199 "ADMA registration failed\n"); 1200 goto rpm_put; 1201 } 1202 1203 ret = of_dma_controller_register(pdev->dev.of_node, 1204 tegra_dma_of_xlate, tdma); 1205 if (ret < 0) { 1206 ret = dev_err_probe(&pdev->dev, ret, 1207 "ADMA OF registration failed\n"); 1208 goto dma_remove; 1209 } 1210 1211 pm_runtime_put(&pdev->dev); 1212 1213 dev_info(&pdev->dev, "Tegra210 ADMA driver registered %d channels\n", 1214 tdma->nr_channels); 1215 1216 return 0; 1217 1218 dma_remove: 1219 dma_async_device_unregister(&tdma->dma_dev); 1220 rpm_put: 1221 pm_runtime_put_sync(&pdev->dev); 1222 rpm_disable: 1223 pm_runtime_disable(&pdev->dev); 1224 irq_dispose: 1225 while (--i >= 0) 1226 irq_dispose_mapping(tdma->channels[i].irq); 1227 1228 return ret; 1229 } 1230 1231 static void tegra_adma_remove(struct platform_device *pdev) 1232 { 1233 struct tegra_adma *tdma = platform_get_drvdata(pdev); 1234 int i; 1235 1236 of_dma_controller_free(pdev->dev.of_node); 1237 dma_async_device_unregister(&tdma->dma_dev); 1238 1239 for (i = 0; i < tdma->nr_channels; ++i) { 1240 if (tdma->channels[i].irq) 1241 irq_dispose_mapping(tdma->channels[i].irq); 1242 } 1243 1244 pm_runtime_disable(&pdev->dev); 1245 } 1246 1247 static const struct dev_pm_ops tegra_adma_dev_pm_ops = { 1248 SET_RUNTIME_PM_OPS(tegra_adma_runtime_suspend, 1249 tegra_adma_runtime_resume, NULL) 1250 SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, 1251 pm_runtime_force_resume) 1252 }; 1253 1254 static struct platform_driver tegra_admac_driver = { 1255 .driver = { 1256 .name = "tegra-adma", 1257 .pm = &tegra_adma_dev_pm_ops, 1258 .of_match_table = tegra_adma_of_match, 1259 }, 1260 .probe = tegra_adma_probe, 1261 .remove = tegra_adma_remove, 1262 }; 1263 1264 module_platform_driver(tegra_admac_driver); 1265 1266 MODULE_DESCRIPTION("NVIDIA Tegra ADMA driver"); 1267 MODULE_AUTHOR("Dara Ramesh <dramesh@nvidia.com>"); 1268 MODULE_AUTHOR("Jon Hunter <jonathanh@nvidia.com>"); 1269 MODULE_LICENSE("GPL v2"); 1270