1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * DMA driver for NVIDIA Tegra GPC DMA controller. 4 * 5 * Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved. 6 */ 7 8 #include <linux/bitfield.h> 9 #include <linux/dmaengine.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/interrupt.h> 12 #include <linux/iommu.h> 13 #include <linux/iopoll.h> 14 #include <linux/minmax.h> 15 #include <linux/module.h> 16 #include <linux/of.h> 17 #include <linux/of_dma.h> 18 #include <linux/of_device.h> 19 #include <linux/platform_device.h> 20 #include <linux/reset.h> 21 #include <linux/slab.h> 22 #include <dt-bindings/memory/tegra186-mc.h> 23 #include "virt-dma.h" 24 25 /* CSR register */ 26 #define TEGRA_GPCDMA_CSR_ENB BIT(31) 27 #define TEGRA_GPCDMA_CSR_IE_EOC BIT(30) 28 #define TEGRA_GPCDMA_CSR_ONCE BIT(27) 29 30 #define TEGRA_GPCDMA_CSR_FC_MODE GENMASK(25, 24) 31 #define TEGRA_GPCDMA_CSR_FC_MODE_NO_MMIO \ 32 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 0) 33 #define TEGRA_GPCDMA_CSR_FC_MODE_ONE_MMIO \ 34 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 1) 35 #define TEGRA_GPCDMA_CSR_FC_MODE_TWO_MMIO \ 36 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 2) 37 #define TEGRA_GPCDMA_CSR_FC_MODE_FOUR_MMIO \ 38 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 3) 39 40 #define TEGRA_GPCDMA_CSR_DMA GENMASK(23, 21) 41 #define TEGRA_GPCDMA_CSR_DMA_IO2MEM_NO_FC \ 42 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 0) 43 #define TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC \ 44 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 1) 45 #define TEGRA_GPCDMA_CSR_DMA_MEM2IO_NO_FC \ 46 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 2) 47 #define TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC \ 48 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 3) 49 #define TEGRA_GPCDMA_CSR_DMA_MEM2MEM \ 50 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 4) 51 #define TEGRA_GPCDMA_CSR_DMA_FIXED_PAT \ 52 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 6) 53 54 #define TEGRA_GPCDMA_CSR_REQ_SEL_MASK GENMASK(20, 16) 55 #define TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED \ 56 FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, 4) 57 #define TEGRA_GPCDMA_CSR_IRQ_MASK BIT(15) 58 #define TEGRA_GPCDMA_CSR_WEIGHT GENMASK(13, 10) 59 60 /* STATUS register */ 61 #define TEGRA_GPCDMA_STATUS_BUSY BIT(31) 62 #define TEGRA_GPCDMA_STATUS_ISE_EOC BIT(30) 63 #define TEGRA_GPCDMA_STATUS_PING_PONG BIT(28) 64 #define TEGRA_GPCDMA_STATUS_DMA_ACTIVITY BIT(27) 65 #define TEGRA_GPCDMA_STATUS_CHANNEL_PAUSE BIT(26) 66 #define TEGRA_GPCDMA_STATUS_CHANNEL_RX BIT(25) 67 #define TEGRA_GPCDMA_STATUS_CHANNEL_TX BIT(24) 68 #define TEGRA_GPCDMA_STATUS_IRQ_INTR_STA BIT(23) 69 #define TEGRA_GPCDMA_STATUS_IRQ_STA BIT(21) 70 #define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA BIT(20) 71 72 #define TEGRA_GPCDMA_CHAN_CSRE_PAUSE BIT(31) 73 74 /* High address pointer */ 75 #define TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR GENMASK(7, 0) 76 #define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR GENMASK(23, 16) 77 78 /* MC sequence register */ 79 #define TEGRA_GPCDMA_MCSEQ_DATA_SWAP BIT(31) 80 #define TEGRA_GPCDMA_MCSEQ_REQ_COUNT GENMASK(30, 25) 81 #define TEGRA_GPCDMA_MCSEQ_BURST GENMASK(24, 23) 82 #define TEGRA_GPCDMA_MCSEQ_BURST_2 \ 83 FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 0) 84 #define TEGRA_GPCDMA_MCSEQ_BURST_16 \ 85 FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 3) 86 #define TEGRA_GPCDMA_MCSEQ_WRAP1 GENMASK(22, 20) 87 #define TEGRA_GPCDMA_MCSEQ_WRAP0 GENMASK(19, 17) 88 #define TEGRA_GPCDMA_MCSEQ_WRAP_NONE 0 89 90 #define TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK GENMASK(13, 7) 91 #define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK GENMASK(6, 0) 92 93 /* MMIO sequence register */ 94 #define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF BIT(31) 95 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH GENMASK(30, 28) 96 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8 \ 97 FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 0) 98 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16 \ 99 FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 1) 100 #define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32 \ 101 FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 2) 102 #define TEGRA_GPCDMA_MMIOSEQ_DATA_SWAP BIT(27) 103 #define TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT 23 104 #define TEGRA_GPCDMA_MMIOSEQ_BURST_MIN 2U 105 #define TEGRA_GPCDMA_MMIOSEQ_BURST_MAX 32U 106 #define TEGRA_GPCDMA_MMIOSEQ_BURST(bs) \ 107 (GENMASK((fls(bs) - 2), 0) << TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT) 108 #define TEGRA_GPCDMA_MMIOSEQ_MASTER_ID GENMASK(22, 19) 109 #define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD GENMASK(18, 16) 110 #define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT GENMASK(8, 7) 111 112 /* Error Status Register */ 113 #define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT 8 114 #define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK 0xF 115 #define TEGRA_GPCDMA_CHAN_ERR_TYPE(err) ( \ 116 ((err) >> TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT) & \ 117 TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK) 118 #define TEGRA_DMA_BM_FIFO_FULL_ERR 0xF 119 #define TEGRA_DMA_PERIPH_FIFO_FULL_ERR 0xE 120 #define TEGRA_DMA_PERIPH_ID_ERR 0xD 121 #define TEGRA_DMA_STREAM_ID_ERR 0xC 122 #define TEGRA_DMA_MC_SLAVE_ERR 0xB 123 #define TEGRA_DMA_MMIO_SLAVE_ERR 0xA 124 125 /* 126 * If any burst is in flight and DMA paused then this is the time to complete 127 * on-flight burst and update DMA status register. 128 */ 129 #define TEGRA_GPCDMA_BURST_COMPLETE_TIME 10 130 #define TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT 5000 /* 5 msec */ 131 132 /* Channel base address offset from GPCDMA base address */ 133 #define TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET 0x10000 134 135 /* Default channel mask reserving channel0 */ 136 #define TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK 0xfffffffe 137 138 struct tegra_dma; 139 struct tegra_dma_channel; 140 141 /* 142 * tegra_dma_chip_data Tegra chip specific DMA data 143 * @nr_channels: Number of channels available in the controller. 144 * @channel_reg_size: Channel register size. 145 * @max_dma_count: Maximum DMA transfer count supported by DMA controller. 146 * @hw_support_pause: DMA HW engine support pause of the channel. 147 */ 148 struct tegra_dma_chip_data { 149 bool hw_support_pause; 150 unsigned int addr_bits; 151 unsigned int nr_channels; 152 unsigned int channel_reg_size; 153 unsigned int max_dma_count; 154 const struct tegra_dma_channel_regs *channel_regs; 155 int (*terminate)(struct tegra_dma_channel *tdc); 156 }; 157 158 /* DMA channel registers */ 159 struct tegra_dma_channel_regs { 160 u32 csr; 161 u32 status; 162 u32 csre; 163 u32 src; 164 u32 dst; 165 u32 high_addr; 166 u32 src_high; 167 u32 dst_high; 168 u32 mc_seq; 169 u32 mmio_seq; 170 u32 wcount; 171 u32 wxfer; 172 u32 wstatus; 173 u32 err_status; 174 u32 fixed_pattern; 175 }; 176 177 /* 178 * tegra_dma_sg_req: DMA request details to configure hardware. This 179 * contains the details for one transfer to configure DMA hw. 180 * The client's request for data transfer can be broken into multiple 181 * sub-transfer as per requester details and hw support. This sub transfer 182 * get added as an array in Tegra DMA desc which manages the transfer details. 183 */ 184 struct tegra_dma_sg_req { 185 unsigned int len; 186 dma_addr_t src; 187 dma_addr_t dst; 188 u32 csr; 189 u32 mc_seq; 190 u32 mmio_seq; 191 u32 wcount; 192 u32 fixed_pattern; 193 }; 194 195 /* 196 * tegra_dma_desc: Tegra DMA descriptors which uses virt_dma_desc to 197 * manage client request and keep track of transfer status, callbacks 198 * and request counts etc. 199 */ 200 struct tegra_dma_desc { 201 bool cyclic; 202 unsigned int bytes_req; 203 unsigned int bytes_xfer; 204 unsigned int sg_idx; 205 unsigned int sg_count; 206 struct virt_dma_desc vd; 207 struct tegra_dma_channel *tdc; 208 struct tegra_dma_sg_req sg_req[] __counted_by(sg_count); 209 }; 210 211 /* 212 * tegra_dma_channel: Channel specific information 213 */ 214 struct tegra_dma_channel { 215 const struct tegra_dma_channel_regs *regs; 216 struct tegra_dma *tdma; 217 struct virt_dma_chan vc; 218 struct tegra_dma_desc *dma_desc; 219 struct dma_slave_config dma_sconfig; 220 enum dma_transfer_direction sid_dir; 221 enum dma_status status; 222 unsigned int stream_id; 223 unsigned long chan_base_offset; 224 bool config_init; 225 char name[30]; 226 int id; 227 int irq; 228 int slave_id; 229 }; 230 231 /* 232 * tegra_dma: Tegra DMA specific information 233 */ 234 struct tegra_dma { 235 const struct tegra_dma_chip_data *chip_data; 236 unsigned long sid_m2d_reserved; 237 unsigned long sid_d2m_reserved; 238 u32 chan_mask; 239 void __iomem *base_addr; 240 struct device *dev; 241 struct dma_device dma_dev; 242 struct reset_control *rst; 243 struct tegra_dma_channel channels[]; 244 }; 245 246 static inline void tdc_write(struct tegra_dma_channel *tdc, 247 u32 reg, u32 val) 248 { 249 writel_relaxed(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg); 250 } 251 252 static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg) 253 { 254 return readl_relaxed(tdc->tdma->base_addr + tdc->chan_base_offset + reg); 255 } 256 257 static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc) 258 { 259 return container_of(dc, struct tegra_dma_channel, vc.chan); 260 } 261 262 static inline struct tegra_dma_desc *vd_to_tegra_dma_desc(struct virt_dma_desc *vd) 263 { 264 return container_of(vd, struct tegra_dma_desc, vd); 265 } 266 267 static inline struct device *tdc2dev(struct tegra_dma_channel *tdc) 268 { 269 return tdc->vc.chan.device->dev; 270 } 271 272 static void tegra_dma_program_addr(struct tegra_dma_channel *tdc, 273 struct tegra_dma_sg_req *sg_req) 274 { 275 tdc_write(tdc, tdc->regs->src, lower_32_bits(sg_req->src)); 276 tdc_write(tdc, tdc->regs->dst, lower_32_bits(sg_req->dst)); 277 278 if (tdc->tdma->chip_data->addr_bits > 39) { 279 tdc_write(tdc, tdc->regs->src_high, upper_32_bits(sg_req->src)); 280 tdc_write(tdc, tdc->regs->dst_high, upper_32_bits(sg_req->dst)); 281 } else { 282 u32 src_high = FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, 283 upper_32_bits(sg_req->src)); 284 u32 dst_high = FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, 285 upper_32_bits(sg_req->dst)); 286 287 tdc_write(tdc, tdc->regs->high_addr, src_high | dst_high); 288 } 289 } 290 291 static void tegra_dma_dump_chan_regs(struct tegra_dma_channel *tdc) 292 { 293 dev_dbg(tdc2dev(tdc), "DMA Channel %d name %s register dump:\n", 294 tdc->id, tdc->name); 295 dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x\n", 296 tdc_read(tdc, tdc->regs->csr), 297 tdc_read(tdc, tdc->regs->status), 298 tdc_read(tdc, tdc->regs->csre)); 299 300 if (tdc->tdma->chip_data->addr_bits > 39) { 301 dev_dbg(tdc2dev(tdc), "SRC %x SRC HI %x DST %x DST HI %x\n", 302 tdc_read(tdc, tdc->regs->src), 303 tdc_read(tdc, tdc->regs->src_high), 304 tdc_read(tdc, tdc->regs->dst), 305 tdc_read(tdc, tdc->regs->dst_high)); 306 } else { 307 dev_dbg(tdc2dev(tdc), "SRC %x DST %x HI ADDR %x\n", 308 tdc_read(tdc, tdc->regs->src), 309 tdc_read(tdc, tdc->regs->dst), 310 tdc_read(tdc, tdc->regs->high_addr)); 311 } 312 313 dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x WSTA %x\n", 314 tdc_read(tdc, tdc->regs->mc_seq), 315 tdc_read(tdc, tdc->regs->mmio_seq), 316 tdc_read(tdc, tdc->regs->wcount), 317 tdc_read(tdc, tdc->regs->wxfer), 318 tdc_read(tdc, tdc->regs->wstatus)); 319 dev_dbg(tdc2dev(tdc), "DMA ERR_STA %x\n", 320 tdc_read(tdc, tdc->regs->err_status)); 321 } 322 323 static int tegra_dma_sid_reserve(struct tegra_dma_channel *tdc, 324 enum dma_transfer_direction direction) 325 { 326 struct tegra_dma *tdma = tdc->tdma; 327 int sid = tdc->slave_id; 328 329 if (!is_slave_direction(direction)) 330 return 0; 331 332 switch (direction) { 333 case DMA_MEM_TO_DEV: 334 if (test_and_set_bit(sid, &tdma->sid_m2d_reserved)) { 335 dev_err(tdma->dev, "slave id already in use\n"); 336 return -EINVAL; 337 } 338 break; 339 case DMA_DEV_TO_MEM: 340 if (test_and_set_bit(sid, &tdma->sid_d2m_reserved)) { 341 dev_err(tdma->dev, "slave id already in use\n"); 342 return -EINVAL; 343 } 344 break; 345 default: 346 break; 347 } 348 349 tdc->sid_dir = direction; 350 351 return 0; 352 } 353 354 static void tegra_dma_sid_free(struct tegra_dma_channel *tdc) 355 { 356 struct tegra_dma *tdma = tdc->tdma; 357 int sid = tdc->slave_id; 358 359 switch (tdc->sid_dir) { 360 case DMA_MEM_TO_DEV: 361 clear_bit(sid, &tdma->sid_m2d_reserved); 362 break; 363 case DMA_DEV_TO_MEM: 364 clear_bit(sid, &tdma->sid_d2m_reserved); 365 break; 366 default: 367 break; 368 } 369 370 tdc->sid_dir = DMA_TRANS_NONE; 371 } 372 373 static void tegra_dma_desc_free(struct virt_dma_desc *vd) 374 { 375 kfree(container_of(vd, struct tegra_dma_desc, vd)); 376 } 377 378 static int tegra_dma_slave_config(struct dma_chan *dc, 379 struct dma_slave_config *sconfig) 380 { 381 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 382 383 memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig)); 384 tdc->config_init = true; 385 386 return 0; 387 } 388 389 static int tegra_dma_pause(struct tegra_dma_channel *tdc) 390 { 391 int ret; 392 u32 val; 393 394 val = tdc_read(tdc, tdc->regs->csre); 395 val |= TEGRA_GPCDMA_CHAN_CSRE_PAUSE; 396 tdc_write(tdc, tdc->regs->csre, val); 397 398 /* Wait until busy bit is de-asserted */ 399 ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr + 400 tdc->chan_base_offset + tdc->regs->status, 401 val, 402 !(val & TEGRA_GPCDMA_STATUS_BUSY), 403 TEGRA_GPCDMA_BURST_COMPLETE_TIME, 404 TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT); 405 406 if (ret) { 407 dev_err(tdc2dev(tdc), "DMA pause timed out\n"); 408 tegra_dma_dump_chan_regs(tdc); 409 } 410 411 tdc->status = DMA_PAUSED; 412 413 return ret; 414 } 415 416 static int tegra_dma_device_pause(struct dma_chan *dc) 417 { 418 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 419 unsigned long flags; 420 int ret; 421 422 if (!tdc->tdma->chip_data->hw_support_pause) 423 return -ENOSYS; 424 425 spin_lock_irqsave(&tdc->vc.lock, flags); 426 ret = tegra_dma_pause(tdc); 427 spin_unlock_irqrestore(&tdc->vc.lock, flags); 428 429 return ret; 430 } 431 432 static void tegra_dma_resume(struct tegra_dma_channel *tdc) 433 { 434 u32 val; 435 436 val = tdc_read(tdc, tdc->regs->csre); 437 val &= ~TEGRA_GPCDMA_CHAN_CSRE_PAUSE; 438 tdc_write(tdc, tdc->regs->csre, val); 439 440 tdc->status = DMA_IN_PROGRESS; 441 } 442 443 static int tegra_dma_device_resume(struct dma_chan *dc) 444 { 445 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 446 unsigned long flags; 447 448 if (!tdc->tdma->chip_data->hw_support_pause) 449 return -ENOSYS; 450 451 spin_lock_irqsave(&tdc->vc.lock, flags); 452 tegra_dma_resume(tdc); 453 spin_unlock_irqrestore(&tdc->vc.lock, flags); 454 455 return 0; 456 } 457 458 static inline int tegra_dma_pause_noerr(struct tegra_dma_channel *tdc) 459 { 460 /* Return 0 irrespective of PAUSE status. 461 * This is useful to recover channels that can exit out of flush 462 * state when the channel is disabled. 463 */ 464 465 tegra_dma_pause(tdc); 466 return 0; 467 } 468 469 static void tegra_dma_disable(struct tegra_dma_channel *tdc) 470 { 471 u32 csr, status; 472 473 csr = tdc_read(tdc, tdc->regs->csr); 474 475 /* Disable interrupts */ 476 csr &= ~TEGRA_GPCDMA_CSR_IE_EOC; 477 478 /* Disable DMA */ 479 csr &= ~TEGRA_GPCDMA_CSR_ENB; 480 tdc_write(tdc, tdc->regs->csr, csr); 481 482 /* Clear interrupt status if it is there */ 483 status = tdc_read(tdc, tdc->regs->status); 484 if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) { 485 dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__); 486 tdc_write(tdc, tdc->regs->status, status); 487 } 488 } 489 490 static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc) 491 { 492 struct tegra_dma_desc *dma_desc = tdc->dma_desc; 493 struct tegra_dma_sg_req *sg_req; 494 int ret; 495 u32 val; 496 497 dma_desc->sg_idx++; 498 499 /* Reset the sg index for cyclic transfers */ 500 if (dma_desc->sg_idx == dma_desc->sg_count) 501 dma_desc->sg_idx = 0; 502 503 /* Configure next transfer immediately after DMA is busy */ 504 ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr + 505 tdc->chan_base_offset + tdc->regs->status, 506 val, 507 (val & TEGRA_GPCDMA_STATUS_BUSY), 0, 508 TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT); 509 if (ret) 510 return; 511 512 sg_req = &dma_desc->sg_req[dma_desc->sg_idx]; 513 514 tdc_write(tdc, tdc->regs->wcount, sg_req->wcount); 515 tegra_dma_program_addr(tdc, sg_req); 516 517 /* Start DMA */ 518 tdc_write(tdc, tdc->regs->csr, 519 sg_req->csr | TEGRA_GPCDMA_CSR_ENB); 520 } 521 522 static void tegra_dma_start(struct tegra_dma_channel *tdc) 523 { 524 struct tegra_dma_desc *dma_desc = tdc->dma_desc; 525 struct tegra_dma_sg_req *sg_req; 526 struct virt_dma_desc *vdesc; 527 528 if (!dma_desc) { 529 vdesc = vchan_next_desc(&tdc->vc); 530 if (!vdesc) 531 return; 532 533 dma_desc = vd_to_tegra_dma_desc(vdesc); 534 list_del(&vdesc->node); 535 dma_desc->tdc = tdc; 536 tdc->dma_desc = dma_desc; 537 538 tegra_dma_resume(tdc); 539 } 540 541 sg_req = &dma_desc->sg_req[dma_desc->sg_idx]; 542 543 tegra_dma_program_addr(tdc, sg_req); 544 tdc_write(tdc, tdc->regs->wcount, sg_req->wcount); 545 tdc_write(tdc, tdc->regs->csr, 0); 546 tdc_write(tdc, tdc->regs->fixed_pattern, sg_req->fixed_pattern); 547 tdc_write(tdc, tdc->regs->mmio_seq, sg_req->mmio_seq); 548 tdc_write(tdc, tdc->regs->mc_seq, sg_req->mc_seq); 549 tdc_write(tdc, tdc->regs->csr, sg_req->csr); 550 551 /* Start DMA */ 552 tdc_write(tdc, tdc->regs->csr, 553 sg_req->csr | TEGRA_GPCDMA_CSR_ENB); 554 } 555 556 static void tegra_dma_xfer_complete(struct tegra_dma_channel *tdc) 557 { 558 vchan_cookie_complete(&tdc->dma_desc->vd); 559 560 tegra_dma_sid_free(tdc); 561 tdc->dma_desc = NULL; 562 tdc->status = DMA_COMPLETE; 563 } 564 565 static void tegra_dma_chan_decode_error(struct tegra_dma_channel *tdc, 566 unsigned int err_status) 567 { 568 switch (TEGRA_GPCDMA_CHAN_ERR_TYPE(err_status)) { 569 case TEGRA_DMA_BM_FIFO_FULL_ERR: 570 dev_err(tdc->tdma->dev, 571 "GPCDMA CH%d bm fifo full\n", tdc->id); 572 break; 573 574 case TEGRA_DMA_PERIPH_FIFO_FULL_ERR: 575 dev_err(tdc->tdma->dev, 576 "GPCDMA CH%d peripheral fifo full\n", tdc->id); 577 break; 578 579 case TEGRA_DMA_PERIPH_ID_ERR: 580 dev_err(tdc->tdma->dev, 581 "GPCDMA CH%d illegal peripheral id\n", tdc->id); 582 break; 583 584 case TEGRA_DMA_STREAM_ID_ERR: 585 dev_err(tdc->tdma->dev, 586 "GPCDMA CH%d illegal stream id\n", tdc->id); 587 break; 588 589 case TEGRA_DMA_MC_SLAVE_ERR: 590 dev_err(tdc->tdma->dev, 591 "GPCDMA CH%d mc slave error\n", tdc->id); 592 break; 593 594 case TEGRA_DMA_MMIO_SLAVE_ERR: 595 dev_err(tdc->tdma->dev, 596 "GPCDMA CH%d mmio slave error\n", tdc->id); 597 break; 598 599 default: 600 dev_err(tdc->tdma->dev, 601 "GPCDMA CH%d security violation %x\n", tdc->id, 602 err_status); 603 } 604 } 605 606 static irqreturn_t tegra_dma_isr(int irq, void *dev_id) 607 { 608 struct tegra_dma_channel *tdc = dev_id; 609 struct tegra_dma_desc *dma_desc = tdc->dma_desc; 610 struct tegra_dma_sg_req *sg_req; 611 u32 status; 612 613 /* Check channel error status register */ 614 status = tdc_read(tdc, tdc->regs->err_status); 615 if (status) { 616 tegra_dma_chan_decode_error(tdc, status); 617 tegra_dma_dump_chan_regs(tdc); 618 tdc_write(tdc, tdc->regs->err_status, 0xFFFFFFFF); 619 } 620 621 spin_lock(&tdc->vc.lock); 622 status = tdc_read(tdc, tdc->regs->status); 623 if (!(status & TEGRA_GPCDMA_STATUS_ISE_EOC)) 624 goto irq_done; 625 626 tdc_write(tdc, tdc->regs->status, 627 TEGRA_GPCDMA_STATUS_ISE_EOC); 628 629 if (!dma_desc) 630 goto irq_done; 631 632 sg_req = dma_desc->sg_req; 633 dma_desc->bytes_xfer += sg_req[dma_desc->sg_idx].len; 634 635 if (dma_desc->cyclic) { 636 vchan_cyclic_callback(&dma_desc->vd); 637 tegra_dma_configure_next_sg(tdc); 638 } else { 639 dma_desc->sg_idx++; 640 if (dma_desc->sg_idx == dma_desc->sg_count) 641 tegra_dma_xfer_complete(tdc); 642 else 643 tegra_dma_start(tdc); 644 } 645 646 irq_done: 647 spin_unlock(&tdc->vc.lock); 648 return IRQ_HANDLED; 649 } 650 651 static void tegra_dma_issue_pending(struct dma_chan *dc) 652 { 653 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 654 unsigned long flags; 655 656 if (tdc->dma_desc) 657 return; 658 659 spin_lock_irqsave(&tdc->vc.lock, flags); 660 if (vchan_issue_pending(&tdc->vc)) 661 tegra_dma_start(tdc); 662 663 /* 664 * For cyclic DMA transfers, program the second 665 * transfer parameters as soon as the first DMA 666 * transfer is started inorder for the DMA 667 * controller to trigger the second transfer 668 * with the correct parameters. 669 */ 670 if (tdc->dma_desc && tdc->dma_desc->cyclic) 671 tegra_dma_configure_next_sg(tdc); 672 673 spin_unlock_irqrestore(&tdc->vc.lock, flags); 674 } 675 676 static int tegra_dma_stop_client(struct tegra_dma_channel *tdc) 677 { 678 int ret; 679 u32 status, csr; 680 681 /* 682 * Change the client associated with the DMA channel 683 * to stop DMA engine from starting any more bursts for 684 * the given client and wait for in flight bursts to complete 685 */ 686 csr = tdc_read(tdc, tdc->regs->csr); 687 csr &= ~(TEGRA_GPCDMA_CSR_REQ_SEL_MASK); 688 csr |= TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED; 689 tdc_write(tdc, tdc->regs->csr, csr); 690 691 /* Wait for in flight data transfer to finish */ 692 udelay(TEGRA_GPCDMA_BURST_COMPLETE_TIME); 693 694 /* If TX/RX path is still active wait till it becomes 695 * inactive 696 */ 697 698 ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr + 699 tdc->chan_base_offset + 700 tdc->regs->status, 701 status, 702 !(status & (TEGRA_GPCDMA_STATUS_CHANNEL_TX | 703 TEGRA_GPCDMA_STATUS_CHANNEL_RX)), 704 5, 705 TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT); 706 if (ret) { 707 dev_err(tdc2dev(tdc), "Timeout waiting for DMA burst completion!\n"); 708 tegra_dma_dump_chan_regs(tdc); 709 } 710 711 return ret; 712 } 713 714 static int tegra_dma_terminate_all(struct dma_chan *dc) 715 { 716 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 717 unsigned long flags; 718 LIST_HEAD(head); 719 int err; 720 721 spin_lock_irqsave(&tdc->vc.lock, flags); 722 723 if (tdc->dma_desc) { 724 err = tdc->tdma->chip_data->terminate(tdc); 725 if (err) { 726 spin_unlock_irqrestore(&tdc->vc.lock, flags); 727 return err; 728 } 729 730 vchan_terminate_vdesc(&tdc->dma_desc->vd); 731 tegra_dma_disable(tdc); 732 tdc->dma_desc = NULL; 733 } 734 735 tdc->status = DMA_COMPLETE; 736 tegra_dma_sid_free(tdc); 737 vchan_get_all_descriptors(&tdc->vc, &head); 738 spin_unlock_irqrestore(&tdc->vc.lock, flags); 739 740 vchan_dma_desc_free_list(&tdc->vc, &head); 741 742 return 0; 743 } 744 745 static int tegra_dma_get_residual(struct tegra_dma_channel *tdc) 746 { 747 struct tegra_dma_desc *dma_desc = tdc->dma_desc; 748 struct tegra_dma_sg_req *sg_req = dma_desc->sg_req; 749 unsigned int bytes_xfer, residual; 750 u32 wcount = 0, status; 751 752 wcount = tdc_read(tdc, tdc->regs->wxfer); 753 754 /* 755 * Set wcount = 0 if EOC bit is set. The transfer would have 756 * already completed and the CHAN_XFER_COUNT could have updated 757 * for the next transfer, specifically in case of cyclic transfers. 758 */ 759 status = tdc_read(tdc, tdc->regs->status); 760 if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) 761 wcount = 0; 762 763 bytes_xfer = dma_desc->bytes_xfer + 764 sg_req[dma_desc->sg_idx].len - (wcount * 4); 765 766 if (dma_desc->bytes_req == bytes_xfer) 767 return 0; 768 769 residual = dma_desc->bytes_req - (bytes_xfer % dma_desc->bytes_req); 770 771 return residual; 772 } 773 774 static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, 775 dma_cookie_t cookie, 776 struct dma_tx_state *txstate) 777 { 778 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 779 struct tegra_dma_desc *dma_desc; 780 struct virt_dma_desc *vd; 781 unsigned int residual; 782 unsigned long flags; 783 enum dma_status ret; 784 785 ret = dma_cookie_status(dc, cookie, txstate); 786 if (ret == DMA_COMPLETE) 787 return ret; 788 789 if (tdc->status == DMA_PAUSED) 790 ret = DMA_PAUSED; 791 792 spin_lock_irqsave(&tdc->vc.lock, flags); 793 vd = vchan_find_desc(&tdc->vc, cookie); 794 if (vd) { 795 dma_desc = vd_to_tegra_dma_desc(vd); 796 residual = dma_desc->bytes_req; 797 dma_set_residue(txstate, residual); 798 } else if (tdc->dma_desc && tdc->dma_desc->vd.tx.cookie == cookie) { 799 residual = tegra_dma_get_residual(tdc); 800 dma_set_residue(txstate, residual); 801 } else { 802 dev_err(tdc2dev(tdc), "cookie %d is not found\n", cookie); 803 } 804 spin_unlock_irqrestore(&tdc->vc.lock, flags); 805 806 return ret; 807 } 808 809 static inline int get_bus_width(struct tegra_dma_channel *tdc, 810 enum dma_slave_buswidth slave_bw) 811 { 812 switch (slave_bw) { 813 case DMA_SLAVE_BUSWIDTH_1_BYTE: 814 return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8; 815 case DMA_SLAVE_BUSWIDTH_2_BYTES: 816 return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16; 817 case DMA_SLAVE_BUSWIDTH_4_BYTES: 818 return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32; 819 default: 820 dev_err(tdc2dev(tdc), "given slave bus width is not supported\n"); 821 return -EINVAL; 822 } 823 } 824 825 static unsigned int get_burst_size(struct tegra_dma_channel *tdc, 826 u32 burst_size, enum dma_slave_buswidth slave_bw, 827 int len) 828 { 829 unsigned int burst_mmio_width, burst_byte; 830 831 /* 832 * burst_size from client is in terms of the bus_width. 833 * convert that into words. 834 * If burst_size is not specified from client, then use 835 * len to calculate the optimum burst size 836 */ 837 burst_byte = burst_size ? burst_size * slave_bw : len; 838 839 /* 840 * Find the largest burst size that evenly divides the transfer length. 841 * The hardware requires the transfer length to be a multiple of the 842 * burst size - partial bursts are not supported. 843 */ 844 burst_byte = min(burst_byte, 1U << __ffs(len)); 845 burst_mmio_width = burst_byte / 4; 846 847 if (burst_mmio_width < TEGRA_GPCDMA_MMIOSEQ_BURST_MIN) 848 return 0; 849 850 burst_mmio_width = min(burst_mmio_width, TEGRA_GPCDMA_MMIOSEQ_BURST_MAX); 851 852 return TEGRA_GPCDMA_MMIOSEQ_BURST(burst_mmio_width); 853 } 854 855 static int get_transfer_param(struct tegra_dma_channel *tdc, 856 enum dma_transfer_direction direction, 857 dma_addr_t *apb_addr, 858 u32 *mmio_seq, 859 u32 *csr, 860 unsigned int *burst_size, 861 enum dma_slave_buswidth *slave_bw) 862 { 863 switch (direction) { 864 case DMA_MEM_TO_DEV: 865 *apb_addr = tdc->dma_sconfig.dst_addr; 866 *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width); 867 *burst_size = tdc->dma_sconfig.dst_maxburst; 868 *slave_bw = tdc->dma_sconfig.dst_addr_width; 869 *csr = TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC; 870 return 0; 871 case DMA_DEV_TO_MEM: 872 *apb_addr = tdc->dma_sconfig.src_addr; 873 *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width); 874 *burst_size = tdc->dma_sconfig.src_maxburst; 875 *slave_bw = tdc->dma_sconfig.src_addr_width; 876 *csr = TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC; 877 return 0; 878 default: 879 dev_err(tdc2dev(tdc), "DMA direction is not supported\n"); 880 } 881 882 return -EINVAL; 883 } 884 885 static struct dma_async_tx_descriptor * 886 tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value, 887 size_t len, unsigned long flags) 888 { 889 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 890 unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count; 891 struct tegra_dma_sg_req *sg_req; 892 struct tegra_dma_desc *dma_desc; 893 u32 csr, mc_seq; 894 895 if ((len & 3) || (dest & 3) || len > max_dma_count) { 896 dev_err(tdc2dev(tdc), 897 "DMA length/memory address is not supported\n"); 898 return NULL; 899 } 900 901 /* Set DMA mode to fixed pattern */ 902 csr = TEGRA_GPCDMA_CSR_DMA_FIXED_PAT; 903 /* Enable once or continuous mode */ 904 csr |= TEGRA_GPCDMA_CSR_ONCE; 905 /* Enable IRQ mask */ 906 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; 907 /* Enable the DMA interrupt */ 908 if (flags & DMA_PREP_INTERRUPT) 909 csr |= TEGRA_GPCDMA_CSR_IE_EOC; 910 /* Configure default priority weight for the channel */ 911 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1); 912 913 mc_seq = tdc_read(tdc, tdc->regs->mc_seq); 914 /* retain stream-id and clean rest */ 915 mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK; 916 917 /* Set the address wrapping */ 918 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0, 919 TEGRA_GPCDMA_MCSEQ_WRAP_NONE); 920 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1, 921 TEGRA_GPCDMA_MCSEQ_WRAP_NONE); 922 923 /* Program outstanding MC requests */ 924 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1); 925 /* Set burst size */ 926 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; 927 928 dma_desc = kzalloc_flex(*dma_desc, sg_req, 1, GFP_NOWAIT); 929 if (!dma_desc) 930 return NULL; 931 932 dma_desc->bytes_req = len; 933 dma_desc->sg_count = 1; 934 sg_req = dma_desc->sg_req; 935 sg_req[0].src = 0; 936 sg_req[0].dst = dest; 937 938 sg_req[0].fixed_pattern = value; 939 /* Word count reg takes value as (N +1) words */ 940 sg_req[0].wcount = ((len - 4) >> 2); 941 sg_req[0].csr = csr; 942 sg_req[0].mmio_seq = 0; 943 sg_req[0].mc_seq = mc_seq; 944 sg_req[0].len = len; 945 946 dma_desc->cyclic = false; 947 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags); 948 } 949 950 static struct dma_async_tx_descriptor * 951 tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest, 952 dma_addr_t src, size_t len, unsigned long flags) 953 { 954 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 955 struct tegra_dma_sg_req *sg_req; 956 struct tegra_dma_desc *dma_desc; 957 unsigned int max_dma_count; 958 u32 csr, mc_seq; 959 960 max_dma_count = tdc->tdma->chip_data->max_dma_count; 961 if ((len & 3) || (src & 3) || (dest & 3) || len > max_dma_count) { 962 dev_err(tdc2dev(tdc), 963 "DMA length/memory address is not supported\n"); 964 return NULL; 965 } 966 967 /* Set DMA mode to memory to memory transfer */ 968 csr = TEGRA_GPCDMA_CSR_DMA_MEM2MEM; 969 /* Enable once or continuous mode */ 970 csr |= TEGRA_GPCDMA_CSR_ONCE; 971 /* Enable IRQ mask */ 972 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; 973 /* Enable the DMA interrupt */ 974 if (flags & DMA_PREP_INTERRUPT) 975 csr |= TEGRA_GPCDMA_CSR_IE_EOC; 976 /* Configure default priority weight for the channel */ 977 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1); 978 979 mc_seq = tdc_read(tdc, tdc->regs->mc_seq); 980 /* retain stream-id and clean rest */ 981 mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK) | 982 (TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK); 983 984 /* Set the address wrapping */ 985 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0, 986 TEGRA_GPCDMA_MCSEQ_WRAP_NONE); 987 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1, 988 TEGRA_GPCDMA_MCSEQ_WRAP_NONE); 989 990 /* Program outstanding MC requests */ 991 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1); 992 /* Set burst size */ 993 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; 994 995 dma_desc = kzalloc_flex(*dma_desc, sg_req, 1, GFP_NOWAIT); 996 if (!dma_desc) 997 return NULL; 998 999 dma_desc->bytes_req = len; 1000 dma_desc->sg_count = 1; 1001 sg_req = dma_desc->sg_req; 1002 1003 sg_req[0].src = src; 1004 sg_req[0].dst = dest; 1005 1006 /* Word count reg takes value as (N +1) words */ 1007 sg_req[0].wcount = ((len - 4) >> 2); 1008 sg_req[0].csr = csr; 1009 sg_req[0].mmio_seq = 0; 1010 sg_req[0].mc_seq = mc_seq; 1011 sg_req[0].len = len; 1012 1013 dma_desc->cyclic = false; 1014 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags); 1015 } 1016 1017 static struct dma_async_tx_descriptor * 1018 tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl, 1019 unsigned int sg_len, enum dma_transfer_direction direction, 1020 unsigned long flags, void *context) 1021 { 1022 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1023 unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count; 1024 enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED; 1025 u32 csr, mc_seq, mmio_seq = 0; 1026 dma_addr_t apb_ptr = 0; 1027 struct tegra_dma_sg_req *sg_req; 1028 struct tegra_dma_desc *dma_desc; 1029 struct scatterlist *sg; 1030 u32 burst_size; 1031 unsigned int i; 1032 int ret; 1033 1034 if (!tdc->config_init) { 1035 dev_err(tdc2dev(tdc), "DMA channel is not configured\n"); 1036 return NULL; 1037 } 1038 if (sg_len < 1) { 1039 dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len); 1040 return NULL; 1041 } 1042 1043 ret = tegra_dma_sid_reserve(tdc, direction); 1044 if (ret) 1045 return NULL; 1046 1047 ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr, 1048 &burst_size, &slave_bw); 1049 if (ret < 0) 1050 return NULL; 1051 1052 /* Enable once or continuous mode */ 1053 csr |= TEGRA_GPCDMA_CSR_ONCE; 1054 /* Program the slave id in requestor select */ 1055 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id); 1056 /* Enable IRQ mask */ 1057 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; 1058 /* Configure default priority weight for the channel*/ 1059 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1); 1060 1061 /* Enable the DMA interrupt */ 1062 if (flags & DMA_PREP_INTERRUPT) 1063 csr |= TEGRA_GPCDMA_CSR_IE_EOC; 1064 1065 mc_seq = tdc_read(tdc, tdc->regs->mc_seq); 1066 /* retain stream-id and clean rest */ 1067 mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK; 1068 1069 /* Set the address wrapping on both MC and MMIO side */ 1070 1071 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0, 1072 TEGRA_GPCDMA_MCSEQ_WRAP_NONE); 1073 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1, 1074 TEGRA_GPCDMA_MCSEQ_WRAP_NONE); 1075 mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1); 1076 1077 /* Program 2 MC outstanding requests by default. */ 1078 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1); 1079 1080 /* Setting MC burst size depending on MMIO burst size */ 1081 if (burst_size == 64) 1082 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; 1083 else 1084 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2; 1085 1086 dma_desc = kzalloc_flex(*dma_desc, sg_req, sg_len, GFP_NOWAIT); 1087 if (!dma_desc) 1088 return NULL; 1089 1090 dma_desc->sg_count = sg_len; 1091 sg_req = dma_desc->sg_req; 1092 1093 /* Make transfer requests */ 1094 for_each_sg(sgl, sg, sg_len, i) { 1095 u32 len; 1096 dma_addr_t mem; 1097 1098 mem = sg_dma_address(sg); 1099 len = sg_dma_len(sg); 1100 1101 if ((len & 3) || (mem & 3) || len > max_dma_count) { 1102 dev_err(tdc2dev(tdc), 1103 "DMA length/memory address is not supported\n"); 1104 kfree(dma_desc); 1105 return NULL; 1106 } 1107 1108 mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len); 1109 dma_desc->bytes_req += len; 1110 1111 if (direction == DMA_MEM_TO_DEV) { 1112 sg_req[i].src = mem; 1113 sg_req[i].dst = apb_ptr; 1114 } else if (direction == DMA_DEV_TO_MEM) { 1115 sg_req[i].src = apb_ptr; 1116 sg_req[i].dst = mem; 1117 } 1118 1119 /* 1120 * Word count register takes input in words. Writing a value 1121 * of N into word count register means a req of (N+1) words. 1122 */ 1123 sg_req[i].wcount = ((len - 4) >> 2); 1124 sg_req[i].csr = csr; 1125 sg_req[i].mmio_seq = mmio_seq; 1126 sg_req[i].mc_seq = mc_seq; 1127 sg_req[i].len = len; 1128 } 1129 1130 dma_desc->cyclic = false; 1131 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags); 1132 } 1133 1134 static struct dma_async_tx_descriptor * 1135 tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, 1136 size_t period_len, enum dma_transfer_direction direction, 1137 unsigned long flags) 1138 { 1139 enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED; 1140 u32 csr, mc_seq, mmio_seq = 0, burst_size; 1141 dma_addr_t apb_ptr = 0; 1142 unsigned int max_dma_count, len, period_count, i; 1143 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1144 struct tegra_dma_desc *dma_desc; 1145 struct tegra_dma_sg_req *sg_req; 1146 dma_addr_t mem = buf_addr; 1147 int ret; 1148 1149 if (!buf_len || !period_len) { 1150 dev_err(tdc2dev(tdc), "Invalid buffer/period len\n"); 1151 return NULL; 1152 } 1153 1154 if (!tdc->config_init) { 1155 dev_err(tdc2dev(tdc), "DMA slave is not configured\n"); 1156 return NULL; 1157 } 1158 1159 ret = tegra_dma_sid_reserve(tdc, direction); 1160 if (ret) 1161 return NULL; 1162 1163 /* 1164 * We only support cycle transfer when buf_len is multiple of 1165 * period_len. 1166 */ 1167 if (buf_len % period_len) { 1168 dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n"); 1169 return NULL; 1170 } 1171 1172 len = period_len; 1173 max_dma_count = tdc->tdma->chip_data->max_dma_count; 1174 if ((len & 3) || (buf_addr & 3) || len > max_dma_count) { 1175 dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n"); 1176 return NULL; 1177 } 1178 1179 ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr, 1180 &burst_size, &slave_bw); 1181 if (ret < 0) 1182 return NULL; 1183 1184 /* Enable once or continuous mode */ 1185 csr &= ~TEGRA_GPCDMA_CSR_ONCE; 1186 /* Program the slave id in requestor select */ 1187 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id); 1188 /* Enable IRQ mask */ 1189 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK; 1190 /* Configure default priority weight for the channel*/ 1191 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1); 1192 1193 /* Enable the DMA interrupt */ 1194 if (flags & DMA_PREP_INTERRUPT) 1195 csr |= TEGRA_GPCDMA_CSR_IE_EOC; 1196 1197 mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1); 1198 1199 mc_seq = tdc_read(tdc, tdc->regs->mc_seq); 1200 /* retain stream-id and clean rest */ 1201 mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK; 1202 1203 /* Set the address wrapping on both MC and MMIO side */ 1204 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0, 1205 TEGRA_GPCDMA_MCSEQ_WRAP_NONE); 1206 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1, 1207 TEGRA_GPCDMA_MCSEQ_WRAP_NONE); 1208 1209 /* Program 2 MC outstanding requests by default. */ 1210 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1); 1211 /* Setting MC burst size depending on MMIO burst size */ 1212 if (burst_size == 64) 1213 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16; 1214 else 1215 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2; 1216 1217 period_count = buf_len / period_len; 1218 dma_desc = kzalloc_flex(*dma_desc, sg_req, period_count, GFP_NOWAIT); 1219 if (!dma_desc) 1220 return NULL; 1221 1222 dma_desc->bytes_req = buf_len; 1223 dma_desc->sg_count = period_count; 1224 sg_req = dma_desc->sg_req; 1225 1226 /* Split transfer equal to period size */ 1227 for (i = 0; i < period_count; i++) { 1228 mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len); 1229 if (direction == DMA_MEM_TO_DEV) { 1230 sg_req[i].src = mem; 1231 sg_req[i].dst = apb_ptr; 1232 } else if (direction == DMA_DEV_TO_MEM) { 1233 sg_req[i].src = apb_ptr; 1234 sg_req[i].dst = mem; 1235 } 1236 /* 1237 * Word count register takes input in words. Writing a value 1238 * of N into word count register means a req of (N+1) words. 1239 */ 1240 sg_req[i].wcount = ((len - 4) >> 2); 1241 sg_req[i].csr = csr; 1242 sg_req[i].mmio_seq = mmio_seq; 1243 sg_req[i].mc_seq = mc_seq; 1244 sg_req[i].len = len; 1245 1246 mem += len; 1247 } 1248 1249 dma_desc->cyclic = true; 1250 1251 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags); 1252 } 1253 1254 static int tegra_dma_alloc_chan_resources(struct dma_chan *dc) 1255 { 1256 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1257 int ret; 1258 1259 ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc); 1260 if (ret) { 1261 dev_err(tdc2dev(tdc), "request_irq failed for %s\n", tdc->name); 1262 return ret; 1263 } 1264 1265 dma_cookie_init(&tdc->vc.chan); 1266 tdc->config_init = false; 1267 return 0; 1268 } 1269 1270 static void tegra_dma_chan_synchronize(struct dma_chan *dc) 1271 { 1272 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1273 1274 synchronize_irq(tdc->irq); 1275 vchan_synchronize(&tdc->vc); 1276 } 1277 1278 static void tegra_dma_free_chan_resources(struct dma_chan *dc) 1279 { 1280 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc); 1281 1282 dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id); 1283 1284 tegra_dma_terminate_all(dc); 1285 synchronize_irq(tdc->irq); 1286 1287 tasklet_kill(&tdc->vc.task); 1288 tdc->config_init = false; 1289 tdc->slave_id = -1; 1290 tdc->sid_dir = DMA_TRANS_NONE; 1291 free_irq(tdc->irq, tdc); 1292 1293 vchan_free_chan_resources(&tdc->vc); 1294 } 1295 1296 static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, 1297 struct of_dma *ofdma) 1298 { 1299 struct tegra_dma *tdma = ofdma->of_dma_data; 1300 struct tegra_dma_channel *tdc; 1301 struct dma_chan *chan; 1302 1303 chan = dma_get_any_slave_channel(&tdma->dma_dev); 1304 if (!chan) 1305 return NULL; 1306 1307 tdc = to_tegra_dma_chan(chan); 1308 tdc->slave_id = dma_spec->args[0]; 1309 1310 return chan; 1311 } 1312 1313 static const struct tegra_dma_channel_regs tegra186_reg_offsets = { 1314 .csr = 0x0, 1315 .status = 0x4, 1316 .csre = 0x8, 1317 .src = 0xc, 1318 .dst = 0x10, 1319 .high_addr = 0x14, 1320 .mc_seq = 0x18, 1321 .mmio_seq = 0x1c, 1322 .wcount = 0x20, 1323 .wxfer = 0x24, 1324 .wstatus = 0x28, 1325 .err_status = 0x30, 1326 .fixed_pattern = 0x34, 1327 }; 1328 1329 static const struct tegra_dma_channel_regs tegra264_reg_offsets = { 1330 .csr = 0x0, 1331 .status = 0x4, 1332 .csre = 0x8, 1333 .src = 0xc, 1334 .dst = 0x10, 1335 .src_high = 0x14, 1336 .dst_high = 0x18, 1337 .mc_seq = 0x1c, 1338 .mmio_seq = 0x20, 1339 .wcount = 0x24, 1340 .wxfer = 0x28, 1341 .wstatus = 0x2c, 1342 .err_status = 0x34, 1343 .fixed_pattern = 0x38, 1344 }; 1345 1346 static const struct tegra_dma_chip_data tegra186_dma_chip_data = { 1347 .nr_channels = 32, 1348 .addr_bits = 39, 1349 .channel_reg_size = SZ_64K, 1350 .max_dma_count = SZ_1G, 1351 .hw_support_pause = false, 1352 .channel_regs = &tegra186_reg_offsets, 1353 .terminate = tegra_dma_stop_client, 1354 }; 1355 1356 static const struct tegra_dma_chip_data tegra194_dma_chip_data = { 1357 .nr_channels = 32, 1358 .addr_bits = 39, 1359 .channel_reg_size = SZ_64K, 1360 .max_dma_count = SZ_1G, 1361 .hw_support_pause = true, 1362 .channel_regs = &tegra186_reg_offsets, 1363 .terminate = tegra_dma_pause, 1364 }; 1365 1366 static const struct tegra_dma_chip_data tegra234_dma_chip_data = { 1367 .nr_channels = 32, 1368 .addr_bits = 39, 1369 .channel_reg_size = SZ_64K, 1370 .max_dma_count = SZ_1G, 1371 .hw_support_pause = true, 1372 .channel_regs = &tegra186_reg_offsets, 1373 .terminate = tegra_dma_pause_noerr, 1374 }; 1375 1376 static const struct tegra_dma_chip_data tegra264_dma_chip_data = { 1377 .nr_channels = 32, 1378 .addr_bits = 41, 1379 .channel_reg_size = SZ_64K, 1380 .max_dma_count = SZ_1G, 1381 .hw_support_pause = true, 1382 .channel_regs = &tegra264_reg_offsets, 1383 .terminate = tegra_dma_pause_noerr, 1384 }; 1385 1386 static const struct of_device_id tegra_dma_of_match[] = { 1387 { 1388 .compatible = "nvidia,tegra186-gpcdma", 1389 .data = &tegra186_dma_chip_data, 1390 }, { 1391 .compatible = "nvidia,tegra194-gpcdma", 1392 .data = &tegra194_dma_chip_data, 1393 }, { 1394 .compatible = "nvidia,tegra234-gpcdma", 1395 .data = &tegra234_dma_chip_data, 1396 }, { 1397 .compatible = "nvidia,tegra264-gpcdma", 1398 .data = &tegra264_dma_chip_data, 1399 }, { 1400 }, 1401 }; 1402 MODULE_DEVICE_TABLE(of, tegra_dma_of_match); 1403 1404 static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id) 1405 { 1406 unsigned int reg_val = tdc_read(tdc, tdc->regs->mc_seq); 1407 1408 reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK); 1409 reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK); 1410 1411 reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK, stream_id); 1412 reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK, stream_id); 1413 1414 tdc_write(tdc, tdc->regs->mc_seq, reg_val); 1415 return 0; 1416 } 1417 1418 static int tegra_dma_probe(struct platform_device *pdev) 1419 { 1420 const struct tegra_dma_chip_data *cdata = NULL; 1421 struct tegra_dma_channel *tdc; 1422 struct tegra_dma *tdma; 1423 struct dma_chan *chan; 1424 struct device *chdev; 1425 bool use_iommu_map = false; 1426 unsigned int i; 1427 u32 stream_id; 1428 int ret; 1429 1430 cdata = of_device_get_match_data(&pdev->dev); 1431 1432 tdma = devm_kzalloc(&pdev->dev, 1433 struct_size(tdma, channels, cdata->nr_channels), 1434 GFP_KERNEL); 1435 if (!tdma) 1436 return -ENOMEM; 1437 1438 tdma->dev = &pdev->dev; 1439 tdma->chip_data = cdata; 1440 platform_set_drvdata(pdev, tdma); 1441 1442 tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); 1443 if (IS_ERR(tdma->base_addr)) 1444 return PTR_ERR(tdma->base_addr); 1445 1446 tdma->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, "gpcdma"); 1447 if (IS_ERR(tdma->rst)) { 1448 return dev_err_probe(&pdev->dev, PTR_ERR(tdma->rst), 1449 "Failed to get controller reset\n"); 1450 } 1451 reset_control_reset(tdma->rst); 1452 1453 tdma->dma_dev.dev = &pdev->dev; 1454 1455 use_iommu_map = of_property_present(pdev->dev.of_node, "iommu-map"); 1456 if (!use_iommu_map) { 1457 if (!tegra_dev_iommu_get_stream_id(&pdev->dev, &stream_id)) 1458 return dev_err_probe(&pdev->dev, -EINVAL, "Missing iommu stream-id\n"); 1459 } 1460 1461 ret = device_property_read_u32(&pdev->dev, "dma-channel-mask", 1462 &tdma->chan_mask); 1463 if (ret) { 1464 dev_warn(&pdev->dev, 1465 "Missing dma-channel-mask property, using default channel mask %#x\n", 1466 TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK); 1467 tdma->chan_mask = TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK; 1468 } 1469 1470 /* Initialize vchan for each channel and populate the channels list */ 1471 INIT_LIST_HEAD(&tdma->dma_dev.channels); 1472 for (i = 0; i < cdata->nr_channels; i++) { 1473 tdc = &tdma->channels[i]; 1474 1475 /* Check for channel mask */ 1476 if (!(tdma->chan_mask & BIT(i))) 1477 continue; 1478 1479 tdc->irq = platform_get_irq(pdev, i); 1480 if (tdc->irq < 0) 1481 return tdc->irq; 1482 1483 tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET + 1484 i * cdata->channel_reg_size; 1485 snprintf(tdc->name, sizeof(tdc->name), "gpcdma.%d", i); 1486 tdc->regs = cdata->channel_regs; 1487 tdc->tdma = tdma; 1488 tdc->id = i; 1489 tdc->slave_id = -1; 1490 1491 vchan_init(&tdc->vc, &tdma->dma_dev); 1492 tdc->vc.desc_free = tegra_dma_desc_free; 1493 } 1494 1495 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(cdata->addr_bits)); 1496 1497 dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask); 1498 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask); 1499 dma_cap_set(DMA_MEMCPY, tdma->dma_dev.cap_mask); 1500 dma_cap_set(DMA_MEMSET, tdma->dma_dev.cap_mask); 1501 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask); 1502 1503 /* 1504 * Only word aligned transfers are supported. Set the copy 1505 * alignment shift. 1506 */ 1507 tdma->dma_dev.copy_align = 2; 1508 tdma->dma_dev.fill_align = 2; 1509 tdma->dma_dev.device_alloc_chan_resources = 1510 tegra_dma_alloc_chan_resources; 1511 tdma->dma_dev.device_free_chan_resources = 1512 tegra_dma_free_chan_resources; 1513 tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg; 1514 tdma->dma_dev.device_prep_dma_memcpy = tegra_dma_prep_dma_memcpy; 1515 tdma->dma_dev.device_prep_dma_memset = tegra_dma_prep_dma_memset; 1516 tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic; 1517 tdma->dma_dev.device_config = tegra_dma_slave_config; 1518 tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all; 1519 tdma->dma_dev.device_tx_status = tegra_dma_tx_status; 1520 tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending; 1521 tdma->dma_dev.device_pause = tegra_dma_device_pause; 1522 tdma->dma_dev.device_resume = tegra_dma_device_resume; 1523 tdma->dma_dev.device_synchronize = tegra_dma_chan_synchronize; 1524 tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 1525 1526 /* Register the DMA device and the channels */ 1527 ret = dmaenginem_async_device_register(&tdma->dma_dev); 1528 if (ret < 0) { 1529 dev_err_probe(&pdev->dev, ret, 1530 "GPC DMA driver registration failed\n"); 1531 return ret; 1532 } 1533 1534 /* 1535 * Configure stream ID for each channel from the channels registered 1536 * above. This is done in a separate iteration to ensure that only 1537 * the channels available and registered for the DMA device are used. 1538 */ 1539 list_for_each_entry(chan, &tdma->dma_dev.channels, device_node) { 1540 chdev = &chan->dev->device; 1541 tdc = to_tegra_dma_chan(chan); 1542 1543 if (use_iommu_map) { 1544 chdev->bus = pdev->dev.bus; 1545 dma_coerce_mask_and_coherent(chdev, DMA_BIT_MASK(cdata->addr_bits)); 1546 1547 ret = of_dma_configure_id(chdev, pdev->dev.of_node, 1548 true, &tdc->id); 1549 if (ret) 1550 return dev_err_probe(chdev, ret, 1551 "Failed to configure IOMMU for channel %d\n", tdc->id); 1552 1553 if (!tegra_dev_iommu_get_stream_id(chdev, &stream_id)) 1554 return dev_err_probe(chdev, -EINVAL, 1555 "Failed to get stream ID for channel %d\n", tdc->id); 1556 1557 chan->dev->chan_dma_dev = true; 1558 } 1559 1560 /* program stream-id for this channel */ 1561 tegra_dma_program_sid(tdc, stream_id); 1562 tdc->stream_id = stream_id; 1563 } 1564 1565 ret = devm_of_dma_controller_register(&pdev->dev, pdev->dev.of_node, 1566 tegra_dma_of_xlate, tdma); 1567 if (ret < 0) { 1568 dev_err_probe(&pdev->dev, ret, 1569 "GPC DMA OF registration failed\n"); 1570 return ret; 1571 } 1572 1573 dev_info(&pdev->dev, "GPC DMA driver registered %lu channels\n", 1574 hweight_long(tdma->chan_mask)); 1575 1576 return 0; 1577 } 1578 1579 static int __maybe_unused tegra_dma_pm_suspend(struct device *dev) 1580 { 1581 struct tegra_dma *tdma = dev_get_drvdata(dev); 1582 unsigned int i; 1583 1584 for (i = 0; i < tdma->chip_data->nr_channels; i++) { 1585 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1586 1587 if (!(tdma->chan_mask & BIT(i))) 1588 continue; 1589 1590 if (tdc->dma_desc) { 1591 dev_err(tdma->dev, "channel %u busy\n", i); 1592 return -EBUSY; 1593 } 1594 } 1595 1596 return 0; 1597 } 1598 1599 static int __maybe_unused tegra_dma_pm_resume(struct device *dev) 1600 { 1601 struct tegra_dma *tdma = dev_get_drvdata(dev); 1602 unsigned int i; 1603 1604 reset_control_reset(tdma->rst); 1605 1606 for (i = 0; i < tdma->chip_data->nr_channels; i++) { 1607 struct tegra_dma_channel *tdc = &tdma->channels[i]; 1608 1609 if (!(tdma->chan_mask & BIT(i))) 1610 continue; 1611 1612 tegra_dma_program_sid(tdc, tdc->stream_id); 1613 } 1614 1615 return 0; 1616 } 1617 1618 static const struct dev_pm_ops tegra_dma_dev_pm_ops = { 1619 SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume) 1620 }; 1621 1622 static struct platform_driver tegra_dma_driver = { 1623 .driver = { 1624 .name = "tegra-gpcdma", 1625 .pm = &tegra_dma_dev_pm_ops, 1626 .of_match_table = tegra_dma_of_match, 1627 }, 1628 .probe = tegra_dma_probe, 1629 }; 1630 1631 module_platform_driver(tegra_dma_driver); 1632 1633 MODULE_DESCRIPTION("NVIDIA Tegra GPC DMA Controller driver"); 1634 MODULE_AUTHOR("Pavan Kunapuli <pkunapuli@nvidia.com>"); 1635 MODULE_AUTHOR("Rajesh Gumasta <rgumasta@nvidia.com>"); 1636 MODULE_LICENSE("GPL"); 1637