1 /* 2 * Ingenic JZ4780 DMA controller 3 * 4 * Copyright (c) 2015 Imagination Technologies 5 * Author: Alex Smith <alex@alex-smith.me.uk> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License as published by the 9 * Free Software Foundation; either version 2 of the License, or (at your 10 * option) any later version. 11 */ 12 13 #include <linux/clk.h> 14 #include <linux/dmapool.h> 15 #include <linux/init.h> 16 #include <linux/interrupt.h> 17 #include <linux/module.h> 18 #include <linux/of.h> 19 #include <linux/of_device.h> 20 #include <linux/of_dma.h> 21 #include <linux/platform_device.h> 22 #include <linux/slab.h> 23 24 #include "dmaengine.h" 25 #include "virt-dma.h" 26 27 /* Global registers. */ 28 #define JZ_DMA_REG_DMAC 0x00 29 #define JZ_DMA_REG_DIRQP 0x04 30 #define JZ_DMA_REG_DDR 0x08 31 #define JZ_DMA_REG_DDRS 0x0c 32 #define JZ_DMA_REG_DCKE 0x10 33 #define JZ_DMA_REG_DCKES 0x14 34 #define JZ_DMA_REG_DCKEC 0x18 35 #define JZ_DMA_REG_DMACP 0x1c 36 #define JZ_DMA_REG_DSIRQP 0x20 37 #define JZ_DMA_REG_DSIRQM 0x24 38 #define JZ_DMA_REG_DCIRQP 0x28 39 #define JZ_DMA_REG_DCIRQM 0x2c 40 41 /* Per-channel registers. */ 42 #define JZ_DMA_REG_CHAN(n) (n * 0x20) 43 #define JZ_DMA_REG_DSA 0x00 44 #define JZ_DMA_REG_DTA 0x04 45 #define JZ_DMA_REG_DTC 0x08 46 #define JZ_DMA_REG_DRT 0x0c 47 #define JZ_DMA_REG_DCS 0x10 48 #define JZ_DMA_REG_DCM 0x14 49 #define JZ_DMA_REG_DDA 0x18 50 #define JZ_DMA_REG_DSD 0x1c 51 52 #define JZ_DMA_DMAC_DMAE BIT(0) 53 #define JZ_DMA_DMAC_AR BIT(2) 54 #define JZ_DMA_DMAC_HLT BIT(3) 55 #define JZ_DMA_DMAC_FAIC BIT(27) 56 #define JZ_DMA_DMAC_FMSC BIT(31) 57 58 #define JZ_DMA_DRT_AUTO 0x8 59 60 #define JZ_DMA_DCS_CTE BIT(0) 61 #define JZ_DMA_DCS_HLT BIT(2) 62 #define JZ_DMA_DCS_TT BIT(3) 63 #define JZ_DMA_DCS_AR BIT(4) 64 #define JZ_DMA_DCS_DES8 BIT(30) 65 66 #define JZ_DMA_DCM_LINK BIT(0) 67 #define JZ_DMA_DCM_TIE BIT(1) 68 #define JZ_DMA_DCM_STDE BIT(2) 69 #define JZ_DMA_DCM_TSZ_SHIFT 8 70 #define JZ_DMA_DCM_TSZ_MASK (0x7 << JZ_DMA_DCM_TSZ_SHIFT) 71 #define JZ_DMA_DCM_DP_SHIFT 12 72 #define JZ_DMA_DCM_SP_SHIFT 14 73 #define JZ_DMA_DCM_DAI BIT(22) 74 #define JZ_DMA_DCM_SAI BIT(23) 75 76 #define JZ_DMA_SIZE_4_BYTE 0x0 77 #define JZ_DMA_SIZE_1_BYTE 0x1 78 #define JZ_DMA_SIZE_2_BYTE 0x2 79 #define JZ_DMA_SIZE_16_BYTE 0x3 80 #define JZ_DMA_SIZE_32_BYTE 0x4 81 #define JZ_DMA_SIZE_64_BYTE 0x5 82 #define JZ_DMA_SIZE_128_BYTE 0x6 83 84 #define JZ_DMA_WIDTH_32_BIT 0x0 85 #define JZ_DMA_WIDTH_8_BIT 0x1 86 #define JZ_DMA_WIDTH_16_BIT 0x2 87 88 #define JZ_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 89 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 90 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) 91 92 #define JZ4780_DMA_CTRL_OFFSET 0x1000 93 94 /* macros for use with jz4780_dma_soc_data.flags */ 95 #define JZ_SOC_DATA_ALLOW_LEGACY_DT BIT(0) 96 #define JZ_SOC_DATA_PROGRAMMABLE_DMA BIT(1) 97 #define JZ_SOC_DATA_PER_CHAN_PM BIT(2) 98 #define JZ_SOC_DATA_NO_DCKES_DCKEC BIT(3) 99 100 /** 101 * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller. 102 * @dcm: value for the DCM (channel command) register 103 * @dsa: source address 104 * @dta: target address 105 * @dtc: transfer count (number of blocks of the transfer size specified in DCM 106 * to transfer) in the low 24 bits, offset of the next descriptor from the 107 * descriptor base address in the upper 8 bits. 108 */ 109 struct jz4780_dma_hwdesc { 110 uint32_t dcm; 111 uint32_t dsa; 112 uint32_t dta; 113 uint32_t dtc; 114 }; 115 116 /* Size of allocations for hardware descriptor blocks. */ 117 #define JZ_DMA_DESC_BLOCK_SIZE PAGE_SIZE 118 #define JZ_DMA_MAX_DESC \ 119 (JZ_DMA_DESC_BLOCK_SIZE / sizeof(struct jz4780_dma_hwdesc)) 120 121 struct jz4780_dma_desc { 122 struct virt_dma_desc vdesc; 123 124 struct jz4780_dma_hwdesc *desc; 125 dma_addr_t desc_phys; 126 unsigned int count; 127 enum dma_transaction_type type; 128 uint32_t status; 129 }; 130 131 struct jz4780_dma_chan { 132 struct virt_dma_chan vchan; 133 unsigned int id; 134 struct dma_pool *desc_pool; 135 136 uint32_t transfer_type; 137 uint32_t transfer_shift; 138 struct dma_slave_config config; 139 140 struct jz4780_dma_desc *desc; 141 unsigned int curr_hwdesc; 142 }; 143 144 struct jz4780_dma_soc_data { 145 unsigned int nb_channels; 146 unsigned int transfer_ord_max; 147 unsigned long flags; 148 }; 149 150 struct jz4780_dma_dev { 151 struct dma_device dma_device; 152 void __iomem *chn_base; 153 void __iomem *ctrl_base; 154 struct clk *clk; 155 unsigned int irq; 156 const struct jz4780_dma_soc_data *soc_data; 157 158 uint32_t chan_reserved; 159 struct jz4780_dma_chan chan[]; 160 }; 161 162 struct jz4780_dma_filter_data { 163 struct device_node *of_node; 164 uint32_t transfer_type; 165 int channel; 166 }; 167 168 static inline struct jz4780_dma_chan *to_jz4780_dma_chan(struct dma_chan *chan) 169 { 170 return container_of(chan, struct jz4780_dma_chan, vchan.chan); 171 } 172 173 static inline struct jz4780_dma_desc *to_jz4780_dma_desc( 174 struct virt_dma_desc *vdesc) 175 { 176 return container_of(vdesc, struct jz4780_dma_desc, vdesc); 177 } 178 179 static inline struct jz4780_dma_dev *jz4780_dma_chan_parent( 180 struct jz4780_dma_chan *jzchan) 181 { 182 return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev, 183 dma_device); 184 } 185 186 static inline uint32_t jz4780_dma_chn_readl(struct jz4780_dma_dev *jzdma, 187 unsigned int chn, unsigned int reg) 188 { 189 return readl(jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); 190 } 191 192 static inline void jz4780_dma_chn_writel(struct jz4780_dma_dev *jzdma, 193 unsigned int chn, unsigned int reg, uint32_t val) 194 { 195 writel(val, jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); 196 } 197 198 static inline uint32_t jz4780_dma_ctrl_readl(struct jz4780_dma_dev *jzdma, 199 unsigned int reg) 200 { 201 return readl(jzdma->ctrl_base + reg); 202 } 203 204 static inline void jz4780_dma_ctrl_writel(struct jz4780_dma_dev *jzdma, 205 unsigned int reg, uint32_t val) 206 { 207 writel(val, jzdma->ctrl_base + reg); 208 } 209 210 static inline void jz4780_dma_chan_enable(struct jz4780_dma_dev *jzdma, 211 unsigned int chn) 212 { 213 if (jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) { 214 unsigned int reg; 215 216 if (jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC) 217 reg = JZ_DMA_REG_DCKE; 218 else 219 reg = JZ_DMA_REG_DCKES; 220 221 jz4780_dma_ctrl_writel(jzdma, reg, BIT(chn)); 222 } 223 } 224 225 static inline void jz4780_dma_chan_disable(struct jz4780_dma_dev *jzdma, 226 unsigned int chn) 227 { 228 if ((jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) && 229 !(jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC)) 230 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DCKEC, BIT(chn)); 231 } 232 233 static struct jz4780_dma_desc *jz4780_dma_desc_alloc( 234 struct jz4780_dma_chan *jzchan, unsigned int count, 235 enum dma_transaction_type type) 236 { 237 struct jz4780_dma_desc *desc; 238 239 if (count > JZ_DMA_MAX_DESC) 240 return NULL; 241 242 desc = kzalloc(sizeof(*desc), GFP_NOWAIT); 243 if (!desc) 244 return NULL; 245 246 desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT, 247 &desc->desc_phys); 248 if (!desc->desc) { 249 kfree(desc); 250 return NULL; 251 } 252 253 desc->count = count; 254 desc->type = type; 255 return desc; 256 } 257 258 static void jz4780_dma_desc_free(struct virt_dma_desc *vdesc) 259 { 260 struct jz4780_dma_desc *desc = to_jz4780_dma_desc(vdesc); 261 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan); 262 263 dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys); 264 kfree(desc); 265 } 266 267 static uint32_t jz4780_dma_transfer_size(struct jz4780_dma_chan *jzchan, 268 unsigned long val, uint32_t *shift) 269 { 270 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 271 int ord = ffs(val) - 1; 272 273 /* 274 * 8 byte transfer sizes unsupported so fall back on 4. If it's larger 275 * than the maximum, just limit it. It is perfectly safe to fall back 276 * in this way since we won't exceed the maximum burst size supported 277 * by the device, the only effect is reduced efficiency. This is better 278 * than refusing to perform the request at all. 279 */ 280 if (ord == 3) 281 ord = 2; 282 else if (ord > jzdma->soc_data->transfer_ord_max) 283 ord = jzdma->soc_data->transfer_ord_max; 284 285 *shift = ord; 286 287 switch (ord) { 288 case 0: 289 return JZ_DMA_SIZE_1_BYTE; 290 case 1: 291 return JZ_DMA_SIZE_2_BYTE; 292 case 2: 293 return JZ_DMA_SIZE_4_BYTE; 294 case 4: 295 return JZ_DMA_SIZE_16_BYTE; 296 case 5: 297 return JZ_DMA_SIZE_32_BYTE; 298 case 6: 299 return JZ_DMA_SIZE_64_BYTE; 300 default: 301 return JZ_DMA_SIZE_128_BYTE; 302 } 303 } 304 305 static int jz4780_dma_setup_hwdesc(struct jz4780_dma_chan *jzchan, 306 struct jz4780_dma_hwdesc *desc, dma_addr_t addr, size_t len, 307 enum dma_transfer_direction direction) 308 { 309 struct dma_slave_config *config = &jzchan->config; 310 uint32_t width, maxburst, tsz; 311 312 if (direction == DMA_MEM_TO_DEV) { 313 desc->dcm = JZ_DMA_DCM_SAI; 314 desc->dsa = addr; 315 desc->dta = config->dst_addr; 316 317 width = config->dst_addr_width; 318 maxburst = config->dst_maxburst; 319 } else { 320 desc->dcm = JZ_DMA_DCM_DAI; 321 desc->dsa = config->src_addr; 322 desc->dta = addr; 323 324 width = config->src_addr_width; 325 maxburst = config->src_maxburst; 326 } 327 328 /* 329 * This calculates the maximum transfer size that can be used with the 330 * given address, length, width and maximum burst size. The address 331 * must be aligned to the transfer size, the total length must be 332 * divisible by the transfer size, and we must not use more than the 333 * maximum burst specified by the user. 334 */ 335 tsz = jz4780_dma_transfer_size(jzchan, addr | len | (width * maxburst), 336 &jzchan->transfer_shift); 337 338 switch (width) { 339 case DMA_SLAVE_BUSWIDTH_1_BYTE: 340 case DMA_SLAVE_BUSWIDTH_2_BYTES: 341 break; 342 case DMA_SLAVE_BUSWIDTH_4_BYTES: 343 width = JZ_DMA_WIDTH_32_BIT; 344 break; 345 default: 346 return -EINVAL; 347 } 348 349 desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT; 350 desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; 351 desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; 352 353 desc->dtc = len >> jzchan->transfer_shift; 354 return 0; 355 } 356 357 static struct dma_async_tx_descriptor *jz4780_dma_prep_slave_sg( 358 struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, 359 enum dma_transfer_direction direction, unsigned long flags, 360 void *context) 361 { 362 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 363 struct jz4780_dma_desc *desc; 364 unsigned int i; 365 int err; 366 367 desc = jz4780_dma_desc_alloc(jzchan, sg_len, DMA_SLAVE); 368 if (!desc) 369 return NULL; 370 371 for (i = 0; i < sg_len; i++) { 372 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], 373 sg_dma_address(&sgl[i]), 374 sg_dma_len(&sgl[i]), 375 direction); 376 if (err < 0) { 377 jz4780_dma_desc_free(&jzchan->desc->vdesc); 378 return NULL; 379 } 380 381 desc->desc[i].dcm |= JZ_DMA_DCM_TIE; 382 383 if (i != (sg_len - 1)) { 384 /* Automatically proceeed to the next descriptor. */ 385 desc->desc[i].dcm |= JZ_DMA_DCM_LINK; 386 387 /* 388 * The upper 8 bits of the DTC field in the descriptor 389 * must be set to (offset from descriptor base of next 390 * descriptor >> 4). 391 */ 392 desc->desc[i].dtc |= 393 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 394 } 395 } 396 397 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 398 } 399 400 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_cyclic( 401 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 402 size_t period_len, enum dma_transfer_direction direction, 403 unsigned long flags) 404 { 405 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 406 struct jz4780_dma_desc *desc; 407 unsigned int periods, i; 408 int err; 409 410 if (buf_len % period_len) 411 return NULL; 412 413 periods = buf_len / period_len; 414 415 desc = jz4780_dma_desc_alloc(jzchan, periods, DMA_CYCLIC); 416 if (!desc) 417 return NULL; 418 419 for (i = 0; i < periods; i++) { 420 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, 421 period_len, direction); 422 if (err < 0) { 423 jz4780_dma_desc_free(&jzchan->desc->vdesc); 424 return NULL; 425 } 426 427 buf_addr += period_len; 428 429 /* 430 * Set the link bit to indicate that the controller should 431 * automatically proceed to the next descriptor. In 432 * jz4780_dma_begin(), this will be cleared if we need to issue 433 * an interrupt after each period. 434 */ 435 desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK; 436 437 /* 438 * The upper 8 bits of the DTC field in the descriptor must be 439 * set to (offset from descriptor base of next descriptor >> 4). 440 * If this is the last descriptor, link it back to the first, 441 * i.e. leave offset set to 0, otherwise point to the next one. 442 */ 443 if (i != (periods - 1)) { 444 desc->desc[i].dtc |= 445 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; 446 } 447 } 448 449 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 450 } 451 452 static struct dma_async_tx_descriptor *jz4780_dma_prep_dma_memcpy( 453 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 454 size_t len, unsigned long flags) 455 { 456 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 457 struct jz4780_dma_desc *desc; 458 uint32_t tsz; 459 460 desc = jz4780_dma_desc_alloc(jzchan, 1, DMA_MEMCPY); 461 if (!desc) 462 return NULL; 463 464 tsz = jz4780_dma_transfer_size(jzchan, dest | src | len, 465 &jzchan->transfer_shift); 466 467 jzchan->transfer_type = JZ_DMA_DRT_AUTO; 468 469 desc->desc[0].dsa = src; 470 desc->desc[0].dta = dest; 471 desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI | 472 tsz << JZ_DMA_DCM_TSZ_SHIFT | 473 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_SP_SHIFT | 474 JZ_DMA_WIDTH_32_BIT << JZ_DMA_DCM_DP_SHIFT; 475 desc->desc[0].dtc = len >> jzchan->transfer_shift; 476 477 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); 478 } 479 480 static void jz4780_dma_begin(struct jz4780_dma_chan *jzchan) 481 { 482 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 483 struct virt_dma_desc *vdesc; 484 unsigned int i; 485 dma_addr_t desc_phys; 486 487 if (!jzchan->desc) { 488 vdesc = vchan_next_desc(&jzchan->vchan); 489 if (!vdesc) 490 return; 491 492 list_del(&vdesc->node); 493 494 jzchan->desc = to_jz4780_dma_desc(vdesc); 495 jzchan->curr_hwdesc = 0; 496 497 if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) { 498 /* 499 * The DMA controller doesn't support triggering an 500 * interrupt after processing each descriptor, only 501 * after processing an entire terminated list of 502 * descriptors. For a cyclic DMA setup the list of 503 * descriptors is not terminated so we can never get an 504 * interrupt. 505 * 506 * If the user requested a callback for a cyclic DMA 507 * setup then we workaround this hardware limitation 508 * here by degrading to a set of unlinked descriptors 509 * which we will submit in sequence in response to the 510 * completion of processing the previous descriptor. 511 */ 512 for (i = 0; i < jzchan->desc->count; i++) 513 jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK; 514 } 515 } else { 516 /* 517 * There is an existing transfer, therefore this must be one 518 * for which we unlinked the descriptors above. Advance to the 519 * next one in the list. 520 */ 521 jzchan->curr_hwdesc = 522 (jzchan->curr_hwdesc + 1) % jzchan->desc->count; 523 } 524 525 /* Enable the channel's clock. */ 526 jz4780_dma_chan_enable(jzdma, jzchan->id); 527 528 /* Use 4-word descriptors. */ 529 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 530 531 /* Set transfer type. */ 532 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DRT, 533 jzchan->transfer_type); 534 535 /* 536 * Set the transfer count. This is redundant for a descriptor-driven 537 * transfer. However, there can be a delay between the transfer start 538 * time and when DTCn reg contains the new transfer count. Setting 539 * it explicitly ensures residue is computed correctly at all times. 540 */ 541 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DTC, 542 jzchan->desc->desc[jzchan->curr_hwdesc].dtc); 543 544 /* Write descriptor address and initiate descriptor fetch. */ 545 desc_phys = jzchan->desc->desc_phys + 546 (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc)); 547 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DDA, desc_phys); 548 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id)); 549 550 /* Enable the channel. */ 551 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 552 JZ_DMA_DCS_CTE); 553 } 554 555 static void jz4780_dma_issue_pending(struct dma_chan *chan) 556 { 557 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 558 unsigned long flags; 559 560 spin_lock_irqsave(&jzchan->vchan.lock, flags); 561 562 if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc) 563 jz4780_dma_begin(jzchan); 564 565 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 566 } 567 568 static int jz4780_dma_terminate_all(struct dma_chan *chan) 569 { 570 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 571 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 572 unsigned long flags; 573 LIST_HEAD(head); 574 575 spin_lock_irqsave(&jzchan->vchan.lock, flags); 576 577 /* Clear the DMA status and stop the transfer. */ 578 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 579 if (jzchan->desc) { 580 vchan_terminate_vdesc(&jzchan->desc->vdesc); 581 jzchan->desc = NULL; 582 } 583 584 jz4780_dma_chan_disable(jzdma, jzchan->id); 585 586 vchan_get_all_descriptors(&jzchan->vchan, &head); 587 588 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 589 590 vchan_dma_desc_free_list(&jzchan->vchan, &head); 591 return 0; 592 } 593 594 static void jz4780_dma_synchronize(struct dma_chan *chan) 595 { 596 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 597 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 598 599 vchan_synchronize(&jzchan->vchan); 600 jz4780_dma_chan_disable(jzdma, jzchan->id); 601 } 602 603 static int jz4780_dma_config(struct dma_chan *chan, 604 struct dma_slave_config *config) 605 { 606 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 607 608 if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 609 || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) 610 return -EINVAL; 611 612 /* Copy the reset of the slave configuration, it is used later. */ 613 memcpy(&jzchan->config, config, sizeof(jzchan->config)); 614 615 return 0; 616 } 617 618 static size_t jz4780_dma_desc_residue(struct jz4780_dma_chan *jzchan, 619 struct jz4780_dma_desc *desc, unsigned int next_sg) 620 { 621 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 622 unsigned int count = 0; 623 unsigned int i; 624 625 for (i = next_sg; i < desc->count; i++) 626 count += desc->desc[i].dtc & GENMASK(23, 0); 627 628 if (next_sg != 0) 629 count += jz4780_dma_chn_readl(jzdma, jzchan->id, 630 JZ_DMA_REG_DTC); 631 632 return count << jzchan->transfer_shift; 633 } 634 635 static enum dma_status jz4780_dma_tx_status(struct dma_chan *chan, 636 dma_cookie_t cookie, struct dma_tx_state *txstate) 637 { 638 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 639 struct virt_dma_desc *vdesc; 640 enum dma_status status; 641 unsigned long flags; 642 unsigned long residue = 0; 643 644 status = dma_cookie_status(chan, cookie, txstate); 645 if ((status == DMA_COMPLETE) || (txstate == NULL)) 646 return status; 647 648 spin_lock_irqsave(&jzchan->vchan.lock, flags); 649 650 vdesc = vchan_find_desc(&jzchan->vchan, cookie); 651 if (vdesc) { 652 /* On the issued list, so hasn't been processed yet */ 653 residue = jz4780_dma_desc_residue(jzchan, 654 to_jz4780_dma_desc(vdesc), 0); 655 } else if (cookie == jzchan->desc->vdesc.tx.cookie) { 656 residue = jz4780_dma_desc_residue(jzchan, jzchan->desc, 657 jzchan->curr_hwdesc + 1); 658 } 659 dma_set_residue(txstate, residue); 660 661 if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc 662 && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) 663 status = DMA_ERROR; 664 665 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); 666 return status; 667 } 668 669 static void jz4780_dma_chan_irq(struct jz4780_dma_dev *jzdma, 670 struct jz4780_dma_chan *jzchan) 671 { 672 uint32_t dcs; 673 674 spin_lock(&jzchan->vchan.lock); 675 676 dcs = jz4780_dma_chn_readl(jzdma, jzchan->id, JZ_DMA_REG_DCS); 677 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); 678 679 if (dcs & JZ_DMA_DCS_AR) { 680 dev_warn(&jzchan->vchan.chan.dev->device, 681 "address error (DCS=0x%x)\n", dcs); 682 } 683 684 if (dcs & JZ_DMA_DCS_HLT) { 685 dev_warn(&jzchan->vchan.chan.dev->device, 686 "channel halt (DCS=0x%x)\n", dcs); 687 } 688 689 if (jzchan->desc) { 690 jzchan->desc->status = dcs; 691 692 if ((dcs & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) == 0) { 693 if (jzchan->desc->type == DMA_CYCLIC) { 694 vchan_cyclic_callback(&jzchan->desc->vdesc); 695 } else { 696 vchan_cookie_complete(&jzchan->desc->vdesc); 697 jzchan->desc = NULL; 698 } 699 700 jz4780_dma_begin(jzchan); 701 } 702 } else { 703 dev_err(&jzchan->vchan.chan.dev->device, 704 "channel IRQ with no active transfer\n"); 705 } 706 707 spin_unlock(&jzchan->vchan.lock); 708 } 709 710 static irqreturn_t jz4780_dma_irq_handler(int irq, void *data) 711 { 712 struct jz4780_dma_dev *jzdma = data; 713 uint32_t pending, dmac; 714 int i; 715 716 pending = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DIRQP); 717 718 for (i = 0; i < jzdma->soc_data->nb_channels; i++) { 719 if (!(pending & (1<<i))) 720 continue; 721 722 jz4780_dma_chan_irq(jzdma, &jzdma->chan[i]); 723 } 724 725 /* Clear halt and address error status of all channels. */ 726 dmac = jz4780_dma_ctrl_readl(jzdma, JZ_DMA_REG_DMAC); 727 dmac &= ~(JZ_DMA_DMAC_HLT | JZ_DMA_DMAC_AR); 728 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, dmac); 729 730 /* Clear interrupt pending status. */ 731 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DIRQP, 0); 732 733 return IRQ_HANDLED; 734 } 735 736 static int jz4780_dma_alloc_chan_resources(struct dma_chan *chan) 737 { 738 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 739 740 jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device), 741 chan->device->dev, 742 JZ_DMA_DESC_BLOCK_SIZE, 743 PAGE_SIZE, 0); 744 if (!jzchan->desc_pool) { 745 dev_err(&chan->dev->device, 746 "failed to allocate descriptor pool\n"); 747 return -ENOMEM; 748 } 749 750 return 0; 751 } 752 753 static void jz4780_dma_free_chan_resources(struct dma_chan *chan) 754 { 755 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 756 757 vchan_free_chan_resources(&jzchan->vchan); 758 dma_pool_destroy(jzchan->desc_pool); 759 jzchan->desc_pool = NULL; 760 } 761 762 static bool jz4780_dma_filter_fn(struct dma_chan *chan, void *param) 763 { 764 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(chan); 765 struct jz4780_dma_dev *jzdma = jz4780_dma_chan_parent(jzchan); 766 struct jz4780_dma_filter_data *data = param; 767 768 if (jzdma->dma_device.dev->of_node != data->of_node) 769 return false; 770 771 if (data->channel > -1) { 772 if (data->channel != jzchan->id) 773 return false; 774 } else if (jzdma->chan_reserved & BIT(jzchan->id)) { 775 return false; 776 } 777 778 jzchan->transfer_type = data->transfer_type; 779 780 return true; 781 } 782 783 static struct dma_chan *jz4780_of_dma_xlate(struct of_phandle_args *dma_spec, 784 struct of_dma *ofdma) 785 { 786 struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; 787 dma_cap_mask_t mask = jzdma->dma_device.cap_mask; 788 struct jz4780_dma_filter_data data; 789 790 if (dma_spec->args_count != 2) 791 return NULL; 792 793 data.of_node = ofdma->of_node; 794 data.transfer_type = dma_spec->args[0]; 795 data.channel = dma_spec->args[1]; 796 797 if (data.channel > -1) { 798 if (data.channel >= jzdma->soc_data->nb_channels) { 799 dev_err(jzdma->dma_device.dev, 800 "device requested non-existent channel %u\n", 801 data.channel); 802 return NULL; 803 } 804 805 /* Can only select a channel marked as reserved. */ 806 if (!(jzdma->chan_reserved & BIT(data.channel))) { 807 dev_err(jzdma->dma_device.dev, 808 "device requested unreserved channel %u\n", 809 data.channel); 810 return NULL; 811 } 812 813 jzdma->chan[data.channel].transfer_type = data.transfer_type; 814 815 return dma_get_slave_channel( 816 &jzdma->chan[data.channel].vchan.chan); 817 } else { 818 return dma_request_channel(mask, jz4780_dma_filter_fn, &data); 819 } 820 } 821 822 static int jz4780_dma_probe(struct platform_device *pdev) 823 { 824 struct device *dev = &pdev->dev; 825 const struct jz4780_dma_soc_data *soc_data; 826 struct jz4780_dma_dev *jzdma; 827 struct jz4780_dma_chan *jzchan; 828 struct dma_device *dd; 829 struct resource *res; 830 int i, ret; 831 832 if (!dev->of_node) { 833 dev_err(dev, "This driver must be probed from devicetree\n"); 834 return -EINVAL; 835 } 836 837 soc_data = device_get_match_data(dev); 838 if (!soc_data) 839 return -EINVAL; 840 841 jzdma = devm_kzalloc(dev, sizeof(*jzdma) 842 + sizeof(*jzdma->chan) * soc_data->nb_channels, 843 GFP_KERNEL); 844 if (!jzdma) 845 return -ENOMEM; 846 847 jzdma->soc_data = soc_data; 848 platform_set_drvdata(pdev, jzdma); 849 850 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 851 if (!res) { 852 dev_err(dev, "failed to get I/O memory\n"); 853 return -EINVAL; 854 } 855 856 jzdma->chn_base = devm_ioremap_resource(dev, res); 857 if (IS_ERR(jzdma->chn_base)) 858 return PTR_ERR(jzdma->chn_base); 859 860 res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 861 if (res) { 862 jzdma->ctrl_base = devm_ioremap_resource(dev, res); 863 if (IS_ERR(jzdma->ctrl_base)) 864 return PTR_ERR(jzdma->ctrl_base); 865 } else if (soc_data->flags & JZ_SOC_DATA_ALLOW_LEGACY_DT) { 866 /* 867 * On JZ4780, if the second memory resource was not supplied, 868 * assume we're using an old devicetree, and calculate the 869 * offset to the control registers. 870 */ 871 jzdma->ctrl_base = jzdma->chn_base + JZ4780_DMA_CTRL_OFFSET; 872 } else { 873 dev_err(dev, "failed to get I/O memory\n"); 874 return -EINVAL; 875 } 876 877 ret = platform_get_irq(pdev, 0); 878 if (ret < 0) { 879 dev_err(dev, "failed to get IRQ: %d\n", ret); 880 return ret; 881 } 882 883 jzdma->irq = ret; 884 885 ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev), 886 jzdma); 887 if (ret) { 888 dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); 889 return ret; 890 } 891 892 jzdma->clk = devm_clk_get(dev, NULL); 893 if (IS_ERR(jzdma->clk)) { 894 dev_err(dev, "failed to get clock\n"); 895 ret = PTR_ERR(jzdma->clk); 896 goto err_free_irq; 897 } 898 899 clk_prepare_enable(jzdma->clk); 900 901 /* Property is optional, if it doesn't exist the value will remain 0. */ 902 of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels", 903 0, &jzdma->chan_reserved); 904 905 dd = &jzdma->dma_device; 906 907 dma_cap_set(DMA_MEMCPY, dd->cap_mask); 908 dma_cap_set(DMA_SLAVE, dd->cap_mask); 909 dma_cap_set(DMA_CYCLIC, dd->cap_mask); 910 911 dd->dev = dev; 912 dd->copy_align = DMAENGINE_ALIGN_4_BYTES; 913 dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; 914 dd->device_free_chan_resources = jz4780_dma_free_chan_resources; 915 dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; 916 dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; 917 dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; 918 dd->device_config = jz4780_dma_config; 919 dd->device_terminate_all = jz4780_dma_terminate_all; 920 dd->device_synchronize = jz4780_dma_synchronize; 921 dd->device_tx_status = jz4780_dma_tx_status; 922 dd->device_issue_pending = jz4780_dma_issue_pending; 923 dd->src_addr_widths = JZ_DMA_BUSWIDTHS; 924 dd->dst_addr_widths = JZ_DMA_BUSWIDTHS; 925 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 926 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; 927 928 /* 929 * Enable DMA controller, mark all channels as not programmable. 930 * Also set the FMSC bit - it increases MSC performance, so it makes 931 * little sense not to enable it. 932 */ 933 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMAC, JZ_DMA_DMAC_DMAE | 934 JZ_DMA_DMAC_FAIC | JZ_DMA_DMAC_FMSC); 935 936 if (soc_data->flags & JZ_SOC_DATA_PROGRAMMABLE_DMA) 937 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DMACP, 0); 938 939 INIT_LIST_HEAD(&dd->channels); 940 941 for (i = 0; i < soc_data->nb_channels; i++) { 942 jzchan = &jzdma->chan[i]; 943 jzchan->id = i; 944 945 vchan_init(&jzchan->vchan, dd); 946 jzchan->vchan.desc_free = jz4780_dma_desc_free; 947 } 948 949 ret = dmaenginem_async_device_register(dd); 950 if (ret) { 951 dev_err(dev, "failed to register device\n"); 952 goto err_disable_clk; 953 } 954 955 /* Register with OF DMA helpers. */ 956 ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate, 957 jzdma); 958 if (ret) { 959 dev_err(dev, "failed to register OF DMA controller\n"); 960 goto err_disable_clk; 961 } 962 963 dev_info(dev, "JZ4780 DMA controller initialised\n"); 964 return 0; 965 966 err_disable_clk: 967 clk_disable_unprepare(jzdma->clk); 968 969 err_free_irq: 970 free_irq(jzdma->irq, jzdma); 971 return ret; 972 } 973 974 static int jz4780_dma_remove(struct platform_device *pdev) 975 { 976 struct jz4780_dma_dev *jzdma = platform_get_drvdata(pdev); 977 int i; 978 979 of_dma_controller_free(pdev->dev.of_node); 980 981 free_irq(jzdma->irq, jzdma); 982 983 for (i = 0; i < jzdma->soc_data->nb_channels; i++) 984 tasklet_kill(&jzdma->chan[i].vchan.task); 985 986 return 0; 987 } 988 989 static const struct jz4780_dma_soc_data jz4740_dma_soc_data = { 990 .nb_channels = 6, 991 .transfer_ord_max = 5, 992 }; 993 994 static const struct jz4780_dma_soc_data jz4725b_dma_soc_data = { 995 .nb_channels = 6, 996 .transfer_ord_max = 5, 997 .flags = JZ_SOC_DATA_PER_CHAN_PM | JZ_SOC_DATA_NO_DCKES_DCKEC, 998 }; 999 1000 static const struct jz4780_dma_soc_data jz4770_dma_soc_data = { 1001 .nb_channels = 6, 1002 .transfer_ord_max = 6, 1003 .flags = JZ_SOC_DATA_PER_CHAN_PM, 1004 }; 1005 1006 static const struct jz4780_dma_soc_data jz4780_dma_soc_data = { 1007 .nb_channels = 32, 1008 .transfer_ord_max = 7, 1009 .flags = JZ_SOC_DATA_ALLOW_LEGACY_DT | JZ_SOC_DATA_PROGRAMMABLE_DMA, 1010 }; 1011 1012 static const struct of_device_id jz4780_dma_dt_match[] = { 1013 { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data }, 1014 { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data }, 1015 { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data }, 1016 { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data }, 1017 {}, 1018 }; 1019 MODULE_DEVICE_TABLE(of, jz4780_dma_dt_match); 1020 1021 static struct platform_driver jz4780_dma_driver = { 1022 .probe = jz4780_dma_probe, 1023 .remove = jz4780_dma_remove, 1024 .driver = { 1025 .name = "jz4780-dma", 1026 .of_match_table = of_match_ptr(jz4780_dma_dt_match), 1027 }, 1028 }; 1029 1030 static int __init jz4780_dma_init(void) 1031 { 1032 return platform_driver_register(&jz4780_dma_driver); 1033 } 1034 subsys_initcall(jz4780_dma_init); 1035 1036 static void __exit jz4780_dma_exit(void) 1037 { 1038 platform_driver_unregister(&jz4780_dma_driver); 1039 } 1040 module_exit(jz4780_dma_exit); 1041 1042 MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>"); 1043 MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver"); 1044 MODULE_LICENSE("GPL"); 1045