1 /* 2 * Copyright (C) 2017 Spreadtrum Communications Inc. 3 * 4 * SPDX-License-Identifier: GPL-2.0 5 */ 6 7 #include <linux/clk.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/dma/sprd-dma.h> 10 #include <linux/errno.h> 11 #include <linux/init.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/of.h> 17 #include <linux/of_dma.h> 18 #include <linux/platform_device.h> 19 #include <linux/pm_runtime.h> 20 #include <linux/slab.h> 21 22 #include "virt-dma.h" 23 24 #define SPRD_DMA_CHN_REG_OFFSET 0x1000 25 #define SPRD_DMA_CHN_REG_LENGTH 0x40 26 #define SPRD_DMA_MEMCPY_MIN_SIZE 64 27 28 /* DMA global registers definition */ 29 #define SPRD_DMA_GLB_PAUSE 0x0 30 #define SPRD_DMA_GLB_FRAG_WAIT 0x4 31 #define SPRD_DMA_GLB_REQ_PEND0_EN 0x8 32 #define SPRD_DMA_GLB_REQ_PEND1_EN 0xc 33 #define SPRD_DMA_GLB_INT_RAW_STS 0x10 34 #define SPRD_DMA_GLB_INT_MSK_STS 0x14 35 #define SPRD_DMA_GLB_REQ_STS 0x18 36 #define SPRD_DMA_GLB_CHN_EN_STS 0x1c 37 #define SPRD_DMA_GLB_DEBUG_STS 0x20 38 #define SPRD_DMA_GLB_ARB_SEL_STS 0x24 39 #define SPRD_DMA_GLB_2STAGE_GRP1 0x28 40 #define SPRD_DMA_GLB_2STAGE_GRP2 0x2c 41 #define SPRD_DMA_GLB_REQ_UID(uid) (0x4 * ((uid) - 1)) 42 #define SPRD_DMA_GLB_REQ_UID_OFFSET 0x2000 43 44 /* DMA channel registers definition */ 45 #define SPRD_DMA_CHN_PAUSE 0x0 46 #define SPRD_DMA_CHN_REQ 0x4 47 #define SPRD_DMA_CHN_CFG 0x8 48 #define SPRD_DMA_CHN_INTC 0xc 49 #define SPRD_DMA_CHN_SRC_ADDR 0x10 50 #define SPRD_DMA_CHN_DES_ADDR 0x14 51 #define SPRD_DMA_CHN_FRG_LEN 0x18 52 #define SPRD_DMA_CHN_BLK_LEN 0x1c 53 #define SPRD_DMA_CHN_TRSC_LEN 0x20 54 #define SPRD_DMA_CHN_TRSF_STEP 0x24 55 #define SPRD_DMA_CHN_WARP_PTR 0x28 56 #define SPRD_DMA_CHN_WARP_TO 0x2c 57 #define SPRD_DMA_CHN_LLIST_PTR 0x30 58 #define SPRD_DMA_CHN_FRAG_STEP 0x34 59 #define SPRD_DMA_CHN_SRC_BLK_STEP 0x38 60 #define SPRD_DMA_CHN_DES_BLK_STEP 0x3c 61 62 /* SPRD_DMA_GLB_2STAGE_GRP register definition */ 63 #define SPRD_DMA_GLB_2STAGE_EN BIT(24) 64 #define SPRD_DMA_GLB_CHN_INT_MASK GENMASK(23, 20) 65 #define SPRD_DMA_GLB_DEST_INT BIT(22) 66 #define SPRD_DMA_GLB_SRC_INT BIT(20) 67 #define SPRD_DMA_GLB_LIST_DONE_TRG BIT(19) 68 #define SPRD_DMA_GLB_TRANS_DONE_TRG BIT(18) 69 #define SPRD_DMA_GLB_BLOCK_DONE_TRG BIT(17) 70 #define SPRD_DMA_GLB_FRAG_DONE_TRG BIT(16) 71 #define SPRD_DMA_GLB_TRG_OFFSET 16 72 #define SPRD_DMA_GLB_DEST_CHN_MASK GENMASK(13, 8) 73 #define SPRD_DMA_GLB_DEST_CHN_OFFSET 8 74 #define SPRD_DMA_GLB_SRC_CHN_MASK GENMASK(5, 0) 75 76 /* SPRD_DMA_CHN_INTC register definition */ 77 #define SPRD_DMA_INT_MASK GENMASK(4, 0) 78 #define SPRD_DMA_INT_CLR_OFFSET 24 79 #define SPRD_DMA_FRAG_INT_EN BIT(0) 80 #define SPRD_DMA_BLK_INT_EN BIT(1) 81 #define SPRD_DMA_TRANS_INT_EN BIT(2) 82 #define SPRD_DMA_LIST_INT_EN BIT(3) 83 #define SPRD_DMA_CFG_ERR_INT_EN BIT(4) 84 85 /* SPRD_DMA_CHN_CFG register definition */ 86 #define SPRD_DMA_CHN_EN BIT(0) 87 #define SPRD_DMA_LINKLIST_EN BIT(4) 88 #define SPRD_DMA_WAIT_BDONE_OFFSET 24 89 #define SPRD_DMA_DONOT_WAIT_BDONE 1 90 91 /* SPRD_DMA_CHN_REQ register definition */ 92 #define SPRD_DMA_REQ_EN BIT(0) 93 94 /* SPRD_DMA_CHN_PAUSE register definition */ 95 #define SPRD_DMA_PAUSE_EN BIT(0) 96 #define SPRD_DMA_PAUSE_STS BIT(2) 97 #define SPRD_DMA_PAUSE_CNT 0x2000 98 99 /* DMA_CHN_WARP_* register definition */ 100 #define SPRD_DMA_HIGH_ADDR_MASK GENMASK(31, 28) 101 #define SPRD_DMA_LOW_ADDR_MASK GENMASK(31, 0) 102 #define SPRD_DMA_WRAP_ADDR_MASK GENMASK(27, 0) 103 #define SPRD_DMA_HIGH_ADDR_OFFSET 4 104 105 /* SPRD_DMA_CHN_INTC register definition */ 106 #define SPRD_DMA_FRAG_INT_STS BIT(16) 107 #define SPRD_DMA_BLK_INT_STS BIT(17) 108 #define SPRD_DMA_TRSC_INT_STS BIT(18) 109 #define SPRD_DMA_LIST_INT_STS BIT(19) 110 #define SPRD_DMA_CFGERR_INT_STS BIT(20) 111 #define SPRD_DMA_CHN_INT_STS \ 112 (SPRD_DMA_FRAG_INT_STS | SPRD_DMA_BLK_INT_STS | \ 113 SPRD_DMA_TRSC_INT_STS | SPRD_DMA_LIST_INT_STS | \ 114 SPRD_DMA_CFGERR_INT_STS) 115 116 /* SPRD_DMA_CHN_FRG_LEN register definition */ 117 #define SPRD_DMA_SRC_DATAWIDTH_OFFSET 30 118 #define SPRD_DMA_DES_DATAWIDTH_OFFSET 28 119 #define SPRD_DMA_SWT_MODE_OFFSET 26 120 #define SPRD_DMA_REQ_MODE_OFFSET 24 121 #define SPRD_DMA_REQ_MODE_MASK GENMASK(1, 0) 122 #define SPRD_DMA_WRAP_SEL_DEST BIT(23) 123 #define SPRD_DMA_WRAP_EN BIT(22) 124 #define SPRD_DMA_FIX_SEL_OFFSET 21 125 #define SPRD_DMA_FIX_EN_OFFSET 20 126 #define SPRD_DMA_LLIST_END BIT(19) 127 #define SPRD_DMA_FRG_LEN_MASK GENMASK(16, 0) 128 129 /* SPRD_DMA_CHN_BLK_LEN register definition */ 130 #define SPRD_DMA_BLK_LEN_MASK GENMASK(16, 0) 131 132 /* SPRD_DMA_CHN_TRSC_LEN register definition */ 133 #define SPRD_DMA_TRSC_LEN_MASK GENMASK(27, 0) 134 135 /* SPRD_DMA_CHN_TRSF_STEP register definition */ 136 #define SPRD_DMA_DEST_TRSF_STEP_OFFSET 16 137 #define SPRD_DMA_SRC_TRSF_STEP_OFFSET 0 138 #define SPRD_DMA_TRSF_STEP_MASK GENMASK(15, 0) 139 140 /* SPRD DMA_SRC_BLK_STEP register definition */ 141 #define SPRD_DMA_LLIST_HIGH_MASK GENMASK(31, 28) 142 #define SPRD_DMA_LLIST_HIGH_SHIFT 28 143 144 /* define DMA channel mode & trigger mode mask */ 145 #define SPRD_DMA_CHN_MODE_MASK GENMASK(7, 0) 146 #define SPRD_DMA_TRG_MODE_MASK GENMASK(7, 0) 147 #define SPRD_DMA_INT_TYPE_MASK GENMASK(7, 0) 148 149 /* define the DMA transfer step type */ 150 #define SPRD_DMA_NONE_STEP 0 151 #define SPRD_DMA_BYTE_STEP 1 152 #define SPRD_DMA_SHORT_STEP 2 153 #define SPRD_DMA_WORD_STEP 4 154 #define SPRD_DMA_DWORD_STEP 8 155 156 #define SPRD_DMA_SOFTWARE_UID 0 157 158 /* dma data width values */ 159 enum sprd_dma_datawidth { 160 SPRD_DMA_DATAWIDTH_1_BYTE, 161 SPRD_DMA_DATAWIDTH_2_BYTES, 162 SPRD_DMA_DATAWIDTH_4_BYTES, 163 SPRD_DMA_DATAWIDTH_8_BYTES, 164 }; 165 166 /* dma channel hardware configuration */ 167 struct sprd_dma_chn_hw { 168 u32 pause; 169 u32 req; 170 u32 cfg; 171 u32 intc; 172 u32 src_addr; 173 u32 des_addr; 174 u32 frg_len; 175 u32 blk_len; 176 u32 trsc_len; 177 u32 trsf_step; 178 u32 wrap_ptr; 179 u32 wrap_to; 180 u32 llist_ptr; 181 u32 frg_step; 182 u32 src_blk_step; 183 u32 des_blk_step; 184 }; 185 186 /* dma request description */ 187 struct sprd_dma_desc { 188 struct virt_dma_desc vd; 189 struct sprd_dma_chn_hw chn_hw; 190 enum dma_transfer_direction dir; 191 }; 192 193 /* dma channel description */ 194 struct sprd_dma_chn { 195 struct virt_dma_chan vc; 196 void __iomem *chn_base; 197 struct sprd_dma_linklist linklist; 198 struct dma_slave_config slave_cfg; 199 u32 chn_num; 200 u32 dev_id; 201 enum sprd_dma_chn_mode chn_mode; 202 enum sprd_dma_trg_mode trg_mode; 203 enum sprd_dma_int_type int_type; 204 struct sprd_dma_desc *cur_desc; 205 }; 206 207 /* SPRD dma device */ 208 struct sprd_dma_dev { 209 struct dma_device dma_dev; 210 void __iomem *glb_base; 211 struct clk_bulk_data clks[2]; 212 int irq; 213 u32 total_chns; 214 struct sprd_dma_chn channels[] __counted_by(total_chns); 215 }; 216 217 static void sprd_dma_free_desc(struct virt_dma_desc *vd); 218 static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param); 219 static struct of_dma_filter_info sprd_dma_info = { 220 .filter_fn = sprd_dma_filter_fn, 221 }; 222 223 static inline struct sprd_dma_chn *to_sprd_dma_chan(struct dma_chan *c) 224 { 225 return container_of(c, struct sprd_dma_chn, vc.chan); 226 } 227 228 static inline struct sprd_dma_dev *to_sprd_dma_dev(struct dma_chan *c) 229 { 230 struct sprd_dma_chn *schan = to_sprd_dma_chan(c); 231 232 return container_of(schan, struct sprd_dma_dev, channels[c->chan_id]); 233 } 234 235 static inline struct sprd_dma_desc *to_sprd_dma_desc(struct virt_dma_desc *vd) 236 { 237 return container_of(vd, struct sprd_dma_desc, vd); 238 } 239 240 static void sprd_dma_glb_update(struct sprd_dma_dev *sdev, u32 reg, 241 u32 mask, u32 val) 242 { 243 u32 orig = readl(sdev->glb_base + reg); 244 u32 tmp; 245 246 tmp = (orig & ~mask) | val; 247 writel(tmp, sdev->glb_base + reg); 248 } 249 250 static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg, 251 u32 mask, u32 val) 252 { 253 u32 orig = readl(schan->chn_base + reg); 254 u32 tmp; 255 256 tmp = (orig & ~mask) | val; 257 writel(tmp, schan->chn_base + reg); 258 } 259 260 static int sprd_dma_enable(struct sprd_dma_dev *sdev) 261 { 262 return clk_bulk_prepare_enable(ARRAY_SIZE(sdev->clks), sdev->clks); 263 } 264 265 static void sprd_dma_disable(struct sprd_dma_dev *sdev) 266 { 267 clk_bulk_disable_unprepare(ARRAY_SIZE(sdev->clks), sdev->clks); 268 } 269 270 static void sprd_dma_set_uid(struct sprd_dma_chn *schan) 271 { 272 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 273 u32 dev_id = schan->dev_id; 274 275 if (dev_id != SPRD_DMA_SOFTWARE_UID) { 276 u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET + 277 SPRD_DMA_GLB_REQ_UID(dev_id); 278 279 writel(schan->chn_num + 1, sdev->glb_base + uid_offset); 280 } 281 } 282 283 static void sprd_dma_unset_uid(struct sprd_dma_chn *schan) 284 { 285 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 286 u32 dev_id = schan->dev_id; 287 288 if (dev_id != SPRD_DMA_SOFTWARE_UID) { 289 u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET + 290 SPRD_DMA_GLB_REQ_UID(dev_id); 291 292 writel(0, sdev->glb_base + uid_offset); 293 } 294 } 295 296 static void sprd_dma_clear_int(struct sprd_dma_chn *schan) 297 { 298 sprd_dma_chn_update(schan, SPRD_DMA_CHN_INTC, 299 SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET, 300 SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET); 301 } 302 303 static void sprd_dma_enable_chn(struct sprd_dma_chn *schan) 304 { 305 sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 306 SPRD_DMA_CHN_EN); 307 } 308 309 static void sprd_dma_disable_chn(struct sprd_dma_chn *schan) 310 { 311 sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 0); 312 } 313 314 static void sprd_dma_soft_request(struct sprd_dma_chn *schan) 315 { 316 sprd_dma_chn_update(schan, SPRD_DMA_CHN_REQ, SPRD_DMA_REQ_EN, 317 SPRD_DMA_REQ_EN); 318 } 319 320 static void sprd_dma_pause_resume(struct sprd_dma_chn *schan, bool enable) 321 { 322 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 323 u32 pause, timeout = SPRD_DMA_PAUSE_CNT; 324 325 if (enable) { 326 sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE, 327 SPRD_DMA_PAUSE_EN, SPRD_DMA_PAUSE_EN); 328 329 do { 330 pause = readl(schan->chn_base + SPRD_DMA_CHN_PAUSE); 331 if (pause & SPRD_DMA_PAUSE_STS) 332 break; 333 334 cpu_relax(); 335 } while (--timeout > 0); 336 337 if (!timeout) 338 dev_warn(sdev->dma_dev.dev, 339 "pause dma controller timeout\n"); 340 } else { 341 sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE, 342 SPRD_DMA_PAUSE_EN, 0); 343 } 344 } 345 346 static void sprd_dma_stop_and_disable(struct sprd_dma_chn *schan) 347 { 348 u32 cfg = readl(schan->chn_base + SPRD_DMA_CHN_CFG); 349 350 if (!(cfg & SPRD_DMA_CHN_EN)) 351 return; 352 353 sprd_dma_pause_resume(schan, true); 354 sprd_dma_disable_chn(schan); 355 } 356 357 static unsigned long sprd_dma_get_src_addr(struct sprd_dma_chn *schan) 358 { 359 unsigned long addr, addr_high; 360 361 addr = readl(schan->chn_base + SPRD_DMA_CHN_SRC_ADDR); 362 addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_PTR) & 363 SPRD_DMA_HIGH_ADDR_MASK; 364 365 return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET); 366 } 367 368 static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *schan) 369 { 370 unsigned long addr, addr_high; 371 372 addr = readl(schan->chn_base + SPRD_DMA_CHN_DES_ADDR); 373 addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_TO) & 374 SPRD_DMA_HIGH_ADDR_MASK; 375 376 return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET); 377 } 378 379 static enum sprd_dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *schan) 380 { 381 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 382 u32 intc_sts = readl(schan->chn_base + SPRD_DMA_CHN_INTC) & 383 SPRD_DMA_CHN_INT_STS; 384 385 switch (intc_sts) { 386 case SPRD_DMA_CFGERR_INT_STS: 387 return SPRD_DMA_CFGERR_INT; 388 389 case SPRD_DMA_LIST_INT_STS: 390 return SPRD_DMA_LIST_INT; 391 392 case SPRD_DMA_TRSC_INT_STS: 393 return SPRD_DMA_TRANS_INT; 394 395 case SPRD_DMA_BLK_INT_STS: 396 return SPRD_DMA_BLK_INT; 397 398 case SPRD_DMA_FRAG_INT_STS: 399 return SPRD_DMA_FRAG_INT; 400 401 default: 402 dev_warn(sdev->dma_dev.dev, "incorrect dma interrupt type\n"); 403 return SPRD_DMA_NO_INT; 404 } 405 } 406 407 static enum sprd_dma_req_mode sprd_dma_get_req_type(struct sprd_dma_chn *schan) 408 { 409 u32 frag_reg = readl(schan->chn_base + SPRD_DMA_CHN_FRG_LEN); 410 411 return (frag_reg >> SPRD_DMA_REQ_MODE_OFFSET) & SPRD_DMA_REQ_MODE_MASK; 412 } 413 414 static int sprd_dma_set_2stage_config(struct sprd_dma_chn *schan) 415 { 416 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 417 u32 val, chn = schan->chn_num + 1; 418 419 switch (schan->chn_mode) { 420 case SPRD_DMA_SRC_CHN0: 421 val = chn & SPRD_DMA_GLB_SRC_CHN_MASK; 422 val |= BIT(schan->trg_mode - 1) << SPRD_DMA_GLB_TRG_OFFSET; 423 val |= SPRD_DMA_GLB_2STAGE_EN; 424 if (schan->int_type != SPRD_DMA_NO_INT) 425 val |= SPRD_DMA_GLB_SRC_INT; 426 427 sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP1, val, val); 428 break; 429 430 case SPRD_DMA_SRC_CHN1: 431 val = chn & SPRD_DMA_GLB_SRC_CHN_MASK; 432 val |= BIT(schan->trg_mode - 1) << SPRD_DMA_GLB_TRG_OFFSET; 433 val |= SPRD_DMA_GLB_2STAGE_EN; 434 if (schan->int_type != SPRD_DMA_NO_INT) 435 val |= SPRD_DMA_GLB_SRC_INT; 436 437 sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP2, val, val); 438 break; 439 440 case SPRD_DMA_DST_CHN0: 441 val = (chn << SPRD_DMA_GLB_DEST_CHN_OFFSET) & 442 SPRD_DMA_GLB_DEST_CHN_MASK; 443 val |= SPRD_DMA_GLB_2STAGE_EN; 444 if (schan->int_type != SPRD_DMA_NO_INT) 445 val |= SPRD_DMA_GLB_DEST_INT; 446 447 sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP1, val, val); 448 break; 449 450 case SPRD_DMA_DST_CHN1: 451 val = (chn << SPRD_DMA_GLB_DEST_CHN_OFFSET) & 452 SPRD_DMA_GLB_DEST_CHN_MASK; 453 val |= SPRD_DMA_GLB_2STAGE_EN; 454 if (schan->int_type != SPRD_DMA_NO_INT) 455 val |= SPRD_DMA_GLB_DEST_INT; 456 457 sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP2, val, val); 458 break; 459 460 default: 461 dev_err(sdev->dma_dev.dev, "invalid channel mode setting %d\n", 462 schan->chn_mode); 463 return -EINVAL; 464 } 465 466 return 0; 467 } 468 469 static void sprd_dma_set_pending(struct sprd_dma_chn *schan, bool enable) 470 { 471 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 472 u32 reg, val, req_id; 473 474 if (schan->dev_id == SPRD_DMA_SOFTWARE_UID) 475 return; 476 477 /* The DMA request id always starts from 0. */ 478 req_id = schan->dev_id - 1; 479 480 if (req_id < 32) { 481 reg = SPRD_DMA_GLB_REQ_PEND0_EN; 482 val = BIT(req_id); 483 } else { 484 reg = SPRD_DMA_GLB_REQ_PEND1_EN; 485 val = BIT(req_id - 32); 486 } 487 488 sprd_dma_glb_update(sdev, reg, val, enable ? val : 0); 489 } 490 491 static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan, 492 struct sprd_dma_desc *sdesc) 493 { 494 struct sprd_dma_chn_hw *cfg = &sdesc->chn_hw; 495 496 writel(cfg->pause, schan->chn_base + SPRD_DMA_CHN_PAUSE); 497 writel(cfg->cfg, schan->chn_base + SPRD_DMA_CHN_CFG); 498 writel(cfg->intc, schan->chn_base + SPRD_DMA_CHN_INTC); 499 writel(cfg->src_addr, schan->chn_base + SPRD_DMA_CHN_SRC_ADDR); 500 writel(cfg->des_addr, schan->chn_base + SPRD_DMA_CHN_DES_ADDR); 501 writel(cfg->frg_len, schan->chn_base + SPRD_DMA_CHN_FRG_LEN); 502 writel(cfg->blk_len, schan->chn_base + SPRD_DMA_CHN_BLK_LEN); 503 writel(cfg->trsc_len, schan->chn_base + SPRD_DMA_CHN_TRSC_LEN); 504 writel(cfg->trsf_step, schan->chn_base + SPRD_DMA_CHN_TRSF_STEP); 505 writel(cfg->wrap_ptr, schan->chn_base + SPRD_DMA_CHN_WARP_PTR); 506 writel(cfg->wrap_to, schan->chn_base + SPRD_DMA_CHN_WARP_TO); 507 writel(cfg->llist_ptr, schan->chn_base + SPRD_DMA_CHN_LLIST_PTR); 508 writel(cfg->frg_step, schan->chn_base + SPRD_DMA_CHN_FRAG_STEP); 509 writel(cfg->src_blk_step, schan->chn_base + SPRD_DMA_CHN_SRC_BLK_STEP); 510 writel(cfg->des_blk_step, schan->chn_base + SPRD_DMA_CHN_DES_BLK_STEP); 511 writel(cfg->req, schan->chn_base + SPRD_DMA_CHN_REQ); 512 } 513 514 static void sprd_dma_start(struct sprd_dma_chn *schan) 515 { 516 struct virt_dma_desc *vd = vchan_next_desc(&schan->vc); 517 518 if (!vd) 519 return; 520 521 list_del(&vd->node); 522 schan->cur_desc = to_sprd_dma_desc(vd); 523 524 /* 525 * Set 2-stage configuration if the channel starts one 2-stage 526 * transfer. 527 */ 528 if (schan->chn_mode && sprd_dma_set_2stage_config(schan)) 529 return; 530 531 /* 532 * Copy the DMA configuration from DMA descriptor to this hardware 533 * channel. 534 */ 535 sprd_dma_set_chn_config(schan, schan->cur_desc); 536 sprd_dma_set_uid(schan); 537 sprd_dma_set_pending(schan, true); 538 sprd_dma_enable_chn(schan); 539 540 if (schan->dev_id == SPRD_DMA_SOFTWARE_UID && 541 schan->chn_mode != SPRD_DMA_DST_CHN0 && 542 schan->chn_mode != SPRD_DMA_DST_CHN1) 543 sprd_dma_soft_request(schan); 544 } 545 546 static void sprd_dma_stop(struct sprd_dma_chn *schan) 547 { 548 sprd_dma_stop_and_disable(schan); 549 sprd_dma_set_pending(schan, false); 550 sprd_dma_unset_uid(schan); 551 sprd_dma_clear_int(schan); 552 schan->cur_desc = NULL; 553 } 554 555 static bool sprd_dma_check_trans_done(enum sprd_dma_int_type int_type, 556 enum sprd_dma_req_mode req_mode) 557 { 558 if (int_type == SPRD_DMA_NO_INT) 559 return false; 560 561 if (int_type >= req_mode + 1) 562 return true; 563 else 564 return false; 565 } 566 567 static irqreturn_t dma_irq_handle(int irq, void *dev_id) 568 { 569 struct sprd_dma_dev *sdev = (struct sprd_dma_dev *)dev_id; 570 u32 irq_status = readl(sdev->glb_base + SPRD_DMA_GLB_INT_MSK_STS); 571 struct sprd_dma_chn *schan; 572 struct sprd_dma_desc *sdesc; 573 enum sprd_dma_req_mode req_type; 574 enum sprd_dma_int_type int_type; 575 bool trans_done = false, cyclic = false; 576 u32 i; 577 578 while (irq_status) { 579 i = __ffs(irq_status); 580 irq_status &= (irq_status - 1); 581 schan = &sdev->channels[i]; 582 583 spin_lock(&schan->vc.lock); 584 585 sdesc = schan->cur_desc; 586 if (!sdesc) { 587 spin_unlock(&schan->vc.lock); 588 return IRQ_HANDLED; 589 } 590 591 int_type = sprd_dma_get_int_type(schan); 592 req_type = sprd_dma_get_req_type(schan); 593 sprd_dma_clear_int(schan); 594 595 /* cyclic mode schedule callback */ 596 cyclic = schan->linklist.phy_addr ? true : false; 597 if (cyclic == true) { 598 vchan_cyclic_callback(&sdesc->vd); 599 } else { 600 /* Check if the dma request descriptor is done. */ 601 trans_done = sprd_dma_check_trans_done(int_type, req_type); 602 if (trans_done == true) { 603 vchan_cookie_complete(&sdesc->vd); 604 schan->cur_desc = NULL; 605 sprd_dma_start(schan); 606 } 607 } 608 spin_unlock(&schan->vc.lock); 609 } 610 611 return IRQ_HANDLED; 612 } 613 614 static int sprd_dma_alloc_chan_resources(struct dma_chan *chan) 615 { 616 return pm_runtime_get_sync(chan->device->dev); 617 } 618 619 static void sprd_dma_free_chan_resources(struct dma_chan *chan) 620 { 621 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 622 struct virt_dma_desc *cur_vd = NULL; 623 unsigned long flags; 624 625 spin_lock_irqsave(&schan->vc.lock, flags); 626 if (schan->cur_desc) 627 cur_vd = &schan->cur_desc->vd; 628 629 sprd_dma_stop(schan); 630 spin_unlock_irqrestore(&schan->vc.lock, flags); 631 632 if (cur_vd) 633 sprd_dma_free_desc(cur_vd); 634 635 vchan_free_chan_resources(&schan->vc); 636 pm_runtime_put(chan->device->dev); 637 } 638 639 static enum dma_status sprd_dma_tx_status(struct dma_chan *chan, 640 dma_cookie_t cookie, 641 struct dma_tx_state *txstate) 642 { 643 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 644 struct virt_dma_desc *vd; 645 unsigned long flags; 646 enum dma_status ret; 647 u32 pos; 648 649 ret = dma_cookie_status(chan, cookie, txstate); 650 if (ret == DMA_COMPLETE || !txstate) 651 return ret; 652 653 spin_lock_irqsave(&schan->vc.lock, flags); 654 vd = vchan_find_desc(&schan->vc, cookie); 655 if (vd) { 656 struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd); 657 struct sprd_dma_chn_hw *hw = &sdesc->chn_hw; 658 659 if (hw->trsc_len > 0) 660 pos = hw->trsc_len; 661 else if (hw->blk_len > 0) 662 pos = hw->blk_len; 663 else if (hw->frg_len > 0) 664 pos = hw->frg_len; 665 else 666 pos = 0; 667 } else if (schan->cur_desc && schan->cur_desc->vd.tx.cookie == cookie) { 668 struct sprd_dma_desc *sdesc = schan->cur_desc; 669 670 if (sdesc->dir == DMA_DEV_TO_MEM) 671 pos = sprd_dma_get_dst_addr(schan); 672 else 673 pos = sprd_dma_get_src_addr(schan); 674 } else { 675 pos = 0; 676 } 677 spin_unlock_irqrestore(&schan->vc.lock, flags); 678 679 dma_set_residue(txstate, pos); 680 return ret; 681 } 682 683 static void sprd_dma_issue_pending(struct dma_chan *chan) 684 { 685 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 686 unsigned long flags; 687 688 spin_lock_irqsave(&schan->vc.lock, flags); 689 if (vchan_issue_pending(&schan->vc) && !schan->cur_desc) 690 sprd_dma_start(schan); 691 spin_unlock_irqrestore(&schan->vc.lock, flags); 692 } 693 694 static int sprd_dma_get_datawidth(enum dma_slave_buswidth buswidth) 695 { 696 switch (buswidth) { 697 case DMA_SLAVE_BUSWIDTH_1_BYTE: 698 case DMA_SLAVE_BUSWIDTH_2_BYTES: 699 case DMA_SLAVE_BUSWIDTH_4_BYTES: 700 case DMA_SLAVE_BUSWIDTH_8_BYTES: 701 return ffs(buswidth) - 1; 702 703 default: 704 return -EINVAL; 705 } 706 } 707 708 static int sprd_dma_get_step(enum dma_slave_buswidth buswidth) 709 { 710 switch (buswidth) { 711 case DMA_SLAVE_BUSWIDTH_1_BYTE: 712 case DMA_SLAVE_BUSWIDTH_2_BYTES: 713 case DMA_SLAVE_BUSWIDTH_4_BYTES: 714 case DMA_SLAVE_BUSWIDTH_8_BYTES: 715 return buswidth; 716 717 default: 718 return -EINVAL; 719 } 720 } 721 722 static int sprd_dma_fill_desc(struct dma_chan *chan, 723 struct sprd_dma_chn_hw *hw, 724 unsigned int sglen, int sg_index, 725 dma_addr_t src, dma_addr_t dst, u32 len, 726 enum dma_transfer_direction dir, 727 unsigned long flags, 728 struct dma_slave_config *slave_cfg) 729 { 730 struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan); 731 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 732 enum sprd_dma_chn_mode chn_mode = schan->chn_mode; 733 u32 req_mode = (flags >> SPRD_DMA_REQ_SHIFT) & SPRD_DMA_REQ_MODE_MASK; 734 u32 int_mode = flags & SPRD_DMA_INT_MASK; 735 int src_datawidth, dst_datawidth, src_step, dst_step; 736 u32 temp, fix_mode = 0, fix_en = 0; 737 phys_addr_t llist_ptr; 738 739 if (dir == DMA_MEM_TO_DEV) { 740 src_step = sprd_dma_get_step(slave_cfg->src_addr_width); 741 if (src_step < 0) { 742 dev_err(sdev->dma_dev.dev, "invalid source step\n"); 743 return src_step; 744 } 745 746 /* 747 * For 2-stage transfer, destination channel step can not be 0, 748 * since destination device is AON IRAM. 749 */ 750 if (chn_mode == SPRD_DMA_DST_CHN0 || 751 chn_mode == SPRD_DMA_DST_CHN1) 752 dst_step = src_step; 753 else 754 dst_step = SPRD_DMA_NONE_STEP; 755 } else { 756 dst_step = sprd_dma_get_step(slave_cfg->dst_addr_width); 757 if (dst_step < 0) { 758 dev_err(sdev->dma_dev.dev, "invalid destination step\n"); 759 return dst_step; 760 } 761 src_step = SPRD_DMA_NONE_STEP; 762 } 763 764 src_datawidth = sprd_dma_get_datawidth(slave_cfg->src_addr_width); 765 if (src_datawidth < 0) { 766 dev_err(sdev->dma_dev.dev, "invalid source datawidth\n"); 767 return src_datawidth; 768 } 769 770 dst_datawidth = sprd_dma_get_datawidth(slave_cfg->dst_addr_width); 771 if (dst_datawidth < 0) { 772 dev_err(sdev->dma_dev.dev, "invalid destination datawidth\n"); 773 return dst_datawidth; 774 } 775 776 hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET; 777 778 /* 779 * wrap_ptr and wrap_to will save the high 4 bits source address and 780 * destination address. 781 */ 782 hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK; 783 hw->wrap_to = (dst >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK; 784 hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK; 785 hw->des_addr = dst & SPRD_DMA_LOW_ADDR_MASK; 786 787 /* 788 * If the src step and dst step both are 0 or both are not 0, that means 789 * we can not enable the fix mode. If one is 0 and another one is not, 790 * we can enable the fix mode. 791 */ 792 if ((src_step != 0 && dst_step != 0) || (src_step | dst_step) == 0) { 793 fix_en = 0; 794 } else { 795 fix_en = 1; 796 if (src_step) 797 fix_mode = 1; 798 else 799 fix_mode = 0; 800 } 801 802 hw->intc = int_mode | SPRD_DMA_CFG_ERR_INT_EN; 803 804 temp = src_datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET; 805 temp |= dst_datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET; 806 temp |= req_mode << SPRD_DMA_REQ_MODE_OFFSET; 807 temp |= fix_mode << SPRD_DMA_FIX_SEL_OFFSET; 808 temp |= fix_en << SPRD_DMA_FIX_EN_OFFSET; 809 temp |= schan->linklist.wrap_addr ? 810 SPRD_DMA_WRAP_EN | SPRD_DMA_WRAP_SEL_DEST : 0; 811 temp |= slave_cfg->src_maxburst & SPRD_DMA_FRG_LEN_MASK; 812 hw->frg_len = temp; 813 814 hw->blk_len = slave_cfg->src_maxburst & SPRD_DMA_BLK_LEN_MASK; 815 hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK; 816 817 temp = (dst_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET; 818 temp |= (src_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET; 819 hw->trsf_step = temp; 820 821 /* link-list configuration */ 822 if (schan->linklist.phy_addr) { 823 hw->cfg |= SPRD_DMA_LINKLIST_EN; 824 825 /* link-list index */ 826 temp = sglen ? (sg_index + 1) % sglen : 0; 827 828 /* Next link-list configuration's physical address offset */ 829 temp = temp * sizeof(*hw) + SPRD_DMA_CHN_SRC_ADDR; 830 /* 831 * Set the link-list pointer point to next link-list 832 * configuration's physical address. 833 */ 834 llist_ptr = schan->linklist.phy_addr + temp; 835 hw->llist_ptr = lower_32_bits(llist_ptr); 836 hw->src_blk_step = (upper_32_bits(llist_ptr) << SPRD_DMA_LLIST_HIGH_SHIFT) & 837 SPRD_DMA_LLIST_HIGH_MASK; 838 839 if (schan->linklist.wrap_addr) { 840 hw->wrap_ptr |= schan->linklist.wrap_addr & 841 SPRD_DMA_WRAP_ADDR_MASK; 842 hw->wrap_to |= dst & SPRD_DMA_WRAP_ADDR_MASK; 843 } 844 } else { 845 hw->llist_ptr = 0; 846 hw->src_blk_step = 0; 847 } 848 849 hw->frg_step = 0; 850 hw->des_blk_step = 0; 851 return 0; 852 } 853 854 static int sprd_dma_fill_linklist_desc(struct dma_chan *chan, 855 unsigned int sglen, int sg_index, 856 dma_addr_t src, dma_addr_t dst, u32 len, 857 enum dma_transfer_direction dir, 858 unsigned long flags, 859 struct dma_slave_config *slave_cfg) 860 { 861 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 862 struct sprd_dma_chn_hw *hw; 863 864 if (!schan->linklist.virt_addr) 865 return -EINVAL; 866 867 hw = (struct sprd_dma_chn_hw *)(schan->linklist.virt_addr + 868 sg_index * sizeof(*hw)); 869 870 return sprd_dma_fill_desc(chan, hw, sglen, sg_index, src, dst, len, 871 dir, flags, slave_cfg); 872 } 873 874 static struct dma_async_tx_descriptor * 875 sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 876 size_t len, unsigned long flags) 877 { 878 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 879 struct sprd_dma_desc *sdesc; 880 struct sprd_dma_chn_hw *hw; 881 enum sprd_dma_datawidth datawidth; 882 u32 step, temp; 883 884 sdesc = kzalloc_obj(*sdesc, GFP_NOWAIT); 885 if (!sdesc) 886 return NULL; 887 888 hw = &sdesc->chn_hw; 889 890 hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET; 891 hw->intc = SPRD_DMA_TRANS_INT | SPRD_DMA_CFG_ERR_INT_EN; 892 hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK; 893 hw->des_addr = dest & SPRD_DMA_LOW_ADDR_MASK; 894 hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) & 895 SPRD_DMA_HIGH_ADDR_MASK; 896 hw->wrap_to = (dest >> SPRD_DMA_HIGH_ADDR_OFFSET) & 897 SPRD_DMA_HIGH_ADDR_MASK; 898 899 if (IS_ALIGNED(len, 8)) { 900 datawidth = SPRD_DMA_DATAWIDTH_8_BYTES; 901 step = SPRD_DMA_DWORD_STEP; 902 } else if (IS_ALIGNED(len, 4)) { 903 datawidth = SPRD_DMA_DATAWIDTH_4_BYTES; 904 step = SPRD_DMA_WORD_STEP; 905 } else if (IS_ALIGNED(len, 2)) { 906 datawidth = SPRD_DMA_DATAWIDTH_2_BYTES; 907 step = SPRD_DMA_SHORT_STEP; 908 } else { 909 datawidth = SPRD_DMA_DATAWIDTH_1_BYTE; 910 step = SPRD_DMA_BYTE_STEP; 911 } 912 913 temp = datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET; 914 temp |= datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET; 915 temp |= SPRD_DMA_TRANS_REQ << SPRD_DMA_REQ_MODE_OFFSET; 916 temp |= len & SPRD_DMA_FRG_LEN_MASK; 917 hw->frg_len = temp; 918 919 hw->blk_len = len & SPRD_DMA_BLK_LEN_MASK; 920 hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK; 921 922 temp = (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET; 923 temp |= (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET; 924 hw->trsf_step = temp; 925 926 return vchan_tx_prep(&schan->vc, &sdesc->vd, flags); 927 } 928 929 static struct dma_async_tx_descriptor * 930 sprd_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 931 unsigned int sglen, enum dma_transfer_direction dir, 932 unsigned long flags, void *context) 933 { 934 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 935 struct dma_slave_config *slave_cfg = &schan->slave_cfg; 936 dma_addr_t src = 0, dst = 0; 937 dma_addr_t start_src = 0, start_dst = 0; 938 struct sprd_dma_desc *sdesc; 939 struct scatterlist *sg; 940 u32 len = 0; 941 int ret, i; 942 943 if (!is_slave_direction(dir)) 944 return NULL; 945 946 if (context) { 947 struct sprd_dma_linklist *ll_cfg = 948 (struct sprd_dma_linklist *)context; 949 950 schan->linklist.phy_addr = ll_cfg->phy_addr; 951 schan->linklist.virt_addr = ll_cfg->virt_addr; 952 schan->linklist.wrap_addr = ll_cfg->wrap_addr; 953 } else { 954 schan->linklist.phy_addr = 0; 955 schan->linklist.virt_addr = 0; 956 schan->linklist.wrap_addr = 0; 957 } 958 959 /* 960 * Set channel mode, interrupt mode and trigger mode for 2-stage 961 * transfer. 962 */ 963 schan->chn_mode = 964 (flags >> SPRD_DMA_CHN_MODE_SHIFT) & SPRD_DMA_CHN_MODE_MASK; 965 schan->trg_mode = 966 (flags >> SPRD_DMA_TRG_MODE_SHIFT) & SPRD_DMA_TRG_MODE_MASK; 967 schan->int_type = flags & SPRD_DMA_INT_TYPE_MASK; 968 969 sdesc = kzalloc_obj(*sdesc, GFP_NOWAIT); 970 if (!sdesc) 971 return NULL; 972 973 sdesc->dir = dir; 974 975 for_each_sg(sgl, sg, sglen, i) { 976 len = sg_dma_len(sg); 977 978 if (dir == DMA_MEM_TO_DEV) { 979 src = sg_dma_address(sg); 980 dst = slave_cfg->dst_addr; 981 } else { 982 src = slave_cfg->src_addr; 983 dst = sg_dma_address(sg); 984 } 985 986 if (!i) { 987 start_src = src; 988 start_dst = dst; 989 } 990 991 /* 992 * The link-list mode needs at least 2 link-list 993 * configurations. If there is only one sg, it doesn't 994 * need to fill the link-list configuration. 995 */ 996 if (sglen < 2) 997 break; 998 999 ret = sprd_dma_fill_linklist_desc(chan, sglen, i, src, dst, len, 1000 dir, flags, slave_cfg); 1001 if (ret) { 1002 kfree(sdesc); 1003 return NULL; 1004 } 1005 } 1006 1007 ret = sprd_dma_fill_desc(chan, &sdesc->chn_hw, 0, 0, start_src, 1008 start_dst, len, dir, flags, slave_cfg); 1009 if (ret) { 1010 kfree(sdesc); 1011 return NULL; 1012 } 1013 1014 return vchan_tx_prep(&schan->vc, &sdesc->vd, flags); 1015 } 1016 1017 static int sprd_dma_slave_config(struct dma_chan *chan, 1018 struct dma_slave_config *config) 1019 { 1020 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 1021 struct dma_slave_config *slave_cfg = &schan->slave_cfg; 1022 1023 memcpy(slave_cfg, config, sizeof(*config)); 1024 return 0; 1025 } 1026 1027 static int sprd_dma_pause(struct dma_chan *chan) 1028 { 1029 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 1030 unsigned long flags; 1031 1032 spin_lock_irqsave(&schan->vc.lock, flags); 1033 sprd_dma_pause_resume(schan, true); 1034 spin_unlock_irqrestore(&schan->vc.lock, flags); 1035 1036 return 0; 1037 } 1038 1039 static int sprd_dma_resume(struct dma_chan *chan) 1040 { 1041 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 1042 unsigned long flags; 1043 1044 spin_lock_irqsave(&schan->vc.lock, flags); 1045 sprd_dma_pause_resume(schan, false); 1046 spin_unlock_irqrestore(&schan->vc.lock, flags); 1047 1048 return 0; 1049 } 1050 1051 static int sprd_dma_terminate_all(struct dma_chan *chan) 1052 { 1053 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 1054 struct virt_dma_desc *cur_vd = NULL; 1055 unsigned long flags; 1056 LIST_HEAD(head); 1057 1058 spin_lock_irqsave(&schan->vc.lock, flags); 1059 if (schan->cur_desc) 1060 cur_vd = &schan->cur_desc->vd; 1061 1062 sprd_dma_stop(schan); 1063 1064 vchan_get_all_descriptors(&schan->vc, &head); 1065 spin_unlock_irqrestore(&schan->vc.lock, flags); 1066 1067 if (cur_vd) 1068 sprd_dma_free_desc(cur_vd); 1069 1070 vchan_dma_desc_free_list(&schan->vc, &head); 1071 return 0; 1072 } 1073 1074 static void sprd_dma_free_desc(struct virt_dma_desc *vd) 1075 { 1076 struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd); 1077 1078 kfree(sdesc); 1079 } 1080 1081 static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param) 1082 { 1083 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 1084 u32 slave_id = *(u32 *)param; 1085 1086 schan->dev_id = slave_id; 1087 return true; 1088 } 1089 1090 static int sprd_dma_probe(struct platform_device *pdev) 1091 { 1092 struct device_node *np = pdev->dev.of_node; 1093 struct sprd_dma_dev *sdev; 1094 struct sprd_dma_chn *dma_chn; 1095 u32 chn_count; 1096 int ret, i; 1097 1098 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36)); 1099 if (ret) { 1100 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1101 if (ret) { 1102 dev_err(&pdev->dev, "unable to set coherent mask to 32\n"); 1103 return ret; 1104 } 1105 } 1106 1107 /* Parse new and deprecated dma-channels properties */ 1108 ret = device_property_read_u32(&pdev->dev, "dma-channels", &chn_count); 1109 if (ret) 1110 ret = device_property_read_u32(&pdev->dev, "#dma-channels", 1111 &chn_count); 1112 if (ret) { 1113 dev_err(&pdev->dev, "get dma channels count failed\n"); 1114 return ret; 1115 } 1116 1117 sdev = devm_kzalloc(&pdev->dev, 1118 struct_size(sdev, channels, chn_count), 1119 GFP_KERNEL); 1120 if (!sdev) 1121 return -ENOMEM; 1122 1123 /* The ashb_eb clock is optional and only present for AGCP DMA */ 1124 sdev->clks[0].id = "enable"; 1125 sdev->clks[1].id = "ashb_eb"; 1126 ret = devm_clk_bulk_get_optional(&pdev->dev, ARRAY_SIZE(sdev->clks), 1127 sdev->clks); 1128 if (ret) 1129 return dev_err_probe(&pdev->dev, ret, "get clocks failed\n"); 1130 1131 if (!sdev->clks[0].clk) 1132 return dev_err_probe(&pdev->dev, -ENOENT, 1133 "get enable clock failed\n"); 1134 1135 /* 1136 * We have three DMA controllers: AP DMA, AON DMA and AGCP DMA. For AGCP 1137 * DMA controller, it can or do not request the irq, which will save 1138 * system power without resuming system by DMA interrupts if AGCP DMA 1139 * does not request the irq. Thus the DMA interrupts property should 1140 * be optional. 1141 */ 1142 sdev->irq = platform_get_irq(pdev, 0); 1143 if (sdev->irq > 0) { 1144 ret = devm_request_irq(&pdev->dev, sdev->irq, dma_irq_handle, 1145 0, "sprd_dma", (void *)sdev); 1146 if (ret < 0) { 1147 dev_err(&pdev->dev, "request dma irq failed\n"); 1148 return ret; 1149 } 1150 } else { 1151 dev_warn(&pdev->dev, "no interrupts for the dma controller\n"); 1152 } 1153 1154 sdev->glb_base = devm_platform_ioremap_resource(pdev, 0); 1155 if (IS_ERR(sdev->glb_base)) 1156 return PTR_ERR(sdev->glb_base); 1157 1158 dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask); 1159 sdev->total_chns = chn_count; 1160 INIT_LIST_HEAD(&sdev->dma_dev.channels); 1161 INIT_LIST_HEAD(&sdev->dma_dev.global_node); 1162 sdev->dma_dev.dev = &pdev->dev; 1163 sdev->dma_dev.device_alloc_chan_resources = sprd_dma_alloc_chan_resources; 1164 sdev->dma_dev.device_free_chan_resources = sprd_dma_free_chan_resources; 1165 sdev->dma_dev.device_tx_status = sprd_dma_tx_status; 1166 sdev->dma_dev.device_issue_pending = sprd_dma_issue_pending; 1167 sdev->dma_dev.device_prep_dma_memcpy = sprd_dma_prep_dma_memcpy; 1168 sdev->dma_dev.device_prep_slave_sg = sprd_dma_prep_slave_sg; 1169 sdev->dma_dev.device_config = sprd_dma_slave_config; 1170 sdev->dma_dev.device_pause = sprd_dma_pause; 1171 sdev->dma_dev.device_resume = sprd_dma_resume; 1172 sdev->dma_dev.device_terminate_all = sprd_dma_terminate_all; 1173 1174 for (i = 0; i < chn_count; i++) { 1175 dma_chn = &sdev->channels[i]; 1176 dma_chn->chn_num = i; 1177 dma_chn->cur_desc = NULL; 1178 /* get each channel's registers base address. */ 1179 dma_chn->chn_base = sdev->glb_base + SPRD_DMA_CHN_REG_OFFSET + 1180 SPRD_DMA_CHN_REG_LENGTH * i; 1181 1182 dma_chn->vc.desc_free = sprd_dma_free_desc; 1183 vchan_init(&dma_chn->vc, &sdev->dma_dev); 1184 } 1185 1186 platform_set_drvdata(pdev, sdev); 1187 ret = sprd_dma_enable(sdev); 1188 if (ret) 1189 return ret; 1190 1191 pm_runtime_set_active(&pdev->dev); 1192 pm_runtime_enable(&pdev->dev); 1193 1194 ret = pm_runtime_get_sync(&pdev->dev); 1195 if (ret < 0) 1196 goto err_register; 1197 1198 ret = dma_async_device_register(&sdev->dma_dev); 1199 if (ret < 0) { 1200 dev_err(&pdev->dev, "register dma device failed:%d\n", ret); 1201 goto err_register; 1202 } 1203 1204 sprd_dma_info.dma_cap = sdev->dma_dev.cap_mask; 1205 ret = of_dma_controller_register(np, of_dma_simple_xlate, 1206 &sprd_dma_info); 1207 if (ret) 1208 goto err_of_register; 1209 1210 pm_runtime_put(&pdev->dev); 1211 return 0; 1212 1213 err_of_register: 1214 dma_async_device_unregister(&sdev->dma_dev); 1215 err_register: 1216 pm_runtime_put_noidle(&pdev->dev); 1217 pm_runtime_disable(&pdev->dev); 1218 sprd_dma_disable(sdev); 1219 return ret; 1220 } 1221 1222 static void sprd_dma_remove(struct platform_device *pdev) 1223 { 1224 struct sprd_dma_dev *sdev = platform_get_drvdata(pdev); 1225 struct sprd_dma_chn *c, *cn; 1226 1227 pm_runtime_get_sync(&pdev->dev); 1228 1229 /* explicitly free the irq */ 1230 if (sdev->irq > 0) 1231 devm_free_irq(&pdev->dev, sdev->irq, sdev); 1232 1233 list_for_each_entry_safe(c, cn, &sdev->dma_dev.channels, 1234 vc.chan.device_node) { 1235 list_del(&c->vc.chan.device_node); 1236 tasklet_kill(&c->vc.task); 1237 } 1238 1239 of_dma_controller_free(pdev->dev.of_node); 1240 dma_async_device_unregister(&sdev->dma_dev); 1241 sprd_dma_disable(sdev); 1242 1243 pm_runtime_put_noidle(&pdev->dev); 1244 pm_runtime_disable(&pdev->dev); 1245 } 1246 1247 static const struct of_device_id sprd_dma_match[] = { 1248 { .compatible = "sprd,sc9860-dma", }, 1249 {}, 1250 }; 1251 MODULE_DEVICE_TABLE(of, sprd_dma_match); 1252 1253 static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev) 1254 { 1255 struct sprd_dma_dev *sdev = dev_get_drvdata(dev); 1256 1257 sprd_dma_disable(sdev); 1258 return 0; 1259 } 1260 1261 static int __maybe_unused sprd_dma_runtime_resume(struct device *dev) 1262 { 1263 struct sprd_dma_dev *sdev = dev_get_drvdata(dev); 1264 int ret; 1265 1266 ret = sprd_dma_enable(sdev); 1267 if (ret) 1268 dev_err(sdev->dma_dev.dev, "enable dma failed\n"); 1269 1270 return ret; 1271 } 1272 1273 static const struct dev_pm_ops sprd_dma_pm_ops = { 1274 SET_RUNTIME_PM_OPS(sprd_dma_runtime_suspend, 1275 sprd_dma_runtime_resume, 1276 NULL) 1277 }; 1278 1279 static struct platform_driver sprd_dma_driver = { 1280 .probe = sprd_dma_probe, 1281 .remove = sprd_dma_remove, 1282 .driver = { 1283 .name = "sprd-dma", 1284 .of_match_table = sprd_dma_match, 1285 .pm = &sprd_dma_pm_ops, 1286 }, 1287 }; 1288 module_platform_driver(sprd_dma_driver); 1289 1290 MODULE_LICENSE("GPL v2"); 1291 MODULE_DESCRIPTION("DMA driver for Spreadtrum"); 1292 MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>"); 1293 MODULE_AUTHOR("Eric Long <eric.long@spreadtrum.com>"); 1294