1 /* 2 * Copyright (C) 2017 Spreadtrum Communications Inc. 3 * 4 * SPDX-License-Identifier: GPL-2.0 5 */ 6 7 #include <linux/clk.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/dma/sprd-dma.h> 10 #include <linux/errno.h> 11 #include <linux/init.h> 12 #include <linux/interrupt.h> 13 #include <linux/io.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/of.h> 17 #include <linux/of_dma.h> 18 #include <linux/of_device.h> 19 #include <linux/pm_runtime.h> 20 #include <linux/slab.h> 21 22 #include "virt-dma.h" 23 24 #define SPRD_DMA_CHN_REG_OFFSET 0x1000 25 #define SPRD_DMA_CHN_REG_LENGTH 0x40 26 #define SPRD_DMA_MEMCPY_MIN_SIZE 64 27 28 /* DMA global registers definition */ 29 #define SPRD_DMA_GLB_PAUSE 0x0 30 #define SPRD_DMA_GLB_FRAG_WAIT 0x4 31 #define SPRD_DMA_GLB_REQ_PEND0_EN 0x8 32 #define SPRD_DMA_GLB_REQ_PEND1_EN 0xc 33 #define SPRD_DMA_GLB_INT_RAW_STS 0x10 34 #define SPRD_DMA_GLB_INT_MSK_STS 0x14 35 #define SPRD_DMA_GLB_REQ_STS 0x18 36 #define SPRD_DMA_GLB_CHN_EN_STS 0x1c 37 #define SPRD_DMA_GLB_DEBUG_STS 0x20 38 #define SPRD_DMA_GLB_ARB_SEL_STS 0x24 39 #define SPRD_DMA_GLB_2STAGE_GRP1 0x28 40 #define SPRD_DMA_GLB_2STAGE_GRP2 0x2c 41 #define SPRD_DMA_GLB_REQ_UID(uid) (0x4 * ((uid) - 1)) 42 #define SPRD_DMA_GLB_REQ_UID_OFFSET 0x2000 43 44 /* DMA channel registers definition */ 45 #define SPRD_DMA_CHN_PAUSE 0x0 46 #define SPRD_DMA_CHN_REQ 0x4 47 #define SPRD_DMA_CHN_CFG 0x8 48 #define SPRD_DMA_CHN_INTC 0xc 49 #define SPRD_DMA_CHN_SRC_ADDR 0x10 50 #define SPRD_DMA_CHN_DES_ADDR 0x14 51 #define SPRD_DMA_CHN_FRG_LEN 0x18 52 #define SPRD_DMA_CHN_BLK_LEN 0x1c 53 #define SPRD_DMA_CHN_TRSC_LEN 0x20 54 #define SPRD_DMA_CHN_TRSF_STEP 0x24 55 #define SPRD_DMA_CHN_WARP_PTR 0x28 56 #define SPRD_DMA_CHN_WARP_TO 0x2c 57 #define SPRD_DMA_CHN_LLIST_PTR 0x30 58 #define SPRD_DMA_CHN_FRAG_STEP 0x34 59 #define SPRD_DMA_CHN_SRC_BLK_STEP 0x38 60 #define SPRD_DMA_CHN_DES_BLK_STEP 0x3c 61 62 /* SPRD_DMA_GLB_2STAGE_GRP register definition */ 63 #define SPRD_DMA_GLB_2STAGE_EN BIT(24) 64 #define SPRD_DMA_GLB_CHN_INT_MASK GENMASK(23, 20) 65 #define SPRD_DMA_GLB_DEST_INT BIT(22) 66 #define SPRD_DMA_GLB_SRC_INT BIT(20) 67 #define SPRD_DMA_GLB_LIST_DONE_TRG BIT(19) 68 #define SPRD_DMA_GLB_TRANS_DONE_TRG BIT(18) 69 #define SPRD_DMA_GLB_BLOCK_DONE_TRG BIT(17) 70 #define SPRD_DMA_GLB_FRAG_DONE_TRG BIT(16) 71 #define SPRD_DMA_GLB_TRG_OFFSET 16 72 #define SPRD_DMA_GLB_DEST_CHN_MASK GENMASK(13, 8) 73 #define SPRD_DMA_GLB_DEST_CHN_OFFSET 8 74 #define SPRD_DMA_GLB_SRC_CHN_MASK GENMASK(5, 0) 75 76 /* SPRD_DMA_CHN_INTC register definition */ 77 #define SPRD_DMA_INT_MASK GENMASK(4, 0) 78 #define SPRD_DMA_INT_CLR_OFFSET 24 79 #define SPRD_DMA_FRAG_INT_EN BIT(0) 80 #define SPRD_DMA_BLK_INT_EN BIT(1) 81 #define SPRD_DMA_TRANS_INT_EN BIT(2) 82 #define SPRD_DMA_LIST_INT_EN BIT(3) 83 #define SPRD_DMA_CFG_ERR_INT_EN BIT(4) 84 85 /* SPRD_DMA_CHN_CFG register definition */ 86 #define SPRD_DMA_CHN_EN BIT(0) 87 #define SPRD_DMA_LINKLIST_EN BIT(4) 88 #define SPRD_DMA_WAIT_BDONE_OFFSET 24 89 #define SPRD_DMA_DONOT_WAIT_BDONE 1 90 91 /* SPRD_DMA_CHN_REQ register definition */ 92 #define SPRD_DMA_REQ_EN BIT(0) 93 94 /* SPRD_DMA_CHN_PAUSE register definition */ 95 #define SPRD_DMA_PAUSE_EN BIT(0) 96 #define SPRD_DMA_PAUSE_STS BIT(2) 97 #define SPRD_DMA_PAUSE_CNT 0x2000 98 99 /* DMA_CHN_WARP_* register definition */ 100 #define SPRD_DMA_HIGH_ADDR_MASK GENMASK(31, 28) 101 #define SPRD_DMA_LOW_ADDR_MASK GENMASK(31, 0) 102 #define SPRD_DMA_HIGH_ADDR_OFFSET 4 103 104 /* SPRD_DMA_CHN_INTC register definition */ 105 #define SPRD_DMA_FRAG_INT_STS BIT(16) 106 #define SPRD_DMA_BLK_INT_STS BIT(17) 107 #define SPRD_DMA_TRSC_INT_STS BIT(18) 108 #define SPRD_DMA_LIST_INT_STS BIT(19) 109 #define SPRD_DMA_CFGERR_INT_STS BIT(20) 110 #define SPRD_DMA_CHN_INT_STS \ 111 (SPRD_DMA_FRAG_INT_STS | SPRD_DMA_BLK_INT_STS | \ 112 SPRD_DMA_TRSC_INT_STS | SPRD_DMA_LIST_INT_STS | \ 113 SPRD_DMA_CFGERR_INT_STS) 114 115 /* SPRD_DMA_CHN_FRG_LEN register definition */ 116 #define SPRD_DMA_SRC_DATAWIDTH_OFFSET 30 117 #define SPRD_DMA_DES_DATAWIDTH_OFFSET 28 118 #define SPRD_DMA_SWT_MODE_OFFSET 26 119 #define SPRD_DMA_REQ_MODE_OFFSET 24 120 #define SPRD_DMA_REQ_MODE_MASK GENMASK(1, 0) 121 #define SPRD_DMA_FIX_SEL_OFFSET 21 122 #define SPRD_DMA_FIX_EN_OFFSET 20 123 #define SPRD_DMA_LLIST_END BIT(19) 124 #define SPRD_DMA_FRG_LEN_MASK GENMASK(16, 0) 125 126 /* SPRD_DMA_CHN_BLK_LEN register definition */ 127 #define SPRD_DMA_BLK_LEN_MASK GENMASK(16, 0) 128 129 /* SPRD_DMA_CHN_TRSC_LEN register definition */ 130 #define SPRD_DMA_TRSC_LEN_MASK GENMASK(27, 0) 131 132 /* SPRD_DMA_CHN_TRSF_STEP register definition */ 133 #define SPRD_DMA_DEST_TRSF_STEP_OFFSET 16 134 #define SPRD_DMA_SRC_TRSF_STEP_OFFSET 0 135 #define SPRD_DMA_TRSF_STEP_MASK GENMASK(15, 0) 136 137 /* define DMA channel mode & trigger mode mask */ 138 #define SPRD_DMA_CHN_MODE_MASK GENMASK(7, 0) 139 #define SPRD_DMA_TRG_MODE_MASK GENMASK(7, 0) 140 #define SPRD_DMA_INT_TYPE_MASK GENMASK(7, 0) 141 142 /* define the DMA transfer step type */ 143 #define SPRD_DMA_NONE_STEP 0 144 #define SPRD_DMA_BYTE_STEP 1 145 #define SPRD_DMA_SHORT_STEP 2 146 #define SPRD_DMA_WORD_STEP 4 147 #define SPRD_DMA_DWORD_STEP 8 148 149 #define SPRD_DMA_SOFTWARE_UID 0 150 151 /* dma data width values */ 152 enum sprd_dma_datawidth { 153 SPRD_DMA_DATAWIDTH_1_BYTE, 154 SPRD_DMA_DATAWIDTH_2_BYTES, 155 SPRD_DMA_DATAWIDTH_4_BYTES, 156 SPRD_DMA_DATAWIDTH_8_BYTES, 157 }; 158 159 /* dma channel hardware configuration */ 160 struct sprd_dma_chn_hw { 161 u32 pause; 162 u32 req; 163 u32 cfg; 164 u32 intc; 165 u32 src_addr; 166 u32 des_addr; 167 u32 frg_len; 168 u32 blk_len; 169 u32 trsc_len; 170 u32 trsf_step; 171 u32 wrap_ptr; 172 u32 wrap_to; 173 u32 llist_ptr; 174 u32 frg_step; 175 u32 src_blk_step; 176 u32 des_blk_step; 177 }; 178 179 /* dma request description */ 180 struct sprd_dma_desc { 181 struct virt_dma_desc vd; 182 struct sprd_dma_chn_hw chn_hw; 183 enum dma_transfer_direction dir; 184 }; 185 186 /* dma channel description */ 187 struct sprd_dma_chn { 188 struct virt_dma_chan vc; 189 void __iomem *chn_base; 190 struct sprd_dma_linklist linklist; 191 struct dma_slave_config slave_cfg; 192 u32 chn_num; 193 u32 dev_id; 194 enum sprd_dma_chn_mode chn_mode; 195 enum sprd_dma_trg_mode trg_mode; 196 enum sprd_dma_int_type int_type; 197 struct sprd_dma_desc *cur_desc; 198 }; 199 200 /* SPRD dma device */ 201 struct sprd_dma_dev { 202 struct dma_device dma_dev; 203 void __iomem *glb_base; 204 struct clk *clk; 205 struct clk *ashb_clk; 206 int irq; 207 u32 total_chns; 208 struct sprd_dma_chn channels[0]; 209 }; 210 211 static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param); 212 static struct of_dma_filter_info sprd_dma_info = { 213 .filter_fn = sprd_dma_filter_fn, 214 }; 215 216 static inline struct sprd_dma_chn *to_sprd_dma_chan(struct dma_chan *c) 217 { 218 return container_of(c, struct sprd_dma_chn, vc.chan); 219 } 220 221 static inline struct sprd_dma_dev *to_sprd_dma_dev(struct dma_chan *c) 222 { 223 struct sprd_dma_chn *schan = to_sprd_dma_chan(c); 224 225 return container_of(schan, struct sprd_dma_dev, channels[c->chan_id]); 226 } 227 228 static inline struct sprd_dma_desc *to_sprd_dma_desc(struct virt_dma_desc *vd) 229 { 230 return container_of(vd, struct sprd_dma_desc, vd); 231 } 232 233 static void sprd_dma_glb_update(struct sprd_dma_dev *sdev, u32 reg, 234 u32 mask, u32 val) 235 { 236 u32 orig = readl(sdev->glb_base + reg); 237 u32 tmp; 238 239 tmp = (orig & ~mask) | val; 240 writel(tmp, sdev->glb_base + reg); 241 } 242 243 static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg, 244 u32 mask, u32 val) 245 { 246 u32 orig = readl(schan->chn_base + reg); 247 u32 tmp; 248 249 tmp = (orig & ~mask) | val; 250 writel(tmp, schan->chn_base + reg); 251 } 252 253 static int sprd_dma_enable(struct sprd_dma_dev *sdev) 254 { 255 int ret; 256 257 ret = clk_prepare_enable(sdev->clk); 258 if (ret) 259 return ret; 260 261 /* 262 * The ashb_clk is optional and only for AGCP DMA controller, so we 263 * need add one condition to check if the ashb_clk need enable. 264 */ 265 if (!IS_ERR(sdev->ashb_clk)) 266 ret = clk_prepare_enable(sdev->ashb_clk); 267 268 return ret; 269 } 270 271 static void sprd_dma_disable(struct sprd_dma_dev *sdev) 272 { 273 clk_disable_unprepare(sdev->clk); 274 275 /* 276 * Need to check if we need disable the optional ashb_clk for AGCP DMA. 277 */ 278 if (!IS_ERR(sdev->ashb_clk)) 279 clk_disable_unprepare(sdev->ashb_clk); 280 } 281 282 static void sprd_dma_set_uid(struct sprd_dma_chn *schan) 283 { 284 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 285 u32 dev_id = schan->dev_id; 286 287 if (dev_id != SPRD_DMA_SOFTWARE_UID) { 288 u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET + 289 SPRD_DMA_GLB_REQ_UID(dev_id); 290 291 writel(schan->chn_num + 1, sdev->glb_base + uid_offset); 292 } 293 } 294 295 static void sprd_dma_unset_uid(struct sprd_dma_chn *schan) 296 { 297 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 298 u32 dev_id = schan->dev_id; 299 300 if (dev_id != SPRD_DMA_SOFTWARE_UID) { 301 u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET + 302 SPRD_DMA_GLB_REQ_UID(dev_id); 303 304 writel(0, sdev->glb_base + uid_offset); 305 } 306 } 307 308 static void sprd_dma_clear_int(struct sprd_dma_chn *schan) 309 { 310 sprd_dma_chn_update(schan, SPRD_DMA_CHN_INTC, 311 SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET, 312 SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET); 313 } 314 315 static void sprd_dma_enable_chn(struct sprd_dma_chn *schan) 316 { 317 sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 318 SPRD_DMA_CHN_EN); 319 } 320 321 static void sprd_dma_disable_chn(struct sprd_dma_chn *schan) 322 { 323 sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 0); 324 } 325 326 static void sprd_dma_soft_request(struct sprd_dma_chn *schan) 327 { 328 sprd_dma_chn_update(schan, SPRD_DMA_CHN_REQ, SPRD_DMA_REQ_EN, 329 SPRD_DMA_REQ_EN); 330 } 331 332 static void sprd_dma_pause_resume(struct sprd_dma_chn *schan, bool enable) 333 { 334 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 335 u32 pause, timeout = SPRD_DMA_PAUSE_CNT; 336 337 if (enable) { 338 sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE, 339 SPRD_DMA_PAUSE_EN, SPRD_DMA_PAUSE_EN); 340 341 do { 342 pause = readl(schan->chn_base + SPRD_DMA_CHN_PAUSE); 343 if (pause & SPRD_DMA_PAUSE_STS) 344 break; 345 346 cpu_relax(); 347 } while (--timeout > 0); 348 349 if (!timeout) 350 dev_warn(sdev->dma_dev.dev, 351 "pause dma controller timeout\n"); 352 } else { 353 sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE, 354 SPRD_DMA_PAUSE_EN, 0); 355 } 356 } 357 358 static void sprd_dma_stop_and_disable(struct sprd_dma_chn *schan) 359 { 360 u32 cfg = readl(schan->chn_base + SPRD_DMA_CHN_CFG); 361 362 if (!(cfg & SPRD_DMA_CHN_EN)) 363 return; 364 365 sprd_dma_pause_resume(schan, true); 366 sprd_dma_disable_chn(schan); 367 } 368 369 static unsigned long sprd_dma_get_src_addr(struct sprd_dma_chn *schan) 370 { 371 unsigned long addr, addr_high; 372 373 addr = readl(schan->chn_base + SPRD_DMA_CHN_SRC_ADDR); 374 addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_PTR) & 375 SPRD_DMA_HIGH_ADDR_MASK; 376 377 return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET); 378 } 379 380 static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *schan) 381 { 382 unsigned long addr, addr_high; 383 384 addr = readl(schan->chn_base + SPRD_DMA_CHN_DES_ADDR); 385 addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_TO) & 386 SPRD_DMA_HIGH_ADDR_MASK; 387 388 return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET); 389 } 390 391 static enum sprd_dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *schan) 392 { 393 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 394 u32 intc_sts = readl(schan->chn_base + SPRD_DMA_CHN_INTC) & 395 SPRD_DMA_CHN_INT_STS; 396 397 switch (intc_sts) { 398 case SPRD_DMA_CFGERR_INT_STS: 399 return SPRD_DMA_CFGERR_INT; 400 401 case SPRD_DMA_LIST_INT_STS: 402 return SPRD_DMA_LIST_INT; 403 404 case SPRD_DMA_TRSC_INT_STS: 405 return SPRD_DMA_TRANS_INT; 406 407 case SPRD_DMA_BLK_INT_STS: 408 return SPRD_DMA_BLK_INT; 409 410 case SPRD_DMA_FRAG_INT_STS: 411 return SPRD_DMA_FRAG_INT; 412 413 default: 414 dev_warn(sdev->dma_dev.dev, "incorrect dma interrupt type\n"); 415 return SPRD_DMA_NO_INT; 416 } 417 } 418 419 static enum sprd_dma_req_mode sprd_dma_get_req_type(struct sprd_dma_chn *schan) 420 { 421 u32 frag_reg = readl(schan->chn_base + SPRD_DMA_CHN_FRG_LEN); 422 423 return (frag_reg >> SPRD_DMA_REQ_MODE_OFFSET) & SPRD_DMA_REQ_MODE_MASK; 424 } 425 426 static int sprd_dma_set_2stage_config(struct sprd_dma_chn *schan) 427 { 428 struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan); 429 u32 val, chn = schan->chn_num + 1; 430 431 switch (schan->chn_mode) { 432 case SPRD_DMA_SRC_CHN0: 433 val = chn & SPRD_DMA_GLB_SRC_CHN_MASK; 434 val |= BIT(schan->trg_mode - 1) << SPRD_DMA_GLB_TRG_OFFSET; 435 val |= SPRD_DMA_GLB_2STAGE_EN; 436 if (schan->int_type != SPRD_DMA_NO_INT) 437 val |= SPRD_DMA_GLB_SRC_INT; 438 439 sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP1, val, val); 440 break; 441 442 case SPRD_DMA_SRC_CHN1: 443 val = chn & SPRD_DMA_GLB_SRC_CHN_MASK; 444 val |= BIT(schan->trg_mode - 1) << SPRD_DMA_GLB_TRG_OFFSET; 445 val |= SPRD_DMA_GLB_2STAGE_EN; 446 if (schan->int_type != SPRD_DMA_NO_INT) 447 val |= SPRD_DMA_GLB_SRC_INT; 448 449 sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP2, val, val); 450 break; 451 452 case SPRD_DMA_DST_CHN0: 453 val = (chn << SPRD_DMA_GLB_DEST_CHN_OFFSET) & 454 SPRD_DMA_GLB_DEST_CHN_MASK; 455 val |= SPRD_DMA_GLB_2STAGE_EN; 456 if (schan->int_type != SPRD_DMA_NO_INT) 457 val |= SPRD_DMA_GLB_DEST_INT; 458 459 sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP1, val, val); 460 break; 461 462 case SPRD_DMA_DST_CHN1: 463 val = (chn << SPRD_DMA_GLB_DEST_CHN_OFFSET) & 464 SPRD_DMA_GLB_DEST_CHN_MASK; 465 val |= SPRD_DMA_GLB_2STAGE_EN; 466 if (schan->int_type != SPRD_DMA_NO_INT) 467 val |= SPRD_DMA_GLB_DEST_INT; 468 469 sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP2, val, val); 470 break; 471 472 default: 473 dev_err(sdev->dma_dev.dev, "invalid channel mode setting %d\n", 474 schan->chn_mode); 475 return -EINVAL; 476 } 477 478 return 0; 479 } 480 481 static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan, 482 struct sprd_dma_desc *sdesc) 483 { 484 struct sprd_dma_chn_hw *cfg = &sdesc->chn_hw; 485 486 writel(cfg->pause, schan->chn_base + SPRD_DMA_CHN_PAUSE); 487 writel(cfg->cfg, schan->chn_base + SPRD_DMA_CHN_CFG); 488 writel(cfg->intc, schan->chn_base + SPRD_DMA_CHN_INTC); 489 writel(cfg->src_addr, schan->chn_base + SPRD_DMA_CHN_SRC_ADDR); 490 writel(cfg->des_addr, schan->chn_base + SPRD_DMA_CHN_DES_ADDR); 491 writel(cfg->frg_len, schan->chn_base + SPRD_DMA_CHN_FRG_LEN); 492 writel(cfg->blk_len, schan->chn_base + SPRD_DMA_CHN_BLK_LEN); 493 writel(cfg->trsc_len, schan->chn_base + SPRD_DMA_CHN_TRSC_LEN); 494 writel(cfg->trsf_step, schan->chn_base + SPRD_DMA_CHN_TRSF_STEP); 495 writel(cfg->wrap_ptr, schan->chn_base + SPRD_DMA_CHN_WARP_PTR); 496 writel(cfg->wrap_to, schan->chn_base + SPRD_DMA_CHN_WARP_TO); 497 writel(cfg->llist_ptr, schan->chn_base + SPRD_DMA_CHN_LLIST_PTR); 498 writel(cfg->frg_step, schan->chn_base + SPRD_DMA_CHN_FRAG_STEP); 499 writel(cfg->src_blk_step, schan->chn_base + SPRD_DMA_CHN_SRC_BLK_STEP); 500 writel(cfg->des_blk_step, schan->chn_base + SPRD_DMA_CHN_DES_BLK_STEP); 501 writel(cfg->req, schan->chn_base + SPRD_DMA_CHN_REQ); 502 } 503 504 static void sprd_dma_start(struct sprd_dma_chn *schan) 505 { 506 struct virt_dma_desc *vd = vchan_next_desc(&schan->vc); 507 508 if (!vd) 509 return; 510 511 list_del(&vd->node); 512 schan->cur_desc = to_sprd_dma_desc(vd); 513 514 /* 515 * Set 2-stage configuration if the channel starts one 2-stage 516 * transfer. 517 */ 518 if (schan->chn_mode && sprd_dma_set_2stage_config(schan)) 519 return; 520 521 /* 522 * Copy the DMA configuration from DMA descriptor to this hardware 523 * channel. 524 */ 525 sprd_dma_set_chn_config(schan, schan->cur_desc); 526 sprd_dma_set_uid(schan); 527 sprd_dma_enable_chn(schan); 528 529 if (schan->dev_id == SPRD_DMA_SOFTWARE_UID && 530 schan->chn_mode != SPRD_DMA_DST_CHN0 && 531 schan->chn_mode != SPRD_DMA_DST_CHN1) 532 sprd_dma_soft_request(schan); 533 } 534 535 static void sprd_dma_stop(struct sprd_dma_chn *schan) 536 { 537 sprd_dma_stop_and_disable(schan); 538 sprd_dma_unset_uid(schan); 539 sprd_dma_clear_int(schan); 540 schan->cur_desc = NULL; 541 } 542 543 static bool sprd_dma_check_trans_done(struct sprd_dma_desc *sdesc, 544 enum sprd_dma_int_type int_type, 545 enum sprd_dma_req_mode req_mode) 546 { 547 if (int_type == SPRD_DMA_NO_INT) 548 return false; 549 550 if (int_type >= req_mode + 1) 551 return true; 552 else 553 return false; 554 } 555 556 static irqreturn_t dma_irq_handle(int irq, void *dev_id) 557 { 558 struct sprd_dma_dev *sdev = (struct sprd_dma_dev *)dev_id; 559 u32 irq_status = readl(sdev->glb_base + SPRD_DMA_GLB_INT_MSK_STS); 560 struct sprd_dma_chn *schan; 561 struct sprd_dma_desc *sdesc; 562 enum sprd_dma_req_mode req_type; 563 enum sprd_dma_int_type int_type; 564 bool trans_done = false, cyclic = false; 565 u32 i; 566 567 while (irq_status) { 568 i = __ffs(irq_status); 569 irq_status &= (irq_status - 1); 570 schan = &sdev->channels[i]; 571 572 spin_lock(&schan->vc.lock); 573 574 sdesc = schan->cur_desc; 575 if (!sdesc) { 576 spin_unlock(&schan->vc.lock); 577 return IRQ_HANDLED; 578 } 579 580 int_type = sprd_dma_get_int_type(schan); 581 req_type = sprd_dma_get_req_type(schan); 582 sprd_dma_clear_int(schan); 583 584 /* cyclic mode schedule callback */ 585 cyclic = schan->linklist.phy_addr ? true : false; 586 if (cyclic == true) { 587 vchan_cyclic_callback(&sdesc->vd); 588 } else { 589 /* Check if the dma request descriptor is done. */ 590 trans_done = sprd_dma_check_trans_done(sdesc, int_type, 591 req_type); 592 if (trans_done == true) { 593 vchan_cookie_complete(&sdesc->vd); 594 schan->cur_desc = NULL; 595 sprd_dma_start(schan); 596 } 597 } 598 spin_unlock(&schan->vc.lock); 599 } 600 601 return IRQ_HANDLED; 602 } 603 604 static int sprd_dma_alloc_chan_resources(struct dma_chan *chan) 605 { 606 return pm_runtime_get_sync(chan->device->dev); 607 } 608 609 static void sprd_dma_free_chan_resources(struct dma_chan *chan) 610 { 611 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 612 unsigned long flags; 613 614 spin_lock_irqsave(&schan->vc.lock, flags); 615 sprd_dma_stop(schan); 616 spin_unlock_irqrestore(&schan->vc.lock, flags); 617 618 vchan_free_chan_resources(&schan->vc); 619 pm_runtime_put(chan->device->dev); 620 } 621 622 static enum dma_status sprd_dma_tx_status(struct dma_chan *chan, 623 dma_cookie_t cookie, 624 struct dma_tx_state *txstate) 625 { 626 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 627 struct virt_dma_desc *vd; 628 unsigned long flags; 629 enum dma_status ret; 630 u32 pos; 631 632 ret = dma_cookie_status(chan, cookie, txstate); 633 if (ret == DMA_COMPLETE || !txstate) 634 return ret; 635 636 spin_lock_irqsave(&schan->vc.lock, flags); 637 vd = vchan_find_desc(&schan->vc, cookie); 638 if (vd) { 639 struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd); 640 struct sprd_dma_chn_hw *hw = &sdesc->chn_hw; 641 642 if (hw->trsc_len > 0) 643 pos = hw->trsc_len; 644 else if (hw->blk_len > 0) 645 pos = hw->blk_len; 646 else if (hw->frg_len > 0) 647 pos = hw->frg_len; 648 else 649 pos = 0; 650 } else if (schan->cur_desc && schan->cur_desc->vd.tx.cookie == cookie) { 651 struct sprd_dma_desc *sdesc = schan->cur_desc; 652 653 if (sdesc->dir == DMA_DEV_TO_MEM) 654 pos = sprd_dma_get_dst_addr(schan); 655 else 656 pos = sprd_dma_get_src_addr(schan); 657 } else { 658 pos = 0; 659 } 660 spin_unlock_irqrestore(&schan->vc.lock, flags); 661 662 dma_set_residue(txstate, pos); 663 return ret; 664 } 665 666 static void sprd_dma_issue_pending(struct dma_chan *chan) 667 { 668 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 669 unsigned long flags; 670 671 spin_lock_irqsave(&schan->vc.lock, flags); 672 if (vchan_issue_pending(&schan->vc) && !schan->cur_desc) 673 sprd_dma_start(schan); 674 spin_unlock_irqrestore(&schan->vc.lock, flags); 675 } 676 677 static int sprd_dma_get_datawidth(enum dma_slave_buswidth buswidth) 678 { 679 switch (buswidth) { 680 case DMA_SLAVE_BUSWIDTH_1_BYTE: 681 case DMA_SLAVE_BUSWIDTH_2_BYTES: 682 case DMA_SLAVE_BUSWIDTH_4_BYTES: 683 case DMA_SLAVE_BUSWIDTH_8_BYTES: 684 return ffs(buswidth) - 1; 685 686 default: 687 return -EINVAL; 688 } 689 } 690 691 static int sprd_dma_get_step(enum dma_slave_buswidth buswidth) 692 { 693 switch (buswidth) { 694 case DMA_SLAVE_BUSWIDTH_1_BYTE: 695 case DMA_SLAVE_BUSWIDTH_2_BYTES: 696 case DMA_SLAVE_BUSWIDTH_4_BYTES: 697 case DMA_SLAVE_BUSWIDTH_8_BYTES: 698 return buswidth; 699 700 default: 701 return -EINVAL; 702 } 703 } 704 705 static int sprd_dma_fill_desc(struct dma_chan *chan, 706 struct sprd_dma_chn_hw *hw, 707 unsigned int sglen, int sg_index, 708 dma_addr_t src, dma_addr_t dst, u32 len, 709 enum dma_transfer_direction dir, 710 unsigned long flags, 711 struct dma_slave_config *slave_cfg) 712 { 713 struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan); 714 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 715 enum sprd_dma_chn_mode chn_mode = schan->chn_mode; 716 u32 req_mode = (flags >> SPRD_DMA_REQ_SHIFT) & SPRD_DMA_REQ_MODE_MASK; 717 u32 int_mode = flags & SPRD_DMA_INT_MASK; 718 int src_datawidth, dst_datawidth, src_step, dst_step; 719 u32 temp, fix_mode = 0, fix_en = 0; 720 721 if (dir == DMA_MEM_TO_DEV) { 722 src_step = sprd_dma_get_step(slave_cfg->src_addr_width); 723 if (src_step < 0) { 724 dev_err(sdev->dma_dev.dev, "invalid source step\n"); 725 return src_step; 726 } 727 728 /* 729 * For 2-stage transfer, destination channel step can not be 0, 730 * since destination device is AON IRAM. 731 */ 732 if (chn_mode == SPRD_DMA_DST_CHN0 || 733 chn_mode == SPRD_DMA_DST_CHN1) 734 dst_step = src_step; 735 else 736 dst_step = SPRD_DMA_NONE_STEP; 737 } else { 738 dst_step = sprd_dma_get_step(slave_cfg->dst_addr_width); 739 if (dst_step < 0) { 740 dev_err(sdev->dma_dev.dev, "invalid destination step\n"); 741 return dst_step; 742 } 743 src_step = SPRD_DMA_NONE_STEP; 744 } 745 746 src_datawidth = sprd_dma_get_datawidth(slave_cfg->src_addr_width); 747 if (src_datawidth < 0) { 748 dev_err(sdev->dma_dev.dev, "invalid source datawidth\n"); 749 return src_datawidth; 750 } 751 752 dst_datawidth = sprd_dma_get_datawidth(slave_cfg->dst_addr_width); 753 if (dst_datawidth < 0) { 754 dev_err(sdev->dma_dev.dev, "invalid destination datawidth\n"); 755 return dst_datawidth; 756 } 757 758 if (slave_cfg->slave_id) 759 schan->dev_id = slave_cfg->slave_id; 760 761 hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET; 762 763 /* 764 * wrap_ptr and wrap_to will save the high 4 bits source address and 765 * destination address. 766 */ 767 hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK; 768 hw->wrap_to = (dst >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK; 769 hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK; 770 hw->des_addr = dst & SPRD_DMA_LOW_ADDR_MASK; 771 772 /* 773 * If the src step and dst step both are 0 or both are not 0, that means 774 * we can not enable the fix mode. If one is 0 and another one is not, 775 * we can enable the fix mode. 776 */ 777 if ((src_step != 0 && dst_step != 0) || (src_step | dst_step) == 0) { 778 fix_en = 0; 779 } else { 780 fix_en = 1; 781 if (src_step) 782 fix_mode = 1; 783 else 784 fix_mode = 0; 785 } 786 787 hw->intc = int_mode | SPRD_DMA_CFG_ERR_INT_EN; 788 789 temp = src_datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET; 790 temp |= dst_datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET; 791 temp |= req_mode << SPRD_DMA_REQ_MODE_OFFSET; 792 temp |= fix_mode << SPRD_DMA_FIX_SEL_OFFSET; 793 temp |= fix_en << SPRD_DMA_FIX_EN_OFFSET; 794 temp |= slave_cfg->src_maxburst & SPRD_DMA_FRG_LEN_MASK; 795 hw->frg_len = temp; 796 797 hw->blk_len = slave_cfg->src_maxburst & SPRD_DMA_BLK_LEN_MASK; 798 hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK; 799 800 temp = (dst_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET; 801 temp |= (src_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET; 802 hw->trsf_step = temp; 803 804 /* link-list configuration */ 805 if (schan->linklist.phy_addr) { 806 hw->cfg |= SPRD_DMA_LINKLIST_EN; 807 808 /* link-list index */ 809 temp = sglen ? (sg_index + 1) % sglen : 0; 810 811 /* Next link-list configuration's physical address offset */ 812 temp = temp * sizeof(*hw) + SPRD_DMA_CHN_SRC_ADDR; 813 /* 814 * Set the link-list pointer point to next link-list 815 * configuration's physical address. 816 */ 817 hw->llist_ptr = schan->linklist.phy_addr + temp; 818 } else { 819 hw->llist_ptr = 0; 820 } 821 822 hw->frg_step = 0; 823 hw->src_blk_step = 0; 824 hw->des_blk_step = 0; 825 return 0; 826 } 827 828 static int sprd_dma_fill_linklist_desc(struct dma_chan *chan, 829 unsigned int sglen, int sg_index, 830 dma_addr_t src, dma_addr_t dst, u32 len, 831 enum dma_transfer_direction dir, 832 unsigned long flags, 833 struct dma_slave_config *slave_cfg) 834 { 835 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 836 struct sprd_dma_chn_hw *hw; 837 838 if (!schan->linklist.virt_addr) 839 return -EINVAL; 840 841 hw = (struct sprd_dma_chn_hw *)(schan->linklist.virt_addr + 842 sg_index * sizeof(*hw)); 843 844 return sprd_dma_fill_desc(chan, hw, sglen, sg_index, src, dst, len, 845 dir, flags, slave_cfg); 846 } 847 848 static struct dma_async_tx_descriptor * 849 sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 850 size_t len, unsigned long flags) 851 { 852 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 853 struct sprd_dma_desc *sdesc; 854 struct sprd_dma_chn_hw *hw; 855 enum sprd_dma_datawidth datawidth; 856 u32 step, temp; 857 858 sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT); 859 if (!sdesc) 860 return NULL; 861 862 hw = &sdesc->chn_hw; 863 864 hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET; 865 hw->intc = SPRD_DMA_TRANS_INT | SPRD_DMA_CFG_ERR_INT_EN; 866 hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK; 867 hw->des_addr = dest & SPRD_DMA_LOW_ADDR_MASK; 868 hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) & 869 SPRD_DMA_HIGH_ADDR_MASK; 870 hw->wrap_to = (dest >> SPRD_DMA_HIGH_ADDR_OFFSET) & 871 SPRD_DMA_HIGH_ADDR_MASK; 872 873 if (IS_ALIGNED(len, 8)) { 874 datawidth = SPRD_DMA_DATAWIDTH_8_BYTES; 875 step = SPRD_DMA_DWORD_STEP; 876 } else if (IS_ALIGNED(len, 4)) { 877 datawidth = SPRD_DMA_DATAWIDTH_4_BYTES; 878 step = SPRD_DMA_WORD_STEP; 879 } else if (IS_ALIGNED(len, 2)) { 880 datawidth = SPRD_DMA_DATAWIDTH_2_BYTES; 881 step = SPRD_DMA_SHORT_STEP; 882 } else { 883 datawidth = SPRD_DMA_DATAWIDTH_1_BYTE; 884 step = SPRD_DMA_BYTE_STEP; 885 } 886 887 temp = datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET; 888 temp |= datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET; 889 temp |= SPRD_DMA_TRANS_REQ << SPRD_DMA_REQ_MODE_OFFSET; 890 temp |= len & SPRD_DMA_FRG_LEN_MASK; 891 hw->frg_len = temp; 892 893 hw->blk_len = len & SPRD_DMA_BLK_LEN_MASK; 894 hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK; 895 896 temp = (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET; 897 temp |= (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET; 898 hw->trsf_step = temp; 899 900 return vchan_tx_prep(&schan->vc, &sdesc->vd, flags); 901 } 902 903 static struct dma_async_tx_descriptor * 904 sprd_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 905 unsigned int sglen, enum dma_transfer_direction dir, 906 unsigned long flags, void *context) 907 { 908 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 909 struct dma_slave_config *slave_cfg = &schan->slave_cfg; 910 dma_addr_t src = 0, dst = 0; 911 struct sprd_dma_desc *sdesc; 912 struct scatterlist *sg; 913 u32 len = 0; 914 int ret, i; 915 916 if (!is_slave_direction(dir)) 917 return NULL; 918 919 if (context) { 920 struct sprd_dma_linklist *ll_cfg = 921 (struct sprd_dma_linklist *)context; 922 923 schan->linklist.phy_addr = ll_cfg->phy_addr; 924 schan->linklist.virt_addr = ll_cfg->virt_addr; 925 } else { 926 schan->linklist.phy_addr = 0; 927 schan->linklist.virt_addr = 0; 928 } 929 930 /* 931 * Set channel mode, interrupt mode and trigger mode for 2-stage 932 * transfer. 933 */ 934 schan->chn_mode = 935 (flags >> SPRD_DMA_CHN_MODE_SHIFT) & SPRD_DMA_CHN_MODE_MASK; 936 schan->trg_mode = 937 (flags >> SPRD_DMA_TRG_MODE_SHIFT) & SPRD_DMA_TRG_MODE_MASK; 938 schan->int_type = flags & SPRD_DMA_INT_TYPE_MASK; 939 940 sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT); 941 if (!sdesc) 942 return NULL; 943 944 sdesc->dir = dir; 945 946 for_each_sg(sgl, sg, sglen, i) { 947 len = sg_dma_len(sg); 948 949 if (dir == DMA_MEM_TO_DEV) { 950 src = sg_dma_address(sg); 951 dst = slave_cfg->dst_addr; 952 } else { 953 src = slave_cfg->src_addr; 954 dst = sg_dma_address(sg); 955 } 956 957 /* 958 * The link-list mode needs at least 2 link-list 959 * configurations. If there is only one sg, it doesn't 960 * need to fill the link-list configuration. 961 */ 962 if (sglen < 2) 963 break; 964 965 ret = sprd_dma_fill_linklist_desc(chan, sglen, i, src, dst, len, 966 dir, flags, slave_cfg); 967 if (ret) { 968 kfree(sdesc); 969 return NULL; 970 } 971 } 972 973 ret = sprd_dma_fill_desc(chan, &sdesc->chn_hw, 0, 0, src, dst, len, 974 dir, flags, slave_cfg); 975 if (ret) { 976 kfree(sdesc); 977 return NULL; 978 } 979 980 return vchan_tx_prep(&schan->vc, &sdesc->vd, flags); 981 } 982 983 static int sprd_dma_slave_config(struct dma_chan *chan, 984 struct dma_slave_config *config) 985 { 986 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 987 struct dma_slave_config *slave_cfg = &schan->slave_cfg; 988 989 memcpy(slave_cfg, config, sizeof(*config)); 990 return 0; 991 } 992 993 static int sprd_dma_pause(struct dma_chan *chan) 994 { 995 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 996 unsigned long flags; 997 998 spin_lock_irqsave(&schan->vc.lock, flags); 999 sprd_dma_pause_resume(schan, true); 1000 spin_unlock_irqrestore(&schan->vc.lock, flags); 1001 1002 return 0; 1003 } 1004 1005 static int sprd_dma_resume(struct dma_chan *chan) 1006 { 1007 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 1008 unsigned long flags; 1009 1010 spin_lock_irqsave(&schan->vc.lock, flags); 1011 sprd_dma_pause_resume(schan, false); 1012 spin_unlock_irqrestore(&schan->vc.lock, flags); 1013 1014 return 0; 1015 } 1016 1017 static int sprd_dma_terminate_all(struct dma_chan *chan) 1018 { 1019 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 1020 unsigned long flags; 1021 LIST_HEAD(head); 1022 1023 spin_lock_irqsave(&schan->vc.lock, flags); 1024 sprd_dma_stop(schan); 1025 1026 vchan_get_all_descriptors(&schan->vc, &head); 1027 spin_unlock_irqrestore(&schan->vc.lock, flags); 1028 1029 vchan_dma_desc_free_list(&schan->vc, &head); 1030 return 0; 1031 } 1032 1033 static void sprd_dma_free_desc(struct virt_dma_desc *vd) 1034 { 1035 struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd); 1036 1037 kfree(sdesc); 1038 } 1039 1040 static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param) 1041 { 1042 struct sprd_dma_chn *schan = to_sprd_dma_chan(chan); 1043 u32 slave_id = *(u32 *)param; 1044 1045 schan->dev_id = slave_id; 1046 return true; 1047 } 1048 1049 static int sprd_dma_probe(struct platform_device *pdev) 1050 { 1051 struct device_node *np = pdev->dev.of_node; 1052 struct sprd_dma_dev *sdev; 1053 struct sprd_dma_chn *dma_chn; 1054 struct resource *res; 1055 u32 chn_count; 1056 int ret, i; 1057 1058 ret = device_property_read_u32(&pdev->dev, "#dma-channels", &chn_count); 1059 if (ret) { 1060 dev_err(&pdev->dev, "get dma channels count failed\n"); 1061 return ret; 1062 } 1063 1064 sdev = devm_kzalloc(&pdev->dev, 1065 struct_size(sdev, channels, chn_count), 1066 GFP_KERNEL); 1067 if (!sdev) 1068 return -ENOMEM; 1069 1070 sdev->clk = devm_clk_get(&pdev->dev, "enable"); 1071 if (IS_ERR(sdev->clk)) { 1072 dev_err(&pdev->dev, "get enable clock failed\n"); 1073 return PTR_ERR(sdev->clk); 1074 } 1075 1076 /* ashb clock is optional for AGCP DMA */ 1077 sdev->ashb_clk = devm_clk_get(&pdev->dev, "ashb_eb"); 1078 if (IS_ERR(sdev->ashb_clk)) 1079 dev_warn(&pdev->dev, "no optional ashb eb clock\n"); 1080 1081 /* 1082 * We have three DMA controllers: AP DMA, AON DMA and AGCP DMA. For AGCP 1083 * DMA controller, it can or do not request the irq, which will save 1084 * system power without resuming system by DMA interrupts if AGCP DMA 1085 * does not request the irq. Thus the DMA interrupts property should 1086 * be optional. 1087 */ 1088 sdev->irq = platform_get_irq(pdev, 0); 1089 if (sdev->irq > 0) { 1090 ret = devm_request_irq(&pdev->dev, sdev->irq, dma_irq_handle, 1091 0, "sprd_dma", (void *)sdev); 1092 if (ret < 0) { 1093 dev_err(&pdev->dev, "request dma irq failed\n"); 1094 return ret; 1095 } 1096 } else { 1097 dev_warn(&pdev->dev, "no interrupts for the dma controller\n"); 1098 } 1099 1100 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1101 sdev->glb_base = devm_ioremap_resource(&pdev->dev, res); 1102 if (IS_ERR(sdev->glb_base)) 1103 return PTR_ERR(sdev->glb_base); 1104 1105 dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask); 1106 sdev->total_chns = chn_count; 1107 sdev->dma_dev.chancnt = chn_count; 1108 INIT_LIST_HEAD(&sdev->dma_dev.channels); 1109 INIT_LIST_HEAD(&sdev->dma_dev.global_node); 1110 sdev->dma_dev.dev = &pdev->dev; 1111 sdev->dma_dev.device_alloc_chan_resources = sprd_dma_alloc_chan_resources; 1112 sdev->dma_dev.device_free_chan_resources = sprd_dma_free_chan_resources; 1113 sdev->dma_dev.device_tx_status = sprd_dma_tx_status; 1114 sdev->dma_dev.device_issue_pending = sprd_dma_issue_pending; 1115 sdev->dma_dev.device_prep_dma_memcpy = sprd_dma_prep_dma_memcpy; 1116 sdev->dma_dev.device_prep_slave_sg = sprd_dma_prep_slave_sg; 1117 sdev->dma_dev.device_config = sprd_dma_slave_config; 1118 sdev->dma_dev.device_pause = sprd_dma_pause; 1119 sdev->dma_dev.device_resume = sprd_dma_resume; 1120 sdev->dma_dev.device_terminate_all = sprd_dma_terminate_all; 1121 1122 for (i = 0; i < chn_count; i++) { 1123 dma_chn = &sdev->channels[i]; 1124 dma_chn->chn_num = i; 1125 dma_chn->cur_desc = NULL; 1126 /* get each channel's registers base address. */ 1127 dma_chn->chn_base = sdev->glb_base + SPRD_DMA_CHN_REG_OFFSET + 1128 SPRD_DMA_CHN_REG_LENGTH * i; 1129 1130 dma_chn->vc.desc_free = sprd_dma_free_desc; 1131 vchan_init(&dma_chn->vc, &sdev->dma_dev); 1132 } 1133 1134 platform_set_drvdata(pdev, sdev); 1135 ret = sprd_dma_enable(sdev); 1136 if (ret) 1137 return ret; 1138 1139 pm_runtime_set_active(&pdev->dev); 1140 pm_runtime_enable(&pdev->dev); 1141 1142 ret = pm_runtime_get_sync(&pdev->dev); 1143 if (ret < 0) 1144 goto err_rpm; 1145 1146 ret = dma_async_device_register(&sdev->dma_dev); 1147 if (ret < 0) { 1148 dev_err(&pdev->dev, "register dma device failed:%d\n", ret); 1149 goto err_register; 1150 } 1151 1152 sprd_dma_info.dma_cap = sdev->dma_dev.cap_mask; 1153 ret = of_dma_controller_register(np, of_dma_simple_xlate, 1154 &sprd_dma_info); 1155 if (ret) 1156 goto err_of_register; 1157 1158 pm_runtime_put(&pdev->dev); 1159 return 0; 1160 1161 err_of_register: 1162 dma_async_device_unregister(&sdev->dma_dev); 1163 err_register: 1164 pm_runtime_put_noidle(&pdev->dev); 1165 pm_runtime_disable(&pdev->dev); 1166 err_rpm: 1167 sprd_dma_disable(sdev); 1168 return ret; 1169 } 1170 1171 static int sprd_dma_remove(struct platform_device *pdev) 1172 { 1173 struct sprd_dma_dev *sdev = platform_get_drvdata(pdev); 1174 struct sprd_dma_chn *c, *cn; 1175 int ret; 1176 1177 ret = pm_runtime_get_sync(&pdev->dev); 1178 if (ret < 0) 1179 return ret; 1180 1181 /* explicitly free the irq */ 1182 if (sdev->irq > 0) 1183 devm_free_irq(&pdev->dev, sdev->irq, sdev); 1184 1185 list_for_each_entry_safe(c, cn, &sdev->dma_dev.channels, 1186 vc.chan.device_node) { 1187 list_del(&c->vc.chan.device_node); 1188 tasklet_kill(&c->vc.task); 1189 } 1190 1191 of_dma_controller_free(pdev->dev.of_node); 1192 dma_async_device_unregister(&sdev->dma_dev); 1193 sprd_dma_disable(sdev); 1194 1195 pm_runtime_put_noidle(&pdev->dev); 1196 pm_runtime_disable(&pdev->dev); 1197 return 0; 1198 } 1199 1200 static const struct of_device_id sprd_dma_match[] = { 1201 { .compatible = "sprd,sc9860-dma", }, 1202 {}, 1203 }; 1204 1205 static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev) 1206 { 1207 struct sprd_dma_dev *sdev = dev_get_drvdata(dev); 1208 1209 sprd_dma_disable(sdev); 1210 return 0; 1211 } 1212 1213 static int __maybe_unused sprd_dma_runtime_resume(struct device *dev) 1214 { 1215 struct sprd_dma_dev *sdev = dev_get_drvdata(dev); 1216 int ret; 1217 1218 ret = sprd_dma_enable(sdev); 1219 if (ret) 1220 dev_err(sdev->dma_dev.dev, "enable dma failed\n"); 1221 1222 return ret; 1223 } 1224 1225 static const struct dev_pm_ops sprd_dma_pm_ops = { 1226 SET_RUNTIME_PM_OPS(sprd_dma_runtime_suspend, 1227 sprd_dma_runtime_resume, 1228 NULL) 1229 }; 1230 1231 static struct platform_driver sprd_dma_driver = { 1232 .probe = sprd_dma_probe, 1233 .remove = sprd_dma_remove, 1234 .driver = { 1235 .name = "sprd-dma", 1236 .of_match_table = sprd_dma_match, 1237 .pm = &sprd_dma_pm_ops, 1238 }, 1239 }; 1240 module_platform_driver(sprd_dma_driver); 1241 1242 MODULE_LICENSE("GPL v2"); 1243 MODULE_DESCRIPTION("DMA driver for Spreadtrum"); 1244 MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>"); 1245 MODULE_AUTHOR("Eric Long <eric.long@spreadtrum.com>"); 1246 MODULE_ALIAS("platform:sprd-dma"); 1247