1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2018-2019 Synopsys, Inc. and/or its affiliates. 4 * Synopsys DesignWare eDMA v0 core 5 * 6 * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> 7 */ 8 9 #include <linux/bitfield.h> 10 #include <linux/irqreturn.h> 11 #include <linux/io-64-nonatomic-lo-hi.h> 12 13 #include "dw-edma-core.h" 14 #include "dw-edma-v0-core.h" 15 #include "dw-edma-v0-regs.h" 16 #include "dw-edma-v0-debugfs.h" 17 18 enum dw_edma_control { 19 DW_EDMA_V0_CB = BIT(0), 20 DW_EDMA_V0_TCB = BIT(1), 21 DW_EDMA_V0_LLP = BIT(2), 22 DW_EDMA_V0_LIE = BIT(3), 23 DW_EDMA_V0_RIE = BIT(4), 24 DW_EDMA_V0_CCS = BIT(8), 25 DW_EDMA_V0_LLE = BIT(9), 26 }; 27 28 static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) 29 { 30 return dw->chip->reg_base; 31 } 32 33 #define SET_32(dw, name, value) \ 34 writel(value, &(__dw_regs(dw)->name)) 35 36 #define GET_32(dw, name) \ 37 readl(&(__dw_regs(dw)->name)) 38 39 #define SET_RW_32(dw, dir, name, value) \ 40 do { \ 41 if ((dir) == EDMA_DIR_WRITE) \ 42 SET_32(dw, wr_##name, value); \ 43 else \ 44 SET_32(dw, rd_##name, value); \ 45 } while (0) 46 47 #define GET_RW_32(dw, dir, name) \ 48 ((dir) == EDMA_DIR_WRITE \ 49 ? GET_32(dw, wr_##name) \ 50 : GET_32(dw, rd_##name)) 51 52 #define SET_BOTH_32(dw, name, value) \ 53 do { \ 54 SET_32(dw, wr_##name, value); \ 55 SET_32(dw, rd_##name, value); \ 56 } while (0) 57 58 #define SET_64(dw, name, value) \ 59 writeq(value, &(__dw_regs(dw)->name)) 60 61 #define GET_64(dw, name) \ 62 readq(&(__dw_regs(dw)->name)) 63 64 #define SET_RW_64(dw, dir, name, value) \ 65 do { \ 66 if ((dir) == EDMA_DIR_WRITE) \ 67 SET_64(dw, wr_##name, value); \ 68 else \ 69 SET_64(dw, rd_##name, value); \ 70 } while (0) 71 72 #define GET_RW_64(dw, dir, name) \ 73 ((dir) == EDMA_DIR_WRITE \ 74 ? GET_64(dw, wr_##name) \ 75 : GET_64(dw, rd_##name)) 76 77 #define SET_BOTH_64(dw, name, value) \ 78 do { \ 79 SET_64(dw, wr_##name, value); \ 80 SET_64(dw, rd_##name, value); \ 81 } while (0) 82 83 #define SET_COMPAT(dw, name, value) \ 84 writel(value, &(__dw_regs(dw)->type.unroll.name)) 85 86 #define SET_RW_COMPAT(dw, dir, name, value) \ 87 do { \ 88 if ((dir) == EDMA_DIR_WRITE) \ 89 SET_COMPAT(dw, wr_##name, value); \ 90 else \ 91 SET_COMPAT(dw, rd_##name, value); \ 92 } while (0) 93 94 static inline struct dw_edma_v0_ch_regs __iomem * 95 __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) 96 { 97 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) 98 return &(__dw_regs(dw)->type.legacy.ch); 99 100 if (dir == EDMA_DIR_WRITE) 101 return &__dw_regs(dw)->type.unroll.ch[ch].wr; 102 103 return &__dw_regs(dw)->type.unroll.ch[ch].rd; 104 } 105 106 static inline void writel_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, 107 u32 value, void __iomem *addr) 108 { 109 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) { 110 u32 viewport_sel; 111 unsigned long flags; 112 113 raw_spin_lock_irqsave(&dw->lock, flags); 114 115 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch); 116 if (dir == EDMA_DIR_READ) 117 viewport_sel |= BIT(31); 118 119 writel(viewport_sel, 120 &(__dw_regs(dw)->type.legacy.viewport_sel)); 121 writel(value, addr); 122 123 raw_spin_unlock_irqrestore(&dw->lock, flags); 124 } else { 125 writel(value, addr); 126 } 127 } 128 129 static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, 130 const void __iomem *addr) 131 { 132 u32 value; 133 134 if (dw->chip->mf == EDMA_MF_EDMA_LEGACY) { 135 u32 viewport_sel; 136 unsigned long flags; 137 138 raw_spin_lock_irqsave(&dw->lock, flags); 139 140 viewport_sel = FIELD_PREP(EDMA_V0_VIEWPORT_MASK, ch); 141 if (dir == EDMA_DIR_READ) 142 viewport_sel |= BIT(31); 143 144 writel(viewport_sel, 145 &(__dw_regs(dw)->type.legacy.viewport_sel)); 146 value = readl(addr); 147 148 raw_spin_unlock_irqrestore(&dw->lock, flags); 149 } else { 150 value = readl(addr); 151 } 152 153 return value; 154 } 155 156 #define SET_CH_32(dw, dir, ch, name, value) \ 157 writel_ch(dw, dir, ch, value, &(__dw_ch_regs(dw, dir, ch)->name)) 158 159 #define GET_CH_32(dw, dir, ch, name) \ 160 readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name)) 161 162 /* eDMA management callbacks */ 163 static void dw_edma_v0_core_off(struct dw_edma *dw) 164 { 165 SET_BOTH_32(dw, int_mask, 166 EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); 167 SET_BOTH_32(dw, int_clear, 168 EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); 169 SET_BOTH_32(dw, engine_en, 0); 170 } 171 172 static u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) 173 { 174 u32 num_ch; 175 176 if (dir == EDMA_DIR_WRITE) 177 num_ch = FIELD_GET(EDMA_V0_WRITE_CH_COUNT_MASK, 178 GET_32(dw, ctrl)); 179 else 180 num_ch = FIELD_GET(EDMA_V0_READ_CH_COUNT_MASK, 181 GET_32(dw, ctrl)); 182 183 if (num_ch > EDMA_V0_MAX_NR_CH) 184 num_ch = EDMA_V0_MAX_NR_CH; 185 186 return (u16)num_ch; 187 } 188 189 static enum dma_status dw_edma_v0_core_ch_status(struct dw_edma_chan *chan) 190 { 191 struct dw_edma *dw = chan->dw; 192 u32 tmp; 193 194 tmp = FIELD_GET(EDMA_V0_CH_STATUS_MASK, 195 GET_CH_32(dw, chan->dir, chan->id, ch_control1)); 196 197 if (tmp == 1) 198 return DMA_IN_PROGRESS; 199 else if (tmp == 3) 200 return DMA_COMPLETE; 201 else 202 return DMA_ERROR; 203 } 204 205 static void dw_edma_v0_core_clear_done_int(struct dw_edma_chan *chan) 206 { 207 struct dw_edma *dw = chan->dw; 208 209 SET_RW_32(dw, chan->dir, int_clear, 210 FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id))); 211 } 212 213 static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan) 214 { 215 struct dw_edma *dw = chan->dw; 216 217 SET_RW_32(dw, chan->dir, int_clear, 218 FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id))); 219 } 220 221 static u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir) 222 { 223 return FIELD_GET(EDMA_V0_DONE_INT_MASK, 224 GET_RW_32(dw, dir, int_status)); 225 } 226 227 static u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir) 228 { 229 return FIELD_GET(EDMA_V0_ABORT_INT_MASK, 230 GET_RW_32(dw, dir, int_status)); 231 } 232 233 static irqreturn_t 234 dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, 235 dw_edma_handler_t done, dw_edma_handler_t abort) 236 { 237 struct dw_edma *dw = dw_irq->dw; 238 unsigned long total, pos, val; 239 irqreturn_t ret = IRQ_NONE; 240 struct dw_edma_chan *chan; 241 unsigned long off; 242 u32 mask; 243 244 if (dir == EDMA_DIR_WRITE) { 245 total = dw->wr_ch_cnt; 246 off = 0; 247 mask = dw_irq->wr_mask; 248 } else { 249 total = dw->rd_ch_cnt; 250 off = dw->wr_ch_cnt; 251 mask = dw_irq->rd_mask; 252 } 253 254 val = dw_edma_v0_core_status_done_int(dw, dir); 255 val &= mask; 256 for_each_set_bit(pos, &val, total) { 257 chan = &dw->chan[pos + off]; 258 259 dw_edma_v0_core_clear_done_int(chan); 260 done(chan); 261 262 ret = IRQ_HANDLED; 263 } 264 265 val = dw_edma_v0_core_status_abort_int(dw, dir); 266 val &= mask; 267 for_each_set_bit(pos, &val, total) { 268 chan = &dw->chan[pos + off]; 269 270 dw_edma_v0_core_clear_abort_int(chan); 271 abort(chan); 272 273 ret = IRQ_HANDLED; 274 } 275 276 return ret; 277 } 278 279 static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, 280 u32 control, u32 size, u64 sar, u64 dar) 281 { 282 ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); 283 284 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 285 struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; 286 287 lli->control = control; 288 lli->transfer_size = size; 289 lli->sar.reg = sar; 290 lli->dar.reg = dar; 291 } else { 292 struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; 293 294 writel(control, &lli->control); 295 writel(size, &lli->transfer_size); 296 writeq(sar, &lli->sar.reg); 297 writeq(dar, &lli->dar.reg); 298 } 299 } 300 301 static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk, 302 int i, u32 control, u64 pointer) 303 { 304 ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); 305 306 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 307 struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; 308 309 llp->control = control; 310 llp->llp.reg = pointer; 311 } else { 312 struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; 313 314 writel(control, &llp->control); 315 writeq(pointer, &llp->llp.reg); 316 } 317 } 318 319 static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) 320 { 321 struct dw_edma_burst *child; 322 struct dw_edma_chan *chan = chunk->chan; 323 u32 control = 0, i = 0; 324 int j; 325 326 if (chunk->cb) 327 control = DW_EDMA_V0_CB; 328 329 j = chunk->bursts_alloc; 330 list_for_each_entry(child, &chunk->burst->list, list) { 331 j--; 332 if (!j) { 333 control |= DW_EDMA_V0_LIE; 334 if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) 335 control |= DW_EDMA_V0_RIE; 336 } 337 338 dw_edma_v0_write_ll_data(chunk, i++, control, child->sz, 339 child->sar, child->dar); 340 } 341 342 control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB; 343 if (!chunk->cb) 344 control |= DW_EDMA_V0_CB; 345 346 dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); 347 } 348 349 static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) 350 { 351 struct dw_edma_chan *chan = chunk->chan; 352 struct dw_edma *dw = chan->dw; 353 u32 tmp; 354 355 dw_edma_v0_core_write_chunk(chunk); 356 357 if (first) { 358 /* Enable engine */ 359 SET_RW_32(dw, chan->dir, engine_en, BIT(0)); 360 if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { 361 switch (chan->id) { 362 case 0: 363 SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en, 364 BIT(0)); 365 break; 366 case 1: 367 SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en, 368 BIT(0)); 369 break; 370 case 2: 371 SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en, 372 BIT(0)); 373 break; 374 case 3: 375 SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en, 376 BIT(0)); 377 break; 378 case 4: 379 SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en, 380 BIT(0)); 381 break; 382 case 5: 383 SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en, 384 BIT(0)); 385 break; 386 case 6: 387 SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en, 388 BIT(0)); 389 break; 390 case 7: 391 SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en, 392 BIT(0)); 393 break; 394 } 395 } 396 /* Interrupt unmask - done, abort */ 397 tmp = GET_RW_32(dw, chan->dir, int_mask); 398 tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); 399 tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); 400 SET_RW_32(dw, chan->dir, int_mask, tmp); 401 /* Linked list error */ 402 tmp = GET_RW_32(dw, chan->dir, linked_list_err_en); 403 tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id)); 404 SET_RW_32(dw, chan->dir, linked_list_err_en, tmp); 405 /* Channel control */ 406 SET_CH_32(dw, chan->dir, chan->id, ch_control1, 407 (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE)); 408 /* Linked list */ 409 /* llp is not aligned on 64bit -> keep 32bit accesses */ 410 SET_CH_32(dw, chan->dir, chan->id, llp.lsb, 411 lower_32_bits(chunk->ll_region.paddr)); 412 SET_CH_32(dw, chan->dir, chan->id, llp.msb, 413 upper_32_bits(chunk->ll_region.paddr)); 414 } 415 /* Doorbell */ 416 SET_RW_32(dw, chan->dir, doorbell, 417 FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id)); 418 } 419 420 static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan) 421 { 422 struct dw_edma *dw = chan->dw; 423 u32 tmp = 0; 424 425 /* MSI done addr - low, high */ 426 SET_RW_32(dw, chan->dir, done_imwr.lsb, chan->msi.address_lo); 427 SET_RW_32(dw, chan->dir, done_imwr.msb, chan->msi.address_hi); 428 /* MSI abort addr - low, high */ 429 SET_RW_32(dw, chan->dir, abort_imwr.lsb, chan->msi.address_lo); 430 SET_RW_32(dw, chan->dir, abort_imwr.msb, chan->msi.address_hi); 431 /* MSI data - low, high */ 432 switch (chan->id) { 433 case 0: 434 case 1: 435 tmp = GET_RW_32(dw, chan->dir, ch01_imwr_data); 436 break; 437 438 case 2: 439 case 3: 440 tmp = GET_RW_32(dw, chan->dir, ch23_imwr_data); 441 break; 442 443 case 4: 444 case 5: 445 tmp = GET_RW_32(dw, chan->dir, ch45_imwr_data); 446 break; 447 448 case 6: 449 case 7: 450 tmp = GET_RW_32(dw, chan->dir, ch67_imwr_data); 451 break; 452 } 453 454 if (chan->id & BIT(0)) { 455 /* Channel odd {1, 3, 5, 7} */ 456 tmp &= EDMA_V0_CH_EVEN_MSI_DATA_MASK; 457 tmp |= FIELD_PREP(EDMA_V0_CH_ODD_MSI_DATA_MASK, 458 chan->msi.data); 459 } else { 460 /* Channel even {0, 2, 4, 6} */ 461 tmp &= EDMA_V0_CH_ODD_MSI_DATA_MASK; 462 tmp |= FIELD_PREP(EDMA_V0_CH_EVEN_MSI_DATA_MASK, 463 chan->msi.data); 464 } 465 466 switch (chan->id) { 467 case 0: 468 case 1: 469 SET_RW_32(dw, chan->dir, ch01_imwr_data, tmp); 470 break; 471 472 case 2: 473 case 3: 474 SET_RW_32(dw, chan->dir, ch23_imwr_data, tmp); 475 break; 476 477 case 4: 478 case 5: 479 SET_RW_32(dw, chan->dir, ch45_imwr_data, tmp); 480 break; 481 482 case 6: 483 case 7: 484 SET_RW_32(dw, chan->dir, ch67_imwr_data, tmp); 485 break; 486 } 487 } 488 489 /* eDMA debugfs callbacks */ 490 static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw) 491 { 492 dw_edma_v0_debugfs_on(dw); 493 } 494 495 static const struct dw_edma_core_ops dw_edma_v0_core = { 496 .off = dw_edma_v0_core_off, 497 .ch_count = dw_edma_v0_core_ch_count, 498 .ch_status = dw_edma_v0_core_ch_status, 499 .handle_int = dw_edma_v0_core_handle_int, 500 .start = dw_edma_v0_core_start, 501 .ch_config = dw_edma_v0_core_ch_config, 502 .debugfs_on = dw_edma_v0_core_debugfs_on, 503 }; 504 505 void dw_edma_v0_core_register(struct dw_edma *dw) 506 { 507 dw->core = &dw_edma_v0_core; 508 } 509