1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2023 Cai Huoqing 4 * Synopsys DesignWare HDMA v0 core 5 */ 6 7 #include <linux/bitfield.h> 8 #include <linux/irqreturn.h> 9 #include <linux/io-64-nonatomic-lo-hi.h> 10 11 #include "dw-edma-core.h" 12 #include "dw-hdma-v0-core.h" 13 #include "dw-hdma-v0-regs.h" 14 #include "dw-hdma-v0-debugfs.h" 15 16 enum dw_hdma_control { 17 DW_HDMA_V0_CB = BIT(0), 18 DW_HDMA_V0_TCB = BIT(1), 19 DW_HDMA_V0_LLP = BIT(2), 20 DW_HDMA_V0_LIE = BIT(3), 21 DW_HDMA_V0_RIE = BIT(4), 22 DW_HDMA_V0_CCS = BIT(8), 23 DW_HDMA_V0_LLE = BIT(9), 24 }; 25 26 static inline struct dw_hdma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) 27 { 28 return dw->chip->reg_base; 29 } 30 31 static inline struct dw_hdma_v0_ch_regs __iomem * 32 __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) 33 { 34 if (dir == EDMA_DIR_WRITE) 35 return &(__dw_regs(dw)->ch[ch].wr); 36 else 37 return &(__dw_regs(dw)->ch[ch].rd); 38 } 39 40 #define SET_CH_32(dw, dir, ch, name, value) \ 41 writel(value, &(__dw_ch_regs(dw, dir, ch)->name)) 42 43 #define GET_CH_32(dw, dir, ch, name) \ 44 readl(&(__dw_ch_regs(dw, dir, ch)->name)) 45 46 #define SET_BOTH_CH_32(dw, ch, name, value) \ 47 do { \ 48 writel(value, &(__dw_ch_regs(dw, EDMA_DIR_WRITE, ch)->name)); \ 49 writel(value, &(__dw_ch_regs(dw, EDMA_DIR_READ, ch)->name)); \ 50 } while (0) 51 52 /* HDMA management callbacks */ 53 static void dw_hdma_v0_core_off(struct dw_edma *dw) 54 { 55 int id; 56 57 for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) { 58 SET_BOTH_CH_32(dw, id, int_setup, 59 HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); 60 SET_BOTH_CH_32(dw, id, int_clear, 61 HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); 62 SET_BOTH_CH_32(dw, id, ch_en, 0); 63 } 64 } 65 66 static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) 67 { 68 u32 num_ch = 0; 69 int id; 70 71 for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) { 72 if (GET_CH_32(dw, id, dir, ch_en) & BIT(0)) 73 num_ch++; 74 } 75 76 if (num_ch > HDMA_V0_MAX_NR_CH) 77 num_ch = HDMA_V0_MAX_NR_CH; 78 79 return (u16)num_ch; 80 } 81 82 static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan) 83 { 84 struct dw_edma *dw = chan->dw; 85 u32 tmp; 86 87 tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK, 88 GET_CH_32(dw, chan->id, chan->dir, ch_stat)); 89 90 if (tmp == 1) 91 return DMA_IN_PROGRESS; 92 else if (tmp == 3) 93 return DMA_COMPLETE; 94 else 95 return DMA_ERROR; 96 } 97 98 static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan) 99 { 100 struct dw_edma *dw = chan->dw; 101 102 SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK); 103 } 104 105 static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan) 106 { 107 struct dw_edma *dw = chan->dw; 108 109 SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_ABORT_INT_MASK); 110 } 111 112 static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan) 113 { 114 struct dw_edma *dw = chan->dw; 115 116 return GET_CH_32(dw, chan->dir, chan->id, int_stat); 117 } 118 119 static irqreturn_t 120 dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, 121 dw_edma_handler_t done, dw_edma_handler_t abort) 122 { 123 struct dw_edma *dw = dw_irq->dw; 124 unsigned long total, pos, val; 125 irqreturn_t ret = IRQ_NONE; 126 struct dw_edma_chan *chan; 127 unsigned long off, mask; 128 129 if (dir == EDMA_DIR_WRITE) { 130 total = dw->wr_ch_cnt; 131 off = 0; 132 mask = dw_irq->wr_mask; 133 } else { 134 total = dw->rd_ch_cnt; 135 off = dw->wr_ch_cnt; 136 mask = dw_irq->rd_mask; 137 } 138 139 for_each_set_bit(pos, &mask, total) { 140 chan = &dw->chan[pos + off]; 141 142 val = dw_hdma_v0_core_status_int(chan); 143 if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) { 144 dw_hdma_v0_core_clear_done_int(chan); 145 done(chan); 146 147 ret = IRQ_HANDLED; 148 } 149 150 if (FIELD_GET(HDMA_V0_ABORT_INT_MASK, val)) { 151 dw_hdma_v0_core_clear_abort_int(chan); 152 abort(chan); 153 154 ret = IRQ_HANDLED; 155 } 156 } 157 158 return ret; 159 } 160 161 static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, 162 u32 control, u32 size, u64 sar, u64 dar) 163 { 164 ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli); 165 166 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 167 struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; 168 169 lli->control = control; 170 lli->transfer_size = size; 171 lli->sar.reg = sar; 172 lli->dar.reg = dar; 173 } else { 174 struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; 175 176 writel(control, &lli->control); 177 writel(size, &lli->transfer_size); 178 writeq(sar, &lli->sar.reg); 179 writeq(dar, &lli->dar.reg); 180 } 181 } 182 183 static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk, 184 int i, u32 control, u64 pointer) 185 { 186 ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli); 187 188 if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { 189 struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; 190 191 llp->control = control; 192 llp->llp.reg = pointer; 193 } else { 194 struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; 195 196 writel(control, &llp->control); 197 writeq(pointer, &llp->llp.reg); 198 } 199 } 200 201 static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) 202 { 203 struct dw_edma_burst *child; 204 struct dw_edma_chan *chan = chunk->chan; 205 u32 control = 0, i = 0; 206 int j; 207 208 if (chunk->cb) 209 control = DW_HDMA_V0_CB; 210 211 j = chunk->bursts_alloc; 212 list_for_each_entry(child, &chunk->burst->list, list) { 213 j--; 214 if (!j) { 215 control |= DW_HDMA_V0_LIE; 216 if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) 217 control |= DW_HDMA_V0_RIE; 218 } 219 220 dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz, 221 child->sar, child->dar); 222 } 223 224 control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB; 225 if (!chunk->cb) 226 control |= DW_HDMA_V0_CB; 227 228 dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); 229 } 230 231 static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) 232 { 233 struct dw_edma_chan *chan = chunk->chan; 234 struct dw_edma *dw = chan->dw; 235 u32 tmp; 236 237 dw_hdma_v0_core_write_chunk(chunk); 238 239 if (first) { 240 /* Enable engine */ 241 SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0)); 242 /* Interrupt enable&unmask - done, abort */ 243 tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup) | 244 HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK | 245 HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_STOP_INT_EN; 246 SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp); 247 /* Channel control */ 248 SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN); 249 /* Linked list */ 250 /* llp is not aligned on 64bit -> keep 32bit accesses */ 251 SET_CH_32(dw, chan->dir, chan->id, llp.lsb, 252 lower_32_bits(chunk->ll_region.paddr)); 253 SET_CH_32(dw, chan->dir, chan->id, llp.msb, 254 upper_32_bits(chunk->ll_region.paddr)); 255 } 256 /* Set consumer cycle */ 257 SET_CH_32(dw, chan->dir, chan->id, cycle_sync, 258 HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); 259 /* Doorbell */ 260 SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); 261 } 262 263 static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan) 264 { 265 struct dw_edma *dw = chan->dw; 266 267 /* MSI done addr - low, high */ 268 SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo); 269 SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi); 270 /* MSI abort addr - low, high */ 271 SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo); 272 SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi); 273 /* config MSI data */ 274 SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data); 275 } 276 277 /* HDMA debugfs callbacks */ 278 static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw) 279 { 280 dw_hdma_v0_debugfs_on(dw); 281 } 282 283 static const struct dw_edma_core_ops dw_hdma_v0_core = { 284 .off = dw_hdma_v0_core_off, 285 .ch_count = dw_hdma_v0_core_ch_count, 286 .ch_status = dw_hdma_v0_core_ch_status, 287 .handle_int = dw_hdma_v0_core_handle_int, 288 .start = dw_hdma_v0_core_start, 289 .ch_config = dw_hdma_v0_core_ch_config, 290 .debugfs_on = dw_hdma_v0_core_debugfs_on, 291 }; 292 293 void dw_hdma_v0_core_register(struct dw_edma *dw) 294 { 295 dw->core = &dw_hdma_v0_core; 296 } 297