1 // SPDX-License-Identifier: GPL-2.0
9 #include <linux/io-64-nonatomic-lo-hi.h>
11 #include "dw-edma-core.h"
12 #include "dw-hdma-v0-core.h"
13 #include "dw-hdma-v0-regs.h"
14 #include "dw-hdma-v0-debugfs.h"
28 	return dw->chip->reg_base;  in __dw_regs()
35 		return &(__dw_regs(dw)->ch[ch].wr);  in __dw_ch_regs()
37 		return &(__dw_regs(dw)->ch[ch].rd);  in __dw_ch_regs()
40 #define SET_CH_32(dw, dir, ch, name, value) \  argument
41 	writel(value, &(__dw_ch_regs(dw, dir, ch)->name))
43 #define GET_CH_32(dw, dir, ch, name) \  argument
44 	readl(&(__dw_ch_regs(dw, dir, ch)->name))
46 #define SET_BOTH_CH_32(dw, ch, name, value) \  argument
48 		writel(value, &(__dw_ch_regs(dw, EDMA_DIR_WRITE, ch)->name));	\
49 		writel(value, &(__dw_ch_regs(dw, EDMA_DIR_READ, ch)->name));	\
76 static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan)  in dw_hdma_v0_core_ch_status()  argument
78 	struct dw_edma *dw = chan->dw;  in dw_hdma_v0_core_ch_status()
82 			GET_CH_32(dw, chan->id, chan->dir, ch_stat));  in dw_hdma_v0_core_ch_status()
92 static void dw_hdma_v0_core_clear_done_int(struct dw_edma_chan *chan)  in dw_hdma_v0_core_clear_done_int()  argument
94 	struct dw_edma *dw = chan->dw;  in dw_hdma_v0_core_clear_done_int()
96 	SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_STOP_INT_MASK);  in dw_hdma_v0_core_clear_done_int()
99 static void dw_hdma_v0_core_clear_abort_int(struct dw_edma_chan *chan)  in dw_hdma_v0_core_clear_abort_int()  argument
101 	struct dw_edma *dw = chan->dw;  in dw_hdma_v0_core_clear_abort_int()
103 	SET_CH_32(dw, chan->dir, chan->id, int_clear, HDMA_V0_ABORT_INT_MASK);  in dw_hdma_v0_core_clear_abort_int()
106 static u32 dw_hdma_v0_core_status_int(struct dw_edma_chan *chan)  in dw_hdma_v0_core_status_int()  argument
108 	struct dw_edma *dw = chan->dw;  in dw_hdma_v0_core_status_int()
110 	return GET_CH_32(dw, chan->dir, chan->id, int_stat);  in dw_hdma_v0_core_status_int()
117 	struct dw_edma *dw = dw_irq->dw;  in dw_hdma_v0_core_handle_int()
120 	struct dw_edma_chan *chan;  in dw_hdma_v0_core_handle_int()  local
124 		total = dw->wr_ch_cnt;  in dw_hdma_v0_core_handle_int()
126 		mask = dw_irq->wr_mask;  in dw_hdma_v0_core_handle_int()
128 		total = dw->rd_ch_cnt;  in dw_hdma_v0_core_handle_int()
129 		off = dw->wr_ch_cnt;  in dw_hdma_v0_core_handle_int()
130 		mask = dw_irq->rd_mask;  in dw_hdma_v0_core_handle_int()
134 		chan = &dw->chan[pos + off];  in dw_hdma_v0_core_handle_int()
136 		val = dw_hdma_v0_core_status_int(chan);  in dw_hdma_v0_core_handle_int()
138 			dw_hdma_v0_core_clear_done_int(chan);  in dw_hdma_v0_core_handle_int()
139 			done(chan);  in dw_hdma_v0_core_handle_int()
145 			dw_hdma_v0_core_clear_abort_int(chan);  in dw_hdma_v0_core_handle_int()
146 			abort(chan);  in dw_hdma_v0_core_handle_int()
160 	if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {  in dw_hdma_v0_write_ll_data()
161 		struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;  in dw_hdma_v0_write_ll_data()
163 		lli->control = control;  in dw_hdma_v0_write_ll_data()
164 		lli->transfer_size = size;  in dw_hdma_v0_write_ll_data()
165 		lli->sar.reg = sar;  in dw_hdma_v0_write_ll_data()
166 		lli->dar.reg = dar;  in dw_hdma_v0_write_ll_data()
168 		struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;  in dw_hdma_v0_write_ll_data()
170 		writel(control, &lli->control);  in dw_hdma_v0_write_ll_data()
171 		writel(size, &lli->transfer_size);  in dw_hdma_v0_write_ll_data()
172 		writeq(sar, &lli->sar.reg);  in dw_hdma_v0_write_ll_data()
173 		writeq(dar, &lli->dar.reg);  in dw_hdma_v0_write_ll_data()
182 	if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {  in dw_hdma_v0_write_ll_link()
183 		struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;  in dw_hdma_v0_write_ll_link()
185 		llp->control = control;  in dw_hdma_v0_write_ll_link()
186 		llp->llp.reg = pointer;  in dw_hdma_v0_write_ll_link()
188 		struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;  in dw_hdma_v0_write_ll_link()
190 		writel(control, &llp->control);  in dw_hdma_v0_write_ll_link()
191 		writeq(pointer, &llp->llp.reg);  in dw_hdma_v0_write_ll_link()
200 	if (chunk->cb)  in dw_hdma_v0_core_write_chunk()
203 	list_for_each_entry(child, &chunk->burst->list, list)  in dw_hdma_v0_core_write_chunk()
204 		dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz,  in dw_hdma_v0_core_write_chunk()
205 					 child->sar, child->dar);  in dw_hdma_v0_core_write_chunk()
208 	if (!chunk->cb)  in dw_hdma_v0_core_write_chunk()
211 	dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);  in dw_hdma_v0_core_write_chunk()
219 	 * over different buses. Ensure LL-data reaches the memory before the  in dw_hdma_v0_sync_ll_data()
220 	 * doorbell register is toggled by issuing the dummy-read from the remote  in dw_hdma_v0_sync_ll_data()
224 	if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))  in dw_hdma_v0_sync_ll_data()
225 		readl(chunk->ll_region.vaddr.io);  in dw_hdma_v0_sync_ll_data()
230 	struct dw_edma_chan *chan = chunk->chan;  in dw_hdma_v0_core_start()  local
231 	struct dw_edma *dw = chan->dw;  in dw_hdma_v0_core_start()
238 		SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));  in dw_hdma_v0_core_start()
239 		/* Interrupt unmask - stop, abort */  in dw_hdma_v0_core_start()
240 		tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);  in dw_hdma_v0_core_start()
242 		/* Interrupt enable - stop, abort */  in dw_hdma_v0_core_start()
244 		if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL))  in dw_hdma_v0_core_start()
246 		SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);  in dw_hdma_v0_core_start()
248 		SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN);  in dw_hdma_v0_core_start()
250 		/* llp is not aligned on 64bit -> keep 32bit accesses */  in dw_hdma_v0_core_start()
251 		SET_CH_32(dw, chan->dir, chan->id, llp.lsb,  in dw_hdma_v0_core_start()
252 			  lower_32_bits(chunk->ll_region.paddr));  in dw_hdma_v0_core_start()
253 		SET_CH_32(dw, chan->dir, chan->id, llp.msb,  in dw_hdma_v0_core_start()
254 			  upper_32_bits(chunk->ll_region.paddr));  in dw_hdma_v0_core_start()
257 	SET_CH_32(dw, chan->dir, chan->id, cycle_sync,  in dw_hdma_v0_core_start()
263 	SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);  in dw_hdma_v0_core_start()
266 static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)  in dw_hdma_v0_core_ch_config()  argument
268 	struct dw_edma *dw = chan->dw;  in dw_hdma_v0_core_ch_config()
270 	/* MSI done addr - low, high */  in dw_hdma_v0_core_ch_config()
271 	SET_CH_32(dw, chan->dir, chan->id, msi_stop.lsb, chan->msi.address_lo);  in dw_hdma_v0_core_ch_config()
272 	SET_CH_32(dw, chan->dir, chan->id, msi_stop.msb, chan->msi.address_hi);  in dw_hdma_v0_core_ch_config()
273 	/* MSI abort addr - low, high */  in dw_hdma_v0_core_ch_config()
274 	SET_CH_32(dw, chan->dir, chan->id, msi_abort.lsb, chan->msi.address_lo);  in dw_hdma_v0_core_ch_config()
275 	SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi);  in dw_hdma_v0_core_ch_config()
277 	SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data);  in dw_hdma_v0_core_ch_config()
298 	dw->core = &dw_hdma_v0_core;  in dw_hdma_v0_core_register()