xref: /linux/drivers/dma/mxs-dma.c (revision 26b0d14106954ae46d2f4f7eec3481828a210f7d)
1 /*
2  * Copyright 2011 Freescale Semiconductor, Inc. All Rights Reserved.
3  *
4  * Refer to drivers/dma/imx-sdma.c
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 
11 #include <linux/init.h>
12 #include <linux/types.h>
13 #include <linux/mm.h>
14 #include <linux/interrupt.h>
15 #include <linux/clk.h>
16 #include <linux/wait.h>
17 #include <linux/sched.h>
18 #include <linux/semaphore.h>
19 #include <linux/device.h>
20 #include <linux/dma-mapping.h>
21 #include <linux/slab.h>
22 #include <linux/platform_device.h>
23 #include <linux/dmaengine.h>
24 #include <linux/delay.h>
25 #include <linux/module.h>
26 #include <linux/fsl/mxs-dma.h>
27 #include <linux/stmp_device.h>
28 #include <linux/of.h>
29 #include <linux/of_device.h>
30 
31 #include <asm/irq.h>
32 #include <mach/mxs.h>
33 
34 #include "dmaengine.h"
35 
36 /*
37  * NOTE: The term "PIO" throughout the mxs-dma implementation means
38  * PIO mode of mxs apbh-dma and apbx-dma.  With this working mode,
39  * dma can program the controller registers of peripheral devices.
40  */
41 
42 #define dma_is_apbh(mxs_dma)	((mxs_dma)->type == MXS_DMA_APBH)
43 #define apbh_is_old(mxs_dma)	((mxs_dma)->dev_id == IMX23_DMA)
44 
45 #define HW_APBHX_CTRL0				0x000
46 #define BM_APBH_CTRL0_APB_BURST8_EN		(1 << 29)
47 #define BM_APBH_CTRL0_APB_BURST_EN		(1 << 28)
48 #define BP_APBH_CTRL0_RESET_CHANNEL		16
49 #define HW_APBHX_CTRL1				0x010
50 #define HW_APBHX_CTRL2				0x020
51 #define HW_APBHX_CHANNEL_CTRL			0x030
52 #define BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL	16
53 /*
54  * The offset of NXTCMDAR register is different per both dma type and version,
55  * while stride for each channel is all the same 0x70.
56  */
57 #define HW_APBHX_CHn_NXTCMDAR(d, n) \
58 	(((dma_is_apbh(d) && apbh_is_old(d)) ? 0x050 : 0x110) + (n) * 0x70)
59 #define HW_APBHX_CHn_SEMA(d, n) \
60 	(((dma_is_apbh(d) && apbh_is_old(d)) ? 0x080 : 0x140) + (n) * 0x70)
61 
62 /*
63  * ccw bits definitions
64  *
65  * COMMAND:		0..1	(2)
66  * CHAIN:		2	(1)
67  * IRQ:			3	(1)
68  * NAND_LOCK:		4	(1) - not implemented
69  * NAND_WAIT4READY:	5	(1) - not implemented
70  * DEC_SEM:		6	(1)
71  * WAIT4END:		7	(1)
72  * HALT_ON_TERMINATE:	8	(1)
73  * TERMINATE_FLUSH:	9	(1)
74  * RESERVED:		10..11	(2)
75  * PIO_NUM:		12..15	(4)
76  */
77 #define BP_CCW_COMMAND		0
78 #define BM_CCW_COMMAND		(3 << 0)
79 #define CCW_CHAIN		(1 << 2)
80 #define CCW_IRQ			(1 << 3)
81 #define CCW_DEC_SEM		(1 << 6)
82 #define CCW_WAIT4END		(1 << 7)
83 #define CCW_HALT_ON_TERM	(1 << 8)
84 #define CCW_TERM_FLUSH		(1 << 9)
85 #define BP_CCW_PIO_NUM		12
86 #define BM_CCW_PIO_NUM		(0xf << 12)
87 
88 #define BF_CCW(value, field)	(((value) << BP_CCW_##field) & BM_CCW_##field)
89 
90 #define MXS_DMA_CMD_NO_XFER	0
91 #define MXS_DMA_CMD_WRITE	1
92 #define MXS_DMA_CMD_READ	2
93 #define MXS_DMA_CMD_DMA_SENSE	3	/* not implemented */
94 
95 struct mxs_dma_ccw {
96 	u32		next;
97 	u16		bits;
98 	u16		xfer_bytes;
99 #define MAX_XFER_BYTES	0xff00
100 	u32		bufaddr;
101 #define MXS_PIO_WORDS	16
102 	u32		pio_words[MXS_PIO_WORDS];
103 };
104 
105 #define NUM_CCW	(int)(PAGE_SIZE / sizeof(struct mxs_dma_ccw))
106 
107 struct mxs_dma_chan {
108 	struct mxs_dma_engine		*mxs_dma;
109 	struct dma_chan			chan;
110 	struct dma_async_tx_descriptor	desc;
111 	struct tasklet_struct		tasklet;
112 	int				chan_irq;
113 	struct mxs_dma_ccw		*ccw;
114 	dma_addr_t			ccw_phys;
115 	int				desc_count;
116 	enum dma_status			status;
117 	unsigned int			flags;
118 #define MXS_DMA_SG_LOOP			(1 << 0)
119 };
120 
121 #define MXS_DMA_CHANNELS		16
122 #define MXS_DMA_CHANNELS_MASK		0xffff
123 
124 enum mxs_dma_devtype {
125 	MXS_DMA_APBH,
126 	MXS_DMA_APBX,
127 };
128 
129 enum mxs_dma_id {
130 	IMX23_DMA,
131 	IMX28_DMA,
132 };
133 
134 struct mxs_dma_engine {
135 	enum mxs_dma_id			dev_id;
136 	enum mxs_dma_devtype		type;
137 	void __iomem			*base;
138 	struct clk			*clk;
139 	struct dma_device		dma_device;
140 	struct device_dma_parameters	dma_parms;
141 	struct mxs_dma_chan		mxs_chans[MXS_DMA_CHANNELS];
142 };
143 
144 struct mxs_dma_type {
145 	enum mxs_dma_id id;
146 	enum mxs_dma_devtype type;
147 };
148 
149 static struct mxs_dma_type mxs_dma_types[] = {
150 	{
151 		.id = IMX23_DMA,
152 		.type = MXS_DMA_APBH,
153 	}, {
154 		.id = IMX23_DMA,
155 		.type = MXS_DMA_APBX,
156 	}, {
157 		.id = IMX28_DMA,
158 		.type = MXS_DMA_APBH,
159 	}, {
160 		.id = IMX28_DMA,
161 		.type = MXS_DMA_APBX,
162 	}
163 };
164 
165 static struct platform_device_id mxs_dma_ids[] = {
166 	{
167 		.name = "imx23-dma-apbh",
168 		.driver_data = (kernel_ulong_t) &mxs_dma_types[0],
169 	}, {
170 		.name = "imx23-dma-apbx",
171 		.driver_data = (kernel_ulong_t) &mxs_dma_types[1],
172 	}, {
173 		.name = "imx28-dma-apbh",
174 		.driver_data = (kernel_ulong_t) &mxs_dma_types[2],
175 	}, {
176 		.name = "imx28-dma-apbx",
177 		.driver_data = (kernel_ulong_t) &mxs_dma_types[3],
178 	}, {
179 		/* end of list */
180 	}
181 };
182 
183 static const struct of_device_id mxs_dma_dt_ids[] = {
184 	{ .compatible = "fsl,imx23-dma-apbh", .data = &mxs_dma_ids[0], },
185 	{ .compatible = "fsl,imx23-dma-apbx", .data = &mxs_dma_ids[1], },
186 	{ .compatible = "fsl,imx28-dma-apbh", .data = &mxs_dma_ids[2], },
187 	{ .compatible = "fsl,imx28-dma-apbx", .data = &mxs_dma_ids[3], },
188 	{ /* sentinel */ }
189 };
190 MODULE_DEVICE_TABLE(of, mxs_dma_dt_ids);
191 
192 static struct mxs_dma_chan *to_mxs_dma_chan(struct dma_chan *chan)
193 {
194 	return container_of(chan, struct mxs_dma_chan, chan);
195 }
196 
197 int mxs_dma_is_apbh(struct dma_chan *chan)
198 {
199 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
200 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
201 
202 	return dma_is_apbh(mxs_dma);
203 }
204 
205 int mxs_dma_is_apbx(struct dma_chan *chan)
206 {
207 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
208 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
209 
210 	return !dma_is_apbh(mxs_dma);
211 }
212 
213 static void mxs_dma_reset_chan(struct mxs_dma_chan *mxs_chan)
214 {
215 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
216 	int chan_id = mxs_chan->chan.chan_id;
217 
218 	if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma))
219 		writel(1 << (chan_id + BP_APBH_CTRL0_RESET_CHANNEL),
220 			mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET);
221 	else
222 		writel(1 << (chan_id + BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL),
223 			mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_SET);
224 }
225 
226 static void mxs_dma_enable_chan(struct mxs_dma_chan *mxs_chan)
227 {
228 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
229 	int chan_id = mxs_chan->chan.chan_id;
230 
231 	/* set cmd_addr up */
232 	writel(mxs_chan->ccw_phys,
233 		mxs_dma->base + HW_APBHX_CHn_NXTCMDAR(mxs_dma, chan_id));
234 
235 	/* write 1 to SEMA to kick off the channel */
236 	writel(1, mxs_dma->base + HW_APBHX_CHn_SEMA(mxs_dma, chan_id));
237 }
238 
239 static void mxs_dma_disable_chan(struct mxs_dma_chan *mxs_chan)
240 {
241 	mxs_chan->status = DMA_SUCCESS;
242 }
243 
244 static void mxs_dma_pause_chan(struct mxs_dma_chan *mxs_chan)
245 {
246 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
247 	int chan_id = mxs_chan->chan.chan_id;
248 
249 	/* freeze the channel */
250 	if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma))
251 		writel(1 << chan_id,
252 			mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET);
253 	else
254 		writel(1 << chan_id,
255 			mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_SET);
256 
257 	mxs_chan->status = DMA_PAUSED;
258 }
259 
260 static void mxs_dma_resume_chan(struct mxs_dma_chan *mxs_chan)
261 {
262 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
263 	int chan_id = mxs_chan->chan.chan_id;
264 
265 	/* unfreeze the channel */
266 	if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma))
267 		writel(1 << chan_id,
268 			mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_CLR);
269 	else
270 		writel(1 << chan_id,
271 			mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_CLR);
272 
273 	mxs_chan->status = DMA_IN_PROGRESS;
274 }
275 
276 static dma_cookie_t mxs_dma_tx_submit(struct dma_async_tx_descriptor *tx)
277 {
278 	return dma_cookie_assign(tx);
279 }
280 
281 static void mxs_dma_tasklet(unsigned long data)
282 {
283 	struct mxs_dma_chan *mxs_chan = (struct mxs_dma_chan *) data;
284 
285 	if (mxs_chan->desc.callback)
286 		mxs_chan->desc.callback(mxs_chan->desc.callback_param);
287 }
288 
289 static irqreturn_t mxs_dma_int_handler(int irq, void *dev_id)
290 {
291 	struct mxs_dma_engine *mxs_dma = dev_id;
292 	u32 stat1, stat2;
293 
294 	/* completion status */
295 	stat1 = readl(mxs_dma->base + HW_APBHX_CTRL1);
296 	stat1 &= MXS_DMA_CHANNELS_MASK;
297 	writel(stat1, mxs_dma->base + HW_APBHX_CTRL1 + STMP_OFFSET_REG_CLR);
298 
299 	/* error status */
300 	stat2 = readl(mxs_dma->base + HW_APBHX_CTRL2);
301 	writel(stat2, mxs_dma->base + HW_APBHX_CTRL2 + STMP_OFFSET_REG_CLR);
302 
303 	/*
304 	 * When both completion and error of termination bits set at the
305 	 * same time, we do not take it as an error.  IOW, it only becomes
306 	 * an error we need to handle here in case of either it's (1) a bus
307 	 * error or (2) a termination error with no completion.
308 	 */
309 	stat2 = ((stat2 >> MXS_DMA_CHANNELS) & stat2) | /* (1) */
310 		(~(stat2 >> MXS_DMA_CHANNELS) & stat2 & ~stat1); /* (2) */
311 
312 	/* combine error and completion status for checking */
313 	stat1 = (stat2 << MXS_DMA_CHANNELS) | stat1;
314 	while (stat1) {
315 		int channel = fls(stat1) - 1;
316 		struct mxs_dma_chan *mxs_chan =
317 			&mxs_dma->mxs_chans[channel % MXS_DMA_CHANNELS];
318 
319 		if (channel >= MXS_DMA_CHANNELS) {
320 			dev_dbg(mxs_dma->dma_device.dev,
321 				"%s: error in channel %d\n", __func__,
322 				channel - MXS_DMA_CHANNELS);
323 			mxs_chan->status = DMA_ERROR;
324 			mxs_dma_reset_chan(mxs_chan);
325 		} else {
326 			if (mxs_chan->flags & MXS_DMA_SG_LOOP)
327 				mxs_chan->status = DMA_IN_PROGRESS;
328 			else
329 				mxs_chan->status = DMA_SUCCESS;
330 		}
331 
332 		stat1 &= ~(1 << channel);
333 
334 		if (mxs_chan->status == DMA_SUCCESS)
335 			dma_cookie_complete(&mxs_chan->desc);
336 
337 		/* schedule tasklet on this channel */
338 		tasklet_schedule(&mxs_chan->tasklet);
339 	}
340 
341 	return IRQ_HANDLED;
342 }
343 
344 static int mxs_dma_alloc_chan_resources(struct dma_chan *chan)
345 {
346 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
347 	struct mxs_dma_data *data = chan->private;
348 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
349 	int ret;
350 
351 	if (!data)
352 		return -EINVAL;
353 
354 	mxs_chan->chan_irq = data->chan_irq;
355 
356 	mxs_chan->ccw = dma_alloc_coherent(mxs_dma->dma_device.dev, PAGE_SIZE,
357 				&mxs_chan->ccw_phys, GFP_KERNEL);
358 	if (!mxs_chan->ccw) {
359 		ret = -ENOMEM;
360 		goto err_alloc;
361 	}
362 
363 	memset(mxs_chan->ccw, 0, PAGE_SIZE);
364 
365 	if (mxs_chan->chan_irq != NO_IRQ) {
366 		ret = request_irq(mxs_chan->chan_irq, mxs_dma_int_handler,
367 					0, "mxs-dma", mxs_dma);
368 		if (ret)
369 			goto err_irq;
370 	}
371 
372 	ret = clk_prepare_enable(mxs_dma->clk);
373 	if (ret)
374 		goto err_clk;
375 
376 	mxs_dma_reset_chan(mxs_chan);
377 
378 	dma_async_tx_descriptor_init(&mxs_chan->desc, chan);
379 	mxs_chan->desc.tx_submit = mxs_dma_tx_submit;
380 
381 	/* the descriptor is ready */
382 	async_tx_ack(&mxs_chan->desc);
383 
384 	return 0;
385 
386 err_clk:
387 	free_irq(mxs_chan->chan_irq, mxs_dma);
388 err_irq:
389 	dma_free_coherent(mxs_dma->dma_device.dev, PAGE_SIZE,
390 			mxs_chan->ccw, mxs_chan->ccw_phys);
391 err_alloc:
392 	return ret;
393 }
394 
395 static void mxs_dma_free_chan_resources(struct dma_chan *chan)
396 {
397 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
398 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
399 
400 	mxs_dma_disable_chan(mxs_chan);
401 
402 	free_irq(mxs_chan->chan_irq, mxs_dma);
403 
404 	dma_free_coherent(mxs_dma->dma_device.dev, PAGE_SIZE,
405 			mxs_chan->ccw, mxs_chan->ccw_phys);
406 
407 	clk_disable_unprepare(mxs_dma->clk);
408 }
409 
410 /*
411  * How to use the flags for ->device_prep_slave_sg() :
412  *    [1] If there is only one DMA command in the DMA chain, the code should be:
413  *            ......
414  *            ->device_prep_slave_sg(DMA_CTRL_ACK);
415  *            ......
416  *    [2] If there are two DMA commands in the DMA chain, the code should be
417  *            ......
418  *            ->device_prep_slave_sg(0);
419  *            ......
420  *            ->device_prep_slave_sg(DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
421  *            ......
422  *    [3] If there are more than two DMA commands in the DMA chain, the code
423  *        should be:
424  *            ......
425  *            ->device_prep_slave_sg(0);                                // First
426  *            ......
427  *            ->device_prep_slave_sg(DMA_PREP_INTERRUPT [| DMA_CTRL_ACK]);
428  *            ......
429  *            ->device_prep_slave_sg(DMA_PREP_INTERRUPT | DMA_CTRL_ACK); // Last
430  *            ......
431  */
432 static struct dma_async_tx_descriptor *mxs_dma_prep_slave_sg(
433 		struct dma_chan *chan, struct scatterlist *sgl,
434 		unsigned int sg_len, enum dma_transfer_direction direction,
435 		unsigned long flags, void *context)
436 {
437 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
438 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
439 	struct mxs_dma_ccw *ccw;
440 	struct scatterlist *sg;
441 	int i, j;
442 	u32 *pio;
443 	bool append = flags & DMA_PREP_INTERRUPT;
444 	int idx = append ? mxs_chan->desc_count : 0;
445 
446 	if (mxs_chan->status == DMA_IN_PROGRESS && !append)
447 		return NULL;
448 
449 	if (sg_len + (append ? idx : 0) > NUM_CCW) {
450 		dev_err(mxs_dma->dma_device.dev,
451 				"maximum number of sg exceeded: %d > %d\n",
452 				sg_len, NUM_CCW);
453 		goto err_out;
454 	}
455 
456 	mxs_chan->status = DMA_IN_PROGRESS;
457 	mxs_chan->flags = 0;
458 
459 	/*
460 	 * If the sg is prepared with append flag set, the sg
461 	 * will be appended to the last prepared sg.
462 	 */
463 	if (append) {
464 		BUG_ON(idx < 1);
465 		ccw = &mxs_chan->ccw[idx - 1];
466 		ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx;
467 		ccw->bits |= CCW_CHAIN;
468 		ccw->bits &= ~CCW_IRQ;
469 		ccw->bits &= ~CCW_DEC_SEM;
470 	} else {
471 		idx = 0;
472 	}
473 
474 	if (direction == DMA_TRANS_NONE) {
475 		ccw = &mxs_chan->ccw[idx++];
476 		pio = (u32 *) sgl;
477 
478 		for (j = 0; j < sg_len;)
479 			ccw->pio_words[j++] = *pio++;
480 
481 		ccw->bits = 0;
482 		ccw->bits |= CCW_IRQ;
483 		ccw->bits |= CCW_DEC_SEM;
484 		if (flags & DMA_CTRL_ACK)
485 			ccw->bits |= CCW_WAIT4END;
486 		ccw->bits |= CCW_HALT_ON_TERM;
487 		ccw->bits |= CCW_TERM_FLUSH;
488 		ccw->bits |= BF_CCW(sg_len, PIO_NUM);
489 		ccw->bits |= BF_CCW(MXS_DMA_CMD_NO_XFER, COMMAND);
490 	} else {
491 		for_each_sg(sgl, sg, sg_len, i) {
492 			if (sg_dma_len(sg) > MAX_XFER_BYTES) {
493 				dev_err(mxs_dma->dma_device.dev, "maximum bytes for sg entry exceeded: %d > %d\n",
494 						sg_dma_len(sg), MAX_XFER_BYTES);
495 				goto err_out;
496 			}
497 
498 			ccw = &mxs_chan->ccw[idx++];
499 
500 			ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx;
501 			ccw->bufaddr = sg->dma_address;
502 			ccw->xfer_bytes = sg_dma_len(sg);
503 
504 			ccw->bits = 0;
505 			ccw->bits |= CCW_CHAIN;
506 			ccw->bits |= CCW_HALT_ON_TERM;
507 			ccw->bits |= CCW_TERM_FLUSH;
508 			ccw->bits |= BF_CCW(direction == DMA_DEV_TO_MEM ?
509 					MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ,
510 					COMMAND);
511 
512 			if (i + 1 == sg_len) {
513 				ccw->bits &= ~CCW_CHAIN;
514 				ccw->bits |= CCW_IRQ;
515 				ccw->bits |= CCW_DEC_SEM;
516 				if (flags & DMA_CTRL_ACK)
517 					ccw->bits |= CCW_WAIT4END;
518 			}
519 		}
520 	}
521 	mxs_chan->desc_count = idx;
522 
523 	return &mxs_chan->desc;
524 
525 err_out:
526 	mxs_chan->status = DMA_ERROR;
527 	return NULL;
528 }
529 
530 static struct dma_async_tx_descriptor *mxs_dma_prep_dma_cyclic(
531 		struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len,
532 		size_t period_len, enum dma_transfer_direction direction,
533 		void *context)
534 {
535 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
536 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;
537 	int num_periods = buf_len / period_len;
538 	int i = 0, buf = 0;
539 
540 	if (mxs_chan->status == DMA_IN_PROGRESS)
541 		return NULL;
542 
543 	mxs_chan->status = DMA_IN_PROGRESS;
544 	mxs_chan->flags |= MXS_DMA_SG_LOOP;
545 
546 	if (num_periods > NUM_CCW) {
547 		dev_err(mxs_dma->dma_device.dev,
548 				"maximum number of sg exceeded: %d > %d\n",
549 				num_periods, NUM_CCW);
550 		goto err_out;
551 	}
552 
553 	if (period_len > MAX_XFER_BYTES) {
554 		dev_err(mxs_dma->dma_device.dev,
555 				"maximum period size exceeded: %d > %d\n",
556 				period_len, MAX_XFER_BYTES);
557 		goto err_out;
558 	}
559 
560 	while (buf < buf_len) {
561 		struct mxs_dma_ccw *ccw = &mxs_chan->ccw[i];
562 
563 		if (i + 1 == num_periods)
564 			ccw->next = mxs_chan->ccw_phys;
565 		else
566 			ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * (i + 1);
567 
568 		ccw->bufaddr = dma_addr;
569 		ccw->xfer_bytes = period_len;
570 
571 		ccw->bits = 0;
572 		ccw->bits |= CCW_CHAIN;
573 		ccw->bits |= CCW_IRQ;
574 		ccw->bits |= CCW_HALT_ON_TERM;
575 		ccw->bits |= CCW_TERM_FLUSH;
576 		ccw->bits |= BF_CCW(direction == DMA_DEV_TO_MEM ?
577 				MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ, COMMAND);
578 
579 		dma_addr += period_len;
580 		buf += period_len;
581 
582 		i++;
583 	}
584 	mxs_chan->desc_count = i;
585 
586 	return &mxs_chan->desc;
587 
588 err_out:
589 	mxs_chan->status = DMA_ERROR;
590 	return NULL;
591 }
592 
593 static int mxs_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
594 		unsigned long arg)
595 {
596 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
597 	int ret = 0;
598 
599 	switch (cmd) {
600 	case DMA_TERMINATE_ALL:
601 		mxs_dma_reset_chan(mxs_chan);
602 		mxs_dma_disable_chan(mxs_chan);
603 		break;
604 	case DMA_PAUSE:
605 		mxs_dma_pause_chan(mxs_chan);
606 		break;
607 	case DMA_RESUME:
608 		mxs_dma_resume_chan(mxs_chan);
609 		break;
610 	default:
611 		ret = -ENOSYS;
612 	}
613 
614 	return ret;
615 }
616 
617 static enum dma_status mxs_dma_tx_status(struct dma_chan *chan,
618 			dma_cookie_t cookie, struct dma_tx_state *txstate)
619 {
620 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
621 	dma_cookie_t last_used;
622 
623 	last_used = chan->cookie;
624 	dma_set_tx_state(txstate, chan->completed_cookie, last_used, 0);
625 
626 	return mxs_chan->status;
627 }
628 
629 static void mxs_dma_issue_pending(struct dma_chan *chan)
630 {
631 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);
632 
633 	mxs_dma_enable_chan(mxs_chan);
634 }
635 
636 static int __init mxs_dma_init(struct mxs_dma_engine *mxs_dma)
637 {
638 	int ret;
639 
640 	ret = clk_prepare_enable(mxs_dma->clk);
641 	if (ret)
642 		return ret;
643 
644 	ret = stmp_reset_block(mxs_dma->base);
645 	if (ret)
646 		goto err_out;
647 
648 	/* enable apbh burst */
649 	if (dma_is_apbh(mxs_dma)) {
650 		writel(BM_APBH_CTRL0_APB_BURST_EN,
651 			mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET);
652 		writel(BM_APBH_CTRL0_APB_BURST8_EN,
653 			mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET);
654 	}
655 
656 	/* enable irq for all the channels */
657 	writel(MXS_DMA_CHANNELS_MASK << MXS_DMA_CHANNELS,
658 		mxs_dma->base + HW_APBHX_CTRL1 + STMP_OFFSET_REG_SET);
659 
660 err_out:
661 	clk_disable_unprepare(mxs_dma->clk);
662 	return ret;
663 }
664 
665 static int __init mxs_dma_probe(struct platform_device *pdev)
666 {
667 	const struct platform_device_id *id_entry;
668 	const struct of_device_id *of_id;
669 	const struct mxs_dma_type *dma_type;
670 	struct mxs_dma_engine *mxs_dma;
671 	struct resource *iores;
672 	int ret, i;
673 
674 	mxs_dma = kzalloc(sizeof(*mxs_dma), GFP_KERNEL);
675 	if (!mxs_dma)
676 		return -ENOMEM;
677 
678 	of_id = of_match_device(mxs_dma_dt_ids, &pdev->dev);
679 	if (of_id)
680 		id_entry = of_id->data;
681 	else
682 		id_entry = platform_get_device_id(pdev);
683 
684 	dma_type = (struct mxs_dma_type *)id_entry->driver_data;
685 	mxs_dma->type = dma_type->type;
686 	mxs_dma->dev_id = dma_type->id;
687 
688 	iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
689 
690 	if (!request_mem_region(iores->start, resource_size(iores),
691 				pdev->name)) {
692 		ret = -EBUSY;
693 		goto err_request_region;
694 	}
695 
696 	mxs_dma->base = ioremap(iores->start, resource_size(iores));
697 	if (!mxs_dma->base) {
698 		ret = -ENOMEM;
699 		goto err_ioremap;
700 	}
701 
702 	mxs_dma->clk = clk_get(&pdev->dev, NULL);
703 	if (IS_ERR(mxs_dma->clk)) {
704 		ret = PTR_ERR(mxs_dma->clk);
705 		goto err_clk;
706 	}
707 
708 	dma_cap_set(DMA_SLAVE, mxs_dma->dma_device.cap_mask);
709 	dma_cap_set(DMA_CYCLIC, mxs_dma->dma_device.cap_mask);
710 
711 	INIT_LIST_HEAD(&mxs_dma->dma_device.channels);
712 
713 	/* Initialize channel parameters */
714 	for (i = 0; i < MXS_DMA_CHANNELS; i++) {
715 		struct mxs_dma_chan *mxs_chan = &mxs_dma->mxs_chans[i];
716 
717 		mxs_chan->mxs_dma = mxs_dma;
718 		mxs_chan->chan.device = &mxs_dma->dma_device;
719 		dma_cookie_init(&mxs_chan->chan);
720 
721 		tasklet_init(&mxs_chan->tasklet, mxs_dma_tasklet,
722 			     (unsigned long) mxs_chan);
723 
724 
725 		/* Add the channel to mxs_chan list */
726 		list_add_tail(&mxs_chan->chan.device_node,
727 			&mxs_dma->dma_device.channels);
728 	}
729 
730 	ret = mxs_dma_init(mxs_dma);
731 	if (ret)
732 		goto err_init;
733 
734 	mxs_dma->dma_device.dev = &pdev->dev;
735 
736 	/* mxs_dma gets 65535 bytes maximum sg size */
737 	mxs_dma->dma_device.dev->dma_parms = &mxs_dma->dma_parms;
738 	dma_set_max_seg_size(mxs_dma->dma_device.dev, MAX_XFER_BYTES);
739 
740 	mxs_dma->dma_device.device_alloc_chan_resources = mxs_dma_alloc_chan_resources;
741 	mxs_dma->dma_device.device_free_chan_resources = mxs_dma_free_chan_resources;
742 	mxs_dma->dma_device.device_tx_status = mxs_dma_tx_status;
743 	mxs_dma->dma_device.device_prep_slave_sg = mxs_dma_prep_slave_sg;
744 	mxs_dma->dma_device.device_prep_dma_cyclic = mxs_dma_prep_dma_cyclic;
745 	mxs_dma->dma_device.device_control = mxs_dma_control;
746 	mxs_dma->dma_device.device_issue_pending = mxs_dma_issue_pending;
747 
748 	ret = dma_async_device_register(&mxs_dma->dma_device);
749 	if (ret) {
750 		dev_err(mxs_dma->dma_device.dev, "unable to register\n");
751 		goto err_init;
752 	}
753 
754 	dev_info(mxs_dma->dma_device.dev, "initialized\n");
755 
756 	return 0;
757 
758 err_init:
759 	clk_put(mxs_dma->clk);
760 err_clk:
761 	iounmap(mxs_dma->base);
762 err_ioremap:
763 	release_mem_region(iores->start, resource_size(iores));
764 err_request_region:
765 	kfree(mxs_dma);
766 	return ret;
767 }
768 
769 static struct platform_driver mxs_dma_driver = {
770 	.driver		= {
771 		.name	= "mxs-dma",
772 		.of_match_table = mxs_dma_dt_ids,
773 	},
774 	.id_table	= mxs_dma_ids,
775 };
776 
777 static int __init mxs_dma_module_init(void)
778 {
779 	return platform_driver_probe(&mxs_dma_driver, mxs_dma_probe);
780 }
781 subsys_initcall(mxs_dma_module_init);
782