xref: /linux/drivers/dma/pl330.c (revision 26b0d14106954ae46d2f4f7eec3481828a210f7d)
1 /*
2  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
3  *		http://www.samsung.com
4  *
5  * Copyright (C) 2010 Samsung Electronics Co. Ltd.
6  *	Jaswinder Singh <jassi.brar@samsung.com>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  */
13 
14 #include <linux/kernel.h>
15 #include <linux/io.h>
16 #include <linux/init.h>
17 #include <linux/slab.h>
18 #include <linux/module.h>
19 #include <linux/string.h>
20 #include <linux/delay.h>
21 #include <linux/interrupt.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/dmaengine.h>
24 #include <linux/amba/bus.h>
25 #include <linux/amba/pl330.h>
26 #include <linux/pm_runtime.h>
27 #include <linux/scatterlist.h>
28 #include <linux/of.h>
29 
30 #include "dmaengine.h"
31 #define PL330_MAX_CHAN		8
32 #define PL330_MAX_IRQS		32
33 #define PL330_MAX_PERI		32
34 
35 enum pl330_srccachectrl {
36 	SCCTRL0,	/* Noncacheable and nonbufferable */
37 	SCCTRL1,	/* Bufferable only */
38 	SCCTRL2,	/* Cacheable, but do not allocate */
39 	SCCTRL3,	/* Cacheable and bufferable, but do not allocate */
40 	SINVALID1,
41 	SINVALID2,
42 	SCCTRL6,	/* Cacheable write-through, allocate on reads only */
43 	SCCTRL7,	/* Cacheable write-back, allocate on reads only */
44 };
45 
46 enum pl330_dstcachectrl {
47 	DCCTRL0,	/* Noncacheable and nonbufferable */
48 	DCCTRL1,	/* Bufferable only */
49 	DCCTRL2,	/* Cacheable, but do not allocate */
50 	DCCTRL3,	/* Cacheable and bufferable, but do not allocate */
51 	DINVALID1,	/* AWCACHE = 0x1000 */
52 	DINVALID2,
53 	DCCTRL6,	/* Cacheable write-through, allocate on writes only */
54 	DCCTRL7,	/* Cacheable write-back, allocate on writes only */
55 };
56 
57 enum pl330_byteswap {
58 	SWAP_NO,
59 	SWAP_2,
60 	SWAP_4,
61 	SWAP_8,
62 	SWAP_16,
63 };
64 
65 enum pl330_reqtype {
66 	MEMTOMEM,
67 	MEMTODEV,
68 	DEVTOMEM,
69 	DEVTODEV,
70 };
71 
72 /* Register and Bit field Definitions */
73 #define DS			0x0
74 #define DS_ST_STOP		0x0
75 #define DS_ST_EXEC		0x1
76 #define DS_ST_CMISS		0x2
77 #define DS_ST_UPDTPC		0x3
78 #define DS_ST_WFE		0x4
79 #define DS_ST_ATBRR		0x5
80 #define DS_ST_QBUSY		0x6
81 #define DS_ST_WFP		0x7
82 #define DS_ST_KILL		0x8
83 #define DS_ST_CMPLT		0x9
84 #define DS_ST_FLTCMP		0xe
85 #define DS_ST_FAULT		0xf
86 
87 #define DPC			0x4
88 #define INTEN			0x20
89 #define ES			0x24
90 #define INTSTATUS		0x28
91 #define INTCLR			0x2c
92 #define FSM			0x30
93 #define FSC			0x34
94 #define FTM			0x38
95 
96 #define _FTC			0x40
97 #define FTC(n)			(_FTC + (n)*0x4)
98 
99 #define _CS			0x100
100 #define CS(n)			(_CS + (n)*0x8)
101 #define CS_CNS			(1 << 21)
102 
103 #define _CPC			0x104
104 #define CPC(n)			(_CPC + (n)*0x8)
105 
106 #define _SA			0x400
107 #define SA(n)			(_SA + (n)*0x20)
108 
109 #define _DA			0x404
110 #define DA(n)			(_DA + (n)*0x20)
111 
112 #define _CC			0x408
113 #define CC(n)			(_CC + (n)*0x20)
114 
115 #define CC_SRCINC		(1 << 0)
116 #define CC_DSTINC		(1 << 14)
117 #define CC_SRCPRI		(1 << 8)
118 #define CC_DSTPRI		(1 << 22)
119 #define CC_SRCNS		(1 << 9)
120 #define CC_DSTNS		(1 << 23)
121 #define CC_SRCIA		(1 << 10)
122 #define CC_DSTIA		(1 << 24)
123 #define CC_SRCBRSTLEN_SHFT	4
124 #define CC_DSTBRSTLEN_SHFT	18
125 #define CC_SRCBRSTSIZE_SHFT	1
126 #define CC_DSTBRSTSIZE_SHFT	15
127 #define CC_SRCCCTRL_SHFT	11
128 #define CC_SRCCCTRL_MASK	0x7
129 #define CC_DSTCCTRL_SHFT	25
130 #define CC_DRCCCTRL_MASK	0x7
131 #define CC_SWAP_SHFT		28
132 
133 #define _LC0			0x40c
134 #define LC0(n)			(_LC0 + (n)*0x20)
135 
136 #define _LC1			0x410
137 #define LC1(n)			(_LC1 + (n)*0x20)
138 
139 #define DBGSTATUS		0xd00
140 #define DBG_BUSY		(1 << 0)
141 
142 #define DBGCMD			0xd04
143 #define DBGINST0		0xd08
144 #define DBGINST1		0xd0c
145 
146 #define CR0			0xe00
147 #define CR1			0xe04
148 #define CR2			0xe08
149 #define CR3			0xe0c
150 #define CR4			0xe10
151 #define CRD			0xe14
152 
153 #define PERIPH_ID		0xfe0
154 #define PERIPH_REV_SHIFT	20
155 #define PERIPH_REV_MASK		0xf
156 #define PERIPH_REV_R0P0		0
157 #define PERIPH_REV_R1P0		1
158 #define PERIPH_REV_R1P1		2
159 #define PCELL_ID		0xff0
160 
161 #define CR0_PERIPH_REQ_SET	(1 << 0)
162 #define CR0_BOOT_EN_SET		(1 << 1)
163 #define CR0_BOOT_MAN_NS		(1 << 2)
164 #define CR0_NUM_CHANS_SHIFT	4
165 #define CR0_NUM_CHANS_MASK	0x7
166 #define CR0_NUM_PERIPH_SHIFT	12
167 #define CR0_NUM_PERIPH_MASK	0x1f
168 #define CR0_NUM_EVENTS_SHIFT	17
169 #define CR0_NUM_EVENTS_MASK	0x1f
170 
171 #define CR1_ICACHE_LEN_SHIFT	0
172 #define CR1_ICACHE_LEN_MASK	0x7
173 #define CR1_NUM_ICACHELINES_SHIFT	4
174 #define CR1_NUM_ICACHELINES_MASK	0xf
175 
176 #define CRD_DATA_WIDTH_SHIFT	0
177 #define CRD_DATA_WIDTH_MASK	0x7
178 #define CRD_WR_CAP_SHIFT	4
179 #define CRD_WR_CAP_MASK		0x7
180 #define CRD_WR_Q_DEP_SHIFT	8
181 #define CRD_WR_Q_DEP_MASK	0xf
182 #define CRD_RD_CAP_SHIFT	12
183 #define CRD_RD_CAP_MASK		0x7
184 #define CRD_RD_Q_DEP_SHIFT	16
185 #define CRD_RD_Q_DEP_MASK	0xf
186 #define CRD_DATA_BUFF_SHIFT	20
187 #define CRD_DATA_BUFF_MASK	0x3ff
188 
189 #define PART			0x330
190 #define DESIGNER		0x41
191 #define REVISION		0x0
192 #define INTEG_CFG		0x0
193 #define PERIPH_ID_VAL		((PART << 0) | (DESIGNER << 12))
194 
195 #define PCELL_ID_VAL		0xb105f00d
196 
197 #define PL330_STATE_STOPPED		(1 << 0)
198 #define PL330_STATE_EXECUTING		(1 << 1)
199 #define PL330_STATE_WFE			(1 << 2)
200 #define PL330_STATE_FAULTING		(1 << 3)
201 #define PL330_STATE_COMPLETING		(1 << 4)
202 #define PL330_STATE_WFP			(1 << 5)
203 #define PL330_STATE_KILLING		(1 << 6)
204 #define PL330_STATE_FAULT_COMPLETING	(1 << 7)
205 #define PL330_STATE_CACHEMISS		(1 << 8)
206 #define PL330_STATE_UPDTPC		(1 << 9)
207 #define PL330_STATE_ATBARRIER		(1 << 10)
208 #define PL330_STATE_QUEUEBUSY		(1 << 11)
209 #define PL330_STATE_INVALID		(1 << 15)
210 
211 #define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \
212 				| PL330_STATE_WFE | PL330_STATE_FAULTING)
213 
214 #define CMD_DMAADDH		0x54
215 #define CMD_DMAEND		0x00
216 #define CMD_DMAFLUSHP		0x35
217 #define CMD_DMAGO		0xa0
218 #define CMD_DMALD		0x04
219 #define CMD_DMALDP		0x25
220 #define CMD_DMALP		0x20
221 #define CMD_DMALPEND		0x28
222 #define CMD_DMAKILL		0x01
223 #define CMD_DMAMOV		0xbc
224 #define CMD_DMANOP		0x18
225 #define CMD_DMARMB		0x12
226 #define CMD_DMASEV		0x34
227 #define CMD_DMAST		0x08
228 #define CMD_DMASTP		0x29
229 #define CMD_DMASTZ		0x0c
230 #define CMD_DMAWFE		0x36
231 #define CMD_DMAWFP		0x30
232 #define CMD_DMAWMB		0x13
233 
234 #define SZ_DMAADDH		3
235 #define SZ_DMAEND		1
236 #define SZ_DMAFLUSHP		2
237 #define SZ_DMALD		1
238 #define SZ_DMALDP		2
239 #define SZ_DMALP		2
240 #define SZ_DMALPEND		2
241 #define SZ_DMAKILL		1
242 #define SZ_DMAMOV		6
243 #define SZ_DMANOP		1
244 #define SZ_DMARMB		1
245 #define SZ_DMASEV		2
246 #define SZ_DMAST		1
247 #define SZ_DMASTP		2
248 #define SZ_DMASTZ		1
249 #define SZ_DMAWFE		2
250 #define SZ_DMAWFP		2
251 #define SZ_DMAWMB		1
252 #define SZ_DMAGO		6
253 
254 #define BRST_LEN(ccr)		((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1)
255 #define BRST_SIZE(ccr)		(1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7))
256 
257 #define BYTE_TO_BURST(b, ccr)	((b) / BRST_SIZE(ccr) / BRST_LEN(ccr))
258 #define BURST_TO_BYTE(c, ccr)	((c) * BRST_SIZE(ccr) * BRST_LEN(ccr))
259 
260 /*
261  * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req
262  * at 1byte/burst for P<->M and M<->M respectively.
263  * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req
264  * should be enough for P<->M and M<->M respectively.
265  */
266 #define MCODE_BUFF_PER_REQ	256
267 
268 /* If the _pl330_req is available to the client */
269 #define IS_FREE(req)	(*((u8 *)((req)->mc_cpu)) == CMD_DMAEND)
270 
271 /* Use this _only_ to wait on transient states */
272 #define UNTIL(t, s)	while (!(_state(t) & (s))) cpu_relax();
273 
274 #ifdef PL330_DEBUG_MCGEN
275 static unsigned cmd_line;
276 #define PL330_DBGCMD_DUMP(off, x...)	do { \
277 						printk("%x:", cmd_line); \
278 						printk(x); \
279 						cmd_line += off; \
280 					} while (0)
281 #define PL330_DBGMC_START(addr)		(cmd_line = addr)
282 #else
283 #define PL330_DBGCMD_DUMP(off, x...)	do {} while (0)
284 #define PL330_DBGMC_START(addr)		do {} while (0)
285 #endif
286 
287 /* The number of default descriptors */
288 
289 #define NR_DEFAULT_DESC	16
290 
291 /* Populated by the PL330 core driver for DMA API driver's info */
292 struct pl330_config {
293 	u32	periph_id;
294 	u32	pcell_id;
295 #define DMAC_MODE_NS	(1 << 0)
296 	unsigned int	mode;
297 	unsigned int	data_bus_width:10; /* In number of bits */
298 	unsigned int	data_buf_dep:10;
299 	unsigned int	num_chan:4;
300 	unsigned int	num_peri:6;
301 	u32		peri_ns;
302 	unsigned int	num_events:6;
303 	u32		irq_ns;
304 };
305 
306 /* Handle to the DMAC provided to the PL330 core */
307 struct pl330_info {
308 	/* Owning device */
309 	struct device *dev;
310 	/* Size of MicroCode buffers for each channel. */
311 	unsigned mcbufsz;
312 	/* ioremap'ed address of PL330 registers. */
313 	void __iomem	*base;
314 	/* Client can freely use it. */
315 	void	*client_data;
316 	/* PL330 core data, Client must not touch it. */
317 	void	*pl330_data;
318 	/* Populated by the PL330 core driver during pl330_add */
319 	struct pl330_config	pcfg;
320 	/*
321 	 * If the DMAC has some reset mechanism, then the
322 	 * client may want to provide pointer to the method.
323 	 */
324 	void (*dmac_reset)(struct pl330_info *pi);
325 };
326 
327 /**
328  * Request Configuration.
329  * The PL330 core does not modify this and uses the last
330  * working configuration if the request doesn't provide any.
331  *
332  * The Client may want to provide this info only for the
333  * first request and a request with new settings.
334  */
335 struct pl330_reqcfg {
336 	/* Address Incrementing */
337 	unsigned dst_inc:1;
338 	unsigned src_inc:1;
339 
340 	/*
341 	 * For now, the SRC & DST protection levels
342 	 * and burst size/length are assumed same.
343 	 */
344 	bool nonsecure;
345 	bool privileged;
346 	bool insnaccess;
347 	unsigned brst_len:5;
348 	unsigned brst_size:3; /* in power of 2 */
349 
350 	enum pl330_dstcachectrl dcctl;
351 	enum pl330_srccachectrl scctl;
352 	enum pl330_byteswap swap;
353 	struct pl330_config *pcfg;
354 };
355 
356 /*
357  * One cycle of DMAC operation.
358  * There may be more than one xfer in a request.
359  */
360 struct pl330_xfer {
361 	u32 src_addr;
362 	u32 dst_addr;
363 	/* Size to xfer */
364 	u32 bytes;
365 	/*
366 	 * Pointer to next xfer in the list.
367 	 * The last xfer in the req must point to NULL.
368 	 */
369 	struct pl330_xfer *next;
370 };
371 
372 /* The xfer callbacks are made with one of these arguments. */
373 enum pl330_op_err {
374 	/* The all xfers in the request were success. */
375 	PL330_ERR_NONE,
376 	/* If req aborted due to global error. */
377 	PL330_ERR_ABORT,
378 	/* If req failed due to problem with Channel. */
379 	PL330_ERR_FAIL,
380 };
381 
382 /* A request defining Scatter-Gather List ending with NULL xfer. */
383 struct pl330_req {
384 	enum pl330_reqtype rqtype;
385 	/* Index of peripheral for the xfer. */
386 	unsigned peri:5;
387 	/* Unique token for this xfer, set by the client. */
388 	void *token;
389 	/* Callback to be called after xfer. */
390 	void (*xfer_cb)(void *token, enum pl330_op_err err);
391 	/* If NULL, req will be done at last set parameters. */
392 	struct pl330_reqcfg *cfg;
393 	/* Pointer to first xfer in the request. */
394 	struct pl330_xfer *x;
395 	/* Hook to attach to DMAC's list of reqs with due callback */
396 	struct list_head rqd;
397 };
398 
399 /*
400  * To know the status of the channel and DMAC, the client
401  * provides a pointer to this structure. The PL330 core
402  * fills it with current information.
403  */
404 struct pl330_chanstatus {
405 	/*
406 	 * If the DMAC engine halted due to some error,
407 	 * the client should remove-add DMAC.
408 	 */
409 	bool dmac_halted;
410 	/*
411 	 * If channel is halted due to some error,
412 	 * the client should ABORT/FLUSH and START the channel.
413 	 */
414 	bool faulting;
415 	/* Location of last load */
416 	u32 src_addr;
417 	/* Location of last store */
418 	u32 dst_addr;
419 	/*
420 	 * Pointer to the currently active req, NULL if channel is
421 	 * inactive, even though the requests may be present.
422 	 */
423 	struct pl330_req *top_req;
424 	/* Pointer to req waiting second in the queue if any. */
425 	struct pl330_req *wait_req;
426 };
427 
428 enum pl330_chan_op {
429 	/* Start the channel */
430 	PL330_OP_START,
431 	/* Abort the active xfer */
432 	PL330_OP_ABORT,
433 	/* Stop xfer and flush queue */
434 	PL330_OP_FLUSH,
435 };
436 
437 struct _xfer_spec {
438 	u32 ccr;
439 	struct pl330_req *r;
440 	struct pl330_xfer *x;
441 };
442 
443 enum dmamov_dst {
444 	SAR = 0,
445 	CCR,
446 	DAR,
447 };
448 
449 enum pl330_dst {
450 	SRC = 0,
451 	DST,
452 };
453 
454 enum pl330_cond {
455 	SINGLE,
456 	BURST,
457 	ALWAYS,
458 };
459 
460 struct _pl330_req {
461 	u32 mc_bus;
462 	void *mc_cpu;
463 	/* Number of bytes taken to setup MC for the req */
464 	u32 mc_len;
465 	struct pl330_req *r;
466 };
467 
468 /* ToBeDone for tasklet */
469 struct _pl330_tbd {
470 	bool reset_dmac;
471 	bool reset_mngr;
472 	u8 reset_chan;
473 };
474 
475 /* A DMAC Thread */
476 struct pl330_thread {
477 	u8 id;
478 	int ev;
479 	/* If the channel is not yet acquired by any client */
480 	bool free;
481 	/* Parent DMAC */
482 	struct pl330_dmac *dmac;
483 	/* Only two at a time */
484 	struct _pl330_req req[2];
485 	/* Index of the last enqueued request */
486 	unsigned lstenq;
487 	/* Index of the last submitted request or -1 if the DMA is stopped */
488 	int req_running;
489 };
490 
491 enum pl330_dmac_state {
492 	UNINIT,
493 	INIT,
494 	DYING,
495 };
496 
497 /* A DMAC */
498 struct pl330_dmac {
499 	spinlock_t		lock;
500 	/* Holds list of reqs with due callbacks */
501 	struct list_head	req_done;
502 	/* Pointer to platform specific stuff */
503 	struct pl330_info	*pinfo;
504 	/* Maximum possible events/irqs */
505 	int			events[32];
506 	/* BUS address of MicroCode buffer */
507 	u32			mcode_bus;
508 	/* CPU address of MicroCode buffer */
509 	void			*mcode_cpu;
510 	/* List of all Channel threads */
511 	struct pl330_thread	*channels;
512 	/* Pointer to the MANAGER thread */
513 	struct pl330_thread	*manager;
514 	/* To handle bad news in interrupt */
515 	struct tasklet_struct	tasks;
516 	struct _pl330_tbd	dmac_tbd;
517 	/* State of DMAC operation */
518 	enum pl330_dmac_state	state;
519 };
520 
521 enum desc_status {
522 	/* In the DMAC pool */
523 	FREE,
524 	/*
525 	 * Allocted to some channel during prep_xxx
526 	 * Also may be sitting on the work_list.
527 	 */
528 	PREP,
529 	/*
530 	 * Sitting on the work_list and already submitted
531 	 * to the PL330 core. Not more than two descriptors
532 	 * of a channel can be BUSY at any time.
533 	 */
534 	BUSY,
535 	/*
536 	 * Sitting on the channel work_list but xfer done
537 	 * by PL330 core
538 	 */
539 	DONE,
540 };
541 
542 struct dma_pl330_chan {
543 	/* Schedule desc completion */
544 	struct tasklet_struct task;
545 
546 	/* DMA-Engine Channel */
547 	struct dma_chan chan;
548 
549 	/* List of to be xfered descriptors */
550 	struct list_head work_list;
551 
552 	/* Pointer to the DMAC that manages this channel,
553 	 * NULL if the channel is available to be acquired.
554 	 * As the parent, this DMAC also provides descriptors
555 	 * to the channel.
556 	 */
557 	struct dma_pl330_dmac *dmac;
558 
559 	/* To protect channel manipulation */
560 	spinlock_t lock;
561 
562 	/* Token of a hardware channel thread of PL330 DMAC
563 	 * NULL if the channel is available to be acquired.
564 	 */
565 	void *pl330_chid;
566 
567 	/* For D-to-M and M-to-D channels */
568 	int burst_sz; /* the peripheral fifo width */
569 	int burst_len; /* the number of burst */
570 	dma_addr_t fifo_addr;
571 
572 	/* for cyclic capability */
573 	bool cyclic;
574 };
575 
576 struct dma_pl330_dmac {
577 	struct pl330_info pif;
578 
579 	/* DMA-Engine Device */
580 	struct dma_device ddma;
581 
582 	/* Pool of descriptors available for the DMAC's channels */
583 	struct list_head desc_pool;
584 	/* To protect desc_pool manipulation */
585 	spinlock_t pool_lock;
586 
587 	/* Peripheral channels connected to this DMAC */
588 	struct dma_pl330_chan *peripherals; /* keep at end */
589 
590 	struct clk *clk;
591 };
592 
593 struct dma_pl330_desc {
594 	/* To attach to a queue as child */
595 	struct list_head node;
596 
597 	/* Descriptor for the DMA Engine API */
598 	struct dma_async_tx_descriptor txd;
599 
600 	/* Xfer for PL330 core */
601 	struct pl330_xfer px;
602 
603 	struct pl330_reqcfg rqcfg;
604 	struct pl330_req req;
605 
606 	enum desc_status status;
607 
608 	/* The channel which currently holds this desc */
609 	struct dma_pl330_chan *pchan;
610 };
611 
612 static inline void _callback(struct pl330_req *r, enum pl330_op_err err)
613 {
614 	if (r && r->xfer_cb)
615 		r->xfer_cb(r->token, err);
616 }
617 
618 static inline bool _queue_empty(struct pl330_thread *thrd)
619 {
620 	return (IS_FREE(&thrd->req[0]) && IS_FREE(&thrd->req[1]))
621 		? true : false;
622 }
623 
624 static inline bool _queue_full(struct pl330_thread *thrd)
625 {
626 	return (IS_FREE(&thrd->req[0]) || IS_FREE(&thrd->req[1]))
627 		? false : true;
628 }
629 
630 static inline bool is_manager(struct pl330_thread *thrd)
631 {
632 	struct pl330_dmac *pl330 = thrd->dmac;
633 
634 	/* MANAGER is indexed at the end */
635 	if (thrd->id == pl330->pinfo->pcfg.num_chan)
636 		return true;
637 	else
638 		return false;
639 }
640 
641 /* If manager of the thread is in Non-Secure mode */
642 static inline bool _manager_ns(struct pl330_thread *thrd)
643 {
644 	struct pl330_dmac *pl330 = thrd->dmac;
645 
646 	return (pl330->pinfo->pcfg.mode & DMAC_MODE_NS) ? true : false;
647 }
648 
649 static inline u32 get_id(struct pl330_info *pi, u32 off)
650 {
651 	void __iomem *regs = pi->base;
652 	u32 id = 0;
653 
654 	id |= (readb(regs + off + 0x0) << 0);
655 	id |= (readb(regs + off + 0x4) << 8);
656 	id |= (readb(regs + off + 0x8) << 16);
657 	id |= (readb(regs + off + 0xc) << 24);
658 
659 	return id;
660 }
661 
662 static inline u32 get_revision(u32 periph_id)
663 {
664 	return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK;
665 }
666 
667 static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[],
668 		enum pl330_dst da, u16 val)
669 {
670 	if (dry_run)
671 		return SZ_DMAADDH;
672 
673 	buf[0] = CMD_DMAADDH;
674 	buf[0] |= (da << 1);
675 	*((u16 *)&buf[1]) = val;
676 
677 	PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n",
678 		da == 1 ? "DA" : "SA", val);
679 
680 	return SZ_DMAADDH;
681 }
682 
683 static inline u32 _emit_END(unsigned dry_run, u8 buf[])
684 {
685 	if (dry_run)
686 		return SZ_DMAEND;
687 
688 	buf[0] = CMD_DMAEND;
689 
690 	PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n");
691 
692 	return SZ_DMAEND;
693 }
694 
695 static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri)
696 {
697 	if (dry_run)
698 		return SZ_DMAFLUSHP;
699 
700 	buf[0] = CMD_DMAFLUSHP;
701 
702 	peri &= 0x1f;
703 	peri <<= 3;
704 	buf[1] = peri;
705 
706 	PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3);
707 
708 	return SZ_DMAFLUSHP;
709 }
710 
711 static inline u32 _emit_LD(unsigned dry_run, u8 buf[],	enum pl330_cond cond)
712 {
713 	if (dry_run)
714 		return SZ_DMALD;
715 
716 	buf[0] = CMD_DMALD;
717 
718 	if (cond == SINGLE)
719 		buf[0] |= (0 << 1) | (1 << 0);
720 	else if (cond == BURST)
721 		buf[0] |= (1 << 1) | (1 << 0);
722 
723 	PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n",
724 		cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
725 
726 	return SZ_DMALD;
727 }
728 
729 static inline u32 _emit_LDP(unsigned dry_run, u8 buf[],
730 		enum pl330_cond cond, u8 peri)
731 {
732 	if (dry_run)
733 		return SZ_DMALDP;
734 
735 	buf[0] = CMD_DMALDP;
736 
737 	if (cond == BURST)
738 		buf[0] |= (1 << 1);
739 
740 	peri &= 0x1f;
741 	peri <<= 3;
742 	buf[1] = peri;
743 
744 	PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n",
745 		cond == SINGLE ? 'S' : 'B', peri >> 3);
746 
747 	return SZ_DMALDP;
748 }
749 
750 static inline u32 _emit_LP(unsigned dry_run, u8 buf[],
751 		unsigned loop, u8 cnt)
752 {
753 	if (dry_run)
754 		return SZ_DMALP;
755 
756 	buf[0] = CMD_DMALP;
757 
758 	if (loop)
759 		buf[0] |= (1 << 1);
760 
761 	cnt--; /* DMAC increments by 1 internally */
762 	buf[1] = cnt;
763 
764 	PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt);
765 
766 	return SZ_DMALP;
767 }
768 
769 struct _arg_LPEND {
770 	enum pl330_cond cond;
771 	bool forever;
772 	unsigned loop;
773 	u8 bjump;
774 };
775 
776 static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[],
777 		const struct _arg_LPEND *arg)
778 {
779 	enum pl330_cond cond = arg->cond;
780 	bool forever = arg->forever;
781 	unsigned loop = arg->loop;
782 	u8 bjump = arg->bjump;
783 
784 	if (dry_run)
785 		return SZ_DMALPEND;
786 
787 	buf[0] = CMD_DMALPEND;
788 
789 	if (loop)
790 		buf[0] |= (1 << 2);
791 
792 	if (!forever)
793 		buf[0] |= (1 << 4);
794 
795 	if (cond == SINGLE)
796 		buf[0] |= (0 << 1) | (1 << 0);
797 	else if (cond == BURST)
798 		buf[0] |= (1 << 1) | (1 << 0);
799 
800 	buf[1] = bjump;
801 
802 	PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n",
803 			forever ? "FE" : "END",
804 			cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'),
805 			loop ? '1' : '0',
806 			bjump);
807 
808 	return SZ_DMALPEND;
809 }
810 
811 static inline u32 _emit_KILL(unsigned dry_run, u8 buf[])
812 {
813 	if (dry_run)
814 		return SZ_DMAKILL;
815 
816 	buf[0] = CMD_DMAKILL;
817 
818 	return SZ_DMAKILL;
819 }
820 
821 static inline u32 _emit_MOV(unsigned dry_run, u8 buf[],
822 		enum dmamov_dst dst, u32 val)
823 {
824 	if (dry_run)
825 		return SZ_DMAMOV;
826 
827 	buf[0] = CMD_DMAMOV;
828 	buf[1] = dst;
829 	*((u32 *)&buf[2]) = val;
830 
831 	PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n",
832 		dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val);
833 
834 	return SZ_DMAMOV;
835 }
836 
837 static inline u32 _emit_NOP(unsigned dry_run, u8 buf[])
838 {
839 	if (dry_run)
840 		return SZ_DMANOP;
841 
842 	buf[0] = CMD_DMANOP;
843 
844 	PL330_DBGCMD_DUMP(SZ_DMANOP, "\tDMANOP\n");
845 
846 	return SZ_DMANOP;
847 }
848 
849 static inline u32 _emit_RMB(unsigned dry_run, u8 buf[])
850 {
851 	if (dry_run)
852 		return SZ_DMARMB;
853 
854 	buf[0] = CMD_DMARMB;
855 
856 	PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n");
857 
858 	return SZ_DMARMB;
859 }
860 
861 static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev)
862 {
863 	if (dry_run)
864 		return SZ_DMASEV;
865 
866 	buf[0] = CMD_DMASEV;
867 
868 	ev &= 0x1f;
869 	ev <<= 3;
870 	buf[1] = ev;
871 
872 	PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3);
873 
874 	return SZ_DMASEV;
875 }
876 
877 static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond)
878 {
879 	if (dry_run)
880 		return SZ_DMAST;
881 
882 	buf[0] = CMD_DMAST;
883 
884 	if (cond == SINGLE)
885 		buf[0] |= (0 << 1) | (1 << 0);
886 	else if (cond == BURST)
887 		buf[0] |= (1 << 1) | (1 << 0);
888 
889 	PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n",
890 		cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'));
891 
892 	return SZ_DMAST;
893 }
894 
895 static inline u32 _emit_STP(unsigned dry_run, u8 buf[],
896 		enum pl330_cond cond, u8 peri)
897 {
898 	if (dry_run)
899 		return SZ_DMASTP;
900 
901 	buf[0] = CMD_DMASTP;
902 
903 	if (cond == BURST)
904 		buf[0] |= (1 << 1);
905 
906 	peri &= 0x1f;
907 	peri <<= 3;
908 	buf[1] = peri;
909 
910 	PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n",
911 		cond == SINGLE ? 'S' : 'B', peri >> 3);
912 
913 	return SZ_DMASTP;
914 }
915 
916 static inline u32 _emit_STZ(unsigned dry_run, u8 buf[])
917 {
918 	if (dry_run)
919 		return SZ_DMASTZ;
920 
921 	buf[0] = CMD_DMASTZ;
922 
923 	PL330_DBGCMD_DUMP(SZ_DMASTZ, "\tDMASTZ\n");
924 
925 	return SZ_DMASTZ;
926 }
927 
928 static inline u32 _emit_WFE(unsigned dry_run, u8 buf[], u8 ev,
929 		unsigned invalidate)
930 {
931 	if (dry_run)
932 		return SZ_DMAWFE;
933 
934 	buf[0] = CMD_DMAWFE;
935 
936 	ev &= 0x1f;
937 	ev <<= 3;
938 	buf[1] = ev;
939 
940 	if (invalidate)
941 		buf[1] |= (1 << 1);
942 
943 	PL330_DBGCMD_DUMP(SZ_DMAWFE, "\tDMAWFE %u%s\n",
944 		ev >> 3, invalidate ? ", I" : "");
945 
946 	return SZ_DMAWFE;
947 }
948 
949 static inline u32 _emit_WFP(unsigned dry_run, u8 buf[],
950 		enum pl330_cond cond, u8 peri)
951 {
952 	if (dry_run)
953 		return SZ_DMAWFP;
954 
955 	buf[0] = CMD_DMAWFP;
956 
957 	if (cond == SINGLE)
958 		buf[0] |= (0 << 1) | (0 << 0);
959 	else if (cond == BURST)
960 		buf[0] |= (1 << 1) | (0 << 0);
961 	else
962 		buf[0] |= (0 << 1) | (1 << 0);
963 
964 	peri &= 0x1f;
965 	peri <<= 3;
966 	buf[1] = peri;
967 
968 	PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n",
969 		cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3);
970 
971 	return SZ_DMAWFP;
972 }
973 
974 static inline u32 _emit_WMB(unsigned dry_run, u8 buf[])
975 {
976 	if (dry_run)
977 		return SZ_DMAWMB;
978 
979 	buf[0] = CMD_DMAWMB;
980 
981 	PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n");
982 
983 	return SZ_DMAWMB;
984 }
985 
986 struct _arg_GO {
987 	u8 chan;
988 	u32 addr;
989 	unsigned ns;
990 };
991 
992 static inline u32 _emit_GO(unsigned dry_run, u8 buf[],
993 		const struct _arg_GO *arg)
994 {
995 	u8 chan = arg->chan;
996 	u32 addr = arg->addr;
997 	unsigned ns = arg->ns;
998 
999 	if (dry_run)
1000 		return SZ_DMAGO;
1001 
1002 	buf[0] = CMD_DMAGO;
1003 	buf[0] |= (ns << 1);
1004 
1005 	buf[1] = chan & 0x7;
1006 
1007 	*((u32 *)&buf[2]) = addr;
1008 
1009 	return SZ_DMAGO;
1010 }
1011 
1012 #define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t)
1013 
1014 /* Returns Time-Out */
1015 static bool _until_dmac_idle(struct pl330_thread *thrd)
1016 {
1017 	void __iomem *regs = thrd->dmac->pinfo->base;
1018 	unsigned long loops = msecs_to_loops(5);
1019 
1020 	do {
1021 		/* Until Manager is Idle */
1022 		if (!(readl(regs + DBGSTATUS) & DBG_BUSY))
1023 			break;
1024 
1025 		cpu_relax();
1026 	} while (--loops);
1027 
1028 	if (!loops)
1029 		return true;
1030 
1031 	return false;
1032 }
1033 
1034 static inline void _execute_DBGINSN(struct pl330_thread *thrd,
1035 		u8 insn[], bool as_manager)
1036 {
1037 	void __iomem *regs = thrd->dmac->pinfo->base;
1038 	u32 val;
1039 
1040 	val = (insn[0] << 16) | (insn[1] << 24);
1041 	if (!as_manager) {
1042 		val |= (1 << 0);
1043 		val |= (thrd->id << 8); /* Channel Number */
1044 	}
1045 	writel(val, regs + DBGINST0);
1046 
1047 	val = *((u32 *)&insn[2]);
1048 	writel(val, regs + DBGINST1);
1049 
1050 	/* If timed out due to halted state-machine */
1051 	if (_until_dmac_idle(thrd)) {
1052 		dev_err(thrd->dmac->pinfo->dev, "DMAC halted!\n");
1053 		return;
1054 	}
1055 
1056 	/* Get going */
1057 	writel(0, regs + DBGCMD);
1058 }
1059 
1060 /*
1061  * Mark a _pl330_req as free.
1062  * We do it by writing DMAEND as the first instruction
1063  * because no valid request is going to have DMAEND as
1064  * its first instruction to execute.
1065  */
1066 static void mark_free(struct pl330_thread *thrd, int idx)
1067 {
1068 	struct _pl330_req *req = &thrd->req[idx];
1069 
1070 	_emit_END(0, req->mc_cpu);
1071 	req->mc_len = 0;
1072 
1073 	thrd->req_running = -1;
1074 }
1075 
1076 static inline u32 _state(struct pl330_thread *thrd)
1077 {
1078 	void __iomem *regs = thrd->dmac->pinfo->base;
1079 	u32 val;
1080 
1081 	if (is_manager(thrd))
1082 		val = readl(regs + DS) & 0xf;
1083 	else
1084 		val = readl(regs + CS(thrd->id)) & 0xf;
1085 
1086 	switch (val) {
1087 	case DS_ST_STOP:
1088 		return PL330_STATE_STOPPED;
1089 	case DS_ST_EXEC:
1090 		return PL330_STATE_EXECUTING;
1091 	case DS_ST_CMISS:
1092 		return PL330_STATE_CACHEMISS;
1093 	case DS_ST_UPDTPC:
1094 		return PL330_STATE_UPDTPC;
1095 	case DS_ST_WFE:
1096 		return PL330_STATE_WFE;
1097 	case DS_ST_FAULT:
1098 		return PL330_STATE_FAULTING;
1099 	case DS_ST_ATBRR:
1100 		if (is_manager(thrd))
1101 			return PL330_STATE_INVALID;
1102 		else
1103 			return PL330_STATE_ATBARRIER;
1104 	case DS_ST_QBUSY:
1105 		if (is_manager(thrd))
1106 			return PL330_STATE_INVALID;
1107 		else
1108 			return PL330_STATE_QUEUEBUSY;
1109 	case DS_ST_WFP:
1110 		if (is_manager(thrd))
1111 			return PL330_STATE_INVALID;
1112 		else
1113 			return PL330_STATE_WFP;
1114 	case DS_ST_KILL:
1115 		if (is_manager(thrd))
1116 			return PL330_STATE_INVALID;
1117 		else
1118 			return PL330_STATE_KILLING;
1119 	case DS_ST_CMPLT:
1120 		if (is_manager(thrd))
1121 			return PL330_STATE_INVALID;
1122 		else
1123 			return PL330_STATE_COMPLETING;
1124 	case DS_ST_FLTCMP:
1125 		if (is_manager(thrd))
1126 			return PL330_STATE_INVALID;
1127 		else
1128 			return PL330_STATE_FAULT_COMPLETING;
1129 	default:
1130 		return PL330_STATE_INVALID;
1131 	}
1132 }
1133 
1134 static void _stop(struct pl330_thread *thrd)
1135 {
1136 	void __iomem *regs = thrd->dmac->pinfo->base;
1137 	u8 insn[6] = {0, 0, 0, 0, 0, 0};
1138 
1139 	if (_state(thrd) == PL330_STATE_FAULT_COMPLETING)
1140 		UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1141 
1142 	/* Return if nothing needs to be done */
1143 	if (_state(thrd) == PL330_STATE_COMPLETING
1144 		  || _state(thrd) == PL330_STATE_KILLING
1145 		  || _state(thrd) == PL330_STATE_STOPPED)
1146 		return;
1147 
1148 	_emit_KILL(0, insn);
1149 
1150 	/* Stop generating interrupts for SEV */
1151 	writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN);
1152 
1153 	_execute_DBGINSN(thrd, insn, is_manager(thrd));
1154 }
1155 
1156 /* Start doing req 'idx' of thread 'thrd' */
1157 static bool _trigger(struct pl330_thread *thrd)
1158 {
1159 	void __iomem *regs = thrd->dmac->pinfo->base;
1160 	struct _pl330_req *req;
1161 	struct pl330_req *r;
1162 	struct _arg_GO go;
1163 	unsigned ns;
1164 	u8 insn[6] = {0, 0, 0, 0, 0, 0};
1165 	int idx;
1166 
1167 	/* Return if already ACTIVE */
1168 	if (_state(thrd) != PL330_STATE_STOPPED)
1169 		return true;
1170 
1171 	idx = 1 - thrd->lstenq;
1172 	if (!IS_FREE(&thrd->req[idx]))
1173 		req = &thrd->req[idx];
1174 	else {
1175 		idx = thrd->lstenq;
1176 		if (!IS_FREE(&thrd->req[idx]))
1177 			req = &thrd->req[idx];
1178 		else
1179 			req = NULL;
1180 	}
1181 
1182 	/* Return if no request */
1183 	if (!req || !req->r)
1184 		return true;
1185 
1186 	r = req->r;
1187 
1188 	if (r->cfg)
1189 		ns = r->cfg->nonsecure ? 1 : 0;
1190 	else if (readl(regs + CS(thrd->id)) & CS_CNS)
1191 		ns = 1;
1192 	else
1193 		ns = 0;
1194 
1195 	/* See 'Abort Sources' point-4 at Page 2-25 */
1196 	if (_manager_ns(thrd) && !ns)
1197 		dev_info(thrd->dmac->pinfo->dev, "%s:%d Recipe for ABORT!\n",
1198 			__func__, __LINE__);
1199 
1200 	go.chan = thrd->id;
1201 	go.addr = req->mc_bus;
1202 	go.ns = ns;
1203 	_emit_GO(0, insn, &go);
1204 
1205 	/* Set to generate interrupts for SEV */
1206 	writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN);
1207 
1208 	/* Only manager can execute GO */
1209 	_execute_DBGINSN(thrd, insn, true);
1210 
1211 	thrd->req_running = idx;
1212 
1213 	return true;
1214 }
1215 
1216 static bool _start(struct pl330_thread *thrd)
1217 {
1218 	switch (_state(thrd)) {
1219 	case PL330_STATE_FAULT_COMPLETING:
1220 		UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING);
1221 
1222 		if (_state(thrd) == PL330_STATE_KILLING)
1223 			UNTIL(thrd, PL330_STATE_STOPPED)
1224 
1225 	case PL330_STATE_FAULTING:
1226 		_stop(thrd);
1227 
1228 	case PL330_STATE_KILLING:
1229 	case PL330_STATE_COMPLETING:
1230 		UNTIL(thrd, PL330_STATE_STOPPED)
1231 
1232 	case PL330_STATE_STOPPED:
1233 		return _trigger(thrd);
1234 
1235 	case PL330_STATE_WFP:
1236 	case PL330_STATE_QUEUEBUSY:
1237 	case PL330_STATE_ATBARRIER:
1238 	case PL330_STATE_UPDTPC:
1239 	case PL330_STATE_CACHEMISS:
1240 	case PL330_STATE_EXECUTING:
1241 		return true;
1242 
1243 	case PL330_STATE_WFE: /* For RESUME, nothing yet */
1244 	default:
1245 		return false;
1246 	}
1247 }
1248 
1249 static inline int _ldst_memtomem(unsigned dry_run, u8 buf[],
1250 		const struct _xfer_spec *pxs, int cyc)
1251 {
1252 	int off = 0;
1253 	struct pl330_config *pcfg = pxs->r->cfg->pcfg;
1254 
1255 	/* check lock-up free version */
1256 	if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) {
1257 		while (cyc--) {
1258 			off += _emit_LD(dry_run, &buf[off], ALWAYS);
1259 			off += _emit_ST(dry_run, &buf[off], ALWAYS);
1260 		}
1261 	} else {
1262 		while (cyc--) {
1263 			off += _emit_LD(dry_run, &buf[off], ALWAYS);
1264 			off += _emit_RMB(dry_run, &buf[off]);
1265 			off += _emit_ST(dry_run, &buf[off], ALWAYS);
1266 			off += _emit_WMB(dry_run, &buf[off]);
1267 		}
1268 	}
1269 
1270 	return off;
1271 }
1272 
1273 static inline int _ldst_devtomem(unsigned dry_run, u8 buf[],
1274 		const struct _xfer_spec *pxs, int cyc)
1275 {
1276 	int off = 0;
1277 
1278 	while (cyc--) {
1279 		off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1280 		off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1281 		off += _emit_ST(dry_run, &buf[off], ALWAYS);
1282 		off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri);
1283 	}
1284 
1285 	return off;
1286 }
1287 
1288 static inline int _ldst_memtodev(unsigned dry_run, u8 buf[],
1289 		const struct _xfer_spec *pxs, int cyc)
1290 {
1291 	int off = 0;
1292 
1293 	while (cyc--) {
1294 		off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1295 		off += _emit_LD(dry_run, &buf[off], ALWAYS);
1296 		off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->r->peri);
1297 		off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri);
1298 	}
1299 
1300 	return off;
1301 }
1302 
1303 static int _bursts(unsigned dry_run, u8 buf[],
1304 		const struct _xfer_spec *pxs, int cyc)
1305 {
1306 	int off = 0;
1307 
1308 	switch (pxs->r->rqtype) {
1309 	case MEMTODEV:
1310 		off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc);
1311 		break;
1312 	case DEVTOMEM:
1313 		off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc);
1314 		break;
1315 	case MEMTOMEM:
1316 		off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc);
1317 		break;
1318 	default:
1319 		off += 0x40000000; /* Scare off the Client */
1320 		break;
1321 	}
1322 
1323 	return off;
1324 }
1325 
1326 /* Returns bytes consumed and updates bursts */
1327 static inline int _loop(unsigned dry_run, u8 buf[],
1328 		unsigned long *bursts, const struct _xfer_spec *pxs)
1329 {
1330 	int cyc, cycmax, szlp, szlpend, szbrst, off;
1331 	unsigned lcnt0, lcnt1, ljmp0, ljmp1;
1332 	struct _arg_LPEND lpend;
1333 
1334 	/* Max iterations possible in DMALP is 256 */
1335 	if (*bursts >= 256*256) {
1336 		lcnt1 = 256;
1337 		lcnt0 = 256;
1338 		cyc = *bursts / lcnt1 / lcnt0;
1339 	} else if (*bursts > 256) {
1340 		lcnt1 = 256;
1341 		lcnt0 = *bursts / lcnt1;
1342 		cyc = 1;
1343 	} else {
1344 		lcnt1 = *bursts;
1345 		lcnt0 = 0;
1346 		cyc = 1;
1347 	}
1348 
1349 	szlp = _emit_LP(1, buf, 0, 0);
1350 	szbrst = _bursts(1, buf, pxs, 1);
1351 
1352 	lpend.cond = ALWAYS;
1353 	lpend.forever = false;
1354 	lpend.loop = 0;
1355 	lpend.bjump = 0;
1356 	szlpend = _emit_LPEND(1, buf, &lpend);
1357 
1358 	if (lcnt0) {
1359 		szlp *= 2;
1360 		szlpend *= 2;
1361 	}
1362 
1363 	/*
1364 	 * Max bursts that we can unroll due to limit on the
1365 	 * size of backward jump that can be encoded in DMALPEND
1366 	 * which is 8-bits and hence 255
1367 	 */
1368 	cycmax = (255 - (szlp + szlpend)) / szbrst;
1369 
1370 	cyc = (cycmax < cyc) ? cycmax : cyc;
1371 
1372 	off = 0;
1373 
1374 	if (lcnt0) {
1375 		off += _emit_LP(dry_run, &buf[off], 0, lcnt0);
1376 		ljmp0 = off;
1377 	}
1378 
1379 	off += _emit_LP(dry_run, &buf[off], 1, lcnt1);
1380 	ljmp1 = off;
1381 
1382 	off += _bursts(dry_run, &buf[off], pxs, cyc);
1383 
1384 	lpend.cond = ALWAYS;
1385 	lpend.forever = false;
1386 	lpend.loop = 1;
1387 	lpend.bjump = off - ljmp1;
1388 	off += _emit_LPEND(dry_run, &buf[off], &lpend);
1389 
1390 	if (lcnt0) {
1391 		lpend.cond = ALWAYS;
1392 		lpend.forever = false;
1393 		lpend.loop = 0;
1394 		lpend.bjump = off - ljmp0;
1395 		off += _emit_LPEND(dry_run, &buf[off], &lpend);
1396 	}
1397 
1398 	*bursts = lcnt1 * cyc;
1399 	if (lcnt0)
1400 		*bursts *= lcnt0;
1401 
1402 	return off;
1403 }
1404 
1405 static inline int _setup_loops(unsigned dry_run, u8 buf[],
1406 		const struct _xfer_spec *pxs)
1407 {
1408 	struct pl330_xfer *x = pxs->x;
1409 	u32 ccr = pxs->ccr;
1410 	unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr);
1411 	int off = 0;
1412 
1413 	while (bursts) {
1414 		c = bursts;
1415 		off += _loop(dry_run, &buf[off], &c, pxs);
1416 		bursts -= c;
1417 	}
1418 
1419 	return off;
1420 }
1421 
1422 static inline int _setup_xfer(unsigned dry_run, u8 buf[],
1423 		const struct _xfer_spec *pxs)
1424 {
1425 	struct pl330_xfer *x = pxs->x;
1426 	int off = 0;
1427 
1428 	/* DMAMOV SAR, x->src_addr */
1429 	off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr);
1430 	/* DMAMOV DAR, x->dst_addr */
1431 	off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr);
1432 
1433 	/* Setup Loop(s) */
1434 	off += _setup_loops(dry_run, &buf[off], pxs);
1435 
1436 	return off;
1437 }
1438 
1439 /*
1440  * A req is a sequence of one or more xfer units.
1441  * Returns the number of bytes taken to setup the MC for the req.
1442  */
1443 static int _setup_req(unsigned dry_run, struct pl330_thread *thrd,
1444 		unsigned index, struct _xfer_spec *pxs)
1445 {
1446 	struct _pl330_req *req = &thrd->req[index];
1447 	struct pl330_xfer *x;
1448 	u8 *buf = req->mc_cpu;
1449 	int off = 0;
1450 
1451 	PL330_DBGMC_START(req->mc_bus);
1452 
1453 	/* DMAMOV CCR, ccr */
1454 	off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr);
1455 
1456 	x = pxs->r->x;
1457 	do {
1458 		/* Error if xfer length is not aligned at burst size */
1459 		if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr)))
1460 			return -EINVAL;
1461 
1462 		pxs->x = x;
1463 		off += _setup_xfer(dry_run, &buf[off], pxs);
1464 
1465 		x = x->next;
1466 	} while (x);
1467 
1468 	/* DMASEV peripheral/event */
1469 	off += _emit_SEV(dry_run, &buf[off], thrd->ev);
1470 	/* DMAEND */
1471 	off += _emit_END(dry_run, &buf[off]);
1472 
1473 	return off;
1474 }
1475 
1476 static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc)
1477 {
1478 	u32 ccr = 0;
1479 
1480 	if (rqc->src_inc)
1481 		ccr |= CC_SRCINC;
1482 
1483 	if (rqc->dst_inc)
1484 		ccr |= CC_DSTINC;
1485 
1486 	/* We set same protection levels for Src and DST for now */
1487 	if (rqc->privileged)
1488 		ccr |= CC_SRCPRI | CC_DSTPRI;
1489 	if (rqc->nonsecure)
1490 		ccr |= CC_SRCNS | CC_DSTNS;
1491 	if (rqc->insnaccess)
1492 		ccr |= CC_SRCIA | CC_DSTIA;
1493 
1494 	ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT);
1495 	ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT);
1496 
1497 	ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT);
1498 	ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT);
1499 
1500 	ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT);
1501 	ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT);
1502 
1503 	ccr |= (rqc->swap << CC_SWAP_SHFT);
1504 
1505 	return ccr;
1506 }
1507 
1508 static inline bool _is_valid(u32 ccr)
1509 {
1510 	enum pl330_dstcachectrl dcctl;
1511 	enum pl330_srccachectrl scctl;
1512 
1513 	dcctl = (ccr >> CC_DSTCCTRL_SHFT) & CC_DRCCCTRL_MASK;
1514 	scctl = (ccr >> CC_SRCCCTRL_SHFT) & CC_SRCCCTRL_MASK;
1515 
1516 	if (dcctl == DINVALID1 || dcctl == DINVALID2
1517 			|| scctl == SINVALID1 || scctl == SINVALID2)
1518 		return false;
1519 	else
1520 		return true;
1521 }
1522 
1523 /*
1524  * Submit a list of xfers after which the client wants notification.
1525  * Client is not notified after each xfer unit, just once after all
1526  * xfer units are done or some error occurs.
1527  */
1528 static int pl330_submit_req(void *ch_id, struct pl330_req *r)
1529 {
1530 	struct pl330_thread *thrd = ch_id;
1531 	struct pl330_dmac *pl330;
1532 	struct pl330_info *pi;
1533 	struct _xfer_spec xs;
1534 	unsigned long flags;
1535 	void __iomem *regs;
1536 	unsigned idx;
1537 	u32 ccr;
1538 	int ret = 0;
1539 
1540 	/* No Req or Unacquired Channel or DMAC */
1541 	if (!r || !thrd || thrd->free)
1542 		return -EINVAL;
1543 
1544 	pl330 = thrd->dmac;
1545 	pi = pl330->pinfo;
1546 	regs = pi->base;
1547 
1548 	if (pl330->state == DYING
1549 		|| pl330->dmac_tbd.reset_chan & (1 << thrd->id)) {
1550 		dev_info(thrd->dmac->pinfo->dev, "%s:%d\n",
1551 			__func__, __LINE__);
1552 		return -EAGAIN;
1553 	}
1554 
1555 	/* If request for non-existing peripheral */
1556 	if (r->rqtype != MEMTOMEM && r->peri >= pi->pcfg.num_peri) {
1557 		dev_info(thrd->dmac->pinfo->dev,
1558 				"%s:%d Invalid peripheral(%u)!\n",
1559 				__func__, __LINE__, r->peri);
1560 		return -EINVAL;
1561 	}
1562 
1563 	spin_lock_irqsave(&pl330->lock, flags);
1564 
1565 	if (_queue_full(thrd)) {
1566 		ret = -EAGAIN;
1567 		goto xfer_exit;
1568 	}
1569 
1570 	/* Prefer Secure Channel */
1571 	if (!_manager_ns(thrd))
1572 		r->cfg->nonsecure = 0;
1573 	else
1574 		r->cfg->nonsecure = 1;
1575 
1576 	/* Use last settings, if not provided */
1577 	if (r->cfg)
1578 		ccr = _prepare_ccr(r->cfg);
1579 	else
1580 		ccr = readl(regs + CC(thrd->id));
1581 
1582 	/* If this req doesn't have valid xfer settings */
1583 	if (!_is_valid(ccr)) {
1584 		ret = -EINVAL;
1585 		dev_info(thrd->dmac->pinfo->dev, "%s:%d Invalid CCR(%x)!\n",
1586 			__func__, __LINE__, ccr);
1587 		goto xfer_exit;
1588 	}
1589 
1590 	idx = IS_FREE(&thrd->req[0]) ? 0 : 1;
1591 
1592 	xs.ccr = ccr;
1593 	xs.r = r;
1594 
1595 	/* First dry run to check if req is acceptable */
1596 	ret = _setup_req(1, thrd, idx, &xs);
1597 	if (ret < 0)
1598 		goto xfer_exit;
1599 
1600 	if (ret > pi->mcbufsz / 2) {
1601 		dev_info(thrd->dmac->pinfo->dev,
1602 			"%s:%d Trying increasing mcbufsz\n",
1603 				__func__, __LINE__);
1604 		ret = -ENOMEM;
1605 		goto xfer_exit;
1606 	}
1607 
1608 	/* Hook the request */
1609 	thrd->lstenq = idx;
1610 	thrd->req[idx].mc_len = _setup_req(0, thrd, idx, &xs);
1611 	thrd->req[idx].r = r;
1612 
1613 	ret = 0;
1614 
1615 xfer_exit:
1616 	spin_unlock_irqrestore(&pl330->lock, flags);
1617 
1618 	return ret;
1619 }
1620 
1621 static void pl330_dotask(unsigned long data)
1622 {
1623 	struct pl330_dmac *pl330 = (struct pl330_dmac *) data;
1624 	struct pl330_info *pi = pl330->pinfo;
1625 	unsigned long flags;
1626 	int i;
1627 
1628 	spin_lock_irqsave(&pl330->lock, flags);
1629 
1630 	/* The DMAC itself gone nuts */
1631 	if (pl330->dmac_tbd.reset_dmac) {
1632 		pl330->state = DYING;
1633 		/* Reset the manager too */
1634 		pl330->dmac_tbd.reset_mngr = true;
1635 		/* Clear the reset flag */
1636 		pl330->dmac_tbd.reset_dmac = false;
1637 	}
1638 
1639 	if (pl330->dmac_tbd.reset_mngr) {
1640 		_stop(pl330->manager);
1641 		/* Reset all channels */
1642 		pl330->dmac_tbd.reset_chan = (1 << pi->pcfg.num_chan) - 1;
1643 		/* Clear the reset flag */
1644 		pl330->dmac_tbd.reset_mngr = false;
1645 	}
1646 
1647 	for (i = 0; i < pi->pcfg.num_chan; i++) {
1648 
1649 		if (pl330->dmac_tbd.reset_chan & (1 << i)) {
1650 			struct pl330_thread *thrd = &pl330->channels[i];
1651 			void __iomem *regs = pi->base;
1652 			enum pl330_op_err err;
1653 
1654 			_stop(thrd);
1655 
1656 			if (readl(regs + FSC) & (1 << thrd->id))
1657 				err = PL330_ERR_FAIL;
1658 			else
1659 				err = PL330_ERR_ABORT;
1660 
1661 			spin_unlock_irqrestore(&pl330->lock, flags);
1662 
1663 			_callback(thrd->req[1 - thrd->lstenq].r, err);
1664 			_callback(thrd->req[thrd->lstenq].r, err);
1665 
1666 			spin_lock_irqsave(&pl330->lock, flags);
1667 
1668 			thrd->req[0].r = NULL;
1669 			thrd->req[1].r = NULL;
1670 			mark_free(thrd, 0);
1671 			mark_free(thrd, 1);
1672 
1673 			/* Clear the reset flag */
1674 			pl330->dmac_tbd.reset_chan &= ~(1 << i);
1675 		}
1676 	}
1677 
1678 	spin_unlock_irqrestore(&pl330->lock, flags);
1679 
1680 	return;
1681 }
1682 
1683 /* Returns 1 if state was updated, 0 otherwise */
1684 static int pl330_update(const struct pl330_info *pi)
1685 {
1686 	struct pl330_req *rqdone, *tmp;
1687 	struct pl330_dmac *pl330;
1688 	unsigned long flags;
1689 	void __iomem *regs;
1690 	u32 val;
1691 	int id, ev, ret = 0;
1692 
1693 	if (!pi || !pi->pl330_data)
1694 		return 0;
1695 
1696 	regs = pi->base;
1697 	pl330 = pi->pl330_data;
1698 
1699 	spin_lock_irqsave(&pl330->lock, flags);
1700 
1701 	val = readl(regs + FSM) & 0x1;
1702 	if (val)
1703 		pl330->dmac_tbd.reset_mngr = true;
1704 	else
1705 		pl330->dmac_tbd.reset_mngr = false;
1706 
1707 	val = readl(regs + FSC) & ((1 << pi->pcfg.num_chan) - 1);
1708 	pl330->dmac_tbd.reset_chan |= val;
1709 	if (val) {
1710 		int i = 0;
1711 		while (i < pi->pcfg.num_chan) {
1712 			if (val & (1 << i)) {
1713 				dev_info(pi->dev,
1714 					"Reset Channel-%d\t CS-%x FTC-%x\n",
1715 						i, readl(regs + CS(i)),
1716 						readl(regs + FTC(i)));
1717 				_stop(&pl330->channels[i]);
1718 			}
1719 			i++;
1720 		}
1721 	}
1722 
1723 	/* Check which event happened i.e, thread notified */
1724 	val = readl(regs + ES);
1725 	if (pi->pcfg.num_events < 32
1726 			&& val & ~((1 << pi->pcfg.num_events) - 1)) {
1727 		pl330->dmac_tbd.reset_dmac = true;
1728 		dev_err(pi->dev, "%s:%d Unexpected!\n", __func__, __LINE__);
1729 		ret = 1;
1730 		goto updt_exit;
1731 	}
1732 
1733 	for (ev = 0; ev < pi->pcfg.num_events; ev++) {
1734 		if (val & (1 << ev)) { /* Event occurred */
1735 			struct pl330_thread *thrd;
1736 			u32 inten = readl(regs + INTEN);
1737 			int active;
1738 
1739 			/* Clear the event */
1740 			if (inten & (1 << ev))
1741 				writel(1 << ev, regs + INTCLR);
1742 
1743 			ret = 1;
1744 
1745 			id = pl330->events[ev];
1746 
1747 			thrd = &pl330->channels[id];
1748 
1749 			active = thrd->req_running;
1750 			if (active == -1) /* Aborted */
1751 				continue;
1752 
1753 			/* Detach the req */
1754 			rqdone = thrd->req[active].r;
1755 			thrd->req[active].r = NULL;
1756 
1757 			mark_free(thrd, active);
1758 
1759 			/* Get going again ASAP */
1760 			_start(thrd);
1761 
1762 			/* For now, just make a list of callbacks to be done */
1763 			list_add_tail(&rqdone->rqd, &pl330->req_done);
1764 		}
1765 	}
1766 
1767 	/* Now that we are in no hurry, do the callbacks */
1768 	list_for_each_entry_safe(rqdone, tmp, &pl330->req_done, rqd) {
1769 		list_del(&rqdone->rqd);
1770 
1771 		spin_unlock_irqrestore(&pl330->lock, flags);
1772 		_callback(rqdone, PL330_ERR_NONE);
1773 		spin_lock_irqsave(&pl330->lock, flags);
1774 	}
1775 
1776 updt_exit:
1777 	spin_unlock_irqrestore(&pl330->lock, flags);
1778 
1779 	if (pl330->dmac_tbd.reset_dmac
1780 			|| pl330->dmac_tbd.reset_mngr
1781 			|| pl330->dmac_tbd.reset_chan) {
1782 		ret = 1;
1783 		tasklet_schedule(&pl330->tasks);
1784 	}
1785 
1786 	return ret;
1787 }
1788 
1789 static int pl330_chan_ctrl(void *ch_id, enum pl330_chan_op op)
1790 {
1791 	struct pl330_thread *thrd = ch_id;
1792 	struct pl330_dmac *pl330;
1793 	unsigned long flags;
1794 	int ret = 0, active;
1795 
1796 	if (!thrd || thrd->free || thrd->dmac->state == DYING)
1797 		return -EINVAL;
1798 
1799 	pl330 = thrd->dmac;
1800 	active = thrd->req_running;
1801 
1802 	spin_lock_irqsave(&pl330->lock, flags);
1803 
1804 	switch (op) {
1805 	case PL330_OP_FLUSH:
1806 		/* Make sure the channel is stopped */
1807 		_stop(thrd);
1808 
1809 		thrd->req[0].r = NULL;
1810 		thrd->req[1].r = NULL;
1811 		mark_free(thrd, 0);
1812 		mark_free(thrd, 1);
1813 		break;
1814 
1815 	case PL330_OP_ABORT:
1816 		/* Make sure the channel is stopped */
1817 		_stop(thrd);
1818 
1819 		/* ABORT is only for the active req */
1820 		if (active == -1)
1821 			break;
1822 
1823 		thrd->req[active].r = NULL;
1824 		mark_free(thrd, active);
1825 
1826 		/* Start the next */
1827 	case PL330_OP_START:
1828 		if ((active == -1) && !_start(thrd))
1829 			ret = -EIO;
1830 		break;
1831 
1832 	default:
1833 		ret = -EINVAL;
1834 	}
1835 
1836 	spin_unlock_irqrestore(&pl330->lock, flags);
1837 	return ret;
1838 }
1839 
1840 /* Reserve an event */
1841 static inline int _alloc_event(struct pl330_thread *thrd)
1842 {
1843 	struct pl330_dmac *pl330 = thrd->dmac;
1844 	struct pl330_info *pi = pl330->pinfo;
1845 	int ev;
1846 
1847 	for (ev = 0; ev < pi->pcfg.num_events; ev++)
1848 		if (pl330->events[ev] == -1) {
1849 			pl330->events[ev] = thrd->id;
1850 			return ev;
1851 		}
1852 
1853 	return -1;
1854 }
1855 
1856 static bool _chan_ns(const struct pl330_info *pi, int i)
1857 {
1858 	return pi->pcfg.irq_ns & (1 << i);
1859 }
1860 
1861 /* Upon success, returns IdentityToken for the
1862  * allocated channel, NULL otherwise.
1863  */
1864 static void *pl330_request_channel(const struct pl330_info *pi)
1865 {
1866 	struct pl330_thread *thrd = NULL;
1867 	struct pl330_dmac *pl330;
1868 	unsigned long flags;
1869 	int chans, i;
1870 
1871 	if (!pi || !pi->pl330_data)
1872 		return NULL;
1873 
1874 	pl330 = pi->pl330_data;
1875 
1876 	if (pl330->state == DYING)
1877 		return NULL;
1878 
1879 	chans = pi->pcfg.num_chan;
1880 
1881 	spin_lock_irqsave(&pl330->lock, flags);
1882 
1883 	for (i = 0; i < chans; i++) {
1884 		thrd = &pl330->channels[i];
1885 		if ((thrd->free) && (!_manager_ns(thrd) ||
1886 					_chan_ns(pi, i))) {
1887 			thrd->ev = _alloc_event(thrd);
1888 			if (thrd->ev >= 0) {
1889 				thrd->free = false;
1890 				thrd->lstenq = 1;
1891 				thrd->req[0].r = NULL;
1892 				mark_free(thrd, 0);
1893 				thrd->req[1].r = NULL;
1894 				mark_free(thrd, 1);
1895 				break;
1896 			}
1897 		}
1898 		thrd = NULL;
1899 	}
1900 
1901 	spin_unlock_irqrestore(&pl330->lock, flags);
1902 
1903 	return thrd;
1904 }
1905 
1906 /* Release an event */
1907 static inline void _free_event(struct pl330_thread *thrd, int ev)
1908 {
1909 	struct pl330_dmac *pl330 = thrd->dmac;
1910 	struct pl330_info *pi = pl330->pinfo;
1911 
1912 	/* If the event is valid and was held by the thread */
1913 	if (ev >= 0 && ev < pi->pcfg.num_events
1914 			&& pl330->events[ev] == thrd->id)
1915 		pl330->events[ev] = -1;
1916 }
1917 
1918 static void pl330_release_channel(void *ch_id)
1919 {
1920 	struct pl330_thread *thrd = ch_id;
1921 	struct pl330_dmac *pl330;
1922 	unsigned long flags;
1923 
1924 	if (!thrd || thrd->free)
1925 		return;
1926 
1927 	_stop(thrd);
1928 
1929 	_callback(thrd->req[1 - thrd->lstenq].r, PL330_ERR_ABORT);
1930 	_callback(thrd->req[thrd->lstenq].r, PL330_ERR_ABORT);
1931 
1932 	pl330 = thrd->dmac;
1933 
1934 	spin_lock_irqsave(&pl330->lock, flags);
1935 	_free_event(thrd, thrd->ev);
1936 	thrd->free = true;
1937 	spin_unlock_irqrestore(&pl330->lock, flags);
1938 }
1939 
1940 /* Initialize the structure for PL330 configuration, that can be used
1941  * by the client driver the make best use of the DMAC
1942  */
1943 static void read_dmac_config(struct pl330_info *pi)
1944 {
1945 	void __iomem *regs = pi->base;
1946 	u32 val;
1947 
1948 	val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT;
1949 	val &= CRD_DATA_WIDTH_MASK;
1950 	pi->pcfg.data_bus_width = 8 * (1 << val);
1951 
1952 	val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT;
1953 	val &= CRD_DATA_BUFF_MASK;
1954 	pi->pcfg.data_buf_dep = val + 1;
1955 
1956 	val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT;
1957 	val &= CR0_NUM_CHANS_MASK;
1958 	val += 1;
1959 	pi->pcfg.num_chan = val;
1960 
1961 	val = readl(regs + CR0);
1962 	if (val & CR0_PERIPH_REQ_SET) {
1963 		val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK;
1964 		val += 1;
1965 		pi->pcfg.num_peri = val;
1966 		pi->pcfg.peri_ns = readl(regs + CR4);
1967 	} else {
1968 		pi->pcfg.num_peri = 0;
1969 	}
1970 
1971 	val = readl(regs + CR0);
1972 	if (val & CR0_BOOT_MAN_NS)
1973 		pi->pcfg.mode |= DMAC_MODE_NS;
1974 	else
1975 		pi->pcfg.mode &= ~DMAC_MODE_NS;
1976 
1977 	val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT;
1978 	val &= CR0_NUM_EVENTS_MASK;
1979 	val += 1;
1980 	pi->pcfg.num_events = val;
1981 
1982 	pi->pcfg.irq_ns = readl(regs + CR3);
1983 
1984 	pi->pcfg.periph_id = get_id(pi, PERIPH_ID);
1985 	pi->pcfg.pcell_id = get_id(pi, PCELL_ID);
1986 }
1987 
1988 static inline void _reset_thread(struct pl330_thread *thrd)
1989 {
1990 	struct pl330_dmac *pl330 = thrd->dmac;
1991 	struct pl330_info *pi = pl330->pinfo;
1992 
1993 	thrd->req[0].mc_cpu = pl330->mcode_cpu
1994 				+ (thrd->id * pi->mcbufsz);
1995 	thrd->req[0].mc_bus = pl330->mcode_bus
1996 				+ (thrd->id * pi->mcbufsz);
1997 	thrd->req[0].r = NULL;
1998 	mark_free(thrd, 0);
1999 
2000 	thrd->req[1].mc_cpu = thrd->req[0].mc_cpu
2001 				+ pi->mcbufsz / 2;
2002 	thrd->req[1].mc_bus = thrd->req[0].mc_bus
2003 				+ pi->mcbufsz / 2;
2004 	thrd->req[1].r = NULL;
2005 	mark_free(thrd, 1);
2006 }
2007 
2008 static int dmac_alloc_threads(struct pl330_dmac *pl330)
2009 {
2010 	struct pl330_info *pi = pl330->pinfo;
2011 	int chans = pi->pcfg.num_chan;
2012 	struct pl330_thread *thrd;
2013 	int i;
2014 
2015 	/* Allocate 1 Manager and 'chans' Channel threads */
2016 	pl330->channels = kzalloc((1 + chans) * sizeof(*thrd),
2017 					GFP_KERNEL);
2018 	if (!pl330->channels)
2019 		return -ENOMEM;
2020 
2021 	/* Init Channel threads */
2022 	for (i = 0; i < chans; i++) {
2023 		thrd = &pl330->channels[i];
2024 		thrd->id = i;
2025 		thrd->dmac = pl330;
2026 		_reset_thread(thrd);
2027 		thrd->free = true;
2028 	}
2029 
2030 	/* MANAGER is indexed at the end */
2031 	thrd = &pl330->channels[chans];
2032 	thrd->id = chans;
2033 	thrd->dmac = pl330;
2034 	thrd->free = false;
2035 	pl330->manager = thrd;
2036 
2037 	return 0;
2038 }
2039 
2040 static int dmac_alloc_resources(struct pl330_dmac *pl330)
2041 {
2042 	struct pl330_info *pi = pl330->pinfo;
2043 	int chans = pi->pcfg.num_chan;
2044 	int ret;
2045 
2046 	/*
2047 	 * Alloc MicroCode buffer for 'chans' Channel threads.
2048 	 * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN)
2049 	 */
2050 	pl330->mcode_cpu = dma_alloc_coherent(pi->dev,
2051 				chans * pi->mcbufsz,
2052 				&pl330->mcode_bus, GFP_KERNEL);
2053 	if (!pl330->mcode_cpu) {
2054 		dev_err(pi->dev, "%s:%d Can't allocate memory!\n",
2055 			__func__, __LINE__);
2056 		return -ENOMEM;
2057 	}
2058 
2059 	ret = dmac_alloc_threads(pl330);
2060 	if (ret) {
2061 		dev_err(pi->dev, "%s:%d Can't to create channels for DMAC!\n",
2062 			__func__, __LINE__);
2063 		dma_free_coherent(pi->dev,
2064 				chans * pi->mcbufsz,
2065 				pl330->mcode_cpu, pl330->mcode_bus);
2066 		return ret;
2067 	}
2068 
2069 	return 0;
2070 }
2071 
2072 static int pl330_add(struct pl330_info *pi)
2073 {
2074 	struct pl330_dmac *pl330;
2075 	void __iomem *regs;
2076 	int i, ret;
2077 
2078 	if (!pi || !pi->dev)
2079 		return -EINVAL;
2080 
2081 	/* If already added */
2082 	if (pi->pl330_data)
2083 		return -EINVAL;
2084 
2085 	/*
2086 	 * If the SoC can perform reset on the DMAC, then do it
2087 	 * before reading its configuration.
2088 	 */
2089 	if (pi->dmac_reset)
2090 		pi->dmac_reset(pi);
2091 
2092 	regs = pi->base;
2093 
2094 	/* Check if we can handle this DMAC */
2095 	if ((get_id(pi, PERIPH_ID) & 0xfffff) != PERIPH_ID_VAL
2096 	   || get_id(pi, PCELL_ID) != PCELL_ID_VAL) {
2097 		dev_err(pi->dev, "PERIPH_ID 0x%x, PCELL_ID 0x%x !\n",
2098 			get_id(pi, PERIPH_ID), get_id(pi, PCELL_ID));
2099 		return -EINVAL;
2100 	}
2101 
2102 	/* Read the configuration of the DMAC */
2103 	read_dmac_config(pi);
2104 
2105 	if (pi->pcfg.num_events == 0) {
2106 		dev_err(pi->dev, "%s:%d Can't work without events!\n",
2107 			__func__, __LINE__);
2108 		return -EINVAL;
2109 	}
2110 
2111 	pl330 = kzalloc(sizeof(*pl330), GFP_KERNEL);
2112 	if (!pl330) {
2113 		dev_err(pi->dev, "%s:%d Can't allocate memory!\n",
2114 			__func__, __LINE__);
2115 		return -ENOMEM;
2116 	}
2117 
2118 	/* Assign the info structure and private data */
2119 	pl330->pinfo = pi;
2120 	pi->pl330_data = pl330;
2121 
2122 	spin_lock_init(&pl330->lock);
2123 
2124 	INIT_LIST_HEAD(&pl330->req_done);
2125 
2126 	/* Use default MC buffer size if not provided */
2127 	if (!pi->mcbufsz)
2128 		pi->mcbufsz = MCODE_BUFF_PER_REQ * 2;
2129 
2130 	/* Mark all events as free */
2131 	for (i = 0; i < pi->pcfg.num_events; i++)
2132 		pl330->events[i] = -1;
2133 
2134 	/* Allocate resources needed by the DMAC */
2135 	ret = dmac_alloc_resources(pl330);
2136 	if (ret) {
2137 		dev_err(pi->dev, "Unable to create channels for DMAC\n");
2138 		kfree(pl330);
2139 		return ret;
2140 	}
2141 
2142 	tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330);
2143 
2144 	pl330->state = INIT;
2145 
2146 	return 0;
2147 }
2148 
2149 static int dmac_free_threads(struct pl330_dmac *pl330)
2150 {
2151 	struct pl330_info *pi = pl330->pinfo;
2152 	int chans = pi->pcfg.num_chan;
2153 	struct pl330_thread *thrd;
2154 	int i;
2155 
2156 	/* Release Channel threads */
2157 	for (i = 0; i < chans; i++) {
2158 		thrd = &pl330->channels[i];
2159 		pl330_release_channel((void *)thrd);
2160 	}
2161 
2162 	/* Free memory */
2163 	kfree(pl330->channels);
2164 
2165 	return 0;
2166 }
2167 
2168 static void dmac_free_resources(struct pl330_dmac *pl330)
2169 {
2170 	struct pl330_info *pi = pl330->pinfo;
2171 	int chans = pi->pcfg.num_chan;
2172 
2173 	dmac_free_threads(pl330);
2174 
2175 	dma_free_coherent(pi->dev, chans * pi->mcbufsz,
2176 				pl330->mcode_cpu, pl330->mcode_bus);
2177 }
2178 
2179 static void pl330_del(struct pl330_info *pi)
2180 {
2181 	struct pl330_dmac *pl330;
2182 
2183 	if (!pi || !pi->pl330_data)
2184 		return;
2185 
2186 	pl330 = pi->pl330_data;
2187 
2188 	pl330->state = UNINIT;
2189 
2190 	tasklet_kill(&pl330->tasks);
2191 
2192 	/* Free DMAC resources */
2193 	dmac_free_resources(pl330);
2194 
2195 	kfree(pl330);
2196 	pi->pl330_data = NULL;
2197 }
2198 
2199 /* forward declaration */
2200 static struct amba_driver pl330_driver;
2201 
2202 static inline struct dma_pl330_chan *
2203 to_pchan(struct dma_chan *ch)
2204 {
2205 	if (!ch)
2206 		return NULL;
2207 
2208 	return container_of(ch, struct dma_pl330_chan, chan);
2209 }
2210 
2211 static inline struct dma_pl330_desc *
2212 to_desc(struct dma_async_tx_descriptor *tx)
2213 {
2214 	return container_of(tx, struct dma_pl330_desc, txd);
2215 }
2216 
2217 static inline void free_desc_list(struct list_head *list)
2218 {
2219 	struct dma_pl330_dmac *pdmac;
2220 	struct dma_pl330_desc *desc;
2221 	struct dma_pl330_chan *pch = NULL;
2222 	unsigned long flags;
2223 
2224 	/* Finish off the work list */
2225 	list_for_each_entry(desc, list, node) {
2226 		dma_async_tx_callback callback;
2227 		void *param;
2228 
2229 		/* All desc in a list belong to same channel */
2230 		pch = desc->pchan;
2231 		callback = desc->txd.callback;
2232 		param = desc->txd.callback_param;
2233 
2234 		if (callback)
2235 			callback(param);
2236 
2237 		desc->pchan = NULL;
2238 	}
2239 
2240 	/* pch will be unset if list was empty */
2241 	if (!pch)
2242 		return;
2243 
2244 	pdmac = pch->dmac;
2245 
2246 	spin_lock_irqsave(&pdmac->pool_lock, flags);
2247 	list_splice_tail_init(list, &pdmac->desc_pool);
2248 	spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2249 }
2250 
2251 static inline void handle_cyclic_desc_list(struct list_head *list)
2252 {
2253 	struct dma_pl330_desc *desc;
2254 	struct dma_pl330_chan *pch = NULL;
2255 	unsigned long flags;
2256 
2257 	list_for_each_entry(desc, list, node) {
2258 		dma_async_tx_callback callback;
2259 
2260 		/* Change status to reload it */
2261 		desc->status = PREP;
2262 		pch = desc->pchan;
2263 		callback = desc->txd.callback;
2264 		if (callback)
2265 			callback(desc->txd.callback_param);
2266 	}
2267 
2268 	/* pch will be unset if list was empty */
2269 	if (!pch)
2270 		return;
2271 
2272 	spin_lock_irqsave(&pch->lock, flags);
2273 	list_splice_tail_init(list, &pch->work_list);
2274 	spin_unlock_irqrestore(&pch->lock, flags);
2275 }
2276 
2277 static inline void fill_queue(struct dma_pl330_chan *pch)
2278 {
2279 	struct dma_pl330_desc *desc;
2280 	int ret;
2281 
2282 	list_for_each_entry(desc, &pch->work_list, node) {
2283 
2284 		/* If already submitted */
2285 		if (desc->status == BUSY)
2286 			break;
2287 
2288 		ret = pl330_submit_req(pch->pl330_chid,
2289 						&desc->req);
2290 		if (!ret) {
2291 			desc->status = BUSY;
2292 			break;
2293 		} else if (ret == -EAGAIN) {
2294 			/* QFull or DMAC Dying */
2295 			break;
2296 		} else {
2297 			/* Unacceptable request */
2298 			desc->status = DONE;
2299 			dev_err(pch->dmac->pif.dev, "%s:%d Bad Desc(%d)\n",
2300 					__func__, __LINE__, desc->txd.cookie);
2301 			tasklet_schedule(&pch->task);
2302 		}
2303 	}
2304 }
2305 
2306 static void pl330_tasklet(unsigned long data)
2307 {
2308 	struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;
2309 	struct dma_pl330_desc *desc, *_dt;
2310 	unsigned long flags;
2311 	LIST_HEAD(list);
2312 
2313 	spin_lock_irqsave(&pch->lock, flags);
2314 
2315 	/* Pick up ripe tomatoes */
2316 	list_for_each_entry_safe(desc, _dt, &pch->work_list, node)
2317 		if (desc->status == DONE) {
2318 			if (!pch->cyclic)
2319 				dma_cookie_complete(&desc->txd);
2320 			list_move_tail(&desc->node, &list);
2321 		}
2322 
2323 	/* Try to submit a req imm. next to the last completed cookie */
2324 	fill_queue(pch);
2325 
2326 	/* Make sure the PL330 Channel thread is active */
2327 	pl330_chan_ctrl(pch->pl330_chid, PL330_OP_START);
2328 
2329 	spin_unlock_irqrestore(&pch->lock, flags);
2330 
2331 	if (pch->cyclic)
2332 		handle_cyclic_desc_list(&list);
2333 	else
2334 		free_desc_list(&list);
2335 }
2336 
2337 static void dma_pl330_rqcb(void *token, enum pl330_op_err err)
2338 {
2339 	struct dma_pl330_desc *desc = token;
2340 	struct dma_pl330_chan *pch = desc->pchan;
2341 	unsigned long flags;
2342 
2343 	/* If desc aborted */
2344 	if (!pch)
2345 		return;
2346 
2347 	spin_lock_irqsave(&pch->lock, flags);
2348 
2349 	desc->status = DONE;
2350 
2351 	spin_unlock_irqrestore(&pch->lock, flags);
2352 
2353 	tasklet_schedule(&pch->task);
2354 }
2355 
2356 bool pl330_filter(struct dma_chan *chan, void *param)
2357 {
2358 	u8 *peri_id;
2359 
2360 	if (chan->device->dev->driver != &pl330_driver.drv)
2361 		return false;
2362 
2363 #ifdef CONFIG_OF
2364 	if (chan->device->dev->of_node) {
2365 		const __be32 *prop_value;
2366 		phandle phandle;
2367 		struct device_node *node;
2368 
2369 		prop_value = ((struct property *)param)->value;
2370 		phandle = be32_to_cpup(prop_value++);
2371 		node = of_find_node_by_phandle(phandle);
2372 		return ((chan->private == node) &&
2373 				(chan->chan_id == be32_to_cpup(prop_value)));
2374 	}
2375 #endif
2376 
2377 	peri_id = chan->private;
2378 	return *peri_id == (unsigned)param;
2379 }
2380 EXPORT_SYMBOL(pl330_filter);
2381 
2382 static int pl330_alloc_chan_resources(struct dma_chan *chan)
2383 {
2384 	struct dma_pl330_chan *pch = to_pchan(chan);
2385 	struct dma_pl330_dmac *pdmac = pch->dmac;
2386 	unsigned long flags;
2387 
2388 	spin_lock_irqsave(&pch->lock, flags);
2389 
2390 	dma_cookie_init(chan);
2391 	pch->cyclic = false;
2392 
2393 	pch->pl330_chid = pl330_request_channel(&pdmac->pif);
2394 	if (!pch->pl330_chid) {
2395 		spin_unlock_irqrestore(&pch->lock, flags);
2396 		return 0;
2397 	}
2398 
2399 	tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);
2400 
2401 	spin_unlock_irqrestore(&pch->lock, flags);
2402 
2403 	return 1;
2404 }
2405 
2406 static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg)
2407 {
2408 	struct dma_pl330_chan *pch = to_pchan(chan);
2409 	struct dma_pl330_desc *desc, *_dt;
2410 	unsigned long flags;
2411 	struct dma_pl330_dmac *pdmac = pch->dmac;
2412 	struct dma_slave_config *slave_config;
2413 	LIST_HEAD(list);
2414 
2415 	switch (cmd) {
2416 	case DMA_TERMINATE_ALL:
2417 		spin_lock_irqsave(&pch->lock, flags);
2418 
2419 		/* FLUSH the PL330 Channel thread */
2420 		pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH);
2421 
2422 		/* Mark all desc done */
2423 		list_for_each_entry_safe(desc, _dt, &pch->work_list , node) {
2424 			desc->status = DONE;
2425 			list_move_tail(&desc->node, &list);
2426 		}
2427 
2428 		list_splice_tail_init(&list, &pdmac->desc_pool);
2429 		spin_unlock_irqrestore(&pch->lock, flags);
2430 		break;
2431 	case DMA_SLAVE_CONFIG:
2432 		slave_config = (struct dma_slave_config *)arg;
2433 
2434 		if (slave_config->direction == DMA_MEM_TO_DEV) {
2435 			if (slave_config->dst_addr)
2436 				pch->fifo_addr = slave_config->dst_addr;
2437 			if (slave_config->dst_addr_width)
2438 				pch->burst_sz = __ffs(slave_config->dst_addr_width);
2439 			if (slave_config->dst_maxburst)
2440 				pch->burst_len = slave_config->dst_maxburst;
2441 		} else if (slave_config->direction == DMA_DEV_TO_MEM) {
2442 			if (slave_config->src_addr)
2443 				pch->fifo_addr = slave_config->src_addr;
2444 			if (slave_config->src_addr_width)
2445 				pch->burst_sz = __ffs(slave_config->src_addr_width);
2446 			if (slave_config->src_maxburst)
2447 				pch->burst_len = slave_config->src_maxburst;
2448 		}
2449 		break;
2450 	default:
2451 		dev_err(pch->dmac->pif.dev, "Not supported command.\n");
2452 		return -ENXIO;
2453 	}
2454 
2455 	return 0;
2456 }
2457 
2458 static void pl330_free_chan_resources(struct dma_chan *chan)
2459 {
2460 	struct dma_pl330_chan *pch = to_pchan(chan);
2461 	unsigned long flags;
2462 
2463 	spin_lock_irqsave(&pch->lock, flags);
2464 
2465 	tasklet_kill(&pch->task);
2466 
2467 	pl330_release_channel(pch->pl330_chid);
2468 	pch->pl330_chid = NULL;
2469 
2470 	if (pch->cyclic)
2471 		list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);
2472 
2473 	spin_unlock_irqrestore(&pch->lock, flags);
2474 }
2475 
2476 static enum dma_status
2477 pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
2478 		 struct dma_tx_state *txstate)
2479 {
2480 	return dma_cookie_status(chan, cookie, txstate);
2481 }
2482 
2483 static void pl330_issue_pending(struct dma_chan *chan)
2484 {
2485 	pl330_tasklet((unsigned long) to_pchan(chan));
2486 }
2487 
2488 /*
2489  * We returned the last one of the circular list of descriptor(s)
2490  * from prep_xxx, so the argument to submit corresponds to the last
2491  * descriptor of the list.
2492  */
2493 static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx)
2494 {
2495 	struct dma_pl330_desc *desc, *last = to_desc(tx);
2496 	struct dma_pl330_chan *pch = to_pchan(tx->chan);
2497 	dma_cookie_t cookie;
2498 	unsigned long flags;
2499 
2500 	spin_lock_irqsave(&pch->lock, flags);
2501 
2502 	/* Assign cookies to all nodes */
2503 	while (!list_empty(&last->node)) {
2504 		desc = list_entry(last->node.next, struct dma_pl330_desc, node);
2505 
2506 		dma_cookie_assign(&desc->txd);
2507 
2508 		list_move_tail(&desc->node, &pch->work_list);
2509 	}
2510 
2511 	cookie = dma_cookie_assign(&last->txd);
2512 	list_add_tail(&last->node, &pch->work_list);
2513 	spin_unlock_irqrestore(&pch->lock, flags);
2514 
2515 	return cookie;
2516 }
2517 
2518 static inline void _init_desc(struct dma_pl330_desc *desc)
2519 {
2520 	desc->pchan = NULL;
2521 	desc->req.x = &desc->px;
2522 	desc->req.token = desc;
2523 	desc->rqcfg.swap = SWAP_NO;
2524 	desc->rqcfg.privileged = 0;
2525 	desc->rqcfg.insnaccess = 0;
2526 	desc->rqcfg.scctl = SCCTRL0;
2527 	desc->rqcfg.dcctl = DCCTRL0;
2528 	desc->req.cfg = &desc->rqcfg;
2529 	desc->req.xfer_cb = dma_pl330_rqcb;
2530 	desc->txd.tx_submit = pl330_tx_submit;
2531 
2532 	INIT_LIST_HEAD(&desc->node);
2533 }
2534 
2535 /* Returns the number of descriptors added to the DMAC pool */
2536 static int add_desc(struct dma_pl330_dmac *pdmac, gfp_t flg, int count)
2537 {
2538 	struct dma_pl330_desc *desc;
2539 	unsigned long flags;
2540 	int i;
2541 
2542 	if (!pdmac)
2543 		return 0;
2544 
2545 	desc = kmalloc(count * sizeof(*desc), flg);
2546 	if (!desc)
2547 		return 0;
2548 
2549 	spin_lock_irqsave(&pdmac->pool_lock, flags);
2550 
2551 	for (i = 0; i < count; i++) {
2552 		_init_desc(&desc[i]);
2553 		list_add_tail(&desc[i].node, &pdmac->desc_pool);
2554 	}
2555 
2556 	spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2557 
2558 	return count;
2559 }
2560 
2561 static struct dma_pl330_desc *
2562 pluck_desc(struct dma_pl330_dmac *pdmac)
2563 {
2564 	struct dma_pl330_desc *desc = NULL;
2565 	unsigned long flags;
2566 
2567 	if (!pdmac)
2568 		return NULL;
2569 
2570 	spin_lock_irqsave(&pdmac->pool_lock, flags);
2571 
2572 	if (!list_empty(&pdmac->desc_pool)) {
2573 		desc = list_entry(pdmac->desc_pool.next,
2574 				struct dma_pl330_desc, node);
2575 
2576 		list_del_init(&desc->node);
2577 
2578 		desc->status = PREP;
2579 		desc->txd.callback = NULL;
2580 	}
2581 
2582 	spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2583 
2584 	return desc;
2585 }
2586 
2587 static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)
2588 {
2589 	struct dma_pl330_dmac *pdmac = pch->dmac;
2590 	u8 *peri_id = pch->chan.private;
2591 	struct dma_pl330_desc *desc;
2592 
2593 	/* Pluck one desc from the pool of DMAC */
2594 	desc = pluck_desc(pdmac);
2595 
2596 	/* If the DMAC pool is empty, alloc new */
2597 	if (!desc) {
2598 		if (!add_desc(pdmac, GFP_ATOMIC, 1))
2599 			return NULL;
2600 
2601 		/* Try again */
2602 		desc = pluck_desc(pdmac);
2603 		if (!desc) {
2604 			dev_err(pch->dmac->pif.dev,
2605 				"%s:%d ALERT!\n", __func__, __LINE__);
2606 			return NULL;
2607 		}
2608 	}
2609 
2610 	/* Initialize the descriptor */
2611 	desc->pchan = pch;
2612 	desc->txd.cookie = 0;
2613 	async_tx_ack(&desc->txd);
2614 
2615 	desc->req.peri = peri_id ? pch->chan.chan_id : 0;
2616 	desc->rqcfg.pcfg = &pch->dmac->pif.pcfg;
2617 
2618 	dma_async_tx_descriptor_init(&desc->txd, &pch->chan);
2619 
2620 	return desc;
2621 }
2622 
2623 static inline void fill_px(struct pl330_xfer *px,
2624 		dma_addr_t dst, dma_addr_t src, size_t len)
2625 {
2626 	px->next = NULL;
2627 	px->bytes = len;
2628 	px->dst_addr = dst;
2629 	px->src_addr = src;
2630 }
2631 
2632 static struct dma_pl330_desc *
2633 __pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,
2634 		dma_addr_t src, size_t len)
2635 {
2636 	struct dma_pl330_desc *desc = pl330_get_desc(pch);
2637 
2638 	if (!desc) {
2639 		dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
2640 			__func__, __LINE__);
2641 		return NULL;
2642 	}
2643 
2644 	/*
2645 	 * Ideally we should lookout for reqs bigger than
2646 	 * those that can be programmed with 256 bytes of
2647 	 * MC buffer, but considering a req size is seldom
2648 	 * going to be word-unaligned and more than 200MB,
2649 	 * we take it easy.
2650 	 * Also, should the limit is reached we'd rather
2651 	 * have the platform increase MC buffer size than
2652 	 * complicating this API driver.
2653 	 */
2654 	fill_px(&desc->px, dst, src, len);
2655 
2656 	return desc;
2657 }
2658 
2659 /* Call after fixing burst size */
2660 static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len)
2661 {
2662 	struct dma_pl330_chan *pch = desc->pchan;
2663 	struct pl330_info *pi = &pch->dmac->pif;
2664 	int burst_len;
2665 
2666 	burst_len = pi->pcfg.data_bus_width / 8;
2667 	burst_len *= pi->pcfg.data_buf_dep;
2668 	burst_len >>= desc->rqcfg.brst_size;
2669 
2670 	/* src/dst_burst_len can't be more than 16 */
2671 	if (burst_len > 16)
2672 		burst_len = 16;
2673 
2674 	while (burst_len > 1) {
2675 		if (!(len % (burst_len << desc->rqcfg.brst_size)))
2676 			break;
2677 		burst_len--;
2678 	}
2679 
2680 	return burst_len;
2681 }
2682 
2683 static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic(
2684 		struct dma_chan *chan, dma_addr_t dma_addr, size_t len,
2685 		size_t period_len, enum dma_transfer_direction direction,
2686 		void *context)
2687 {
2688 	struct dma_pl330_desc *desc;
2689 	struct dma_pl330_chan *pch = to_pchan(chan);
2690 	dma_addr_t dst;
2691 	dma_addr_t src;
2692 
2693 	desc = pl330_get_desc(pch);
2694 	if (!desc) {
2695 		dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n",
2696 			__func__, __LINE__);
2697 		return NULL;
2698 	}
2699 
2700 	switch (direction) {
2701 	case DMA_MEM_TO_DEV:
2702 		desc->rqcfg.src_inc = 1;
2703 		desc->rqcfg.dst_inc = 0;
2704 		desc->req.rqtype = MEMTODEV;
2705 		src = dma_addr;
2706 		dst = pch->fifo_addr;
2707 		break;
2708 	case DMA_DEV_TO_MEM:
2709 		desc->rqcfg.src_inc = 0;
2710 		desc->rqcfg.dst_inc = 1;
2711 		desc->req.rqtype = DEVTOMEM;
2712 		src = pch->fifo_addr;
2713 		dst = dma_addr;
2714 		break;
2715 	default:
2716 		dev_err(pch->dmac->pif.dev, "%s:%d Invalid dma direction\n",
2717 		__func__, __LINE__);
2718 		return NULL;
2719 	}
2720 
2721 	desc->rqcfg.brst_size = pch->burst_sz;
2722 	desc->rqcfg.brst_len = 1;
2723 
2724 	pch->cyclic = true;
2725 
2726 	fill_px(&desc->px, dst, src, period_len);
2727 
2728 	return &desc->txd;
2729 }
2730 
2731 static struct dma_async_tx_descriptor *
2732 pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst,
2733 		dma_addr_t src, size_t len, unsigned long flags)
2734 {
2735 	struct dma_pl330_desc *desc;
2736 	struct dma_pl330_chan *pch = to_pchan(chan);
2737 	struct pl330_info *pi;
2738 	int burst;
2739 
2740 	if (unlikely(!pch || !len))
2741 		return NULL;
2742 
2743 	pi = &pch->dmac->pif;
2744 
2745 	desc = __pl330_prep_dma_memcpy(pch, dst, src, len);
2746 	if (!desc)
2747 		return NULL;
2748 
2749 	desc->rqcfg.src_inc = 1;
2750 	desc->rqcfg.dst_inc = 1;
2751 	desc->req.rqtype = MEMTOMEM;
2752 
2753 	/* Select max possible burst size */
2754 	burst = pi->pcfg.data_bus_width / 8;
2755 
2756 	while (burst > 1) {
2757 		if (!(len % burst))
2758 			break;
2759 		burst /= 2;
2760 	}
2761 
2762 	desc->rqcfg.brst_size = 0;
2763 	while (burst != (1 << desc->rqcfg.brst_size))
2764 		desc->rqcfg.brst_size++;
2765 
2766 	desc->rqcfg.brst_len = get_burst_len(desc, len);
2767 
2768 	desc->txd.flags = flags;
2769 
2770 	return &desc->txd;
2771 }
2772 
2773 static struct dma_async_tx_descriptor *
2774 pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
2775 		unsigned int sg_len, enum dma_transfer_direction direction,
2776 		unsigned long flg, void *context)
2777 {
2778 	struct dma_pl330_desc *first, *desc = NULL;
2779 	struct dma_pl330_chan *pch = to_pchan(chan);
2780 	struct scatterlist *sg;
2781 	unsigned long flags;
2782 	int i;
2783 	dma_addr_t addr;
2784 
2785 	if (unlikely(!pch || !sgl || !sg_len))
2786 		return NULL;
2787 
2788 	addr = pch->fifo_addr;
2789 
2790 	first = NULL;
2791 
2792 	for_each_sg(sgl, sg, sg_len, i) {
2793 
2794 		desc = pl330_get_desc(pch);
2795 		if (!desc) {
2796 			struct dma_pl330_dmac *pdmac = pch->dmac;
2797 
2798 			dev_err(pch->dmac->pif.dev,
2799 				"%s:%d Unable to fetch desc\n",
2800 				__func__, __LINE__);
2801 			if (!first)
2802 				return NULL;
2803 
2804 			spin_lock_irqsave(&pdmac->pool_lock, flags);
2805 
2806 			while (!list_empty(&first->node)) {
2807 				desc = list_entry(first->node.next,
2808 						struct dma_pl330_desc, node);
2809 				list_move_tail(&desc->node, &pdmac->desc_pool);
2810 			}
2811 
2812 			list_move_tail(&first->node, &pdmac->desc_pool);
2813 
2814 			spin_unlock_irqrestore(&pdmac->pool_lock, flags);
2815 
2816 			return NULL;
2817 		}
2818 
2819 		if (!first)
2820 			first = desc;
2821 		else
2822 			list_add_tail(&desc->node, &first->node);
2823 
2824 		if (direction == DMA_MEM_TO_DEV) {
2825 			desc->rqcfg.src_inc = 1;
2826 			desc->rqcfg.dst_inc = 0;
2827 			desc->req.rqtype = MEMTODEV;
2828 			fill_px(&desc->px,
2829 				addr, sg_dma_address(sg), sg_dma_len(sg));
2830 		} else {
2831 			desc->rqcfg.src_inc = 0;
2832 			desc->rqcfg.dst_inc = 1;
2833 			desc->req.rqtype = DEVTOMEM;
2834 			fill_px(&desc->px,
2835 				sg_dma_address(sg), addr, sg_dma_len(sg));
2836 		}
2837 
2838 		desc->rqcfg.brst_size = pch->burst_sz;
2839 		desc->rqcfg.brst_len = 1;
2840 	}
2841 
2842 	/* Return the last desc in the chain */
2843 	desc->txd.flags = flg;
2844 	return &desc->txd;
2845 }
2846 
2847 static irqreturn_t pl330_irq_handler(int irq, void *data)
2848 {
2849 	if (pl330_update(data))
2850 		return IRQ_HANDLED;
2851 	else
2852 		return IRQ_NONE;
2853 }
2854 
2855 static int __devinit
2856 pl330_probe(struct amba_device *adev, const struct amba_id *id)
2857 {
2858 	struct dma_pl330_platdata *pdat;
2859 	struct dma_pl330_dmac *pdmac;
2860 	struct dma_pl330_chan *pch;
2861 	struct pl330_info *pi;
2862 	struct dma_device *pd;
2863 	struct resource *res;
2864 	int i, ret, irq;
2865 	int num_chan;
2866 
2867 	pdat = adev->dev.platform_data;
2868 
2869 	/* Allocate a new DMAC and its Channels */
2870 	pdmac = kzalloc(sizeof(*pdmac), GFP_KERNEL);
2871 	if (!pdmac) {
2872 		dev_err(&adev->dev, "unable to allocate mem\n");
2873 		return -ENOMEM;
2874 	}
2875 
2876 	pi = &pdmac->pif;
2877 	pi->dev = &adev->dev;
2878 	pi->pl330_data = NULL;
2879 	pi->mcbufsz = pdat ? pdat->mcbuf_sz : 0;
2880 
2881 	res = &adev->res;
2882 	request_mem_region(res->start, resource_size(res), "dma-pl330");
2883 
2884 	pi->base = ioremap(res->start, resource_size(res));
2885 	if (!pi->base) {
2886 		ret = -ENXIO;
2887 		goto probe_err1;
2888 	}
2889 
2890 	pdmac->clk = clk_get(&adev->dev, "dma");
2891 	if (IS_ERR(pdmac->clk)) {
2892 		dev_err(&adev->dev, "Cannot get operation clock.\n");
2893 		ret = -EINVAL;
2894 		goto probe_err2;
2895 	}
2896 
2897 	amba_set_drvdata(adev, pdmac);
2898 
2899 #ifndef CONFIG_PM_RUNTIME
2900 	/* enable dma clk */
2901 	clk_enable(pdmac->clk);
2902 #endif
2903 
2904 	irq = adev->irq[0];
2905 	ret = request_irq(irq, pl330_irq_handler, 0,
2906 			dev_name(&adev->dev), pi);
2907 	if (ret)
2908 		goto probe_err3;
2909 
2910 	ret = pl330_add(pi);
2911 	if (ret)
2912 		goto probe_err4;
2913 
2914 	INIT_LIST_HEAD(&pdmac->desc_pool);
2915 	spin_lock_init(&pdmac->pool_lock);
2916 
2917 	/* Create a descriptor pool of default size */
2918 	if (!add_desc(pdmac, GFP_KERNEL, NR_DEFAULT_DESC))
2919 		dev_warn(&adev->dev, "unable to allocate desc\n");
2920 
2921 	pd = &pdmac->ddma;
2922 	INIT_LIST_HEAD(&pd->channels);
2923 
2924 	/* Initialize channel parameters */
2925 	if (pdat)
2926 		num_chan = max_t(int, pdat->nr_valid_peri, pi->pcfg.num_chan);
2927 	else
2928 		num_chan = max_t(int, pi->pcfg.num_peri, pi->pcfg.num_chan);
2929 
2930 	pdmac->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL);
2931 
2932 	for (i = 0; i < num_chan; i++) {
2933 		pch = &pdmac->peripherals[i];
2934 		if (!adev->dev.of_node)
2935 			pch->chan.private = pdat ? &pdat->peri_id[i] : NULL;
2936 		else
2937 			pch->chan.private = adev->dev.of_node;
2938 
2939 		INIT_LIST_HEAD(&pch->work_list);
2940 		spin_lock_init(&pch->lock);
2941 		pch->pl330_chid = NULL;
2942 		pch->chan.device = pd;
2943 		pch->dmac = pdmac;
2944 
2945 		/* Add the channel to the DMAC list */
2946 		list_add_tail(&pch->chan.device_node, &pd->channels);
2947 	}
2948 
2949 	pd->dev = &adev->dev;
2950 	if (pdat) {
2951 		pd->cap_mask = pdat->cap_mask;
2952 	} else {
2953 		dma_cap_set(DMA_MEMCPY, pd->cap_mask);
2954 		if (pi->pcfg.num_peri) {
2955 			dma_cap_set(DMA_SLAVE, pd->cap_mask);
2956 			dma_cap_set(DMA_CYCLIC, pd->cap_mask);
2957 		}
2958 	}
2959 
2960 	pd->device_alloc_chan_resources = pl330_alloc_chan_resources;
2961 	pd->device_free_chan_resources = pl330_free_chan_resources;
2962 	pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy;
2963 	pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic;
2964 	pd->device_tx_status = pl330_tx_status;
2965 	pd->device_prep_slave_sg = pl330_prep_slave_sg;
2966 	pd->device_control = pl330_control;
2967 	pd->device_issue_pending = pl330_issue_pending;
2968 
2969 	ret = dma_async_device_register(pd);
2970 	if (ret) {
2971 		dev_err(&adev->dev, "unable to register DMAC\n");
2972 		goto probe_err5;
2973 	}
2974 
2975 	dev_info(&adev->dev,
2976 		"Loaded driver for PL330 DMAC-%d\n", adev->periphid);
2977 	dev_info(&adev->dev,
2978 		"\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n",
2979 		pi->pcfg.data_buf_dep,
2980 		pi->pcfg.data_bus_width / 8, pi->pcfg.num_chan,
2981 		pi->pcfg.num_peri, pi->pcfg.num_events);
2982 
2983 	return 0;
2984 
2985 probe_err5:
2986 	pl330_del(pi);
2987 probe_err4:
2988 	free_irq(irq, pi);
2989 probe_err3:
2990 #ifndef CONFIG_PM_RUNTIME
2991 	clk_disable(pdmac->clk);
2992 #endif
2993 	clk_put(pdmac->clk);
2994 probe_err2:
2995 	iounmap(pi->base);
2996 probe_err1:
2997 	release_mem_region(res->start, resource_size(res));
2998 	kfree(pdmac);
2999 
3000 	return ret;
3001 }
3002 
3003 static int __devexit pl330_remove(struct amba_device *adev)
3004 {
3005 	struct dma_pl330_dmac *pdmac = amba_get_drvdata(adev);
3006 	struct dma_pl330_chan *pch, *_p;
3007 	struct pl330_info *pi;
3008 	struct resource *res;
3009 	int irq;
3010 
3011 	if (!pdmac)
3012 		return 0;
3013 
3014 	amba_set_drvdata(adev, NULL);
3015 
3016 	/* Idle the DMAC */
3017 	list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels,
3018 			chan.device_node) {
3019 
3020 		/* Remove the channel */
3021 		list_del(&pch->chan.device_node);
3022 
3023 		/* Flush the channel */
3024 		pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0);
3025 		pl330_free_chan_resources(&pch->chan);
3026 	}
3027 
3028 	pi = &pdmac->pif;
3029 
3030 	pl330_del(pi);
3031 
3032 	irq = adev->irq[0];
3033 	free_irq(irq, pi);
3034 
3035 	iounmap(pi->base);
3036 
3037 	res = &adev->res;
3038 	release_mem_region(res->start, resource_size(res));
3039 
3040 #ifndef CONFIG_PM_RUNTIME
3041 	clk_disable(pdmac->clk);
3042 #endif
3043 
3044 	kfree(pdmac);
3045 
3046 	return 0;
3047 }
3048 
3049 static struct amba_id pl330_ids[] = {
3050 	{
3051 		.id	= 0x00041330,
3052 		.mask	= 0x000fffff,
3053 	},
3054 	{ 0, 0 },
3055 };
3056 
3057 MODULE_DEVICE_TABLE(amba, pl330_ids);
3058 
3059 #ifdef CONFIG_PM_RUNTIME
3060 static int pl330_runtime_suspend(struct device *dev)
3061 {
3062 	struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev);
3063 
3064 	if (!pdmac) {
3065 		dev_err(dev, "failed to get dmac\n");
3066 		return -ENODEV;
3067 	}
3068 
3069 	clk_disable(pdmac->clk);
3070 
3071 	return 0;
3072 }
3073 
3074 static int pl330_runtime_resume(struct device *dev)
3075 {
3076 	struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev);
3077 
3078 	if (!pdmac) {
3079 		dev_err(dev, "failed to get dmac\n");
3080 		return -ENODEV;
3081 	}
3082 
3083 	clk_enable(pdmac->clk);
3084 
3085 	return 0;
3086 }
3087 #else
3088 #define pl330_runtime_suspend	NULL
3089 #define pl330_runtime_resume	NULL
3090 #endif /* CONFIG_PM_RUNTIME */
3091 
3092 static const struct dev_pm_ops pl330_pm_ops = {
3093 	.runtime_suspend = pl330_runtime_suspend,
3094 	.runtime_resume = pl330_runtime_resume,
3095 };
3096 
3097 static struct amba_driver pl330_driver = {
3098 	.drv = {
3099 		.owner = THIS_MODULE,
3100 		.name = "dma-pl330",
3101 		.pm = &pl330_pm_ops,
3102 	},
3103 	.id_table = pl330_ids,
3104 	.probe = pl330_probe,
3105 	.remove = pl330_remove,
3106 };
3107 
3108 module_amba_driver(pl330_driver);
3109 
3110 MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>");
3111 MODULE_DESCRIPTION("API Driver for PL330 DMAC");
3112 MODULE_LICENSE("GPL");
3113