xref: /linux/drivers/dma/qcom/hidma_ll.c (revision fa84cf094ef9667e2b91c104b0a788fd1896f482)
1 /*
2  * Qualcomm Technologies HIDMA DMA engine low level code
3  *
4  * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 and
8  * only version 2 as published by the Free Software Foundation.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  */
15 
16 #include <linux/dmaengine.h>
17 #include <linux/slab.h>
18 #include <linux/interrupt.h>
19 #include <linux/mm.h>
20 #include <linux/highmem.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/delay.h>
23 #include <linux/atomic.h>
24 #include <linux/iopoll.h>
25 #include <linux/kfifo.h>
26 #include <linux/bitops.h>
27 
28 #include "hidma.h"
29 
30 #define HIDMA_EVRE_SIZE			16	/* each EVRE is 16 bytes */
31 
32 #define HIDMA_TRCA_CTRLSTS_REG			0x000
33 #define HIDMA_TRCA_RING_LOW_REG		0x008
34 #define HIDMA_TRCA_RING_HIGH_REG		0x00C
35 #define HIDMA_TRCA_RING_LEN_REG		0x010
36 #define HIDMA_TRCA_DOORBELL_REG		0x400
37 
38 #define HIDMA_EVCA_CTRLSTS_REG			0x000
39 #define HIDMA_EVCA_INTCTRL_REG			0x004
40 #define HIDMA_EVCA_RING_LOW_REG		0x008
41 #define HIDMA_EVCA_RING_HIGH_REG		0x00C
42 #define HIDMA_EVCA_RING_LEN_REG		0x010
43 #define HIDMA_EVCA_WRITE_PTR_REG		0x020
44 #define HIDMA_EVCA_DOORBELL_REG		0x400
45 
46 #define HIDMA_EVCA_IRQ_STAT_REG		0x100
47 #define HIDMA_EVCA_IRQ_CLR_REG			0x108
48 #define HIDMA_EVCA_IRQ_EN_REG			0x110
49 
50 #define HIDMA_EVRE_CFG_IDX			0
51 
52 #define HIDMA_EVRE_ERRINFO_BIT_POS		24
53 #define HIDMA_EVRE_CODE_BIT_POS		28
54 
55 #define HIDMA_EVRE_ERRINFO_MASK		GENMASK(3, 0)
56 #define HIDMA_EVRE_CODE_MASK			GENMASK(3, 0)
57 
58 #define HIDMA_CH_CONTROL_MASK			GENMASK(7, 0)
59 #define HIDMA_CH_STATE_MASK			GENMASK(7, 0)
60 #define HIDMA_CH_STATE_BIT_POS			0x8
61 
62 #define HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS	0
63 #define HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS	1
64 #define HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS	9
65 #define HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS	10
66 #define HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS	11
67 #define HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS	14
68 
69 #define ENABLE_IRQS (BIT(HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS)	| \
70 		     BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS)	| \
71 		     BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS)	| \
72 		     BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS)	| \
73 		     BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS)	| \
74 		     BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS))
75 
76 #define HIDMA_INCREMENT_ITERATOR(iter, size, ring_size)	\
77 do {								\
78 	iter += size;						\
79 	if (iter >= ring_size)					\
80 		iter -= ring_size;				\
81 } while (0)
82 
83 #define HIDMA_CH_STATE(val)	\
84 	((val >> HIDMA_CH_STATE_BIT_POS) & HIDMA_CH_STATE_MASK)
85 
86 #define HIDMA_ERR_INT_MASK				\
87 	(BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS)   |	\
88 	 BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) |	\
89 	 BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS)	    |	\
90 	 BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS)    |	\
91 	 BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS))
92 
93 enum ch_command {
94 	HIDMA_CH_DISABLE = 0,
95 	HIDMA_CH_ENABLE = 1,
96 	HIDMA_CH_SUSPEND = 2,
97 	HIDMA_CH_RESET = 9,
98 };
99 
100 enum ch_state {
101 	HIDMA_CH_DISABLED = 0,
102 	HIDMA_CH_ENABLED = 1,
103 	HIDMA_CH_RUNNING = 2,
104 	HIDMA_CH_SUSPENDED = 3,
105 	HIDMA_CH_STOPPED = 4,
106 };
107 
108 enum err_code {
109 	HIDMA_EVRE_STATUS_COMPLETE = 1,
110 	HIDMA_EVRE_STATUS_ERROR = 4,
111 };
112 
113 static int hidma_is_chan_enabled(int state)
114 {
115 	switch (state) {
116 	case HIDMA_CH_ENABLED:
117 	case HIDMA_CH_RUNNING:
118 		return true;
119 	default:
120 		return false;
121 	}
122 }
123 
124 void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch)
125 {
126 	struct hidma_tre *tre;
127 
128 	if (tre_ch >= lldev->nr_tres) {
129 		dev_err(lldev->dev, "invalid TRE number in free:%d", tre_ch);
130 		return;
131 	}
132 
133 	tre = &lldev->trepool[tre_ch];
134 	if (atomic_read(&tre->allocated) != true) {
135 		dev_err(lldev->dev, "trying to free an unused TRE:%d", tre_ch);
136 		return;
137 	}
138 
139 	atomic_set(&tre->allocated, 0);
140 }
141 
142 int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name,
143 		     void (*callback)(void *data), void *data, u32 *tre_ch)
144 {
145 	unsigned int i;
146 	struct hidma_tre *tre;
147 	u32 *tre_local;
148 
149 	if (!tre_ch || !lldev)
150 		return -EINVAL;
151 
152 	/* need to have at least one empty spot in the queue */
153 	for (i = 0; i < lldev->nr_tres - 1; i++) {
154 		if (atomic_add_unless(&lldev->trepool[i].allocated, 1, 1))
155 			break;
156 	}
157 
158 	if (i == (lldev->nr_tres - 1))
159 		return -ENOMEM;
160 
161 	tre = &lldev->trepool[i];
162 	tre->dma_sig = sig;
163 	tre->dev_name = dev_name;
164 	tre->callback = callback;
165 	tre->data = data;
166 	tre->idx = i;
167 	tre->status = 0;
168 	tre->queued = 0;
169 	tre->err_code = 0;
170 	tre->err_info = 0;
171 	tre->lldev = lldev;
172 	tre_local = &tre->tre_local[0];
173 	tre_local[HIDMA_TRE_CFG_IDX] = (lldev->chidx & 0xFF) << 8;
174 	tre_local[HIDMA_TRE_CFG_IDX] |= BIT(16);	/* set IEOB */
175 	*tre_ch = i;
176 	if (callback)
177 		callback(data);
178 	return 0;
179 }
180 
181 /*
182  * Multiple TREs may be queued and waiting in the pending queue.
183  */
184 static void hidma_ll_tre_complete(unsigned long arg)
185 {
186 	struct hidma_lldev *lldev = (struct hidma_lldev *)arg;
187 	struct hidma_tre *tre;
188 
189 	while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) {
190 		/* call the user if it has been read by the hardware */
191 		if (tre->callback)
192 			tre->callback(tre->data);
193 	}
194 }
195 
196 static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info,
197 				u8 err_code)
198 {
199 	struct hidma_tre *tre;
200 	unsigned long flags;
201 	u32 tre_iterator;
202 
203 	spin_lock_irqsave(&lldev->lock, flags);
204 
205 	tre_iterator = lldev->tre_processed_off;
206 	tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE];
207 	if (!tre) {
208 		spin_unlock_irqrestore(&lldev->lock, flags);
209 		dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n",
210 			 tre_iterator / HIDMA_TRE_SIZE);
211 		return -EINVAL;
212 	}
213 	lldev->pending_tre_list[tre->tre_index] = NULL;
214 
215 	/*
216 	 * Keep track of pending TREs that SW is expecting to receive
217 	 * from HW. We got one now. Decrement our counter.
218 	 */
219 	if (atomic_dec_return(&lldev->pending_tre_count) < 0) {
220 		dev_warn(lldev->dev, "tre count mismatch on completion");
221 		atomic_set(&lldev->pending_tre_count, 0);
222 	}
223 
224 	HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE,
225 				 lldev->tre_ring_size);
226 	lldev->tre_processed_off = tre_iterator;
227 	spin_unlock_irqrestore(&lldev->lock, flags);
228 
229 	tre->err_info = err_info;
230 	tre->err_code = err_code;
231 	tre->queued = 0;
232 
233 	kfifo_put(&lldev->handoff_fifo, tre);
234 	tasklet_schedule(&lldev->task);
235 
236 	return 0;
237 }
238 
239 /*
240  * Called to handle the interrupt for the channel.
241  * Return a positive number if TRE or EVRE were consumed on this run.
242  * Return a positive number if there are pending TREs or EVREs.
243  * Return 0 if there is nothing to consume or no pending TREs/EVREs found.
244  */
245 static int hidma_handle_tre_completion(struct hidma_lldev *lldev)
246 {
247 	u32 evre_ring_size = lldev->evre_ring_size;
248 	u32 err_info, err_code, evre_write_off;
249 	u32 evre_iterator;
250 	u32 num_completed = 0;
251 
252 	evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);
253 	evre_iterator = lldev->evre_processed_off;
254 
255 	if ((evre_write_off > evre_ring_size) ||
256 	    (evre_write_off % HIDMA_EVRE_SIZE)) {
257 		dev_err(lldev->dev, "HW reports invalid EVRE write offset\n");
258 		return 0;
259 	}
260 
261 	/*
262 	 * By the time control reaches here the number of EVREs and TREs
263 	 * may not match. Only consume the ones that hardware told us.
264 	 */
265 	while ((evre_iterator != evre_write_off)) {
266 		u32 *current_evre = lldev->evre_ring + evre_iterator;
267 		u32 cfg;
268 
269 		cfg = current_evre[HIDMA_EVRE_CFG_IDX];
270 		err_info = cfg >> HIDMA_EVRE_ERRINFO_BIT_POS;
271 		err_info &= HIDMA_EVRE_ERRINFO_MASK;
272 		err_code =
273 		    (cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK;
274 
275 		if (hidma_post_completed(lldev, err_info, err_code))
276 			break;
277 
278 		HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE,
279 					 evre_ring_size);
280 
281 		/*
282 		 * Read the new event descriptor written by the HW.
283 		 * As we are processing the delivered events, other events
284 		 * get queued to the SW for processing.
285 		 */
286 		evre_write_off =
287 		    readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);
288 		num_completed++;
289 
290 		/*
291 		 * An error interrupt might have arrived while we are processing
292 		 * the completed interrupt.
293 		 */
294 		if (!hidma_ll_isenabled(lldev))
295 			break;
296 	}
297 
298 	if (num_completed) {
299 		u32 evre_read_off = (lldev->evre_processed_off +
300 				     HIDMA_EVRE_SIZE * num_completed);
301 		evre_read_off = evre_read_off % evre_ring_size;
302 		writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG);
303 
304 		/* record the last processed tre offset */
305 		lldev->evre_processed_off = evre_read_off;
306 	}
307 
308 	return num_completed;
309 }
310 
311 void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info,
312 			       u8 err_code)
313 {
314 	while (atomic_read(&lldev->pending_tre_count)) {
315 		if (hidma_post_completed(lldev, err_info, err_code))
316 			break;
317 	}
318 }
319 
320 static int hidma_ll_reset(struct hidma_lldev *lldev)
321 {
322 	u32 val;
323 	int ret;
324 
325 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
326 	val &= ~(HIDMA_CH_CONTROL_MASK << 16);
327 	val |= HIDMA_CH_RESET << 16;
328 	writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
329 
330 	/*
331 	 * Delay 10ms after reset to allow DMA logic to quiesce.
332 	 * Do a polled read up to 1ms and 10ms maximum.
333 	 */
334 	ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
335 				 HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED,
336 				 1000, 10000);
337 	if (ret) {
338 		dev_err(lldev->dev, "transfer channel did not reset\n");
339 		return ret;
340 	}
341 
342 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
343 	val &= ~(HIDMA_CH_CONTROL_MASK << 16);
344 	val |= HIDMA_CH_RESET << 16;
345 	writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
346 
347 	/*
348 	 * Delay 10ms after reset to allow DMA logic to quiesce.
349 	 * Do a polled read up to 1ms and 10ms maximum.
350 	 */
351 	ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
352 				 HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED,
353 				 1000, 10000);
354 	if (ret)
355 		return ret;
356 
357 	lldev->trch_state = HIDMA_CH_DISABLED;
358 	lldev->evch_state = HIDMA_CH_DISABLED;
359 	return 0;
360 }
361 
362 /*
363  * The interrupt handler for HIDMA will try to consume as many pending
364  * EVRE from the event queue as possible. Each EVRE has an associated
365  * TRE that holds the user interface parameters. EVRE reports the
366  * result of the transaction. Hardware guarantees ordering between EVREs
367  * and TREs. We use last processed offset to figure out which TRE is
368  * associated with which EVRE. If two TREs are consumed by HW, the EVREs
369  * are in order in the event ring.
370  *
371  * This handler will do a one pass for consuming EVREs. Other EVREs may
372  * be delivered while we are working. It will try to consume incoming
373  * EVREs one more time and return.
374  *
375  * For unprocessed EVREs, hardware will trigger another interrupt until
376  * all the interrupt bits are cleared.
377  *
378  * Hardware guarantees that by the time interrupt is observed, all data
379  * transactions in flight are delivered to their respective places and
380  * are visible to the CPU.
381  *
382  * On demand paging for IOMMU is only supported for PCIe via PRI
383  * (Page Request Interface) not for HIDMA. All other hardware instances
384  * including HIDMA work on pinned DMA addresses.
385  *
386  * HIDMA is not aware of IOMMU presence since it follows the DMA API. All
387  * IOMMU latency will be built into the data movement time. By the time
388  * interrupt happens, IOMMU lookups + data movement has already taken place.
389  *
390  * While the first read in a typical PCI endpoint ISR flushes all outstanding
391  * requests traditionally to the destination, this concept does not apply
392  * here for this HW.
393  */
394 static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause)
395 {
396 	unsigned long irqflags;
397 
398 	if (cause & HIDMA_ERR_INT_MASK) {
399 		dev_err(lldev->dev, "error 0x%x, disabling...\n",
400 				cause);
401 
402 		/* Clear out pending interrupts */
403 		writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
404 
405 		/* No further submissions. */
406 		hidma_ll_disable(lldev);
407 
408 		/* Driver completes the txn and intimates the client.*/
409 		hidma_cleanup_pending_tre(lldev, 0xFF,
410 					  HIDMA_EVRE_STATUS_ERROR);
411 
412 		return;
413 	}
414 
415 	spin_lock_irqsave(&lldev->lock, irqflags);
416 	writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
417 	spin_unlock_irqrestore(&lldev->lock, irqflags);
418 
419 	/*
420 	 * Fine tuned for this HW...
421 	 *
422 	 * This ISR has been designed for this particular hardware. Relaxed
423 	 * read and write accessors are used for performance reasons due to
424 	 * interrupt delivery guarantees. Do not copy this code blindly and
425 	 * expect that to work.
426 	 *
427 	 * Try to consume as many EVREs as possible.
428 	 */
429 	hidma_handle_tre_completion(lldev);
430 }
431 
432 irqreturn_t hidma_ll_inthandler(int chirq, void *arg)
433 {
434 	struct hidma_lldev *lldev = arg;
435 	u32 status;
436 	u32 enable;
437 	u32 cause;
438 
439 	status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
440 	enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
441 	cause = status & enable;
442 
443 	while (cause) {
444 		hidma_ll_int_handler_internal(lldev, cause);
445 
446 		/*
447 		 * Another interrupt might have arrived while we are
448 		 * processing this one. Read the new cause.
449 		 */
450 		status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
451 		enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
452 		cause = status & enable;
453 	}
454 
455 	return IRQ_HANDLED;
456 }
457 
458 irqreturn_t hidma_ll_inthandler_msi(int chirq, void *arg, int cause)
459 {
460 	struct hidma_lldev *lldev = arg;
461 
462 	hidma_ll_int_handler_internal(lldev, cause);
463 	return IRQ_HANDLED;
464 }
465 
466 int hidma_ll_enable(struct hidma_lldev *lldev)
467 {
468 	u32 val;
469 	int ret;
470 
471 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
472 	val &= ~(HIDMA_CH_CONTROL_MASK << 16);
473 	val |= HIDMA_CH_ENABLE << 16;
474 	writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
475 
476 	ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
477 				 hidma_is_chan_enabled(HIDMA_CH_STATE(val)),
478 				 1000, 10000);
479 	if (ret) {
480 		dev_err(lldev->dev, "event channel did not get enabled\n");
481 		return ret;
482 	}
483 
484 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
485 	val &= ~(HIDMA_CH_CONTROL_MASK << 16);
486 	val |= HIDMA_CH_ENABLE << 16;
487 	writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
488 
489 	ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
490 				 hidma_is_chan_enabled(HIDMA_CH_STATE(val)),
491 				 1000, 10000);
492 	if (ret) {
493 		dev_err(lldev->dev, "transfer channel did not get enabled\n");
494 		return ret;
495 	}
496 
497 	lldev->trch_state = HIDMA_CH_ENABLED;
498 	lldev->evch_state = HIDMA_CH_ENABLED;
499 
500 	/* enable irqs */
501 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
502 
503 	return 0;
504 }
505 
506 void hidma_ll_start(struct hidma_lldev *lldev)
507 {
508 	unsigned long irqflags;
509 
510 	spin_lock_irqsave(&lldev->lock, irqflags);
511 	writel(lldev->tre_write_offset, lldev->trca + HIDMA_TRCA_DOORBELL_REG);
512 	spin_unlock_irqrestore(&lldev->lock, irqflags);
513 }
514 
515 bool hidma_ll_isenabled(struct hidma_lldev *lldev)
516 {
517 	u32 val;
518 
519 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
520 	lldev->trch_state = HIDMA_CH_STATE(val);
521 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
522 	lldev->evch_state = HIDMA_CH_STATE(val);
523 
524 	/* both channels have to be enabled before calling this function */
525 	if (hidma_is_chan_enabled(lldev->trch_state) &&
526 	    hidma_is_chan_enabled(lldev->evch_state))
527 		return true;
528 
529 	return false;
530 }
531 
532 void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch)
533 {
534 	struct hidma_tre *tre;
535 	unsigned long flags;
536 
537 	tre = &lldev->trepool[tre_ch];
538 
539 	/* copy the TRE into its location in the TRE ring */
540 	spin_lock_irqsave(&lldev->lock, flags);
541 	tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE;
542 	lldev->pending_tre_list[tre->tre_index] = tre;
543 	memcpy(lldev->tre_ring + lldev->tre_write_offset,
544 			&tre->tre_local[0], HIDMA_TRE_SIZE);
545 	tre->err_code = 0;
546 	tre->err_info = 0;
547 	tre->queued = 1;
548 	atomic_inc(&lldev->pending_tre_count);
549 	lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE)
550 					% lldev->tre_ring_size;
551 	spin_unlock_irqrestore(&lldev->lock, flags);
552 }
553 
554 /*
555  * Note that even though we stop this channel if there is a pending transaction
556  * in flight it will complete and follow the callback. This request will
557  * prevent further requests to be made.
558  */
559 int hidma_ll_disable(struct hidma_lldev *lldev)
560 {
561 	u32 val;
562 	int ret;
563 
564 	/* The channel needs to be in working state */
565 	if (!hidma_ll_isenabled(lldev))
566 		return 0;
567 
568 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
569 	val &= ~(HIDMA_CH_CONTROL_MASK << 16);
570 	val |= HIDMA_CH_SUSPEND << 16;
571 	writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
572 
573 	/*
574 	 * Start the wait right after the suspend is confirmed.
575 	 * Do a polled read up to 1ms and 10ms maximum.
576 	 */
577 	ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
578 				 HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED,
579 				 1000, 10000);
580 	if (ret)
581 		return ret;
582 
583 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
584 	val &= ~(HIDMA_CH_CONTROL_MASK << 16);
585 	val |= HIDMA_CH_SUSPEND << 16;
586 	writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
587 
588 	/*
589 	 * Start the wait right after the suspend is confirmed
590 	 * Delay up to 10ms after reset to allow DMA logic to quiesce.
591 	 */
592 	ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
593 				 HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED,
594 				 1000, 10000);
595 	if (ret)
596 		return ret;
597 
598 	lldev->trch_state = HIDMA_CH_SUSPENDED;
599 	lldev->evch_state = HIDMA_CH_SUSPENDED;
600 
601 	/* disable interrupts */
602 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
603 	return 0;
604 }
605 
606 void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch,
607 				  dma_addr_t src, dma_addr_t dest, u32 len,
608 				  u32 flags, u32 txntype)
609 {
610 	struct hidma_tre *tre;
611 	u32 *tre_local;
612 
613 	if (tre_ch >= lldev->nr_tres) {
614 		dev_err(lldev->dev, "invalid TRE number in transfer params:%d",
615 			tre_ch);
616 		return;
617 	}
618 
619 	tre = &lldev->trepool[tre_ch];
620 	if (atomic_read(&tre->allocated) != true) {
621 		dev_err(lldev->dev, "trying to set params on an unused TRE:%d",
622 			tre_ch);
623 		return;
624 	}
625 
626 	tre_local = &tre->tre_local[0];
627 	tre_local[HIDMA_TRE_CFG_IDX] &= ~GENMASK(7, 0);
628 	tre_local[HIDMA_TRE_CFG_IDX] |= txntype;
629 	tre_local[HIDMA_TRE_LEN_IDX] = len;
630 	tre_local[HIDMA_TRE_SRC_LOW_IDX] = lower_32_bits(src);
631 	tre_local[HIDMA_TRE_SRC_HI_IDX] = upper_32_bits(src);
632 	tre_local[HIDMA_TRE_DEST_LOW_IDX] = lower_32_bits(dest);
633 	tre_local[HIDMA_TRE_DEST_HI_IDX] = upper_32_bits(dest);
634 	tre->int_flags = flags;
635 }
636 
637 /*
638  * Called during initialization and after an error condition
639  * to restore hardware state.
640  */
641 int hidma_ll_setup(struct hidma_lldev *lldev)
642 {
643 	int rc;
644 	u64 addr;
645 	u32 val;
646 	u32 nr_tres = lldev->nr_tres;
647 
648 	atomic_set(&lldev->pending_tre_count, 0);
649 	lldev->tre_processed_off = 0;
650 	lldev->evre_processed_off = 0;
651 	lldev->tre_write_offset = 0;
652 
653 	/* disable interrupts */
654 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
655 
656 	/* clear all pending interrupts */
657 	val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
658 	writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
659 
660 	rc = hidma_ll_reset(lldev);
661 	if (rc)
662 		return rc;
663 
664 	/*
665 	 * Clear all pending interrupts again.
666 	 * Otherwise, we observe reset complete interrupts.
667 	 */
668 	val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
669 	writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
670 
671 	/* disable interrupts again after reset */
672 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
673 
674 	addr = lldev->tre_dma;
675 	writel(lower_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_LOW_REG);
676 	writel(upper_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_HIGH_REG);
677 	writel(lldev->tre_ring_size, lldev->trca + HIDMA_TRCA_RING_LEN_REG);
678 
679 	addr = lldev->evre_dma;
680 	writel(lower_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_LOW_REG);
681 	writel(upper_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_HIGH_REG);
682 	writel(HIDMA_EVRE_SIZE * nr_tres,
683 			lldev->evca + HIDMA_EVCA_RING_LEN_REG);
684 
685 	/* configure interrupts */
686 	hidma_ll_setup_irq(lldev, lldev->msi_support);
687 
688 	rc = hidma_ll_enable(lldev);
689 	if (rc)
690 		return rc;
691 
692 	return rc;
693 }
694 
695 void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi)
696 {
697 	u32 val;
698 
699 	lldev->msi_support = msi;
700 
701 	/* disable interrupts again after reset */
702 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
703 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
704 
705 	/* support IRQ by default */
706 	val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG);
707 	val &= ~0xF;
708 	if (!lldev->msi_support)
709 		val = val | 0x1;
710 	writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG);
711 
712 	/* clear all pending interrupts and enable them */
713 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
714 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
715 }
716 
717 struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres,
718 				  void __iomem *trca, void __iomem *evca,
719 				  u8 chidx)
720 {
721 	u32 required_bytes;
722 	struct hidma_lldev *lldev;
723 	int rc;
724 	size_t sz;
725 
726 	if (!trca || !evca || !dev || !nr_tres)
727 		return NULL;
728 
729 	/* need at least four TREs */
730 	if (nr_tres < 4)
731 		return NULL;
732 
733 	/* need an extra space */
734 	nr_tres += 1;
735 
736 	lldev = devm_kzalloc(dev, sizeof(struct hidma_lldev), GFP_KERNEL);
737 	if (!lldev)
738 		return NULL;
739 
740 	lldev->evca = evca;
741 	lldev->trca = trca;
742 	lldev->dev = dev;
743 	sz = sizeof(struct hidma_tre);
744 	lldev->trepool = devm_kcalloc(lldev->dev, nr_tres, sz, GFP_KERNEL);
745 	if (!lldev->trepool)
746 		return NULL;
747 
748 	required_bytes = sizeof(lldev->pending_tre_list[0]);
749 	lldev->pending_tre_list = devm_kcalloc(dev, nr_tres, required_bytes,
750 					       GFP_KERNEL);
751 	if (!lldev->pending_tre_list)
752 		return NULL;
753 
754 	sz = (HIDMA_TRE_SIZE + 1) * nr_tres;
755 	lldev->tre_ring = dmam_alloc_coherent(dev, sz, &lldev->tre_dma,
756 					      GFP_KERNEL);
757 	if (!lldev->tre_ring)
758 		return NULL;
759 
760 	memset(lldev->tre_ring, 0, (HIDMA_TRE_SIZE + 1) * nr_tres);
761 	lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres;
762 	lldev->nr_tres = nr_tres;
763 
764 	/* the TRE ring has to be TRE_SIZE aligned */
765 	if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) {
766 		u8 tre_ring_shift;
767 
768 		tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE;
769 		tre_ring_shift = HIDMA_TRE_SIZE - tre_ring_shift;
770 		lldev->tre_dma += tre_ring_shift;
771 		lldev->tre_ring += tre_ring_shift;
772 	}
773 
774 	sz = (HIDMA_EVRE_SIZE + 1) * nr_tres;
775 	lldev->evre_ring = dmam_alloc_coherent(dev, sz, &lldev->evre_dma,
776 					       GFP_KERNEL);
777 	if (!lldev->evre_ring)
778 		return NULL;
779 
780 	memset(lldev->evre_ring, 0, (HIDMA_EVRE_SIZE + 1) * nr_tres);
781 	lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres;
782 
783 	/* the EVRE ring has to be EVRE_SIZE aligned */
784 	if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) {
785 		u8 evre_ring_shift;
786 
787 		evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE;
788 		evre_ring_shift = HIDMA_EVRE_SIZE - evre_ring_shift;
789 		lldev->evre_dma += evre_ring_shift;
790 		lldev->evre_ring += evre_ring_shift;
791 	}
792 	lldev->nr_tres = nr_tres;
793 	lldev->chidx = chidx;
794 
795 	sz = nr_tres * sizeof(struct hidma_tre *);
796 	rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL);
797 	if (rc)
798 		return NULL;
799 
800 	rc = hidma_ll_setup(lldev);
801 	if (rc)
802 		return NULL;
803 
804 	spin_lock_init(&lldev->lock);
805 	tasklet_init(&lldev->task, hidma_ll_tre_complete, (unsigned long)lldev);
806 	lldev->initialized = 1;
807 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
808 	return lldev;
809 }
810 
811 int hidma_ll_uninit(struct hidma_lldev *lldev)
812 {
813 	u32 required_bytes;
814 	int rc = 0;
815 	u32 val;
816 
817 	if (!lldev)
818 		return -ENODEV;
819 
820 	if (!lldev->initialized)
821 		return 0;
822 
823 	lldev->initialized = 0;
824 
825 	required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres;
826 	tasklet_kill(&lldev->task);
827 	memset(lldev->trepool, 0, required_bytes);
828 	lldev->trepool = NULL;
829 	atomic_set(&lldev->pending_tre_count, 0);
830 	lldev->tre_write_offset = 0;
831 
832 	rc = hidma_ll_reset(lldev);
833 
834 	/*
835 	 * Clear all pending interrupts again.
836 	 * Otherwise, we observe reset complete interrupts.
837 	 */
838 	val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
839 	writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
840 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
841 	return rc;
842 }
843 
844 enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch)
845 {
846 	enum dma_status ret = DMA_ERROR;
847 	struct hidma_tre *tre;
848 	unsigned long flags;
849 	u8 err_code;
850 
851 	spin_lock_irqsave(&lldev->lock, flags);
852 
853 	tre = &lldev->trepool[tre_ch];
854 	err_code = tre->err_code;
855 
856 	if (err_code & HIDMA_EVRE_STATUS_COMPLETE)
857 		ret = DMA_COMPLETE;
858 	else if (err_code & HIDMA_EVRE_STATUS_ERROR)
859 		ret = DMA_ERROR;
860 	else
861 		ret = DMA_IN_PROGRESS;
862 	spin_unlock_irqrestore(&lldev->lock, flags);
863 
864 	return ret;
865 }
866