xref: /linux/drivers/usb/gadget/udc/cdns2/cdns2-gadget.c (revision cc8b10fa70682218c2a318fc44f71f3175a23cc0)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Cadence USBHS-DEV Driver - gadget side.
4  *
5  * Copyright (C) 2023 Cadence Design Systems.
6  *
7  * Authors: Pawel Laszczak <pawell@cadence.com>
8  */
9 
10 /*
11  * Work around 1:
12  * At some situations, the controller may get stale data address in TRB
13  * at below sequences:
14  * 1. Controller read TRB includes data address
15  * 2. Software updates TRBs includes data address and Cycle bit
16  * 3. Controller read TRB which includes Cycle bit
17  * 4. DMA run with stale data address
18  *
19  * To fix this problem, driver needs to make the first TRB in TD as invalid.
20  * After preparing all TRBs driver needs to check the position of DMA and
21  * if the DMA point to the first just added TRB and doorbell is 1,
22  * then driver must defer making this TRB as valid. This TRB will be make
23  * as valid during adding next TRB only if DMA is stopped or at TRBERR
24  * interrupt.
25  *
26  */
27 
28 #include <linux/dma-mapping.h>
29 #include <linux/pm_runtime.h>
30 #include <linux/interrupt.h>
31 #include <linux/property.h>
32 #include <linux/string_choices.h>
33 #include <linux/dmapool.h>
34 #include <linux/iopoll.h>
35 
36 #include "cdns2-gadget.h"
37 #include "cdns2-trace.h"
38 
39 /**
40  * set_reg_bit_32 - set bit in given 32 bits register.
41  * @ptr: register address.
42  * @mask: bits to set.
43  */
44 static void set_reg_bit_32(void __iomem *ptr, u32 mask)
45 {
46 	mask = readl(ptr) | mask;
47 	writel(mask, ptr);
48 }
49 
50 /*
51  * clear_reg_bit_32 - clear bit in given 32 bits register.
52  * @ptr: register address.
53  * @mask: bits to clear.
54  */
55 static void clear_reg_bit_32(void __iomem *ptr, u32 mask)
56 {
57 	mask = readl(ptr) & ~mask;
58 	writel(mask, ptr);
59 }
60 
61 /* Clear bit in given 8 bits register. */
62 static void clear_reg_bit_8(void __iomem *ptr, u8 mask)
63 {
64 	mask = readb(ptr) & ~mask;
65 	writeb(mask, ptr);
66 }
67 
68 /* Set bit in given 16 bits register. */
69 void set_reg_bit_8(void __iomem *ptr, u8 mask)
70 {
71 	mask = readb(ptr) | mask;
72 	writeb(mask, ptr);
73 }
74 
75 static int cdns2_get_dma_pos(struct cdns2_device *pdev,
76 			     struct cdns2_endpoint *pep)
77 {
78 	int dma_index;
79 
80 	dma_index = readl(&pdev->adma_regs->ep_traddr) - pep->ring.dma;
81 
82 	return dma_index / TRB_SIZE;
83 }
84 
85 /* Get next private request from list. */
86 struct cdns2_request *cdns2_next_preq(struct list_head *list)
87 {
88 	return list_first_entry_or_null(list, struct cdns2_request, list);
89 }
90 
91 void cdns2_select_ep(struct cdns2_device *pdev, u32 ep)
92 {
93 	if (pdev->selected_ep == ep)
94 		return;
95 
96 	pdev->selected_ep = ep;
97 	writel(ep, &pdev->adma_regs->ep_sel);
98 }
99 
100 dma_addr_t cdns2_trb_virt_to_dma(struct cdns2_endpoint *pep,
101 				 struct cdns2_trb *trb)
102 {
103 	u32 offset = (char *)trb - (char *)pep->ring.trbs;
104 
105 	return pep->ring.dma + offset;
106 }
107 
108 static void cdns2_free_tr_segment(struct cdns2_endpoint *pep)
109 {
110 	struct cdns2_device *pdev = pep->pdev;
111 	struct cdns2_ring *ring = &pep->ring;
112 
113 	if (pep->ring.trbs) {
114 		dma_pool_free(pdev->eps_dma_pool, ring->trbs, ring->dma);
115 		memset(ring, 0, sizeof(*ring));
116 	}
117 }
118 
119 /* Allocates Transfer Ring segment. */
120 static int cdns2_alloc_tr_segment(struct cdns2_endpoint *pep)
121 {
122 	struct cdns2_device *pdev = pep->pdev;
123 	struct cdns2_trb *link_trb;
124 	struct cdns2_ring *ring;
125 
126 	ring = &pep->ring;
127 
128 	if (!ring->trbs) {
129 		ring->trbs = dma_pool_alloc(pdev->eps_dma_pool,
130 					    GFP_DMA32 | GFP_ATOMIC,
131 					    &ring->dma);
132 		if (!ring->trbs)
133 			return -ENOMEM;
134 	}
135 
136 	memset(ring->trbs, 0, TR_SEG_SIZE);
137 
138 	if (!pep->num)
139 		return 0;
140 
141 	/* Initialize the last TRB as Link TRB */
142 	link_trb = (ring->trbs + (TRBS_PER_SEGMENT - 1));
143 	link_trb->buffer = cpu_to_le32(TRB_BUFFER(ring->dma));
144 	link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) |
145 					TRB_TOGGLE);
146 
147 	return 0;
148 }
149 
150 /*
151  * Stalls and flushes selected endpoint.
152  * Endpoint must be selected before invoking this function.
153  */
154 static void cdns2_ep_stall_flush(struct cdns2_endpoint *pep)
155 {
156 	struct cdns2_device *pdev = pep->pdev;
157 	int val;
158 
159 	trace_cdns2_ep_halt(pep, 1, 1);
160 
161 	writel(DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd);
162 
163 	/* Wait for DFLUSH cleared. */
164 	readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
165 				  !(val & DMA_EP_CMD_DFLUSH), 1, 1000);
166 	pep->ep_state |= EP_STALLED;
167 	pep->ep_state &= ~EP_STALL_PENDING;
168 }
169 
170 /*
171  * Increment a trb index.
172  *
173  * The index should never point to the last link TRB in TR. After incrementing,
174  * if it point to the link TRB, wrap around to the beginning and revert
175  * cycle state bit. The link TRB is always at the last TRB entry.
176  */
177 static void cdns2_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
178 {
179 	(*index)++;
180 	if (*index == (trb_in_seg - 1)) {
181 		*index = 0;
182 		*cs ^=  1;
183 	}
184 }
185 
186 static void cdns2_ep_inc_enq(struct cdns2_ring *ring)
187 {
188 	ring->free_trbs--;
189 	cdns2_ep_inc_trb(&ring->enqueue, &ring->pcs, TRBS_PER_SEGMENT);
190 }
191 
192 static void cdns2_ep_inc_deq(struct cdns2_ring *ring)
193 {
194 	ring->free_trbs++;
195 	cdns2_ep_inc_trb(&ring->dequeue, &ring->ccs, TRBS_PER_SEGMENT);
196 }
197 
198 /*
199  * Enable/disable LPM.
200  *
201  * If bit USBCS_LPMNYET is not set and device receive Extended Token packet,
202  * then controller answer with ACK handshake.
203  * If bit USBCS_LPMNYET is set and device receive Extended Token packet,
204  * then controller answer with NYET handshake.
205  */
206 static void cdns2_enable_l1(struct cdns2_device *pdev, int enable)
207 {
208 	if (enable) {
209 		clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET);
210 		writeb(LPMCLOCK_SLEEP_ENTRY, &pdev->usb_regs->lpmclock);
211 	} else {
212 		set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET);
213 	}
214 }
215 
216 static enum usb_device_speed cdns2_get_speed(struct cdns2_device *pdev)
217 {
218 	u8 speed = readb(&pdev->usb_regs->speedctrl);
219 
220 	if (speed & SPEEDCTRL_HS)
221 		return USB_SPEED_HIGH;
222 	else if (speed & SPEEDCTRL_FS)
223 		return USB_SPEED_FULL;
224 
225 	return USB_SPEED_UNKNOWN;
226 }
227 
228 static struct cdns2_trb *cdns2_next_trb(struct cdns2_endpoint *pep,
229 					struct cdns2_trb *trb)
230 {
231 	if (trb == (pep->ring.trbs + (TRBS_PER_SEGMENT - 1)))
232 		return pep->ring.trbs;
233 	else
234 		return ++trb;
235 }
236 
237 void cdns2_gadget_giveback(struct cdns2_endpoint *pep,
238 			   struct cdns2_request *preq,
239 			   int status)
240 {
241 	struct usb_request *request = &preq->request;
242 	struct cdns2_device *pdev = pep->pdev;
243 
244 	list_del_init(&preq->list);
245 
246 	if (request->status == -EINPROGRESS)
247 		request->status = status;
248 
249 	usb_gadget_unmap_request_by_dev(pdev->dev, request, pep->dir);
250 
251 	/* All TRBs have finished, clear the counter. */
252 	preq->finished_trb = 0;
253 
254 	trace_cdns2_request_giveback(preq);
255 
256 	if (request->complete) {
257 		spin_unlock(&pdev->lock);
258 		usb_gadget_giveback_request(&pep->endpoint, request);
259 		spin_lock(&pdev->lock);
260 	}
261 
262 	if (request->buf == pdev->zlp_buf)
263 		cdns2_gadget_ep_free_request(&pep->endpoint, request);
264 }
265 
266 static void cdns2_wa1_restore_cycle_bit(struct cdns2_endpoint *pep)
267 {
268 	/* Work around for stale data address in TRB. */
269 	if (pep->wa1_set) {
270 		trace_cdns2_wa1(pep, "restore cycle bit");
271 
272 		pep->wa1_set = 0;
273 		pep->wa1_trb_index = 0xFFFF;
274 		if (pep->wa1_cycle_bit)
275 			pep->wa1_trb->control |= cpu_to_le32(0x1);
276 		else
277 			pep->wa1_trb->control &= cpu_to_le32(~0x1);
278 	}
279 }
280 
281 static int cdns2_wa1_update_guard(struct cdns2_endpoint *pep,
282 				  struct cdns2_trb *trb)
283 {
284 	struct cdns2_device *pdev = pep->pdev;
285 
286 	if (!pep->wa1_set) {
287 		u32 doorbell;
288 
289 		doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
290 
291 		if (doorbell) {
292 			pep->wa1_cycle_bit = pep->ring.pcs ? TRB_CYCLE : 0;
293 			pep->wa1_set = 1;
294 			pep->wa1_trb = trb;
295 			pep->wa1_trb_index = pep->ring.enqueue;
296 			trace_cdns2_wa1(pep, "set guard");
297 			return 0;
298 		}
299 	}
300 	return 1;
301 }
302 
303 static void cdns2_wa1_tray_restore_cycle_bit(struct cdns2_device *pdev,
304 					     struct cdns2_endpoint *pep)
305 {
306 	int dma_index;
307 	u32 doorbell;
308 
309 	doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
310 	dma_index = cdns2_get_dma_pos(pdev, pep);
311 
312 	if (!doorbell || dma_index != pep->wa1_trb_index)
313 		cdns2_wa1_restore_cycle_bit(pep);
314 }
315 
316 static int cdns2_prepare_ring(struct cdns2_device *pdev,
317 			      struct cdns2_endpoint *pep,
318 			      int num_trbs)
319 {
320 	struct cdns2_trb *link_trb = NULL;
321 	int doorbell, dma_index;
322 	struct cdns2_ring *ring;
323 	u32 ch_bit = 0;
324 
325 	ring = &pep->ring;
326 
327 	if (num_trbs > ring->free_trbs) {
328 		pep->ep_state |= EP_RING_FULL;
329 		trace_cdns2_no_room_on_ring("Ring full\n");
330 		return -ENOBUFS;
331 	}
332 
333 	if ((ring->enqueue + num_trbs)  >= (TRBS_PER_SEGMENT - 1)) {
334 		doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
335 		dma_index = cdns2_get_dma_pos(pdev, pep);
336 
337 		/* Driver can't update LINK TRB if it is current processed. */
338 		if (doorbell && dma_index == TRBS_PER_SEGMENT - 1) {
339 			pep->ep_state |= EP_DEFERRED_DRDY;
340 			return -ENOBUFS;
341 		}
342 
343 		/* Update C bt in Link TRB before starting DMA. */
344 		link_trb = ring->trbs + (TRBS_PER_SEGMENT - 1);
345 
346 		/*
347 		 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
348 		 * that DMA stuck at the LINK TRB.
349 		 * On the other hand, removing TRB_CHAIN for longer TRs for
350 		 * epXout cause that DMA stuck after handling LINK TRB.
351 		 * To eliminate this strange behavioral driver set TRB_CHAIN
352 		 * bit only for TR size > 2.
353 		 */
354 		if (pep->type == USB_ENDPOINT_XFER_ISOC || TRBS_PER_SEGMENT > 2)
355 			ch_bit = TRB_CHAIN;
356 
357 		link_trb->control = cpu_to_le32(((ring->pcs) ? TRB_CYCLE : 0) |
358 				    TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
359 	}
360 
361 	return 0;
362 }
363 
364 static void cdns2_dbg_request_trbs(struct cdns2_endpoint *pep,
365 				   struct cdns2_request *preq)
366 {
367 	struct cdns2_trb *link_trb = pep->ring.trbs + (TRBS_PER_SEGMENT - 1);
368 	struct cdns2_trb *trb = preq->trb;
369 	int num_trbs = preq->num_of_trb;
370 	int i = 0;
371 
372 	while (i < num_trbs) {
373 		trace_cdns2_queue_trb(pep, trb + i);
374 		if (trb + i == link_trb) {
375 			trb = pep->ring.trbs;
376 			num_trbs = num_trbs - i;
377 			i = 0;
378 		} else {
379 			i++;
380 		}
381 	}
382 }
383 
384 static unsigned int cdns2_count_trbs(struct cdns2_endpoint *pep,
385 				     u64 addr, u64 len)
386 {
387 	unsigned int num_trbs = 1;
388 
389 	if (pep->type == USB_ENDPOINT_XFER_ISOC) {
390 		/*
391 		 * To speed up DMA performance address should not exceed 4KB.
392 		 * for high bandwidth transfer and driver will split
393 		 * such buffer into two TRBs.
394 		 */
395 		num_trbs = DIV_ROUND_UP(len +
396 					(addr & (TRB_MAX_ISO_BUFF_SIZE - 1)),
397 					TRB_MAX_ISO_BUFF_SIZE);
398 
399 		if (pep->interval > 1)
400 			num_trbs = pep->dir ? num_trbs * pep->interval : 1;
401 	} else if (pep->dir) {
402 		/*
403 		 * One extra link trb for IN direction.
404 		 * Sometimes DMA doesn't want advance to next TD and transfer
405 		 * hangs. This extra Link TRB force DMA to advance to next TD.
406 		 */
407 		num_trbs++;
408 	}
409 
410 	return num_trbs;
411 }
412 
413 static unsigned int cdns2_count_sg_trbs(struct cdns2_endpoint *pep,
414 					struct usb_request *req)
415 {
416 	unsigned int i, len, full_len, num_trbs = 0;
417 	struct scatterlist *sg;
418 	int trb_len = 0;
419 
420 	full_len = req->length;
421 
422 	for_each_sg(req->sg, sg, req->num_sgs, i) {
423 		len = sg_dma_len(sg);
424 		num_trbs += cdns2_count_trbs(pep, sg_dma_address(sg), len);
425 		len = min(len, full_len);
426 
427 		/*
428 		 * For HS ISO transfer TRBs should not exceed max packet size.
429 		 * When DMA is working, and data exceed max packet size then
430 		 * some data will be read in single mode instead burst mode.
431 		 * This behavior will drastically reduce the copying speed.
432 		 * To avoid this we need one or two extra TRBs.
433 		 * This issue occurs for UVC class with sg_supported = 1
434 		 * because buffers addresses are not aligned to 1024.
435 		 */
436 		if (pep->type == USB_ENDPOINT_XFER_ISOC) {
437 			u8 temp;
438 
439 			trb_len += len;
440 			temp = trb_len >> 10;
441 
442 			if (temp) {
443 				if (trb_len % 1024)
444 					num_trbs = num_trbs + temp;
445 				else
446 					num_trbs = num_trbs + temp - 1;
447 
448 				trb_len = trb_len - (temp << 10);
449 			}
450 		}
451 
452 		full_len -= len;
453 		if (full_len == 0)
454 			break;
455 	}
456 
457 	return num_trbs;
458 }
459 
460 /*
461  * Function prepares the array with optimized AXI burst value for different
462  * transfer lengths. Controller handles the final data which are less
463  * then AXI burst size as single byte transactions.
464  * e.g.:
465  * Let's assume that driver prepares trb with trb->length 700 and burst size
466  * will be set to 128. In this case the controller will handle a first 512 as
467  * single AXI transaction but the next 188 bytes will be handled
468  * as 47 separate AXI transaction.
469  * The better solution is to use the burst size equal 16 and then we will
470  * have only 25 AXI transaction (10 * 64 + 15 *4).
471  */
472 static void cdsn2_isoc_burst_opt(struct cdns2_device *pdev)
473 {
474 	int axi_burst_option[]  =  {1, 2, 4, 8, 16, 32, 64, 128};
475 	int best_burst;
476 	int array_size;
477 	int opt_burst;
478 	int trb_size;
479 	int i, j;
480 
481 	array_size = ARRAY_SIZE(axi_burst_option);
482 
483 	for (i = 0; i <= MAX_ISO_SIZE; i++) {
484 		trb_size = i / 4;
485 		best_burst = trb_size ? trb_size : 1;
486 
487 		for (j = 0; j < array_size; j++) {
488 			opt_burst = trb_size / axi_burst_option[j];
489 			opt_burst += trb_size % axi_burst_option[j];
490 
491 			if (opt_burst < best_burst) {
492 				best_burst = opt_burst;
493 				pdev->burst_opt[i] = axi_burst_option[j];
494 			}
495 		}
496 	}
497 }
498 
499 static void cdns2_ep_tx_isoc(struct cdns2_endpoint *pep,
500 			     struct cdns2_request *preq,
501 			     int num_trbs)
502 {
503 	struct scatterlist *sg = NULL;
504 	u32 remaining_packet_size = 0;
505 	struct cdns2_trb *trb;
506 	bool first_trb = true;
507 	dma_addr_t trb_dma;
508 	u32 trb_buff_len;
509 	u32 block_length;
510 	int td_idx = 0;
511 	int split_size;
512 	u32 full_len;
513 	int enqd_len;
514 	int sent_len;
515 	int sg_iter;
516 	u32 control;
517 	int num_tds;
518 	u32 length;
519 
520 	/*
521 	 * For OUT direction 1 TD per interval is enough
522 	 * because TRBs are not dumped by controller.
523 	 */
524 	num_tds = pep->dir ? pep->interval : 1;
525 	split_size = preq->request.num_sgs ? 1024 : 3072;
526 
527 	for (td_idx = 0; td_idx < num_tds; td_idx++) {
528 		if (preq->request.num_sgs) {
529 			sg = preq->request.sg;
530 			trb_dma = sg_dma_address(sg);
531 			block_length = sg_dma_len(sg);
532 		} else {
533 			trb_dma = preq->request.dma;
534 			block_length = preq->request.length;
535 		}
536 
537 		full_len = preq->request.length;
538 		sg_iter = preq->request.num_sgs ? preq->request.num_sgs : 1;
539 		remaining_packet_size = split_size;
540 
541 		for (enqd_len = 0;  enqd_len < full_len;
542 		     enqd_len += trb_buff_len) {
543 			if (remaining_packet_size == 0)
544 				remaining_packet_size = split_size;
545 
546 			/*
547 			 * Calculate TRB length.- buffer can't across 4KB
548 			 * and max packet size.
549 			 */
550 			trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(trb_dma);
551 			trb_buff_len = min(trb_buff_len, remaining_packet_size);
552 			trb_buff_len = min(trb_buff_len, block_length);
553 
554 			if (trb_buff_len > full_len - enqd_len)
555 				trb_buff_len = full_len - enqd_len;
556 
557 			control = TRB_TYPE(TRB_NORMAL);
558 
559 			/*
560 			 * For IN direction driver has to set the IOC for
561 			 * last TRB in last TD.
562 			 * For OUT direction driver must set IOC and ISP
563 			 * only for last TRB in each TDs.
564 			 */
565 			if (enqd_len + trb_buff_len >= full_len || !pep->dir)
566 				control |= TRB_IOC | TRB_ISP;
567 
568 			/*
569 			 * Don't give the first TRB to the hardware (by toggling
570 			 * the cycle bit) until we've finished creating all the
571 			 * other TRBs.
572 			 */
573 			if (first_trb) {
574 				first_trb = false;
575 				if (pep->ring.pcs == 0)
576 					control |= TRB_CYCLE;
577 			} else {
578 				control |= pep->ring.pcs;
579 			}
580 
581 			if (enqd_len + trb_buff_len < full_len)
582 				control |= TRB_CHAIN;
583 
584 			length = TRB_LEN(trb_buff_len) |
585 				 TRB_BURST(pep->pdev->burst_opt[trb_buff_len]);
586 
587 			trb = pep->ring.trbs + pep->ring.enqueue;
588 			trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
589 			trb->length = cpu_to_le32(length);
590 			trb->control = cpu_to_le32(control);
591 
592 			trb_dma += trb_buff_len;
593 			sent_len = trb_buff_len;
594 
595 			if (sg && sent_len >= block_length) {
596 				/* New sg entry */
597 				--sg_iter;
598 				sent_len -= block_length;
599 				if (sg_iter != 0) {
600 					sg = sg_next(sg);
601 					trb_dma = sg_dma_address(sg);
602 					block_length = sg_dma_len(sg);
603 				}
604 			}
605 
606 			remaining_packet_size -= trb_buff_len;
607 			block_length -= sent_len;
608 			preq->end_trb = pep->ring.enqueue;
609 
610 			cdns2_ep_inc_enq(&pep->ring);
611 		}
612 	}
613 }
614 
615 static void cdns2_ep_tx_bulk(struct cdns2_endpoint *pep,
616 			     struct cdns2_request *preq,
617 			     int trbs_per_td)
618 {
619 	struct scatterlist *sg = NULL;
620 	struct cdns2_ring *ring;
621 	struct cdns2_trb *trb;
622 	dma_addr_t trb_dma;
623 	int sg_iter = 0;
624 	u32 control;
625 	u32 length;
626 
627 	if (preq->request.num_sgs) {
628 		sg = preq->request.sg;
629 		trb_dma = sg_dma_address(sg);
630 		length = sg_dma_len(sg);
631 	} else {
632 		trb_dma = preq->request.dma;
633 		length = preq->request.length;
634 	}
635 
636 	ring = &pep->ring;
637 
638 	for (sg_iter = 0; sg_iter < trbs_per_td; sg_iter++) {
639 		control = TRB_TYPE(TRB_NORMAL) | ring->pcs | TRB_ISP;
640 		trb = pep->ring.trbs + ring->enqueue;
641 
642 		if (pep->dir && sg_iter == trbs_per_td - 1) {
643 			preq->end_trb = ring->enqueue;
644 			control = ring->pcs | TRB_TYPE(TRB_LINK) | TRB_CHAIN
645 				  | TRB_IOC;
646 			cdns2_ep_inc_enq(&pep->ring);
647 
648 			if (ring->enqueue == 0)
649 				control |= TRB_TOGGLE;
650 
651 			/* Point to next bad TRB. */
652 			trb->buffer = cpu_to_le32(pep->ring.dma +
653 						  (ring->enqueue * TRB_SIZE));
654 			trb->length = 0;
655 			trb->control = cpu_to_le32(control);
656 			break;
657 		}
658 
659 		/*
660 		 * Don't give the first TRB to the hardware (by toggling
661 		 * the cycle bit) until we've finished creating all the
662 		 * other TRBs.
663 		 */
664 		if (sg_iter == 0)
665 			control = control ^ TRB_CYCLE;
666 
667 		/* For last TRB in TD. */
668 		if (sg_iter == (trbs_per_td - (pep->dir ? 2 : 1)))
669 			control |= TRB_IOC;
670 		else
671 			control |= TRB_CHAIN;
672 
673 		trb->buffer = cpu_to_le32(trb_dma);
674 		trb->length = cpu_to_le32(TRB_BURST(pep->trb_burst_size) |
675 					   TRB_LEN(length));
676 		trb->control = cpu_to_le32(control);
677 
678 		if (sg && sg_iter < (trbs_per_td - 1)) {
679 			sg = sg_next(sg);
680 			trb_dma = sg_dma_address(sg);
681 			length = sg_dma_len(sg);
682 		}
683 
684 		preq->end_trb = ring->enqueue;
685 		cdns2_ep_inc_enq(&pep->ring);
686 	}
687 }
688 
689 static void cdns2_set_drdy(struct cdns2_device *pdev,
690 			   struct cdns2_endpoint *pep)
691 {
692 	trace_cdns2_ring(pep);
693 
694 	/*
695 	 * Memory barrier - Cycle Bit must be set before doorbell.
696 	 */
697 	dma_wmb();
698 
699 	/* Clearing TRBERR and DESCMIS before setting DRDY. */
700 	writel(DMA_EP_STS_TRBERR | DMA_EP_STS_DESCMIS,
701 	       &pdev->adma_regs->ep_sts);
702 	writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
703 
704 	if (readl(&pdev->adma_regs->ep_sts) & DMA_EP_STS_TRBERR) {
705 		writel(DMA_EP_STS_TRBERR, &pdev->adma_regs->ep_sts);
706 		writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
707 	}
708 
709 	trace_cdns2_doorbell_epx(pep, readl(&pdev->adma_regs->ep_traddr));
710 }
711 
712 static int cdns2_prepare_first_isoc_transfer(struct cdns2_device *pdev,
713 					     struct cdns2_endpoint *pep)
714 {
715 	struct cdns2_trb *trb;
716 	u32 buffer;
717 	u8 hw_ccs;
718 
719 	if ((readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY))
720 		return -EBUSY;
721 
722 	if (!pep->dir) {
723 		set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
724 		writel(pep->ring.dma + pep->ring.dequeue,
725 		       &pdev->adma_regs->ep_traddr);
726 		return 0;
727 	}
728 
729 	/*
730 	 * The first packet after doorbell can be corrupted so,
731 	 * driver prepares 0 length packet as first packet.
732 	 */
733 	buffer = pep->ring.dma + pep->ring.dequeue * TRB_SIZE;
734 	hw_ccs = !!DMA_EP_STS_CCS(readl(&pdev->adma_regs->ep_sts));
735 
736 	trb = &pep->ring.trbs[TRBS_PER_SEGMENT];
737 	trb->length = 0;
738 	trb->buffer = cpu_to_le32(TRB_BUFFER(buffer));
739 	trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) | TRB_TYPE(TRB_NORMAL));
740 
741 	/*
742 	 * LINK TRB is used to force updating cycle bit in controller and
743 	 * move to correct place in transfer ring.
744 	 */
745 	trb++;
746 	trb->length = 0;
747 	trb->buffer = cpu_to_le32(TRB_BUFFER(buffer));
748 	trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) |
749 				    TRB_TYPE(TRB_LINK) | TRB_CHAIN);
750 
751 	if (hw_ccs !=  pep->ring.ccs)
752 		trb->control |= cpu_to_le32(TRB_TOGGLE);
753 
754 	set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
755 	writel(pep->ring.dma + (TRBS_PER_SEGMENT * TRB_SIZE),
756 	       &pdev->adma_regs->ep_traddr);
757 
758 	return 0;
759 }
760 
761 /* Prepare and start transfer on no-default endpoint. */
762 static int cdns2_ep_run_transfer(struct cdns2_endpoint *pep,
763 				 struct cdns2_request *preq)
764 {
765 	struct cdns2_device *pdev = pep->pdev;
766 	struct cdns2_ring *ring;
767 	u32 togle_pcs = 1;
768 	int num_trbs;
769 	int ret;
770 
771 	cdns2_select_ep(pdev, pep->endpoint.address);
772 
773 	if (preq->request.sg)
774 		num_trbs = cdns2_count_sg_trbs(pep, &preq->request);
775 	else
776 		num_trbs = cdns2_count_trbs(pep, preq->request.dma,
777 					    preq->request.length);
778 
779 	ret = cdns2_prepare_ring(pdev, pep, num_trbs);
780 	if (ret)
781 		return ret;
782 
783 	ring = &pep->ring;
784 	preq->start_trb = ring->enqueue;
785 	preq->trb = ring->trbs + ring->enqueue;
786 
787 	if (usb_endpoint_xfer_isoc(pep->endpoint.desc)) {
788 		cdns2_ep_tx_isoc(pep, preq, num_trbs);
789 	} else {
790 		togle_pcs = cdns2_wa1_update_guard(pep, ring->trbs + ring->enqueue);
791 		cdns2_ep_tx_bulk(pep, preq, num_trbs);
792 	}
793 
794 	preq->num_of_trb = num_trbs;
795 
796 	/*
797 	 * Memory barrier - cycle bit must be set as the last operation.
798 	 */
799 	dma_wmb();
800 
801 	/* Give the TD to the consumer. */
802 	if (togle_pcs)
803 		preq->trb->control = preq->trb->control ^ cpu_to_le32(1);
804 
805 	cdns2_wa1_tray_restore_cycle_bit(pdev, pep);
806 	cdns2_dbg_request_trbs(pep, preq);
807 
808 	if (!pep->wa1_set && !(pep->ep_state & EP_STALLED) && !pep->skip) {
809 		if (pep->type == USB_ENDPOINT_XFER_ISOC) {
810 			ret = cdns2_prepare_first_isoc_transfer(pdev, pep);
811 			if (ret)
812 				return 0;
813 		}
814 
815 		cdns2_set_drdy(pdev, pep);
816 	}
817 
818 	return 0;
819 }
820 
821 /* Prepare and start transfer for all not started requests. */
822 static int cdns2_start_all_request(struct cdns2_device *pdev,
823 				   struct cdns2_endpoint *pep)
824 {
825 	struct cdns2_request *preq;
826 	int ret;
827 
828 	while (!list_empty(&pep->deferred_list)) {
829 		preq = cdns2_next_preq(&pep->deferred_list);
830 
831 		ret = cdns2_ep_run_transfer(pep, preq);
832 		if (ret)
833 			return ret;
834 
835 		list_move_tail(&preq->list, &pep->pending_list);
836 	}
837 
838 	pep->ep_state &= ~EP_RING_FULL;
839 
840 	return 0;
841 }
842 
843 /*
844  * Check whether trb has been handled by DMA.
845  *
846  * Endpoint must be selected before invoking this function.
847  *
848  * Returns false if request has not been handled by DMA, else returns true.
849  *
850  * SR - start ring
851  * ER - end ring
852  * DQ = ring->dequeue - dequeue position
853  * EQ = ring->enqueue - enqueue position
854  * ST = preq->start_trb - index of first TRB in transfer ring
855  * ET = preq->end_trb - index of last TRB in transfer ring
856  * CI = current_index - index of processed TRB by DMA.
857  *
858  * As first step, we check if the TRB between the ST and ET.
859  * Then, we check if cycle bit for index pep->dequeue
860  * is correct.
861  *
862  * some rules:
863  * 1. ring->dequeue never equals to current_index.
864  * 2  ring->enqueue never exceed ring->dequeue
865  * 3. exception: ring->enqueue == ring->dequeue
866  *    and ring->free_trbs is zero.
867  *    This case indicate that TR is full.
868  *
869  * At below two cases, the request have been handled.
870  * Case 1 - ring->dequeue < current_index
871  *      SR ... EQ ... DQ ... CI ... ER
872  *      SR ... DQ ... CI ... EQ ... ER
873  *
874  * Case 2 - ring->dequeue > current_index
875  * This situation takes place when CI go through the LINK TRB at the end of
876  * transfer ring.
877  *      SR ... CI ... EQ ... DQ ... ER
878  */
879 static bool cdns2_trb_handled(struct cdns2_endpoint *pep,
880 			      struct cdns2_request *preq)
881 {
882 	struct cdns2_device *pdev = pep->pdev;
883 	struct cdns2_ring *ring;
884 	struct cdns2_trb *trb;
885 	int current_index = 0;
886 	int handled = 0;
887 	int doorbell;
888 
889 	ring = &pep->ring;
890 	current_index = cdns2_get_dma_pos(pdev, pep);
891 	doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
892 
893 	/*
894 	 * Only ISO transfer can use 2 entries outside the standard
895 	 * Transfer Ring. First of them is used as zero length packet and the
896 	 * second as LINK TRB.
897 	 */
898 	if (current_index >= TRBS_PER_SEGMENT)
899 		goto finish;
900 
901 	/* Current trb doesn't belong to this request. */
902 	if (preq->start_trb < preq->end_trb) {
903 		if (ring->dequeue > preq->end_trb)
904 			goto finish;
905 
906 		if (ring->dequeue < preq->start_trb)
907 			goto finish;
908 	}
909 
910 	if (preq->start_trb > preq->end_trb && ring->dequeue > preq->end_trb &&
911 	    ring->dequeue < preq->start_trb)
912 		goto finish;
913 
914 	if (preq->start_trb == preq->end_trb && ring->dequeue != preq->end_trb)
915 		goto finish;
916 
917 	trb = &ring->trbs[ring->dequeue];
918 
919 	if ((le32_to_cpu(trb->control) & TRB_CYCLE) != ring->ccs)
920 		goto finish;
921 
922 	if (doorbell == 1 && current_index == ring->dequeue)
923 		goto finish;
924 
925 	/* The corner case for TRBS_PER_SEGMENT equal 2). */
926 	if (TRBS_PER_SEGMENT == 2 && pep->type != USB_ENDPOINT_XFER_ISOC) {
927 		handled = 1;
928 		goto finish;
929 	}
930 
931 	if (ring->enqueue == ring->dequeue &&
932 	    ring->free_trbs == 0) {
933 		handled = 1;
934 	} else if (ring->dequeue < current_index) {
935 		if ((current_index == (TRBS_PER_SEGMENT - 1)) &&
936 		    !ring->dequeue)
937 			goto finish;
938 
939 		handled = 1;
940 	} else if (ring->dequeue  > current_index) {
941 		handled = 1;
942 	}
943 
944 finish:
945 	trace_cdns2_request_handled(preq, current_index, handled);
946 
947 	return handled;
948 }
949 
950 static void cdns2_skip_isoc_td(struct cdns2_device *pdev,
951 			       struct cdns2_endpoint *pep,
952 			       struct cdns2_request *preq)
953 {
954 	struct cdns2_trb *trb;
955 	int i;
956 
957 	trb = pep->ring.trbs + pep->ring.dequeue;
958 
959 	for (i = preq->finished_trb ; i < preq->num_of_trb; i++) {
960 		preq->finished_trb++;
961 		trace_cdns2_complete_trb(pep, trb);
962 		cdns2_ep_inc_deq(&pep->ring);
963 		trb = cdns2_next_trb(pep, trb);
964 	}
965 
966 	cdns2_gadget_giveback(pep, preq, 0);
967 	cdns2_prepare_first_isoc_transfer(pdev, pep);
968 	pep->skip = false;
969 	cdns2_set_drdy(pdev, pep);
970 }
971 
972 static void cdns2_transfer_completed(struct cdns2_device *pdev,
973 				     struct cdns2_endpoint *pep)
974 {
975 	struct cdns2_request *preq = NULL;
976 	bool request_handled = false;
977 	struct cdns2_trb *trb;
978 
979 	while (!list_empty(&pep->pending_list)) {
980 		preq = cdns2_next_preq(&pep->pending_list);
981 		trb = pep->ring.trbs + pep->ring.dequeue;
982 
983 		/*
984 		 * The TRB was changed as link TRB, and the request
985 		 * was handled at ep_dequeue.
986 		 */
987 		while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK &&
988 		       le32_to_cpu(trb->length)) {
989 			trace_cdns2_complete_trb(pep, trb);
990 			cdns2_ep_inc_deq(&pep->ring);
991 			trb = pep->ring.trbs + pep->ring.dequeue;
992 		}
993 
994 		/*
995 		 * Re-select endpoint. It could be changed by other CPU
996 		 * during handling usb_gadget_giveback_request.
997 		 */
998 		cdns2_select_ep(pdev, pep->endpoint.address);
999 
1000 		while (cdns2_trb_handled(pep, preq)) {
1001 			preq->finished_trb++;
1002 
1003 			if (preq->finished_trb >= preq->num_of_trb)
1004 				request_handled = true;
1005 
1006 			trb = pep->ring.trbs + pep->ring.dequeue;
1007 			trace_cdns2_complete_trb(pep, trb);
1008 
1009 			if (pep->dir && pep->type == USB_ENDPOINT_XFER_ISOC)
1010 				/*
1011 				 * For ISOC IN controller doens't update the
1012 				 * trb->length.
1013 				 */
1014 				preq->request.actual = preq->request.length;
1015 			else
1016 				preq->request.actual +=
1017 					TRB_LEN(le32_to_cpu(trb->length));
1018 
1019 			cdns2_ep_inc_deq(&pep->ring);
1020 		}
1021 
1022 		if (request_handled) {
1023 			cdns2_gadget_giveback(pep, preq, 0);
1024 			request_handled = false;
1025 		} else {
1026 			goto prepare_next_td;
1027 		}
1028 
1029 		if (pep->type != USB_ENDPOINT_XFER_ISOC &&
1030 		    TRBS_PER_SEGMENT == 2)
1031 			break;
1032 	}
1033 
1034 prepare_next_td:
1035 	if (pep->skip && preq)
1036 		cdns2_skip_isoc_td(pdev, pep, preq);
1037 
1038 	if (!(pep->ep_state & EP_STALLED) &&
1039 	    !(pep->ep_state & EP_STALL_PENDING))
1040 		cdns2_start_all_request(pdev, pep);
1041 }
1042 
1043 static void cdns2_wakeup(struct cdns2_device *pdev)
1044 {
1045 	if (!pdev->may_wakeup)
1046 		return;
1047 
1048 	/* Start driving resume signaling to indicate remote wakeup. */
1049 	set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_SIGRSUME);
1050 }
1051 
1052 static void cdns2_rearm_transfer(struct cdns2_endpoint *pep, u8 rearm)
1053 {
1054 	struct cdns2_device *pdev = pep->pdev;
1055 
1056 	cdns2_wa1_restore_cycle_bit(pep);
1057 
1058 	if (rearm) {
1059 		trace_cdns2_ring(pep);
1060 
1061 		/* Cycle Bit must be updated before arming DMA. */
1062 		dma_wmb();
1063 
1064 		writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
1065 
1066 		cdns2_wakeup(pdev);
1067 
1068 		trace_cdns2_doorbell_epx(pep,
1069 					 readl(&pdev->adma_regs->ep_traddr));
1070 	}
1071 }
1072 
1073 static void cdns2_handle_epx_interrupt(struct cdns2_endpoint *pep)
1074 {
1075 	struct cdns2_device *pdev = pep->pdev;
1076 	u8 isoerror = 0;
1077 	u32 ep_sts_reg;
1078 	u32 val;
1079 
1080 	cdns2_select_ep(pdev, pep->endpoint.address);
1081 
1082 	trace_cdns2_epx_irq(pdev, pep);
1083 
1084 	ep_sts_reg = readl(&pdev->adma_regs->ep_sts);
1085 	writel(ep_sts_reg, &pdev->adma_regs->ep_sts);
1086 
1087 	if (pep->type == USB_ENDPOINT_XFER_ISOC) {
1088 		u8 mult;
1089 		u8 cs;
1090 
1091 		mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize);
1092 		cs = pep->dir ? readb(&pdev->epx_regs->ep[pep->num - 1].txcs) :
1093 				readb(&pdev->epx_regs->ep[pep->num - 1].rxcs);
1094 		if (mult > 0)
1095 			isoerror = EPX_CS_ERR(cs);
1096 	}
1097 
1098 	/*
1099 	 * Sometimes ISO Error for mult=1 or mult=2 is not propagated on time
1100 	 * from USB module to DMA module. To protect against this driver
1101 	 * checks also the txcs/rxcs registers.
1102 	 */
1103 	if ((ep_sts_reg & DMA_EP_STS_ISOERR) || isoerror) {
1104 		clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
1105 
1106 		/* Wait for DBUSY cleared. */
1107 		readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val,
1108 					  !(val & DMA_EP_STS_DBUSY), 1, 125);
1109 
1110 		writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd);
1111 
1112 		/* Wait for DFLUSH cleared. */
1113 		readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val,
1114 					  !(val & DMA_EP_CMD_DFLUSH), 1, 10);
1115 
1116 		pep->skip = true;
1117 	}
1118 
1119 	if (ep_sts_reg & DMA_EP_STS_TRBERR || pep->skip) {
1120 		if (pep->ep_state & EP_STALL_PENDING &&
1121 		    !(ep_sts_reg & DMA_EP_STS_DESCMIS))
1122 			cdns2_ep_stall_flush(pep);
1123 
1124 		/*
1125 		 * For isochronous transfer driver completes request on
1126 		 * IOC or on TRBERR. IOC appears only when device receive
1127 		 * OUT data packet. If host disable stream or lost some packet
1128 		 * then the only way to finish all queued transfer is to do it
1129 		 * on TRBERR event.
1130 		 */
1131 		if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->wa1_set) {
1132 			if (!pep->dir)
1133 				clear_reg_bit_32(&pdev->adma_regs->ep_cfg,
1134 						 DMA_EP_CFG_ENABLE);
1135 
1136 			cdns2_transfer_completed(pdev, pep);
1137 			if (pep->ep_state & EP_DEFERRED_DRDY) {
1138 				pep->ep_state &= ~EP_DEFERRED_DRDY;
1139 				cdns2_set_drdy(pdev, pep);
1140 			}
1141 
1142 			return;
1143 		}
1144 
1145 		cdns2_transfer_completed(pdev, pep);
1146 
1147 		if (!(pep->ep_state & EP_STALLED) &&
1148 		    !(pep->ep_state & EP_STALL_PENDING)) {
1149 			if (pep->ep_state & EP_DEFERRED_DRDY) {
1150 				pep->ep_state &= ~EP_DEFERRED_DRDY;
1151 				cdns2_start_all_request(pdev, pep);
1152 			} else {
1153 				cdns2_rearm_transfer(pep, pep->wa1_set);
1154 			}
1155 		}
1156 
1157 		return;
1158 	}
1159 
1160 	if ((ep_sts_reg & DMA_EP_STS_IOC) || (ep_sts_reg & DMA_EP_STS_ISP))
1161 		cdns2_transfer_completed(pdev, pep);
1162 }
1163 
1164 static void cdns2_disconnect_gadget(struct cdns2_device *pdev)
1165 {
1166 	if (pdev->gadget_driver && pdev->gadget_driver->disconnect)
1167 		pdev->gadget_driver->disconnect(&pdev->gadget);
1168 }
1169 
1170 static irqreturn_t cdns2_usb_irq_handler(int irq, void *data)
1171 {
1172 	struct cdns2_device *pdev = data;
1173 	unsigned long reg_ep_ists;
1174 	u8 reg_usb_irq_m;
1175 	u8 reg_ext_irq_m;
1176 	u8 reg_usb_irq;
1177 	u8 reg_ext_irq;
1178 
1179 	if (pdev->in_lpm)
1180 		return IRQ_NONE;
1181 
1182 	reg_usb_irq_m = readb(&pdev->interrupt_regs->usbien);
1183 	reg_ext_irq_m = readb(&pdev->interrupt_regs->extien);
1184 
1185 	/* Mask all sources of interrupt. */
1186 	writeb(0, &pdev->interrupt_regs->usbien);
1187 	writeb(0, &pdev->interrupt_regs->extien);
1188 	writel(0, &pdev->adma_regs->ep_ien);
1189 
1190 	/* Clear interrupt sources. */
1191 	writel(0, &pdev->adma_regs->ep_sts);
1192 	writeb(0, &pdev->interrupt_regs->usbirq);
1193 	writeb(0, &pdev->interrupt_regs->extirq);
1194 
1195 	reg_ep_ists = readl(&pdev->adma_regs->ep_ists);
1196 	reg_usb_irq = readb(&pdev->interrupt_regs->usbirq);
1197 	reg_ext_irq = readb(&pdev->interrupt_regs->extirq);
1198 
1199 	if (reg_ep_ists || (reg_usb_irq & reg_usb_irq_m) ||
1200 	    (reg_ext_irq & reg_ext_irq_m))
1201 		return IRQ_WAKE_THREAD;
1202 
1203 	writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
1204 	writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
1205 	writel(~0, &pdev->adma_regs->ep_ien);
1206 
1207 	return IRQ_NONE;
1208 }
1209 
1210 static irqreturn_t cdns2_thread_usb_irq_handler(struct cdns2_device *pdev)
1211 {
1212 	u8 usb_irq, ext_irq;
1213 	int speed;
1214 	int i;
1215 
1216 	ext_irq = readb(&pdev->interrupt_regs->extirq) & EXTIRQ_WAKEUP;
1217 	writeb(ext_irq, &pdev->interrupt_regs->extirq);
1218 
1219 	usb_irq = readb(&pdev->interrupt_regs->usbirq) & USB_IEN_INIT;
1220 	writeb(usb_irq, &pdev->interrupt_regs->usbirq);
1221 
1222 	if (!ext_irq && !usb_irq)
1223 		return IRQ_NONE;
1224 
1225 	trace_cdns2_usb_irq(usb_irq, ext_irq);
1226 
1227 	if (ext_irq & EXTIRQ_WAKEUP) {
1228 		if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1229 			spin_unlock(&pdev->lock);
1230 			pdev->gadget_driver->resume(&pdev->gadget);
1231 			spin_lock(&pdev->lock);
1232 		}
1233 	}
1234 
1235 	if (usb_irq & USBIRQ_LPM) {
1236 		u8 reg = readb(&pdev->usb_regs->lpmctrl);
1237 
1238 		/* LPM1 enter */
1239 		if (!(reg & LPMCTRLLH_LPMNYET))
1240 			writeb(0, &pdev->usb_regs->sleep_clkgate);
1241 	}
1242 
1243 	if (usb_irq & USBIRQ_SUSPEND) {
1244 		if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1245 			spin_unlock(&pdev->lock);
1246 			pdev->gadget_driver->suspend(&pdev->gadget);
1247 			spin_lock(&pdev->lock);
1248 		}
1249 	}
1250 
1251 	if (usb_irq & USBIRQ_URESET) {
1252 		if (pdev->gadget_driver) {
1253 			pdev->dev_address = 0;
1254 
1255 			spin_unlock(&pdev->lock);
1256 			usb_gadget_udc_reset(&pdev->gadget,
1257 					     pdev->gadget_driver);
1258 			spin_lock(&pdev->lock);
1259 
1260 			/*
1261 			 * The USBIRQ_URESET is reported at the beginning of
1262 			 * reset signal. 100ms is enough time to finish reset
1263 			 * process. For high-speed reset procedure is completed
1264 			 * when controller detect HS mode.
1265 			 */
1266 			for (i = 0; i < 100; i++) {
1267 				mdelay(1);
1268 				speed = cdns2_get_speed(pdev);
1269 				if (speed == USB_SPEED_HIGH)
1270 					break;
1271 			}
1272 
1273 			pdev->gadget.speed = speed;
1274 			cdns2_enable_l1(pdev, 0);
1275 			cdns2_ep0_config(pdev);
1276 			pdev->may_wakeup = 0;
1277 		}
1278 	}
1279 
1280 	if (usb_irq & USBIRQ_SUDAV) {
1281 		pdev->ep0_stage = CDNS2_SETUP_STAGE;
1282 		cdns2_handle_setup_packet(pdev);
1283 	}
1284 
1285 	return IRQ_HANDLED;
1286 }
1287 
1288 /* Deferred USB interrupt handler. */
1289 static irqreturn_t cdns2_thread_irq_handler(int irq, void *data)
1290 {
1291 	struct cdns2_device *pdev = data;
1292 	unsigned long  dma_ep_ists;
1293 	unsigned long flags;
1294 	unsigned int bit;
1295 
1296 	local_bh_disable();
1297 	spin_lock_irqsave(&pdev->lock, flags);
1298 
1299 	cdns2_thread_usb_irq_handler(pdev);
1300 
1301 	dma_ep_ists = readl(&pdev->adma_regs->ep_ists);
1302 	if (!dma_ep_ists)
1303 		goto unlock;
1304 
1305 	trace_cdns2_dma_ep_ists(dma_ep_ists);
1306 
1307 	/* Handle default endpoint OUT. */
1308 	if (dma_ep_ists & DMA_EP_ISTS_EP_OUT0)
1309 		cdns2_handle_ep0_interrupt(pdev, USB_DIR_OUT);
1310 
1311 	/* Handle default endpoint IN. */
1312 	if (dma_ep_ists & DMA_EP_ISTS_EP_IN0)
1313 		cdns2_handle_ep0_interrupt(pdev, USB_DIR_IN);
1314 
1315 	dma_ep_ists &= ~(DMA_EP_ISTS_EP_OUT0 | DMA_EP_ISTS_EP_IN0);
1316 
1317 	for_each_set_bit(bit, &dma_ep_ists, sizeof(u32) * BITS_PER_BYTE) {
1318 		u8 ep_idx = bit > 16 ? (bit - 16) * 2 : (bit * 2) - 1;
1319 
1320 		/*
1321 		 * Endpoints in pdev->eps[] are held in order:
1322 		 * ep0, ep1out, ep1in, ep2out, ep2in... ep15out, ep15in.
1323 		 * but in dma_ep_ists in order:
1324 		 * ep0 ep1out ep2out ... ep15out ep0in ep1in .. ep15in
1325 		 */
1326 		cdns2_handle_epx_interrupt(&pdev->eps[ep_idx]);
1327 	}
1328 
1329 unlock:
1330 	writel(~0, &pdev->adma_regs->ep_ien);
1331 	writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
1332 	writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
1333 
1334 	spin_unlock_irqrestore(&pdev->lock, flags);
1335 	local_bh_enable();
1336 
1337 	return IRQ_HANDLED;
1338 }
1339 
1340 /* Calculates and assigns onchip memory for endpoints. */
1341 static void cdns2_eps_onchip_buffer_init(struct cdns2_device *pdev)
1342 {
1343 	struct cdns2_endpoint *pep;
1344 	int min_buf_tx = 0;
1345 	int min_buf_rx = 0;
1346 	u16 tx_offset = 0;
1347 	u16 rx_offset = 0;
1348 	int free;
1349 	int i;
1350 
1351 	for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
1352 		pep = &pdev->eps[i];
1353 
1354 		if (!(pep->ep_state & EP_CLAIMED))
1355 			continue;
1356 
1357 		if (pep->dir)
1358 			min_buf_tx += pep->buffering;
1359 		else
1360 			min_buf_rx += pep->buffering;
1361 	}
1362 
1363 	for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
1364 		pep = &pdev->eps[i];
1365 
1366 		if (!(pep->ep_state & EP_CLAIMED))
1367 			continue;
1368 
1369 		if (pep->dir) {
1370 			free = pdev->onchip_tx_buf - min_buf_tx;
1371 
1372 			if (free + pep->buffering >= 4)
1373 				free = 4;
1374 			else
1375 				free = free + pep->buffering;
1376 
1377 			min_buf_tx = min_buf_tx - pep->buffering + free;
1378 
1379 			pep->buffering = free;
1380 
1381 			writel(tx_offset,
1382 			       &pdev->epx_regs->txstaddr[pep->num - 1]);
1383 			pdev->epx_regs->txstaddr[pep->num - 1] = tx_offset;
1384 
1385 			dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n",
1386 				pep->name, tx_offset, pep->buffering);
1387 
1388 			tx_offset += pep->buffering * 1024;
1389 		} else {
1390 			free = pdev->onchip_rx_buf - min_buf_rx;
1391 
1392 			if (free + pep->buffering >= 4)
1393 				free = 4;
1394 			else
1395 				free = free + pep->buffering;
1396 
1397 			min_buf_rx = min_buf_rx - pep->buffering + free;
1398 
1399 			pep->buffering = free;
1400 			writel(rx_offset,
1401 			       &pdev->epx_regs->rxstaddr[pep->num - 1]);
1402 
1403 			dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n",
1404 				pep->name, rx_offset, pep->buffering);
1405 
1406 			rx_offset += pep->buffering * 1024;
1407 		}
1408 	}
1409 }
1410 
1411 /* Configure hardware endpoint. */
1412 static int cdns2_ep_config(struct cdns2_endpoint *pep, bool enable)
1413 {
1414 	bool is_iso_ep = (pep->type == USB_ENDPOINT_XFER_ISOC);
1415 	struct cdns2_device *pdev = pep->pdev;
1416 	u32 max_packet_size;
1417 	u8 dir = 0;
1418 	u8 ep_cfg;
1419 	u8 mult;
1420 	u32 val;
1421 	int ret;
1422 
1423 	switch (pep->type) {
1424 	case USB_ENDPOINT_XFER_INT:
1425 		ep_cfg = EPX_CON_TYPE_INT;
1426 		break;
1427 	case USB_ENDPOINT_XFER_BULK:
1428 		ep_cfg = EPX_CON_TYPE_BULK;
1429 		break;
1430 	default:
1431 		mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize);
1432 		ep_cfg = mult << EPX_CON_ISOD_SHIFT;
1433 		ep_cfg |= EPX_CON_TYPE_ISOC;
1434 
1435 		if (pep->dir) {
1436 			set_reg_bit_8(&pdev->epx_regs->isoautoarm, BIT(pep->num));
1437 			set_reg_bit_8(&pdev->epx_regs->isoautodump, BIT(pep->num));
1438 			set_reg_bit_8(&pdev->epx_regs->isodctrl, BIT(pep->num));
1439 		}
1440 	}
1441 
1442 	switch (pdev->gadget.speed) {
1443 	case USB_SPEED_FULL:
1444 		max_packet_size = is_iso_ep ? 1023 : 64;
1445 		break;
1446 	case USB_SPEED_HIGH:
1447 		max_packet_size = is_iso_ep ? 1024 : 512;
1448 		break;
1449 	default:
1450 		/* All other speed are not supported. */
1451 		return -EINVAL;
1452 	}
1453 
1454 	ep_cfg |= (EPX_CON_VAL | (pep->buffering - 1));
1455 
1456 	if (pep->dir) {
1457 		dir = FIFOCTRL_IO_TX;
1458 		writew(max_packet_size, &pdev->epx_regs->txmaxpack[pep->num - 1]);
1459 		writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].txcon);
1460 	} else {
1461 		writew(max_packet_size, &pdev->epx_regs->rxmaxpack[pep->num - 1]);
1462 		writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].rxcon);
1463 	}
1464 
1465 	writeb(pep->num | dir | FIFOCTRL_FIFOAUTO,
1466 	       &pdev->usb_regs->fifoctrl);
1467 	writeb(pep->num | dir, &pdev->epx_regs->endprst);
1468 	writeb(pep->num | ENDPRST_FIFORST | ENDPRST_TOGRST | dir,
1469 	       &pdev->epx_regs->endprst);
1470 
1471 	if (max_packet_size == 1024)
1472 		pep->trb_burst_size = 128;
1473 	else if (max_packet_size >= 512)
1474 		pep->trb_burst_size = 64;
1475 	else
1476 		pep->trb_burst_size = 16;
1477 
1478 	cdns2_select_ep(pdev, pep->num | pep->dir);
1479 	writel(DMA_EP_CMD_EPRST | DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd);
1480 
1481 	ret = readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
1482 					!(val & (DMA_EP_CMD_DFLUSH |
1483 					DMA_EP_CMD_EPRST)),
1484 					1, 1000);
1485 
1486 	if (ret)
1487 		return ret;
1488 
1489 	writel(DMA_EP_STS_TRBERR | DMA_EP_STS_ISOERR, &pdev->adma_regs->ep_sts_en);
1490 
1491 	if (enable)
1492 		writel(DMA_EP_CFG_ENABLE, &pdev->adma_regs->ep_cfg);
1493 
1494 	trace_cdns2_epx_hw_cfg(pdev, pep);
1495 
1496 	dev_dbg(pdev->dev, "Configure %s: with MPS: %08x, ep con: %02x\n",
1497 		pep->name, max_packet_size, ep_cfg);
1498 
1499 	return 0;
1500 }
1501 
1502 struct usb_request *cdns2_gadget_ep_alloc_request(struct usb_ep *ep,
1503 						  gfp_t gfp_flags)
1504 {
1505 	struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
1506 	struct cdns2_request *preq;
1507 
1508 	preq = kzalloc(sizeof(*preq), gfp_flags);
1509 	if (!preq)
1510 		return NULL;
1511 
1512 	preq->pep = pep;
1513 
1514 	trace_cdns2_alloc_request(preq);
1515 
1516 	return &preq->request;
1517 }
1518 
1519 void cdns2_gadget_ep_free_request(struct usb_ep *ep,
1520 				  struct usb_request *request)
1521 {
1522 	struct cdns2_request *preq = to_cdns2_request(request);
1523 
1524 	trace_cdns2_free_request(preq);
1525 	kfree(preq);
1526 }
1527 
1528 static int cdns2_gadget_ep_enable(struct usb_ep *ep,
1529 				  const struct usb_endpoint_descriptor *desc)
1530 {
1531 	u32 reg = DMA_EP_STS_EN_TRBERREN;
1532 	struct cdns2_endpoint *pep;
1533 	struct cdns2_device *pdev;
1534 	unsigned long flags;
1535 	int enable = 1;
1536 	int ret = 0;
1537 
1538 	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
1539 	    !desc->wMaxPacketSize) {
1540 		return -EINVAL;
1541 	}
1542 
1543 	pep = ep_to_cdns2_ep(ep);
1544 	pdev = pep->pdev;
1545 
1546 	if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
1547 			  "%s is already enabled\n", pep->name))
1548 		return 0;
1549 
1550 	spin_lock_irqsave(&pdev->lock, flags);
1551 
1552 	pep->type = usb_endpoint_type(desc);
1553 	pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1554 
1555 	if (pdev->gadget.speed == USB_SPEED_FULL)
1556 		if (pep->type == USB_ENDPOINT_XFER_INT)
1557 			pep->interval = desc->bInterval;
1558 
1559 	if (pep->interval > ISO_MAX_INTERVAL &&
1560 	    pep->type == USB_ENDPOINT_XFER_ISOC) {
1561 		dev_err(pdev->dev, "ISO period is limited to %d (current: %d)\n",
1562 			ISO_MAX_INTERVAL, pep->interval);
1563 
1564 		ret =  -EINVAL;
1565 		goto exit;
1566 	}
1567 
1568 	/*
1569 	 * During ISO OUT traffic DMA reads Transfer Ring for the EP which has
1570 	 * never got doorbell.
1571 	 * This issue was detected only on simulation, but to avoid this issue
1572 	 * driver add protection against it. To fix it driver enable ISO OUT
1573 	 * endpoint before setting DRBL. This special treatment of ISO OUT
1574 	 * endpoints are recommended by controller specification.
1575 	 */
1576 	if (pep->type == USB_ENDPOINT_XFER_ISOC  && !pep->dir)
1577 		enable = 0;
1578 
1579 	ret = cdns2_alloc_tr_segment(pep);
1580 	if (ret)
1581 		goto exit;
1582 
1583 	ret = cdns2_ep_config(pep, enable);
1584 	if (ret) {
1585 		cdns2_free_tr_segment(pep);
1586 		ret =  -EINVAL;
1587 		goto exit;
1588 	}
1589 
1590 	trace_cdns2_gadget_ep_enable(pep);
1591 
1592 	pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING);
1593 	pep->ep_state |= EP_ENABLED;
1594 	pep->wa1_set = 0;
1595 	pep->ring.enqueue = 0;
1596 	pep->ring.dequeue = 0;
1597 	reg = readl(&pdev->adma_regs->ep_sts);
1598 	pep->ring.pcs = !!DMA_EP_STS_CCS(reg);
1599 	pep->ring.ccs = !!DMA_EP_STS_CCS(reg);
1600 
1601 	writel(pep->ring.dma, &pdev->adma_regs->ep_traddr);
1602 
1603 	/* one TRB is reserved for link TRB used in DMULT mode*/
1604 	pep->ring.free_trbs = TRBS_PER_SEGMENT - 1;
1605 
1606 exit:
1607 	spin_unlock_irqrestore(&pdev->lock, flags);
1608 
1609 	return ret;
1610 }
1611 
1612 static int cdns2_gadget_ep_disable(struct usb_ep *ep)
1613 {
1614 	struct cdns2_endpoint *pep;
1615 	struct cdns2_request *preq;
1616 	struct cdns2_device *pdev;
1617 	unsigned long flags;
1618 	int val;
1619 
1620 	if (!ep)
1621 		return -EINVAL;
1622 
1623 	pep = ep_to_cdns2_ep(ep);
1624 	pdev = pep->pdev;
1625 
1626 	if (dev_WARN_ONCE(pdev->dev, !(pep->ep_state & EP_ENABLED),
1627 			  "%s is already disabled\n", pep->name))
1628 		return 0;
1629 
1630 	spin_lock_irqsave(&pdev->lock, flags);
1631 
1632 	trace_cdns2_gadget_ep_disable(pep);
1633 
1634 	cdns2_select_ep(pdev, ep->desc->bEndpointAddress);
1635 
1636 	clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
1637 
1638 	/*
1639 	 * Driver needs some time before resetting endpoint.
1640 	 * It need waits for clearing DBUSY bit or for timeout expired.
1641 	 * 10us is enough time for controller to stop transfer.
1642 	 */
1643 	readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val,
1644 				  !(val & DMA_EP_STS_DBUSY), 1, 10);
1645 	writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd);
1646 
1647 	readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
1648 				  !(val & (DMA_EP_CMD_DFLUSH | DMA_EP_CMD_EPRST)),
1649 				  1, 1000);
1650 
1651 	while (!list_empty(&pep->pending_list)) {
1652 		preq = cdns2_next_preq(&pep->pending_list);
1653 		cdns2_gadget_giveback(pep, preq, -ESHUTDOWN);
1654 	}
1655 
1656 	while (!list_empty(&pep->deferred_list)) {
1657 		preq = cdns2_next_preq(&pep->deferred_list);
1658 		cdns2_gadget_giveback(pep, preq, -ESHUTDOWN);
1659 	}
1660 
1661 	ep->desc = NULL;
1662 	pep->ep_state &= ~EP_ENABLED;
1663 
1664 	spin_unlock_irqrestore(&pdev->lock, flags);
1665 
1666 	return 0;
1667 }
1668 
1669 static int cdns2_ep_enqueue(struct cdns2_endpoint *pep,
1670 			    struct cdns2_request *preq,
1671 			    gfp_t gfp_flags)
1672 {
1673 	struct cdns2_device *pdev = pep->pdev;
1674 	struct usb_request *request;
1675 	int ret;
1676 
1677 	request = &preq->request;
1678 	request->actual = 0;
1679 	request->status = -EINPROGRESS;
1680 
1681 	ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->dir);
1682 	if (ret) {
1683 		trace_cdns2_request_enqueue_error(preq);
1684 		return ret;
1685 	}
1686 
1687 	list_add_tail(&preq->list, &pep->deferred_list);
1688 	trace_cdns2_request_enqueue(preq);
1689 
1690 	if (!(pep->ep_state & EP_STALLED) && !(pep->ep_state & EP_STALL_PENDING))
1691 		cdns2_start_all_request(pdev, pep);
1692 
1693 	return 0;
1694 }
1695 
1696 static int cdns2_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1697 				 gfp_t gfp_flags)
1698 {
1699 	struct usb_request *zlp_request;
1700 	struct cdns2_request *preq;
1701 	struct cdns2_endpoint *pep;
1702 	struct cdns2_device *pdev;
1703 	unsigned long flags;
1704 	int ret;
1705 
1706 	if (!request || !ep)
1707 		return -EINVAL;
1708 
1709 	pep = ep_to_cdns2_ep(ep);
1710 	pdev = pep->pdev;
1711 
1712 	if (!(pep->ep_state & EP_ENABLED)) {
1713 		dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1714 			pep->name);
1715 		return -EINVAL;
1716 	}
1717 
1718 	spin_lock_irqsave(&pdev->lock, flags);
1719 
1720 	preq =  to_cdns2_request(request);
1721 	ret = cdns2_ep_enqueue(pep, preq, gfp_flags);
1722 
1723 	if (ret == 0 && request->zero && request->length &&
1724 	    (request->length % ep->maxpacket == 0)) {
1725 		struct cdns2_request *preq;
1726 
1727 		zlp_request = cdns2_gadget_ep_alloc_request(ep, GFP_ATOMIC);
1728 		zlp_request->buf = pdev->zlp_buf;
1729 		zlp_request->length = 0;
1730 
1731 		preq = to_cdns2_request(zlp_request);
1732 		ret = cdns2_ep_enqueue(pep, preq, gfp_flags);
1733 	}
1734 
1735 	spin_unlock_irqrestore(&pdev->lock, flags);
1736 	return ret;
1737 }
1738 
1739 int cdns2_gadget_ep_dequeue(struct usb_ep *ep,
1740 			    struct usb_request *request)
1741 {
1742 	struct cdns2_request *preq, *preq_temp, *cur_preq;
1743 	struct cdns2_endpoint *pep;
1744 	struct cdns2_trb *link_trb;
1745 	u8 req_on_hw_ring = 0;
1746 	unsigned long flags;
1747 	u32 buffer;
1748 	int val, i;
1749 
1750 	if (!ep || !request || !ep->desc)
1751 		return -EINVAL;
1752 
1753 	pep = ep_to_cdns2_ep(ep);
1754 	if (!pep->endpoint.desc) {
1755 		dev_err(pep->pdev->dev, "%s: can't dequeue to disabled endpoint\n",
1756 			pep->name);
1757 		return -ESHUTDOWN;
1758 	}
1759 
1760 	/* Requests has been dequeued during disabling endpoint. */
1761 	if (!(pep->ep_state & EP_ENABLED))
1762 		return 0;
1763 
1764 	spin_lock_irqsave(&pep->pdev->lock, flags);
1765 
1766 	cur_preq = to_cdns2_request(request);
1767 	trace_cdns2_request_dequeue(cur_preq);
1768 
1769 	list_for_each_entry_safe(preq, preq_temp, &pep->pending_list, list) {
1770 		if (cur_preq == preq) {
1771 			req_on_hw_ring = 1;
1772 			goto found;
1773 		}
1774 	}
1775 
1776 	list_for_each_entry_safe(preq, preq_temp, &pep->deferred_list, list) {
1777 		if (cur_preq == preq)
1778 			goto found;
1779 	}
1780 
1781 	goto not_found;
1782 
1783 found:
1784 	link_trb = preq->trb;
1785 
1786 	/* Update ring only if removed request is on pending_req_list list. */
1787 	if (req_on_hw_ring && link_trb) {
1788 		/* Stop DMA */
1789 		writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd);
1790 
1791 		/* Wait for DFLUSH cleared. */
1792 		readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val,
1793 					  !(val & DMA_EP_CMD_DFLUSH), 1, 1000);
1794 
1795 		buffer = cpu_to_le32(TRB_BUFFER(pep->ring.dma +
1796 				    ((preq->end_trb + 1) * TRB_SIZE)));
1797 
1798 		for (i = 0; i < preq->num_of_trb; i++) {
1799 			link_trb->buffer = buffer;
1800 			link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control)
1801 					    & TRB_CYCLE) | TRB_CHAIN |
1802 					    TRB_TYPE(TRB_LINK));
1803 
1804 			trace_cdns2_queue_trb(pep, link_trb);
1805 			link_trb = cdns2_next_trb(pep, link_trb);
1806 		}
1807 
1808 		if (pep->wa1_trb == preq->trb)
1809 			cdns2_wa1_restore_cycle_bit(pep);
1810 	}
1811 
1812 	cdns2_gadget_giveback(pep, cur_preq, -ECONNRESET);
1813 
1814 	preq = cdns2_next_preq(&pep->pending_list);
1815 	if (preq)
1816 		cdns2_rearm_transfer(pep, 1);
1817 
1818 not_found:
1819 	spin_unlock_irqrestore(&pep->pdev->lock, flags);
1820 	return 0;
1821 }
1822 
1823 int cdns2_halt_endpoint(struct cdns2_device *pdev,
1824 			struct cdns2_endpoint *pep,
1825 			int value)
1826 {
1827 	u8 __iomem *conf;
1828 	int dir = 0;
1829 
1830 	if (!(pep->ep_state & EP_ENABLED))
1831 		return -EPERM;
1832 
1833 	if (pep->dir) {
1834 		dir = ENDPRST_IO_TX;
1835 		conf = &pdev->epx_regs->ep[pep->num - 1].txcon;
1836 	} else {
1837 		conf = &pdev->epx_regs->ep[pep->num - 1].rxcon;
1838 	}
1839 
1840 	if (!value) {
1841 		struct cdns2_trb *trb = NULL;
1842 		struct cdns2_request *preq;
1843 		struct cdns2_trb trb_tmp;
1844 
1845 		preq = cdns2_next_preq(&pep->pending_list);
1846 		if (preq) {
1847 			trb = preq->trb;
1848 			if (trb) {
1849 				trb_tmp = *trb;
1850 				trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
1851 			}
1852 		}
1853 
1854 		trace_cdns2_ep_halt(pep, 0, 0);
1855 
1856 		/* Resets Sequence Number */
1857 		writeb(dir | pep->num, &pdev->epx_regs->endprst);
1858 		writeb(dir | ENDPRST_TOGRST | pep->num,
1859 		       &pdev->epx_regs->endprst);
1860 
1861 		clear_reg_bit_8(conf, EPX_CON_STALL);
1862 
1863 		pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING);
1864 
1865 		if (preq) {
1866 			if (trb)
1867 				*trb = trb_tmp;
1868 
1869 			cdns2_rearm_transfer(pep, 1);
1870 		}
1871 
1872 		cdns2_start_all_request(pdev, pep);
1873 	} else {
1874 		trace_cdns2_ep_halt(pep, 1, 0);
1875 		set_reg_bit_8(conf, EPX_CON_STALL);
1876 		writeb(dir | pep->num, &pdev->epx_regs->endprst);
1877 		writeb(dir | ENDPRST_FIFORST | pep->num,
1878 		       &pdev->epx_regs->endprst);
1879 		pep->ep_state |= EP_STALLED;
1880 	}
1881 
1882 	return 0;
1883 }
1884 
1885 /* Sets/clears stall on selected endpoint. */
1886 static int cdns2_gadget_ep_set_halt(struct usb_ep *ep, int value)
1887 {
1888 	struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
1889 	struct cdns2_device *pdev = pep->pdev;
1890 	struct cdns2_request *preq;
1891 	unsigned long flags = 0;
1892 	int ret;
1893 
1894 	spin_lock_irqsave(&pdev->lock, flags);
1895 
1896 	preq = cdns2_next_preq(&pep->pending_list);
1897 	if (value && preq) {
1898 		trace_cdns2_ep_busy_try_halt_again(pep);
1899 		ret = -EAGAIN;
1900 		goto done;
1901 	}
1902 
1903 	if (!value)
1904 		pep->ep_state &= ~EP_WEDGE;
1905 
1906 	ret = cdns2_halt_endpoint(pdev, pep, value);
1907 
1908 done:
1909 	spin_unlock_irqrestore(&pdev->lock, flags);
1910 	return ret;
1911 }
1912 
1913 static int cdns2_gadget_ep_set_wedge(struct usb_ep *ep)
1914 {
1915 	struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
1916 
1917 	cdns2_gadget_ep_set_halt(ep, 1);
1918 	pep->ep_state |= EP_WEDGE;
1919 
1920 	return 0;
1921 }
1922 
1923 static struct
1924 cdns2_endpoint *cdns2_find_available_ep(struct cdns2_device *pdev,
1925 					struct usb_endpoint_descriptor *desc)
1926 {
1927 	struct cdns2_endpoint *pep;
1928 	struct usb_ep *ep;
1929 	int ep_correct;
1930 
1931 	list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) {
1932 		unsigned long num;
1933 		int ret;
1934 		/* ep name pattern likes epXin or epXout. */
1935 		char c[2] = {ep->name[2], '\0'};
1936 
1937 		ret = kstrtoul(c, 10, &num);
1938 		if (ret)
1939 			return ERR_PTR(ret);
1940 		pep = ep_to_cdns2_ep(ep);
1941 
1942 		if (pep->num != num)
1943 			continue;
1944 
1945 		ep_correct = (pep->endpoint.caps.dir_in &&
1946 			      usb_endpoint_dir_in(desc)) ||
1947 			     (pep->endpoint.caps.dir_out &&
1948 			      usb_endpoint_dir_out(desc));
1949 
1950 		if (ep_correct && !(pep->ep_state & EP_CLAIMED))
1951 			return pep;
1952 	}
1953 
1954 	return ERR_PTR(-ENOENT);
1955 }
1956 
1957 /*
1958  * Function used to recognize which endpoints will be used to optimize
1959  * on-chip memory usage.
1960  */
1961 static struct
1962 usb_ep *cdns2_gadget_match_ep(struct usb_gadget *gadget,
1963 			      struct usb_endpoint_descriptor *desc,
1964 			      struct usb_ss_ep_comp_descriptor *comp_desc)
1965 {
1966 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
1967 	struct cdns2_endpoint *pep;
1968 	unsigned long flags;
1969 
1970 	pep = cdns2_find_available_ep(pdev, desc);
1971 	if (IS_ERR(pep)) {
1972 		dev_err(pdev->dev, "no available ep\n");
1973 		return NULL;
1974 	}
1975 
1976 	spin_lock_irqsave(&pdev->lock, flags);
1977 
1978 	if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
1979 		pep->buffering = 4;
1980 	else
1981 		pep->buffering = 1;
1982 
1983 	pep->ep_state |= EP_CLAIMED;
1984 	spin_unlock_irqrestore(&pdev->lock, flags);
1985 
1986 	return &pep->endpoint;
1987 }
1988 
1989 static const struct usb_ep_ops cdns2_gadget_ep_ops = {
1990 	.enable = cdns2_gadget_ep_enable,
1991 	.disable = cdns2_gadget_ep_disable,
1992 	.alloc_request = cdns2_gadget_ep_alloc_request,
1993 	.free_request = cdns2_gadget_ep_free_request,
1994 	.queue = cdns2_gadget_ep_queue,
1995 	.dequeue = cdns2_gadget_ep_dequeue,
1996 	.set_halt = cdns2_gadget_ep_set_halt,
1997 	.set_wedge = cdns2_gadget_ep_set_wedge,
1998 };
1999 
2000 static int cdns2_gadget_get_frame(struct usb_gadget *gadget)
2001 {
2002 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2003 
2004 	return readw(&pdev->usb_regs->frmnr);
2005 }
2006 
2007 static int cdns2_gadget_wakeup(struct usb_gadget *gadget)
2008 {
2009 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2010 	unsigned long flags;
2011 
2012 	spin_lock_irqsave(&pdev->lock, flags);
2013 	cdns2_wakeup(pdev);
2014 	spin_unlock_irqrestore(&pdev->lock, flags);
2015 
2016 	return 0;
2017 }
2018 
2019 static int cdns2_gadget_set_selfpowered(struct usb_gadget *gadget,
2020 					int is_selfpowered)
2021 {
2022 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2023 	unsigned long flags;
2024 
2025 	spin_lock_irqsave(&pdev->lock, flags);
2026 	pdev->is_selfpowered = !!is_selfpowered;
2027 	spin_unlock_irqrestore(&pdev->lock, flags);
2028 	return 0;
2029 }
2030 
2031 /*  Disable interrupts and begin the controller halting process. */
2032 static void cdns2_quiesce(struct cdns2_device *pdev)
2033 {
2034 	set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2035 
2036 	/* Disable interrupt. */
2037 	writeb(0, &pdev->interrupt_regs->extien);
2038 	writeb(0, &pdev->interrupt_regs->usbien);
2039 	writew(0, &pdev->adma_regs->ep_ien);
2040 
2041 	/* Clear interrupt line. */
2042 	writeb(0x0, &pdev->interrupt_regs->usbirq);
2043 }
2044 
2045 static void cdns2_gadget_config(struct cdns2_device *pdev)
2046 {
2047 	cdns2_ep0_config(pdev);
2048 
2049 	/* Enable DMA interrupts for all endpoints. */
2050 	writel(~0x0, &pdev->adma_regs->ep_ien);
2051 	cdns2_enable_l1(pdev, 0);
2052 	writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
2053 	writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
2054 	writel(DMA_CONF_DMULT, &pdev->adma_regs->conf);
2055 }
2056 
2057 static int cdns2_gadget_pullup(struct usb_gadget *gadget, int is_on)
2058 {
2059 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2060 	unsigned long flags;
2061 
2062 	trace_cdns2_pullup(is_on);
2063 
2064 	/*
2065 	 * Disable events handling while controller is being
2066 	 * enabled/disabled.
2067 	 */
2068 	disable_irq(pdev->irq);
2069 	spin_lock_irqsave(&pdev->lock, flags);
2070 
2071 	if (is_on) {
2072 		cdns2_gadget_config(pdev);
2073 		clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2074 	} else {
2075 		cdns2_quiesce(pdev);
2076 	}
2077 
2078 	spin_unlock_irqrestore(&pdev->lock, flags);
2079 	enable_irq(pdev->irq);
2080 
2081 	return 0;
2082 }
2083 
2084 static int cdns2_gadget_udc_start(struct usb_gadget *gadget,
2085 				  struct usb_gadget_driver *driver)
2086 {
2087 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2088 	enum usb_device_speed max_speed = driver->max_speed;
2089 	unsigned long flags;
2090 
2091 	spin_lock_irqsave(&pdev->lock, flags);
2092 	pdev->gadget_driver = driver;
2093 
2094 	/* Limit speed if necessary. */
2095 	max_speed = min(driver->max_speed, gadget->max_speed);
2096 
2097 	switch (max_speed) {
2098 	case USB_SPEED_FULL:
2099 		writeb(SPEEDCTRL_HSDISABLE, &pdev->usb_regs->speedctrl);
2100 		break;
2101 	case USB_SPEED_HIGH:
2102 		writeb(0, &pdev->usb_regs->speedctrl);
2103 		break;
2104 	default:
2105 		dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
2106 			max_speed);
2107 		fallthrough;
2108 	case USB_SPEED_UNKNOWN:
2109 		/* Default to highspeed. */
2110 		max_speed = USB_SPEED_HIGH;
2111 		break;
2112 	}
2113 
2114 	/* Reset all USB endpoints. */
2115 	writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst);
2116 	writeb(ENDPRST_FIFORST | ENDPRST_TOGRST | ENDPRST_IO_TX,
2117 	       &pdev->usb_regs->endprst);
2118 	writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->usb_regs->endprst);
2119 
2120 	cdns2_eps_onchip_buffer_init(pdev);
2121 
2122 	cdns2_gadget_config(pdev);
2123 	spin_unlock_irqrestore(&pdev->lock, flags);
2124 
2125 	return 0;
2126 }
2127 
2128 static int cdns2_gadget_udc_stop(struct usb_gadget *gadget)
2129 {
2130 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2131 	struct cdns2_endpoint *pep;
2132 	u32 bEndpointAddress;
2133 	struct usb_ep *ep;
2134 	int val;
2135 
2136 	pdev->gadget_driver = NULL;
2137 	pdev->gadget.speed = USB_SPEED_UNKNOWN;
2138 
2139 	list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) {
2140 		pep = ep_to_cdns2_ep(ep);
2141 		bEndpointAddress = pep->num | pep->dir;
2142 		cdns2_select_ep(pdev, bEndpointAddress);
2143 		writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd);
2144 		readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
2145 					  !(val & DMA_EP_CMD_EPRST), 1, 100);
2146 	}
2147 
2148 	cdns2_quiesce(pdev);
2149 
2150 	writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst);
2151 	writeb(ENDPRST_FIFORST | ENDPRST_TOGRST | ENDPRST_IO_TX,
2152 	       &pdev->epx_regs->endprst);
2153 	writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->epx_regs->endprst);
2154 
2155 	return 0;
2156 }
2157 
2158 static const struct usb_gadget_ops cdns2_gadget_ops = {
2159 	.get_frame = cdns2_gadget_get_frame,
2160 	.wakeup = cdns2_gadget_wakeup,
2161 	.set_selfpowered = cdns2_gadget_set_selfpowered,
2162 	.pullup = cdns2_gadget_pullup,
2163 	.udc_start = cdns2_gadget_udc_start,
2164 	.udc_stop = cdns2_gadget_udc_stop,
2165 	.match_ep = cdns2_gadget_match_ep,
2166 };
2167 
2168 static void cdns2_free_all_eps(struct cdns2_device *pdev)
2169 {
2170 	int i;
2171 
2172 	for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++)
2173 		cdns2_free_tr_segment(&pdev->eps[i]);
2174 }
2175 
2176 /* Initializes software endpoints of gadget. */
2177 static int cdns2_init_eps(struct cdns2_device *pdev)
2178 {
2179 	struct cdns2_endpoint *pep;
2180 	int i;
2181 
2182 	for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
2183 		bool direction = !(i & 1); /* Start from OUT endpoint. */
2184 		u8 epnum = ((i + 1) >> 1);
2185 
2186 		/*
2187 		 * Endpoints are being held in pdev->eps[] in form:
2188 		 * ep0, ep1out, ep1in ... ep15out, ep15in.
2189 		 */
2190 		if (!CDNS2_IF_EP_EXIST(pdev, epnum, direction))
2191 			continue;
2192 
2193 		pep = &pdev->eps[i];
2194 		pep->pdev = pdev;
2195 		pep->num = epnum;
2196 		/* 0 for OUT, 1 for IN. */
2197 		pep->dir = direction ? USB_DIR_IN : USB_DIR_OUT;
2198 		pep->idx = i;
2199 
2200 		/* Ep0in and ep0out are represented by pdev->eps[0]. */
2201 		if (!epnum) {
2202 			int ret;
2203 
2204 			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
2205 				 epnum, "BiDir");
2206 
2207 			cdns2_init_ep0(pdev, pep);
2208 
2209 			ret = cdns2_alloc_tr_segment(pep);
2210 			if (ret) {
2211 				dev_err(pdev->dev, "Failed to init ep0\n");
2212 				return ret;
2213 			}
2214 		} else {
2215 			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
2216 				 epnum, !!direction ? "in" : "out");
2217 			pep->endpoint.name = pep->name;
2218 
2219 			usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
2220 			pep->endpoint.ops = &cdns2_gadget_ep_ops;
2221 			list_add_tail(&pep->endpoint.ep_list, &pdev->gadget.ep_list);
2222 
2223 			pep->endpoint.caps.dir_in = direction;
2224 			pep->endpoint.caps.dir_out = !direction;
2225 
2226 			pep->endpoint.caps.type_iso = 1;
2227 			pep->endpoint.caps.type_bulk = 1;
2228 			pep->endpoint.caps.type_int = 1;
2229 		}
2230 
2231 		pep->endpoint.name = pep->name;
2232 		pep->ep_state = 0;
2233 
2234 		dev_dbg(pdev->dev, "Init %s, SupType: CTRL: %s, INT: %s, "
2235 			"BULK: %s, ISOC %s, SupDir IN: %s, OUT: %s\n",
2236 			pep->name,
2237 			str_yes_no(pep->endpoint.caps.type_control),
2238 			str_yes_no(pep->endpoint.caps.type_int),
2239 			str_yes_no(pep->endpoint.caps.type_bulk),
2240 			str_yes_no(pep->endpoint.caps.type_iso),
2241 			str_yes_no(pep->endpoint.caps.dir_in),
2242 			str_yes_no(pep->endpoint.caps.dir_out));
2243 
2244 		INIT_LIST_HEAD(&pep->pending_list);
2245 		INIT_LIST_HEAD(&pep->deferred_list);
2246 	}
2247 
2248 	return 0;
2249 }
2250 
2251 static int cdns2_gadget_start(struct cdns2_device *pdev)
2252 {
2253 	u32 max_speed;
2254 	void *buf;
2255 	int ret;
2256 
2257 	pdev->usb_regs = pdev->regs;
2258 	pdev->ep0_regs = pdev->regs;
2259 	pdev->epx_regs = pdev->regs;
2260 	pdev->interrupt_regs = pdev->regs;
2261 	pdev->adma_regs = pdev->regs + CDNS2_ADMA_REGS_OFFSET;
2262 
2263 	/* Reset controller. */
2264 	writeb(CPUCTRL_SW_RST | CPUCTRL_UPCLK | CPUCTRL_WUEN,
2265 	       &pdev->usb_regs->cpuctrl);
2266 	usleep_range(5, 10);
2267 
2268 	usb_initialize_gadget(pdev->dev, &pdev->gadget, NULL);
2269 
2270 	device_property_read_u16(pdev->dev, "cdns,on-chip-tx-buff-size",
2271 				 &pdev->onchip_tx_buf);
2272 	device_property_read_u16(pdev->dev, "cdns,on-chip-rx-buff-size",
2273 				 &pdev->onchip_rx_buf);
2274 	device_property_read_u32(pdev->dev, "cdns,avail-endpoints",
2275 				 &pdev->eps_supported);
2276 
2277 	/*
2278 	 * Driver assumes that each USBHS controller has at least
2279 	 * one IN and one OUT non control endpoint.
2280 	 */
2281 	if (!pdev->onchip_tx_buf && !pdev->onchip_rx_buf) {
2282 		ret = -EINVAL;
2283 		dev_err(pdev->dev, "Invalid on-chip memory configuration\n");
2284 		goto put_gadget;
2285 	}
2286 
2287 	if (!(pdev->eps_supported & ~0x00010001)) {
2288 		ret = -EINVAL;
2289 		dev_err(pdev->dev, "No hardware endpoints available\n");
2290 		goto put_gadget;
2291 	}
2292 
2293 	max_speed = usb_get_maximum_speed(pdev->dev);
2294 
2295 	switch (max_speed) {
2296 	case USB_SPEED_FULL:
2297 	case USB_SPEED_HIGH:
2298 		break;
2299 	default:
2300 		dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
2301 			max_speed);
2302 		fallthrough;
2303 	case USB_SPEED_UNKNOWN:
2304 		max_speed = USB_SPEED_HIGH;
2305 		break;
2306 	}
2307 
2308 	pdev->gadget.max_speed = max_speed;
2309 	pdev->gadget.speed = USB_SPEED_UNKNOWN;
2310 	pdev->gadget.ops = &cdns2_gadget_ops;
2311 	pdev->gadget.name = "usbhs-gadget";
2312 	pdev->gadget.quirk_avoids_skb_reserve = 1;
2313 	pdev->gadget.irq = pdev->irq;
2314 
2315 	spin_lock_init(&pdev->lock);
2316 	INIT_WORK(&pdev->pending_status_wq, cdns2_pending_setup_status_handler);
2317 
2318 	/* Initialize endpoint container. */
2319 	INIT_LIST_HEAD(&pdev->gadget.ep_list);
2320 	pdev->eps_dma_pool = dma_pool_create("cdns2_eps_dma_pool", pdev->dev,
2321 					     TR_SEG_SIZE, 8, 0);
2322 	if (!pdev->eps_dma_pool) {
2323 		dev_err(pdev->dev, "Failed to create TRB dma pool\n");
2324 		ret = -ENOMEM;
2325 		goto put_gadget;
2326 	}
2327 
2328 	ret = cdns2_init_eps(pdev);
2329 	if (ret) {
2330 		dev_err(pdev->dev, "Failed to create endpoints\n");
2331 		goto destroy_dma_pool;
2332 	}
2333 
2334 	pdev->gadget.sg_supported = 1;
2335 
2336 	pdev->zlp_buf = kzalloc(CDNS2_EP_ZLP_BUF_SIZE, GFP_KERNEL);
2337 	if (!pdev->zlp_buf) {
2338 		ret = -ENOMEM;
2339 		goto destroy_dma_pool;
2340 	}
2341 
2342 	/* Allocate memory for setup packet buffer. */
2343 	buf = dma_alloc_coherent(pdev->dev, 8, &pdev->ep0_preq.request.dma,
2344 				 GFP_DMA);
2345 	pdev->ep0_preq.request.buf = buf;
2346 
2347 	if (!pdev->ep0_preq.request.buf) {
2348 		ret = -ENOMEM;
2349 		goto free_zlp_buf;
2350 	}
2351 
2352 	/* Add USB gadget device. */
2353 	ret = usb_add_gadget(&pdev->gadget);
2354 	if (ret < 0) {
2355 		dev_err(pdev->dev, "Failed to add gadget\n");
2356 		goto free_ep0_buf;
2357 	}
2358 
2359 	return 0;
2360 
2361 free_ep0_buf:
2362 	dma_free_coherent(pdev->dev, 8, pdev->ep0_preq.request.buf,
2363 			  pdev->ep0_preq.request.dma);
2364 free_zlp_buf:
2365 	kfree(pdev->zlp_buf);
2366 destroy_dma_pool:
2367 	dma_pool_destroy(pdev->eps_dma_pool);
2368 put_gadget:
2369 	usb_put_gadget(&pdev->gadget);
2370 
2371 	return ret;
2372 }
2373 
2374 int cdns2_gadget_suspend(struct cdns2_device *pdev)
2375 {
2376 	unsigned long flags;
2377 
2378 	cdns2_disconnect_gadget(pdev);
2379 
2380 	spin_lock_irqsave(&pdev->lock, flags);
2381 	pdev->gadget.speed = USB_SPEED_UNKNOWN;
2382 
2383 	trace_cdns2_device_state("notattached");
2384 	usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
2385 	cdns2_enable_l1(pdev, 0);
2386 
2387 	/* Disable interrupt for device. */
2388 	writeb(0, &pdev->interrupt_regs->usbien);
2389 	writel(0, &pdev->adma_regs->ep_ien);
2390 	spin_unlock_irqrestore(&pdev->lock, flags);
2391 
2392 	return 0;
2393 }
2394 
2395 int cdns2_gadget_resume(struct cdns2_device *pdev, bool hibernated)
2396 {
2397 	unsigned long flags;
2398 
2399 	spin_lock_irqsave(&pdev->lock, flags);
2400 
2401 	if (!pdev->gadget_driver) {
2402 		spin_unlock_irqrestore(&pdev->lock, flags);
2403 		return 0;
2404 	}
2405 
2406 	cdns2_gadget_config(pdev);
2407 
2408 	if (hibernated)
2409 		clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2410 
2411 	spin_unlock_irqrestore(&pdev->lock, flags);
2412 
2413 	return 0;
2414 }
2415 
2416 void cdns2_gadget_remove(struct cdns2_device *pdev)
2417 {
2418 	pm_runtime_mark_last_busy(pdev->dev);
2419 	pm_runtime_put_autosuspend(pdev->dev);
2420 
2421 	usb_del_gadget(&pdev->gadget);
2422 	cdns2_free_all_eps(pdev);
2423 
2424 	dma_pool_destroy(pdev->eps_dma_pool);
2425 	kfree(pdev->zlp_buf);
2426 	usb_put_gadget(&pdev->gadget);
2427 }
2428 
2429 int cdns2_gadget_init(struct cdns2_device *pdev)
2430 {
2431 	int ret;
2432 
2433 	/* Ensure 32-bit DMA Mask. */
2434 	ret = dma_set_mask_and_coherent(pdev->dev, DMA_BIT_MASK(32));
2435 	if (ret) {
2436 		dev_err(pdev->dev, "Failed to set dma mask: %d\n", ret);
2437 		return ret;
2438 	}
2439 
2440 	pm_runtime_get_sync(pdev->dev);
2441 
2442 	cdsn2_isoc_burst_opt(pdev);
2443 
2444 	ret = cdns2_gadget_start(pdev);
2445 	if (ret) {
2446 		pm_runtime_put_sync(pdev->dev);
2447 		return ret;
2448 	}
2449 
2450 	/*
2451 	 * Because interrupt line can be shared with other components in
2452 	 * driver it can't use IRQF_ONESHOT flag here.
2453 	 */
2454 	ret = devm_request_threaded_irq(pdev->dev, pdev->irq,
2455 					cdns2_usb_irq_handler,
2456 					cdns2_thread_irq_handler,
2457 					IRQF_SHARED,
2458 					dev_name(pdev->dev),
2459 					pdev);
2460 	if (ret)
2461 		goto err0;
2462 
2463 	return 0;
2464 
2465 err0:
2466 	cdns2_gadget_remove(pdev);
2467 
2468 	return ret;
2469 }
2470