xref: /linux/drivers/usb/dwc3/gadget.c (revision c532de5a67a70f8533d495f8f2aaa9a0491c3ad0)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4  *
5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6  *
7  * Authors: Felipe Balbi <balbi@ti.com>,
8  *	    Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9  */
10 
11 #include <linux/kernel.h>
12 #include <linux/delay.h>
13 #include <linux/slab.h>
14 #include <linux/spinlock.h>
15 #include <linux/platform_device.h>
16 #include <linux/pm_runtime.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/list.h>
20 #include <linux/dma-mapping.h>
21 
22 #include <linux/usb/ch9.h>
23 #include <linux/usb/gadget.h>
24 
25 #include "debug.h"
26 #include "core.h"
27 #include "gadget.h"
28 #include "io.h"
29 
30 #define DWC3_ALIGN_FRAME(d, n)	(((d)->frame_number + ((d)->interval * (n))) \
31 					& ~((d)->interval - 1))
32 
33 /**
34  * dwc3_gadget_set_test_mode - enables usb2 test modes
35  * @dwc: pointer to our context structure
36  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
37  *
38  * Caller should take care of locking. This function will return 0 on
39  * success or -EINVAL if wrong Test Selector is passed.
40  */
41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42 {
43 	u32		reg;
44 
45 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
46 	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47 
48 	switch (mode) {
49 	case USB_TEST_J:
50 	case USB_TEST_K:
51 	case USB_TEST_SE0_NAK:
52 	case USB_TEST_PACKET:
53 	case USB_TEST_FORCE_ENABLE:
54 		reg |= mode << 1;
55 		break;
56 	default:
57 		return -EINVAL;
58 	}
59 
60 	dwc3_gadget_dctl_write_safe(dwc, reg);
61 
62 	return 0;
63 }
64 
65 /**
66  * dwc3_gadget_get_link_state - gets current state of usb link
67  * @dwc: pointer to our context structure
68  *
69  * Caller should take care of locking. This function will
70  * return the link state on success (>= 0) or -ETIMEDOUT.
71  */
72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73 {
74 	u32		reg;
75 
76 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
77 
78 	return DWC3_DSTS_USBLNKST(reg);
79 }
80 
81 /**
82  * dwc3_gadget_set_link_state - sets usb link to a particular state
83  * @dwc: pointer to our context structure
84  * @state: the state to put link into
85  *
86  * Caller should take care of locking. This function will
87  * return 0 on success or -ETIMEDOUT.
88  */
89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90 {
91 	int		retries = 10000;
92 	u32		reg;
93 
94 	/*
95 	 * Wait until device controller is ready. Only applies to 1.94a and
96 	 * later RTL.
97 	 */
98 	if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99 		while (--retries) {
100 			reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101 			if (reg & DWC3_DSTS_DCNRD)
102 				udelay(5);
103 			else
104 				break;
105 		}
106 
107 		if (retries <= 0)
108 			return -ETIMEDOUT;
109 	}
110 
111 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112 	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113 
114 	/* set no action before sending new link state change */
115 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
116 
117 	/* set requested state */
118 	reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
120 
121 	/*
122 	 * The following code is racy when called from dwc3_gadget_wakeup,
123 	 * and is not needed, at least on newer versions
124 	 */
125 	if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126 		return 0;
127 
128 	/* wait for a change in DSTS */
129 	retries = 10000;
130 	while (--retries) {
131 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
132 
133 		if (DWC3_DSTS_USBLNKST(reg) == state)
134 			return 0;
135 
136 		udelay(5);
137 	}
138 
139 	return -ETIMEDOUT;
140 }
141 
142 static void dwc3_ep0_reset_state(struct dwc3 *dwc)
143 {
144 	unsigned int	dir;
145 
146 	if (dwc->ep0state != EP0_SETUP_PHASE) {
147 		dir = !!dwc->ep0_expect_in;
148 		if (dwc->ep0state == EP0_DATA_PHASE)
149 			dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
150 		else
151 			dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
152 
153 		dwc->eps[0]->trb_enqueue = 0;
154 		dwc->eps[1]->trb_enqueue = 0;
155 
156 		dwc3_ep0_stall_and_restart(dwc);
157 	}
158 }
159 
160 /**
161  * dwc3_ep_inc_trb - increment a trb index.
162  * @index: Pointer to the TRB index to increment.
163  *
164  * The index should never point to the link TRB. After incrementing,
165  * if it is point to the link TRB, wrap around to the beginning. The
166  * link TRB is always at the last TRB entry.
167  */
168 static void dwc3_ep_inc_trb(u8 *index)
169 {
170 	(*index)++;
171 	if (*index == (DWC3_TRB_NUM - 1))
172 		*index = 0;
173 }
174 
175 /**
176  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
177  * @dep: The endpoint whose enqueue pointer we're incrementing
178  */
179 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
180 {
181 	dwc3_ep_inc_trb(&dep->trb_enqueue);
182 }
183 
184 /**
185  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
186  * @dep: The endpoint whose enqueue pointer we're incrementing
187  */
188 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
189 {
190 	dwc3_ep_inc_trb(&dep->trb_dequeue);
191 }
192 
193 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
194 		struct dwc3_request *req, int status)
195 {
196 	struct dwc3			*dwc = dep->dwc;
197 
198 	list_del(&req->list);
199 	req->remaining = 0;
200 	req->needs_extra_trb = false;
201 	req->num_trbs = 0;
202 
203 	if (req->request.status == -EINPROGRESS)
204 		req->request.status = status;
205 
206 	if (req->trb)
207 		usb_gadget_unmap_request_by_dev(dwc->sysdev,
208 				&req->request, req->direction);
209 
210 	req->trb = NULL;
211 	trace_dwc3_gadget_giveback(req);
212 
213 	if (dep->number > 1)
214 		pm_runtime_put(dwc->dev);
215 }
216 
217 /**
218  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
219  * @dep: The endpoint to whom the request belongs to
220  * @req: The request we're giving back
221  * @status: completion code for the request
222  *
223  * Must be called with controller's lock held and interrupts disabled. This
224  * function will unmap @req and call its ->complete() callback to notify upper
225  * layers that it has completed.
226  */
227 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
228 		int status)
229 {
230 	struct dwc3			*dwc = dep->dwc;
231 
232 	dwc3_gadget_del_and_unmap_request(dep, req, status);
233 	req->status = DWC3_REQUEST_STATUS_COMPLETED;
234 
235 	spin_unlock(&dwc->lock);
236 	usb_gadget_giveback_request(&dep->endpoint, &req->request);
237 	spin_lock(&dwc->lock);
238 }
239 
240 /**
241  * dwc3_send_gadget_generic_command - issue a generic command for the controller
242  * @dwc: pointer to the controller context
243  * @cmd: the command to be issued
244  * @param: command parameter
245  *
246  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
247  * and wait for its completion.
248  */
249 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
250 		u32 param)
251 {
252 	u32		timeout = 500;
253 	int		status = 0;
254 	int		ret = 0;
255 	u32		reg;
256 
257 	dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
258 	dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
259 
260 	do {
261 		reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
262 		if (!(reg & DWC3_DGCMD_CMDACT)) {
263 			status = DWC3_DGCMD_STATUS(reg);
264 			if (status)
265 				ret = -EINVAL;
266 			break;
267 		}
268 	} while (--timeout);
269 
270 	if (!timeout) {
271 		ret = -ETIMEDOUT;
272 		status = -ETIMEDOUT;
273 	}
274 
275 	trace_dwc3_gadget_generic_cmd(cmd, param, status);
276 
277 	return ret;
278 }
279 
280 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async);
281 
282 /**
283  * dwc3_send_gadget_ep_cmd - issue an endpoint command
284  * @dep: the endpoint to which the command is going to be issued
285  * @cmd: the command to be issued
286  * @params: parameters to the command
287  *
288  * Caller should handle locking. This function will issue @cmd with given
289  * @params to @dep and wait for its completion.
290  *
291  * According to the programming guide, if the link state is in L1/L2/U3,
292  * then sending the Start Transfer command may not complete. The
293  * programming guide suggested to bring the link state back to ON/U0 by
294  * performing remote wakeup prior to sending the command. However, don't
295  * initiate remote wakeup when the user/function does not send wakeup
296  * request via wakeup ops. Send the command when it's allowed.
297  *
298  * Notes:
299  * For L1 link state, issuing a command requires the clearing of
300  * GUSB2PHYCFG.SUSPENDUSB2, which turns on the signal required to complete
301  * the given command (usually within 50us). This should happen within the
302  * command timeout set by driver. No additional step is needed.
303  *
304  * For L2 or U3 link state, the gadget is in USB suspend. Care should be
305  * taken when sending Start Transfer command to ensure that it's done after
306  * USB resume.
307  */
308 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
309 		struct dwc3_gadget_ep_cmd_params *params)
310 {
311 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
312 	struct dwc3		*dwc = dep->dwc;
313 	u32			timeout = 5000;
314 	u32			saved_config = 0;
315 	u32			reg;
316 
317 	int			cmd_status = 0;
318 	int			ret = -EINVAL;
319 
320 	/*
321 	 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
322 	 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
323 	 * endpoint command.
324 	 *
325 	 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
326 	 * settings. Restore them after the command is completed.
327 	 *
328 	 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
329 	 */
330 	if (dwc->gadget->speed <= USB_SPEED_HIGH ||
331 	    DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) {
332 		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
333 		if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
334 			saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
335 			reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
336 		}
337 
338 		if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
339 			saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
340 			reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
341 		}
342 
343 		if (saved_config)
344 			dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
345 	}
346 
347 	/*
348 	 * For some commands such as Update Transfer command, DEPCMDPARn
349 	 * registers are reserved. Since the driver often sends Update Transfer
350 	 * command, don't write to DEPCMDPARn to avoid register write delays and
351 	 * improve performance.
352 	 */
353 	if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
354 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
355 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
356 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
357 	}
358 
359 	/*
360 	 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
361 	 * not relying on XferNotReady, we can make use of a special "No
362 	 * Response Update Transfer" command where we should clear both CmdAct
363 	 * and CmdIOC bits.
364 	 *
365 	 * With this, we don't need to wait for command completion and can
366 	 * straight away issue further commands to the endpoint.
367 	 *
368 	 * NOTICE: We're making an assumption that control endpoints will never
369 	 * make use of Update Transfer command. This is a safe assumption
370 	 * because we can never have more than one request at a time with
371 	 * Control Endpoints. If anybody changes that assumption, this chunk
372 	 * needs to be updated accordingly.
373 	 */
374 	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
375 			!usb_endpoint_xfer_isoc(desc))
376 		cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
377 	else
378 		cmd |= DWC3_DEPCMD_CMDACT;
379 
380 	dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
381 
382 	if (!(cmd & DWC3_DEPCMD_CMDACT) ||
383 		(DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
384 		!(cmd & DWC3_DEPCMD_CMDIOC))) {
385 		ret = 0;
386 		goto skip_status;
387 	}
388 
389 	do {
390 		reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
391 		if (!(reg & DWC3_DEPCMD_CMDACT)) {
392 			cmd_status = DWC3_DEPCMD_STATUS(reg);
393 
394 			switch (cmd_status) {
395 			case 0:
396 				ret = 0;
397 				break;
398 			case DEPEVT_TRANSFER_NO_RESOURCE:
399 				dev_WARN(dwc->dev, "No resource for %s\n",
400 					 dep->name);
401 				ret = -EINVAL;
402 				break;
403 			case DEPEVT_TRANSFER_BUS_EXPIRY:
404 				/*
405 				 * SW issues START TRANSFER command to
406 				 * isochronous ep with future frame interval. If
407 				 * future interval time has already passed when
408 				 * core receives the command, it will respond
409 				 * with an error status of 'Bus Expiry'.
410 				 *
411 				 * Instead of always returning -EINVAL, let's
412 				 * give a hint to the gadget driver that this is
413 				 * the case by returning -EAGAIN.
414 				 */
415 				ret = -EAGAIN;
416 				break;
417 			default:
418 				dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
419 			}
420 
421 			break;
422 		}
423 	} while (--timeout);
424 
425 	if (timeout == 0) {
426 		ret = -ETIMEDOUT;
427 		cmd_status = -ETIMEDOUT;
428 	}
429 
430 skip_status:
431 	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
432 
433 	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
434 		if (ret == 0)
435 			dep->flags |= DWC3_EP_TRANSFER_STARTED;
436 
437 		if (ret != -ETIMEDOUT)
438 			dwc3_gadget_ep_get_transfer_index(dep);
439 	}
440 
441 	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
442 	    !(cmd & DWC3_DEPCMD_CMDIOC))
443 		mdelay(1);
444 
445 	if (saved_config) {
446 		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
447 		reg |= saved_config;
448 		dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
449 	}
450 
451 	return ret;
452 }
453 
454 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
455 {
456 	struct dwc3 *dwc = dep->dwc;
457 	struct dwc3_gadget_ep_cmd_params params;
458 	u32 cmd = DWC3_DEPCMD_CLEARSTALL;
459 
460 	/*
461 	 * As of core revision 2.60a the recommended programming model
462 	 * is to set the ClearPendIN bit when issuing a Clear Stall EP
463 	 * command for IN endpoints. This is to prevent an issue where
464 	 * some (non-compliant) hosts may not send ACK TPs for pending
465 	 * IN transfers due to a mishandled error condition. Synopsys
466 	 * STAR 9000614252.
467 	 */
468 	if (dep->direction &&
469 	    !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
470 	    (dwc->gadget->speed >= USB_SPEED_SUPER))
471 		cmd |= DWC3_DEPCMD_CLEARPENDIN;
472 
473 	memset(&params, 0, sizeof(params));
474 
475 	return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
476 }
477 
478 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
479 		struct dwc3_trb *trb)
480 {
481 	u32		offset = (char *) trb - (char *) dep->trb_pool;
482 
483 	return dep->trb_pool_dma + offset;
484 }
485 
486 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
487 {
488 	struct dwc3		*dwc = dep->dwc;
489 
490 	if (dep->trb_pool)
491 		return 0;
492 
493 	dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
494 			sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
495 			&dep->trb_pool_dma, GFP_KERNEL);
496 	if (!dep->trb_pool) {
497 		dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
498 				dep->name);
499 		return -ENOMEM;
500 	}
501 
502 	return 0;
503 }
504 
505 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
506 {
507 	struct dwc3		*dwc = dep->dwc;
508 
509 	dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
510 			dep->trb_pool, dep->trb_pool_dma);
511 
512 	dep->trb_pool = NULL;
513 	dep->trb_pool_dma = 0;
514 }
515 
516 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
517 {
518 	struct dwc3_gadget_ep_cmd_params params;
519 	int ret;
520 
521 	if (dep->flags & DWC3_EP_RESOURCE_ALLOCATED)
522 		return 0;
523 
524 	memset(&params, 0x00, sizeof(params));
525 
526 	params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
527 
528 	ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
529 			&params);
530 	if (ret)
531 		return ret;
532 
533 	dep->flags |= DWC3_EP_RESOURCE_ALLOCATED;
534 	return 0;
535 }
536 
537 /**
538  * dwc3_gadget_start_config - reset endpoint resources
539  * @dwc: pointer to the DWC3 context
540  * @resource_index: DEPSTARTCFG.XferRscIdx value (must be 0 or 2)
541  *
542  * Set resource_index=0 to reset all endpoints' resources allocation. Do this as
543  * part of the power-on/soft-reset initialization.
544  *
545  * Set resource_index=2 to reset only non-control endpoints' resources. Do this
546  * on receiving the SET_CONFIGURATION request or hibernation resume.
547  */
548 int dwc3_gadget_start_config(struct dwc3 *dwc, unsigned int resource_index)
549 {
550 	struct dwc3_gadget_ep_cmd_params params;
551 	u32			cmd;
552 	int			i;
553 	int			ret;
554 
555 	if (resource_index != 0 && resource_index != 2)
556 		return -EINVAL;
557 
558 	memset(&params, 0x00, sizeof(params));
559 	cmd = DWC3_DEPCMD_DEPSTARTCFG;
560 	cmd |= DWC3_DEPCMD_PARAM(resource_index);
561 
562 	ret = dwc3_send_gadget_ep_cmd(dwc->eps[0], cmd, &params);
563 	if (ret)
564 		return ret;
565 
566 	/* Reset resource allocation flags */
567 	for (i = resource_index; i < dwc->num_eps && dwc->eps[i]; i++)
568 		dwc->eps[i]->flags &= ~DWC3_EP_RESOURCE_ALLOCATED;
569 
570 	return 0;
571 }
572 
573 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
574 {
575 	const struct usb_ss_ep_comp_descriptor *comp_desc;
576 	const struct usb_endpoint_descriptor *desc;
577 	struct dwc3_gadget_ep_cmd_params params;
578 	struct dwc3 *dwc = dep->dwc;
579 
580 	comp_desc = dep->endpoint.comp_desc;
581 	desc = dep->endpoint.desc;
582 
583 	memset(&params, 0x00, sizeof(params));
584 
585 	params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
586 		| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
587 
588 	/* Burst size is only needed in SuperSpeed mode */
589 	if (dwc->gadget->speed >= USB_SPEED_SUPER) {
590 		u32 burst = dep->endpoint.maxburst;
591 
592 		params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
593 	}
594 
595 	params.param0 |= action;
596 	if (action == DWC3_DEPCFG_ACTION_RESTORE)
597 		params.param2 |= dep->saved_state;
598 
599 	if (usb_endpoint_xfer_control(desc))
600 		params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
601 
602 	if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
603 		params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
604 
605 	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
606 		params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
607 			| DWC3_DEPCFG_XFER_COMPLETE_EN
608 			| DWC3_DEPCFG_STREAM_EVENT_EN;
609 		dep->stream_capable = true;
610 	}
611 
612 	if (!usb_endpoint_xfer_control(desc))
613 		params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
614 
615 	/*
616 	 * We are doing 1:1 mapping for endpoints, meaning
617 	 * Physical Endpoints 2 maps to Logical Endpoint 2 and
618 	 * so on. We consider the direction bit as part of the physical
619 	 * endpoint number. So USB endpoint 0x81 is 0x03.
620 	 */
621 	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
622 
623 	/*
624 	 * We must use the lower 16 TX FIFOs even though
625 	 * HW might have more
626 	 */
627 	if (dep->direction)
628 		params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
629 
630 	if (desc->bInterval) {
631 		u8 bInterval_m1;
632 
633 		/*
634 		 * Valid range for DEPCFG.bInterval_m1 is from 0 to 13.
635 		 *
636 		 * NOTE: The programming guide incorrectly stated bInterval_m1
637 		 * must be set to 0 when operating in fullspeed. Internally the
638 		 * controller does not have this limitation. See DWC_usb3x
639 		 * programming guide section 3.2.2.1.
640 		 */
641 		bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
642 
643 		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
644 		    dwc->gadget->speed == USB_SPEED_FULL)
645 			dep->interval = desc->bInterval;
646 		else
647 			dep->interval = 1 << (desc->bInterval - 1);
648 
649 		params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
650 	}
651 
652 	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
653 }
654 
655 /**
656  * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
657  * @dwc: pointer to the DWC3 context
658  * @mult: multiplier to be used when calculating the fifo_size
659  *
660  * Calculates the size value based on the equation below:
661  *
662  * DWC3 revision 280A and prior:
663  * fifo_size = mult * (max_packet / mdwidth) + 1;
664  *
665  * DWC3 revision 290A and onwards:
666  * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
667  *
668  * The max packet size is set to 1024, as the txfifo requirements mainly apply
669  * to super speed USB use cases.  However, it is safe to overestimate the fifo
670  * allocations for other scenarios, i.e. high speed USB.
671  */
672 static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
673 {
674 	int max_packet = 1024;
675 	int fifo_size;
676 	int mdwidth;
677 
678 	mdwidth = dwc3_mdwidth(dwc);
679 
680 	/* MDWIDTH is represented in bits, we need it in bytes */
681 	mdwidth >>= 3;
682 
683 	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
684 		fifo_size = mult * (max_packet / mdwidth) + 1;
685 	else
686 		fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
687 	return fifo_size;
688 }
689 
690 /**
691  * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
692  * @dwc: pointer to the DWC3 context
693  *
694  * Iterates through all the endpoint registers and clears the previous txfifo
695  * allocations.
696  */
697 void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
698 {
699 	struct dwc3_ep *dep;
700 	int fifo_depth;
701 	int size;
702 	int num;
703 
704 	if (!dwc->do_fifo_resize)
705 		return;
706 
707 	/* Read ep0IN related TXFIFO size */
708 	dep = dwc->eps[1];
709 	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
710 	if (DWC3_IP_IS(DWC3))
711 		fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
712 	else
713 		fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
714 
715 	dwc->last_fifo_depth = fifo_depth;
716 	/* Clear existing TXFIFO for all IN eps except ep0 */
717 	for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
718 	     num += 2) {
719 		dep = dwc->eps[num];
720 		/* Don't change TXFRAMNUM on usb31 version */
721 		size = DWC3_IP_IS(DWC3) ? 0 :
722 			dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
723 				   DWC31_GTXFIFOSIZ_TXFRAMNUM;
724 
725 		dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
726 		dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
727 	}
728 	dwc->num_ep_resized = 0;
729 }
730 
731 /*
732  * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
733  * @dwc: pointer to our context structure
734  *
735  * This function will a best effort FIFO allocation in order
736  * to improve FIFO usage and throughput, while still allowing
737  * us to enable as many endpoints as possible.
738  *
739  * Keep in mind that this operation will be highly dependent
740  * on the configured size for RAM1 - which contains TxFifo -,
741  * the amount of endpoints enabled on coreConsultant tool, and
742  * the width of the Master Bus.
743  *
744  * In general, FIFO depths are represented with the following equation:
745  *
746  * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
747  *
748  * In conjunction with dwc3_gadget_check_config(), this resizing logic will
749  * ensure that all endpoints will have enough internal memory for one max
750  * packet per endpoint.
751  */
752 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
753 {
754 	struct dwc3 *dwc = dep->dwc;
755 	int fifo_0_start;
756 	int ram1_depth;
757 	int fifo_size;
758 	int min_depth;
759 	int num_in_ep;
760 	int remaining;
761 	int num_fifos = 1;
762 	int fifo;
763 	int tmp;
764 
765 	if (!dwc->do_fifo_resize)
766 		return 0;
767 
768 	/* resize IN endpoints except ep0 */
769 	if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)
770 		return 0;
771 
772 	/* bail if already resized */
773 	if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
774 		return 0;
775 
776 	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
777 
778 	if ((dep->endpoint.maxburst > 1 &&
779 	     usb_endpoint_xfer_bulk(dep->endpoint.desc)) ||
780 	    usb_endpoint_xfer_isoc(dep->endpoint.desc))
781 		num_fifos = 3;
782 
783 	if (dep->endpoint.maxburst > 6 &&
784 	    (usb_endpoint_xfer_bulk(dep->endpoint.desc) ||
785 	     usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
786 		num_fifos = dwc->tx_fifo_resize_max_num;
787 
788 	/* FIFO size for a single buffer */
789 	fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1);
790 
791 	/* Calculate the number of remaining EPs w/o any FIFO */
792 	num_in_ep = dwc->max_cfg_eps;
793 	num_in_ep -= dwc->num_ep_resized;
794 
795 	/* Reserve at least one FIFO for the number of IN EPs */
796 	min_depth = num_in_ep * (fifo + 1);
797 	remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
798 	remaining = max_t(int, 0, remaining);
799 	/*
800 	 * We've already reserved 1 FIFO per EP, so check what we can fit in
801 	 * addition to it.  If there is not enough remaining space, allocate
802 	 * all the remaining space to the EP.
803 	 */
804 	fifo_size = (num_fifos - 1) * fifo;
805 	if (remaining < fifo_size)
806 		fifo_size = remaining;
807 
808 	fifo_size += fifo;
809 	/* Last increment according to the TX FIFO size equation */
810 	fifo_size++;
811 
812 	/* Check if TXFIFOs start at non-zero addr */
813 	tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
814 	fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
815 
816 	fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
817 	if (DWC3_IP_IS(DWC3))
818 		dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
819 	else
820 		dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
821 
822 	/* Check fifo size allocation doesn't exceed available RAM size. */
823 	if (dwc->last_fifo_depth >= ram1_depth) {
824 		dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
825 			dwc->last_fifo_depth, ram1_depth,
826 			dep->endpoint.name, fifo_size);
827 		if (DWC3_IP_IS(DWC3))
828 			fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
829 		else
830 			fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
831 
832 		dwc->last_fifo_depth -= fifo_size;
833 		return -ENOMEM;
834 	}
835 
836 	dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
837 	dep->flags |= DWC3_EP_TXFIFO_RESIZED;
838 	dwc->num_ep_resized++;
839 
840 	return 0;
841 }
842 
843 /**
844  * __dwc3_gadget_ep_enable - initializes a hw endpoint
845  * @dep: endpoint to be initialized
846  * @action: one of INIT, MODIFY or RESTORE
847  *
848  * Caller should take care of locking. Execute all necessary commands to
849  * initialize a HW endpoint so it can be used by a gadget driver.
850  */
851 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
852 {
853 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
854 	struct dwc3		*dwc = dep->dwc;
855 
856 	u32			reg;
857 	int			ret;
858 
859 	if (!(dep->flags & DWC3_EP_ENABLED)) {
860 		ret = dwc3_gadget_resize_tx_fifos(dep);
861 		if (ret)
862 			return ret;
863 	}
864 
865 	ret = dwc3_gadget_set_ep_config(dep, action);
866 	if (ret)
867 		return ret;
868 
869 	if (!(dep->flags & DWC3_EP_RESOURCE_ALLOCATED)) {
870 		ret = dwc3_gadget_set_xfer_resource(dep);
871 		if (ret)
872 			return ret;
873 	}
874 
875 	if (!(dep->flags & DWC3_EP_ENABLED)) {
876 		struct dwc3_trb	*trb_st_hw;
877 		struct dwc3_trb	*trb_link;
878 
879 		dep->type = usb_endpoint_type(desc);
880 		dep->flags |= DWC3_EP_ENABLED;
881 
882 		reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
883 		reg |= DWC3_DALEPENA_EP(dep->number);
884 		dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
885 
886 		dep->trb_dequeue = 0;
887 		dep->trb_enqueue = 0;
888 
889 		if (usb_endpoint_xfer_control(desc))
890 			goto out;
891 
892 		/* Initialize the TRB ring */
893 		memset(dep->trb_pool, 0,
894 		       sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
895 
896 		/* Link TRB. The HWO bit is never reset */
897 		trb_st_hw = &dep->trb_pool[0];
898 
899 		trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
900 		trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
901 		trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
902 		trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
903 		trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
904 	}
905 
906 	/*
907 	 * Issue StartTransfer here with no-op TRB so we can always rely on No
908 	 * Response Update Transfer command.
909 	 */
910 	if (usb_endpoint_xfer_bulk(desc) ||
911 			usb_endpoint_xfer_int(desc)) {
912 		struct dwc3_gadget_ep_cmd_params params;
913 		struct dwc3_trb	*trb;
914 		dma_addr_t trb_dma;
915 		u32 cmd;
916 
917 		memset(&params, 0, sizeof(params));
918 		trb = &dep->trb_pool[0];
919 		trb_dma = dwc3_trb_dma_offset(dep, trb);
920 
921 		params.param0 = upper_32_bits(trb_dma);
922 		params.param1 = lower_32_bits(trb_dma);
923 
924 		cmd = DWC3_DEPCMD_STARTTRANSFER;
925 
926 		ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
927 		if (ret < 0)
928 			return ret;
929 
930 		if (dep->stream_capable) {
931 			/*
932 			 * For streams, at start, there maybe a race where the
933 			 * host primes the endpoint before the function driver
934 			 * queues a request to initiate a stream. In that case,
935 			 * the controller will not see the prime to generate the
936 			 * ERDY and start stream. To workaround this, issue a
937 			 * no-op TRB as normal, but end it immediately. As a
938 			 * result, when the function driver queues the request,
939 			 * the next START_TRANSFER command will cause the
940 			 * controller to generate an ERDY to initiate the
941 			 * stream.
942 			 */
943 			dwc3_stop_active_transfer(dep, true, true);
944 
945 			/*
946 			 * All stream eps will reinitiate stream on NoStream
947 			 * rejection until we can determine that the host can
948 			 * prime after the first transfer.
949 			 *
950 			 * However, if the controller is capable of
951 			 * TXF_FLUSH_BYPASS, then IN direction endpoints will
952 			 * automatically restart the stream without the driver
953 			 * initiation.
954 			 */
955 			if (!dep->direction ||
956 			    !(dwc->hwparams.hwparams9 &
957 			      DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
958 				dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
959 		}
960 	}
961 
962 out:
963 	trace_dwc3_gadget_ep_enable(dep);
964 
965 	return 0;
966 }
967 
968 void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status)
969 {
970 	struct dwc3_request		*req;
971 
972 	dwc3_stop_active_transfer(dep, true, false);
973 
974 	/* If endxfer is delayed, avoid unmapping requests */
975 	if (dep->flags & DWC3_EP_DELAY_STOP)
976 		return;
977 
978 	/* - giveback all requests to gadget driver */
979 	while (!list_empty(&dep->started_list)) {
980 		req = next_request(&dep->started_list);
981 
982 		dwc3_gadget_giveback(dep, req, status);
983 	}
984 
985 	while (!list_empty(&dep->pending_list)) {
986 		req = next_request(&dep->pending_list);
987 
988 		dwc3_gadget_giveback(dep, req, status);
989 	}
990 
991 	while (!list_empty(&dep->cancelled_list)) {
992 		req = next_request(&dep->cancelled_list);
993 
994 		dwc3_gadget_giveback(dep, req, status);
995 	}
996 }
997 
998 /**
999  * __dwc3_gadget_ep_disable - disables a hw endpoint
1000  * @dep: the endpoint to disable
1001  *
1002  * This function undoes what __dwc3_gadget_ep_enable did and also removes
1003  * requests which are currently being processed by the hardware and those which
1004  * are not yet scheduled.
1005  *
1006  * Caller should take care of locking.
1007  */
1008 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1009 {
1010 	struct dwc3		*dwc = dep->dwc;
1011 	u32			reg;
1012 	u32			mask;
1013 
1014 	trace_dwc3_gadget_ep_disable(dep);
1015 
1016 	/* make sure HW endpoint isn't stalled */
1017 	if (dep->flags & DWC3_EP_STALL)
1018 		__dwc3_gadget_ep_set_halt(dep, 0, false);
1019 
1020 	reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
1021 	reg &= ~DWC3_DALEPENA_EP(dep->number);
1022 	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
1023 
1024 	dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
1025 
1026 	dep->stream_capable = false;
1027 	dep->type = 0;
1028 	mask = DWC3_EP_TXFIFO_RESIZED | DWC3_EP_RESOURCE_ALLOCATED;
1029 	/*
1030 	 * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is
1031 	 * set.  Do not clear DEP flags, so that the end transfer command will
1032 	 * be reattempted during the next SETUP stage.
1033 	 */
1034 	if (dep->flags & DWC3_EP_DELAY_STOP)
1035 		mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED);
1036 	dep->flags &= mask;
1037 
1038 	/* Clear out the ep descriptors for non-ep0 */
1039 	if (dep->number > 1) {
1040 		dep->endpoint.comp_desc = NULL;
1041 		dep->endpoint.desc = NULL;
1042 	}
1043 
1044 	return 0;
1045 }
1046 
1047 /* -------------------------------------------------------------------------- */
1048 
1049 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1050 		const struct usb_endpoint_descriptor *desc)
1051 {
1052 	return -EINVAL;
1053 }
1054 
1055 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1056 {
1057 	return -EINVAL;
1058 }
1059 
1060 /* -------------------------------------------------------------------------- */
1061 
1062 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1063 		const struct usb_endpoint_descriptor *desc)
1064 {
1065 	struct dwc3_ep			*dep;
1066 	struct dwc3			*dwc;
1067 	unsigned long			flags;
1068 	int				ret;
1069 
1070 	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1071 		pr_debug("dwc3: invalid parameters\n");
1072 		return -EINVAL;
1073 	}
1074 
1075 	if (!desc->wMaxPacketSize) {
1076 		pr_debug("dwc3: missing wMaxPacketSize\n");
1077 		return -EINVAL;
1078 	}
1079 
1080 	dep = to_dwc3_ep(ep);
1081 	dwc = dep->dwc;
1082 
1083 	if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1084 					"%s is already enabled\n",
1085 					dep->name))
1086 		return 0;
1087 
1088 	spin_lock_irqsave(&dwc->lock, flags);
1089 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1090 	spin_unlock_irqrestore(&dwc->lock, flags);
1091 
1092 	return ret;
1093 }
1094 
1095 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1096 {
1097 	struct dwc3_ep			*dep;
1098 	struct dwc3			*dwc;
1099 	unsigned long			flags;
1100 	int				ret;
1101 
1102 	if (!ep) {
1103 		pr_debug("dwc3: invalid parameters\n");
1104 		return -EINVAL;
1105 	}
1106 
1107 	dep = to_dwc3_ep(ep);
1108 	dwc = dep->dwc;
1109 
1110 	if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1111 					"%s is already disabled\n",
1112 					dep->name))
1113 		return 0;
1114 
1115 	spin_lock_irqsave(&dwc->lock, flags);
1116 	ret = __dwc3_gadget_ep_disable(dep);
1117 	spin_unlock_irqrestore(&dwc->lock, flags);
1118 
1119 	return ret;
1120 }
1121 
1122 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1123 		gfp_t gfp_flags)
1124 {
1125 	struct dwc3_request		*req;
1126 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1127 
1128 	req = kzalloc(sizeof(*req), gfp_flags);
1129 	if (!req)
1130 		return NULL;
1131 
1132 	req->direction	= dep->direction;
1133 	req->epnum	= dep->number;
1134 	req->dep	= dep;
1135 	req->status	= DWC3_REQUEST_STATUS_UNKNOWN;
1136 
1137 	trace_dwc3_alloc_request(req);
1138 
1139 	return &req->request;
1140 }
1141 
1142 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1143 		struct usb_request *request)
1144 {
1145 	struct dwc3_request		*req = to_dwc3_request(request);
1146 
1147 	trace_dwc3_free_request(req);
1148 	kfree(req);
1149 }
1150 
1151 /**
1152  * dwc3_ep_prev_trb - returns the previous TRB in the ring
1153  * @dep: The endpoint with the TRB ring
1154  * @index: The index of the current TRB in the ring
1155  *
1156  * Returns the TRB prior to the one pointed to by the index. If the
1157  * index is 0, we will wrap backwards, skip the link TRB, and return
1158  * the one just before that.
1159  */
1160 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1161 {
1162 	u8 tmp = index;
1163 
1164 	if (!tmp)
1165 		tmp = DWC3_TRB_NUM - 1;
1166 
1167 	return &dep->trb_pool[tmp - 1];
1168 }
1169 
1170 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1171 {
1172 	u8			trbs_left;
1173 
1174 	/*
1175 	 * If the enqueue & dequeue are equal then the TRB ring is either full
1176 	 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs
1177 	 * pending to be processed by the driver.
1178 	 */
1179 	if (dep->trb_enqueue == dep->trb_dequeue) {
1180 		/*
1181 		 * If there is any request remained in the started_list at
1182 		 * this point, that means there is no TRB available.
1183 		 */
1184 		if (!list_empty(&dep->started_list))
1185 			return 0;
1186 
1187 		return DWC3_TRB_NUM - 1;
1188 	}
1189 
1190 	trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1191 	trbs_left &= (DWC3_TRB_NUM - 1);
1192 
1193 	if (dep->trb_dequeue < dep->trb_enqueue)
1194 		trbs_left--;
1195 
1196 	return trbs_left;
1197 }
1198 
1199 /**
1200  * dwc3_prepare_one_trb - setup one TRB from one request
1201  * @dep: endpoint for which this request is prepared
1202  * @req: dwc3_request pointer
1203  * @trb_length: buffer size of the TRB
1204  * @chain: should this TRB be chained to the next?
1205  * @node: only for isochronous endpoints. First TRB needs different type.
1206  * @use_bounce_buffer: set to use bounce buffer
1207  * @must_interrupt: set to interrupt on TRB completion
1208  */
1209 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1210 		struct dwc3_request *req, unsigned int trb_length,
1211 		unsigned int chain, unsigned int node, bool use_bounce_buffer,
1212 		bool must_interrupt)
1213 {
1214 	struct dwc3_trb		*trb;
1215 	dma_addr_t		dma;
1216 	unsigned int		stream_id = req->request.stream_id;
1217 	unsigned int		short_not_ok = req->request.short_not_ok;
1218 	unsigned int		no_interrupt = req->request.no_interrupt;
1219 	unsigned int		is_last = req->request.is_last;
1220 	struct dwc3		*dwc = dep->dwc;
1221 	struct usb_gadget	*gadget = dwc->gadget;
1222 	enum usb_device_speed	speed = gadget->speed;
1223 
1224 	if (use_bounce_buffer)
1225 		dma = dep->dwc->bounce_addr;
1226 	else if (req->request.num_sgs > 0)
1227 		dma = sg_dma_address(req->start_sg);
1228 	else
1229 		dma = req->request.dma;
1230 
1231 	trb = &dep->trb_pool[dep->trb_enqueue];
1232 
1233 	if (!req->trb) {
1234 		dwc3_gadget_move_started_request(req);
1235 		req->trb = trb;
1236 		req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1237 	}
1238 
1239 	req->num_trbs++;
1240 
1241 	trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1242 	trb->bpl = lower_32_bits(dma);
1243 	trb->bph = upper_32_bits(dma);
1244 
1245 	switch (usb_endpoint_type(dep->endpoint.desc)) {
1246 	case USB_ENDPOINT_XFER_CONTROL:
1247 		trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1248 		break;
1249 
1250 	case USB_ENDPOINT_XFER_ISOC:
1251 		if (!node) {
1252 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1253 
1254 			/*
1255 			 * USB Specification 2.0 Section 5.9.2 states that: "If
1256 			 * there is only a single transaction in the microframe,
1257 			 * only a DATA0 data packet PID is used.  If there are
1258 			 * two transactions per microframe, DATA1 is used for
1259 			 * the first transaction data packet and DATA0 is used
1260 			 * for the second transaction data packet.  If there are
1261 			 * three transactions per microframe, DATA2 is used for
1262 			 * the first transaction data packet, DATA1 is used for
1263 			 * the second, and DATA0 is used for the third."
1264 			 *
1265 			 * IOW, we should satisfy the following cases:
1266 			 *
1267 			 * 1) length <= maxpacket
1268 			 *	- DATA0
1269 			 *
1270 			 * 2) maxpacket < length <= (2 * maxpacket)
1271 			 *	- DATA1, DATA0
1272 			 *
1273 			 * 3) (2 * maxpacket) < length <= (3 * maxpacket)
1274 			 *	- DATA2, DATA1, DATA0
1275 			 */
1276 			if (speed == USB_SPEED_HIGH) {
1277 				struct usb_ep *ep = &dep->endpoint;
1278 				unsigned int mult = 2;
1279 				unsigned int maxp = usb_endpoint_maxp(ep->desc);
1280 
1281 				if (req->request.length <= (2 * maxp))
1282 					mult--;
1283 
1284 				if (req->request.length <= maxp)
1285 					mult--;
1286 
1287 				trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1288 			}
1289 		} else {
1290 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1291 		}
1292 
1293 		if (!no_interrupt && !chain)
1294 			trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1295 		break;
1296 
1297 	case USB_ENDPOINT_XFER_BULK:
1298 	case USB_ENDPOINT_XFER_INT:
1299 		trb->ctrl = DWC3_TRBCTL_NORMAL;
1300 		break;
1301 	default:
1302 		/*
1303 		 * This is only possible with faulty memory because we
1304 		 * checked it already :)
1305 		 */
1306 		dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1307 				usb_endpoint_type(dep->endpoint.desc));
1308 	}
1309 
1310 	/*
1311 	 * Enable Continue on Short Packet
1312 	 * when endpoint is not a stream capable
1313 	 */
1314 	if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1315 		if (!dep->stream_capable)
1316 			trb->ctrl |= DWC3_TRB_CTRL_CSP;
1317 
1318 		if (short_not_ok)
1319 			trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1320 	}
1321 
1322 	/* All TRBs setup for MST must set CSP=1 when LST=0 */
1323 	if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1324 		trb->ctrl |= DWC3_TRB_CTRL_CSP;
1325 
1326 	if ((!no_interrupt && !chain) || must_interrupt)
1327 		trb->ctrl |= DWC3_TRB_CTRL_IOC;
1328 
1329 	if (chain)
1330 		trb->ctrl |= DWC3_TRB_CTRL_CHN;
1331 	else if (dep->stream_capable && is_last &&
1332 		 !DWC3_MST_CAPABLE(&dwc->hwparams))
1333 		trb->ctrl |= DWC3_TRB_CTRL_LST;
1334 
1335 	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1336 		trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1337 
1338 	/*
1339 	 * As per data book 4.2.3.2TRB Control Bit Rules section
1340 	 *
1341 	 * The controller autonomously checks the HWO field of a TRB to determine if the
1342 	 * entire TRB is valid. Therefore, software must ensure that the rest of the TRB
1343 	 * is valid before setting the HWO field to '1'. In most systems, this means that
1344 	 * software must update the fourth DWORD of a TRB last.
1345 	 *
1346 	 * However there is a possibility of CPU re-ordering here which can cause
1347 	 * controller to observe the HWO bit set prematurely.
1348 	 * Add a write memory barrier to prevent CPU re-ordering.
1349 	 */
1350 	wmb();
1351 	trb->ctrl |= DWC3_TRB_CTRL_HWO;
1352 
1353 	dwc3_ep_inc_enq(dep);
1354 
1355 	trace_dwc3_prepare_trb(dep, trb);
1356 }
1357 
1358 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1359 {
1360 	unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1361 	unsigned int rem = req->request.length % maxp;
1362 
1363 	if ((req->request.length && req->request.zero && !rem &&
1364 			!usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1365 			(!req->direction && rem))
1366 		return true;
1367 
1368 	return false;
1369 }
1370 
1371 /**
1372  * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1373  * @dep: The endpoint that the request belongs to
1374  * @req: The request to prepare
1375  * @entry_length: The last SG entry size
1376  * @node: Indicates whether this is not the first entry (for isoc only)
1377  *
1378  * Return the number of TRBs prepared.
1379  */
1380 static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1381 		struct dwc3_request *req, unsigned int entry_length,
1382 		unsigned int node)
1383 {
1384 	unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1385 	unsigned int rem = req->request.length % maxp;
1386 	unsigned int num_trbs = 1;
1387 
1388 	if (dwc3_needs_extra_trb(dep, req))
1389 		num_trbs++;
1390 
1391 	if (dwc3_calc_trbs_left(dep) < num_trbs)
1392 		return 0;
1393 
1394 	req->needs_extra_trb = num_trbs > 1;
1395 
1396 	/* Prepare a normal TRB */
1397 	if (req->direction || req->request.length)
1398 		dwc3_prepare_one_trb(dep, req, entry_length,
1399 				req->needs_extra_trb, node, false, false);
1400 
1401 	/* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
1402 	if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1403 		dwc3_prepare_one_trb(dep, req,
1404 				req->direction ? 0 : maxp - rem,
1405 				false, 1, true, false);
1406 
1407 	return num_trbs;
1408 }
1409 
1410 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1411 		struct dwc3_request *req)
1412 {
1413 	struct scatterlist *sg = req->start_sg;
1414 	struct scatterlist *s;
1415 	int		i;
1416 	unsigned int length = req->request.length;
1417 	unsigned int remaining = req->request.num_mapped_sgs
1418 		- req->num_queued_sgs;
1419 	unsigned int num_trbs = req->num_trbs;
1420 	bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1421 
1422 	/*
1423 	 * If we resume preparing the request, then get the remaining length of
1424 	 * the request and resume where we left off.
1425 	 */
1426 	for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1427 		length -= sg_dma_len(s);
1428 
1429 	for_each_sg(sg, s, remaining, i) {
1430 		unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1431 		unsigned int trb_length;
1432 		bool must_interrupt = false;
1433 		bool last_sg = false;
1434 
1435 		trb_length = min_t(unsigned int, length, sg_dma_len(s));
1436 
1437 		length -= trb_length;
1438 
1439 		/*
1440 		 * IOMMU driver is coalescing the list of sgs which shares a
1441 		 * page boundary into one and giving it to USB driver. With
1442 		 * this the number of sgs mapped is not equal to the number of
1443 		 * sgs passed. So mark the chain bit to false if it isthe last
1444 		 * mapped sg.
1445 		 */
1446 		if ((i == remaining - 1) || !length)
1447 			last_sg = true;
1448 
1449 		if (!num_trbs_left)
1450 			break;
1451 
1452 		if (last_sg) {
1453 			if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1454 				break;
1455 		} else {
1456 			/*
1457 			 * Look ahead to check if we have enough TRBs for the
1458 			 * next SG entry. If not, set interrupt on this TRB to
1459 			 * resume preparing the next SG entry when more TRBs are
1460 			 * free.
1461 			 */
1462 			if (num_trbs_left == 1 || (needs_extra_trb &&
1463 					num_trbs_left <= 2 &&
1464 					sg_dma_len(sg_next(s)) >= length)) {
1465 				struct dwc3_request *r;
1466 
1467 				/* Check if previous requests already set IOC */
1468 				list_for_each_entry(r, &dep->started_list, list) {
1469 					if (r != req && !r->request.no_interrupt)
1470 						break;
1471 
1472 					if (r == req)
1473 						must_interrupt = true;
1474 				}
1475 			}
1476 
1477 			dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1478 					must_interrupt);
1479 		}
1480 
1481 		/*
1482 		 * There can be a situation where all sgs in sglist are not
1483 		 * queued because of insufficient trb number. To handle this
1484 		 * case, update start_sg to next sg to be queued, so that
1485 		 * we have free trbs we can continue queuing from where we
1486 		 * previously stopped
1487 		 */
1488 		if (!last_sg)
1489 			req->start_sg = sg_next(s);
1490 
1491 		req->num_queued_sgs++;
1492 		req->num_pending_sgs--;
1493 
1494 		/*
1495 		 * The number of pending SG entries may not correspond to the
1496 		 * number of mapped SG entries. If all the data are queued, then
1497 		 * don't include unused SG entries.
1498 		 */
1499 		if (length == 0) {
1500 			req->num_pending_sgs = 0;
1501 			break;
1502 		}
1503 
1504 		if (must_interrupt)
1505 			break;
1506 	}
1507 
1508 	return req->num_trbs - num_trbs;
1509 }
1510 
1511 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1512 		struct dwc3_request *req)
1513 {
1514 	return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1515 }
1516 
1517 /*
1518  * dwc3_prepare_trbs - setup TRBs from requests
1519  * @dep: endpoint for which requests are being prepared
1520  *
1521  * The function goes through the requests list and sets up TRBs for the
1522  * transfers. The function returns once there are no more TRBs available or
1523  * it runs out of requests.
1524  *
1525  * Returns the number of TRBs prepared or negative errno.
1526  */
1527 static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1528 {
1529 	struct dwc3_request	*req, *n;
1530 	int			ret = 0;
1531 
1532 	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1533 
1534 	/*
1535 	 * We can get in a situation where there's a request in the started list
1536 	 * but there weren't enough TRBs to fully kick it in the first time
1537 	 * around, so it has been waiting for more TRBs to be freed up.
1538 	 *
1539 	 * In that case, we should check if we have a request with pending_sgs
1540 	 * in the started list and prepare TRBs for that request first,
1541 	 * otherwise we will prepare TRBs completely out of order and that will
1542 	 * break things.
1543 	 */
1544 	list_for_each_entry(req, &dep->started_list, list) {
1545 		if (req->num_pending_sgs > 0) {
1546 			ret = dwc3_prepare_trbs_sg(dep, req);
1547 			if (!ret || req->num_pending_sgs)
1548 				return ret;
1549 		}
1550 
1551 		if (!dwc3_calc_trbs_left(dep))
1552 			return ret;
1553 
1554 		/*
1555 		 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1556 		 * burst capability may try to read and use TRBs beyond the
1557 		 * active transfer instead of stopping.
1558 		 */
1559 		if (dep->stream_capable && req->request.is_last &&
1560 		    !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1561 			return ret;
1562 	}
1563 
1564 	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1565 		struct dwc3	*dwc = dep->dwc;
1566 
1567 		ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1568 						    dep->direction);
1569 		if (ret)
1570 			return ret;
1571 
1572 		req->sg			= req->request.sg;
1573 		req->start_sg		= req->sg;
1574 		req->num_queued_sgs	= 0;
1575 		req->num_pending_sgs	= req->request.num_mapped_sgs;
1576 
1577 		if (req->num_pending_sgs > 0) {
1578 			ret = dwc3_prepare_trbs_sg(dep, req);
1579 			if (req->num_pending_sgs)
1580 				return ret;
1581 		} else {
1582 			ret = dwc3_prepare_trbs_linear(dep, req);
1583 		}
1584 
1585 		if (!ret || !dwc3_calc_trbs_left(dep))
1586 			return ret;
1587 
1588 		/*
1589 		 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1590 		 * burst capability may try to read and use TRBs beyond the
1591 		 * active transfer instead of stopping.
1592 		 */
1593 		if (dep->stream_capable && req->request.is_last &&
1594 		    !DWC3_MST_CAPABLE(&dwc->hwparams))
1595 			return ret;
1596 	}
1597 
1598 	return ret;
1599 }
1600 
1601 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1602 
1603 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1604 {
1605 	struct dwc3_gadget_ep_cmd_params params;
1606 	struct dwc3_request		*req;
1607 	int				starting;
1608 	int				ret;
1609 	u32				cmd;
1610 
1611 	/*
1612 	 * Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
1613 	 * This happens when we need to stop and restart a transfer such as in
1614 	 * the case of reinitiating a stream or retrying an isoc transfer.
1615 	 */
1616 	ret = dwc3_prepare_trbs(dep);
1617 	if (ret < 0)
1618 		return ret;
1619 
1620 	starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1621 
1622 	/*
1623 	 * If there's no new TRB prepared and we don't need to restart a
1624 	 * transfer, there's no need to update the transfer.
1625 	 */
1626 	if (!ret && !starting)
1627 		return ret;
1628 
1629 	req = next_request(&dep->started_list);
1630 	if (!req) {
1631 		dep->flags |= DWC3_EP_PENDING_REQUEST;
1632 		return 0;
1633 	}
1634 
1635 	memset(&params, 0, sizeof(params));
1636 
1637 	if (starting) {
1638 		params.param0 = upper_32_bits(req->trb_dma);
1639 		params.param1 = lower_32_bits(req->trb_dma);
1640 		cmd = DWC3_DEPCMD_STARTTRANSFER;
1641 
1642 		if (dep->stream_capable)
1643 			cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1644 
1645 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1646 			cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1647 	} else {
1648 		cmd = DWC3_DEPCMD_UPDATETRANSFER |
1649 			DWC3_DEPCMD_PARAM(dep->resource_index);
1650 	}
1651 
1652 	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1653 	if (ret < 0) {
1654 		struct dwc3_request *tmp;
1655 
1656 		if (ret == -EAGAIN)
1657 			return ret;
1658 
1659 		dwc3_stop_active_transfer(dep, true, true);
1660 
1661 		list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1662 			dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1663 
1664 		/* If ep isn't started, then there's no end transfer pending */
1665 		if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1666 			dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1667 
1668 		return ret;
1669 	}
1670 
1671 	if (dep->stream_capable && req->request.is_last &&
1672 	    !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1673 		dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1674 
1675 	return 0;
1676 }
1677 
1678 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1679 {
1680 	u32			reg;
1681 
1682 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1683 	return DWC3_DSTS_SOFFN(reg);
1684 }
1685 
1686 /**
1687  * __dwc3_stop_active_transfer - stop the current active transfer
1688  * @dep: isoc endpoint
1689  * @force: set forcerm bit in the command
1690  * @interrupt: command complete interrupt after End Transfer command
1691  *
1692  * When setting force, the ForceRM bit will be set. In that case
1693  * the controller won't update the TRB progress on command
1694  * completion. It also won't clear the HWO bit in the TRB.
1695  * The command will also not complete immediately in that case.
1696  */
1697 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1698 {
1699 	struct dwc3_gadget_ep_cmd_params params;
1700 	u32 cmd;
1701 	int ret;
1702 
1703 	cmd = DWC3_DEPCMD_ENDTRANSFER;
1704 	cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1705 	cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
1706 	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1707 	memset(&params, 0, sizeof(params));
1708 	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1709 	/*
1710 	 * If the End Transfer command was timed out while the device is
1711 	 * not in SETUP phase, it's possible that an incoming Setup packet
1712 	 * may prevent the command's completion. Let's retry when the
1713 	 * ep0state returns to EP0_SETUP_PHASE.
1714 	 */
1715 	if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
1716 		dep->flags |= DWC3_EP_DELAY_STOP;
1717 		return 0;
1718 	}
1719 	WARN_ON_ONCE(ret);
1720 	dep->resource_index = 0;
1721 
1722 	if (!interrupt)
1723 		dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1724 	else if (!ret)
1725 		dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1726 
1727 	dep->flags &= ~DWC3_EP_DELAY_STOP;
1728 	return ret;
1729 }
1730 
1731 /**
1732  * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1733  * @dep: isoc endpoint
1734  *
1735  * This function tests for the correct combination of BIT[15:14] from the 16-bit
1736  * microframe number reported by the XferNotReady event for the future frame
1737  * number to start the isoc transfer.
1738  *
1739  * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1740  * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1741  * XferNotReady event are invalid. The driver uses this number to schedule the
1742  * isochronous transfer and passes it to the START TRANSFER command. Because
1743  * this number is invalid, the command may fail. If BIT[15:14] matches the
1744  * internal 16-bit microframe, the START TRANSFER command will pass and the
1745  * transfer will start at the scheduled time, if it is off by 1, the command
1746  * will still pass, but the transfer will start 2 seconds in the future. For all
1747  * other conditions, the START TRANSFER command will fail with bus-expiry.
1748  *
1749  * In order to workaround this issue, we can test for the correct combination of
1750  * BIT[15:14] by sending START TRANSFER commands with different values of
1751  * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1752  * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1753  * As the result, within the 4 possible combinations for BIT[15:14], there will
1754  * be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1755  * command status will result in a 2-second delay start. The smaller BIT[15:14]
1756  * value is the correct combination.
1757  *
1758  * Since there are only 4 outcomes and the results are ordered, we can simply
1759  * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1760  * deduce the smaller successful combination.
1761  *
1762  * Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1763  * of BIT[15:14]. The correct combination is as follow:
1764  *
1765  * if test0 fails and test1 passes, BIT[15:14] is 'b01
1766  * if test0 fails and test1 fails, BIT[15:14] is 'b10
1767  * if test0 passes and test1 fails, BIT[15:14] is 'b11
1768  * if test0 passes and test1 passes, BIT[15:14] is 'b00
1769  *
1770  * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1771  * endpoints.
1772  */
1773 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1774 {
1775 	int cmd_status = 0;
1776 	bool test0;
1777 	bool test1;
1778 
1779 	while (dep->combo_num < 2) {
1780 		struct dwc3_gadget_ep_cmd_params params;
1781 		u32 test_frame_number;
1782 		u32 cmd;
1783 
1784 		/*
1785 		 * Check if we can start isoc transfer on the next interval or
1786 		 * 4 uframes in the future with BIT[15:14] as dep->combo_num
1787 		 */
1788 		test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1789 		test_frame_number |= dep->combo_num << 14;
1790 		test_frame_number += max_t(u32, 4, dep->interval);
1791 
1792 		params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1793 		params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1794 
1795 		cmd = DWC3_DEPCMD_STARTTRANSFER;
1796 		cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1797 		cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1798 
1799 		/* Redo if some other failure beside bus-expiry is received */
1800 		if (cmd_status && cmd_status != -EAGAIN) {
1801 			dep->start_cmd_status = 0;
1802 			dep->combo_num = 0;
1803 			return 0;
1804 		}
1805 
1806 		/* Store the first test status */
1807 		if (dep->combo_num == 0)
1808 			dep->start_cmd_status = cmd_status;
1809 
1810 		dep->combo_num++;
1811 
1812 		/*
1813 		 * End the transfer if the START_TRANSFER command is successful
1814 		 * to wait for the next XferNotReady to test the command again
1815 		 */
1816 		if (cmd_status == 0) {
1817 			dwc3_stop_active_transfer(dep, true, true);
1818 			return 0;
1819 		}
1820 	}
1821 
1822 	/* test0 and test1 are both completed at this point */
1823 	test0 = (dep->start_cmd_status == 0);
1824 	test1 = (cmd_status == 0);
1825 
1826 	if (!test0 && test1)
1827 		dep->combo_num = 1;
1828 	else if (!test0 && !test1)
1829 		dep->combo_num = 2;
1830 	else if (test0 && !test1)
1831 		dep->combo_num = 3;
1832 	else if (test0 && test1)
1833 		dep->combo_num = 0;
1834 
1835 	dep->frame_number &= DWC3_FRNUMBER_MASK;
1836 	dep->frame_number |= dep->combo_num << 14;
1837 	dep->frame_number += max_t(u32, 4, dep->interval);
1838 
1839 	/* Reinitialize test variables */
1840 	dep->start_cmd_status = 0;
1841 	dep->combo_num = 0;
1842 
1843 	return __dwc3_gadget_kick_transfer(dep);
1844 }
1845 
1846 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1847 {
1848 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1849 	struct dwc3 *dwc = dep->dwc;
1850 	int ret;
1851 	int i;
1852 
1853 	if (list_empty(&dep->pending_list) &&
1854 	    list_empty(&dep->started_list)) {
1855 		dep->flags |= DWC3_EP_PENDING_REQUEST;
1856 		return -EAGAIN;
1857 	}
1858 
1859 	if (!dwc->dis_start_transfer_quirk &&
1860 	    (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1861 	     DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1862 		if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1863 			return dwc3_gadget_start_isoc_quirk(dep);
1864 	}
1865 
1866 	if (desc->bInterval <= 14 &&
1867 	    dwc->gadget->speed >= USB_SPEED_HIGH) {
1868 		u32 frame = __dwc3_gadget_get_frame(dwc);
1869 		bool rollover = frame <
1870 				(dep->frame_number & DWC3_FRNUMBER_MASK);
1871 
1872 		/*
1873 		 * frame_number is set from XferNotReady and may be already
1874 		 * out of date. DSTS only provides the lower 14 bit of the
1875 		 * current frame number. So add the upper two bits of
1876 		 * frame_number and handle a possible rollover.
1877 		 * This will provide the correct frame_number unless more than
1878 		 * rollover has happened since XferNotReady.
1879 		 */
1880 
1881 		dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1882 				     frame;
1883 		if (rollover)
1884 			dep->frame_number += BIT(14);
1885 	}
1886 
1887 	for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1888 		int future_interval = i + 1;
1889 
1890 		/* Give the controller at least 500us to schedule transfers */
1891 		if (desc->bInterval < 3)
1892 			future_interval += 3 - desc->bInterval;
1893 
1894 		dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1895 
1896 		ret = __dwc3_gadget_kick_transfer(dep);
1897 		if (ret != -EAGAIN)
1898 			break;
1899 	}
1900 
1901 	/*
1902 	 * After a number of unsuccessful start attempts due to bus-expiry
1903 	 * status, issue END_TRANSFER command and retry on the next XferNotReady
1904 	 * event.
1905 	 */
1906 	if (ret == -EAGAIN)
1907 		ret = __dwc3_stop_active_transfer(dep, false, true);
1908 
1909 	return ret;
1910 }
1911 
1912 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1913 {
1914 	struct dwc3		*dwc = dep->dwc;
1915 
1916 	if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1917 		dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1918 				dep->name);
1919 		return -ESHUTDOWN;
1920 	}
1921 
1922 	if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1923 				&req->request, req->dep->name))
1924 		return -EINVAL;
1925 
1926 	if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1927 				"%s: request %pK already in flight\n",
1928 				dep->name, &req->request))
1929 		return -EINVAL;
1930 
1931 	pm_runtime_get(dwc->dev);
1932 
1933 	req->request.actual	= 0;
1934 	req->request.status	= -EINPROGRESS;
1935 
1936 	trace_dwc3_ep_queue(req);
1937 
1938 	list_add_tail(&req->list, &dep->pending_list);
1939 	req->status = DWC3_REQUEST_STATUS_QUEUED;
1940 
1941 	if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1942 		return 0;
1943 
1944 	/*
1945 	 * Start the transfer only after the END_TRANSFER is completed
1946 	 * and endpoint STALL is cleared.
1947 	 */
1948 	if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1949 	    (dep->flags & DWC3_EP_WEDGE) ||
1950 	    (dep->flags & DWC3_EP_DELAY_STOP) ||
1951 	    (dep->flags & DWC3_EP_STALL)) {
1952 		dep->flags |= DWC3_EP_DELAY_START;
1953 		return 0;
1954 	}
1955 
1956 	/*
1957 	 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1958 	 * wait for a XferNotReady event so we will know what's the current
1959 	 * (micro-)frame number.
1960 	 *
1961 	 * Without this trick, we are very, very likely gonna get Bus Expiry
1962 	 * errors which will force us issue EndTransfer command.
1963 	 */
1964 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1965 		if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1966 			if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1967 				return __dwc3_gadget_start_isoc(dep);
1968 
1969 			return 0;
1970 		}
1971 	}
1972 
1973 	__dwc3_gadget_kick_transfer(dep);
1974 
1975 	return 0;
1976 }
1977 
1978 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1979 	gfp_t gfp_flags)
1980 {
1981 	struct dwc3_request		*req = to_dwc3_request(request);
1982 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1983 	struct dwc3			*dwc = dep->dwc;
1984 
1985 	unsigned long			flags;
1986 
1987 	int				ret;
1988 
1989 	spin_lock_irqsave(&dwc->lock, flags);
1990 	ret = __dwc3_gadget_ep_queue(dep, req);
1991 	spin_unlock_irqrestore(&dwc->lock, flags);
1992 
1993 	return ret;
1994 }
1995 
1996 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1997 {
1998 	int i;
1999 
2000 	/* If req->trb is not set, then the request has not started */
2001 	if (!req->trb)
2002 		return;
2003 
2004 	/*
2005 	 * If request was already started, this means we had to
2006 	 * stop the transfer. With that we also need to ignore
2007 	 * all TRBs used by the request, however TRBs can only
2008 	 * be modified after completion of END_TRANSFER
2009 	 * command. So what we do here is that we wait for
2010 	 * END_TRANSFER completion and only after that, we jump
2011 	 * over TRBs by clearing HWO and incrementing dequeue
2012 	 * pointer.
2013 	 */
2014 	for (i = 0; i < req->num_trbs; i++) {
2015 		struct dwc3_trb *trb;
2016 
2017 		trb = &dep->trb_pool[dep->trb_dequeue];
2018 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2019 		dwc3_ep_inc_deq(dep);
2020 	}
2021 
2022 	req->num_trbs = 0;
2023 }
2024 
2025 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
2026 {
2027 	struct dwc3_request		*req;
2028 	struct dwc3			*dwc = dep->dwc;
2029 
2030 	while (!list_empty(&dep->cancelled_list)) {
2031 		req = next_request(&dep->cancelled_list);
2032 		dwc3_gadget_ep_skip_trbs(dep, req);
2033 		switch (req->status) {
2034 		case DWC3_REQUEST_STATUS_DISCONNECTED:
2035 			dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
2036 			break;
2037 		case DWC3_REQUEST_STATUS_DEQUEUED:
2038 			dwc3_gadget_giveback(dep, req, -ECONNRESET);
2039 			break;
2040 		case DWC3_REQUEST_STATUS_STALLED:
2041 			dwc3_gadget_giveback(dep, req, -EPIPE);
2042 			break;
2043 		default:
2044 			dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
2045 			dwc3_gadget_giveback(dep, req, -ECONNRESET);
2046 			break;
2047 		}
2048 		/*
2049 		 * The endpoint is disabled, let the dwc3_remove_requests()
2050 		 * handle the cleanup.
2051 		 */
2052 		if (!dep->endpoint.desc)
2053 			break;
2054 	}
2055 }
2056 
2057 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2058 		struct usb_request *request)
2059 {
2060 	struct dwc3_request		*req = to_dwc3_request(request);
2061 	struct dwc3_request		*r = NULL;
2062 
2063 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
2064 	struct dwc3			*dwc = dep->dwc;
2065 
2066 	unsigned long			flags;
2067 	int				ret = 0;
2068 
2069 	trace_dwc3_ep_dequeue(req);
2070 
2071 	spin_lock_irqsave(&dwc->lock, flags);
2072 
2073 	list_for_each_entry(r, &dep->cancelled_list, list) {
2074 		if (r == req)
2075 			goto out;
2076 	}
2077 
2078 	list_for_each_entry(r, &dep->pending_list, list) {
2079 		if (r == req) {
2080 			/*
2081 			 * Explicitly check for EP0/1 as dequeue for those
2082 			 * EPs need to be handled differently.  Control EP
2083 			 * only deals with one USB req, and giveback will
2084 			 * occur during dwc3_ep0_stall_and_restart().  EP0
2085 			 * requests are never added to started_list.
2086 			 */
2087 			if (dep->number > 1)
2088 				dwc3_gadget_giveback(dep, req, -ECONNRESET);
2089 			else
2090 				dwc3_ep0_reset_state(dwc);
2091 			goto out;
2092 		}
2093 	}
2094 
2095 	list_for_each_entry(r, &dep->started_list, list) {
2096 		if (r == req) {
2097 			struct dwc3_request *t;
2098 
2099 			/* wait until it is processed */
2100 			dwc3_stop_active_transfer(dep, true, true);
2101 
2102 			/*
2103 			 * Remove any started request if the transfer is
2104 			 * cancelled.
2105 			 */
2106 			list_for_each_entry_safe(r, t, &dep->started_list, list)
2107 				dwc3_gadget_move_cancelled_request(r,
2108 						DWC3_REQUEST_STATUS_DEQUEUED);
2109 
2110 			dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2111 
2112 			goto out;
2113 		}
2114 	}
2115 
2116 	dev_err(dwc->dev, "request %pK was not queued to %s\n",
2117 		request, ep->name);
2118 	ret = -EINVAL;
2119 out:
2120 	spin_unlock_irqrestore(&dwc->lock, flags);
2121 
2122 	return ret;
2123 }
2124 
2125 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2126 {
2127 	struct dwc3_gadget_ep_cmd_params	params;
2128 	struct dwc3				*dwc = dep->dwc;
2129 	struct dwc3_request			*req;
2130 	struct dwc3_request			*tmp;
2131 	int					ret;
2132 
2133 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2134 		dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2135 		return -EINVAL;
2136 	}
2137 
2138 	memset(&params, 0x00, sizeof(params));
2139 
2140 	if (value) {
2141 		struct dwc3_trb *trb;
2142 
2143 		unsigned int transfer_in_flight;
2144 		unsigned int started;
2145 
2146 		if (dep->number > 1)
2147 			trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
2148 		else
2149 			trb = &dwc->ep0_trb[dep->trb_enqueue];
2150 
2151 		transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2152 		started = !list_empty(&dep->started_list);
2153 
2154 		if (!protocol && ((dep->direction && transfer_in_flight) ||
2155 				(!dep->direction && started))) {
2156 			return -EAGAIN;
2157 		}
2158 
2159 		ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2160 				&params);
2161 		if (ret)
2162 			dev_err(dwc->dev, "failed to set STALL on %s\n",
2163 					dep->name);
2164 		else
2165 			dep->flags |= DWC3_EP_STALL;
2166 	} else {
2167 		/*
2168 		 * Don't issue CLEAR_STALL command to control endpoints. The
2169 		 * controller automatically clears the STALL when it receives
2170 		 * the SETUP token.
2171 		 */
2172 		if (dep->number <= 1) {
2173 			dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2174 			return 0;
2175 		}
2176 
2177 		dwc3_stop_active_transfer(dep, true, true);
2178 
2179 		list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2180 			dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2181 
2182 		if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2183 		    (dep->flags & DWC3_EP_DELAY_STOP)) {
2184 			dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2185 			if (protocol)
2186 				dwc->clear_stall_protocol = dep->number;
2187 
2188 			return 0;
2189 		}
2190 
2191 		dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2192 
2193 		ret = dwc3_send_clear_stall_ep_cmd(dep);
2194 		if (ret) {
2195 			dev_err(dwc->dev, "failed to clear STALL on %s\n",
2196 					dep->name);
2197 			return ret;
2198 		}
2199 
2200 		dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2201 
2202 		if ((dep->flags & DWC3_EP_DELAY_START) &&
2203 		    !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2204 			__dwc3_gadget_kick_transfer(dep);
2205 
2206 		dep->flags &= ~DWC3_EP_DELAY_START;
2207 	}
2208 
2209 	return ret;
2210 }
2211 
2212 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2213 {
2214 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
2215 	struct dwc3			*dwc = dep->dwc;
2216 
2217 	unsigned long			flags;
2218 
2219 	int				ret;
2220 
2221 	spin_lock_irqsave(&dwc->lock, flags);
2222 	ret = __dwc3_gadget_ep_set_halt(dep, value, false);
2223 	spin_unlock_irqrestore(&dwc->lock, flags);
2224 
2225 	return ret;
2226 }
2227 
2228 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2229 {
2230 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
2231 	struct dwc3			*dwc = dep->dwc;
2232 	unsigned long			flags;
2233 	int				ret;
2234 
2235 	spin_lock_irqsave(&dwc->lock, flags);
2236 	dep->flags |= DWC3_EP_WEDGE;
2237 
2238 	if (dep->number == 0 || dep->number == 1)
2239 		ret = __dwc3_gadget_ep0_set_halt(ep, 1);
2240 	else
2241 		ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
2242 	spin_unlock_irqrestore(&dwc->lock, flags);
2243 
2244 	return ret;
2245 }
2246 
2247 /* -------------------------------------------------------------------------- */
2248 
2249 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2250 	.bLength	= USB_DT_ENDPOINT_SIZE,
2251 	.bDescriptorType = USB_DT_ENDPOINT,
2252 	.bmAttributes	= USB_ENDPOINT_XFER_CONTROL,
2253 };
2254 
2255 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2256 	.enable		= dwc3_gadget_ep0_enable,
2257 	.disable	= dwc3_gadget_ep0_disable,
2258 	.alloc_request	= dwc3_gadget_ep_alloc_request,
2259 	.free_request	= dwc3_gadget_ep_free_request,
2260 	.queue		= dwc3_gadget_ep0_queue,
2261 	.dequeue	= dwc3_gadget_ep_dequeue,
2262 	.set_halt	= dwc3_gadget_ep0_set_halt,
2263 	.set_wedge	= dwc3_gadget_ep_set_wedge,
2264 };
2265 
2266 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2267 	.enable		= dwc3_gadget_ep_enable,
2268 	.disable	= dwc3_gadget_ep_disable,
2269 	.alloc_request	= dwc3_gadget_ep_alloc_request,
2270 	.free_request	= dwc3_gadget_ep_free_request,
2271 	.queue		= dwc3_gadget_ep_queue,
2272 	.dequeue	= dwc3_gadget_ep_dequeue,
2273 	.set_halt	= dwc3_gadget_ep_set_halt,
2274 	.set_wedge	= dwc3_gadget_ep_set_wedge,
2275 };
2276 
2277 /* -------------------------------------------------------------------------- */
2278 
2279 static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set)
2280 {
2281 	u32 reg;
2282 
2283 	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2284 		return;
2285 
2286 	reg = dwc3_readl(dwc->regs, DWC3_DEVTEN);
2287 	if (set)
2288 		reg |= DWC3_DEVTEN_ULSTCNGEN;
2289 	else
2290 		reg &= ~DWC3_DEVTEN_ULSTCNGEN;
2291 
2292 	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2293 }
2294 
2295 static int dwc3_gadget_get_frame(struct usb_gadget *g)
2296 {
2297 	struct dwc3		*dwc = gadget_to_dwc(g);
2298 
2299 	return __dwc3_gadget_get_frame(dwc);
2300 }
2301 
2302 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async)
2303 {
2304 	int			retries;
2305 
2306 	int			ret;
2307 	u32			reg;
2308 
2309 	u8			link_state;
2310 
2311 	/*
2312 	 * According to the Databook Remote wakeup request should
2313 	 * be issued only when the device is in early suspend state.
2314 	 *
2315 	 * We can check that via USB Link State bits in DSTS register.
2316 	 */
2317 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2318 
2319 	link_state = DWC3_DSTS_USBLNKST(reg);
2320 
2321 	switch (link_state) {
2322 	case DWC3_LINK_STATE_RESET:
2323 	case DWC3_LINK_STATE_RX_DET:	/* in HS, means Early Suspend */
2324 	case DWC3_LINK_STATE_U3:	/* in HS, means SUSPEND */
2325 	case DWC3_LINK_STATE_U2:	/* in HS, means Sleep (L1) */
2326 	case DWC3_LINK_STATE_U1:
2327 	case DWC3_LINK_STATE_RESUME:
2328 		break;
2329 	default:
2330 		return -EINVAL;
2331 	}
2332 
2333 	if (async)
2334 		dwc3_gadget_enable_linksts_evts(dwc, true);
2335 
2336 	ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2337 	if (ret < 0) {
2338 		dev_err(dwc->dev, "failed to put link in Recovery\n");
2339 		dwc3_gadget_enable_linksts_evts(dwc, false);
2340 		return ret;
2341 	}
2342 
2343 	/* Recent versions do this automatically */
2344 	if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2345 		/* write zeroes to Link Change Request */
2346 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2347 		reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2348 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2349 	}
2350 
2351 	/*
2352 	 * Since link status change events are enabled we will receive
2353 	 * an U0 event when wakeup is successful. So bail out.
2354 	 */
2355 	if (async)
2356 		return 0;
2357 
2358 	/* poll until Link State changes to ON */
2359 	retries = 20000;
2360 
2361 	while (retries--) {
2362 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2363 
2364 		/* in HS, means ON */
2365 		if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2366 			break;
2367 	}
2368 
2369 	if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2370 		dev_err(dwc->dev, "failed to send remote wakeup\n");
2371 		return -EINVAL;
2372 	}
2373 
2374 	return 0;
2375 }
2376 
2377 static int dwc3_gadget_wakeup(struct usb_gadget *g)
2378 {
2379 	struct dwc3		*dwc = gadget_to_dwc(g);
2380 	unsigned long		flags;
2381 	int			ret;
2382 
2383 	if (!dwc->wakeup_configured) {
2384 		dev_err(dwc->dev, "remote wakeup not configured\n");
2385 		return -EINVAL;
2386 	}
2387 
2388 	spin_lock_irqsave(&dwc->lock, flags);
2389 	if (!dwc->gadget->wakeup_armed) {
2390 		dev_err(dwc->dev, "not armed for remote wakeup\n");
2391 		spin_unlock_irqrestore(&dwc->lock, flags);
2392 		return -EINVAL;
2393 	}
2394 	ret = __dwc3_gadget_wakeup(dwc, true);
2395 
2396 	spin_unlock_irqrestore(&dwc->lock, flags);
2397 
2398 	return ret;
2399 }
2400 
2401 static void dwc3_resume_gadget(struct dwc3 *dwc);
2402 
2403 static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id)
2404 {
2405 	struct  dwc3		*dwc = gadget_to_dwc(g);
2406 	unsigned long		flags;
2407 	int			ret;
2408 	int			link_state;
2409 
2410 	if (!dwc->wakeup_configured) {
2411 		dev_err(dwc->dev, "remote wakeup not configured\n");
2412 		return -EINVAL;
2413 	}
2414 
2415 	spin_lock_irqsave(&dwc->lock, flags);
2416 	/*
2417 	 * If the link is in U3, signal for remote wakeup and wait for the
2418 	 * link to transition to U0 before sending device notification.
2419 	 */
2420 	link_state = dwc3_gadget_get_link_state(dwc);
2421 	if (link_state == DWC3_LINK_STATE_U3) {
2422 		ret = __dwc3_gadget_wakeup(dwc, false);
2423 		if (ret) {
2424 			spin_unlock_irqrestore(&dwc->lock, flags);
2425 			return -EINVAL;
2426 		}
2427 		dwc3_resume_gadget(dwc);
2428 		dwc->suspended = false;
2429 		dwc->link_state = DWC3_LINK_STATE_U0;
2430 	}
2431 
2432 	ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION,
2433 					       DWC3_DGCMDPAR_DN_FUNC_WAKE |
2434 					       DWC3_DGCMDPAR_INTF_SEL(intf_id));
2435 	if (ret)
2436 		dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret);
2437 
2438 	spin_unlock_irqrestore(&dwc->lock, flags);
2439 
2440 	return ret;
2441 }
2442 
2443 static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set)
2444 {
2445 	struct dwc3		*dwc = gadget_to_dwc(g);
2446 	unsigned long		flags;
2447 
2448 	spin_lock_irqsave(&dwc->lock, flags);
2449 	dwc->wakeup_configured = !!set;
2450 	spin_unlock_irqrestore(&dwc->lock, flags);
2451 
2452 	return 0;
2453 }
2454 
2455 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2456 		int is_selfpowered)
2457 {
2458 	struct dwc3		*dwc = gadget_to_dwc(g);
2459 	unsigned long		flags;
2460 
2461 	spin_lock_irqsave(&dwc->lock, flags);
2462 	g->is_selfpowered = !!is_selfpowered;
2463 	spin_unlock_irqrestore(&dwc->lock, flags);
2464 
2465 	return 0;
2466 }
2467 
2468 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2469 {
2470 	u32 epnum;
2471 
2472 	for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2473 		struct dwc3_ep *dep;
2474 
2475 		dep = dwc->eps[epnum];
2476 		if (!dep)
2477 			continue;
2478 
2479 		dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
2480 	}
2481 }
2482 
2483 static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2484 {
2485 	enum usb_ssp_rate	ssp_rate = dwc->gadget_ssp_rate;
2486 	u32			reg;
2487 
2488 	if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2489 		ssp_rate = dwc->max_ssp_rate;
2490 
2491 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2492 	reg &= ~DWC3_DCFG_SPEED_MASK;
2493 	reg &= ~DWC3_DCFG_NUMLANES(~0);
2494 
2495 	if (ssp_rate == USB_SSP_GEN_1x2)
2496 		reg |= DWC3_DCFG_SUPERSPEED;
2497 	else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2498 		reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2499 
2500 	if (ssp_rate != USB_SSP_GEN_2x1 &&
2501 	    dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2502 		reg |= DWC3_DCFG_NUMLANES(1);
2503 
2504 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2505 }
2506 
2507 static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2508 {
2509 	enum usb_device_speed	speed;
2510 	u32			reg;
2511 
2512 	speed = dwc->gadget_max_speed;
2513 	if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2514 		speed = dwc->maximum_speed;
2515 
2516 	if (speed == USB_SPEED_SUPER_PLUS &&
2517 	    DWC3_IP_IS(DWC32)) {
2518 		__dwc3_gadget_set_ssp_rate(dwc);
2519 		return;
2520 	}
2521 
2522 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2523 	reg &= ~(DWC3_DCFG_SPEED_MASK);
2524 
2525 	/*
2526 	 * WORKAROUND: DWC3 revision < 2.20a have an issue
2527 	 * which would cause metastability state on Run/Stop
2528 	 * bit if we try to force the IP to USB2-only mode.
2529 	 *
2530 	 * Because of that, we cannot configure the IP to any
2531 	 * speed other than the SuperSpeed
2532 	 *
2533 	 * Refers to:
2534 	 *
2535 	 * STAR#9000525659: Clock Domain Crossing on DCTL in
2536 	 * USB 2.0 Mode
2537 	 */
2538 	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2539 	    !dwc->dis_metastability_quirk) {
2540 		reg |= DWC3_DCFG_SUPERSPEED;
2541 	} else {
2542 		switch (speed) {
2543 		case USB_SPEED_FULL:
2544 			reg |= DWC3_DCFG_FULLSPEED;
2545 			break;
2546 		case USB_SPEED_HIGH:
2547 			reg |= DWC3_DCFG_HIGHSPEED;
2548 			break;
2549 		case USB_SPEED_SUPER:
2550 			reg |= DWC3_DCFG_SUPERSPEED;
2551 			break;
2552 		case USB_SPEED_SUPER_PLUS:
2553 			if (DWC3_IP_IS(DWC3))
2554 				reg |= DWC3_DCFG_SUPERSPEED;
2555 			else
2556 				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2557 			break;
2558 		default:
2559 			dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2560 
2561 			if (DWC3_IP_IS(DWC3))
2562 				reg |= DWC3_DCFG_SUPERSPEED;
2563 			else
2564 				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2565 		}
2566 	}
2567 
2568 	if (DWC3_IP_IS(DWC32) &&
2569 	    speed > USB_SPEED_UNKNOWN &&
2570 	    speed < USB_SPEED_SUPER_PLUS)
2571 		reg &= ~DWC3_DCFG_NUMLANES(~0);
2572 
2573 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2574 }
2575 
2576 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
2577 {
2578 	u32			reg;
2579 	u32			timeout = 2000;
2580 
2581 	if (pm_runtime_suspended(dwc->dev))
2582 		return 0;
2583 
2584 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2585 	if (is_on) {
2586 		if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2587 			reg &= ~DWC3_DCTL_TRGTULST_MASK;
2588 			reg |= DWC3_DCTL_TRGTULST_RX_DET;
2589 		}
2590 
2591 		if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2592 			reg &= ~DWC3_DCTL_KEEP_CONNECT;
2593 		reg |= DWC3_DCTL_RUN_STOP;
2594 
2595 		__dwc3_gadget_set_speed(dwc);
2596 		dwc->pullups_connected = true;
2597 	} else {
2598 		reg &= ~DWC3_DCTL_RUN_STOP;
2599 
2600 		dwc->pullups_connected = false;
2601 	}
2602 
2603 	dwc3_gadget_dctl_write_safe(dwc, reg);
2604 
2605 	do {
2606 		usleep_range(1000, 2000);
2607 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2608 		reg &= DWC3_DSTS_DEVCTRLHLT;
2609 	} while (--timeout && !(!is_on ^ !reg));
2610 
2611 	if (!timeout)
2612 		return -ETIMEDOUT;
2613 
2614 	return 0;
2615 }
2616 
2617 static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2618 static void __dwc3_gadget_stop(struct dwc3 *dwc);
2619 static int __dwc3_gadget_start(struct dwc3 *dwc);
2620 
2621 static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
2622 {
2623 	unsigned long flags;
2624 	int ret;
2625 
2626 	spin_lock_irqsave(&dwc->lock, flags);
2627 	if (!dwc->pullups_connected) {
2628 		spin_unlock_irqrestore(&dwc->lock, flags);
2629 		return 0;
2630 	}
2631 
2632 	dwc->connected = false;
2633 
2634 	/*
2635 	 * Attempt to end pending SETUP status phase, and not wait for the
2636 	 * function to do so.
2637 	 */
2638 	if (dwc->delayed_status)
2639 		dwc3_ep0_send_delayed_status(dwc);
2640 
2641 	/*
2642 	 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
2643 	 * Section 4.1.8 Table 4-7, it states that for a device-initiated
2644 	 * disconnect, the SW needs to ensure that it sends "a DEPENDXFER
2645 	 * command for any active transfers" before clearing the RunStop
2646 	 * bit.
2647 	 */
2648 	dwc3_stop_active_transfers(dwc);
2649 	spin_unlock_irqrestore(&dwc->lock, flags);
2650 
2651 	/*
2652 	 * Per databook, when we want to stop the gadget, if a control transfer
2653 	 * is still in process, complete it and get the core into setup phase.
2654 	 * In case the host is unresponsive to a SETUP transaction, forcefully
2655 	 * stall the transfer, and move back to the SETUP phase, so that any
2656 	 * pending endxfers can be executed.
2657 	 */
2658 	if (dwc->ep0state != EP0_SETUP_PHASE) {
2659 		reinit_completion(&dwc->ep0_in_setup);
2660 
2661 		ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2662 				msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2663 		if (ret == 0) {
2664 			dev_warn(dwc->dev, "wait for SETUP phase timed out\n");
2665 			spin_lock_irqsave(&dwc->lock, flags);
2666 			dwc3_ep0_reset_state(dwc);
2667 			spin_unlock_irqrestore(&dwc->lock, flags);
2668 		}
2669 	}
2670 
2671 	/*
2672 	 * Note: if the GEVNTCOUNT indicates events in the event buffer, the
2673 	 * driver needs to acknowledge them before the controller can halt.
2674 	 * Simply let the interrupt handler acknowledges and handle the
2675 	 * remaining event generated by the controller while polling for
2676 	 * DSTS.DEVCTLHLT.
2677 	 */
2678 	ret = dwc3_gadget_run_stop(dwc, false);
2679 
2680 	/*
2681 	 * Stop the gadget after controller is halted, so that if needed, the
2682 	 * events to update EP0 state can still occur while the run/stop
2683 	 * routine polls for the halted state.  DEVTEN is cleared as part of
2684 	 * gadget stop.
2685 	 */
2686 	spin_lock_irqsave(&dwc->lock, flags);
2687 	__dwc3_gadget_stop(dwc);
2688 	spin_unlock_irqrestore(&dwc->lock, flags);
2689 
2690 	return ret;
2691 }
2692 
2693 static int dwc3_gadget_soft_connect(struct dwc3 *dwc)
2694 {
2695 	int ret;
2696 
2697 	/*
2698 	 * In the Synopsys DWC_usb31 1.90a programming guide section
2699 	 * 4.1.9, it specifies that for a reconnect after a
2700 	 * device-initiated disconnect requires a core soft reset
2701 	 * (DCTL.CSftRst) before enabling the run/stop bit.
2702 	 */
2703 	ret = dwc3_core_soft_reset(dwc);
2704 	if (ret)
2705 		return ret;
2706 
2707 	dwc3_event_buffers_setup(dwc);
2708 	__dwc3_gadget_start(dwc);
2709 	return dwc3_gadget_run_stop(dwc, true);
2710 }
2711 
2712 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2713 {
2714 	struct dwc3		*dwc = gadget_to_dwc(g);
2715 	int			ret;
2716 
2717 	is_on = !!is_on;
2718 
2719 	dwc->softconnect = is_on;
2720 
2721 	/*
2722 	 * Avoid issuing a runtime resume if the device is already in the
2723 	 * suspended state during gadget disconnect.  DWC3 gadget was already
2724 	 * halted/stopped during runtime suspend.
2725 	 */
2726 	if (!is_on) {
2727 		pm_runtime_barrier(dwc->dev);
2728 		if (pm_runtime_suspended(dwc->dev))
2729 			return 0;
2730 	}
2731 
2732 	/*
2733 	 * Check the return value for successful resume, or error.  For a
2734 	 * successful resume, the DWC3 runtime PM resume routine will handle
2735 	 * the run stop sequence, so avoid duplicate operations here.
2736 	 */
2737 	ret = pm_runtime_get_sync(dwc->dev);
2738 	if (!ret || ret < 0) {
2739 		pm_runtime_put(dwc->dev);
2740 		if (ret < 0)
2741 			pm_runtime_set_suspended(dwc->dev);
2742 		return ret;
2743 	}
2744 
2745 	if (dwc->pullups_connected == is_on) {
2746 		pm_runtime_put(dwc->dev);
2747 		return 0;
2748 	}
2749 
2750 	synchronize_irq(dwc->irq_gadget);
2751 
2752 	if (!is_on)
2753 		ret = dwc3_gadget_soft_disconnect(dwc);
2754 	else
2755 		ret = dwc3_gadget_soft_connect(dwc);
2756 
2757 	pm_runtime_put(dwc->dev);
2758 
2759 	return ret;
2760 }
2761 
2762 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2763 {
2764 	u32			reg;
2765 
2766 	/* Enable all but Start and End of Frame IRQs */
2767 	reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2768 			DWC3_DEVTEN_CMDCMPLTEN |
2769 			DWC3_DEVTEN_ERRTICERREN |
2770 			DWC3_DEVTEN_WKUPEVTEN |
2771 			DWC3_DEVTEN_CONNECTDONEEN |
2772 			DWC3_DEVTEN_USBRSTEN |
2773 			DWC3_DEVTEN_DISCONNEVTEN);
2774 
2775 	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2776 		reg |= DWC3_DEVTEN_ULSTCNGEN;
2777 
2778 	/* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2779 	if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2780 		reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2781 
2782 	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2783 }
2784 
2785 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2786 {
2787 	/* mask all interrupts */
2788 	dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2789 }
2790 
2791 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2792 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2793 
2794 /**
2795  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2796  * @dwc: pointer to our context structure
2797  *
2798  * The following looks like complex but it's actually very simple. In order to
2799  * calculate the number of packets we can burst at once on OUT transfers, we're
2800  * gonna use RxFIFO size.
2801  *
2802  * To calculate RxFIFO size we need two numbers:
2803  * MDWIDTH = size, in bits, of the internal memory bus
2804  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2805  *
2806  * Given these two numbers, the formula is simple:
2807  *
2808  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2809  *
2810  * 24 bytes is for 3x SETUP packets
2811  * 16 bytes is a clock domain crossing tolerance
2812  *
2813  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2814  */
2815 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2816 {
2817 	u32 ram2_depth;
2818 	u32 mdwidth;
2819 	u32 nump;
2820 	u32 reg;
2821 
2822 	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2823 	mdwidth = dwc3_mdwidth(dwc);
2824 
2825 	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2826 	nump = min_t(u32, nump, 16);
2827 
2828 	/* update NumP */
2829 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2830 	reg &= ~DWC3_DCFG_NUMP_MASK;
2831 	reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2832 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2833 }
2834 
2835 static int __dwc3_gadget_start(struct dwc3 *dwc)
2836 {
2837 	struct dwc3_ep		*dep;
2838 	int			ret = 0;
2839 	u32			reg;
2840 
2841 	/*
2842 	 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2843 	 * the core supports IMOD, disable it.
2844 	 */
2845 	if (dwc->imod_interval) {
2846 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
2847 		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2848 	} else if (dwc3_has_imod(dwc)) {
2849 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2850 	}
2851 
2852 	/*
2853 	 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2854 	 * field instead of letting dwc3 itself calculate that automatically.
2855 	 *
2856 	 * This way, we maximize the chances that we'll be able to get several
2857 	 * bursts of data without going through any sort of endpoint throttling.
2858 	 */
2859 	reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2860 	if (DWC3_IP_IS(DWC3))
2861 		reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2862 	else
2863 		reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2864 
2865 	dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2866 
2867 	dwc3_gadget_setup_nump(dwc);
2868 
2869 	/*
2870 	 * Currently the controller handles single stream only. So, Ignore
2871 	 * Packet Pending bit for stream selection and don't search for another
2872 	 * stream if the host sends Data Packet with PP=0 (for OUT direction) or
2873 	 * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
2874 	 * the stream performance.
2875 	 */
2876 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2877 	reg |= DWC3_DCFG_IGNSTRMPP;
2878 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2879 
2880 	/* Enable MST by default if the device is capable of MST */
2881 	if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2882 		reg = dwc3_readl(dwc->regs, DWC3_DCFG1);
2883 		reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2884 		dwc3_writel(dwc->regs, DWC3_DCFG1, reg);
2885 	}
2886 
2887 	/* Start with SuperSpeed Default */
2888 	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2889 
2890 	ret = dwc3_gadget_start_config(dwc, 0);
2891 	if (ret) {
2892 		dev_err(dwc->dev, "failed to config endpoints\n");
2893 		return ret;
2894 	}
2895 
2896 	dep = dwc->eps[0];
2897 	dep->flags = 0;
2898 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2899 	if (ret) {
2900 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2901 		goto err0;
2902 	}
2903 
2904 	dep = dwc->eps[1];
2905 	dep->flags = 0;
2906 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2907 	if (ret) {
2908 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2909 		goto err1;
2910 	}
2911 
2912 	/* begin to receive SETUP packets */
2913 	dwc->ep0state = EP0_SETUP_PHASE;
2914 	dwc->ep0_bounced = false;
2915 	dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2916 	dwc->delayed_status = false;
2917 	dwc3_ep0_out_start(dwc);
2918 
2919 	dwc3_gadget_enable_irq(dwc);
2920 	dwc3_enable_susphy(dwc, true);
2921 
2922 	return 0;
2923 
2924 err1:
2925 	__dwc3_gadget_ep_disable(dwc->eps[0]);
2926 
2927 err0:
2928 	return ret;
2929 }
2930 
2931 static int dwc3_gadget_start(struct usb_gadget *g,
2932 		struct usb_gadget_driver *driver)
2933 {
2934 	struct dwc3		*dwc = gadget_to_dwc(g);
2935 	unsigned long		flags;
2936 	int			ret;
2937 	int			irq;
2938 
2939 	irq = dwc->irq_gadget;
2940 	ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2941 			IRQF_SHARED, "dwc3", dwc->ev_buf);
2942 	if (ret) {
2943 		dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2944 				irq, ret);
2945 		return ret;
2946 	}
2947 
2948 	spin_lock_irqsave(&dwc->lock, flags);
2949 	dwc->gadget_driver	= driver;
2950 	spin_unlock_irqrestore(&dwc->lock, flags);
2951 
2952 	if (dwc->sys_wakeup)
2953 		device_wakeup_enable(dwc->sysdev);
2954 
2955 	return 0;
2956 }
2957 
2958 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2959 {
2960 	dwc3_gadget_disable_irq(dwc);
2961 	__dwc3_gadget_ep_disable(dwc->eps[0]);
2962 	__dwc3_gadget_ep_disable(dwc->eps[1]);
2963 }
2964 
2965 static int dwc3_gadget_stop(struct usb_gadget *g)
2966 {
2967 	struct dwc3		*dwc = gadget_to_dwc(g);
2968 	unsigned long		flags;
2969 
2970 	if (dwc->sys_wakeup)
2971 		device_wakeup_disable(dwc->sysdev);
2972 
2973 	spin_lock_irqsave(&dwc->lock, flags);
2974 	dwc->gadget_driver	= NULL;
2975 	dwc->max_cfg_eps = 0;
2976 	spin_unlock_irqrestore(&dwc->lock, flags);
2977 
2978 	free_irq(dwc->irq_gadget, dwc->ev_buf);
2979 
2980 	return 0;
2981 }
2982 
2983 static void dwc3_gadget_config_params(struct usb_gadget *g,
2984 				      struct usb_dcd_config_params *params)
2985 {
2986 	struct dwc3		*dwc = gadget_to_dwc(g);
2987 
2988 	params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2989 	params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2990 
2991 	/* Recommended BESL */
2992 	if (!dwc->dis_enblslpm_quirk) {
2993 		/*
2994 		 * If the recommended BESL baseline is 0 or if the BESL deep is
2995 		 * less than 2, Microsoft's Windows 10 host usb stack will issue
2996 		 * a usb reset immediately after it receives the extended BOS
2997 		 * descriptor and the enumeration will fail. To maintain
2998 		 * compatibility with the Windows' usb stack, let's set the
2999 		 * recommended BESL baseline to 1 and clamp the BESL deep to be
3000 		 * within 2 to 15.
3001 		 */
3002 		params->besl_baseline = 1;
3003 		if (dwc->is_utmi_l1_suspend)
3004 			params->besl_deep =
3005 				clamp_t(u8, dwc->hird_threshold, 2, 15);
3006 	}
3007 
3008 	/* U1 Device exit Latency */
3009 	if (dwc->dis_u1_entry_quirk)
3010 		params->bU1devExitLat = 0;
3011 	else
3012 		params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
3013 
3014 	/* U2 Device exit Latency */
3015 	if (dwc->dis_u2_entry_quirk)
3016 		params->bU2DevExitLat = 0;
3017 	else
3018 		params->bU2DevExitLat =
3019 				cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
3020 }
3021 
3022 static void dwc3_gadget_set_speed(struct usb_gadget *g,
3023 				  enum usb_device_speed speed)
3024 {
3025 	struct dwc3		*dwc = gadget_to_dwc(g);
3026 	unsigned long		flags;
3027 
3028 	spin_lock_irqsave(&dwc->lock, flags);
3029 	dwc->gadget_max_speed = speed;
3030 	spin_unlock_irqrestore(&dwc->lock, flags);
3031 }
3032 
3033 static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
3034 				     enum usb_ssp_rate rate)
3035 {
3036 	struct dwc3		*dwc = gadget_to_dwc(g);
3037 	unsigned long		flags;
3038 
3039 	spin_lock_irqsave(&dwc->lock, flags);
3040 	dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
3041 	dwc->gadget_ssp_rate = rate;
3042 	spin_unlock_irqrestore(&dwc->lock, flags);
3043 }
3044 
3045 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
3046 {
3047 	struct dwc3		*dwc = gadget_to_dwc(g);
3048 	union power_supply_propval	val = {0};
3049 	int				ret;
3050 
3051 	if (dwc->usb2_phy)
3052 		return usb_phy_set_power(dwc->usb2_phy, mA);
3053 
3054 	if (!dwc->usb_psy)
3055 		return -EOPNOTSUPP;
3056 
3057 	val.intval = 1000 * mA;
3058 	ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
3059 
3060 	return ret;
3061 }
3062 
3063 /**
3064  * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3065  * @g: pointer to the USB gadget
3066  *
3067  * Used to record the maximum number of endpoints being used in a USB composite
3068  * device. (across all configurations)  This is to be used in the calculation
3069  * of the TXFIFO sizes when resizing internal memory for individual endpoints.
3070  * It will help ensured that the resizing logic reserves enough space for at
3071  * least one max packet.
3072  */
3073 static int dwc3_gadget_check_config(struct usb_gadget *g)
3074 {
3075 	struct dwc3 *dwc = gadget_to_dwc(g);
3076 	struct usb_ep *ep;
3077 	int fifo_size = 0;
3078 	int ram1_depth;
3079 	int ep_num = 0;
3080 
3081 	if (!dwc->do_fifo_resize)
3082 		return 0;
3083 
3084 	list_for_each_entry(ep, &g->ep_list, ep_list) {
3085 		/* Only interested in the IN endpoints */
3086 		if (ep->claimed && (ep->address & USB_DIR_IN))
3087 			ep_num++;
3088 	}
3089 
3090 	if (ep_num <= dwc->max_cfg_eps)
3091 		return 0;
3092 
3093 	/* Update the max number of eps in the composition */
3094 	dwc->max_cfg_eps = ep_num;
3095 
3096 	fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);
3097 	/* Based on the equation, increment by one for every ep */
3098 	fifo_size += dwc->max_cfg_eps;
3099 
3100 	/* Check if we can fit a single fifo per endpoint */
3101 	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
3102 	if (fifo_size > ram1_depth)
3103 		return -ENOMEM;
3104 
3105 	return 0;
3106 }
3107 
3108 static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
3109 {
3110 	struct dwc3		*dwc = gadget_to_dwc(g);
3111 	unsigned long		flags;
3112 
3113 	spin_lock_irqsave(&dwc->lock, flags);
3114 	dwc->async_callbacks = enable;
3115 	spin_unlock_irqrestore(&dwc->lock, flags);
3116 }
3117 
3118 static const struct usb_gadget_ops dwc3_gadget_ops = {
3119 	.get_frame		= dwc3_gadget_get_frame,
3120 	.wakeup			= dwc3_gadget_wakeup,
3121 	.func_wakeup		= dwc3_gadget_func_wakeup,
3122 	.set_remote_wakeup	= dwc3_gadget_set_remote_wakeup,
3123 	.set_selfpowered	= dwc3_gadget_set_selfpowered,
3124 	.pullup			= dwc3_gadget_pullup,
3125 	.udc_start		= dwc3_gadget_start,
3126 	.udc_stop		= dwc3_gadget_stop,
3127 	.udc_set_speed		= dwc3_gadget_set_speed,
3128 	.udc_set_ssp_rate	= dwc3_gadget_set_ssp_rate,
3129 	.get_config_params	= dwc3_gadget_config_params,
3130 	.vbus_draw		= dwc3_gadget_vbus_draw,
3131 	.check_config		= dwc3_gadget_check_config,
3132 	.udc_async_callbacks	= dwc3_gadget_async_callbacks,
3133 };
3134 
3135 /* -------------------------------------------------------------------------- */
3136 
3137 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
3138 {
3139 	struct dwc3 *dwc = dep->dwc;
3140 
3141 	usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
3142 	dep->endpoint.maxburst = 1;
3143 	dep->endpoint.ops = &dwc3_gadget_ep0_ops;
3144 	if (!dep->direction)
3145 		dwc->gadget->ep0 = &dep->endpoint;
3146 
3147 	dep->endpoint.caps.type_control = true;
3148 
3149 	return 0;
3150 }
3151 
3152 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
3153 {
3154 	struct dwc3 *dwc = dep->dwc;
3155 	u32 mdwidth;
3156 	int size;
3157 	int maxpacket;
3158 
3159 	mdwidth = dwc3_mdwidth(dwc);
3160 
3161 	/* MDWIDTH is represented in bits, we need it in bytes */
3162 	mdwidth /= 8;
3163 
3164 	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
3165 	if (DWC3_IP_IS(DWC3))
3166 		size = DWC3_GTXFIFOSIZ_TXFDEP(size);
3167 	else
3168 		size = DWC31_GTXFIFOSIZ_TXFDEP(size);
3169 
3170 	/*
3171 	 * maxpacket size is determined as part of the following, after assuming
3172 	 * a mult value of one maxpacket:
3173 	 * DWC3 revision 280A and prior:
3174 	 * fifo_size = mult * (max_packet / mdwidth) + 1;
3175 	 * maxpacket = mdwidth * (fifo_size - 1);
3176 	 *
3177 	 * DWC3 revision 290A and onwards:
3178 	 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
3179 	 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
3180 	 */
3181 	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
3182 		maxpacket = mdwidth * (size - 1);
3183 	else
3184 		maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
3185 
3186 	/* Functionally, space for one max packet is sufficient */
3187 	size = min_t(int, maxpacket, 1024);
3188 	usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3189 
3190 	dep->endpoint.max_streams = 16;
3191 	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3192 	list_add_tail(&dep->endpoint.ep_list,
3193 			&dwc->gadget->ep_list);
3194 	dep->endpoint.caps.type_iso = true;
3195 	dep->endpoint.caps.type_bulk = true;
3196 	dep->endpoint.caps.type_int = true;
3197 
3198 	return dwc3_alloc_trb_pool(dep);
3199 }
3200 
3201 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3202 {
3203 	struct dwc3 *dwc = dep->dwc;
3204 	u32 mdwidth;
3205 	int size;
3206 
3207 	mdwidth = dwc3_mdwidth(dwc);
3208 
3209 	/* MDWIDTH is represented in bits, convert to bytes */
3210 	mdwidth /= 8;
3211 
3212 	/* All OUT endpoints share a single RxFIFO space */
3213 	size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
3214 	if (DWC3_IP_IS(DWC3))
3215 		size = DWC3_GRXFIFOSIZ_RXFDEP(size);
3216 	else
3217 		size = DWC31_GRXFIFOSIZ_RXFDEP(size);
3218 
3219 	/* FIFO depth is in MDWDITH bytes */
3220 	size *= mdwidth;
3221 
3222 	/*
3223 	 * To meet performance requirement, a minimum recommended RxFIFO size
3224 	 * is defined as follow:
3225 	 * RxFIFO size >= (3 x MaxPacketSize) +
3226 	 * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
3227 	 *
3228 	 * Then calculate the max packet limit as below.
3229 	 */
3230 	size -= (3 * 8) + 16;
3231 	if (size < 0)
3232 		size = 0;
3233 	else
3234 		size /= 3;
3235 
3236 	usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3237 	dep->endpoint.max_streams = 16;
3238 	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3239 	list_add_tail(&dep->endpoint.ep_list,
3240 			&dwc->gadget->ep_list);
3241 	dep->endpoint.caps.type_iso = true;
3242 	dep->endpoint.caps.type_bulk = true;
3243 	dep->endpoint.caps.type_int = true;
3244 
3245 	return dwc3_alloc_trb_pool(dep);
3246 }
3247 
3248 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3249 {
3250 	struct dwc3_ep			*dep;
3251 	bool				direction = epnum & 1;
3252 	int				ret;
3253 	u8				num = epnum >> 1;
3254 
3255 	dep = kzalloc(sizeof(*dep), GFP_KERNEL);
3256 	if (!dep)
3257 		return -ENOMEM;
3258 
3259 	dep->dwc = dwc;
3260 	dep->number = epnum;
3261 	dep->direction = direction;
3262 	dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3263 	dwc->eps[epnum] = dep;
3264 	dep->combo_num = 0;
3265 	dep->start_cmd_status = 0;
3266 
3267 	snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
3268 			direction ? "in" : "out");
3269 
3270 	dep->endpoint.name = dep->name;
3271 
3272 	if (!(dep->number > 1)) {
3273 		dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3274 		dep->endpoint.comp_desc = NULL;
3275 	}
3276 
3277 	if (num == 0)
3278 		ret = dwc3_gadget_init_control_endpoint(dep);
3279 	else if (direction)
3280 		ret = dwc3_gadget_init_in_endpoint(dep);
3281 	else
3282 		ret = dwc3_gadget_init_out_endpoint(dep);
3283 
3284 	if (ret)
3285 		return ret;
3286 
3287 	dep->endpoint.caps.dir_in = direction;
3288 	dep->endpoint.caps.dir_out = !direction;
3289 
3290 	INIT_LIST_HEAD(&dep->pending_list);
3291 	INIT_LIST_HEAD(&dep->started_list);
3292 	INIT_LIST_HEAD(&dep->cancelled_list);
3293 
3294 	dwc3_debugfs_create_endpoint_dir(dep);
3295 
3296 	return 0;
3297 }
3298 
3299 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3300 {
3301 	u8				epnum;
3302 
3303 	INIT_LIST_HEAD(&dwc->gadget->ep_list);
3304 
3305 	for (epnum = 0; epnum < total; epnum++) {
3306 		int			ret;
3307 
3308 		ret = dwc3_gadget_init_endpoint(dwc, epnum);
3309 		if (ret)
3310 			return ret;
3311 	}
3312 
3313 	return 0;
3314 }
3315 
3316 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3317 {
3318 	struct dwc3_ep			*dep;
3319 	u8				epnum;
3320 
3321 	for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3322 		dep = dwc->eps[epnum];
3323 		if (!dep)
3324 			continue;
3325 		/*
3326 		 * Physical endpoints 0 and 1 are special; they form the
3327 		 * bi-directional USB endpoint 0.
3328 		 *
3329 		 * For those two physical endpoints, we don't allocate a TRB
3330 		 * pool nor do we add them the endpoints list. Due to that, we
3331 		 * shouldn't do these two operations otherwise we would end up
3332 		 * with all sorts of bugs when removing dwc3.ko.
3333 		 */
3334 		if (epnum != 0 && epnum != 1) {
3335 			dwc3_free_trb_pool(dep);
3336 			list_del(&dep->endpoint.ep_list);
3337 		}
3338 
3339 		dwc3_debugfs_remove_endpoint_dir(dep);
3340 		kfree(dep);
3341 	}
3342 }
3343 
3344 /* -------------------------------------------------------------------------- */
3345 
3346 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3347 		struct dwc3_request *req, struct dwc3_trb *trb,
3348 		const struct dwc3_event_depevt *event, int status, int chain)
3349 {
3350 	unsigned int		count;
3351 
3352 	dwc3_ep_inc_deq(dep);
3353 
3354 	trace_dwc3_complete_trb(dep, trb);
3355 	req->num_trbs--;
3356 
3357 	/*
3358 	 * If we're in the middle of series of chained TRBs and we
3359 	 * receive a short transfer along the way, DWC3 will skip
3360 	 * through all TRBs including the last TRB in the chain (the
3361 	 * where CHN bit is zero. DWC3 will also avoid clearing HWO
3362 	 * bit and SW has to do it manually.
3363 	 *
3364 	 * We're going to do that here to avoid problems of HW trying
3365 	 * to use bogus TRBs for transfers.
3366 	 */
3367 	if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
3368 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3369 
3370 	/*
3371 	 * For isochronous transfers, the first TRB in a service interval must
3372 	 * have the Isoc-First type. Track and report its interval frame number.
3373 	 */
3374 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3375 	    (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3376 		unsigned int frame_number;
3377 
3378 		frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3379 		frame_number &= ~(dep->interval - 1);
3380 		req->request.frame_number = frame_number;
3381 	}
3382 
3383 	/*
3384 	 * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
3385 	 * this TRB points to the bounce buffer address, it's a MPS alignment
3386 	 * TRB. Don't add it to req->remaining calculation.
3387 	 */
3388 	if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3389 	    trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3390 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3391 		return 1;
3392 	}
3393 
3394 	count = trb->size & DWC3_TRB_SIZE_MASK;
3395 	req->remaining += count;
3396 
3397 	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3398 		return 1;
3399 
3400 	if (event->status & DEPEVT_STATUS_SHORT && !chain)
3401 		return 1;
3402 
3403 	if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&
3404 	    DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)
3405 		return 1;
3406 
3407 	if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3408 	    (trb->ctrl & DWC3_TRB_CTRL_LST))
3409 		return 1;
3410 
3411 	return 0;
3412 }
3413 
3414 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3415 		struct dwc3_request *req, const struct dwc3_event_depevt *event,
3416 		int status)
3417 {
3418 	struct dwc3_trb *trb;
3419 	struct scatterlist *sg = req->sg;
3420 	struct scatterlist *s;
3421 	unsigned int num_queued = req->num_queued_sgs;
3422 	unsigned int i;
3423 	int ret = 0;
3424 
3425 	for_each_sg(sg, s, num_queued, i) {
3426 		trb = &dep->trb_pool[dep->trb_dequeue];
3427 
3428 		req->sg = sg_next(s);
3429 		req->num_queued_sgs--;
3430 
3431 		ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3432 				trb, event, status, true);
3433 		if (ret)
3434 			break;
3435 	}
3436 
3437 	return ret;
3438 }
3439 
3440 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3441 		struct dwc3_request *req, const struct dwc3_event_depevt *event,
3442 		int status)
3443 {
3444 	struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3445 
3446 	return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3447 			event, status, false);
3448 }
3449 
3450 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3451 {
3452 	return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;
3453 }
3454 
3455 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3456 		const struct dwc3_event_depevt *event,
3457 		struct dwc3_request *req, int status)
3458 {
3459 	int request_status;
3460 	int ret;
3461 
3462 	if (req->request.num_mapped_sgs)
3463 		ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3464 				status);
3465 	else
3466 		ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3467 				status);
3468 
3469 	req->request.actual = req->request.length - req->remaining;
3470 
3471 	if (!dwc3_gadget_ep_request_completed(req))
3472 		goto out;
3473 
3474 	if (req->needs_extra_trb) {
3475 		ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3476 				status);
3477 		req->needs_extra_trb = false;
3478 	}
3479 
3480 	/*
3481 	 * The event status only reflects the status of the TRB with IOC set.
3482 	 * For the requests that don't set interrupt on completion, the driver
3483 	 * needs to check and return the status of the completed TRBs associated
3484 	 * with the request. Use the status of the last TRB of the request.
3485 	 */
3486 	if (req->request.no_interrupt) {
3487 		struct dwc3_trb *trb;
3488 
3489 		trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
3490 		switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
3491 		case DWC3_TRBSTS_MISSED_ISOC:
3492 			/* Isoc endpoint only */
3493 			request_status = -EXDEV;
3494 			break;
3495 		case DWC3_TRB_STS_XFER_IN_PROG:
3496 			/* Applicable when End Transfer with ForceRM=0 */
3497 		case DWC3_TRBSTS_SETUP_PENDING:
3498 			/* Control endpoint only */
3499 		case DWC3_TRBSTS_OK:
3500 		default:
3501 			request_status = 0;
3502 			break;
3503 		}
3504 	} else {
3505 		request_status = status;
3506 	}
3507 
3508 	dwc3_gadget_giveback(dep, req, request_status);
3509 
3510 out:
3511 	return ret;
3512 }
3513 
3514 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3515 		const struct dwc3_event_depevt *event, int status)
3516 {
3517 	struct dwc3_request	*req;
3518 
3519 	while (!list_empty(&dep->started_list)) {
3520 		int ret;
3521 
3522 		req = next_request(&dep->started_list);
3523 		ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3524 				req, status);
3525 		if (ret)
3526 			break;
3527 		/*
3528 		 * The endpoint is disabled, let the dwc3_remove_requests()
3529 		 * handle the cleanup.
3530 		 */
3531 		if (!dep->endpoint.desc)
3532 			break;
3533 	}
3534 }
3535 
3536 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3537 {
3538 	struct dwc3_request	*req;
3539 	struct dwc3		*dwc = dep->dwc;
3540 
3541 	if (!dep->endpoint.desc || !dwc->pullups_connected ||
3542 	    !dwc->connected)
3543 		return false;
3544 
3545 	if (!list_empty(&dep->pending_list))
3546 		return true;
3547 
3548 	/*
3549 	 * We only need to check the first entry of the started list. We can
3550 	 * assume the completed requests are removed from the started list.
3551 	 */
3552 	req = next_request(&dep->started_list);
3553 	if (!req)
3554 		return false;
3555 
3556 	return !dwc3_gadget_ep_request_completed(req);
3557 }
3558 
3559 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3560 		const struct dwc3_event_depevt *event)
3561 {
3562 	dep->frame_number = event->parameters;
3563 }
3564 
3565 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3566 		const struct dwc3_event_depevt *event, int status)
3567 {
3568 	struct dwc3		*dwc = dep->dwc;
3569 	bool			no_started_trb = true;
3570 
3571 	dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3572 
3573 	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3574 		goto out;
3575 
3576 	if (!dep->endpoint.desc)
3577 		return no_started_trb;
3578 
3579 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3580 		list_empty(&dep->started_list) &&
3581 		(list_empty(&dep->pending_list) || status == -EXDEV))
3582 		dwc3_stop_active_transfer(dep, true, true);
3583 	else if (dwc3_gadget_ep_should_continue(dep))
3584 		if (__dwc3_gadget_kick_transfer(dep) == 0)
3585 			no_started_trb = false;
3586 
3587 out:
3588 	/*
3589 	 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
3590 	 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
3591 	 */
3592 	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3593 		u32		reg;
3594 		int		i;
3595 
3596 		for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3597 			dep = dwc->eps[i];
3598 
3599 			if (!(dep->flags & DWC3_EP_ENABLED))
3600 				continue;
3601 
3602 			if (!list_empty(&dep->started_list))
3603 				return no_started_trb;
3604 		}
3605 
3606 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3607 		reg |= dwc->u1u2;
3608 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3609 
3610 		dwc->u1u2 = 0;
3611 	}
3612 
3613 	return no_started_trb;
3614 }
3615 
3616 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3617 		const struct dwc3_event_depevt *event)
3618 {
3619 	int status = 0;
3620 
3621 	if (!dep->endpoint.desc)
3622 		return;
3623 
3624 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3625 		dwc3_gadget_endpoint_frame_from_event(dep, event);
3626 
3627 	if (event->status & DEPEVT_STATUS_BUSERR)
3628 		status = -ECONNRESET;
3629 
3630 	if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3631 		status = -EXDEV;
3632 
3633 	dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3634 }
3635 
3636 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3637 		const struct dwc3_event_depevt *event)
3638 {
3639 	int status = 0;
3640 
3641 	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3642 
3643 	if (event->status & DEPEVT_STATUS_BUSERR)
3644 		status = -ECONNRESET;
3645 
3646 	if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3647 		dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3648 }
3649 
3650 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3651 		const struct dwc3_event_depevt *event)
3652 {
3653 	dwc3_gadget_endpoint_frame_from_event(dep, event);
3654 
3655 	/*
3656 	 * The XferNotReady event is generated only once before the endpoint
3657 	 * starts. It will be generated again when END_TRANSFER command is
3658 	 * issued. For some controller versions, the XferNotReady event may be
3659 	 * generated while the END_TRANSFER command is still in process. Ignore
3660 	 * it and wait for the next XferNotReady event after the command is
3661 	 * completed.
3662 	 */
3663 	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3664 		return;
3665 
3666 	(void) __dwc3_gadget_start_isoc(dep);
3667 }
3668 
3669 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3670 		const struct dwc3_event_depevt *event)
3671 {
3672 	u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3673 
3674 	if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3675 		return;
3676 
3677 	/*
3678 	 * The END_TRANSFER command will cause the controller to generate a
3679 	 * NoStream Event, and it's not due to the host DP NoStream rejection.
3680 	 * Ignore the next NoStream event.
3681 	 */
3682 	if (dep->stream_capable)
3683 		dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3684 
3685 	dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3686 	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3687 	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3688 
3689 	if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3690 		struct dwc3 *dwc = dep->dwc;
3691 
3692 		dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3693 		if (dwc3_send_clear_stall_ep_cmd(dep)) {
3694 			struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3695 
3696 			dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3697 			if (dwc->delayed_status)
3698 				__dwc3_gadget_ep0_set_halt(ep0, 1);
3699 			return;
3700 		}
3701 
3702 		dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3703 		if (dwc->clear_stall_protocol == dep->number)
3704 			dwc3_ep0_send_delayed_status(dwc);
3705 	}
3706 
3707 	if ((dep->flags & DWC3_EP_DELAY_START) &&
3708 	    !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3709 		__dwc3_gadget_kick_transfer(dep);
3710 
3711 	dep->flags &= ~DWC3_EP_DELAY_START;
3712 }
3713 
3714 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3715 		const struct dwc3_event_depevt *event)
3716 {
3717 	struct dwc3 *dwc = dep->dwc;
3718 
3719 	if (event->status == DEPEVT_STREAMEVT_FOUND) {
3720 		dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3721 		goto out;
3722 	}
3723 
3724 	/* Note: NoStream rejection event param value is 0 and not 0xFFFF */
3725 	switch (event->parameters) {
3726 	case DEPEVT_STREAM_PRIME:
3727 		/*
3728 		 * If the host can properly transition the endpoint state from
3729 		 * idle to prime after a NoStream rejection, there's no need to
3730 		 * force restarting the endpoint to reinitiate the stream. To
3731 		 * simplify the check, assume the host follows the USB spec if
3732 		 * it primed the endpoint more than once.
3733 		 */
3734 		if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3735 			if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3736 				dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3737 			else
3738 				dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3739 		}
3740 
3741 		break;
3742 	case DEPEVT_STREAM_NOSTREAM:
3743 		if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3744 		    !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3745 		    (!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3746 		     !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3747 			break;
3748 
3749 		/*
3750 		 * If the host rejects a stream due to no active stream, by the
3751 		 * USB and xHCI spec, the endpoint will be put back to idle
3752 		 * state. When the host is ready (buffer added/updated), it will
3753 		 * prime the endpoint to inform the usb device controller. This
3754 		 * triggers the device controller to issue ERDY to restart the
3755 		 * stream. However, some hosts don't follow this and keep the
3756 		 * endpoint in the idle state. No prime will come despite host
3757 		 * streams are updated, and the device controller will not be
3758 		 * triggered to generate ERDY to move the next stream data. To
3759 		 * workaround this and maintain compatibility with various
3760 		 * hosts, force to reinitiate the stream until the host is ready
3761 		 * instead of waiting for the host to prime the endpoint.
3762 		 */
3763 		if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3764 			unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3765 
3766 			dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3767 		} else {
3768 			dep->flags |= DWC3_EP_DELAY_START;
3769 			dwc3_stop_active_transfer(dep, true, true);
3770 			return;
3771 		}
3772 		break;
3773 	}
3774 
3775 out:
3776 	dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3777 }
3778 
3779 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3780 		const struct dwc3_event_depevt *event)
3781 {
3782 	struct dwc3_ep		*dep;
3783 	u8			epnum = event->endpoint_number;
3784 
3785 	dep = dwc->eps[epnum];
3786 
3787 	if (!(dep->flags & DWC3_EP_ENABLED)) {
3788 		if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3789 			return;
3790 
3791 		/* Handle only EPCMDCMPLT when EP disabled */
3792 		if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&
3793 			!(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))
3794 			return;
3795 	}
3796 
3797 	if (epnum == 0 || epnum == 1) {
3798 		dwc3_ep0_interrupt(dwc, event);
3799 		return;
3800 	}
3801 
3802 	switch (event->endpoint_event) {
3803 	case DWC3_DEPEVT_XFERINPROGRESS:
3804 		dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3805 		break;
3806 	case DWC3_DEPEVT_XFERNOTREADY:
3807 		dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3808 		break;
3809 	case DWC3_DEPEVT_EPCMDCMPLT:
3810 		dwc3_gadget_endpoint_command_complete(dep, event);
3811 		break;
3812 	case DWC3_DEPEVT_XFERCOMPLETE:
3813 		dwc3_gadget_endpoint_transfer_complete(dep, event);
3814 		break;
3815 	case DWC3_DEPEVT_STREAMEVT:
3816 		dwc3_gadget_endpoint_stream_event(dep, event);
3817 		break;
3818 	case DWC3_DEPEVT_RXTXFIFOEVT:
3819 		break;
3820 	default:
3821 		dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event);
3822 		break;
3823 	}
3824 }
3825 
3826 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3827 {
3828 	if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3829 		spin_unlock(&dwc->lock);
3830 		dwc->gadget_driver->disconnect(dwc->gadget);
3831 		spin_lock(&dwc->lock);
3832 	}
3833 }
3834 
3835 static void dwc3_suspend_gadget(struct dwc3 *dwc)
3836 {
3837 	if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3838 		spin_unlock(&dwc->lock);
3839 		dwc->gadget_driver->suspend(dwc->gadget);
3840 		spin_lock(&dwc->lock);
3841 	}
3842 }
3843 
3844 static void dwc3_resume_gadget(struct dwc3 *dwc)
3845 {
3846 	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3847 		spin_unlock(&dwc->lock);
3848 		dwc->gadget_driver->resume(dwc->gadget);
3849 		spin_lock(&dwc->lock);
3850 	}
3851 }
3852 
3853 static void dwc3_reset_gadget(struct dwc3 *dwc)
3854 {
3855 	if (!dwc->gadget_driver)
3856 		return;
3857 
3858 	if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3859 		spin_unlock(&dwc->lock);
3860 		usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3861 		spin_lock(&dwc->lock);
3862 	}
3863 }
3864 
3865 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3866 	bool interrupt)
3867 {
3868 	struct dwc3 *dwc = dep->dwc;
3869 
3870 	/*
3871 	 * Only issue End Transfer command to the control endpoint of a started
3872 	 * Data Phase. Typically we should only do so in error cases such as
3873 	 * invalid/unexpected direction as described in the control transfer
3874 	 * flow of the programming guide.
3875 	 */
3876 	if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
3877 		return;
3878 
3879 	if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))
3880 		return;
3881 
3882 	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3883 	    (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3884 		return;
3885 
3886 	/*
3887 	 * If a Setup packet is received but yet to DMA out, the controller will
3888 	 * not process the End Transfer command of any endpoint. Polling of its
3889 	 * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
3890 	 * timeout. Delay issuing the End Transfer command until the Setup TRB is
3891 	 * prepared.
3892 	 */
3893 	if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
3894 		dep->flags |= DWC3_EP_DELAY_STOP;
3895 		return;
3896 	}
3897 
3898 	/*
3899 	 * NOTICE: We are violating what the Databook says about the
3900 	 * EndTransfer command. Ideally we would _always_ wait for the
3901 	 * EndTransfer Command Completion IRQ, but that's causing too
3902 	 * much trouble synchronizing between us and gadget driver.
3903 	 *
3904 	 * We have discussed this with the IP Provider and it was
3905 	 * suggested to giveback all requests here.
3906 	 *
3907 	 * Note also that a similar handling was tested by Synopsys
3908 	 * (thanks a lot Paul) and nothing bad has come out of it.
3909 	 * In short, what we're doing is issuing EndTransfer with
3910 	 * CMDIOC bit set and delay kicking transfer until the
3911 	 * EndTransfer command had completed.
3912 	 *
3913 	 * As of IP version 3.10a of the DWC_usb3 IP, the controller
3914 	 * supports a mode to work around the above limitation. The
3915 	 * software can poll the CMDACT bit in the DEPCMD register
3916 	 * after issuing a EndTransfer command. This mode is enabled
3917 	 * by writing GUCTL2[14]. This polling is already done in the
3918 	 * dwc3_send_gadget_ep_cmd() function so if the mode is
3919 	 * enabled, the EndTransfer command will have completed upon
3920 	 * returning from this function.
3921 	 *
3922 	 * This mode is NOT available on the DWC_usb31 IP.  In this
3923 	 * case, if the IOC bit is not set, then delay by 1ms
3924 	 * after issuing the EndTransfer command.  This allows for the
3925 	 * controller to handle the command completely before DWC3
3926 	 * remove requests attempts to unmap USB request buffers.
3927 	 */
3928 
3929 	__dwc3_stop_active_transfer(dep, force, interrupt);
3930 }
3931 
3932 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3933 {
3934 	u32 epnum;
3935 
3936 	for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3937 		struct dwc3_ep *dep;
3938 		int ret;
3939 
3940 		dep = dwc->eps[epnum];
3941 		if (!dep)
3942 			continue;
3943 
3944 		if (!(dep->flags & DWC3_EP_STALL))
3945 			continue;
3946 
3947 		dep->flags &= ~DWC3_EP_STALL;
3948 
3949 		ret = dwc3_send_clear_stall_ep_cmd(dep);
3950 		WARN_ON_ONCE(ret);
3951 	}
3952 }
3953 
3954 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3955 {
3956 	int			reg;
3957 
3958 	dwc->suspended = false;
3959 
3960 	dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
3961 
3962 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3963 	reg &= ~DWC3_DCTL_INITU1ENA;
3964 	reg &= ~DWC3_DCTL_INITU2ENA;
3965 	dwc3_gadget_dctl_write_safe(dwc, reg);
3966 
3967 	dwc->connected = false;
3968 
3969 	dwc3_disconnect_gadget(dwc);
3970 
3971 	dwc->gadget->speed = USB_SPEED_UNKNOWN;
3972 	dwc->setup_packet_pending = false;
3973 	dwc->gadget->wakeup_armed = false;
3974 	dwc3_gadget_enable_linksts_evts(dwc, false);
3975 	usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3976 
3977 	dwc3_ep0_reset_state(dwc);
3978 
3979 	/*
3980 	 * Request PM idle to address condition where usage count is
3981 	 * already decremented to zero, but waiting for the disconnect
3982 	 * interrupt to set dwc->connected to FALSE.
3983 	 */
3984 	pm_request_idle(dwc->dev);
3985 }
3986 
3987 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3988 {
3989 	u32			reg;
3990 
3991 	dwc->suspended = false;
3992 
3993 	/*
3994 	 * Ideally, dwc3_reset_gadget() would trigger the function
3995 	 * drivers to stop any active transfers through ep disable.
3996 	 * However, for functions which defer ep disable, such as mass
3997 	 * storage, we will need to rely on the call to stop active
3998 	 * transfers here, and avoid allowing of request queuing.
3999 	 */
4000 	dwc->connected = false;
4001 
4002 	/*
4003 	 * WORKAROUND: DWC3 revisions <1.88a have an issue which
4004 	 * would cause a missing Disconnect Event if there's a
4005 	 * pending Setup Packet in the FIFO.
4006 	 *
4007 	 * There's no suggested workaround on the official Bug
4008 	 * report, which states that "unless the driver/application
4009 	 * is doing any special handling of a disconnect event,
4010 	 * there is no functional issue".
4011 	 *
4012 	 * Unfortunately, it turns out that we _do_ some special
4013 	 * handling of a disconnect event, namely complete all
4014 	 * pending transfers, notify gadget driver of the
4015 	 * disconnection, and so on.
4016 	 *
4017 	 * Our suggested workaround is to follow the Disconnect
4018 	 * Event steps here, instead, based on a setup_packet_pending
4019 	 * flag. Such flag gets set whenever we have a SETUP_PENDING
4020 	 * status for EP0 TRBs and gets cleared on XferComplete for the
4021 	 * same endpoint.
4022 	 *
4023 	 * Refers to:
4024 	 *
4025 	 * STAR#9000466709: RTL: Device : Disconnect event not
4026 	 * generated if setup packet pending in FIFO
4027 	 */
4028 	if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
4029 		if (dwc->setup_packet_pending)
4030 			dwc3_gadget_disconnect_interrupt(dwc);
4031 	}
4032 
4033 	dwc3_reset_gadget(dwc);
4034 
4035 	/*
4036 	 * From SNPS databook section 8.1.2, the EP0 should be in setup
4037 	 * phase. So ensure that EP0 is in setup phase by issuing a stall
4038 	 * and restart if EP0 is not in setup phase.
4039 	 */
4040 	dwc3_ep0_reset_state(dwc);
4041 
4042 	/*
4043 	 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
4044 	 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
4045 	 * needs to ensure that it sends "a DEPENDXFER command for any active
4046 	 * transfers."
4047 	 */
4048 	dwc3_stop_active_transfers(dwc);
4049 	dwc->connected = true;
4050 
4051 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4052 	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
4053 	dwc3_gadget_dctl_write_safe(dwc, reg);
4054 	dwc->test_mode = false;
4055 	dwc->gadget->wakeup_armed = false;
4056 	dwc3_gadget_enable_linksts_evts(dwc, false);
4057 	dwc3_clear_stall_all_ep(dwc);
4058 
4059 	/* Reset device address to zero */
4060 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4061 	reg &= ~(DWC3_DCFG_DEVADDR_MASK);
4062 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4063 }
4064 
4065 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
4066 {
4067 	struct dwc3_ep		*dep;
4068 	int			ret;
4069 	u32			reg;
4070 	u8			lanes = 1;
4071 	u8			speed;
4072 
4073 	if (!dwc->softconnect)
4074 		return;
4075 
4076 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
4077 	speed = reg & DWC3_DSTS_CONNECTSPD;
4078 	dwc->speed = speed;
4079 
4080 	if (DWC3_IP_IS(DWC32))
4081 		lanes = DWC3_DSTS_CONNLANES(reg) + 1;
4082 
4083 	dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4084 
4085 	/*
4086 	 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
4087 	 * each time on Connect Done.
4088 	 *
4089 	 * Currently we always use the reset value. If any platform
4090 	 * wants to set this to a different value, we need to add a
4091 	 * setting and update GCTL.RAMCLKSEL here.
4092 	 */
4093 
4094 	switch (speed) {
4095 	case DWC3_DSTS_SUPERSPEED_PLUS:
4096 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4097 		dwc->gadget->ep0->maxpacket = 512;
4098 		dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4099 
4100 		if (lanes > 1)
4101 			dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
4102 		else
4103 			dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
4104 		break;
4105 	case DWC3_DSTS_SUPERSPEED:
4106 		/*
4107 		 * WORKAROUND: DWC3 revisions <1.90a have an issue which
4108 		 * would cause a missing USB3 Reset event.
4109 		 *
4110 		 * In such situations, we should force a USB3 Reset
4111 		 * event by calling our dwc3_gadget_reset_interrupt()
4112 		 * routine.
4113 		 *
4114 		 * Refers to:
4115 		 *
4116 		 * STAR#9000483510: RTL: SS : USB3 reset event may
4117 		 * not be generated always when the link enters poll
4118 		 */
4119 		if (DWC3_VER_IS_PRIOR(DWC3, 190A))
4120 			dwc3_gadget_reset_interrupt(dwc);
4121 
4122 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4123 		dwc->gadget->ep0->maxpacket = 512;
4124 		dwc->gadget->speed = USB_SPEED_SUPER;
4125 
4126 		if (lanes > 1) {
4127 			dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4128 			dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
4129 		}
4130 		break;
4131 	case DWC3_DSTS_HIGHSPEED:
4132 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4133 		dwc->gadget->ep0->maxpacket = 64;
4134 		dwc->gadget->speed = USB_SPEED_HIGH;
4135 		break;
4136 	case DWC3_DSTS_FULLSPEED:
4137 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4138 		dwc->gadget->ep0->maxpacket = 64;
4139 		dwc->gadget->speed = USB_SPEED_FULL;
4140 		break;
4141 	}
4142 
4143 	dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
4144 
4145 	/* Enable USB2 LPM Capability */
4146 
4147 	if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
4148 	    !dwc->usb2_gadget_lpm_disable &&
4149 	    (speed != DWC3_DSTS_SUPERSPEED) &&
4150 	    (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
4151 		reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4152 		reg |= DWC3_DCFG_LPM_CAP;
4153 		dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4154 
4155 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4156 		reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
4157 
4158 		reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
4159 					    (dwc->is_utmi_l1_suspend << 4));
4160 
4161 		/*
4162 		 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
4163 		 * DCFG.LPMCap is set, core responses with an ACK and the
4164 		 * BESL value in the LPM token is less than or equal to LPM
4165 		 * NYET threshold.
4166 		 */
4167 		WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
4168 				"LPM Erratum not available on dwc3 revisions < 2.40a\n");
4169 
4170 		if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
4171 			reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
4172 
4173 		dwc3_gadget_dctl_write_safe(dwc, reg);
4174 	} else {
4175 		if (dwc->usb2_gadget_lpm_disable) {
4176 			reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4177 			reg &= ~DWC3_DCFG_LPM_CAP;
4178 			dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4179 		}
4180 
4181 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4182 		reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
4183 		dwc3_gadget_dctl_write_safe(dwc, reg);
4184 	}
4185 
4186 	dep = dwc->eps[0];
4187 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4188 	if (ret) {
4189 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4190 		return;
4191 	}
4192 
4193 	dep = dwc->eps[1];
4194 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4195 	if (ret) {
4196 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4197 		return;
4198 	}
4199 
4200 	/*
4201 	 * Configure PHY via GUSB3PIPECTLn if required.
4202 	 *
4203 	 * Update GTXFIFOSIZn
4204 	 *
4205 	 * In both cases reset values should be sufficient.
4206 	 */
4207 }
4208 
4209 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
4210 {
4211 	dwc->suspended = false;
4212 
4213 	/*
4214 	 * TODO take core out of low power mode when that's
4215 	 * implemented.
4216 	 */
4217 
4218 	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
4219 		spin_unlock(&dwc->lock);
4220 		dwc->gadget_driver->resume(dwc->gadget);
4221 		spin_lock(&dwc->lock);
4222 	}
4223 
4224 	dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
4225 }
4226 
4227 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
4228 		unsigned int evtinfo)
4229 {
4230 	enum dwc3_link_state	next = evtinfo & DWC3_LINK_STATE_MASK;
4231 	unsigned int		pwropt;
4232 
4233 	/*
4234 	 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
4235 	 * Hibernation mode enabled which would show up when device detects
4236 	 * host-initiated U3 exit.
4237 	 *
4238 	 * In that case, device will generate a Link State Change Interrupt
4239 	 * from U3 to RESUME which is only necessary if Hibernation is
4240 	 * configured in.
4241 	 *
4242 	 * There are no functional changes due to such spurious event and we
4243 	 * just need to ignore it.
4244 	 *
4245 	 * Refers to:
4246 	 *
4247 	 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
4248 	 * operational mode
4249 	 */
4250 	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
4251 	if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
4252 			(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
4253 		if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
4254 				(next == DWC3_LINK_STATE_RESUME)) {
4255 			return;
4256 		}
4257 	}
4258 
4259 	/*
4260 	 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
4261 	 * on the link partner, the USB session might do multiple entry/exit
4262 	 * of low power states before a transfer takes place.
4263 	 *
4264 	 * Due to this problem, we might experience lower throughput. The
4265 	 * suggested workaround is to disable DCTL[12:9] bits if we're
4266 	 * transitioning from U1/U2 to U0 and enable those bits again
4267 	 * after a transfer completes and there are no pending transfers
4268 	 * on any of the enabled endpoints.
4269 	 *
4270 	 * This is the first half of that workaround.
4271 	 *
4272 	 * Refers to:
4273 	 *
4274 	 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
4275 	 * core send LGO_Ux entering U0
4276 	 */
4277 	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
4278 		if (next == DWC3_LINK_STATE_U0) {
4279 			u32	u1u2;
4280 			u32	reg;
4281 
4282 			switch (dwc->link_state) {
4283 			case DWC3_LINK_STATE_U1:
4284 			case DWC3_LINK_STATE_U2:
4285 				reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4286 				u1u2 = reg & (DWC3_DCTL_INITU2ENA
4287 						| DWC3_DCTL_ACCEPTU2ENA
4288 						| DWC3_DCTL_INITU1ENA
4289 						| DWC3_DCTL_ACCEPTU1ENA);
4290 
4291 				if (!dwc->u1u2)
4292 					dwc->u1u2 = reg & u1u2;
4293 
4294 				reg &= ~u1u2;
4295 
4296 				dwc3_gadget_dctl_write_safe(dwc, reg);
4297 				break;
4298 			default:
4299 				/* do nothing */
4300 				break;
4301 			}
4302 		}
4303 	}
4304 
4305 	switch (next) {
4306 	case DWC3_LINK_STATE_U0:
4307 		if (dwc->gadget->wakeup_armed) {
4308 			dwc3_gadget_enable_linksts_evts(dwc, false);
4309 			dwc3_resume_gadget(dwc);
4310 			dwc->suspended = false;
4311 		}
4312 		break;
4313 	case DWC3_LINK_STATE_U1:
4314 		if (dwc->speed == USB_SPEED_SUPER)
4315 			dwc3_suspend_gadget(dwc);
4316 		break;
4317 	case DWC3_LINK_STATE_U2:
4318 	case DWC3_LINK_STATE_U3:
4319 		dwc3_suspend_gadget(dwc);
4320 		break;
4321 	case DWC3_LINK_STATE_RESUME:
4322 		dwc3_resume_gadget(dwc);
4323 		break;
4324 	default:
4325 		/* do nothing */
4326 		break;
4327 	}
4328 
4329 	dwc->link_state = next;
4330 }
4331 
4332 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4333 					  unsigned int evtinfo)
4334 {
4335 	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4336 
4337 	if (!dwc->suspended && next == DWC3_LINK_STATE_U3) {
4338 		dwc->suspended = true;
4339 		dwc3_suspend_gadget(dwc);
4340 	}
4341 
4342 	dwc->link_state = next;
4343 }
4344 
4345 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4346 		const struct dwc3_event_devt *event)
4347 {
4348 	switch (event->type) {
4349 	case DWC3_DEVICE_EVENT_DISCONNECT:
4350 		dwc3_gadget_disconnect_interrupt(dwc);
4351 		break;
4352 	case DWC3_DEVICE_EVENT_RESET:
4353 		dwc3_gadget_reset_interrupt(dwc);
4354 		break;
4355 	case DWC3_DEVICE_EVENT_CONNECT_DONE:
4356 		dwc3_gadget_conndone_interrupt(dwc);
4357 		break;
4358 	case DWC3_DEVICE_EVENT_WAKEUP:
4359 		dwc3_gadget_wakeup_interrupt(dwc, event->event_info);
4360 		break;
4361 	case DWC3_DEVICE_EVENT_HIBER_REQ:
4362 		dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n");
4363 		break;
4364 	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4365 		dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
4366 		break;
4367 	case DWC3_DEVICE_EVENT_SUSPEND:
4368 		/* It changed to be suspend event for version 2.30a and above */
4369 		if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
4370 			dwc3_gadget_suspend_interrupt(dwc, event->event_info);
4371 		break;
4372 	case DWC3_DEVICE_EVENT_SOF:
4373 	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4374 	case DWC3_DEVICE_EVENT_CMD_CMPL:
4375 	case DWC3_DEVICE_EVENT_OVERFLOW:
4376 		break;
4377 	default:
4378 		dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4379 	}
4380 }
4381 
4382 static void dwc3_process_event_entry(struct dwc3 *dwc,
4383 		const union dwc3_event *event)
4384 {
4385 	trace_dwc3_event(event->raw, dwc);
4386 
4387 	if (!event->type.is_devspec)
4388 		dwc3_endpoint_interrupt(dwc, &event->depevt);
4389 	else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4390 		dwc3_gadget_interrupt(dwc, &event->devt);
4391 	else
4392 		dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4393 }
4394 
4395 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4396 {
4397 	struct dwc3 *dwc = evt->dwc;
4398 	irqreturn_t ret = IRQ_NONE;
4399 	int left;
4400 
4401 	left = evt->count;
4402 
4403 	if (!(evt->flags & DWC3_EVENT_PENDING))
4404 		return IRQ_NONE;
4405 
4406 	while (left > 0) {
4407 		union dwc3_event event;
4408 
4409 		event.raw = *(u32 *) (evt->cache + evt->lpos);
4410 
4411 		dwc3_process_event_entry(dwc, &event);
4412 
4413 		/*
4414 		 * FIXME we wrap around correctly to the next entry as
4415 		 * almost all entries are 4 bytes in size. There is one
4416 		 * entry which has 12 bytes which is a regular entry
4417 		 * followed by 8 bytes data. ATM I don't know how
4418 		 * things are organized if we get next to the a
4419 		 * boundary so I worry about that once we try to handle
4420 		 * that.
4421 		 */
4422 		evt->lpos = (evt->lpos + 4) % evt->length;
4423 		left -= 4;
4424 	}
4425 
4426 	evt->count = 0;
4427 	ret = IRQ_HANDLED;
4428 
4429 	/* Unmask interrupt */
4430 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4431 		    DWC3_GEVNTSIZ_SIZE(evt->length));
4432 
4433 	if (dwc->imod_interval) {
4434 		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4435 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
4436 	}
4437 
4438 	/* Keep the clearing of DWC3_EVENT_PENDING at the end */
4439 	evt->flags &= ~DWC3_EVENT_PENDING;
4440 
4441 	return ret;
4442 }
4443 
4444 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4445 {
4446 	struct dwc3_event_buffer *evt = _evt;
4447 	struct dwc3 *dwc = evt->dwc;
4448 	unsigned long flags;
4449 	irqreturn_t ret = IRQ_NONE;
4450 
4451 	local_bh_disable();
4452 	spin_lock_irqsave(&dwc->lock, flags);
4453 	ret = dwc3_process_event_buf(evt);
4454 	spin_unlock_irqrestore(&dwc->lock, flags);
4455 	local_bh_enable();
4456 
4457 	return ret;
4458 }
4459 
4460 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4461 {
4462 	struct dwc3 *dwc = evt->dwc;
4463 	u32 amount;
4464 	u32 count;
4465 
4466 	if (pm_runtime_suspended(dwc->dev)) {
4467 		dwc->pending_events = true;
4468 		/*
4469 		 * Trigger runtime resume. The get() function will be balanced
4470 		 * after processing the pending events in dwc3_process_pending
4471 		 * events().
4472 		 */
4473 		pm_runtime_get(dwc->dev);
4474 		disable_irq_nosync(dwc->irq_gadget);
4475 		return IRQ_HANDLED;
4476 	}
4477 
4478 	/*
4479 	 * With PCIe legacy interrupt, test shows that top-half irq handler can
4480 	 * be called again after HW interrupt deassertion. Check if bottom-half
4481 	 * irq event handler completes before caching new event to prevent
4482 	 * losing events.
4483 	 */
4484 	if (evt->flags & DWC3_EVENT_PENDING)
4485 		return IRQ_HANDLED;
4486 
4487 	count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
4488 	count &= DWC3_GEVNTCOUNT_MASK;
4489 	if (!count)
4490 		return IRQ_NONE;
4491 
4492 	evt->count = count;
4493 	evt->flags |= DWC3_EVENT_PENDING;
4494 
4495 	/* Mask interrupt */
4496 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4497 		    DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4498 
4499 	amount = min(count, evt->length - evt->lpos);
4500 	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4501 
4502 	if (amount < count)
4503 		memcpy(evt->cache, evt->buf, count - amount);
4504 
4505 	dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
4506 
4507 	return IRQ_WAKE_THREAD;
4508 }
4509 
4510 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4511 {
4512 	struct dwc3_event_buffer	*evt = _evt;
4513 
4514 	return dwc3_check_event_buf(evt);
4515 }
4516 
4517 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4518 {
4519 	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4520 	int irq;
4521 
4522 	irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
4523 	if (irq > 0)
4524 		goto out;
4525 
4526 	if (irq == -EPROBE_DEFER)
4527 		goto out;
4528 
4529 	irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
4530 	if (irq > 0)
4531 		goto out;
4532 
4533 	if (irq == -EPROBE_DEFER)
4534 		goto out;
4535 
4536 	irq = platform_get_irq(dwc3_pdev, 0);
4537 
4538 out:
4539 	return irq;
4540 }
4541 
4542 static void dwc_gadget_release(struct device *dev)
4543 {
4544 	struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4545 
4546 	kfree(gadget);
4547 }
4548 
4549 /**
4550  * dwc3_gadget_init - initializes gadget related registers
4551  * @dwc: pointer to our controller context structure
4552  *
4553  * Returns 0 on success otherwise negative errno.
4554  */
4555 int dwc3_gadget_init(struct dwc3 *dwc)
4556 {
4557 	int ret;
4558 	int irq;
4559 	struct device *dev;
4560 
4561 	irq = dwc3_gadget_get_irq(dwc);
4562 	if (irq < 0) {
4563 		ret = irq;
4564 		goto err0;
4565 	}
4566 
4567 	dwc->irq_gadget = irq;
4568 
4569 	dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
4570 					  sizeof(*dwc->ep0_trb) * 2,
4571 					  &dwc->ep0_trb_addr, GFP_KERNEL);
4572 	if (!dwc->ep0_trb) {
4573 		dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4574 		ret = -ENOMEM;
4575 		goto err0;
4576 	}
4577 
4578 	dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4579 	if (!dwc->setup_buf) {
4580 		ret = -ENOMEM;
4581 		goto err1;
4582 	}
4583 
4584 	dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
4585 			&dwc->bounce_addr, GFP_KERNEL);
4586 	if (!dwc->bounce) {
4587 		ret = -ENOMEM;
4588 		goto err2;
4589 	}
4590 
4591 	init_completion(&dwc->ep0_in_setup);
4592 	dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
4593 	if (!dwc->gadget) {
4594 		ret = -ENOMEM;
4595 		goto err3;
4596 	}
4597 
4598 
4599 	usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
4600 	dev				= &dwc->gadget->dev;
4601 	dev->platform_data		= dwc;
4602 	dwc->gadget->ops		= &dwc3_gadget_ops;
4603 	dwc->gadget->speed		= USB_SPEED_UNKNOWN;
4604 	dwc->gadget->ssp_rate		= USB_SSP_GEN_UNKNOWN;
4605 	dwc->gadget->sg_supported	= true;
4606 	dwc->gadget->name		= "dwc3-gadget";
4607 	dwc->gadget->lpm_capable	= !dwc->usb2_gadget_lpm_disable;
4608 	dwc->gadget->wakeup_capable	= true;
4609 
4610 	/*
4611 	 * FIXME We might be setting max_speed to <SUPER, however versions
4612 	 * <2.20a of dwc3 have an issue with metastability (documented
4613 	 * elsewhere in this driver) which tells us we can't set max speed to
4614 	 * anything lower than SUPER.
4615 	 *
4616 	 * Because gadget.max_speed is only used by composite.c and function
4617 	 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
4618 	 * to happen so we avoid sending SuperSpeed Capability descriptor
4619 	 * together with our BOS descriptor as that could confuse host into
4620 	 * thinking we can handle super speed.
4621 	 *
4622 	 * Note that, in fact, we won't even support GetBOS requests when speed
4623 	 * is less than super speed because we don't have means, yet, to tell
4624 	 * composite.c that we are USB 2.0 + LPM ECN.
4625 	 */
4626 	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4627 	    !dwc->dis_metastability_quirk)
4628 		dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4629 				dwc->revision);
4630 
4631 	dwc->gadget->max_speed		= dwc->maximum_speed;
4632 	dwc->gadget->max_ssp_rate	= dwc->max_ssp_rate;
4633 
4634 	/*
4635 	 * REVISIT: Here we should clear all pending IRQs to be
4636 	 * sure we're starting from a well known location.
4637 	 */
4638 
4639 	ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
4640 	if (ret)
4641 		goto err4;
4642 
4643 	ret = usb_add_gadget(dwc->gadget);
4644 	if (ret) {
4645 		dev_err(dwc->dev, "failed to add gadget\n");
4646 		goto err5;
4647 	}
4648 
4649 	if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4650 		dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);
4651 	else
4652 		dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
4653 
4654 	/* No system wakeup if no gadget driver bound */
4655 	if (dwc->sys_wakeup)
4656 		device_wakeup_disable(dwc->sysdev);
4657 
4658 	return 0;
4659 
4660 err5:
4661 	dwc3_gadget_free_endpoints(dwc);
4662 err4:
4663 	usb_put_gadget(dwc->gadget);
4664 	dwc->gadget = NULL;
4665 err3:
4666 	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4667 			dwc->bounce_addr);
4668 
4669 err2:
4670 	kfree(dwc->setup_buf);
4671 
4672 err1:
4673 	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4674 			dwc->ep0_trb, dwc->ep0_trb_addr);
4675 
4676 err0:
4677 	return ret;
4678 }
4679 
4680 /* -------------------------------------------------------------------------- */
4681 
4682 void dwc3_gadget_exit(struct dwc3 *dwc)
4683 {
4684 	if (!dwc->gadget)
4685 		return;
4686 
4687 	dwc3_enable_susphy(dwc, false);
4688 	usb_del_gadget(dwc->gadget);
4689 	dwc3_gadget_free_endpoints(dwc);
4690 	usb_put_gadget(dwc->gadget);
4691 	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4692 			  dwc->bounce_addr);
4693 	kfree(dwc->setup_buf);
4694 	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4695 			  dwc->ep0_trb, dwc->ep0_trb_addr);
4696 }
4697 
4698 int dwc3_gadget_suspend(struct dwc3 *dwc)
4699 {
4700 	unsigned long flags;
4701 	int ret;
4702 
4703 	ret = dwc3_gadget_soft_disconnect(dwc);
4704 	if (ret)
4705 		goto err;
4706 
4707 	spin_lock_irqsave(&dwc->lock, flags);
4708 	if (dwc->gadget_driver)
4709 		dwc3_disconnect_gadget(dwc);
4710 	spin_unlock_irqrestore(&dwc->lock, flags);
4711 
4712 	return 0;
4713 
4714 err:
4715 	/*
4716 	 * Attempt to reset the controller's state. Likely no
4717 	 * communication can be established until the host
4718 	 * performs a port reset.
4719 	 */
4720 	if (dwc->softconnect)
4721 		dwc3_gadget_soft_connect(dwc);
4722 
4723 	return ret;
4724 }
4725 
4726 int dwc3_gadget_resume(struct dwc3 *dwc)
4727 {
4728 	if (!dwc->gadget_driver || !dwc->softconnect)
4729 		return 0;
4730 
4731 	return dwc3_gadget_soft_connect(dwc);
4732 }
4733