1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4 *
5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6 *
7 * Authors: Felipe Balbi <balbi@ti.com>,
8 * Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9 */
10
11 #include <linux/kernel.h>
12 #include <linux/delay.h>
13 #include <linux/slab.h>
14 #include <linux/spinlock.h>
15 #include <linux/platform_device.h>
16 #include <linux/pm_runtime.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/list.h>
20 #include <linux/dma-mapping.h>
21
22 #include <linux/usb/ch9.h>
23 #include <linux/usb/gadget.h>
24
25 #include "debug.h"
26 #include "core.h"
27 #include "gadget.h"
28 #include "io.h"
29
30 #define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
32
33 /**
34 * dwc3_gadget_set_test_mode - enables usb2 test modes
35 * @dwc: pointer to our context structure
36 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
37 *
38 * Caller should take care of locking. This function will return 0 on
39 * success or -EINVAL if wrong Test Selector is passed.
40 */
dwc3_gadget_set_test_mode(struct dwc3 * dwc,int mode)41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42 {
43 u32 reg;
44
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
46 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47
48 switch (mode) {
49 case USB_TEST_J:
50 case USB_TEST_K:
51 case USB_TEST_SE0_NAK:
52 case USB_TEST_PACKET:
53 case USB_TEST_FORCE_ENABLE:
54 reg |= mode << 1;
55 break;
56 default:
57 return -EINVAL;
58 }
59
60 dwc3_gadget_dctl_write_safe(dwc, reg);
61
62 return 0;
63 }
64
65 /**
66 * dwc3_gadget_get_link_state - gets current state of usb link
67 * @dwc: pointer to our context structure
68 *
69 * Caller should take care of locking. This function will
70 * return the link state on success (>= 0) or -ETIMEDOUT.
71 */
dwc3_gadget_get_link_state(struct dwc3 * dwc)72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73 {
74 u32 reg;
75
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
77
78 return DWC3_DSTS_USBLNKST(reg);
79 }
80
81 /**
82 * dwc3_gadget_set_link_state - sets usb link to a particular state
83 * @dwc: pointer to our context structure
84 * @state: the state to put link into
85 *
86 * Caller should take care of locking. This function will
87 * return 0 on success or -ETIMEDOUT.
88 */
dwc3_gadget_set_link_state(struct dwc3 * dwc,enum dwc3_link_state state)89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90 {
91 int retries = 10000;
92 u32 reg;
93
94 /*
95 * Wait until device controller is ready. Only applies to 1.94a and
96 * later RTL.
97 */
98 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99 while (--retries) {
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101 if (reg & DWC3_DSTS_DCNRD)
102 udelay(5);
103 else
104 break;
105 }
106
107 if (retries <= 0)
108 return -ETIMEDOUT;
109 }
110
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113
114 /* set no action before sending new link state change */
115 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
116
117 /* set requested state */
118 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
120
121 /*
122 * The following code is racy when called from dwc3_gadget_wakeup,
123 * and is not needed, at least on newer versions
124 */
125 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126 return 0;
127
128 /* wait for a change in DSTS */
129 retries = 10000;
130 while (--retries) {
131 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
132
133 if (DWC3_DSTS_USBLNKST(reg) == state)
134 return 0;
135
136 udelay(5);
137 }
138
139 return -ETIMEDOUT;
140 }
141
dwc3_ep0_reset_state(struct dwc3 * dwc)142 static void dwc3_ep0_reset_state(struct dwc3 *dwc)
143 {
144 unsigned int dir;
145
146 if (dwc->ep0state != EP0_SETUP_PHASE) {
147 dir = !!dwc->ep0_expect_in;
148 if (dwc->ep0state == EP0_DATA_PHASE)
149 dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
150 else
151 dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
152
153 dwc->eps[0]->trb_enqueue = 0;
154 dwc->eps[1]->trb_enqueue = 0;
155
156 dwc3_ep0_stall_and_restart(dwc);
157 }
158 }
159
160 /**
161 * dwc3_ep_inc_trb - increment a trb index.
162 * @index: Pointer to the TRB index to increment.
163 *
164 * The index should never point to the link TRB. After incrementing,
165 * if it is point to the link TRB, wrap around to the beginning. The
166 * link TRB is always at the last TRB entry.
167 */
dwc3_ep_inc_trb(u8 * index)168 static void dwc3_ep_inc_trb(u8 *index)
169 {
170 (*index)++;
171 if (*index == (DWC3_TRB_NUM - 1))
172 *index = 0;
173 }
174
175 /**
176 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
177 * @dep: The endpoint whose enqueue pointer we're incrementing
178 */
dwc3_ep_inc_enq(struct dwc3_ep * dep)179 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
180 {
181 dwc3_ep_inc_trb(&dep->trb_enqueue);
182 }
183
184 /**
185 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
186 * @dep: The endpoint whose enqueue pointer we're incrementing
187 */
dwc3_ep_inc_deq(struct dwc3_ep * dep)188 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
189 {
190 dwc3_ep_inc_trb(&dep->trb_dequeue);
191 }
192
dwc3_gadget_del_and_unmap_request(struct dwc3_ep * dep,struct dwc3_request * req,int status)193 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
194 struct dwc3_request *req, int status)
195 {
196 struct dwc3 *dwc = dep->dwc;
197
198 list_del(&req->list);
199 req->remaining = 0;
200 req->needs_extra_trb = false;
201 req->num_trbs = 0;
202
203 if (req->request.status == -EINPROGRESS)
204 req->request.status = status;
205
206 if (req->trb)
207 usb_gadget_unmap_request_by_dev(dwc->sysdev,
208 &req->request, req->direction);
209
210 req->trb = NULL;
211 trace_dwc3_gadget_giveback(req);
212
213 if (dep->number > 1)
214 pm_runtime_put(dwc->dev);
215 }
216
217 /**
218 * dwc3_gadget_giveback - call struct usb_request's ->complete callback
219 * @dep: The endpoint to whom the request belongs to
220 * @req: The request we're giving back
221 * @status: completion code for the request
222 *
223 * Must be called with controller's lock held and interrupts disabled. This
224 * function will unmap @req and call its ->complete() callback to notify upper
225 * layers that it has completed.
226 */
dwc3_gadget_giveback(struct dwc3_ep * dep,struct dwc3_request * req,int status)227 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
228 int status)
229 {
230 struct dwc3 *dwc = dep->dwc;
231
232 dwc3_gadget_del_and_unmap_request(dep, req, status);
233 req->status = DWC3_REQUEST_STATUS_COMPLETED;
234
235 spin_unlock(&dwc->lock);
236 usb_gadget_giveback_request(&dep->endpoint, &req->request);
237 spin_lock(&dwc->lock);
238 }
239
240 /**
241 * dwc3_send_gadget_generic_command - issue a generic command for the controller
242 * @dwc: pointer to the controller context
243 * @cmd: the command to be issued
244 * @param: command parameter
245 *
246 * Caller should take care of locking. Issue @cmd with a given @param to @dwc
247 * and wait for its completion.
248 */
dwc3_send_gadget_generic_command(struct dwc3 * dwc,unsigned int cmd,u32 param)249 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
250 u32 param)
251 {
252 u32 timeout = 500;
253 int status = 0;
254 int ret = 0;
255 u32 reg;
256
257 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
258 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
259
260 do {
261 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
262 if (!(reg & DWC3_DGCMD_CMDACT)) {
263 status = DWC3_DGCMD_STATUS(reg);
264 if (status)
265 ret = -EINVAL;
266 break;
267 }
268 } while (--timeout);
269
270 if (!timeout) {
271 ret = -ETIMEDOUT;
272 status = -ETIMEDOUT;
273 }
274
275 trace_dwc3_gadget_generic_cmd(cmd, param, status);
276
277 return ret;
278 }
279
280 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async);
281
282 /**
283 * dwc3_send_gadget_ep_cmd - issue an endpoint command
284 * @dep: the endpoint to which the command is going to be issued
285 * @cmd: the command to be issued
286 * @params: parameters to the command
287 *
288 * Caller should handle locking. This function will issue @cmd with given
289 * @params to @dep and wait for its completion.
290 *
291 * According to the programming guide, if the link state is in L1/L2/U3,
292 * then sending the Start Transfer command may not complete. The
293 * programming guide suggested to bring the link state back to ON/U0 by
294 * performing remote wakeup prior to sending the command. However, don't
295 * initiate remote wakeup when the user/function does not send wakeup
296 * request via wakeup ops. Send the command when it's allowed.
297 *
298 * Notes:
299 * For L1 link state, issuing a command requires the clearing of
300 * GUSB2PHYCFG.SUSPENDUSB2, which turns on the signal required to complete
301 * the given command (usually within 50us). This should happen within the
302 * command timeout set by driver. No additional step is needed.
303 *
304 * For L2 or U3 link state, the gadget is in USB suspend. Care should be
305 * taken when sending Start Transfer command to ensure that it's done after
306 * USB resume.
307 */
dwc3_send_gadget_ep_cmd(struct dwc3_ep * dep,unsigned int cmd,struct dwc3_gadget_ep_cmd_params * params)308 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
309 struct dwc3_gadget_ep_cmd_params *params)
310 {
311 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
312 struct dwc3 *dwc = dep->dwc;
313 u32 timeout = 5000;
314 u32 saved_config = 0;
315 u32 reg;
316
317 int cmd_status = 0;
318 int ret = -EINVAL;
319
320 /*
321 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
322 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
323 * endpoint command.
324 *
325 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
326 * settings. Restore them after the command is completed.
327 *
328 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
329 */
330 if (dwc->gadget->speed <= USB_SPEED_HIGH ||
331 DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) {
332 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
333 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
334 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
335 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
336 }
337
338 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
339 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
340 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
341 }
342
343 if (saved_config)
344 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
345 }
346
347 /*
348 * For some commands such as Update Transfer command, DEPCMDPARn
349 * registers are reserved. Since the driver often sends Update Transfer
350 * command, don't write to DEPCMDPARn to avoid register write delays and
351 * improve performance.
352 */
353 if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
354 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
355 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
356 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
357 }
358
359 /*
360 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
361 * not relying on XferNotReady, we can make use of a special "No
362 * Response Update Transfer" command where we should clear both CmdAct
363 * and CmdIOC bits.
364 *
365 * With this, we don't need to wait for command completion and can
366 * straight away issue further commands to the endpoint.
367 *
368 * NOTICE: We're making an assumption that control endpoints will never
369 * make use of Update Transfer command. This is a safe assumption
370 * because we can never have more than one request at a time with
371 * Control Endpoints. If anybody changes that assumption, this chunk
372 * needs to be updated accordingly.
373 */
374 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
375 !usb_endpoint_xfer_isoc(desc))
376 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
377 else
378 cmd |= DWC3_DEPCMD_CMDACT;
379
380 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
381
382 if (!(cmd & DWC3_DEPCMD_CMDACT) ||
383 (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
384 !(cmd & DWC3_DEPCMD_CMDIOC))) {
385 ret = 0;
386 goto skip_status;
387 }
388
389 do {
390 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
391 if (!(reg & DWC3_DEPCMD_CMDACT)) {
392 cmd_status = DWC3_DEPCMD_STATUS(reg);
393
394 switch (cmd_status) {
395 case 0:
396 ret = 0;
397 break;
398 case DEPEVT_TRANSFER_NO_RESOURCE:
399 dev_WARN(dwc->dev, "No resource for %s\n",
400 dep->name);
401 ret = -EINVAL;
402 break;
403 case DEPEVT_TRANSFER_BUS_EXPIRY:
404 /*
405 * SW issues START TRANSFER command to
406 * isochronous ep with future frame interval. If
407 * future interval time has already passed when
408 * core receives the command, it will respond
409 * with an error status of 'Bus Expiry'.
410 *
411 * Instead of always returning -EINVAL, let's
412 * give a hint to the gadget driver that this is
413 * the case by returning -EAGAIN.
414 */
415 ret = -EAGAIN;
416 break;
417 default:
418 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
419 }
420
421 break;
422 }
423 } while (--timeout);
424
425 if (timeout == 0) {
426 ret = -ETIMEDOUT;
427 cmd_status = -ETIMEDOUT;
428 }
429
430 skip_status:
431 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
432
433 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
434 if (ret == 0)
435 dep->flags |= DWC3_EP_TRANSFER_STARTED;
436
437 if (ret != -ETIMEDOUT)
438 dwc3_gadget_ep_get_transfer_index(dep);
439 }
440
441 if (saved_config) {
442 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
443 reg |= saved_config;
444 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
445 }
446
447 return ret;
448 }
449
dwc3_send_clear_stall_ep_cmd(struct dwc3_ep * dep)450 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
451 {
452 struct dwc3 *dwc = dep->dwc;
453 struct dwc3_gadget_ep_cmd_params params;
454 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
455
456 /*
457 * As of core revision 2.60a the recommended programming model
458 * is to set the ClearPendIN bit when issuing a Clear Stall EP
459 * command for IN endpoints. This is to prevent an issue where
460 * some (non-compliant) hosts may not send ACK TPs for pending
461 * IN transfers due to a mishandled error condition. Synopsys
462 * STAR 9000614252.
463 */
464 if (dep->direction &&
465 !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
466 (dwc->gadget->speed >= USB_SPEED_SUPER))
467 cmd |= DWC3_DEPCMD_CLEARPENDIN;
468
469 memset(¶ms, 0, sizeof(params));
470
471 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
472 }
473
dwc3_trb_dma_offset(struct dwc3_ep * dep,struct dwc3_trb * trb)474 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
475 struct dwc3_trb *trb)
476 {
477 u32 offset = (char *) trb - (char *) dep->trb_pool;
478
479 return dep->trb_pool_dma + offset;
480 }
481
dwc3_alloc_trb_pool(struct dwc3_ep * dep)482 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
483 {
484 struct dwc3 *dwc = dep->dwc;
485
486 if (dep->trb_pool)
487 return 0;
488
489 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
490 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
491 &dep->trb_pool_dma, GFP_KERNEL);
492 if (!dep->trb_pool) {
493 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
494 dep->name);
495 return -ENOMEM;
496 }
497
498 return 0;
499 }
500
dwc3_free_trb_pool(struct dwc3_ep * dep)501 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
502 {
503 struct dwc3 *dwc = dep->dwc;
504
505 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
506 dep->trb_pool, dep->trb_pool_dma);
507
508 dep->trb_pool = NULL;
509 dep->trb_pool_dma = 0;
510 }
511
dwc3_gadget_set_xfer_resource(struct dwc3_ep * dep)512 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
513 {
514 struct dwc3_gadget_ep_cmd_params params;
515 int ret;
516
517 if (dep->flags & DWC3_EP_RESOURCE_ALLOCATED)
518 return 0;
519
520 memset(¶ms, 0x00, sizeof(params));
521
522 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
523
524 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
525 ¶ms);
526 if (ret)
527 return ret;
528
529 dep->flags |= DWC3_EP_RESOURCE_ALLOCATED;
530 return 0;
531 }
532
533 /**
534 * dwc3_gadget_start_config - reset endpoint resources
535 * @dwc: pointer to the DWC3 context
536 * @resource_index: DEPSTARTCFG.XferRscIdx value (must be 0 or 2)
537 *
538 * Set resource_index=0 to reset all endpoints' resources allocation. Do this as
539 * part of the power-on/soft-reset initialization.
540 *
541 * Set resource_index=2 to reset only non-control endpoints' resources. Do this
542 * on receiving the SET_CONFIGURATION request or hibernation resume.
543 */
dwc3_gadget_start_config(struct dwc3 * dwc,unsigned int resource_index)544 int dwc3_gadget_start_config(struct dwc3 *dwc, unsigned int resource_index)
545 {
546 struct dwc3_gadget_ep_cmd_params params;
547 u32 cmd;
548 int i;
549 int ret;
550
551 if (resource_index != 0 && resource_index != 2)
552 return -EINVAL;
553
554 memset(¶ms, 0x00, sizeof(params));
555 cmd = DWC3_DEPCMD_DEPSTARTCFG;
556 cmd |= DWC3_DEPCMD_PARAM(resource_index);
557
558 ret = dwc3_send_gadget_ep_cmd(dwc->eps[0], cmd, ¶ms);
559 if (ret)
560 return ret;
561
562 /* Reset resource allocation flags */
563 for (i = resource_index; i < dwc->num_eps && dwc->eps[i]; i++)
564 dwc->eps[i]->flags &= ~DWC3_EP_RESOURCE_ALLOCATED;
565
566 return 0;
567 }
568
dwc3_gadget_set_ep_config(struct dwc3_ep * dep,unsigned int action)569 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
570 {
571 const struct usb_ss_ep_comp_descriptor *comp_desc;
572 const struct usb_endpoint_descriptor *desc;
573 struct dwc3_gadget_ep_cmd_params params;
574 struct dwc3 *dwc = dep->dwc;
575
576 comp_desc = dep->endpoint.comp_desc;
577 desc = dep->endpoint.desc;
578
579 memset(¶ms, 0x00, sizeof(params));
580
581 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
582 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
583
584 /* Burst size is only needed in SuperSpeed mode */
585 if (dwc->gadget->speed >= USB_SPEED_SUPER) {
586 u32 burst = dep->endpoint.maxburst;
587
588 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
589 }
590
591 params.param0 |= action;
592 if (action == DWC3_DEPCFG_ACTION_RESTORE)
593 params.param2 |= dep->saved_state;
594
595 if (usb_endpoint_xfer_control(desc))
596 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
597
598 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
599 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
600
601 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
602 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
603 | DWC3_DEPCFG_XFER_COMPLETE_EN
604 | DWC3_DEPCFG_STREAM_EVENT_EN;
605 dep->stream_capable = true;
606 }
607
608 if (!usb_endpoint_xfer_control(desc))
609 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
610
611 /*
612 * We are doing 1:1 mapping for endpoints, meaning
613 * Physical Endpoints 2 maps to Logical Endpoint 2 and
614 * so on. We consider the direction bit as part of the physical
615 * endpoint number. So USB endpoint 0x81 is 0x03.
616 */
617 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
618
619 /*
620 * We must use the lower 16 TX FIFOs even though
621 * HW might have more
622 */
623 if (dep->direction)
624 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
625
626 if (desc->bInterval) {
627 u8 bInterval_m1;
628
629 /*
630 * Valid range for DEPCFG.bInterval_m1 is from 0 to 13.
631 *
632 * NOTE: The programming guide incorrectly stated bInterval_m1
633 * must be set to 0 when operating in fullspeed. Internally the
634 * controller does not have this limitation. See DWC_usb3x
635 * programming guide section 3.2.2.1.
636 */
637 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
638
639 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
640 dwc->gadget->speed == USB_SPEED_FULL)
641 dep->interval = desc->bInterval;
642 else
643 dep->interval = 1 << (desc->bInterval - 1);
644
645 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
646 }
647
648 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms);
649 }
650
651 /**
652 * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
653 * @dwc: pointer to the DWC3 context
654 * @mult: multiplier to be used when calculating the fifo_size
655 *
656 * Calculates the size value based on the equation below:
657 *
658 * DWC3 revision 280A and prior:
659 * fifo_size = mult * (max_packet / mdwidth) + 1;
660 *
661 * DWC3 revision 290A and onwards:
662 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
663 *
664 * The max packet size is set to 1024, as the txfifo requirements mainly apply
665 * to super speed USB use cases. However, it is safe to overestimate the fifo
666 * allocations for other scenarios, i.e. high speed USB.
667 */
dwc3_gadget_calc_tx_fifo_size(struct dwc3 * dwc,int mult)668 static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
669 {
670 int max_packet = 1024;
671 int fifo_size;
672 int mdwidth;
673
674 mdwidth = dwc3_mdwidth(dwc);
675
676 /* MDWIDTH is represented in bits, we need it in bytes */
677 mdwidth >>= 3;
678
679 if (DWC3_VER_IS_PRIOR(DWC3, 290A))
680 fifo_size = mult * (max_packet / mdwidth) + 1;
681 else
682 fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
683 return fifo_size;
684 }
685
686 /**
687 * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
688 * @dwc: pointer to the DWC3 context
689 *
690 * Iterates through all the endpoint registers and clears the previous txfifo
691 * allocations.
692 */
dwc3_gadget_clear_tx_fifos(struct dwc3 * dwc)693 void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
694 {
695 struct dwc3_ep *dep;
696 int fifo_depth;
697 int size;
698 int num;
699
700 if (!dwc->do_fifo_resize)
701 return;
702
703 /* Read ep0IN related TXFIFO size */
704 dep = dwc->eps[1];
705 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
706 if (DWC3_IP_IS(DWC3))
707 fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
708 else
709 fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
710
711 dwc->last_fifo_depth = fifo_depth;
712 /* Clear existing TXFIFO for all IN eps except ep0 */
713 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
714 num += 2) {
715 dep = dwc->eps[num];
716 /* Don't change TXFRAMNUM on usb31 version */
717 size = DWC3_IP_IS(DWC3) ? 0 :
718 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
719 DWC31_GTXFIFOSIZ_TXFRAMNUM;
720
721 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
722 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
723 }
724 dwc->num_ep_resized = 0;
725 }
726
727 /*
728 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
729 * @dwc: pointer to our context structure
730 *
731 * This function will a best effort FIFO allocation in order
732 * to improve FIFO usage and throughput, while still allowing
733 * us to enable as many endpoints as possible.
734 *
735 * Keep in mind that this operation will be highly dependent
736 * on the configured size for RAM1 - which contains TxFifo -,
737 * the amount of endpoints enabled on coreConsultant tool, and
738 * the width of the Master Bus.
739 *
740 * In general, FIFO depths are represented with the following equation:
741 *
742 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
743 *
744 * In conjunction with dwc3_gadget_check_config(), this resizing logic will
745 * ensure that all endpoints will have enough internal memory for one max
746 * packet per endpoint.
747 */
dwc3_gadget_resize_tx_fifos(struct dwc3_ep * dep)748 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
749 {
750 struct dwc3 *dwc = dep->dwc;
751 int fifo_0_start;
752 int ram1_depth;
753 int fifo_size;
754 int min_depth;
755 int num_in_ep;
756 int remaining;
757 int num_fifos = 1;
758 int fifo;
759 int tmp;
760
761 if (!dwc->do_fifo_resize)
762 return 0;
763
764 /* resize IN endpoints except ep0 */
765 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)
766 return 0;
767
768 /* bail if already resized */
769 if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
770 return 0;
771
772 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
773
774 if ((dep->endpoint.maxburst > 1 &&
775 usb_endpoint_xfer_bulk(dep->endpoint.desc)) ||
776 usb_endpoint_xfer_isoc(dep->endpoint.desc))
777 num_fifos = 3;
778
779 if (dep->endpoint.maxburst > 6 &&
780 (usb_endpoint_xfer_bulk(dep->endpoint.desc) ||
781 usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
782 num_fifos = dwc->tx_fifo_resize_max_num;
783
784 /* FIFO size for a single buffer */
785 fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1);
786
787 /* Calculate the number of remaining EPs w/o any FIFO */
788 num_in_ep = dwc->max_cfg_eps;
789 num_in_ep -= dwc->num_ep_resized;
790
791 /* Reserve at least one FIFO for the number of IN EPs */
792 min_depth = num_in_ep * (fifo + 1);
793 remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
794 remaining = max_t(int, 0, remaining);
795 /*
796 * We've already reserved 1 FIFO per EP, so check what we can fit in
797 * addition to it. If there is not enough remaining space, allocate
798 * all the remaining space to the EP.
799 */
800 fifo_size = (num_fifos - 1) * fifo;
801 if (remaining < fifo_size)
802 fifo_size = remaining;
803
804 fifo_size += fifo;
805 /* Last increment according to the TX FIFO size equation */
806 fifo_size++;
807
808 /* Check if TXFIFOs start at non-zero addr */
809 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
810 fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
811
812 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
813 if (DWC3_IP_IS(DWC3))
814 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
815 else
816 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
817
818 /* Check fifo size allocation doesn't exceed available RAM size. */
819 if (dwc->last_fifo_depth >= ram1_depth) {
820 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
821 dwc->last_fifo_depth, ram1_depth,
822 dep->endpoint.name, fifo_size);
823 if (DWC3_IP_IS(DWC3))
824 fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
825 else
826 fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
827
828 dwc->last_fifo_depth -= fifo_size;
829 return -ENOMEM;
830 }
831
832 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
833 dep->flags |= DWC3_EP_TXFIFO_RESIZED;
834 dwc->num_ep_resized++;
835
836 return 0;
837 }
838
839 /**
840 * __dwc3_gadget_ep_enable - initializes a hw endpoint
841 * @dep: endpoint to be initialized
842 * @action: one of INIT, MODIFY or RESTORE
843 *
844 * Caller should take care of locking. Execute all necessary commands to
845 * initialize a HW endpoint so it can be used by a gadget driver.
846 */
__dwc3_gadget_ep_enable(struct dwc3_ep * dep,unsigned int action)847 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
848 {
849 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
850 struct dwc3 *dwc = dep->dwc;
851
852 u32 reg;
853 int ret;
854
855 if (!(dep->flags & DWC3_EP_ENABLED)) {
856 ret = dwc3_gadget_resize_tx_fifos(dep);
857 if (ret)
858 return ret;
859 }
860
861 ret = dwc3_gadget_set_ep_config(dep, action);
862 if (ret)
863 return ret;
864
865 if (!(dep->flags & DWC3_EP_RESOURCE_ALLOCATED)) {
866 ret = dwc3_gadget_set_xfer_resource(dep);
867 if (ret)
868 return ret;
869 }
870
871 if (!(dep->flags & DWC3_EP_ENABLED)) {
872 struct dwc3_trb *trb_st_hw;
873 struct dwc3_trb *trb_link;
874
875 dep->type = usb_endpoint_type(desc);
876 dep->flags |= DWC3_EP_ENABLED;
877
878 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
879 reg |= DWC3_DALEPENA_EP(dep->number);
880 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
881
882 dep->trb_dequeue = 0;
883 dep->trb_enqueue = 0;
884
885 if (usb_endpoint_xfer_control(desc))
886 goto out;
887
888 /* Initialize the TRB ring */
889 memset(dep->trb_pool, 0,
890 sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
891
892 /* Link TRB. The HWO bit is never reset */
893 trb_st_hw = &dep->trb_pool[0];
894
895 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
896 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
897 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
898 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
899 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
900 }
901
902 /*
903 * Issue StartTransfer here with no-op TRB so we can always rely on No
904 * Response Update Transfer command.
905 */
906 if (usb_endpoint_xfer_bulk(desc) ||
907 usb_endpoint_xfer_int(desc)) {
908 struct dwc3_gadget_ep_cmd_params params;
909 struct dwc3_trb *trb;
910 dma_addr_t trb_dma;
911 u32 cmd;
912
913 memset(¶ms, 0, sizeof(params));
914 trb = &dep->trb_pool[0];
915 trb_dma = dwc3_trb_dma_offset(dep, trb);
916
917 params.param0 = upper_32_bits(trb_dma);
918 params.param1 = lower_32_bits(trb_dma);
919
920 cmd = DWC3_DEPCMD_STARTTRANSFER;
921
922 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
923 if (ret < 0)
924 return ret;
925
926 if (dep->stream_capable) {
927 /*
928 * For streams, at start, there maybe a race where the
929 * host primes the endpoint before the function driver
930 * queues a request to initiate a stream. In that case,
931 * the controller will not see the prime to generate the
932 * ERDY and start stream. To workaround this, issue a
933 * no-op TRB as normal, but end it immediately. As a
934 * result, when the function driver queues the request,
935 * the next START_TRANSFER command will cause the
936 * controller to generate an ERDY to initiate the
937 * stream.
938 */
939 dwc3_stop_active_transfer(dep, true, true);
940
941 /*
942 * All stream eps will reinitiate stream on NoStream
943 * rejection until we can determine that the host can
944 * prime after the first transfer.
945 *
946 * However, if the controller is capable of
947 * TXF_FLUSH_BYPASS, then IN direction endpoints will
948 * automatically restart the stream without the driver
949 * initiation.
950 */
951 if (!dep->direction ||
952 !(dwc->hwparams.hwparams9 &
953 DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
954 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
955 }
956 }
957
958 out:
959 trace_dwc3_gadget_ep_enable(dep);
960
961 return 0;
962 }
963
dwc3_remove_requests(struct dwc3 * dwc,struct dwc3_ep * dep,int status)964 void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status)
965 {
966 struct dwc3_request *req;
967
968 dwc3_stop_active_transfer(dep, true, false);
969
970 /* If endxfer is delayed, avoid unmapping requests */
971 if (dep->flags & DWC3_EP_DELAY_STOP)
972 return;
973
974 /* - giveback all requests to gadget driver */
975 while (!list_empty(&dep->started_list)) {
976 req = next_request(&dep->started_list);
977
978 dwc3_gadget_giveback(dep, req, status);
979 }
980
981 while (!list_empty(&dep->pending_list)) {
982 req = next_request(&dep->pending_list);
983
984 dwc3_gadget_giveback(dep, req, status);
985 }
986
987 while (!list_empty(&dep->cancelled_list)) {
988 req = next_request(&dep->cancelled_list);
989
990 dwc3_gadget_giveback(dep, req, status);
991 }
992 }
993
994 /**
995 * __dwc3_gadget_ep_disable - disables a hw endpoint
996 * @dep: the endpoint to disable
997 *
998 * This function undoes what __dwc3_gadget_ep_enable did and also removes
999 * requests which are currently being processed by the hardware and those which
1000 * are not yet scheduled.
1001 *
1002 * Caller should take care of locking.
1003 */
__dwc3_gadget_ep_disable(struct dwc3_ep * dep)1004 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1005 {
1006 struct dwc3 *dwc = dep->dwc;
1007 u32 reg;
1008 u32 mask;
1009
1010 trace_dwc3_gadget_ep_disable(dep);
1011
1012 /* make sure HW endpoint isn't stalled */
1013 if (dep->flags & DWC3_EP_STALL)
1014 __dwc3_gadget_ep_set_halt(dep, 0, false);
1015
1016 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
1017 reg &= ~DWC3_DALEPENA_EP(dep->number);
1018 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
1019
1020 dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
1021
1022 dep->stream_capable = false;
1023 dep->type = 0;
1024 mask = DWC3_EP_TXFIFO_RESIZED | DWC3_EP_RESOURCE_ALLOCATED;
1025 /*
1026 * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is
1027 * set. Do not clear DEP flags, so that the end transfer command will
1028 * be reattempted during the next SETUP stage.
1029 */
1030 if (dep->flags & DWC3_EP_DELAY_STOP)
1031 mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED);
1032 dep->flags &= mask;
1033
1034 /* Clear out the ep descriptors for non-ep0 */
1035 if (dep->number > 1) {
1036 dep->endpoint.comp_desc = NULL;
1037 dep->endpoint.desc = NULL;
1038 }
1039
1040 return 0;
1041 }
1042
1043 /* -------------------------------------------------------------------------- */
1044
dwc3_gadget_ep0_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)1045 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1046 const struct usb_endpoint_descriptor *desc)
1047 {
1048 return -EINVAL;
1049 }
1050
dwc3_gadget_ep0_disable(struct usb_ep * ep)1051 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1052 {
1053 return -EINVAL;
1054 }
1055
1056 /* -------------------------------------------------------------------------- */
1057
dwc3_gadget_ep_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)1058 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1059 const struct usb_endpoint_descriptor *desc)
1060 {
1061 struct dwc3_ep *dep;
1062 struct dwc3 *dwc;
1063 unsigned long flags;
1064 int ret;
1065
1066 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1067 pr_debug("dwc3: invalid parameters\n");
1068 return -EINVAL;
1069 }
1070
1071 if (!desc->wMaxPacketSize) {
1072 pr_debug("dwc3: missing wMaxPacketSize\n");
1073 return -EINVAL;
1074 }
1075
1076 dep = to_dwc3_ep(ep);
1077 dwc = dep->dwc;
1078
1079 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1080 "%s is already enabled\n",
1081 dep->name))
1082 return 0;
1083
1084 spin_lock_irqsave(&dwc->lock, flags);
1085 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1086 spin_unlock_irqrestore(&dwc->lock, flags);
1087
1088 return ret;
1089 }
1090
dwc3_gadget_ep_disable(struct usb_ep * ep)1091 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1092 {
1093 struct dwc3_ep *dep;
1094 struct dwc3 *dwc;
1095 unsigned long flags;
1096 int ret;
1097
1098 if (!ep) {
1099 pr_debug("dwc3: invalid parameters\n");
1100 return -EINVAL;
1101 }
1102
1103 dep = to_dwc3_ep(ep);
1104 dwc = dep->dwc;
1105
1106 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1107 "%s is already disabled\n",
1108 dep->name))
1109 return 0;
1110
1111 spin_lock_irqsave(&dwc->lock, flags);
1112 ret = __dwc3_gadget_ep_disable(dep);
1113 spin_unlock_irqrestore(&dwc->lock, flags);
1114
1115 return ret;
1116 }
1117
dwc3_gadget_ep_alloc_request(struct usb_ep * ep,gfp_t gfp_flags)1118 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1119 gfp_t gfp_flags)
1120 {
1121 struct dwc3_request *req;
1122 struct dwc3_ep *dep = to_dwc3_ep(ep);
1123
1124 req = kzalloc(sizeof(*req), gfp_flags);
1125 if (!req)
1126 return NULL;
1127
1128 req->direction = dep->direction;
1129 req->epnum = dep->number;
1130 req->dep = dep;
1131 req->status = DWC3_REQUEST_STATUS_UNKNOWN;
1132
1133 trace_dwc3_alloc_request(req);
1134
1135 return &req->request;
1136 }
1137
dwc3_gadget_ep_free_request(struct usb_ep * ep,struct usb_request * request)1138 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1139 struct usb_request *request)
1140 {
1141 struct dwc3_request *req = to_dwc3_request(request);
1142
1143 trace_dwc3_free_request(req);
1144 kfree(req);
1145 }
1146
1147 /**
1148 * dwc3_ep_prev_trb - returns the previous TRB in the ring
1149 * @dep: The endpoint with the TRB ring
1150 * @index: The index of the current TRB in the ring
1151 *
1152 * Returns the TRB prior to the one pointed to by the index. If the
1153 * index is 0, we will wrap backwards, skip the link TRB, and return
1154 * the one just before that.
1155 */
dwc3_ep_prev_trb(struct dwc3_ep * dep,u8 index)1156 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1157 {
1158 u8 tmp = index;
1159
1160 if (!tmp)
1161 tmp = DWC3_TRB_NUM - 1;
1162
1163 return &dep->trb_pool[tmp - 1];
1164 }
1165
dwc3_calc_trbs_left(struct dwc3_ep * dep)1166 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1167 {
1168 u8 trbs_left;
1169
1170 /*
1171 * If the enqueue & dequeue are equal then the TRB ring is either full
1172 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs
1173 * pending to be processed by the driver.
1174 */
1175 if (dep->trb_enqueue == dep->trb_dequeue) {
1176 /*
1177 * If there is any request remained in the started_list at
1178 * this point, that means there is no TRB available.
1179 */
1180 if (!list_empty(&dep->started_list))
1181 return 0;
1182
1183 return DWC3_TRB_NUM - 1;
1184 }
1185
1186 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1187 trbs_left &= (DWC3_TRB_NUM - 1);
1188
1189 if (dep->trb_dequeue < dep->trb_enqueue)
1190 trbs_left--;
1191
1192 return trbs_left;
1193 }
1194
1195 /**
1196 * dwc3_prepare_one_trb - setup one TRB from one request
1197 * @dep: endpoint for which this request is prepared
1198 * @req: dwc3_request pointer
1199 * @trb_length: buffer size of the TRB
1200 * @chain: should this TRB be chained to the next?
1201 * @node: only for isochronous endpoints. First TRB needs different type.
1202 * @use_bounce_buffer: set to use bounce buffer
1203 * @must_interrupt: set to interrupt on TRB completion
1204 */
dwc3_prepare_one_trb(struct dwc3_ep * dep,struct dwc3_request * req,unsigned int trb_length,unsigned int chain,unsigned int node,bool use_bounce_buffer,bool must_interrupt)1205 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1206 struct dwc3_request *req, unsigned int trb_length,
1207 unsigned int chain, unsigned int node, bool use_bounce_buffer,
1208 bool must_interrupt)
1209 {
1210 struct dwc3_trb *trb;
1211 dma_addr_t dma;
1212 unsigned int stream_id = req->request.stream_id;
1213 unsigned int short_not_ok = req->request.short_not_ok;
1214 unsigned int no_interrupt = req->request.no_interrupt;
1215 unsigned int is_last = req->request.is_last;
1216 struct dwc3 *dwc = dep->dwc;
1217 struct usb_gadget *gadget = dwc->gadget;
1218 enum usb_device_speed speed = gadget->speed;
1219
1220 if (use_bounce_buffer)
1221 dma = dep->dwc->bounce_addr;
1222 else if (req->request.num_sgs > 0)
1223 dma = sg_dma_address(req->start_sg);
1224 else
1225 dma = req->request.dma;
1226
1227 trb = &dep->trb_pool[dep->trb_enqueue];
1228
1229 if (!req->trb) {
1230 dwc3_gadget_move_started_request(req);
1231 req->trb = trb;
1232 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1233 }
1234
1235 req->num_trbs++;
1236
1237 trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1238 trb->bpl = lower_32_bits(dma);
1239 trb->bph = upper_32_bits(dma);
1240
1241 switch (usb_endpoint_type(dep->endpoint.desc)) {
1242 case USB_ENDPOINT_XFER_CONTROL:
1243 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1244 break;
1245
1246 case USB_ENDPOINT_XFER_ISOC:
1247 if (!node) {
1248 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1249
1250 /*
1251 * USB Specification 2.0 Section 5.9.2 states that: "If
1252 * there is only a single transaction in the microframe,
1253 * only a DATA0 data packet PID is used. If there are
1254 * two transactions per microframe, DATA1 is used for
1255 * the first transaction data packet and DATA0 is used
1256 * for the second transaction data packet. If there are
1257 * three transactions per microframe, DATA2 is used for
1258 * the first transaction data packet, DATA1 is used for
1259 * the second, and DATA0 is used for the third."
1260 *
1261 * IOW, we should satisfy the following cases:
1262 *
1263 * 1) length <= maxpacket
1264 * - DATA0
1265 *
1266 * 2) maxpacket < length <= (2 * maxpacket)
1267 * - DATA1, DATA0
1268 *
1269 * 3) (2 * maxpacket) < length <= (3 * maxpacket)
1270 * - DATA2, DATA1, DATA0
1271 */
1272 if (speed == USB_SPEED_HIGH) {
1273 struct usb_ep *ep = &dep->endpoint;
1274 unsigned int mult = 2;
1275 unsigned int maxp = usb_endpoint_maxp(ep->desc);
1276
1277 if (req->request.length <= (2 * maxp))
1278 mult--;
1279
1280 if (req->request.length <= maxp)
1281 mult--;
1282
1283 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1284 }
1285 } else {
1286 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1287 }
1288
1289 if (!no_interrupt && !chain)
1290 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1291 break;
1292
1293 case USB_ENDPOINT_XFER_BULK:
1294 case USB_ENDPOINT_XFER_INT:
1295 trb->ctrl = DWC3_TRBCTL_NORMAL;
1296 break;
1297 default:
1298 /*
1299 * This is only possible with faulty memory because we
1300 * checked it already :)
1301 */
1302 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1303 usb_endpoint_type(dep->endpoint.desc));
1304 }
1305
1306 /*
1307 * Enable Continue on Short Packet
1308 * when endpoint is not a stream capable
1309 */
1310 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1311 if (!dep->stream_capable)
1312 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1313
1314 if (short_not_ok)
1315 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1316 }
1317
1318 /* All TRBs setup for MST must set CSP=1 when LST=0 */
1319 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1320 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1321
1322 if ((!no_interrupt && !chain) || must_interrupt)
1323 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1324
1325 if (chain)
1326 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1327 else if (dep->stream_capable && is_last &&
1328 !DWC3_MST_CAPABLE(&dwc->hwparams))
1329 trb->ctrl |= DWC3_TRB_CTRL_LST;
1330
1331 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1332 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1333
1334 /*
1335 * As per data book 4.2.3.2TRB Control Bit Rules section
1336 *
1337 * The controller autonomously checks the HWO field of a TRB to determine if the
1338 * entire TRB is valid. Therefore, software must ensure that the rest of the TRB
1339 * is valid before setting the HWO field to '1'. In most systems, this means that
1340 * software must update the fourth DWORD of a TRB last.
1341 *
1342 * However there is a possibility of CPU re-ordering here which can cause
1343 * controller to observe the HWO bit set prematurely.
1344 * Add a write memory barrier to prevent CPU re-ordering.
1345 */
1346 wmb();
1347 trb->ctrl |= DWC3_TRB_CTRL_HWO;
1348
1349 dwc3_ep_inc_enq(dep);
1350
1351 trace_dwc3_prepare_trb(dep, trb);
1352 }
1353
dwc3_needs_extra_trb(struct dwc3_ep * dep,struct dwc3_request * req)1354 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1355 {
1356 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1357 unsigned int rem = req->request.length % maxp;
1358
1359 if ((req->request.length && req->request.zero && !rem &&
1360 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1361 (!req->direction && rem))
1362 return true;
1363
1364 return false;
1365 }
1366
1367 /**
1368 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1369 * @dep: The endpoint that the request belongs to
1370 * @req: The request to prepare
1371 * @entry_length: The last SG entry size
1372 * @node: Indicates whether this is not the first entry (for isoc only)
1373 *
1374 * Return the number of TRBs prepared.
1375 */
dwc3_prepare_last_sg(struct dwc3_ep * dep,struct dwc3_request * req,unsigned int entry_length,unsigned int node)1376 static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1377 struct dwc3_request *req, unsigned int entry_length,
1378 unsigned int node)
1379 {
1380 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1381 unsigned int rem = req->request.length % maxp;
1382 unsigned int num_trbs = 1;
1383
1384 if (dwc3_needs_extra_trb(dep, req))
1385 num_trbs++;
1386
1387 if (dwc3_calc_trbs_left(dep) < num_trbs)
1388 return 0;
1389
1390 req->needs_extra_trb = num_trbs > 1;
1391
1392 /* Prepare a normal TRB */
1393 if (req->direction || req->request.length)
1394 dwc3_prepare_one_trb(dep, req, entry_length,
1395 req->needs_extra_trb, node, false, false);
1396
1397 /* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
1398 if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1399 dwc3_prepare_one_trb(dep, req,
1400 req->direction ? 0 : maxp - rem,
1401 false, 1, true, false);
1402
1403 return num_trbs;
1404 }
1405
dwc3_prepare_trbs_sg(struct dwc3_ep * dep,struct dwc3_request * req)1406 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1407 struct dwc3_request *req)
1408 {
1409 struct scatterlist *sg = req->start_sg;
1410 struct scatterlist *s;
1411 int i;
1412 unsigned int length = req->request.length;
1413 unsigned int remaining = req->request.num_mapped_sgs
1414 - req->num_queued_sgs;
1415 unsigned int num_trbs = req->num_trbs;
1416 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1417
1418 /*
1419 * If we resume preparing the request, then get the remaining length of
1420 * the request and resume where we left off.
1421 */
1422 for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1423 length -= sg_dma_len(s);
1424
1425 for_each_sg(sg, s, remaining, i) {
1426 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1427 unsigned int trb_length;
1428 bool must_interrupt = false;
1429 bool last_sg = false;
1430
1431 trb_length = min_t(unsigned int, length, sg_dma_len(s));
1432
1433 length -= trb_length;
1434
1435 /*
1436 * IOMMU driver is coalescing the list of sgs which shares a
1437 * page boundary into one and giving it to USB driver. With
1438 * this the number of sgs mapped is not equal to the number of
1439 * sgs passed. So mark the chain bit to false if it isthe last
1440 * mapped sg.
1441 */
1442 if ((i == remaining - 1) || !length)
1443 last_sg = true;
1444
1445 if (!num_trbs_left)
1446 break;
1447
1448 if (last_sg) {
1449 if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1450 break;
1451 } else {
1452 /*
1453 * Look ahead to check if we have enough TRBs for the
1454 * next SG entry. If not, set interrupt on this TRB to
1455 * resume preparing the next SG entry when more TRBs are
1456 * free.
1457 */
1458 if (num_trbs_left == 1 || (needs_extra_trb &&
1459 num_trbs_left <= 2 &&
1460 sg_dma_len(sg_next(s)) >= length)) {
1461 struct dwc3_request *r;
1462
1463 /* Check if previous requests already set IOC */
1464 list_for_each_entry(r, &dep->started_list, list) {
1465 if (r != req && !r->request.no_interrupt)
1466 break;
1467
1468 if (r == req)
1469 must_interrupt = true;
1470 }
1471 }
1472
1473 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1474 must_interrupt);
1475 }
1476
1477 /*
1478 * There can be a situation where all sgs in sglist are not
1479 * queued because of insufficient trb number. To handle this
1480 * case, update start_sg to next sg to be queued, so that
1481 * we have free trbs we can continue queuing from where we
1482 * previously stopped
1483 */
1484 if (!last_sg)
1485 req->start_sg = sg_next(s);
1486
1487 req->num_queued_sgs++;
1488 req->num_pending_sgs--;
1489
1490 /*
1491 * The number of pending SG entries may not correspond to the
1492 * number of mapped SG entries. If all the data are queued, then
1493 * don't include unused SG entries.
1494 */
1495 if (length == 0) {
1496 req->num_pending_sgs = 0;
1497 break;
1498 }
1499
1500 if (must_interrupt)
1501 break;
1502 }
1503
1504 return req->num_trbs - num_trbs;
1505 }
1506
dwc3_prepare_trbs_linear(struct dwc3_ep * dep,struct dwc3_request * req)1507 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1508 struct dwc3_request *req)
1509 {
1510 return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1511 }
1512
1513 /*
1514 * dwc3_prepare_trbs - setup TRBs from requests
1515 * @dep: endpoint for which requests are being prepared
1516 *
1517 * The function goes through the requests list and sets up TRBs for the
1518 * transfers. The function returns once there are no more TRBs available or
1519 * it runs out of requests.
1520 *
1521 * Returns the number of TRBs prepared or negative errno.
1522 */
dwc3_prepare_trbs(struct dwc3_ep * dep)1523 static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1524 {
1525 struct dwc3_request *req, *n;
1526 int ret = 0;
1527
1528 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1529
1530 /*
1531 * We can get in a situation where there's a request in the started list
1532 * but there weren't enough TRBs to fully kick it in the first time
1533 * around, so it has been waiting for more TRBs to be freed up.
1534 *
1535 * In that case, we should check if we have a request with pending_sgs
1536 * in the started list and prepare TRBs for that request first,
1537 * otherwise we will prepare TRBs completely out of order and that will
1538 * break things.
1539 */
1540 list_for_each_entry(req, &dep->started_list, list) {
1541 if (req->num_pending_sgs > 0) {
1542 ret = dwc3_prepare_trbs_sg(dep, req);
1543 if (!ret || req->num_pending_sgs)
1544 return ret;
1545 }
1546
1547 if (!dwc3_calc_trbs_left(dep))
1548 return ret;
1549
1550 /*
1551 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1552 * burst capability may try to read and use TRBs beyond the
1553 * active transfer instead of stopping.
1554 */
1555 if (dep->stream_capable && req->request.is_last &&
1556 !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1557 return ret;
1558 }
1559
1560 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1561 struct dwc3 *dwc = dep->dwc;
1562
1563 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1564 dep->direction);
1565 if (ret)
1566 return ret;
1567
1568 req->sg = req->request.sg;
1569 req->start_sg = req->sg;
1570 req->num_queued_sgs = 0;
1571 req->num_pending_sgs = req->request.num_mapped_sgs;
1572
1573 if (req->num_pending_sgs > 0) {
1574 ret = dwc3_prepare_trbs_sg(dep, req);
1575 if (req->num_pending_sgs)
1576 return ret;
1577 } else {
1578 ret = dwc3_prepare_trbs_linear(dep, req);
1579 }
1580
1581 if (!ret || !dwc3_calc_trbs_left(dep))
1582 return ret;
1583
1584 /*
1585 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1586 * burst capability may try to read and use TRBs beyond the
1587 * active transfer instead of stopping.
1588 */
1589 if (dep->stream_capable && req->request.is_last &&
1590 !DWC3_MST_CAPABLE(&dwc->hwparams))
1591 return ret;
1592 }
1593
1594 return ret;
1595 }
1596
1597 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1598
__dwc3_gadget_kick_transfer(struct dwc3_ep * dep)1599 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1600 {
1601 struct dwc3_gadget_ep_cmd_params params;
1602 struct dwc3_request *req;
1603 int starting;
1604 int ret;
1605 u32 cmd;
1606
1607 /*
1608 * Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
1609 * This happens when we need to stop and restart a transfer such as in
1610 * the case of reinitiating a stream or retrying an isoc transfer.
1611 */
1612 ret = dwc3_prepare_trbs(dep);
1613 if (ret < 0)
1614 return ret;
1615
1616 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1617
1618 /*
1619 * If there's no new TRB prepared and we don't need to restart a
1620 * transfer, there's no need to update the transfer.
1621 */
1622 if (!ret && !starting)
1623 return ret;
1624
1625 req = next_request(&dep->started_list);
1626 if (!req) {
1627 dep->flags |= DWC3_EP_PENDING_REQUEST;
1628 return 0;
1629 }
1630
1631 memset(¶ms, 0, sizeof(params));
1632
1633 if (starting) {
1634 params.param0 = upper_32_bits(req->trb_dma);
1635 params.param1 = lower_32_bits(req->trb_dma);
1636 cmd = DWC3_DEPCMD_STARTTRANSFER;
1637
1638 if (dep->stream_capable)
1639 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1640
1641 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1642 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1643 } else {
1644 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1645 DWC3_DEPCMD_PARAM(dep->resource_index);
1646 }
1647
1648 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1649 if (ret < 0) {
1650 struct dwc3_request *tmp;
1651
1652 if (ret == -EAGAIN)
1653 return ret;
1654
1655 dwc3_stop_active_transfer(dep, true, true);
1656
1657 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1658 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1659
1660 /* If ep isn't started, then there's no end transfer pending */
1661 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1662 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1663
1664 return ret;
1665 }
1666
1667 if (dep->stream_capable && req->request.is_last &&
1668 !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1669 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1670
1671 return 0;
1672 }
1673
__dwc3_gadget_get_frame(struct dwc3 * dwc)1674 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1675 {
1676 u32 reg;
1677
1678 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1679 return DWC3_DSTS_SOFFN(reg);
1680 }
1681
1682 /**
1683 * __dwc3_stop_active_transfer - stop the current active transfer
1684 * @dep: isoc endpoint
1685 * @force: set forcerm bit in the command
1686 * @interrupt: command complete interrupt after End Transfer command
1687 *
1688 * When setting force, the ForceRM bit will be set. In that case
1689 * the controller won't update the TRB progress on command
1690 * completion. It also won't clear the HWO bit in the TRB.
1691 * The command will also not complete immediately in that case.
1692 */
__dwc3_stop_active_transfer(struct dwc3_ep * dep,bool force,bool interrupt)1693 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1694 {
1695 struct dwc3_gadget_ep_cmd_params params;
1696 u32 cmd;
1697 int ret;
1698
1699 cmd = DWC3_DEPCMD_ENDTRANSFER;
1700 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1701 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
1702 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1703 memset(¶ms, 0, sizeof(params));
1704 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1705 /*
1706 * If the End Transfer command was timed out while the device is
1707 * not in SETUP phase, it's possible that an incoming Setup packet
1708 * may prevent the command's completion. Let's retry when the
1709 * ep0state returns to EP0_SETUP_PHASE.
1710 */
1711 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
1712 dep->flags |= DWC3_EP_DELAY_STOP;
1713 return 0;
1714 }
1715 WARN_ON_ONCE(ret);
1716 dep->resource_index = 0;
1717
1718 if (!interrupt) {
1719 mdelay(1);
1720 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1721 } else if (!ret) {
1722 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1723 }
1724
1725 dep->flags &= ~DWC3_EP_DELAY_STOP;
1726 return ret;
1727 }
1728
1729 /**
1730 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1731 * @dep: isoc endpoint
1732 *
1733 * This function tests for the correct combination of BIT[15:14] from the 16-bit
1734 * microframe number reported by the XferNotReady event for the future frame
1735 * number to start the isoc transfer.
1736 *
1737 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1738 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1739 * XferNotReady event are invalid. The driver uses this number to schedule the
1740 * isochronous transfer and passes it to the START TRANSFER command. Because
1741 * this number is invalid, the command may fail. If BIT[15:14] matches the
1742 * internal 16-bit microframe, the START TRANSFER command will pass and the
1743 * transfer will start at the scheduled time, if it is off by 1, the command
1744 * will still pass, but the transfer will start 2 seconds in the future. For all
1745 * other conditions, the START TRANSFER command will fail with bus-expiry.
1746 *
1747 * In order to workaround this issue, we can test for the correct combination of
1748 * BIT[15:14] by sending START TRANSFER commands with different values of
1749 * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1750 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1751 * As the result, within the 4 possible combinations for BIT[15:14], there will
1752 * be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1753 * command status will result in a 2-second delay start. The smaller BIT[15:14]
1754 * value is the correct combination.
1755 *
1756 * Since there are only 4 outcomes and the results are ordered, we can simply
1757 * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1758 * deduce the smaller successful combination.
1759 *
1760 * Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1761 * of BIT[15:14]. The correct combination is as follow:
1762 *
1763 * if test0 fails and test1 passes, BIT[15:14] is 'b01
1764 * if test0 fails and test1 fails, BIT[15:14] is 'b10
1765 * if test0 passes and test1 fails, BIT[15:14] is 'b11
1766 * if test0 passes and test1 passes, BIT[15:14] is 'b00
1767 *
1768 * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1769 * endpoints.
1770 */
dwc3_gadget_start_isoc_quirk(struct dwc3_ep * dep)1771 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1772 {
1773 int cmd_status = 0;
1774 bool test0;
1775 bool test1;
1776
1777 while (dep->combo_num < 2) {
1778 struct dwc3_gadget_ep_cmd_params params;
1779 u32 test_frame_number;
1780 u32 cmd;
1781
1782 /*
1783 * Check if we can start isoc transfer on the next interval or
1784 * 4 uframes in the future with BIT[15:14] as dep->combo_num
1785 */
1786 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1787 test_frame_number |= dep->combo_num << 14;
1788 test_frame_number += max_t(u32, 4, dep->interval);
1789
1790 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1791 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1792
1793 cmd = DWC3_DEPCMD_STARTTRANSFER;
1794 cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1795 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1796
1797 /* Redo if some other failure beside bus-expiry is received */
1798 if (cmd_status && cmd_status != -EAGAIN) {
1799 dep->start_cmd_status = 0;
1800 dep->combo_num = 0;
1801 return 0;
1802 }
1803
1804 /* Store the first test status */
1805 if (dep->combo_num == 0)
1806 dep->start_cmd_status = cmd_status;
1807
1808 dep->combo_num++;
1809
1810 /*
1811 * End the transfer if the START_TRANSFER command is successful
1812 * to wait for the next XferNotReady to test the command again
1813 */
1814 if (cmd_status == 0) {
1815 dwc3_stop_active_transfer(dep, true, true);
1816 return 0;
1817 }
1818 }
1819
1820 /* test0 and test1 are both completed at this point */
1821 test0 = (dep->start_cmd_status == 0);
1822 test1 = (cmd_status == 0);
1823
1824 if (!test0 && test1)
1825 dep->combo_num = 1;
1826 else if (!test0 && !test1)
1827 dep->combo_num = 2;
1828 else if (test0 && !test1)
1829 dep->combo_num = 3;
1830 else if (test0 && test1)
1831 dep->combo_num = 0;
1832
1833 dep->frame_number &= DWC3_FRNUMBER_MASK;
1834 dep->frame_number |= dep->combo_num << 14;
1835 dep->frame_number += max_t(u32, 4, dep->interval);
1836
1837 /* Reinitialize test variables */
1838 dep->start_cmd_status = 0;
1839 dep->combo_num = 0;
1840
1841 return __dwc3_gadget_kick_transfer(dep);
1842 }
1843
__dwc3_gadget_start_isoc(struct dwc3_ep * dep)1844 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1845 {
1846 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1847 struct dwc3 *dwc = dep->dwc;
1848 int ret;
1849 int i;
1850
1851 if (list_empty(&dep->pending_list) &&
1852 list_empty(&dep->started_list)) {
1853 dep->flags |= DWC3_EP_PENDING_REQUEST;
1854 return -EAGAIN;
1855 }
1856
1857 if (!dwc->dis_start_transfer_quirk &&
1858 (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1859 DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1860 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1861 return dwc3_gadget_start_isoc_quirk(dep);
1862 }
1863
1864 if (desc->bInterval <= 14 &&
1865 dwc->gadget->speed >= USB_SPEED_HIGH) {
1866 u32 frame = __dwc3_gadget_get_frame(dwc);
1867 bool rollover = frame <
1868 (dep->frame_number & DWC3_FRNUMBER_MASK);
1869
1870 /*
1871 * frame_number is set from XferNotReady and may be already
1872 * out of date. DSTS only provides the lower 14 bit of the
1873 * current frame number. So add the upper two bits of
1874 * frame_number and handle a possible rollover.
1875 * This will provide the correct frame_number unless more than
1876 * rollover has happened since XferNotReady.
1877 */
1878
1879 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1880 frame;
1881 if (rollover)
1882 dep->frame_number += BIT(14);
1883 }
1884
1885 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1886 int future_interval = i + 1;
1887
1888 /* Give the controller at least 500us to schedule transfers */
1889 if (desc->bInterval < 3)
1890 future_interval += 3 - desc->bInterval;
1891
1892 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1893
1894 ret = __dwc3_gadget_kick_transfer(dep);
1895 if (ret != -EAGAIN)
1896 break;
1897 }
1898
1899 /*
1900 * After a number of unsuccessful start attempts due to bus-expiry
1901 * status, issue END_TRANSFER command and retry on the next XferNotReady
1902 * event.
1903 */
1904 if (ret == -EAGAIN)
1905 ret = __dwc3_stop_active_transfer(dep, false, true);
1906
1907 return ret;
1908 }
1909
__dwc3_gadget_ep_queue(struct dwc3_ep * dep,struct dwc3_request * req)1910 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1911 {
1912 struct dwc3 *dwc = dep->dwc;
1913
1914 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1915 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1916 dep->name);
1917 return -ESHUTDOWN;
1918 }
1919
1920 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1921 &req->request, req->dep->name))
1922 return -EINVAL;
1923
1924 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1925 "%s: request %pK already in flight\n",
1926 dep->name, &req->request))
1927 return -EINVAL;
1928
1929 pm_runtime_get(dwc->dev);
1930
1931 req->request.actual = 0;
1932 req->request.status = -EINPROGRESS;
1933
1934 trace_dwc3_ep_queue(req);
1935
1936 list_add_tail(&req->list, &dep->pending_list);
1937 req->status = DWC3_REQUEST_STATUS_QUEUED;
1938
1939 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1940 return 0;
1941
1942 /*
1943 * Start the transfer only after the END_TRANSFER is completed
1944 * and endpoint STALL is cleared.
1945 */
1946 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1947 (dep->flags & DWC3_EP_WEDGE) ||
1948 (dep->flags & DWC3_EP_DELAY_STOP) ||
1949 (dep->flags & DWC3_EP_STALL)) {
1950 dep->flags |= DWC3_EP_DELAY_START;
1951 return 0;
1952 }
1953
1954 /*
1955 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1956 * wait for a XferNotReady event so we will know what's the current
1957 * (micro-)frame number.
1958 *
1959 * Without this trick, we are very, very likely gonna get Bus Expiry
1960 * errors which will force us issue EndTransfer command.
1961 */
1962 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1963 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1964 if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1965 return __dwc3_gadget_start_isoc(dep);
1966
1967 return 0;
1968 }
1969 }
1970
1971 __dwc3_gadget_kick_transfer(dep);
1972
1973 return 0;
1974 }
1975
dwc3_gadget_ep_queue(struct usb_ep * ep,struct usb_request * request,gfp_t gfp_flags)1976 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1977 gfp_t gfp_flags)
1978 {
1979 struct dwc3_request *req = to_dwc3_request(request);
1980 struct dwc3_ep *dep = to_dwc3_ep(ep);
1981 struct dwc3 *dwc = dep->dwc;
1982
1983 unsigned long flags;
1984
1985 int ret;
1986
1987 spin_lock_irqsave(&dwc->lock, flags);
1988 ret = __dwc3_gadget_ep_queue(dep, req);
1989 spin_unlock_irqrestore(&dwc->lock, flags);
1990
1991 return ret;
1992 }
1993
dwc3_gadget_ep_skip_trbs(struct dwc3_ep * dep,struct dwc3_request * req)1994 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1995 {
1996 int i;
1997
1998 /* If req->trb is not set, then the request has not started */
1999 if (!req->trb)
2000 return;
2001
2002 /*
2003 * If request was already started, this means we had to
2004 * stop the transfer. With that we also need to ignore
2005 * all TRBs used by the request, however TRBs can only
2006 * be modified after completion of END_TRANSFER
2007 * command. So what we do here is that we wait for
2008 * END_TRANSFER completion and only after that, we jump
2009 * over TRBs by clearing HWO and incrementing dequeue
2010 * pointer.
2011 */
2012 for (i = 0; i < req->num_trbs; i++) {
2013 struct dwc3_trb *trb;
2014
2015 trb = &dep->trb_pool[dep->trb_dequeue];
2016 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2017 dwc3_ep_inc_deq(dep);
2018 }
2019
2020 req->num_trbs = 0;
2021 }
2022
dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep * dep)2023 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
2024 {
2025 struct dwc3_request *req;
2026 struct dwc3 *dwc = dep->dwc;
2027
2028 while (!list_empty(&dep->cancelled_list)) {
2029 req = next_request(&dep->cancelled_list);
2030 dwc3_gadget_ep_skip_trbs(dep, req);
2031 switch (req->status) {
2032 case DWC3_REQUEST_STATUS_DISCONNECTED:
2033 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
2034 break;
2035 case DWC3_REQUEST_STATUS_DEQUEUED:
2036 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2037 break;
2038 case DWC3_REQUEST_STATUS_STALLED:
2039 dwc3_gadget_giveback(dep, req, -EPIPE);
2040 break;
2041 default:
2042 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
2043 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2044 break;
2045 }
2046 /*
2047 * The endpoint is disabled, let the dwc3_remove_requests()
2048 * handle the cleanup.
2049 */
2050 if (!dep->endpoint.desc)
2051 break;
2052 }
2053 }
2054
dwc3_gadget_ep_dequeue(struct usb_ep * ep,struct usb_request * request)2055 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2056 struct usb_request *request)
2057 {
2058 struct dwc3_request *req = to_dwc3_request(request);
2059 struct dwc3_request *r = NULL;
2060
2061 struct dwc3_ep *dep = to_dwc3_ep(ep);
2062 struct dwc3 *dwc = dep->dwc;
2063
2064 unsigned long flags;
2065 int ret = 0;
2066
2067 trace_dwc3_ep_dequeue(req);
2068
2069 spin_lock_irqsave(&dwc->lock, flags);
2070
2071 list_for_each_entry(r, &dep->cancelled_list, list) {
2072 if (r == req)
2073 goto out;
2074 }
2075
2076 list_for_each_entry(r, &dep->pending_list, list) {
2077 if (r == req) {
2078 /*
2079 * Explicitly check for EP0/1 as dequeue for those
2080 * EPs need to be handled differently. Control EP
2081 * only deals with one USB req, and giveback will
2082 * occur during dwc3_ep0_stall_and_restart(). EP0
2083 * requests are never added to started_list.
2084 */
2085 if (dep->number > 1)
2086 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2087 else
2088 dwc3_ep0_reset_state(dwc);
2089 goto out;
2090 }
2091 }
2092
2093 list_for_each_entry(r, &dep->started_list, list) {
2094 if (r == req) {
2095 struct dwc3_request *t;
2096
2097 /* wait until it is processed */
2098 dwc3_stop_active_transfer(dep, true, true);
2099
2100 /*
2101 * Remove any started request if the transfer is
2102 * cancelled.
2103 */
2104 list_for_each_entry_safe(r, t, &dep->started_list, list)
2105 dwc3_gadget_move_cancelled_request(r,
2106 DWC3_REQUEST_STATUS_DEQUEUED);
2107
2108 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2109
2110 goto out;
2111 }
2112 }
2113
2114 dev_err(dwc->dev, "request %pK was not queued to %s\n",
2115 request, ep->name);
2116 ret = -EINVAL;
2117 out:
2118 spin_unlock_irqrestore(&dwc->lock, flags);
2119
2120 return ret;
2121 }
2122
__dwc3_gadget_ep_set_halt(struct dwc3_ep * dep,int value,int protocol)2123 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2124 {
2125 struct dwc3_gadget_ep_cmd_params params;
2126 struct dwc3 *dwc = dep->dwc;
2127 struct dwc3_request *req;
2128 struct dwc3_request *tmp;
2129 int ret;
2130
2131 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2132 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2133 return -EINVAL;
2134 }
2135
2136 memset(¶ms, 0x00, sizeof(params));
2137
2138 if (value) {
2139 struct dwc3_trb *trb;
2140
2141 unsigned int transfer_in_flight;
2142 unsigned int started;
2143
2144 if (dep->number > 1)
2145 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
2146 else
2147 trb = &dwc->ep0_trb[dep->trb_enqueue];
2148
2149 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2150 started = !list_empty(&dep->started_list);
2151
2152 if (!protocol && ((dep->direction && transfer_in_flight) ||
2153 (!dep->direction && started))) {
2154 return -EAGAIN;
2155 }
2156
2157 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2158 ¶ms);
2159 if (ret)
2160 dev_err(dwc->dev, "failed to set STALL on %s\n",
2161 dep->name);
2162 else
2163 dep->flags |= DWC3_EP_STALL;
2164 } else {
2165 /*
2166 * Don't issue CLEAR_STALL command to control endpoints. The
2167 * controller automatically clears the STALL when it receives
2168 * the SETUP token.
2169 */
2170 if (dep->number <= 1) {
2171 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2172 return 0;
2173 }
2174
2175 dwc3_stop_active_transfer(dep, true, true);
2176
2177 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2178 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2179
2180 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2181 (dep->flags & DWC3_EP_DELAY_STOP)) {
2182 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2183 if (protocol)
2184 dwc->clear_stall_protocol = dep->number;
2185
2186 return 0;
2187 }
2188
2189 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2190
2191 ret = dwc3_send_clear_stall_ep_cmd(dep);
2192 if (ret) {
2193 dev_err(dwc->dev, "failed to clear STALL on %s\n",
2194 dep->name);
2195 return ret;
2196 }
2197
2198 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2199
2200 if ((dep->flags & DWC3_EP_DELAY_START) &&
2201 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2202 __dwc3_gadget_kick_transfer(dep);
2203
2204 dep->flags &= ~DWC3_EP_DELAY_START;
2205 }
2206
2207 return ret;
2208 }
2209
dwc3_gadget_ep_set_halt(struct usb_ep * ep,int value)2210 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2211 {
2212 struct dwc3_ep *dep = to_dwc3_ep(ep);
2213 struct dwc3 *dwc = dep->dwc;
2214
2215 unsigned long flags;
2216
2217 int ret;
2218
2219 spin_lock_irqsave(&dwc->lock, flags);
2220 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
2221 spin_unlock_irqrestore(&dwc->lock, flags);
2222
2223 return ret;
2224 }
2225
dwc3_gadget_ep_set_wedge(struct usb_ep * ep)2226 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2227 {
2228 struct dwc3_ep *dep = to_dwc3_ep(ep);
2229 struct dwc3 *dwc = dep->dwc;
2230 unsigned long flags;
2231 int ret;
2232
2233 spin_lock_irqsave(&dwc->lock, flags);
2234 dep->flags |= DWC3_EP_WEDGE;
2235
2236 if (dep->number == 0 || dep->number == 1)
2237 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
2238 else
2239 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
2240 spin_unlock_irqrestore(&dwc->lock, flags);
2241
2242 return ret;
2243 }
2244
2245 /* -------------------------------------------------------------------------- */
2246
2247 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2248 .bLength = USB_DT_ENDPOINT_SIZE,
2249 .bDescriptorType = USB_DT_ENDPOINT,
2250 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
2251 };
2252
2253 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2254 .enable = dwc3_gadget_ep0_enable,
2255 .disable = dwc3_gadget_ep0_disable,
2256 .alloc_request = dwc3_gadget_ep_alloc_request,
2257 .free_request = dwc3_gadget_ep_free_request,
2258 .queue = dwc3_gadget_ep0_queue,
2259 .dequeue = dwc3_gadget_ep_dequeue,
2260 .set_halt = dwc3_gadget_ep0_set_halt,
2261 .set_wedge = dwc3_gadget_ep_set_wedge,
2262 };
2263
2264 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2265 .enable = dwc3_gadget_ep_enable,
2266 .disable = dwc3_gadget_ep_disable,
2267 .alloc_request = dwc3_gadget_ep_alloc_request,
2268 .free_request = dwc3_gadget_ep_free_request,
2269 .queue = dwc3_gadget_ep_queue,
2270 .dequeue = dwc3_gadget_ep_dequeue,
2271 .set_halt = dwc3_gadget_ep_set_halt,
2272 .set_wedge = dwc3_gadget_ep_set_wedge,
2273 };
2274
2275 /* -------------------------------------------------------------------------- */
2276
dwc3_gadget_enable_linksts_evts(struct dwc3 * dwc,bool set)2277 static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set)
2278 {
2279 u32 reg;
2280
2281 if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2282 return;
2283
2284 reg = dwc3_readl(dwc->regs, DWC3_DEVTEN);
2285 if (set)
2286 reg |= DWC3_DEVTEN_ULSTCNGEN;
2287 else
2288 reg &= ~DWC3_DEVTEN_ULSTCNGEN;
2289
2290 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2291 }
2292
dwc3_gadget_get_frame(struct usb_gadget * g)2293 static int dwc3_gadget_get_frame(struct usb_gadget *g)
2294 {
2295 struct dwc3 *dwc = gadget_to_dwc(g);
2296
2297 return __dwc3_gadget_get_frame(dwc);
2298 }
2299
__dwc3_gadget_wakeup(struct dwc3 * dwc,bool async)2300 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async)
2301 {
2302 int retries;
2303
2304 int ret;
2305 u32 reg;
2306
2307 u8 link_state;
2308
2309 /*
2310 * According to the Databook Remote wakeup request should
2311 * be issued only when the device is in early suspend state.
2312 *
2313 * We can check that via USB Link State bits in DSTS register.
2314 */
2315 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2316
2317 link_state = DWC3_DSTS_USBLNKST(reg);
2318
2319 switch (link_state) {
2320 case DWC3_LINK_STATE_RESET:
2321 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
2322 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
2323 case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */
2324 case DWC3_LINK_STATE_U1:
2325 case DWC3_LINK_STATE_RESUME:
2326 break;
2327 default:
2328 return -EINVAL;
2329 }
2330
2331 if (async)
2332 dwc3_gadget_enable_linksts_evts(dwc, true);
2333
2334 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2335 if (ret < 0) {
2336 dev_err(dwc->dev, "failed to put link in Recovery\n");
2337 dwc3_gadget_enable_linksts_evts(dwc, false);
2338 return ret;
2339 }
2340
2341 /* Recent versions do this automatically */
2342 if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2343 /* write zeroes to Link Change Request */
2344 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2345 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2346 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2347 }
2348
2349 /*
2350 * Since link status change events are enabled we will receive
2351 * an U0 event when wakeup is successful. So bail out.
2352 */
2353 if (async)
2354 return 0;
2355
2356 /* poll until Link State changes to ON */
2357 retries = 20000;
2358
2359 while (retries--) {
2360 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2361
2362 /* in HS, means ON */
2363 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2364 break;
2365 }
2366
2367 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2368 dev_err(dwc->dev, "failed to send remote wakeup\n");
2369 return -EINVAL;
2370 }
2371
2372 return 0;
2373 }
2374
dwc3_gadget_wakeup(struct usb_gadget * g)2375 static int dwc3_gadget_wakeup(struct usb_gadget *g)
2376 {
2377 struct dwc3 *dwc = gadget_to_dwc(g);
2378 unsigned long flags;
2379 int ret;
2380
2381 if (!dwc->wakeup_configured) {
2382 dev_err(dwc->dev, "remote wakeup not configured\n");
2383 return -EINVAL;
2384 }
2385
2386 spin_lock_irqsave(&dwc->lock, flags);
2387 if (!dwc->gadget->wakeup_armed) {
2388 dev_err(dwc->dev, "not armed for remote wakeup\n");
2389 spin_unlock_irqrestore(&dwc->lock, flags);
2390 return -EINVAL;
2391 }
2392 ret = __dwc3_gadget_wakeup(dwc, true);
2393
2394 spin_unlock_irqrestore(&dwc->lock, flags);
2395
2396 return ret;
2397 }
2398
2399 static void dwc3_resume_gadget(struct dwc3 *dwc);
2400
dwc3_gadget_func_wakeup(struct usb_gadget * g,int intf_id)2401 static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id)
2402 {
2403 struct dwc3 *dwc = gadget_to_dwc(g);
2404 unsigned long flags;
2405 int ret;
2406 int link_state;
2407
2408 if (!dwc->wakeup_configured) {
2409 dev_err(dwc->dev, "remote wakeup not configured\n");
2410 return -EINVAL;
2411 }
2412
2413 spin_lock_irqsave(&dwc->lock, flags);
2414 /*
2415 * If the link is in U3, signal for remote wakeup and wait for the
2416 * link to transition to U0 before sending device notification.
2417 */
2418 link_state = dwc3_gadget_get_link_state(dwc);
2419 if (link_state == DWC3_LINK_STATE_U3) {
2420 ret = __dwc3_gadget_wakeup(dwc, false);
2421 if (ret) {
2422 spin_unlock_irqrestore(&dwc->lock, flags);
2423 return -EINVAL;
2424 }
2425 dwc3_resume_gadget(dwc);
2426 dwc->suspended = false;
2427 dwc->link_state = DWC3_LINK_STATE_U0;
2428 }
2429
2430 ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION,
2431 DWC3_DGCMDPAR_DN_FUNC_WAKE |
2432 DWC3_DGCMDPAR_INTF_SEL(intf_id));
2433 if (ret)
2434 dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret);
2435
2436 spin_unlock_irqrestore(&dwc->lock, flags);
2437
2438 return ret;
2439 }
2440
dwc3_gadget_set_remote_wakeup(struct usb_gadget * g,int set)2441 static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set)
2442 {
2443 struct dwc3 *dwc = gadget_to_dwc(g);
2444 unsigned long flags;
2445
2446 spin_lock_irqsave(&dwc->lock, flags);
2447 dwc->wakeup_configured = !!set;
2448 spin_unlock_irqrestore(&dwc->lock, flags);
2449
2450 return 0;
2451 }
2452
dwc3_gadget_set_selfpowered(struct usb_gadget * g,int is_selfpowered)2453 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2454 int is_selfpowered)
2455 {
2456 struct dwc3 *dwc = gadget_to_dwc(g);
2457 unsigned long flags;
2458
2459 spin_lock_irqsave(&dwc->lock, flags);
2460 g->is_selfpowered = !!is_selfpowered;
2461 spin_unlock_irqrestore(&dwc->lock, flags);
2462
2463 return 0;
2464 }
2465
dwc3_stop_active_transfers(struct dwc3 * dwc)2466 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2467 {
2468 u32 epnum;
2469
2470 for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2471 struct dwc3_ep *dep;
2472
2473 dep = dwc->eps[epnum];
2474 if (!dep)
2475 continue;
2476
2477 dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
2478 }
2479 }
2480
__dwc3_gadget_set_ssp_rate(struct dwc3 * dwc)2481 static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2482 {
2483 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate;
2484 u32 reg;
2485
2486 if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2487 ssp_rate = dwc->max_ssp_rate;
2488
2489 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2490 reg &= ~DWC3_DCFG_SPEED_MASK;
2491 reg &= ~DWC3_DCFG_NUMLANES(~0);
2492
2493 if (ssp_rate == USB_SSP_GEN_1x2)
2494 reg |= DWC3_DCFG_SUPERSPEED;
2495 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2496 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2497
2498 if (ssp_rate != USB_SSP_GEN_2x1 &&
2499 dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2500 reg |= DWC3_DCFG_NUMLANES(1);
2501
2502 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2503 }
2504
__dwc3_gadget_set_speed(struct dwc3 * dwc)2505 static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2506 {
2507 enum usb_device_speed speed;
2508 u32 reg;
2509
2510 speed = dwc->gadget_max_speed;
2511 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2512 speed = dwc->maximum_speed;
2513
2514 if (speed == USB_SPEED_SUPER_PLUS &&
2515 DWC3_IP_IS(DWC32)) {
2516 __dwc3_gadget_set_ssp_rate(dwc);
2517 return;
2518 }
2519
2520 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2521 reg &= ~(DWC3_DCFG_SPEED_MASK);
2522
2523 /*
2524 * WORKAROUND: DWC3 revision < 2.20a have an issue
2525 * which would cause metastability state on Run/Stop
2526 * bit if we try to force the IP to USB2-only mode.
2527 *
2528 * Because of that, we cannot configure the IP to any
2529 * speed other than the SuperSpeed
2530 *
2531 * Refers to:
2532 *
2533 * STAR#9000525659: Clock Domain Crossing on DCTL in
2534 * USB 2.0 Mode
2535 */
2536 if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2537 !dwc->dis_metastability_quirk) {
2538 reg |= DWC3_DCFG_SUPERSPEED;
2539 } else {
2540 switch (speed) {
2541 case USB_SPEED_FULL:
2542 reg |= DWC3_DCFG_FULLSPEED;
2543 break;
2544 case USB_SPEED_HIGH:
2545 reg |= DWC3_DCFG_HIGHSPEED;
2546 break;
2547 case USB_SPEED_SUPER:
2548 reg |= DWC3_DCFG_SUPERSPEED;
2549 break;
2550 case USB_SPEED_SUPER_PLUS:
2551 if (DWC3_IP_IS(DWC3))
2552 reg |= DWC3_DCFG_SUPERSPEED;
2553 else
2554 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2555 break;
2556 default:
2557 dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2558
2559 if (DWC3_IP_IS(DWC3))
2560 reg |= DWC3_DCFG_SUPERSPEED;
2561 else
2562 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2563 }
2564 }
2565
2566 if (DWC3_IP_IS(DWC32) &&
2567 speed > USB_SPEED_UNKNOWN &&
2568 speed < USB_SPEED_SUPER_PLUS)
2569 reg &= ~DWC3_DCFG_NUMLANES(~0);
2570
2571 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2572 }
2573
dwc3_gadget_run_stop(struct dwc3 * dwc,int is_on)2574 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
2575 {
2576 u32 reg;
2577 u32 timeout = 2000;
2578
2579 if (pm_runtime_suspended(dwc->dev))
2580 return 0;
2581
2582 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2583 if (is_on) {
2584 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2585 reg &= ~DWC3_DCTL_TRGTULST_MASK;
2586 reg |= DWC3_DCTL_TRGTULST_RX_DET;
2587 }
2588
2589 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2590 reg &= ~DWC3_DCTL_KEEP_CONNECT;
2591 reg |= DWC3_DCTL_RUN_STOP;
2592
2593 __dwc3_gadget_set_speed(dwc);
2594 dwc->pullups_connected = true;
2595 } else {
2596 reg &= ~DWC3_DCTL_RUN_STOP;
2597
2598 dwc->pullups_connected = false;
2599 }
2600
2601 dwc3_gadget_dctl_write_safe(dwc, reg);
2602
2603 do {
2604 usleep_range(1000, 2000);
2605 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2606 reg &= DWC3_DSTS_DEVCTRLHLT;
2607 } while (--timeout && !(!is_on ^ !reg));
2608
2609 if (!timeout)
2610 return -ETIMEDOUT;
2611
2612 return 0;
2613 }
2614
2615 static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2616 static void __dwc3_gadget_stop(struct dwc3 *dwc);
2617 static int __dwc3_gadget_start(struct dwc3 *dwc);
2618
dwc3_gadget_soft_disconnect(struct dwc3 * dwc)2619 static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
2620 {
2621 unsigned long flags;
2622 int ret;
2623
2624 spin_lock_irqsave(&dwc->lock, flags);
2625 if (!dwc->pullups_connected) {
2626 spin_unlock_irqrestore(&dwc->lock, flags);
2627 return 0;
2628 }
2629
2630 dwc->connected = false;
2631
2632 /*
2633 * Attempt to end pending SETUP status phase, and not wait for the
2634 * function to do so.
2635 */
2636 if (dwc->delayed_status)
2637 dwc3_ep0_send_delayed_status(dwc);
2638
2639 /*
2640 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
2641 * Section 4.1.8 Table 4-7, it states that for a device-initiated
2642 * disconnect, the SW needs to ensure that it sends "a DEPENDXFER
2643 * command for any active transfers" before clearing the RunStop
2644 * bit.
2645 */
2646 dwc3_stop_active_transfers(dwc);
2647 spin_unlock_irqrestore(&dwc->lock, flags);
2648
2649 /*
2650 * Per databook, when we want to stop the gadget, if a control transfer
2651 * is still in process, complete it and get the core into setup phase.
2652 * In case the host is unresponsive to a SETUP transaction, forcefully
2653 * stall the transfer, and move back to the SETUP phase, so that any
2654 * pending endxfers can be executed.
2655 */
2656 if (dwc->ep0state != EP0_SETUP_PHASE) {
2657 reinit_completion(&dwc->ep0_in_setup);
2658
2659 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2660 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2661 if (ret == 0) {
2662 dev_warn(dwc->dev, "wait for SETUP phase timed out\n");
2663 spin_lock_irqsave(&dwc->lock, flags);
2664 dwc3_ep0_reset_state(dwc);
2665 spin_unlock_irqrestore(&dwc->lock, flags);
2666 }
2667 }
2668
2669 /*
2670 * Note: if the GEVNTCOUNT indicates events in the event buffer, the
2671 * driver needs to acknowledge them before the controller can halt.
2672 * Simply let the interrupt handler acknowledges and handle the
2673 * remaining event generated by the controller while polling for
2674 * DSTS.DEVCTLHLT.
2675 */
2676 ret = dwc3_gadget_run_stop(dwc, false);
2677
2678 /*
2679 * Stop the gadget after controller is halted, so that if needed, the
2680 * events to update EP0 state can still occur while the run/stop
2681 * routine polls for the halted state. DEVTEN is cleared as part of
2682 * gadget stop.
2683 */
2684 spin_lock_irqsave(&dwc->lock, flags);
2685 __dwc3_gadget_stop(dwc);
2686 spin_unlock_irqrestore(&dwc->lock, flags);
2687
2688 return ret;
2689 }
2690
dwc3_gadget_soft_connect(struct dwc3 * dwc)2691 static int dwc3_gadget_soft_connect(struct dwc3 *dwc)
2692 {
2693 int ret;
2694
2695 /*
2696 * In the Synopsys DWC_usb31 1.90a programming guide section
2697 * 4.1.9, it specifies that for a reconnect after a
2698 * device-initiated disconnect requires a core soft reset
2699 * (DCTL.CSftRst) before enabling the run/stop bit.
2700 */
2701 ret = dwc3_core_soft_reset(dwc);
2702 if (ret)
2703 return ret;
2704
2705 dwc3_event_buffers_setup(dwc);
2706 __dwc3_gadget_start(dwc);
2707 return dwc3_gadget_run_stop(dwc, true);
2708 }
2709
dwc3_gadget_pullup(struct usb_gadget * g,int is_on)2710 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2711 {
2712 struct dwc3 *dwc = gadget_to_dwc(g);
2713 int ret;
2714
2715 is_on = !!is_on;
2716
2717 dwc->softconnect = is_on;
2718
2719 /*
2720 * Avoid issuing a runtime resume if the device is already in the
2721 * suspended state during gadget disconnect. DWC3 gadget was already
2722 * halted/stopped during runtime suspend.
2723 */
2724 if (!is_on) {
2725 pm_runtime_barrier(dwc->dev);
2726 if (pm_runtime_suspended(dwc->dev))
2727 return 0;
2728 }
2729
2730 /*
2731 * Check the return value for successful resume, or error. For a
2732 * successful resume, the DWC3 runtime PM resume routine will handle
2733 * the run stop sequence, so avoid duplicate operations here.
2734 */
2735 ret = pm_runtime_get_sync(dwc->dev);
2736 if (!ret || ret < 0) {
2737 pm_runtime_put(dwc->dev);
2738 if (ret < 0)
2739 pm_runtime_set_suspended(dwc->dev);
2740 return ret;
2741 }
2742
2743 if (dwc->pullups_connected == is_on) {
2744 pm_runtime_put(dwc->dev);
2745 return 0;
2746 }
2747
2748 synchronize_irq(dwc->irq_gadget);
2749
2750 if (!is_on)
2751 ret = dwc3_gadget_soft_disconnect(dwc);
2752 else
2753 ret = dwc3_gadget_soft_connect(dwc);
2754
2755 pm_runtime_put(dwc->dev);
2756
2757 return ret;
2758 }
2759
dwc3_gadget_enable_irq(struct dwc3 * dwc)2760 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2761 {
2762 u32 reg;
2763
2764 /* Enable all but Start and End of Frame IRQs */
2765 reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2766 DWC3_DEVTEN_CMDCMPLTEN |
2767 DWC3_DEVTEN_ERRTICERREN |
2768 DWC3_DEVTEN_WKUPEVTEN |
2769 DWC3_DEVTEN_CONNECTDONEEN |
2770 DWC3_DEVTEN_USBRSTEN |
2771 DWC3_DEVTEN_DISCONNEVTEN);
2772
2773 if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2774 reg |= DWC3_DEVTEN_ULSTCNGEN;
2775
2776 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2777 if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2778 reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2779
2780 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2781 }
2782
dwc3_gadget_disable_irq(struct dwc3 * dwc)2783 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2784 {
2785 /* mask all interrupts */
2786 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2787 }
2788
2789 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2790 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2791
2792 /**
2793 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2794 * @dwc: pointer to our context structure
2795 *
2796 * The following looks like complex but it's actually very simple. In order to
2797 * calculate the number of packets we can burst at once on OUT transfers, we're
2798 * gonna use RxFIFO size.
2799 *
2800 * To calculate RxFIFO size we need two numbers:
2801 * MDWIDTH = size, in bits, of the internal memory bus
2802 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2803 *
2804 * Given these two numbers, the formula is simple:
2805 *
2806 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2807 *
2808 * 24 bytes is for 3x SETUP packets
2809 * 16 bytes is a clock domain crossing tolerance
2810 *
2811 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2812 */
dwc3_gadget_setup_nump(struct dwc3 * dwc)2813 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2814 {
2815 u32 ram2_depth;
2816 u32 mdwidth;
2817 u32 nump;
2818 u32 reg;
2819
2820 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2821 mdwidth = dwc3_mdwidth(dwc);
2822
2823 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2824 nump = min_t(u32, nump, 16);
2825
2826 /* update NumP */
2827 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2828 reg &= ~DWC3_DCFG_NUMP_MASK;
2829 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2830 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2831 }
2832
__dwc3_gadget_start(struct dwc3 * dwc)2833 static int __dwc3_gadget_start(struct dwc3 *dwc)
2834 {
2835 struct dwc3_ep *dep;
2836 int ret = 0;
2837 u32 reg;
2838
2839 /*
2840 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2841 * the core supports IMOD, disable it.
2842 */
2843 if (dwc->imod_interval) {
2844 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
2845 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2846 } else if (dwc3_has_imod(dwc)) {
2847 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2848 }
2849
2850 /*
2851 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2852 * field instead of letting dwc3 itself calculate that automatically.
2853 *
2854 * This way, we maximize the chances that we'll be able to get several
2855 * bursts of data without going through any sort of endpoint throttling.
2856 */
2857 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2858 if (DWC3_IP_IS(DWC3))
2859 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2860 else
2861 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2862
2863 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2864
2865 dwc3_gadget_setup_nump(dwc);
2866
2867 /*
2868 * Currently the controller handles single stream only. So, Ignore
2869 * Packet Pending bit for stream selection and don't search for another
2870 * stream if the host sends Data Packet with PP=0 (for OUT direction) or
2871 * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
2872 * the stream performance.
2873 */
2874 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2875 reg |= DWC3_DCFG_IGNSTRMPP;
2876 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2877
2878 /* Enable MST by default if the device is capable of MST */
2879 if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2880 reg = dwc3_readl(dwc->regs, DWC3_DCFG1);
2881 reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2882 dwc3_writel(dwc->regs, DWC3_DCFG1, reg);
2883 }
2884
2885 /* Start with SuperSpeed Default */
2886 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2887
2888 ret = dwc3_gadget_start_config(dwc, 0);
2889 if (ret) {
2890 dev_err(dwc->dev, "failed to config endpoints\n");
2891 return ret;
2892 }
2893
2894 dep = dwc->eps[0];
2895 dep->flags = 0;
2896 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2897 if (ret) {
2898 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2899 goto err0;
2900 }
2901
2902 dep = dwc->eps[1];
2903 dep->flags = 0;
2904 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2905 if (ret) {
2906 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2907 goto err1;
2908 }
2909
2910 /* begin to receive SETUP packets */
2911 dwc->ep0state = EP0_SETUP_PHASE;
2912 dwc->ep0_bounced = false;
2913 dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2914 dwc->delayed_status = false;
2915 dwc3_ep0_out_start(dwc);
2916
2917 dwc3_gadget_enable_irq(dwc);
2918 dwc3_enable_susphy(dwc, true);
2919
2920 return 0;
2921
2922 err1:
2923 __dwc3_gadget_ep_disable(dwc->eps[0]);
2924
2925 err0:
2926 return ret;
2927 }
2928
dwc3_gadget_start(struct usb_gadget * g,struct usb_gadget_driver * driver)2929 static int dwc3_gadget_start(struct usb_gadget *g,
2930 struct usb_gadget_driver *driver)
2931 {
2932 struct dwc3 *dwc = gadget_to_dwc(g);
2933 unsigned long flags;
2934 int ret;
2935 int irq;
2936
2937 irq = dwc->irq_gadget;
2938 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2939 IRQF_SHARED, "dwc3", dwc->ev_buf);
2940 if (ret) {
2941 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2942 irq, ret);
2943 return ret;
2944 }
2945
2946 spin_lock_irqsave(&dwc->lock, flags);
2947 dwc->gadget_driver = driver;
2948 spin_unlock_irqrestore(&dwc->lock, flags);
2949
2950 if (dwc->sys_wakeup)
2951 device_wakeup_enable(dwc->sysdev);
2952
2953 return 0;
2954 }
2955
__dwc3_gadget_stop(struct dwc3 * dwc)2956 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2957 {
2958 dwc3_gadget_disable_irq(dwc);
2959 __dwc3_gadget_ep_disable(dwc->eps[0]);
2960 __dwc3_gadget_ep_disable(dwc->eps[1]);
2961 }
2962
dwc3_gadget_stop(struct usb_gadget * g)2963 static int dwc3_gadget_stop(struct usb_gadget *g)
2964 {
2965 struct dwc3 *dwc = gadget_to_dwc(g);
2966 unsigned long flags;
2967
2968 if (dwc->sys_wakeup)
2969 device_wakeup_disable(dwc->sysdev);
2970
2971 spin_lock_irqsave(&dwc->lock, flags);
2972 dwc->gadget_driver = NULL;
2973 dwc->max_cfg_eps = 0;
2974 spin_unlock_irqrestore(&dwc->lock, flags);
2975
2976 free_irq(dwc->irq_gadget, dwc->ev_buf);
2977
2978 return 0;
2979 }
2980
dwc3_gadget_config_params(struct usb_gadget * g,struct usb_dcd_config_params * params)2981 static void dwc3_gadget_config_params(struct usb_gadget *g,
2982 struct usb_dcd_config_params *params)
2983 {
2984 struct dwc3 *dwc = gadget_to_dwc(g);
2985
2986 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2987 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2988
2989 /* Recommended BESL */
2990 if (!dwc->dis_enblslpm_quirk) {
2991 /*
2992 * If the recommended BESL baseline is 0 or if the BESL deep is
2993 * less than 2, Microsoft's Windows 10 host usb stack will issue
2994 * a usb reset immediately after it receives the extended BOS
2995 * descriptor and the enumeration will fail. To maintain
2996 * compatibility with the Windows' usb stack, let's set the
2997 * recommended BESL baseline to 1 and clamp the BESL deep to be
2998 * within 2 to 15.
2999 */
3000 params->besl_baseline = 1;
3001 if (dwc->is_utmi_l1_suspend)
3002 params->besl_deep =
3003 clamp_t(u8, dwc->hird_threshold, 2, 15);
3004 }
3005
3006 /* U1 Device exit Latency */
3007 if (dwc->dis_u1_entry_quirk)
3008 params->bU1devExitLat = 0;
3009 else
3010 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
3011
3012 /* U2 Device exit Latency */
3013 if (dwc->dis_u2_entry_quirk)
3014 params->bU2DevExitLat = 0;
3015 else
3016 params->bU2DevExitLat =
3017 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
3018 }
3019
dwc3_gadget_set_speed(struct usb_gadget * g,enum usb_device_speed speed)3020 static void dwc3_gadget_set_speed(struct usb_gadget *g,
3021 enum usb_device_speed speed)
3022 {
3023 struct dwc3 *dwc = gadget_to_dwc(g);
3024 unsigned long flags;
3025
3026 spin_lock_irqsave(&dwc->lock, flags);
3027 dwc->gadget_max_speed = speed;
3028 spin_unlock_irqrestore(&dwc->lock, flags);
3029 }
3030
dwc3_gadget_set_ssp_rate(struct usb_gadget * g,enum usb_ssp_rate rate)3031 static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
3032 enum usb_ssp_rate rate)
3033 {
3034 struct dwc3 *dwc = gadget_to_dwc(g);
3035 unsigned long flags;
3036
3037 spin_lock_irqsave(&dwc->lock, flags);
3038 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
3039 dwc->gadget_ssp_rate = rate;
3040 spin_unlock_irqrestore(&dwc->lock, flags);
3041 }
3042
dwc3_gadget_vbus_draw(struct usb_gadget * g,unsigned int mA)3043 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
3044 {
3045 struct dwc3 *dwc = gadget_to_dwc(g);
3046 union power_supply_propval val = {0};
3047 int ret;
3048
3049 if (dwc->usb2_phy)
3050 return usb_phy_set_power(dwc->usb2_phy, mA);
3051
3052 if (!dwc->usb_psy)
3053 return -EOPNOTSUPP;
3054
3055 val.intval = 1000 * mA;
3056 ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
3057
3058 return ret;
3059 }
3060
3061 /**
3062 * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3063 * @g: pointer to the USB gadget
3064 *
3065 * Used to record the maximum number of endpoints being used in a USB composite
3066 * device. (across all configurations) This is to be used in the calculation
3067 * of the TXFIFO sizes when resizing internal memory for individual endpoints.
3068 * It will help ensured that the resizing logic reserves enough space for at
3069 * least one max packet.
3070 */
dwc3_gadget_check_config(struct usb_gadget * g)3071 static int dwc3_gadget_check_config(struct usb_gadget *g)
3072 {
3073 struct dwc3 *dwc = gadget_to_dwc(g);
3074 struct usb_ep *ep;
3075 int fifo_size = 0;
3076 int ram1_depth;
3077 int ep_num = 0;
3078
3079 if (!dwc->do_fifo_resize)
3080 return 0;
3081
3082 list_for_each_entry(ep, &g->ep_list, ep_list) {
3083 /* Only interested in the IN endpoints */
3084 if (ep->claimed && (ep->address & USB_DIR_IN))
3085 ep_num++;
3086 }
3087
3088 if (ep_num <= dwc->max_cfg_eps)
3089 return 0;
3090
3091 /* Update the max number of eps in the composition */
3092 dwc->max_cfg_eps = ep_num;
3093
3094 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);
3095 /* Based on the equation, increment by one for every ep */
3096 fifo_size += dwc->max_cfg_eps;
3097
3098 /* Check if we can fit a single fifo per endpoint */
3099 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
3100 if (fifo_size > ram1_depth)
3101 return -ENOMEM;
3102
3103 return 0;
3104 }
3105
dwc3_gadget_async_callbacks(struct usb_gadget * g,bool enable)3106 static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
3107 {
3108 struct dwc3 *dwc = gadget_to_dwc(g);
3109 unsigned long flags;
3110
3111 spin_lock_irqsave(&dwc->lock, flags);
3112 dwc->async_callbacks = enable;
3113 spin_unlock_irqrestore(&dwc->lock, flags);
3114 }
3115
3116 static const struct usb_gadget_ops dwc3_gadget_ops = {
3117 .get_frame = dwc3_gadget_get_frame,
3118 .wakeup = dwc3_gadget_wakeup,
3119 .func_wakeup = dwc3_gadget_func_wakeup,
3120 .set_remote_wakeup = dwc3_gadget_set_remote_wakeup,
3121 .set_selfpowered = dwc3_gadget_set_selfpowered,
3122 .pullup = dwc3_gadget_pullup,
3123 .udc_start = dwc3_gadget_start,
3124 .udc_stop = dwc3_gadget_stop,
3125 .udc_set_speed = dwc3_gadget_set_speed,
3126 .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate,
3127 .get_config_params = dwc3_gadget_config_params,
3128 .vbus_draw = dwc3_gadget_vbus_draw,
3129 .check_config = dwc3_gadget_check_config,
3130 .udc_async_callbacks = dwc3_gadget_async_callbacks,
3131 };
3132
3133 /* -------------------------------------------------------------------------- */
3134
dwc3_gadget_init_control_endpoint(struct dwc3_ep * dep)3135 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
3136 {
3137 struct dwc3 *dwc = dep->dwc;
3138
3139 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
3140 dep->endpoint.maxburst = 1;
3141 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
3142 if (!dep->direction)
3143 dwc->gadget->ep0 = &dep->endpoint;
3144
3145 dep->endpoint.caps.type_control = true;
3146
3147 return 0;
3148 }
3149
dwc3_gadget_init_in_endpoint(struct dwc3_ep * dep)3150 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
3151 {
3152 struct dwc3 *dwc = dep->dwc;
3153 u32 mdwidth;
3154 int size;
3155 int maxpacket;
3156
3157 mdwidth = dwc3_mdwidth(dwc);
3158
3159 /* MDWIDTH is represented in bits, we need it in bytes */
3160 mdwidth /= 8;
3161
3162 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
3163 if (DWC3_IP_IS(DWC3))
3164 size = DWC3_GTXFIFOSIZ_TXFDEP(size);
3165 else
3166 size = DWC31_GTXFIFOSIZ_TXFDEP(size);
3167
3168 /*
3169 * maxpacket size is determined as part of the following, after assuming
3170 * a mult value of one maxpacket:
3171 * DWC3 revision 280A and prior:
3172 * fifo_size = mult * (max_packet / mdwidth) + 1;
3173 * maxpacket = mdwidth * (fifo_size - 1);
3174 *
3175 * DWC3 revision 290A and onwards:
3176 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
3177 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
3178 */
3179 if (DWC3_VER_IS_PRIOR(DWC3, 290A))
3180 maxpacket = mdwidth * (size - 1);
3181 else
3182 maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
3183
3184 /* Functionally, space for one max packet is sufficient */
3185 size = min_t(int, maxpacket, 1024);
3186 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3187
3188 dep->endpoint.max_streams = 16;
3189 dep->endpoint.ops = &dwc3_gadget_ep_ops;
3190 list_add_tail(&dep->endpoint.ep_list,
3191 &dwc->gadget->ep_list);
3192 dep->endpoint.caps.type_iso = true;
3193 dep->endpoint.caps.type_bulk = true;
3194 dep->endpoint.caps.type_int = true;
3195
3196 return dwc3_alloc_trb_pool(dep);
3197 }
3198
dwc3_gadget_init_out_endpoint(struct dwc3_ep * dep)3199 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3200 {
3201 struct dwc3 *dwc = dep->dwc;
3202 u32 mdwidth;
3203 int size;
3204
3205 mdwidth = dwc3_mdwidth(dwc);
3206
3207 /* MDWIDTH is represented in bits, convert to bytes */
3208 mdwidth /= 8;
3209
3210 /* All OUT endpoints share a single RxFIFO space */
3211 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
3212 if (DWC3_IP_IS(DWC3))
3213 size = DWC3_GRXFIFOSIZ_RXFDEP(size);
3214 else
3215 size = DWC31_GRXFIFOSIZ_RXFDEP(size);
3216
3217 /* FIFO depth is in MDWDITH bytes */
3218 size *= mdwidth;
3219
3220 /*
3221 * To meet performance requirement, a minimum recommended RxFIFO size
3222 * is defined as follow:
3223 * RxFIFO size >= (3 x MaxPacketSize) +
3224 * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
3225 *
3226 * Then calculate the max packet limit as below.
3227 */
3228 size -= (3 * 8) + 16;
3229 if (size < 0)
3230 size = 0;
3231 else
3232 size /= 3;
3233
3234 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3235 dep->endpoint.max_streams = 16;
3236 dep->endpoint.ops = &dwc3_gadget_ep_ops;
3237 list_add_tail(&dep->endpoint.ep_list,
3238 &dwc->gadget->ep_list);
3239 dep->endpoint.caps.type_iso = true;
3240 dep->endpoint.caps.type_bulk = true;
3241 dep->endpoint.caps.type_int = true;
3242
3243 return dwc3_alloc_trb_pool(dep);
3244 }
3245
dwc3_gadget_init_endpoint(struct dwc3 * dwc,u8 epnum)3246 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3247 {
3248 struct dwc3_ep *dep;
3249 bool direction = epnum & 1;
3250 int ret;
3251 u8 num = epnum >> 1;
3252
3253 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
3254 if (!dep)
3255 return -ENOMEM;
3256
3257 dep->dwc = dwc;
3258 dep->number = epnum;
3259 dep->direction = direction;
3260 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3261 dwc->eps[epnum] = dep;
3262 dep->combo_num = 0;
3263 dep->start_cmd_status = 0;
3264
3265 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
3266 direction ? "in" : "out");
3267
3268 dep->endpoint.name = dep->name;
3269
3270 if (!(dep->number > 1)) {
3271 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3272 dep->endpoint.comp_desc = NULL;
3273 }
3274
3275 if (num == 0)
3276 ret = dwc3_gadget_init_control_endpoint(dep);
3277 else if (direction)
3278 ret = dwc3_gadget_init_in_endpoint(dep);
3279 else
3280 ret = dwc3_gadget_init_out_endpoint(dep);
3281
3282 if (ret)
3283 return ret;
3284
3285 dep->endpoint.caps.dir_in = direction;
3286 dep->endpoint.caps.dir_out = !direction;
3287
3288 INIT_LIST_HEAD(&dep->pending_list);
3289 INIT_LIST_HEAD(&dep->started_list);
3290 INIT_LIST_HEAD(&dep->cancelled_list);
3291
3292 dwc3_debugfs_create_endpoint_dir(dep);
3293
3294 return 0;
3295 }
3296
dwc3_gadget_init_endpoints(struct dwc3 * dwc,u8 total)3297 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3298 {
3299 u8 epnum;
3300
3301 INIT_LIST_HEAD(&dwc->gadget->ep_list);
3302
3303 for (epnum = 0; epnum < total; epnum++) {
3304 int ret;
3305
3306 ret = dwc3_gadget_init_endpoint(dwc, epnum);
3307 if (ret)
3308 return ret;
3309 }
3310
3311 return 0;
3312 }
3313
dwc3_gadget_free_endpoints(struct dwc3 * dwc)3314 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3315 {
3316 struct dwc3_ep *dep;
3317 u8 epnum;
3318
3319 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3320 dep = dwc->eps[epnum];
3321 if (!dep)
3322 continue;
3323 /*
3324 * Physical endpoints 0 and 1 are special; they form the
3325 * bi-directional USB endpoint 0.
3326 *
3327 * For those two physical endpoints, we don't allocate a TRB
3328 * pool nor do we add them the endpoints list. Due to that, we
3329 * shouldn't do these two operations otherwise we would end up
3330 * with all sorts of bugs when removing dwc3.ko.
3331 */
3332 if (epnum != 0 && epnum != 1) {
3333 dwc3_free_trb_pool(dep);
3334 list_del(&dep->endpoint.ep_list);
3335 }
3336
3337 dwc3_debugfs_remove_endpoint_dir(dep);
3338 kfree(dep);
3339 }
3340 }
3341
3342 /* -------------------------------------------------------------------------- */
3343
dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep * dep,struct dwc3_request * req,struct dwc3_trb * trb,const struct dwc3_event_depevt * event,int status,int chain)3344 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3345 struct dwc3_request *req, struct dwc3_trb *trb,
3346 const struct dwc3_event_depevt *event, int status, int chain)
3347 {
3348 unsigned int count;
3349
3350 dwc3_ep_inc_deq(dep);
3351
3352 trace_dwc3_complete_trb(dep, trb);
3353 req->num_trbs--;
3354
3355 /*
3356 * If we're in the middle of series of chained TRBs and we
3357 * receive a short transfer along the way, DWC3 will skip
3358 * through all TRBs including the last TRB in the chain (the
3359 * where CHN bit is zero. DWC3 will also avoid clearing HWO
3360 * bit and SW has to do it manually.
3361 *
3362 * We're going to do that here to avoid problems of HW trying
3363 * to use bogus TRBs for transfers.
3364 */
3365 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
3366 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3367
3368 /*
3369 * For isochronous transfers, the first TRB in a service interval must
3370 * have the Isoc-First type. Track and report its interval frame number.
3371 */
3372 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3373 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3374 unsigned int frame_number;
3375
3376 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3377 frame_number &= ~(dep->interval - 1);
3378 req->request.frame_number = frame_number;
3379 }
3380
3381 /*
3382 * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
3383 * this TRB points to the bounce buffer address, it's a MPS alignment
3384 * TRB. Don't add it to req->remaining calculation.
3385 */
3386 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3387 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3388 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3389 return 1;
3390 }
3391
3392 count = trb->size & DWC3_TRB_SIZE_MASK;
3393 req->remaining += count;
3394
3395 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3396 return 1;
3397
3398 if (event->status & DEPEVT_STATUS_SHORT && !chain)
3399 return 1;
3400
3401 if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&
3402 DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)
3403 return 1;
3404
3405 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3406 (trb->ctrl & DWC3_TRB_CTRL_LST))
3407 return 1;
3408
3409 return 0;
3410 }
3411
dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep * dep,struct dwc3_request * req,const struct dwc3_event_depevt * event,int status)3412 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3413 struct dwc3_request *req, const struct dwc3_event_depevt *event,
3414 int status)
3415 {
3416 struct dwc3_trb *trb;
3417 struct scatterlist *sg = req->sg;
3418 struct scatterlist *s;
3419 unsigned int num_queued = req->num_queued_sgs;
3420 unsigned int i;
3421 int ret = 0;
3422
3423 for_each_sg(sg, s, num_queued, i) {
3424 trb = &dep->trb_pool[dep->trb_dequeue];
3425
3426 req->sg = sg_next(s);
3427 req->num_queued_sgs--;
3428
3429 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3430 trb, event, status, true);
3431 if (ret)
3432 break;
3433 }
3434
3435 return ret;
3436 }
3437
dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep * dep,struct dwc3_request * req,const struct dwc3_event_depevt * event,int status)3438 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3439 struct dwc3_request *req, const struct dwc3_event_depevt *event,
3440 int status)
3441 {
3442 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3443
3444 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3445 event, status, false);
3446 }
3447
dwc3_gadget_ep_request_completed(struct dwc3_request * req)3448 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3449 {
3450 return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;
3451 }
3452
dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,struct dwc3_request * req,int status)3453 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3454 const struct dwc3_event_depevt *event,
3455 struct dwc3_request *req, int status)
3456 {
3457 int request_status;
3458 int ret;
3459
3460 if (req->request.num_mapped_sgs)
3461 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3462 status);
3463 else
3464 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3465 status);
3466
3467 req->request.actual = req->request.length - req->remaining;
3468
3469 if (!dwc3_gadget_ep_request_completed(req))
3470 goto out;
3471
3472 if (req->needs_extra_trb) {
3473 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3474 status);
3475 req->needs_extra_trb = false;
3476 }
3477
3478 /*
3479 * The event status only reflects the status of the TRB with IOC set.
3480 * For the requests that don't set interrupt on completion, the driver
3481 * needs to check and return the status of the completed TRBs associated
3482 * with the request. Use the status of the last TRB of the request.
3483 */
3484 if (req->request.no_interrupt) {
3485 struct dwc3_trb *trb;
3486
3487 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
3488 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
3489 case DWC3_TRBSTS_MISSED_ISOC:
3490 /* Isoc endpoint only */
3491 request_status = -EXDEV;
3492 break;
3493 case DWC3_TRB_STS_XFER_IN_PROG:
3494 /* Applicable when End Transfer with ForceRM=0 */
3495 case DWC3_TRBSTS_SETUP_PENDING:
3496 /* Control endpoint only */
3497 case DWC3_TRBSTS_OK:
3498 default:
3499 request_status = 0;
3500 break;
3501 }
3502 } else {
3503 request_status = status;
3504 }
3505
3506 dwc3_gadget_giveback(dep, req, request_status);
3507
3508 out:
3509 return ret;
3510 }
3511
dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,int status)3512 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3513 const struct dwc3_event_depevt *event, int status)
3514 {
3515 struct dwc3_request *req;
3516
3517 while (!list_empty(&dep->started_list)) {
3518 int ret;
3519
3520 req = next_request(&dep->started_list);
3521 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3522 req, status);
3523 if (ret)
3524 break;
3525 /*
3526 * The endpoint is disabled, let the dwc3_remove_requests()
3527 * handle the cleanup.
3528 */
3529 if (!dep->endpoint.desc)
3530 break;
3531 }
3532 }
3533
dwc3_gadget_ep_should_continue(struct dwc3_ep * dep)3534 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3535 {
3536 struct dwc3_request *req;
3537 struct dwc3 *dwc = dep->dwc;
3538
3539 if (!dep->endpoint.desc || !dwc->pullups_connected ||
3540 !dwc->connected)
3541 return false;
3542
3543 if (!list_empty(&dep->pending_list))
3544 return true;
3545
3546 /*
3547 * We only need to check the first entry of the started list. We can
3548 * assume the completed requests are removed from the started list.
3549 */
3550 req = next_request(&dep->started_list);
3551 if (!req)
3552 return false;
3553
3554 return !dwc3_gadget_ep_request_completed(req);
3555 }
3556
dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3557 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3558 const struct dwc3_event_depevt *event)
3559 {
3560 dep->frame_number = event->parameters;
3561 }
3562
dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,int status)3563 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3564 const struct dwc3_event_depevt *event, int status)
3565 {
3566 struct dwc3 *dwc = dep->dwc;
3567 bool no_started_trb = true;
3568
3569 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3570
3571 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3572 goto out;
3573
3574 if (!dep->endpoint.desc)
3575 return no_started_trb;
3576
3577 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3578 list_empty(&dep->started_list) &&
3579 (list_empty(&dep->pending_list) || status == -EXDEV))
3580 dwc3_stop_active_transfer(dep, true, true);
3581 else if (dwc3_gadget_ep_should_continue(dep))
3582 if (__dwc3_gadget_kick_transfer(dep) == 0)
3583 no_started_trb = false;
3584
3585 out:
3586 /*
3587 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
3588 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
3589 */
3590 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3591 u32 reg;
3592 int i;
3593
3594 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3595 dep = dwc->eps[i];
3596
3597 if (!(dep->flags & DWC3_EP_ENABLED))
3598 continue;
3599
3600 if (!list_empty(&dep->started_list))
3601 return no_started_trb;
3602 }
3603
3604 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3605 reg |= dwc->u1u2;
3606 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3607
3608 dwc->u1u2 = 0;
3609 }
3610
3611 return no_started_trb;
3612 }
3613
dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3614 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3615 const struct dwc3_event_depevt *event)
3616 {
3617 int status = 0;
3618
3619 if (!dep->endpoint.desc)
3620 return;
3621
3622 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3623 dwc3_gadget_endpoint_frame_from_event(dep, event);
3624
3625 if (event->status & DEPEVT_STATUS_BUSERR)
3626 status = -ECONNRESET;
3627
3628 if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3629 status = -EXDEV;
3630
3631 dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3632 }
3633
dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3634 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3635 const struct dwc3_event_depevt *event)
3636 {
3637 int status = 0;
3638
3639 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3640
3641 if (event->status & DEPEVT_STATUS_BUSERR)
3642 status = -ECONNRESET;
3643
3644 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3645 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3646 }
3647
dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3648 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3649 const struct dwc3_event_depevt *event)
3650 {
3651 dwc3_gadget_endpoint_frame_from_event(dep, event);
3652
3653 /*
3654 * The XferNotReady event is generated only once before the endpoint
3655 * starts. It will be generated again when END_TRANSFER command is
3656 * issued. For some controller versions, the XferNotReady event may be
3657 * generated while the END_TRANSFER command is still in process. Ignore
3658 * it and wait for the next XferNotReady event after the command is
3659 * completed.
3660 */
3661 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3662 return;
3663
3664 (void) __dwc3_gadget_start_isoc(dep);
3665 }
3666
dwc3_gadget_endpoint_command_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3667 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3668 const struct dwc3_event_depevt *event)
3669 {
3670 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3671
3672 if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3673 return;
3674
3675 /*
3676 * The END_TRANSFER command will cause the controller to generate a
3677 * NoStream Event, and it's not due to the host DP NoStream rejection.
3678 * Ignore the next NoStream event.
3679 */
3680 if (dep->stream_capable)
3681 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3682
3683 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3684 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3685 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3686
3687 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3688 struct dwc3 *dwc = dep->dwc;
3689
3690 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3691 if (dwc3_send_clear_stall_ep_cmd(dep)) {
3692 struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3693
3694 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3695 if (dwc->delayed_status)
3696 __dwc3_gadget_ep0_set_halt(ep0, 1);
3697 return;
3698 }
3699
3700 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3701 if (dwc->clear_stall_protocol == dep->number)
3702 dwc3_ep0_send_delayed_status(dwc);
3703 }
3704
3705 if ((dep->flags & DWC3_EP_DELAY_START) &&
3706 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3707 __dwc3_gadget_kick_transfer(dep);
3708
3709 dep->flags &= ~DWC3_EP_DELAY_START;
3710 }
3711
dwc3_gadget_endpoint_stream_event(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3712 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3713 const struct dwc3_event_depevt *event)
3714 {
3715 struct dwc3 *dwc = dep->dwc;
3716
3717 if (event->status == DEPEVT_STREAMEVT_FOUND) {
3718 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3719 goto out;
3720 }
3721
3722 /* Note: NoStream rejection event param value is 0 and not 0xFFFF */
3723 switch (event->parameters) {
3724 case DEPEVT_STREAM_PRIME:
3725 /*
3726 * If the host can properly transition the endpoint state from
3727 * idle to prime after a NoStream rejection, there's no need to
3728 * force restarting the endpoint to reinitiate the stream. To
3729 * simplify the check, assume the host follows the USB spec if
3730 * it primed the endpoint more than once.
3731 */
3732 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3733 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3734 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3735 else
3736 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3737 }
3738
3739 break;
3740 case DEPEVT_STREAM_NOSTREAM:
3741 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3742 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3743 (!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3744 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3745 break;
3746
3747 /*
3748 * If the host rejects a stream due to no active stream, by the
3749 * USB and xHCI spec, the endpoint will be put back to idle
3750 * state. When the host is ready (buffer added/updated), it will
3751 * prime the endpoint to inform the usb device controller. This
3752 * triggers the device controller to issue ERDY to restart the
3753 * stream. However, some hosts don't follow this and keep the
3754 * endpoint in the idle state. No prime will come despite host
3755 * streams are updated, and the device controller will not be
3756 * triggered to generate ERDY to move the next stream data. To
3757 * workaround this and maintain compatibility with various
3758 * hosts, force to reinitiate the stream until the host is ready
3759 * instead of waiting for the host to prime the endpoint.
3760 */
3761 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3762 unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3763
3764 dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3765 } else {
3766 dep->flags |= DWC3_EP_DELAY_START;
3767 dwc3_stop_active_transfer(dep, true, true);
3768 return;
3769 }
3770 break;
3771 }
3772
3773 out:
3774 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3775 }
3776
dwc3_endpoint_interrupt(struct dwc3 * dwc,const struct dwc3_event_depevt * event)3777 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3778 const struct dwc3_event_depevt *event)
3779 {
3780 struct dwc3_ep *dep;
3781 u8 epnum = event->endpoint_number;
3782
3783 dep = dwc->eps[epnum];
3784
3785 if (!(dep->flags & DWC3_EP_ENABLED)) {
3786 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3787 return;
3788
3789 /* Handle only EPCMDCMPLT when EP disabled */
3790 if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&
3791 !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))
3792 return;
3793 }
3794
3795 if (epnum == 0 || epnum == 1) {
3796 dwc3_ep0_interrupt(dwc, event);
3797 return;
3798 }
3799
3800 switch (event->endpoint_event) {
3801 case DWC3_DEPEVT_XFERINPROGRESS:
3802 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3803 break;
3804 case DWC3_DEPEVT_XFERNOTREADY:
3805 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3806 break;
3807 case DWC3_DEPEVT_EPCMDCMPLT:
3808 dwc3_gadget_endpoint_command_complete(dep, event);
3809 break;
3810 case DWC3_DEPEVT_XFERCOMPLETE:
3811 dwc3_gadget_endpoint_transfer_complete(dep, event);
3812 break;
3813 case DWC3_DEPEVT_STREAMEVT:
3814 dwc3_gadget_endpoint_stream_event(dep, event);
3815 break;
3816 case DWC3_DEPEVT_RXTXFIFOEVT:
3817 break;
3818 default:
3819 dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event);
3820 break;
3821 }
3822 }
3823
dwc3_disconnect_gadget(struct dwc3 * dwc)3824 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3825 {
3826 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3827 spin_unlock(&dwc->lock);
3828 dwc->gadget_driver->disconnect(dwc->gadget);
3829 spin_lock(&dwc->lock);
3830 }
3831 }
3832
dwc3_suspend_gadget(struct dwc3 * dwc)3833 static void dwc3_suspend_gadget(struct dwc3 *dwc)
3834 {
3835 if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3836 spin_unlock(&dwc->lock);
3837 dwc->gadget_driver->suspend(dwc->gadget);
3838 spin_lock(&dwc->lock);
3839 }
3840 }
3841
dwc3_resume_gadget(struct dwc3 * dwc)3842 static void dwc3_resume_gadget(struct dwc3 *dwc)
3843 {
3844 if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3845 spin_unlock(&dwc->lock);
3846 dwc->gadget_driver->resume(dwc->gadget);
3847 spin_lock(&dwc->lock);
3848 }
3849 }
3850
dwc3_reset_gadget(struct dwc3 * dwc)3851 static void dwc3_reset_gadget(struct dwc3 *dwc)
3852 {
3853 if (!dwc->gadget_driver)
3854 return;
3855
3856 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3857 spin_unlock(&dwc->lock);
3858 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3859 spin_lock(&dwc->lock);
3860 }
3861 }
3862
dwc3_stop_active_transfer(struct dwc3_ep * dep,bool force,bool interrupt)3863 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3864 bool interrupt)
3865 {
3866 struct dwc3 *dwc = dep->dwc;
3867
3868 /*
3869 * Only issue End Transfer command to the control endpoint of a started
3870 * Data Phase. Typically we should only do so in error cases such as
3871 * invalid/unexpected direction as described in the control transfer
3872 * flow of the programming guide.
3873 */
3874 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
3875 return;
3876
3877 if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))
3878 return;
3879
3880 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3881 (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3882 return;
3883
3884 /*
3885 * If a Setup packet is received but yet to DMA out, the controller will
3886 * not process the End Transfer command of any endpoint. Polling of its
3887 * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
3888 * timeout. Delay issuing the End Transfer command until the Setup TRB is
3889 * prepared.
3890 */
3891 if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
3892 dep->flags |= DWC3_EP_DELAY_STOP;
3893 return;
3894 }
3895
3896 /*
3897 * NOTICE: We are violating what the Databook says about the
3898 * EndTransfer command. Ideally we would _always_ wait for the
3899 * EndTransfer Command Completion IRQ, but that's causing too
3900 * much trouble synchronizing between us and gadget driver.
3901 *
3902 * We have discussed this with the IP Provider and it was
3903 * suggested to giveback all requests here.
3904 *
3905 * Note also that a similar handling was tested by Synopsys
3906 * (thanks a lot Paul) and nothing bad has come out of it.
3907 * In short, what we're doing is issuing EndTransfer with
3908 * CMDIOC bit set and delay kicking transfer until the
3909 * EndTransfer command had completed.
3910 *
3911 * As of IP version 3.10a of the DWC_usb3 IP, the controller
3912 * supports a mode to work around the above limitation. The
3913 * software can poll the CMDACT bit in the DEPCMD register
3914 * after issuing a EndTransfer command. This mode is enabled
3915 * by writing GUCTL2[14]. This polling is already done in the
3916 * dwc3_send_gadget_ep_cmd() function so if the mode is
3917 * enabled, the EndTransfer command will have completed upon
3918 * returning from this function.
3919 *
3920 * This mode is NOT available on the DWC_usb31 IP. In this
3921 * case, if the IOC bit is not set, then delay by 1ms
3922 * after issuing the EndTransfer command. This allows for the
3923 * controller to handle the command completely before DWC3
3924 * remove requests attempts to unmap USB request buffers.
3925 */
3926
3927 __dwc3_stop_active_transfer(dep, force, interrupt);
3928 }
3929
dwc3_clear_stall_all_ep(struct dwc3 * dwc)3930 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3931 {
3932 u32 epnum;
3933
3934 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3935 struct dwc3_ep *dep;
3936 int ret;
3937
3938 dep = dwc->eps[epnum];
3939 if (!dep)
3940 continue;
3941
3942 if (!(dep->flags & DWC3_EP_STALL))
3943 continue;
3944
3945 dep->flags &= ~DWC3_EP_STALL;
3946
3947 ret = dwc3_send_clear_stall_ep_cmd(dep);
3948 WARN_ON_ONCE(ret);
3949 }
3950 }
3951
dwc3_gadget_disconnect_interrupt(struct dwc3 * dwc)3952 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3953 {
3954 int reg;
3955
3956 dwc->suspended = false;
3957
3958 dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
3959
3960 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3961 reg &= ~DWC3_DCTL_INITU1ENA;
3962 reg &= ~DWC3_DCTL_INITU2ENA;
3963 dwc3_gadget_dctl_write_safe(dwc, reg);
3964
3965 dwc->connected = false;
3966
3967 dwc3_disconnect_gadget(dwc);
3968
3969 dwc->gadget->speed = USB_SPEED_UNKNOWN;
3970 dwc->setup_packet_pending = false;
3971 dwc->gadget->wakeup_armed = false;
3972 dwc3_gadget_enable_linksts_evts(dwc, false);
3973 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3974
3975 dwc3_ep0_reset_state(dwc);
3976
3977 /*
3978 * Request PM idle to address condition where usage count is
3979 * already decremented to zero, but waiting for the disconnect
3980 * interrupt to set dwc->connected to FALSE.
3981 */
3982 pm_request_idle(dwc->dev);
3983 }
3984
dwc3_gadget_reset_interrupt(struct dwc3 * dwc)3985 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3986 {
3987 u32 reg;
3988
3989 dwc->suspended = false;
3990
3991 /*
3992 * Ideally, dwc3_reset_gadget() would trigger the function
3993 * drivers to stop any active transfers through ep disable.
3994 * However, for functions which defer ep disable, such as mass
3995 * storage, we will need to rely on the call to stop active
3996 * transfers here, and avoid allowing of request queuing.
3997 */
3998 dwc->connected = false;
3999
4000 /*
4001 * WORKAROUND: DWC3 revisions <1.88a have an issue which
4002 * would cause a missing Disconnect Event if there's a
4003 * pending Setup Packet in the FIFO.
4004 *
4005 * There's no suggested workaround on the official Bug
4006 * report, which states that "unless the driver/application
4007 * is doing any special handling of a disconnect event,
4008 * there is no functional issue".
4009 *
4010 * Unfortunately, it turns out that we _do_ some special
4011 * handling of a disconnect event, namely complete all
4012 * pending transfers, notify gadget driver of the
4013 * disconnection, and so on.
4014 *
4015 * Our suggested workaround is to follow the Disconnect
4016 * Event steps here, instead, based on a setup_packet_pending
4017 * flag. Such flag gets set whenever we have a SETUP_PENDING
4018 * status for EP0 TRBs and gets cleared on XferComplete for the
4019 * same endpoint.
4020 *
4021 * Refers to:
4022 *
4023 * STAR#9000466709: RTL: Device : Disconnect event not
4024 * generated if setup packet pending in FIFO
4025 */
4026 if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
4027 if (dwc->setup_packet_pending)
4028 dwc3_gadget_disconnect_interrupt(dwc);
4029 }
4030
4031 dwc3_reset_gadget(dwc);
4032
4033 /*
4034 * From SNPS databook section 8.1.2, the EP0 should be in setup
4035 * phase. So ensure that EP0 is in setup phase by issuing a stall
4036 * and restart if EP0 is not in setup phase.
4037 */
4038 dwc3_ep0_reset_state(dwc);
4039
4040 /*
4041 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
4042 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
4043 * needs to ensure that it sends "a DEPENDXFER command for any active
4044 * transfers."
4045 */
4046 dwc3_stop_active_transfers(dwc);
4047 dwc->connected = true;
4048
4049 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4050 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
4051 dwc3_gadget_dctl_write_safe(dwc, reg);
4052 dwc->test_mode = false;
4053 dwc->gadget->wakeup_armed = false;
4054 dwc3_gadget_enable_linksts_evts(dwc, false);
4055 dwc3_clear_stall_all_ep(dwc);
4056
4057 /* Reset device address to zero */
4058 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4059 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
4060 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4061 }
4062
dwc3_gadget_conndone_interrupt(struct dwc3 * dwc)4063 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
4064 {
4065 struct dwc3_ep *dep;
4066 int ret;
4067 u32 reg;
4068 u8 lanes = 1;
4069 u8 speed;
4070
4071 if (!dwc->softconnect)
4072 return;
4073
4074 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
4075 speed = reg & DWC3_DSTS_CONNECTSPD;
4076 dwc->speed = speed;
4077
4078 if (DWC3_IP_IS(DWC32))
4079 lanes = DWC3_DSTS_CONNLANES(reg) + 1;
4080
4081 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4082
4083 /*
4084 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
4085 * each time on Connect Done.
4086 *
4087 * Currently we always use the reset value. If any platform
4088 * wants to set this to a different value, we need to add a
4089 * setting and update GCTL.RAMCLKSEL here.
4090 */
4091
4092 switch (speed) {
4093 case DWC3_DSTS_SUPERSPEED_PLUS:
4094 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4095 dwc->gadget->ep0->maxpacket = 512;
4096 dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4097
4098 if (lanes > 1)
4099 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
4100 else
4101 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
4102 break;
4103 case DWC3_DSTS_SUPERSPEED:
4104 /*
4105 * WORKAROUND: DWC3 revisions <1.90a have an issue which
4106 * would cause a missing USB3 Reset event.
4107 *
4108 * In such situations, we should force a USB3 Reset
4109 * event by calling our dwc3_gadget_reset_interrupt()
4110 * routine.
4111 *
4112 * Refers to:
4113 *
4114 * STAR#9000483510: RTL: SS : USB3 reset event may
4115 * not be generated always when the link enters poll
4116 */
4117 if (DWC3_VER_IS_PRIOR(DWC3, 190A))
4118 dwc3_gadget_reset_interrupt(dwc);
4119
4120 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4121 dwc->gadget->ep0->maxpacket = 512;
4122 dwc->gadget->speed = USB_SPEED_SUPER;
4123
4124 if (lanes > 1) {
4125 dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4126 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
4127 }
4128 break;
4129 case DWC3_DSTS_HIGHSPEED:
4130 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4131 dwc->gadget->ep0->maxpacket = 64;
4132 dwc->gadget->speed = USB_SPEED_HIGH;
4133 break;
4134 case DWC3_DSTS_FULLSPEED:
4135 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4136 dwc->gadget->ep0->maxpacket = 64;
4137 dwc->gadget->speed = USB_SPEED_FULL;
4138 break;
4139 }
4140
4141 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
4142
4143 /* Enable USB2 LPM Capability */
4144
4145 if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
4146 !dwc->usb2_gadget_lpm_disable &&
4147 (speed != DWC3_DSTS_SUPERSPEED) &&
4148 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
4149 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4150 reg |= DWC3_DCFG_LPM_CAP;
4151 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4152
4153 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4154 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
4155
4156 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
4157 (dwc->is_utmi_l1_suspend << 4));
4158
4159 /*
4160 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
4161 * DCFG.LPMCap is set, core responses with an ACK and the
4162 * BESL value in the LPM token is less than or equal to LPM
4163 * NYET threshold.
4164 */
4165 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
4166 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
4167
4168 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
4169 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
4170
4171 dwc3_gadget_dctl_write_safe(dwc, reg);
4172 } else {
4173 if (dwc->usb2_gadget_lpm_disable) {
4174 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4175 reg &= ~DWC3_DCFG_LPM_CAP;
4176 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4177 }
4178
4179 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4180 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
4181 dwc3_gadget_dctl_write_safe(dwc, reg);
4182 }
4183
4184 dep = dwc->eps[0];
4185 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4186 if (ret) {
4187 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4188 return;
4189 }
4190
4191 dep = dwc->eps[1];
4192 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4193 if (ret) {
4194 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4195 return;
4196 }
4197
4198 /*
4199 * Configure PHY via GUSB3PIPECTLn if required.
4200 *
4201 * Update GTXFIFOSIZn
4202 *
4203 * In both cases reset values should be sufficient.
4204 */
4205 }
4206
dwc3_gadget_wakeup_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4207 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
4208 {
4209 dwc->suspended = false;
4210
4211 /*
4212 * TODO take core out of low power mode when that's
4213 * implemented.
4214 */
4215
4216 if (dwc->async_callbacks && dwc->gadget_driver->resume) {
4217 spin_unlock(&dwc->lock);
4218 dwc->gadget_driver->resume(dwc->gadget);
4219 spin_lock(&dwc->lock);
4220 }
4221
4222 dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
4223 }
4224
dwc3_gadget_linksts_change_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4225 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
4226 unsigned int evtinfo)
4227 {
4228 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4229 unsigned int pwropt;
4230
4231 /*
4232 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
4233 * Hibernation mode enabled which would show up when device detects
4234 * host-initiated U3 exit.
4235 *
4236 * In that case, device will generate a Link State Change Interrupt
4237 * from U3 to RESUME which is only necessary if Hibernation is
4238 * configured in.
4239 *
4240 * There are no functional changes due to such spurious event and we
4241 * just need to ignore it.
4242 *
4243 * Refers to:
4244 *
4245 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
4246 * operational mode
4247 */
4248 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
4249 if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
4250 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
4251 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
4252 (next == DWC3_LINK_STATE_RESUME)) {
4253 return;
4254 }
4255 }
4256
4257 /*
4258 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
4259 * on the link partner, the USB session might do multiple entry/exit
4260 * of low power states before a transfer takes place.
4261 *
4262 * Due to this problem, we might experience lower throughput. The
4263 * suggested workaround is to disable DCTL[12:9] bits if we're
4264 * transitioning from U1/U2 to U0 and enable those bits again
4265 * after a transfer completes and there are no pending transfers
4266 * on any of the enabled endpoints.
4267 *
4268 * This is the first half of that workaround.
4269 *
4270 * Refers to:
4271 *
4272 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
4273 * core send LGO_Ux entering U0
4274 */
4275 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
4276 if (next == DWC3_LINK_STATE_U0) {
4277 u32 u1u2;
4278 u32 reg;
4279
4280 switch (dwc->link_state) {
4281 case DWC3_LINK_STATE_U1:
4282 case DWC3_LINK_STATE_U2:
4283 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4284 u1u2 = reg & (DWC3_DCTL_INITU2ENA
4285 | DWC3_DCTL_ACCEPTU2ENA
4286 | DWC3_DCTL_INITU1ENA
4287 | DWC3_DCTL_ACCEPTU1ENA);
4288
4289 if (!dwc->u1u2)
4290 dwc->u1u2 = reg & u1u2;
4291
4292 reg &= ~u1u2;
4293
4294 dwc3_gadget_dctl_write_safe(dwc, reg);
4295 break;
4296 default:
4297 /* do nothing */
4298 break;
4299 }
4300 }
4301 }
4302
4303 switch (next) {
4304 case DWC3_LINK_STATE_U0:
4305 if (dwc->gadget->wakeup_armed) {
4306 dwc3_gadget_enable_linksts_evts(dwc, false);
4307 dwc3_resume_gadget(dwc);
4308 dwc->suspended = false;
4309 }
4310 break;
4311 case DWC3_LINK_STATE_U1:
4312 if (dwc->speed == USB_SPEED_SUPER)
4313 dwc3_suspend_gadget(dwc);
4314 break;
4315 case DWC3_LINK_STATE_U2:
4316 case DWC3_LINK_STATE_U3:
4317 dwc3_suspend_gadget(dwc);
4318 break;
4319 case DWC3_LINK_STATE_RESUME:
4320 dwc3_resume_gadget(dwc);
4321 break;
4322 default:
4323 /* do nothing */
4324 break;
4325 }
4326
4327 dwc->link_state = next;
4328 }
4329
dwc3_gadget_suspend_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4330 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4331 unsigned int evtinfo)
4332 {
4333 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4334
4335 if (!dwc->suspended && next == DWC3_LINK_STATE_U3) {
4336 dwc->suspended = true;
4337 dwc3_suspend_gadget(dwc);
4338 }
4339
4340 dwc->link_state = next;
4341 }
4342
dwc3_gadget_interrupt(struct dwc3 * dwc,const struct dwc3_event_devt * event)4343 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4344 const struct dwc3_event_devt *event)
4345 {
4346 switch (event->type) {
4347 case DWC3_DEVICE_EVENT_DISCONNECT:
4348 dwc3_gadget_disconnect_interrupt(dwc);
4349 break;
4350 case DWC3_DEVICE_EVENT_RESET:
4351 dwc3_gadget_reset_interrupt(dwc);
4352 break;
4353 case DWC3_DEVICE_EVENT_CONNECT_DONE:
4354 dwc3_gadget_conndone_interrupt(dwc);
4355 break;
4356 case DWC3_DEVICE_EVENT_WAKEUP:
4357 dwc3_gadget_wakeup_interrupt(dwc, event->event_info);
4358 break;
4359 case DWC3_DEVICE_EVENT_HIBER_REQ:
4360 dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n");
4361 break;
4362 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4363 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
4364 break;
4365 case DWC3_DEVICE_EVENT_SUSPEND:
4366 /* It changed to be suspend event for version 2.30a and above */
4367 if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
4368 dwc3_gadget_suspend_interrupt(dwc, event->event_info);
4369 break;
4370 case DWC3_DEVICE_EVENT_SOF:
4371 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4372 case DWC3_DEVICE_EVENT_CMD_CMPL:
4373 case DWC3_DEVICE_EVENT_OVERFLOW:
4374 break;
4375 default:
4376 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4377 }
4378 }
4379
dwc3_process_event_entry(struct dwc3 * dwc,const union dwc3_event * event)4380 static void dwc3_process_event_entry(struct dwc3 *dwc,
4381 const union dwc3_event *event)
4382 {
4383 trace_dwc3_event(event->raw, dwc);
4384
4385 if (!event->type.is_devspec)
4386 dwc3_endpoint_interrupt(dwc, &event->depevt);
4387 else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4388 dwc3_gadget_interrupt(dwc, &event->devt);
4389 else
4390 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4391 }
4392
dwc3_process_event_buf(struct dwc3_event_buffer * evt)4393 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4394 {
4395 struct dwc3 *dwc = evt->dwc;
4396 irqreturn_t ret = IRQ_NONE;
4397 int left;
4398
4399 left = evt->count;
4400
4401 if (!(evt->flags & DWC3_EVENT_PENDING))
4402 return IRQ_NONE;
4403
4404 while (left > 0) {
4405 union dwc3_event event;
4406
4407 event.raw = *(u32 *) (evt->cache + evt->lpos);
4408
4409 dwc3_process_event_entry(dwc, &event);
4410
4411 /*
4412 * FIXME we wrap around correctly to the next entry as
4413 * almost all entries are 4 bytes in size. There is one
4414 * entry which has 12 bytes which is a regular entry
4415 * followed by 8 bytes data. ATM I don't know how
4416 * things are organized if we get next to the a
4417 * boundary so I worry about that once we try to handle
4418 * that.
4419 */
4420 evt->lpos = (evt->lpos + 4) % evt->length;
4421 left -= 4;
4422 }
4423
4424 evt->count = 0;
4425 ret = IRQ_HANDLED;
4426
4427 /* Unmask interrupt */
4428 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4429 DWC3_GEVNTSIZ_SIZE(evt->length));
4430
4431 if (dwc->imod_interval) {
4432 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4433 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
4434 }
4435
4436 /* Keep the clearing of DWC3_EVENT_PENDING at the end */
4437 evt->flags &= ~DWC3_EVENT_PENDING;
4438
4439 return ret;
4440 }
4441
dwc3_thread_interrupt(int irq,void * _evt)4442 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4443 {
4444 struct dwc3_event_buffer *evt = _evt;
4445 struct dwc3 *dwc = evt->dwc;
4446 unsigned long flags;
4447 irqreturn_t ret = IRQ_NONE;
4448
4449 local_bh_disable();
4450 spin_lock_irqsave(&dwc->lock, flags);
4451 ret = dwc3_process_event_buf(evt);
4452 spin_unlock_irqrestore(&dwc->lock, flags);
4453 local_bh_enable();
4454
4455 return ret;
4456 }
4457
dwc3_check_event_buf(struct dwc3_event_buffer * evt)4458 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4459 {
4460 struct dwc3 *dwc = evt->dwc;
4461 u32 amount;
4462 u32 count;
4463
4464 if (pm_runtime_suspended(dwc->dev)) {
4465 dwc->pending_events = true;
4466 /*
4467 * Trigger runtime resume. The get() function will be balanced
4468 * after processing the pending events in dwc3_process_pending
4469 * events().
4470 */
4471 pm_runtime_get(dwc->dev);
4472 disable_irq_nosync(dwc->irq_gadget);
4473 return IRQ_HANDLED;
4474 }
4475
4476 /*
4477 * With PCIe legacy interrupt, test shows that top-half irq handler can
4478 * be called again after HW interrupt deassertion. Check if bottom-half
4479 * irq event handler completes before caching new event to prevent
4480 * losing events.
4481 */
4482 if (evt->flags & DWC3_EVENT_PENDING)
4483 return IRQ_HANDLED;
4484
4485 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
4486 count &= DWC3_GEVNTCOUNT_MASK;
4487 if (!count)
4488 return IRQ_NONE;
4489
4490 evt->count = count;
4491 evt->flags |= DWC3_EVENT_PENDING;
4492
4493 /* Mask interrupt */
4494 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4495 DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4496
4497 amount = min(count, evt->length - evt->lpos);
4498 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4499
4500 if (amount < count)
4501 memcpy(evt->cache, evt->buf, count - amount);
4502
4503 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
4504
4505 return IRQ_WAKE_THREAD;
4506 }
4507
dwc3_interrupt(int irq,void * _evt)4508 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4509 {
4510 struct dwc3_event_buffer *evt = _evt;
4511
4512 return dwc3_check_event_buf(evt);
4513 }
4514
dwc3_gadget_get_irq(struct dwc3 * dwc)4515 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4516 {
4517 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4518 int irq;
4519
4520 irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
4521 if (irq > 0)
4522 goto out;
4523
4524 if (irq == -EPROBE_DEFER)
4525 goto out;
4526
4527 irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
4528 if (irq > 0)
4529 goto out;
4530
4531 if (irq == -EPROBE_DEFER)
4532 goto out;
4533
4534 irq = platform_get_irq(dwc3_pdev, 0);
4535
4536 out:
4537 return irq;
4538 }
4539
dwc_gadget_release(struct device * dev)4540 static void dwc_gadget_release(struct device *dev)
4541 {
4542 struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4543
4544 kfree(gadget);
4545 }
4546
4547 /**
4548 * dwc3_gadget_init - initializes gadget related registers
4549 * @dwc: pointer to our controller context structure
4550 *
4551 * Returns 0 on success otherwise negative errno.
4552 */
dwc3_gadget_init(struct dwc3 * dwc)4553 int dwc3_gadget_init(struct dwc3 *dwc)
4554 {
4555 int ret;
4556 int irq;
4557 struct device *dev;
4558
4559 irq = dwc3_gadget_get_irq(dwc);
4560 if (irq < 0) {
4561 ret = irq;
4562 goto err0;
4563 }
4564
4565 dwc->irq_gadget = irq;
4566
4567 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
4568 sizeof(*dwc->ep0_trb) * 2,
4569 &dwc->ep0_trb_addr, GFP_KERNEL);
4570 if (!dwc->ep0_trb) {
4571 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4572 ret = -ENOMEM;
4573 goto err0;
4574 }
4575
4576 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4577 if (!dwc->setup_buf) {
4578 ret = -ENOMEM;
4579 goto err1;
4580 }
4581
4582 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
4583 &dwc->bounce_addr, GFP_KERNEL);
4584 if (!dwc->bounce) {
4585 ret = -ENOMEM;
4586 goto err2;
4587 }
4588
4589 init_completion(&dwc->ep0_in_setup);
4590 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
4591 if (!dwc->gadget) {
4592 ret = -ENOMEM;
4593 goto err3;
4594 }
4595
4596
4597 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
4598 dev = &dwc->gadget->dev;
4599 dev->platform_data = dwc;
4600 dwc->gadget->ops = &dwc3_gadget_ops;
4601 dwc->gadget->speed = USB_SPEED_UNKNOWN;
4602 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4603 dwc->gadget->sg_supported = true;
4604 dwc->gadget->name = "dwc3-gadget";
4605 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable;
4606 dwc->gadget->wakeup_capable = true;
4607
4608 /*
4609 * FIXME We might be setting max_speed to <SUPER, however versions
4610 * <2.20a of dwc3 have an issue with metastability (documented
4611 * elsewhere in this driver) which tells us we can't set max speed to
4612 * anything lower than SUPER.
4613 *
4614 * Because gadget.max_speed is only used by composite.c and function
4615 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
4616 * to happen so we avoid sending SuperSpeed Capability descriptor
4617 * together with our BOS descriptor as that could confuse host into
4618 * thinking we can handle super speed.
4619 *
4620 * Note that, in fact, we won't even support GetBOS requests when speed
4621 * is less than super speed because we don't have means, yet, to tell
4622 * composite.c that we are USB 2.0 + LPM ECN.
4623 */
4624 if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4625 !dwc->dis_metastability_quirk)
4626 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4627 dwc->revision);
4628
4629 dwc->gadget->max_speed = dwc->maximum_speed;
4630 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate;
4631
4632 /*
4633 * REVISIT: Here we should clear all pending IRQs to be
4634 * sure we're starting from a well known location.
4635 */
4636
4637 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
4638 if (ret)
4639 goto err4;
4640
4641 ret = usb_add_gadget(dwc->gadget);
4642 if (ret) {
4643 dev_err(dwc->dev, "failed to add gadget\n");
4644 goto err5;
4645 }
4646
4647 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4648 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);
4649 else
4650 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
4651
4652 /* No system wakeup if no gadget driver bound */
4653 if (dwc->sys_wakeup)
4654 device_wakeup_disable(dwc->sysdev);
4655
4656 return 0;
4657
4658 err5:
4659 dwc3_gadget_free_endpoints(dwc);
4660 err4:
4661 usb_put_gadget(dwc->gadget);
4662 dwc->gadget = NULL;
4663 err3:
4664 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4665 dwc->bounce_addr);
4666
4667 err2:
4668 kfree(dwc->setup_buf);
4669
4670 err1:
4671 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4672 dwc->ep0_trb, dwc->ep0_trb_addr);
4673
4674 err0:
4675 return ret;
4676 }
4677
4678 /* -------------------------------------------------------------------------- */
4679
dwc3_gadget_exit(struct dwc3 * dwc)4680 void dwc3_gadget_exit(struct dwc3 *dwc)
4681 {
4682 if (!dwc->gadget)
4683 return;
4684
4685 dwc3_enable_susphy(dwc, false);
4686 usb_del_gadget(dwc->gadget);
4687 dwc3_gadget_free_endpoints(dwc);
4688 usb_put_gadget(dwc->gadget);
4689 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4690 dwc->bounce_addr);
4691 kfree(dwc->setup_buf);
4692 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4693 dwc->ep0_trb, dwc->ep0_trb_addr);
4694 }
4695
dwc3_gadget_suspend(struct dwc3 * dwc)4696 int dwc3_gadget_suspend(struct dwc3 *dwc)
4697 {
4698 unsigned long flags;
4699 int ret;
4700
4701 ret = dwc3_gadget_soft_disconnect(dwc);
4702 if (ret)
4703 goto err;
4704
4705 spin_lock_irqsave(&dwc->lock, flags);
4706 if (dwc->gadget_driver)
4707 dwc3_disconnect_gadget(dwc);
4708 spin_unlock_irqrestore(&dwc->lock, flags);
4709
4710 return 0;
4711
4712 err:
4713 /*
4714 * Attempt to reset the controller's state. Likely no
4715 * communication can be established until the host
4716 * performs a port reset.
4717 */
4718 if (dwc->softconnect)
4719 dwc3_gadget_soft_connect(dwc);
4720
4721 return ret;
4722 }
4723
dwc3_gadget_resume(struct dwc3 * dwc)4724 int dwc3_gadget_resume(struct dwc3 *dwc)
4725 {
4726 if (!dwc->gadget_driver || !dwc->softconnect)
4727 return 0;
4728
4729 return dwc3_gadget_soft_connect(dwc);
4730 }
4731
dwc3_gadget_process_pending_events(struct dwc3 * dwc)4732 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
4733 {
4734 if (dwc->pending_events) {
4735 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
4736 dwc3_thread_interrupt(dwc->irq_gadget, dwc->ev_buf);
4737 pm_runtime_put(dwc->dev);
4738 dwc->pending_events = false;
4739 enable_irq(dwc->irq_gadget);
4740 }
4741 }
4742