1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link 4 * 5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com 6 * 7 * Authors: Felipe Balbi <balbi@ti.com>, 8 * Sebastian Andrzej Siewior <bigeasy@linutronix.de> 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/delay.h> 13 #include <linux/slab.h> 14 #include <linux/spinlock.h> 15 #include <linux/platform_device.h> 16 #include <linux/pm_runtime.h> 17 #include <linux/interrupt.h> 18 #include <linux/io.h> 19 #include <linux/list.h> 20 #include <linux/dma-mapping.h> 21 22 #include <linux/usb/ch9.h> 23 #include <linux/usb/gadget.h> 24 25 #include "debug.h" 26 #include "core.h" 27 #include "gadget.h" 28 #include "io.h" 29 30 #define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \ 31 & ~((d)->interval - 1)) 32 33 /** 34 * dwc3_gadget_set_test_mode - enables usb2 test modes 35 * @dwc: pointer to our context structure 36 * @mode: the mode to set (J, K SE0 NAK, Force Enable) 37 * 38 * Caller should take care of locking. This function will return 0 on 39 * success or -EINVAL if wrong Test Selector is passed. 40 */ 41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode) 42 { 43 u32 reg; 44 45 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 46 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 47 48 switch (mode) { 49 case USB_TEST_J: 50 case USB_TEST_K: 51 case USB_TEST_SE0_NAK: 52 case USB_TEST_PACKET: 53 case USB_TEST_FORCE_ENABLE: 54 reg |= mode << 1; 55 break; 56 default: 57 return -EINVAL; 58 } 59 60 dwc3_gadget_dctl_write_safe(dwc, reg); 61 62 return 0; 63 } 64 65 /** 66 * dwc3_gadget_get_link_state - gets current state of usb link 67 * @dwc: pointer to our context structure 68 * 69 * Caller should take care of locking. This function will 70 * return the link state on success (>= 0) or -ETIMEDOUT. 71 */ 72 int dwc3_gadget_get_link_state(struct dwc3 *dwc) 73 { 74 u32 reg; 75 76 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 77 78 return DWC3_DSTS_USBLNKST(reg); 79 } 80 81 /** 82 * dwc3_gadget_set_link_state - sets usb link to a particular state 83 * @dwc: pointer to our context structure 84 * @state: the state to put link into 85 * 86 * Caller should take care of locking. This function will 87 * return 0 on success or -ETIMEDOUT. 88 */ 89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state) 90 { 91 int retries = 10000; 92 u32 reg; 93 94 /* 95 * Wait until device controller is ready. Only applies to 1.94a and 96 * later RTL. 97 */ 98 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) { 99 while (--retries) { 100 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 101 if (reg & DWC3_DSTS_DCNRD) 102 udelay(5); 103 else 104 break; 105 } 106 107 if (retries <= 0) 108 return -ETIMEDOUT; 109 } 110 111 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 113 114 /* set no action before sending new link state change */ 115 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 116 117 /* set requested state */ 118 reg |= DWC3_DCTL_ULSTCHNGREQ(state); 119 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 120 121 /* 122 * The following code is racy when called from dwc3_gadget_wakeup, 123 * and is not needed, at least on newer versions 124 */ 125 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) 126 return 0; 127 128 /* wait for a change in DSTS */ 129 retries = 10000; 130 while (--retries) { 131 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 132 133 if (DWC3_DSTS_USBLNKST(reg) == state) 134 return 0; 135 136 udelay(5); 137 } 138 139 return -ETIMEDOUT; 140 } 141 142 static void dwc3_ep0_reset_state(struct dwc3 *dwc) 143 { 144 unsigned int dir; 145 146 if (dwc->ep0state != EP0_SETUP_PHASE) { 147 dir = !!dwc->ep0_expect_in; 148 if (dwc->ep0state == EP0_DATA_PHASE) 149 dwc3_ep0_end_control_data(dwc, dwc->eps[dir]); 150 else 151 dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]); 152 153 dwc->eps[0]->trb_enqueue = 0; 154 dwc->eps[1]->trb_enqueue = 0; 155 156 dwc3_ep0_stall_and_restart(dwc); 157 } 158 } 159 160 /** 161 * dwc3_ep_inc_trb - increment a trb index. 162 * @index: Pointer to the TRB index to increment. 163 * 164 * The index should never point to the link TRB. After incrementing, 165 * if it is point to the link TRB, wrap around to the beginning. The 166 * link TRB is always at the last TRB entry. 167 */ 168 static void dwc3_ep_inc_trb(u8 *index) 169 { 170 (*index)++; 171 if (*index == (DWC3_TRB_NUM - 1)) 172 *index = 0; 173 } 174 175 /** 176 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer 177 * @dep: The endpoint whose enqueue pointer we're incrementing 178 */ 179 static void dwc3_ep_inc_enq(struct dwc3_ep *dep) 180 { 181 dwc3_ep_inc_trb(&dep->trb_enqueue); 182 } 183 184 /** 185 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer 186 * @dep: The endpoint whose enqueue pointer we're incrementing 187 */ 188 static void dwc3_ep_inc_deq(struct dwc3_ep *dep) 189 { 190 dwc3_ep_inc_trb(&dep->trb_dequeue); 191 } 192 193 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep, 194 struct dwc3_request *req, int status) 195 { 196 struct dwc3 *dwc = dep->dwc; 197 198 list_del(&req->list); 199 req->remaining = 0; 200 req->needs_extra_trb = false; 201 req->num_trbs = 0; 202 203 if (req->request.status == -EINPROGRESS) 204 req->request.status = status; 205 206 if (req->trb) 207 usb_gadget_unmap_request_by_dev(dwc->sysdev, 208 &req->request, req->direction); 209 210 req->trb = NULL; 211 trace_dwc3_gadget_giveback(req); 212 213 if (dep->number > 1) 214 pm_runtime_put(dwc->dev); 215 } 216 217 /** 218 * dwc3_gadget_giveback - call struct usb_request's ->complete callback 219 * @dep: The endpoint to whom the request belongs to 220 * @req: The request we're giving back 221 * @status: completion code for the request 222 * 223 * Must be called with controller's lock held and interrupts disabled. This 224 * function will unmap @req and call its ->complete() callback to notify upper 225 * layers that it has completed. 226 */ 227 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, 228 int status) 229 { 230 struct dwc3 *dwc = dep->dwc; 231 232 dwc3_gadget_del_and_unmap_request(dep, req, status); 233 req->status = DWC3_REQUEST_STATUS_COMPLETED; 234 235 spin_unlock(&dwc->lock); 236 usb_gadget_giveback_request(&dep->endpoint, &req->request); 237 spin_lock(&dwc->lock); 238 } 239 240 /** 241 * dwc3_send_gadget_generic_command - issue a generic command for the controller 242 * @dwc: pointer to the controller context 243 * @cmd: the command to be issued 244 * @param: command parameter 245 * 246 * Caller should take care of locking. Issue @cmd with a given @param to @dwc 247 * and wait for its completion. 248 */ 249 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd, 250 u32 param) 251 { 252 u32 timeout = 500; 253 int status = 0; 254 int ret = 0; 255 u32 reg; 256 257 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param); 258 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT); 259 260 do { 261 reg = dwc3_readl(dwc->regs, DWC3_DGCMD); 262 if (!(reg & DWC3_DGCMD_CMDACT)) { 263 status = DWC3_DGCMD_STATUS(reg); 264 if (status) 265 ret = -EINVAL; 266 break; 267 } 268 } while (--timeout); 269 270 if (!timeout) { 271 ret = -ETIMEDOUT; 272 status = -ETIMEDOUT; 273 } 274 275 trace_dwc3_gadget_generic_cmd(cmd, param, status); 276 277 return ret; 278 } 279 280 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async); 281 282 /** 283 * dwc3_send_gadget_ep_cmd - issue an endpoint command 284 * @dep: the endpoint to which the command is going to be issued 285 * @cmd: the command to be issued 286 * @params: parameters to the command 287 * 288 * Caller should handle locking. This function will issue @cmd with given 289 * @params to @dep and wait for its completion. 290 * 291 * According to the programming guide, if the link state is in L1/L2/U3, 292 * then sending the Start Transfer command may not complete. The 293 * programming guide suggested to bring the link state back to ON/U0 by 294 * performing remote wakeup prior to sending the command. However, don't 295 * initiate remote wakeup when the user/function does not send wakeup 296 * request via wakeup ops. Send the command when it's allowed. 297 * 298 * Notes: 299 * For L1 link state, issuing a command requires the clearing of 300 * GUSB2PHYCFG.SUSPENDUSB2, which turns on the signal required to complete 301 * the given command (usually within 50us). This should happen within the 302 * command timeout set by driver. No additional step is needed. 303 * 304 * For L2 or U3 link state, the gadget is in USB suspend. Care should be 305 * taken when sending Start Transfer command to ensure that it's done after 306 * USB resume. 307 */ 308 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, 309 struct dwc3_gadget_ep_cmd_params *params) 310 { 311 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 312 struct dwc3 *dwc = dep->dwc; 313 u32 timeout = 5000; 314 u32 saved_config = 0; 315 u32 reg; 316 317 int cmd_status = 0; 318 int ret = -EINVAL; 319 320 /* 321 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or 322 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an 323 * endpoint command. 324 * 325 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY 326 * settings. Restore them after the command is completed. 327 * 328 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2 329 */ 330 if (dwc->gadget->speed <= USB_SPEED_HIGH || 331 DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) { 332 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 333 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) { 334 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY; 335 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY; 336 } 337 338 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) { 339 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM; 340 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM; 341 } 342 343 if (saved_config) 344 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 345 } 346 347 /* 348 * For some commands such as Update Transfer command, DEPCMDPARn 349 * registers are reserved. Since the driver often sends Update Transfer 350 * command, don't write to DEPCMDPARn to avoid register write delays and 351 * improve performance. 352 */ 353 if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) { 354 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); 355 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); 356 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); 357 } 358 359 /* 360 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're 361 * not relying on XferNotReady, we can make use of a special "No 362 * Response Update Transfer" command where we should clear both CmdAct 363 * and CmdIOC bits. 364 * 365 * With this, we don't need to wait for command completion and can 366 * straight away issue further commands to the endpoint. 367 * 368 * NOTICE: We're making an assumption that control endpoints will never 369 * make use of Update Transfer command. This is a safe assumption 370 * because we can never have more than one request at a time with 371 * Control Endpoints. If anybody changes that assumption, this chunk 372 * needs to be updated accordingly. 373 */ 374 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER && 375 !usb_endpoint_xfer_isoc(desc)) 376 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT); 377 else 378 cmd |= DWC3_DEPCMD_CMDACT; 379 380 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); 381 382 if (!(cmd & DWC3_DEPCMD_CMDACT) || 383 (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER && 384 !(cmd & DWC3_DEPCMD_CMDIOC))) { 385 ret = 0; 386 goto skip_status; 387 } 388 389 do { 390 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); 391 if (!(reg & DWC3_DEPCMD_CMDACT)) { 392 cmd_status = DWC3_DEPCMD_STATUS(reg); 393 394 switch (cmd_status) { 395 case 0: 396 ret = 0; 397 break; 398 case DEPEVT_TRANSFER_NO_RESOURCE: 399 dev_WARN(dwc->dev, "No resource for %s\n", 400 dep->name); 401 ret = -EINVAL; 402 break; 403 case DEPEVT_TRANSFER_BUS_EXPIRY: 404 /* 405 * SW issues START TRANSFER command to 406 * isochronous ep with future frame interval. If 407 * future interval time has already passed when 408 * core receives the command, it will respond 409 * with an error status of 'Bus Expiry'. 410 * 411 * Instead of always returning -EINVAL, let's 412 * give a hint to the gadget driver that this is 413 * the case by returning -EAGAIN. 414 */ 415 ret = -EAGAIN; 416 break; 417 default: 418 dev_WARN(dwc->dev, "UNKNOWN cmd status\n"); 419 } 420 421 break; 422 } 423 } while (--timeout); 424 425 if (timeout == 0) { 426 ret = -ETIMEDOUT; 427 cmd_status = -ETIMEDOUT; 428 } 429 430 skip_status: 431 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status); 432 433 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) { 434 if (ret == 0) 435 dep->flags |= DWC3_EP_TRANSFER_STARTED; 436 437 if (ret != -ETIMEDOUT) 438 dwc3_gadget_ep_get_transfer_index(dep); 439 } 440 441 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER && 442 !(cmd & DWC3_DEPCMD_CMDIOC)) 443 mdelay(1); 444 445 if (saved_config) { 446 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 447 reg |= saved_config; 448 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 449 } 450 451 return ret; 452 } 453 454 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep) 455 { 456 struct dwc3 *dwc = dep->dwc; 457 struct dwc3_gadget_ep_cmd_params params; 458 u32 cmd = DWC3_DEPCMD_CLEARSTALL; 459 460 /* 461 * As of core revision 2.60a the recommended programming model 462 * is to set the ClearPendIN bit when issuing a Clear Stall EP 463 * command for IN endpoints. This is to prevent an issue where 464 * some (non-compliant) hosts may not send ACK TPs for pending 465 * IN transfers due to a mishandled error condition. Synopsys 466 * STAR 9000614252. 467 */ 468 if (dep->direction && 469 !DWC3_VER_IS_PRIOR(DWC3, 260A) && 470 (dwc->gadget->speed >= USB_SPEED_SUPER)) 471 cmd |= DWC3_DEPCMD_CLEARPENDIN; 472 473 memset(¶ms, 0, sizeof(params)); 474 475 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 476 } 477 478 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, 479 struct dwc3_trb *trb) 480 { 481 u32 offset = (char *) trb - (char *) dep->trb_pool; 482 483 return dep->trb_pool_dma + offset; 484 } 485 486 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep) 487 { 488 struct dwc3 *dwc = dep->dwc; 489 490 if (dep->trb_pool) 491 return 0; 492 493 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, 494 sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 495 &dep->trb_pool_dma, GFP_KERNEL); 496 if (!dep->trb_pool) { 497 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", 498 dep->name); 499 return -ENOMEM; 500 } 501 502 return 0; 503 } 504 505 static void dwc3_free_trb_pool(struct dwc3_ep *dep) 506 { 507 struct dwc3 *dwc = dep->dwc; 508 509 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 510 dep->trb_pool, dep->trb_pool_dma); 511 512 dep->trb_pool = NULL; 513 dep->trb_pool_dma = 0; 514 } 515 516 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep) 517 { 518 struct dwc3_gadget_ep_cmd_params params; 519 int ret; 520 521 if (dep->flags & DWC3_EP_RESOURCE_ALLOCATED) 522 return 0; 523 524 memset(¶ms, 0x00, sizeof(params)); 525 526 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1); 527 528 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE, 529 ¶ms); 530 if (ret) 531 return ret; 532 533 dep->flags |= DWC3_EP_RESOURCE_ALLOCATED; 534 return 0; 535 } 536 537 /** 538 * dwc3_gadget_start_config - reset endpoint resources 539 * @dwc: pointer to the DWC3 context 540 * @resource_index: DEPSTARTCFG.XferRscIdx value (must be 0 or 2) 541 * 542 * Set resource_index=0 to reset all endpoints' resources allocation. Do this as 543 * part of the power-on/soft-reset initialization. 544 * 545 * Set resource_index=2 to reset only non-control endpoints' resources. Do this 546 * on receiving the SET_CONFIGURATION request or hibernation resume. 547 */ 548 int dwc3_gadget_start_config(struct dwc3 *dwc, unsigned int resource_index) 549 { 550 struct dwc3_gadget_ep_cmd_params params; 551 u32 cmd; 552 int i; 553 int ret; 554 555 if (resource_index != 0 && resource_index != 2) 556 return -EINVAL; 557 558 memset(¶ms, 0x00, sizeof(params)); 559 cmd = DWC3_DEPCMD_DEPSTARTCFG; 560 cmd |= DWC3_DEPCMD_PARAM(resource_index); 561 562 ret = dwc3_send_gadget_ep_cmd(dwc->eps[0], cmd, ¶ms); 563 if (ret) 564 return ret; 565 566 /* Reset resource allocation flags */ 567 for (i = resource_index; i < dwc->num_eps && dwc->eps[i]; i++) 568 dwc->eps[i]->flags &= ~DWC3_EP_RESOURCE_ALLOCATED; 569 570 return 0; 571 } 572 573 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action) 574 { 575 const struct usb_ss_ep_comp_descriptor *comp_desc; 576 const struct usb_endpoint_descriptor *desc; 577 struct dwc3_gadget_ep_cmd_params params; 578 struct dwc3 *dwc = dep->dwc; 579 580 comp_desc = dep->endpoint.comp_desc; 581 desc = dep->endpoint.desc; 582 583 memset(¶ms, 0x00, sizeof(params)); 584 585 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc)) 586 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc)); 587 588 /* Burst size is only needed in SuperSpeed mode */ 589 if (dwc->gadget->speed >= USB_SPEED_SUPER) { 590 u32 burst = dep->endpoint.maxburst; 591 592 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); 593 } 594 595 params.param0 |= action; 596 if (action == DWC3_DEPCFG_ACTION_RESTORE) 597 params.param2 |= dep->saved_state; 598 599 if (usb_endpoint_xfer_control(desc)) 600 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN; 601 602 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) 603 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN; 604 605 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) { 606 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE 607 | DWC3_DEPCFG_XFER_COMPLETE_EN 608 | DWC3_DEPCFG_STREAM_EVENT_EN; 609 dep->stream_capable = true; 610 } 611 612 if (!usb_endpoint_xfer_control(desc)) 613 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN; 614 615 /* 616 * We are doing 1:1 mapping for endpoints, meaning 617 * Physical Endpoints 2 maps to Logical Endpoint 2 and 618 * so on. We consider the direction bit as part of the physical 619 * endpoint number. So USB endpoint 0x81 is 0x03. 620 */ 621 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); 622 623 /* 624 * We must use the lower 16 TX FIFOs even though 625 * HW might have more 626 */ 627 if (dep->direction) 628 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); 629 630 if (desc->bInterval) { 631 u8 bInterval_m1; 632 633 /* 634 * Valid range for DEPCFG.bInterval_m1 is from 0 to 13. 635 * 636 * NOTE: The programming guide incorrectly stated bInterval_m1 637 * must be set to 0 when operating in fullspeed. Internally the 638 * controller does not have this limitation. See DWC_usb3x 639 * programming guide section 3.2.2.1. 640 */ 641 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13); 642 643 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT && 644 dwc->gadget->speed == USB_SPEED_FULL) 645 dep->interval = desc->bInterval; 646 else 647 dep->interval = 1 << (desc->bInterval - 1); 648 649 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1); 650 } 651 652 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms); 653 } 654 655 /** 656 * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value 657 * @dwc: pointer to the DWC3 context 658 * @mult: multiplier to be used when calculating the fifo_size 659 * 660 * Calculates the size value based on the equation below: 661 * 662 * DWC3 revision 280A and prior: 663 * fifo_size = mult * (max_packet / mdwidth) + 1; 664 * 665 * DWC3 revision 290A and onwards: 666 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 667 * 668 * The max packet size is set to 1024, as the txfifo requirements mainly apply 669 * to super speed USB use cases. However, it is safe to overestimate the fifo 670 * allocations for other scenarios, i.e. high speed USB. 671 */ 672 static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult) 673 { 674 int max_packet = 1024; 675 int fifo_size; 676 int mdwidth; 677 678 mdwidth = dwc3_mdwidth(dwc); 679 680 /* MDWIDTH is represented in bits, we need it in bytes */ 681 mdwidth >>= 3; 682 683 if (DWC3_VER_IS_PRIOR(DWC3, 290A)) 684 fifo_size = mult * (max_packet / mdwidth) + 1; 685 else 686 fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1; 687 return fifo_size; 688 } 689 690 /** 691 * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation 692 * @dwc: pointer to the DWC3 context 693 * 694 * Iterates through all the endpoint registers and clears the previous txfifo 695 * allocations. 696 */ 697 void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc) 698 { 699 struct dwc3_ep *dep; 700 int fifo_depth; 701 int size; 702 int num; 703 704 if (!dwc->do_fifo_resize) 705 return; 706 707 /* Read ep0IN related TXFIFO size */ 708 dep = dwc->eps[1]; 709 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); 710 if (DWC3_IP_IS(DWC3)) 711 fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size); 712 else 713 fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size); 714 715 dwc->last_fifo_depth = fifo_depth; 716 /* Clear existing TXFIFO for all IN eps except ep0 */ 717 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM); 718 num += 2) { 719 dep = dwc->eps[num]; 720 /* Don't change TXFRAMNUM on usb31 version */ 721 size = DWC3_IP_IS(DWC3) ? 0 : 722 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) & 723 DWC31_GTXFIFOSIZ_TXFRAMNUM; 724 725 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size); 726 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED; 727 } 728 dwc->num_ep_resized = 0; 729 } 730 731 /* 732 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case 733 * @dwc: pointer to our context structure 734 * 735 * This function will a best effort FIFO allocation in order 736 * to improve FIFO usage and throughput, while still allowing 737 * us to enable as many endpoints as possible. 738 * 739 * Keep in mind that this operation will be highly dependent 740 * on the configured size for RAM1 - which contains TxFifo -, 741 * the amount of endpoints enabled on coreConsultant tool, and 742 * the width of the Master Bus. 743 * 744 * In general, FIFO depths are represented with the following equation: 745 * 746 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 747 * 748 * In conjunction with dwc3_gadget_check_config(), this resizing logic will 749 * ensure that all endpoints will have enough internal memory for one max 750 * packet per endpoint. 751 */ 752 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep) 753 { 754 struct dwc3 *dwc = dep->dwc; 755 int fifo_0_start; 756 int ram1_depth; 757 int fifo_size; 758 int min_depth; 759 int num_in_ep; 760 int remaining; 761 int num_fifos = 1; 762 int fifo; 763 int tmp; 764 765 if (!dwc->do_fifo_resize) 766 return 0; 767 768 /* resize IN endpoints except ep0 */ 769 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1) 770 return 0; 771 772 /* bail if already resized */ 773 if (dep->flags & DWC3_EP_TXFIFO_RESIZED) 774 return 0; 775 776 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); 777 778 if ((dep->endpoint.maxburst > 1 && 779 usb_endpoint_xfer_bulk(dep->endpoint.desc)) || 780 usb_endpoint_xfer_isoc(dep->endpoint.desc)) 781 num_fifos = 3; 782 783 if (dep->endpoint.maxburst > 6 && 784 (usb_endpoint_xfer_bulk(dep->endpoint.desc) || 785 usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31)) 786 num_fifos = dwc->tx_fifo_resize_max_num; 787 788 /* FIFO size for a single buffer */ 789 fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1); 790 791 /* Calculate the number of remaining EPs w/o any FIFO */ 792 num_in_ep = dwc->max_cfg_eps; 793 num_in_ep -= dwc->num_ep_resized; 794 795 /* Reserve at least one FIFO for the number of IN EPs */ 796 min_depth = num_in_ep * (fifo + 1); 797 remaining = ram1_depth - min_depth - dwc->last_fifo_depth; 798 remaining = max_t(int, 0, remaining); 799 /* 800 * We've already reserved 1 FIFO per EP, so check what we can fit in 801 * addition to it. If there is not enough remaining space, allocate 802 * all the remaining space to the EP. 803 */ 804 fifo_size = (num_fifos - 1) * fifo; 805 if (remaining < fifo_size) 806 fifo_size = remaining; 807 808 fifo_size += fifo; 809 /* Last increment according to the TX FIFO size equation */ 810 fifo_size++; 811 812 /* Check if TXFIFOs start at non-zero addr */ 813 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); 814 fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp); 815 816 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16)); 817 if (DWC3_IP_IS(DWC3)) 818 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); 819 else 820 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); 821 822 /* Check fifo size allocation doesn't exceed available RAM size. */ 823 if (dwc->last_fifo_depth >= ram1_depth) { 824 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n", 825 dwc->last_fifo_depth, ram1_depth, 826 dep->endpoint.name, fifo_size); 827 if (DWC3_IP_IS(DWC3)) 828 fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); 829 else 830 fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); 831 832 dwc->last_fifo_depth -= fifo_size; 833 return -ENOMEM; 834 } 835 836 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); 837 dep->flags |= DWC3_EP_TXFIFO_RESIZED; 838 dwc->num_ep_resized++; 839 840 return 0; 841 } 842 843 /** 844 * __dwc3_gadget_ep_enable - initializes a hw endpoint 845 * @dep: endpoint to be initialized 846 * @action: one of INIT, MODIFY or RESTORE 847 * 848 * Caller should take care of locking. Execute all necessary commands to 849 * initialize a HW endpoint so it can be used by a gadget driver. 850 */ 851 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) 852 { 853 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 854 struct dwc3 *dwc = dep->dwc; 855 856 u32 reg; 857 int ret; 858 859 if (!(dep->flags & DWC3_EP_ENABLED)) { 860 ret = dwc3_gadget_resize_tx_fifos(dep); 861 if (ret) 862 return ret; 863 } 864 865 ret = dwc3_gadget_set_ep_config(dep, action); 866 if (ret) 867 return ret; 868 869 if (!(dep->flags & DWC3_EP_RESOURCE_ALLOCATED)) { 870 ret = dwc3_gadget_set_xfer_resource(dep); 871 if (ret) 872 return ret; 873 } 874 875 if (!(dep->flags & DWC3_EP_ENABLED)) { 876 struct dwc3_trb *trb_st_hw; 877 struct dwc3_trb *trb_link; 878 879 dep->type = usb_endpoint_type(desc); 880 dep->flags |= DWC3_EP_ENABLED; 881 882 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 883 reg |= DWC3_DALEPENA_EP(dep->number); 884 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 885 886 dep->trb_dequeue = 0; 887 dep->trb_enqueue = 0; 888 889 if (usb_endpoint_xfer_control(desc)) 890 goto out; 891 892 /* Initialize the TRB ring */ 893 memset(dep->trb_pool, 0, 894 sizeof(struct dwc3_trb) * DWC3_TRB_NUM); 895 896 /* Link TRB. The HWO bit is never reset */ 897 trb_st_hw = &dep->trb_pool[0]; 898 899 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; 900 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 901 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 902 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB; 903 trb_link->ctrl |= DWC3_TRB_CTRL_HWO; 904 } 905 906 /* 907 * Issue StartTransfer here with no-op TRB so we can always rely on No 908 * Response Update Transfer command. 909 */ 910 if (usb_endpoint_xfer_bulk(desc) || 911 usb_endpoint_xfer_int(desc)) { 912 struct dwc3_gadget_ep_cmd_params params; 913 struct dwc3_trb *trb; 914 dma_addr_t trb_dma; 915 u32 cmd; 916 917 memset(¶ms, 0, sizeof(params)); 918 trb = &dep->trb_pool[0]; 919 trb_dma = dwc3_trb_dma_offset(dep, trb); 920 921 params.param0 = upper_32_bits(trb_dma); 922 params.param1 = lower_32_bits(trb_dma); 923 924 cmd = DWC3_DEPCMD_STARTTRANSFER; 925 926 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 927 if (ret < 0) 928 return ret; 929 930 if (dep->stream_capable) { 931 /* 932 * For streams, at start, there maybe a race where the 933 * host primes the endpoint before the function driver 934 * queues a request to initiate a stream. In that case, 935 * the controller will not see the prime to generate the 936 * ERDY and start stream. To workaround this, issue a 937 * no-op TRB as normal, but end it immediately. As a 938 * result, when the function driver queues the request, 939 * the next START_TRANSFER command will cause the 940 * controller to generate an ERDY to initiate the 941 * stream. 942 */ 943 dwc3_stop_active_transfer(dep, true, true); 944 945 /* 946 * All stream eps will reinitiate stream on NoStream 947 * rejection until we can determine that the host can 948 * prime after the first transfer. 949 * 950 * However, if the controller is capable of 951 * TXF_FLUSH_BYPASS, then IN direction endpoints will 952 * automatically restart the stream without the driver 953 * initiation. 954 */ 955 if (!dep->direction || 956 !(dwc->hwparams.hwparams9 & 957 DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS)) 958 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM; 959 } 960 } 961 962 out: 963 trace_dwc3_gadget_ep_enable(dep); 964 965 return 0; 966 } 967 968 void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status) 969 { 970 struct dwc3_request *req; 971 972 dwc3_stop_active_transfer(dep, true, false); 973 974 /* If endxfer is delayed, avoid unmapping requests */ 975 if (dep->flags & DWC3_EP_DELAY_STOP) 976 return; 977 978 /* - giveback all requests to gadget driver */ 979 while (!list_empty(&dep->started_list)) { 980 req = next_request(&dep->started_list); 981 982 dwc3_gadget_giveback(dep, req, status); 983 } 984 985 while (!list_empty(&dep->pending_list)) { 986 req = next_request(&dep->pending_list); 987 988 dwc3_gadget_giveback(dep, req, status); 989 } 990 991 while (!list_empty(&dep->cancelled_list)) { 992 req = next_request(&dep->cancelled_list); 993 994 dwc3_gadget_giveback(dep, req, status); 995 } 996 } 997 998 /** 999 * __dwc3_gadget_ep_disable - disables a hw endpoint 1000 * @dep: the endpoint to disable 1001 * 1002 * This function undoes what __dwc3_gadget_ep_enable did and also removes 1003 * requests which are currently being processed by the hardware and those which 1004 * are not yet scheduled. 1005 * 1006 * Caller should take care of locking. 1007 */ 1008 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) 1009 { 1010 struct dwc3 *dwc = dep->dwc; 1011 u32 reg; 1012 u32 mask; 1013 1014 trace_dwc3_gadget_ep_disable(dep); 1015 1016 /* make sure HW endpoint isn't stalled */ 1017 if (dep->flags & DWC3_EP_STALL) 1018 __dwc3_gadget_ep_set_halt(dep, 0, false); 1019 1020 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 1021 reg &= ~DWC3_DALEPENA_EP(dep->number); 1022 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 1023 1024 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); 1025 1026 dep->stream_capable = false; 1027 dep->type = 0; 1028 mask = DWC3_EP_TXFIFO_RESIZED | DWC3_EP_RESOURCE_ALLOCATED; 1029 /* 1030 * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is 1031 * set. Do not clear DEP flags, so that the end transfer command will 1032 * be reattempted during the next SETUP stage. 1033 */ 1034 if (dep->flags & DWC3_EP_DELAY_STOP) 1035 mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED); 1036 dep->flags &= mask; 1037 1038 /* Clear out the ep descriptors for non-ep0 */ 1039 if (dep->number > 1) { 1040 dep->endpoint.comp_desc = NULL; 1041 dep->endpoint.desc = NULL; 1042 } 1043 1044 return 0; 1045 } 1046 1047 /* -------------------------------------------------------------------------- */ 1048 1049 static int dwc3_gadget_ep0_enable(struct usb_ep *ep, 1050 const struct usb_endpoint_descriptor *desc) 1051 { 1052 return -EINVAL; 1053 } 1054 1055 static int dwc3_gadget_ep0_disable(struct usb_ep *ep) 1056 { 1057 return -EINVAL; 1058 } 1059 1060 /* -------------------------------------------------------------------------- */ 1061 1062 static int dwc3_gadget_ep_enable(struct usb_ep *ep, 1063 const struct usb_endpoint_descriptor *desc) 1064 { 1065 struct dwc3_ep *dep; 1066 struct dwc3 *dwc; 1067 unsigned long flags; 1068 int ret; 1069 1070 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { 1071 pr_debug("dwc3: invalid parameters\n"); 1072 return -EINVAL; 1073 } 1074 1075 if (!desc->wMaxPacketSize) { 1076 pr_debug("dwc3: missing wMaxPacketSize\n"); 1077 return -EINVAL; 1078 } 1079 1080 dep = to_dwc3_ep(ep); 1081 dwc = dep->dwc; 1082 1083 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, 1084 "%s is already enabled\n", 1085 dep->name)) 1086 return 0; 1087 1088 spin_lock_irqsave(&dwc->lock, flags); 1089 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 1090 spin_unlock_irqrestore(&dwc->lock, flags); 1091 1092 return ret; 1093 } 1094 1095 static int dwc3_gadget_ep_disable(struct usb_ep *ep) 1096 { 1097 struct dwc3_ep *dep; 1098 struct dwc3 *dwc; 1099 unsigned long flags; 1100 int ret; 1101 1102 if (!ep) { 1103 pr_debug("dwc3: invalid parameters\n"); 1104 return -EINVAL; 1105 } 1106 1107 dep = to_dwc3_ep(ep); 1108 dwc = dep->dwc; 1109 1110 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), 1111 "%s is already disabled\n", 1112 dep->name)) 1113 return 0; 1114 1115 spin_lock_irqsave(&dwc->lock, flags); 1116 ret = __dwc3_gadget_ep_disable(dep); 1117 spin_unlock_irqrestore(&dwc->lock, flags); 1118 1119 return ret; 1120 } 1121 1122 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep, 1123 gfp_t gfp_flags) 1124 { 1125 struct dwc3_request *req; 1126 struct dwc3_ep *dep = to_dwc3_ep(ep); 1127 1128 req = kzalloc(sizeof(*req), gfp_flags); 1129 if (!req) 1130 return NULL; 1131 1132 req->direction = dep->direction; 1133 req->epnum = dep->number; 1134 req->dep = dep; 1135 req->status = DWC3_REQUEST_STATUS_UNKNOWN; 1136 1137 trace_dwc3_alloc_request(req); 1138 1139 return &req->request; 1140 } 1141 1142 static void dwc3_gadget_ep_free_request(struct usb_ep *ep, 1143 struct usb_request *request) 1144 { 1145 struct dwc3_request *req = to_dwc3_request(request); 1146 1147 trace_dwc3_free_request(req); 1148 kfree(req); 1149 } 1150 1151 /** 1152 * dwc3_ep_prev_trb - returns the previous TRB in the ring 1153 * @dep: The endpoint with the TRB ring 1154 * @index: The index of the current TRB in the ring 1155 * 1156 * Returns the TRB prior to the one pointed to by the index. If the 1157 * index is 0, we will wrap backwards, skip the link TRB, and return 1158 * the one just before that. 1159 */ 1160 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index) 1161 { 1162 u8 tmp = index; 1163 1164 if (!tmp) 1165 tmp = DWC3_TRB_NUM - 1; 1166 1167 return &dep->trb_pool[tmp - 1]; 1168 } 1169 1170 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep) 1171 { 1172 u8 trbs_left; 1173 1174 /* 1175 * If the enqueue & dequeue are equal then the TRB ring is either full 1176 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs 1177 * pending to be processed by the driver. 1178 */ 1179 if (dep->trb_enqueue == dep->trb_dequeue) { 1180 /* 1181 * If there is any request remained in the started_list at 1182 * this point, that means there is no TRB available. 1183 */ 1184 if (!list_empty(&dep->started_list)) 1185 return 0; 1186 1187 return DWC3_TRB_NUM - 1; 1188 } 1189 1190 trbs_left = dep->trb_dequeue - dep->trb_enqueue; 1191 trbs_left &= (DWC3_TRB_NUM - 1); 1192 1193 if (dep->trb_dequeue < dep->trb_enqueue) 1194 trbs_left--; 1195 1196 return trbs_left; 1197 } 1198 1199 /** 1200 * dwc3_prepare_one_trb - setup one TRB from one request 1201 * @dep: endpoint for which this request is prepared 1202 * @req: dwc3_request pointer 1203 * @trb_length: buffer size of the TRB 1204 * @chain: should this TRB be chained to the next? 1205 * @node: only for isochronous endpoints. First TRB needs different type. 1206 * @use_bounce_buffer: set to use bounce buffer 1207 * @must_interrupt: set to interrupt on TRB completion 1208 */ 1209 static void dwc3_prepare_one_trb(struct dwc3_ep *dep, 1210 struct dwc3_request *req, unsigned int trb_length, 1211 unsigned int chain, unsigned int node, bool use_bounce_buffer, 1212 bool must_interrupt) 1213 { 1214 struct dwc3_trb *trb; 1215 dma_addr_t dma; 1216 unsigned int stream_id = req->request.stream_id; 1217 unsigned int short_not_ok = req->request.short_not_ok; 1218 unsigned int no_interrupt = req->request.no_interrupt; 1219 unsigned int is_last = req->request.is_last; 1220 struct dwc3 *dwc = dep->dwc; 1221 struct usb_gadget *gadget = dwc->gadget; 1222 enum usb_device_speed speed = gadget->speed; 1223 1224 if (use_bounce_buffer) 1225 dma = dep->dwc->bounce_addr; 1226 else if (req->request.num_sgs > 0) 1227 dma = sg_dma_address(req->start_sg); 1228 else 1229 dma = req->request.dma; 1230 1231 trb = &dep->trb_pool[dep->trb_enqueue]; 1232 1233 if (!req->trb) { 1234 dwc3_gadget_move_started_request(req); 1235 req->trb = trb; 1236 req->trb_dma = dwc3_trb_dma_offset(dep, trb); 1237 } 1238 1239 req->num_trbs++; 1240 1241 trb->size = DWC3_TRB_SIZE_LENGTH(trb_length); 1242 trb->bpl = lower_32_bits(dma); 1243 trb->bph = upper_32_bits(dma); 1244 1245 switch (usb_endpoint_type(dep->endpoint.desc)) { 1246 case USB_ENDPOINT_XFER_CONTROL: 1247 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP; 1248 break; 1249 1250 case USB_ENDPOINT_XFER_ISOC: 1251 if (!node) { 1252 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; 1253 1254 /* 1255 * USB Specification 2.0 Section 5.9.2 states that: "If 1256 * there is only a single transaction in the microframe, 1257 * only a DATA0 data packet PID is used. If there are 1258 * two transactions per microframe, DATA1 is used for 1259 * the first transaction data packet and DATA0 is used 1260 * for the second transaction data packet. If there are 1261 * three transactions per microframe, DATA2 is used for 1262 * the first transaction data packet, DATA1 is used for 1263 * the second, and DATA0 is used for the third." 1264 * 1265 * IOW, we should satisfy the following cases: 1266 * 1267 * 1) length <= maxpacket 1268 * - DATA0 1269 * 1270 * 2) maxpacket < length <= (2 * maxpacket) 1271 * - DATA1, DATA0 1272 * 1273 * 3) (2 * maxpacket) < length <= (3 * maxpacket) 1274 * - DATA2, DATA1, DATA0 1275 */ 1276 if (speed == USB_SPEED_HIGH) { 1277 struct usb_ep *ep = &dep->endpoint; 1278 unsigned int mult = 2; 1279 unsigned int maxp = usb_endpoint_maxp(ep->desc); 1280 1281 if (req->request.length <= (2 * maxp)) 1282 mult--; 1283 1284 if (req->request.length <= maxp) 1285 mult--; 1286 1287 trb->size |= DWC3_TRB_SIZE_PCM1(mult); 1288 } 1289 } else { 1290 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; 1291 } 1292 1293 if (!no_interrupt && !chain) 1294 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 1295 break; 1296 1297 case USB_ENDPOINT_XFER_BULK: 1298 case USB_ENDPOINT_XFER_INT: 1299 trb->ctrl = DWC3_TRBCTL_NORMAL; 1300 break; 1301 default: 1302 /* 1303 * This is only possible with faulty memory because we 1304 * checked it already :) 1305 */ 1306 dev_WARN(dwc->dev, "Unknown endpoint type %d\n", 1307 usb_endpoint_type(dep->endpoint.desc)); 1308 } 1309 1310 /* 1311 * Enable Continue on Short Packet 1312 * when endpoint is not a stream capable 1313 */ 1314 if (usb_endpoint_dir_out(dep->endpoint.desc)) { 1315 if (!dep->stream_capable) 1316 trb->ctrl |= DWC3_TRB_CTRL_CSP; 1317 1318 if (short_not_ok) 1319 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 1320 } 1321 1322 /* All TRBs setup for MST must set CSP=1 when LST=0 */ 1323 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams)) 1324 trb->ctrl |= DWC3_TRB_CTRL_CSP; 1325 1326 if ((!no_interrupt && !chain) || must_interrupt) 1327 trb->ctrl |= DWC3_TRB_CTRL_IOC; 1328 1329 if (chain) 1330 trb->ctrl |= DWC3_TRB_CTRL_CHN; 1331 else if (dep->stream_capable && is_last && 1332 !DWC3_MST_CAPABLE(&dwc->hwparams)) 1333 trb->ctrl |= DWC3_TRB_CTRL_LST; 1334 1335 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) 1336 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id); 1337 1338 /* 1339 * As per data book 4.2.3.2TRB Control Bit Rules section 1340 * 1341 * The controller autonomously checks the HWO field of a TRB to determine if the 1342 * entire TRB is valid. Therefore, software must ensure that the rest of the TRB 1343 * is valid before setting the HWO field to '1'. In most systems, this means that 1344 * software must update the fourth DWORD of a TRB last. 1345 * 1346 * However there is a possibility of CPU re-ordering here which can cause 1347 * controller to observe the HWO bit set prematurely. 1348 * Add a write memory barrier to prevent CPU re-ordering. 1349 */ 1350 wmb(); 1351 trb->ctrl |= DWC3_TRB_CTRL_HWO; 1352 1353 dwc3_ep_inc_enq(dep); 1354 1355 trace_dwc3_prepare_trb(dep, trb); 1356 } 1357 1358 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req) 1359 { 1360 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); 1361 unsigned int rem = req->request.length % maxp; 1362 1363 if ((req->request.length && req->request.zero && !rem && 1364 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) || 1365 (!req->direction && rem)) 1366 return true; 1367 1368 return false; 1369 } 1370 1371 /** 1372 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry 1373 * @dep: The endpoint that the request belongs to 1374 * @req: The request to prepare 1375 * @entry_length: The last SG entry size 1376 * @node: Indicates whether this is not the first entry (for isoc only) 1377 * 1378 * Return the number of TRBs prepared. 1379 */ 1380 static int dwc3_prepare_last_sg(struct dwc3_ep *dep, 1381 struct dwc3_request *req, unsigned int entry_length, 1382 unsigned int node) 1383 { 1384 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); 1385 unsigned int rem = req->request.length % maxp; 1386 unsigned int num_trbs = 1; 1387 1388 if (dwc3_needs_extra_trb(dep, req)) 1389 num_trbs++; 1390 1391 if (dwc3_calc_trbs_left(dep) < num_trbs) 1392 return 0; 1393 1394 req->needs_extra_trb = num_trbs > 1; 1395 1396 /* Prepare a normal TRB */ 1397 if (req->direction || req->request.length) 1398 dwc3_prepare_one_trb(dep, req, entry_length, 1399 req->needs_extra_trb, node, false, false); 1400 1401 /* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */ 1402 if ((!req->direction && !req->request.length) || req->needs_extra_trb) 1403 dwc3_prepare_one_trb(dep, req, 1404 req->direction ? 0 : maxp - rem, 1405 false, 1, true, false); 1406 1407 return num_trbs; 1408 } 1409 1410 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep, 1411 struct dwc3_request *req) 1412 { 1413 struct scatterlist *sg = req->start_sg; 1414 struct scatterlist *s; 1415 int i; 1416 unsigned int length = req->request.length; 1417 unsigned int remaining = req->request.num_mapped_sgs 1418 - req->num_queued_sgs; 1419 unsigned int num_trbs = req->num_trbs; 1420 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req); 1421 1422 /* 1423 * If we resume preparing the request, then get the remaining length of 1424 * the request and resume where we left off. 1425 */ 1426 for_each_sg(req->request.sg, s, req->num_queued_sgs, i) 1427 length -= sg_dma_len(s); 1428 1429 for_each_sg(sg, s, remaining, i) { 1430 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep); 1431 unsigned int trb_length; 1432 bool must_interrupt = false; 1433 bool last_sg = false; 1434 1435 trb_length = min_t(unsigned int, length, sg_dma_len(s)); 1436 1437 length -= trb_length; 1438 1439 /* 1440 * IOMMU driver is coalescing the list of sgs which shares a 1441 * page boundary into one and giving it to USB driver. With 1442 * this the number of sgs mapped is not equal to the number of 1443 * sgs passed. So mark the chain bit to false if it isthe last 1444 * mapped sg. 1445 */ 1446 if ((i == remaining - 1) || !length) 1447 last_sg = true; 1448 1449 if (!num_trbs_left) 1450 break; 1451 1452 if (last_sg) { 1453 if (!dwc3_prepare_last_sg(dep, req, trb_length, i)) 1454 break; 1455 } else { 1456 /* 1457 * Look ahead to check if we have enough TRBs for the 1458 * next SG entry. If not, set interrupt on this TRB to 1459 * resume preparing the next SG entry when more TRBs are 1460 * free. 1461 */ 1462 if (num_trbs_left == 1 || (needs_extra_trb && 1463 num_trbs_left <= 2 && 1464 sg_dma_len(sg_next(s)) >= length)) { 1465 struct dwc3_request *r; 1466 1467 /* Check if previous requests already set IOC */ 1468 list_for_each_entry(r, &dep->started_list, list) { 1469 if (r != req && !r->request.no_interrupt) 1470 break; 1471 1472 if (r == req) 1473 must_interrupt = true; 1474 } 1475 } 1476 1477 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false, 1478 must_interrupt); 1479 } 1480 1481 /* 1482 * There can be a situation where all sgs in sglist are not 1483 * queued because of insufficient trb number. To handle this 1484 * case, update start_sg to next sg to be queued, so that 1485 * we have free trbs we can continue queuing from where we 1486 * previously stopped 1487 */ 1488 if (!last_sg) 1489 req->start_sg = sg_next(s); 1490 1491 req->num_queued_sgs++; 1492 req->num_pending_sgs--; 1493 1494 /* 1495 * The number of pending SG entries may not correspond to the 1496 * number of mapped SG entries. If all the data are queued, then 1497 * don't include unused SG entries. 1498 */ 1499 if (length == 0) { 1500 req->num_pending_sgs = 0; 1501 break; 1502 } 1503 1504 if (must_interrupt) 1505 break; 1506 } 1507 1508 return req->num_trbs - num_trbs; 1509 } 1510 1511 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep, 1512 struct dwc3_request *req) 1513 { 1514 return dwc3_prepare_last_sg(dep, req, req->request.length, 0); 1515 } 1516 1517 /* 1518 * dwc3_prepare_trbs - setup TRBs from requests 1519 * @dep: endpoint for which requests are being prepared 1520 * 1521 * The function goes through the requests list and sets up TRBs for the 1522 * transfers. The function returns once there are no more TRBs available or 1523 * it runs out of requests. 1524 * 1525 * Returns the number of TRBs prepared or negative errno. 1526 */ 1527 static int dwc3_prepare_trbs(struct dwc3_ep *dep) 1528 { 1529 struct dwc3_request *req, *n; 1530 int ret = 0; 1531 1532 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM); 1533 1534 /* 1535 * We can get in a situation where there's a request in the started list 1536 * but there weren't enough TRBs to fully kick it in the first time 1537 * around, so it has been waiting for more TRBs to be freed up. 1538 * 1539 * In that case, we should check if we have a request with pending_sgs 1540 * in the started list and prepare TRBs for that request first, 1541 * otherwise we will prepare TRBs completely out of order and that will 1542 * break things. 1543 */ 1544 list_for_each_entry(req, &dep->started_list, list) { 1545 if (req->num_pending_sgs > 0) { 1546 ret = dwc3_prepare_trbs_sg(dep, req); 1547 if (!ret || req->num_pending_sgs) 1548 return ret; 1549 } 1550 1551 if (!dwc3_calc_trbs_left(dep)) 1552 return ret; 1553 1554 /* 1555 * Don't prepare beyond a transfer. In DWC_usb32, its transfer 1556 * burst capability may try to read and use TRBs beyond the 1557 * active transfer instead of stopping. 1558 */ 1559 if (dep->stream_capable && req->request.is_last && 1560 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) 1561 return ret; 1562 } 1563 1564 list_for_each_entry_safe(req, n, &dep->pending_list, list) { 1565 struct dwc3 *dwc = dep->dwc; 1566 1567 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request, 1568 dep->direction); 1569 if (ret) 1570 return ret; 1571 1572 req->sg = req->request.sg; 1573 req->start_sg = req->sg; 1574 req->num_queued_sgs = 0; 1575 req->num_pending_sgs = req->request.num_mapped_sgs; 1576 1577 if (req->num_pending_sgs > 0) { 1578 ret = dwc3_prepare_trbs_sg(dep, req); 1579 if (req->num_pending_sgs) 1580 return ret; 1581 } else { 1582 ret = dwc3_prepare_trbs_linear(dep, req); 1583 } 1584 1585 if (!ret || !dwc3_calc_trbs_left(dep)) 1586 return ret; 1587 1588 /* 1589 * Don't prepare beyond a transfer. In DWC_usb32, its transfer 1590 * burst capability may try to read and use TRBs beyond the 1591 * active transfer instead of stopping. 1592 */ 1593 if (dep->stream_capable && req->request.is_last && 1594 !DWC3_MST_CAPABLE(&dwc->hwparams)) 1595 return ret; 1596 } 1597 1598 return ret; 1599 } 1600 1601 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep); 1602 1603 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) 1604 { 1605 struct dwc3_gadget_ep_cmd_params params; 1606 struct dwc3_request *req; 1607 int starting; 1608 int ret; 1609 u32 cmd; 1610 1611 /* 1612 * Note that it's normal to have no new TRBs prepared (i.e. ret == 0). 1613 * This happens when we need to stop and restart a transfer such as in 1614 * the case of reinitiating a stream or retrying an isoc transfer. 1615 */ 1616 ret = dwc3_prepare_trbs(dep); 1617 if (ret < 0) 1618 return ret; 1619 1620 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); 1621 1622 /* 1623 * If there's no new TRB prepared and we don't need to restart a 1624 * transfer, there's no need to update the transfer. 1625 */ 1626 if (!ret && !starting) 1627 return ret; 1628 1629 req = next_request(&dep->started_list); 1630 if (!req) { 1631 dep->flags |= DWC3_EP_PENDING_REQUEST; 1632 return 0; 1633 } 1634 1635 memset(¶ms, 0, sizeof(params)); 1636 1637 if (starting) { 1638 params.param0 = upper_32_bits(req->trb_dma); 1639 params.param1 = lower_32_bits(req->trb_dma); 1640 cmd = DWC3_DEPCMD_STARTTRANSFER; 1641 1642 if (dep->stream_capable) 1643 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id); 1644 1645 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) 1646 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); 1647 } else { 1648 cmd = DWC3_DEPCMD_UPDATETRANSFER | 1649 DWC3_DEPCMD_PARAM(dep->resource_index); 1650 } 1651 1652 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1653 if (ret < 0) { 1654 struct dwc3_request *tmp; 1655 1656 if (ret == -EAGAIN) 1657 return ret; 1658 1659 dwc3_stop_active_transfer(dep, true, true); 1660 1661 list_for_each_entry_safe(req, tmp, &dep->started_list, list) 1662 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED); 1663 1664 /* If ep isn't started, then there's no end transfer pending */ 1665 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) 1666 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 1667 1668 return ret; 1669 } 1670 1671 if (dep->stream_capable && req->request.is_last && 1672 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) 1673 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE; 1674 1675 return 0; 1676 } 1677 1678 static int __dwc3_gadget_get_frame(struct dwc3 *dwc) 1679 { 1680 u32 reg; 1681 1682 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1683 return DWC3_DSTS_SOFFN(reg); 1684 } 1685 1686 /** 1687 * __dwc3_stop_active_transfer - stop the current active transfer 1688 * @dep: isoc endpoint 1689 * @force: set forcerm bit in the command 1690 * @interrupt: command complete interrupt after End Transfer command 1691 * 1692 * When setting force, the ForceRM bit will be set. In that case 1693 * the controller won't update the TRB progress on command 1694 * completion. It also won't clear the HWO bit in the TRB. 1695 * The command will also not complete immediately in that case. 1696 */ 1697 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) 1698 { 1699 struct dwc3_gadget_ep_cmd_params params; 1700 u32 cmd; 1701 int ret; 1702 1703 cmd = DWC3_DEPCMD_ENDTRANSFER; 1704 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0; 1705 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0; 1706 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); 1707 memset(¶ms, 0, sizeof(params)); 1708 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1709 /* 1710 * If the End Transfer command was timed out while the device is 1711 * not in SETUP phase, it's possible that an incoming Setup packet 1712 * may prevent the command's completion. Let's retry when the 1713 * ep0state returns to EP0_SETUP_PHASE. 1714 */ 1715 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) { 1716 dep->flags |= DWC3_EP_DELAY_STOP; 1717 return 0; 1718 } 1719 WARN_ON_ONCE(ret); 1720 dep->resource_index = 0; 1721 1722 if (!interrupt) 1723 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 1724 else if (!ret) 1725 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; 1726 1727 dep->flags &= ~DWC3_EP_DELAY_STOP; 1728 return ret; 1729 } 1730 1731 /** 1732 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number 1733 * @dep: isoc endpoint 1734 * 1735 * This function tests for the correct combination of BIT[15:14] from the 16-bit 1736 * microframe number reported by the XferNotReady event for the future frame 1737 * number to start the isoc transfer. 1738 * 1739 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed 1740 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the 1741 * XferNotReady event are invalid. The driver uses this number to schedule the 1742 * isochronous transfer and passes it to the START TRANSFER command. Because 1743 * this number is invalid, the command may fail. If BIT[15:14] matches the 1744 * internal 16-bit microframe, the START TRANSFER command will pass and the 1745 * transfer will start at the scheduled time, if it is off by 1, the command 1746 * will still pass, but the transfer will start 2 seconds in the future. For all 1747 * other conditions, the START TRANSFER command will fail with bus-expiry. 1748 * 1749 * In order to workaround this issue, we can test for the correct combination of 1750 * BIT[15:14] by sending START TRANSFER commands with different values of 1751 * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart 1752 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status. 1753 * As the result, within the 4 possible combinations for BIT[15:14], there will 1754 * be 2 successful and 2 failure START COMMAND status. One of the 2 successful 1755 * command status will result in a 2-second delay start. The smaller BIT[15:14] 1756 * value is the correct combination. 1757 * 1758 * Since there are only 4 outcomes and the results are ordered, we can simply 1759 * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to 1760 * deduce the smaller successful combination. 1761 * 1762 * Let test0 = test status for combination 'b00 and test1 = test status for 'b01 1763 * of BIT[15:14]. The correct combination is as follow: 1764 * 1765 * if test0 fails and test1 passes, BIT[15:14] is 'b01 1766 * if test0 fails and test1 fails, BIT[15:14] is 'b10 1767 * if test0 passes and test1 fails, BIT[15:14] is 'b11 1768 * if test0 passes and test1 passes, BIT[15:14] is 'b00 1769 * 1770 * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN 1771 * endpoints. 1772 */ 1773 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep) 1774 { 1775 int cmd_status = 0; 1776 bool test0; 1777 bool test1; 1778 1779 while (dep->combo_num < 2) { 1780 struct dwc3_gadget_ep_cmd_params params; 1781 u32 test_frame_number; 1782 u32 cmd; 1783 1784 /* 1785 * Check if we can start isoc transfer on the next interval or 1786 * 4 uframes in the future with BIT[15:14] as dep->combo_num 1787 */ 1788 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK; 1789 test_frame_number |= dep->combo_num << 14; 1790 test_frame_number += max_t(u32, 4, dep->interval); 1791 1792 params.param0 = upper_32_bits(dep->dwc->bounce_addr); 1793 params.param1 = lower_32_bits(dep->dwc->bounce_addr); 1794 1795 cmd = DWC3_DEPCMD_STARTTRANSFER; 1796 cmd |= DWC3_DEPCMD_PARAM(test_frame_number); 1797 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1798 1799 /* Redo if some other failure beside bus-expiry is received */ 1800 if (cmd_status && cmd_status != -EAGAIN) { 1801 dep->start_cmd_status = 0; 1802 dep->combo_num = 0; 1803 return 0; 1804 } 1805 1806 /* Store the first test status */ 1807 if (dep->combo_num == 0) 1808 dep->start_cmd_status = cmd_status; 1809 1810 dep->combo_num++; 1811 1812 /* 1813 * End the transfer if the START_TRANSFER command is successful 1814 * to wait for the next XferNotReady to test the command again 1815 */ 1816 if (cmd_status == 0) { 1817 dwc3_stop_active_transfer(dep, true, true); 1818 return 0; 1819 } 1820 } 1821 1822 /* test0 and test1 are both completed at this point */ 1823 test0 = (dep->start_cmd_status == 0); 1824 test1 = (cmd_status == 0); 1825 1826 if (!test0 && test1) 1827 dep->combo_num = 1; 1828 else if (!test0 && !test1) 1829 dep->combo_num = 2; 1830 else if (test0 && !test1) 1831 dep->combo_num = 3; 1832 else if (test0 && test1) 1833 dep->combo_num = 0; 1834 1835 dep->frame_number &= DWC3_FRNUMBER_MASK; 1836 dep->frame_number |= dep->combo_num << 14; 1837 dep->frame_number += max_t(u32, 4, dep->interval); 1838 1839 /* Reinitialize test variables */ 1840 dep->start_cmd_status = 0; 1841 dep->combo_num = 0; 1842 1843 return __dwc3_gadget_kick_transfer(dep); 1844 } 1845 1846 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep) 1847 { 1848 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 1849 struct dwc3 *dwc = dep->dwc; 1850 int ret; 1851 int i; 1852 1853 if (list_empty(&dep->pending_list) && 1854 list_empty(&dep->started_list)) { 1855 dep->flags |= DWC3_EP_PENDING_REQUEST; 1856 return -EAGAIN; 1857 } 1858 1859 if (!dwc->dis_start_transfer_quirk && 1860 (DWC3_VER_IS_PRIOR(DWC31, 170A) || 1861 DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) { 1862 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction) 1863 return dwc3_gadget_start_isoc_quirk(dep); 1864 } 1865 1866 if (desc->bInterval <= 14 && 1867 dwc->gadget->speed >= USB_SPEED_HIGH) { 1868 u32 frame = __dwc3_gadget_get_frame(dwc); 1869 bool rollover = frame < 1870 (dep->frame_number & DWC3_FRNUMBER_MASK); 1871 1872 /* 1873 * frame_number is set from XferNotReady and may be already 1874 * out of date. DSTS only provides the lower 14 bit of the 1875 * current frame number. So add the upper two bits of 1876 * frame_number and handle a possible rollover. 1877 * This will provide the correct frame_number unless more than 1878 * rollover has happened since XferNotReady. 1879 */ 1880 1881 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) | 1882 frame; 1883 if (rollover) 1884 dep->frame_number += BIT(14); 1885 } 1886 1887 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) { 1888 int future_interval = i + 1; 1889 1890 /* Give the controller at least 500us to schedule transfers */ 1891 if (desc->bInterval < 3) 1892 future_interval += 3 - desc->bInterval; 1893 1894 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval); 1895 1896 ret = __dwc3_gadget_kick_transfer(dep); 1897 if (ret != -EAGAIN) 1898 break; 1899 } 1900 1901 /* 1902 * After a number of unsuccessful start attempts due to bus-expiry 1903 * status, issue END_TRANSFER command and retry on the next XferNotReady 1904 * event. 1905 */ 1906 if (ret == -EAGAIN) 1907 ret = __dwc3_stop_active_transfer(dep, false, true); 1908 1909 return ret; 1910 } 1911 1912 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) 1913 { 1914 struct dwc3 *dwc = dep->dwc; 1915 1916 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { 1917 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n", 1918 dep->name); 1919 return -ESHUTDOWN; 1920 } 1921 1922 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n", 1923 &req->request, req->dep->name)) 1924 return -EINVAL; 1925 1926 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED, 1927 "%s: request %pK already in flight\n", 1928 dep->name, &req->request)) 1929 return -EINVAL; 1930 1931 pm_runtime_get(dwc->dev); 1932 1933 req->request.actual = 0; 1934 req->request.status = -EINPROGRESS; 1935 1936 trace_dwc3_ep_queue(req); 1937 1938 list_add_tail(&req->list, &dep->pending_list); 1939 req->status = DWC3_REQUEST_STATUS_QUEUED; 1940 1941 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE) 1942 return 0; 1943 1944 /* 1945 * Start the transfer only after the END_TRANSFER is completed 1946 * and endpoint STALL is cleared. 1947 */ 1948 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || 1949 (dep->flags & DWC3_EP_WEDGE) || 1950 (dep->flags & DWC3_EP_DELAY_STOP) || 1951 (dep->flags & DWC3_EP_STALL)) { 1952 dep->flags |= DWC3_EP_DELAY_START; 1953 return 0; 1954 } 1955 1956 /* 1957 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must 1958 * wait for a XferNotReady event so we will know what's the current 1959 * (micro-)frame number. 1960 * 1961 * Without this trick, we are very, very likely gonna get Bus Expiry 1962 * errors which will force us issue EndTransfer command. 1963 */ 1964 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 1965 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { 1966 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) 1967 return __dwc3_gadget_start_isoc(dep); 1968 1969 return 0; 1970 } 1971 } 1972 1973 __dwc3_gadget_kick_transfer(dep); 1974 1975 return 0; 1976 } 1977 1978 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, 1979 gfp_t gfp_flags) 1980 { 1981 struct dwc3_request *req = to_dwc3_request(request); 1982 struct dwc3_ep *dep = to_dwc3_ep(ep); 1983 struct dwc3 *dwc = dep->dwc; 1984 1985 unsigned long flags; 1986 1987 int ret; 1988 1989 spin_lock_irqsave(&dwc->lock, flags); 1990 ret = __dwc3_gadget_ep_queue(dep, req); 1991 spin_unlock_irqrestore(&dwc->lock, flags); 1992 1993 return ret; 1994 } 1995 1996 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req) 1997 { 1998 int i; 1999 2000 /* If req->trb is not set, then the request has not started */ 2001 if (!req->trb) 2002 return; 2003 2004 /* 2005 * If request was already started, this means we had to 2006 * stop the transfer. With that we also need to ignore 2007 * all TRBs used by the request, however TRBs can only 2008 * be modified after completion of END_TRANSFER 2009 * command. So what we do here is that we wait for 2010 * END_TRANSFER completion and only after that, we jump 2011 * over TRBs by clearing HWO and incrementing dequeue 2012 * pointer. 2013 */ 2014 for (i = 0; i < req->num_trbs; i++) { 2015 struct dwc3_trb *trb; 2016 2017 trb = &dep->trb_pool[dep->trb_dequeue]; 2018 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 2019 dwc3_ep_inc_deq(dep); 2020 } 2021 2022 req->num_trbs = 0; 2023 } 2024 2025 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep) 2026 { 2027 struct dwc3_request *req; 2028 struct dwc3 *dwc = dep->dwc; 2029 2030 while (!list_empty(&dep->cancelled_list)) { 2031 req = next_request(&dep->cancelled_list); 2032 dwc3_gadget_ep_skip_trbs(dep, req); 2033 switch (req->status) { 2034 case DWC3_REQUEST_STATUS_DISCONNECTED: 2035 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 2036 break; 2037 case DWC3_REQUEST_STATUS_DEQUEUED: 2038 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2039 break; 2040 case DWC3_REQUEST_STATUS_STALLED: 2041 dwc3_gadget_giveback(dep, req, -EPIPE); 2042 break; 2043 default: 2044 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status); 2045 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2046 break; 2047 } 2048 /* 2049 * The endpoint is disabled, let the dwc3_remove_requests() 2050 * handle the cleanup. 2051 */ 2052 if (!dep->endpoint.desc) 2053 break; 2054 } 2055 } 2056 2057 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, 2058 struct usb_request *request) 2059 { 2060 struct dwc3_request *req = to_dwc3_request(request); 2061 struct dwc3_request *r = NULL; 2062 2063 struct dwc3_ep *dep = to_dwc3_ep(ep); 2064 struct dwc3 *dwc = dep->dwc; 2065 2066 unsigned long flags; 2067 int ret = 0; 2068 2069 trace_dwc3_ep_dequeue(req); 2070 2071 spin_lock_irqsave(&dwc->lock, flags); 2072 2073 list_for_each_entry(r, &dep->cancelled_list, list) { 2074 if (r == req) 2075 goto out; 2076 } 2077 2078 list_for_each_entry(r, &dep->pending_list, list) { 2079 if (r == req) { 2080 /* 2081 * Explicitly check for EP0/1 as dequeue for those 2082 * EPs need to be handled differently. Control EP 2083 * only deals with one USB req, and giveback will 2084 * occur during dwc3_ep0_stall_and_restart(). EP0 2085 * requests are never added to started_list. 2086 */ 2087 if (dep->number > 1) 2088 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2089 else 2090 dwc3_ep0_reset_state(dwc); 2091 goto out; 2092 } 2093 } 2094 2095 list_for_each_entry(r, &dep->started_list, list) { 2096 if (r == req) { 2097 struct dwc3_request *t; 2098 2099 /* wait until it is processed */ 2100 dwc3_stop_active_transfer(dep, true, true); 2101 2102 /* 2103 * Remove any started request if the transfer is 2104 * cancelled. 2105 */ 2106 list_for_each_entry_safe(r, t, &dep->started_list, list) 2107 dwc3_gadget_move_cancelled_request(r, 2108 DWC3_REQUEST_STATUS_DEQUEUED); 2109 2110 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; 2111 2112 goto out; 2113 } 2114 } 2115 2116 dev_err(dwc->dev, "request %pK was not queued to %s\n", 2117 request, ep->name); 2118 ret = -EINVAL; 2119 out: 2120 spin_unlock_irqrestore(&dwc->lock, flags); 2121 2122 return ret; 2123 } 2124 2125 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) 2126 { 2127 struct dwc3_gadget_ep_cmd_params params; 2128 struct dwc3 *dwc = dep->dwc; 2129 struct dwc3_request *req; 2130 struct dwc3_request *tmp; 2131 int ret; 2132 2133 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 2134 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); 2135 return -EINVAL; 2136 } 2137 2138 memset(¶ms, 0x00, sizeof(params)); 2139 2140 if (value) { 2141 struct dwc3_trb *trb; 2142 2143 unsigned int transfer_in_flight; 2144 unsigned int started; 2145 2146 if (dep->number > 1) 2147 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); 2148 else 2149 trb = &dwc->ep0_trb[dep->trb_enqueue]; 2150 2151 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO; 2152 started = !list_empty(&dep->started_list); 2153 2154 if (!protocol && ((dep->direction && transfer_in_flight) || 2155 (!dep->direction && started))) { 2156 return -EAGAIN; 2157 } 2158 2159 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL, 2160 ¶ms); 2161 if (ret) 2162 dev_err(dwc->dev, "failed to set STALL on %s\n", 2163 dep->name); 2164 else 2165 dep->flags |= DWC3_EP_STALL; 2166 } else { 2167 /* 2168 * Don't issue CLEAR_STALL command to control endpoints. The 2169 * controller automatically clears the STALL when it receives 2170 * the SETUP token. 2171 */ 2172 if (dep->number <= 1) { 2173 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 2174 return 0; 2175 } 2176 2177 dwc3_stop_active_transfer(dep, true, true); 2178 2179 list_for_each_entry_safe(req, tmp, &dep->started_list, list) 2180 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED); 2181 2182 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING || 2183 (dep->flags & DWC3_EP_DELAY_STOP)) { 2184 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; 2185 if (protocol) 2186 dwc->clear_stall_protocol = dep->number; 2187 2188 return 0; 2189 } 2190 2191 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 2192 2193 ret = dwc3_send_clear_stall_ep_cmd(dep); 2194 if (ret) { 2195 dev_err(dwc->dev, "failed to clear STALL on %s\n", 2196 dep->name); 2197 return ret; 2198 } 2199 2200 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 2201 2202 if ((dep->flags & DWC3_EP_DELAY_START) && 2203 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) 2204 __dwc3_gadget_kick_transfer(dep); 2205 2206 dep->flags &= ~DWC3_EP_DELAY_START; 2207 } 2208 2209 return ret; 2210 } 2211 2212 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value) 2213 { 2214 struct dwc3_ep *dep = to_dwc3_ep(ep); 2215 struct dwc3 *dwc = dep->dwc; 2216 2217 unsigned long flags; 2218 2219 int ret; 2220 2221 spin_lock_irqsave(&dwc->lock, flags); 2222 ret = __dwc3_gadget_ep_set_halt(dep, value, false); 2223 spin_unlock_irqrestore(&dwc->lock, flags); 2224 2225 return ret; 2226 } 2227 2228 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep) 2229 { 2230 struct dwc3_ep *dep = to_dwc3_ep(ep); 2231 struct dwc3 *dwc = dep->dwc; 2232 unsigned long flags; 2233 int ret; 2234 2235 spin_lock_irqsave(&dwc->lock, flags); 2236 dep->flags |= DWC3_EP_WEDGE; 2237 2238 if (dep->number == 0 || dep->number == 1) 2239 ret = __dwc3_gadget_ep0_set_halt(ep, 1); 2240 else 2241 ret = __dwc3_gadget_ep_set_halt(dep, 1, false); 2242 spin_unlock_irqrestore(&dwc->lock, flags); 2243 2244 return ret; 2245 } 2246 2247 /* -------------------------------------------------------------------------- */ 2248 2249 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = { 2250 .bLength = USB_DT_ENDPOINT_SIZE, 2251 .bDescriptorType = USB_DT_ENDPOINT, 2252 .bmAttributes = USB_ENDPOINT_XFER_CONTROL, 2253 }; 2254 2255 static const struct usb_ep_ops dwc3_gadget_ep0_ops = { 2256 .enable = dwc3_gadget_ep0_enable, 2257 .disable = dwc3_gadget_ep0_disable, 2258 .alloc_request = dwc3_gadget_ep_alloc_request, 2259 .free_request = dwc3_gadget_ep_free_request, 2260 .queue = dwc3_gadget_ep0_queue, 2261 .dequeue = dwc3_gadget_ep_dequeue, 2262 .set_halt = dwc3_gadget_ep0_set_halt, 2263 .set_wedge = dwc3_gadget_ep_set_wedge, 2264 }; 2265 2266 static const struct usb_ep_ops dwc3_gadget_ep_ops = { 2267 .enable = dwc3_gadget_ep_enable, 2268 .disable = dwc3_gadget_ep_disable, 2269 .alloc_request = dwc3_gadget_ep_alloc_request, 2270 .free_request = dwc3_gadget_ep_free_request, 2271 .queue = dwc3_gadget_ep_queue, 2272 .dequeue = dwc3_gadget_ep_dequeue, 2273 .set_halt = dwc3_gadget_ep_set_halt, 2274 .set_wedge = dwc3_gadget_ep_set_wedge, 2275 }; 2276 2277 /* -------------------------------------------------------------------------- */ 2278 2279 static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set) 2280 { 2281 u32 reg; 2282 2283 if (DWC3_VER_IS_PRIOR(DWC3, 250A)) 2284 return; 2285 2286 reg = dwc3_readl(dwc->regs, DWC3_DEVTEN); 2287 if (set) 2288 reg |= DWC3_DEVTEN_ULSTCNGEN; 2289 else 2290 reg &= ~DWC3_DEVTEN_ULSTCNGEN; 2291 2292 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); 2293 } 2294 2295 static int dwc3_gadget_get_frame(struct usb_gadget *g) 2296 { 2297 struct dwc3 *dwc = gadget_to_dwc(g); 2298 2299 return __dwc3_gadget_get_frame(dwc); 2300 } 2301 2302 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async) 2303 { 2304 int retries; 2305 2306 int ret; 2307 u32 reg; 2308 2309 u8 link_state; 2310 2311 /* 2312 * According to the Databook Remote wakeup request should 2313 * be issued only when the device is in early suspend state. 2314 * 2315 * We can check that via USB Link State bits in DSTS register. 2316 */ 2317 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2318 2319 link_state = DWC3_DSTS_USBLNKST(reg); 2320 2321 switch (link_state) { 2322 case DWC3_LINK_STATE_RESET: 2323 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */ 2324 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */ 2325 case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */ 2326 case DWC3_LINK_STATE_U1: 2327 case DWC3_LINK_STATE_RESUME: 2328 break; 2329 default: 2330 return -EINVAL; 2331 } 2332 2333 if (async) 2334 dwc3_gadget_enable_linksts_evts(dwc, true); 2335 2336 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV); 2337 if (ret < 0) { 2338 dev_err(dwc->dev, "failed to put link in Recovery\n"); 2339 dwc3_gadget_enable_linksts_evts(dwc, false); 2340 return ret; 2341 } 2342 2343 /* Recent versions do this automatically */ 2344 if (DWC3_VER_IS_PRIOR(DWC3, 194A)) { 2345 /* write zeroes to Link Change Request */ 2346 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2347 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 2348 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2349 } 2350 2351 /* 2352 * Since link status change events are enabled we will receive 2353 * an U0 event when wakeup is successful. So bail out. 2354 */ 2355 if (async) 2356 return 0; 2357 2358 /* poll until Link State changes to ON */ 2359 retries = 20000; 2360 2361 while (retries--) { 2362 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2363 2364 /* in HS, means ON */ 2365 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0) 2366 break; 2367 } 2368 2369 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) { 2370 dev_err(dwc->dev, "failed to send remote wakeup\n"); 2371 return -EINVAL; 2372 } 2373 2374 return 0; 2375 } 2376 2377 static int dwc3_gadget_wakeup(struct usb_gadget *g) 2378 { 2379 struct dwc3 *dwc = gadget_to_dwc(g); 2380 unsigned long flags; 2381 int ret; 2382 2383 if (!dwc->wakeup_configured) { 2384 dev_err(dwc->dev, "remote wakeup not configured\n"); 2385 return -EINVAL; 2386 } 2387 2388 spin_lock_irqsave(&dwc->lock, flags); 2389 if (!dwc->gadget->wakeup_armed) { 2390 dev_err(dwc->dev, "not armed for remote wakeup\n"); 2391 spin_unlock_irqrestore(&dwc->lock, flags); 2392 return -EINVAL; 2393 } 2394 ret = __dwc3_gadget_wakeup(dwc, true); 2395 2396 spin_unlock_irqrestore(&dwc->lock, flags); 2397 2398 return ret; 2399 } 2400 2401 static void dwc3_resume_gadget(struct dwc3 *dwc); 2402 2403 static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id) 2404 { 2405 struct dwc3 *dwc = gadget_to_dwc(g); 2406 unsigned long flags; 2407 int ret; 2408 int link_state; 2409 2410 if (!dwc->wakeup_configured) { 2411 dev_err(dwc->dev, "remote wakeup not configured\n"); 2412 return -EINVAL; 2413 } 2414 2415 spin_lock_irqsave(&dwc->lock, flags); 2416 /* 2417 * If the link is in U3, signal for remote wakeup and wait for the 2418 * link to transition to U0 before sending device notification. 2419 */ 2420 link_state = dwc3_gadget_get_link_state(dwc); 2421 if (link_state == DWC3_LINK_STATE_U3) { 2422 ret = __dwc3_gadget_wakeup(dwc, false); 2423 if (ret) { 2424 spin_unlock_irqrestore(&dwc->lock, flags); 2425 return -EINVAL; 2426 } 2427 dwc3_resume_gadget(dwc); 2428 dwc->suspended = false; 2429 dwc->link_state = DWC3_LINK_STATE_U0; 2430 } 2431 2432 ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION, 2433 DWC3_DGCMDPAR_DN_FUNC_WAKE | 2434 DWC3_DGCMDPAR_INTF_SEL(intf_id)); 2435 if (ret) 2436 dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret); 2437 2438 spin_unlock_irqrestore(&dwc->lock, flags); 2439 2440 return ret; 2441 } 2442 2443 static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set) 2444 { 2445 struct dwc3 *dwc = gadget_to_dwc(g); 2446 unsigned long flags; 2447 2448 spin_lock_irqsave(&dwc->lock, flags); 2449 dwc->wakeup_configured = !!set; 2450 spin_unlock_irqrestore(&dwc->lock, flags); 2451 2452 return 0; 2453 } 2454 2455 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g, 2456 int is_selfpowered) 2457 { 2458 struct dwc3 *dwc = gadget_to_dwc(g); 2459 unsigned long flags; 2460 2461 spin_lock_irqsave(&dwc->lock, flags); 2462 g->is_selfpowered = !!is_selfpowered; 2463 spin_unlock_irqrestore(&dwc->lock, flags); 2464 2465 return 0; 2466 } 2467 2468 static void dwc3_stop_active_transfers(struct dwc3 *dwc) 2469 { 2470 u32 epnum; 2471 2472 for (epnum = 2; epnum < dwc->num_eps; epnum++) { 2473 struct dwc3_ep *dep; 2474 2475 dep = dwc->eps[epnum]; 2476 if (!dep) 2477 continue; 2478 2479 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); 2480 } 2481 } 2482 2483 static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc) 2484 { 2485 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate; 2486 u32 reg; 2487 2488 if (ssp_rate == USB_SSP_GEN_UNKNOWN) 2489 ssp_rate = dwc->max_ssp_rate; 2490 2491 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2492 reg &= ~DWC3_DCFG_SPEED_MASK; 2493 reg &= ~DWC3_DCFG_NUMLANES(~0); 2494 2495 if (ssp_rate == USB_SSP_GEN_1x2) 2496 reg |= DWC3_DCFG_SUPERSPEED; 2497 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2) 2498 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2499 2500 if (ssp_rate != USB_SSP_GEN_2x1 && 2501 dwc->max_ssp_rate != USB_SSP_GEN_2x1) 2502 reg |= DWC3_DCFG_NUMLANES(1); 2503 2504 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2505 } 2506 2507 static void __dwc3_gadget_set_speed(struct dwc3 *dwc) 2508 { 2509 enum usb_device_speed speed; 2510 u32 reg; 2511 2512 speed = dwc->gadget_max_speed; 2513 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed) 2514 speed = dwc->maximum_speed; 2515 2516 if (speed == USB_SPEED_SUPER_PLUS && 2517 DWC3_IP_IS(DWC32)) { 2518 __dwc3_gadget_set_ssp_rate(dwc); 2519 return; 2520 } 2521 2522 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2523 reg &= ~(DWC3_DCFG_SPEED_MASK); 2524 2525 /* 2526 * WORKAROUND: DWC3 revision < 2.20a have an issue 2527 * which would cause metastability state on Run/Stop 2528 * bit if we try to force the IP to USB2-only mode. 2529 * 2530 * Because of that, we cannot configure the IP to any 2531 * speed other than the SuperSpeed 2532 * 2533 * Refers to: 2534 * 2535 * STAR#9000525659: Clock Domain Crossing on DCTL in 2536 * USB 2.0 Mode 2537 */ 2538 if (DWC3_VER_IS_PRIOR(DWC3, 220A) && 2539 !dwc->dis_metastability_quirk) { 2540 reg |= DWC3_DCFG_SUPERSPEED; 2541 } else { 2542 switch (speed) { 2543 case USB_SPEED_FULL: 2544 reg |= DWC3_DCFG_FULLSPEED; 2545 break; 2546 case USB_SPEED_HIGH: 2547 reg |= DWC3_DCFG_HIGHSPEED; 2548 break; 2549 case USB_SPEED_SUPER: 2550 reg |= DWC3_DCFG_SUPERSPEED; 2551 break; 2552 case USB_SPEED_SUPER_PLUS: 2553 if (DWC3_IP_IS(DWC3)) 2554 reg |= DWC3_DCFG_SUPERSPEED; 2555 else 2556 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2557 break; 2558 default: 2559 dev_err(dwc->dev, "invalid speed (%d)\n", speed); 2560 2561 if (DWC3_IP_IS(DWC3)) 2562 reg |= DWC3_DCFG_SUPERSPEED; 2563 else 2564 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2565 } 2566 } 2567 2568 if (DWC3_IP_IS(DWC32) && 2569 speed > USB_SPEED_UNKNOWN && 2570 speed < USB_SPEED_SUPER_PLUS) 2571 reg &= ~DWC3_DCFG_NUMLANES(~0); 2572 2573 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2574 } 2575 2576 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on) 2577 { 2578 u32 reg; 2579 u32 timeout = 2000; 2580 2581 if (pm_runtime_suspended(dwc->dev)) 2582 return 0; 2583 2584 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2585 if (is_on) { 2586 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) { 2587 reg &= ~DWC3_DCTL_TRGTULST_MASK; 2588 reg |= DWC3_DCTL_TRGTULST_RX_DET; 2589 } 2590 2591 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) 2592 reg &= ~DWC3_DCTL_KEEP_CONNECT; 2593 reg |= DWC3_DCTL_RUN_STOP; 2594 2595 __dwc3_gadget_set_speed(dwc); 2596 dwc->pullups_connected = true; 2597 } else { 2598 reg &= ~DWC3_DCTL_RUN_STOP; 2599 2600 dwc->pullups_connected = false; 2601 } 2602 2603 dwc3_gadget_dctl_write_safe(dwc, reg); 2604 2605 do { 2606 usleep_range(1000, 2000); 2607 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2608 reg &= DWC3_DSTS_DEVCTRLHLT; 2609 } while (--timeout && !(!is_on ^ !reg)); 2610 2611 if (!timeout) 2612 return -ETIMEDOUT; 2613 2614 return 0; 2615 } 2616 2617 static void dwc3_gadget_disable_irq(struct dwc3 *dwc); 2618 static void __dwc3_gadget_stop(struct dwc3 *dwc); 2619 static int __dwc3_gadget_start(struct dwc3 *dwc); 2620 2621 static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc) 2622 { 2623 unsigned long flags; 2624 int ret; 2625 2626 spin_lock_irqsave(&dwc->lock, flags); 2627 if (!dwc->pullups_connected) { 2628 spin_unlock_irqrestore(&dwc->lock, flags); 2629 return 0; 2630 } 2631 2632 dwc->connected = false; 2633 2634 /* 2635 * Attempt to end pending SETUP status phase, and not wait for the 2636 * function to do so. 2637 */ 2638 if (dwc->delayed_status) 2639 dwc3_ep0_send_delayed_status(dwc); 2640 2641 /* 2642 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a 2643 * Section 4.1.8 Table 4-7, it states that for a device-initiated 2644 * disconnect, the SW needs to ensure that it sends "a DEPENDXFER 2645 * command for any active transfers" before clearing the RunStop 2646 * bit. 2647 */ 2648 dwc3_stop_active_transfers(dwc); 2649 spin_unlock_irqrestore(&dwc->lock, flags); 2650 2651 /* 2652 * Per databook, when we want to stop the gadget, if a control transfer 2653 * is still in process, complete it and get the core into setup phase. 2654 * In case the host is unresponsive to a SETUP transaction, forcefully 2655 * stall the transfer, and move back to the SETUP phase, so that any 2656 * pending endxfers can be executed. 2657 */ 2658 if (dwc->ep0state != EP0_SETUP_PHASE) { 2659 reinit_completion(&dwc->ep0_in_setup); 2660 2661 ret = wait_for_completion_timeout(&dwc->ep0_in_setup, 2662 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT)); 2663 if (ret == 0) { 2664 dev_warn(dwc->dev, "wait for SETUP phase timed out\n"); 2665 spin_lock_irqsave(&dwc->lock, flags); 2666 dwc3_ep0_reset_state(dwc); 2667 spin_unlock_irqrestore(&dwc->lock, flags); 2668 } 2669 } 2670 2671 /* 2672 * Note: if the GEVNTCOUNT indicates events in the event buffer, the 2673 * driver needs to acknowledge them before the controller can halt. 2674 * Simply let the interrupt handler acknowledges and handle the 2675 * remaining event generated by the controller while polling for 2676 * DSTS.DEVCTLHLT. 2677 */ 2678 ret = dwc3_gadget_run_stop(dwc, false); 2679 2680 /* 2681 * Stop the gadget after controller is halted, so that if needed, the 2682 * events to update EP0 state can still occur while the run/stop 2683 * routine polls for the halted state. DEVTEN is cleared as part of 2684 * gadget stop. 2685 */ 2686 spin_lock_irqsave(&dwc->lock, flags); 2687 __dwc3_gadget_stop(dwc); 2688 spin_unlock_irqrestore(&dwc->lock, flags); 2689 2690 return ret; 2691 } 2692 2693 static int dwc3_gadget_soft_connect(struct dwc3 *dwc) 2694 { 2695 int ret; 2696 2697 /* 2698 * In the Synopsys DWC_usb31 1.90a programming guide section 2699 * 4.1.9, it specifies that for a reconnect after a 2700 * device-initiated disconnect requires a core soft reset 2701 * (DCTL.CSftRst) before enabling the run/stop bit. 2702 */ 2703 ret = dwc3_core_soft_reset(dwc); 2704 if (ret) 2705 return ret; 2706 2707 dwc3_event_buffers_setup(dwc); 2708 __dwc3_gadget_start(dwc); 2709 return dwc3_gadget_run_stop(dwc, true); 2710 } 2711 2712 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) 2713 { 2714 struct dwc3 *dwc = gadget_to_dwc(g); 2715 int ret; 2716 2717 is_on = !!is_on; 2718 2719 dwc->softconnect = is_on; 2720 2721 /* 2722 * Avoid issuing a runtime resume if the device is already in the 2723 * suspended state during gadget disconnect. DWC3 gadget was already 2724 * halted/stopped during runtime suspend. 2725 */ 2726 if (!is_on) { 2727 pm_runtime_barrier(dwc->dev); 2728 if (pm_runtime_suspended(dwc->dev)) 2729 return 0; 2730 } 2731 2732 /* 2733 * Check the return value for successful resume, or error. For a 2734 * successful resume, the DWC3 runtime PM resume routine will handle 2735 * the run stop sequence, so avoid duplicate operations here. 2736 */ 2737 ret = pm_runtime_get_sync(dwc->dev); 2738 if (!ret || ret < 0) { 2739 pm_runtime_put(dwc->dev); 2740 if (ret < 0) 2741 pm_runtime_set_suspended(dwc->dev); 2742 return ret; 2743 } 2744 2745 if (dwc->pullups_connected == is_on) { 2746 pm_runtime_put(dwc->dev); 2747 return 0; 2748 } 2749 2750 synchronize_irq(dwc->irq_gadget); 2751 2752 if (!is_on) 2753 ret = dwc3_gadget_soft_disconnect(dwc); 2754 else 2755 ret = dwc3_gadget_soft_connect(dwc); 2756 2757 pm_runtime_put(dwc->dev); 2758 2759 return ret; 2760 } 2761 2762 static void dwc3_gadget_enable_irq(struct dwc3 *dwc) 2763 { 2764 u32 reg; 2765 2766 /* Enable all but Start and End of Frame IRQs */ 2767 reg = (DWC3_DEVTEN_EVNTOVERFLOWEN | 2768 DWC3_DEVTEN_CMDCMPLTEN | 2769 DWC3_DEVTEN_ERRTICERREN | 2770 DWC3_DEVTEN_WKUPEVTEN | 2771 DWC3_DEVTEN_CONNECTDONEEN | 2772 DWC3_DEVTEN_USBRSTEN | 2773 DWC3_DEVTEN_DISCONNEVTEN); 2774 2775 if (DWC3_VER_IS_PRIOR(DWC3, 250A)) 2776 reg |= DWC3_DEVTEN_ULSTCNGEN; 2777 2778 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */ 2779 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) 2780 reg |= DWC3_DEVTEN_U3L2L1SUSPEN; 2781 2782 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); 2783 } 2784 2785 static void dwc3_gadget_disable_irq(struct dwc3 *dwc) 2786 { 2787 /* mask all interrupts */ 2788 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00); 2789 } 2790 2791 static irqreturn_t dwc3_interrupt(int irq, void *_dwc); 2792 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc); 2793 2794 /** 2795 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG 2796 * @dwc: pointer to our context structure 2797 * 2798 * The following looks like complex but it's actually very simple. In order to 2799 * calculate the number of packets we can burst at once on OUT transfers, we're 2800 * gonna use RxFIFO size. 2801 * 2802 * To calculate RxFIFO size we need two numbers: 2803 * MDWIDTH = size, in bits, of the internal memory bus 2804 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits) 2805 * 2806 * Given these two numbers, the formula is simple: 2807 * 2808 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16; 2809 * 2810 * 24 bytes is for 3x SETUP packets 2811 * 16 bytes is a clock domain crossing tolerance 2812 * 2813 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024; 2814 */ 2815 static void dwc3_gadget_setup_nump(struct dwc3 *dwc) 2816 { 2817 u32 ram2_depth; 2818 u32 mdwidth; 2819 u32 nump; 2820 u32 reg; 2821 2822 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7); 2823 mdwidth = dwc3_mdwidth(dwc); 2824 2825 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024; 2826 nump = min_t(u32, nump, 16); 2827 2828 /* update NumP */ 2829 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2830 reg &= ~DWC3_DCFG_NUMP_MASK; 2831 reg |= nump << DWC3_DCFG_NUMP_SHIFT; 2832 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2833 } 2834 2835 static int __dwc3_gadget_start(struct dwc3 *dwc) 2836 { 2837 struct dwc3_ep *dep; 2838 int ret = 0; 2839 u32 reg; 2840 2841 /* 2842 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if 2843 * the core supports IMOD, disable it. 2844 */ 2845 if (dwc->imod_interval) { 2846 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); 2847 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); 2848 } else if (dwc3_has_imod(dwc)) { 2849 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0); 2850 } 2851 2852 /* 2853 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP 2854 * field instead of letting dwc3 itself calculate that automatically. 2855 * 2856 * This way, we maximize the chances that we'll be able to get several 2857 * bursts of data without going through any sort of endpoint throttling. 2858 */ 2859 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG); 2860 if (DWC3_IP_IS(DWC3)) 2861 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL; 2862 else 2863 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL; 2864 2865 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg); 2866 2867 dwc3_gadget_setup_nump(dwc); 2868 2869 /* 2870 * Currently the controller handles single stream only. So, Ignore 2871 * Packet Pending bit for stream selection and don't search for another 2872 * stream if the host sends Data Packet with PP=0 (for OUT direction) or 2873 * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves 2874 * the stream performance. 2875 */ 2876 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2877 reg |= DWC3_DCFG_IGNSTRMPP; 2878 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2879 2880 /* Enable MST by default if the device is capable of MST */ 2881 if (DWC3_MST_CAPABLE(&dwc->hwparams)) { 2882 reg = dwc3_readl(dwc->regs, DWC3_DCFG1); 2883 reg &= ~DWC3_DCFG1_DIS_MST_ENH; 2884 dwc3_writel(dwc->regs, DWC3_DCFG1, reg); 2885 } 2886 2887 /* Start with SuperSpeed Default */ 2888 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 2889 2890 ret = dwc3_gadget_start_config(dwc, 0); 2891 if (ret) { 2892 dev_err(dwc->dev, "failed to config endpoints\n"); 2893 return ret; 2894 } 2895 2896 dep = dwc->eps[0]; 2897 dep->flags = 0; 2898 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 2899 if (ret) { 2900 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2901 goto err0; 2902 } 2903 2904 dep = dwc->eps[1]; 2905 dep->flags = 0; 2906 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 2907 if (ret) { 2908 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2909 goto err1; 2910 } 2911 2912 /* begin to receive SETUP packets */ 2913 dwc->ep0state = EP0_SETUP_PHASE; 2914 dwc->ep0_bounced = false; 2915 dwc->link_state = DWC3_LINK_STATE_SS_DIS; 2916 dwc->delayed_status = false; 2917 dwc3_ep0_out_start(dwc); 2918 2919 dwc3_gadget_enable_irq(dwc); 2920 dwc3_enable_susphy(dwc, true); 2921 2922 return 0; 2923 2924 err1: 2925 __dwc3_gadget_ep_disable(dwc->eps[0]); 2926 2927 err0: 2928 return ret; 2929 } 2930 2931 static int dwc3_gadget_start(struct usb_gadget *g, 2932 struct usb_gadget_driver *driver) 2933 { 2934 struct dwc3 *dwc = gadget_to_dwc(g); 2935 unsigned long flags; 2936 int ret; 2937 int irq; 2938 2939 irq = dwc->irq_gadget; 2940 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt, 2941 IRQF_SHARED, "dwc3", dwc->ev_buf); 2942 if (ret) { 2943 dev_err(dwc->dev, "failed to request irq #%d --> %d\n", 2944 irq, ret); 2945 return ret; 2946 } 2947 2948 spin_lock_irqsave(&dwc->lock, flags); 2949 dwc->gadget_driver = driver; 2950 spin_unlock_irqrestore(&dwc->lock, flags); 2951 2952 if (dwc->sys_wakeup) 2953 device_wakeup_enable(dwc->sysdev); 2954 2955 return 0; 2956 } 2957 2958 static void __dwc3_gadget_stop(struct dwc3 *dwc) 2959 { 2960 dwc3_gadget_disable_irq(dwc); 2961 __dwc3_gadget_ep_disable(dwc->eps[0]); 2962 __dwc3_gadget_ep_disable(dwc->eps[1]); 2963 } 2964 2965 static int dwc3_gadget_stop(struct usb_gadget *g) 2966 { 2967 struct dwc3 *dwc = gadget_to_dwc(g); 2968 unsigned long flags; 2969 2970 if (dwc->sys_wakeup) 2971 device_wakeup_disable(dwc->sysdev); 2972 2973 spin_lock_irqsave(&dwc->lock, flags); 2974 dwc->gadget_driver = NULL; 2975 dwc->max_cfg_eps = 0; 2976 spin_unlock_irqrestore(&dwc->lock, flags); 2977 2978 free_irq(dwc->irq_gadget, dwc->ev_buf); 2979 2980 return 0; 2981 } 2982 2983 static void dwc3_gadget_config_params(struct usb_gadget *g, 2984 struct usb_dcd_config_params *params) 2985 { 2986 struct dwc3 *dwc = gadget_to_dwc(g); 2987 2988 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED; 2989 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED; 2990 2991 /* Recommended BESL */ 2992 if (!dwc->dis_enblslpm_quirk) { 2993 /* 2994 * If the recommended BESL baseline is 0 or if the BESL deep is 2995 * less than 2, Microsoft's Windows 10 host usb stack will issue 2996 * a usb reset immediately after it receives the extended BOS 2997 * descriptor and the enumeration will fail. To maintain 2998 * compatibility with the Windows' usb stack, let's set the 2999 * recommended BESL baseline to 1 and clamp the BESL deep to be 3000 * within 2 to 15. 3001 */ 3002 params->besl_baseline = 1; 3003 if (dwc->is_utmi_l1_suspend) 3004 params->besl_deep = 3005 clamp_t(u8, dwc->hird_threshold, 2, 15); 3006 } 3007 3008 /* U1 Device exit Latency */ 3009 if (dwc->dis_u1_entry_quirk) 3010 params->bU1devExitLat = 0; 3011 else 3012 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT; 3013 3014 /* U2 Device exit Latency */ 3015 if (dwc->dis_u2_entry_quirk) 3016 params->bU2DevExitLat = 0; 3017 else 3018 params->bU2DevExitLat = 3019 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT); 3020 } 3021 3022 static void dwc3_gadget_set_speed(struct usb_gadget *g, 3023 enum usb_device_speed speed) 3024 { 3025 struct dwc3 *dwc = gadget_to_dwc(g); 3026 unsigned long flags; 3027 3028 spin_lock_irqsave(&dwc->lock, flags); 3029 dwc->gadget_max_speed = speed; 3030 spin_unlock_irqrestore(&dwc->lock, flags); 3031 } 3032 3033 static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g, 3034 enum usb_ssp_rate rate) 3035 { 3036 struct dwc3 *dwc = gadget_to_dwc(g); 3037 unsigned long flags; 3038 3039 spin_lock_irqsave(&dwc->lock, flags); 3040 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS; 3041 dwc->gadget_ssp_rate = rate; 3042 spin_unlock_irqrestore(&dwc->lock, flags); 3043 } 3044 3045 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) 3046 { 3047 struct dwc3 *dwc = gadget_to_dwc(g); 3048 union power_supply_propval val = {0}; 3049 int ret; 3050 3051 if (dwc->usb2_phy) 3052 return usb_phy_set_power(dwc->usb2_phy, mA); 3053 3054 if (!dwc->usb_psy) 3055 return -EOPNOTSUPP; 3056 3057 val.intval = 1000 * mA; 3058 ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); 3059 3060 return ret; 3061 } 3062 3063 /** 3064 * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration 3065 * @g: pointer to the USB gadget 3066 * 3067 * Used to record the maximum number of endpoints being used in a USB composite 3068 * device. (across all configurations) This is to be used in the calculation 3069 * of the TXFIFO sizes when resizing internal memory for individual endpoints. 3070 * It will help ensured that the resizing logic reserves enough space for at 3071 * least one max packet. 3072 */ 3073 static int dwc3_gadget_check_config(struct usb_gadget *g) 3074 { 3075 struct dwc3 *dwc = gadget_to_dwc(g); 3076 struct usb_ep *ep; 3077 int fifo_size = 0; 3078 int ram1_depth; 3079 int ep_num = 0; 3080 3081 if (!dwc->do_fifo_resize) 3082 return 0; 3083 3084 list_for_each_entry(ep, &g->ep_list, ep_list) { 3085 /* Only interested in the IN endpoints */ 3086 if (ep->claimed && (ep->address & USB_DIR_IN)) 3087 ep_num++; 3088 } 3089 3090 if (ep_num <= dwc->max_cfg_eps) 3091 return 0; 3092 3093 /* Update the max number of eps in the composition */ 3094 dwc->max_cfg_eps = ep_num; 3095 3096 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps); 3097 /* Based on the equation, increment by one for every ep */ 3098 fifo_size += dwc->max_cfg_eps; 3099 3100 /* Check if we can fit a single fifo per endpoint */ 3101 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); 3102 if (fifo_size > ram1_depth) 3103 return -ENOMEM; 3104 3105 return 0; 3106 } 3107 3108 static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable) 3109 { 3110 struct dwc3 *dwc = gadget_to_dwc(g); 3111 unsigned long flags; 3112 3113 spin_lock_irqsave(&dwc->lock, flags); 3114 dwc->async_callbacks = enable; 3115 spin_unlock_irqrestore(&dwc->lock, flags); 3116 } 3117 3118 static const struct usb_gadget_ops dwc3_gadget_ops = { 3119 .get_frame = dwc3_gadget_get_frame, 3120 .wakeup = dwc3_gadget_wakeup, 3121 .func_wakeup = dwc3_gadget_func_wakeup, 3122 .set_remote_wakeup = dwc3_gadget_set_remote_wakeup, 3123 .set_selfpowered = dwc3_gadget_set_selfpowered, 3124 .pullup = dwc3_gadget_pullup, 3125 .udc_start = dwc3_gadget_start, 3126 .udc_stop = dwc3_gadget_stop, 3127 .udc_set_speed = dwc3_gadget_set_speed, 3128 .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate, 3129 .get_config_params = dwc3_gadget_config_params, 3130 .vbus_draw = dwc3_gadget_vbus_draw, 3131 .check_config = dwc3_gadget_check_config, 3132 .udc_async_callbacks = dwc3_gadget_async_callbacks, 3133 }; 3134 3135 /* -------------------------------------------------------------------------- */ 3136 3137 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep) 3138 { 3139 struct dwc3 *dwc = dep->dwc; 3140 3141 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); 3142 dep->endpoint.maxburst = 1; 3143 dep->endpoint.ops = &dwc3_gadget_ep0_ops; 3144 if (!dep->direction) 3145 dwc->gadget->ep0 = &dep->endpoint; 3146 3147 dep->endpoint.caps.type_control = true; 3148 3149 return 0; 3150 } 3151 3152 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep) 3153 { 3154 struct dwc3 *dwc = dep->dwc; 3155 u32 mdwidth; 3156 int size; 3157 int maxpacket; 3158 3159 mdwidth = dwc3_mdwidth(dwc); 3160 3161 /* MDWIDTH is represented in bits, we need it in bytes */ 3162 mdwidth /= 8; 3163 3164 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); 3165 if (DWC3_IP_IS(DWC3)) 3166 size = DWC3_GTXFIFOSIZ_TXFDEP(size); 3167 else 3168 size = DWC31_GTXFIFOSIZ_TXFDEP(size); 3169 3170 /* 3171 * maxpacket size is determined as part of the following, after assuming 3172 * a mult value of one maxpacket: 3173 * DWC3 revision 280A and prior: 3174 * fifo_size = mult * (max_packet / mdwidth) + 1; 3175 * maxpacket = mdwidth * (fifo_size - 1); 3176 * 3177 * DWC3 revision 290A and onwards: 3178 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 3179 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth; 3180 */ 3181 if (DWC3_VER_IS_PRIOR(DWC3, 290A)) 3182 maxpacket = mdwidth * (size - 1); 3183 else 3184 maxpacket = mdwidth * ((size - 1) - 1) - mdwidth; 3185 3186 /* Functionally, space for one max packet is sufficient */ 3187 size = min_t(int, maxpacket, 1024); 3188 usb_ep_set_maxpacket_limit(&dep->endpoint, size); 3189 3190 dep->endpoint.max_streams = 16; 3191 dep->endpoint.ops = &dwc3_gadget_ep_ops; 3192 list_add_tail(&dep->endpoint.ep_list, 3193 &dwc->gadget->ep_list); 3194 dep->endpoint.caps.type_iso = true; 3195 dep->endpoint.caps.type_bulk = true; 3196 dep->endpoint.caps.type_int = true; 3197 3198 return dwc3_alloc_trb_pool(dep); 3199 } 3200 3201 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep) 3202 { 3203 struct dwc3 *dwc = dep->dwc; 3204 u32 mdwidth; 3205 int size; 3206 3207 mdwidth = dwc3_mdwidth(dwc); 3208 3209 /* MDWIDTH is represented in bits, convert to bytes */ 3210 mdwidth /= 8; 3211 3212 /* All OUT endpoints share a single RxFIFO space */ 3213 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0)); 3214 if (DWC3_IP_IS(DWC3)) 3215 size = DWC3_GRXFIFOSIZ_RXFDEP(size); 3216 else 3217 size = DWC31_GRXFIFOSIZ_RXFDEP(size); 3218 3219 /* FIFO depth is in MDWDITH bytes */ 3220 size *= mdwidth; 3221 3222 /* 3223 * To meet performance requirement, a minimum recommended RxFIFO size 3224 * is defined as follow: 3225 * RxFIFO size >= (3 x MaxPacketSize) + 3226 * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin) 3227 * 3228 * Then calculate the max packet limit as below. 3229 */ 3230 size -= (3 * 8) + 16; 3231 if (size < 0) 3232 size = 0; 3233 else 3234 size /= 3; 3235 3236 usb_ep_set_maxpacket_limit(&dep->endpoint, size); 3237 dep->endpoint.max_streams = 16; 3238 dep->endpoint.ops = &dwc3_gadget_ep_ops; 3239 list_add_tail(&dep->endpoint.ep_list, 3240 &dwc->gadget->ep_list); 3241 dep->endpoint.caps.type_iso = true; 3242 dep->endpoint.caps.type_bulk = true; 3243 dep->endpoint.caps.type_int = true; 3244 3245 return dwc3_alloc_trb_pool(dep); 3246 } 3247 3248 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum) 3249 { 3250 struct dwc3_ep *dep; 3251 bool direction = epnum & 1; 3252 int ret; 3253 u8 num = epnum >> 1; 3254 3255 dep = kzalloc(sizeof(*dep), GFP_KERNEL); 3256 if (!dep) 3257 return -ENOMEM; 3258 3259 dep->dwc = dwc; 3260 dep->number = epnum; 3261 dep->direction = direction; 3262 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); 3263 dwc->eps[epnum] = dep; 3264 dep->combo_num = 0; 3265 dep->start_cmd_status = 0; 3266 3267 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, 3268 direction ? "in" : "out"); 3269 3270 dep->endpoint.name = dep->name; 3271 3272 if (!(dep->number > 1)) { 3273 dep->endpoint.desc = &dwc3_gadget_ep0_desc; 3274 dep->endpoint.comp_desc = NULL; 3275 } 3276 3277 if (num == 0) 3278 ret = dwc3_gadget_init_control_endpoint(dep); 3279 else if (direction) 3280 ret = dwc3_gadget_init_in_endpoint(dep); 3281 else 3282 ret = dwc3_gadget_init_out_endpoint(dep); 3283 3284 if (ret) 3285 return ret; 3286 3287 dep->endpoint.caps.dir_in = direction; 3288 dep->endpoint.caps.dir_out = !direction; 3289 3290 INIT_LIST_HEAD(&dep->pending_list); 3291 INIT_LIST_HEAD(&dep->started_list); 3292 INIT_LIST_HEAD(&dep->cancelled_list); 3293 3294 dwc3_debugfs_create_endpoint_dir(dep); 3295 3296 return 0; 3297 } 3298 3299 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total) 3300 { 3301 u8 epnum; 3302 3303 INIT_LIST_HEAD(&dwc->gadget->ep_list); 3304 3305 for (epnum = 0; epnum < total; epnum++) { 3306 int ret; 3307 3308 ret = dwc3_gadget_init_endpoint(dwc, epnum); 3309 if (ret) 3310 return ret; 3311 } 3312 3313 return 0; 3314 } 3315 3316 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc) 3317 { 3318 struct dwc3_ep *dep; 3319 u8 epnum; 3320 3321 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 3322 dep = dwc->eps[epnum]; 3323 if (!dep) 3324 continue; 3325 /* 3326 * Physical endpoints 0 and 1 are special; they form the 3327 * bi-directional USB endpoint 0. 3328 * 3329 * For those two physical endpoints, we don't allocate a TRB 3330 * pool nor do we add them the endpoints list. Due to that, we 3331 * shouldn't do these two operations otherwise we would end up 3332 * with all sorts of bugs when removing dwc3.ko. 3333 */ 3334 if (epnum != 0 && epnum != 1) { 3335 dwc3_free_trb_pool(dep); 3336 list_del(&dep->endpoint.ep_list); 3337 } 3338 3339 dwc3_debugfs_remove_endpoint_dir(dep); 3340 kfree(dep); 3341 } 3342 } 3343 3344 /* -------------------------------------------------------------------------- */ 3345 3346 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep, 3347 struct dwc3_request *req, struct dwc3_trb *trb, 3348 const struct dwc3_event_depevt *event, int status, int chain) 3349 { 3350 unsigned int count; 3351 3352 dwc3_ep_inc_deq(dep); 3353 3354 trace_dwc3_complete_trb(dep, trb); 3355 req->num_trbs--; 3356 3357 /* 3358 * If we're in the middle of series of chained TRBs and we 3359 * receive a short transfer along the way, DWC3 will skip 3360 * through all TRBs including the last TRB in the chain (the 3361 * where CHN bit is zero. DWC3 will also avoid clearing HWO 3362 * bit and SW has to do it manually. 3363 * 3364 * We're going to do that here to avoid problems of HW trying 3365 * to use bogus TRBs for transfers. 3366 */ 3367 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO)) 3368 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 3369 3370 /* 3371 * For isochronous transfers, the first TRB in a service interval must 3372 * have the Isoc-First type. Track and report its interval frame number. 3373 */ 3374 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && 3375 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) { 3376 unsigned int frame_number; 3377 3378 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl); 3379 frame_number &= ~(dep->interval - 1); 3380 req->request.frame_number = frame_number; 3381 } 3382 3383 /* 3384 * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If 3385 * this TRB points to the bounce buffer address, it's a MPS alignment 3386 * TRB. Don't add it to req->remaining calculation. 3387 */ 3388 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) && 3389 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) { 3390 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 3391 return 1; 3392 } 3393 3394 count = trb->size & DWC3_TRB_SIZE_MASK; 3395 req->remaining += count; 3396 3397 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) 3398 return 1; 3399 3400 if (event->status & DEPEVT_STATUS_SHORT && !chain) 3401 return 1; 3402 3403 if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) && 3404 DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC) 3405 return 1; 3406 3407 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) || 3408 (trb->ctrl & DWC3_TRB_CTRL_LST)) 3409 return 1; 3410 3411 return 0; 3412 } 3413 3414 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep, 3415 struct dwc3_request *req, const struct dwc3_event_depevt *event, 3416 int status) 3417 { 3418 struct dwc3_trb *trb; 3419 struct scatterlist *sg = req->sg; 3420 struct scatterlist *s; 3421 unsigned int num_queued = req->num_queued_sgs; 3422 unsigned int i; 3423 int ret = 0; 3424 3425 for_each_sg(sg, s, num_queued, i) { 3426 trb = &dep->trb_pool[dep->trb_dequeue]; 3427 3428 req->sg = sg_next(s); 3429 req->num_queued_sgs--; 3430 3431 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req, 3432 trb, event, status, true); 3433 if (ret) 3434 break; 3435 } 3436 3437 return ret; 3438 } 3439 3440 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep, 3441 struct dwc3_request *req, const struct dwc3_event_depevt *event, 3442 int status) 3443 { 3444 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; 3445 3446 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb, 3447 event, status, false); 3448 } 3449 3450 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req) 3451 { 3452 return req->num_pending_sgs == 0 && req->num_queued_sgs == 0; 3453 } 3454 3455 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep, 3456 const struct dwc3_event_depevt *event, 3457 struct dwc3_request *req, int status) 3458 { 3459 int request_status; 3460 int ret; 3461 3462 if (req->request.num_mapped_sgs) 3463 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event, 3464 status); 3465 else 3466 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, 3467 status); 3468 3469 req->request.actual = req->request.length - req->remaining; 3470 3471 if (!dwc3_gadget_ep_request_completed(req)) 3472 goto out; 3473 3474 if (req->needs_extra_trb) { 3475 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, 3476 status); 3477 req->needs_extra_trb = false; 3478 } 3479 3480 /* 3481 * The event status only reflects the status of the TRB with IOC set. 3482 * For the requests that don't set interrupt on completion, the driver 3483 * needs to check and return the status of the completed TRBs associated 3484 * with the request. Use the status of the last TRB of the request. 3485 */ 3486 if (req->request.no_interrupt) { 3487 struct dwc3_trb *trb; 3488 3489 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue); 3490 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) { 3491 case DWC3_TRBSTS_MISSED_ISOC: 3492 /* Isoc endpoint only */ 3493 request_status = -EXDEV; 3494 break; 3495 case DWC3_TRB_STS_XFER_IN_PROG: 3496 /* Applicable when End Transfer with ForceRM=0 */ 3497 case DWC3_TRBSTS_SETUP_PENDING: 3498 /* Control endpoint only */ 3499 case DWC3_TRBSTS_OK: 3500 default: 3501 request_status = 0; 3502 break; 3503 } 3504 } else { 3505 request_status = status; 3506 } 3507 3508 dwc3_gadget_giveback(dep, req, request_status); 3509 3510 out: 3511 return ret; 3512 } 3513 3514 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep, 3515 const struct dwc3_event_depevt *event, int status) 3516 { 3517 struct dwc3_request *req; 3518 3519 while (!list_empty(&dep->started_list)) { 3520 int ret; 3521 3522 req = next_request(&dep->started_list); 3523 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event, 3524 req, status); 3525 if (ret) 3526 break; 3527 /* 3528 * The endpoint is disabled, let the dwc3_remove_requests() 3529 * handle the cleanup. 3530 */ 3531 if (!dep->endpoint.desc) 3532 break; 3533 } 3534 } 3535 3536 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep) 3537 { 3538 struct dwc3_request *req; 3539 struct dwc3 *dwc = dep->dwc; 3540 3541 if (!dep->endpoint.desc || !dwc->pullups_connected || 3542 !dwc->connected) 3543 return false; 3544 3545 if (!list_empty(&dep->pending_list)) 3546 return true; 3547 3548 /* 3549 * We only need to check the first entry of the started list. We can 3550 * assume the completed requests are removed from the started list. 3551 */ 3552 req = next_request(&dep->started_list); 3553 if (!req) 3554 return false; 3555 3556 return !dwc3_gadget_ep_request_completed(req); 3557 } 3558 3559 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep, 3560 const struct dwc3_event_depevt *event) 3561 { 3562 dep->frame_number = event->parameters; 3563 } 3564 3565 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep, 3566 const struct dwc3_event_depevt *event, int status) 3567 { 3568 struct dwc3 *dwc = dep->dwc; 3569 bool no_started_trb = true; 3570 3571 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status); 3572 3573 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) 3574 goto out; 3575 3576 if (!dep->endpoint.desc) 3577 return no_started_trb; 3578 3579 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && 3580 list_empty(&dep->started_list) && 3581 (list_empty(&dep->pending_list) || status == -EXDEV)) 3582 dwc3_stop_active_transfer(dep, true, true); 3583 else if (dwc3_gadget_ep_should_continue(dep)) 3584 if (__dwc3_gadget_kick_transfer(dep) == 0) 3585 no_started_trb = false; 3586 3587 out: 3588 /* 3589 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. 3590 * See dwc3_gadget_linksts_change_interrupt() for 1st half. 3591 */ 3592 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) { 3593 u32 reg; 3594 int i; 3595 3596 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) { 3597 dep = dwc->eps[i]; 3598 3599 if (!(dep->flags & DWC3_EP_ENABLED)) 3600 continue; 3601 3602 if (!list_empty(&dep->started_list)) 3603 return no_started_trb; 3604 } 3605 3606 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 3607 reg |= dwc->u1u2; 3608 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 3609 3610 dwc->u1u2 = 0; 3611 } 3612 3613 return no_started_trb; 3614 } 3615 3616 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep, 3617 const struct dwc3_event_depevt *event) 3618 { 3619 int status = 0; 3620 3621 if (!dep->endpoint.desc) 3622 return; 3623 3624 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) 3625 dwc3_gadget_endpoint_frame_from_event(dep, event); 3626 3627 if (event->status & DEPEVT_STATUS_BUSERR) 3628 status = -ECONNRESET; 3629 3630 if (event->status & DEPEVT_STATUS_MISSED_ISOC) 3631 status = -EXDEV; 3632 3633 dwc3_gadget_endpoint_trbs_complete(dep, event, status); 3634 } 3635 3636 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep, 3637 const struct dwc3_event_depevt *event) 3638 { 3639 int status = 0; 3640 3641 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 3642 3643 if (event->status & DEPEVT_STATUS_BUSERR) 3644 status = -ECONNRESET; 3645 3646 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status)) 3647 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; 3648 } 3649 3650 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep, 3651 const struct dwc3_event_depevt *event) 3652 { 3653 dwc3_gadget_endpoint_frame_from_event(dep, event); 3654 3655 /* 3656 * The XferNotReady event is generated only once before the endpoint 3657 * starts. It will be generated again when END_TRANSFER command is 3658 * issued. For some controller versions, the XferNotReady event may be 3659 * generated while the END_TRANSFER command is still in process. Ignore 3660 * it and wait for the next XferNotReady event after the command is 3661 * completed. 3662 */ 3663 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) 3664 return; 3665 3666 (void) __dwc3_gadget_start_isoc(dep); 3667 } 3668 3669 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep, 3670 const struct dwc3_event_depevt *event) 3671 { 3672 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters); 3673 3674 if (cmd != DWC3_DEPCMD_ENDTRANSFER) 3675 return; 3676 3677 /* 3678 * The END_TRANSFER command will cause the controller to generate a 3679 * NoStream Event, and it's not due to the host DP NoStream rejection. 3680 * Ignore the next NoStream event. 3681 */ 3682 if (dep->stream_capable) 3683 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM; 3684 3685 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; 3686 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 3687 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 3688 3689 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) { 3690 struct dwc3 *dwc = dep->dwc; 3691 3692 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL; 3693 if (dwc3_send_clear_stall_ep_cmd(dep)) { 3694 struct usb_ep *ep0 = &dwc->eps[0]->endpoint; 3695 3696 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name); 3697 if (dwc->delayed_status) 3698 __dwc3_gadget_ep0_set_halt(ep0, 1); 3699 return; 3700 } 3701 3702 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 3703 if (dwc->clear_stall_protocol == dep->number) 3704 dwc3_ep0_send_delayed_status(dwc); 3705 } 3706 3707 if ((dep->flags & DWC3_EP_DELAY_START) && 3708 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) 3709 __dwc3_gadget_kick_transfer(dep); 3710 3711 dep->flags &= ~DWC3_EP_DELAY_START; 3712 } 3713 3714 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep, 3715 const struct dwc3_event_depevt *event) 3716 { 3717 struct dwc3 *dwc = dep->dwc; 3718 3719 if (event->status == DEPEVT_STREAMEVT_FOUND) { 3720 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; 3721 goto out; 3722 } 3723 3724 /* Note: NoStream rejection event param value is 0 and not 0xFFFF */ 3725 switch (event->parameters) { 3726 case DEPEVT_STREAM_PRIME: 3727 /* 3728 * If the host can properly transition the endpoint state from 3729 * idle to prime after a NoStream rejection, there's no need to 3730 * force restarting the endpoint to reinitiate the stream. To 3731 * simplify the check, assume the host follows the USB spec if 3732 * it primed the endpoint more than once. 3733 */ 3734 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) { 3735 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED) 3736 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM; 3737 else 3738 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; 3739 } 3740 3741 break; 3742 case DEPEVT_STREAM_NOSTREAM: 3743 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) || 3744 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) || 3745 (!DWC3_MST_CAPABLE(&dwc->hwparams) && 3746 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))) 3747 break; 3748 3749 /* 3750 * If the host rejects a stream due to no active stream, by the 3751 * USB and xHCI spec, the endpoint will be put back to idle 3752 * state. When the host is ready (buffer added/updated), it will 3753 * prime the endpoint to inform the usb device controller. This 3754 * triggers the device controller to issue ERDY to restart the 3755 * stream. However, some hosts don't follow this and keep the 3756 * endpoint in the idle state. No prime will come despite host 3757 * streams are updated, and the device controller will not be 3758 * triggered to generate ERDY to move the next stream data. To 3759 * workaround this and maintain compatibility with various 3760 * hosts, force to reinitiate the stream until the host is ready 3761 * instead of waiting for the host to prime the endpoint. 3762 */ 3763 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) { 3764 unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME; 3765 3766 dwc3_send_gadget_generic_command(dwc, cmd, dep->number); 3767 } else { 3768 dep->flags |= DWC3_EP_DELAY_START; 3769 dwc3_stop_active_transfer(dep, true, true); 3770 return; 3771 } 3772 break; 3773 } 3774 3775 out: 3776 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; 3777 } 3778 3779 static void dwc3_endpoint_interrupt(struct dwc3 *dwc, 3780 const struct dwc3_event_depevt *event) 3781 { 3782 struct dwc3_ep *dep; 3783 u8 epnum = event->endpoint_number; 3784 3785 dep = dwc->eps[epnum]; 3786 3787 if (!(dep->flags & DWC3_EP_ENABLED)) { 3788 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED)) 3789 return; 3790 3791 /* Handle only EPCMDCMPLT when EP disabled */ 3792 if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) && 3793 !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE)) 3794 return; 3795 } 3796 3797 if (epnum == 0 || epnum == 1) { 3798 dwc3_ep0_interrupt(dwc, event); 3799 return; 3800 } 3801 3802 switch (event->endpoint_event) { 3803 case DWC3_DEPEVT_XFERINPROGRESS: 3804 dwc3_gadget_endpoint_transfer_in_progress(dep, event); 3805 break; 3806 case DWC3_DEPEVT_XFERNOTREADY: 3807 dwc3_gadget_endpoint_transfer_not_ready(dep, event); 3808 break; 3809 case DWC3_DEPEVT_EPCMDCMPLT: 3810 dwc3_gadget_endpoint_command_complete(dep, event); 3811 break; 3812 case DWC3_DEPEVT_XFERCOMPLETE: 3813 dwc3_gadget_endpoint_transfer_complete(dep, event); 3814 break; 3815 case DWC3_DEPEVT_STREAMEVT: 3816 dwc3_gadget_endpoint_stream_event(dep, event); 3817 break; 3818 case DWC3_DEPEVT_RXTXFIFOEVT: 3819 break; 3820 default: 3821 dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event); 3822 break; 3823 } 3824 } 3825 3826 static void dwc3_disconnect_gadget(struct dwc3 *dwc) 3827 { 3828 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) { 3829 spin_unlock(&dwc->lock); 3830 dwc->gadget_driver->disconnect(dwc->gadget); 3831 spin_lock(&dwc->lock); 3832 } 3833 } 3834 3835 static void dwc3_suspend_gadget(struct dwc3 *dwc) 3836 { 3837 if (dwc->async_callbacks && dwc->gadget_driver->suspend) { 3838 spin_unlock(&dwc->lock); 3839 dwc->gadget_driver->suspend(dwc->gadget); 3840 spin_lock(&dwc->lock); 3841 } 3842 } 3843 3844 static void dwc3_resume_gadget(struct dwc3 *dwc) 3845 { 3846 if (dwc->async_callbacks && dwc->gadget_driver->resume) { 3847 spin_unlock(&dwc->lock); 3848 dwc->gadget_driver->resume(dwc->gadget); 3849 spin_lock(&dwc->lock); 3850 } 3851 } 3852 3853 static void dwc3_reset_gadget(struct dwc3 *dwc) 3854 { 3855 if (!dwc->gadget_driver) 3856 return; 3857 3858 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) { 3859 spin_unlock(&dwc->lock); 3860 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver); 3861 spin_lock(&dwc->lock); 3862 } 3863 } 3864 3865 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, 3866 bool interrupt) 3867 { 3868 struct dwc3 *dwc = dep->dwc; 3869 3870 /* 3871 * Only issue End Transfer command to the control endpoint of a started 3872 * Data Phase. Typically we should only do so in error cases such as 3873 * invalid/unexpected direction as described in the control transfer 3874 * flow of the programming guide. 3875 */ 3876 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE) 3877 return; 3878 3879 if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP)) 3880 return; 3881 3882 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || 3883 (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) 3884 return; 3885 3886 /* 3887 * If a Setup packet is received but yet to DMA out, the controller will 3888 * not process the End Transfer command of any endpoint. Polling of its 3889 * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a 3890 * timeout. Delay issuing the End Transfer command until the Setup TRB is 3891 * prepared. 3892 */ 3893 if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) { 3894 dep->flags |= DWC3_EP_DELAY_STOP; 3895 return; 3896 } 3897 3898 /* 3899 * NOTICE: We are violating what the Databook says about the 3900 * EndTransfer command. Ideally we would _always_ wait for the 3901 * EndTransfer Command Completion IRQ, but that's causing too 3902 * much trouble synchronizing between us and gadget driver. 3903 * 3904 * We have discussed this with the IP Provider and it was 3905 * suggested to giveback all requests here. 3906 * 3907 * Note also that a similar handling was tested by Synopsys 3908 * (thanks a lot Paul) and nothing bad has come out of it. 3909 * In short, what we're doing is issuing EndTransfer with 3910 * CMDIOC bit set and delay kicking transfer until the 3911 * EndTransfer command had completed. 3912 * 3913 * As of IP version 3.10a of the DWC_usb3 IP, the controller 3914 * supports a mode to work around the above limitation. The 3915 * software can poll the CMDACT bit in the DEPCMD register 3916 * after issuing a EndTransfer command. This mode is enabled 3917 * by writing GUCTL2[14]. This polling is already done in the 3918 * dwc3_send_gadget_ep_cmd() function so if the mode is 3919 * enabled, the EndTransfer command will have completed upon 3920 * returning from this function. 3921 * 3922 * This mode is NOT available on the DWC_usb31 IP. In this 3923 * case, if the IOC bit is not set, then delay by 1ms 3924 * after issuing the EndTransfer command. This allows for the 3925 * controller to handle the command completely before DWC3 3926 * remove requests attempts to unmap USB request buffers. 3927 */ 3928 3929 __dwc3_stop_active_transfer(dep, force, interrupt); 3930 } 3931 3932 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc) 3933 { 3934 u32 epnum; 3935 3936 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 3937 struct dwc3_ep *dep; 3938 int ret; 3939 3940 dep = dwc->eps[epnum]; 3941 if (!dep) 3942 continue; 3943 3944 if (!(dep->flags & DWC3_EP_STALL)) 3945 continue; 3946 3947 dep->flags &= ~DWC3_EP_STALL; 3948 3949 ret = dwc3_send_clear_stall_ep_cmd(dep); 3950 WARN_ON_ONCE(ret); 3951 } 3952 } 3953 3954 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc) 3955 { 3956 int reg; 3957 3958 dwc->suspended = false; 3959 3960 dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET); 3961 3962 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 3963 reg &= ~DWC3_DCTL_INITU1ENA; 3964 reg &= ~DWC3_DCTL_INITU2ENA; 3965 dwc3_gadget_dctl_write_safe(dwc, reg); 3966 3967 dwc->connected = false; 3968 3969 dwc3_disconnect_gadget(dwc); 3970 3971 dwc->gadget->speed = USB_SPEED_UNKNOWN; 3972 dwc->setup_packet_pending = false; 3973 dwc->gadget->wakeup_armed = false; 3974 dwc3_gadget_enable_linksts_evts(dwc, false); 3975 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); 3976 3977 dwc3_ep0_reset_state(dwc); 3978 3979 /* 3980 * Request PM idle to address condition where usage count is 3981 * already decremented to zero, but waiting for the disconnect 3982 * interrupt to set dwc->connected to FALSE. 3983 */ 3984 pm_request_idle(dwc->dev); 3985 } 3986 3987 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) 3988 { 3989 u32 reg; 3990 3991 dwc->suspended = false; 3992 3993 /* 3994 * Ideally, dwc3_reset_gadget() would trigger the function 3995 * drivers to stop any active transfers through ep disable. 3996 * However, for functions which defer ep disable, such as mass 3997 * storage, we will need to rely on the call to stop active 3998 * transfers here, and avoid allowing of request queuing. 3999 */ 4000 dwc->connected = false; 4001 4002 /* 4003 * WORKAROUND: DWC3 revisions <1.88a have an issue which 4004 * would cause a missing Disconnect Event if there's a 4005 * pending Setup Packet in the FIFO. 4006 * 4007 * There's no suggested workaround on the official Bug 4008 * report, which states that "unless the driver/application 4009 * is doing any special handling of a disconnect event, 4010 * there is no functional issue". 4011 * 4012 * Unfortunately, it turns out that we _do_ some special 4013 * handling of a disconnect event, namely complete all 4014 * pending transfers, notify gadget driver of the 4015 * disconnection, and so on. 4016 * 4017 * Our suggested workaround is to follow the Disconnect 4018 * Event steps here, instead, based on a setup_packet_pending 4019 * flag. Such flag gets set whenever we have a SETUP_PENDING 4020 * status for EP0 TRBs and gets cleared on XferComplete for the 4021 * same endpoint. 4022 * 4023 * Refers to: 4024 * 4025 * STAR#9000466709: RTL: Device : Disconnect event not 4026 * generated if setup packet pending in FIFO 4027 */ 4028 if (DWC3_VER_IS_PRIOR(DWC3, 188A)) { 4029 if (dwc->setup_packet_pending) 4030 dwc3_gadget_disconnect_interrupt(dwc); 4031 } 4032 4033 dwc3_reset_gadget(dwc); 4034 4035 /* 4036 * From SNPS databook section 8.1.2, the EP0 should be in setup 4037 * phase. So ensure that EP0 is in setup phase by issuing a stall 4038 * and restart if EP0 is not in setup phase. 4039 */ 4040 dwc3_ep0_reset_state(dwc); 4041 4042 /* 4043 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a 4044 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW 4045 * needs to ensure that it sends "a DEPENDXFER command for any active 4046 * transfers." 4047 */ 4048 dwc3_stop_active_transfers(dwc); 4049 dwc->connected = true; 4050 4051 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4052 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 4053 dwc3_gadget_dctl_write_safe(dwc, reg); 4054 dwc->test_mode = false; 4055 dwc->gadget->wakeup_armed = false; 4056 dwc3_gadget_enable_linksts_evts(dwc, false); 4057 dwc3_clear_stall_all_ep(dwc); 4058 4059 /* Reset device address to zero */ 4060 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 4061 reg &= ~(DWC3_DCFG_DEVADDR_MASK); 4062 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 4063 } 4064 4065 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) 4066 { 4067 struct dwc3_ep *dep; 4068 int ret; 4069 u32 reg; 4070 u8 lanes = 1; 4071 u8 speed; 4072 4073 if (!dwc->softconnect) 4074 return; 4075 4076 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 4077 speed = reg & DWC3_DSTS_CONNECTSPD; 4078 dwc->speed = speed; 4079 4080 if (DWC3_IP_IS(DWC32)) 4081 lanes = DWC3_DSTS_CONNLANES(reg) + 1; 4082 4083 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; 4084 4085 /* 4086 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed 4087 * each time on Connect Done. 4088 * 4089 * Currently we always use the reset value. If any platform 4090 * wants to set this to a different value, we need to add a 4091 * setting and update GCTL.RAMCLKSEL here. 4092 */ 4093 4094 switch (speed) { 4095 case DWC3_DSTS_SUPERSPEED_PLUS: 4096 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 4097 dwc->gadget->ep0->maxpacket = 512; 4098 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; 4099 4100 if (lanes > 1) 4101 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2; 4102 else 4103 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1; 4104 break; 4105 case DWC3_DSTS_SUPERSPEED: 4106 /* 4107 * WORKAROUND: DWC3 revisions <1.90a have an issue which 4108 * would cause a missing USB3 Reset event. 4109 * 4110 * In such situations, we should force a USB3 Reset 4111 * event by calling our dwc3_gadget_reset_interrupt() 4112 * routine. 4113 * 4114 * Refers to: 4115 * 4116 * STAR#9000483510: RTL: SS : USB3 reset event may 4117 * not be generated always when the link enters poll 4118 */ 4119 if (DWC3_VER_IS_PRIOR(DWC3, 190A)) 4120 dwc3_gadget_reset_interrupt(dwc); 4121 4122 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 4123 dwc->gadget->ep0->maxpacket = 512; 4124 dwc->gadget->speed = USB_SPEED_SUPER; 4125 4126 if (lanes > 1) { 4127 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; 4128 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2; 4129 } 4130 break; 4131 case DWC3_DSTS_HIGHSPEED: 4132 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 4133 dwc->gadget->ep0->maxpacket = 64; 4134 dwc->gadget->speed = USB_SPEED_HIGH; 4135 break; 4136 case DWC3_DSTS_FULLSPEED: 4137 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 4138 dwc->gadget->ep0->maxpacket = 64; 4139 dwc->gadget->speed = USB_SPEED_FULL; 4140 break; 4141 } 4142 4143 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket; 4144 4145 /* Enable USB2 LPM Capability */ 4146 4147 if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) && 4148 !dwc->usb2_gadget_lpm_disable && 4149 (speed != DWC3_DSTS_SUPERSPEED) && 4150 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) { 4151 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 4152 reg |= DWC3_DCFG_LPM_CAP; 4153 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 4154 4155 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4156 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN); 4157 4158 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold | 4159 (dwc->is_utmi_l1_suspend << 4)); 4160 4161 /* 4162 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and 4163 * DCFG.LPMCap is set, core responses with an ACK and the 4164 * BESL value in the LPM token is less than or equal to LPM 4165 * NYET threshold. 4166 */ 4167 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum, 4168 "LPM Erratum not available on dwc3 revisions < 2.40a\n"); 4169 4170 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A)) 4171 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold); 4172 4173 dwc3_gadget_dctl_write_safe(dwc, reg); 4174 } else { 4175 if (dwc->usb2_gadget_lpm_disable) { 4176 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 4177 reg &= ~DWC3_DCFG_LPM_CAP; 4178 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 4179 } 4180 4181 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4182 reg &= ~DWC3_DCTL_HIRD_THRES_MASK; 4183 dwc3_gadget_dctl_write_safe(dwc, reg); 4184 } 4185 4186 dep = dwc->eps[0]; 4187 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); 4188 if (ret) { 4189 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 4190 return; 4191 } 4192 4193 dep = dwc->eps[1]; 4194 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); 4195 if (ret) { 4196 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 4197 return; 4198 } 4199 4200 /* 4201 * Configure PHY via GUSB3PIPECTLn if required. 4202 * 4203 * Update GTXFIFOSIZn 4204 * 4205 * In both cases reset values should be sufficient. 4206 */ 4207 } 4208 4209 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo) 4210 { 4211 dwc->suspended = false; 4212 4213 /* 4214 * TODO take core out of low power mode when that's 4215 * implemented. 4216 */ 4217 4218 if (dwc->async_callbacks && dwc->gadget_driver->resume) { 4219 spin_unlock(&dwc->lock); 4220 dwc->gadget_driver->resume(dwc->gadget); 4221 spin_lock(&dwc->lock); 4222 } 4223 4224 dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK; 4225 } 4226 4227 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc, 4228 unsigned int evtinfo) 4229 { 4230 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 4231 unsigned int pwropt; 4232 4233 /* 4234 * WORKAROUND: DWC3 < 2.50a have an issue when configured without 4235 * Hibernation mode enabled which would show up when device detects 4236 * host-initiated U3 exit. 4237 * 4238 * In that case, device will generate a Link State Change Interrupt 4239 * from U3 to RESUME which is only necessary if Hibernation is 4240 * configured in. 4241 * 4242 * There are no functional changes due to such spurious event and we 4243 * just need to ignore it. 4244 * 4245 * Refers to: 4246 * 4247 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation 4248 * operational mode 4249 */ 4250 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1); 4251 if (DWC3_VER_IS_PRIOR(DWC3, 250A) && 4252 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) { 4253 if ((dwc->link_state == DWC3_LINK_STATE_U3) && 4254 (next == DWC3_LINK_STATE_RESUME)) { 4255 return; 4256 } 4257 } 4258 4259 /* 4260 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending 4261 * on the link partner, the USB session might do multiple entry/exit 4262 * of low power states before a transfer takes place. 4263 * 4264 * Due to this problem, we might experience lower throughput. The 4265 * suggested workaround is to disable DCTL[12:9] bits if we're 4266 * transitioning from U1/U2 to U0 and enable those bits again 4267 * after a transfer completes and there are no pending transfers 4268 * on any of the enabled endpoints. 4269 * 4270 * This is the first half of that workaround. 4271 * 4272 * Refers to: 4273 * 4274 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us 4275 * core send LGO_Ux entering U0 4276 */ 4277 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) { 4278 if (next == DWC3_LINK_STATE_U0) { 4279 u32 u1u2; 4280 u32 reg; 4281 4282 switch (dwc->link_state) { 4283 case DWC3_LINK_STATE_U1: 4284 case DWC3_LINK_STATE_U2: 4285 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4286 u1u2 = reg & (DWC3_DCTL_INITU2ENA 4287 | DWC3_DCTL_ACCEPTU2ENA 4288 | DWC3_DCTL_INITU1ENA 4289 | DWC3_DCTL_ACCEPTU1ENA); 4290 4291 if (!dwc->u1u2) 4292 dwc->u1u2 = reg & u1u2; 4293 4294 reg &= ~u1u2; 4295 4296 dwc3_gadget_dctl_write_safe(dwc, reg); 4297 break; 4298 default: 4299 /* do nothing */ 4300 break; 4301 } 4302 } 4303 } 4304 4305 switch (next) { 4306 case DWC3_LINK_STATE_U0: 4307 if (dwc->gadget->wakeup_armed) { 4308 dwc3_gadget_enable_linksts_evts(dwc, false); 4309 dwc3_resume_gadget(dwc); 4310 dwc->suspended = false; 4311 } 4312 break; 4313 case DWC3_LINK_STATE_U1: 4314 if (dwc->speed == USB_SPEED_SUPER) 4315 dwc3_suspend_gadget(dwc); 4316 break; 4317 case DWC3_LINK_STATE_U2: 4318 case DWC3_LINK_STATE_U3: 4319 dwc3_suspend_gadget(dwc); 4320 break; 4321 case DWC3_LINK_STATE_RESUME: 4322 dwc3_resume_gadget(dwc); 4323 break; 4324 default: 4325 /* do nothing */ 4326 break; 4327 } 4328 4329 dwc->link_state = next; 4330 } 4331 4332 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc, 4333 unsigned int evtinfo) 4334 { 4335 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 4336 4337 if (!dwc->suspended && next == DWC3_LINK_STATE_U3) { 4338 dwc->suspended = true; 4339 dwc3_suspend_gadget(dwc); 4340 } 4341 4342 dwc->link_state = next; 4343 } 4344 4345 static void dwc3_gadget_interrupt(struct dwc3 *dwc, 4346 const struct dwc3_event_devt *event) 4347 { 4348 switch (event->type) { 4349 case DWC3_DEVICE_EVENT_DISCONNECT: 4350 dwc3_gadget_disconnect_interrupt(dwc); 4351 break; 4352 case DWC3_DEVICE_EVENT_RESET: 4353 dwc3_gadget_reset_interrupt(dwc); 4354 break; 4355 case DWC3_DEVICE_EVENT_CONNECT_DONE: 4356 dwc3_gadget_conndone_interrupt(dwc); 4357 break; 4358 case DWC3_DEVICE_EVENT_WAKEUP: 4359 dwc3_gadget_wakeup_interrupt(dwc, event->event_info); 4360 break; 4361 case DWC3_DEVICE_EVENT_HIBER_REQ: 4362 dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n"); 4363 break; 4364 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE: 4365 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info); 4366 break; 4367 case DWC3_DEVICE_EVENT_SUSPEND: 4368 /* It changed to be suspend event for version 2.30a and above */ 4369 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) 4370 dwc3_gadget_suspend_interrupt(dwc, event->event_info); 4371 break; 4372 case DWC3_DEVICE_EVENT_SOF: 4373 case DWC3_DEVICE_EVENT_ERRATIC_ERROR: 4374 case DWC3_DEVICE_EVENT_CMD_CMPL: 4375 case DWC3_DEVICE_EVENT_OVERFLOW: 4376 break; 4377 default: 4378 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type); 4379 } 4380 } 4381 4382 static void dwc3_process_event_entry(struct dwc3 *dwc, 4383 const union dwc3_event *event) 4384 { 4385 trace_dwc3_event(event->raw, dwc); 4386 4387 if (!event->type.is_devspec) 4388 dwc3_endpoint_interrupt(dwc, &event->depevt); 4389 else if (event->type.type == DWC3_EVENT_TYPE_DEV) 4390 dwc3_gadget_interrupt(dwc, &event->devt); 4391 else 4392 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw); 4393 } 4394 4395 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt) 4396 { 4397 struct dwc3 *dwc = evt->dwc; 4398 irqreturn_t ret = IRQ_NONE; 4399 int left; 4400 4401 left = evt->count; 4402 4403 if (!(evt->flags & DWC3_EVENT_PENDING)) 4404 return IRQ_NONE; 4405 4406 while (left > 0) { 4407 union dwc3_event event; 4408 4409 event.raw = *(u32 *) (evt->cache + evt->lpos); 4410 4411 dwc3_process_event_entry(dwc, &event); 4412 4413 /* 4414 * FIXME we wrap around correctly to the next entry as 4415 * almost all entries are 4 bytes in size. There is one 4416 * entry which has 12 bytes which is a regular entry 4417 * followed by 8 bytes data. ATM I don't know how 4418 * things are organized if we get next to the a 4419 * boundary so I worry about that once we try to handle 4420 * that. 4421 */ 4422 evt->lpos = (evt->lpos + 4) % evt->length; 4423 left -= 4; 4424 } 4425 4426 evt->count = 0; 4427 ret = IRQ_HANDLED; 4428 4429 /* Unmask interrupt */ 4430 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), 4431 DWC3_GEVNTSIZ_SIZE(evt->length)); 4432 4433 if (dwc->imod_interval) { 4434 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); 4435 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); 4436 } 4437 4438 /* Keep the clearing of DWC3_EVENT_PENDING at the end */ 4439 evt->flags &= ~DWC3_EVENT_PENDING; 4440 4441 return ret; 4442 } 4443 4444 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt) 4445 { 4446 struct dwc3_event_buffer *evt = _evt; 4447 struct dwc3 *dwc = evt->dwc; 4448 unsigned long flags; 4449 irqreturn_t ret = IRQ_NONE; 4450 4451 local_bh_disable(); 4452 spin_lock_irqsave(&dwc->lock, flags); 4453 ret = dwc3_process_event_buf(evt); 4454 spin_unlock_irqrestore(&dwc->lock, flags); 4455 local_bh_enable(); 4456 4457 return ret; 4458 } 4459 4460 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt) 4461 { 4462 struct dwc3 *dwc = evt->dwc; 4463 u32 amount; 4464 u32 count; 4465 4466 if (pm_runtime_suspended(dwc->dev)) { 4467 dwc->pending_events = true; 4468 /* 4469 * Trigger runtime resume. The get() function will be balanced 4470 * after processing the pending events in dwc3_process_pending 4471 * events(). 4472 */ 4473 pm_runtime_get(dwc->dev); 4474 disable_irq_nosync(dwc->irq_gadget); 4475 return IRQ_HANDLED; 4476 } 4477 4478 /* 4479 * With PCIe legacy interrupt, test shows that top-half irq handler can 4480 * be called again after HW interrupt deassertion. Check if bottom-half 4481 * irq event handler completes before caching new event to prevent 4482 * losing events. 4483 */ 4484 if (evt->flags & DWC3_EVENT_PENDING) 4485 return IRQ_HANDLED; 4486 4487 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); 4488 count &= DWC3_GEVNTCOUNT_MASK; 4489 if (!count) 4490 return IRQ_NONE; 4491 4492 evt->count = count; 4493 evt->flags |= DWC3_EVENT_PENDING; 4494 4495 /* Mask interrupt */ 4496 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), 4497 DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length)); 4498 4499 amount = min(count, evt->length - evt->lpos); 4500 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount); 4501 4502 if (amount < count) 4503 memcpy(evt->cache, evt->buf, count - amount); 4504 4505 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count); 4506 4507 return IRQ_WAKE_THREAD; 4508 } 4509 4510 static irqreturn_t dwc3_interrupt(int irq, void *_evt) 4511 { 4512 struct dwc3_event_buffer *evt = _evt; 4513 4514 return dwc3_check_event_buf(evt); 4515 } 4516 4517 static int dwc3_gadget_get_irq(struct dwc3 *dwc) 4518 { 4519 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev); 4520 int irq; 4521 4522 irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral"); 4523 if (irq > 0) 4524 goto out; 4525 4526 if (irq == -EPROBE_DEFER) 4527 goto out; 4528 4529 irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3"); 4530 if (irq > 0) 4531 goto out; 4532 4533 if (irq == -EPROBE_DEFER) 4534 goto out; 4535 4536 irq = platform_get_irq(dwc3_pdev, 0); 4537 4538 out: 4539 return irq; 4540 } 4541 4542 static void dwc_gadget_release(struct device *dev) 4543 { 4544 struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev); 4545 4546 kfree(gadget); 4547 } 4548 4549 /** 4550 * dwc3_gadget_init - initializes gadget related registers 4551 * @dwc: pointer to our controller context structure 4552 * 4553 * Returns 0 on success otherwise negative errno. 4554 */ 4555 int dwc3_gadget_init(struct dwc3 *dwc) 4556 { 4557 int ret; 4558 int irq; 4559 struct device *dev; 4560 4561 irq = dwc3_gadget_get_irq(dwc); 4562 if (irq < 0) { 4563 ret = irq; 4564 goto err0; 4565 } 4566 4567 dwc->irq_gadget = irq; 4568 4569 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev, 4570 sizeof(*dwc->ep0_trb) * 2, 4571 &dwc->ep0_trb_addr, GFP_KERNEL); 4572 if (!dwc->ep0_trb) { 4573 dev_err(dwc->dev, "failed to allocate ep0 trb\n"); 4574 ret = -ENOMEM; 4575 goto err0; 4576 } 4577 4578 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL); 4579 if (!dwc->setup_buf) { 4580 ret = -ENOMEM; 4581 goto err1; 4582 } 4583 4584 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, 4585 &dwc->bounce_addr, GFP_KERNEL); 4586 if (!dwc->bounce) { 4587 ret = -ENOMEM; 4588 goto err2; 4589 } 4590 4591 init_completion(&dwc->ep0_in_setup); 4592 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL); 4593 if (!dwc->gadget) { 4594 ret = -ENOMEM; 4595 goto err3; 4596 } 4597 4598 4599 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release); 4600 dev = &dwc->gadget->dev; 4601 dev->platform_data = dwc; 4602 dwc->gadget->ops = &dwc3_gadget_ops; 4603 dwc->gadget->speed = USB_SPEED_UNKNOWN; 4604 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; 4605 dwc->gadget->sg_supported = true; 4606 dwc->gadget->name = "dwc3-gadget"; 4607 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable; 4608 dwc->gadget->wakeup_capable = true; 4609 4610 /* 4611 * FIXME We might be setting max_speed to <SUPER, however versions 4612 * <2.20a of dwc3 have an issue with metastability (documented 4613 * elsewhere in this driver) which tells us we can't set max speed to 4614 * anything lower than SUPER. 4615 * 4616 * Because gadget.max_speed is only used by composite.c and function 4617 * drivers (i.e. it won't go into dwc3's registers) we are allowing this 4618 * to happen so we avoid sending SuperSpeed Capability descriptor 4619 * together with our BOS descriptor as that could confuse host into 4620 * thinking we can handle super speed. 4621 * 4622 * Note that, in fact, we won't even support GetBOS requests when speed 4623 * is less than super speed because we don't have means, yet, to tell 4624 * composite.c that we are USB 2.0 + LPM ECN. 4625 */ 4626 if (DWC3_VER_IS_PRIOR(DWC3, 220A) && 4627 !dwc->dis_metastability_quirk) 4628 dev_info(dwc->dev, "changing max_speed on rev %08x\n", 4629 dwc->revision); 4630 4631 dwc->gadget->max_speed = dwc->maximum_speed; 4632 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate; 4633 4634 /* 4635 * REVISIT: Here we should clear all pending IRQs to be 4636 * sure we're starting from a well known location. 4637 */ 4638 4639 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps); 4640 if (ret) 4641 goto err4; 4642 4643 ret = usb_add_gadget(dwc->gadget); 4644 if (ret) { 4645 dev_err(dwc->dev, "failed to add gadget\n"); 4646 goto err5; 4647 } 4648 4649 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS) 4650 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate); 4651 else 4652 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed); 4653 4654 /* No system wakeup if no gadget driver bound */ 4655 if (dwc->sys_wakeup) 4656 device_wakeup_disable(dwc->sysdev); 4657 4658 return 0; 4659 4660 err5: 4661 dwc3_gadget_free_endpoints(dwc); 4662 err4: 4663 usb_put_gadget(dwc->gadget); 4664 dwc->gadget = NULL; 4665 err3: 4666 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, 4667 dwc->bounce_addr); 4668 4669 err2: 4670 kfree(dwc->setup_buf); 4671 4672 err1: 4673 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, 4674 dwc->ep0_trb, dwc->ep0_trb_addr); 4675 4676 err0: 4677 return ret; 4678 } 4679 4680 /* -------------------------------------------------------------------------- */ 4681 4682 void dwc3_gadget_exit(struct dwc3 *dwc) 4683 { 4684 if (!dwc->gadget) 4685 return; 4686 4687 dwc3_enable_susphy(dwc, false); 4688 usb_del_gadget(dwc->gadget); 4689 dwc3_gadget_free_endpoints(dwc); 4690 usb_put_gadget(dwc->gadget); 4691 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, 4692 dwc->bounce_addr); 4693 kfree(dwc->setup_buf); 4694 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, 4695 dwc->ep0_trb, dwc->ep0_trb_addr); 4696 } 4697 4698 int dwc3_gadget_suspend(struct dwc3 *dwc) 4699 { 4700 unsigned long flags; 4701 int ret; 4702 4703 ret = dwc3_gadget_soft_disconnect(dwc); 4704 if (ret) 4705 goto err; 4706 4707 spin_lock_irqsave(&dwc->lock, flags); 4708 if (dwc->gadget_driver) 4709 dwc3_disconnect_gadget(dwc); 4710 spin_unlock_irqrestore(&dwc->lock, flags); 4711 4712 return 0; 4713 4714 err: 4715 /* 4716 * Attempt to reset the controller's state. Likely no 4717 * communication can be established until the host 4718 * performs a port reset. 4719 */ 4720 if (dwc->softconnect) 4721 dwc3_gadget_soft_connect(dwc); 4722 4723 return ret; 4724 } 4725 4726 int dwc3_gadget_resume(struct dwc3 *dwc) 4727 { 4728 if (!dwc->gadget_driver || !dwc->softconnect) 4729 return 0; 4730 4731 return dwc3_gadget_soft_connect(dwc); 4732 } 4733