1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link 4 * 5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com 6 * 7 * Authors: Felipe Balbi <balbi@ti.com>, 8 * Sebastian Andrzej Siewior <bigeasy@linutronix.de> 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/delay.h> 13 #include <linux/slab.h> 14 #include <linux/spinlock.h> 15 #include <linux/platform_device.h> 16 #include <linux/pm_runtime.h> 17 #include <linux/interrupt.h> 18 #include <linux/io.h> 19 #include <linux/list.h> 20 #include <linux/dma-mapping.h> 21 22 #include <linux/usb/ch9.h> 23 #include <linux/usb/gadget.h> 24 25 #include "debug.h" 26 #include "core.h" 27 #include "gadget.h" 28 #include "io.h" 29 30 #define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \ 31 & ~((d)->interval - 1)) 32 33 /** 34 * dwc3_gadget_set_test_mode - enables usb2 test modes 35 * @dwc: pointer to our context structure 36 * @mode: the mode to set (J, K SE0 NAK, Force Enable) 37 * 38 * Caller should take care of locking. This function will return 0 on 39 * success or -EINVAL if wrong Test Selector is passed. 40 */ 41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode) 42 { 43 u32 reg; 44 45 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 46 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 47 48 switch (mode) { 49 case USB_TEST_J: 50 case USB_TEST_K: 51 case USB_TEST_SE0_NAK: 52 case USB_TEST_PACKET: 53 case USB_TEST_FORCE_ENABLE: 54 reg |= mode << 1; 55 break; 56 default: 57 return -EINVAL; 58 } 59 60 dwc3_gadget_dctl_write_safe(dwc, reg); 61 62 return 0; 63 } 64 65 /** 66 * dwc3_gadget_get_link_state - gets current state of usb link 67 * @dwc: pointer to our context structure 68 * 69 * Caller should take care of locking. This function will 70 * return the link state on success (>= 0) or -ETIMEDOUT. 71 */ 72 int dwc3_gadget_get_link_state(struct dwc3 *dwc) 73 { 74 u32 reg; 75 76 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 77 78 return DWC3_DSTS_USBLNKST(reg); 79 } 80 81 /** 82 * dwc3_gadget_set_link_state - sets usb link to a particular state 83 * @dwc: pointer to our context structure 84 * @state: the state to put link into 85 * 86 * Caller should take care of locking. This function will 87 * return 0 on success or -ETIMEDOUT. 88 */ 89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state) 90 { 91 int retries = 10000; 92 u32 reg; 93 94 /* 95 * Wait until device controller is ready. Only applies to 1.94a and 96 * later RTL. 97 */ 98 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) { 99 while (--retries) { 100 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 101 if (reg & DWC3_DSTS_DCNRD) 102 udelay(5); 103 else 104 break; 105 } 106 107 if (retries <= 0) 108 return -ETIMEDOUT; 109 } 110 111 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 113 114 /* set no action before sending new link state change */ 115 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 116 117 /* set requested state */ 118 reg |= DWC3_DCTL_ULSTCHNGREQ(state); 119 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 120 121 /* 122 * The following code is racy when called from dwc3_gadget_wakeup, 123 * and is not needed, at least on newer versions 124 */ 125 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) 126 return 0; 127 128 /* wait for a change in DSTS */ 129 retries = 10000; 130 while (--retries) { 131 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 132 133 if (DWC3_DSTS_USBLNKST(reg) == state) 134 return 0; 135 136 udelay(5); 137 } 138 139 return -ETIMEDOUT; 140 } 141 142 static void dwc3_ep0_reset_state(struct dwc3 *dwc) 143 { 144 unsigned int dir; 145 146 if (dwc->ep0state != EP0_SETUP_PHASE) { 147 dir = !!dwc->ep0_expect_in; 148 if (dwc->ep0state == EP0_DATA_PHASE) 149 dwc3_ep0_end_control_data(dwc, dwc->eps[dir]); 150 else 151 dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]); 152 153 dwc->eps[0]->trb_enqueue = 0; 154 dwc->eps[1]->trb_enqueue = 0; 155 156 dwc3_ep0_stall_and_restart(dwc); 157 } 158 } 159 160 /** 161 * dwc3_ep_inc_trb - increment a trb index. 162 * @index: Pointer to the TRB index to increment. 163 * 164 * The index should never point to the link TRB. After incrementing, 165 * if it is point to the link TRB, wrap around to the beginning. The 166 * link TRB is always at the last TRB entry. 167 */ 168 static void dwc3_ep_inc_trb(u8 *index) 169 { 170 (*index)++; 171 if (*index == (DWC3_TRB_NUM - 1)) 172 *index = 0; 173 } 174 175 /** 176 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer 177 * @dep: The endpoint whose enqueue pointer we're incrementing 178 */ 179 static void dwc3_ep_inc_enq(struct dwc3_ep *dep) 180 { 181 dwc3_ep_inc_trb(&dep->trb_enqueue); 182 } 183 184 /** 185 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer 186 * @dep: The endpoint whose enqueue pointer we're incrementing 187 */ 188 static void dwc3_ep_inc_deq(struct dwc3_ep *dep) 189 { 190 dwc3_ep_inc_trb(&dep->trb_dequeue); 191 } 192 193 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep, 194 struct dwc3_request *req, int status) 195 { 196 struct dwc3 *dwc = dep->dwc; 197 198 list_del(&req->list); 199 req->remaining = 0; 200 req->needs_extra_trb = false; 201 req->num_trbs = 0; 202 203 if (req->request.status == -EINPROGRESS) 204 req->request.status = status; 205 206 if (req->trb) 207 usb_gadget_unmap_request_by_dev(dwc->sysdev, 208 &req->request, req->direction); 209 210 req->trb = NULL; 211 trace_dwc3_gadget_giveback(req); 212 213 if (dep->number > 1) 214 pm_runtime_put(dwc->dev); 215 } 216 217 /** 218 * dwc3_gadget_giveback - call struct usb_request's ->complete callback 219 * @dep: The endpoint to whom the request belongs to 220 * @req: The request we're giving back 221 * @status: completion code for the request 222 * 223 * Must be called with controller's lock held and interrupts disabled. This 224 * function will unmap @req and call its ->complete() callback to notify upper 225 * layers that it has completed. 226 */ 227 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, 228 int status) 229 { 230 struct dwc3 *dwc = dep->dwc; 231 232 dwc3_gadget_del_and_unmap_request(dep, req, status); 233 req->status = DWC3_REQUEST_STATUS_COMPLETED; 234 235 spin_unlock(&dwc->lock); 236 usb_gadget_giveback_request(&dep->endpoint, &req->request); 237 spin_lock(&dwc->lock); 238 } 239 240 /** 241 * dwc3_send_gadget_generic_command - issue a generic command for the controller 242 * @dwc: pointer to the controller context 243 * @cmd: the command to be issued 244 * @param: command parameter 245 * 246 * Caller should take care of locking. Issue @cmd with a given @param to @dwc 247 * and wait for its completion. 248 */ 249 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd, 250 u32 param) 251 { 252 u32 timeout = 500; 253 int status = 0; 254 int ret = 0; 255 u32 reg; 256 257 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param); 258 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT); 259 260 do { 261 reg = dwc3_readl(dwc->regs, DWC3_DGCMD); 262 if (!(reg & DWC3_DGCMD_CMDACT)) { 263 status = DWC3_DGCMD_STATUS(reg); 264 if (status) 265 ret = -EINVAL; 266 break; 267 } 268 } while (--timeout); 269 270 if (!timeout) { 271 ret = -ETIMEDOUT; 272 status = -ETIMEDOUT; 273 } 274 275 trace_dwc3_gadget_generic_cmd(cmd, param, status); 276 277 return ret; 278 } 279 280 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async); 281 282 /** 283 * dwc3_send_gadget_ep_cmd - issue an endpoint command 284 * @dep: the endpoint to which the command is going to be issued 285 * @cmd: the command to be issued 286 * @params: parameters to the command 287 * 288 * Caller should handle locking. This function will issue @cmd with given 289 * @params to @dep and wait for its completion. 290 */ 291 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, 292 struct dwc3_gadget_ep_cmd_params *params) 293 { 294 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 295 struct dwc3 *dwc = dep->dwc; 296 u32 timeout = 5000; 297 u32 saved_config = 0; 298 u32 reg; 299 300 int cmd_status = 0; 301 int ret = -EINVAL; 302 303 /* 304 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or 305 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an 306 * endpoint command. 307 * 308 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY 309 * settings. Restore them after the command is completed. 310 * 311 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2 312 */ 313 if (dwc->gadget->speed <= USB_SPEED_HIGH || 314 DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) { 315 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 316 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) { 317 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY; 318 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY; 319 } 320 321 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) { 322 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM; 323 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM; 324 } 325 326 if (saved_config) 327 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 328 } 329 330 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) { 331 int link_state; 332 333 /* 334 * Initiate remote wakeup if the link state is in U3 when 335 * operating in SS/SSP or L1/L2 when operating in HS/FS. If the 336 * link state is in U1/U2, no remote wakeup is needed. The Start 337 * Transfer command will initiate the link recovery. 338 */ 339 link_state = dwc3_gadget_get_link_state(dwc); 340 switch (link_state) { 341 case DWC3_LINK_STATE_U2: 342 if (dwc->gadget->speed >= USB_SPEED_SUPER) 343 break; 344 345 fallthrough; 346 case DWC3_LINK_STATE_U3: 347 ret = __dwc3_gadget_wakeup(dwc, false); 348 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n", 349 ret); 350 break; 351 } 352 } 353 354 /* 355 * For some commands such as Update Transfer command, DEPCMDPARn 356 * registers are reserved. Since the driver often sends Update Transfer 357 * command, don't write to DEPCMDPARn to avoid register write delays and 358 * improve performance. 359 */ 360 if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) { 361 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); 362 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); 363 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); 364 } 365 366 /* 367 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're 368 * not relying on XferNotReady, we can make use of a special "No 369 * Response Update Transfer" command where we should clear both CmdAct 370 * and CmdIOC bits. 371 * 372 * With this, we don't need to wait for command completion and can 373 * straight away issue further commands to the endpoint. 374 * 375 * NOTICE: We're making an assumption that control endpoints will never 376 * make use of Update Transfer command. This is a safe assumption 377 * because we can never have more than one request at a time with 378 * Control Endpoints. If anybody changes that assumption, this chunk 379 * needs to be updated accordingly. 380 */ 381 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER && 382 !usb_endpoint_xfer_isoc(desc)) 383 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT); 384 else 385 cmd |= DWC3_DEPCMD_CMDACT; 386 387 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); 388 389 if (!(cmd & DWC3_DEPCMD_CMDACT) || 390 (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER && 391 !(cmd & DWC3_DEPCMD_CMDIOC))) { 392 ret = 0; 393 goto skip_status; 394 } 395 396 do { 397 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); 398 if (!(reg & DWC3_DEPCMD_CMDACT)) { 399 cmd_status = DWC3_DEPCMD_STATUS(reg); 400 401 switch (cmd_status) { 402 case 0: 403 ret = 0; 404 break; 405 case DEPEVT_TRANSFER_NO_RESOURCE: 406 dev_WARN(dwc->dev, "No resource for %s\n", 407 dep->name); 408 ret = -EINVAL; 409 break; 410 case DEPEVT_TRANSFER_BUS_EXPIRY: 411 /* 412 * SW issues START TRANSFER command to 413 * isochronous ep with future frame interval. If 414 * future interval time has already passed when 415 * core receives the command, it will respond 416 * with an error status of 'Bus Expiry'. 417 * 418 * Instead of always returning -EINVAL, let's 419 * give a hint to the gadget driver that this is 420 * the case by returning -EAGAIN. 421 */ 422 ret = -EAGAIN; 423 break; 424 default: 425 dev_WARN(dwc->dev, "UNKNOWN cmd status\n"); 426 } 427 428 break; 429 } 430 } while (--timeout); 431 432 if (timeout == 0) { 433 ret = -ETIMEDOUT; 434 cmd_status = -ETIMEDOUT; 435 } 436 437 skip_status: 438 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status); 439 440 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) { 441 if (ret == 0) 442 dep->flags |= DWC3_EP_TRANSFER_STARTED; 443 444 if (ret != -ETIMEDOUT) 445 dwc3_gadget_ep_get_transfer_index(dep); 446 } 447 448 if (saved_config) { 449 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 450 reg |= saved_config; 451 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 452 } 453 454 return ret; 455 } 456 457 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep) 458 { 459 struct dwc3 *dwc = dep->dwc; 460 struct dwc3_gadget_ep_cmd_params params; 461 u32 cmd = DWC3_DEPCMD_CLEARSTALL; 462 463 /* 464 * As of core revision 2.60a the recommended programming model 465 * is to set the ClearPendIN bit when issuing a Clear Stall EP 466 * command for IN endpoints. This is to prevent an issue where 467 * some (non-compliant) hosts may not send ACK TPs for pending 468 * IN transfers due to a mishandled error condition. Synopsys 469 * STAR 9000614252. 470 */ 471 if (dep->direction && 472 !DWC3_VER_IS_PRIOR(DWC3, 260A) && 473 (dwc->gadget->speed >= USB_SPEED_SUPER)) 474 cmd |= DWC3_DEPCMD_CLEARPENDIN; 475 476 memset(¶ms, 0, sizeof(params)); 477 478 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 479 } 480 481 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, 482 struct dwc3_trb *trb) 483 { 484 u32 offset = (char *) trb - (char *) dep->trb_pool; 485 486 return dep->trb_pool_dma + offset; 487 } 488 489 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep) 490 { 491 struct dwc3 *dwc = dep->dwc; 492 493 if (dep->trb_pool) 494 return 0; 495 496 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, 497 sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 498 &dep->trb_pool_dma, GFP_KERNEL); 499 if (!dep->trb_pool) { 500 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", 501 dep->name); 502 return -ENOMEM; 503 } 504 505 return 0; 506 } 507 508 static void dwc3_free_trb_pool(struct dwc3_ep *dep) 509 { 510 struct dwc3 *dwc = dep->dwc; 511 512 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 513 dep->trb_pool, dep->trb_pool_dma); 514 515 dep->trb_pool = NULL; 516 dep->trb_pool_dma = 0; 517 } 518 519 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep) 520 { 521 struct dwc3_gadget_ep_cmd_params params; 522 523 memset(¶ms, 0x00, sizeof(params)); 524 525 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1); 526 527 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE, 528 ¶ms); 529 } 530 531 /** 532 * dwc3_gadget_start_config - configure ep resources 533 * @dep: endpoint that is being enabled 534 * 535 * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's 536 * completion, it will set Transfer Resource for all available endpoints. 537 * 538 * The assignment of transfer resources cannot perfectly follow the data book 539 * due to the fact that the controller driver does not have all knowledge of the 540 * configuration in advance. It is given this information piecemeal by the 541 * composite gadget framework after every SET_CONFIGURATION and 542 * SET_INTERFACE. Trying to follow the databook programming model in this 543 * scenario can cause errors. For two reasons: 544 * 545 * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every 546 * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is 547 * incorrect in the scenario of multiple interfaces. 548 * 549 * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new 550 * endpoint on alt setting (8.1.6). 551 * 552 * The following simplified method is used instead: 553 * 554 * All hardware endpoints can be assigned a transfer resource and this setting 555 * will stay persistent until either a core reset or hibernation. So whenever we 556 * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do 557 * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are 558 * guaranteed that there are as many transfer resources as endpoints. 559 * 560 * This function is called for each endpoint when it is being enabled but is 561 * triggered only when called for EP0-out, which always happens first, and which 562 * should only happen in one of the above conditions. 563 */ 564 static int dwc3_gadget_start_config(struct dwc3_ep *dep) 565 { 566 struct dwc3_gadget_ep_cmd_params params; 567 struct dwc3 *dwc; 568 u32 cmd; 569 int i; 570 int ret; 571 572 if (dep->number) 573 return 0; 574 575 memset(¶ms, 0x00, sizeof(params)); 576 cmd = DWC3_DEPCMD_DEPSTARTCFG; 577 dwc = dep->dwc; 578 579 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 580 if (ret) 581 return ret; 582 583 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) { 584 struct dwc3_ep *dep = dwc->eps[i]; 585 586 if (!dep) 587 continue; 588 589 ret = dwc3_gadget_set_xfer_resource(dep); 590 if (ret) 591 return ret; 592 } 593 594 return 0; 595 } 596 597 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action) 598 { 599 const struct usb_ss_ep_comp_descriptor *comp_desc; 600 const struct usb_endpoint_descriptor *desc; 601 struct dwc3_gadget_ep_cmd_params params; 602 struct dwc3 *dwc = dep->dwc; 603 604 comp_desc = dep->endpoint.comp_desc; 605 desc = dep->endpoint.desc; 606 607 memset(¶ms, 0x00, sizeof(params)); 608 609 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc)) 610 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc)); 611 612 /* Burst size is only needed in SuperSpeed mode */ 613 if (dwc->gadget->speed >= USB_SPEED_SUPER) { 614 u32 burst = dep->endpoint.maxburst; 615 616 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); 617 } 618 619 params.param0 |= action; 620 if (action == DWC3_DEPCFG_ACTION_RESTORE) 621 params.param2 |= dep->saved_state; 622 623 if (usb_endpoint_xfer_control(desc)) 624 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN; 625 626 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) 627 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN; 628 629 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) { 630 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE 631 | DWC3_DEPCFG_XFER_COMPLETE_EN 632 | DWC3_DEPCFG_STREAM_EVENT_EN; 633 dep->stream_capable = true; 634 } 635 636 if (!usb_endpoint_xfer_control(desc)) 637 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN; 638 639 /* 640 * We are doing 1:1 mapping for endpoints, meaning 641 * Physical Endpoints 2 maps to Logical Endpoint 2 and 642 * so on. We consider the direction bit as part of the physical 643 * endpoint number. So USB endpoint 0x81 is 0x03. 644 */ 645 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); 646 647 /* 648 * We must use the lower 16 TX FIFOs even though 649 * HW might have more 650 */ 651 if (dep->direction) 652 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); 653 654 if (desc->bInterval) { 655 u8 bInterval_m1; 656 657 /* 658 * Valid range for DEPCFG.bInterval_m1 is from 0 to 13. 659 * 660 * NOTE: The programming guide incorrectly stated bInterval_m1 661 * must be set to 0 when operating in fullspeed. Internally the 662 * controller does not have this limitation. See DWC_usb3x 663 * programming guide section 3.2.2.1. 664 */ 665 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13); 666 667 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT && 668 dwc->gadget->speed == USB_SPEED_FULL) 669 dep->interval = desc->bInterval; 670 else 671 dep->interval = 1 << (desc->bInterval - 1); 672 673 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1); 674 } 675 676 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms); 677 } 678 679 /** 680 * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value 681 * @dwc: pointer to the DWC3 context 682 * @mult: multiplier to be used when calculating the fifo_size 683 * 684 * Calculates the size value based on the equation below: 685 * 686 * DWC3 revision 280A and prior: 687 * fifo_size = mult * (max_packet / mdwidth) + 1; 688 * 689 * DWC3 revision 290A and onwards: 690 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 691 * 692 * The max packet size is set to 1024, as the txfifo requirements mainly apply 693 * to super speed USB use cases. However, it is safe to overestimate the fifo 694 * allocations for other scenarios, i.e. high speed USB. 695 */ 696 static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult) 697 { 698 int max_packet = 1024; 699 int fifo_size; 700 int mdwidth; 701 702 mdwidth = dwc3_mdwidth(dwc); 703 704 /* MDWIDTH is represented in bits, we need it in bytes */ 705 mdwidth >>= 3; 706 707 if (DWC3_VER_IS_PRIOR(DWC3, 290A)) 708 fifo_size = mult * (max_packet / mdwidth) + 1; 709 else 710 fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1; 711 return fifo_size; 712 } 713 714 /** 715 * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation 716 * @dwc: pointer to the DWC3 context 717 * 718 * Iterates through all the endpoint registers and clears the previous txfifo 719 * allocations. 720 */ 721 void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc) 722 { 723 struct dwc3_ep *dep; 724 int fifo_depth; 725 int size; 726 int num; 727 728 if (!dwc->do_fifo_resize) 729 return; 730 731 /* Read ep0IN related TXFIFO size */ 732 dep = dwc->eps[1]; 733 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); 734 if (DWC3_IP_IS(DWC3)) 735 fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size); 736 else 737 fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size); 738 739 dwc->last_fifo_depth = fifo_depth; 740 /* Clear existing TXFIFO for all IN eps except ep0 */ 741 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM); 742 num += 2) { 743 dep = dwc->eps[num]; 744 /* Don't change TXFRAMNUM on usb31 version */ 745 size = DWC3_IP_IS(DWC3) ? 0 : 746 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) & 747 DWC31_GTXFIFOSIZ_TXFRAMNUM; 748 749 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size); 750 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED; 751 } 752 dwc->num_ep_resized = 0; 753 } 754 755 /* 756 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case 757 * @dwc: pointer to our context structure 758 * 759 * This function will a best effort FIFO allocation in order 760 * to improve FIFO usage and throughput, while still allowing 761 * us to enable as many endpoints as possible. 762 * 763 * Keep in mind that this operation will be highly dependent 764 * on the configured size for RAM1 - which contains TxFifo -, 765 * the amount of endpoints enabled on coreConsultant tool, and 766 * the width of the Master Bus. 767 * 768 * In general, FIFO depths are represented with the following equation: 769 * 770 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 771 * 772 * In conjunction with dwc3_gadget_check_config(), this resizing logic will 773 * ensure that all endpoints will have enough internal memory for one max 774 * packet per endpoint. 775 */ 776 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep) 777 { 778 struct dwc3 *dwc = dep->dwc; 779 int fifo_0_start; 780 int ram1_depth; 781 int fifo_size; 782 int min_depth; 783 int num_in_ep; 784 int remaining; 785 int num_fifos = 1; 786 int fifo; 787 int tmp; 788 789 if (!dwc->do_fifo_resize) 790 return 0; 791 792 /* resize IN endpoints except ep0 */ 793 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1) 794 return 0; 795 796 /* bail if already resized */ 797 if (dep->flags & DWC3_EP_TXFIFO_RESIZED) 798 return 0; 799 800 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); 801 802 if ((dep->endpoint.maxburst > 1 && 803 usb_endpoint_xfer_bulk(dep->endpoint.desc)) || 804 usb_endpoint_xfer_isoc(dep->endpoint.desc)) 805 num_fifos = 3; 806 807 if (dep->endpoint.maxburst > 6 && 808 (usb_endpoint_xfer_bulk(dep->endpoint.desc) || 809 usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31)) 810 num_fifos = dwc->tx_fifo_resize_max_num; 811 812 /* FIFO size for a single buffer */ 813 fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1); 814 815 /* Calculate the number of remaining EPs w/o any FIFO */ 816 num_in_ep = dwc->max_cfg_eps; 817 num_in_ep -= dwc->num_ep_resized; 818 819 /* Reserve at least one FIFO for the number of IN EPs */ 820 min_depth = num_in_ep * (fifo + 1); 821 remaining = ram1_depth - min_depth - dwc->last_fifo_depth; 822 remaining = max_t(int, 0, remaining); 823 /* 824 * We've already reserved 1 FIFO per EP, so check what we can fit in 825 * addition to it. If there is not enough remaining space, allocate 826 * all the remaining space to the EP. 827 */ 828 fifo_size = (num_fifos - 1) * fifo; 829 if (remaining < fifo_size) 830 fifo_size = remaining; 831 832 fifo_size += fifo; 833 /* Last increment according to the TX FIFO size equation */ 834 fifo_size++; 835 836 /* Check if TXFIFOs start at non-zero addr */ 837 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); 838 fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp); 839 840 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16)); 841 if (DWC3_IP_IS(DWC3)) 842 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); 843 else 844 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); 845 846 /* Check fifo size allocation doesn't exceed available RAM size. */ 847 if (dwc->last_fifo_depth >= ram1_depth) { 848 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n", 849 dwc->last_fifo_depth, ram1_depth, 850 dep->endpoint.name, fifo_size); 851 if (DWC3_IP_IS(DWC3)) 852 fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); 853 else 854 fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); 855 856 dwc->last_fifo_depth -= fifo_size; 857 return -ENOMEM; 858 } 859 860 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); 861 dep->flags |= DWC3_EP_TXFIFO_RESIZED; 862 dwc->num_ep_resized++; 863 864 return 0; 865 } 866 867 /** 868 * __dwc3_gadget_ep_enable - initializes a hw endpoint 869 * @dep: endpoint to be initialized 870 * @action: one of INIT, MODIFY or RESTORE 871 * 872 * Caller should take care of locking. Execute all necessary commands to 873 * initialize a HW endpoint so it can be used by a gadget driver. 874 */ 875 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) 876 { 877 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 878 struct dwc3 *dwc = dep->dwc; 879 880 u32 reg; 881 int ret; 882 883 if (!(dep->flags & DWC3_EP_ENABLED)) { 884 ret = dwc3_gadget_resize_tx_fifos(dep); 885 if (ret) 886 return ret; 887 888 ret = dwc3_gadget_start_config(dep); 889 if (ret) 890 return ret; 891 } 892 893 ret = dwc3_gadget_set_ep_config(dep, action); 894 if (ret) 895 return ret; 896 897 if (!(dep->flags & DWC3_EP_ENABLED)) { 898 struct dwc3_trb *trb_st_hw; 899 struct dwc3_trb *trb_link; 900 901 dep->type = usb_endpoint_type(desc); 902 dep->flags |= DWC3_EP_ENABLED; 903 904 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 905 reg |= DWC3_DALEPENA_EP(dep->number); 906 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 907 908 dep->trb_dequeue = 0; 909 dep->trb_enqueue = 0; 910 911 if (usb_endpoint_xfer_control(desc)) 912 goto out; 913 914 /* Initialize the TRB ring */ 915 memset(dep->trb_pool, 0, 916 sizeof(struct dwc3_trb) * DWC3_TRB_NUM); 917 918 /* Link TRB. The HWO bit is never reset */ 919 trb_st_hw = &dep->trb_pool[0]; 920 921 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; 922 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 923 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 924 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB; 925 trb_link->ctrl |= DWC3_TRB_CTRL_HWO; 926 } 927 928 /* 929 * Issue StartTransfer here with no-op TRB so we can always rely on No 930 * Response Update Transfer command. 931 */ 932 if (usb_endpoint_xfer_bulk(desc) || 933 usb_endpoint_xfer_int(desc)) { 934 struct dwc3_gadget_ep_cmd_params params; 935 struct dwc3_trb *trb; 936 dma_addr_t trb_dma; 937 u32 cmd; 938 939 memset(¶ms, 0, sizeof(params)); 940 trb = &dep->trb_pool[0]; 941 trb_dma = dwc3_trb_dma_offset(dep, trb); 942 943 params.param0 = upper_32_bits(trb_dma); 944 params.param1 = lower_32_bits(trb_dma); 945 946 cmd = DWC3_DEPCMD_STARTTRANSFER; 947 948 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 949 if (ret < 0) 950 return ret; 951 952 if (dep->stream_capable) { 953 /* 954 * For streams, at start, there maybe a race where the 955 * host primes the endpoint before the function driver 956 * queues a request to initiate a stream. In that case, 957 * the controller will not see the prime to generate the 958 * ERDY and start stream. To workaround this, issue a 959 * no-op TRB as normal, but end it immediately. As a 960 * result, when the function driver queues the request, 961 * the next START_TRANSFER command will cause the 962 * controller to generate an ERDY to initiate the 963 * stream. 964 */ 965 dwc3_stop_active_transfer(dep, true, true); 966 967 /* 968 * All stream eps will reinitiate stream on NoStream 969 * rejection until we can determine that the host can 970 * prime after the first transfer. 971 * 972 * However, if the controller is capable of 973 * TXF_FLUSH_BYPASS, then IN direction endpoints will 974 * automatically restart the stream without the driver 975 * initiation. 976 */ 977 if (!dep->direction || 978 !(dwc->hwparams.hwparams9 & 979 DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS)) 980 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM; 981 } 982 } 983 984 out: 985 trace_dwc3_gadget_ep_enable(dep); 986 987 return 0; 988 } 989 990 void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status) 991 { 992 struct dwc3_request *req; 993 994 dwc3_stop_active_transfer(dep, true, false); 995 996 /* If endxfer is delayed, avoid unmapping requests */ 997 if (dep->flags & DWC3_EP_DELAY_STOP) 998 return; 999 1000 /* - giveback all requests to gadget driver */ 1001 while (!list_empty(&dep->started_list)) { 1002 req = next_request(&dep->started_list); 1003 1004 dwc3_gadget_giveback(dep, req, status); 1005 } 1006 1007 while (!list_empty(&dep->pending_list)) { 1008 req = next_request(&dep->pending_list); 1009 1010 dwc3_gadget_giveback(dep, req, status); 1011 } 1012 1013 while (!list_empty(&dep->cancelled_list)) { 1014 req = next_request(&dep->cancelled_list); 1015 1016 dwc3_gadget_giveback(dep, req, status); 1017 } 1018 } 1019 1020 /** 1021 * __dwc3_gadget_ep_disable - disables a hw endpoint 1022 * @dep: the endpoint to disable 1023 * 1024 * This function undoes what __dwc3_gadget_ep_enable did and also removes 1025 * requests which are currently being processed by the hardware and those which 1026 * are not yet scheduled. 1027 * 1028 * Caller should take care of locking. 1029 */ 1030 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) 1031 { 1032 struct dwc3 *dwc = dep->dwc; 1033 u32 reg; 1034 u32 mask; 1035 1036 trace_dwc3_gadget_ep_disable(dep); 1037 1038 /* make sure HW endpoint isn't stalled */ 1039 if (dep->flags & DWC3_EP_STALL) 1040 __dwc3_gadget_ep_set_halt(dep, 0, false); 1041 1042 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 1043 reg &= ~DWC3_DALEPENA_EP(dep->number); 1044 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 1045 1046 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); 1047 1048 dep->stream_capable = false; 1049 dep->type = 0; 1050 mask = DWC3_EP_TXFIFO_RESIZED; 1051 /* 1052 * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is 1053 * set. Do not clear DEP flags, so that the end transfer command will 1054 * be reattempted during the next SETUP stage. 1055 */ 1056 if (dep->flags & DWC3_EP_DELAY_STOP) 1057 mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED); 1058 dep->flags &= mask; 1059 1060 /* Clear out the ep descriptors for non-ep0 */ 1061 if (dep->number > 1) { 1062 dep->endpoint.comp_desc = NULL; 1063 dep->endpoint.desc = NULL; 1064 } 1065 1066 return 0; 1067 } 1068 1069 /* -------------------------------------------------------------------------- */ 1070 1071 static int dwc3_gadget_ep0_enable(struct usb_ep *ep, 1072 const struct usb_endpoint_descriptor *desc) 1073 { 1074 return -EINVAL; 1075 } 1076 1077 static int dwc3_gadget_ep0_disable(struct usb_ep *ep) 1078 { 1079 return -EINVAL; 1080 } 1081 1082 /* -------------------------------------------------------------------------- */ 1083 1084 static int dwc3_gadget_ep_enable(struct usb_ep *ep, 1085 const struct usb_endpoint_descriptor *desc) 1086 { 1087 struct dwc3_ep *dep; 1088 struct dwc3 *dwc; 1089 unsigned long flags; 1090 int ret; 1091 1092 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { 1093 pr_debug("dwc3: invalid parameters\n"); 1094 return -EINVAL; 1095 } 1096 1097 if (!desc->wMaxPacketSize) { 1098 pr_debug("dwc3: missing wMaxPacketSize\n"); 1099 return -EINVAL; 1100 } 1101 1102 dep = to_dwc3_ep(ep); 1103 dwc = dep->dwc; 1104 1105 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, 1106 "%s is already enabled\n", 1107 dep->name)) 1108 return 0; 1109 1110 spin_lock_irqsave(&dwc->lock, flags); 1111 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 1112 spin_unlock_irqrestore(&dwc->lock, flags); 1113 1114 return ret; 1115 } 1116 1117 static int dwc3_gadget_ep_disable(struct usb_ep *ep) 1118 { 1119 struct dwc3_ep *dep; 1120 struct dwc3 *dwc; 1121 unsigned long flags; 1122 int ret; 1123 1124 if (!ep) { 1125 pr_debug("dwc3: invalid parameters\n"); 1126 return -EINVAL; 1127 } 1128 1129 dep = to_dwc3_ep(ep); 1130 dwc = dep->dwc; 1131 1132 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), 1133 "%s is already disabled\n", 1134 dep->name)) 1135 return 0; 1136 1137 spin_lock_irqsave(&dwc->lock, flags); 1138 ret = __dwc3_gadget_ep_disable(dep); 1139 spin_unlock_irqrestore(&dwc->lock, flags); 1140 1141 return ret; 1142 } 1143 1144 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep, 1145 gfp_t gfp_flags) 1146 { 1147 struct dwc3_request *req; 1148 struct dwc3_ep *dep = to_dwc3_ep(ep); 1149 1150 req = kzalloc(sizeof(*req), gfp_flags); 1151 if (!req) 1152 return NULL; 1153 1154 req->direction = dep->direction; 1155 req->epnum = dep->number; 1156 req->dep = dep; 1157 req->status = DWC3_REQUEST_STATUS_UNKNOWN; 1158 1159 trace_dwc3_alloc_request(req); 1160 1161 return &req->request; 1162 } 1163 1164 static void dwc3_gadget_ep_free_request(struct usb_ep *ep, 1165 struct usb_request *request) 1166 { 1167 struct dwc3_request *req = to_dwc3_request(request); 1168 1169 trace_dwc3_free_request(req); 1170 kfree(req); 1171 } 1172 1173 /** 1174 * dwc3_ep_prev_trb - returns the previous TRB in the ring 1175 * @dep: The endpoint with the TRB ring 1176 * @index: The index of the current TRB in the ring 1177 * 1178 * Returns the TRB prior to the one pointed to by the index. If the 1179 * index is 0, we will wrap backwards, skip the link TRB, and return 1180 * the one just before that. 1181 */ 1182 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index) 1183 { 1184 u8 tmp = index; 1185 1186 if (!tmp) 1187 tmp = DWC3_TRB_NUM - 1; 1188 1189 return &dep->trb_pool[tmp - 1]; 1190 } 1191 1192 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep) 1193 { 1194 u8 trbs_left; 1195 1196 /* 1197 * If the enqueue & dequeue are equal then the TRB ring is either full 1198 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs 1199 * pending to be processed by the driver. 1200 */ 1201 if (dep->trb_enqueue == dep->trb_dequeue) { 1202 /* 1203 * If there is any request remained in the started_list at 1204 * this point, that means there is no TRB available. 1205 */ 1206 if (!list_empty(&dep->started_list)) 1207 return 0; 1208 1209 return DWC3_TRB_NUM - 1; 1210 } 1211 1212 trbs_left = dep->trb_dequeue - dep->trb_enqueue; 1213 trbs_left &= (DWC3_TRB_NUM - 1); 1214 1215 if (dep->trb_dequeue < dep->trb_enqueue) 1216 trbs_left--; 1217 1218 return trbs_left; 1219 } 1220 1221 /** 1222 * dwc3_prepare_one_trb - setup one TRB from one request 1223 * @dep: endpoint for which this request is prepared 1224 * @req: dwc3_request pointer 1225 * @trb_length: buffer size of the TRB 1226 * @chain: should this TRB be chained to the next? 1227 * @node: only for isochronous endpoints. First TRB needs different type. 1228 * @use_bounce_buffer: set to use bounce buffer 1229 * @must_interrupt: set to interrupt on TRB completion 1230 */ 1231 static void dwc3_prepare_one_trb(struct dwc3_ep *dep, 1232 struct dwc3_request *req, unsigned int trb_length, 1233 unsigned int chain, unsigned int node, bool use_bounce_buffer, 1234 bool must_interrupt) 1235 { 1236 struct dwc3_trb *trb; 1237 dma_addr_t dma; 1238 unsigned int stream_id = req->request.stream_id; 1239 unsigned int short_not_ok = req->request.short_not_ok; 1240 unsigned int no_interrupt = req->request.no_interrupt; 1241 unsigned int is_last = req->request.is_last; 1242 struct dwc3 *dwc = dep->dwc; 1243 struct usb_gadget *gadget = dwc->gadget; 1244 enum usb_device_speed speed = gadget->speed; 1245 1246 if (use_bounce_buffer) 1247 dma = dep->dwc->bounce_addr; 1248 else if (req->request.num_sgs > 0) 1249 dma = sg_dma_address(req->start_sg); 1250 else 1251 dma = req->request.dma; 1252 1253 trb = &dep->trb_pool[dep->trb_enqueue]; 1254 1255 if (!req->trb) { 1256 dwc3_gadget_move_started_request(req); 1257 req->trb = trb; 1258 req->trb_dma = dwc3_trb_dma_offset(dep, trb); 1259 } 1260 1261 req->num_trbs++; 1262 1263 trb->size = DWC3_TRB_SIZE_LENGTH(trb_length); 1264 trb->bpl = lower_32_bits(dma); 1265 trb->bph = upper_32_bits(dma); 1266 1267 switch (usb_endpoint_type(dep->endpoint.desc)) { 1268 case USB_ENDPOINT_XFER_CONTROL: 1269 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP; 1270 break; 1271 1272 case USB_ENDPOINT_XFER_ISOC: 1273 if (!node) { 1274 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; 1275 1276 /* 1277 * USB Specification 2.0 Section 5.9.2 states that: "If 1278 * there is only a single transaction in the microframe, 1279 * only a DATA0 data packet PID is used. If there are 1280 * two transactions per microframe, DATA1 is used for 1281 * the first transaction data packet and DATA0 is used 1282 * for the second transaction data packet. If there are 1283 * three transactions per microframe, DATA2 is used for 1284 * the first transaction data packet, DATA1 is used for 1285 * the second, and DATA0 is used for the third." 1286 * 1287 * IOW, we should satisfy the following cases: 1288 * 1289 * 1) length <= maxpacket 1290 * - DATA0 1291 * 1292 * 2) maxpacket < length <= (2 * maxpacket) 1293 * - DATA1, DATA0 1294 * 1295 * 3) (2 * maxpacket) < length <= (3 * maxpacket) 1296 * - DATA2, DATA1, DATA0 1297 */ 1298 if (speed == USB_SPEED_HIGH) { 1299 struct usb_ep *ep = &dep->endpoint; 1300 unsigned int mult = 2; 1301 unsigned int maxp = usb_endpoint_maxp(ep->desc); 1302 1303 if (req->request.length <= (2 * maxp)) 1304 mult--; 1305 1306 if (req->request.length <= maxp) 1307 mult--; 1308 1309 trb->size |= DWC3_TRB_SIZE_PCM1(mult); 1310 } 1311 } else { 1312 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; 1313 } 1314 1315 if (!no_interrupt && !chain) 1316 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 1317 break; 1318 1319 case USB_ENDPOINT_XFER_BULK: 1320 case USB_ENDPOINT_XFER_INT: 1321 trb->ctrl = DWC3_TRBCTL_NORMAL; 1322 break; 1323 default: 1324 /* 1325 * This is only possible with faulty memory because we 1326 * checked it already :) 1327 */ 1328 dev_WARN(dwc->dev, "Unknown endpoint type %d\n", 1329 usb_endpoint_type(dep->endpoint.desc)); 1330 } 1331 1332 /* 1333 * Enable Continue on Short Packet 1334 * when endpoint is not a stream capable 1335 */ 1336 if (usb_endpoint_dir_out(dep->endpoint.desc)) { 1337 if (!dep->stream_capable) 1338 trb->ctrl |= DWC3_TRB_CTRL_CSP; 1339 1340 if (short_not_ok) 1341 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 1342 } 1343 1344 /* All TRBs setup for MST must set CSP=1 when LST=0 */ 1345 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams)) 1346 trb->ctrl |= DWC3_TRB_CTRL_CSP; 1347 1348 if ((!no_interrupt && !chain) || must_interrupt) 1349 trb->ctrl |= DWC3_TRB_CTRL_IOC; 1350 1351 if (chain) 1352 trb->ctrl |= DWC3_TRB_CTRL_CHN; 1353 else if (dep->stream_capable && is_last && 1354 !DWC3_MST_CAPABLE(&dwc->hwparams)) 1355 trb->ctrl |= DWC3_TRB_CTRL_LST; 1356 1357 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) 1358 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id); 1359 1360 /* 1361 * As per data book 4.2.3.2TRB Control Bit Rules section 1362 * 1363 * The controller autonomously checks the HWO field of a TRB to determine if the 1364 * entire TRB is valid. Therefore, software must ensure that the rest of the TRB 1365 * is valid before setting the HWO field to '1'. In most systems, this means that 1366 * software must update the fourth DWORD of a TRB last. 1367 * 1368 * However there is a possibility of CPU re-ordering here which can cause 1369 * controller to observe the HWO bit set prematurely. 1370 * Add a write memory barrier to prevent CPU re-ordering. 1371 */ 1372 wmb(); 1373 trb->ctrl |= DWC3_TRB_CTRL_HWO; 1374 1375 dwc3_ep_inc_enq(dep); 1376 1377 trace_dwc3_prepare_trb(dep, trb); 1378 } 1379 1380 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req) 1381 { 1382 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); 1383 unsigned int rem = req->request.length % maxp; 1384 1385 if ((req->request.length && req->request.zero && !rem && 1386 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) || 1387 (!req->direction && rem)) 1388 return true; 1389 1390 return false; 1391 } 1392 1393 /** 1394 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry 1395 * @dep: The endpoint that the request belongs to 1396 * @req: The request to prepare 1397 * @entry_length: The last SG entry size 1398 * @node: Indicates whether this is not the first entry (for isoc only) 1399 * 1400 * Return the number of TRBs prepared. 1401 */ 1402 static int dwc3_prepare_last_sg(struct dwc3_ep *dep, 1403 struct dwc3_request *req, unsigned int entry_length, 1404 unsigned int node) 1405 { 1406 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); 1407 unsigned int rem = req->request.length % maxp; 1408 unsigned int num_trbs = 1; 1409 1410 if (dwc3_needs_extra_trb(dep, req)) 1411 num_trbs++; 1412 1413 if (dwc3_calc_trbs_left(dep) < num_trbs) 1414 return 0; 1415 1416 req->needs_extra_trb = num_trbs > 1; 1417 1418 /* Prepare a normal TRB */ 1419 if (req->direction || req->request.length) 1420 dwc3_prepare_one_trb(dep, req, entry_length, 1421 req->needs_extra_trb, node, false, false); 1422 1423 /* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */ 1424 if ((!req->direction && !req->request.length) || req->needs_extra_trb) 1425 dwc3_prepare_one_trb(dep, req, 1426 req->direction ? 0 : maxp - rem, 1427 false, 1, true, false); 1428 1429 return num_trbs; 1430 } 1431 1432 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep, 1433 struct dwc3_request *req) 1434 { 1435 struct scatterlist *sg = req->start_sg; 1436 struct scatterlist *s; 1437 int i; 1438 unsigned int length = req->request.length; 1439 unsigned int remaining = req->request.num_mapped_sgs 1440 - req->num_queued_sgs; 1441 unsigned int num_trbs = req->num_trbs; 1442 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req); 1443 1444 /* 1445 * If we resume preparing the request, then get the remaining length of 1446 * the request and resume where we left off. 1447 */ 1448 for_each_sg(req->request.sg, s, req->num_queued_sgs, i) 1449 length -= sg_dma_len(s); 1450 1451 for_each_sg(sg, s, remaining, i) { 1452 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep); 1453 unsigned int trb_length; 1454 bool must_interrupt = false; 1455 bool last_sg = false; 1456 1457 trb_length = min_t(unsigned int, length, sg_dma_len(s)); 1458 1459 length -= trb_length; 1460 1461 /* 1462 * IOMMU driver is coalescing the list of sgs which shares a 1463 * page boundary into one and giving it to USB driver. With 1464 * this the number of sgs mapped is not equal to the number of 1465 * sgs passed. So mark the chain bit to false if it isthe last 1466 * mapped sg. 1467 */ 1468 if ((i == remaining - 1) || !length) 1469 last_sg = true; 1470 1471 if (!num_trbs_left) 1472 break; 1473 1474 if (last_sg) { 1475 if (!dwc3_prepare_last_sg(dep, req, trb_length, i)) 1476 break; 1477 } else { 1478 /* 1479 * Look ahead to check if we have enough TRBs for the 1480 * next SG entry. If not, set interrupt on this TRB to 1481 * resume preparing the next SG entry when more TRBs are 1482 * free. 1483 */ 1484 if (num_trbs_left == 1 || (needs_extra_trb && 1485 num_trbs_left <= 2 && 1486 sg_dma_len(sg_next(s)) >= length)) { 1487 struct dwc3_request *r; 1488 1489 /* Check if previous requests already set IOC */ 1490 list_for_each_entry(r, &dep->started_list, list) { 1491 if (r != req && !r->request.no_interrupt) 1492 break; 1493 1494 if (r == req) 1495 must_interrupt = true; 1496 } 1497 } 1498 1499 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false, 1500 must_interrupt); 1501 } 1502 1503 /* 1504 * There can be a situation where all sgs in sglist are not 1505 * queued because of insufficient trb number. To handle this 1506 * case, update start_sg to next sg to be queued, so that 1507 * we have free trbs we can continue queuing from where we 1508 * previously stopped 1509 */ 1510 if (!last_sg) 1511 req->start_sg = sg_next(s); 1512 1513 req->num_queued_sgs++; 1514 req->num_pending_sgs--; 1515 1516 /* 1517 * The number of pending SG entries may not correspond to the 1518 * number of mapped SG entries. If all the data are queued, then 1519 * don't include unused SG entries. 1520 */ 1521 if (length == 0) { 1522 req->num_pending_sgs = 0; 1523 break; 1524 } 1525 1526 if (must_interrupt) 1527 break; 1528 } 1529 1530 return req->num_trbs - num_trbs; 1531 } 1532 1533 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep, 1534 struct dwc3_request *req) 1535 { 1536 return dwc3_prepare_last_sg(dep, req, req->request.length, 0); 1537 } 1538 1539 /* 1540 * dwc3_prepare_trbs - setup TRBs from requests 1541 * @dep: endpoint for which requests are being prepared 1542 * 1543 * The function goes through the requests list and sets up TRBs for the 1544 * transfers. The function returns once there are no more TRBs available or 1545 * it runs out of requests. 1546 * 1547 * Returns the number of TRBs prepared or negative errno. 1548 */ 1549 static int dwc3_prepare_trbs(struct dwc3_ep *dep) 1550 { 1551 struct dwc3_request *req, *n; 1552 int ret = 0; 1553 1554 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM); 1555 1556 /* 1557 * We can get in a situation where there's a request in the started list 1558 * but there weren't enough TRBs to fully kick it in the first time 1559 * around, so it has been waiting for more TRBs to be freed up. 1560 * 1561 * In that case, we should check if we have a request with pending_sgs 1562 * in the started list and prepare TRBs for that request first, 1563 * otherwise we will prepare TRBs completely out of order and that will 1564 * break things. 1565 */ 1566 list_for_each_entry(req, &dep->started_list, list) { 1567 if (req->num_pending_sgs > 0) { 1568 ret = dwc3_prepare_trbs_sg(dep, req); 1569 if (!ret || req->num_pending_sgs) 1570 return ret; 1571 } 1572 1573 if (!dwc3_calc_trbs_left(dep)) 1574 return ret; 1575 1576 /* 1577 * Don't prepare beyond a transfer. In DWC_usb32, its transfer 1578 * burst capability may try to read and use TRBs beyond the 1579 * active transfer instead of stopping. 1580 */ 1581 if (dep->stream_capable && req->request.is_last && 1582 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) 1583 return ret; 1584 } 1585 1586 list_for_each_entry_safe(req, n, &dep->pending_list, list) { 1587 struct dwc3 *dwc = dep->dwc; 1588 1589 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request, 1590 dep->direction); 1591 if (ret) 1592 return ret; 1593 1594 req->sg = req->request.sg; 1595 req->start_sg = req->sg; 1596 req->num_queued_sgs = 0; 1597 req->num_pending_sgs = req->request.num_mapped_sgs; 1598 1599 if (req->num_pending_sgs > 0) { 1600 ret = dwc3_prepare_trbs_sg(dep, req); 1601 if (req->num_pending_sgs) 1602 return ret; 1603 } else { 1604 ret = dwc3_prepare_trbs_linear(dep, req); 1605 } 1606 1607 if (!ret || !dwc3_calc_trbs_left(dep)) 1608 return ret; 1609 1610 /* 1611 * Don't prepare beyond a transfer. In DWC_usb32, its transfer 1612 * burst capability may try to read and use TRBs beyond the 1613 * active transfer instead of stopping. 1614 */ 1615 if (dep->stream_capable && req->request.is_last && 1616 !DWC3_MST_CAPABLE(&dwc->hwparams)) 1617 return ret; 1618 } 1619 1620 return ret; 1621 } 1622 1623 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep); 1624 1625 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) 1626 { 1627 struct dwc3_gadget_ep_cmd_params params; 1628 struct dwc3_request *req; 1629 int starting; 1630 int ret; 1631 u32 cmd; 1632 1633 /* 1634 * Note that it's normal to have no new TRBs prepared (i.e. ret == 0). 1635 * This happens when we need to stop and restart a transfer such as in 1636 * the case of reinitiating a stream or retrying an isoc transfer. 1637 */ 1638 ret = dwc3_prepare_trbs(dep); 1639 if (ret < 0) 1640 return ret; 1641 1642 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); 1643 1644 /* 1645 * If there's no new TRB prepared and we don't need to restart a 1646 * transfer, there's no need to update the transfer. 1647 */ 1648 if (!ret && !starting) 1649 return ret; 1650 1651 req = next_request(&dep->started_list); 1652 if (!req) { 1653 dep->flags |= DWC3_EP_PENDING_REQUEST; 1654 return 0; 1655 } 1656 1657 memset(¶ms, 0, sizeof(params)); 1658 1659 if (starting) { 1660 params.param0 = upper_32_bits(req->trb_dma); 1661 params.param1 = lower_32_bits(req->trb_dma); 1662 cmd = DWC3_DEPCMD_STARTTRANSFER; 1663 1664 if (dep->stream_capable) 1665 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id); 1666 1667 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) 1668 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); 1669 } else { 1670 cmd = DWC3_DEPCMD_UPDATETRANSFER | 1671 DWC3_DEPCMD_PARAM(dep->resource_index); 1672 } 1673 1674 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1675 if (ret < 0) { 1676 struct dwc3_request *tmp; 1677 1678 if (ret == -EAGAIN) 1679 return ret; 1680 1681 dwc3_stop_active_transfer(dep, true, true); 1682 1683 list_for_each_entry_safe(req, tmp, &dep->started_list, list) 1684 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED); 1685 1686 /* If ep isn't started, then there's no end transfer pending */ 1687 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) 1688 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 1689 1690 return ret; 1691 } 1692 1693 if (dep->stream_capable && req->request.is_last && 1694 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) 1695 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE; 1696 1697 return 0; 1698 } 1699 1700 static int __dwc3_gadget_get_frame(struct dwc3 *dwc) 1701 { 1702 u32 reg; 1703 1704 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1705 return DWC3_DSTS_SOFFN(reg); 1706 } 1707 1708 /** 1709 * __dwc3_stop_active_transfer - stop the current active transfer 1710 * @dep: isoc endpoint 1711 * @force: set forcerm bit in the command 1712 * @interrupt: command complete interrupt after End Transfer command 1713 * 1714 * When setting force, the ForceRM bit will be set. In that case 1715 * the controller won't update the TRB progress on command 1716 * completion. It also won't clear the HWO bit in the TRB. 1717 * The command will also not complete immediately in that case. 1718 */ 1719 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) 1720 { 1721 struct dwc3 *dwc = dep->dwc; 1722 struct dwc3_gadget_ep_cmd_params params; 1723 u32 cmd; 1724 int ret; 1725 1726 cmd = DWC3_DEPCMD_ENDTRANSFER; 1727 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0; 1728 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0; 1729 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); 1730 memset(¶ms, 0, sizeof(params)); 1731 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1732 /* 1733 * If the End Transfer command was timed out while the device is 1734 * not in SETUP phase, it's possible that an incoming Setup packet 1735 * may prevent the command's completion. Let's retry when the 1736 * ep0state returns to EP0_SETUP_PHASE. 1737 */ 1738 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) { 1739 dep->flags |= DWC3_EP_DELAY_STOP; 1740 return 0; 1741 } 1742 WARN_ON_ONCE(ret); 1743 dep->resource_index = 0; 1744 1745 if (!interrupt) { 1746 if (!DWC3_IP_IS(DWC3) || DWC3_VER_IS_PRIOR(DWC3, 310A)) 1747 mdelay(1); 1748 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 1749 } else if (!ret) { 1750 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; 1751 } 1752 1753 dep->flags &= ~DWC3_EP_DELAY_STOP; 1754 return ret; 1755 } 1756 1757 /** 1758 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number 1759 * @dep: isoc endpoint 1760 * 1761 * This function tests for the correct combination of BIT[15:14] from the 16-bit 1762 * microframe number reported by the XferNotReady event for the future frame 1763 * number to start the isoc transfer. 1764 * 1765 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed 1766 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the 1767 * XferNotReady event are invalid. The driver uses this number to schedule the 1768 * isochronous transfer and passes it to the START TRANSFER command. Because 1769 * this number is invalid, the command may fail. If BIT[15:14] matches the 1770 * internal 16-bit microframe, the START TRANSFER command will pass and the 1771 * transfer will start at the scheduled time, if it is off by 1, the command 1772 * will still pass, but the transfer will start 2 seconds in the future. For all 1773 * other conditions, the START TRANSFER command will fail with bus-expiry. 1774 * 1775 * In order to workaround this issue, we can test for the correct combination of 1776 * BIT[15:14] by sending START TRANSFER commands with different values of 1777 * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart 1778 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status. 1779 * As the result, within the 4 possible combinations for BIT[15:14], there will 1780 * be 2 successful and 2 failure START COMMAND status. One of the 2 successful 1781 * command status will result in a 2-second delay start. The smaller BIT[15:14] 1782 * value is the correct combination. 1783 * 1784 * Since there are only 4 outcomes and the results are ordered, we can simply 1785 * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to 1786 * deduce the smaller successful combination. 1787 * 1788 * Let test0 = test status for combination 'b00 and test1 = test status for 'b01 1789 * of BIT[15:14]. The correct combination is as follow: 1790 * 1791 * if test0 fails and test1 passes, BIT[15:14] is 'b01 1792 * if test0 fails and test1 fails, BIT[15:14] is 'b10 1793 * if test0 passes and test1 fails, BIT[15:14] is 'b11 1794 * if test0 passes and test1 passes, BIT[15:14] is 'b00 1795 * 1796 * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN 1797 * endpoints. 1798 */ 1799 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep) 1800 { 1801 int cmd_status = 0; 1802 bool test0; 1803 bool test1; 1804 1805 while (dep->combo_num < 2) { 1806 struct dwc3_gadget_ep_cmd_params params; 1807 u32 test_frame_number; 1808 u32 cmd; 1809 1810 /* 1811 * Check if we can start isoc transfer on the next interval or 1812 * 4 uframes in the future with BIT[15:14] as dep->combo_num 1813 */ 1814 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK; 1815 test_frame_number |= dep->combo_num << 14; 1816 test_frame_number += max_t(u32, 4, dep->interval); 1817 1818 params.param0 = upper_32_bits(dep->dwc->bounce_addr); 1819 params.param1 = lower_32_bits(dep->dwc->bounce_addr); 1820 1821 cmd = DWC3_DEPCMD_STARTTRANSFER; 1822 cmd |= DWC3_DEPCMD_PARAM(test_frame_number); 1823 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1824 1825 /* Redo if some other failure beside bus-expiry is received */ 1826 if (cmd_status && cmd_status != -EAGAIN) { 1827 dep->start_cmd_status = 0; 1828 dep->combo_num = 0; 1829 return 0; 1830 } 1831 1832 /* Store the first test status */ 1833 if (dep->combo_num == 0) 1834 dep->start_cmd_status = cmd_status; 1835 1836 dep->combo_num++; 1837 1838 /* 1839 * End the transfer if the START_TRANSFER command is successful 1840 * to wait for the next XferNotReady to test the command again 1841 */ 1842 if (cmd_status == 0) { 1843 dwc3_stop_active_transfer(dep, true, true); 1844 return 0; 1845 } 1846 } 1847 1848 /* test0 and test1 are both completed at this point */ 1849 test0 = (dep->start_cmd_status == 0); 1850 test1 = (cmd_status == 0); 1851 1852 if (!test0 && test1) 1853 dep->combo_num = 1; 1854 else if (!test0 && !test1) 1855 dep->combo_num = 2; 1856 else if (test0 && !test1) 1857 dep->combo_num = 3; 1858 else if (test0 && test1) 1859 dep->combo_num = 0; 1860 1861 dep->frame_number &= DWC3_FRNUMBER_MASK; 1862 dep->frame_number |= dep->combo_num << 14; 1863 dep->frame_number += max_t(u32, 4, dep->interval); 1864 1865 /* Reinitialize test variables */ 1866 dep->start_cmd_status = 0; 1867 dep->combo_num = 0; 1868 1869 return __dwc3_gadget_kick_transfer(dep); 1870 } 1871 1872 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep) 1873 { 1874 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 1875 struct dwc3 *dwc = dep->dwc; 1876 int ret; 1877 int i; 1878 1879 if (list_empty(&dep->pending_list) && 1880 list_empty(&dep->started_list)) { 1881 dep->flags |= DWC3_EP_PENDING_REQUEST; 1882 return -EAGAIN; 1883 } 1884 1885 if (!dwc->dis_start_transfer_quirk && 1886 (DWC3_VER_IS_PRIOR(DWC31, 170A) || 1887 DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) { 1888 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction) 1889 return dwc3_gadget_start_isoc_quirk(dep); 1890 } 1891 1892 if (desc->bInterval <= 14 && 1893 dwc->gadget->speed >= USB_SPEED_HIGH) { 1894 u32 frame = __dwc3_gadget_get_frame(dwc); 1895 bool rollover = frame < 1896 (dep->frame_number & DWC3_FRNUMBER_MASK); 1897 1898 /* 1899 * frame_number is set from XferNotReady and may be already 1900 * out of date. DSTS only provides the lower 14 bit of the 1901 * current frame number. So add the upper two bits of 1902 * frame_number and handle a possible rollover. 1903 * This will provide the correct frame_number unless more than 1904 * rollover has happened since XferNotReady. 1905 */ 1906 1907 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) | 1908 frame; 1909 if (rollover) 1910 dep->frame_number += BIT(14); 1911 } 1912 1913 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) { 1914 int future_interval = i + 1; 1915 1916 /* Give the controller at least 500us to schedule transfers */ 1917 if (desc->bInterval < 3) 1918 future_interval += 3 - desc->bInterval; 1919 1920 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval); 1921 1922 ret = __dwc3_gadget_kick_transfer(dep); 1923 if (ret != -EAGAIN) 1924 break; 1925 } 1926 1927 /* 1928 * After a number of unsuccessful start attempts due to bus-expiry 1929 * status, issue END_TRANSFER command and retry on the next XferNotReady 1930 * event. 1931 */ 1932 if (ret == -EAGAIN) 1933 ret = __dwc3_stop_active_transfer(dep, false, true); 1934 1935 return ret; 1936 } 1937 1938 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) 1939 { 1940 struct dwc3 *dwc = dep->dwc; 1941 1942 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { 1943 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n", 1944 dep->name); 1945 return -ESHUTDOWN; 1946 } 1947 1948 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n", 1949 &req->request, req->dep->name)) 1950 return -EINVAL; 1951 1952 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED, 1953 "%s: request %pK already in flight\n", 1954 dep->name, &req->request)) 1955 return -EINVAL; 1956 1957 pm_runtime_get(dwc->dev); 1958 1959 req->request.actual = 0; 1960 req->request.status = -EINPROGRESS; 1961 1962 trace_dwc3_ep_queue(req); 1963 1964 list_add_tail(&req->list, &dep->pending_list); 1965 req->status = DWC3_REQUEST_STATUS_QUEUED; 1966 1967 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE) 1968 return 0; 1969 1970 /* 1971 * Start the transfer only after the END_TRANSFER is completed 1972 * and endpoint STALL is cleared. 1973 */ 1974 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || 1975 (dep->flags & DWC3_EP_WEDGE) || 1976 (dep->flags & DWC3_EP_DELAY_STOP) || 1977 (dep->flags & DWC3_EP_STALL)) { 1978 dep->flags |= DWC3_EP_DELAY_START; 1979 return 0; 1980 } 1981 1982 /* 1983 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must 1984 * wait for a XferNotReady event so we will know what's the current 1985 * (micro-)frame number. 1986 * 1987 * Without this trick, we are very, very likely gonna get Bus Expiry 1988 * errors which will force us issue EndTransfer command. 1989 */ 1990 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 1991 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { 1992 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) 1993 return __dwc3_gadget_start_isoc(dep); 1994 1995 return 0; 1996 } 1997 } 1998 1999 __dwc3_gadget_kick_transfer(dep); 2000 2001 return 0; 2002 } 2003 2004 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, 2005 gfp_t gfp_flags) 2006 { 2007 struct dwc3_request *req = to_dwc3_request(request); 2008 struct dwc3_ep *dep = to_dwc3_ep(ep); 2009 struct dwc3 *dwc = dep->dwc; 2010 2011 unsigned long flags; 2012 2013 int ret; 2014 2015 spin_lock_irqsave(&dwc->lock, flags); 2016 ret = __dwc3_gadget_ep_queue(dep, req); 2017 spin_unlock_irqrestore(&dwc->lock, flags); 2018 2019 return ret; 2020 } 2021 2022 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req) 2023 { 2024 int i; 2025 2026 /* If req->trb is not set, then the request has not started */ 2027 if (!req->trb) 2028 return; 2029 2030 /* 2031 * If request was already started, this means we had to 2032 * stop the transfer. With that we also need to ignore 2033 * all TRBs used by the request, however TRBs can only 2034 * be modified after completion of END_TRANSFER 2035 * command. So what we do here is that we wait for 2036 * END_TRANSFER completion and only after that, we jump 2037 * over TRBs by clearing HWO and incrementing dequeue 2038 * pointer. 2039 */ 2040 for (i = 0; i < req->num_trbs; i++) { 2041 struct dwc3_trb *trb; 2042 2043 trb = &dep->trb_pool[dep->trb_dequeue]; 2044 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 2045 dwc3_ep_inc_deq(dep); 2046 } 2047 2048 req->num_trbs = 0; 2049 } 2050 2051 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep) 2052 { 2053 struct dwc3_request *req; 2054 struct dwc3 *dwc = dep->dwc; 2055 2056 while (!list_empty(&dep->cancelled_list)) { 2057 req = next_request(&dep->cancelled_list); 2058 dwc3_gadget_ep_skip_trbs(dep, req); 2059 switch (req->status) { 2060 case DWC3_REQUEST_STATUS_DISCONNECTED: 2061 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 2062 break; 2063 case DWC3_REQUEST_STATUS_DEQUEUED: 2064 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2065 break; 2066 case DWC3_REQUEST_STATUS_STALLED: 2067 dwc3_gadget_giveback(dep, req, -EPIPE); 2068 break; 2069 default: 2070 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status); 2071 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2072 break; 2073 } 2074 /* 2075 * The endpoint is disabled, let the dwc3_remove_requests() 2076 * handle the cleanup. 2077 */ 2078 if (!dep->endpoint.desc) 2079 break; 2080 } 2081 } 2082 2083 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, 2084 struct usb_request *request) 2085 { 2086 struct dwc3_request *req = to_dwc3_request(request); 2087 struct dwc3_request *r = NULL; 2088 2089 struct dwc3_ep *dep = to_dwc3_ep(ep); 2090 struct dwc3 *dwc = dep->dwc; 2091 2092 unsigned long flags; 2093 int ret = 0; 2094 2095 trace_dwc3_ep_dequeue(req); 2096 2097 spin_lock_irqsave(&dwc->lock, flags); 2098 2099 list_for_each_entry(r, &dep->cancelled_list, list) { 2100 if (r == req) 2101 goto out; 2102 } 2103 2104 list_for_each_entry(r, &dep->pending_list, list) { 2105 if (r == req) { 2106 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2107 goto out; 2108 } 2109 } 2110 2111 list_for_each_entry(r, &dep->started_list, list) { 2112 if (r == req) { 2113 struct dwc3_request *t; 2114 2115 /* wait until it is processed */ 2116 dwc3_stop_active_transfer(dep, true, true); 2117 2118 /* 2119 * Remove any started request if the transfer is 2120 * cancelled. 2121 */ 2122 list_for_each_entry_safe(r, t, &dep->started_list, list) 2123 dwc3_gadget_move_cancelled_request(r, 2124 DWC3_REQUEST_STATUS_DEQUEUED); 2125 2126 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; 2127 2128 goto out; 2129 } 2130 } 2131 2132 dev_err(dwc->dev, "request %pK was not queued to %s\n", 2133 request, ep->name); 2134 ret = -EINVAL; 2135 out: 2136 spin_unlock_irqrestore(&dwc->lock, flags); 2137 2138 return ret; 2139 } 2140 2141 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) 2142 { 2143 struct dwc3_gadget_ep_cmd_params params; 2144 struct dwc3 *dwc = dep->dwc; 2145 struct dwc3_request *req; 2146 struct dwc3_request *tmp; 2147 int ret; 2148 2149 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 2150 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); 2151 return -EINVAL; 2152 } 2153 2154 memset(¶ms, 0x00, sizeof(params)); 2155 2156 if (value) { 2157 struct dwc3_trb *trb; 2158 2159 unsigned int transfer_in_flight; 2160 unsigned int started; 2161 2162 if (dep->number > 1) 2163 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); 2164 else 2165 trb = &dwc->ep0_trb[dep->trb_enqueue]; 2166 2167 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO; 2168 started = !list_empty(&dep->started_list); 2169 2170 if (!protocol && ((dep->direction && transfer_in_flight) || 2171 (!dep->direction && started))) { 2172 return -EAGAIN; 2173 } 2174 2175 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL, 2176 ¶ms); 2177 if (ret) 2178 dev_err(dwc->dev, "failed to set STALL on %s\n", 2179 dep->name); 2180 else 2181 dep->flags |= DWC3_EP_STALL; 2182 } else { 2183 /* 2184 * Don't issue CLEAR_STALL command to control endpoints. The 2185 * controller automatically clears the STALL when it receives 2186 * the SETUP token. 2187 */ 2188 if (dep->number <= 1) { 2189 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 2190 return 0; 2191 } 2192 2193 dwc3_stop_active_transfer(dep, true, true); 2194 2195 list_for_each_entry_safe(req, tmp, &dep->started_list, list) 2196 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED); 2197 2198 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING || 2199 (dep->flags & DWC3_EP_DELAY_STOP)) { 2200 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; 2201 if (protocol) 2202 dwc->clear_stall_protocol = dep->number; 2203 2204 return 0; 2205 } 2206 2207 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 2208 2209 ret = dwc3_send_clear_stall_ep_cmd(dep); 2210 if (ret) { 2211 dev_err(dwc->dev, "failed to clear STALL on %s\n", 2212 dep->name); 2213 return ret; 2214 } 2215 2216 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 2217 2218 if ((dep->flags & DWC3_EP_DELAY_START) && 2219 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) 2220 __dwc3_gadget_kick_transfer(dep); 2221 2222 dep->flags &= ~DWC3_EP_DELAY_START; 2223 } 2224 2225 return ret; 2226 } 2227 2228 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value) 2229 { 2230 struct dwc3_ep *dep = to_dwc3_ep(ep); 2231 struct dwc3 *dwc = dep->dwc; 2232 2233 unsigned long flags; 2234 2235 int ret; 2236 2237 spin_lock_irqsave(&dwc->lock, flags); 2238 ret = __dwc3_gadget_ep_set_halt(dep, value, false); 2239 spin_unlock_irqrestore(&dwc->lock, flags); 2240 2241 return ret; 2242 } 2243 2244 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep) 2245 { 2246 struct dwc3_ep *dep = to_dwc3_ep(ep); 2247 struct dwc3 *dwc = dep->dwc; 2248 unsigned long flags; 2249 int ret; 2250 2251 spin_lock_irqsave(&dwc->lock, flags); 2252 dep->flags |= DWC3_EP_WEDGE; 2253 2254 if (dep->number == 0 || dep->number == 1) 2255 ret = __dwc3_gadget_ep0_set_halt(ep, 1); 2256 else 2257 ret = __dwc3_gadget_ep_set_halt(dep, 1, false); 2258 spin_unlock_irqrestore(&dwc->lock, flags); 2259 2260 return ret; 2261 } 2262 2263 /* -------------------------------------------------------------------------- */ 2264 2265 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = { 2266 .bLength = USB_DT_ENDPOINT_SIZE, 2267 .bDescriptorType = USB_DT_ENDPOINT, 2268 .bmAttributes = USB_ENDPOINT_XFER_CONTROL, 2269 }; 2270 2271 static const struct usb_ep_ops dwc3_gadget_ep0_ops = { 2272 .enable = dwc3_gadget_ep0_enable, 2273 .disable = dwc3_gadget_ep0_disable, 2274 .alloc_request = dwc3_gadget_ep_alloc_request, 2275 .free_request = dwc3_gadget_ep_free_request, 2276 .queue = dwc3_gadget_ep0_queue, 2277 .dequeue = dwc3_gadget_ep_dequeue, 2278 .set_halt = dwc3_gadget_ep0_set_halt, 2279 .set_wedge = dwc3_gadget_ep_set_wedge, 2280 }; 2281 2282 static const struct usb_ep_ops dwc3_gadget_ep_ops = { 2283 .enable = dwc3_gadget_ep_enable, 2284 .disable = dwc3_gadget_ep_disable, 2285 .alloc_request = dwc3_gadget_ep_alloc_request, 2286 .free_request = dwc3_gadget_ep_free_request, 2287 .queue = dwc3_gadget_ep_queue, 2288 .dequeue = dwc3_gadget_ep_dequeue, 2289 .set_halt = dwc3_gadget_ep_set_halt, 2290 .set_wedge = dwc3_gadget_ep_set_wedge, 2291 }; 2292 2293 /* -------------------------------------------------------------------------- */ 2294 2295 static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set) 2296 { 2297 u32 reg; 2298 2299 if (DWC3_VER_IS_PRIOR(DWC3, 250A)) 2300 return; 2301 2302 reg = dwc3_readl(dwc->regs, DWC3_DEVTEN); 2303 if (set) 2304 reg |= DWC3_DEVTEN_ULSTCNGEN; 2305 else 2306 reg &= ~DWC3_DEVTEN_ULSTCNGEN; 2307 2308 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); 2309 } 2310 2311 static int dwc3_gadget_get_frame(struct usb_gadget *g) 2312 { 2313 struct dwc3 *dwc = gadget_to_dwc(g); 2314 2315 return __dwc3_gadget_get_frame(dwc); 2316 } 2317 2318 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async) 2319 { 2320 int retries; 2321 2322 int ret; 2323 u32 reg; 2324 2325 u8 link_state; 2326 2327 /* 2328 * According to the Databook Remote wakeup request should 2329 * be issued only when the device is in early suspend state. 2330 * 2331 * We can check that via USB Link State bits in DSTS register. 2332 */ 2333 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2334 2335 link_state = DWC3_DSTS_USBLNKST(reg); 2336 2337 switch (link_state) { 2338 case DWC3_LINK_STATE_RESET: 2339 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */ 2340 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */ 2341 case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */ 2342 case DWC3_LINK_STATE_U1: 2343 case DWC3_LINK_STATE_RESUME: 2344 break; 2345 default: 2346 return -EINVAL; 2347 } 2348 2349 if (async) 2350 dwc3_gadget_enable_linksts_evts(dwc, true); 2351 2352 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV); 2353 if (ret < 0) { 2354 dev_err(dwc->dev, "failed to put link in Recovery\n"); 2355 dwc3_gadget_enable_linksts_evts(dwc, false); 2356 return ret; 2357 } 2358 2359 /* Recent versions do this automatically */ 2360 if (DWC3_VER_IS_PRIOR(DWC3, 194A)) { 2361 /* write zeroes to Link Change Request */ 2362 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2363 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 2364 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2365 } 2366 2367 /* 2368 * Since link status change events are enabled we will receive 2369 * an U0 event when wakeup is successful. So bail out. 2370 */ 2371 if (async) 2372 return 0; 2373 2374 /* poll until Link State changes to ON */ 2375 retries = 20000; 2376 2377 while (retries--) { 2378 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2379 2380 /* in HS, means ON */ 2381 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0) 2382 break; 2383 } 2384 2385 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) { 2386 dev_err(dwc->dev, "failed to send remote wakeup\n"); 2387 return -EINVAL; 2388 } 2389 2390 return 0; 2391 } 2392 2393 static int dwc3_gadget_wakeup(struct usb_gadget *g) 2394 { 2395 struct dwc3 *dwc = gadget_to_dwc(g); 2396 unsigned long flags; 2397 int ret; 2398 2399 if (!dwc->wakeup_configured) { 2400 dev_err(dwc->dev, "remote wakeup not configured\n"); 2401 return -EINVAL; 2402 } 2403 2404 spin_lock_irqsave(&dwc->lock, flags); 2405 if (!dwc->gadget->wakeup_armed) { 2406 dev_err(dwc->dev, "not armed for remote wakeup\n"); 2407 spin_unlock_irqrestore(&dwc->lock, flags); 2408 return -EINVAL; 2409 } 2410 ret = __dwc3_gadget_wakeup(dwc, true); 2411 2412 spin_unlock_irqrestore(&dwc->lock, flags); 2413 2414 return ret; 2415 } 2416 2417 static void dwc3_resume_gadget(struct dwc3 *dwc); 2418 2419 static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id) 2420 { 2421 struct dwc3 *dwc = gadget_to_dwc(g); 2422 unsigned long flags; 2423 int ret; 2424 int link_state; 2425 2426 if (!dwc->wakeup_configured) { 2427 dev_err(dwc->dev, "remote wakeup not configured\n"); 2428 return -EINVAL; 2429 } 2430 2431 spin_lock_irqsave(&dwc->lock, flags); 2432 /* 2433 * If the link is in U3, signal for remote wakeup and wait for the 2434 * link to transition to U0 before sending device notification. 2435 */ 2436 link_state = dwc3_gadget_get_link_state(dwc); 2437 if (link_state == DWC3_LINK_STATE_U3) { 2438 ret = __dwc3_gadget_wakeup(dwc, false); 2439 if (ret) { 2440 spin_unlock_irqrestore(&dwc->lock, flags); 2441 return -EINVAL; 2442 } 2443 dwc3_resume_gadget(dwc); 2444 dwc->suspended = false; 2445 dwc->link_state = DWC3_LINK_STATE_U0; 2446 } 2447 2448 ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION, 2449 DWC3_DGCMDPAR_DN_FUNC_WAKE | 2450 DWC3_DGCMDPAR_INTF_SEL(intf_id)); 2451 if (ret) 2452 dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret); 2453 2454 spin_unlock_irqrestore(&dwc->lock, flags); 2455 2456 return ret; 2457 } 2458 2459 static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set) 2460 { 2461 struct dwc3 *dwc = gadget_to_dwc(g); 2462 unsigned long flags; 2463 2464 spin_lock_irqsave(&dwc->lock, flags); 2465 dwc->wakeup_configured = !!set; 2466 spin_unlock_irqrestore(&dwc->lock, flags); 2467 2468 return 0; 2469 } 2470 2471 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g, 2472 int is_selfpowered) 2473 { 2474 struct dwc3 *dwc = gadget_to_dwc(g); 2475 unsigned long flags; 2476 2477 spin_lock_irqsave(&dwc->lock, flags); 2478 g->is_selfpowered = !!is_selfpowered; 2479 spin_unlock_irqrestore(&dwc->lock, flags); 2480 2481 return 0; 2482 } 2483 2484 static void dwc3_stop_active_transfers(struct dwc3 *dwc) 2485 { 2486 u32 epnum; 2487 2488 for (epnum = 2; epnum < dwc->num_eps; epnum++) { 2489 struct dwc3_ep *dep; 2490 2491 dep = dwc->eps[epnum]; 2492 if (!dep) 2493 continue; 2494 2495 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); 2496 } 2497 } 2498 2499 static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc) 2500 { 2501 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate; 2502 u32 reg; 2503 2504 if (ssp_rate == USB_SSP_GEN_UNKNOWN) 2505 ssp_rate = dwc->max_ssp_rate; 2506 2507 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2508 reg &= ~DWC3_DCFG_SPEED_MASK; 2509 reg &= ~DWC3_DCFG_NUMLANES(~0); 2510 2511 if (ssp_rate == USB_SSP_GEN_1x2) 2512 reg |= DWC3_DCFG_SUPERSPEED; 2513 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2) 2514 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2515 2516 if (ssp_rate != USB_SSP_GEN_2x1 && 2517 dwc->max_ssp_rate != USB_SSP_GEN_2x1) 2518 reg |= DWC3_DCFG_NUMLANES(1); 2519 2520 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2521 } 2522 2523 static void __dwc3_gadget_set_speed(struct dwc3 *dwc) 2524 { 2525 enum usb_device_speed speed; 2526 u32 reg; 2527 2528 speed = dwc->gadget_max_speed; 2529 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed) 2530 speed = dwc->maximum_speed; 2531 2532 if (speed == USB_SPEED_SUPER_PLUS && 2533 DWC3_IP_IS(DWC32)) { 2534 __dwc3_gadget_set_ssp_rate(dwc); 2535 return; 2536 } 2537 2538 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2539 reg &= ~(DWC3_DCFG_SPEED_MASK); 2540 2541 /* 2542 * WORKAROUND: DWC3 revision < 2.20a have an issue 2543 * which would cause metastability state on Run/Stop 2544 * bit if we try to force the IP to USB2-only mode. 2545 * 2546 * Because of that, we cannot configure the IP to any 2547 * speed other than the SuperSpeed 2548 * 2549 * Refers to: 2550 * 2551 * STAR#9000525659: Clock Domain Crossing on DCTL in 2552 * USB 2.0 Mode 2553 */ 2554 if (DWC3_VER_IS_PRIOR(DWC3, 220A) && 2555 !dwc->dis_metastability_quirk) { 2556 reg |= DWC3_DCFG_SUPERSPEED; 2557 } else { 2558 switch (speed) { 2559 case USB_SPEED_FULL: 2560 reg |= DWC3_DCFG_FULLSPEED; 2561 break; 2562 case USB_SPEED_HIGH: 2563 reg |= DWC3_DCFG_HIGHSPEED; 2564 break; 2565 case USB_SPEED_SUPER: 2566 reg |= DWC3_DCFG_SUPERSPEED; 2567 break; 2568 case USB_SPEED_SUPER_PLUS: 2569 if (DWC3_IP_IS(DWC3)) 2570 reg |= DWC3_DCFG_SUPERSPEED; 2571 else 2572 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2573 break; 2574 default: 2575 dev_err(dwc->dev, "invalid speed (%d)\n", speed); 2576 2577 if (DWC3_IP_IS(DWC3)) 2578 reg |= DWC3_DCFG_SUPERSPEED; 2579 else 2580 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2581 } 2582 } 2583 2584 if (DWC3_IP_IS(DWC32) && 2585 speed > USB_SPEED_UNKNOWN && 2586 speed < USB_SPEED_SUPER_PLUS) 2587 reg &= ~DWC3_DCFG_NUMLANES(~0); 2588 2589 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2590 } 2591 2592 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on) 2593 { 2594 u32 reg; 2595 u32 timeout = 2000; 2596 2597 if (pm_runtime_suspended(dwc->dev)) 2598 return 0; 2599 2600 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2601 if (is_on) { 2602 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) { 2603 reg &= ~DWC3_DCTL_TRGTULST_MASK; 2604 reg |= DWC3_DCTL_TRGTULST_RX_DET; 2605 } 2606 2607 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) 2608 reg &= ~DWC3_DCTL_KEEP_CONNECT; 2609 reg |= DWC3_DCTL_RUN_STOP; 2610 2611 __dwc3_gadget_set_speed(dwc); 2612 dwc->pullups_connected = true; 2613 } else { 2614 reg &= ~DWC3_DCTL_RUN_STOP; 2615 2616 dwc->pullups_connected = false; 2617 } 2618 2619 dwc3_gadget_dctl_write_safe(dwc, reg); 2620 2621 do { 2622 usleep_range(1000, 2000); 2623 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2624 reg &= DWC3_DSTS_DEVCTRLHLT; 2625 } while (--timeout && !(!is_on ^ !reg)); 2626 2627 if (!timeout) 2628 return -ETIMEDOUT; 2629 2630 return 0; 2631 } 2632 2633 static void dwc3_gadget_disable_irq(struct dwc3 *dwc); 2634 static void __dwc3_gadget_stop(struct dwc3 *dwc); 2635 static int __dwc3_gadget_start(struct dwc3 *dwc); 2636 2637 static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc) 2638 { 2639 unsigned long flags; 2640 int ret; 2641 2642 spin_lock_irqsave(&dwc->lock, flags); 2643 dwc->connected = false; 2644 2645 /* 2646 * Attempt to end pending SETUP status phase, and not wait for the 2647 * function to do so. 2648 */ 2649 if (dwc->delayed_status) 2650 dwc3_ep0_send_delayed_status(dwc); 2651 2652 /* 2653 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a 2654 * Section 4.1.8 Table 4-7, it states that for a device-initiated 2655 * disconnect, the SW needs to ensure that it sends "a DEPENDXFER 2656 * command for any active transfers" before clearing the RunStop 2657 * bit. 2658 */ 2659 dwc3_stop_active_transfers(dwc); 2660 spin_unlock_irqrestore(&dwc->lock, flags); 2661 2662 /* 2663 * Per databook, when we want to stop the gadget, if a control transfer 2664 * is still in process, complete it and get the core into setup phase. 2665 * In case the host is unresponsive to a SETUP transaction, forcefully 2666 * stall the transfer, and move back to the SETUP phase, so that any 2667 * pending endxfers can be executed. 2668 */ 2669 if (dwc->ep0state != EP0_SETUP_PHASE) { 2670 reinit_completion(&dwc->ep0_in_setup); 2671 2672 ret = wait_for_completion_timeout(&dwc->ep0_in_setup, 2673 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT)); 2674 if (ret == 0) { 2675 dev_warn(dwc->dev, "wait for SETUP phase timed out\n"); 2676 spin_lock_irqsave(&dwc->lock, flags); 2677 dwc3_ep0_reset_state(dwc); 2678 spin_unlock_irqrestore(&dwc->lock, flags); 2679 } 2680 } 2681 2682 /* 2683 * Note: if the GEVNTCOUNT indicates events in the event buffer, the 2684 * driver needs to acknowledge them before the controller can halt. 2685 * Simply let the interrupt handler acknowledges and handle the 2686 * remaining event generated by the controller while polling for 2687 * DSTS.DEVCTLHLT. 2688 */ 2689 ret = dwc3_gadget_run_stop(dwc, false); 2690 2691 /* 2692 * Stop the gadget after controller is halted, so that if needed, the 2693 * events to update EP0 state can still occur while the run/stop 2694 * routine polls for the halted state. DEVTEN is cleared as part of 2695 * gadget stop. 2696 */ 2697 spin_lock_irqsave(&dwc->lock, flags); 2698 __dwc3_gadget_stop(dwc); 2699 spin_unlock_irqrestore(&dwc->lock, flags); 2700 2701 return ret; 2702 } 2703 2704 static int dwc3_gadget_soft_connect(struct dwc3 *dwc) 2705 { 2706 int ret; 2707 2708 /* 2709 * In the Synopsys DWC_usb31 1.90a programming guide section 2710 * 4.1.9, it specifies that for a reconnect after a 2711 * device-initiated disconnect requires a core soft reset 2712 * (DCTL.CSftRst) before enabling the run/stop bit. 2713 */ 2714 ret = dwc3_core_soft_reset(dwc); 2715 if (ret) 2716 return ret; 2717 2718 dwc3_event_buffers_setup(dwc); 2719 __dwc3_gadget_start(dwc); 2720 return dwc3_gadget_run_stop(dwc, true); 2721 } 2722 2723 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) 2724 { 2725 struct dwc3 *dwc = gadget_to_dwc(g); 2726 int ret; 2727 2728 is_on = !!is_on; 2729 2730 dwc->softconnect = is_on; 2731 2732 /* 2733 * Avoid issuing a runtime resume if the device is already in the 2734 * suspended state during gadget disconnect. DWC3 gadget was already 2735 * halted/stopped during runtime suspend. 2736 */ 2737 if (!is_on) { 2738 pm_runtime_barrier(dwc->dev); 2739 if (pm_runtime_suspended(dwc->dev)) 2740 return 0; 2741 } 2742 2743 /* 2744 * Check the return value for successful resume, or error. For a 2745 * successful resume, the DWC3 runtime PM resume routine will handle 2746 * the run stop sequence, so avoid duplicate operations here. 2747 */ 2748 ret = pm_runtime_get_sync(dwc->dev); 2749 if (!ret || ret < 0) { 2750 pm_runtime_put(dwc->dev); 2751 if (ret < 0) 2752 pm_runtime_set_suspended(dwc->dev); 2753 return ret; 2754 } 2755 2756 if (dwc->pullups_connected == is_on) { 2757 pm_runtime_put(dwc->dev); 2758 return 0; 2759 } 2760 2761 synchronize_irq(dwc->irq_gadget); 2762 2763 if (!is_on) 2764 ret = dwc3_gadget_soft_disconnect(dwc); 2765 else 2766 ret = dwc3_gadget_soft_connect(dwc); 2767 2768 pm_runtime_put(dwc->dev); 2769 2770 return ret; 2771 } 2772 2773 static void dwc3_gadget_enable_irq(struct dwc3 *dwc) 2774 { 2775 u32 reg; 2776 2777 /* Enable all but Start and End of Frame IRQs */ 2778 reg = (DWC3_DEVTEN_EVNTOVERFLOWEN | 2779 DWC3_DEVTEN_CMDCMPLTEN | 2780 DWC3_DEVTEN_ERRTICERREN | 2781 DWC3_DEVTEN_WKUPEVTEN | 2782 DWC3_DEVTEN_CONNECTDONEEN | 2783 DWC3_DEVTEN_USBRSTEN | 2784 DWC3_DEVTEN_DISCONNEVTEN); 2785 2786 if (DWC3_VER_IS_PRIOR(DWC3, 250A)) 2787 reg |= DWC3_DEVTEN_ULSTCNGEN; 2788 2789 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */ 2790 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) 2791 reg |= DWC3_DEVTEN_U3L2L1SUSPEN; 2792 2793 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); 2794 } 2795 2796 static void dwc3_gadget_disable_irq(struct dwc3 *dwc) 2797 { 2798 /* mask all interrupts */ 2799 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00); 2800 } 2801 2802 static irqreturn_t dwc3_interrupt(int irq, void *_dwc); 2803 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc); 2804 2805 /** 2806 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG 2807 * @dwc: pointer to our context structure 2808 * 2809 * The following looks like complex but it's actually very simple. In order to 2810 * calculate the number of packets we can burst at once on OUT transfers, we're 2811 * gonna use RxFIFO size. 2812 * 2813 * To calculate RxFIFO size we need two numbers: 2814 * MDWIDTH = size, in bits, of the internal memory bus 2815 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits) 2816 * 2817 * Given these two numbers, the formula is simple: 2818 * 2819 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16; 2820 * 2821 * 24 bytes is for 3x SETUP packets 2822 * 16 bytes is a clock domain crossing tolerance 2823 * 2824 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024; 2825 */ 2826 static void dwc3_gadget_setup_nump(struct dwc3 *dwc) 2827 { 2828 u32 ram2_depth; 2829 u32 mdwidth; 2830 u32 nump; 2831 u32 reg; 2832 2833 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7); 2834 mdwidth = dwc3_mdwidth(dwc); 2835 2836 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024; 2837 nump = min_t(u32, nump, 16); 2838 2839 /* update NumP */ 2840 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2841 reg &= ~DWC3_DCFG_NUMP_MASK; 2842 reg |= nump << DWC3_DCFG_NUMP_SHIFT; 2843 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2844 } 2845 2846 static int __dwc3_gadget_start(struct dwc3 *dwc) 2847 { 2848 struct dwc3_ep *dep; 2849 int ret = 0; 2850 u32 reg; 2851 2852 /* 2853 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if 2854 * the core supports IMOD, disable it. 2855 */ 2856 if (dwc->imod_interval) { 2857 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); 2858 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); 2859 } else if (dwc3_has_imod(dwc)) { 2860 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0); 2861 } 2862 2863 /* 2864 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP 2865 * field instead of letting dwc3 itself calculate that automatically. 2866 * 2867 * This way, we maximize the chances that we'll be able to get several 2868 * bursts of data without going through any sort of endpoint throttling. 2869 */ 2870 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG); 2871 if (DWC3_IP_IS(DWC3)) 2872 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL; 2873 else 2874 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL; 2875 2876 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg); 2877 2878 dwc3_gadget_setup_nump(dwc); 2879 2880 /* 2881 * Currently the controller handles single stream only. So, Ignore 2882 * Packet Pending bit for stream selection and don't search for another 2883 * stream if the host sends Data Packet with PP=0 (for OUT direction) or 2884 * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves 2885 * the stream performance. 2886 */ 2887 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2888 reg |= DWC3_DCFG_IGNSTRMPP; 2889 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2890 2891 /* Enable MST by default if the device is capable of MST */ 2892 if (DWC3_MST_CAPABLE(&dwc->hwparams)) { 2893 reg = dwc3_readl(dwc->regs, DWC3_DCFG1); 2894 reg &= ~DWC3_DCFG1_DIS_MST_ENH; 2895 dwc3_writel(dwc->regs, DWC3_DCFG1, reg); 2896 } 2897 2898 /* Start with SuperSpeed Default */ 2899 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 2900 2901 dep = dwc->eps[0]; 2902 dep->flags = 0; 2903 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 2904 if (ret) { 2905 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2906 goto err0; 2907 } 2908 2909 dep = dwc->eps[1]; 2910 dep->flags = 0; 2911 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 2912 if (ret) { 2913 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2914 goto err1; 2915 } 2916 2917 /* begin to receive SETUP packets */ 2918 dwc->ep0state = EP0_SETUP_PHASE; 2919 dwc->ep0_bounced = false; 2920 dwc->link_state = DWC3_LINK_STATE_SS_DIS; 2921 dwc->delayed_status = false; 2922 dwc3_ep0_out_start(dwc); 2923 2924 dwc3_gadget_enable_irq(dwc); 2925 2926 return 0; 2927 2928 err1: 2929 __dwc3_gadget_ep_disable(dwc->eps[0]); 2930 2931 err0: 2932 return ret; 2933 } 2934 2935 static int dwc3_gadget_start(struct usb_gadget *g, 2936 struct usb_gadget_driver *driver) 2937 { 2938 struct dwc3 *dwc = gadget_to_dwc(g); 2939 unsigned long flags; 2940 int ret; 2941 int irq; 2942 2943 irq = dwc->irq_gadget; 2944 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt, 2945 IRQF_SHARED, "dwc3", dwc->ev_buf); 2946 if (ret) { 2947 dev_err(dwc->dev, "failed to request irq #%d --> %d\n", 2948 irq, ret); 2949 return ret; 2950 } 2951 2952 spin_lock_irqsave(&dwc->lock, flags); 2953 dwc->gadget_driver = driver; 2954 spin_unlock_irqrestore(&dwc->lock, flags); 2955 2956 return 0; 2957 } 2958 2959 static void __dwc3_gadget_stop(struct dwc3 *dwc) 2960 { 2961 dwc3_gadget_disable_irq(dwc); 2962 __dwc3_gadget_ep_disable(dwc->eps[0]); 2963 __dwc3_gadget_ep_disable(dwc->eps[1]); 2964 } 2965 2966 static int dwc3_gadget_stop(struct usb_gadget *g) 2967 { 2968 struct dwc3 *dwc = gadget_to_dwc(g); 2969 unsigned long flags; 2970 2971 spin_lock_irqsave(&dwc->lock, flags); 2972 dwc->gadget_driver = NULL; 2973 dwc->max_cfg_eps = 0; 2974 spin_unlock_irqrestore(&dwc->lock, flags); 2975 2976 free_irq(dwc->irq_gadget, dwc->ev_buf); 2977 2978 return 0; 2979 } 2980 2981 static void dwc3_gadget_config_params(struct usb_gadget *g, 2982 struct usb_dcd_config_params *params) 2983 { 2984 struct dwc3 *dwc = gadget_to_dwc(g); 2985 2986 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED; 2987 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED; 2988 2989 /* Recommended BESL */ 2990 if (!dwc->dis_enblslpm_quirk) { 2991 /* 2992 * If the recommended BESL baseline is 0 or if the BESL deep is 2993 * less than 2, Microsoft's Windows 10 host usb stack will issue 2994 * a usb reset immediately after it receives the extended BOS 2995 * descriptor and the enumeration will fail. To maintain 2996 * compatibility with the Windows' usb stack, let's set the 2997 * recommended BESL baseline to 1 and clamp the BESL deep to be 2998 * within 2 to 15. 2999 */ 3000 params->besl_baseline = 1; 3001 if (dwc->is_utmi_l1_suspend) 3002 params->besl_deep = 3003 clamp_t(u8, dwc->hird_threshold, 2, 15); 3004 } 3005 3006 /* U1 Device exit Latency */ 3007 if (dwc->dis_u1_entry_quirk) 3008 params->bU1devExitLat = 0; 3009 else 3010 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT; 3011 3012 /* U2 Device exit Latency */ 3013 if (dwc->dis_u2_entry_quirk) 3014 params->bU2DevExitLat = 0; 3015 else 3016 params->bU2DevExitLat = 3017 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT); 3018 } 3019 3020 static void dwc3_gadget_set_speed(struct usb_gadget *g, 3021 enum usb_device_speed speed) 3022 { 3023 struct dwc3 *dwc = gadget_to_dwc(g); 3024 unsigned long flags; 3025 3026 spin_lock_irqsave(&dwc->lock, flags); 3027 dwc->gadget_max_speed = speed; 3028 spin_unlock_irqrestore(&dwc->lock, flags); 3029 } 3030 3031 static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g, 3032 enum usb_ssp_rate rate) 3033 { 3034 struct dwc3 *dwc = gadget_to_dwc(g); 3035 unsigned long flags; 3036 3037 spin_lock_irqsave(&dwc->lock, flags); 3038 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS; 3039 dwc->gadget_ssp_rate = rate; 3040 spin_unlock_irqrestore(&dwc->lock, flags); 3041 } 3042 3043 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) 3044 { 3045 struct dwc3 *dwc = gadget_to_dwc(g); 3046 union power_supply_propval val = {0}; 3047 int ret; 3048 3049 if (dwc->usb2_phy) 3050 return usb_phy_set_power(dwc->usb2_phy, mA); 3051 3052 if (!dwc->usb_psy) 3053 return -EOPNOTSUPP; 3054 3055 val.intval = 1000 * mA; 3056 ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); 3057 3058 return ret; 3059 } 3060 3061 /** 3062 * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration 3063 * @g: pointer to the USB gadget 3064 * 3065 * Used to record the maximum number of endpoints being used in a USB composite 3066 * device. (across all configurations) This is to be used in the calculation 3067 * of the TXFIFO sizes when resizing internal memory for individual endpoints. 3068 * It will help ensured that the resizing logic reserves enough space for at 3069 * least one max packet. 3070 */ 3071 static int dwc3_gadget_check_config(struct usb_gadget *g) 3072 { 3073 struct dwc3 *dwc = gadget_to_dwc(g); 3074 struct usb_ep *ep; 3075 int fifo_size = 0; 3076 int ram1_depth; 3077 int ep_num = 0; 3078 3079 if (!dwc->do_fifo_resize) 3080 return 0; 3081 3082 list_for_each_entry(ep, &g->ep_list, ep_list) { 3083 /* Only interested in the IN endpoints */ 3084 if (ep->claimed && (ep->address & USB_DIR_IN)) 3085 ep_num++; 3086 } 3087 3088 if (ep_num <= dwc->max_cfg_eps) 3089 return 0; 3090 3091 /* Update the max number of eps in the composition */ 3092 dwc->max_cfg_eps = ep_num; 3093 3094 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps); 3095 /* Based on the equation, increment by one for every ep */ 3096 fifo_size += dwc->max_cfg_eps; 3097 3098 /* Check if we can fit a single fifo per endpoint */ 3099 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); 3100 if (fifo_size > ram1_depth) 3101 return -ENOMEM; 3102 3103 return 0; 3104 } 3105 3106 static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable) 3107 { 3108 struct dwc3 *dwc = gadget_to_dwc(g); 3109 unsigned long flags; 3110 3111 spin_lock_irqsave(&dwc->lock, flags); 3112 dwc->async_callbacks = enable; 3113 spin_unlock_irqrestore(&dwc->lock, flags); 3114 } 3115 3116 static const struct usb_gadget_ops dwc3_gadget_ops = { 3117 .get_frame = dwc3_gadget_get_frame, 3118 .wakeup = dwc3_gadget_wakeup, 3119 .func_wakeup = dwc3_gadget_func_wakeup, 3120 .set_remote_wakeup = dwc3_gadget_set_remote_wakeup, 3121 .set_selfpowered = dwc3_gadget_set_selfpowered, 3122 .pullup = dwc3_gadget_pullup, 3123 .udc_start = dwc3_gadget_start, 3124 .udc_stop = dwc3_gadget_stop, 3125 .udc_set_speed = dwc3_gadget_set_speed, 3126 .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate, 3127 .get_config_params = dwc3_gadget_config_params, 3128 .vbus_draw = dwc3_gadget_vbus_draw, 3129 .check_config = dwc3_gadget_check_config, 3130 .udc_async_callbacks = dwc3_gadget_async_callbacks, 3131 }; 3132 3133 /* -------------------------------------------------------------------------- */ 3134 3135 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep) 3136 { 3137 struct dwc3 *dwc = dep->dwc; 3138 3139 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); 3140 dep->endpoint.maxburst = 1; 3141 dep->endpoint.ops = &dwc3_gadget_ep0_ops; 3142 if (!dep->direction) 3143 dwc->gadget->ep0 = &dep->endpoint; 3144 3145 dep->endpoint.caps.type_control = true; 3146 3147 return 0; 3148 } 3149 3150 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep) 3151 { 3152 struct dwc3 *dwc = dep->dwc; 3153 u32 mdwidth; 3154 int size; 3155 int maxpacket; 3156 3157 mdwidth = dwc3_mdwidth(dwc); 3158 3159 /* MDWIDTH is represented in bits, we need it in bytes */ 3160 mdwidth /= 8; 3161 3162 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); 3163 if (DWC3_IP_IS(DWC3)) 3164 size = DWC3_GTXFIFOSIZ_TXFDEP(size); 3165 else 3166 size = DWC31_GTXFIFOSIZ_TXFDEP(size); 3167 3168 /* 3169 * maxpacket size is determined as part of the following, after assuming 3170 * a mult value of one maxpacket: 3171 * DWC3 revision 280A and prior: 3172 * fifo_size = mult * (max_packet / mdwidth) + 1; 3173 * maxpacket = mdwidth * (fifo_size - 1); 3174 * 3175 * DWC3 revision 290A and onwards: 3176 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 3177 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth; 3178 */ 3179 if (DWC3_VER_IS_PRIOR(DWC3, 290A)) 3180 maxpacket = mdwidth * (size - 1); 3181 else 3182 maxpacket = mdwidth * ((size - 1) - 1) - mdwidth; 3183 3184 /* Functionally, space for one max packet is sufficient */ 3185 size = min_t(int, maxpacket, 1024); 3186 usb_ep_set_maxpacket_limit(&dep->endpoint, size); 3187 3188 dep->endpoint.max_streams = 16; 3189 dep->endpoint.ops = &dwc3_gadget_ep_ops; 3190 list_add_tail(&dep->endpoint.ep_list, 3191 &dwc->gadget->ep_list); 3192 dep->endpoint.caps.type_iso = true; 3193 dep->endpoint.caps.type_bulk = true; 3194 dep->endpoint.caps.type_int = true; 3195 3196 return dwc3_alloc_trb_pool(dep); 3197 } 3198 3199 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep) 3200 { 3201 struct dwc3 *dwc = dep->dwc; 3202 u32 mdwidth; 3203 int size; 3204 3205 mdwidth = dwc3_mdwidth(dwc); 3206 3207 /* MDWIDTH is represented in bits, convert to bytes */ 3208 mdwidth /= 8; 3209 3210 /* All OUT endpoints share a single RxFIFO space */ 3211 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0)); 3212 if (DWC3_IP_IS(DWC3)) 3213 size = DWC3_GRXFIFOSIZ_RXFDEP(size); 3214 else 3215 size = DWC31_GRXFIFOSIZ_RXFDEP(size); 3216 3217 /* FIFO depth is in MDWDITH bytes */ 3218 size *= mdwidth; 3219 3220 /* 3221 * To meet performance requirement, a minimum recommended RxFIFO size 3222 * is defined as follow: 3223 * RxFIFO size >= (3 x MaxPacketSize) + 3224 * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin) 3225 * 3226 * Then calculate the max packet limit as below. 3227 */ 3228 size -= (3 * 8) + 16; 3229 if (size < 0) 3230 size = 0; 3231 else 3232 size /= 3; 3233 3234 usb_ep_set_maxpacket_limit(&dep->endpoint, size); 3235 dep->endpoint.max_streams = 16; 3236 dep->endpoint.ops = &dwc3_gadget_ep_ops; 3237 list_add_tail(&dep->endpoint.ep_list, 3238 &dwc->gadget->ep_list); 3239 dep->endpoint.caps.type_iso = true; 3240 dep->endpoint.caps.type_bulk = true; 3241 dep->endpoint.caps.type_int = true; 3242 3243 return dwc3_alloc_trb_pool(dep); 3244 } 3245 3246 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum) 3247 { 3248 struct dwc3_ep *dep; 3249 bool direction = epnum & 1; 3250 int ret; 3251 u8 num = epnum >> 1; 3252 3253 dep = kzalloc(sizeof(*dep), GFP_KERNEL); 3254 if (!dep) 3255 return -ENOMEM; 3256 3257 dep->dwc = dwc; 3258 dep->number = epnum; 3259 dep->direction = direction; 3260 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); 3261 dwc->eps[epnum] = dep; 3262 dep->combo_num = 0; 3263 dep->start_cmd_status = 0; 3264 3265 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, 3266 direction ? "in" : "out"); 3267 3268 dep->endpoint.name = dep->name; 3269 3270 if (!(dep->number > 1)) { 3271 dep->endpoint.desc = &dwc3_gadget_ep0_desc; 3272 dep->endpoint.comp_desc = NULL; 3273 } 3274 3275 if (num == 0) 3276 ret = dwc3_gadget_init_control_endpoint(dep); 3277 else if (direction) 3278 ret = dwc3_gadget_init_in_endpoint(dep); 3279 else 3280 ret = dwc3_gadget_init_out_endpoint(dep); 3281 3282 if (ret) 3283 return ret; 3284 3285 dep->endpoint.caps.dir_in = direction; 3286 dep->endpoint.caps.dir_out = !direction; 3287 3288 INIT_LIST_HEAD(&dep->pending_list); 3289 INIT_LIST_HEAD(&dep->started_list); 3290 INIT_LIST_HEAD(&dep->cancelled_list); 3291 3292 dwc3_debugfs_create_endpoint_dir(dep); 3293 3294 return 0; 3295 } 3296 3297 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total) 3298 { 3299 u8 epnum; 3300 3301 INIT_LIST_HEAD(&dwc->gadget->ep_list); 3302 3303 for (epnum = 0; epnum < total; epnum++) { 3304 int ret; 3305 3306 ret = dwc3_gadget_init_endpoint(dwc, epnum); 3307 if (ret) 3308 return ret; 3309 } 3310 3311 return 0; 3312 } 3313 3314 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc) 3315 { 3316 struct dwc3_ep *dep; 3317 u8 epnum; 3318 3319 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 3320 dep = dwc->eps[epnum]; 3321 if (!dep) 3322 continue; 3323 /* 3324 * Physical endpoints 0 and 1 are special; they form the 3325 * bi-directional USB endpoint 0. 3326 * 3327 * For those two physical endpoints, we don't allocate a TRB 3328 * pool nor do we add them the endpoints list. Due to that, we 3329 * shouldn't do these two operations otherwise we would end up 3330 * with all sorts of bugs when removing dwc3.ko. 3331 */ 3332 if (epnum != 0 && epnum != 1) { 3333 dwc3_free_trb_pool(dep); 3334 list_del(&dep->endpoint.ep_list); 3335 } 3336 3337 dwc3_debugfs_remove_endpoint_dir(dep); 3338 kfree(dep); 3339 } 3340 } 3341 3342 /* -------------------------------------------------------------------------- */ 3343 3344 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep, 3345 struct dwc3_request *req, struct dwc3_trb *trb, 3346 const struct dwc3_event_depevt *event, int status, int chain) 3347 { 3348 unsigned int count; 3349 3350 dwc3_ep_inc_deq(dep); 3351 3352 trace_dwc3_complete_trb(dep, trb); 3353 req->num_trbs--; 3354 3355 /* 3356 * If we're in the middle of series of chained TRBs and we 3357 * receive a short transfer along the way, DWC3 will skip 3358 * through all TRBs including the last TRB in the chain (the 3359 * where CHN bit is zero. DWC3 will also avoid clearing HWO 3360 * bit and SW has to do it manually. 3361 * 3362 * We're going to do that here to avoid problems of HW trying 3363 * to use bogus TRBs for transfers. 3364 */ 3365 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO)) 3366 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 3367 3368 /* 3369 * For isochronous transfers, the first TRB in a service interval must 3370 * have the Isoc-First type. Track and report its interval frame number. 3371 */ 3372 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && 3373 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) { 3374 unsigned int frame_number; 3375 3376 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl); 3377 frame_number &= ~(dep->interval - 1); 3378 req->request.frame_number = frame_number; 3379 } 3380 3381 /* 3382 * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If 3383 * this TRB points to the bounce buffer address, it's a MPS alignment 3384 * TRB. Don't add it to req->remaining calculation. 3385 */ 3386 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) && 3387 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) { 3388 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 3389 return 1; 3390 } 3391 3392 count = trb->size & DWC3_TRB_SIZE_MASK; 3393 req->remaining += count; 3394 3395 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) 3396 return 1; 3397 3398 if (event->status & DEPEVT_STATUS_SHORT && !chain) 3399 return 1; 3400 3401 if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) && 3402 DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC) 3403 return 1; 3404 3405 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) || 3406 (trb->ctrl & DWC3_TRB_CTRL_LST)) 3407 return 1; 3408 3409 return 0; 3410 } 3411 3412 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep, 3413 struct dwc3_request *req, const struct dwc3_event_depevt *event, 3414 int status) 3415 { 3416 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; 3417 struct scatterlist *sg = req->sg; 3418 struct scatterlist *s; 3419 unsigned int num_queued = req->num_queued_sgs; 3420 unsigned int i; 3421 int ret = 0; 3422 3423 for_each_sg(sg, s, num_queued, i) { 3424 trb = &dep->trb_pool[dep->trb_dequeue]; 3425 3426 req->sg = sg_next(s); 3427 req->num_queued_sgs--; 3428 3429 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req, 3430 trb, event, status, true); 3431 if (ret) 3432 break; 3433 } 3434 3435 return ret; 3436 } 3437 3438 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep, 3439 struct dwc3_request *req, const struct dwc3_event_depevt *event, 3440 int status) 3441 { 3442 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; 3443 3444 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb, 3445 event, status, false); 3446 } 3447 3448 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req) 3449 { 3450 return req->num_pending_sgs == 0 && req->num_queued_sgs == 0; 3451 } 3452 3453 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep, 3454 const struct dwc3_event_depevt *event, 3455 struct dwc3_request *req, int status) 3456 { 3457 int request_status; 3458 int ret; 3459 3460 if (req->request.num_mapped_sgs) 3461 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event, 3462 status); 3463 else 3464 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, 3465 status); 3466 3467 req->request.actual = req->request.length - req->remaining; 3468 3469 if (!dwc3_gadget_ep_request_completed(req)) 3470 goto out; 3471 3472 if (req->needs_extra_trb) { 3473 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, 3474 status); 3475 req->needs_extra_trb = false; 3476 } 3477 3478 /* 3479 * The event status only reflects the status of the TRB with IOC set. 3480 * For the requests that don't set interrupt on completion, the driver 3481 * needs to check and return the status of the completed TRBs associated 3482 * with the request. Use the status of the last TRB of the request. 3483 */ 3484 if (req->request.no_interrupt) { 3485 struct dwc3_trb *trb; 3486 3487 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue); 3488 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) { 3489 case DWC3_TRBSTS_MISSED_ISOC: 3490 /* Isoc endpoint only */ 3491 request_status = -EXDEV; 3492 break; 3493 case DWC3_TRB_STS_XFER_IN_PROG: 3494 /* Applicable when End Transfer with ForceRM=0 */ 3495 case DWC3_TRBSTS_SETUP_PENDING: 3496 /* Control endpoint only */ 3497 case DWC3_TRBSTS_OK: 3498 default: 3499 request_status = 0; 3500 break; 3501 } 3502 } else { 3503 request_status = status; 3504 } 3505 3506 dwc3_gadget_giveback(dep, req, request_status); 3507 3508 out: 3509 return ret; 3510 } 3511 3512 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep, 3513 const struct dwc3_event_depevt *event, int status) 3514 { 3515 struct dwc3_request *req; 3516 3517 while (!list_empty(&dep->started_list)) { 3518 int ret; 3519 3520 req = next_request(&dep->started_list); 3521 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event, 3522 req, status); 3523 if (ret) 3524 break; 3525 /* 3526 * The endpoint is disabled, let the dwc3_remove_requests() 3527 * handle the cleanup. 3528 */ 3529 if (!dep->endpoint.desc) 3530 break; 3531 } 3532 } 3533 3534 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep) 3535 { 3536 struct dwc3_request *req; 3537 struct dwc3 *dwc = dep->dwc; 3538 3539 if (!dep->endpoint.desc || !dwc->pullups_connected || 3540 !dwc->connected) 3541 return false; 3542 3543 if (!list_empty(&dep->pending_list)) 3544 return true; 3545 3546 /* 3547 * We only need to check the first entry of the started list. We can 3548 * assume the completed requests are removed from the started list. 3549 */ 3550 req = next_request(&dep->started_list); 3551 if (!req) 3552 return false; 3553 3554 return !dwc3_gadget_ep_request_completed(req); 3555 } 3556 3557 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep, 3558 const struct dwc3_event_depevt *event) 3559 { 3560 dep->frame_number = event->parameters; 3561 } 3562 3563 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep, 3564 const struct dwc3_event_depevt *event, int status) 3565 { 3566 struct dwc3 *dwc = dep->dwc; 3567 bool no_started_trb = true; 3568 3569 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status); 3570 3571 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) 3572 goto out; 3573 3574 if (!dep->endpoint.desc) 3575 return no_started_trb; 3576 3577 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && 3578 list_empty(&dep->started_list) && 3579 (list_empty(&dep->pending_list) || status == -EXDEV)) 3580 dwc3_stop_active_transfer(dep, true, true); 3581 else if (dwc3_gadget_ep_should_continue(dep)) 3582 if (__dwc3_gadget_kick_transfer(dep) == 0) 3583 no_started_trb = false; 3584 3585 out: 3586 /* 3587 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. 3588 * See dwc3_gadget_linksts_change_interrupt() for 1st half. 3589 */ 3590 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) { 3591 u32 reg; 3592 int i; 3593 3594 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) { 3595 dep = dwc->eps[i]; 3596 3597 if (!(dep->flags & DWC3_EP_ENABLED)) 3598 continue; 3599 3600 if (!list_empty(&dep->started_list)) 3601 return no_started_trb; 3602 } 3603 3604 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 3605 reg |= dwc->u1u2; 3606 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 3607 3608 dwc->u1u2 = 0; 3609 } 3610 3611 return no_started_trb; 3612 } 3613 3614 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep, 3615 const struct dwc3_event_depevt *event) 3616 { 3617 int status = 0; 3618 3619 if (!dep->endpoint.desc) 3620 return; 3621 3622 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) 3623 dwc3_gadget_endpoint_frame_from_event(dep, event); 3624 3625 if (event->status & DEPEVT_STATUS_BUSERR) 3626 status = -ECONNRESET; 3627 3628 if (event->status & DEPEVT_STATUS_MISSED_ISOC) 3629 status = -EXDEV; 3630 3631 dwc3_gadget_endpoint_trbs_complete(dep, event, status); 3632 } 3633 3634 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep, 3635 const struct dwc3_event_depevt *event) 3636 { 3637 int status = 0; 3638 3639 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 3640 3641 if (event->status & DEPEVT_STATUS_BUSERR) 3642 status = -ECONNRESET; 3643 3644 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status)) 3645 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; 3646 } 3647 3648 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep, 3649 const struct dwc3_event_depevt *event) 3650 { 3651 dwc3_gadget_endpoint_frame_from_event(dep, event); 3652 3653 /* 3654 * The XferNotReady event is generated only once before the endpoint 3655 * starts. It will be generated again when END_TRANSFER command is 3656 * issued. For some controller versions, the XferNotReady event may be 3657 * generated while the END_TRANSFER command is still in process. Ignore 3658 * it and wait for the next XferNotReady event after the command is 3659 * completed. 3660 */ 3661 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) 3662 return; 3663 3664 (void) __dwc3_gadget_start_isoc(dep); 3665 } 3666 3667 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep, 3668 const struct dwc3_event_depevt *event) 3669 { 3670 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters); 3671 3672 if (cmd != DWC3_DEPCMD_ENDTRANSFER) 3673 return; 3674 3675 /* 3676 * The END_TRANSFER command will cause the controller to generate a 3677 * NoStream Event, and it's not due to the host DP NoStream rejection. 3678 * Ignore the next NoStream event. 3679 */ 3680 if (dep->stream_capable) 3681 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM; 3682 3683 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; 3684 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 3685 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 3686 3687 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) { 3688 struct dwc3 *dwc = dep->dwc; 3689 3690 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL; 3691 if (dwc3_send_clear_stall_ep_cmd(dep)) { 3692 struct usb_ep *ep0 = &dwc->eps[0]->endpoint; 3693 3694 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name); 3695 if (dwc->delayed_status) 3696 __dwc3_gadget_ep0_set_halt(ep0, 1); 3697 return; 3698 } 3699 3700 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 3701 if (dwc->clear_stall_protocol == dep->number) 3702 dwc3_ep0_send_delayed_status(dwc); 3703 } 3704 3705 if ((dep->flags & DWC3_EP_DELAY_START) && 3706 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) 3707 __dwc3_gadget_kick_transfer(dep); 3708 3709 dep->flags &= ~DWC3_EP_DELAY_START; 3710 } 3711 3712 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep, 3713 const struct dwc3_event_depevt *event) 3714 { 3715 struct dwc3 *dwc = dep->dwc; 3716 3717 if (event->status == DEPEVT_STREAMEVT_FOUND) { 3718 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; 3719 goto out; 3720 } 3721 3722 /* Note: NoStream rejection event param value is 0 and not 0xFFFF */ 3723 switch (event->parameters) { 3724 case DEPEVT_STREAM_PRIME: 3725 /* 3726 * If the host can properly transition the endpoint state from 3727 * idle to prime after a NoStream rejection, there's no need to 3728 * force restarting the endpoint to reinitiate the stream. To 3729 * simplify the check, assume the host follows the USB spec if 3730 * it primed the endpoint more than once. 3731 */ 3732 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) { 3733 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED) 3734 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM; 3735 else 3736 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; 3737 } 3738 3739 break; 3740 case DEPEVT_STREAM_NOSTREAM: 3741 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) || 3742 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) || 3743 (!DWC3_MST_CAPABLE(&dwc->hwparams) && 3744 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))) 3745 break; 3746 3747 /* 3748 * If the host rejects a stream due to no active stream, by the 3749 * USB and xHCI spec, the endpoint will be put back to idle 3750 * state. When the host is ready (buffer added/updated), it will 3751 * prime the endpoint to inform the usb device controller. This 3752 * triggers the device controller to issue ERDY to restart the 3753 * stream. However, some hosts don't follow this and keep the 3754 * endpoint in the idle state. No prime will come despite host 3755 * streams are updated, and the device controller will not be 3756 * triggered to generate ERDY to move the next stream data. To 3757 * workaround this and maintain compatibility with various 3758 * hosts, force to reinitiate the stream until the host is ready 3759 * instead of waiting for the host to prime the endpoint. 3760 */ 3761 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) { 3762 unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME; 3763 3764 dwc3_send_gadget_generic_command(dwc, cmd, dep->number); 3765 } else { 3766 dep->flags |= DWC3_EP_DELAY_START; 3767 dwc3_stop_active_transfer(dep, true, true); 3768 return; 3769 } 3770 break; 3771 } 3772 3773 out: 3774 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; 3775 } 3776 3777 static void dwc3_endpoint_interrupt(struct dwc3 *dwc, 3778 const struct dwc3_event_depevt *event) 3779 { 3780 struct dwc3_ep *dep; 3781 u8 epnum = event->endpoint_number; 3782 3783 dep = dwc->eps[epnum]; 3784 3785 if (!(dep->flags & DWC3_EP_ENABLED)) { 3786 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED)) 3787 return; 3788 3789 /* Handle only EPCMDCMPLT when EP disabled */ 3790 if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) && 3791 !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE)) 3792 return; 3793 } 3794 3795 if (epnum == 0 || epnum == 1) { 3796 dwc3_ep0_interrupt(dwc, event); 3797 return; 3798 } 3799 3800 switch (event->endpoint_event) { 3801 case DWC3_DEPEVT_XFERINPROGRESS: 3802 dwc3_gadget_endpoint_transfer_in_progress(dep, event); 3803 break; 3804 case DWC3_DEPEVT_XFERNOTREADY: 3805 dwc3_gadget_endpoint_transfer_not_ready(dep, event); 3806 break; 3807 case DWC3_DEPEVT_EPCMDCMPLT: 3808 dwc3_gadget_endpoint_command_complete(dep, event); 3809 break; 3810 case DWC3_DEPEVT_XFERCOMPLETE: 3811 dwc3_gadget_endpoint_transfer_complete(dep, event); 3812 break; 3813 case DWC3_DEPEVT_STREAMEVT: 3814 dwc3_gadget_endpoint_stream_event(dep, event); 3815 break; 3816 case DWC3_DEPEVT_RXTXFIFOEVT: 3817 break; 3818 default: 3819 dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event); 3820 break; 3821 } 3822 } 3823 3824 static void dwc3_disconnect_gadget(struct dwc3 *dwc) 3825 { 3826 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) { 3827 spin_unlock(&dwc->lock); 3828 dwc->gadget_driver->disconnect(dwc->gadget); 3829 spin_lock(&dwc->lock); 3830 } 3831 } 3832 3833 static void dwc3_suspend_gadget(struct dwc3 *dwc) 3834 { 3835 if (dwc->async_callbacks && dwc->gadget_driver->suspend) { 3836 spin_unlock(&dwc->lock); 3837 dwc->gadget_driver->suspend(dwc->gadget); 3838 spin_lock(&dwc->lock); 3839 } 3840 } 3841 3842 static void dwc3_resume_gadget(struct dwc3 *dwc) 3843 { 3844 if (dwc->async_callbacks && dwc->gadget_driver->resume) { 3845 spin_unlock(&dwc->lock); 3846 dwc->gadget_driver->resume(dwc->gadget); 3847 spin_lock(&dwc->lock); 3848 } 3849 } 3850 3851 static void dwc3_reset_gadget(struct dwc3 *dwc) 3852 { 3853 if (!dwc->gadget_driver) 3854 return; 3855 3856 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) { 3857 spin_unlock(&dwc->lock); 3858 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver); 3859 spin_lock(&dwc->lock); 3860 } 3861 } 3862 3863 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, 3864 bool interrupt) 3865 { 3866 struct dwc3 *dwc = dep->dwc; 3867 3868 /* 3869 * Only issue End Transfer command to the control endpoint of a started 3870 * Data Phase. Typically we should only do so in error cases such as 3871 * invalid/unexpected direction as described in the control transfer 3872 * flow of the programming guide. 3873 */ 3874 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE) 3875 return; 3876 3877 if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP)) 3878 return; 3879 3880 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || 3881 (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) 3882 return; 3883 3884 /* 3885 * If a Setup packet is received but yet to DMA out, the controller will 3886 * not process the End Transfer command of any endpoint. Polling of its 3887 * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a 3888 * timeout. Delay issuing the End Transfer command until the Setup TRB is 3889 * prepared. 3890 */ 3891 if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) { 3892 dep->flags |= DWC3_EP_DELAY_STOP; 3893 return; 3894 } 3895 3896 /* 3897 * NOTICE: We are violating what the Databook says about the 3898 * EndTransfer command. Ideally we would _always_ wait for the 3899 * EndTransfer Command Completion IRQ, but that's causing too 3900 * much trouble synchronizing between us and gadget driver. 3901 * 3902 * We have discussed this with the IP Provider and it was 3903 * suggested to giveback all requests here. 3904 * 3905 * Note also that a similar handling was tested by Synopsys 3906 * (thanks a lot Paul) and nothing bad has come out of it. 3907 * In short, what we're doing is issuing EndTransfer with 3908 * CMDIOC bit set and delay kicking transfer until the 3909 * EndTransfer command had completed. 3910 * 3911 * As of IP version 3.10a of the DWC_usb3 IP, the controller 3912 * supports a mode to work around the above limitation. The 3913 * software can poll the CMDACT bit in the DEPCMD register 3914 * after issuing a EndTransfer command. This mode is enabled 3915 * by writing GUCTL2[14]. This polling is already done in the 3916 * dwc3_send_gadget_ep_cmd() function so if the mode is 3917 * enabled, the EndTransfer command will have completed upon 3918 * returning from this function. 3919 * 3920 * This mode is NOT available on the DWC_usb31 IP. In this 3921 * case, if the IOC bit is not set, then delay by 1ms 3922 * after issuing the EndTransfer command. This allows for the 3923 * controller to handle the command completely before DWC3 3924 * remove requests attempts to unmap USB request buffers. 3925 */ 3926 3927 __dwc3_stop_active_transfer(dep, force, interrupt); 3928 } 3929 3930 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc) 3931 { 3932 u32 epnum; 3933 3934 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 3935 struct dwc3_ep *dep; 3936 int ret; 3937 3938 dep = dwc->eps[epnum]; 3939 if (!dep) 3940 continue; 3941 3942 if (!(dep->flags & DWC3_EP_STALL)) 3943 continue; 3944 3945 dep->flags &= ~DWC3_EP_STALL; 3946 3947 ret = dwc3_send_clear_stall_ep_cmd(dep); 3948 WARN_ON_ONCE(ret); 3949 } 3950 } 3951 3952 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc) 3953 { 3954 int reg; 3955 3956 dwc->suspended = false; 3957 3958 dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET); 3959 3960 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 3961 reg &= ~DWC3_DCTL_INITU1ENA; 3962 reg &= ~DWC3_DCTL_INITU2ENA; 3963 dwc3_gadget_dctl_write_safe(dwc, reg); 3964 3965 dwc->connected = false; 3966 3967 dwc3_disconnect_gadget(dwc); 3968 3969 dwc->gadget->speed = USB_SPEED_UNKNOWN; 3970 dwc->setup_packet_pending = false; 3971 dwc->gadget->wakeup_armed = false; 3972 dwc3_gadget_enable_linksts_evts(dwc, false); 3973 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); 3974 3975 dwc3_ep0_reset_state(dwc); 3976 } 3977 3978 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) 3979 { 3980 u32 reg; 3981 3982 dwc->suspended = false; 3983 3984 /* 3985 * Ideally, dwc3_reset_gadget() would trigger the function 3986 * drivers to stop any active transfers through ep disable. 3987 * However, for functions which defer ep disable, such as mass 3988 * storage, we will need to rely on the call to stop active 3989 * transfers here, and avoid allowing of request queuing. 3990 */ 3991 dwc->connected = false; 3992 3993 /* 3994 * WORKAROUND: DWC3 revisions <1.88a have an issue which 3995 * would cause a missing Disconnect Event if there's a 3996 * pending Setup Packet in the FIFO. 3997 * 3998 * There's no suggested workaround on the official Bug 3999 * report, which states that "unless the driver/application 4000 * is doing any special handling of a disconnect event, 4001 * there is no functional issue". 4002 * 4003 * Unfortunately, it turns out that we _do_ some special 4004 * handling of a disconnect event, namely complete all 4005 * pending transfers, notify gadget driver of the 4006 * disconnection, and so on. 4007 * 4008 * Our suggested workaround is to follow the Disconnect 4009 * Event steps here, instead, based on a setup_packet_pending 4010 * flag. Such flag gets set whenever we have a SETUP_PENDING 4011 * status for EP0 TRBs and gets cleared on XferComplete for the 4012 * same endpoint. 4013 * 4014 * Refers to: 4015 * 4016 * STAR#9000466709: RTL: Device : Disconnect event not 4017 * generated if setup packet pending in FIFO 4018 */ 4019 if (DWC3_VER_IS_PRIOR(DWC3, 188A)) { 4020 if (dwc->setup_packet_pending) 4021 dwc3_gadget_disconnect_interrupt(dwc); 4022 } 4023 4024 dwc3_reset_gadget(dwc); 4025 4026 /* 4027 * From SNPS databook section 8.1.2, the EP0 should be in setup 4028 * phase. So ensure that EP0 is in setup phase by issuing a stall 4029 * and restart if EP0 is not in setup phase. 4030 */ 4031 dwc3_ep0_reset_state(dwc); 4032 4033 /* 4034 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a 4035 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW 4036 * needs to ensure that it sends "a DEPENDXFER command for any active 4037 * transfers." 4038 */ 4039 dwc3_stop_active_transfers(dwc); 4040 dwc->connected = true; 4041 4042 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4043 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 4044 dwc3_gadget_dctl_write_safe(dwc, reg); 4045 dwc->test_mode = false; 4046 dwc->gadget->wakeup_armed = false; 4047 dwc3_gadget_enable_linksts_evts(dwc, false); 4048 dwc3_clear_stall_all_ep(dwc); 4049 4050 /* Reset device address to zero */ 4051 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 4052 reg &= ~(DWC3_DCFG_DEVADDR_MASK); 4053 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 4054 } 4055 4056 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) 4057 { 4058 struct dwc3_ep *dep; 4059 int ret; 4060 u32 reg; 4061 u8 lanes = 1; 4062 u8 speed; 4063 4064 if (!dwc->softconnect) 4065 return; 4066 4067 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 4068 speed = reg & DWC3_DSTS_CONNECTSPD; 4069 dwc->speed = speed; 4070 4071 if (DWC3_IP_IS(DWC32)) 4072 lanes = DWC3_DSTS_CONNLANES(reg) + 1; 4073 4074 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; 4075 4076 /* 4077 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed 4078 * each time on Connect Done. 4079 * 4080 * Currently we always use the reset value. If any platform 4081 * wants to set this to a different value, we need to add a 4082 * setting and update GCTL.RAMCLKSEL here. 4083 */ 4084 4085 switch (speed) { 4086 case DWC3_DSTS_SUPERSPEED_PLUS: 4087 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 4088 dwc->gadget->ep0->maxpacket = 512; 4089 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; 4090 4091 if (lanes > 1) 4092 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2; 4093 else 4094 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1; 4095 break; 4096 case DWC3_DSTS_SUPERSPEED: 4097 /* 4098 * WORKAROUND: DWC3 revisions <1.90a have an issue which 4099 * would cause a missing USB3 Reset event. 4100 * 4101 * In such situations, we should force a USB3 Reset 4102 * event by calling our dwc3_gadget_reset_interrupt() 4103 * routine. 4104 * 4105 * Refers to: 4106 * 4107 * STAR#9000483510: RTL: SS : USB3 reset event may 4108 * not be generated always when the link enters poll 4109 */ 4110 if (DWC3_VER_IS_PRIOR(DWC3, 190A)) 4111 dwc3_gadget_reset_interrupt(dwc); 4112 4113 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 4114 dwc->gadget->ep0->maxpacket = 512; 4115 dwc->gadget->speed = USB_SPEED_SUPER; 4116 4117 if (lanes > 1) { 4118 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; 4119 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2; 4120 } 4121 break; 4122 case DWC3_DSTS_HIGHSPEED: 4123 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 4124 dwc->gadget->ep0->maxpacket = 64; 4125 dwc->gadget->speed = USB_SPEED_HIGH; 4126 break; 4127 case DWC3_DSTS_FULLSPEED: 4128 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 4129 dwc->gadget->ep0->maxpacket = 64; 4130 dwc->gadget->speed = USB_SPEED_FULL; 4131 break; 4132 } 4133 4134 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket; 4135 4136 /* Enable USB2 LPM Capability */ 4137 4138 if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) && 4139 !dwc->usb2_gadget_lpm_disable && 4140 (speed != DWC3_DSTS_SUPERSPEED) && 4141 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) { 4142 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 4143 reg |= DWC3_DCFG_LPM_CAP; 4144 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 4145 4146 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4147 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN); 4148 4149 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold | 4150 (dwc->is_utmi_l1_suspend << 4)); 4151 4152 /* 4153 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and 4154 * DCFG.LPMCap is set, core responses with an ACK and the 4155 * BESL value in the LPM token is less than or equal to LPM 4156 * NYET threshold. 4157 */ 4158 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum, 4159 "LPM Erratum not available on dwc3 revisions < 2.40a\n"); 4160 4161 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A)) 4162 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold); 4163 4164 dwc3_gadget_dctl_write_safe(dwc, reg); 4165 } else { 4166 if (dwc->usb2_gadget_lpm_disable) { 4167 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 4168 reg &= ~DWC3_DCFG_LPM_CAP; 4169 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 4170 } 4171 4172 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4173 reg &= ~DWC3_DCTL_HIRD_THRES_MASK; 4174 dwc3_gadget_dctl_write_safe(dwc, reg); 4175 } 4176 4177 dep = dwc->eps[0]; 4178 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); 4179 if (ret) { 4180 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 4181 return; 4182 } 4183 4184 dep = dwc->eps[1]; 4185 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); 4186 if (ret) { 4187 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 4188 return; 4189 } 4190 4191 /* 4192 * Configure PHY via GUSB3PIPECTLn if required. 4193 * 4194 * Update GTXFIFOSIZn 4195 * 4196 * In both cases reset values should be sufficient. 4197 */ 4198 } 4199 4200 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo) 4201 { 4202 dwc->suspended = false; 4203 4204 /* 4205 * TODO take core out of low power mode when that's 4206 * implemented. 4207 */ 4208 4209 if (dwc->async_callbacks && dwc->gadget_driver->resume) { 4210 spin_unlock(&dwc->lock); 4211 dwc->gadget_driver->resume(dwc->gadget); 4212 spin_lock(&dwc->lock); 4213 } 4214 4215 dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK; 4216 } 4217 4218 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc, 4219 unsigned int evtinfo) 4220 { 4221 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 4222 unsigned int pwropt; 4223 4224 /* 4225 * WORKAROUND: DWC3 < 2.50a have an issue when configured without 4226 * Hibernation mode enabled which would show up when device detects 4227 * host-initiated U3 exit. 4228 * 4229 * In that case, device will generate a Link State Change Interrupt 4230 * from U3 to RESUME which is only necessary if Hibernation is 4231 * configured in. 4232 * 4233 * There are no functional changes due to such spurious event and we 4234 * just need to ignore it. 4235 * 4236 * Refers to: 4237 * 4238 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation 4239 * operational mode 4240 */ 4241 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1); 4242 if (DWC3_VER_IS_PRIOR(DWC3, 250A) && 4243 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) { 4244 if ((dwc->link_state == DWC3_LINK_STATE_U3) && 4245 (next == DWC3_LINK_STATE_RESUME)) { 4246 return; 4247 } 4248 } 4249 4250 /* 4251 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending 4252 * on the link partner, the USB session might do multiple entry/exit 4253 * of low power states before a transfer takes place. 4254 * 4255 * Due to this problem, we might experience lower throughput. The 4256 * suggested workaround is to disable DCTL[12:9] bits if we're 4257 * transitioning from U1/U2 to U0 and enable those bits again 4258 * after a transfer completes and there are no pending transfers 4259 * on any of the enabled endpoints. 4260 * 4261 * This is the first half of that workaround. 4262 * 4263 * Refers to: 4264 * 4265 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us 4266 * core send LGO_Ux entering U0 4267 */ 4268 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) { 4269 if (next == DWC3_LINK_STATE_U0) { 4270 u32 u1u2; 4271 u32 reg; 4272 4273 switch (dwc->link_state) { 4274 case DWC3_LINK_STATE_U1: 4275 case DWC3_LINK_STATE_U2: 4276 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4277 u1u2 = reg & (DWC3_DCTL_INITU2ENA 4278 | DWC3_DCTL_ACCEPTU2ENA 4279 | DWC3_DCTL_INITU1ENA 4280 | DWC3_DCTL_ACCEPTU1ENA); 4281 4282 if (!dwc->u1u2) 4283 dwc->u1u2 = reg & u1u2; 4284 4285 reg &= ~u1u2; 4286 4287 dwc3_gadget_dctl_write_safe(dwc, reg); 4288 break; 4289 default: 4290 /* do nothing */ 4291 break; 4292 } 4293 } 4294 } 4295 4296 switch (next) { 4297 case DWC3_LINK_STATE_U0: 4298 if (dwc->gadget->wakeup_armed) { 4299 dwc3_gadget_enable_linksts_evts(dwc, false); 4300 dwc3_resume_gadget(dwc); 4301 dwc->suspended = false; 4302 } 4303 break; 4304 case DWC3_LINK_STATE_U1: 4305 if (dwc->speed == USB_SPEED_SUPER) 4306 dwc3_suspend_gadget(dwc); 4307 break; 4308 case DWC3_LINK_STATE_U2: 4309 case DWC3_LINK_STATE_U3: 4310 dwc3_suspend_gadget(dwc); 4311 break; 4312 case DWC3_LINK_STATE_RESUME: 4313 dwc3_resume_gadget(dwc); 4314 break; 4315 default: 4316 /* do nothing */ 4317 break; 4318 } 4319 4320 dwc->link_state = next; 4321 } 4322 4323 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc, 4324 unsigned int evtinfo) 4325 { 4326 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 4327 4328 if (!dwc->suspended && next == DWC3_LINK_STATE_U3) { 4329 dwc->suspended = true; 4330 dwc3_suspend_gadget(dwc); 4331 } 4332 4333 dwc->link_state = next; 4334 } 4335 4336 static void dwc3_gadget_interrupt(struct dwc3 *dwc, 4337 const struct dwc3_event_devt *event) 4338 { 4339 switch (event->type) { 4340 case DWC3_DEVICE_EVENT_DISCONNECT: 4341 dwc3_gadget_disconnect_interrupt(dwc); 4342 break; 4343 case DWC3_DEVICE_EVENT_RESET: 4344 dwc3_gadget_reset_interrupt(dwc); 4345 break; 4346 case DWC3_DEVICE_EVENT_CONNECT_DONE: 4347 dwc3_gadget_conndone_interrupt(dwc); 4348 break; 4349 case DWC3_DEVICE_EVENT_WAKEUP: 4350 dwc3_gadget_wakeup_interrupt(dwc, event->event_info); 4351 break; 4352 case DWC3_DEVICE_EVENT_HIBER_REQ: 4353 dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n"); 4354 break; 4355 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE: 4356 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info); 4357 break; 4358 case DWC3_DEVICE_EVENT_SUSPEND: 4359 /* It changed to be suspend event for version 2.30a and above */ 4360 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) 4361 dwc3_gadget_suspend_interrupt(dwc, event->event_info); 4362 break; 4363 case DWC3_DEVICE_EVENT_SOF: 4364 case DWC3_DEVICE_EVENT_ERRATIC_ERROR: 4365 case DWC3_DEVICE_EVENT_CMD_CMPL: 4366 case DWC3_DEVICE_EVENT_OVERFLOW: 4367 break; 4368 default: 4369 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type); 4370 } 4371 } 4372 4373 static void dwc3_process_event_entry(struct dwc3 *dwc, 4374 const union dwc3_event *event) 4375 { 4376 trace_dwc3_event(event->raw, dwc); 4377 4378 if (!event->type.is_devspec) 4379 dwc3_endpoint_interrupt(dwc, &event->depevt); 4380 else if (event->type.type == DWC3_EVENT_TYPE_DEV) 4381 dwc3_gadget_interrupt(dwc, &event->devt); 4382 else 4383 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw); 4384 } 4385 4386 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt) 4387 { 4388 struct dwc3 *dwc = evt->dwc; 4389 irqreturn_t ret = IRQ_NONE; 4390 int left; 4391 4392 left = evt->count; 4393 4394 if (!(evt->flags & DWC3_EVENT_PENDING)) 4395 return IRQ_NONE; 4396 4397 while (left > 0) { 4398 union dwc3_event event; 4399 4400 event.raw = *(u32 *) (evt->cache + evt->lpos); 4401 4402 dwc3_process_event_entry(dwc, &event); 4403 4404 /* 4405 * FIXME we wrap around correctly to the next entry as 4406 * almost all entries are 4 bytes in size. There is one 4407 * entry which has 12 bytes which is a regular entry 4408 * followed by 8 bytes data. ATM I don't know how 4409 * things are organized if we get next to the a 4410 * boundary so I worry about that once we try to handle 4411 * that. 4412 */ 4413 evt->lpos = (evt->lpos + 4) % evt->length; 4414 left -= 4; 4415 } 4416 4417 evt->count = 0; 4418 ret = IRQ_HANDLED; 4419 4420 /* Unmask interrupt */ 4421 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), 4422 DWC3_GEVNTSIZ_SIZE(evt->length)); 4423 4424 if (dwc->imod_interval) { 4425 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); 4426 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); 4427 } 4428 4429 /* Keep the clearing of DWC3_EVENT_PENDING at the end */ 4430 evt->flags &= ~DWC3_EVENT_PENDING; 4431 4432 return ret; 4433 } 4434 4435 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt) 4436 { 4437 struct dwc3_event_buffer *evt = _evt; 4438 struct dwc3 *dwc = evt->dwc; 4439 unsigned long flags; 4440 irqreturn_t ret = IRQ_NONE; 4441 4442 local_bh_disable(); 4443 spin_lock_irqsave(&dwc->lock, flags); 4444 ret = dwc3_process_event_buf(evt); 4445 spin_unlock_irqrestore(&dwc->lock, flags); 4446 local_bh_enable(); 4447 4448 return ret; 4449 } 4450 4451 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt) 4452 { 4453 struct dwc3 *dwc = evt->dwc; 4454 u32 amount; 4455 u32 count; 4456 4457 if (pm_runtime_suspended(dwc->dev)) { 4458 dwc->pending_events = true; 4459 /* 4460 * Trigger runtime resume. The get() function will be balanced 4461 * after processing the pending events in dwc3_process_pending 4462 * events(). 4463 */ 4464 pm_runtime_get(dwc->dev); 4465 disable_irq_nosync(dwc->irq_gadget); 4466 return IRQ_HANDLED; 4467 } 4468 4469 /* 4470 * With PCIe legacy interrupt, test shows that top-half irq handler can 4471 * be called again after HW interrupt deassertion. Check if bottom-half 4472 * irq event handler completes before caching new event to prevent 4473 * losing events. 4474 */ 4475 if (evt->flags & DWC3_EVENT_PENDING) 4476 return IRQ_HANDLED; 4477 4478 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); 4479 count &= DWC3_GEVNTCOUNT_MASK; 4480 if (!count) 4481 return IRQ_NONE; 4482 4483 evt->count = count; 4484 evt->flags |= DWC3_EVENT_PENDING; 4485 4486 /* Mask interrupt */ 4487 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), 4488 DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length)); 4489 4490 amount = min(count, evt->length - evt->lpos); 4491 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount); 4492 4493 if (amount < count) 4494 memcpy(evt->cache, evt->buf, count - amount); 4495 4496 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count); 4497 4498 return IRQ_WAKE_THREAD; 4499 } 4500 4501 static irqreturn_t dwc3_interrupt(int irq, void *_evt) 4502 { 4503 struct dwc3_event_buffer *evt = _evt; 4504 4505 return dwc3_check_event_buf(evt); 4506 } 4507 4508 static int dwc3_gadget_get_irq(struct dwc3 *dwc) 4509 { 4510 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev); 4511 int irq; 4512 4513 irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral"); 4514 if (irq > 0) 4515 goto out; 4516 4517 if (irq == -EPROBE_DEFER) 4518 goto out; 4519 4520 irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3"); 4521 if (irq > 0) 4522 goto out; 4523 4524 if (irq == -EPROBE_DEFER) 4525 goto out; 4526 4527 irq = platform_get_irq(dwc3_pdev, 0); 4528 4529 out: 4530 return irq; 4531 } 4532 4533 static void dwc_gadget_release(struct device *dev) 4534 { 4535 struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev); 4536 4537 kfree(gadget); 4538 } 4539 4540 /** 4541 * dwc3_gadget_init - initializes gadget related registers 4542 * @dwc: pointer to our controller context structure 4543 * 4544 * Returns 0 on success otherwise negative errno. 4545 */ 4546 int dwc3_gadget_init(struct dwc3 *dwc) 4547 { 4548 int ret; 4549 int irq; 4550 struct device *dev; 4551 4552 irq = dwc3_gadget_get_irq(dwc); 4553 if (irq < 0) { 4554 ret = irq; 4555 goto err0; 4556 } 4557 4558 dwc->irq_gadget = irq; 4559 4560 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev, 4561 sizeof(*dwc->ep0_trb) * 2, 4562 &dwc->ep0_trb_addr, GFP_KERNEL); 4563 if (!dwc->ep0_trb) { 4564 dev_err(dwc->dev, "failed to allocate ep0 trb\n"); 4565 ret = -ENOMEM; 4566 goto err0; 4567 } 4568 4569 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL); 4570 if (!dwc->setup_buf) { 4571 ret = -ENOMEM; 4572 goto err1; 4573 } 4574 4575 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, 4576 &dwc->bounce_addr, GFP_KERNEL); 4577 if (!dwc->bounce) { 4578 ret = -ENOMEM; 4579 goto err2; 4580 } 4581 4582 init_completion(&dwc->ep0_in_setup); 4583 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL); 4584 if (!dwc->gadget) { 4585 ret = -ENOMEM; 4586 goto err3; 4587 } 4588 4589 4590 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release); 4591 dev = &dwc->gadget->dev; 4592 dev->platform_data = dwc; 4593 dwc->gadget->ops = &dwc3_gadget_ops; 4594 dwc->gadget->speed = USB_SPEED_UNKNOWN; 4595 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; 4596 dwc->gadget->sg_supported = true; 4597 dwc->gadget->name = "dwc3-gadget"; 4598 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable; 4599 dwc->gadget->wakeup_capable = true; 4600 4601 /* 4602 * FIXME We might be setting max_speed to <SUPER, however versions 4603 * <2.20a of dwc3 have an issue with metastability (documented 4604 * elsewhere in this driver) which tells us we can't set max speed to 4605 * anything lower than SUPER. 4606 * 4607 * Because gadget.max_speed is only used by composite.c and function 4608 * drivers (i.e. it won't go into dwc3's registers) we are allowing this 4609 * to happen so we avoid sending SuperSpeed Capability descriptor 4610 * together with our BOS descriptor as that could confuse host into 4611 * thinking we can handle super speed. 4612 * 4613 * Note that, in fact, we won't even support GetBOS requests when speed 4614 * is less than super speed because we don't have means, yet, to tell 4615 * composite.c that we are USB 2.0 + LPM ECN. 4616 */ 4617 if (DWC3_VER_IS_PRIOR(DWC3, 220A) && 4618 !dwc->dis_metastability_quirk) 4619 dev_info(dwc->dev, "changing max_speed on rev %08x\n", 4620 dwc->revision); 4621 4622 dwc->gadget->max_speed = dwc->maximum_speed; 4623 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate; 4624 4625 /* 4626 * REVISIT: Here we should clear all pending IRQs to be 4627 * sure we're starting from a well known location. 4628 */ 4629 4630 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps); 4631 if (ret) 4632 goto err4; 4633 4634 ret = usb_add_gadget(dwc->gadget); 4635 if (ret) { 4636 dev_err(dwc->dev, "failed to add gadget\n"); 4637 goto err5; 4638 } 4639 4640 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS) 4641 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate); 4642 else 4643 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed); 4644 4645 return 0; 4646 4647 err5: 4648 dwc3_gadget_free_endpoints(dwc); 4649 err4: 4650 usb_put_gadget(dwc->gadget); 4651 dwc->gadget = NULL; 4652 err3: 4653 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, 4654 dwc->bounce_addr); 4655 4656 err2: 4657 kfree(dwc->setup_buf); 4658 4659 err1: 4660 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, 4661 dwc->ep0_trb, dwc->ep0_trb_addr); 4662 4663 err0: 4664 return ret; 4665 } 4666 4667 /* -------------------------------------------------------------------------- */ 4668 4669 void dwc3_gadget_exit(struct dwc3 *dwc) 4670 { 4671 if (!dwc->gadget) 4672 return; 4673 4674 usb_del_gadget(dwc->gadget); 4675 dwc3_gadget_free_endpoints(dwc); 4676 usb_put_gadget(dwc->gadget); 4677 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, 4678 dwc->bounce_addr); 4679 kfree(dwc->setup_buf); 4680 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, 4681 dwc->ep0_trb, dwc->ep0_trb_addr); 4682 } 4683 4684 int dwc3_gadget_suspend(struct dwc3 *dwc) 4685 { 4686 unsigned long flags; 4687 int ret; 4688 4689 if (!dwc->gadget_driver) 4690 return 0; 4691 4692 ret = dwc3_gadget_soft_disconnect(dwc); 4693 if (ret) 4694 goto err; 4695 4696 spin_lock_irqsave(&dwc->lock, flags); 4697 dwc3_disconnect_gadget(dwc); 4698 spin_unlock_irqrestore(&dwc->lock, flags); 4699 4700 return 0; 4701 4702 err: 4703 /* 4704 * Attempt to reset the controller's state. Likely no 4705 * communication can be established until the host 4706 * performs a port reset. 4707 */ 4708 if (dwc->softconnect) 4709 dwc3_gadget_soft_connect(dwc); 4710 4711 return ret; 4712 } 4713 4714 int dwc3_gadget_resume(struct dwc3 *dwc) 4715 { 4716 if (!dwc->gadget_driver || !dwc->softconnect) 4717 return 0; 4718 4719 return dwc3_gadget_soft_connect(dwc); 4720 } 4721 4722 void dwc3_gadget_process_pending_events(struct dwc3 *dwc) 4723 { 4724 if (dwc->pending_events) { 4725 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf); 4726 dwc3_thread_interrupt(dwc->irq_gadget, dwc->ev_buf); 4727 pm_runtime_put(dwc->dev); 4728 dwc->pending_events = false; 4729 enable_irq(dwc->irq_gadget); 4730 } 4731 } 4732