1 /** 2 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link 3 * 4 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com 5 * 6 * Authors: Felipe Balbi <balbi@ti.com>, 7 * Sebastian Andrzej Siewior <bigeasy@linutronix.de> 8 * 9 * This program is free software: you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 of 11 * the License as published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 */ 18 19 #include <linux/kernel.h> 20 #include <linux/delay.h> 21 #include <linux/slab.h> 22 #include <linux/spinlock.h> 23 #include <linux/platform_device.h> 24 #include <linux/pm_runtime.h> 25 #include <linux/interrupt.h> 26 #include <linux/io.h> 27 #include <linux/list.h> 28 #include <linux/dma-mapping.h> 29 30 #include <linux/usb/ch9.h> 31 #include <linux/usb/gadget.h> 32 33 #include "debug.h" 34 #include "core.h" 35 #include "gadget.h" 36 #include "io.h" 37 38 /** 39 * dwc3_gadget_set_test_mode - Enables USB2 Test Modes 40 * @dwc: pointer to our context structure 41 * @mode: the mode to set (J, K SE0 NAK, Force Enable) 42 * 43 * Caller should take care of locking. This function will 44 * return 0 on success or -EINVAL if wrong Test Selector 45 * is passed 46 */ 47 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode) 48 { 49 u32 reg; 50 51 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 52 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 53 54 switch (mode) { 55 case TEST_J: 56 case TEST_K: 57 case TEST_SE0_NAK: 58 case TEST_PACKET: 59 case TEST_FORCE_EN: 60 reg |= mode << 1; 61 break; 62 default: 63 return -EINVAL; 64 } 65 66 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 67 68 return 0; 69 } 70 71 /** 72 * dwc3_gadget_get_link_state - Gets current state of USB Link 73 * @dwc: pointer to our context structure 74 * 75 * Caller should take care of locking. This function will 76 * return the link state on success (>= 0) or -ETIMEDOUT. 77 */ 78 int dwc3_gadget_get_link_state(struct dwc3 *dwc) 79 { 80 u32 reg; 81 82 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 83 84 return DWC3_DSTS_USBLNKST(reg); 85 } 86 87 /** 88 * dwc3_gadget_set_link_state - Sets USB Link to a particular State 89 * @dwc: pointer to our context structure 90 * @state: the state to put link into 91 * 92 * Caller should take care of locking. This function will 93 * return 0 on success or -ETIMEDOUT. 94 */ 95 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state) 96 { 97 int retries = 10000; 98 u32 reg; 99 100 /* 101 * Wait until device controller is ready. Only applies to 1.94a and 102 * later RTL. 103 */ 104 if (dwc->revision >= DWC3_REVISION_194A) { 105 while (--retries) { 106 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 107 if (reg & DWC3_DSTS_DCNRD) 108 udelay(5); 109 else 110 break; 111 } 112 113 if (retries <= 0) 114 return -ETIMEDOUT; 115 } 116 117 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 118 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 119 120 /* set requested state */ 121 reg |= DWC3_DCTL_ULSTCHNGREQ(state); 122 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 123 124 /* 125 * The following code is racy when called from dwc3_gadget_wakeup, 126 * and is not needed, at least on newer versions 127 */ 128 if (dwc->revision >= DWC3_REVISION_194A) 129 return 0; 130 131 /* wait for a change in DSTS */ 132 retries = 10000; 133 while (--retries) { 134 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 135 136 if (DWC3_DSTS_USBLNKST(reg) == state) 137 return 0; 138 139 udelay(5); 140 } 141 142 dwc3_trace(trace_dwc3_gadget, 143 "link state change request timed out"); 144 145 return -ETIMEDOUT; 146 } 147 148 static void dwc3_ep_inc_enq(struct dwc3_ep *dep) 149 { 150 dep->trb_enqueue++; 151 dep->trb_enqueue %= DWC3_TRB_NUM; 152 } 153 154 static void dwc3_ep_inc_deq(struct dwc3_ep *dep) 155 { 156 dep->trb_dequeue++; 157 dep->trb_dequeue %= DWC3_TRB_NUM; 158 } 159 160 static int dwc3_ep_is_last_trb(unsigned int index) 161 { 162 return index == DWC3_TRB_NUM - 1; 163 } 164 165 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, 166 int status) 167 { 168 struct dwc3 *dwc = dep->dwc; 169 int i; 170 171 if (req->started) { 172 i = 0; 173 do { 174 dwc3_ep_inc_deq(dep); 175 /* 176 * Skip LINK TRB. We can't use req->trb and check for 177 * DWC3_TRBCTL_LINK_TRB because it points the TRB we 178 * just completed (not the LINK TRB). 179 */ 180 if (dwc3_ep_is_last_trb(dep->trb_dequeue)) 181 dwc3_ep_inc_deq(dep); 182 } while(++i < req->request.num_mapped_sgs); 183 req->started = false; 184 } 185 list_del(&req->list); 186 req->trb = NULL; 187 188 if (req->request.status == -EINPROGRESS) 189 req->request.status = status; 190 191 if (dwc->ep0_bounced && dep->number == 0) 192 dwc->ep0_bounced = false; 193 else 194 usb_gadget_unmap_request(&dwc->gadget, &req->request, 195 req->direction); 196 197 trace_dwc3_gadget_giveback(req); 198 199 spin_unlock(&dwc->lock); 200 usb_gadget_giveback_request(&dep->endpoint, &req->request); 201 spin_lock(&dwc->lock); 202 } 203 204 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param) 205 { 206 u32 timeout = 500; 207 u32 reg; 208 209 trace_dwc3_gadget_generic_cmd(cmd, param); 210 211 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param); 212 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT); 213 214 do { 215 reg = dwc3_readl(dwc->regs, DWC3_DGCMD); 216 if (!(reg & DWC3_DGCMD_CMDACT)) { 217 dwc3_trace(trace_dwc3_gadget, 218 "Command Complete --> %d", 219 DWC3_DGCMD_STATUS(reg)); 220 if (DWC3_DGCMD_STATUS(reg)) 221 return -EINVAL; 222 return 0; 223 } 224 225 /* 226 * We can't sleep here, because it's also called from 227 * interrupt context. 228 */ 229 timeout--; 230 if (!timeout) { 231 dwc3_trace(trace_dwc3_gadget, 232 "Command Timed Out"); 233 return -ETIMEDOUT; 234 } 235 udelay(1); 236 } while (1); 237 } 238 239 static int __dwc3_gadget_wakeup(struct dwc3 *dwc); 240 241 int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep, 242 unsigned cmd, struct dwc3_gadget_ep_cmd_params *params) 243 { 244 struct dwc3_ep *dep = dwc->eps[ep]; 245 u32 timeout = 500; 246 u32 reg; 247 248 int susphy = false; 249 int ret = -EINVAL; 250 251 trace_dwc3_gadget_ep_cmd(dep, cmd, params); 252 253 /* 254 * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if 255 * we're issuing an endpoint command, we must check if 256 * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it. 257 * 258 * We will also set SUSPHY bit to what it was before returning as stated 259 * by the same section on Synopsys databook. 260 */ 261 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 262 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) { 263 susphy = true; 264 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY; 265 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 266 } 267 268 if (cmd == DWC3_DEPCMD_STARTTRANSFER) { 269 int needs_wakeup; 270 271 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 || 272 dwc->link_state == DWC3_LINK_STATE_U2 || 273 dwc->link_state == DWC3_LINK_STATE_U3); 274 275 if (unlikely(needs_wakeup)) { 276 ret = __dwc3_gadget_wakeup(dwc); 277 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n", 278 ret); 279 } 280 } 281 282 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0); 283 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1); 284 dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2); 285 286 dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT); 287 do { 288 reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep)); 289 if (!(reg & DWC3_DEPCMD_CMDACT)) { 290 int cmd_status = DWC3_DEPCMD_STATUS(reg); 291 292 dwc3_trace(trace_dwc3_gadget, 293 "Command Complete --> %d", 294 cmd_status); 295 296 switch (cmd_status) { 297 case 0: 298 ret = 0; 299 break; 300 case DEPEVT_TRANSFER_NO_RESOURCE: 301 dwc3_trace(trace_dwc3_gadget, "%s: no resource available"); 302 ret = -EINVAL; 303 break; 304 case DEPEVT_TRANSFER_BUS_EXPIRY: 305 /* 306 * SW issues START TRANSFER command to 307 * isochronous ep with future frame interval. If 308 * future interval time has already passed when 309 * core receives the command, it will respond 310 * with an error status of 'Bus Expiry'. 311 * 312 * Instead of always returning -EINVAL, let's 313 * give a hint to the gadget driver that this is 314 * the case by returning -EAGAIN. 315 */ 316 dwc3_trace(trace_dwc3_gadget, "%s: bus expiry"); 317 ret = -EAGAIN; 318 break; 319 default: 320 dev_WARN(dwc->dev, "UNKNOWN cmd status\n"); 321 } 322 323 break; 324 } 325 326 /* 327 * We can't sleep here, because it is also called from 328 * interrupt context. 329 */ 330 timeout--; 331 if (!timeout) { 332 dwc3_trace(trace_dwc3_gadget, 333 "Command Timed Out"); 334 ret = -ETIMEDOUT; 335 break; 336 } 337 338 udelay(1); 339 } while (1); 340 341 if (unlikely(susphy)) { 342 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 343 reg |= DWC3_GUSB2PHYCFG_SUSPHY; 344 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 345 } 346 347 return ret; 348 } 349 350 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, 351 struct dwc3_trb *trb) 352 { 353 u32 offset = (char *) trb - (char *) dep->trb_pool; 354 355 return dep->trb_pool_dma + offset; 356 } 357 358 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep) 359 { 360 struct dwc3 *dwc = dep->dwc; 361 362 if (dep->trb_pool) 363 return 0; 364 365 dep->trb_pool = dma_alloc_coherent(dwc->dev, 366 sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 367 &dep->trb_pool_dma, GFP_KERNEL); 368 if (!dep->trb_pool) { 369 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", 370 dep->name); 371 return -ENOMEM; 372 } 373 374 return 0; 375 } 376 377 static void dwc3_free_trb_pool(struct dwc3_ep *dep) 378 { 379 struct dwc3 *dwc = dep->dwc; 380 381 dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 382 dep->trb_pool, dep->trb_pool_dma); 383 384 dep->trb_pool = NULL; 385 dep->trb_pool_dma = 0; 386 } 387 388 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep); 389 390 /** 391 * dwc3_gadget_start_config - Configure EP resources 392 * @dwc: pointer to our controller context structure 393 * @dep: endpoint that is being enabled 394 * 395 * The assignment of transfer resources cannot perfectly follow the 396 * data book due to the fact that the controller driver does not have 397 * all knowledge of the configuration in advance. It is given this 398 * information piecemeal by the composite gadget framework after every 399 * SET_CONFIGURATION and SET_INTERFACE. Trying to follow the databook 400 * programming model in this scenario can cause errors. For two 401 * reasons: 402 * 403 * 1) The databook says to do DEPSTARTCFG for every SET_CONFIGURATION 404 * and SET_INTERFACE (8.1.5). This is incorrect in the scenario of 405 * multiple interfaces. 406 * 407 * 2) The databook does not mention doing more DEPXFERCFG for new 408 * endpoint on alt setting (8.1.6). 409 * 410 * The following simplified method is used instead: 411 * 412 * All hardware endpoints can be assigned a transfer resource and this 413 * setting will stay persistent until either a core reset or 414 * hibernation. So whenever we do a DEPSTARTCFG(0) we can go ahead and 415 * do DEPXFERCFG for every hardware endpoint as well. We are 416 * guaranteed that there are as many transfer resources as endpoints. 417 * 418 * This function is called for each endpoint when it is being enabled 419 * but is triggered only when called for EP0-out, which always happens 420 * first, and which should only happen in one of the above conditions. 421 */ 422 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep) 423 { 424 struct dwc3_gadget_ep_cmd_params params; 425 u32 cmd; 426 int i; 427 int ret; 428 429 if (dep->number) 430 return 0; 431 432 memset(¶ms, 0x00, sizeof(params)); 433 cmd = DWC3_DEPCMD_DEPSTARTCFG; 434 435 ret = dwc3_send_gadget_ep_cmd(dwc, 0, cmd, ¶ms); 436 if (ret) 437 return ret; 438 439 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) { 440 struct dwc3_ep *dep = dwc->eps[i]; 441 442 if (!dep) 443 continue; 444 445 ret = dwc3_gadget_set_xfer_resource(dwc, dep); 446 if (ret) 447 return ret; 448 } 449 450 return 0; 451 } 452 453 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep, 454 const struct usb_endpoint_descriptor *desc, 455 const struct usb_ss_ep_comp_descriptor *comp_desc, 456 bool ignore, bool restore) 457 { 458 struct dwc3_gadget_ep_cmd_params params; 459 460 memset(¶ms, 0x00, sizeof(params)); 461 462 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc)) 463 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc)); 464 465 /* Burst size is only needed in SuperSpeed mode */ 466 if (dwc->gadget.speed >= USB_SPEED_SUPER) { 467 u32 burst = dep->endpoint.maxburst; 468 u32 nump; 469 u32 reg; 470 471 /* update NumP */ 472 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 473 nump = DWC3_DCFG_NUMP(reg); 474 nump = max(nump, burst); 475 reg &= ~DWC3_DCFG_NUMP_MASK; 476 reg |= nump << DWC3_DCFG_NUMP_SHIFT; 477 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 478 479 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); 480 } 481 482 if (ignore) 483 params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM; 484 485 if (restore) { 486 params.param0 |= DWC3_DEPCFG_ACTION_RESTORE; 487 params.param2 |= dep->saved_state; 488 } 489 490 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN 491 | DWC3_DEPCFG_XFER_NOT_READY_EN; 492 493 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) { 494 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE 495 | DWC3_DEPCFG_STREAM_EVENT_EN; 496 dep->stream_capable = true; 497 } 498 499 if (!usb_endpoint_xfer_control(desc)) 500 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN; 501 502 /* 503 * We are doing 1:1 mapping for endpoints, meaning 504 * Physical Endpoints 2 maps to Logical Endpoint 2 and 505 * so on. We consider the direction bit as part of the physical 506 * endpoint number. So USB endpoint 0x81 is 0x03. 507 */ 508 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); 509 510 /* 511 * We must use the lower 16 TX FIFOs even though 512 * HW might have more 513 */ 514 if (dep->direction) 515 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); 516 517 if (desc->bInterval) { 518 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1); 519 dep->interval = 1 << (desc->bInterval - 1); 520 } 521 522 return dwc3_send_gadget_ep_cmd(dwc, dep->number, 523 DWC3_DEPCMD_SETEPCONFIG, ¶ms); 524 } 525 526 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep) 527 { 528 struct dwc3_gadget_ep_cmd_params params; 529 530 memset(¶ms, 0x00, sizeof(params)); 531 532 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1); 533 534 return dwc3_send_gadget_ep_cmd(dwc, dep->number, 535 DWC3_DEPCMD_SETTRANSFRESOURCE, ¶ms); 536 } 537 538 /** 539 * __dwc3_gadget_ep_enable - Initializes a HW endpoint 540 * @dep: endpoint to be initialized 541 * @desc: USB Endpoint Descriptor 542 * 543 * Caller should take care of locking 544 */ 545 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, 546 const struct usb_endpoint_descriptor *desc, 547 const struct usb_ss_ep_comp_descriptor *comp_desc, 548 bool ignore, bool restore) 549 { 550 struct dwc3 *dwc = dep->dwc; 551 u32 reg; 552 int ret; 553 554 dwc3_trace(trace_dwc3_gadget, "Enabling %s", dep->name); 555 556 if (!(dep->flags & DWC3_EP_ENABLED)) { 557 ret = dwc3_gadget_start_config(dwc, dep); 558 if (ret) 559 return ret; 560 } 561 562 ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore, 563 restore); 564 if (ret) 565 return ret; 566 567 if (!(dep->flags & DWC3_EP_ENABLED)) { 568 struct dwc3_trb *trb_st_hw; 569 struct dwc3_trb *trb_link; 570 571 dep->endpoint.desc = desc; 572 dep->comp_desc = comp_desc; 573 dep->type = usb_endpoint_type(desc); 574 dep->flags |= DWC3_EP_ENABLED; 575 576 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 577 reg |= DWC3_DALEPENA_EP(dep->number); 578 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 579 580 if (usb_endpoint_xfer_control(desc)) 581 goto out; 582 583 /* Link TRB. The HWO bit is never reset */ 584 trb_st_hw = &dep->trb_pool[0]; 585 586 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; 587 memset(trb_link, 0, sizeof(*trb_link)); 588 589 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 590 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 591 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB; 592 trb_link->ctrl |= DWC3_TRB_CTRL_HWO; 593 } 594 595 out: 596 switch (usb_endpoint_type(desc)) { 597 case USB_ENDPOINT_XFER_CONTROL: 598 /* don't change name */ 599 break; 600 case USB_ENDPOINT_XFER_ISOC: 601 strlcat(dep->name, "-isoc", sizeof(dep->name)); 602 break; 603 case USB_ENDPOINT_XFER_BULK: 604 strlcat(dep->name, "-bulk", sizeof(dep->name)); 605 break; 606 case USB_ENDPOINT_XFER_INT: 607 strlcat(dep->name, "-int", sizeof(dep->name)); 608 break; 609 default: 610 dev_err(dwc->dev, "invalid endpoint transfer type\n"); 611 } 612 613 return 0; 614 } 615 616 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force); 617 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep) 618 { 619 struct dwc3_request *req; 620 621 if (!list_empty(&dep->started_list)) { 622 dwc3_stop_active_transfer(dwc, dep->number, true); 623 624 /* - giveback all requests to gadget driver */ 625 while (!list_empty(&dep->started_list)) { 626 req = next_request(&dep->started_list); 627 628 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 629 } 630 } 631 632 while (!list_empty(&dep->pending_list)) { 633 req = next_request(&dep->pending_list); 634 635 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 636 } 637 } 638 639 /** 640 * __dwc3_gadget_ep_disable - Disables a HW endpoint 641 * @dep: the endpoint to disable 642 * 643 * This function also removes requests which are currently processed ny the 644 * hardware and those which are not yet scheduled. 645 * Caller should take care of locking. 646 */ 647 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) 648 { 649 struct dwc3 *dwc = dep->dwc; 650 u32 reg; 651 652 dwc3_trace(trace_dwc3_gadget, "Disabling %s", dep->name); 653 654 dwc3_remove_requests(dwc, dep); 655 656 /* make sure HW endpoint isn't stalled */ 657 if (dep->flags & DWC3_EP_STALL) 658 __dwc3_gadget_ep_set_halt(dep, 0, false); 659 660 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 661 reg &= ~DWC3_DALEPENA_EP(dep->number); 662 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 663 664 dep->stream_capable = false; 665 dep->endpoint.desc = NULL; 666 dep->comp_desc = NULL; 667 dep->type = 0; 668 dep->flags = 0; 669 670 snprintf(dep->name, sizeof(dep->name), "ep%d%s", 671 dep->number >> 1, 672 (dep->number & 1) ? "in" : "out"); 673 674 return 0; 675 } 676 677 /* -------------------------------------------------------------------------- */ 678 679 static int dwc3_gadget_ep0_enable(struct usb_ep *ep, 680 const struct usb_endpoint_descriptor *desc) 681 { 682 return -EINVAL; 683 } 684 685 static int dwc3_gadget_ep0_disable(struct usb_ep *ep) 686 { 687 return -EINVAL; 688 } 689 690 /* -------------------------------------------------------------------------- */ 691 692 static int dwc3_gadget_ep_enable(struct usb_ep *ep, 693 const struct usb_endpoint_descriptor *desc) 694 { 695 struct dwc3_ep *dep; 696 struct dwc3 *dwc; 697 unsigned long flags; 698 int ret; 699 700 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { 701 pr_debug("dwc3: invalid parameters\n"); 702 return -EINVAL; 703 } 704 705 if (!desc->wMaxPacketSize) { 706 pr_debug("dwc3: missing wMaxPacketSize\n"); 707 return -EINVAL; 708 } 709 710 dep = to_dwc3_ep(ep); 711 dwc = dep->dwc; 712 713 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, 714 "%s is already enabled\n", 715 dep->name)) 716 return 0; 717 718 spin_lock_irqsave(&dwc->lock, flags); 719 ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false); 720 spin_unlock_irqrestore(&dwc->lock, flags); 721 722 return ret; 723 } 724 725 static int dwc3_gadget_ep_disable(struct usb_ep *ep) 726 { 727 struct dwc3_ep *dep; 728 struct dwc3 *dwc; 729 unsigned long flags; 730 int ret; 731 732 if (!ep) { 733 pr_debug("dwc3: invalid parameters\n"); 734 return -EINVAL; 735 } 736 737 dep = to_dwc3_ep(ep); 738 dwc = dep->dwc; 739 740 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), 741 "%s is already disabled\n", 742 dep->name)) 743 return 0; 744 745 spin_lock_irqsave(&dwc->lock, flags); 746 ret = __dwc3_gadget_ep_disable(dep); 747 spin_unlock_irqrestore(&dwc->lock, flags); 748 749 return ret; 750 } 751 752 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep, 753 gfp_t gfp_flags) 754 { 755 struct dwc3_request *req; 756 struct dwc3_ep *dep = to_dwc3_ep(ep); 757 758 req = kzalloc(sizeof(*req), gfp_flags); 759 if (!req) 760 return NULL; 761 762 req->epnum = dep->number; 763 req->dep = dep; 764 765 trace_dwc3_alloc_request(req); 766 767 return &req->request; 768 } 769 770 static void dwc3_gadget_ep_free_request(struct usb_ep *ep, 771 struct usb_request *request) 772 { 773 struct dwc3_request *req = to_dwc3_request(request); 774 775 trace_dwc3_free_request(req); 776 kfree(req); 777 } 778 779 /** 780 * dwc3_prepare_one_trb - setup one TRB from one request 781 * @dep: endpoint for which this request is prepared 782 * @req: dwc3_request pointer 783 */ 784 static void dwc3_prepare_one_trb(struct dwc3_ep *dep, 785 struct dwc3_request *req, dma_addr_t dma, 786 unsigned length, unsigned last, unsigned chain, unsigned node) 787 { 788 struct dwc3_trb *trb; 789 790 dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s%s", 791 dep->name, req, (unsigned long long) dma, 792 length, last ? " last" : "", 793 chain ? " chain" : ""); 794 795 796 trb = &dep->trb_pool[dep->trb_enqueue]; 797 798 if (!req->trb) { 799 dwc3_gadget_move_started_request(req); 800 req->trb = trb; 801 req->trb_dma = dwc3_trb_dma_offset(dep, trb); 802 req->first_trb_index = dep->trb_enqueue; 803 } 804 805 dwc3_ep_inc_enq(dep); 806 /* Skip the LINK-TRB */ 807 if (dwc3_ep_is_last_trb(dep->trb_enqueue)) 808 dwc3_ep_inc_enq(dep); 809 810 trb->size = DWC3_TRB_SIZE_LENGTH(length); 811 trb->bpl = lower_32_bits(dma); 812 trb->bph = upper_32_bits(dma); 813 814 switch (usb_endpoint_type(dep->endpoint.desc)) { 815 case USB_ENDPOINT_XFER_CONTROL: 816 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP; 817 break; 818 819 case USB_ENDPOINT_XFER_ISOC: 820 if (!node) 821 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; 822 else 823 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; 824 825 /* always enable Interrupt on Missed ISOC */ 826 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 827 break; 828 829 case USB_ENDPOINT_XFER_BULK: 830 case USB_ENDPOINT_XFER_INT: 831 trb->ctrl = DWC3_TRBCTL_NORMAL; 832 break; 833 default: 834 /* 835 * This is only possible with faulty memory because we 836 * checked it already :) 837 */ 838 BUG(); 839 } 840 841 /* always enable Continue on Short Packet */ 842 trb->ctrl |= DWC3_TRB_CTRL_CSP; 843 844 if (!req->request.no_interrupt && !chain) 845 trb->ctrl |= DWC3_TRB_CTRL_IOC | DWC3_TRB_CTRL_ISP_IMI; 846 847 if (last) 848 trb->ctrl |= DWC3_TRB_CTRL_LST; 849 850 if (chain) 851 trb->ctrl |= DWC3_TRB_CTRL_CHN; 852 853 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) 854 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id); 855 856 trb->ctrl |= DWC3_TRB_CTRL_HWO; 857 858 trace_dwc3_prepare_trb(dep, trb); 859 } 860 861 /* 862 * dwc3_prepare_trbs - setup TRBs from requests 863 * @dep: endpoint for which requests are being prepared 864 * @starting: true if the endpoint is idle and no requests are queued. 865 * 866 * The function goes through the requests list and sets up TRBs for the 867 * transfers. The function returns once there are no more TRBs available or 868 * it runs out of requests. 869 */ 870 static void dwc3_prepare_trbs(struct dwc3_ep *dep, bool starting) 871 { 872 struct dwc3_request *req, *n; 873 u32 trbs_left; 874 unsigned int last_one = 0; 875 876 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM); 877 878 trbs_left = dep->trb_dequeue - dep->trb_enqueue; 879 880 /* 881 * If enqueue & dequeue are equal than it is either full or empty. If we 882 * are starting to process requests then we are empty. Otherwise we are 883 * full and don't do anything 884 */ 885 if (!trbs_left) { 886 if (!starting) 887 return; 888 889 trbs_left = DWC3_TRB_NUM; 890 } 891 892 /* The last TRB is a link TRB, not used for xfer */ 893 if (trbs_left <= 1) 894 return; 895 896 list_for_each_entry_safe(req, n, &dep->pending_list, list) { 897 unsigned length; 898 dma_addr_t dma; 899 last_one = false; 900 901 if (req->request.num_mapped_sgs > 0) { 902 struct usb_request *request = &req->request; 903 struct scatterlist *sg = request->sg; 904 struct scatterlist *s; 905 int i; 906 907 for_each_sg(sg, s, request->num_mapped_sgs, i) { 908 unsigned chain = true; 909 910 length = sg_dma_len(s); 911 dma = sg_dma_address(s); 912 913 if (i == (request->num_mapped_sgs - 1) || 914 sg_is_last(s)) { 915 if (list_empty(&dep->pending_list)) 916 last_one = true; 917 chain = false; 918 } 919 920 trbs_left--; 921 if (!trbs_left) 922 last_one = true; 923 924 if (last_one) 925 chain = false; 926 927 dwc3_prepare_one_trb(dep, req, dma, length, 928 last_one, chain, i); 929 930 if (last_one) 931 break; 932 } 933 934 if (last_one) 935 break; 936 } else { 937 dma = req->request.dma; 938 length = req->request.length; 939 trbs_left--; 940 941 if (!trbs_left) 942 last_one = 1; 943 944 /* Is this the last request? */ 945 if (list_is_last(&req->list, &dep->pending_list)) 946 last_one = 1; 947 948 dwc3_prepare_one_trb(dep, req, dma, length, 949 last_one, false, 0); 950 951 if (last_one) 952 break; 953 } 954 } 955 } 956 957 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param, 958 int start_new) 959 { 960 struct dwc3_gadget_ep_cmd_params params; 961 struct dwc3_request *req; 962 struct dwc3 *dwc = dep->dwc; 963 int ret; 964 u32 cmd; 965 966 if (start_new && (dep->flags & DWC3_EP_BUSY)) { 967 dwc3_trace(trace_dwc3_gadget, "%s: endpoint busy", dep->name); 968 return -EBUSY; 969 } 970 971 /* 972 * If we are getting here after a short-out-packet we don't enqueue any 973 * new requests as we try to set the IOC bit only on the last request. 974 */ 975 if (start_new) { 976 if (list_empty(&dep->started_list)) 977 dwc3_prepare_trbs(dep, start_new); 978 979 /* req points to the first request which will be sent */ 980 req = next_request(&dep->started_list); 981 } else { 982 dwc3_prepare_trbs(dep, start_new); 983 984 /* 985 * req points to the first request where HWO changed from 0 to 1 986 */ 987 req = next_request(&dep->started_list); 988 } 989 if (!req) { 990 dep->flags |= DWC3_EP_PENDING_REQUEST; 991 return 0; 992 } 993 994 memset(¶ms, 0, sizeof(params)); 995 996 if (start_new) { 997 params.param0 = upper_32_bits(req->trb_dma); 998 params.param1 = lower_32_bits(req->trb_dma); 999 cmd = DWC3_DEPCMD_STARTTRANSFER; 1000 } else { 1001 cmd = DWC3_DEPCMD_UPDATETRANSFER; 1002 } 1003 1004 cmd |= DWC3_DEPCMD_PARAM(cmd_param); 1005 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, ¶ms); 1006 if (ret < 0) { 1007 /* 1008 * FIXME we need to iterate over the list of requests 1009 * here and stop, unmap, free and del each of the linked 1010 * requests instead of what we do now. 1011 */ 1012 usb_gadget_unmap_request(&dwc->gadget, &req->request, 1013 req->direction); 1014 list_del(&req->list); 1015 return ret; 1016 } 1017 1018 dep->flags |= DWC3_EP_BUSY; 1019 1020 if (start_new) { 1021 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dwc, 1022 dep->number); 1023 WARN_ON_ONCE(!dep->resource_index); 1024 } 1025 1026 return 0; 1027 } 1028 1029 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc, 1030 struct dwc3_ep *dep, u32 cur_uf) 1031 { 1032 u32 uf; 1033 1034 if (list_empty(&dep->pending_list)) { 1035 dwc3_trace(trace_dwc3_gadget, 1036 "ISOC ep %s run out for requests", 1037 dep->name); 1038 dep->flags |= DWC3_EP_PENDING_REQUEST; 1039 return; 1040 } 1041 1042 /* 4 micro frames in the future */ 1043 uf = cur_uf + dep->interval * 4; 1044 1045 __dwc3_gadget_kick_transfer(dep, uf, 1); 1046 } 1047 1048 static void dwc3_gadget_start_isoc(struct dwc3 *dwc, 1049 struct dwc3_ep *dep, const struct dwc3_event_depevt *event) 1050 { 1051 u32 cur_uf, mask; 1052 1053 mask = ~(dep->interval - 1); 1054 cur_uf = event->parameters & mask; 1055 1056 __dwc3_gadget_start_isoc(dwc, dep, cur_uf); 1057 } 1058 1059 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) 1060 { 1061 struct dwc3 *dwc = dep->dwc; 1062 int ret; 1063 1064 if (!dep->endpoint.desc) { 1065 dwc3_trace(trace_dwc3_gadget, 1066 "trying to queue request %p to disabled %s\n", 1067 &req->request, dep->endpoint.name); 1068 return -ESHUTDOWN; 1069 } 1070 1071 if (WARN(req->dep != dep, "request %p belongs to '%s'\n", 1072 &req->request, req->dep->name)) { 1073 dwc3_trace(trace_dwc3_gadget, "request %p belongs to '%s'\n", 1074 &req->request, req->dep->name); 1075 return -EINVAL; 1076 } 1077 1078 req->request.actual = 0; 1079 req->request.status = -EINPROGRESS; 1080 req->direction = dep->direction; 1081 req->epnum = dep->number; 1082 1083 trace_dwc3_ep_queue(req); 1084 1085 /* 1086 * We only add to our list of requests now and 1087 * start consuming the list once we get XferNotReady 1088 * IRQ. 1089 * 1090 * That way, we avoid doing anything that we don't need 1091 * to do now and defer it until the point we receive a 1092 * particular token from the Host side. 1093 * 1094 * This will also avoid Host cancelling URBs due to too 1095 * many NAKs. 1096 */ 1097 ret = usb_gadget_map_request(&dwc->gadget, &req->request, 1098 dep->direction); 1099 if (ret) 1100 return ret; 1101 1102 list_add_tail(&req->list, &dep->pending_list); 1103 1104 /* 1105 * If there are no pending requests and the endpoint isn't already 1106 * busy, we will just start the request straight away. 1107 * 1108 * This will save one IRQ (XFER_NOT_READY) and possibly make it a 1109 * little bit faster. 1110 */ 1111 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) && 1112 !usb_endpoint_xfer_int(dep->endpoint.desc) && 1113 !(dep->flags & DWC3_EP_BUSY)) { 1114 ret = __dwc3_gadget_kick_transfer(dep, 0, true); 1115 goto out; 1116 } 1117 1118 /* 1119 * There are a few special cases: 1120 * 1121 * 1. XferNotReady with empty list of requests. We need to kick the 1122 * transfer here in that situation, otherwise we will be NAKing 1123 * forever. If we get XferNotReady before gadget driver has a 1124 * chance to queue a request, we will ACK the IRQ but won't be 1125 * able to receive the data until the next request is queued. 1126 * The following code is handling exactly that. 1127 * 1128 */ 1129 if (dep->flags & DWC3_EP_PENDING_REQUEST) { 1130 /* 1131 * If xfernotready is already elapsed and it is a case 1132 * of isoc transfer, then issue END TRANSFER, so that 1133 * you can receive xfernotready again and can have 1134 * notion of current microframe. 1135 */ 1136 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 1137 if (list_empty(&dep->started_list)) { 1138 dwc3_stop_active_transfer(dwc, dep->number, true); 1139 dep->flags = DWC3_EP_ENABLED; 1140 } 1141 return 0; 1142 } 1143 1144 ret = __dwc3_gadget_kick_transfer(dep, 0, true); 1145 if (!ret) 1146 dep->flags &= ~DWC3_EP_PENDING_REQUEST; 1147 1148 goto out; 1149 } 1150 1151 /* 1152 * 2. XferInProgress on Isoc EP with an active transfer. We need to 1153 * kick the transfer here after queuing a request, otherwise the 1154 * core may not see the modified TRB(s). 1155 */ 1156 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && 1157 (dep->flags & DWC3_EP_BUSY) && 1158 !(dep->flags & DWC3_EP_MISSED_ISOC)) { 1159 WARN_ON_ONCE(!dep->resource_index); 1160 ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index, 1161 false); 1162 goto out; 1163 } 1164 1165 /* 1166 * 4. Stream Capable Bulk Endpoints. We need to start the transfer 1167 * right away, otherwise host will not know we have streams to be 1168 * handled. 1169 */ 1170 if (dep->stream_capable) 1171 ret = __dwc3_gadget_kick_transfer(dep, 0, true); 1172 1173 out: 1174 if (ret && ret != -EBUSY) 1175 dwc3_trace(trace_dwc3_gadget, 1176 "%s: failed to kick transfers\n", 1177 dep->name); 1178 if (ret == -EBUSY) 1179 ret = 0; 1180 1181 return ret; 1182 } 1183 1184 static void __dwc3_gadget_ep_zlp_complete(struct usb_ep *ep, 1185 struct usb_request *request) 1186 { 1187 dwc3_gadget_ep_free_request(ep, request); 1188 } 1189 1190 static int __dwc3_gadget_ep_queue_zlp(struct dwc3 *dwc, struct dwc3_ep *dep) 1191 { 1192 struct dwc3_request *req; 1193 struct usb_request *request; 1194 struct usb_ep *ep = &dep->endpoint; 1195 1196 dwc3_trace(trace_dwc3_gadget, "queueing ZLP\n"); 1197 request = dwc3_gadget_ep_alloc_request(ep, GFP_ATOMIC); 1198 if (!request) 1199 return -ENOMEM; 1200 1201 request->length = 0; 1202 request->buf = dwc->zlp_buf; 1203 request->complete = __dwc3_gadget_ep_zlp_complete; 1204 1205 req = to_dwc3_request(request); 1206 1207 return __dwc3_gadget_ep_queue(dep, req); 1208 } 1209 1210 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, 1211 gfp_t gfp_flags) 1212 { 1213 struct dwc3_request *req = to_dwc3_request(request); 1214 struct dwc3_ep *dep = to_dwc3_ep(ep); 1215 struct dwc3 *dwc = dep->dwc; 1216 1217 unsigned long flags; 1218 1219 int ret; 1220 1221 spin_lock_irqsave(&dwc->lock, flags); 1222 ret = __dwc3_gadget_ep_queue(dep, req); 1223 1224 /* 1225 * Okay, here's the thing, if gadget driver has requested for a ZLP by 1226 * setting request->zero, instead of doing magic, we will just queue an 1227 * extra usb_request ourselves so that it gets handled the same way as 1228 * any other request. 1229 */ 1230 if (ret == 0 && request->zero && request->length && 1231 (request->length % ep->maxpacket == 0)) 1232 ret = __dwc3_gadget_ep_queue_zlp(dwc, dep); 1233 1234 spin_unlock_irqrestore(&dwc->lock, flags); 1235 1236 return ret; 1237 } 1238 1239 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, 1240 struct usb_request *request) 1241 { 1242 struct dwc3_request *req = to_dwc3_request(request); 1243 struct dwc3_request *r = NULL; 1244 1245 struct dwc3_ep *dep = to_dwc3_ep(ep); 1246 struct dwc3 *dwc = dep->dwc; 1247 1248 unsigned long flags; 1249 int ret = 0; 1250 1251 trace_dwc3_ep_dequeue(req); 1252 1253 spin_lock_irqsave(&dwc->lock, flags); 1254 1255 list_for_each_entry(r, &dep->pending_list, list) { 1256 if (r == req) 1257 break; 1258 } 1259 1260 if (r != req) { 1261 list_for_each_entry(r, &dep->started_list, list) { 1262 if (r == req) 1263 break; 1264 } 1265 if (r == req) { 1266 /* wait until it is processed */ 1267 dwc3_stop_active_transfer(dwc, dep->number, true); 1268 goto out1; 1269 } 1270 dev_err(dwc->dev, "request %p was not queued to %s\n", 1271 request, ep->name); 1272 ret = -EINVAL; 1273 goto out0; 1274 } 1275 1276 out1: 1277 /* giveback the request */ 1278 dwc3_gadget_giveback(dep, req, -ECONNRESET); 1279 1280 out0: 1281 spin_unlock_irqrestore(&dwc->lock, flags); 1282 1283 return ret; 1284 } 1285 1286 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) 1287 { 1288 struct dwc3_gadget_ep_cmd_params params; 1289 struct dwc3 *dwc = dep->dwc; 1290 int ret; 1291 1292 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 1293 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); 1294 return -EINVAL; 1295 } 1296 1297 memset(¶ms, 0x00, sizeof(params)); 1298 1299 if (value) { 1300 if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) || 1301 (!list_empty(&dep->started_list) || 1302 !list_empty(&dep->pending_list)))) { 1303 dwc3_trace(trace_dwc3_gadget, 1304 "%s: pending request, cannot halt", 1305 dep->name); 1306 return -EAGAIN; 1307 } 1308 1309 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, 1310 DWC3_DEPCMD_SETSTALL, ¶ms); 1311 if (ret) 1312 dev_err(dwc->dev, "failed to set STALL on %s\n", 1313 dep->name); 1314 else 1315 dep->flags |= DWC3_EP_STALL; 1316 } else { 1317 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, 1318 DWC3_DEPCMD_CLEARSTALL, ¶ms); 1319 if (ret) 1320 dev_err(dwc->dev, "failed to clear STALL on %s\n", 1321 dep->name); 1322 else 1323 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 1324 } 1325 1326 return ret; 1327 } 1328 1329 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value) 1330 { 1331 struct dwc3_ep *dep = to_dwc3_ep(ep); 1332 struct dwc3 *dwc = dep->dwc; 1333 1334 unsigned long flags; 1335 1336 int ret; 1337 1338 spin_lock_irqsave(&dwc->lock, flags); 1339 ret = __dwc3_gadget_ep_set_halt(dep, value, false); 1340 spin_unlock_irqrestore(&dwc->lock, flags); 1341 1342 return ret; 1343 } 1344 1345 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep) 1346 { 1347 struct dwc3_ep *dep = to_dwc3_ep(ep); 1348 struct dwc3 *dwc = dep->dwc; 1349 unsigned long flags; 1350 int ret; 1351 1352 spin_lock_irqsave(&dwc->lock, flags); 1353 dep->flags |= DWC3_EP_WEDGE; 1354 1355 if (dep->number == 0 || dep->number == 1) 1356 ret = __dwc3_gadget_ep0_set_halt(ep, 1); 1357 else 1358 ret = __dwc3_gadget_ep_set_halt(dep, 1, false); 1359 spin_unlock_irqrestore(&dwc->lock, flags); 1360 1361 return ret; 1362 } 1363 1364 /* -------------------------------------------------------------------------- */ 1365 1366 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = { 1367 .bLength = USB_DT_ENDPOINT_SIZE, 1368 .bDescriptorType = USB_DT_ENDPOINT, 1369 .bmAttributes = USB_ENDPOINT_XFER_CONTROL, 1370 }; 1371 1372 static const struct usb_ep_ops dwc3_gadget_ep0_ops = { 1373 .enable = dwc3_gadget_ep0_enable, 1374 .disable = dwc3_gadget_ep0_disable, 1375 .alloc_request = dwc3_gadget_ep_alloc_request, 1376 .free_request = dwc3_gadget_ep_free_request, 1377 .queue = dwc3_gadget_ep0_queue, 1378 .dequeue = dwc3_gadget_ep_dequeue, 1379 .set_halt = dwc3_gadget_ep0_set_halt, 1380 .set_wedge = dwc3_gadget_ep_set_wedge, 1381 }; 1382 1383 static const struct usb_ep_ops dwc3_gadget_ep_ops = { 1384 .enable = dwc3_gadget_ep_enable, 1385 .disable = dwc3_gadget_ep_disable, 1386 .alloc_request = dwc3_gadget_ep_alloc_request, 1387 .free_request = dwc3_gadget_ep_free_request, 1388 .queue = dwc3_gadget_ep_queue, 1389 .dequeue = dwc3_gadget_ep_dequeue, 1390 .set_halt = dwc3_gadget_ep_set_halt, 1391 .set_wedge = dwc3_gadget_ep_set_wedge, 1392 }; 1393 1394 /* -------------------------------------------------------------------------- */ 1395 1396 static int dwc3_gadget_get_frame(struct usb_gadget *g) 1397 { 1398 struct dwc3 *dwc = gadget_to_dwc(g); 1399 u32 reg; 1400 1401 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1402 return DWC3_DSTS_SOFFN(reg); 1403 } 1404 1405 static int __dwc3_gadget_wakeup(struct dwc3 *dwc) 1406 { 1407 unsigned long timeout; 1408 1409 int ret; 1410 u32 reg; 1411 1412 u8 link_state; 1413 u8 speed; 1414 1415 /* 1416 * According to the Databook Remote wakeup request should 1417 * be issued only when the device is in early suspend state. 1418 * 1419 * We can check that via USB Link State bits in DSTS register. 1420 */ 1421 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1422 1423 speed = reg & DWC3_DSTS_CONNECTSPD; 1424 if ((speed == DWC3_DSTS_SUPERSPEED) || 1425 (speed == DWC3_DSTS_SUPERSPEED_PLUS)) { 1426 dwc3_trace(trace_dwc3_gadget, "no wakeup on SuperSpeed\n"); 1427 return -EINVAL; 1428 } 1429 1430 link_state = DWC3_DSTS_USBLNKST(reg); 1431 1432 switch (link_state) { 1433 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */ 1434 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */ 1435 break; 1436 default: 1437 dwc3_trace(trace_dwc3_gadget, 1438 "can't wakeup from '%s'\n", 1439 dwc3_gadget_link_string(link_state)); 1440 return -EINVAL; 1441 } 1442 1443 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV); 1444 if (ret < 0) { 1445 dev_err(dwc->dev, "failed to put link in Recovery\n"); 1446 return ret; 1447 } 1448 1449 /* Recent versions do this automatically */ 1450 if (dwc->revision < DWC3_REVISION_194A) { 1451 /* write zeroes to Link Change Request */ 1452 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 1453 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 1454 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 1455 } 1456 1457 /* poll until Link State changes to ON */ 1458 timeout = jiffies + msecs_to_jiffies(100); 1459 1460 while (!time_after(jiffies, timeout)) { 1461 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1462 1463 /* in HS, means ON */ 1464 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0) 1465 break; 1466 } 1467 1468 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) { 1469 dev_err(dwc->dev, "failed to send remote wakeup\n"); 1470 return -EINVAL; 1471 } 1472 1473 return 0; 1474 } 1475 1476 static int dwc3_gadget_wakeup(struct usb_gadget *g) 1477 { 1478 struct dwc3 *dwc = gadget_to_dwc(g); 1479 unsigned long flags; 1480 int ret; 1481 1482 spin_lock_irqsave(&dwc->lock, flags); 1483 ret = __dwc3_gadget_wakeup(dwc); 1484 spin_unlock_irqrestore(&dwc->lock, flags); 1485 1486 return ret; 1487 } 1488 1489 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g, 1490 int is_selfpowered) 1491 { 1492 struct dwc3 *dwc = gadget_to_dwc(g); 1493 unsigned long flags; 1494 1495 spin_lock_irqsave(&dwc->lock, flags); 1496 g->is_selfpowered = !!is_selfpowered; 1497 spin_unlock_irqrestore(&dwc->lock, flags); 1498 1499 return 0; 1500 } 1501 1502 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend) 1503 { 1504 u32 reg; 1505 u32 timeout = 500; 1506 1507 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 1508 if (is_on) { 1509 if (dwc->revision <= DWC3_REVISION_187A) { 1510 reg &= ~DWC3_DCTL_TRGTULST_MASK; 1511 reg |= DWC3_DCTL_TRGTULST_RX_DET; 1512 } 1513 1514 if (dwc->revision >= DWC3_REVISION_194A) 1515 reg &= ~DWC3_DCTL_KEEP_CONNECT; 1516 reg |= DWC3_DCTL_RUN_STOP; 1517 1518 if (dwc->has_hibernation) 1519 reg |= DWC3_DCTL_KEEP_CONNECT; 1520 1521 dwc->pullups_connected = true; 1522 } else { 1523 reg &= ~DWC3_DCTL_RUN_STOP; 1524 1525 if (dwc->has_hibernation && !suspend) 1526 reg &= ~DWC3_DCTL_KEEP_CONNECT; 1527 1528 dwc->pullups_connected = false; 1529 } 1530 1531 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 1532 1533 do { 1534 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1535 if (is_on) { 1536 if (!(reg & DWC3_DSTS_DEVCTRLHLT)) 1537 break; 1538 } else { 1539 if (reg & DWC3_DSTS_DEVCTRLHLT) 1540 break; 1541 } 1542 timeout--; 1543 if (!timeout) 1544 return -ETIMEDOUT; 1545 udelay(1); 1546 } while (1); 1547 1548 dwc3_trace(trace_dwc3_gadget, "gadget %s data soft-%s", 1549 dwc->gadget_driver 1550 ? dwc->gadget_driver->function : "no-function", 1551 is_on ? "connect" : "disconnect"); 1552 1553 return 0; 1554 } 1555 1556 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) 1557 { 1558 struct dwc3 *dwc = gadget_to_dwc(g); 1559 unsigned long flags; 1560 int ret; 1561 1562 is_on = !!is_on; 1563 1564 spin_lock_irqsave(&dwc->lock, flags); 1565 ret = dwc3_gadget_run_stop(dwc, is_on, false); 1566 spin_unlock_irqrestore(&dwc->lock, flags); 1567 1568 return ret; 1569 } 1570 1571 static void dwc3_gadget_enable_irq(struct dwc3 *dwc) 1572 { 1573 u32 reg; 1574 1575 /* Enable all but Start and End of Frame IRQs */ 1576 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN | 1577 DWC3_DEVTEN_EVNTOVERFLOWEN | 1578 DWC3_DEVTEN_CMDCMPLTEN | 1579 DWC3_DEVTEN_ERRTICERREN | 1580 DWC3_DEVTEN_WKUPEVTEN | 1581 DWC3_DEVTEN_ULSTCNGEN | 1582 DWC3_DEVTEN_CONNECTDONEEN | 1583 DWC3_DEVTEN_USBRSTEN | 1584 DWC3_DEVTEN_DISCONNEVTEN); 1585 1586 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); 1587 } 1588 1589 static void dwc3_gadget_disable_irq(struct dwc3 *dwc) 1590 { 1591 /* mask all interrupts */ 1592 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00); 1593 } 1594 1595 static irqreturn_t dwc3_interrupt(int irq, void *_dwc); 1596 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc); 1597 1598 static int dwc3_gadget_start(struct usb_gadget *g, 1599 struct usb_gadget_driver *driver) 1600 { 1601 struct dwc3 *dwc = gadget_to_dwc(g); 1602 struct dwc3_ep *dep; 1603 unsigned long flags; 1604 int ret = 0; 1605 int irq; 1606 u32 reg; 1607 1608 irq = platform_get_irq(to_platform_device(dwc->dev), 0); 1609 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt, 1610 IRQF_SHARED, "dwc3", dwc->ev_buf); 1611 if (ret) { 1612 dev_err(dwc->dev, "failed to request irq #%d --> %d\n", 1613 irq, ret); 1614 goto err0; 1615 } 1616 1617 spin_lock_irqsave(&dwc->lock, flags); 1618 1619 if (dwc->gadget_driver) { 1620 dev_err(dwc->dev, "%s is already bound to %s\n", 1621 dwc->gadget.name, 1622 dwc->gadget_driver->driver.name); 1623 ret = -EBUSY; 1624 goto err1; 1625 } 1626 1627 dwc->gadget_driver = driver; 1628 1629 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 1630 reg &= ~(DWC3_DCFG_SPEED_MASK); 1631 1632 /** 1633 * WORKAROUND: DWC3 revision < 2.20a have an issue 1634 * which would cause metastability state on Run/Stop 1635 * bit if we try to force the IP to USB2-only mode. 1636 * 1637 * Because of that, we cannot configure the IP to any 1638 * speed other than the SuperSpeed 1639 * 1640 * Refers to: 1641 * 1642 * STAR#9000525659: Clock Domain Crossing on DCTL in 1643 * USB 2.0 Mode 1644 */ 1645 if (dwc->revision < DWC3_REVISION_220A) { 1646 reg |= DWC3_DCFG_SUPERSPEED; 1647 } else { 1648 switch (dwc->maximum_speed) { 1649 case USB_SPEED_LOW: 1650 reg |= DWC3_DSTS_LOWSPEED; 1651 break; 1652 case USB_SPEED_FULL: 1653 reg |= DWC3_DSTS_FULLSPEED1; 1654 break; 1655 case USB_SPEED_HIGH: 1656 reg |= DWC3_DSTS_HIGHSPEED; 1657 break; 1658 case USB_SPEED_SUPER_PLUS: 1659 reg |= DWC3_DSTS_SUPERSPEED_PLUS; 1660 break; 1661 default: 1662 dev_err(dwc->dev, "invalid dwc->maximum_speed (%d)\n", 1663 dwc->maximum_speed); 1664 /* fall through */ 1665 case USB_SPEED_SUPER: 1666 reg |= DWC3_DCFG_SUPERSPEED; 1667 break; 1668 } 1669 } 1670 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 1671 1672 /* 1673 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP 1674 * field instead of letting dwc3 itself calculate that automatically. 1675 * 1676 * This way, we maximize the chances that we'll be able to get several 1677 * bursts of data without going through any sort of endpoint throttling. 1678 */ 1679 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG); 1680 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL; 1681 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg); 1682 1683 /* Start with SuperSpeed Default */ 1684 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 1685 1686 dep = dwc->eps[0]; 1687 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false, 1688 false); 1689 if (ret) { 1690 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 1691 goto err2; 1692 } 1693 1694 dep = dwc->eps[1]; 1695 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false, 1696 false); 1697 if (ret) { 1698 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 1699 goto err3; 1700 } 1701 1702 /* begin to receive SETUP packets */ 1703 dwc->ep0state = EP0_SETUP_PHASE; 1704 dwc3_ep0_out_start(dwc); 1705 1706 dwc3_gadget_enable_irq(dwc); 1707 1708 spin_unlock_irqrestore(&dwc->lock, flags); 1709 1710 return 0; 1711 1712 err3: 1713 __dwc3_gadget_ep_disable(dwc->eps[0]); 1714 1715 err2: 1716 dwc->gadget_driver = NULL; 1717 1718 err1: 1719 spin_unlock_irqrestore(&dwc->lock, flags); 1720 1721 free_irq(irq, dwc->ev_buf); 1722 1723 err0: 1724 return ret; 1725 } 1726 1727 static int dwc3_gadget_stop(struct usb_gadget *g) 1728 { 1729 struct dwc3 *dwc = gadget_to_dwc(g); 1730 unsigned long flags; 1731 int irq; 1732 1733 spin_lock_irqsave(&dwc->lock, flags); 1734 1735 dwc3_gadget_disable_irq(dwc); 1736 __dwc3_gadget_ep_disable(dwc->eps[0]); 1737 __dwc3_gadget_ep_disable(dwc->eps[1]); 1738 1739 dwc->gadget_driver = NULL; 1740 1741 spin_unlock_irqrestore(&dwc->lock, flags); 1742 1743 irq = platform_get_irq(to_platform_device(dwc->dev), 0); 1744 free_irq(irq, dwc->ev_buf); 1745 1746 return 0; 1747 } 1748 1749 static const struct usb_gadget_ops dwc3_gadget_ops = { 1750 .get_frame = dwc3_gadget_get_frame, 1751 .wakeup = dwc3_gadget_wakeup, 1752 .set_selfpowered = dwc3_gadget_set_selfpowered, 1753 .pullup = dwc3_gadget_pullup, 1754 .udc_start = dwc3_gadget_start, 1755 .udc_stop = dwc3_gadget_stop, 1756 }; 1757 1758 /* -------------------------------------------------------------------------- */ 1759 1760 static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc, 1761 u8 num, u32 direction) 1762 { 1763 struct dwc3_ep *dep; 1764 u8 i; 1765 1766 for (i = 0; i < num; i++) { 1767 u8 epnum = (i << 1) | (!!direction); 1768 1769 dep = kzalloc(sizeof(*dep), GFP_KERNEL); 1770 if (!dep) 1771 return -ENOMEM; 1772 1773 dep->dwc = dwc; 1774 dep->number = epnum; 1775 dep->direction = !!direction; 1776 dwc->eps[epnum] = dep; 1777 1778 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1, 1779 (epnum & 1) ? "in" : "out"); 1780 1781 dep->endpoint.name = dep->name; 1782 1783 dwc3_trace(trace_dwc3_gadget, "initializing %s", dep->name); 1784 1785 if (epnum == 0 || epnum == 1) { 1786 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); 1787 dep->endpoint.maxburst = 1; 1788 dep->endpoint.ops = &dwc3_gadget_ep0_ops; 1789 if (!epnum) 1790 dwc->gadget.ep0 = &dep->endpoint; 1791 } else { 1792 int ret; 1793 1794 usb_ep_set_maxpacket_limit(&dep->endpoint, 1024); 1795 dep->endpoint.max_streams = 15; 1796 dep->endpoint.ops = &dwc3_gadget_ep_ops; 1797 list_add_tail(&dep->endpoint.ep_list, 1798 &dwc->gadget.ep_list); 1799 1800 ret = dwc3_alloc_trb_pool(dep); 1801 if (ret) 1802 return ret; 1803 } 1804 1805 if (epnum == 0 || epnum == 1) { 1806 dep->endpoint.caps.type_control = true; 1807 } else { 1808 dep->endpoint.caps.type_iso = true; 1809 dep->endpoint.caps.type_bulk = true; 1810 dep->endpoint.caps.type_int = true; 1811 } 1812 1813 dep->endpoint.caps.dir_in = !!direction; 1814 dep->endpoint.caps.dir_out = !direction; 1815 1816 INIT_LIST_HEAD(&dep->pending_list); 1817 INIT_LIST_HEAD(&dep->started_list); 1818 } 1819 1820 return 0; 1821 } 1822 1823 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc) 1824 { 1825 int ret; 1826 1827 INIT_LIST_HEAD(&dwc->gadget.ep_list); 1828 1829 ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0); 1830 if (ret < 0) { 1831 dwc3_trace(trace_dwc3_gadget, 1832 "failed to allocate OUT endpoints"); 1833 return ret; 1834 } 1835 1836 ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1); 1837 if (ret < 0) { 1838 dwc3_trace(trace_dwc3_gadget, 1839 "failed to allocate IN endpoints"); 1840 return ret; 1841 } 1842 1843 return 0; 1844 } 1845 1846 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc) 1847 { 1848 struct dwc3_ep *dep; 1849 u8 epnum; 1850 1851 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 1852 dep = dwc->eps[epnum]; 1853 if (!dep) 1854 continue; 1855 /* 1856 * Physical endpoints 0 and 1 are special; they form the 1857 * bi-directional USB endpoint 0. 1858 * 1859 * For those two physical endpoints, we don't allocate a TRB 1860 * pool nor do we add them the endpoints list. Due to that, we 1861 * shouldn't do these two operations otherwise we would end up 1862 * with all sorts of bugs when removing dwc3.ko. 1863 */ 1864 if (epnum != 0 && epnum != 1) { 1865 dwc3_free_trb_pool(dep); 1866 list_del(&dep->endpoint.ep_list); 1867 } 1868 1869 kfree(dep); 1870 } 1871 } 1872 1873 /* -------------------------------------------------------------------------- */ 1874 1875 static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep, 1876 struct dwc3_request *req, struct dwc3_trb *trb, 1877 const struct dwc3_event_depevt *event, int status) 1878 { 1879 unsigned int count; 1880 unsigned int s_pkt = 0; 1881 unsigned int trb_status; 1882 1883 trace_dwc3_complete_trb(dep, trb); 1884 1885 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) 1886 /* 1887 * We continue despite the error. There is not much we 1888 * can do. If we don't clean it up we loop forever. If 1889 * we skip the TRB then it gets overwritten after a 1890 * while since we use them in a ring buffer. A BUG() 1891 * would help. Lets hope that if this occurs, someone 1892 * fixes the root cause instead of looking away :) 1893 */ 1894 dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n", 1895 dep->name, trb); 1896 count = trb->size & DWC3_TRB_SIZE_MASK; 1897 1898 if (dep->direction) { 1899 if (count) { 1900 trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size); 1901 if (trb_status == DWC3_TRBSTS_MISSED_ISOC) { 1902 dwc3_trace(trace_dwc3_gadget, 1903 "%s: incomplete IN transfer\n", 1904 dep->name); 1905 /* 1906 * If missed isoc occurred and there is 1907 * no request queued then issue END 1908 * TRANSFER, so that core generates 1909 * next xfernotready and we will issue 1910 * a fresh START TRANSFER. 1911 * If there are still queued request 1912 * then wait, do not issue either END 1913 * or UPDATE TRANSFER, just attach next 1914 * request in pending_list during 1915 * giveback.If any future queued request 1916 * is successfully transferred then we 1917 * will issue UPDATE TRANSFER for all 1918 * request in the pending_list. 1919 */ 1920 dep->flags |= DWC3_EP_MISSED_ISOC; 1921 } else { 1922 dev_err(dwc->dev, "incomplete IN transfer %s\n", 1923 dep->name); 1924 status = -ECONNRESET; 1925 } 1926 } else { 1927 dep->flags &= ~DWC3_EP_MISSED_ISOC; 1928 } 1929 } else { 1930 if (count && (event->status & DEPEVT_STATUS_SHORT)) 1931 s_pkt = 1; 1932 } 1933 1934 /* 1935 * We assume here we will always receive the entire data block 1936 * which we should receive. Meaning, if we program RX to 1937 * receive 4K but we receive only 2K, we assume that's all we 1938 * should receive and we simply bounce the request back to the 1939 * gadget driver for further processing. 1940 */ 1941 req->request.actual += req->request.length - count; 1942 if (s_pkt) 1943 return 1; 1944 if ((event->status & DEPEVT_STATUS_LST) && 1945 (trb->ctrl & (DWC3_TRB_CTRL_LST | 1946 DWC3_TRB_CTRL_HWO))) 1947 return 1; 1948 if ((event->status & DEPEVT_STATUS_IOC) && 1949 (trb->ctrl & DWC3_TRB_CTRL_IOC)) 1950 return 1; 1951 return 0; 1952 } 1953 1954 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep, 1955 const struct dwc3_event_depevt *event, int status) 1956 { 1957 struct dwc3_request *req; 1958 struct dwc3_trb *trb; 1959 unsigned int slot; 1960 unsigned int i; 1961 int ret; 1962 1963 do { 1964 req = next_request(&dep->started_list); 1965 if (WARN_ON_ONCE(!req)) 1966 return 1; 1967 1968 i = 0; 1969 do { 1970 slot = req->first_trb_index + i; 1971 if (slot == DWC3_TRB_NUM - 1) 1972 slot++; 1973 slot %= DWC3_TRB_NUM; 1974 trb = &dep->trb_pool[slot]; 1975 1976 ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb, 1977 event, status); 1978 if (ret) 1979 break; 1980 } while (++i < req->request.num_mapped_sgs); 1981 1982 dwc3_gadget_giveback(dep, req, status); 1983 1984 if (ret) 1985 break; 1986 } while (1); 1987 1988 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && 1989 list_empty(&dep->started_list)) { 1990 if (list_empty(&dep->pending_list)) { 1991 /* 1992 * If there is no entry in request list then do 1993 * not issue END TRANSFER now. Just set PENDING 1994 * flag, so that END TRANSFER is issued when an 1995 * entry is added into request list. 1996 */ 1997 dep->flags = DWC3_EP_PENDING_REQUEST; 1998 } else { 1999 dwc3_stop_active_transfer(dwc, dep->number, true); 2000 dep->flags = DWC3_EP_ENABLED; 2001 } 2002 return 1; 2003 } 2004 2005 return 1; 2006 } 2007 2008 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc, 2009 struct dwc3_ep *dep, const struct dwc3_event_depevt *event) 2010 { 2011 unsigned status = 0; 2012 int clean_busy; 2013 u32 is_xfer_complete; 2014 2015 is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE); 2016 2017 if (event->status & DEPEVT_STATUS_BUSERR) 2018 status = -ECONNRESET; 2019 2020 clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status); 2021 if (clean_busy && (is_xfer_complete || 2022 usb_endpoint_xfer_isoc(dep->endpoint.desc))) 2023 dep->flags &= ~DWC3_EP_BUSY; 2024 2025 /* 2026 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. 2027 * See dwc3_gadget_linksts_change_interrupt() for 1st half. 2028 */ 2029 if (dwc->revision < DWC3_REVISION_183A) { 2030 u32 reg; 2031 int i; 2032 2033 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) { 2034 dep = dwc->eps[i]; 2035 2036 if (!(dep->flags & DWC3_EP_ENABLED)) 2037 continue; 2038 2039 if (!list_empty(&dep->started_list)) 2040 return; 2041 } 2042 2043 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2044 reg |= dwc->u1u2; 2045 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2046 2047 dwc->u1u2 = 0; 2048 } 2049 2050 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 2051 int ret; 2052 2053 ret = __dwc3_gadget_kick_transfer(dep, 0, is_xfer_complete); 2054 if (!ret || ret == -EBUSY) 2055 return; 2056 } 2057 } 2058 2059 static void dwc3_endpoint_interrupt(struct dwc3 *dwc, 2060 const struct dwc3_event_depevt *event) 2061 { 2062 struct dwc3_ep *dep; 2063 u8 epnum = event->endpoint_number; 2064 2065 dep = dwc->eps[epnum]; 2066 2067 if (!(dep->flags & DWC3_EP_ENABLED)) 2068 return; 2069 2070 if (epnum == 0 || epnum == 1) { 2071 dwc3_ep0_interrupt(dwc, event); 2072 return; 2073 } 2074 2075 switch (event->endpoint_event) { 2076 case DWC3_DEPEVT_XFERCOMPLETE: 2077 dep->resource_index = 0; 2078 2079 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 2080 dwc3_trace(trace_dwc3_gadget, 2081 "%s is an Isochronous endpoint\n", 2082 dep->name); 2083 return; 2084 } 2085 2086 dwc3_endpoint_transfer_complete(dwc, dep, event); 2087 break; 2088 case DWC3_DEPEVT_XFERINPROGRESS: 2089 dwc3_endpoint_transfer_complete(dwc, dep, event); 2090 break; 2091 case DWC3_DEPEVT_XFERNOTREADY: 2092 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 2093 dwc3_gadget_start_isoc(dwc, dep, event); 2094 } else { 2095 int active; 2096 int ret; 2097 2098 active = event->status & DEPEVT_STATUS_TRANSFER_ACTIVE; 2099 2100 dwc3_trace(trace_dwc3_gadget, "%s: reason %s", 2101 dep->name, active ? "Transfer Active" 2102 : "Transfer Not Active"); 2103 2104 ret = __dwc3_gadget_kick_transfer(dep, 0, !active); 2105 if (!ret || ret == -EBUSY) 2106 return; 2107 2108 dwc3_trace(trace_dwc3_gadget, 2109 "%s: failed to kick transfers\n", 2110 dep->name); 2111 } 2112 2113 break; 2114 case DWC3_DEPEVT_STREAMEVT: 2115 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) { 2116 dev_err(dwc->dev, "Stream event for non-Bulk %s\n", 2117 dep->name); 2118 return; 2119 } 2120 2121 switch (event->status) { 2122 case DEPEVT_STREAMEVT_FOUND: 2123 dwc3_trace(trace_dwc3_gadget, 2124 "Stream %d found and started", 2125 event->parameters); 2126 2127 break; 2128 case DEPEVT_STREAMEVT_NOTFOUND: 2129 /* FALLTHROUGH */ 2130 default: 2131 dwc3_trace(trace_dwc3_gadget, 2132 "unable to find suitable stream\n"); 2133 } 2134 break; 2135 case DWC3_DEPEVT_RXTXFIFOEVT: 2136 dwc3_trace(trace_dwc3_gadget, "%s FIFO Overrun\n", dep->name); 2137 break; 2138 case DWC3_DEPEVT_EPCMDCMPLT: 2139 dwc3_trace(trace_dwc3_gadget, "Endpoint Command Complete"); 2140 break; 2141 } 2142 } 2143 2144 static void dwc3_disconnect_gadget(struct dwc3 *dwc) 2145 { 2146 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) { 2147 spin_unlock(&dwc->lock); 2148 dwc->gadget_driver->disconnect(&dwc->gadget); 2149 spin_lock(&dwc->lock); 2150 } 2151 } 2152 2153 static void dwc3_suspend_gadget(struct dwc3 *dwc) 2154 { 2155 if (dwc->gadget_driver && dwc->gadget_driver->suspend) { 2156 spin_unlock(&dwc->lock); 2157 dwc->gadget_driver->suspend(&dwc->gadget); 2158 spin_lock(&dwc->lock); 2159 } 2160 } 2161 2162 static void dwc3_resume_gadget(struct dwc3 *dwc) 2163 { 2164 if (dwc->gadget_driver && dwc->gadget_driver->resume) { 2165 spin_unlock(&dwc->lock); 2166 dwc->gadget_driver->resume(&dwc->gadget); 2167 spin_lock(&dwc->lock); 2168 } 2169 } 2170 2171 static void dwc3_reset_gadget(struct dwc3 *dwc) 2172 { 2173 if (!dwc->gadget_driver) 2174 return; 2175 2176 if (dwc->gadget.speed != USB_SPEED_UNKNOWN) { 2177 spin_unlock(&dwc->lock); 2178 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver); 2179 spin_lock(&dwc->lock); 2180 } 2181 } 2182 2183 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force) 2184 { 2185 struct dwc3_ep *dep; 2186 struct dwc3_gadget_ep_cmd_params params; 2187 u32 cmd; 2188 int ret; 2189 2190 dep = dwc->eps[epnum]; 2191 2192 if (!dep->resource_index) 2193 return; 2194 2195 /* 2196 * NOTICE: We are violating what the Databook says about the 2197 * EndTransfer command. Ideally we would _always_ wait for the 2198 * EndTransfer Command Completion IRQ, but that's causing too 2199 * much trouble synchronizing between us and gadget driver. 2200 * 2201 * We have discussed this with the IP Provider and it was 2202 * suggested to giveback all requests here, but give HW some 2203 * extra time to synchronize with the interconnect. We're using 2204 * an arbitrary 100us delay for that. 2205 * 2206 * Note also that a similar handling was tested by Synopsys 2207 * (thanks a lot Paul) and nothing bad has come out of it. 2208 * In short, what we're doing is: 2209 * 2210 * - Issue EndTransfer WITH CMDIOC bit set 2211 * - Wait 100us 2212 */ 2213 2214 cmd = DWC3_DEPCMD_ENDTRANSFER; 2215 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0; 2216 cmd |= DWC3_DEPCMD_CMDIOC; 2217 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); 2218 memset(¶ms, 0, sizeof(params)); 2219 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, ¶ms); 2220 WARN_ON_ONCE(ret); 2221 dep->resource_index = 0; 2222 dep->flags &= ~DWC3_EP_BUSY; 2223 udelay(100); 2224 } 2225 2226 static void dwc3_stop_active_transfers(struct dwc3 *dwc) 2227 { 2228 u32 epnum; 2229 2230 for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 2231 struct dwc3_ep *dep; 2232 2233 dep = dwc->eps[epnum]; 2234 if (!dep) 2235 continue; 2236 2237 if (!(dep->flags & DWC3_EP_ENABLED)) 2238 continue; 2239 2240 dwc3_remove_requests(dwc, dep); 2241 } 2242 } 2243 2244 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc) 2245 { 2246 u32 epnum; 2247 2248 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 2249 struct dwc3_ep *dep; 2250 struct dwc3_gadget_ep_cmd_params params; 2251 int ret; 2252 2253 dep = dwc->eps[epnum]; 2254 if (!dep) 2255 continue; 2256 2257 if (!(dep->flags & DWC3_EP_STALL)) 2258 continue; 2259 2260 dep->flags &= ~DWC3_EP_STALL; 2261 2262 memset(¶ms, 0, sizeof(params)); 2263 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, 2264 DWC3_DEPCMD_CLEARSTALL, ¶ms); 2265 WARN_ON_ONCE(ret); 2266 } 2267 } 2268 2269 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc) 2270 { 2271 int reg; 2272 2273 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2274 reg &= ~DWC3_DCTL_INITU1ENA; 2275 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2276 2277 reg &= ~DWC3_DCTL_INITU2ENA; 2278 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2279 2280 dwc3_disconnect_gadget(dwc); 2281 2282 dwc->gadget.speed = USB_SPEED_UNKNOWN; 2283 dwc->setup_packet_pending = false; 2284 usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED); 2285 } 2286 2287 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) 2288 { 2289 u32 reg; 2290 2291 /* 2292 * WORKAROUND: DWC3 revisions <1.88a have an issue which 2293 * would cause a missing Disconnect Event if there's a 2294 * pending Setup Packet in the FIFO. 2295 * 2296 * There's no suggested workaround on the official Bug 2297 * report, which states that "unless the driver/application 2298 * is doing any special handling of a disconnect event, 2299 * there is no functional issue". 2300 * 2301 * Unfortunately, it turns out that we _do_ some special 2302 * handling of a disconnect event, namely complete all 2303 * pending transfers, notify gadget driver of the 2304 * disconnection, and so on. 2305 * 2306 * Our suggested workaround is to follow the Disconnect 2307 * Event steps here, instead, based on a setup_packet_pending 2308 * flag. Such flag gets set whenever we have a SETUP_PENDING 2309 * status for EP0 TRBs and gets cleared on XferComplete for the 2310 * same endpoint. 2311 * 2312 * Refers to: 2313 * 2314 * STAR#9000466709: RTL: Device : Disconnect event not 2315 * generated if setup packet pending in FIFO 2316 */ 2317 if (dwc->revision < DWC3_REVISION_188A) { 2318 if (dwc->setup_packet_pending) 2319 dwc3_gadget_disconnect_interrupt(dwc); 2320 } 2321 2322 dwc3_reset_gadget(dwc); 2323 2324 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2325 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 2326 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2327 dwc->test_mode = false; 2328 2329 dwc3_stop_active_transfers(dwc); 2330 dwc3_clear_stall_all_ep(dwc); 2331 2332 /* Reset device address to zero */ 2333 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2334 reg &= ~(DWC3_DCFG_DEVADDR_MASK); 2335 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2336 } 2337 2338 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed) 2339 { 2340 u32 reg; 2341 u32 usb30_clock = DWC3_GCTL_CLK_BUS; 2342 2343 /* 2344 * We change the clock only at SS but I dunno why I would want to do 2345 * this. Maybe it becomes part of the power saving plan. 2346 */ 2347 2348 if ((speed != DWC3_DSTS_SUPERSPEED) && 2349 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) 2350 return; 2351 2352 /* 2353 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed 2354 * each time on Connect Done. 2355 */ 2356 if (!usb30_clock) 2357 return; 2358 2359 reg = dwc3_readl(dwc->regs, DWC3_GCTL); 2360 reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock); 2361 dwc3_writel(dwc->regs, DWC3_GCTL, reg); 2362 } 2363 2364 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) 2365 { 2366 struct dwc3_ep *dep; 2367 int ret; 2368 u32 reg; 2369 u8 speed; 2370 2371 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2372 speed = reg & DWC3_DSTS_CONNECTSPD; 2373 dwc->speed = speed; 2374 2375 dwc3_update_ram_clk_sel(dwc, speed); 2376 2377 switch (speed) { 2378 case DWC3_DCFG_SUPERSPEED_PLUS: 2379 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 2380 dwc->gadget.ep0->maxpacket = 512; 2381 dwc->gadget.speed = USB_SPEED_SUPER_PLUS; 2382 break; 2383 case DWC3_DCFG_SUPERSPEED: 2384 /* 2385 * WORKAROUND: DWC3 revisions <1.90a have an issue which 2386 * would cause a missing USB3 Reset event. 2387 * 2388 * In such situations, we should force a USB3 Reset 2389 * event by calling our dwc3_gadget_reset_interrupt() 2390 * routine. 2391 * 2392 * Refers to: 2393 * 2394 * STAR#9000483510: RTL: SS : USB3 reset event may 2395 * not be generated always when the link enters poll 2396 */ 2397 if (dwc->revision < DWC3_REVISION_190A) 2398 dwc3_gadget_reset_interrupt(dwc); 2399 2400 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 2401 dwc->gadget.ep0->maxpacket = 512; 2402 dwc->gadget.speed = USB_SPEED_SUPER; 2403 break; 2404 case DWC3_DCFG_HIGHSPEED: 2405 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 2406 dwc->gadget.ep0->maxpacket = 64; 2407 dwc->gadget.speed = USB_SPEED_HIGH; 2408 break; 2409 case DWC3_DCFG_FULLSPEED2: 2410 case DWC3_DCFG_FULLSPEED1: 2411 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 2412 dwc->gadget.ep0->maxpacket = 64; 2413 dwc->gadget.speed = USB_SPEED_FULL; 2414 break; 2415 case DWC3_DCFG_LOWSPEED: 2416 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8); 2417 dwc->gadget.ep0->maxpacket = 8; 2418 dwc->gadget.speed = USB_SPEED_LOW; 2419 break; 2420 } 2421 2422 /* Enable USB2 LPM Capability */ 2423 2424 if ((dwc->revision > DWC3_REVISION_194A) && 2425 (speed != DWC3_DCFG_SUPERSPEED) && 2426 (speed != DWC3_DCFG_SUPERSPEED_PLUS)) { 2427 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2428 reg |= DWC3_DCFG_LPM_CAP; 2429 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2430 2431 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2432 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN); 2433 2434 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold); 2435 2436 /* 2437 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and 2438 * DCFG.LPMCap is set, core responses with an ACK and the 2439 * BESL value in the LPM token is less than or equal to LPM 2440 * NYET threshold. 2441 */ 2442 WARN_ONCE(dwc->revision < DWC3_REVISION_240A 2443 && dwc->has_lpm_erratum, 2444 "LPM Erratum not available on dwc3 revisisions < 2.40a\n"); 2445 2446 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A) 2447 reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold); 2448 2449 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2450 } else { 2451 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2452 reg &= ~DWC3_DCTL_HIRD_THRES_MASK; 2453 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2454 } 2455 2456 dep = dwc->eps[0]; 2457 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true, 2458 false); 2459 if (ret) { 2460 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2461 return; 2462 } 2463 2464 dep = dwc->eps[1]; 2465 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true, 2466 false); 2467 if (ret) { 2468 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2469 return; 2470 } 2471 2472 /* 2473 * Configure PHY via GUSB3PIPECTLn if required. 2474 * 2475 * Update GTXFIFOSIZn 2476 * 2477 * In both cases reset values should be sufficient. 2478 */ 2479 } 2480 2481 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc) 2482 { 2483 /* 2484 * TODO take core out of low power mode when that's 2485 * implemented. 2486 */ 2487 2488 if (dwc->gadget_driver && dwc->gadget_driver->resume) { 2489 spin_unlock(&dwc->lock); 2490 dwc->gadget_driver->resume(&dwc->gadget); 2491 spin_lock(&dwc->lock); 2492 } 2493 } 2494 2495 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc, 2496 unsigned int evtinfo) 2497 { 2498 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 2499 unsigned int pwropt; 2500 2501 /* 2502 * WORKAROUND: DWC3 < 2.50a have an issue when configured without 2503 * Hibernation mode enabled which would show up when device detects 2504 * host-initiated U3 exit. 2505 * 2506 * In that case, device will generate a Link State Change Interrupt 2507 * from U3 to RESUME which is only necessary if Hibernation is 2508 * configured in. 2509 * 2510 * There are no functional changes due to such spurious event and we 2511 * just need to ignore it. 2512 * 2513 * Refers to: 2514 * 2515 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation 2516 * operational mode 2517 */ 2518 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1); 2519 if ((dwc->revision < DWC3_REVISION_250A) && 2520 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) { 2521 if ((dwc->link_state == DWC3_LINK_STATE_U3) && 2522 (next == DWC3_LINK_STATE_RESUME)) { 2523 dwc3_trace(trace_dwc3_gadget, 2524 "ignoring transition U3 -> Resume"); 2525 return; 2526 } 2527 } 2528 2529 /* 2530 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending 2531 * on the link partner, the USB session might do multiple entry/exit 2532 * of low power states before a transfer takes place. 2533 * 2534 * Due to this problem, we might experience lower throughput. The 2535 * suggested workaround is to disable DCTL[12:9] bits if we're 2536 * transitioning from U1/U2 to U0 and enable those bits again 2537 * after a transfer completes and there are no pending transfers 2538 * on any of the enabled endpoints. 2539 * 2540 * This is the first half of that workaround. 2541 * 2542 * Refers to: 2543 * 2544 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us 2545 * core send LGO_Ux entering U0 2546 */ 2547 if (dwc->revision < DWC3_REVISION_183A) { 2548 if (next == DWC3_LINK_STATE_U0) { 2549 u32 u1u2; 2550 u32 reg; 2551 2552 switch (dwc->link_state) { 2553 case DWC3_LINK_STATE_U1: 2554 case DWC3_LINK_STATE_U2: 2555 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2556 u1u2 = reg & (DWC3_DCTL_INITU2ENA 2557 | DWC3_DCTL_ACCEPTU2ENA 2558 | DWC3_DCTL_INITU1ENA 2559 | DWC3_DCTL_ACCEPTU1ENA); 2560 2561 if (!dwc->u1u2) 2562 dwc->u1u2 = reg & u1u2; 2563 2564 reg &= ~u1u2; 2565 2566 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2567 break; 2568 default: 2569 /* do nothing */ 2570 break; 2571 } 2572 } 2573 } 2574 2575 switch (next) { 2576 case DWC3_LINK_STATE_U1: 2577 if (dwc->speed == USB_SPEED_SUPER) 2578 dwc3_suspend_gadget(dwc); 2579 break; 2580 case DWC3_LINK_STATE_U2: 2581 case DWC3_LINK_STATE_U3: 2582 dwc3_suspend_gadget(dwc); 2583 break; 2584 case DWC3_LINK_STATE_RESUME: 2585 dwc3_resume_gadget(dwc); 2586 break; 2587 default: 2588 /* do nothing */ 2589 break; 2590 } 2591 2592 dwc->link_state = next; 2593 } 2594 2595 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc, 2596 unsigned int evtinfo) 2597 { 2598 unsigned int is_ss = evtinfo & BIT(4); 2599 2600 /** 2601 * WORKAROUND: DWC3 revison 2.20a with hibernation support 2602 * have a known issue which can cause USB CV TD.9.23 to fail 2603 * randomly. 2604 * 2605 * Because of this issue, core could generate bogus hibernation 2606 * events which SW needs to ignore. 2607 * 2608 * Refers to: 2609 * 2610 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0 2611 * Device Fallback from SuperSpeed 2612 */ 2613 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER)) 2614 return; 2615 2616 /* enter hibernation here */ 2617 } 2618 2619 static void dwc3_gadget_interrupt(struct dwc3 *dwc, 2620 const struct dwc3_event_devt *event) 2621 { 2622 switch (event->type) { 2623 case DWC3_DEVICE_EVENT_DISCONNECT: 2624 dwc3_gadget_disconnect_interrupt(dwc); 2625 break; 2626 case DWC3_DEVICE_EVENT_RESET: 2627 dwc3_gadget_reset_interrupt(dwc); 2628 break; 2629 case DWC3_DEVICE_EVENT_CONNECT_DONE: 2630 dwc3_gadget_conndone_interrupt(dwc); 2631 break; 2632 case DWC3_DEVICE_EVENT_WAKEUP: 2633 dwc3_gadget_wakeup_interrupt(dwc); 2634 break; 2635 case DWC3_DEVICE_EVENT_HIBER_REQ: 2636 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation, 2637 "unexpected hibernation event\n")) 2638 break; 2639 2640 dwc3_gadget_hibernation_interrupt(dwc, event->event_info); 2641 break; 2642 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE: 2643 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info); 2644 break; 2645 case DWC3_DEVICE_EVENT_EOPF: 2646 dwc3_trace(trace_dwc3_gadget, "End of Periodic Frame"); 2647 break; 2648 case DWC3_DEVICE_EVENT_SOF: 2649 dwc3_trace(trace_dwc3_gadget, "Start of Periodic Frame"); 2650 break; 2651 case DWC3_DEVICE_EVENT_ERRATIC_ERROR: 2652 dwc3_trace(trace_dwc3_gadget, "Erratic Error"); 2653 break; 2654 case DWC3_DEVICE_EVENT_CMD_CMPL: 2655 dwc3_trace(trace_dwc3_gadget, "Command Complete"); 2656 break; 2657 case DWC3_DEVICE_EVENT_OVERFLOW: 2658 dwc3_trace(trace_dwc3_gadget, "Overflow"); 2659 break; 2660 default: 2661 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type); 2662 } 2663 } 2664 2665 static void dwc3_process_event_entry(struct dwc3 *dwc, 2666 const union dwc3_event *event) 2667 { 2668 trace_dwc3_event(event->raw); 2669 2670 /* Endpoint IRQ, handle it and return early */ 2671 if (event->type.is_devspec == 0) { 2672 /* depevt */ 2673 return dwc3_endpoint_interrupt(dwc, &event->depevt); 2674 } 2675 2676 switch (event->type.type) { 2677 case DWC3_EVENT_TYPE_DEV: 2678 dwc3_gadget_interrupt(dwc, &event->devt); 2679 break; 2680 /* REVISIT what to do with Carkit and I2C events ? */ 2681 default: 2682 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw); 2683 } 2684 } 2685 2686 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt) 2687 { 2688 struct dwc3 *dwc = evt->dwc; 2689 irqreturn_t ret = IRQ_NONE; 2690 int left; 2691 u32 reg; 2692 2693 left = evt->count; 2694 2695 if (!(evt->flags & DWC3_EVENT_PENDING)) 2696 return IRQ_NONE; 2697 2698 while (left > 0) { 2699 union dwc3_event event; 2700 2701 event.raw = *(u32 *) (evt->buf + evt->lpos); 2702 2703 dwc3_process_event_entry(dwc, &event); 2704 2705 /* 2706 * FIXME we wrap around correctly to the next entry as 2707 * almost all entries are 4 bytes in size. There is one 2708 * entry which has 12 bytes which is a regular entry 2709 * followed by 8 bytes data. ATM I don't know how 2710 * things are organized if we get next to the a 2711 * boundary so I worry about that once we try to handle 2712 * that. 2713 */ 2714 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE; 2715 left -= 4; 2716 2717 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), 4); 2718 } 2719 2720 evt->count = 0; 2721 evt->flags &= ~DWC3_EVENT_PENDING; 2722 ret = IRQ_HANDLED; 2723 2724 /* Unmask interrupt */ 2725 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0)); 2726 reg &= ~DWC3_GEVNTSIZ_INTMASK; 2727 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg); 2728 2729 return ret; 2730 } 2731 2732 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt) 2733 { 2734 struct dwc3_event_buffer *evt = _evt; 2735 struct dwc3 *dwc = evt->dwc; 2736 unsigned long flags; 2737 irqreturn_t ret = IRQ_NONE; 2738 2739 spin_lock_irqsave(&dwc->lock, flags); 2740 ret = dwc3_process_event_buf(evt); 2741 spin_unlock_irqrestore(&dwc->lock, flags); 2742 2743 return ret; 2744 } 2745 2746 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt) 2747 { 2748 struct dwc3 *dwc = evt->dwc; 2749 u32 count; 2750 u32 reg; 2751 2752 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); 2753 count &= DWC3_GEVNTCOUNT_MASK; 2754 if (!count) 2755 return IRQ_NONE; 2756 2757 evt->count = count; 2758 evt->flags |= DWC3_EVENT_PENDING; 2759 2760 /* Mask interrupt */ 2761 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0)); 2762 reg |= DWC3_GEVNTSIZ_INTMASK; 2763 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg); 2764 2765 return IRQ_WAKE_THREAD; 2766 } 2767 2768 static irqreturn_t dwc3_interrupt(int irq, void *_evt) 2769 { 2770 struct dwc3_event_buffer *evt = _evt; 2771 2772 return dwc3_check_event_buf(evt); 2773 } 2774 2775 /** 2776 * dwc3_gadget_init - Initializes gadget related registers 2777 * @dwc: pointer to our controller context structure 2778 * 2779 * Returns 0 on success otherwise negative errno. 2780 */ 2781 int dwc3_gadget_init(struct dwc3 *dwc) 2782 { 2783 int ret; 2784 2785 dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req), 2786 &dwc->ctrl_req_addr, GFP_KERNEL); 2787 if (!dwc->ctrl_req) { 2788 dev_err(dwc->dev, "failed to allocate ctrl request\n"); 2789 ret = -ENOMEM; 2790 goto err0; 2791 } 2792 2793 dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2, 2794 &dwc->ep0_trb_addr, GFP_KERNEL); 2795 if (!dwc->ep0_trb) { 2796 dev_err(dwc->dev, "failed to allocate ep0 trb\n"); 2797 ret = -ENOMEM; 2798 goto err1; 2799 } 2800 2801 dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL); 2802 if (!dwc->setup_buf) { 2803 ret = -ENOMEM; 2804 goto err2; 2805 } 2806 2807 dwc->ep0_bounce = dma_alloc_coherent(dwc->dev, 2808 DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr, 2809 GFP_KERNEL); 2810 if (!dwc->ep0_bounce) { 2811 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n"); 2812 ret = -ENOMEM; 2813 goto err3; 2814 } 2815 2816 dwc->zlp_buf = kzalloc(DWC3_ZLP_BUF_SIZE, GFP_KERNEL); 2817 if (!dwc->zlp_buf) { 2818 ret = -ENOMEM; 2819 goto err4; 2820 } 2821 2822 dwc->gadget.ops = &dwc3_gadget_ops; 2823 dwc->gadget.speed = USB_SPEED_UNKNOWN; 2824 dwc->gadget.sg_supported = true; 2825 dwc->gadget.name = "dwc3-gadget"; 2826 dwc->gadget.is_otg = dwc->dr_mode == USB_DR_MODE_OTG; 2827 2828 /* 2829 * FIXME We might be setting max_speed to <SUPER, however versions 2830 * <2.20a of dwc3 have an issue with metastability (documented 2831 * elsewhere in this driver) which tells us we can't set max speed to 2832 * anything lower than SUPER. 2833 * 2834 * Because gadget.max_speed is only used by composite.c and function 2835 * drivers (i.e. it won't go into dwc3's registers) we are allowing this 2836 * to happen so we avoid sending SuperSpeed Capability descriptor 2837 * together with our BOS descriptor as that could confuse host into 2838 * thinking we can handle super speed. 2839 * 2840 * Note that, in fact, we won't even support GetBOS requests when speed 2841 * is less than super speed because we don't have means, yet, to tell 2842 * composite.c that we are USB 2.0 + LPM ECN. 2843 */ 2844 if (dwc->revision < DWC3_REVISION_220A) 2845 dwc3_trace(trace_dwc3_gadget, 2846 "Changing max_speed on rev %08x\n", 2847 dwc->revision); 2848 2849 dwc->gadget.max_speed = dwc->maximum_speed; 2850 2851 /* 2852 * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize 2853 * on ep out. 2854 */ 2855 dwc->gadget.quirk_ep_out_aligned_size = true; 2856 2857 /* 2858 * REVISIT: Here we should clear all pending IRQs to be 2859 * sure we're starting from a well known location. 2860 */ 2861 2862 ret = dwc3_gadget_init_endpoints(dwc); 2863 if (ret) 2864 goto err5; 2865 2866 ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget); 2867 if (ret) { 2868 dev_err(dwc->dev, "failed to register udc\n"); 2869 goto err5; 2870 } 2871 2872 return 0; 2873 2874 err5: 2875 kfree(dwc->zlp_buf); 2876 2877 err4: 2878 dwc3_gadget_free_endpoints(dwc); 2879 dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE, 2880 dwc->ep0_bounce, dwc->ep0_bounce_addr); 2881 2882 err3: 2883 kfree(dwc->setup_buf); 2884 2885 err2: 2886 dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb), 2887 dwc->ep0_trb, dwc->ep0_trb_addr); 2888 2889 err1: 2890 dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req), 2891 dwc->ctrl_req, dwc->ctrl_req_addr); 2892 2893 err0: 2894 return ret; 2895 } 2896 2897 /* -------------------------------------------------------------------------- */ 2898 2899 void dwc3_gadget_exit(struct dwc3 *dwc) 2900 { 2901 usb_del_gadget_udc(&dwc->gadget); 2902 2903 dwc3_gadget_free_endpoints(dwc); 2904 2905 dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE, 2906 dwc->ep0_bounce, dwc->ep0_bounce_addr); 2907 2908 kfree(dwc->setup_buf); 2909 kfree(dwc->zlp_buf); 2910 2911 dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb), 2912 dwc->ep0_trb, dwc->ep0_trb_addr); 2913 2914 dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req), 2915 dwc->ctrl_req, dwc->ctrl_req_addr); 2916 } 2917 2918 int dwc3_gadget_suspend(struct dwc3 *dwc) 2919 { 2920 if (!dwc->gadget_driver) 2921 return 0; 2922 2923 if (dwc->pullups_connected) { 2924 dwc3_gadget_disable_irq(dwc); 2925 dwc3_gadget_run_stop(dwc, true, true); 2926 } 2927 2928 __dwc3_gadget_ep_disable(dwc->eps[0]); 2929 __dwc3_gadget_ep_disable(dwc->eps[1]); 2930 2931 dwc->dcfg = dwc3_readl(dwc->regs, DWC3_DCFG); 2932 2933 return 0; 2934 } 2935 2936 int dwc3_gadget_resume(struct dwc3 *dwc) 2937 { 2938 struct dwc3_ep *dep; 2939 int ret; 2940 2941 if (!dwc->gadget_driver) 2942 return 0; 2943 2944 /* Start with SuperSpeed Default */ 2945 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 2946 2947 dep = dwc->eps[0]; 2948 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false, 2949 false); 2950 if (ret) 2951 goto err0; 2952 2953 dep = dwc->eps[1]; 2954 ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false, 2955 false); 2956 if (ret) 2957 goto err1; 2958 2959 /* begin to receive SETUP packets */ 2960 dwc->ep0state = EP0_SETUP_PHASE; 2961 dwc3_ep0_out_start(dwc); 2962 2963 dwc3_writel(dwc->regs, DWC3_DCFG, dwc->dcfg); 2964 2965 if (dwc->pullups_connected) { 2966 dwc3_gadget_enable_irq(dwc); 2967 dwc3_gadget_run_stop(dwc, true, false); 2968 } 2969 2970 return 0; 2971 2972 err1: 2973 __dwc3_gadget_ep_disable(dwc->eps[0]); 2974 2975 err0: 2976 return ret; 2977 } 2978