1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cadence CDNSP DRD Driver. 4 * 5 * Copyright (C) 2020 Cadence. 6 * 7 * Author: Pawel Laszczak <pawell@cadence.com> 8 * 9 */ 10 11 #include <linux/moduleparam.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/module.h> 14 #include <linux/iopoll.h> 15 #include <linux/delay.h> 16 #include <linux/log2.h> 17 #include <linux/slab.h> 18 #include <linux/string_choices.h> 19 #include <linux/pci.h> 20 #include <linux/irq.h> 21 #include <linux/dmi.h> 22 23 #include "core.h" 24 #include "gadget-export.h" 25 #include "drd.h" 26 #include "cdnsp-gadget.h" 27 #include "cdnsp-trace.h" 28 29 unsigned int cdnsp_port_speed(unsigned int port_status) 30 { 31 /*Detect gadget speed based on PORTSC register*/ 32 if (DEV_SUPERSPEEDPLUS(port_status)) 33 return USB_SPEED_SUPER_PLUS; 34 else if (DEV_SUPERSPEED(port_status)) 35 return USB_SPEED_SUPER; 36 else if (DEV_HIGHSPEED(port_status)) 37 return USB_SPEED_HIGH; 38 else if (DEV_FULLSPEED(port_status)) 39 return USB_SPEED_FULL; 40 41 /* If device is detached then speed will be USB_SPEED_UNKNOWN.*/ 42 return USB_SPEED_UNKNOWN; 43 } 44 45 /* 46 * Given a port state, this function returns a value that would result in the 47 * port being in the same state, if the value was written to the port status 48 * control register. 49 * Save Read Only (RO) bits and save read/write bits where 50 * writing a 0 clears the bit and writing a 1 sets the bit (RWS). 51 * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect. 52 */ 53 u32 cdnsp_port_state_to_neutral(u32 state) 54 { 55 /* Save read-only status and port state. */ 56 return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS); 57 } 58 59 /** 60 * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities 61 * with capability ID id. 62 * @base: PCI MMIO registers base address. 63 * @start: Address at which to start looking, (0 or HCC_PARAMS to start at 64 * beginning of list) 65 * @id: Extended capability ID to search for. 66 * 67 * Returns the offset of the next matching extended capability structure. 68 * Some capabilities can occur several times, 69 * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all. 70 */ 71 int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id) 72 { 73 u32 offset = start; 74 u32 next; 75 u32 val; 76 77 if (!start || start == HCC_PARAMS_OFFSET) { 78 val = readl(base + HCC_PARAMS_OFFSET); 79 if (val == ~0) 80 return 0; 81 82 offset = HCC_EXT_CAPS(val) << 2; 83 if (!offset) 84 return 0; 85 } 86 87 do { 88 val = readl(base + offset); 89 if (val == ~0) 90 return 0; 91 92 if (EXT_CAPS_ID(val) == id && offset != start) 93 return offset; 94 95 next = EXT_CAPS_NEXT(val); 96 offset += next << 2; 97 } while (next); 98 99 return 0; 100 } 101 102 void cdnsp_set_link_state(struct cdnsp_device *pdev, 103 __le32 __iomem *port_regs, 104 u32 link_state) 105 { 106 int port_num = 0xFF; 107 u32 temp; 108 109 temp = readl(port_regs); 110 temp = cdnsp_port_state_to_neutral(temp); 111 temp |= PORT_WKCONN_E | PORT_WKDISC_E; 112 writel(temp, port_regs); 113 114 temp &= ~PORT_PLS_MASK; 115 temp |= PORT_LINK_STROBE | link_state; 116 117 if (pdev->active_port) 118 port_num = pdev->active_port->port_num; 119 120 trace_cdnsp_handle_port_status(port_num, readl(port_regs)); 121 writel(temp, port_regs); 122 trace_cdnsp_link_state_changed(port_num, readl(port_regs)); 123 } 124 125 static void cdnsp_disable_port(struct cdnsp_device *pdev, 126 __le32 __iomem *port_regs) 127 { 128 u32 temp = cdnsp_port_state_to_neutral(readl(port_regs)); 129 130 writel(temp | PORT_PED, port_regs); 131 } 132 133 static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev, 134 __le32 __iomem *port_regs) 135 { 136 u32 portsc = readl(port_regs); 137 138 writel(cdnsp_port_state_to_neutral(portsc) | 139 (portsc & PORT_CHANGE_BITS), port_regs); 140 } 141 142 static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit) 143 { 144 __le32 __iomem *reg; 145 void __iomem *base; 146 u32 offset = 0; 147 148 base = &pdev->cap_regs->hc_capbase; 149 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP); 150 reg = base + offset + REG_CHICKEN_BITS_2_OFFSET; 151 152 bit = readl(reg) | bit; 153 writel(bit, reg); 154 } 155 156 static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit) 157 { 158 __le32 __iomem *reg; 159 void __iomem *base; 160 u32 offset = 0; 161 162 base = &pdev->cap_regs->hc_capbase; 163 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP); 164 reg = base + offset + REG_CHICKEN_BITS_2_OFFSET; 165 166 bit = readl(reg) & ~bit; 167 writel(bit, reg); 168 } 169 170 /* 171 * Disable interrupts and begin the controller halting process. 172 */ 173 static void cdnsp_quiesce(struct cdnsp_device *pdev) 174 { 175 u32 halted; 176 u32 mask; 177 u32 cmd; 178 179 mask = ~(u32)(CDNSP_IRQS); 180 181 halted = readl(&pdev->op_regs->status) & STS_HALT; 182 if (!halted) 183 mask &= ~(CMD_R_S | CMD_DEVEN); 184 185 cmd = readl(&pdev->op_regs->command); 186 cmd &= mask; 187 writel(cmd, &pdev->op_regs->command); 188 } 189 190 /* 191 * Force controller into halt state. 192 * 193 * Disable any IRQs and clear the run/stop bit. 194 * Controller will complete any current and actively pipelined transactions, and 195 * should halt within 16 ms of the run/stop bit being cleared. 196 * Read controller Halted bit in the status register to see when the 197 * controller is finished. 198 */ 199 int cdnsp_halt(struct cdnsp_device *pdev) 200 { 201 int ret; 202 u32 val; 203 204 cdnsp_quiesce(pdev); 205 206 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val, 207 val & STS_HALT, 1, 208 CDNSP_MAX_HALT_USEC); 209 if (ret) { 210 dev_err(pdev->dev, "ERROR: Device halt failed\n"); 211 return ret; 212 } 213 214 pdev->cdnsp_state |= CDNSP_STATE_HALTED; 215 216 return 0; 217 } 218 219 /* 220 * device controller died, register read returns 0xffffffff, or command never 221 * ends. 222 */ 223 void cdnsp_died(struct cdnsp_device *pdev) 224 { 225 dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n"); 226 pdev->cdnsp_state |= CDNSP_STATE_DYING; 227 cdnsp_halt(pdev); 228 } 229 230 /* 231 * Set the run bit and wait for the device to be running. 232 */ 233 static int cdnsp_start(struct cdnsp_device *pdev) 234 { 235 u32 temp; 236 int ret; 237 238 temp = readl(&pdev->op_regs->command); 239 temp |= (CMD_R_S | CMD_DEVEN); 240 writel(temp, &pdev->op_regs->command); 241 242 pdev->cdnsp_state = 0; 243 244 /* 245 * Wait for the STS_HALT Status bit to be 0 to indicate the device is 246 * running. 247 */ 248 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp, 249 !(temp & STS_HALT), 1, 250 CDNSP_MAX_HALT_USEC); 251 if (ret) { 252 pdev->cdnsp_state = CDNSP_STATE_DYING; 253 dev_err(pdev->dev, "ERROR: Controller run failed\n"); 254 } 255 256 return ret; 257 } 258 259 /* 260 * Reset a halted controller. 261 * 262 * This resets pipelines, timers, counters, state machines, etc. 263 * Transactions will be terminated immediately, and operational registers 264 * will be set to their defaults. 265 */ 266 int cdnsp_reset(struct cdnsp_device *pdev) 267 { 268 u32 command; 269 u32 temp; 270 int ret; 271 272 temp = readl(&pdev->op_regs->status); 273 274 if (temp == ~(u32)0) { 275 dev_err(pdev->dev, "Device not accessible, reset failed.\n"); 276 return -ENODEV; 277 } 278 279 if ((temp & STS_HALT) == 0) { 280 dev_err(pdev->dev, "Controller not halted, aborting reset.\n"); 281 return -EINVAL; 282 } 283 284 command = readl(&pdev->op_regs->command); 285 command |= CMD_RESET; 286 writel(command, &pdev->op_regs->command); 287 288 ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp, 289 !(temp & CMD_RESET), 1, 290 10 * 1000); 291 if (ret) { 292 dev_err(pdev->dev, "ERROR: Controller reset failed\n"); 293 return ret; 294 } 295 296 /* 297 * CDNSP cannot write any doorbells or operational registers other 298 * than status until the "Controller Not Ready" flag is cleared. 299 */ 300 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp, 301 !(temp & STS_CNR), 1, 302 10 * 1000); 303 304 if (ret) { 305 dev_err(pdev->dev, "ERROR: Controller not ready to work\n"); 306 return ret; 307 } 308 309 dev_dbg(pdev->dev, "Controller ready to work"); 310 311 return ret; 312 } 313 314 /* 315 * cdnsp_get_endpoint_index - Find the index for an endpoint given its 316 * descriptor.Use the return value to right shift 1 for the bitmask. 317 * 318 * Index = (epnum * 2) + direction - 1, 319 * where direction = 0 for OUT, 1 for IN. 320 * For control endpoints, the IN index is used (OUT index is unused), so 321 * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2) 322 */ 323 static unsigned int 324 cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc) 325 { 326 unsigned int index = (unsigned int)usb_endpoint_num(desc); 327 328 if (usb_endpoint_xfer_control(desc)) 329 return index * 2; 330 331 return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1; 332 } 333 334 /* 335 * Find the flag for this endpoint (for use in the control context). Use the 336 * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is 337 * bit 1, etc. 338 */ 339 static unsigned int 340 cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc) 341 { 342 return 1 << (cdnsp_get_endpoint_index(desc) + 1); 343 } 344 345 int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq) 346 { 347 struct cdnsp_device *pdev = pep->pdev; 348 struct usb_request *request; 349 int ret; 350 351 if (preq->epnum == 0 && !list_empty(&pep->pending_list)) { 352 trace_cdnsp_request_enqueue_busy(preq); 353 return -EBUSY; 354 } 355 356 request = &preq->request; 357 request->actual = 0; 358 request->status = -EINPROGRESS; 359 preq->direction = pep->direction; 360 preq->epnum = pep->number; 361 preq->td.drbl = 0; 362 363 ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction); 364 if (ret) { 365 trace_cdnsp_request_enqueue_error(preq); 366 return ret; 367 } 368 369 list_add_tail(&preq->list, &pep->pending_list); 370 371 trace_cdnsp_request_enqueue(preq); 372 373 switch (usb_endpoint_type(pep->endpoint.desc)) { 374 case USB_ENDPOINT_XFER_CONTROL: 375 ret = cdnsp_queue_ctrl_tx(pdev, preq); 376 break; 377 case USB_ENDPOINT_XFER_BULK: 378 case USB_ENDPOINT_XFER_INT: 379 ret = cdnsp_queue_bulk_tx(pdev, preq); 380 break; 381 case USB_ENDPOINT_XFER_ISOC: 382 ret = cdnsp_queue_isoc_tx(pdev, preq); 383 } 384 385 if (ret) 386 goto unmap; 387 388 return 0; 389 390 unmap: 391 usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request, 392 pep->direction); 393 list_del(&preq->list); 394 trace_cdnsp_request_enqueue_error(preq); 395 396 return ret; 397 } 398 399 /* 400 * Remove the request's TD from the endpoint ring. This may cause the 401 * controller to stop USB transfers, potentially stopping in the middle of a 402 * TRB buffer. The controller should pick up where it left off in the TD, 403 * unless a Set Transfer Ring Dequeue Pointer is issued. 404 * 405 * The TRBs that make up the buffers for the canceled request will be "removed" 406 * from the ring. Since the ring is a contiguous structure, they can't be 407 * physically removed. Instead, there are two options: 408 * 409 * 1) If the controller is in the middle of processing the request to be 410 * canceled, we simply move the ring's dequeue pointer past those TRBs 411 * using the Set Transfer Ring Dequeue Pointer command. This will be 412 * the common case, when drivers timeout on the last submitted request 413 * and attempt to cancel. 414 * 415 * 2) If the controller is in the middle of a different TD, we turn the TRBs 416 * into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained. 417 * The controller will need to invalidate the any TRBs it has cached after 418 * the stop endpoint command. 419 * 420 * 3) The TD may have completed by the time the Stop Endpoint Command 421 * completes, so software needs to handle that case too. 422 * 423 */ 424 int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq) 425 { 426 struct cdnsp_device *pdev = pep->pdev; 427 int ret_stop = 0; 428 int ret_rem; 429 430 trace_cdnsp_request_dequeue(preq); 431 432 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING) 433 ret_stop = cdnsp_cmd_stop_ep(pdev, pep); 434 435 ret_rem = cdnsp_remove_request(pdev, preq, pep); 436 437 return ret_rem ? ret_rem : ret_stop; 438 } 439 440 static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev) 441 { 442 struct cdnsp_input_control_ctx *ctrl_ctx; 443 struct cdnsp_slot_ctx *slot_ctx; 444 struct cdnsp_ep_ctx *ep_ctx; 445 int i; 446 447 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx); 448 449 /* 450 * When a device's add flag and drop flag are zero, any subsequent 451 * configure endpoint command will leave that endpoint's state 452 * untouched. Make sure we don't leave any old state in the input 453 * endpoint contexts. 454 */ 455 ctrl_ctx->drop_flags = 0; 456 ctrl_ctx->add_flags = 0; 457 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx); 458 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK); 459 460 /* Endpoint 0 is always valid */ 461 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1)); 462 for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) { 463 ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i); 464 ep_ctx->ep_info = 0; 465 ep_ctx->ep_info2 = 0; 466 ep_ctx->deq = 0; 467 ep_ctx->tx_info = 0; 468 } 469 } 470 471 /* Issue a configure endpoint command and wait for it to finish. */ 472 static int cdnsp_configure_endpoint(struct cdnsp_device *pdev) 473 { 474 int ret; 475 476 cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma); 477 cdnsp_ring_cmd_db(pdev); 478 ret = cdnsp_wait_for_cmd_compl(pdev); 479 if (ret) { 480 dev_err(pdev->dev, 481 "ERR: unexpected command completion code 0x%x.\n", ret); 482 return -EINVAL; 483 } 484 485 return ret; 486 } 487 488 static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev, 489 struct cdnsp_ep *pep) 490 { 491 struct cdnsp_segment *segment; 492 union cdnsp_trb *event; 493 u32 cycle_state; 494 u32 data; 495 496 event = pdev->event_ring->dequeue; 497 segment = pdev->event_ring->deq_seg; 498 cycle_state = pdev->event_ring->cycle_state; 499 500 while (1) { 501 data = le32_to_cpu(event->trans_event.flags); 502 503 /* Check the owner of the TRB. */ 504 if ((data & TRB_CYCLE) != cycle_state) 505 break; 506 507 if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER && 508 TRB_TO_EP_ID(data) == (pep->idx + 1)) { 509 data |= TRB_EVENT_INVALIDATE; 510 event->trans_event.flags = cpu_to_le32(data); 511 } 512 513 if (cdnsp_last_trb_on_seg(segment, event)) { 514 cycle_state ^= 1; 515 segment = pdev->event_ring->deq_seg->next; 516 event = segment->trbs; 517 } else { 518 event++; 519 } 520 } 521 } 522 523 int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev) 524 { 525 struct cdnsp_segment *event_deq_seg; 526 union cdnsp_trb *cmd_trb; 527 dma_addr_t cmd_deq_dma; 528 union cdnsp_trb *event; 529 u32 cycle_state; 530 int ret, val; 531 u64 cmd_dma; 532 u32 flags; 533 534 cmd_trb = pdev->cmd.command_trb; 535 pdev->cmd.status = 0; 536 537 trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic); 538 539 ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val, 540 !CMD_RING_BUSY(val), 1, 541 CDNSP_CMD_TIMEOUT); 542 if (ret) { 543 dev_err(pdev->dev, "ERR: Timeout while waiting for command\n"); 544 trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic); 545 pdev->cdnsp_state = CDNSP_STATE_DYING; 546 return -ETIMEDOUT; 547 } 548 549 event = pdev->event_ring->dequeue; 550 event_deq_seg = pdev->event_ring->deq_seg; 551 cycle_state = pdev->event_ring->cycle_state; 552 553 cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb); 554 if (!cmd_deq_dma) 555 return -EINVAL; 556 557 while (1) { 558 flags = le32_to_cpu(event->event_cmd.flags); 559 560 /* Check the owner of the TRB. */ 561 if ((flags & TRB_CYCLE) != cycle_state) 562 return -EINVAL; 563 564 cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb); 565 566 /* 567 * Check whether the completion event is for last queued 568 * command. 569 */ 570 if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION || 571 cmd_dma != (u64)cmd_deq_dma) { 572 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) { 573 event++; 574 continue; 575 } 576 577 if (cdnsp_last_trb_on_ring(pdev->event_ring, 578 event_deq_seg, event)) 579 cycle_state ^= 1; 580 581 event_deq_seg = event_deq_seg->next; 582 event = event_deq_seg->trbs; 583 continue; 584 } 585 586 trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic); 587 588 pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status)); 589 if (pdev->cmd.status == COMP_SUCCESS) 590 return 0; 591 592 return -pdev->cmd.status; 593 } 594 } 595 596 int cdnsp_halt_endpoint(struct cdnsp_device *pdev, 597 struct cdnsp_ep *pep, 598 int value) 599 { 600 int ret; 601 602 trace_cdnsp_ep_halt(value ? "Set" : "Clear"); 603 604 ret = cdnsp_cmd_stop_ep(pdev, pep); 605 if (ret) 606 return ret; 607 608 if (value) { 609 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) { 610 cdnsp_queue_halt_endpoint(pdev, pep->idx); 611 cdnsp_ring_cmd_db(pdev); 612 ret = cdnsp_wait_for_cmd_compl(pdev); 613 } 614 615 pep->ep_state |= EP_HALTED; 616 } else { 617 cdnsp_queue_reset_ep(pdev, pep->idx); 618 cdnsp_ring_cmd_db(pdev); 619 ret = cdnsp_wait_for_cmd_compl(pdev); 620 trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx); 621 622 if (ret) 623 return ret; 624 625 pep->ep_state &= ~EP_HALTED; 626 627 if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE)) 628 cdnsp_ring_doorbell_for_active_rings(pdev, pep); 629 630 pep->ep_state &= ~EP_WEDGE; 631 } 632 633 return 0; 634 } 635 636 static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev, 637 struct cdnsp_ep *pep) 638 { 639 struct cdnsp_input_control_ctx *ctrl_ctx; 640 struct cdnsp_slot_ctx *slot_ctx; 641 int ret = 0; 642 u32 ep_sts; 643 int i; 644 645 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx); 646 647 /* Don't issue the command if there's no endpoints to update. */ 648 if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0) 649 return 0; 650 651 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG); 652 ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG); 653 ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG)); 654 655 /* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */ 656 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx); 657 for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) { 658 __le32 le32 = cpu_to_le32(BIT(i)); 659 660 if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) || 661 (ctrl_ctx->add_flags & le32) || i == 1) { 662 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK); 663 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i)); 664 break; 665 } 666 } 667 668 ep_sts = GET_EP_CTX_STATE(pep->out_ctx); 669 670 if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) && 671 ep_sts == EP_STATE_DISABLED) || 672 (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags)) 673 ret = cdnsp_configure_endpoint(pdev); 674 675 trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx)); 676 trace_cdnsp_handle_cmd_config_ep(pep->out_ctx); 677 678 cdnsp_zero_in_ctx(pdev); 679 680 return ret; 681 } 682 683 /* 684 * This submits a Reset Device Command, which will set the device state to 0, 685 * set the device address to 0, and disable all the endpoints except the default 686 * control endpoint. The USB core should come back and call 687 * cdnsp_setup_device(), and then re-set up the configuration. 688 */ 689 int cdnsp_reset_device(struct cdnsp_device *pdev) 690 { 691 struct cdnsp_slot_ctx *slot_ctx; 692 int slot_state; 693 int ret, i; 694 695 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx); 696 slot_ctx->dev_info = 0; 697 pdev->device_address = 0; 698 699 /* If device is not setup, there is no point in resetting it. */ 700 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx); 701 slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)); 702 trace_cdnsp_reset_device(slot_ctx); 703 704 if (slot_state <= SLOT_STATE_DEFAULT && 705 pdev->eps[0].ep_state & EP_HALTED) { 706 cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0); 707 } 708 709 /* 710 * During Reset Device command controller shall transition the 711 * endpoint ep0 to the Running State. 712 */ 713 pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED); 714 pdev->eps[0].ep_state |= EP_ENABLED; 715 716 if (slot_state <= SLOT_STATE_DEFAULT) 717 return 0; 718 719 cdnsp_queue_reset_device(pdev); 720 cdnsp_ring_cmd_db(pdev); 721 ret = cdnsp_wait_for_cmd_compl(pdev); 722 723 /* 724 * After Reset Device command all not default endpoints 725 * are in Disabled state. 726 */ 727 for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) 728 pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED; 729 730 trace_cdnsp_handle_cmd_reset_dev(slot_ctx); 731 732 if (ret) 733 dev_err(pdev->dev, "Reset device failed with error code %d", 734 ret); 735 736 return ret; 737 } 738 739 /* 740 * Sets the MaxPStreams field and the Linear Stream Array field. 741 * Sets the dequeue pointer to the stream context array. 742 */ 743 static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev, 744 struct cdnsp_ep_ctx *ep_ctx, 745 struct cdnsp_stream_info *stream_info) 746 { 747 u32 max_primary_streams; 748 749 /* MaxPStreams is the number of stream context array entries, not the 750 * number we're actually using. Must be in 2^(MaxPstreams + 1) format. 751 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc. 752 */ 753 max_primary_streams = fls(stream_info->num_stream_ctxs) - 2; 754 ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK); 755 ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams) 756 | EP_HAS_LSA); 757 ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma); 758 } 759 760 /* 761 * The drivers use this function to prepare a bulk endpoints to use streams. 762 * 763 * Don't allow the call to succeed if endpoint only supports one stream 764 * (which means it doesn't support streams at all). 765 */ 766 int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep) 767 { 768 unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc); 769 unsigned int num_stream_ctxs; 770 int ret; 771 772 if (num_streams == 0) 773 return 0; 774 775 if (num_streams > STREAM_NUM_STREAMS) 776 return -EINVAL; 777 778 /* 779 * Add two to the number of streams requested to account for 780 * stream 0 that is reserved for controller usage and one additional 781 * for TASK SET FULL response. 782 */ 783 num_streams += 2; 784 785 /* The stream context array size must be a power of two */ 786 num_stream_ctxs = roundup_pow_of_two(num_streams); 787 788 trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams); 789 790 ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams); 791 if (ret) 792 return ret; 793 794 cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info); 795 796 pep->ep_state |= EP_HAS_STREAMS; 797 pep->stream_info.td_count = 0; 798 pep->stream_info.first_prime_det = 0; 799 800 /* Subtract 1 for stream 0, which drivers can't use. */ 801 return num_streams - 1; 802 } 803 804 int cdnsp_disable_slot(struct cdnsp_device *pdev) 805 { 806 int ret; 807 808 cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT); 809 cdnsp_ring_cmd_db(pdev); 810 ret = cdnsp_wait_for_cmd_compl(pdev); 811 812 pdev->slot_id = 0; 813 pdev->active_port = NULL; 814 815 trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx)); 816 817 memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE); 818 memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE); 819 820 return ret; 821 } 822 823 int cdnsp_enable_slot(struct cdnsp_device *pdev) 824 { 825 struct cdnsp_slot_ctx *slot_ctx; 826 int slot_state; 827 int ret; 828 829 /* If device is not setup, there is no point in resetting it */ 830 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx); 831 slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)); 832 833 if (slot_state != SLOT_STATE_DISABLED) 834 return 0; 835 836 cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT); 837 cdnsp_ring_cmd_db(pdev); 838 ret = cdnsp_wait_for_cmd_compl(pdev); 839 if (ret) 840 goto show_trace; 841 842 pdev->slot_id = 1; 843 844 show_trace: 845 trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx)); 846 847 return ret; 848 } 849 850 /* 851 * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY 852 * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS. 853 */ 854 int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup) 855 { 856 struct cdnsp_input_control_ctx *ctrl_ctx; 857 struct cdnsp_slot_ctx *slot_ctx; 858 int dev_state = 0; 859 int ret; 860 861 if (!pdev->slot_id) { 862 trace_cdnsp_slot_id("incorrect"); 863 return -EINVAL; 864 } 865 866 if (!pdev->active_port->port_num) 867 return -EINVAL; 868 869 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx); 870 dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)); 871 872 if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) { 873 trace_cdnsp_slot_already_in_default(slot_ctx); 874 return 0; 875 } 876 877 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx); 878 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx); 879 880 if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) { 881 ret = cdnsp_setup_addressable_priv_dev(pdev); 882 if (ret) 883 return ret; 884 } 885 886 cdnsp_copy_ep0_dequeue_into_input_ctx(pdev); 887 888 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG); 889 ctrl_ctx->drop_flags = 0; 890 891 trace_cdnsp_setup_device_slot(slot_ctx); 892 893 cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup); 894 cdnsp_ring_cmd_db(pdev); 895 ret = cdnsp_wait_for_cmd_compl(pdev); 896 897 trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx)); 898 899 /* Zero the input context control for later use. */ 900 ctrl_ctx->add_flags = 0; 901 ctrl_ctx->drop_flags = 0; 902 903 return ret; 904 } 905 906 void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev, 907 struct usb_request *req, 908 int enable) 909 { 910 if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable) 911 return; 912 913 trace_cdnsp_lpm(enable); 914 915 if (enable) 916 writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE, 917 &pdev->active_port->regs->portpmsc); 918 else 919 writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc); 920 } 921 922 static int cdnsp_get_frame(struct cdnsp_device *pdev) 923 { 924 return readl(&pdev->run_regs->microframe_index) >> 3; 925 } 926 927 static int cdnsp_gadget_ep_enable(struct usb_ep *ep, 928 const struct usb_endpoint_descriptor *desc) 929 { 930 struct cdnsp_input_control_ctx *ctrl_ctx; 931 struct cdnsp_device *pdev; 932 struct cdnsp_ep *pep; 933 unsigned long flags; 934 u32 added_ctxs; 935 int ret; 936 937 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT || 938 !desc->wMaxPacketSize) 939 return -EINVAL; 940 941 pep = to_cdnsp_ep(ep); 942 pdev = pep->pdev; 943 pep->ep_state &= ~EP_UNCONFIGURED; 944 945 if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED, 946 "%s is already enabled\n", pep->name)) 947 return 0; 948 949 spin_lock_irqsave(&pdev->lock, flags); 950 951 added_ctxs = cdnsp_get_endpoint_flag(desc); 952 if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) { 953 dev_err(pdev->dev, "ERROR: Bad endpoint number\n"); 954 ret = -EINVAL; 955 goto unlock; 956 } 957 958 pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; 959 960 if (pdev->gadget.speed == USB_SPEED_FULL) { 961 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT) 962 pep->interval = desc->bInterval << 3; 963 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) 964 pep->interval = BIT(desc->bInterval - 1) << 3; 965 } 966 967 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) { 968 if (pep->interval > BIT(12)) { 969 dev_err(pdev->dev, "bInterval %d not supported\n", 970 desc->bInterval); 971 ret = -EINVAL; 972 goto unlock; 973 } 974 cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS); 975 } 976 977 ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC); 978 if (ret) 979 goto unlock; 980 981 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx); 982 ctrl_ctx->add_flags = cpu_to_le32(added_ctxs); 983 ctrl_ctx->drop_flags = 0; 984 985 ret = cdnsp_update_eps_configuration(pdev, pep); 986 if (ret) { 987 cdnsp_free_endpoint_rings(pdev, pep); 988 goto unlock; 989 } 990 991 pep->ep_state |= EP_ENABLED; 992 pep->ep_state &= ~EP_STOPPED; 993 994 unlock: 995 trace_cdnsp_ep_enable_end(pep, 0); 996 spin_unlock_irqrestore(&pdev->lock, flags); 997 998 return ret; 999 } 1000 1001 static int cdnsp_gadget_ep_disable(struct usb_ep *ep) 1002 { 1003 struct cdnsp_input_control_ctx *ctrl_ctx; 1004 struct cdnsp_request *preq; 1005 struct cdnsp_device *pdev; 1006 struct cdnsp_ep *pep; 1007 unsigned long flags; 1008 u32 drop_flag; 1009 int ret = 0; 1010 1011 if (!ep) 1012 return -EINVAL; 1013 1014 pep = to_cdnsp_ep(ep); 1015 pdev = pep->pdev; 1016 1017 spin_lock_irqsave(&pdev->lock, flags); 1018 1019 if (!(pep->ep_state & EP_ENABLED)) { 1020 dev_err(pdev->dev, "%s is already disabled\n", pep->name); 1021 ret = -EINVAL; 1022 goto finish; 1023 } 1024 1025 pep->ep_state |= EP_DIS_IN_RROGRESS; 1026 1027 /* Endpoint was unconfigured by Reset Device command. */ 1028 if (!(pep->ep_state & EP_UNCONFIGURED)) 1029 cdnsp_cmd_stop_ep(pdev, pep); 1030 1031 /* Remove all queued USB requests. */ 1032 while (!list_empty(&pep->pending_list)) { 1033 preq = next_request(&pep->pending_list); 1034 cdnsp_ep_dequeue(pep, preq); 1035 } 1036 1037 cdnsp_invalidate_ep_events(pdev, pep); 1038 1039 pep->ep_state &= ~EP_DIS_IN_RROGRESS; 1040 drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc); 1041 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx); 1042 ctrl_ctx->drop_flags = cpu_to_le32(drop_flag); 1043 ctrl_ctx->add_flags = 0; 1044 1045 cdnsp_endpoint_zero(pdev, pep); 1046 1047 if (!(pep->ep_state & EP_UNCONFIGURED)) 1048 ret = cdnsp_update_eps_configuration(pdev, pep); 1049 1050 cdnsp_free_endpoint_rings(pdev, pep); 1051 1052 pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED); 1053 pep->ep_state |= EP_STOPPED; 1054 1055 finish: 1056 trace_cdnsp_ep_disable_end(pep, 0); 1057 spin_unlock_irqrestore(&pdev->lock, flags); 1058 1059 return ret; 1060 } 1061 1062 static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep, 1063 gfp_t gfp_flags) 1064 { 1065 struct cdnsp_ep *pep = to_cdnsp_ep(ep); 1066 struct cdnsp_request *preq; 1067 1068 preq = kzalloc(sizeof(*preq), gfp_flags); 1069 if (!preq) 1070 return NULL; 1071 1072 preq->epnum = pep->number; 1073 preq->pep = pep; 1074 1075 trace_cdnsp_alloc_request(preq); 1076 1077 return &preq->request; 1078 } 1079 1080 static void cdnsp_gadget_ep_free_request(struct usb_ep *ep, 1081 struct usb_request *request) 1082 { 1083 struct cdnsp_request *preq = to_cdnsp_request(request); 1084 1085 trace_cdnsp_free_request(preq); 1086 kfree(preq); 1087 } 1088 1089 static int cdnsp_gadget_ep_queue(struct usb_ep *ep, 1090 struct usb_request *request, 1091 gfp_t gfp_flags) 1092 { 1093 struct cdnsp_request *preq; 1094 struct cdnsp_device *pdev; 1095 struct cdnsp_ep *pep; 1096 unsigned long flags; 1097 int ret; 1098 1099 if (!request || !ep) 1100 return -EINVAL; 1101 1102 pep = to_cdnsp_ep(ep); 1103 pdev = pep->pdev; 1104 1105 if (!(pep->ep_state & EP_ENABLED)) { 1106 dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n", 1107 pep->name); 1108 return -EINVAL; 1109 } 1110 1111 preq = to_cdnsp_request(request); 1112 spin_lock_irqsave(&pdev->lock, flags); 1113 ret = cdnsp_ep_enqueue(pep, preq); 1114 spin_unlock_irqrestore(&pdev->lock, flags); 1115 1116 return ret; 1117 } 1118 1119 static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep, 1120 struct usb_request *request) 1121 { 1122 struct cdnsp_ep *pep = to_cdnsp_ep(ep); 1123 struct cdnsp_device *pdev = pep->pdev; 1124 unsigned long flags; 1125 int ret; 1126 1127 if (request->status != -EINPROGRESS) 1128 return 0; 1129 1130 if (!pep->endpoint.desc) { 1131 dev_err(pdev->dev, 1132 "%s: can't dequeue to disabled endpoint\n", 1133 pep->name); 1134 return -ESHUTDOWN; 1135 } 1136 1137 /* Requests has been dequeued during disabling endpoint. */ 1138 if (!(pep->ep_state & EP_ENABLED)) 1139 return 0; 1140 1141 spin_lock_irqsave(&pdev->lock, flags); 1142 ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request)); 1143 spin_unlock_irqrestore(&pdev->lock, flags); 1144 1145 return ret; 1146 } 1147 1148 static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value) 1149 { 1150 struct cdnsp_ep *pep = to_cdnsp_ep(ep); 1151 struct cdnsp_device *pdev = pep->pdev; 1152 struct cdnsp_request *preq; 1153 unsigned long flags; 1154 int ret; 1155 1156 spin_lock_irqsave(&pdev->lock, flags); 1157 1158 preq = next_request(&pep->pending_list); 1159 if (value) { 1160 if (preq) { 1161 trace_cdnsp_ep_busy_try_halt_again(pep, 0); 1162 ret = -EAGAIN; 1163 goto done; 1164 } 1165 } 1166 1167 ret = cdnsp_halt_endpoint(pdev, pep, value); 1168 1169 done: 1170 spin_unlock_irqrestore(&pdev->lock, flags); 1171 return ret; 1172 } 1173 1174 static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep) 1175 { 1176 struct cdnsp_ep *pep = to_cdnsp_ep(ep); 1177 struct cdnsp_device *pdev = pep->pdev; 1178 unsigned long flags; 1179 int ret; 1180 1181 spin_lock_irqsave(&pdev->lock, flags); 1182 pep->ep_state |= EP_WEDGE; 1183 ret = cdnsp_halt_endpoint(pdev, pep, 1); 1184 spin_unlock_irqrestore(&pdev->lock, flags); 1185 1186 return ret; 1187 } 1188 1189 static const struct usb_ep_ops cdnsp_gadget_ep0_ops = { 1190 .enable = cdnsp_gadget_ep_enable, 1191 .disable = cdnsp_gadget_ep_disable, 1192 .alloc_request = cdnsp_gadget_ep_alloc_request, 1193 .free_request = cdnsp_gadget_ep_free_request, 1194 .queue = cdnsp_gadget_ep_queue, 1195 .dequeue = cdnsp_gadget_ep_dequeue, 1196 .set_halt = cdnsp_gadget_ep_set_halt, 1197 .set_wedge = cdnsp_gadget_ep_set_wedge, 1198 }; 1199 1200 static const struct usb_ep_ops cdnsp_gadget_ep_ops = { 1201 .enable = cdnsp_gadget_ep_enable, 1202 .disable = cdnsp_gadget_ep_disable, 1203 .alloc_request = cdnsp_gadget_ep_alloc_request, 1204 .free_request = cdnsp_gadget_ep_free_request, 1205 .queue = cdnsp_gadget_ep_queue, 1206 .dequeue = cdnsp_gadget_ep_dequeue, 1207 .set_halt = cdnsp_gadget_ep_set_halt, 1208 .set_wedge = cdnsp_gadget_ep_set_wedge, 1209 }; 1210 1211 void cdnsp_gadget_giveback(struct cdnsp_ep *pep, 1212 struct cdnsp_request *preq, 1213 int status) 1214 { 1215 struct cdnsp_device *pdev = pep->pdev; 1216 1217 list_del(&preq->list); 1218 1219 if (preq->request.status == -EINPROGRESS) 1220 preq->request.status = status; 1221 1222 usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request, 1223 preq->direction); 1224 1225 trace_cdnsp_request_giveback(preq); 1226 1227 if (preq != &pdev->ep0_preq) { 1228 spin_unlock(&pdev->lock); 1229 usb_gadget_giveback_request(&pep->endpoint, &preq->request); 1230 spin_lock(&pdev->lock); 1231 } 1232 } 1233 1234 static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = { 1235 .bLength = USB_DT_ENDPOINT_SIZE, 1236 .bDescriptorType = USB_DT_ENDPOINT, 1237 .bmAttributes = USB_ENDPOINT_XFER_CONTROL, 1238 }; 1239 1240 static int cdnsp_run(struct cdnsp_device *pdev, 1241 enum usb_device_speed speed) 1242 { 1243 u32 fs_speed = 0; 1244 u32 temp; 1245 int ret; 1246 1247 temp = readl(&pdev->ir_set->irq_control); 1248 temp &= ~IMOD_INTERVAL_MASK; 1249 temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK); 1250 writel(temp, &pdev->ir_set->irq_control); 1251 1252 temp = readl(&pdev->port3x_regs->mode_addr); 1253 1254 switch (speed) { 1255 case USB_SPEED_SUPER_PLUS: 1256 temp |= CFG_3XPORT_SSP_SUPPORT; 1257 break; 1258 case USB_SPEED_SUPER: 1259 temp &= ~CFG_3XPORT_SSP_SUPPORT; 1260 break; 1261 case USB_SPEED_HIGH: 1262 break; 1263 case USB_SPEED_FULL: 1264 fs_speed = PORT_REG6_FORCE_FS; 1265 break; 1266 default: 1267 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n", 1268 speed); 1269 fallthrough; 1270 case USB_SPEED_UNKNOWN: 1271 /* Default to superspeed. */ 1272 speed = USB_SPEED_SUPER; 1273 break; 1274 } 1275 1276 if (speed >= USB_SPEED_SUPER) { 1277 writel(temp, &pdev->port3x_regs->mode_addr); 1278 cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc, 1279 XDEV_RXDETECT); 1280 } else { 1281 cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc); 1282 } 1283 1284 cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc, 1285 XDEV_RXDETECT); 1286 1287 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 1288 1289 writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6); 1290 1291 ret = cdnsp_start(pdev); 1292 if (ret) { 1293 ret = -ENODEV; 1294 goto err; 1295 } 1296 1297 temp = readl(&pdev->op_regs->command); 1298 temp |= (CMD_INTE); 1299 writel(temp, &pdev->op_regs->command); 1300 1301 temp = readl(&pdev->ir_set->irq_pending); 1302 writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending); 1303 1304 trace_cdnsp_init("Controller ready to work"); 1305 return 0; 1306 err: 1307 cdnsp_halt(pdev); 1308 return ret; 1309 } 1310 1311 static int cdnsp_gadget_udc_start(struct usb_gadget *g, 1312 struct usb_gadget_driver *driver) 1313 { 1314 enum usb_device_speed max_speed = driver->max_speed; 1315 struct cdnsp_device *pdev = gadget_to_cdnsp(g); 1316 unsigned long flags; 1317 int ret; 1318 1319 spin_lock_irqsave(&pdev->lock, flags); 1320 pdev->gadget_driver = driver; 1321 1322 /* limit speed if necessary */ 1323 max_speed = min(driver->max_speed, g->max_speed); 1324 ret = cdnsp_run(pdev, max_speed); 1325 1326 spin_unlock_irqrestore(&pdev->lock, flags); 1327 1328 return ret; 1329 } 1330 1331 /* 1332 * Update Event Ring Dequeue Pointer: 1333 * - When all events have finished 1334 * - To avoid "Event Ring Full Error" condition 1335 */ 1336 void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev, 1337 union cdnsp_trb *event_ring_deq, 1338 u8 clear_ehb) 1339 { 1340 u64 temp_64; 1341 dma_addr_t deq; 1342 1343 temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue); 1344 1345 /* If necessary, update the HW's version of the event ring deq ptr. */ 1346 if (event_ring_deq != pdev->event_ring->dequeue) { 1347 deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg, 1348 pdev->event_ring->dequeue); 1349 temp_64 &= ERST_PTR_MASK; 1350 temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK); 1351 } 1352 1353 /* Clear the event handler busy flag (RW1C). */ 1354 if (clear_ehb) 1355 temp_64 |= ERST_EHB; 1356 else 1357 temp_64 &= ~ERST_EHB; 1358 1359 cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue); 1360 } 1361 1362 static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev) 1363 { 1364 struct cdnsp_segment *seg; 1365 u64 val_64; 1366 int i; 1367 1368 cdnsp_initialize_ring_info(pdev->cmd_ring); 1369 1370 seg = pdev->cmd_ring->first_seg; 1371 for (i = 0; i < pdev->cmd_ring->num_segs; i++) { 1372 memset(seg->trbs, 0, 1373 sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1)); 1374 seg = seg->next; 1375 } 1376 1377 /* Set the address in the Command Ring Control register. */ 1378 val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring); 1379 val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) | 1380 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) | 1381 pdev->cmd_ring->cycle_state; 1382 cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring); 1383 } 1384 1385 static void cdnsp_consume_all_events(struct cdnsp_device *pdev) 1386 { 1387 struct cdnsp_segment *event_deq_seg; 1388 union cdnsp_trb *event_ring_deq; 1389 union cdnsp_trb *event; 1390 u32 cycle_bit; 1391 1392 event_ring_deq = pdev->event_ring->dequeue; 1393 event_deq_seg = pdev->event_ring->deq_seg; 1394 event = pdev->event_ring->dequeue; 1395 1396 /* Update ring dequeue pointer. */ 1397 while (1) { 1398 cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE); 1399 1400 /* Does the controller or driver own the TRB? */ 1401 if (cycle_bit != pdev->event_ring->cycle_state) 1402 break; 1403 1404 cdnsp_inc_deq(pdev, pdev->event_ring); 1405 1406 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) { 1407 event++; 1408 continue; 1409 } 1410 1411 if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg, 1412 event)) 1413 cycle_bit ^= 1; 1414 1415 event_deq_seg = event_deq_seg->next; 1416 event = event_deq_seg->trbs; 1417 } 1418 1419 cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1); 1420 } 1421 1422 static void cdnsp_stop(struct cdnsp_device *pdev) 1423 { 1424 u32 temp; 1425 1426 /* Remove internally queued request for ep0. */ 1427 if (!list_empty(&pdev->eps[0].pending_list)) { 1428 struct cdnsp_request *req; 1429 1430 req = next_request(&pdev->eps[0].pending_list); 1431 if (req == &pdev->ep0_preq) 1432 cdnsp_ep_dequeue(&pdev->eps[0], req); 1433 } 1434 1435 cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc); 1436 cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc); 1437 cdnsp_disable_slot(pdev); 1438 cdnsp_halt(pdev); 1439 1440 temp = readl(&pdev->op_regs->status); 1441 writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status); 1442 temp = readl(&pdev->ir_set->irq_pending); 1443 writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending); 1444 1445 cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc); 1446 cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc); 1447 1448 /* Clear interrupt line */ 1449 temp = readl(&pdev->ir_set->irq_pending); 1450 temp |= IMAN_IP; 1451 writel(temp, &pdev->ir_set->irq_pending); 1452 1453 cdnsp_consume_all_events(pdev); 1454 cdnsp_clear_cmd_ring(pdev); 1455 1456 trace_cdnsp_exit("Controller stopped."); 1457 } 1458 1459 /* 1460 * Stop controller. 1461 * This function is called by the gadget core when the driver is removed. 1462 * Disable slot, disable IRQs, and quiesce the controller. 1463 */ 1464 static int cdnsp_gadget_udc_stop(struct usb_gadget *g) 1465 { 1466 struct cdnsp_device *pdev = gadget_to_cdnsp(g); 1467 unsigned long flags; 1468 1469 spin_lock_irqsave(&pdev->lock, flags); 1470 cdnsp_stop(pdev); 1471 pdev->gadget_driver = NULL; 1472 spin_unlock_irqrestore(&pdev->lock, flags); 1473 1474 return 0; 1475 } 1476 1477 static int cdnsp_gadget_get_frame(struct usb_gadget *g) 1478 { 1479 struct cdnsp_device *pdev = gadget_to_cdnsp(g); 1480 1481 return cdnsp_get_frame(pdev); 1482 } 1483 1484 static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev) 1485 { 1486 struct cdnsp_port_regs __iomem *port_regs; 1487 u32 portpm, portsc; 1488 1489 port_regs = pdev->active_port->regs; 1490 portsc = readl(&port_regs->portsc) & PORT_PLS_MASK; 1491 1492 /* Remote wakeup feature is not enabled by host. */ 1493 if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) { 1494 portpm = readl(&port_regs->portpmsc); 1495 1496 if (!(portpm & PORT_RWE)) 1497 return; 1498 } 1499 1500 if (portsc == XDEV_U3 && !pdev->may_wakeup) 1501 return; 1502 1503 cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0); 1504 1505 pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING; 1506 } 1507 1508 static int cdnsp_gadget_wakeup(struct usb_gadget *g) 1509 { 1510 struct cdnsp_device *pdev = gadget_to_cdnsp(g); 1511 unsigned long flags; 1512 1513 spin_lock_irqsave(&pdev->lock, flags); 1514 __cdnsp_gadget_wakeup(pdev); 1515 spin_unlock_irqrestore(&pdev->lock, flags); 1516 1517 return 0; 1518 } 1519 1520 static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g, 1521 int is_selfpowered) 1522 { 1523 struct cdnsp_device *pdev = gadget_to_cdnsp(g); 1524 unsigned long flags; 1525 1526 spin_lock_irqsave(&pdev->lock, flags); 1527 g->is_selfpowered = !!is_selfpowered; 1528 spin_unlock_irqrestore(&pdev->lock, flags); 1529 1530 return 0; 1531 } 1532 1533 static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on) 1534 { 1535 struct cdnsp_device *pdev = gadget_to_cdnsp(gadget); 1536 struct cdns *cdns = dev_get_drvdata(pdev->dev); 1537 unsigned long flags; 1538 1539 trace_cdnsp_pullup(is_on); 1540 1541 /* 1542 * Disable events handling while controller is being 1543 * enabled/disabled. 1544 */ 1545 disable_irq(cdns->dev_irq); 1546 spin_lock_irqsave(&pdev->lock, flags); 1547 1548 if (!is_on) { 1549 cdnsp_reset_device(pdev); 1550 cdns_clear_vbus(cdns); 1551 } else { 1552 cdns_set_vbus(cdns); 1553 } 1554 1555 spin_unlock_irqrestore(&pdev->lock, flags); 1556 enable_irq(cdns->dev_irq); 1557 1558 return 0; 1559 } 1560 1561 static const struct usb_gadget_ops cdnsp_gadget_ops = { 1562 .get_frame = cdnsp_gadget_get_frame, 1563 .wakeup = cdnsp_gadget_wakeup, 1564 .set_selfpowered = cdnsp_gadget_set_selfpowered, 1565 .pullup = cdnsp_gadget_pullup, 1566 .udc_start = cdnsp_gadget_udc_start, 1567 .udc_stop = cdnsp_gadget_udc_stop, 1568 }; 1569 1570 static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev, 1571 struct cdnsp_ep *pep) 1572 { 1573 void __iomem *reg = &pdev->cap_regs->hc_capbase; 1574 int endpoints; 1575 1576 reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID); 1577 1578 if (!pep->direction) { 1579 pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET); 1580 pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET); 1581 pep->buffering = (pep->buffering + 1) / 2; 1582 pep->buffering_period = (pep->buffering_period + 1) / 2; 1583 return; 1584 } 1585 1586 endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2; 1587 1588 /* Set to XBUF_TX_TAG_MASK_0 register. */ 1589 reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32); 1590 /* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */ 1591 reg += pep->number * sizeof(u32) * 2; 1592 1593 pep->buffering = (readl(reg) + 1) / 2; 1594 pep->buffering_period = pep->buffering; 1595 } 1596 1597 static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev) 1598 { 1599 int max_streams = HCC_MAX_PSA(pdev->hcc_params); 1600 struct cdnsp_ep *pep; 1601 int i; 1602 1603 INIT_LIST_HEAD(&pdev->gadget.ep_list); 1604 1605 if (max_streams < STREAM_LOG_STREAMS) { 1606 dev_err(pdev->dev, "Stream size %d not supported\n", 1607 max_streams); 1608 return -EINVAL; 1609 } 1610 1611 max_streams = STREAM_LOG_STREAMS; 1612 1613 for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) { 1614 bool direction = !(i & 1); /* Start from OUT endpoint. */ 1615 u8 epnum = ((i + 1) >> 1); 1616 1617 if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction)) 1618 continue; 1619 1620 pep = &pdev->eps[i]; 1621 pep->pdev = pdev; 1622 pep->number = epnum; 1623 pep->direction = direction; /* 0 for OUT, 1 for IN. */ 1624 1625 /* 1626 * Ep0 is bidirectional, so ep0in and ep0out are represented by 1627 * pdev->eps[0] 1628 */ 1629 if (epnum == 0) { 1630 snprintf(pep->name, sizeof(pep->name), "ep%d%s", 1631 epnum, "BiDir"); 1632 1633 pep->idx = 0; 1634 usb_ep_set_maxpacket_limit(&pep->endpoint, 512); 1635 pep->endpoint.maxburst = 1; 1636 pep->endpoint.ops = &cdnsp_gadget_ep0_ops; 1637 pep->endpoint.desc = &cdnsp_gadget_ep0_desc; 1638 pep->endpoint.comp_desc = NULL; 1639 pep->endpoint.caps.type_control = true; 1640 pep->endpoint.caps.dir_in = true; 1641 pep->endpoint.caps.dir_out = true; 1642 1643 pdev->ep0_preq.epnum = pep->number; 1644 pdev->ep0_preq.pep = pep; 1645 pdev->gadget.ep0 = &pep->endpoint; 1646 } else { 1647 snprintf(pep->name, sizeof(pep->name), "ep%d%s", 1648 epnum, (pep->direction) ? "in" : "out"); 1649 1650 pep->idx = (epnum * 2 + (direction ? 1 : 0)) - 1; 1651 usb_ep_set_maxpacket_limit(&pep->endpoint, 1024); 1652 1653 pep->endpoint.max_streams = max_streams; 1654 pep->endpoint.ops = &cdnsp_gadget_ep_ops; 1655 list_add_tail(&pep->endpoint.ep_list, 1656 &pdev->gadget.ep_list); 1657 1658 pep->endpoint.caps.type_iso = true; 1659 pep->endpoint.caps.type_bulk = true; 1660 pep->endpoint.caps.type_int = true; 1661 1662 pep->endpoint.caps.dir_in = direction; 1663 pep->endpoint.caps.dir_out = !direction; 1664 } 1665 1666 pep->endpoint.name = pep->name; 1667 pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx); 1668 pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx); 1669 cdnsp_get_ep_buffering(pdev, pep); 1670 1671 dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: " 1672 "CTRL: %s, INT: %s, BULK: %s, ISOC %s, " 1673 "SupDir IN: %s, OUT: %s\n", 1674 pep->name, 1024, 1675 str_yes_no(pep->endpoint.caps.type_control), 1676 str_yes_no(pep->endpoint.caps.type_int), 1677 str_yes_no(pep->endpoint.caps.type_bulk), 1678 str_yes_no(pep->endpoint.caps.type_iso), 1679 str_yes_no(pep->endpoint.caps.dir_in), 1680 str_yes_no(pep->endpoint.caps.dir_out)); 1681 1682 INIT_LIST_HEAD(&pep->pending_list); 1683 } 1684 1685 return 0; 1686 } 1687 1688 static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev) 1689 { 1690 struct cdnsp_ep *pep; 1691 int i; 1692 1693 for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) { 1694 pep = &pdev->eps[i]; 1695 if (pep->number != 0 && pep->out_ctx) 1696 list_del(&pep->endpoint.ep_list); 1697 } 1698 } 1699 1700 void cdnsp_disconnect_gadget(struct cdnsp_device *pdev) 1701 { 1702 pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING; 1703 1704 if (pdev->gadget_driver && pdev->gadget_driver->disconnect) { 1705 spin_unlock(&pdev->lock); 1706 pdev->gadget_driver->disconnect(&pdev->gadget); 1707 spin_lock(&pdev->lock); 1708 } 1709 1710 pdev->gadget.speed = USB_SPEED_UNKNOWN; 1711 usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED); 1712 1713 pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING; 1714 } 1715 1716 void cdnsp_suspend_gadget(struct cdnsp_device *pdev) 1717 { 1718 if (pdev->gadget_driver && pdev->gadget_driver->suspend) { 1719 spin_unlock(&pdev->lock); 1720 pdev->gadget_driver->suspend(&pdev->gadget); 1721 spin_lock(&pdev->lock); 1722 } 1723 } 1724 1725 void cdnsp_resume_gadget(struct cdnsp_device *pdev) 1726 { 1727 if (pdev->gadget_driver && pdev->gadget_driver->resume) { 1728 spin_unlock(&pdev->lock); 1729 pdev->gadget_driver->resume(&pdev->gadget); 1730 spin_lock(&pdev->lock); 1731 } 1732 } 1733 1734 void cdnsp_irq_reset(struct cdnsp_device *pdev) 1735 { 1736 struct cdnsp_port_regs __iomem *port_regs; 1737 1738 cdnsp_reset_device(pdev); 1739 1740 port_regs = pdev->active_port->regs; 1741 pdev->gadget.speed = cdnsp_port_speed(readl(port_regs)); 1742 1743 spin_unlock(&pdev->lock); 1744 usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver); 1745 spin_lock(&pdev->lock); 1746 1747 switch (pdev->gadget.speed) { 1748 case USB_SPEED_SUPER_PLUS: 1749 case USB_SPEED_SUPER: 1750 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 1751 pdev->gadget.ep0->maxpacket = 512; 1752 break; 1753 case USB_SPEED_HIGH: 1754 case USB_SPEED_FULL: 1755 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 1756 pdev->gadget.ep0->maxpacket = 64; 1757 break; 1758 default: 1759 /* Low speed is not supported. */ 1760 dev_err(pdev->dev, "Unknown device speed\n"); 1761 break; 1762 } 1763 1764 cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS); 1765 cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY); 1766 usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT); 1767 } 1768 1769 static void cdnsp_get_rev_cap(struct cdnsp_device *pdev) 1770 { 1771 void __iomem *reg = &pdev->cap_regs->hc_capbase; 1772 1773 reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP); 1774 pdev->rev_cap = reg; 1775 1776 dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n", 1777 readl(&pdev->rev_cap->ctrl_revision), 1778 readl(&pdev->rev_cap->rtl_revision), 1779 readl(&pdev->rev_cap->ep_supported), 1780 readl(&pdev->rev_cap->rx_buff_size), 1781 readl(&pdev->rev_cap->tx_buff_size)); 1782 } 1783 1784 static int cdnsp_gen_setup(struct cdnsp_device *pdev) 1785 { 1786 int ret; 1787 u32 reg; 1788 1789 pdev->cap_regs = pdev->regs; 1790 pdev->op_regs = pdev->regs + 1791 HC_LENGTH(readl(&pdev->cap_regs->hc_capbase)); 1792 pdev->run_regs = pdev->regs + 1793 (readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK); 1794 1795 /* Cache read-only capability registers */ 1796 pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1); 1797 pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase); 1798 pdev->hci_version = HC_VERSION(pdev->hcc_params); 1799 pdev->hcc_params = readl(&pdev->cap_regs->hcc_params); 1800 1801 cdnsp_get_rev_cap(pdev); 1802 1803 /* Make sure the Device Controller is halted. */ 1804 ret = cdnsp_halt(pdev); 1805 if (ret) 1806 return ret; 1807 1808 /* Reset the internal controller memory state and registers. */ 1809 ret = cdnsp_reset(pdev); 1810 if (ret) 1811 return ret; 1812 1813 /* 1814 * Set dma_mask and coherent_dma_mask to 64-bits, 1815 * if controller supports 64-bit addressing. 1816 */ 1817 if (HCC_64BIT_ADDR(pdev->hcc_params) && 1818 !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) { 1819 dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n"); 1820 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64)); 1821 } else { 1822 /* 1823 * This is to avoid error in cases where a 32-bit USB 1824 * controller is used on a 64-bit capable system. 1825 */ 1826 ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32)); 1827 if (ret) 1828 return ret; 1829 1830 dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n"); 1831 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32)); 1832 } 1833 1834 spin_lock_init(&pdev->lock); 1835 1836 ret = cdnsp_mem_init(pdev); 1837 if (ret) 1838 return ret; 1839 1840 /* 1841 * Software workaround for U1: after transition 1842 * to U1 the controller starts gating clock, and in some cases, 1843 * it causes that controller stack. 1844 */ 1845 reg = readl(&pdev->port3x_regs->mode_2); 1846 reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN; 1847 writel(reg, &pdev->port3x_regs->mode_2); 1848 1849 return 0; 1850 } 1851 1852 static int __cdnsp_gadget_init(struct cdns *cdns) 1853 { 1854 struct cdnsp_device *pdev; 1855 u32 max_speed; 1856 int ret = -ENOMEM; 1857 1858 cdns_drd_gadget_on(cdns); 1859 1860 pdev = kzalloc(sizeof(*pdev), GFP_KERNEL); 1861 if (!pdev) 1862 return -ENOMEM; 1863 1864 pm_runtime_get_sync(cdns->dev); 1865 1866 cdns->gadget_dev = pdev; 1867 pdev->dev = cdns->dev; 1868 pdev->regs = cdns->dev_regs; 1869 max_speed = usb_get_maximum_speed(cdns->dev); 1870 1871 switch (max_speed) { 1872 case USB_SPEED_FULL: 1873 case USB_SPEED_HIGH: 1874 case USB_SPEED_SUPER: 1875 case USB_SPEED_SUPER_PLUS: 1876 break; 1877 default: 1878 dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed); 1879 fallthrough; 1880 case USB_SPEED_UNKNOWN: 1881 /* Default to SSP */ 1882 max_speed = USB_SPEED_SUPER_PLUS; 1883 break; 1884 } 1885 1886 pdev->gadget.ops = &cdnsp_gadget_ops; 1887 pdev->gadget.name = "cdnsp-gadget"; 1888 pdev->gadget.speed = USB_SPEED_UNKNOWN; 1889 pdev->gadget.sg_supported = 1; 1890 pdev->gadget.max_speed = max_speed; 1891 pdev->gadget.lpm_capable = 1; 1892 1893 pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL); 1894 if (!pdev->setup_buf) 1895 goto free_pdev; 1896 1897 /* 1898 * Controller supports not aligned buffer but it should improve 1899 * performance. 1900 */ 1901 pdev->gadget.quirk_ep_out_aligned_size = true; 1902 1903 ret = cdnsp_gen_setup(pdev); 1904 if (ret) { 1905 dev_err(pdev->dev, "Generic initialization failed %d\n", ret); 1906 goto free_setup; 1907 } 1908 1909 ret = cdnsp_gadget_init_endpoints(pdev); 1910 if (ret) { 1911 dev_err(pdev->dev, "failed to initialize endpoints\n"); 1912 goto halt_pdev; 1913 } 1914 1915 ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget); 1916 if (ret) { 1917 dev_err(pdev->dev, "failed to register udc\n"); 1918 goto free_endpoints; 1919 } 1920 1921 ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq, 1922 cdnsp_irq_handler, 1923 cdnsp_thread_irq_handler, IRQF_SHARED, 1924 dev_name(pdev->dev), pdev); 1925 if (ret) 1926 goto del_gadget; 1927 1928 return 0; 1929 1930 del_gadget: 1931 usb_del_gadget_udc(&pdev->gadget); 1932 free_endpoints: 1933 cdnsp_gadget_free_endpoints(pdev); 1934 halt_pdev: 1935 cdnsp_halt(pdev); 1936 cdnsp_reset(pdev); 1937 cdnsp_mem_cleanup(pdev); 1938 free_setup: 1939 kfree(pdev->setup_buf); 1940 free_pdev: 1941 kfree(pdev); 1942 1943 return ret; 1944 } 1945 1946 static void cdnsp_gadget_exit(struct cdns *cdns) 1947 { 1948 struct cdnsp_device *pdev = cdns->gadget_dev; 1949 1950 devm_free_irq(pdev->dev, cdns->dev_irq, pdev); 1951 pm_runtime_mark_last_busy(cdns->dev); 1952 pm_runtime_put_autosuspend(cdns->dev); 1953 usb_del_gadget_udc(&pdev->gadget); 1954 cdnsp_gadget_free_endpoints(pdev); 1955 cdnsp_mem_cleanup(pdev); 1956 kfree(pdev); 1957 cdns->gadget_dev = NULL; 1958 cdns_drd_gadget_off(cdns); 1959 } 1960 1961 static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup) 1962 { 1963 struct cdnsp_device *pdev = cdns->gadget_dev; 1964 unsigned long flags; 1965 1966 if (pdev->link_state == XDEV_U3) 1967 return 0; 1968 1969 spin_lock_irqsave(&pdev->lock, flags); 1970 cdnsp_disconnect_gadget(pdev); 1971 cdnsp_stop(pdev); 1972 spin_unlock_irqrestore(&pdev->lock, flags); 1973 1974 return 0; 1975 } 1976 1977 static int cdnsp_gadget_resume(struct cdns *cdns, bool hibernated) 1978 { 1979 struct cdnsp_device *pdev = cdns->gadget_dev; 1980 enum usb_device_speed max_speed; 1981 unsigned long flags; 1982 int ret; 1983 1984 if (!pdev->gadget_driver) 1985 return 0; 1986 1987 spin_lock_irqsave(&pdev->lock, flags); 1988 max_speed = pdev->gadget_driver->max_speed; 1989 1990 /* Limit speed if necessary. */ 1991 max_speed = min(max_speed, pdev->gadget.max_speed); 1992 1993 ret = cdnsp_run(pdev, max_speed); 1994 1995 if (pdev->link_state == XDEV_U3) 1996 __cdnsp_gadget_wakeup(pdev); 1997 1998 spin_unlock_irqrestore(&pdev->lock, flags); 1999 2000 return ret; 2001 } 2002 2003 /** 2004 * cdnsp_gadget_init - initialize device structure 2005 * @cdns: cdnsp instance 2006 * 2007 * This function initializes the gadget. 2008 */ 2009 int cdnsp_gadget_init(struct cdns *cdns) 2010 { 2011 struct cdns_role_driver *rdrv; 2012 2013 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL); 2014 if (!rdrv) 2015 return -ENOMEM; 2016 2017 rdrv->start = __cdnsp_gadget_init; 2018 rdrv->stop = cdnsp_gadget_exit; 2019 rdrv->suspend = cdnsp_gadget_suspend; 2020 rdrv->resume = cdnsp_gadget_resume; 2021 rdrv->state = CDNS_ROLE_STATE_INACTIVE; 2022 rdrv->name = "gadget"; 2023 cdns->roles[USB_ROLE_DEVICE] = rdrv; 2024 2025 return 0; 2026 } 2027