Lines Matching +full:disable +full:- +full:hibernation +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0
3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
20 #include <linux/dma-mapping.h>
30 #define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
34 * dwc3_gadget_set_test_mode - enables usb2 test modes
36 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
39 * success or -EINVAL if wrong Test Selector is passed.
41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode) in dwc3_gadget_set_test_mode() argument
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_set_test_mode()
48 switch (mode) { in dwc3_gadget_set_test_mode()
54 reg |= mode << 1; in dwc3_gadget_set_test_mode()
57 return -EINVAL; in dwc3_gadget_set_test_mode()
66 * dwc3_gadget_get_link_state - gets current state of usb link
70 * return the link state on success (>= 0) or -ETIMEDOUT.
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_get_link_state()
82 * dwc3_gadget_set_link_state - sets usb link to a particular state
87 * return 0 on success or -ETIMEDOUT.
99 while (--retries) { in dwc3_gadget_set_link_state()
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_set_link_state()
108 return -ETIMEDOUT; in dwc3_gadget_set_link_state()
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_set_link_state()
115 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_set_link_state()
119 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_set_link_state()
130 while (--retries) { in dwc3_gadget_set_link_state()
131 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_set_link_state()
139 return -ETIMEDOUT; in dwc3_gadget_set_link_state()
146 if (dwc->ep0state != EP0_SETUP_PHASE) { in dwc3_ep0_reset_state()
147 dir = !!dwc->ep0_expect_in; in dwc3_ep0_reset_state()
148 if (dwc->ep0state == EP0_DATA_PHASE) in dwc3_ep0_reset_state()
149 dwc3_ep0_end_control_data(dwc, dwc->eps[dir]); in dwc3_ep0_reset_state()
151 dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]); in dwc3_ep0_reset_state()
153 dwc->eps[0]->trb_enqueue = 0; in dwc3_ep0_reset_state()
154 dwc->eps[1]->trb_enqueue = 0; in dwc3_ep0_reset_state()
161 * dwc3_ep_inc_trb - increment a trb index.
171 if (*index == (DWC3_TRB_NUM - 1)) in dwc3_ep_inc_trb()
176 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
181 dwc3_ep_inc_trb(&dep->trb_enqueue); in dwc3_ep_inc_enq()
185 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
190 dwc3_ep_inc_trb(&dep->trb_dequeue); in dwc3_ep_inc_deq()
196 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_del_and_unmap_request()
198 list_del(&req->list); in dwc3_gadget_del_and_unmap_request()
199 req->remaining = 0; in dwc3_gadget_del_and_unmap_request()
200 req->num_trbs = 0; in dwc3_gadget_del_and_unmap_request()
202 if (req->request.status == -EINPROGRESS) in dwc3_gadget_del_and_unmap_request()
203 req->request.status = status; in dwc3_gadget_del_and_unmap_request()
205 if (req->trb) in dwc3_gadget_del_and_unmap_request()
206 usb_gadget_unmap_request_by_dev(dwc->sysdev, in dwc3_gadget_del_and_unmap_request()
207 &req->request, req->direction); in dwc3_gadget_del_and_unmap_request()
209 req->trb = NULL; in dwc3_gadget_del_and_unmap_request()
212 if (dep->number > 1) in dwc3_gadget_del_and_unmap_request()
213 pm_runtime_put(dwc->dev); in dwc3_gadget_del_and_unmap_request()
217 * dwc3_gadget_giveback - call struct usb_request's ->complete callback
223 * function will unmap @req and call its ->complete() callback to notify upper
229 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_giveback()
235 if (req->status == DWC3_REQUEST_STATUS_COMPLETED) in dwc3_gadget_giveback()
239 req->status = DWC3_REQUEST_STATUS_COMPLETED; in dwc3_gadget_giveback()
241 spin_unlock(&dwc->lock); in dwc3_gadget_giveback()
242 usb_gadget_giveback_request(&dep->endpoint, &req->request); in dwc3_gadget_giveback()
243 spin_lock(&dwc->lock); in dwc3_gadget_giveback()
247 * dwc3_send_gadget_generic_command - issue a generic command for the controller
263 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param); in dwc3_send_gadget_generic_command()
264 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT); in dwc3_send_gadget_generic_command()
267 reg = dwc3_readl(dwc->regs, DWC3_DGCMD); in dwc3_send_gadget_generic_command()
271 ret = -EINVAL; in dwc3_send_gadget_generic_command()
274 } while (--timeout); in dwc3_send_gadget_generic_command()
277 ret = -ETIMEDOUT; in dwc3_send_gadget_generic_command()
278 status = -ETIMEDOUT; in dwc3_send_gadget_generic_command()
287 * dwc3_send_gadget_ep_cmd - issue an endpoint command
315 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in dwc3_send_gadget_ep_cmd()
316 struct dwc3 *dwc = dep->dwc; in dwc3_send_gadget_ep_cmd()
322 int ret = -EINVAL; in dwc3_send_gadget_ep_cmd()
334 if (dwc->gadget->speed <= USB_SPEED_HIGH || in dwc3_send_gadget_ep_cmd()
336 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_send_gadget_ep_cmd()
348 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_send_gadget_ep_cmd()
358 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); in dwc3_send_gadget_ep_cmd()
359 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); in dwc3_send_gadget_ep_cmd()
360 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); in dwc3_send_gadget_ep_cmd()
384 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); in dwc3_send_gadget_ep_cmd()
394 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); in dwc3_send_gadget_ep_cmd()
403 dev_WARN(dwc->dev, "No resource for %s\n", in dwc3_send_gadget_ep_cmd()
404 dep->name); in dwc3_send_gadget_ep_cmd()
405 ret = -EINVAL; in dwc3_send_gadget_ep_cmd()
415 * Instead of always returning -EINVAL, let's in dwc3_send_gadget_ep_cmd()
417 * the case by returning -EAGAIN. in dwc3_send_gadget_ep_cmd()
419 ret = -EAGAIN; in dwc3_send_gadget_ep_cmd()
422 dev_WARN(dwc->dev, "UNKNOWN cmd status\n"); in dwc3_send_gadget_ep_cmd()
427 } while (--timeout); in dwc3_send_gadget_ep_cmd()
430 ret = -ETIMEDOUT; in dwc3_send_gadget_ep_cmd()
431 cmd_status = -ETIMEDOUT; in dwc3_send_gadget_ep_cmd()
439 dep->flags |= DWC3_EP_TRANSFER_STARTED; in dwc3_send_gadget_ep_cmd()
441 if (ret != -ETIMEDOUT) in dwc3_send_gadget_ep_cmd()
450 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_send_gadget_ep_cmd()
452 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_send_gadget_ep_cmd()
460 struct dwc3 *dwc = dep->dwc; in dwc3_send_clear_stall_ep_cmd()
468 * some (non-compliant) hosts may not send ACK TPs for pending in dwc3_send_clear_stall_ep_cmd()
472 if (dep->direction && in dwc3_send_clear_stall_ep_cmd()
474 (dwc->gadget->speed >= USB_SPEED_SUPER)) in dwc3_send_clear_stall_ep_cmd()
485 u32 offset = (char *) trb - (char *) dep->trb_pool; in dwc3_trb_dma_offset()
487 return dep->trb_pool_dma + offset; in dwc3_trb_dma_offset()
492 struct dwc3 *dwc = dep->dwc; in dwc3_alloc_trb_pool()
494 if (dep->trb_pool) in dwc3_alloc_trb_pool()
497 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, in dwc3_alloc_trb_pool()
499 &dep->trb_pool_dma, GFP_KERNEL); in dwc3_alloc_trb_pool()
500 if (!dep->trb_pool) { in dwc3_alloc_trb_pool()
501 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", in dwc3_alloc_trb_pool()
502 dep->name); in dwc3_alloc_trb_pool()
503 return -ENOMEM; in dwc3_alloc_trb_pool()
511 struct dwc3 *dwc = dep->dwc; in dwc3_free_trb_pool()
513 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM, in dwc3_free_trb_pool()
514 dep->trb_pool, dep->trb_pool_dma); in dwc3_free_trb_pool()
516 dep->trb_pool = NULL; in dwc3_free_trb_pool()
517 dep->trb_pool_dma = 0; in dwc3_free_trb_pool()
525 if (dep->flags & DWC3_EP_RESOURCE_ALLOCATED) in dwc3_gadget_set_xfer_resource()
537 dep->flags |= DWC3_EP_RESOURCE_ALLOCATED; in dwc3_gadget_set_xfer_resource()
542 * dwc3_gadget_start_config - reset endpoint resources
547 * part of the power-on/soft-reset initialization.
549 * Set resource_index=2 to reset only non-control endpoints' resources. Do this
550 * on receiving the SET_CONFIGURATION request or hibernation resume.
561 return -EINVAL; in dwc3_gadget_start_config()
567 ret = dwc3_send_gadget_ep_cmd(dwc->eps[0], cmd, ¶ms); in dwc3_gadget_start_config()
572 for (i = resource_index; i < dwc->num_eps; i++) { in dwc3_gadget_start_config()
573 dep = dwc->eps[i]; in dwc3_gadget_start_config()
577 dep->flags &= ~DWC3_EP_RESOURCE_ALLOCATED; in dwc3_gadget_start_config()
588 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_set_ep_config()
590 comp_desc = dep->endpoint.comp_desc; in dwc3_gadget_set_ep_config()
591 desc = dep->endpoint.desc; in dwc3_gadget_set_ep_config()
598 /* Burst size is only needed in SuperSpeed mode */ in dwc3_gadget_set_ep_config()
599 if (dwc->gadget->speed >= USB_SPEED_SUPER) { in dwc3_gadget_set_ep_config()
600 u32 burst = dep->endpoint.maxburst; in dwc3_gadget_set_ep_config()
602 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); in dwc3_gadget_set_ep_config()
607 params.param2 |= dep->saved_state; in dwc3_gadget_set_ep_config()
612 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) in dwc3_gadget_set_ep_config()
619 dep->stream_capable = true; in dwc3_gadget_set_ep_config()
631 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); in dwc3_gadget_set_ep_config()
637 if (dep->direction) in dwc3_gadget_set_ep_config()
638 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); in dwc3_gadget_set_ep_config()
640 if (desc->bInterval) { in dwc3_gadget_set_ep_config()
651 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13); in dwc3_gadget_set_ep_config()
654 dwc->gadget->speed == USB_SPEED_FULL) in dwc3_gadget_set_ep_config()
655 dep->interval = desc->bInterval; in dwc3_gadget_set_ep_config()
657 dep->interval = 1 << (desc->bInterval - 1); in dwc3_gadget_set_ep_config()
666 * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
701 * dwc3_gadget_calc_ram_depth - calculates the ram depth for txfifo
711 is_single_port_ram = DWC3_SPRAM_TYPE(dwc->hwparams.hwparams1); in dwc3_gadget_calc_ram_depth()
717 ram_depth = is_single_port_ram ? DWC3_RAM0_DEPTH(dwc->hwparams.hwparams6) : in dwc3_gadget_calc_ram_depth()
718 DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); in dwc3_gadget_calc_ram_depth()
723 * at a non-zero address. in dwc3_gadget_calc_ram_depth()
728 /* Check if TXFIFOs start at non-zero addr */ in dwc3_gadget_calc_ram_depth()
729 reg = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); in dwc3_gadget_calc_ram_depth()
732 ram_depth -= (fifo_0_start >> 16); in dwc3_gadget_calc_ram_depth()
739 * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
752 if (!dwc->do_fifo_resize) in dwc3_gadget_clear_tx_fifos()
756 dep = dwc->eps[1]; in dwc3_gadget_clear_tx_fifos()
757 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); in dwc3_gadget_clear_tx_fifos()
763 dwc->last_fifo_depth = fifo_depth; in dwc3_gadget_clear_tx_fifos()
765 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM); num += 2) { in dwc3_gadget_clear_tx_fifos()
766 dep = dwc->eps[num]; in dwc3_gadget_clear_tx_fifos()
772 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) & in dwc3_gadget_clear_tx_fifos()
775 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size); in dwc3_gadget_clear_tx_fifos()
776 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED; in dwc3_gadget_clear_tx_fifos()
778 dwc->num_ep_resized = 0; in dwc3_gadget_clear_tx_fifos()
782 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
790 * on the configured size for RAM1 - which contains TxFifo -,
804 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_resize_tx_fifos()
815 if (!dwc->do_fifo_resize) in dwc3_gadget_resize_tx_fifos()
819 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1) in dwc3_gadget_resize_tx_fifos()
823 if (dep->flags & DWC3_EP_TXFIFO_RESIZED) in dwc3_gadget_resize_tx_fifos()
828 switch (dwc->gadget->speed) { in dwc3_gadget_resize_tx_fifos()
831 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) || in dwc3_gadget_resize_tx_fifos()
832 usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_resize_tx_fifos()
834 dep->endpoint.maxburst, in dwc3_gadget_resize_tx_fifos()
835 dwc->tx_fifo_resize_max_num); in dwc3_gadget_resize_tx_fifos()
838 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in dwc3_gadget_resize_tx_fifos()
840 usb_endpoint_maxp_mult(dep->endpoint.desc) + 1, in dwc3_gadget_resize_tx_fifos()
841 dwc->tx_fifo_resize_max_num); in dwc3_gadget_resize_tx_fifos()
846 if (usb_endpoint_xfer_bulk(dep->endpoint.desc)) in dwc3_gadget_resize_tx_fifos()
857 num_in_ep = dwc->max_cfg_eps; in dwc3_gadget_resize_tx_fifos()
858 num_in_ep -= dwc->num_ep_resized; in dwc3_gadget_resize_tx_fifos()
862 remaining = ram_depth - min_depth - dwc->last_fifo_depth; in dwc3_gadget_resize_tx_fifos()
869 fifo_size = (num_fifos - 1) * fifo; in dwc3_gadget_resize_tx_fifos()
877 /* Check if TXFIFOs start at non-zero addr */ in dwc3_gadget_resize_tx_fifos()
878 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); in dwc3_gadget_resize_tx_fifos()
881 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16)); in dwc3_gadget_resize_tx_fifos()
883 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); in dwc3_gadget_resize_tx_fifos()
885 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); in dwc3_gadget_resize_tx_fifos()
888 if (dwc->last_fifo_depth >= ram_depth) { in dwc3_gadget_resize_tx_fifos()
889 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n", in dwc3_gadget_resize_tx_fifos()
890 dwc->last_fifo_depth, ram_depth, in dwc3_gadget_resize_tx_fifos()
891 dep->endpoint.name, fifo_size); in dwc3_gadget_resize_tx_fifos()
897 dwc->last_fifo_depth -= fifo_size; in dwc3_gadget_resize_tx_fifos()
898 return -ENOMEM; in dwc3_gadget_resize_tx_fifos()
901 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); in dwc3_gadget_resize_tx_fifos()
902 dep->flags |= DWC3_EP_TXFIFO_RESIZED; in dwc3_gadget_resize_tx_fifos()
903 dwc->num_ep_resized++; in dwc3_gadget_resize_tx_fifos()
909 * __dwc3_gadget_ep_enable - initializes a hw endpoint
918 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_ep_enable()
919 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_enable()
924 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
938 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
942 dep->type = usb_endpoint_type(desc); in __dwc3_gadget_ep_enable()
943 dep->flags |= DWC3_EP_ENABLED; in __dwc3_gadget_ep_enable()
945 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); in __dwc3_gadget_ep_enable()
946 reg |= DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_enable()
947 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); in __dwc3_gadget_ep_enable()
949 dep->trb_dequeue = 0; in __dwc3_gadget_ep_enable()
950 dep->trb_enqueue = 0; in __dwc3_gadget_ep_enable()
956 memset(dep->trb_pool, 0, in __dwc3_gadget_ep_enable()
960 trb_st_hw = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
962 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; in __dwc3_gadget_ep_enable()
963 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
964 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
965 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB; in __dwc3_gadget_ep_enable()
966 trb_link->ctrl |= DWC3_TRB_CTRL_HWO; in __dwc3_gadget_ep_enable()
970 * Issue StartTransfer here with no-op TRB so we can always rely on No in __dwc3_gadget_ep_enable()
981 trb = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
993 if (dep->stream_capable) { in __dwc3_gadget_ep_enable()
1000 * no-op TRB as normal, but end it immediately. As a in __dwc3_gadget_ep_enable()
1017 if (!dep->direction || in __dwc3_gadget_ep_enable()
1018 !(dwc->hwparams.hwparams9 & in __dwc3_gadget_ep_enable()
1020 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM; in __dwc3_gadget_ep_enable()
1037 if (dep->flags & DWC3_EP_DELAY_STOP) in dwc3_remove_requests()
1040 /* - giveback all requests to gadget driver */ in dwc3_remove_requests()
1041 while (!list_empty(&dep->started_list)) { in dwc3_remove_requests()
1042 req = next_request(&dep->started_list); in dwc3_remove_requests()
1047 while (!list_empty(&dep->pending_list)) { in dwc3_remove_requests()
1048 req = next_request(&dep->pending_list); in dwc3_remove_requests()
1053 while (!list_empty(&dep->cancelled_list)) { in dwc3_remove_requests()
1054 req = next_request(&dep->cancelled_list); in dwc3_remove_requests()
1061 * __dwc3_gadget_ep_disable - disables a hw endpoint
1062 * @dep: the endpoint to disable
1072 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_disable()
1079 if (dep->flags & DWC3_EP_STALL) in __dwc3_gadget_ep_disable()
1082 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); in __dwc3_gadget_ep_disable()
1083 reg &= ~DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_disable()
1084 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); in __dwc3_gadget_ep_disable()
1086 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); in __dwc3_gadget_ep_disable()
1088 dep->stream_capable = false; in __dwc3_gadget_ep_disable()
1089 dep->type = 0; in __dwc3_gadget_ep_disable()
1096 if (dep->flags & DWC3_EP_DELAY_STOP) in __dwc3_gadget_ep_disable()
1098 dep->flags &= mask; in __dwc3_gadget_ep_disable()
1100 /* Clear out the ep descriptors for non-ep0 */ in __dwc3_gadget_ep_disable()
1101 if (dep->number > 1) { in __dwc3_gadget_ep_disable()
1102 dep->endpoint.comp_desc = NULL; in __dwc3_gadget_ep_disable()
1103 dep->endpoint.desc = NULL; in __dwc3_gadget_ep_disable()
1109 /* -------------------------------------------------------------------------- */
1114 return -EINVAL; in dwc3_gadget_ep0_enable()
1119 return -EINVAL; in dwc3_gadget_ep0_disable()
1122 /* -------------------------------------------------------------------------- */
1132 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { in dwc3_gadget_ep_enable()
1134 return -EINVAL; in dwc3_gadget_ep_enable()
1137 if (!desc->wMaxPacketSize) { in dwc3_gadget_ep_enable()
1139 return -EINVAL; in dwc3_gadget_ep_enable()
1143 dwc = dep->dwc; in dwc3_gadget_ep_enable()
1145 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, in dwc3_gadget_ep_enable()
1147 dep->name)) in dwc3_gadget_ep_enable()
1150 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_enable()
1152 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_enable()
1166 return -EINVAL; in dwc3_gadget_ep_disable()
1170 dwc = dep->dwc; in dwc3_gadget_ep_disable()
1172 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), in dwc3_gadget_ep_disable()
1174 dep->name)) in dwc3_gadget_ep_disable()
1177 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_disable()
1179 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_disable()
1194 req->direction = dep->direction; in dwc3_gadget_ep_alloc_request()
1195 req->epnum = dep->number; in dwc3_gadget_ep_alloc_request()
1196 req->dep = dep; in dwc3_gadget_ep_alloc_request()
1197 req->status = DWC3_REQUEST_STATUS_UNKNOWN; in dwc3_gadget_ep_alloc_request()
1201 return &req->request; in dwc3_gadget_ep_alloc_request()
1214 * dwc3_ep_prev_trb - returns the previous TRB in the ring
1227 tmp = DWC3_TRB_NUM - 1; in dwc3_ep_prev_trb()
1229 return &dep->trb_pool[tmp - 1]; in dwc3_ep_prev_trb()
1238 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs in dwc3_calc_trbs_left()
1241 if (dep->trb_enqueue == dep->trb_dequeue) { in dwc3_calc_trbs_left()
1248 req = next_request(&dep->started_list); in dwc3_calc_trbs_left()
1249 if (req && req->num_trbs) in dwc3_calc_trbs_left()
1252 return DWC3_TRB_NUM - 1; in dwc3_calc_trbs_left()
1255 trbs_left = dep->trb_dequeue - dep->trb_enqueue; in dwc3_calc_trbs_left()
1256 trbs_left &= (DWC3_TRB_NUM - 1); in dwc3_calc_trbs_left()
1258 if (dep->trb_dequeue < dep->trb_enqueue) in dwc3_calc_trbs_left()
1259 trbs_left--; in dwc3_calc_trbs_left()
1265 * dwc3_prepare_one_trb - setup one TRB from one request
1281 unsigned int stream_id = req->request.stream_id; in dwc3_prepare_one_trb()
1282 unsigned int short_not_ok = req->request.short_not_ok; in dwc3_prepare_one_trb()
1283 unsigned int no_interrupt = req->request.no_interrupt; in dwc3_prepare_one_trb()
1284 unsigned int is_last = req->request.is_last; in dwc3_prepare_one_trb()
1285 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_one_trb()
1286 struct usb_gadget *gadget = dwc->gadget; in dwc3_prepare_one_trb()
1287 enum usb_device_speed speed = gadget->speed; in dwc3_prepare_one_trb()
1290 dma = dep->dwc->bounce_addr; in dwc3_prepare_one_trb()
1291 else if (req->request.num_sgs > 0) in dwc3_prepare_one_trb()
1292 dma = sg_dma_address(req->start_sg); in dwc3_prepare_one_trb()
1294 dma = req->request.dma; in dwc3_prepare_one_trb()
1296 trb = &dep->trb_pool[dep->trb_enqueue]; in dwc3_prepare_one_trb()
1298 if (!req->trb) { in dwc3_prepare_one_trb()
1300 req->trb = trb; in dwc3_prepare_one_trb()
1301 req->trb_dma = dwc3_trb_dma_offset(dep, trb); in dwc3_prepare_one_trb()
1304 req->num_trbs++; in dwc3_prepare_one_trb()
1306 trb->size = DWC3_TRB_SIZE_LENGTH(trb_length); in dwc3_prepare_one_trb()
1307 trb->bpl = lower_32_bits(dma); in dwc3_prepare_one_trb()
1308 trb->bph = upper_32_bits(dma); in dwc3_prepare_one_trb()
1310 switch (usb_endpoint_type(dep->endpoint.desc)) { in dwc3_prepare_one_trb()
1312 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP; in dwc3_prepare_one_trb()
1317 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; in dwc3_prepare_one_trb()
1333 * - DATA0 in dwc3_prepare_one_trb()
1336 * - DATA1, DATA0 in dwc3_prepare_one_trb()
1339 * - DATA2, DATA1, DATA0 in dwc3_prepare_one_trb()
1342 struct usb_ep *ep = &dep->endpoint; in dwc3_prepare_one_trb()
1344 unsigned int maxp = usb_endpoint_maxp(ep->desc); in dwc3_prepare_one_trb()
1346 if (req->request.length <= (2 * maxp)) in dwc3_prepare_one_trb()
1347 mult--; in dwc3_prepare_one_trb()
1349 if (req->request.length <= maxp) in dwc3_prepare_one_trb()
1350 mult--; in dwc3_prepare_one_trb()
1352 trb->size |= DWC3_TRB_SIZE_PCM1(mult); in dwc3_prepare_one_trb()
1355 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; in dwc3_prepare_one_trb()
1359 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; in dwc3_prepare_one_trb()
1364 trb->ctrl = DWC3_TRBCTL_NORMAL; in dwc3_prepare_one_trb()
1371 dev_WARN(dwc->dev, "Unknown endpoint type %d\n", in dwc3_prepare_one_trb()
1372 usb_endpoint_type(dep->endpoint.desc)); in dwc3_prepare_one_trb()
1379 if (usb_endpoint_dir_out(dep->endpoint.desc)) { in dwc3_prepare_one_trb()
1380 if (!dep->stream_capable) in dwc3_prepare_one_trb()
1381 trb->ctrl |= DWC3_TRB_CTRL_CSP; in dwc3_prepare_one_trb()
1384 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; in dwc3_prepare_one_trb()
1388 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams)) in dwc3_prepare_one_trb()
1389 trb->ctrl |= DWC3_TRB_CTRL_CSP; in dwc3_prepare_one_trb()
1392 trb->ctrl |= DWC3_TRB_CTRL_IOC; in dwc3_prepare_one_trb()
1395 trb->ctrl |= DWC3_TRB_CTRL_CHN; in dwc3_prepare_one_trb()
1396 else if (dep->stream_capable && is_last && in dwc3_prepare_one_trb()
1397 !DWC3_MST_CAPABLE(&dwc->hwparams)) in dwc3_prepare_one_trb()
1398 trb->ctrl |= DWC3_TRB_CTRL_LST; in dwc3_prepare_one_trb()
1400 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) in dwc3_prepare_one_trb()
1401 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id); in dwc3_prepare_one_trb()
1411 * However there is a possibility of CPU re-ordering here which can cause in dwc3_prepare_one_trb()
1413 * Add a write memory barrier to prevent CPU re-ordering. in dwc3_prepare_one_trb()
1416 trb->ctrl |= DWC3_TRB_CTRL_HWO; in dwc3_prepare_one_trb()
1425 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_needs_extra_trb()
1426 unsigned int rem = req->request.length % maxp; in dwc3_needs_extra_trb()
1428 if ((req->request.length && req->request.zero && !rem && in dwc3_needs_extra_trb()
1429 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) || in dwc3_needs_extra_trb()
1430 (!req->direction && rem)) in dwc3_needs_extra_trb()
1437 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1449 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_prepare_last_sg()
1450 unsigned int rem = req->request.length % maxp; in dwc3_prepare_last_sg()
1463 if (req->direction || req->request.length) in dwc3_prepare_last_sg()
1468 if ((!req->direction && !req->request.length) || needs_extra_trb) in dwc3_prepare_last_sg()
1470 req->direction ? 0 : maxp - rem, in dwc3_prepare_last_sg()
1479 struct scatterlist *sg = req->start_sg; in dwc3_prepare_trbs_sg()
1482 unsigned int length = req->request.length; in dwc3_prepare_trbs_sg()
1483 unsigned int remaining = req->num_pending_sgs; in dwc3_prepare_trbs_sg()
1484 unsigned int num_queued_sgs = req->request.num_mapped_sgs - remaining; in dwc3_prepare_trbs_sg()
1485 unsigned int num_trbs = req->num_trbs; in dwc3_prepare_trbs_sg()
1492 for_each_sg(req->request.sg, s, num_queued_sgs, i) in dwc3_prepare_trbs_sg()
1493 length -= sg_dma_len(s); in dwc3_prepare_trbs_sg()
1503 length -= trb_length; in dwc3_prepare_trbs_sg()
1512 if ((i == remaining - 1) || !length) in dwc3_prepare_trbs_sg()
1534 list_for_each_entry(r, &dep->started_list, list) { in dwc3_prepare_trbs_sg()
1535 if (r != req && !r->request.no_interrupt) in dwc3_prepare_trbs_sg()
1555 req->start_sg = sg_next(s); in dwc3_prepare_trbs_sg()
1557 req->num_pending_sgs--; in dwc3_prepare_trbs_sg()
1565 req->num_pending_sgs = 0; in dwc3_prepare_trbs_sg()
1573 return req->num_trbs - num_trbs; in dwc3_prepare_trbs_sg()
1579 return dwc3_prepare_last_sg(dep, req, req->request.length, 0); in dwc3_prepare_trbs_linear()
1583 * dwc3_prepare_trbs - setup TRBs from requests
1609 list_for_each_entry(req, &dep->started_list, list) { in dwc3_prepare_trbs()
1610 if (req->num_pending_sgs > 0) { in dwc3_prepare_trbs()
1612 if (!ret || req->num_pending_sgs) in dwc3_prepare_trbs()
1624 if (dep->stream_capable && req->request.is_last && in dwc3_prepare_trbs()
1625 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) in dwc3_prepare_trbs()
1629 list_for_each_entry_safe(req, n, &dep->pending_list, list) { in dwc3_prepare_trbs()
1630 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_trbs()
1632 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request, in dwc3_prepare_trbs()
1633 dep->direction); in dwc3_prepare_trbs()
1637 req->start_sg = req->request.sg; in dwc3_prepare_trbs()
1638 req->num_pending_sgs = req->request.num_mapped_sgs; in dwc3_prepare_trbs()
1640 if (req->num_pending_sgs > 0) { in dwc3_prepare_trbs()
1642 if (req->num_pending_sgs) in dwc3_prepare_trbs()
1656 if (dep->stream_capable && req->request.is_last && in dwc3_prepare_trbs()
1657 !DWC3_MST_CAPABLE(&dwc->hwparams)) in dwc3_prepare_trbs()
1683 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); in __dwc3_gadget_kick_transfer()
1692 req = next_request(&dep->started_list); in __dwc3_gadget_kick_transfer()
1694 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_kick_transfer()
1701 params.param0 = upper_32_bits(req->trb_dma); in __dwc3_gadget_kick_transfer()
1702 params.param1 = lower_32_bits(req->trb_dma); in __dwc3_gadget_kick_transfer()
1705 if (dep->stream_capable) in __dwc3_gadget_kick_transfer()
1706 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id); in __dwc3_gadget_kick_transfer()
1708 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_kick_transfer()
1709 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); in __dwc3_gadget_kick_transfer()
1712 DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_gadget_kick_transfer()
1719 if (ret == -EAGAIN) in __dwc3_gadget_kick_transfer()
1724 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_kick_transfer()
1728 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in __dwc3_gadget_kick_transfer()
1734 if (dep->stream_capable && req->request.is_last && in __dwc3_gadget_kick_transfer()
1735 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) in __dwc3_gadget_kick_transfer()
1736 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE; in __dwc3_gadget_kick_transfer()
1745 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in __dwc3_gadget_get_frame()
1750 * __dwc3_stop_active_transfer - stop the current active transfer
1769 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_stop_active_transfer()
1778 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) { in __dwc3_stop_active_transfer()
1779 dep->flags |= DWC3_EP_DELAY_STOP; in __dwc3_stop_active_transfer()
1784 dev_err_ratelimited(dep->dwc->dev, in __dwc3_stop_active_transfer()
1787 dep->resource_index = 0; in __dwc3_stop_active_transfer()
1790 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in __dwc3_stop_active_transfer()
1792 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; in __dwc3_stop_active_transfer()
1794 dep->flags &= ~DWC3_EP_DELAY_STOP; in __dwc3_stop_active_transfer()
1799 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1802 * This function tests for the correct combination of BIT[15:14] from the 16-bit
1806 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1807 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1811 * internal 16-bit microframe, the START TRANSFER command will pass and the
1814 * other conditions, the START TRANSFER command will fail with bus-expiry.
1819 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1822 * command status will result in a 2-second delay start. The smaller BIT[15:14]
1846 while (dep->combo_num < 2) { in dwc3_gadget_start_isoc_quirk()
1853 * 4 uframes in the future with BIT[15:14] as dep->combo_num in dwc3_gadget_start_isoc_quirk()
1855 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1856 test_frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1857 test_frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1859 params.param0 = upper_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1860 params.param1 = lower_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1866 /* Redo if some other failure beside bus-expiry is received */ in dwc3_gadget_start_isoc_quirk()
1867 if (cmd_status && cmd_status != -EAGAIN) { in dwc3_gadget_start_isoc_quirk()
1868 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1869 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1874 if (dep->combo_num == 0) in dwc3_gadget_start_isoc_quirk()
1875 dep->start_cmd_status = cmd_status; in dwc3_gadget_start_isoc_quirk()
1877 dep->combo_num++; in dwc3_gadget_start_isoc_quirk()
1890 test0 = (dep->start_cmd_status == 0); in dwc3_gadget_start_isoc_quirk()
1894 dep->combo_num = 1; in dwc3_gadget_start_isoc_quirk()
1896 dep->combo_num = 2; in dwc3_gadget_start_isoc_quirk()
1898 dep->combo_num = 3; in dwc3_gadget_start_isoc_quirk()
1900 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1902 dep->frame_number &= DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1903 dep->frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1904 dep->frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1907 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1908 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1915 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_start_isoc()
1916 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_start_isoc()
1920 if (list_empty(&dep->pending_list) && in __dwc3_gadget_start_isoc()
1921 list_empty(&dep->started_list)) { in __dwc3_gadget_start_isoc()
1922 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_start_isoc()
1923 return -EAGAIN; in __dwc3_gadget_start_isoc()
1926 if (!dwc->dis_start_transfer_quirk && in __dwc3_gadget_start_isoc()
1929 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction) in __dwc3_gadget_start_isoc()
1933 if (desc->bInterval <= 14 && in __dwc3_gadget_start_isoc()
1934 dwc->gadget->speed >= USB_SPEED_HIGH) { in __dwc3_gadget_start_isoc()
1937 (dep->frame_number & DWC3_FRNUMBER_MASK); in __dwc3_gadget_start_isoc()
1948 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) | in __dwc3_gadget_start_isoc()
1951 dep->frame_number += BIT(14); in __dwc3_gadget_start_isoc()
1958 if (desc->bInterval < 3) in __dwc3_gadget_start_isoc()
1959 future_interval += 3 - desc->bInterval; in __dwc3_gadget_start_isoc()
1961 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval); in __dwc3_gadget_start_isoc()
1964 if (ret != -EAGAIN) in __dwc3_gadget_start_isoc()
1969 * After a number of unsuccessful start attempts due to bus-expiry in __dwc3_gadget_start_isoc()
1973 if (ret == -EAGAIN) in __dwc3_gadget_start_isoc()
1981 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_queue()
1983 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { in __dwc3_gadget_ep_queue()
1984 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n", in __dwc3_gadget_ep_queue()
1985 dep->name); in __dwc3_gadget_ep_queue()
1986 return -ESHUTDOWN; in __dwc3_gadget_ep_queue()
1989 if (WARN(req->dep != dep, "request %p belongs to '%s'\n", in __dwc3_gadget_ep_queue()
1990 &req->request, req->dep->name)) in __dwc3_gadget_ep_queue()
1991 return -EINVAL; in __dwc3_gadget_ep_queue()
1993 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED, in __dwc3_gadget_ep_queue()
1995 dep->name, &req->request)) in __dwc3_gadget_ep_queue()
1996 return -EINVAL; in __dwc3_gadget_ep_queue()
1998 pm_runtime_get(dwc->dev); in __dwc3_gadget_ep_queue()
2000 req->request.actual = 0; in __dwc3_gadget_ep_queue()
2001 req->request.status = -EINPROGRESS; in __dwc3_gadget_ep_queue()
2005 list_add_tail(&req->list, &dep->pending_list); in __dwc3_gadget_ep_queue()
2006 req->status = DWC3_REQUEST_STATUS_QUEUED; in __dwc3_gadget_ep_queue()
2008 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE) in __dwc3_gadget_ep_queue()
2015 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || in __dwc3_gadget_ep_queue()
2016 (dep->flags & DWC3_EP_WEDGE) || in __dwc3_gadget_ep_queue()
2017 (dep->flags & DWC3_EP_DELAY_STOP) || in __dwc3_gadget_ep_queue()
2018 (dep->flags & DWC3_EP_STALL)) { in __dwc3_gadget_ep_queue()
2019 dep->flags |= DWC3_EP_DELAY_START; in __dwc3_gadget_ep_queue()
2026 * (micro-)frame number. in __dwc3_gadget_ep_queue()
2031 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_queue()
2032 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { in __dwc3_gadget_ep_queue()
2033 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) in __dwc3_gadget_ep_queue()
2050 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_queue()
2056 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_queue()
2058 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_queue()
2067 /* If req->trb is not set, then the request has not started */ in dwc3_gadget_ep_skip_trbs()
2068 if (!req->trb) in dwc3_gadget_ep_skip_trbs()
2081 for (i = 0; i < req->num_trbs; i++) { in dwc3_gadget_ep_skip_trbs()
2084 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_skip_trbs()
2085 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_skip_trbs()
2089 req->num_trbs = 0; in dwc3_gadget_ep_skip_trbs()
2095 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_cleanup_cancelled_requests()
2097 while (!list_empty(&dep->cancelled_list)) { in dwc3_gadget_ep_cleanup_cancelled_requests()
2098 req = next_request(&dep->cancelled_list); in dwc3_gadget_ep_cleanup_cancelled_requests()
2100 switch (req->status) { in dwc3_gadget_ep_cleanup_cancelled_requests()
2102 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); in dwc3_gadget_ep_cleanup_cancelled_requests()
2105 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
2108 dwc3_gadget_giveback(dep, req, -EPIPE); in dwc3_gadget_ep_cleanup_cancelled_requests()
2111 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status); in dwc3_gadget_ep_cleanup_cancelled_requests()
2112 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
2119 if (!dep->endpoint.desc) in dwc3_gadget_ep_cleanup_cancelled_requests()
2131 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_dequeue()
2138 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_dequeue()
2140 list_for_each_entry(r, &dep->cancelled_list, list) { in dwc3_gadget_ep_dequeue()
2145 list_for_each_entry(r, &dep->pending_list, list) { in dwc3_gadget_ep_dequeue()
2154 if (dep->number > 1) in dwc3_gadget_ep_dequeue()
2155 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_dequeue()
2162 list_for_each_entry(r, &dep->started_list, list) { in dwc3_gadget_ep_dequeue()
2173 list_for_each_entry_safe(r, t, &dep->started_list, list) in dwc3_gadget_ep_dequeue()
2177 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_ep_dequeue()
2183 dev_err(dwc->dev, "request %p was not queued to %s\n", in dwc3_gadget_ep_dequeue()
2184 request, ep->name); in dwc3_gadget_ep_dequeue()
2185 ret = -EINVAL; in dwc3_gadget_ep_dequeue()
2187 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_dequeue()
2195 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_set_halt()
2200 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_set_halt()
2201 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); in __dwc3_gadget_ep_set_halt()
2202 return -EINVAL; in __dwc3_gadget_ep_set_halt()
2213 if (dep->number > 1) in __dwc3_gadget_ep_set_halt()
2214 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); in __dwc3_gadget_ep_set_halt()
2216 trb = &dwc->ep0_trb[dep->trb_enqueue]; in __dwc3_gadget_ep_set_halt()
2218 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO; in __dwc3_gadget_ep_set_halt()
2219 started = !list_empty(&dep->started_list); in __dwc3_gadget_ep_set_halt()
2221 if (!protocol && ((dep->direction && transfer_in_flight) || in __dwc3_gadget_ep_set_halt()
2222 (!dep->direction && started))) { in __dwc3_gadget_ep_set_halt()
2223 return -EAGAIN; in __dwc3_gadget_ep_set_halt()
2229 dev_err(dwc->dev, "failed to set STALL on %s\n", in __dwc3_gadget_ep_set_halt()
2230 dep->name); in __dwc3_gadget_ep_set_halt()
2232 dep->flags |= DWC3_EP_STALL; in __dwc3_gadget_ep_set_halt()
2239 if (dep->number <= 1) { in __dwc3_gadget_ep_set_halt()
2240 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
2246 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_ep_set_halt()
2249 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING || in __dwc3_gadget_ep_set_halt()
2250 (dep->flags & DWC3_EP_DELAY_STOP)) { in __dwc3_gadget_ep_set_halt()
2251 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; in __dwc3_gadget_ep_set_halt()
2253 dwc->clear_stall_protocol = dep->number; in __dwc3_gadget_ep_set_halt()
2262 dev_err(dwc->dev, "failed to clear STALL on %s\n", in __dwc3_gadget_ep_set_halt()
2263 dep->name); in __dwc3_gadget_ep_set_halt()
2267 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
2269 if ((dep->flags & DWC3_EP_DELAY_START) && in __dwc3_gadget_ep_set_halt()
2270 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_ep_set_halt()
2273 dep->flags &= ~DWC3_EP_DELAY_START; in __dwc3_gadget_ep_set_halt()
2282 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_halt()
2288 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_set_halt()
2290 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_set_halt()
2298 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_wedge()
2302 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_set_wedge()
2303 dep->flags |= DWC3_EP_WEDGE; in dwc3_gadget_ep_set_wedge()
2305 if (dep->number == 0 || dep->number == 1) in dwc3_gadget_ep_set_wedge()
2309 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_set_wedge()
2314 /* -------------------------------------------------------------------------- */
2324 .disable = dwc3_gadget_ep0_disable,
2335 .disable = dwc3_gadget_ep_disable,
2344 /* -------------------------------------------------------------------------- */
2353 reg = dwc3_readl(dwc->regs, DWC3_DEVTEN); in dwc3_gadget_enable_linksts_evts()
2359 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); in dwc3_gadget_enable_linksts_evts()
2382 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in __dwc3_gadget_wakeup()
2395 return -EINVAL; in __dwc3_gadget_wakeup()
2402 dev_err(dwc->dev, "failed to put link in Recovery\n"); in __dwc3_gadget_wakeup()
2410 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in __dwc3_gadget_wakeup()
2412 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in __dwc3_gadget_wakeup()
2428 if (!dwc->wakeup_configured) { in dwc3_gadget_wakeup()
2429 dev_err(dwc->dev, "remote wakeup not configured\n"); in dwc3_gadget_wakeup()
2430 return -EINVAL; in dwc3_gadget_wakeup()
2433 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_wakeup()
2434 if (!dwc->gadget->wakeup_armed) { in dwc3_gadget_wakeup()
2435 dev_err(dwc->dev, "not armed for remote wakeup\n"); in dwc3_gadget_wakeup()
2436 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_wakeup()
2437 return -EINVAL; in dwc3_gadget_wakeup()
2441 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_wakeup()
2455 if (!dwc->wakeup_configured) { in dwc3_gadget_func_wakeup()
2456 dev_err(dwc->dev, "remote wakeup not configured\n"); in dwc3_gadget_func_wakeup()
2457 return -EINVAL; in dwc3_gadget_func_wakeup()
2460 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_func_wakeup()
2467 dwc->wakeup_pending_funcs |= BIT(intf_id); in dwc3_gadget_func_wakeup()
2469 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_func_wakeup()
2477 dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret); in dwc3_gadget_func_wakeup()
2479 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_func_wakeup()
2489 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_remote_wakeup()
2490 dwc->wakeup_configured = !!set; in dwc3_gadget_set_remote_wakeup()
2491 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_remote_wakeup()
2502 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_selfpowered()
2503 g->is_selfpowered = !!is_selfpowered; in dwc3_gadget_set_selfpowered()
2504 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_selfpowered()
2513 for (epnum = 2; epnum < dwc->num_eps; epnum++) { in dwc3_stop_active_transfers()
2516 dep = dwc->eps[epnum]; in dwc3_stop_active_transfers()
2520 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); in dwc3_stop_active_transfers()
2526 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate; in __dwc3_gadget_set_ssp_rate()
2530 ssp_rate = dwc->max_ssp_rate; in __dwc3_gadget_set_ssp_rate()
2532 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in __dwc3_gadget_set_ssp_rate()
2538 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2) in __dwc3_gadget_set_ssp_rate()
2542 dwc->max_ssp_rate != USB_SSP_GEN_2x1) in __dwc3_gadget_set_ssp_rate()
2545 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in __dwc3_gadget_set_ssp_rate()
2553 speed = dwc->gadget_max_speed; in __dwc3_gadget_set_speed()
2554 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed) in __dwc3_gadget_set_speed()
2555 speed = dwc->maximum_speed; in __dwc3_gadget_set_speed()
2563 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in __dwc3_gadget_set_speed()
2569 * bit if we try to force the IP to USB2-only mode. in __dwc3_gadget_set_speed()
2577 * USB 2.0 Mode in __dwc3_gadget_set_speed()
2580 !dwc->dis_metastability_quirk) { in __dwc3_gadget_set_speed()
2600 dev_err(dwc->dev, "invalid speed (%d)\n", speed); in __dwc3_gadget_set_speed()
2614 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in __dwc3_gadget_set_speed()
2623 if (pm_runtime_suspended(dwc->dev)) in dwc3_gadget_run_stop()
2639 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_gadget_run_stop()
2651 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_gadget_run_stop()
2653 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_run_stop()
2665 dwc->pullups_connected = true; in dwc3_gadget_run_stop()
2669 dwc->pullups_connected = false; in dwc3_gadget_run_stop()
2677 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_run_stop()
2679 } while (--timeout && !(!is_on ^ !reg)); in dwc3_gadget_run_stop()
2682 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_gadget_run_stop()
2684 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_gadget_run_stop()
2688 return -ETIMEDOUT; in dwc3_gadget_run_stop()
2702 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2703 if (!dwc->pullups_connected) { in dwc3_gadget_soft_disconnect()
2704 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2708 dwc->connected = false; in dwc3_gadget_soft_disconnect()
2714 if (dwc->delayed_status) in dwc3_gadget_soft_disconnect()
2719 * Section 4.1.8 Table 4-7, it states that for a device-initiated in dwc3_gadget_soft_disconnect()
2725 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2734 if (dwc->ep0state != EP0_SETUP_PHASE) { in dwc3_gadget_soft_disconnect()
2735 reinit_completion(&dwc->ep0_in_setup); in dwc3_gadget_soft_disconnect()
2737 ret = wait_for_completion_timeout(&dwc->ep0_in_setup, in dwc3_gadget_soft_disconnect()
2740 dev_warn(dwc->dev, "wait for SETUP phase timed out\n"); in dwc3_gadget_soft_disconnect()
2741 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2743 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2762 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2764 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2766 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); in dwc3_gadget_soft_disconnect()
2778 * device-initiated disconnect requires a core soft reset in dwc3_gadget_soft_connect()
2797 dwc->softconnect = is_on; in dwc3_gadget_pullup()
2805 pm_runtime_barrier(dwc->dev); in dwc3_gadget_pullup()
2806 if (pm_runtime_suspended(dwc->dev)) in dwc3_gadget_pullup()
2815 ret = pm_runtime_get_sync(dwc->dev); in dwc3_gadget_pullup()
2817 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2819 pm_runtime_set_suspended(dwc->dev); in dwc3_gadget_pullup()
2823 if (dwc->pullups_connected == is_on) { in dwc3_gadget_pullup()
2824 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2828 synchronize_irq(dwc->irq_gadget); in dwc3_gadget_pullup()
2835 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2856 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */ in dwc3_gadget_enable_irq()
2860 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); in dwc3_gadget_enable_irq()
2866 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00); in dwc3_gadget_disable_irq()
2873 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2886 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2900 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7); in dwc3_gadget_setup_nump()
2903 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024; in dwc3_gadget_setup_nump()
2907 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_setup_nump()
2910 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_setup_nump()
2920 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if in __dwc3_gadget_start()
2921 * the core supports IMOD, disable it. in __dwc3_gadget_start()
2923 if (dwc->imod_interval) { in __dwc3_gadget_start()
2924 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); in __dwc3_gadget_start()
2925 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); in __dwc3_gadget_start()
2927 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0); in __dwc3_gadget_start()
2937 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG); in __dwc3_gadget_start()
2943 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg); in __dwc3_gadget_start()
2954 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in __dwc3_gadget_start()
2956 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in __dwc3_gadget_start()
2959 if (DWC3_MST_CAPABLE(&dwc->hwparams)) { in __dwc3_gadget_start()
2960 reg = dwc3_readl(dwc->regs, DWC3_DCFG1); in __dwc3_gadget_start()
2962 dwc3_writel(dwc->regs, DWC3_DCFG1, reg); in __dwc3_gadget_start()
2970 dev_err(dwc->dev, "failed to config endpoints\n"); in __dwc3_gadget_start()
2974 dep = dwc->eps[0]; in __dwc3_gadget_start()
2975 dep->flags = 0; in __dwc3_gadget_start()
2978 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
2982 dep = dwc->eps[1]; in __dwc3_gadget_start()
2983 dep->flags = 0; in __dwc3_gadget_start()
2986 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
2991 dwc->ep0state = EP0_SETUP_PHASE; in __dwc3_gadget_start()
2992 dwc->ep0_bounced = false; in __dwc3_gadget_start()
2993 dwc->link_state = DWC3_LINK_STATE_SS_DIS; in __dwc3_gadget_start()
2994 dwc->delayed_status = false; in __dwc3_gadget_start()
3003 __dwc3_gadget_ep_disable(dwc->eps[0]); in __dwc3_gadget_start()
3017 irq = dwc->irq_gadget; in dwc3_gadget_start()
3019 IRQF_SHARED, "dwc3", dwc->ev_buf); in dwc3_gadget_start()
3021 dev_err(dwc->dev, "failed to request irq #%d --> %d\n", in dwc3_gadget_start()
3026 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_start()
3027 dwc->gadget_driver = driver; in dwc3_gadget_start()
3028 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_start()
3030 if (dwc->sys_wakeup) in dwc3_gadget_start()
3031 device_wakeup_enable(dwc->sysdev); in dwc3_gadget_start()
3039 __dwc3_gadget_ep_disable(dwc->eps[0]); in __dwc3_gadget_stop()
3040 __dwc3_gadget_ep_disable(dwc->eps[1]); in __dwc3_gadget_stop()
3048 if (dwc->sys_wakeup) in dwc3_gadget_stop()
3049 device_wakeup_disable(dwc->sysdev); in dwc3_gadget_stop()
3051 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_stop()
3052 dwc->gadget_driver = NULL; in dwc3_gadget_stop()
3053 dwc->max_cfg_eps = 0; in dwc3_gadget_stop()
3054 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_stop()
3056 free_irq(dwc->irq_gadget, dwc->ev_buf); in dwc3_gadget_stop()
3066 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED; in dwc3_gadget_config_params()
3067 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED; in dwc3_gadget_config_params()
3070 if (!dwc->dis_enblslpm_quirk) { in dwc3_gadget_config_params()
3080 params->besl_baseline = 1; in dwc3_gadget_config_params()
3081 if (dwc->is_utmi_l1_suspend) in dwc3_gadget_config_params()
3082 params->besl_deep = in dwc3_gadget_config_params()
3083 clamp_t(u8, dwc->hird_threshold, 2, 15); in dwc3_gadget_config_params()
3087 if (dwc->dis_u1_entry_quirk) in dwc3_gadget_config_params()
3088 params->bU1devExitLat = 0; in dwc3_gadget_config_params()
3090 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT; in dwc3_gadget_config_params()
3093 if (dwc->dis_u2_entry_quirk) in dwc3_gadget_config_params()
3094 params->bU2DevExitLat = 0; in dwc3_gadget_config_params()
3096 params->bU2DevExitLat = in dwc3_gadget_config_params()
3106 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_speed()
3107 dwc->gadget_max_speed = speed; in dwc3_gadget_set_speed()
3108 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_speed()
3117 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_ssp_rate()
3118 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS; in dwc3_gadget_set_ssp_rate()
3119 dwc->gadget_ssp_rate = rate; in dwc3_gadget_set_ssp_rate()
3120 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_ssp_rate()
3129 if (dwc->usb2_phy) in dwc3_gadget_vbus_draw()
3130 return usb_phy_set_power(dwc->usb2_phy, mA); in dwc3_gadget_vbus_draw()
3132 if (!dwc->usb_psy) in dwc3_gadget_vbus_draw()
3133 return -EOPNOTSUPP; in dwc3_gadget_vbus_draw()
3136 ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); in dwc3_gadget_vbus_draw()
3142 * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3159 if (!dwc->do_fifo_resize) in dwc3_gadget_check_config()
3162 list_for_each_entry(ep, &g->ep_list, ep_list) { in dwc3_gadget_check_config()
3164 if (ep->claimed && (ep->address & USB_DIR_IN)) in dwc3_gadget_check_config()
3168 if (ep_num <= dwc->max_cfg_eps) in dwc3_gadget_check_config()
3172 dwc->max_cfg_eps = ep_num; in dwc3_gadget_check_config()
3174 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps); in dwc3_gadget_check_config()
3176 fifo_size += dwc->max_cfg_eps; in dwc3_gadget_check_config()
3181 return -ENOMEM; in dwc3_gadget_check_config()
3191 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_async_callbacks()
3192 dwc->async_callbacks = enable; in dwc3_gadget_async_callbacks()
3193 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_async_callbacks()
3213 /* -------------------------------------------------------------------------- */
3217 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_control_endpoint()
3219 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); in dwc3_gadget_init_control_endpoint()
3220 dep->endpoint.maxburst = 1; in dwc3_gadget_init_control_endpoint()
3221 dep->endpoint.ops = &dwc3_gadget_ep0_ops; in dwc3_gadget_init_control_endpoint()
3222 if (!dep->direction) in dwc3_gadget_init_control_endpoint()
3223 dwc->gadget->ep0 = &dep->endpoint; in dwc3_gadget_init_control_endpoint()
3225 dep->endpoint.caps.type_control = true; in dwc3_gadget_init_control_endpoint()
3232 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_in_endpoint()
3242 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); in dwc3_gadget_init_in_endpoint()
3253 * maxpacket = mdwidth * (fifo_size - 1); in dwc3_gadget_init_in_endpoint()
3257 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth; in dwc3_gadget_init_in_endpoint()
3260 maxpacket = mdwidth * (size - 1); in dwc3_gadget_init_in_endpoint()
3262 maxpacket = mdwidth * ((size - 1) - 1) - mdwidth; in dwc3_gadget_init_in_endpoint()
3266 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_in_endpoint()
3268 dep->endpoint.max_streams = 16; in dwc3_gadget_init_in_endpoint()
3269 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_in_endpoint()
3270 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_in_endpoint()
3271 &dwc->gadget->ep_list); in dwc3_gadget_init_in_endpoint()
3272 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_in_endpoint()
3273 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_in_endpoint()
3274 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_in_endpoint()
3281 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_out_endpoint()
3291 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0)); in dwc3_gadget_init_out_endpoint()
3308 size -= (3 * 8) + 16; in dwc3_gadget_init_out_endpoint()
3314 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_out_endpoint()
3315 dep->endpoint.max_streams = 16; in dwc3_gadget_init_out_endpoint()
3316 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_out_endpoint()
3317 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_out_endpoint()
3318 &dwc->gadget->ep_list); in dwc3_gadget_init_out_endpoint()
3319 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_out_endpoint()
3320 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_out_endpoint()
3321 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_out_endpoint()
3330 struct dwc3 *dwc = dep->dwc; in dwc3_nostream_work()
3333 spin_lock_irqsave(&dwc->lock, flags); in dwc3_nostream_work()
3334 if (dep->flags & DWC3_EP_STREAM_PRIMED) in dwc3_nostream_work()
3337 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) || in dwc3_nostream_work()
3338 (!DWC3_MST_CAPABLE(&dwc->hwparams) && in dwc3_nostream_work()
3339 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))) in dwc3_nostream_work()
3358 dwc3_send_gadget_generic_command(dwc, cmd, dep->number); in dwc3_nostream_work()
3360 dep->flags |= DWC3_EP_DELAY_START; in dwc3_nostream_work()
3362 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_nostream_work()
3366 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_nostream_work()
3367 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_nostream_work()
3379 return -ENOMEM; in dwc3_gadget_init_endpoint()
3381 dep->dwc = dwc; in dwc3_gadget_init_endpoint()
3382 dep->number = epnum; in dwc3_gadget_init_endpoint()
3383 dep->direction = direction; in dwc3_gadget_init_endpoint()
3384 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); in dwc3_gadget_init_endpoint()
3385 dwc->eps[epnum] = dep; in dwc3_gadget_init_endpoint()
3386 dep->combo_num = 0; in dwc3_gadget_init_endpoint()
3387 dep->start_cmd_status = 0; in dwc3_gadget_init_endpoint()
3389 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, in dwc3_gadget_init_endpoint()
3392 dep->endpoint.name = dep->name; in dwc3_gadget_init_endpoint()
3394 if (!(dep->number > 1)) { in dwc3_gadget_init_endpoint()
3395 dep->endpoint.desc = &dwc3_gadget_ep0_desc; in dwc3_gadget_init_endpoint()
3396 dep->endpoint.comp_desc = NULL; in dwc3_gadget_init_endpoint()
3409 dep->endpoint.caps.dir_in = direction; in dwc3_gadget_init_endpoint()
3410 dep->endpoint.caps.dir_out = !direction; in dwc3_gadget_init_endpoint()
3412 INIT_LIST_HEAD(&dep->pending_list); in dwc3_gadget_init_endpoint()
3413 INIT_LIST_HEAD(&dep->started_list); in dwc3_gadget_init_endpoint()
3414 INIT_LIST_HEAD(&dep->cancelled_list); in dwc3_gadget_init_endpoint()
3415 INIT_DELAYED_WORK(&dep->nostream_work, dwc3_nostream_work); in dwc3_gadget_init_endpoint()
3428 if (!device_property_present(dwc->dev, propname)) in dwc3_gadget_get_reserved_endpoints()
3431 ret = device_property_count_u8(dwc->dev, propname); in dwc3_gadget_get_reserved_endpoints()
3436 ret = device_property_read_u8_array(dwc->dev, propname, eps, min(num, count)); in dwc3_gadget_get_reserved_endpoints()
3445 const char *propname = "snps,reserved-endpoints"; in dwc3_gadget_init_endpoints()
3452 INIT_LIST_HEAD(&dwc->gadget->ep_list); in dwc3_gadget_init_endpoints()
3457 dev_err(dwc->dev, "failed to read %s\n", propname); in dwc3_gadget_init_endpoints()
3484 dep = dwc->eps[epnum]; in dwc3_gadget_free_endpoints()
3489 * bi-directional USB endpoint 0. in dwc3_gadget_free_endpoints()
3498 list_del(&dep->endpoint.ep_list); in dwc3_gadget_free_endpoints()
3506 /* -------------------------------------------------------------------------- */
3517 req->num_trbs--; in dwc3_gadget_ep_reclaim_completed_trb()
3529 if (trb->ctrl & DWC3_TRB_CTRL_HWO) in dwc3_gadget_ep_reclaim_completed_trb()
3530 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_reclaim_completed_trb()
3534 * have the Isoc-First type. Track and report its interval frame number. in dwc3_gadget_ep_reclaim_completed_trb()
3536 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_ep_reclaim_completed_trb()
3537 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) { in dwc3_gadget_ep_reclaim_completed_trb()
3540 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl); in dwc3_gadget_ep_reclaim_completed_trb()
3541 frame_number &= ~(dep->interval - 1); in dwc3_gadget_ep_reclaim_completed_trb()
3542 req->request.frame_number = frame_number; in dwc3_gadget_ep_reclaim_completed_trb()
3548 * TRB. Don't add it to req->remaining calculation. in dwc3_gadget_ep_reclaim_completed_trb()
3550 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) && in dwc3_gadget_ep_reclaim_completed_trb()
3551 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) { in dwc3_gadget_ep_reclaim_completed_trb()
3552 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_reclaim_completed_trb()
3556 count = trb->size & DWC3_TRB_SIZE_MASK; in dwc3_gadget_ep_reclaim_completed_trb()
3557 req->remaining += count; in dwc3_gadget_ep_reclaim_completed_trb()
3559 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) in dwc3_gadget_ep_reclaim_completed_trb()
3562 if (event->status & DEPEVT_STATUS_SHORT && in dwc3_gadget_ep_reclaim_completed_trb()
3563 !(trb->ctrl & DWC3_TRB_CTRL_CHN)) in dwc3_gadget_ep_reclaim_completed_trb()
3566 if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) && in dwc3_gadget_ep_reclaim_completed_trb()
3567 DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC) in dwc3_gadget_ep_reclaim_completed_trb()
3570 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) || in dwc3_gadget_ep_reclaim_completed_trb()
3571 (trb->ctrl & DWC3_TRB_CTRL_LST)) in dwc3_gadget_ep_reclaim_completed_trb()
3582 unsigned int num_completed_trbs = req->num_trbs; in dwc3_gadget_ep_reclaim_trb_sg()
3587 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_sg()
3600 return req->num_pending_sgs == 0 && req->num_trbs == 0; in dwc3_gadget_ep_request_completed()
3612 req->request.actual = req->request.length - req->remaining; in dwc3_gadget_ep_cleanup_completed_request()
3623 if (req->request.no_interrupt) { in dwc3_gadget_ep_cleanup_completed_request()
3626 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue); in dwc3_gadget_ep_cleanup_completed_request()
3627 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) { in dwc3_gadget_ep_cleanup_completed_request()
3630 request_status = -EXDEV; in dwc3_gadget_ep_cleanup_completed_request()
3656 while (!list_empty(&dep->started_list)) { in dwc3_gadget_ep_cleanup_completed_requests()
3659 req = next_request(&dep->started_list); in dwc3_gadget_ep_cleanup_completed_requests()
3668 if (!dep->endpoint.desc) in dwc3_gadget_ep_cleanup_completed_requests()
3676 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_should_continue()
3678 if (!dep->endpoint.desc || !dwc->pullups_connected || in dwc3_gadget_ep_should_continue()
3679 !dwc->connected) in dwc3_gadget_ep_should_continue()
3682 if (!list_empty(&dep->pending_list)) in dwc3_gadget_ep_should_continue()
3689 req = next_request(&dep->started_list); in dwc3_gadget_ep_should_continue()
3699 dep->frame_number = event->parameters; in dwc3_gadget_endpoint_frame_from_event()
3705 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_trbs_complete()
3710 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_trbs_complete()
3713 if (!dep->endpoint.desc) in dwc3_gadget_endpoint_trbs_complete()
3716 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_endpoint_trbs_complete()
3717 list_empty(&dep->started_list) && in dwc3_gadget_endpoint_trbs_complete()
3718 (list_empty(&dep->pending_list) || status == -EXDEV)) in dwc3_gadget_endpoint_trbs_complete()
3726 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. in dwc3_gadget_endpoint_trbs_complete()
3734 dep = dwc->eps[i]; in dwc3_gadget_endpoint_trbs_complete()
3738 if (!(dep->flags & DWC3_EP_ENABLED)) in dwc3_gadget_endpoint_trbs_complete()
3741 if (!list_empty(&dep->started_list)) in dwc3_gadget_endpoint_trbs_complete()
3745 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_endpoint_trbs_complete()
3746 reg |= dwc->u1u2; in dwc3_gadget_endpoint_trbs_complete()
3747 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_endpoint_trbs_complete()
3749 dwc->u1u2 = 0; in dwc3_gadget_endpoint_trbs_complete()
3760 if (!dep->endpoint.desc) in dwc3_gadget_endpoint_transfer_in_progress()
3763 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_transfer_in_progress()
3766 if (event->status & DEPEVT_STATUS_BUSERR) in dwc3_gadget_endpoint_transfer_in_progress()
3767 status = -ECONNRESET; in dwc3_gadget_endpoint_transfer_in_progress()
3769 if (event->status & DEPEVT_STATUS_MISSED_ISOC) in dwc3_gadget_endpoint_transfer_in_progress()
3770 status = -EXDEV; in dwc3_gadget_endpoint_transfer_in_progress()
3780 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_transfer_complete()
3782 if (event->status & DEPEVT_STATUS_BUSERR) in dwc3_gadget_endpoint_transfer_complete()
3783 status = -ECONNRESET; in dwc3_gadget_endpoint_transfer_complete()
3786 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_endpoint_transfer_complete()
3793 * During a device-initiated disconnect, a late xferNotReady event can in dwc3_gadget_endpoint_transfer_not_ready()
3798 if (!dep->dwc->connected) in dwc3_gadget_endpoint_transfer_not_ready()
3811 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_transfer_not_ready()
3820 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters); in dwc3_gadget_endpoint_command_complete()
3830 if (dep->stream_capable) in dwc3_gadget_endpoint_command_complete()
3831 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_command_complete()
3833 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; in dwc3_gadget_endpoint_command_complete()
3834 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_command_complete()
3837 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) { in dwc3_gadget_endpoint_command_complete()
3838 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_command_complete()
3840 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL; in dwc3_gadget_endpoint_command_complete()
3842 struct usb_ep *ep0 = &dwc->eps[0]->endpoint; in dwc3_gadget_endpoint_command_complete()
3844 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name); in dwc3_gadget_endpoint_command_complete()
3845 if (dwc->delayed_status) in dwc3_gadget_endpoint_command_complete()
3850 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in dwc3_gadget_endpoint_command_complete()
3851 if (dwc->clear_stall_protocol == dep->number) in dwc3_gadget_endpoint_command_complete()
3855 if ((dep->flags & DWC3_EP_DELAY_START) && in dwc3_gadget_endpoint_command_complete()
3856 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_command_complete()
3859 dep->flags &= ~DWC3_EP_DELAY_START; in dwc3_gadget_endpoint_command_complete()
3865 if (event->status == DEPEVT_STREAMEVT_FOUND) { in dwc3_gadget_endpoint_stream_event()
3866 cancel_delayed_work(&dep->nostream_work); in dwc3_gadget_endpoint_stream_event()
3867 dep->flags |= DWC3_EP_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3868 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_stream_event()
3873 switch (event->parameters) { in dwc3_gadget_endpoint_stream_event()
3875 cancel_delayed_work(&dep->nostream_work); in dwc3_gadget_endpoint_stream_event()
3876 dep->flags |= DWC3_EP_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3877 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_stream_event()
3880 dep->flags &= ~DWC3_EP_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3881 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) in dwc3_gadget_endpoint_stream_event()
3882 queue_delayed_work(system_wq, &dep->nostream_work, in dwc3_gadget_endpoint_stream_event()
3892 u8 epnum = event->endpoint_number; in dwc3_endpoint_interrupt()
3894 dep = dwc->eps[epnum]; in dwc3_endpoint_interrupt()
3896 dev_warn(dwc->dev, "spurious event, endpoint %u is not allocated\n", epnum); in dwc3_endpoint_interrupt()
3900 if (!(dep->flags & DWC3_EP_ENABLED)) { in dwc3_endpoint_interrupt()
3901 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED)) in dwc3_endpoint_interrupt()
3905 if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) && in dwc3_endpoint_interrupt()
3906 !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE)) in dwc3_endpoint_interrupt()
3915 switch (event->endpoint_event) { in dwc3_endpoint_interrupt()
3934 dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event); in dwc3_endpoint_interrupt()
3941 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) { in dwc3_disconnect_gadget()
3942 spin_unlock(&dwc->lock); in dwc3_disconnect_gadget()
3943 dwc->gadget_driver->disconnect(dwc->gadget); in dwc3_disconnect_gadget()
3944 spin_lock(&dwc->lock); in dwc3_disconnect_gadget()
3950 if (dwc->async_callbacks && dwc->gadget_driver->suspend) { in dwc3_suspend_gadget()
3951 spin_unlock(&dwc->lock); in dwc3_suspend_gadget()
3952 dwc->gadget_driver->suspend(dwc->gadget); in dwc3_suspend_gadget()
3953 spin_lock(&dwc->lock); in dwc3_suspend_gadget()
3959 if (dwc->async_callbacks && dwc->gadget_driver->resume) { in dwc3_resume_gadget()
3960 spin_unlock(&dwc->lock); in dwc3_resume_gadget()
3961 dwc->gadget_driver->resume(dwc->gadget); in dwc3_resume_gadget()
3962 spin_lock(&dwc->lock); in dwc3_resume_gadget()
3968 if (!dwc->gadget_driver) in dwc3_reset_gadget()
3971 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) { in dwc3_reset_gadget()
3972 spin_unlock(&dwc->lock); in dwc3_reset_gadget()
3973 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver); in dwc3_reset_gadget()
3974 spin_lock(&dwc->lock); in dwc3_reset_gadget()
3981 struct dwc3 *dwc = dep->dwc; in dwc3_stop_active_transfer()
3989 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE) in dwc3_stop_active_transfer()
3992 if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP)) in dwc3_stop_active_transfer()
3995 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || in dwc3_stop_active_transfer()
3996 (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in dwc3_stop_active_transfer()
4006 if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) { in dwc3_stop_active_transfer()
4007 dep->flags |= DWC3_EP_DELAY_STOP; in dwc3_stop_active_transfer()
4027 * supports a mode to work around the above limitation. The in dwc3_stop_active_transfer()
4029 * after issuing a EndTransfer command. This mode is enabled in dwc3_stop_active_transfer()
4031 * dwc3_send_gadget_ep_cmd() function so if the mode is in dwc3_stop_active_transfer()
4035 * This mode is NOT available on the DWC_usb31 IP. In this in dwc3_stop_active_transfer()
4053 dep = dwc->eps[epnum]; in dwc3_clear_stall_all_ep()
4057 if (!(dep->flags & DWC3_EP_STALL)) in dwc3_clear_stall_all_ep()
4060 dep->flags &= ~DWC3_EP_STALL; in dwc3_clear_stall_all_ep()
4064 dev_err_ratelimited(dwc->dev, in dwc3_clear_stall_all_ep()
4065 "failed to clear STALL on %s\n", dep->name); in dwc3_clear_stall_all_ep()
4073 dwc->suspended = false; in dwc3_gadget_disconnect_interrupt()
4077 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_disconnect_interrupt()
4082 dwc->connected = false; in dwc3_gadget_disconnect_interrupt()
4086 dwc->gadget->speed = USB_SPEED_UNKNOWN; in dwc3_gadget_disconnect_interrupt()
4087 dwc->setup_packet_pending = false; in dwc3_gadget_disconnect_interrupt()
4088 dwc->gadget->wakeup_armed = false; in dwc3_gadget_disconnect_interrupt()
4090 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); in dwc3_gadget_disconnect_interrupt()
4097 * interrupt to set dwc->connected to FALSE. in dwc3_gadget_disconnect_interrupt()
4099 pm_request_idle(dwc->dev); in dwc3_gadget_disconnect_interrupt()
4106 dwc->suspended = false; in dwc3_gadget_reset_interrupt()
4110 * drivers to stop any active transfers through ep disable. in dwc3_gadget_reset_interrupt()
4111 * However, for functions which defer ep disable, such as mass in dwc3_gadget_reset_interrupt()
4115 dwc->connected = false; in dwc3_gadget_reset_interrupt()
4144 if (dwc->setup_packet_pending) in dwc3_gadget_reset_interrupt()
4159 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW in dwc3_gadget_reset_interrupt()
4164 dwc->connected = true; in dwc3_gadget_reset_interrupt()
4166 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_reset_interrupt()
4169 dwc->test_mode = false; in dwc3_gadget_reset_interrupt()
4170 dwc->gadget->wakeup_armed = false; in dwc3_gadget_reset_interrupt()
4175 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_reset_interrupt()
4177 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_reset_interrupt()
4188 if (!dwc->softconnect) in dwc3_gadget_conndone_interrupt()
4191 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_conndone_interrupt()
4193 dwc->speed = speed; in dwc3_gadget_conndone_interrupt()
4198 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; in dwc3_gadget_conndone_interrupt()
4212 dwc->gadget->ep0->maxpacket = 512; in dwc3_gadget_conndone_interrupt()
4213 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; in dwc3_gadget_conndone_interrupt()
4216 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2; in dwc3_gadget_conndone_interrupt()
4218 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1; in dwc3_gadget_conndone_interrupt()
4238 dwc->gadget->ep0->maxpacket = 512; in dwc3_gadget_conndone_interrupt()
4239 dwc->gadget->speed = USB_SPEED_SUPER; in dwc3_gadget_conndone_interrupt()
4242 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; in dwc3_gadget_conndone_interrupt()
4243 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2; in dwc3_gadget_conndone_interrupt()
4248 dwc->gadget->ep0->maxpacket = 64; in dwc3_gadget_conndone_interrupt()
4249 dwc->gadget->speed = USB_SPEED_HIGH; in dwc3_gadget_conndone_interrupt()
4253 dwc->gadget->ep0->maxpacket = 64; in dwc3_gadget_conndone_interrupt()
4254 dwc->gadget->speed = USB_SPEED_FULL; in dwc3_gadget_conndone_interrupt()
4258 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket; in dwc3_gadget_conndone_interrupt()
4263 !dwc->usb2_gadget_lpm_disable && in dwc3_gadget_conndone_interrupt()
4266 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_conndone_interrupt()
4268 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_conndone_interrupt()
4270 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_conndone_interrupt()
4273 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold | in dwc3_gadget_conndone_interrupt()
4274 (dwc->is_utmi_l1_suspend << 4)); in dwc3_gadget_conndone_interrupt()
4282 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum, in dwc3_gadget_conndone_interrupt()
4285 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A)) { in dwc3_gadget_conndone_interrupt()
4287 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold); in dwc3_gadget_conndone_interrupt()
4292 if (dwc->usb2_gadget_lpm_disable) { in dwc3_gadget_conndone_interrupt()
4293 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_conndone_interrupt()
4295 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_conndone_interrupt()
4298 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_conndone_interrupt()
4303 dep = dwc->eps[0]; in dwc3_gadget_conndone_interrupt()
4306 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()
4310 dep = dwc->eps[1]; in dwc3_gadget_conndone_interrupt()
4313 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()
4328 dwc->suspended = false; in dwc3_gadget_wakeup_interrupt()
4331 * TODO take core out of low power mode when that's in dwc3_gadget_wakeup_interrupt()
4335 if (dwc->async_callbacks && dwc->gadget_driver->resume) { in dwc3_gadget_wakeup_interrupt()
4336 spin_unlock(&dwc->lock); in dwc3_gadget_wakeup_interrupt()
4337 dwc->gadget_driver->resume(dwc->gadget); in dwc3_gadget_wakeup_interrupt()
4338 spin_lock(&dwc->lock); in dwc3_gadget_wakeup_interrupt()
4341 dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK; in dwc3_gadget_wakeup_interrupt()
4354 * Hibernation mode enabled which would show up when device detects in dwc3_gadget_linksts_change_interrupt()
4355 * host-initiated U3 exit. in dwc3_gadget_linksts_change_interrupt()
4358 * from U3 to RESUME which is only necessary if Hibernation is in dwc3_gadget_linksts_change_interrupt()
4366 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation in dwc3_gadget_linksts_change_interrupt()
4367 * operational mode in dwc3_gadget_linksts_change_interrupt()
4369 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1); in dwc3_gadget_linksts_change_interrupt()
4372 if ((dwc->link_state == DWC3_LINK_STATE_U3) && in dwc3_gadget_linksts_change_interrupt()
4384 * suggested workaround is to disable DCTL[12:9] bits if we're in dwc3_gadget_linksts_change_interrupt()
4393 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us in dwc3_gadget_linksts_change_interrupt()
4401 switch (dwc->link_state) { in dwc3_gadget_linksts_change_interrupt()
4404 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_linksts_change_interrupt()
4410 if (!dwc->u1u2) in dwc3_gadget_linksts_change_interrupt()
4411 dwc->u1u2 = reg & u1u2; in dwc3_gadget_linksts_change_interrupt()
4426 if (dwc->gadget->wakeup_armed || dwc->wakeup_pending_funcs) { in dwc3_gadget_linksts_change_interrupt()
4429 dwc->suspended = false; in dwc3_gadget_linksts_change_interrupt()
4433 if (dwc->speed == USB_SPEED_SUPER) in dwc3_gadget_linksts_change_interrupt()
4448 dwc->link_state = next; in dwc3_gadget_linksts_change_interrupt()
4451 while (dwc->wakeup_pending_funcs && (next == DWC3_LINK_STATE_U0)) { in dwc3_gadget_linksts_change_interrupt()
4452 intf_id = ffs(dwc->wakeup_pending_funcs) - 1; in dwc3_gadget_linksts_change_interrupt()
4457 dev_err(dwc->dev, "Failed to send DN wake for intf %d\n", intf_id); in dwc3_gadget_linksts_change_interrupt()
4459 dwc->wakeup_pending_funcs &= ~BIT(intf_id); in dwc3_gadget_linksts_change_interrupt()
4468 if (!dwc->suspended && next == DWC3_LINK_STATE_U3) { in dwc3_gadget_suspend_interrupt()
4469 dwc->suspended = true; in dwc3_gadget_suspend_interrupt()
4473 dwc->link_state = next; in dwc3_gadget_suspend_interrupt()
4479 switch (event->type) { in dwc3_gadget_interrupt()
4490 dwc3_gadget_wakeup_interrupt(dwc, event->event_info); in dwc3_gadget_interrupt()
4493 dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n"); in dwc3_gadget_interrupt()
4496 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info); in dwc3_gadget_interrupt()
4501 dwc3_gadget_suspend_interrupt(dwc, event->event_info); in dwc3_gadget_interrupt()
4509 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type); in dwc3_gadget_interrupt()
4516 trace_dwc3_event(event->raw, dwc); in dwc3_process_event_entry()
4518 if (!event->type.is_devspec) in dwc3_process_event_entry()
4519 dwc3_endpoint_interrupt(dwc, &event->depevt); in dwc3_process_event_entry()
4520 else if (event->type.type == DWC3_EVENT_TYPE_DEV) in dwc3_process_event_entry()
4521 dwc3_gadget_interrupt(dwc, &event->devt); in dwc3_process_event_entry()
4523 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw); in dwc3_process_event_entry()
4528 struct dwc3 *dwc = evt->dwc; in dwc3_process_event_buf()
4532 left = evt->count; in dwc3_process_event_buf()
4534 if (!(evt->flags & DWC3_EVENT_PENDING)) in dwc3_process_event_buf()
4540 event.raw = *(u32 *) (evt->cache + evt->lpos); in dwc3_process_event_buf()
4553 evt->lpos = (evt->lpos + 4) % evt->length; in dwc3_process_event_buf()
4554 left -= 4; in dwc3_process_event_buf()
4557 evt->count = 0; in dwc3_process_event_buf()
4561 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), in dwc3_process_event_buf()
4562 DWC3_GEVNTSIZ_SIZE(evt->length)); in dwc3_process_event_buf()
4564 evt->flags &= ~DWC3_EVENT_PENDING; in dwc3_process_event_buf()
4571 if (dwc->imod_interval) { in dwc3_process_event_buf()
4572 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); in dwc3_process_event_buf()
4573 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); in dwc3_process_event_buf()
4582 struct dwc3 *dwc = evt->dwc; in dwc3_thread_interrupt()
4587 spin_lock_irqsave(&dwc->lock, flags); in dwc3_thread_interrupt()
4589 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_thread_interrupt()
4597 struct dwc3 *dwc = evt->dwc; in dwc3_check_event_buf()
4601 if (pm_runtime_suspended(dwc->dev)) { in dwc3_check_event_buf()
4602 dwc->pending_events = true; in dwc3_check_event_buf()
4608 pm_runtime_get(dwc->dev); in dwc3_check_event_buf()
4609 disable_irq_nosync(dwc->irq_gadget); in dwc3_check_event_buf()
4614 * With PCIe legacy interrupt, test shows that top-half irq handler can in dwc3_check_event_buf()
4615 * be called again after HW interrupt deassertion. Check if bottom-half in dwc3_check_event_buf()
4619 if (evt->flags & DWC3_EVENT_PENDING) in dwc3_check_event_buf()
4622 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); in dwc3_check_event_buf()
4627 if (count > evt->length) { in dwc3_check_event_buf()
4628 dev_err_ratelimited(dwc->dev, "invalid count(%u) > evt->length(%u)\n", in dwc3_check_event_buf()
4629 count, evt->length); in dwc3_check_event_buf()
4633 evt->count = count; in dwc3_check_event_buf()
4634 evt->flags |= DWC3_EVENT_PENDING; in dwc3_check_event_buf()
4637 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), in dwc3_check_event_buf()
4638 DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length)); in dwc3_check_event_buf()
4640 amount = min(count, evt->length - evt->lpos); in dwc3_check_event_buf()
4641 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount); in dwc3_check_event_buf()
4644 memcpy(evt->cache, evt->buf, count - amount); in dwc3_check_event_buf()
4646 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count); in dwc3_check_event_buf()
4660 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev); in dwc3_gadget_get_irq()
4667 if (irq == -EPROBE_DEFER) in dwc3_gadget_get_irq()
4674 if (irq == -EPROBE_DEFER) in dwc3_gadget_get_irq()
4691 * dwc3_gadget_init - initializes gadget related registers
4708 dwc->irq_gadget = irq; in dwc3_gadget_init()
4710 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev, in dwc3_gadget_init()
4711 sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_init()
4712 &dwc->ep0_trb_addr, GFP_KERNEL); in dwc3_gadget_init()
4713 if (!dwc->ep0_trb) { in dwc3_gadget_init()
4714 dev_err(dwc->dev, "failed to allocate ep0 trb\n"); in dwc3_gadget_init()
4715 ret = -ENOMEM; in dwc3_gadget_init()
4719 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL); in dwc3_gadget_init()
4720 if (!dwc->setup_buf) { in dwc3_gadget_init()
4721 ret = -ENOMEM; in dwc3_gadget_init()
4725 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, in dwc3_gadget_init()
4726 &dwc->bounce_addr, GFP_KERNEL); in dwc3_gadget_init()
4727 if (!dwc->bounce) { in dwc3_gadget_init()
4728 ret = -ENOMEM; in dwc3_gadget_init()
4732 init_completion(&dwc->ep0_in_setup); in dwc3_gadget_init()
4733 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL); in dwc3_gadget_init()
4734 if (!dwc->gadget) { in dwc3_gadget_init()
4735 ret = -ENOMEM; in dwc3_gadget_init()
4740 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release); in dwc3_gadget_init()
4741 dev = &dwc->gadget->dev; in dwc3_gadget_init()
4742 dev->platform_data = dwc; in dwc3_gadget_init()
4743 dwc->gadget->ops = &dwc3_gadget_ops; in dwc3_gadget_init()
4744 dwc->gadget->speed = USB_SPEED_UNKNOWN; in dwc3_gadget_init()
4745 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; in dwc3_gadget_init()
4746 dwc->gadget->sg_supported = true; in dwc3_gadget_init()
4747 dwc->gadget->name = "dwc3-gadget"; in dwc3_gadget_init()
4748 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable; in dwc3_gadget_init()
4749 dwc->gadget->wakeup_capable = true; in dwc3_gadget_init()
4768 !dwc->dis_metastability_quirk) in dwc3_gadget_init()
4769 dev_info(dwc->dev, "changing max_speed on rev %08x\n", in dwc3_gadget_init()
4770 dwc->revision); in dwc3_gadget_init()
4772 dwc->gadget->max_speed = dwc->maximum_speed; in dwc3_gadget_init()
4773 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate; in dwc3_gadget_init()
4780 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps); in dwc3_gadget_init()
4784 ret = usb_add_gadget(dwc->gadget); in dwc3_gadget_init()
4786 dev_err(dwc->dev, "failed to add gadget\n"); in dwc3_gadget_init()
4790 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS) in dwc3_gadget_init()
4791 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate); in dwc3_gadget_init()
4793 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed); in dwc3_gadget_init()
4796 if (dwc->sys_wakeup) in dwc3_gadget_init()
4797 device_wakeup_disable(dwc->sysdev); in dwc3_gadget_init()
4804 usb_put_gadget(dwc->gadget); in dwc3_gadget_init()
4805 dwc->gadget = NULL; in dwc3_gadget_init()
4807 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, in dwc3_gadget_init()
4808 dwc->bounce_addr); in dwc3_gadget_init()
4811 kfree(dwc->setup_buf); in dwc3_gadget_init()
4814 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_init()
4815 dwc->ep0_trb, dwc->ep0_trb_addr); in dwc3_gadget_init()
4821 /* -------------------------------------------------------------------------- */
4825 if (!dwc->gadget) in dwc3_gadget_exit()
4829 usb_del_gadget(dwc->gadget); in dwc3_gadget_exit()
4831 usb_put_gadget(dwc->gadget); in dwc3_gadget_exit()
4832 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, in dwc3_gadget_exit()
4833 dwc->bounce_addr); in dwc3_gadget_exit()
4834 kfree(dwc->setup_buf); in dwc3_gadget_exit()
4835 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_exit()
4836 dwc->ep0_trb, dwc->ep0_trb_addr); in dwc3_gadget_exit()
4850 if (ret && dwc->softconnect) { in dwc3_gadget_suspend()
4852 return -EAGAIN; in dwc3_gadget_suspend()
4855 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_suspend()
4856 if (dwc->gadget_driver) in dwc3_gadget_suspend()
4858 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_suspend()
4865 if (!dwc->gadget_driver || !dwc->softconnect) in dwc3_gadget_resume()