Lines Matching +full:axi +full:- +full:max +full:- +full:burst +full:- +full:len
1 // SPDX-License-Identifier: GPL-2.0
3 * Cadence USBSS DRD Driver - gadget side.
5 * Copyright (C) 2018-2019 Cadence Design Systems.
6 * Copyright (C) 2017-2018 NXP
32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
37 * Additionally the packets directed to one endpoint can block entire on-chip
59 #include <linux/dma-mapping.h>
67 #include "gadget-export.h"
68 #include "cdns3-gadget.h"
69 #include "cdns3-trace.h"
83 * cdns3_clear_register_bit - clear bit in given register.
94 * cdns3_set_register_bit - set bit in given register.
105 * cdns3_ep_addr_to_index - Macro converts endpoint address to
120 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma; in cdns3_get_dma_pos()
126 * cdns3_next_request - returns next request from list
137 * cdns3_next_align_buf - returns next buffer from list
148 * cdns3_next_priv_request - returns next request from list
159 * cdns3_select_ep - selects endpoint
165 if (priv_dev->selected_ep == ep) in cdns3_select_ep()
168 priv_dev->selected_ep = ep; in cdns3_select_ep()
169 writel(ep, &priv_dev->regs->ep_sel); in cdns3_select_ep()
173 * cdns3_get_tdl - gets current tdl for selected endpoint.
181 if (priv_dev->dev_ver < DEV_VER_V3) in cdns3_get_tdl()
182 return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); in cdns3_get_tdl()
184 return readl(&priv_dev->regs->ep_tdl); in cdns3_get_tdl()
190 u32 offset = (char *)trb - (char *)priv_ep->trb_pool; in cdns3_trb_virt_to_dma()
192 return priv_ep->trb_pool_dma + offset; in cdns3_trb_virt_to_dma()
197 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_free_trb_pool()
199 if (priv_ep->trb_pool) { in cdns3_free_trb_pool()
200 dma_pool_free(priv_dev->eps_dma_pool, in cdns3_free_trb_pool()
201 priv_ep->trb_pool, priv_ep->trb_pool_dma); in cdns3_free_trb_pool()
202 priv_ep->trb_pool = NULL; in cdns3_free_trb_pool()
207 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
210 * Function will return 0 on success or -ENOMEM on allocation error
214 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_allocate_trb_pool()
219 if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size) in cdns3_allocate_trb_pool()
222 if (!priv_ep->trb_pool) { in cdns3_allocate_trb_pool()
223 priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool, in cdns3_allocate_trb_pool()
225 &priv_ep->trb_pool_dma); in cdns3_allocate_trb_pool()
227 if (!priv_ep->trb_pool) in cdns3_allocate_trb_pool()
228 return -ENOMEM; in cdns3_allocate_trb_pool()
230 priv_ep->alloc_ring_size = ring_size; in cdns3_allocate_trb_pool()
233 memset(priv_ep->trb_pool, 0, ring_size); in cdns3_allocate_trb_pool()
235 priv_ep->num_trbs = num_trbs; in cdns3_allocate_trb_pool()
237 if (!priv_ep->num) in cdns3_allocate_trb_pool()
241 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1)); in cdns3_allocate_trb_pool()
243 if (priv_ep->use_streams) { in cdns3_allocate_trb_pool()
248 link_trb->control = 0; in cdns3_allocate_trb_pool()
250 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma)); in cdns3_allocate_trb_pool()
251 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE); in cdns3_allocate_trb_pool()
257 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
264 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_ep_stall_flush()
270 &priv_dev->regs->ep_cmd); in cdns3_ep_stall_flush()
273 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in cdns3_ep_stall_flush()
275 priv_ep->flags |= EP_STALLED; in cdns3_ep_stall_flush()
276 priv_ep->flags &= ~EP_STALL_PENDING; in cdns3_ep_stall_flush()
280 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
287 writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf); in cdns3_hw_reset_eps_config()
290 priv_dev->hw_configured_flag = 0; in cdns3_hw_reset_eps_config()
291 priv_dev->onchip_used_size = 0; in cdns3_hw_reset_eps_config()
292 priv_dev->out_mem_is_allocated = 0; in cdns3_hw_reset_eps_config()
293 priv_dev->wait_for_setup = 0; in cdns3_hw_reset_eps_config()
294 priv_dev->using_streams = 0; in cdns3_hw_reset_eps_config()
297 if (priv_dev->eps[i]) in cdns3_hw_reset_eps_config()
298 priv_dev->eps[i]->flags &= ~EP_CONFIGURED; in cdns3_hw_reset_eps_config()
302 * cdns3_ep_inc_trb - increment a trb index.
315 if (*index == (trb_in_seg - 1)) { in cdns3_ep_inc_trb()
322 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
327 priv_ep->free_trbs--; in cdns3_ep_inc_enq()
328 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs); in cdns3_ep_inc_enq()
332 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
337 priv_ep->free_trbs++; in cdns3_ep_inc_deq()
338 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs); in cdns3_ep_inc_deq()
342 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
354 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf); in cdns3_allow_enable_l1()
356 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf); in cdns3_allow_enable_l1()
363 reg = readl(&priv_dev->regs->usb_sts); in cdns3_get_speed()
377 * cdns3_start_all_request - add to ring all request not started
389 u8 pending_empty = list_empty(&priv_ep->pending_req_list); in cdns3_start_all_request()
399 request = cdns3_next_request(&priv_ep->pending_req_list); in cdns3_start_all_request()
401 if ((priv_req->flags & REQUEST_INTERNAL) || in cdns3_start_all_request()
402 (priv_ep->flags & EP_TDLCHK_EN) || in cdns3_start_all_request()
403 priv_ep->use_streams) { in cdns3_start_all_request()
404 dev_dbg(priv_dev->dev, "Blocking external request\n"); in cdns3_start_all_request()
409 while (!list_empty(&priv_ep->deferred_req_list)) { in cdns3_start_all_request()
410 request = cdns3_next_request(&priv_ep->deferred_req_list); in cdns3_start_all_request()
412 if (!priv_ep->use_streams) { in cdns3_start_all_request()
415 priv_ep->stream_sg_idx = 0; in cdns3_start_all_request()
421 list_move_tail(&request->list, &priv_ep->pending_req_list); in cdns3_start_all_request()
422 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN)) in cdns3_start_all_request()
426 priv_ep->flags &= ~EP_RING_FULL; in cdns3_start_all_request()
433 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
437 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
438 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
445 int length = request->actual + descmiss_req->actual; in __cdns3_descmiss_copy_data()
446 struct scatterlist *s = request->sg; in __cdns3_descmiss_copy_data()
449 if (length <= request->length) { in __cdns3_descmiss_copy_data()
450 memcpy(&((u8 *)request->buf)[request->actual], in __cdns3_descmiss_copy_data()
451 descmiss_req->buf, in __cdns3_descmiss_copy_data()
452 descmiss_req->actual); in __cdns3_descmiss_copy_data()
453 request->actual = length; in __cdns3_descmiss_copy_data()
456 request->status = -ENOMEM; in __cdns3_descmiss_copy_data()
462 memcpy(&((u8 *)p)[request->actual], in __cdns3_descmiss_copy_data()
463 descmiss_req->buf, in __cdns3_descmiss_copy_data()
464 descmiss_req->actual); in __cdns3_descmiss_copy_data()
465 request->actual = length; in __cdns3_descmiss_copy_data()
467 request->status = -ENOMEM; in __cdns3_descmiss_copy_data()
473 * cdns3_wa2_descmiss_copy_data - copy data from internal requests to
484 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { in cdns3_wa2_descmiss_copy_data()
488 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); in cdns3_wa2_descmiss_copy_data()
489 descmiss_req = &descmiss_priv_req->request; in cdns3_wa2_descmiss_copy_data()
492 if (descmiss_priv_req->flags & REQUEST_PENDING) in cdns3_wa2_descmiss_copy_data()
495 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH; in cdns3_wa2_descmiss_copy_data()
496 request->status = descmiss_req->status; in cdns3_wa2_descmiss_copy_data()
498 list_del_init(&descmiss_priv_req->list); in cdns3_wa2_descmiss_copy_data()
499 kfree(descmiss_req->buf); in cdns3_wa2_descmiss_copy_data()
500 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req); in cdns3_wa2_descmiss_copy_data()
501 --priv_ep->wa2_counter; in cdns3_wa2_descmiss_copy_data()
512 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN && in cdns3_wa2_gadget_giveback()
513 priv_req->flags & REQUEST_INTERNAL) { in cdns3_wa2_gadget_giveback()
516 req = cdns3_next_request(&priv_ep->deferred_req_list); in cdns3_wa2_gadget_giveback()
518 priv_ep->descmis_req = NULL; in cdns3_wa2_gadget_giveback()
524 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req, in cdns3_wa2_gadget_giveback()
525 priv_ep->dir); in cdns3_wa2_gadget_giveback()
528 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) && in cdns3_wa2_gadget_giveback()
529 req->length != req->actual) { in cdns3_wa2_gadget_giveback()
531 /* re-map the gadget request buffer*/ in cdns3_wa2_gadget_giveback()
532 usb_gadget_map_request_by_dev(priv_dev->sysdev, req, in cdns3_wa2_gadget_giveback()
533 usb_endpoint_dir_in(priv_ep->endpoint.desc)); in cdns3_wa2_gadget_giveback()
537 if (req->status == -EINPROGRESS) in cdns3_wa2_gadget_giveback()
538 req->status = 0; in cdns3_wa2_gadget_giveback()
540 list_del_init(&req->list); in cdns3_wa2_gadget_giveback()
545 return &priv_req->request; in cdns3_wa2_gadget_giveback()
559 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) { in cdns3_wa2_gadget_ep_queue()
562 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir); in cdns3_wa2_gadget_ep_queue()
563 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET; in cdns3_wa2_gadget_ep_queue()
564 reg = readl(&priv_dev->regs->ep_sts_en); in cdns3_wa2_gadget_ep_queue()
567 writel(reg, &priv_dev->regs->ep_sts_en); in cdns3_wa2_gadget_ep_queue()
570 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) { in cdns3_wa2_gadget_ep_queue()
571 u8 pending_empty = list_empty(&priv_ep->pending_req_list); in cdns3_wa2_gadget_ep_queue()
572 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list); in cdns3_wa2_gadget_ep_queue()
580 !(priv_req->flags & REQUEST_INTERNAL)) { in cdns3_wa2_gadget_ep_queue()
582 &priv_req->request); in cdns3_wa2_gadget_ep_queue()
586 list_add_tail(&priv_req->request.list, in cdns3_wa2_gadget_ep_queue()
587 &priv_ep->pending_req_list); in cdns3_wa2_gadget_ep_queue()
589 priv_req->request.status); in cdns3_wa2_gadget_ep_queue()
608 if (priv_req->flags & REQUEST_INTERNAL) in cdns3_wa2_gadget_ep_queue()
609 list_add_tail(&priv_req->list, in cdns3_wa2_gadget_ep_queue()
610 &priv_ep->wa2_descmiss_req_list); in cdns3_wa2_gadget_ep_queue()
620 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { in cdns3_wa2_remove_old_request()
623 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); in cdns3_wa2_remove_old_request()
624 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH); in cdns3_wa2_remove_old_request()
628 kfree(priv_req->request.buf); in cdns3_wa2_remove_old_request()
629 list_del_init(&priv_req->list); in cdns3_wa2_remove_old_request()
630 cdns3_gadget_ep_free_request(&priv_ep->endpoint, in cdns3_wa2_remove_old_request()
631 &priv_req->request); in cdns3_wa2_remove_old_request()
632 --priv_ep->wa2_counter; in cdns3_wa2_remove_old_request()
640 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
650 u8 pending_empty = list_empty(&priv_ep->pending_req_list); in cdns3_wa2_descmissing_packet()
658 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) { in cdns3_wa2_descmissing_packet()
659 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET; in cdns3_wa2_descmissing_packet()
660 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN; in cdns3_wa2_descmissing_packet()
665 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) { in cdns3_wa2_descmissing_packet()
670 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint, in cdns3_wa2_descmissing_packet()
676 priv_req->flags |= REQUEST_INTERNAL; in cdns3_wa2_descmissing_packet()
684 if (priv_ep->descmis_req) in cdns3_wa2_descmissing_packet()
685 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH; in cdns3_wa2_descmissing_packet()
687 priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE, in cdns3_wa2_descmissing_packet()
689 priv_ep->wa2_counter++; in cdns3_wa2_descmissing_packet()
691 if (!priv_req->request.buf) { in cdns3_wa2_descmissing_packet()
692 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request); in cdns3_wa2_descmissing_packet()
696 priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE; in cdns3_wa2_descmissing_packet()
697 priv_ep->descmis_req = priv_req; in cdns3_wa2_descmissing_packet()
699 __cdns3_gadget_ep_queue(&priv_ep->endpoint, in cdns3_wa2_descmissing_packet()
700 &priv_ep->descmis_req->request, in cdns3_wa2_descmissing_packet()
706 dev_err(priv_ep->cdns3_dev->dev, in cdns3_wa2_descmissing_packet()
712 u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); in cdns3_wa2_reset_tdl()
715 u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl; in cdns3_wa2_reset_tdl()
718 &priv_dev->regs->ep_cmd); in cdns3_wa2_reset_tdl()
726 /* select EP0-out */ in cdns3_wa2_check_outq_status()
729 ep_sts_reg = readl(&priv_dev->regs->ep_sts); in cdns3_wa2_check_outq_status()
733 struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num]; in cdns3_wa2_check_outq_status()
735 if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) && in cdns3_wa2_check_outq_status()
736 outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) { in cdns3_wa2_check_outq_status()
737 u8 pending_empty = list_empty(&outq_ep->pending_req_list); in cdns3_wa2_check_outq_status()
739 if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) || in cdns3_wa2_check_outq_status()
740 (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) || in cdns3_wa2_check_outq_status()
746 cdns3_select_ep(priv_dev, outq_ep->num | in cdns3_wa2_check_outq_status()
747 outq_ep->dir); in cdns3_wa2_check_outq_status()
748 ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en); in cdns3_wa2_check_outq_status()
749 ep_cmd_reg = readl(&priv_dev->regs->ep_cmd); in cdns3_wa2_check_outq_status()
751 outq_ep->flags |= EP_TDLCHK_EN; in cdns3_wa2_check_outq_status()
752 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, in cdns3_wa2_check_outq_status()
758 &priv_dev->regs->ep_sts_en); in cdns3_wa2_check_outq_status()
762 * Memory barrier - Reset tdl before ringing the in cdns3_wa2_check_outq_status()
775 &priv_dev->regs->ep_cmd); in cdns3_wa2_check_outq_status()
783 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
789 * function will unmap @req and call its ->complete() callback to notify upper
796 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_giveback()
797 struct usb_request *request = &priv_req->request; in cdns3_gadget_giveback()
799 list_del_init(&request->list); in cdns3_gadget_giveback()
801 if (request->status == -EINPROGRESS) in cdns3_gadget_giveback()
802 request->status = status; in cdns3_gadget_giveback()
804 if (likely(!(priv_req->flags & REQUEST_UNALIGNED))) in cdns3_gadget_giveback()
805 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request, in cdns3_gadget_giveback()
806 priv_ep->dir); in cdns3_gadget_giveback()
808 if ((priv_req->flags & REQUEST_UNALIGNED) && in cdns3_gadget_giveback()
809 priv_ep->dir == USB_DIR_OUT && !request->status) { in cdns3_gadget_giveback()
811 dma_sync_single_for_cpu(priv_dev->sysdev, in cdns3_gadget_giveback()
812 priv_req->aligned_buf->dma, in cdns3_gadget_giveback()
813 request->actual, in cdns3_gadget_giveback()
814 priv_req->aligned_buf->dir); in cdns3_gadget_giveback()
815 memcpy(request->buf, priv_req->aligned_buf->buf, in cdns3_gadget_giveback()
816 request->actual); in cdns3_gadget_giveback()
819 priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED); in cdns3_gadget_giveback()
821 priv_req->finished_trb = 0; in cdns3_gadget_giveback()
824 if (priv_dev->dev_ver < DEV_VER_V2) { in cdns3_gadget_giveback()
835 if (request->complete && request->buf != priv_dev->zlp_buf) { in cdns3_gadget_giveback()
836 spin_unlock(&priv_dev->lock); in cdns3_gadget_giveback()
837 usb_gadget_giveback_request(&priv_ep->endpoint, in cdns3_gadget_giveback()
839 spin_lock(&priv_dev->lock); in cdns3_gadget_giveback()
842 if (request->buf == priv_dev->zlp_buf) in cdns3_gadget_giveback()
843 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request); in cdns3_gadget_giveback()
849 if (priv_ep->wa1_set) { in cdns3_wa1_restore_cycle_bit()
852 priv_ep->wa1_set = 0; in cdns3_wa1_restore_cycle_bit()
853 priv_ep->wa1_trb_index = 0xFFFF; in cdns3_wa1_restore_cycle_bit()
854 if (priv_ep->wa1_cycle_bit) { in cdns3_wa1_restore_cycle_bit()
855 priv_ep->wa1_trb->control = in cdns3_wa1_restore_cycle_bit()
856 priv_ep->wa1_trb->control | cpu_to_le32(0x1); in cdns3_wa1_restore_cycle_bit()
858 priv_ep->wa1_trb->control = in cdns3_wa1_restore_cycle_bit()
859 priv_ep->wa1_trb->control & cpu_to_le32(~0x1); in cdns3_wa1_restore_cycle_bit()
871 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_free_aligned_request_buf()
873 list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) { in cdns3_free_aligned_request_buf()
874 if (!buf->in_use) { in cdns3_free_aligned_request_buf()
875 list_del(&buf->list); in cdns3_free_aligned_request_buf()
878 * Re-enable interrupts to free DMA capable memory. in cdns3_free_aligned_request_buf()
882 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_free_aligned_request_buf()
883 dma_free_noncoherent(priv_dev->sysdev, buf->size, in cdns3_free_aligned_request_buf()
884 buf->buf, buf->dma, buf->dir); in cdns3_free_aligned_request_buf()
886 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_free_aligned_request_buf()
890 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_free_aligned_request_buf()
895 struct cdns3_endpoint *priv_ep = priv_req->priv_ep; in cdns3_prepare_aligned_request_buf()
896 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_prepare_aligned_request_buf()
900 if (!((uintptr_t)priv_req->request.buf & 0x7)) in cdns3_prepare_aligned_request_buf()
903 buf = priv_req->aligned_buf; in cdns3_prepare_aligned_request_buf()
905 if (!buf || priv_req->request.length > buf->size) { in cdns3_prepare_aligned_request_buf()
908 return -ENOMEM; in cdns3_prepare_aligned_request_buf()
910 buf->size = priv_req->request.length; in cdns3_prepare_aligned_request_buf()
911 buf->dir = usb_endpoint_dir_in(priv_ep->endpoint.desc) ? in cdns3_prepare_aligned_request_buf()
914 buf->buf = dma_alloc_noncoherent(priv_dev->sysdev, in cdns3_prepare_aligned_request_buf()
915 buf->size, in cdns3_prepare_aligned_request_buf()
916 &buf->dma, in cdns3_prepare_aligned_request_buf()
917 buf->dir, in cdns3_prepare_aligned_request_buf()
919 if (!buf->buf) { in cdns3_prepare_aligned_request_buf()
921 return -ENOMEM; in cdns3_prepare_aligned_request_buf()
924 if (priv_req->aligned_buf) { in cdns3_prepare_aligned_request_buf()
926 priv_req->aligned_buf->in_use = 0; in cdns3_prepare_aligned_request_buf()
928 &priv_dev->aligned_buf_wq); in cdns3_prepare_aligned_request_buf()
931 buf->in_use = 1; in cdns3_prepare_aligned_request_buf()
932 priv_req->aligned_buf = buf; in cdns3_prepare_aligned_request_buf()
934 list_add_tail(&buf->list, in cdns3_prepare_aligned_request_buf()
935 &priv_dev->aligned_buf_list); in cdns3_prepare_aligned_request_buf()
938 if (priv_ep->dir == USB_DIR_IN) { in cdns3_prepare_aligned_request_buf()
940 dma_sync_single_for_cpu(priv_dev->sysdev, in cdns3_prepare_aligned_request_buf()
941 buf->dma, buf->size, buf->dir); in cdns3_prepare_aligned_request_buf()
942 memcpy(buf->buf, priv_req->request.buf, in cdns3_prepare_aligned_request_buf()
943 priv_req->request.length); in cdns3_prepare_aligned_request_buf()
947 dma_sync_single_for_device(priv_dev->sysdev, in cdns3_prepare_aligned_request_buf()
948 buf->dma, buf->size, buf->dir); in cdns3_prepare_aligned_request_buf()
950 priv_req->flags |= REQUEST_UNALIGNED; in cdns3_prepare_aligned_request_buf()
959 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_wa1_update_guard()
961 if (!priv_ep->wa1_set) { in cdns3_wa1_update_guard()
964 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); in cdns3_wa1_update_guard()
967 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0; in cdns3_wa1_update_guard()
968 priv_ep->wa1_set = 1; in cdns3_wa1_update_guard()
969 priv_ep->wa1_trb = trb; in cdns3_wa1_update_guard()
970 priv_ep->wa1_trb_index = priv_ep->enqueue; in cdns3_wa1_update_guard()
984 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); in cdns3_wa1_tray_restore_cycle_bit()
987 if (!doorbell || dma_index != priv_ep->wa1_trb_index) in cdns3_wa1_tray_restore_cycle_bit()
994 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_ep_run_stream_transfer()
1002 unsigned int sg_idx = priv_ep->stream_sg_idx; in cdns3_ep_run_stream_transfer()
1005 address = priv_ep->endpoint.desc->bEndpointAddress; in cdns3_ep_run_stream_transfer()
1007 priv_ep->flags |= EP_PENDING_REQUEST; in cdns3_ep_run_stream_transfer()
1010 if (priv_req->flags & REQUEST_UNALIGNED) in cdns3_ep_run_stream_transfer()
1011 trb_dma = priv_req->aligned_buf->dma; in cdns3_ep_run_stream_transfer()
1013 trb_dma = request->dma; in cdns3_ep_run_stream_transfer()
1016 trb = priv_ep->trb_pool + priv_ep->enqueue; in cdns3_ep_run_stream_transfer()
1017 priv_req->start_trb = priv_ep->enqueue; in cdns3_ep_run_stream_transfer()
1018 priv_req->end_trb = priv_req->start_trb; in cdns3_ep_run_stream_transfer()
1019 priv_req->trb = trb; in cdns3_ep_run_stream_transfer()
1021 cdns3_select_ep(priv_ep->cdns3_dev, address); in cdns3_ep_run_stream_transfer()
1024 TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP; in cdns3_ep_run_stream_transfer()
1026 if (!request->num_sgs) { in cdns3_ep_run_stream_transfer()
1027 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma)); in cdns3_ep_run_stream_transfer()
1028 length = request->length; in cdns3_ep_run_stream_transfer()
1030 trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address)); in cdns3_ep_run_stream_transfer()
1031 length = request->sg[sg_idx].length; in cdns3_ep_run_stream_transfer()
1034 tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket); in cdns3_ep_run_stream_transfer()
1036 trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length)); in cdns3_ep_run_stream_transfer()
1043 if (priv_dev->dev_ver >= DEV_VER_V2) { in cdns3_ep_run_stream_transfer()
1044 if (priv_dev->gadget.speed == USB_SPEED_SUPER) in cdns3_ep_run_stream_transfer()
1045 trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl)); in cdns3_ep_run_stream_transfer()
1047 priv_req->flags |= REQUEST_PENDING; in cdns3_ep_run_stream_transfer()
1049 trb->control = cpu_to_le32(control); in cdns3_ep_run_stream_transfer()
1051 trace_cdns3_prepare_trb(priv_ep, priv_req->trb); in cdns3_ep_run_stream_transfer()
1054 * Memory barrier - Cycle Bit must be set before trb->length and in cdns3_ep_run_stream_transfer()
1055 * trb->buffer fields. in cdns3_ep_run_stream_transfer()
1060 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma), in cdns3_ep_run_stream_transfer()
1061 &priv_dev->regs->ep_traddr); in cdns3_ep_run_stream_transfer()
1063 if (!(priv_ep->flags & EP_STALLED)) { in cdns3_ep_run_stream_transfer()
1066 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts); in cdns3_ep_run_stream_transfer()
1068 priv_ep->prime_flag = false; in cdns3_ep_run_stream_transfer()
1075 if (priv_dev->dev_ver < DEV_VER_V2) in cdns3_ep_run_stream_transfer()
1077 &priv_dev->regs->ep_cmd); in cdns3_ep_run_stream_transfer()
1078 else if (priv_dev->dev_ver > DEV_VER_V2) in cdns3_ep_run_stream_transfer()
1079 writel(tdl, &priv_dev->regs->ep_tdl); in cdns3_ep_run_stream_transfer()
1081 priv_ep->last_stream_id = priv_req->request.stream_id; in cdns3_ep_run_stream_transfer()
1082 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); in cdns3_ep_run_stream_transfer()
1083 writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) | in cdns3_ep_run_stream_transfer()
1084 EP_CMD_ERDY, &priv_dev->regs->ep_cmd); in cdns3_ep_run_stream_transfer()
1086 trace_cdns3_doorbell_epx(priv_ep->name, in cdns3_ep_run_stream_transfer()
1087 readl(&priv_dev->regs->ep_traddr)); in cdns3_ep_run_stream_transfer()
1098 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_rearm_drdy_if_needed()
1100 if (priv_dev->dev_ver < DEV_VER_V3) in cdns3_rearm_drdy_if_needed()
1103 if (readl(&priv_dev->regs->ep_sts) & EP_STS_TRBERR) { in cdns3_rearm_drdy_if_needed()
1104 writel(EP_STS_TRBERR, &priv_dev->regs->ep_sts); in cdns3_rearm_drdy_if_needed()
1105 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); in cdns3_rearm_drdy_if_needed()
1110 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1119 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_ep_run_transfer()
1134 bool sg_supported = !!(request->num_mapped_sgs); in cdns3_ep_run_transfer()
1135 u32 ioc = request->no_interrupt ? 0 : TRB_IOC; in cdns3_ep_run_transfer()
1137 num_trb_req = sg_supported ? request->num_mapped_sgs : 1; in cdns3_ep_run_transfer()
1140 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) in cdns3_ep_run_transfer()
1141 num_trb = priv_ep->interval * num_trb_req; in cdns3_ep_run_transfer()
1146 address = priv_ep->endpoint.desc->bEndpointAddress; in cdns3_ep_run_transfer()
1148 priv_ep->flags |= EP_PENDING_REQUEST; in cdns3_ep_run_transfer()
1151 if (priv_req->flags & REQUEST_UNALIGNED) in cdns3_ep_run_transfer()
1152 trb_dma = priv_req->aligned_buf->dma; in cdns3_ep_run_transfer()
1154 trb_dma = request->dma; in cdns3_ep_run_transfer()
1156 trb = priv_ep->trb_pool + priv_ep->enqueue; in cdns3_ep_run_transfer()
1157 priv_req->start_trb = priv_ep->enqueue; in cdns3_ep_run_transfer()
1158 priv_req->trb = trb; in cdns3_ep_run_transfer()
1160 cdns3_select_ep(priv_ep->cdns3_dev, address); in cdns3_ep_run_transfer()
1163 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) { in cdns3_ep_run_transfer()
1167 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); in cdns3_ep_run_transfer()
1171 if (doorbell && dma_index == priv_ep->num_trbs - 1) { in cdns3_ep_run_transfer()
1172 priv_ep->flags |= EP_DEFERRED_DRDY; in cdns3_ep_run_transfer()
1173 return -ENOBUFS; in cdns3_ep_run_transfer()
1177 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1); in cdns3_ep_run_transfer()
1186 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC || in cdns3_ep_run_transfer()
1190 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) | in cdns3_ep_run_transfer()
1193 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) { in cdns3_ep_run_transfer()
1198 while (priv_ep->enqueue) { in cdns3_ep_run_transfer()
1203 trb = priv_ep->trb_pool + priv_ep->enqueue; in cdns3_ep_run_transfer()
1204 priv_req->trb = trb; in cdns3_ep_run_transfer()
1209 if (num_trb > priv_ep->free_trbs) { in cdns3_ep_run_transfer()
1210 priv_ep->flags |= EP_RING_FULL; in cdns3_ep_run_transfer()
1211 return -ENOBUFS; in cdns3_ep_run_transfer()
1214 if (priv_dev->dev_ver <= DEV_VER_V2) in cdns3_ep_run_transfer()
1218 control = priv_ep->pcs ? 0 : TRB_CYCLE; in cdns3_ep_run_transfer()
1219 trb->length = 0; in cdns3_ep_run_transfer()
1220 if (priv_dev->dev_ver >= DEV_VER_V2) { in cdns3_ep_run_transfer()
1223 td_size = DIV_ROUND_UP(request->length, in cdns3_ep_run_transfer()
1224 priv_ep->endpoint.maxpacket); in cdns3_ep_run_transfer()
1225 if (priv_dev->gadget.speed == USB_SPEED_SUPER) in cdns3_ep_run_transfer()
1226 trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size)); in cdns3_ep_run_transfer()
1235 s = request->sg; in cdns3_ep_run_transfer()
1240 trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s))); in cdns3_ep_run_transfer()
1243 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma)); in cdns3_ep_run_transfer()
1244 length = request->length; in cdns3_ep_run_transfer()
1247 if (priv_ep->flags & EP_TDLCHK_EN) in cdns3_ep_run_transfer()
1249 priv_ep->endpoint.maxpacket); in cdns3_ep_run_transfer()
1251 trb_burst = priv_ep->trb_burst_size; in cdns3_ep_run_transfer()
1258 * [UVC Header(8B) ] [data(3k - 8)] ... in cdns3_ep_run_transfer()
1260 * The received data at offset 0xd000 will get 0xc000 data, len 0x70. Error happen in cdns3_ep_run_transfer()
1273 * To avoid DMA cross 4k bounder at ISO transfer, reduce burst len according to 16. in cdns3_ep_run_transfer()
1275 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_dev->dev_ver <= DEV_VER_V2) in cdns3_ep_run_transfer()
1276 if (ALIGN_DOWN(trb->buffer, SZ_4K) != in cdns3_ep_run_transfer()
1277 ALIGN_DOWN(trb->buffer + length, SZ_4K)) in cdns3_ep_run_transfer()
1280 trb->length |= cpu_to_le32(TRB_BURST_LEN(trb_burst) | in cdns3_ep_run_transfer()
1282 pcs = priv_ep->pcs ? TRB_CYCLE : 0; in cdns3_ep_run_transfer()
1291 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) { in cdns3_ep_run_transfer()
1295 if (sg_iter == (num_trb - 1) && sg_iter != 0) in cdns3_ep_run_transfer()
1300 trb->control = cpu_to_le32(control); in cdns3_ep_run_transfer()
1302 priv_req->trb->control = cpu_to_le32(control); in cdns3_ep_run_transfer()
1305 trb->control |= cpu_to_le32(TRB_ISP); in cdns3_ep_run_transfer()
1307 if ((sg_iter % num_trb_req) < num_trb_req - 1) in cdns3_ep_run_transfer()
1308 trb->control |= cpu_to_le32(TRB_CHAIN); in cdns3_ep_run_transfer()
1315 priv_req->end_trb = priv_ep->enqueue; in cdns3_ep_run_transfer()
1317 trb = priv_ep->trb_pool + priv_ep->enqueue; in cdns3_ep_run_transfer()
1318 trb->length = 0; in cdns3_ep_run_transfer()
1321 trb = priv_req->trb; in cdns3_ep_run_transfer()
1323 priv_req->flags |= REQUEST_PENDING; in cdns3_ep_run_transfer()
1324 priv_req->num_of_trb = num_trb; in cdns3_ep_run_transfer()
1327 trb->control |= cpu_to_le32(ioc | TRB_ISP); in cdns3_ep_run_transfer()
1329 if (priv_dev->dev_ver < DEV_VER_V2 && in cdns3_ep_run_transfer()
1330 (priv_ep->flags & EP_TDLCHK_EN)) { in cdns3_ep_run_transfer()
1332 u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); in cdns3_ep_run_transfer()
1336 priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX; in cdns3_ep_run_transfer()
1340 tdl -= old_tdl; in cdns3_ep_run_transfer()
1342 &priv_dev->regs->ep_cmd); in cdns3_ep_run_transfer()
1347 * Memory barrier - cycle bit must be set before other filds in trb. in cdns3_ep_run_transfer()
1353 trb->control = trb->control ^ cpu_to_le32(1); in cdns3_ep_run_transfer()
1355 if (priv_dev->dev_ver <= DEV_VER_V2) in cdns3_ep_run_transfer()
1364 trb = priv_ep->trb_pool; in cdns3_ep_run_transfer()
1365 num_trb = num_trb - i; in cdns3_ep_run_transfer()
1372 trace_cdns3_prepare_trb(priv_ep, priv_req->trb); in cdns3_ep_run_transfer()
1376 * Memory barrier - Cycle Bit must be set before trb->length and in cdns3_ep_run_transfer()
1377 * trb->buffer fields. in cdns3_ep_run_transfer()
1385 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) { in cdns3_ep_run_transfer()
1391 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir && in cdns3_ep_run_transfer()
1392 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) { in cdns3_ep_run_transfer()
1393 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN; in cdns3_ep_run_transfer()
1394 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, in cdns3_ep_run_transfer()
1398 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma + in cdns3_ep_run_transfer()
1399 priv_req->start_trb * TRB_SIZE), in cdns3_ep_run_transfer()
1400 &priv_dev->regs->ep_traddr); in cdns3_ep_run_transfer()
1402 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR; in cdns3_ep_run_transfer()
1405 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) { in cdns3_ep_run_transfer()
1408 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts); in cdns3_ep_run_transfer()
1409 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); in cdns3_ep_run_transfer()
1411 trace_cdns3_doorbell_epx(priv_ep->name, in cdns3_ep_run_transfer()
1412 readl(&priv_dev->regs->ep_traddr)); in cdns3_ep_run_transfer()
1426 if (priv_dev->hw_configured_flag) in cdns3_set_hw_configuration()
1429 writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf); in cdns3_set_hw_configuration()
1431 cdns3_set_register_bit(&priv_dev->regs->usb_conf, in cdns3_set_hw_configuration()
1434 priv_dev->hw_configured_flag = 1; in cdns3_set_hw_configuration()
1436 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { in cdns3_set_hw_configuration()
1437 if (ep->enabled) { in cdns3_set_hw_configuration()
1447 * cdns3_trb_handled - check whether trb has been handled by DMA
1456 * SR - start ring
1457 * ER - end ring
1458 * DQ = priv_ep->dequeue - dequeue position
1459 * EQ = priv_ep->enqueue - enqueue position
1460 * ST = priv_req->start_trb - index of first TRB in transfer ring
1461 * ET = priv_req->end_trb - index of last TRB in transfer ring
1462 * CI = current_index - index of processed TRB by DMA.
1465 * Then, we check if cycle bit for index priv_ep->dequeue
1469 * 1. priv_ep->dequeue never equals to current_index.
1470 * 2 priv_ep->enqueue never exceed priv_ep->dequeue
1471 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1472 * and priv_ep->free_trbs is zero.
1476 * Case 1 - priv_ep->dequeue < current_index
1480 * Case 2 - priv_ep->dequeue > current_index
1488 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_trb_handled()
1495 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); in cdns3_trb_handled()
1498 if (priv_req->start_trb < priv_req->end_trb) { in cdns3_trb_handled()
1499 if (priv_ep->dequeue > priv_req->end_trb) in cdns3_trb_handled()
1502 if (priv_ep->dequeue < priv_req->start_trb) in cdns3_trb_handled()
1506 if ((priv_req->start_trb > priv_req->end_trb) && in cdns3_trb_handled()
1507 (priv_ep->dequeue > priv_req->end_trb) && in cdns3_trb_handled()
1508 (priv_ep->dequeue < priv_req->start_trb)) in cdns3_trb_handled()
1511 if ((priv_req->start_trb == priv_req->end_trb) && in cdns3_trb_handled()
1512 (priv_ep->dequeue != priv_req->end_trb)) in cdns3_trb_handled()
1515 trb = &priv_ep->trb_pool[priv_ep->dequeue]; in cdns3_trb_handled()
1517 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs) in cdns3_trb_handled()
1520 if (doorbell == 1 && current_index == priv_ep->dequeue) in cdns3_trb_handled()
1524 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { in cdns3_trb_handled()
1529 if (priv_ep->enqueue == priv_ep->dequeue && in cdns3_trb_handled()
1530 priv_ep->free_trbs == 0) { in cdns3_trb_handled()
1532 } else if (priv_ep->dequeue < current_index) { in cdns3_trb_handled()
1533 if ((current_index == (priv_ep->num_trbs - 1)) && in cdns3_trb_handled()
1534 !priv_ep->dequeue) in cdns3_trb_handled()
1538 } else if (priv_ep->dequeue > current_index) { in cdns3_trb_handled()
1557 while (!list_empty(&priv_ep->pending_req_list)) { in cdns3_transfer_completed()
1558 request = cdns3_next_request(&priv_ep->pending_req_list); in cdns3_transfer_completed()
1561 trb = priv_ep->trb_pool + priv_ep->dequeue; in cdns3_transfer_completed()
1564 while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) { in cdns3_transfer_completed()
1567 if (priv_ep->dequeue == cdns3_get_dma_pos(priv_dev, priv_ep) && in cdns3_transfer_completed()
1568 priv_ep->type == USB_ENDPOINT_XFER_ISOC) in cdns3_transfer_completed()
1573 trb = priv_ep->trb_pool + priv_ep->dequeue; in cdns3_transfer_completed()
1576 if (!request->stream_id) { in cdns3_transfer_completed()
1577 /* Re-select endpoint. It could be changed by other CPU in cdns3_transfer_completed()
1580 cdns3_select_ep(priv_dev, priv_ep->endpoint.address); in cdns3_transfer_completed()
1583 priv_req->finished_trb++; in cdns3_transfer_completed()
1584 if (priv_req->finished_trb >= priv_req->num_of_trb) in cdns3_transfer_completed()
1587 trb = priv_ep->trb_pool + priv_ep->dequeue; in cdns3_transfer_completed()
1591 request->actual += in cdns3_transfer_completed()
1592 TRB_LEN(le32_to_cpu(trb->length)); in cdns3_transfer_completed()
1594 if (priv_req->num_of_trb > 1 && in cdns3_transfer_completed()
1595 le32_to_cpu(trb->control) & TRB_SMM && in cdns3_transfer_completed()
1596 le32_to_cpu(trb->control) & TRB_CHAIN) in cdns3_transfer_completed()
1603 /* TRBs are duplicated by priv_ep->interval time for ISO IN */ in cdns3_transfer_completed()
1604 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && priv_ep->dir) in cdns3_transfer_completed()
1605 request->actual /= priv_ep->interval; in cdns3_transfer_completed()
1614 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && in cdns3_transfer_completed()
1618 /* Re-select endpoint. It could be changed by other CPU in cdns3_transfer_completed()
1621 cdns3_select_ep(priv_dev, priv_ep->endpoint.address); in cdns3_transfer_completed()
1623 trb = priv_ep->trb_pool; in cdns3_transfer_completed()
1626 if (trb != priv_req->trb) in cdns3_transfer_completed()
1627 dev_warn(priv_dev->dev, in cdns3_transfer_completed()
1629 priv_req->trb, trb); in cdns3_transfer_completed()
1631 request->actual += TRB_LEN(le32_to_cpu(trb->length)); in cdns3_transfer_completed()
1633 if (!request->num_sgs || in cdns3_transfer_completed()
1634 (request->num_sgs == (priv_ep->stream_sg_idx + 1))) { in cdns3_transfer_completed()
1635 priv_ep->stream_sg_idx = 0; in cdns3_transfer_completed()
1638 priv_ep->stream_sg_idx++; in cdns3_transfer_completed()
1644 priv_ep->flags &= ~EP_PENDING_REQUEST; in cdns3_transfer_completed()
1647 if (!(priv_ep->flags & EP_STALLED) && in cdns3_transfer_completed()
1648 !(priv_ep->flags & EP_STALL_PENDING)) in cdns3_transfer_completed()
1654 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_rearm_transfer()
1663 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); in cdns3_rearm_transfer()
1667 trace_cdns3_doorbell_epx(priv_ep->name, in cdns3_rearm_transfer()
1668 readl(&priv_dev->regs->ep_traddr)); in cdns3_rearm_transfer()
1674 u16 tdl = priv_ep->pending_tdl; in cdns3_reprogram_tdl()
1675 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_reprogram_tdl()
1679 priv_ep->pending_tdl -= EP_CMD_TDL_MAX; in cdns3_reprogram_tdl()
1681 priv_ep->pending_tdl = 0; in cdns3_reprogram_tdl()
1684 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd); in cdns3_reprogram_tdl()
1688 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1695 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_check_ep_interrupt_proceed()
1701 cdns3_select_ep(priv_dev, priv_ep->endpoint.address); in cdns3_check_ep_interrupt_proceed()
1705 ep_sts_reg = readl(&priv_dev->regs->ep_sts); in cdns3_check_ep_interrupt_proceed()
1706 writel(ep_sts_reg, &priv_dev->regs->ep_sts); in cdns3_check_ep_interrupt_proceed()
1708 if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) { in cdns3_check_ep_interrupt_proceed()
1723 EP_CMD_ERDY_SID(priv_ep->last_stream_id), in cdns3_check_ep_interrupt_proceed()
1724 &priv_dev->regs->ep_cmd); in cdns3_check_ep_interrupt_proceed()
1727 priv_ep->prime_flag = true; in cdns3_check_ep_interrupt_proceed()
1729 pending_request = cdns3_next_request(&priv_ep->pending_req_list); in cdns3_check_ep_interrupt_proceed()
1730 deferred_request = cdns3_next_request(&priv_ep->deferred_req_list); in cdns3_check_ep_interrupt_proceed()
1739 if (priv_ep->flags & EP_STALL_PENDING && in cdns3_check_ep_interrupt_proceed()
1741 priv_dev->dev_ver < DEV_VER_V2)) { in cdns3_check_ep_interrupt_proceed()
1752 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && in cdns3_check_ep_interrupt_proceed()
1753 !priv_ep->wa1_set) { in cdns3_check_ep_interrupt_proceed()
1754 if (!priv_ep->dir) { in cdns3_check_ep_interrupt_proceed()
1755 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg); in cdns3_check_ep_interrupt_proceed()
1758 writel(ep_cfg, &priv_dev->regs->ep_cfg); in cdns3_check_ep_interrupt_proceed()
1759 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN; in cdns3_check_ep_interrupt_proceed()
1760 priv_ep->flags |= EP_UPDATE_EP_TRBADDR; in cdns3_check_ep_interrupt_proceed()
1763 } else if (!(priv_ep->flags & EP_STALLED) && in cdns3_check_ep_interrupt_proceed()
1764 !(priv_ep->flags & EP_STALL_PENDING)) { in cdns3_check_ep_interrupt_proceed()
1765 if (priv_ep->flags & EP_DEFERRED_DRDY) { in cdns3_check_ep_interrupt_proceed()
1766 priv_ep->flags &= ~EP_DEFERRED_DRDY; in cdns3_check_ep_interrupt_proceed()
1770 priv_ep->wa1_set); in cdns3_check_ep_interrupt_proceed()
1777 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) { in cdns3_check_ep_interrupt_proceed()
1779 priv_ep->flags |= EP_QUIRK_END_TRANSFER; in cdns3_check_ep_interrupt_proceed()
1781 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER; in cdns3_check_ep_interrupt_proceed()
1784 if (!priv_ep->use_streams) { in cdns3_check_ep_interrupt_proceed()
1788 } else if ((priv_ep->flags & EP_TDLCHK_EN) & in cdns3_check_ep_interrupt_proceed()
1789 priv_ep->pending_tdl) { in cdns3_check_ep_interrupt_proceed()
1793 } else if (priv_ep->dir == USB_DIR_OUT) { in cdns3_check_ep_interrupt_proceed()
1794 priv_ep->ep_sts_pending |= ep_sts_reg; in cdns3_check_ep_interrupt_proceed()
1804 if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) && in cdns3_check_ep_interrupt_proceed()
1805 (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) { in cdns3_check_ep_interrupt_proceed()
1806 priv_ep->ep_sts_pending = 0; in cdns3_check_ep_interrupt_proceed()
1812 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or in cdns3_check_ep_interrupt_proceed()
1813 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN. in cdns3_check_ep_interrupt_proceed()
1816 if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 && in cdns3_check_ep_interrupt_proceed()
1817 !(priv_ep->flags & EP_STALLED)) in cdns3_check_ep_interrupt_proceed()
1825 if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) in cdns3_disconnect_gadget()
1826 priv_dev->gadget_driver->disconnect(&priv_dev->gadget); in cdns3_disconnect_gadget()
1830 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1837 __must_hold(&priv_dev->lock) in cdns3_check_usb_interrupt_proceed()
1848 if (readl(&priv_dev->regs->drbl)) in cdns3_check_usb_interrupt_proceed()
1855 priv_dev->gadget.speed = speed; in cdns3_check_usb_interrupt_proceed()
1856 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED); in cdns3_check_usb_interrupt_proceed()
1862 spin_unlock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1864 spin_lock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1865 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; in cdns3_check_usb_interrupt_proceed()
1866 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED); in cdns3_check_usb_interrupt_proceed()
1871 if (priv_dev->gadget_driver && in cdns3_check_usb_interrupt_proceed()
1872 priv_dev->gadget_driver->suspend) { in cdns3_check_usb_interrupt_proceed()
1873 spin_unlock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1874 priv_dev->gadget_driver->suspend(&priv_dev->gadget); in cdns3_check_usb_interrupt_proceed()
1875 spin_lock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1880 if (priv_dev->gadget_driver && in cdns3_check_usb_interrupt_proceed()
1881 priv_dev->gadget_driver->resume) { in cdns3_check_usb_interrupt_proceed()
1882 spin_unlock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1883 priv_dev->gadget_driver->resume(&priv_dev->gadget); in cdns3_check_usb_interrupt_proceed()
1884 spin_lock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1890 if (priv_dev->gadget_driver) { in cdns3_check_usb_interrupt_proceed()
1891 spin_unlock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1892 usb_gadget_udc_reset(&priv_dev->gadget, in cdns3_check_usb_interrupt_proceed()
1893 priv_dev->gadget_driver); in cdns3_check_usb_interrupt_proceed()
1894 spin_lock(&priv_dev->lock); in cdns3_check_usb_interrupt_proceed()
1898 priv_dev->gadget.speed = speed; in cdns3_check_usb_interrupt_proceed()
1906 * cdns3_device_irq_handler - interrupt handler for device part of controller
1916 struct cdns *cdns = dev_get_drvdata(priv_dev->dev); in cdns3_device_irq_handler()
1920 if (cdns->in_lpm) in cdns3_device_irq_handler()
1924 reg = readl(&priv_dev->regs->usb_ists); in cdns3_device_irq_handler()
1933 reg = ~reg & readl(&priv_dev->regs->usb_ien); in cdns3_device_irq_handler()
1935 writel(reg, &priv_dev->regs->usb_ien); in cdns3_device_irq_handler()
1940 reg = readl(&priv_dev->regs->ep_ists); in cdns3_device_irq_handler()
1942 writel(0, &priv_dev->regs->ep_ien); in cdns3_device_irq_handler()
1950 * cdns3_device_thread_irq_handler - interrupt handler for device part
1966 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_device_thread_irq_handler()
1968 reg = readl(&priv_dev->regs->usb_ists); in cdns3_device_thread_irq_handler()
1970 writel(reg, &priv_dev->regs->usb_ists); in cdns3_device_thread_irq_handler()
1971 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien); in cdns3_device_thread_irq_handler()
1976 reg = readl(&priv_dev->regs->ep_ists); in cdns3_device_thread_irq_handler()
1997 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]); in cdns3_device_thread_irq_handler()
2001 if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams) in cdns3_device_thread_irq_handler()
2005 writel(~0, &priv_dev->regs->ep_ien); in cdns3_device_thread_irq_handler()
2006 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_device_thread_irq_handler()
2012 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
2029 remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2; in cdns3_ep_onchip_buffer_reserve()
2033 return -EPERM; in cdns3_ep_onchip_buffer_reserve()
2035 priv_dev->onchip_used_size += size; in cdns3_ep_onchip_buffer_reserve()
2043 if (priv_dev->out_mem_is_allocated >= size) in cdns3_ep_onchip_buffer_reserve()
2046 required = size - priv_dev->out_mem_is_allocated; in cdns3_ep_onchip_buffer_reserve()
2049 return -EPERM; in cdns3_ep_onchip_buffer_reserve()
2051 priv_dev->out_mem_is_allocated += required; in cdns3_ep_onchip_buffer_reserve()
2052 priv_dev->onchip_used_size += required; in cdns3_ep_onchip_buffer_reserve()
2061 struct cdns3_usb_regs __iomem *regs = priv_dev->regs; in cdns3_configure_dmult()
2064 if (priv_dev->dev_ver <= DEV_VER_V2) in cdns3_configure_dmult()
2065 writel(USB_CONF_DMULT, ®s->usb_conf); in cdns3_configure_dmult()
2067 if (priv_dev->dev_ver == DEV_VER_V2) in cdns3_configure_dmult()
2068 writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2); in cdns3_configure_dmult()
2070 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) { in cdns3_configure_dmult()
2073 if (priv_ep->dir) in cdns3_configure_dmult()
2074 mask = BIT(priv_ep->num + 16); in cdns3_configure_dmult()
2076 mask = BIT(priv_ep->num); in cdns3_configure_dmult()
2078 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) { in cdns3_configure_dmult()
2079 cdns3_set_register_bit(®s->tdl_from_trb, mask); in cdns3_configure_dmult()
2080 cdns3_set_register_bit(®s->tdl_beh, mask); in cdns3_configure_dmult()
2081 cdns3_set_register_bit(®s->tdl_beh2, mask); in cdns3_configure_dmult()
2082 cdns3_set_register_bit(®s->dma_adv_td, mask); in cdns3_configure_dmult()
2085 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) in cdns3_configure_dmult()
2086 cdns3_set_register_bit(®s->tdl_from_trb, mask); in cdns3_configure_dmult()
2088 cdns3_set_register_bit(®s->dtrans, mask); in cdns3_configure_dmult()
2093 * cdns3_ep_config - Configure hardware endpoint
2099 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC); in cdns3_ep_config()
2100 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_ep_config()
2101 u32 bEndpointAddress = priv_ep->num | priv_ep->dir; in cdns3_ep_config()
2102 u32 max_packet_size = priv_ep->wMaxPacketSize; in cdns3_ep_config()
2103 u8 maxburst = priv_ep->bMaxBurst; in cdns3_ep_config()
2108 buffering = priv_dev->ep_buf_size - 1; in cdns3_ep_config()
2112 switch (priv_ep->type) { in cdns3_ep_config()
2116 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir) in cdns3_ep_config()
2122 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir) in cdns3_ep_config()
2127 buffering = (priv_ep->bMaxBurst + 1) * (priv_ep->mult + 1) - 1; in cdns3_ep_config()
2130 switch (priv_dev->gadget.speed) { in cdns3_ep_config()
2138 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) { in cdns3_ep_config()
2140 maxburst = priv_dev->ep_buf_size - 1; in cdns3_ep_config()
2145 return -EINVAL; in cdns3_ep_config()
2149 priv_ep->trb_burst_size = 128; in cdns3_ep_config()
2151 priv_ep->trb_burst_size = 64; in cdns3_ep_config()
2153 priv_ep->trb_burst_size = 16; in cdns3_ep_config()
2158 * address is not aligned to 128 Bytes (which is a product of the 64-bit AXI in cdns3_ep_config()
2159 * and AXI maximum burst length of 16 or 0xF+1, dma_axi_ctrl0[3:0]). This in cdns3_ep_config()
2161 * specifically occurs from the position (4K - (address & 0x7F)) to 4K. in cdns3_ep_config()
2165 if (priv_dev->dev_ver < DEV_VER_V2) in cdns3_ep_config()
2166 priv_ep->trb_burst_size = 16; in cdns3_ep_config()
2172 if (!priv_dev->hw_configured_flag) { in cdns3_ep_config()
2174 !!priv_ep->dir); in cdns3_ep_config()
2176 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n"); in cdns3_ep_config()
2184 if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) { in cdns3_ep_config()
2185 if (priv_dev->dev_ver >= DEV_VER_V3) { in cdns3_ep_config()
2186 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0)); in cdns3_ep_config()
2192 cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb, in cdns3_ep_config()
2201 EP_CFG_MULT(priv_ep->mult) | /* must match EP setting */ in cdns3_ep_config()
2206 writel(ep_cfg, &priv_dev->regs->ep_cfg); in cdns3_ep_config()
2207 priv_ep->flags |= EP_CONFIGURED; in cdns3_ep_config()
2209 dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n", in cdns3_ep_config()
2210 priv_ep->name, ep_cfg); in cdns3_ep_config()
2219 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) || in cdns3_ep_dir_is_correct()
2220 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc)); in cdns3_ep_dir_is_correct()
2230 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { in cdns3_find_available_ep()
2234 char c[2] = {ep->name[2], '\0'}; in cdns3_find_available_ep()
2242 if (!(priv_ep->flags & EP_CLAIMED)) { in cdns3_find_available_ep()
2243 priv_ep->num = num; in cdns3_find_available_ep()
2249 return ERR_PTR(-ENOENT); in cdns3_find_available_ep()
2279 dev_err(priv_dev->dev, "no available ep\n"); in cdns3_gadget_match_ep()
2283 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name); in cdns3_gadget_match_ep()
2285 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_match_ep()
2286 priv_ep->endpoint.desc = desc; in cdns3_gadget_match_ep()
2287 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT; in cdns3_gadget_match_ep()
2288 priv_ep->type = usb_endpoint_type(desc); in cdns3_gadget_match_ep()
2289 priv_ep->flags |= EP_CLAIMED; in cdns3_gadget_match_ep()
2290 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; in cdns3_gadget_match_ep()
2291 priv_ep->wMaxPacketSize = usb_endpoint_maxp(desc); in cdns3_gadget_match_ep()
2292 priv_ep->mult = USB_EP_MAXP_MULT(priv_ep->wMaxPacketSize); in cdns3_gadget_match_ep()
2293 priv_ep->wMaxPacketSize &= USB_ENDPOINT_MAXP_MASK; in cdns3_gadget_match_ep()
2294 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && comp_desc) { in cdns3_gadget_match_ep()
2295 priv_ep->mult = USB_SS_MULT(comp_desc->bmAttributes) - 1; in cdns3_gadget_match_ep()
2296 priv_ep->bMaxBurst = comp_desc->bMaxBurst; in cdns3_gadget_match_ep()
2299 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_match_ep()
2300 return &priv_ep->endpoint; in cdns3_gadget_match_ep()
2304 * cdns3_gadget_ep_alloc_request - Allocates request
2320 priv_req->priv_ep = priv_ep; in cdns3_gadget_ep_alloc_request()
2323 return &priv_req->request; in cdns3_gadget_ep_alloc_request()
2327 * cdns3_gadget_ep_free_request - Free memory occupied by request
2336 if (priv_req->aligned_buf) in cdns3_gadget_ep_free_request()
2337 priv_req->aligned_buf->in_use = 0; in cdns3_gadget_ep_free_request()
2344 * cdns3_gadget_ep_enable - Enable endpoint
2365 return -EINVAL; in cdns3_gadget_ep_enable()
2369 priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_ep_enable()
2370 comp_desc = priv_ep->endpoint.comp_desc; in cdns3_gadget_ep_enable()
2372 if (!desc || desc->bDescriptorType != USB_DT_ENDPOINT) { in cdns3_gadget_ep_enable()
2373 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n"); in cdns3_gadget_ep_enable()
2374 return -EINVAL; in cdns3_gadget_ep_enable()
2377 if (!desc->wMaxPacketSize) { in cdns3_gadget_ep_enable()
2378 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n"); in cdns3_gadget_ep_enable()
2379 return -EINVAL; in cdns3_gadget_ep_enable()
2382 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED, in cdns3_gadget_ep_enable()
2383 "%s is already enabled\n", priv_ep->name)) in cdns3_gadget_ep_enable()
2386 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_ep_enable()
2388 priv_ep->endpoint.desc = desc; in cdns3_gadget_ep_enable()
2389 priv_ep->type = usb_endpoint_type(desc); in cdns3_gadget_ep_enable()
2390 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; in cdns3_gadget_ep_enable()
2392 if (priv_ep->interval > ISO_MAX_INTERVAL && in cdns3_gadget_ep_enable()
2393 priv_ep->type == USB_ENDPOINT_XFER_ISOC) { in cdns3_gadget_ep_enable()
2394 dev_err(priv_dev->dev, "Driver is limited to %d period\n", in cdns3_gadget_ep_enable()
2397 ret = -EINVAL; in cdns3_gadget_ep_enable()
2401 bEndpointAddress = priv_ep->num | priv_ep->dir; in cdns3_gadget_ep_enable()
2412 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) in cdns3_gadget_ep_enable()
2420 if (priv_dev->gadget.speed >= USB_SPEED_SUPER) { in cdns3_gadget_ep_enable()
2424 priv_ep->use_streams = true; in cdns3_gadget_ep_enable()
2426 priv_dev->using_streams |= true; in cdns3_gadget_ep_enable()
2439 bEndpointAddress = priv_ep->num | priv_ep->dir; in cdns3_gadget_ep_enable()
2444 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); in cdns3_gadget_ep_enable()
2446 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in cdns3_gadget_ep_enable()
2452 ret = -EINVAL; in cdns3_gadget_ep_enable()
2457 cdns3_set_register_bit(&priv_dev->regs->ep_ien, in cdns3_gadget_ep_enable()
2460 if (priv_dev->dev_ver < DEV_VER_V2) in cdns3_gadget_ep_enable()
2463 writel(reg, &priv_dev->regs->ep_sts_en); in cdns3_gadget_ep_enable()
2465 ep->desc = desc; in cdns3_gadget_ep_enable()
2466 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING | in cdns3_gadget_ep_enable()
2468 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR; in cdns3_gadget_ep_enable()
2469 priv_ep->wa1_set = 0; in cdns3_gadget_ep_enable()
2470 priv_ep->enqueue = 0; in cdns3_gadget_ep_enable()
2471 priv_ep->dequeue = 0; in cdns3_gadget_ep_enable()
2472 reg = readl(&priv_dev->regs->ep_sts); in cdns3_gadget_ep_enable()
2473 priv_ep->pcs = !!EP_STS_CCS(reg); in cdns3_gadget_ep_enable()
2474 priv_ep->ccs = !!EP_STS_CCS(reg); in cdns3_gadget_ep_enable()
2476 priv_ep->free_trbs = priv_ep->num_trbs - 1; in cdns3_gadget_ep_enable()
2478 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_ep_enable()
2484 * cdns3_gadget_ep_disable - Disable endpoint
2502 return -EINVAL; in cdns3_gadget_ep_disable()
2506 priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_ep_disable()
2508 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED), in cdns3_gadget_ep_disable()
2509 "%s is already disabled\n", priv_ep->name)) in cdns3_gadget_ep_disable()
2512 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_ep_disable()
2516 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); in cdns3_gadget_ep_disable()
2518 ep_cfg = readl(&priv_dev->regs->ep_cfg); in cdns3_gadget_ep_disable()
2520 writel(ep_cfg, &priv_dev->regs->ep_cfg); in cdns3_gadget_ep_disable()
2527 readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val, in cdns3_gadget_ep_disable()
2529 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); in cdns3_gadget_ep_disable()
2531 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in cdns3_gadget_ep_disable()
2535 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n", in cdns3_gadget_ep_disable()
2536 priv_ep->name); in cdns3_gadget_ep_disable()
2538 while (!list_empty(&priv_ep->pending_req_list)) { in cdns3_gadget_ep_disable()
2539 request = cdns3_next_request(&priv_ep->pending_req_list); in cdns3_gadget_ep_disable()
2542 -ESHUTDOWN); in cdns3_gadget_ep_disable()
2545 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { in cdns3_gadget_ep_disable()
2546 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); in cdns3_gadget_ep_disable()
2547 list_del_init(&priv_req->list); in cdns3_gadget_ep_disable()
2549 kfree(priv_req->request.buf); in cdns3_gadget_ep_disable()
2550 cdns3_gadget_ep_free_request(&priv_ep->endpoint, in cdns3_gadget_ep_disable()
2551 &priv_req->request); in cdns3_gadget_ep_disable()
2552 --priv_ep->wa2_counter; in cdns3_gadget_ep_disable()
2555 while (!list_empty(&priv_ep->deferred_req_list)) { in cdns3_gadget_ep_disable()
2556 request = cdns3_next_request(&priv_ep->deferred_req_list); in cdns3_gadget_ep_disable()
2559 -ESHUTDOWN); in cdns3_gadget_ep_disable()
2562 priv_ep->descmis_req = NULL; in cdns3_gadget_ep_disable()
2564 ep->desc = NULL; in cdns3_gadget_ep_disable()
2565 priv_ep->flags &= ~EP_ENABLED; in cdns3_gadget_ep_disable()
2566 priv_ep->use_streams = false; in cdns3_gadget_ep_disable()
2568 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_ep_disable()
2574 * __cdns3_gadget_ep_queue - Transfer data on endpoint
2586 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in __cdns3_gadget_ep_queue()
2590 request->actual = 0; in __cdns3_gadget_ep_queue()
2591 request->status = -EINPROGRESS; in __cdns3_gadget_ep_queue()
2595 if (priv_dev->dev_ver < DEV_VER_V2) { in __cdns3_gadget_ep_queue()
2607 if (likely(!(priv_req->flags & REQUEST_UNALIGNED))) { in __cdns3_gadget_ep_queue()
2608 ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request, in __cdns3_gadget_ep_queue()
2609 usb_endpoint_dir_in(ep->desc)); in __cdns3_gadget_ep_queue()
2614 list_add_tail(&request->list, &priv_ep->deferred_req_list); in __cdns3_gadget_ep_queue()
2623 if (!request->stream_id) { in __cdns3_gadget_ep_queue()
2624 if (priv_dev->hw_configured_flag && in __cdns3_gadget_ep_queue()
2625 !(priv_ep->flags & EP_STALLED) && in __cdns3_gadget_ep_queue()
2626 !(priv_ep->flags & EP_STALL_PENDING)) in __cdns3_gadget_ep_queue()
2629 if (priv_dev->hw_configured_flag && priv_ep->prime_flag) in __cdns3_gadget_ep_queue()
2646 return -EINVAL; in cdns3_gadget_ep_queue()
2649 priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_ep_queue()
2651 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_ep_queue()
2655 if (ret == 0 && request->zero && request->length && in cdns3_gadget_ep_queue()
2656 (request->length % ep->maxpacket == 0)) { in cdns3_gadget_ep_queue()
2660 zlp_request->buf = priv_dev->zlp_buf; in cdns3_gadget_ep_queue()
2661 zlp_request->length = 0; in cdns3_gadget_ep_queue()
2664 priv_req->flags |= REQUEST_ZLP; in cdns3_gadget_ep_queue()
2666 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n", in cdns3_gadget_ep_queue()
2667 priv_ep->name); in cdns3_gadget_ep_queue()
2671 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_ep_queue()
2676 * cdns3_gadget_ep_dequeue - Remove request from transfer queue
2695 if (!ep || !request || !ep->desc) in cdns3_gadget_ep_dequeue()
2696 return -EINVAL; in cdns3_gadget_ep_dequeue()
2698 priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_ep_dequeue()
2700 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_ep_dequeue()
2706 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); in cdns3_gadget_ep_dequeue()
2708 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list, in cdns3_gadget_ep_dequeue()
2716 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list, in cdns3_gadget_ep_dequeue()
2725 link_trb = priv_req->trb; in cdns3_gadget_ep_dequeue()
2730 writel(EP_CMD_DFLUSH, &priv_dev->regs->ep_cmd); in cdns3_gadget_ep_dequeue()
2733 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in cdns3_gadget_ep_dequeue()
2736 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma + in cdns3_gadget_ep_dequeue()
2737 ((priv_req->end_trb + 1) * TRB_SIZE))); in cdns3_gadget_ep_dequeue()
2738 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) | in cdns3_gadget_ep_dequeue()
2741 if (priv_ep->wa1_trb == priv_req->trb) in cdns3_gadget_ep_dequeue()
2745 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET); in cdns3_gadget_ep_dequeue()
2747 req = cdns3_next_request(&priv_ep->pending_req_list); in cdns3_gadget_ep_dequeue()
2752 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_ep_dequeue()
2757 * __cdns3_gadget_ep_set_halt - Sets stall on selected endpoint
2763 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in __cdns3_gadget_ep_set_halt()
2767 if (!(priv_ep->flags & EP_STALLED)) { in __cdns3_gadget_ep_set_halt()
2768 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts); in __cdns3_gadget_ep_set_halt()
2773 priv_ep->flags |= EP_STALL_PENDING; in __cdns3_gadget_ep_set_halt()
2778 * __cdns3_gadget_ep_clear_halt - Clears stall on selected endpoint
2784 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in __cdns3_gadget_ep_clear_halt()
2794 request = cdns3_next_request(&priv_ep->pending_req_list); in __cdns3_gadget_ep_clear_halt()
2797 trb = priv_req->trb; in __cdns3_gadget_ep_clear_halt()
2800 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE); in __cdns3_gadget_ep_clear_halt()
2804 writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd); in __cdns3_gadget_ep_clear_halt()
2807 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in __cdns3_gadget_ep_clear_halt()
2810 return -EINVAL; in __cdns3_gadget_ep_clear_halt()
2812 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING); in __cdns3_gadget_ep_clear_halt()
2826 * cdns3_gadget_ep_set_halt - Sets/clears stall on selected endpoint
2835 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; in cdns3_gadget_ep_set_halt()
2839 if (!(priv_ep->flags & EP_ENABLED)) in cdns3_gadget_ep_set_halt()
2840 return -EPERM; in cdns3_gadget_ep_set_halt()
2842 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_ep_set_halt()
2844 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); in cdns3_gadget_ep_set_halt()
2847 priv_ep->flags &= ~EP_WEDGE; in cdns3_gadget_ep_set_halt()
2853 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_ep_set_halt()
2872 * cdns3_gadget_get_frame - Returns number of actual ITP frame
2881 return readl(&priv_dev->regs->usb_itpn); in cdns3_gadget_get_frame()
2894 writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf); in __cdns3_gadget_wakeup()
2905 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_wakeup()
2907 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_wakeup()
2917 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_set_selfpowered()
2918 priv_dev->is_selfpowered = !!is_selfpowered; in cdns3_gadget_set_selfpowered()
2919 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_set_selfpowered()
2928 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf); in cdns3_gadget_pullup()
2930 writel(~0, &priv_dev->regs->ep_ists); in cdns3_gadget_pullup()
2931 writel(~0, &priv_dev->regs->usb_ists); in cdns3_gadget_pullup()
2932 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf); in cdns3_gadget_pullup()
2940 struct cdns3_usb_regs __iomem *regs = priv_dev->regs; in cdns3_gadget_config()
2946 writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien); in cdns3_gadget_config()
2952 if (priv_dev->dev_ver == DEV_VER_TI_V1) { in cdns3_gadget_config()
2953 reg = readl(®s->dbg_link1); in cdns3_gadget_config()
2958 writel(reg, ®s->dbg_link1); in cdns3_gadget_config()
2963 * This cause problem with cache, so driver restore non-secure in cdns3_gadget_config()
2966 reg = readl(®s->dma_axi_ctrl); in cdns3_gadget_config()
2969 writel(reg, ®s->dma_axi_ctrl); in cdns3_gadget_config()
2972 writel(USB_IEN_INIT, ®s->usb_ien); in cdns3_gadget_config()
2973 writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf); in cdns3_gadget_config()
2975 writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr); in cdns3_gadget_config()
2981 * cdns3_gadget_udc_start - Gadget start
2992 enum usb_device_speed max_speed = driver->max_speed; in cdns3_gadget_udc_start()
2994 spin_lock_irqsave(&priv_dev->lock, flags); in cdns3_gadget_udc_start()
2995 priv_dev->gadget_driver = driver; in cdns3_gadget_udc_start()
2998 max_speed = min(driver->max_speed, gadget->max_speed); in cdns3_gadget_udc_start()
3002 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf); in cdns3_gadget_udc_start()
3003 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf); in cdns3_gadget_udc_start()
3006 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf); in cdns3_gadget_udc_start()
3011 dev_err(priv_dev->dev, in cdns3_gadget_udc_start()
3022 spin_unlock_irqrestore(&priv_dev->lock, flags); in cdns3_gadget_udc_start()
3027 * cdns3_gadget_udc_stop - Stops gadget
3040 priv_dev->gadget_driver = NULL; in cdns3_gadget_udc_stop()
3042 priv_dev->onchip_used_size = 0; in cdns3_gadget_udc_stop()
3043 priv_dev->out_mem_is_allocated = 0; in cdns3_gadget_udc_stop()
3044 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; in cdns3_gadget_udc_stop()
3046 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { in cdns3_gadget_udc_stop()
3048 bEndpointAddress = priv_ep->num | priv_ep->dir; in cdns3_gadget_udc_stop()
3050 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); in cdns3_gadget_udc_stop()
3051 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, in cdns3_gadget_udc_stop()
3054 priv_ep->flags &= ~EP_CLAIMED; in cdns3_gadget_udc_stop()
3058 writel(0, &priv_dev->regs->usb_ien); in cdns3_gadget_udc_stop()
3059 writel(0, &priv_dev->regs->usb_pwr); in cdns3_gadget_udc_stop()
3060 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf); in cdns3_gadget_udc_stop()
3066 * cdns3_gadget_check_config - ensure cdns3 can support the USB configuration
3073 * least one max packet.
3086 list_for_each_entry(ep, &gadget->ep_list, ep_list) { in cdns3_gadget_check_config()
3088 if (!(priv_ep->flags & EP_CLAIMED)) in cdns3_gadget_check_config()
3091 n = (priv_ep->mult + 1) * (priv_ep->bMaxBurst + 1); in cdns3_gadget_check_config()
3092 if (ep->address & USB_DIR_IN) { in cdns3_gadget_check_config()
3098 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) in cdns3_gadget_check_config()
3103 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) in cdns3_gadget_check_config()
3111 if (total > priv_dev->onchip_buffers) in cdns3_gadget_check_config()
3112 return -ENOMEM; in cdns3_gadget_check_config()
3114 priv_dev->ep_buf_size = (priv_dev->onchip_buffers - 2 - iso) / (n_in + out); in cdns3_gadget_check_config()
3135 priv_dev->eps[16] = NULL; in cdns3_free_all_eps()
3138 if (priv_dev->eps[i]) { in cdns3_free_all_eps()
3139 cdns3_free_trb_pool(priv_dev->eps[i]); in cdns3_free_all_eps()
3140 devm_kfree(priv_dev->dev, priv_dev->eps[i]); in cdns3_free_all_eps()
3145 * cdns3_init_eps - Initializes software endpoints of gadget
3160 ep_enabled_reg = readl(&priv_dev->regs->usb_cap3); in cdns3_init_eps()
3161 iso_ep_reg = readl(&priv_dev->regs->usb_cap4); in cdns3_init_eps()
3163 dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n"); in cdns3_init_eps()
3174 priv_dev->eps[i] = priv_dev->eps[0]; in cdns3_init_eps()
3178 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep), in cdns3_init_eps()
3184 priv_ep->cdns3_dev = priv_dev; in cdns3_init_eps()
3185 priv_dev->eps[i] = priv_ep; in cdns3_init_eps()
3186 priv_ep->num = ep_number; in cdns3_init_eps()
3187 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT; in cdns3_init_eps()
3192 dev_err(priv_dev->dev, "Failed to init ep0\n"); in cdns3_init_eps()
3196 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s", in cdns3_init_eps()
3198 priv_ep->endpoint.name = priv_ep->name; in cdns3_init_eps()
3200 usb_ep_set_maxpacket_limit(&priv_ep->endpoint, in cdns3_init_eps()
3202 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS; in cdns3_init_eps()
3203 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops; in cdns3_init_eps()
3205 priv_ep->endpoint.caps.dir_in = 1; in cdns3_init_eps()
3207 priv_ep->endpoint.caps.dir_out = 1; in cdns3_init_eps()
3210 priv_ep->endpoint.caps.type_iso = 1; in cdns3_init_eps()
3212 priv_ep->endpoint.caps.type_bulk = 1; in cdns3_init_eps()
3213 priv_ep->endpoint.caps.type_int = 1; in cdns3_init_eps()
3215 list_add_tail(&priv_ep->endpoint.ep_list, in cdns3_init_eps()
3216 &priv_dev->gadget.ep_list); in cdns3_init_eps()
3219 priv_ep->flags = 0; in cdns3_init_eps()
3221 dev_dbg(priv_dev->dev, "Initialized %s support: %s %s\n", in cdns3_init_eps()
3222 priv_ep->name, in cdns3_init_eps()
3223 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "", in cdns3_init_eps()
3224 priv_ep->endpoint.caps.type_iso ? "ISO" : ""); in cdns3_init_eps()
3226 INIT_LIST_HEAD(&priv_ep->pending_req_list); in cdns3_init_eps()
3227 INIT_LIST_HEAD(&priv_ep->deferred_req_list); in cdns3_init_eps()
3228 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list); in cdns3_init_eps()
3234 return -ENOMEM; in cdns3_init_eps()
3249 priv_dev = cdns->gadget_dev; in cdns3_gadget_exit()
3252 pm_runtime_mark_last_busy(cdns->dev); in cdns3_gadget_exit()
3253 pm_runtime_put_autosuspend(cdns->dev); in cdns3_gadget_exit()
3255 usb_del_gadget(&priv_dev->gadget); in cdns3_gadget_exit()
3256 devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev); in cdns3_gadget_exit()
3260 while (!list_empty(&priv_dev->aligned_buf_list)) { in cdns3_gadget_exit()
3263 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list); in cdns3_gadget_exit()
3264 dma_free_noncoherent(priv_dev->sysdev, buf->size, in cdns3_gadget_exit()
3265 buf->buf, in cdns3_gadget_exit()
3266 buf->dma, in cdns3_gadget_exit()
3267 buf->dir); in cdns3_gadget_exit()
3269 list_del(&buf->list); in cdns3_gadget_exit()
3273 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf, in cdns3_gadget_exit()
3274 priv_dev->setup_dma); in cdns3_gadget_exit()
3275 dma_pool_destroy(priv_dev->eps_dma_pool); in cdns3_gadget_exit()
3277 kfree(priv_dev->zlp_buf); in cdns3_gadget_exit()
3278 usb_put_gadget(&priv_dev->gadget); in cdns3_gadget_exit()
3279 cdns->gadget_dev = NULL; in cdns3_gadget_exit()
3291 return -ENOMEM; in cdns3_gadget_start()
3293 usb_initialize_gadget(cdns->dev, &priv_dev->gadget, in cdns3_gadget_start()
3295 cdns->gadget_dev = priv_dev; in cdns3_gadget_start()
3296 priv_dev->sysdev = cdns->dev; in cdns3_gadget_start()
3297 priv_dev->dev = cdns->dev; in cdns3_gadget_start()
3298 priv_dev->regs = cdns->dev_regs; in cdns3_gadget_start()
3300 device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size", in cdns3_gadget_start()
3301 &priv_dev->onchip_buffers); in cdns3_gadget_start()
3303 if (priv_dev->onchip_buffers <= 0) { in cdns3_gadget_start()
3304 u32 reg = readl(&priv_dev->regs->usb_cap2); in cdns3_gadget_start()
3306 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg); in cdns3_gadget_start()
3309 if (!priv_dev->onchip_buffers) in cdns3_gadget_start()
3310 priv_dev->onchip_buffers = 256; in cdns3_gadget_start()
3312 max_speed = usb_get_maximum_speed(cdns->dev); in cdns3_gadget_start()
3321 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n", in cdns3_gadget_start()
3331 priv_dev->gadget.max_speed = max_speed; in cdns3_gadget_start()
3332 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; in cdns3_gadget_start()
3333 priv_dev->gadget.ops = &cdns3_gadget_ops; in cdns3_gadget_start()
3334 priv_dev->gadget.name = "usb-ss-gadget"; in cdns3_gadget_start()
3335 priv_dev->gadget.quirk_avoids_skb_reserve = 1; in cdns3_gadget_start()
3336 priv_dev->gadget.irq = cdns->dev_irq; in cdns3_gadget_start()
3338 spin_lock_init(&priv_dev->lock); in cdns3_gadget_start()
3339 INIT_WORK(&priv_dev->pending_status_wq, in cdns3_gadget_start()
3342 INIT_WORK(&priv_dev->aligned_buf_wq, in cdns3_gadget_start()
3346 INIT_LIST_HEAD(&priv_dev->gadget.ep_list); in cdns3_gadget_start()
3347 INIT_LIST_HEAD(&priv_dev->aligned_buf_list); in cdns3_gadget_start()
3348 priv_dev->eps_dma_pool = dma_pool_create("cdns3_eps_dma_pool", in cdns3_gadget_start()
3349 priv_dev->sysdev, in cdns3_gadget_start()
3351 if (!priv_dev->eps_dma_pool) { in cdns3_gadget_start()
3352 dev_err(priv_dev->dev, "Failed to create TRB dma pool\n"); in cdns3_gadget_start()
3353 ret = -ENOMEM; in cdns3_gadget_start()
3359 dev_err(priv_dev->dev, "Failed to create endpoints\n"); in cdns3_gadget_start()
3364 priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8, in cdns3_gadget_start()
3365 &priv_dev->setup_dma, GFP_DMA); in cdns3_gadget_start()
3366 if (!priv_dev->setup_buf) { in cdns3_gadget_start()
3367 ret = -ENOMEM; in cdns3_gadget_start()
3371 priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6); in cdns3_gadget_start()
3373 dev_dbg(priv_dev->dev, "Device Controller version: %08x\n", in cdns3_gadget_start()
3374 readl(&priv_dev->regs->usb_cap6)); in cdns3_gadget_start()
3375 dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n", in cdns3_gadget_start()
3376 readl(&priv_dev->regs->usb_cap1)); in cdns3_gadget_start()
3377 dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n", in cdns3_gadget_start()
3378 readl(&priv_dev->regs->usb_cap2)); in cdns3_gadget_start()
3380 priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver); in cdns3_gadget_start()
3381 if (priv_dev->dev_ver >= DEV_VER_V2) in cdns3_gadget_start()
3382 priv_dev->gadget.sg_supported = 1; in cdns3_gadget_start()
3384 priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL); in cdns3_gadget_start()
3385 if (!priv_dev->zlp_buf) { in cdns3_gadget_start()
3386 ret = -ENOMEM; in cdns3_gadget_start()
3391 ret = usb_add_gadget(&priv_dev->gadget); in cdns3_gadget_start()
3393 dev_err(priv_dev->dev, "Failed to add gadget\n"); in cdns3_gadget_start()
3399 kfree(priv_dev->zlp_buf); in cdns3_gadget_start()
3401 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf, in cdns3_gadget_start()
3402 priv_dev->setup_dma); in cdns3_gadget_start()
3406 dma_pool_destroy(priv_dev->eps_dma_pool); in cdns3_gadget_start()
3408 usb_put_gadget(&priv_dev->gadget); in cdns3_gadget_start()
3409 cdns->gadget_dev = NULL; in cdns3_gadget_start()
3417 /* Ensure 32-bit DMA Mask in case we switched back from Host mode */ in __cdns3_gadget_init()
3418 ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32)); in __cdns3_gadget_init()
3420 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret); in __cdns3_gadget_init()
3425 pm_runtime_get_sync(cdns->dev); in __cdns3_gadget_init()
3429 pm_runtime_put_sync(cdns->dev); in __cdns3_gadget_init()
3437 ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq, in __cdns3_gadget_init()
3440 IRQF_SHARED, dev_name(cdns->dev), in __cdns3_gadget_init()
3441 cdns->gadget_dev); in __cdns3_gadget_init()
3453 __must_hold(&cdns->lock) in cdns3_gadget_suspend()
3455 struct cdns3_device *priv_dev = cdns->gadget_dev; in cdns3_gadget_suspend()
3457 spin_unlock(&cdns->lock); in cdns3_gadget_suspend()
3459 spin_lock(&cdns->lock); in cdns3_gadget_suspend()
3461 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; in cdns3_gadget_suspend()
3462 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED); in cdns3_gadget_suspend()
3466 writel(0, &priv_dev->regs->usb_ien); in cdns3_gadget_suspend()
3473 struct cdns3_device *priv_dev = cdns->gadget_dev; in cdns3_gadget_resume()
3475 if (!priv_dev->gadget_driver) in cdns3_gadget_resume()
3480 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf); in cdns3_gadget_resume()
3486 * cdns3_gadget_init - initialize device structure
3496 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL); in cdns3_gadget_init()
3498 return -ENOMEM; in cdns3_gadget_init()
3500 rdrv->start = __cdns3_gadget_init; in cdns3_gadget_init()
3501 rdrv->stop = cdns3_gadget_exit; in cdns3_gadget_init()
3502 rdrv->suspend = cdns3_gadget_suspend; in cdns3_gadget_init()
3503 rdrv->resume = cdns3_gadget_resume; in cdns3_gadget_init()
3504 rdrv->state = CDNS_ROLE_STATE_INACTIVE; in cdns3_gadget_init()
3505 rdrv->name = "gadget"; in cdns3_gadget_init()
3506 cdns->roles[USB_ROLE_DEVICE] = rdrv; in cdns3_gadget_init()