Lines Matching +full:num +full:- +full:transfer +full:- +full:bits
1 // SPDX-License-Identifier: GPL-2.0
3 * Cadence USBHS-DEV Driver - gadget side.
28 #include <linux/dma-mapping.h>
36 #include "cdns2-gadget.h"
37 #include "cdns2-trace.h"
40 * set_reg_bit_32 - set bit in given 32 bits register.
42 * @mask: bits to set.
51 * clear_reg_bit_32 - clear bit in given 32 bits register.
53 * @mask: bits to clear.
61 /* Clear bit in given 8 bits register. */
68 /* Set bit in given 16 bits register. */
80 dma_index = readl(&pdev->adma_regs->ep_traddr) - pep->ring.dma;
93 if (pdev->selected_ep == ep)
96 pdev->selected_ep = ep;
97 writel(ep, &pdev->adma_regs->ep_sel);
103 u32 offset = (char *)trb - (char *)pep->ring.trbs;
105 return pep->ring.dma + offset;
110 struct cdns2_device *pdev = pep->pdev;
111 struct cdns2_ring *ring = &pep->ring;
113 if (pep->ring.trbs) {
114 dma_pool_free(pdev->eps_dma_pool, ring->trbs, ring->dma);
119 /* Allocates Transfer Ring segment. */
122 struct cdns2_device *pdev = pep->pdev;
126 ring = &pep->ring;
128 if (!ring->trbs) {
129 ring->trbs = dma_pool_alloc(pdev->eps_dma_pool,
131 &ring->dma);
132 if (!ring->trbs)
133 return -ENOMEM;
136 memset(ring->trbs, 0, TR_SEG_SIZE);
138 if (!pep->num)
142 link_trb = (ring->trbs + (TRBS_PER_SEGMENT - 1));
143 link_trb->buffer = cpu_to_le32(TRB_BUFFER(ring->dma));
144 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) |
156 struct cdns2_device *pdev = pep->pdev;
161 writel(DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd);
164 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
166 pep->ep_state |= EP_STALLED;
167 pep->ep_state &= ~EP_STALL_PENDING;
180 if (*index == (trb_in_seg - 1)) {
188 ring->free_trbs--;
189 cdns2_ep_inc_trb(&ring->enqueue, &ring->pcs, TRBS_PER_SEGMENT);
194 ring->free_trbs++;
195 cdns2_ep_inc_trb(&ring->dequeue, &ring->ccs, TRBS_PER_SEGMENT);
209 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET);
210 writeb(LPMCLOCK_SLEEP_ENTRY, &pdev->usb_regs->lpmclock);
212 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET);
218 u8 speed = readb(&pdev->usb_regs->speedctrl);
231 if (trb == (pep->ring.trbs + (TRBS_PER_SEGMENT - 1)))
232 return pep->ring.trbs;
241 struct usb_request *request = &preq->request;
242 struct cdns2_device *pdev = pep->pdev;
244 list_del_init(&preq->list);
246 if (request->status == -EINPROGRESS)
247 request->status = status;
249 usb_gadget_unmap_request_by_dev(pdev->dev, request, pep->dir);
252 preq->finished_trb = 0;
256 if (request->complete) {
257 spin_unlock(&pdev->lock);
258 usb_gadget_giveback_request(&pep->endpoint, request);
259 spin_lock(&pdev->lock);
262 if (request->buf == pdev->zlp_buf)
263 cdns2_gadget_ep_free_request(&pep->endpoint, request);
269 if (pep->wa1_set) {
272 pep->wa1_set = 0;
273 pep->wa1_trb_index = 0xFFFF;
274 if (pep->wa1_cycle_bit)
275 pep->wa1_trb->control |= cpu_to_le32(0x1);
277 pep->wa1_trb->control &= cpu_to_le32(~0x1);
284 struct cdns2_device *pdev = pep->pdev;
286 if (!pep->wa1_set) {
289 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
292 pep->wa1_cycle_bit = pep->ring.pcs ? TRB_CYCLE : 0;
293 pep->wa1_set = 1;
294 pep->wa1_trb = trb;
295 pep->wa1_trb_index = pep->ring.enqueue;
309 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
312 if (!doorbell || dma_index != pep->wa1_trb_index)
325 ring = &pep->ring;
327 if (num_trbs > ring->free_trbs) {
328 pep->ep_state |= EP_RING_FULL;
330 return -ENOBUFS;
333 if ((ring->enqueue + num_trbs) >= (TRBS_PER_SEGMENT - 1)) {
334 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
338 if (doorbell && dma_index == TRBS_PER_SEGMENT - 1) {
339 pep->ep_state |= EP_DEFERRED_DRDY;
340 return -ENOBUFS;
344 link_trb = ring->trbs + (TRBS_PER_SEGMENT - 1);
354 if (pep->type == USB_ENDPOINT_XFER_ISOC || TRBS_PER_SEGMENT > 2)
357 link_trb->control = cpu_to_le32(((ring->pcs) ? TRB_CYCLE : 0) |
367 struct cdns2_trb *link_trb = pep->ring.trbs + (TRBS_PER_SEGMENT - 1);
368 struct cdns2_trb *trb = preq->trb;
369 int num_trbs = preq->num_of_trb;
375 trb = pep->ring.trbs;
376 num_trbs = num_trbs - i;
389 if (pep->type == USB_ENDPOINT_XFER_ISOC) {
392 * for high bandwidth transfer and driver will split
396 (addr & (TRB_MAX_ISO_BUFF_SIZE - 1)),
399 if (pep->interval > 1)
400 num_trbs = pep->dir ? num_trbs * pep->interval : 1;
401 } else if (pep->dir) {
404 * Sometimes DMA doesn't want advance to next TD and transfer
420 full_len = req->length;
422 for_each_sg(req->sg, sg, req->num_sgs, i) {
428 * For HS ISO transfer TRBs should not exceed max packet size.
436 if (pep->type == USB_ENDPOINT_XFER_ISOC) {
446 num_trbs = num_trbs + temp - 1;
448 trb_len = trb_len - (temp << 10);
452 full_len -= len;
462 * transfer lengths. Controller handles the final data which are less
465 * Let's assume that driver prepares trb with trb->length 700 and burst size
493 pdev->burst_opt[i] = axi_burst_option[j];
524 num_tds = pep->dir ? pep->interval : 1;
525 split_size = preq->request.num_sgs ? 1024 : 3072;
528 if (preq->request.num_sgs) {
529 sg = preq->request.sg;
533 trb_dma = preq->request.dma;
534 block_length = preq->request.length;
537 full_len = preq->request.length;
538 sg_iter = preq->request.num_sgs ? preq->request.num_sgs : 1;
547 * Calculate TRB length.- buffer can't across 4KB
554 if (trb_buff_len > full_len - enqd_len)
555 trb_buff_len = full_len - enqd_len;
565 if (enqd_len + trb_buff_len >= full_len || !pep->dir)
575 if (pep->ring.pcs == 0)
578 control |= pep->ring.pcs;
585 TRB_BURST(pep->pdev->burst_opt[trb_buff_len]);
587 trb = pep->ring.trbs + pep->ring.enqueue;
588 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
589 trb->length = cpu_to_le32(length);
590 trb->control = cpu_to_le32(control);
597 --sg_iter;
598 sent_len -= block_length;
606 remaining_packet_size -= trb_buff_len;
607 block_length -= sent_len;
608 preq->end_trb = pep->ring.enqueue;
610 cdns2_ep_inc_enq(&pep->ring);
627 if (preq->request.num_sgs) {
628 sg = preq->request.sg;
632 trb_dma = preq->request.dma;
633 length = preq->request.length;
636 ring = &pep->ring;
639 control = TRB_TYPE(TRB_NORMAL) | ring->pcs | TRB_ISP;
640 trb = pep->ring.trbs + ring->enqueue;
642 if (pep->dir && sg_iter == trbs_per_td - 1) {
643 preq->end_trb = ring->enqueue;
644 control = ring->pcs | TRB_TYPE(TRB_LINK) | TRB_CHAIN
646 cdns2_ep_inc_enq(&pep->ring);
648 if (ring->enqueue == 0)
652 trb->buffer = cpu_to_le32(pep->ring.dma +
653 (ring->enqueue * TRB_SIZE));
654 trb->length = 0;
655 trb->control = cpu_to_le32(control);
668 if (sg_iter == (trbs_per_td - (pep->dir ? 2 : 1)))
673 trb->buffer = cpu_to_le32(trb_dma);
674 trb->length = cpu_to_le32(TRB_BURST(pep->trb_burst_size) |
676 trb->control = cpu_to_le32(control);
678 if (sg && sg_iter < (trbs_per_td - 1)) {
684 preq->end_trb = ring->enqueue;
685 cdns2_ep_inc_enq(&pep->ring);
695 * Memory barrier - Cycle Bit must be set before doorbell.
701 &pdev->adma_regs->ep_sts);
702 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
704 if (readl(&pdev->adma_regs->ep_sts) & DMA_EP_STS_TRBERR) {
705 writel(DMA_EP_STS_TRBERR, &pdev->adma_regs->ep_sts);
706 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
709 trace_cdns2_doorbell_epx(pep, readl(&pdev->adma_regs->ep_traddr));
719 if ((readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY))
720 return -EBUSY;
722 if (!pep->dir) {
723 set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
724 writel(pep->ring.dma + pep->ring.dequeue,
725 &pdev->adma_regs->ep_traddr);
733 buffer = pep->ring.dma + pep->ring.dequeue * TRB_SIZE;
734 hw_ccs = !!DMA_EP_STS_CCS(readl(&pdev->adma_regs->ep_sts));
736 trb = &pep->ring.trbs[TRBS_PER_SEGMENT];
737 trb->length = 0;
738 trb->buffer = cpu_to_le32(TRB_BUFFER(buffer));
739 trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) | TRB_TYPE(TRB_NORMAL));
743 * move to correct place in transfer ring.
746 trb->length = 0;
747 trb->buffer = cpu_to_le32(TRB_BUFFER(buffer));
748 trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) |
751 if (hw_ccs != pep->ring.ccs)
752 trb->control |= cpu_to_le32(TRB_TOGGLE);
754 set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
755 writel(pep->ring.dma + (TRBS_PER_SEGMENT * TRB_SIZE),
756 &pdev->adma_regs->ep_traddr);
761 /* Prepare and start transfer on no-default endpoint. */
765 struct cdns2_device *pdev = pep->pdev;
771 cdns2_select_ep(pdev, pep->endpoint.address);
773 if (preq->request.sg)
774 num_trbs = cdns2_count_sg_trbs(pep, &preq->request);
776 num_trbs = cdns2_count_trbs(pep, preq->request.dma,
777 preq->request.length);
783 ring = &pep->ring;
784 preq->start_trb = ring->enqueue;
785 preq->trb = ring->trbs + ring->enqueue;
787 if (usb_endpoint_xfer_isoc(pep->endpoint.desc)) {
790 togle_pcs = cdns2_wa1_update_guard(pep, ring->trbs + ring->enqueue);
794 preq->num_of_trb = num_trbs;
797 * Memory barrier - cycle bit must be set as the last operation.
803 preq->trb->control = preq->trb->control ^ cpu_to_le32(1);
808 if (!pep->wa1_set && !(pep->ep_state & EP_STALLED) && !pep->skip) {
809 if (pep->type == USB_ENDPOINT_XFER_ISOC) {
821 /* Prepare and start transfer for all not started requests. */
828 while (!list_empty(&pep->deferred_list)) {
829 preq = cdns2_next_preq(&pep->deferred_list);
835 list_move_tail(&preq->list, &pep->pending_list);
838 pep->ep_state &= ~EP_RING_FULL;
850 * SR - start ring
851 * ER - end ring
852 * DQ = ring->dequeue - dequeue position
853 * EQ = ring->enqueue - enqueue position
854 * ST = preq->start_trb - index of first TRB in transfer ring
855 * ET = preq->end_trb - index of last TRB in transfer ring
856 * CI = current_index - index of processed TRB by DMA.
859 * Then, we check if cycle bit for index pep->dequeue
863 * 1. ring->dequeue never equals to current_index.
864 * 2 ring->enqueue never exceed ring->dequeue
865 * 3. exception: ring->enqueue == ring->dequeue
866 * and ring->free_trbs is zero.
870 * Case 1 - ring->dequeue < current_index
874 * Case 2 - ring->dequeue > current_index
876 * transfer ring.
882 struct cdns2_device *pdev = pep->pdev;
889 ring = &pep->ring;
891 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
894 * Only ISO transfer can use 2 entries outside the standard
895 * Transfer Ring. First of them is used as zero length packet and the
902 if (preq->start_trb < preq->end_trb) {
903 if (ring->dequeue > preq->end_trb)
906 if (ring->dequeue < preq->start_trb)
910 if (preq->start_trb > preq->end_trb && ring->dequeue > preq->end_trb &&
911 ring->dequeue < preq->start_trb)
914 if (preq->start_trb == preq->end_trb && ring->dequeue != preq->end_trb)
917 trb = &ring->trbs[ring->dequeue];
919 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != ring->ccs)
922 if (doorbell == 1 && current_index == ring->dequeue)
926 if (TRBS_PER_SEGMENT == 2 && pep->type != USB_ENDPOINT_XFER_ISOC) {
931 if (ring->enqueue == ring->dequeue &&
932 ring->free_trbs == 0) {
934 } else if (ring->dequeue < current_index) {
935 if ((current_index == (TRBS_PER_SEGMENT - 1)) &&
936 !ring->dequeue)
940 } else if (ring->dequeue > current_index) {
957 trb = pep->ring.trbs + pep->ring.dequeue;
959 for (i = preq->finished_trb ; i < preq->num_of_trb; i++) {
960 preq->finished_trb++;
962 cdns2_ep_inc_deq(&pep->ring);
968 pep->skip = false;
979 while (!list_empty(&pep->pending_list)) {
980 preq = cdns2_next_preq(&pep->pending_list);
981 trb = pep->ring.trbs + pep->ring.dequeue;
987 while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK &&
988 le32_to_cpu(trb->length)) {
990 cdns2_ep_inc_deq(&pep->ring);
991 trb = pep->ring.trbs + pep->ring.dequeue;
995 * Re-select endpoint. It could be changed by other CPU
998 cdns2_select_ep(pdev, pep->endpoint.address);
1001 preq->finished_trb++;
1003 if (preq->finished_trb >= preq->num_of_trb)
1006 trb = pep->ring.trbs + pep->ring.dequeue;
1009 if (pep->dir && pep->type == USB_ENDPOINT_XFER_ISOC)
1012 * trb->length.
1014 preq->request.actual = preq->request.length;
1016 preq->request.actual +=
1017 TRB_LEN(le32_to_cpu(trb->length));
1019 cdns2_ep_inc_deq(&pep->ring);
1029 if (pep->type != USB_ENDPOINT_XFER_ISOC &&
1035 if (pep->skip && preq)
1038 if (!(pep->ep_state & EP_STALLED) &&
1039 !(pep->ep_state & EP_STALL_PENDING))
1045 if (!pdev->may_wakeup)
1049 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_SIGRSUME);
1054 struct cdns2_device *pdev = pep->pdev;
1064 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
1069 readl(&pdev->adma_regs->ep_traddr));
1075 struct cdns2_device *pdev = pep->pdev;
1080 cdns2_select_ep(pdev, pep->endpoint.address);
1084 ep_sts_reg = readl(&pdev->adma_regs->ep_sts);
1085 writel(ep_sts_reg, &pdev->adma_regs->ep_sts);
1087 if (pep->type == USB_ENDPOINT_XFER_ISOC) {
1091 mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize);
1092 cs = pep->dir ? readb(&pdev->epx_regs->ep[pep->num - 1].txcs) :
1093 readb(&pdev->epx_regs->ep[pep->num - 1].rxcs);
1104 clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
1107 readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val,
1110 writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd);
1113 readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val,
1116 pep->skip = true;
1119 if (ep_sts_reg & DMA_EP_STS_TRBERR || pep->skip) {
1120 if (pep->ep_state & EP_STALL_PENDING &&
1125 * For isochronous transfer driver completes request on
1128 * then the only way to finish all queued transfer is to do it
1131 if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->wa1_set) {
1132 if (!pep->dir)
1133 clear_reg_bit_32(&pdev->adma_regs->ep_cfg,
1137 if (pep->ep_state & EP_DEFERRED_DRDY) {
1138 pep->ep_state &= ~EP_DEFERRED_DRDY;
1147 if (!(pep->ep_state & EP_STALLED) &&
1148 !(pep->ep_state & EP_STALL_PENDING)) {
1149 if (pep->ep_state & EP_DEFERRED_DRDY) {
1150 pep->ep_state &= ~EP_DEFERRED_DRDY;
1153 cdns2_rearm_transfer(pep, pep->wa1_set);
1166 if (pdev->gadget_driver && pdev->gadget_driver->disconnect)
1167 pdev->gadget_driver->disconnect(&pdev->gadget);
1179 if (pdev->in_lpm)
1182 reg_usb_irq_m = readb(&pdev->interrupt_regs->usbien);
1183 reg_ext_irq_m = readb(&pdev->interrupt_regs->extien);
1186 writeb(0, &pdev->interrupt_regs->usbien);
1187 writeb(0, &pdev->interrupt_regs->extien);
1188 writel(0, &pdev->adma_regs->ep_ien);
1191 writel(0, &pdev->adma_regs->ep_sts);
1192 writeb(0, &pdev->interrupt_regs->usbirq);
1193 writeb(0, &pdev->interrupt_regs->extirq);
1195 reg_ep_ists = readl(&pdev->adma_regs->ep_ists);
1196 reg_usb_irq = readb(&pdev->interrupt_regs->usbirq);
1197 reg_ext_irq = readb(&pdev->interrupt_regs->extirq);
1203 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
1204 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
1205 writel(~0, &pdev->adma_regs->ep_ien);
1216 ext_irq = readb(&pdev->interrupt_regs->extirq) & EXTIRQ_WAKEUP;
1217 writeb(ext_irq, &pdev->interrupt_regs->extirq);
1219 usb_irq = readb(&pdev->interrupt_regs->usbirq) & USB_IEN_INIT;
1220 writeb(usb_irq, &pdev->interrupt_regs->usbirq);
1228 if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1229 spin_unlock(&pdev->lock);
1230 pdev->gadget_driver->resume(&pdev->gadget);
1231 spin_lock(&pdev->lock);
1236 u8 reg = readb(&pdev->usb_regs->lpmctrl);
1240 writeb(0, &pdev->usb_regs->sleep_clkgate);
1244 if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1245 spin_unlock(&pdev->lock);
1246 pdev->gadget_driver->suspend(&pdev->gadget);
1247 spin_lock(&pdev->lock);
1252 if (pdev->gadget_driver) {
1253 pdev->dev_address = 0;
1255 spin_unlock(&pdev->lock);
1256 usb_gadget_udc_reset(&pdev->gadget,
1257 pdev->gadget_driver);
1258 spin_lock(&pdev->lock);
1263 * process. For high-speed reset procedure is completed
1273 pdev->gadget.speed = speed;
1276 pdev->may_wakeup = 0;
1281 pdev->ep0_stage = CDNS2_SETUP_STAGE;
1297 spin_lock_irqsave(&pdev->lock, flags);
1301 dma_ep_ists = readl(&pdev->adma_regs->ep_ists);
1318 u8 ep_idx = bit > 16 ? (bit - 16) * 2 : (bit * 2) - 1;
1321 * Endpoints in pdev->eps[] are held in order:
1326 cdns2_handle_epx_interrupt(&pdev->eps[ep_idx]);
1330 writel(~0, &pdev->adma_regs->ep_ien);
1331 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
1332 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
1334 spin_unlock_irqrestore(&pdev->lock, flags);
1352 pep = &pdev->eps[i];
1354 if (!(pep->ep_state & EP_CLAIMED))
1357 if (pep->dir)
1358 min_buf_tx += pep->buffering;
1360 min_buf_rx += pep->buffering;
1364 pep = &pdev->eps[i];
1366 if (!(pep->ep_state & EP_CLAIMED))
1369 if (pep->dir) {
1370 free = pdev->onchip_tx_buf - min_buf_tx;
1372 if (free + pep->buffering >= 4)
1375 free = free + pep->buffering;
1377 min_buf_tx = min_buf_tx - pep->buffering + free;
1379 pep->buffering = free;
1382 &pdev->epx_regs->txstaddr[pep->num - 1]);
1383 pdev->epx_regs->txstaddr[pep->num - 1] = tx_offset;
1385 dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n",
1386 pep->name, tx_offset, pep->buffering);
1388 tx_offset += pep->buffering * 1024;
1390 free = pdev->onchip_rx_buf - min_buf_rx;
1392 if (free + pep->buffering >= 4)
1395 free = free + pep->buffering;
1397 min_buf_rx = min_buf_rx - pep->buffering + free;
1399 pep->buffering = free;
1401 &pdev->epx_regs->rxstaddr[pep->num - 1]);
1403 dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n",
1404 pep->name, rx_offset, pep->buffering);
1406 rx_offset += pep->buffering * 1024;
1414 bool is_iso_ep = (pep->type == USB_ENDPOINT_XFER_ISOC);
1415 struct cdns2_device *pdev = pep->pdev;
1423 switch (pep->type) {
1431 mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize);
1435 if (pep->dir) {
1436 set_reg_bit_8(&pdev->epx_regs->isoautoarm, BIT(pep->num));
1437 set_reg_bit_8(&pdev->epx_regs->isoautodump, BIT(pep->num));
1438 set_reg_bit_8(&pdev->epx_regs->isodctrl, BIT(pep->num));
1442 switch (pdev->gadget.speed) {
1451 return -EINVAL;
1454 ep_cfg |= (EPX_CON_VAL | (pep->buffering - 1));
1456 if (pep->dir) {
1458 writew(max_packet_size, &pdev->epx_regs->txmaxpack[pep->num - 1]);
1459 writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].txcon);
1461 writew(max_packet_size, &pdev->epx_regs->rxmaxpack[pep->num - 1]);
1462 writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].rxcon);
1465 writeb(pep->num | dir | FIFOCTRL_FIFOAUTO,
1466 &pdev->usb_regs->fifoctrl);
1467 writeb(pep->num | dir, &pdev->epx_regs->endprst);
1468 writeb(pep->num | ENDPRST_FIFORST | ENDPRST_TOGRST | dir,
1469 &pdev->epx_regs->endprst);
1472 pep->trb_burst_size = 128;
1474 pep->trb_burst_size = 64;
1476 pep->trb_burst_size = 16;
1478 cdns2_select_ep(pdev, pep->num | pep->dir);
1479 writel(DMA_EP_CMD_EPRST | DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd);
1481 ret = readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
1489 writel(DMA_EP_STS_TRBERR | DMA_EP_STS_ISOERR, &pdev->adma_regs->ep_sts_en);
1492 writel(DMA_EP_CFG_ENABLE, &pdev->adma_regs->ep_cfg);
1496 dev_dbg(pdev->dev, "Configure %s: with MPS: %08x, ep con: %02x\n",
1497 pep->name, max_packet_size, ep_cfg);
1512 preq->pep = pep;
1516 return &preq->request;
1538 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
1539 !desc->wMaxPacketSize) {
1540 return -EINVAL;
1544 pdev = pep->pdev;
1546 if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
1547 "%s is already enabled\n", pep->name))
1550 spin_lock_irqsave(&pdev->lock, flags);
1552 pep->type = usb_endpoint_type(desc);
1553 pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1555 if (pdev->gadget.speed == USB_SPEED_FULL)
1556 if (pep->type == USB_ENDPOINT_XFER_INT)
1557 pep->interval = desc->bInterval;
1559 if (pep->interval > ISO_MAX_INTERVAL &&
1560 pep->type == USB_ENDPOINT_XFER_ISOC) {
1561 dev_err(pdev->dev, "ISO period is limited to %d (current: %d)\n",
1562 ISO_MAX_INTERVAL, pep->interval);
1564 ret = -EINVAL;
1569 * During ISO OUT traffic DMA reads Transfer Ring for the EP which has
1576 if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->dir)
1586 ret = -EINVAL;
1592 pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING);
1593 pep->ep_state |= EP_ENABLED;
1594 pep->wa1_set = 0;
1595 pep->ring.enqueue = 0;
1596 pep->ring.dequeue = 0;
1597 reg = readl(&pdev->adma_regs->ep_sts);
1598 pep->ring.pcs = !!DMA_EP_STS_CCS(reg);
1599 pep->ring.ccs = !!DMA_EP_STS_CCS(reg);
1601 writel(pep->ring.dma, &pdev->adma_regs->ep_traddr);
1604 pep->ring.free_trbs = TRBS_PER_SEGMENT - 1;
1607 spin_unlock_irqrestore(&pdev->lock, flags);
1621 return -EINVAL;
1624 pdev = pep->pdev;
1626 if (dev_WARN_ONCE(pdev->dev, !(pep->ep_state & EP_ENABLED),
1627 "%s is already disabled\n", pep->name))
1630 spin_lock_irqsave(&pdev->lock, flags);
1634 cdns2_select_ep(pdev, ep->desc->bEndpointAddress);
1636 clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
1641 * 10us is enough time for controller to stop transfer.
1643 readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val,
1645 writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd);
1647 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
1651 while (!list_empty(&pep->pending_list)) {
1652 preq = cdns2_next_preq(&pep->pending_list);
1653 cdns2_gadget_giveback(pep, preq, -ESHUTDOWN);
1656 while (!list_empty(&pep->deferred_list)) {
1657 preq = cdns2_next_preq(&pep->deferred_list);
1658 cdns2_gadget_giveback(pep, preq, -ESHUTDOWN);
1661 ep->desc = NULL;
1662 pep->ep_state &= ~EP_ENABLED;
1664 spin_unlock_irqrestore(&pdev->lock, flags);
1673 struct cdns2_device *pdev = pep->pdev;
1677 request = &preq->request;
1678 request->actual = 0;
1679 request->status = -EINPROGRESS;
1681 ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->dir);
1687 list_add_tail(&preq->list, &pep->deferred_list);
1690 if (!(pep->ep_state & EP_STALLED) && !(pep->ep_state & EP_STALL_PENDING))
1707 return -EINVAL;
1710 pdev = pep->pdev;
1712 if (!(pep->ep_state & EP_ENABLED)) {
1713 dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1714 pep->name);
1715 return -EINVAL;
1718 spin_lock_irqsave(&pdev->lock, flags);
1723 if (ret == 0 && request->zero && request->length &&
1724 (request->length % ep->maxpacket == 0)) {
1728 zlp_request->buf = pdev->zlp_buf;
1729 zlp_request->length = 0;
1735 spin_unlock_irqrestore(&pdev->lock, flags);
1750 if (!ep || !request || !ep->desc)
1751 return -EINVAL;
1754 if (!pep->endpoint.desc) {
1755 dev_err(pep->pdev->dev, "%s: can't dequeue to disabled endpoint\n",
1756 pep->name);
1757 return -ESHUTDOWN;
1761 if (!(pep->ep_state & EP_ENABLED))
1764 spin_lock_irqsave(&pep->pdev->lock, flags);
1769 list_for_each_entry_safe(preq, preq_temp, &pep->pending_list, list) {
1776 list_for_each_entry_safe(preq, preq_temp, &pep->deferred_list, list) {
1784 link_trb = preq->trb;
1789 writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd);
1792 readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val,
1795 buffer = cpu_to_le32(TRB_BUFFER(pep->ring.dma +
1796 ((preq->end_trb + 1) * TRB_SIZE)));
1798 for (i = 0; i < preq->num_of_trb; i++) {
1799 link_trb->buffer = buffer;
1800 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control)
1808 if (pep->wa1_trb == preq->trb)
1812 cdns2_gadget_giveback(pep, cur_preq, -ECONNRESET);
1814 preq = cdns2_next_preq(&pep->pending_list);
1819 spin_unlock_irqrestore(&pep->pdev->lock, flags);
1830 if (!(pep->ep_state & EP_ENABLED))
1831 return -EPERM;
1833 if (pep->dir) {
1835 conf = &pdev->epx_regs->ep[pep->num - 1].txcon;
1837 conf = &pdev->epx_regs->ep[pep->num - 1].rxcon;
1845 preq = cdns2_next_preq(&pep->pending_list);
1847 trb = preq->trb;
1850 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
1857 writeb(dir | pep->num, &pdev->epx_regs->endprst);
1858 writeb(dir | ENDPRST_TOGRST | pep->num,
1859 &pdev->epx_regs->endprst);
1863 pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING);
1876 writeb(dir | pep->num, &pdev->epx_regs->endprst);
1877 writeb(dir | ENDPRST_FIFORST | pep->num,
1878 &pdev->epx_regs->endprst);
1879 pep->ep_state |= EP_STALLED;
1889 struct cdns2_device *pdev = pep->pdev;
1894 spin_lock_irqsave(&pdev->lock, flags);
1896 preq = cdns2_next_preq(&pep->pending_list);
1899 ret = -EAGAIN;
1904 pep->ep_state &= ~EP_WEDGE;
1909 spin_unlock_irqrestore(&pdev->lock, flags);
1918 pep->ep_state |= EP_WEDGE;
1931 list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) {
1932 unsigned long num;
1935 char c[2] = {ep->name[2], '\0'};
1937 ret = kstrtoul(c, 10, &num);
1942 if (pep->num != num)
1945 ep_correct = (pep->endpoint.caps.dir_in &&
1947 (pep->endpoint.caps.dir_out &&
1950 if (ep_correct && !(pep->ep_state & EP_CLAIMED))
1954 return ERR_PTR(-ENOENT);
1959 * on-chip memory usage.
1972 dev_err(pdev->dev, "no available ep\n");
1976 spin_lock_irqsave(&pdev->lock, flags);
1979 pep->buffering = 4;
1981 pep->buffering = 1;
1983 pep->ep_state |= EP_CLAIMED;
1984 spin_unlock_irqrestore(&pdev->lock, flags);
1986 return &pep->endpoint;
2004 return readw(&pdev->usb_regs->frmnr);
2012 spin_lock_irqsave(&pdev->lock, flags);
2014 spin_unlock_irqrestore(&pdev->lock, flags);
2025 spin_lock_irqsave(&pdev->lock, flags);
2026 pdev->is_selfpowered = !!is_selfpowered;
2027 spin_unlock_irqrestore(&pdev->lock, flags);
2034 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2037 writeb(0, &pdev->interrupt_regs->extien);
2038 writeb(0, &pdev->interrupt_regs->usbien);
2039 writew(0, &pdev->adma_regs->ep_ien);
2042 writeb(0x0, &pdev->interrupt_regs->usbirq);
2050 writel(~0x0, &pdev->adma_regs->ep_ien);
2052 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
2053 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
2054 writel(DMA_CONF_DMULT, &pdev->adma_regs->conf);
2068 disable_irq(pdev->irq);
2069 spin_lock_irqsave(&pdev->lock, flags);
2073 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2078 spin_unlock_irqrestore(&pdev->lock, flags);
2079 enable_irq(pdev->irq);
2088 enum usb_device_speed max_speed = driver->max_speed;
2091 spin_lock_irqsave(&pdev->lock, flags);
2092 pdev->gadget_driver = driver;
2095 max_speed = min(driver->max_speed, gadget->max_speed);
2099 writeb(SPEEDCTRL_HSDISABLE, &pdev->usb_regs->speedctrl);
2102 writeb(0, &pdev->usb_regs->speedctrl);
2105 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
2115 writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst);
2117 &pdev->usb_regs->endprst);
2118 writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->usb_regs->endprst);
2123 spin_unlock_irqrestore(&pdev->lock, flags);
2136 pdev->gadget_driver = NULL;
2137 pdev->gadget.speed = USB_SPEED_UNKNOWN;
2139 list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) {
2141 bEndpointAddress = pep->num | pep->dir;
2143 writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd);
2144 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
2150 writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst);
2152 &pdev->epx_regs->endprst);
2153 writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->epx_regs->endprst);
2173 cdns2_free_tr_segment(&pdev->eps[i]);
2187 * Endpoints are being held in pdev->eps[] in form:
2193 pep = &pdev->eps[i];
2194 pep->pdev = pdev;
2195 pep->num = epnum;
2197 pep->dir = direction ? USB_DIR_IN : USB_DIR_OUT;
2198 pep->idx = i;
2200 /* Ep0in and ep0out are represented by pdev->eps[0]. */
2204 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
2211 dev_err(pdev->dev, "Failed to init ep0\n");
2215 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
2217 pep->endpoint.name = pep->name;
2219 usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
2220 pep->endpoint.ops = &cdns2_gadget_ep_ops;
2221 list_add_tail(&pep->endpoint.ep_list, &pdev->gadget.ep_list);
2223 pep->endpoint.caps.dir_in = direction;
2224 pep->endpoint.caps.dir_out = !direction;
2226 pep->endpoint.caps.type_iso = 1;
2227 pep->endpoint.caps.type_bulk = 1;
2228 pep->endpoint.caps.type_int = 1;
2231 pep->endpoint.name = pep->name;
2232 pep->ep_state = 0;
2234 dev_dbg(pdev->dev, "Init %s, SupType: CTRL: %s, INT: %s, "
2236 pep->name,
2237 str_yes_no(pep->endpoint.caps.type_control),
2238 str_yes_no(pep->endpoint.caps.type_int),
2239 str_yes_no(pep->endpoint.caps.type_bulk),
2240 str_yes_no(pep->endpoint.caps.type_iso),
2241 str_yes_no(pep->endpoint.caps.dir_in),
2242 str_yes_no(pep->endpoint.caps.dir_out));
2244 INIT_LIST_HEAD(&pep->pending_list);
2245 INIT_LIST_HEAD(&pep->deferred_list);
2257 pdev->usb_regs = pdev->regs;
2258 pdev->ep0_regs = pdev->regs;
2259 pdev->epx_regs = pdev->regs;
2260 pdev->interrupt_regs = pdev->regs;
2261 pdev->adma_regs = pdev->regs + CDNS2_ADMA_REGS_OFFSET;
2265 &pdev->usb_regs->cpuctrl);
2268 usb_initialize_gadget(pdev->dev, &pdev->gadget, NULL);
2270 device_property_read_u16(pdev->dev, "cdns,on-chip-tx-buff-size",
2271 &pdev->onchip_tx_buf);
2272 device_property_read_u16(pdev->dev, "cdns,on-chip-rx-buff-size",
2273 &pdev->onchip_rx_buf);
2274 device_property_read_u32(pdev->dev, "cdns,avail-endpoints",
2275 &pdev->eps_supported);
2281 if (!pdev->onchip_tx_buf && !pdev->onchip_rx_buf) {
2282 ret = -EINVAL;
2283 dev_err(pdev->dev, "Invalid on-chip memory configuration\n");
2287 if (!(pdev->eps_supported & ~0x00010001)) {
2288 ret = -EINVAL;
2289 dev_err(pdev->dev, "No hardware endpoints available\n");
2293 max_speed = usb_get_maximum_speed(pdev->dev);
2300 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
2308 pdev->gadget.max_speed = max_speed;
2309 pdev->gadget.speed = USB_SPEED_UNKNOWN;
2310 pdev->gadget.ops = &cdns2_gadget_ops;
2311 pdev->gadget.name = "usbhs-gadget";
2312 pdev->gadget.quirk_avoids_skb_reserve = 1;
2313 pdev->gadget.irq = pdev->irq;
2315 spin_lock_init(&pdev->lock);
2316 INIT_WORK(&pdev->pending_status_wq, cdns2_pending_setup_status_handler);
2319 INIT_LIST_HEAD(&pdev->gadget.ep_list);
2320 pdev->eps_dma_pool = dma_pool_create("cdns2_eps_dma_pool", pdev->dev,
2322 if (!pdev->eps_dma_pool) {
2323 dev_err(pdev->dev, "Failed to create TRB dma pool\n");
2324 ret = -ENOMEM;
2330 dev_err(pdev->dev, "Failed to create endpoints\n");
2334 pdev->gadget.sg_supported = 1;
2336 pdev->zlp_buf = kzalloc(CDNS2_EP_ZLP_BUF_SIZE, GFP_KERNEL);
2337 if (!pdev->zlp_buf) {
2338 ret = -ENOMEM;
2343 buf = dma_alloc_coherent(pdev->dev, 8, &pdev->ep0_preq.request.dma,
2345 pdev->ep0_preq.request.buf = buf;
2347 if (!pdev->ep0_preq.request.buf) {
2348 ret = -ENOMEM;
2353 ret = usb_add_gadget(&pdev->gadget);
2355 dev_err(pdev->dev, "Failed to add gadget\n");
2362 dma_free_coherent(pdev->dev, 8, pdev->ep0_preq.request.buf,
2363 pdev->ep0_preq.request.dma);
2365 kfree(pdev->zlp_buf);
2367 dma_pool_destroy(pdev->eps_dma_pool);
2369 usb_put_gadget(&pdev->gadget);
2380 spin_lock_irqsave(&pdev->lock, flags);
2381 pdev->gadget.speed = USB_SPEED_UNKNOWN;
2384 usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
2388 writeb(0, &pdev->interrupt_regs->usbien);
2389 writel(0, &pdev->adma_regs->ep_ien);
2390 spin_unlock_irqrestore(&pdev->lock, flags);
2399 spin_lock_irqsave(&pdev->lock, flags);
2401 if (!pdev->gadget_driver) {
2402 spin_unlock_irqrestore(&pdev->lock, flags);
2409 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2411 spin_unlock_irqrestore(&pdev->lock, flags);
2418 pm_runtime_mark_last_busy(pdev->dev);
2419 pm_runtime_put_autosuspend(pdev->dev);
2421 usb_del_gadget(&pdev->gadget);
2424 dma_pool_destroy(pdev->eps_dma_pool);
2425 kfree(pdev->zlp_buf);
2426 usb_put_gadget(&pdev->gadget);
2433 /* Ensure 32-bit DMA Mask. */
2434 ret = dma_set_mask_and_coherent(pdev->dev, DMA_BIT_MASK(32));
2436 dev_err(pdev->dev, "Failed to set dma mask: %d\n", ret);
2440 pm_runtime_get_sync(pdev->dev);
2446 pm_runtime_put_sync(pdev->dev);
2454 ret = devm_request_threaded_irq(pdev->dev, pdev->irq,
2458 dev_name(pdev->dev),