1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Cadence CDNSP DRD Driver.
4 *
5 * Copyright (C) 2020 Cadence.
6 *
7 * Author: Pawel Laszczak <pawell@cadence.com>
8 *
9 */
10
11 #include <linux/moduleparam.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/module.h>
14 #include <linux/iopoll.h>
15 #include <linux/delay.h>
16 #include <linux/log2.h>
17 #include <linux/slab.h>
18 #include <linux/string_choices.h>
19 #include <linux/pci.h>
20 #include <linux/irq.h>
21 #include <linux/dmi.h>
22
23 #include "core.h"
24 #include "gadget-export.h"
25 #include "drd.h"
26 #include "cdnsp-gadget.h"
27 #include "cdnsp-trace.h"
28
cdnsp_port_speed(unsigned int port_status)29 unsigned int cdnsp_port_speed(unsigned int port_status)
30 {
31 /*Detect gadget speed based on PORTSC register*/
32 if (DEV_SUPERSPEEDPLUS(port_status) ||
33 DEV_SSP_GEN1x2(port_status) || DEV_SSP_GEN2x2(port_status))
34 return USB_SPEED_SUPER_PLUS;
35 else if (DEV_SUPERSPEED(port_status))
36 return USB_SPEED_SUPER;
37 else if (DEV_HIGHSPEED(port_status))
38 return USB_SPEED_HIGH;
39 else if (DEV_FULLSPEED(port_status))
40 return USB_SPEED_FULL;
41
42 /* If device is detached then speed will be USB_SPEED_UNKNOWN.*/
43 return USB_SPEED_UNKNOWN;
44 }
45
46 /*
47 * Given a port state, this function returns a value that would result in the
48 * port being in the same state, if the value was written to the port status
49 * control register.
50 * Save Read Only (RO) bits and save read/write bits where
51 * writing a 0 clears the bit and writing a 1 sets the bit (RWS).
52 * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect.
53 */
cdnsp_port_state_to_neutral(u32 state)54 u32 cdnsp_port_state_to_neutral(u32 state)
55 {
56 /* Save read-only status and port state. */
57 return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS);
58 }
59
60 /**
61 * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities
62 * with capability ID id.
63 * @base: PCI MMIO registers base address.
64 * @start: Address at which to start looking, (0 or HCC_PARAMS to start at
65 * beginning of list)
66 * @id: Extended capability ID to search for.
67 *
68 * Returns the offset of the next matching extended capability structure.
69 * Some capabilities can occur several times,
70 * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all.
71 */
cdnsp_find_next_ext_cap(void __iomem * base,u32 start,int id)72 int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id)
73 {
74 u32 offset = start;
75 u32 next;
76 u32 val;
77
78 if (!start || start == HCC_PARAMS_OFFSET) {
79 val = readl(base + HCC_PARAMS_OFFSET);
80 if (val == ~0)
81 return 0;
82
83 offset = HCC_EXT_CAPS(val) << 2;
84 if (!offset)
85 return 0;
86 }
87
88 do {
89 val = readl(base + offset);
90 if (val == ~0)
91 return 0;
92
93 if (EXT_CAPS_ID(val) == id && offset != start)
94 return offset;
95
96 next = EXT_CAPS_NEXT(val);
97 offset += next << 2;
98 } while (next);
99
100 return 0;
101 }
102
cdnsp_set_link_state(struct cdnsp_device * pdev,__le32 __iomem * port_regs,u32 link_state)103 void cdnsp_set_link_state(struct cdnsp_device *pdev,
104 __le32 __iomem *port_regs,
105 u32 link_state)
106 {
107 int port_num = 0xFF;
108 u32 temp;
109
110 temp = readl(port_regs);
111 temp = cdnsp_port_state_to_neutral(temp);
112 temp |= PORT_WKCONN_E | PORT_WKDISC_E;
113 writel(temp, port_regs);
114
115 temp &= ~PORT_PLS_MASK;
116 temp |= PORT_LINK_STROBE | link_state;
117
118 if (pdev->active_port)
119 port_num = pdev->active_port->port_num;
120
121 trace_cdnsp_handle_port_status(port_num, readl(port_regs));
122 writel(temp, port_regs);
123 trace_cdnsp_link_state_changed(port_num, readl(port_regs));
124 }
125
cdnsp_disable_port(struct cdnsp_device * pdev,struct cdnsp_port * port)126 static void cdnsp_disable_port(struct cdnsp_device *pdev,
127 struct cdnsp_port *port)
128 {
129 u32 temp;
130
131 if (!port->exist)
132 return;
133
134 temp = cdnsp_port_state_to_neutral(readl(&port->regs->portsc));
135 writel(temp | PORT_PED, &port->regs->portsc);
136 }
137
cdnsp_clear_port_change_bit(struct cdnsp_device * pdev,struct cdnsp_port * port)138 static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
139 struct cdnsp_port *port)
140 {
141 u32 portsc;
142
143 if (!port->exist)
144 return;
145
146 portsc = readl(&port->regs->portsc);
147 writel(cdnsp_port_state_to_neutral(portsc) |
148 (portsc & PORT_CHANGE_BITS), &port->regs->portsc);
149 }
150
cdnsp_set_apb_timeout_value(struct cdnsp_device * pdev)151 static void cdnsp_set_apb_timeout_value(struct cdnsp_device *pdev)
152 {
153 struct cdns *cdns = dev_get_drvdata(pdev->dev);
154 __le32 __iomem *reg;
155 void __iomem *base;
156 u32 offset = 0;
157 u32 val;
158
159 if (!cdns->override_apb_timeout)
160 return;
161
162 base = &pdev->cap_regs->hc_capbase;
163 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
164 reg = base + offset + REG_CHICKEN_BITS_3_OFFSET;
165
166 val = readl(reg);
167 val = CHICKEN_APB_TIMEOUT_SET(val, cdns->override_apb_timeout);
168 writel(val, reg);
169 }
170
cdnsp_set_chicken_bits_2(struct cdnsp_device * pdev,u32 bit)171 static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
172 {
173 __le32 __iomem *reg;
174 void __iomem *base;
175 u32 offset = 0;
176
177 base = &pdev->cap_regs->hc_capbase;
178 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
179 reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
180
181 bit = readl(reg) | bit;
182 writel(bit, reg);
183 }
184
cdnsp_clear_chicken_bits_2(struct cdnsp_device * pdev,u32 bit)185 static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
186 {
187 __le32 __iomem *reg;
188 void __iomem *base;
189 u32 offset = 0;
190
191 base = &pdev->cap_regs->hc_capbase;
192 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
193 reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
194
195 bit = readl(reg) & ~bit;
196 writel(bit, reg);
197 }
198
199 /*
200 * Disable interrupts and begin the controller halting process.
201 */
cdnsp_quiesce(struct cdnsp_device * pdev)202 static void cdnsp_quiesce(struct cdnsp_device *pdev)
203 {
204 u32 halted;
205 u32 mask;
206 u32 cmd;
207
208 mask = ~(u32)(CDNSP_IRQS);
209
210 halted = readl(&pdev->op_regs->status) & STS_HALT;
211 if (!halted)
212 mask &= ~(CMD_R_S | CMD_DEVEN);
213
214 cmd = readl(&pdev->op_regs->command);
215 cmd &= mask;
216 writel(cmd, &pdev->op_regs->command);
217 }
218
219 /*
220 * Force controller into halt state.
221 *
222 * Disable any IRQs and clear the run/stop bit.
223 * Controller will complete any current and actively pipelined transactions, and
224 * should halt within 16 ms of the run/stop bit being cleared.
225 * Read controller Halted bit in the status register to see when the
226 * controller is finished.
227 */
cdnsp_halt(struct cdnsp_device * pdev)228 int cdnsp_halt(struct cdnsp_device *pdev)
229 {
230 int ret;
231 u32 val;
232
233 cdnsp_quiesce(pdev);
234
235 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val,
236 val & STS_HALT, 1,
237 CDNSP_MAX_HALT_USEC);
238 if (ret) {
239 dev_err(pdev->dev, "ERROR: Device halt failed\n");
240 return ret;
241 }
242
243 pdev->cdnsp_state |= CDNSP_STATE_HALTED;
244
245 return 0;
246 }
247
248 /*
249 * device controller died, register read returns 0xffffffff, or command never
250 * ends.
251 */
cdnsp_died(struct cdnsp_device * pdev)252 void cdnsp_died(struct cdnsp_device *pdev)
253 {
254 dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n");
255 pdev->cdnsp_state |= CDNSP_STATE_DYING;
256 cdnsp_halt(pdev);
257 }
258
259 /*
260 * Set the run bit and wait for the device to be running.
261 */
cdnsp_start(struct cdnsp_device * pdev)262 static int cdnsp_start(struct cdnsp_device *pdev)
263 {
264 u32 temp;
265 int ret;
266
267 temp = readl(&pdev->op_regs->command);
268 temp |= (CMD_R_S | CMD_DEVEN);
269 writel(temp, &pdev->op_regs->command);
270
271 pdev->cdnsp_state = 0;
272
273 /*
274 * Wait for the STS_HALT Status bit to be 0 to indicate the device is
275 * running.
276 */
277 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
278 !(temp & STS_HALT), 1,
279 CDNSP_MAX_HALT_USEC);
280 if (ret) {
281 pdev->cdnsp_state = CDNSP_STATE_DYING;
282 dev_err(pdev->dev, "ERROR: Controller run failed\n");
283 }
284
285 return ret;
286 }
287
288 /*
289 * Reset a halted controller.
290 *
291 * This resets pipelines, timers, counters, state machines, etc.
292 * Transactions will be terminated immediately, and operational registers
293 * will be set to their defaults.
294 */
cdnsp_reset(struct cdnsp_device * pdev)295 int cdnsp_reset(struct cdnsp_device *pdev)
296 {
297 u32 command;
298 u32 temp;
299 int ret;
300
301 temp = readl(&pdev->op_regs->status);
302
303 if (temp == ~(u32)0) {
304 dev_err(pdev->dev, "Device not accessible, reset failed.\n");
305 return -ENODEV;
306 }
307
308 if ((temp & STS_HALT) == 0) {
309 dev_err(pdev->dev, "Controller not halted, aborting reset.\n");
310 return -EINVAL;
311 }
312
313 command = readl(&pdev->op_regs->command);
314 command |= CMD_RESET;
315 writel(command, &pdev->op_regs->command);
316
317 ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp,
318 !(temp & CMD_RESET), 1,
319 10 * 1000);
320 if (ret) {
321 dev_err(pdev->dev, "ERROR: Controller reset failed\n");
322 return ret;
323 }
324
325 /*
326 * CDNSP cannot write any doorbells or operational registers other
327 * than status until the "Controller Not Ready" flag is cleared.
328 */
329 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
330 !(temp & STS_CNR), 1,
331 10 * 1000);
332
333 if (ret) {
334 dev_err(pdev->dev, "ERROR: Controller not ready to work\n");
335 return ret;
336 }
337
338 dev_dbg(pdev->dev, "Controller ready to work");
339
340 return ret;
341 }
342
343 /*
344 * cdnsp_get_endpoint_index - Find the index for an endpoint given its
345 * descriptor.Use the return value to right shift 1 for the bitmask.
346 *
347 * Index = (epnum * 2) + direction - 1,
348 * where direction = 0 for OUT, 1 for IN.
349 * For control endpoints, the IN index is used (OUT index is unused), so
350 * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
351 */
352 static unsigned int
cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor * desc)353 cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc)
354 {
355 unsigned int index = (unsigned int)usb_endpoint_num(desc);
356
357 if (usb_endpoint_xfer_control(desc))
358 return index * 2;
359
360 return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
361 }
362
363 /*
364 * Find the flag for this endpoint (for use in the control context). Use the
365 * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is
366 * bit 1, etc.
367 */
368 static unsigned int
cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor * desc)369 cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc)
370 {
371 return 1 << (cdnsp_get_endpoint_index(desc) + 1);
372 }
373
cdnsp_ep_enqueue(struct cdnsp_ep * pep,struct cdnsp_request * preq)374 int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
375 {
376 struct cdnsp_device *pdev = pep->pdev;
377 struct usb_request *request;
378 int ret;
379
380 if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {
381 trace_cdnsp_request_enqueue_busy(preq);
382 return -EBUSY;
383 }
384
385 request = &preq->request;
386 request->actual = 0;
387 request->status = -EINPROGRESS;
388 preq->direction = pep->direction;
389 preq->epnum = pep->number;
390 preq->td.drbl = 0;
391
392 ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);
393 if (ret) {
394 trace_cdnsp_request_enqueue_error(preq);
395 return ret;
396 }
397
398 list_add_tail(&preq->list, &pep->pending_list);
399
400 trace_cdnsp_request_enqueue(preq);
401
402 switch (usb_endpoint_type(pep->endpoint.desc)) {
403 case USB_ENDPOINT_XFER_CONTROL:
404 ret = cdnsp_queue_ctrl_tx(pdev, preq);
405 break;
406 case USB_ENDPOINT_XFER_BULK:
407 case USB_ENDPOINT_XFER_INT:
408 ret = cdnsp_queue_bulk_tx(pdev, preq);
409 break;
410 case USB_ENDPOINT_XFER_ISOC:
411 ret = cdnsp_queue_isoc_tx(pdev, preq);
412 }
413
414 if (ret)
415 goto unmap;
416
417 return 0;
418
419 unmap:
420 usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
421 pep->direction);
422 list_del(&preq->list);
423 trace_cdnsp_request_enqueue_error(preq);
424
425 return ret;
426 }
427
428 /*
429 * Remove the request's TD from the endpoint ring. This may cause the
430 * controller to stop USB transfers, potentially stopping in the middle of a
431 * TRB buffer. The controller should pick up where it left off in the TD,
432 * unless a Set Transfer Ring Dequeue Pointer is issued.
433 *
434 * The TRBs that make up the buffers for the canceled request will be "removed"
435 * from the ring. Since the ring is a contiguous structure, they can't be
436 * physically removed. Instead, there are two options:
437 *
438 * 1) If the controller is in the middle of processing the request to be
439 * canceled, we simply move the ring's dequeue pointer past those TRBs
440 * using the Set Transfer Ring Dequeue Pointer command. This will be
441 * the common case, when drivers timeout on the last submitted request
442 * and attempt to cancel.
443 *
444 * 2) If the controller is in the middle of a different TD, we turn the TRBs
445 * into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained.
446 * The controller will need to invalidate the any TRBs it has cached after
447 * the stop endpoint command.
448 *
449 * 3) The TD may have completed by the time the Stop Endpoint Command
450 * completes, so software needs to handle that case too.
451 *
452 */
cdnsp_ep_dequeue(struct cdnsp_ep * pep,struct cdnsp_request * preq)453 int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
454 {
455 struct cdnsp_device *pdev = pep->pdev;
456 int ret_stop = 0;
457 int ret_rem;
458
459 trace_cdnsp_request_dequeue(preq);
460
461 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)
462 ret_stop = cdnsp_cmd_stop_ep(pdev, pep);
463
464 ret_rem = cdnsp_remove_request(pdev, preq, pep);
465
466 return ret_rem ? ret_rem : ret_stop;
467 }
468
cdnsp_zero_in_ctx(struct cdnsp_device * pdev)469 static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev)
470 {
471 struct cdnsp_input_control_ctx *ctrl_ctx;
472 struct cdnsp_slot_ctx *slot_ctx;
473 struct cdnsp_ep_ctx *ep_ctx;
474 int i;
475
476 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
477
478 /*
479 * When a device's add flag and drop flag are zero, any subsequent
480 * configure endpoint command will leave that endpoint's state
481 * untouched. Make sure we don't leave any old state in the input
482 * endpoint contexts.
483 */
484 ctrl_ctx->drop_flags = 0;
485 ctrl_ctx->add_flags = 0;
486 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
487 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
488
489 /* Endpoint 0 is always valid */
490 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
491 for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) {
492 ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i);
493 ep_ctx->ep_info = 0;
494 ep_ctx->ep_info2 = 0;
495 ep_ctx->deq = 0;
496 ep_ctx->tx_info = 0;
497 }
498 }
499
500 /* Issue a configure endpoint command and wait for it to finish. */
cdnsp_configure_endpoint(struct cdnsp_device * pdev)501 static int cdnsp_configure_endpoint(struct cdnsp_device *pdev)
502 {
503 int ret;
504
505 cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma);
506 cdnsp_ring_cmd_db(pdev);
507 ret = cdnsp_wait_for_cmd_compl(pdev);
508 if (ret) {
509 dev_err(pdev->dev,
510 "ERR: unexpected command completion code 0x%x.\n", ret);
511 return -EINVAL;
512 }
513
514 return ret;
515 }
516
cdnsp_invalidate_ep_events(struct cdnsp_device * pdev,struct cdnsp_ep * pep)517 static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev,
518 struct cdnsp_ep *pep)
519 {
520 struct cdnsp_segment *segment;
521 union cdnsp_trb *event;
522 u32 cycle_state;
523 u32 data;
524
525 event = pdev->event_ring->dequeue;
526 segment = pdev->event_ring->deq_seg;
527 cycle_state = pdev->event_ring->cycle_state;
528
529 while (1) {
530 data = le32_to_cpu(event->trans_event.flags);
531
532 /* Check the owner of the TRB. */
533 if ((data & TRB_CYCLE) != cycle_state)
534 break;
535
536 if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER &&
537 TRB_TO_EP_ID(data) == (pep->idx + 1)) {
538 data |= TRB_EVENT_INVALIDATE;
539 event->trans_event.flags = cpu_to_le32(data);
540 }
541
542 if (cdnsp_last_trb_on_seg(segment, event)) {
543 cycle_state ^= 1;
544 segment = pdev->event_ring->deq_seg->next;
545 event = segment->trbs;
546 } else {
547 event++;
548 }
549 }
550 }
551
cdnsp_wait_for_cmd_compl(struct cdnsp_device * pdev)552 int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev)
553 {
554 struct cdnsp_segment *event_deq_seg;
555 union cdnsp_trb *cmd_trb;
556 dma_addr_t cmd_deq_dma;
557 union cdnsp_trb *event;
558 u32 cycle_state;
559 u32 retry = 10;
560 int ret, val;
561 u64 cmd_dma;
562 u32 flags;
563
564 cmd_trb = pdev->cmd.command_trb;
565 pdev->cmd.status = 0;
566
567 trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic);
568
569 ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val,
570 !CMD_RING_BUSY(val), 1,
571 CDNSP_CMD_TIMEOUT);
572 if (ret) {
573 dev_err(pdev->dev, "ERR: Timeout while waiting for command\n");
574 trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic);
575 pdev->cdnsp_state = CDNSP_STATE_DYING;
576 return -ETIMEDOUT;
577 }
578
579 event = pdev->event_ring->dequeue;
580 event_deq_seg = pdev->event_ring->deq_seg;
581 cycle_state = pdev->event_ring->cycle_state;
582
583 cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb);
584 if (!cmd_deq_dma)
585 return -EINVAL;
586
587 while (1) {
588 flags = le32_to_cpu(event->event_cmd.flags);
589
590 /* Check the owner of the TRB. */
591 if ((flags & TRB_CYCLE) != cycle_state) {
592 /*
593 * Give some extra time to get chance controller
594 * to finish command before returning error code.
595 * Checking CMD_RING_BUSY is not sufficient because
596 * this bit is cleared to '0' when the Command
597 * Descriptor has been executed by controller
598 * and not when command completion event has
599 * be added to event ring.
600 */
601 if (retry--) {
602 udelay(20);
603 continue;
604 }
605
606 return -EINVAL;
607 }
608
609 cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb);
610
611 /*
612 * Check whether the completion event is for last queued
613 * command.
614 */
615 if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION ||
616 cmd_dma != (u64)cmd_deq_dma) {
617 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
618 event++;
619 continue;
620 }
621
622 if (cdnsp_last_trb_on_ring(pdev->event_ring,
623 event_deq_seg, event))
624 cycle_state ^= 1;
625
626 event_deq_seg = event_deq_seg->next;
627 event = event_deq_seg->trbs;
628 continue;
629 }
630
631 trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic);
632
633 pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status));
634 if (pdev->cmd.status == COMP_SUCCESS)
635 return 0;
636
637 return -pdev->cmd.status;
638 }
639 }
640
cdnsp_halt_endpoint(struct cdnsp_device * pdev,struct cdnsp_ep * pep,int value)641 int cdnsp_halt_endpoint(struct cdnsp_device *pdev,
642 struct cdnsp_ep *pep,
643 int value)
644 {
645 int ret;
646
647 trace_cdnsp_ep_halt(value ? "Set" : "Clear");
648
649 ret = cdnsp_cmd_stop_ep(pdev, pep);
650 if (ret)
651 return ret;
652
653 if (value) {
654 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {
655 cdnsp_queue_halt_endpoint(pdev, pep->idx);
656 cdnsp_ring_cmd_db(pdev);
657 ret = cdnsp_wait_for_cmd_compl(pdev);
658 }
659
660 pep->ep_state |= EP_HALTED;
661 } else {
662 cdnsp_queue_reset_ep(pdev, pep->idx);
663 cdnsp_ring_cmd_db(pdev);
664 ret = cdnsp_wait_for_cmd_compl(pdev);
665 trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);
666
667 if (ret)
668 return ret;
669
670 pep->ep_state &= ~EP_HALTED;
671
672 if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))
673 cdnsp_ring_doorbell_for_active_rings(pdev, pep);
674
675 pep->ep_state &= ~EP_WEDGE;
676 }
677
678 return 0;
679 }
680
cdnsp_update_eps_configuration(struct cdnsp_device * pdev,struct cdnsp_ep * pep)681 static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev,
682 struct cdnsp_ep *pep)
683 {
684 struct cdnsp_input_control_ctx *ctrl_ctx;
685 struct cdnsp_slot_ctx *slot_ctx;
686 int ret = 0;
687 u32 ep_sts;
688 int i;
689
690 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
691
692 /* Don't issue the command if there's no endpoints to update. */
693 if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0)
694 return 0;
695
696 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
697 ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
698 ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
699
700 /* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
701 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
702 for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) {
703 __le32 le32 = cpu_to_le32(BIT(i));
704
705 if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) ||
706 (ctrl_ctx->add_flags & le32) || i == 1) {
707 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
708 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
709 break;
710 }
711 }
712
713 ep_sts = GET_EP_CTX_STATE(pep->out_ctx);
714
715 if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) &&
716 ep_sts == EP_STATE_DISABLED) ||
717 (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags))
718 ret = cdnsp_configure_endpoint(pdev);
719
720 trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx));
721 trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);
722
723 cdnsp_zero_in_ctx(pdev);
724
725 return ret;
726 }
727
728 /*
729 * This submits a Reset Device Command, which will set the device state to 0,
730 * set the device address to 0, and disable all the endpoints except the default
731 * control endpoint. The USB core should come back and call
732 * cdnsp_setup_device(), and then re-set up the configuration.
733 */
cdnsp_reset_device(struct cdnsp_device * pdev)734 int cdnsp_reset_device(struct cdnsp_device *pdev)
735 {
736 struct cdnsp_slot_ctx *slot_ctx;
737 int slot_state;
738 int ret, i;
739
740 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
741 slot_ctx->dev_info = 0;
742 pdev->device_address = 0;
743
744 /* If device is not setup, there is no point in resetting it. */
745 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
746 slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
747 trace_cdnsp_reset_device(slot_ctx);
748
749 if (slot_state <= SLOT_STATE_DEFAULT &&
750 pdev->eps[0].ep_state & EP_HALTED) {
751 cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0);
752 }
753
754 /*
755 * During Reset Device command controller shall transition the
756 * endpoint ep0 to the Running State.
757 */
758 pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED);
759 pdev->eps[0].ep_state |= EP_ENABLED;
760
761 if (slot_state <= SLOT_STATE_DEFAULT)
762 return 0;
763
764 cdnsp_queue_reset_device(pdev);
765 cdnsp_ring_cmd_db(pdev);
766 ret = cdnsp_wait_for_cmd_compl(pdev);
767
768 /*
769 * After Reset Device command all not default endpoints
770 * are in Disabled state.
771 */
772 for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i)
773 pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED;
774
775 trace_cdnsp_handle_cmd_reset_dev(slot_ctx);
776
777 if (ret)
778 dev_err(pdev->dev, "Reset device failed with error code %d",
779 ret);
780
781 return ret;
782 }
783
784 /*
785 * Sets the MaxPStreams field and the Linear Stream Array field.
786 * Sets the dequeue pointer to the stream context array.
787 */
cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device * pdev,struct cdnsp_ep_ctx * ep_ctx,struct cdnsp_stream_info * stream_info)788 static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev,
789 struct cdnsp_ep_ctx *ep_ctx,
790 struct cdnsp_stream_info *stream_info)
791 {
792 u32 max_primary_streams;
793
794 /* MaxPStreams is the number of stream context array entries, not the
795 * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
796 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
797 */
798 max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
799 ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
800 ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
801 | EP_HAS_LSA);
802 ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma);
803 }
804
805 /*
806 * The drivers use this function to prepare a bulk endpoints to use streams.
807 *
808 * Don't allow the call to succeed if endpoint only supports one stream
809 * (which means it doesn't support streams at all).
810 */
cdnsp_alloc_streams(struct cdnsp_device * pdev,struct cdnsp_ep * pep)811 int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
812 {
813 unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);
814 unsigned int num_stream_ctxs;
815 int ret;
816
817 if (num_streams == 0)
818 return 0;
819
820 if (num_streams > STREAM_NUM_STREAMS)
821 return -EINVAL;
822
823 /*
824 * Add two to the number of streams requested to account for
825 * stream 0 that is reserved for controller usage and one additional
826 * for TASK SET FULL response.
827 */
828 num_streams += 2;
829
830 /* The stream context array size must be a power of two */
831 num_stream_ctxs = roundup_pow_of_two(num_streams);
832
833 trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);
834
835 ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);
836 if (ret)
837 return ret;
838
839 cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);
840
841 pep->ep_state |= EP_HAS_STREAMS;
842 pep->stream_info.td_count = 0;
843 pep->stream_info.first_prime_det = 0;
844
845 /* Subtract 1 for stream 0, which drivers can't use. */
846 return num_streams - 1;
847 }
848
cdnsp_disable_slot(struct cdnsp_device * pdev)849 int cdnsp_disable_slot(struct cdnsp_device *pdev)
850 {
851 int ret;
852
853 cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT);
854 cdnsp_ring_cmd_db(pdev);
855 ret = cdnsp_wait_for_cmd_compl(pdev);
856
857 pdev->slot_id = 0;
858 pdev->active_port = NULL;
859
860 trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
861
862 memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE);
863 memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE);
864
865 return ret;
866 }
867
cdnsp_enable_slot(struct cdnsp_device * pdev)868 int cdnsp_enable_slot(struct cdnsp_device *pdev)
869 {
870 struct cdnsp_slot_ctx *slot_ctx;
871 int slot_state;
872 int ret;
873
874 /* If device is not setup, there is no point in resetting it */
875 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
876 slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
877
878 if (slot_state != SLOT_STATE_DISABLED)
879 return 0;
880
881 cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT);
882 cdnsp_ring_cmd_db(pdev);
883 ret = cdnsp_wait_for_cmd_compl(pdev);
884 if (ret)
885 goto show_trace;
886
887 pdev->slot_id = 1;
888
889 show_trace:
890 trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
891
892 return ret;
893 }
894
895 /*
896 * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY
897 * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS.
898 */
cdnsp_setup_device(struct cdnsp_device * pdev,enum cdnsp_setup_dev setup)899 int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup)
900 {
901 struct cdnsp_input_control_ctx *ctrl_ctx;
902 struct cdnsp_slot_ctx *slot_ctx;
903 int dev_state = 0;
904 int ret;
905
906 if (!pdev->slot_id) {
907 trace_cdnsp_slot_id("incorrect");
908 return -EINVAL;
909 }
910
911 if (!pdev->active_port->port_num)
912 return -EINVAL;
913
914 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
915 dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
916
917 if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) {
918 trace_cdnsp_slot_already_in_default(slot_ctx);
919 return 0;
920 }
921
922 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
923 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
924
925 if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) {
926 ret = cdnsp_setup_addressable_priv_dev(pdev);
927 if (ret)
928 return ret;
929 }
930
931 cdnsp_copy_ep0_dequeue_into_input_ctx(pdev);
932
933 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
934 ctrl_ctx->drop_flags = 0;
935
936 trace_cdnsp_setup_device_slot(slot_ctx);
937
938 cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup);
939 cdnsp_ring_cmd_db(pdev);
940 ret = cdnsp_wait_for_cmd_compl(pdev);
941
942 trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx));
943
944 /* Zero the input context control for later use. */
945 ctrl_ctx->add_flags = 0;
946 ctrl_ctx->drop_flags = 0;
947
948 return ret;
949 }
950
cdnsp_set_usb2_hardware_lpm(struct cdnsp_device * pdev,struct usb_request * req,int enable)951 void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
952 struct usb_request *req,
953 int enable)
954 {
955 if (pdev->active_port == &pdev->usb3_port || !pdev->gadget.lpm_capable)
956 return;
957
958 trace_cdnsp_lpm(enable);
959
960 if (enable)
961 writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE,
962 &pdev->active_port->regs->portpmsc);
963 else
964 writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc);
965 }
966
cdnsp_get_frame(struct cdnsp_device * pdev)967 static int cdnsp_get_frame(struct cdnsp_device *pdev)
968 {
969 return readl(&pdev->run_regs->microframe_index) >> 3;
970 }
971
cdnsp_gadget_ep_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)972 static int cdnsp_gadget_ep_enable(struct usb_ep *ep,
973 const struct usb_endpoint_descriptor *desc)
974 {
975 struct cdnsp_input_control_ctx *ctrl_ctx;
976 struct cdnsp_device *pdev;
977 struct cdnsp_ep *pep;
978 unsigned long flags;
979 u32 added_ctxs;
980 int ret;
981
982 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
983 !desc->wMaxPacketSize)
984 return -EINVAL;
985
986 pep = to_cdnsp_ep(ep);
987 pdev = pep->pdev;
988 pep->ep_state &= ~EP_UNCONFIGURED;
989
990 if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
991 "%s is already enabled\n", pep->name))
992 return 0;
993
994 spin_lock_irqsave(&pdev->lock, flags);
995
996 added_ctxs = cdnsp_get_endpoint_flag(desc);
997 if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
998 dev_err(pdev->dev, "ERROR: Bad endpoint number\n");
999 ret = -EINVAL;
1000 goto unlock;
1001 }
1002
1003 pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1004
1005 if (pdev->gadget.speed == USB_SPEED_FULL) {
1006 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT)
1007 pep->interval = desc->bInterval << 3;
1008 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
1009 pep->interval = BIT(desc->bInterval - 1) << 3;
1010 }
1011
1012 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) {
1013 if (pep->interval > BIT(12)) {
1014 dev_err(pdev->dev, "bInterval %d not supported\n",
1015 desc->bInterval);
1016 ret = -EINVAL;
1017 goto unlock;
1018 }
1019 cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1020 }
1021
1022 ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);
1023 if (ret)
1024 goto unlock;
1025
1026 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1027 ctrl_ctx->add_flags = cpu_to_le32(added_ctxs);
1028 ctrl_ctx->drop_flags = 0;
1029
1030 ret = cdnsp_update_eps_configuration(pdev, pep);
1031 if (ret) {
1032 cdnsp_free_endpoint_rings(pdev, pep);
1033 goto unlock;
1034 }
1035
1036 pep->ep_state |= EP_ENABLED;
1037 pep->ep_state &= ~EP_STOPPED;
1038
1039 unlock:
1040 trace_cdnsp_ep_enable_end(pep, 0);
1041 spin_unlock_irqrestore(&pdev->lock, flags);
1042
1043 return ret;
1044 }
1045
cdnsp_gadget_ep_disable(struct usb_ep * ep)1046 static int cdnsp_gadget_ep_disable(struct usb_ep *ep)
1047 {
1048 struct cdnsp_input_control_ctx *ctrl_ctx;
1049 struct cdnsp_request *preq;
1050 struct cdnsp_device *pdev;
1051 struct cdnsp_ep *pep;
1052 unsigned long flags;
1053 u32 drop_flag;
1054 int ret = 0;
1055
1056 if (!ep)
1057 return -EINVAL;
1058
1059 pep = to_cdnsp_ep(ep);
1060 pdev = pep->pdev;
1061
1062 spin_lock_irqsave(&pdev->lock, flags);
1063
1064 if (!(pep->ep_state & EP_ENABLED)) {
1065 dev_err(pdev->dev, "%s is already disabled\n", pep->name);
1066 ret = -EINVAL;
1067 goto finish;
1068 }
1069
1070 pep->ep_state |= EP_DIS_IN_RROGRESS;
1071
1072 /* Endpoint was unconfigured by Reset Device command. */
1073 if (!(pep->ep_state & EP_UNCONFIGURED))
1074 cdnsp_cmd_stop_ep(pdev, pep);
1075
1076 /* Remove all queued USB requests. */
1077 while (!list_empty(&pep->pending_list)) {
1078 preq = next_request(&pep->pending_list);
1079 cdnsp_ep_dequeue(pep, preq);
1080 }
1081
1082 cdnsp_invalidate_ep_events(pdev, pep);
1083
1084 pep->ep_state &= ~EP_DIS_IN_RROGRESS;
1085 drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);
1086 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1087 ctrl_ctx->drop_flags = cpu_to_le32(drop_flag);
1088 ctrl_ctx->add_flags = 0;
1089
1090 cdnsp_endpoint_zero(pdev, pep);
1091
1092 if (!(pep->ep_state & EP_UNCONFIGURED))
1093 ret = cdnsp_update_eps_configuration(pdev, pep);
1094
1095 cdnsp_free_endpoint_rings(pdev, pep);
1096
1097 pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);
1098 pep->ep_state |= EP_STOPPED;
1099
1100 finish:
1101 trace_cdnsp_ep_disable_end(pep, 0);
1102 spin_unlock_irqrestore(&pdev->lock, flags);
1103
1104 return ret;
1105 }
1106
cdnsp_gadget_ep_alloc_request(struct usb_ep * ep,gfp_t gfp_flags)1107 static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep,
1108 gfp_t gfp_flags)
1109 {
1110 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1111 struct cdnsp_request *preq;
1112
1113 preq = kzalloc_obj(*preq, gfp_flags);
1114 if (!preq)
1115 return NULL;
1116
1117 preq->epnum = pep->number;
1118 preq->pep = pep;
1119
1120 trace_cdnsp_alloc_request(preq);
1121
1122 return &preq->request;
1123 }
1124
cdnsp_gadget_ep_free_request(struct usb_ep * ep,struct usb_request * request)1125 static void cdnsp_gadget_ep_free_request(struct usb_ep *ep,
1126 struct usb_request *request)
1127 {
1128 struct cdnsp_request *preq = to_cdnsp_request(request);
1129
1130 trace_cdnsp_free_request(preq);
1131 kfree(preq);
1132 }
1133
cdnsp_gadget_ep_queue(struct usb_ep * ep,struct usb_request * request,gfp_t gfp_flags)1134 static int cdnsp_gadget_ep_queue(struct usb_ep *ep,
1135 struct usb_request *request,
1136 gfp_t gfp_flags)
1137 {
1138 struct cdnsp_request *preq;
1139 struct cdnsp_device *pdev;
1140 struct cdnsp_ep *pep;
1141 unsigned long flags;
1142 int ret;
1143
1144 if (!request || !ep)
1145 return -EINVAL;
1146
1147 pep = to_cdnsp_ep(ep);
1148 pdev = pep->pdev;
1149
1150 if (!(pep->ep_state & EP_ENABLED)) {
1151 dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1152 pep->name);
1153 return -EINVAL;
1154 }
1155
1156 preq = to_cdnsp_request(request);
1157 spin_lock_irqsave(&pdev->lock, flags);
1158 ret = cdnsp_ep_enqueue(pep, preq);
1159 spin_unlock_irqrestore(&pdev->lock, flags);
1160
1161 return ret;
1162 }
1163
cdnsp_gadget_ep_dequeue(struct usb_ep * ep,struct usb_request * request)1164 static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep,
1165 struct usb_request *request)
1166 {
1167 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1168 struct cdnsp_device *pdev = pep->pdev;
1169 unsigned long flags;
1170 int ret;
1171
1172 if (request->status != -EINPROGRESS)
1173 return 0;
1174
1175 if (!pep->endpoint.desc) {
1176 dev_err(pdev->dev,
1177 "%s: can't dequeue to disabled endpoint\n",
1178 pep->name);
1179 return -ESHUTDOWN;
1180 }
1181
1182 /* Requests has been dequeued during disabling endpoint. */
1183 if (!(pep->ep_state & EP_ENABLED))
1184 return 0;
1185
1186 spin_lock_irqsave(&pdev->lock, flags);
1187 ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));
1188 spin_unlock_irqrestore(&pdev->lock, flags);
1189
1190 return ret;
1191 }
1192
cdnsp_gadget_ep_set_halt(struct usb_ep * ep,int value)1193 static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value)
1194 {
1195 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1196 struct cdnsp_device *pdev = pep->pdev;
1197 struct cdnsp_request *preq;
1198 unsigned long flags;
1199 int ret;
1200
1201 spin_lock_irqsave(&pdev->lock, flags);
1202
1203 preq = next_request(&pep->pending_list);
1204 if (value) {
1205 if (preq) {
1206 trace_cdnsp_ep_busy_try_halt_again(pep, 0);
1207 ret = -EAGAIN;
1208 goto done;
1209 }
1210 }
1211
1212 ret = cdnsp_halt_endpoint(pdev, pep, value);
1213
1214 done:
1215 spin_unlock_irqrestore(&pdev->lock, flags);
1216 return ret;
1217 }
1218
cdnsp_gadget_ep_set_wedge(struct usb_ep * ep)1219 static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep)
1220 {
1221 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1222 struct cdnsp_device *pdev = pep->pdev;
1223 unsigned long flags;
1224 int ret;
1225
1226 spin_lock_irqsave(&pdev->lock, flags);
1227 pep->ep_state |= EP_WEDGE;
1228 ret = cdnsp_halt_endpoint(pdev, pep, 1);
1229 spin_unlock_irqrestore(&pdev->lock, flags);
1230
1231 return ret;
1232 }
1233
1234 static const struct usb_ep_ops cdnsp_gadget_ep0_ops = {
1235 .enable = cdnsp_gadget_ep_enable,
1236 .disable = cdnsp_gadget_ep_disable,
1237 .alloc_request = cdnsp_gadget_ep_alloc_request,
1238 .free_request = cdnsp_gadget_ep_free_request,
1239 .queue = cdnsp_gadget_ep_queue,
1240 .dequeue = cdnsp_gadget_ep_dequeue,
1241 .set_halt = cdnsp_gadget_ep_set_halt,
1242 .set_wedge = cdnsp_gadget_ep_set_wedge,
1243 };
1244
1245 static const struct usb_ep_ops cdnsp_gadget_ep_ops = {
1246 .enable = cdnsp_gadget_ep_enable,
1247 .disable = cdnsp_gadget_ep_disable,
1248 .alloc_request = cdnsp_gadget_ep_alloc_request,
1249 .free_request = cdnsp_gadget_ep_free_request,
1250 .queue = cdnsp_gadget_ep_queue,
1251 .dequeue = cdnsp_gadget_ep_dequeue,
1252 .set_halt = cdnsp_gadget_ep_set_halt,
1253 .set_wedge = cdnsp_gadget_ep_set_wedge,
1254 };
1255
cdnsp_gadget_giveback(struct cdnsp_ep * pep,struct cdnsp_request * preq,int status)1256 void cdnsp_gadget_giveback(struct cdnsp_ep *pep,
1257 struct cdnsp_request *preq,
1258 int status)
1259 {
1260 struct cdnsp_device *pdev = pep->pdev;
1261
1262 list_del(&preq->list);
1263
1264 if (preq->request.status == -EINPROGRESS)
1265 preq->request.status = status;
1266
1267 usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
1268 preq->direction);
1269
1270 trace_cdnsp_request_giveback(preq);
1271
1272 if (preq != &pdev->ep0_preq) {
1273 spin_unlock(&pdev->lock);
1274 usb_gadget_giveback_request(&pep->endpoint, &preq->request);
1275 spin_lock(&pdev->lock);
1276 }
1277 }
1278
1279 static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = {
1280 .bLength = USB_DT_ENDPOINT_SIZE,
1281 .bDescriptorType = USB_DT_ENDPOINT,
1282 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1283 };
1284
cdnsp_run(struct cdnsp_device * pdev,enum usb_device_speed speed)1285 static int cdnsp_run(struct cdnsp_device *pdev,
1286 enum usb_device_speed speed)
1287 {
1288 u32 fs_speed = 0;
1289 u32 temp;
1290 int ret;
1291
1292 temp = readl(&pdev->ir_set->irq_control);
1293 temp &= ~IMOD_INTERVAL_MASK;
1294 temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK);
1295 writel(temp, &pdev->ir_set->irq_control);
1296
1297 temp = readl(&pdev->port3x_regs->mode_addr);
1298
1299 switch (speed) {
1300 case USB_SPEED_SUPER_PLUS:
1301 temp |= CFG_3XPORT_SSP_SUPPORT;
1302 break;
1303 case USB_SPEED_SUPER:
1304 temp &= ~CFG_3XPORT_SSP_SUPPORT;
1305 break;
1306 case USB_SPEED_HIGH:
1307 break;
1308 case USB_SPEED_FULL:
1309 fs_speed = PORT_REG6_FORCE_FS;
1310 break;
1311 default:
1312 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
1313 speed);
1314 fallthrough;
1315 case USB_SPEED_UNKNOWN:
1316 /* Default to superspeed. */
1317 speed = USB_SPEED_SUPER;
1318 break;
1319 }
1320
1321 if (pdev->usb3_port.exist && speed >= USB_SPEED_SUPER) {
1322 writel(temp, &pdev->port3x_regs->mode_addr);
1323 cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
1324 XDEV_RXDETECT);
1325 } else {
1326 cdnsp_disable_port(pdev, &pdev->usb3_port);
1327 }
1328
1329 if (pdev->usb2_port.exist) {
1330 cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
1331 XDEV_RXDETECT);
1332 writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
1333 }
1334
1335 if (pdev->eusb_port.exist)
1336 cdnsp_set_link_state(pdev, &pdev->eusb_port.regs->portsc,
1337 XDEV_RXDETECT);
1338
1339 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1340
1341
1342 ret = cdnsp_start(pdev);
1343 if (ret) {
1344 ret = -ENODEV;
1345 goto err;
1346 }
1347
1348 temp = readl(&pdev->op_regs->command);
1349 temp |= (CMD_INTE);
1350 writel(temp, &pdev->op_regs->command);
1351
1352 temp = readl(&pdev->ir_set->irq_pending);
1353 writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending);
1354
1355 trace_cdnsp_init("Controller ready to work");
1356 return 0;
1357 err:
1358 cdnsp_halt(pdev);
1359 return ret;
1360 }
1361
cdnsp_gadget_udc_start(struct usb_gadget * g,struct usb_gadget_driver * driver)1362 static int cdnsp_gadget_udc_start(struct usb_gadget *g,
1363 struct usb_gadget_driver *driver)
1364 {
1365 enum usb_device_speed max_speed = driver->max_speed;
1366 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1367 unsigned long flags;
1368 int ret;
1369
1370 spin_lock_irqsave(&pdev->lock, flags);
1371 pdev->gadget_driver = driver;
1372
1373 /* limit speed if necessary */
1374 max_speed = min(driver->max_speed, g->max_speed);
1375 ret = cdnsp_run(pdev, max_speed);
1376
1377 spin_unlock_irqrestore(&pdev->lock, flags);
1378
1379 return ret;
1380 }
1381
1382 /*
1383 * Update Event Ring Dequeue Pointer:
1384 * - When all events have finished
1385 * - To avoid "Event Ring Full Error" condition
1386 */
cdnsp_update_erst_dequeue(struct cdnsp_device * pdev,union cdnsp_trb * event_ring_deq,u8 clear_ehb)1387 void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev,
1388 union cdnsp_trb *event_ring_deq,
1389 u8 clear_ehb)
1390 {
1391 u64 temp_64;
1392 dma_addr_t deq;
1393
1394 temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1395
1396 /* If necessary, update the HW's version of the event ring deq ptr. */
1397 if (event_ring_deq != pdev->event_ring->dequeue) {
1398 deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1399 pdev->event_ring->dequeue);
1400 temp_64 &= ERST_PTR_MASK;
1401 temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK);
1402 }
1403
1404 /* Clear the event handler busy flag (RW1C). */
1405 if (clear_ehb)
1406 temp_64 |= ERST_EHB;
1407 else
1408 temp_64 &= ~ERST_EHB;
1409
1410 cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue);
1411 }
1412
cdnsp_clear_cmd_ring(struct cdnsp_device * pdev)1413 static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev)
1414 {
1415 struct cdnsp_segment *seg;
1416 u64 val_64;
1417 int i;
1418
1419 cdnsp_initialize_ring_info(pdev->cmd_ring);
1420
1421 seg = pdev->cmd_ring->first_seg;
1422 for (i = 0; i < pdev->cmd_ring->num_segs; i++) {
1423 memset(seg->trbs, 0,
1424 sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1));
1425 seg = seg->next;
1426 }
1427
1428 /* Set the address in the Command Ring Control register. */
1429 val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1430 val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1431 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1432 pdev->cmd_ring->cycle_state;
1433 cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1434 }
1435
cdnsp_consume_all_events(struct cdnsp_device * pdev)1436 static void cdnsp_consume_all_events(struct cdnsp_device *pdev)
1437 {
1438 struct cdnsp_segment *event_deq_seg;
1439 union cdnsp_trb *event_ring_deq;
1440 union cdnsp_trb *event;
1441 u32 cycle_bit;
1442
1443 event_ring_deq = pdev->event_ring->dequeue;
1444 event_deq_seg = pdev->event_ring->deq_seg;
1445 event = pdev->event_ring->dequeue;
1446
1447 /* Update ring dequeue pointer. */
1448 while (1) {
1449 cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE);
1450
1451 /* Does the controller or driver own the TRB? */
1452 if (cycle_bit != pdev->event_ring->cycle_state)
1453 break;
1454
1455 cdnsp_inc_deq(pdev, pdev->event_ring);
1456
1457 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
1458 event++;
1459 continue;
1460 }
1461
1462 if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg,
1463 event))
1464 cycle_bit ^= 1;
1465
1466 event_deq_seg = event_deq_seg->next;
1467 event = event_deq_seg->trbs;
1468 }
1469
1470 cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1);
1471 }
1472
cdnsp_stop(struct cdnsp_device * pdev)1473 static void cdnsp_stop(struct cdnsp_device *pdev)
1474 {
1475 u32 temp;
1476
1477 /* Remove internally queued request for ep0. */
1478 if (!list_empty(&pdev->eps[0].pending_list)) {
1479 struct cdnsp_request *req;
1480
1481 req = next_request(&pdev->eps[0].pending_list);
1482 if (req == &pdev->ep0_preq)
1483 cdnsp_ep_dequeue(&pdev->eps[0], req);
1484 }
1485
1486 cdnsp_disable_port(pdev, &pdev->usb2_port);
1487 cdnsp_disable_port(pdev, &pdev->usb3_port);
1488 cdnsp_disable_port(pdev, &pdev->eusb_port);
1489
1490 cdnsp_disable_slot(pdev);
1491 cdnsp_halt(pdev);
1492
1493 temp = readl(&pdev->op_regs->status);
1494 writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status);
1495 temp = readl(&pdev->ir_set->irq_pending);
1496 writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
1497
1498 cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port);
1499 cdnsp_clear_port_change_bit(pdev, &pdev->eusb_port);
1500 cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port);
1501
1502 /* Clear interrupt line */
1503 temp = readl(&pdev->ir_set->irq_pending);
1504 temp |= IMAN_IP;
1505 writel(temp, &pdev->ir_set->irq_pending);
1506
1507 cdnsp_consume_all_events(pdev);
1508 cdnsp_clear_cmd_ring(pdev);
1509
1510 trace_cdnsp_exit("Controller stopped.");
1511 }
1512
1513 /*
1514 * Stop controller.
1515 * This function is called by the gadget core when the driver is removed.
1516 * Disable slot, disable IRQs, and quiesce the controller.
1517 */
cdnsp_gadget_udc_stop(struct usb_gadget * g)1518 static int cdnsp_gadget_udc_stop(struct usb_gadget *g)
1519 {
1520 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1521 unsigned long flags;
1522
1523 spin_lock_irqsave(&pdev->lock, flags);
1524 cdnsp_stop(pdev);
1525 pdev->gadget_driver = NULL;
1526 spin_unlock_irqrestore(&pdev->lock, flags);
1527
1528 return 0;
1529 }
1530
cdnsp_gadget_get_frame(struct usb_gadget * g)1531 static int cdnsp_gadget_get_frame(struct usb_gadget *g)
1532 {
1533 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1534
1535 return cdnsp_get_frame(pdev);
1536 }
1537
__cdnsp_gadget_wakeup(struct cdnsp_device * pdev)1538 static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev)
1539 {
1540 struct cdnsp_port_regs __iomem *port_regs;
1541 u32 portpm, portsc;
1542
1543 port_regs = pdev->active_port->regs;
1544 portsc = readl(&port_regs->portsc) & PORT_PLS_MASK;
1545
1546 /* Remote wakeup feature is not enabled by host. */
1547 if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) {
1548 portpm = readl(&port_regs->portpmsc);
1549
1550 if (!(portpm & PORT_RWE))
1551 return;
1552 }
1553
1554 if (portsc == XDEV_U3 && !pdev->may_wakeup)
1555 return;
1556
1557 cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0);
1558
1559 pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING;
1560 }
1561
cdnsp_gadget_wakeup(struct usb_gadget * g)1562 static int cdnsp_gadget_wakeup(struct usb_gadget *g)
1563 {
1564 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1565 unsigned long flags;
1566
1567 spin_lock_irqsave(&pdev->lock, flags);
1568 __cdnsp_gadget_wakeup(pdev);
1569 spin_unlock_irqrestore(&pdev->lock, flags);
1570
1571 return 0;
1572 }
1573
cdnsp_gadget_set_selfpowered(struct usb_gadget * g,int is_selfpowered)1574 static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g,
1575 int is_selfpowered)
1576 {
1577 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1578 unsigned long flags;
1579
1580 spin_lock_irqsave(&pdev->lock, flags);
1581 g->is_selfpowered = !!is_selfpowered;
1582 spin_unlock_irqrestore(&pdev->lock, flags);
1583
1584 return 0;
1585 }
1586
cdnsp_gadget_pullup(struct usb_gadget * gadget,int is_on)1587 static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
1588 {
1589 struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
1590 struct cdns *cdns = dev_get_drvdata(pdev->dev);
1591 unsigned long flags;
1592
1593 trace_cdnsp_pullup(is_on);
1594
1595 /*
1596 * Disable events handling while controller is being
1597 * enabled/disabled.
1598 */
1599 disable_irq(cdns->dev_irq);
1600 spin_lock_irqsave(&pdev->lock, flags);
1601
1602 if (!is_on) {
1603 cdnsp_reset_device(pdev);
1604 cdns_clear_vbus(cdns);
1605 } else {
1606 cdns_set_vbus(cdns);
1607 }
1608
1609 spin_unlock_irqrestore(&pdev->lock, flags);
1610 enable_irq(cdns->dev_irq);
1611
1612 return 0;
1613 }
1614
1615 static const struct usb_gadget_ops cdnsp_gadget_ops = {
1616 .get_frame = cdnsp_gadget_get_frame,
1617 .wakeup = cdnsp_gadget_wakeup,
1618 .set_selfpowered = cdnsp_gadget_set_selfpowered,
1619 .pullup = cdnsp_gadget_pullup,
1620 .udc_start = cdnsp_gadget_udc_start,
1621 .udc_stop = cdnsp_gadget_udc_stop,
1622 };
1623
cdnsp_get_ep_buffering(struct cdnsp_device * pdev,struct cdnsp_ep * pep)1624 static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev,
1625 struct cdnsp_ep *pep)
1626 {
1627 void __iomem *reg = &pdev->cap_regs->hc_capbase;
1628 int endpoints;
1629
1630 reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID);
1631
1632 if (!pep->direction) {
1633 pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);
1634 pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);
1635 pep->buffering = (pep->buffering + 1) / 2;
1636 pep->buffering_period = (pep->buffering_period + 1) / 2;
1637 return;
1638 }
1639
1640 endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2;
1641
1642 /* Set to XBUF_TX_TAG_MASK_0 register. */
1643 reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32);
1644 /* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */
1645 reg += pep->number * sizeof(u32) * 2;
1646
1647 pep->buffering = (readl(reg) + 1) / 2;
1648 pep->buffering_period = pep->buffering;
1649 }
1650
cdnsp_gadget_init_endpoints(struct cdnsp_device * pdev)1651 static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev)
1652 {
1653 int max_streams = HCC_MAX_PSA(pdev->hcc_params);
1654 struct cdnsp_ep *pep;
1655 int i;
1656
1657 INIT_LIST_HEAD(&pdev->gadget.ep_list);
1658
1659 if (max_streams < STREAM_LOG_STREAMS) {
1660 dev_err(pdev->dev, "Stream size %d not supported\n",
1661 max_streams);
1662 return -EINVAL;
1663 }
1664
1665 max_streams = STREAM_LOG_STREAMS;
1666
1667 for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1668 bool direction = !(i & 1); /* Start from OUT endpoint. */
1669 u8 epnum = ((i + 1) >> 1);
1670
1671 if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction))
1672 continue;
1673
1674 pep = &pdev->eps[i];
1675 pep->pdev = pdev;
1676 pep->number = epnum;
1677 pep->direction = direction; /* 0 for OUT, 1 for IN. */
1678
1679 /*
1680 * Ep0 is bidirectional, so ep0in and ep0out are represented by
1681 * pdev->eps[0]
1682 */
1683 if (epnum == 0) {
1684 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1685 epnum, "BiDir");
1686
1687 pep->idx = 0;
1688 usb_ep_set_maxpacket_limit(&pep->endpoint, 512);
1689 pep->endpoint.maxburst = 1;
1690 pep->endpoint.ops = &cdnsp_gadget_ep0_ops;
1691 pep->endpoint.desc = &cdnsp_gadget_ep0_desc;
1692 pep->endpoint.comp_desc = NULL;
1693 pep->endpoint.caps.type_control = true;
1694 pep->endpoint.caps.dir_in = true;
1695 pep->endpoint.caps.dir_out = true;
1696
1697 pdev->ep0_preq.epnum = pep->number;
1698 pdev->ep0_preq.pep = pep;
1699 pdev->gadget.ep0 = &pep->endpoint;
1700 } else {
1701 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1702 epnum, (pep->direction) ? "in" : "out");
1703
1704 pep->idx = (epnum * 2 + (direction ? 1 : 0)) - 1;
1705 usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
1706
1707 pep->endpoint.max_streams = max_streams;
1708 pep->endpoint.ops = &cdnsp_gadget_ep_ops;
1709 list_add_tail(&pep->endpoint.ep_list,
1710 &pdev->gadget.ep_list);
1711
1712 pep->endpoint.caps.type_iso = true;
1713 pep->endpoint.caps.type_bulk = true;
1714 pep->endpoint.caps.type_int = true;
1715
1716 pep->endpoint.caps.dir_in = direction;
1717 pep->endpoint.caps.dir_out = !direction;
1718 }
1719
1720 pep->endpoint.name = pep->name;
1721 pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);
1722 pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);
1723 cdnsp_get_ep_buffering(pdev, pep);
1724
1725 dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: "
1726 "CTRL: %s, INT: %s, BULK: %s, ISOC %s, "
1727 "SupDir IN: %s, OUT: %s\n",
1728 pep->name, 1024,
1729 str_yes_no(pep->endpoint.caps.type_control),
1730 str_yes_no(pep->endpoint.caps.type_int),
1731 str_yes_no(pep->endpoint.caps.type_bulk),
1732 str_yes_no(pep->endpoint.caps.type_iso),
1733 str_yes_no(pep->endpoint.caps.dir_in),
1734 str_yes_no(pep->endpoint.caps.dir_out));
1735
1736 INIT_LIST_HEAD(&pep->pending_list);
1737 }
1738
1739 return 0;
1740 }
1741
cdnsp_gadget_free_endpoints(struct cdnsp_device * pdev)1742 static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev)
1743 {
1744 struct cdnsp_ep *pep;
1745 int i;
1746
1747 for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1748 pep = &pdev->eps[i];
1749 if (pep->number != 0 && pep->out_ctx)
1750 list_del(&pep->endpoint.ep_list);
1751 }
1752 }
1753
cdnsp_disconnect_gadget(struct cdnsp_device * pdev)1754 void cdnsp_disconnect_gadget(struct cdnsp_device *pdev)
1755 {
1756 pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING;
1757
1758 if (pdev->gadget_driver && pdev->gadget_driver->disconnect) {
1759 spin_unlock(&pdev->lock);
1760 pdev->gadget_driver->disconnect(&pdev->gadget);
1761 spin_lock(&pdev->lock);
1762 }
1763
1764 pdev->gadget.speed = USB_SPEED_UNKNOWN;
1765 usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
1766
1767 pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING;
1768 }
1769
cdnsp_suspend_gadget(struct cdnsp_device * pdev)1770 void cdnsp_suspend_gadget(struct cdnsp_device *pdev)
1771 {
1772 if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1773 spin_unlock(&pdev->lock);
1774 pdev->gadget_driver->suspend(&pdev->gadget);
1775 spin_lock(&pdev->lock);
1776 }
1777 }
1778
cdnsp_resume_gadget(struct cdnsp_device * pdev)1779 void cdnsp_resume_gadget(struct cdnsp_device *pdev)
1780 {
1781 if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1782 spin_unlock(&pdev->lock);
1783 pdev->gadget_driver->resume(&pdev->gadget);
1784 spin_lock(&pdev->lock);
1785 }
1786 }
1787
cdnsp_irq_reset(struct cdnsp_device * pdev)1788 void cdnsp_irq_reset(struct cdnsp_device *pdev)
1789 {
1790 struct cdnsp_port_regs __iomem *port_regs;
1791
1792 cdnsp_reset_device(pdev);
1793
1794 port_regs = pdev->active_port->regs;
1795 pdev->gadget.speed = cdnsp_port_speed(readl(port_regs));
1796
1797 spin_unlock(&pdev->lock);
1798 usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver);
1799 spin_lock(&pdev->lock);
1800
1801 switch (pdev->gadget.speed) {
1802 case USB_SPEED_SUPER_PLUS:
1803 case USB_SPEED_SUPER:
1804 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1805 pdev->gadget.ep0->maxpacket = 512;
1806 break;
1807 case USB_SPEED_HIGH:
1808 case USB_SPEED_FULL:
1809 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1810 pdev->gadget.ep0->maxpacket = 64;
1811 break;
1812 default:
1813 /* Low speed is not supported. */
1814 dev_err(pdev->dev, "Unknown device speed\n");
1815 break;
1816 }
1817
1818 cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1819 cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY);
1820 usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT);
1821 }
1822
cdnsp_get_rev_cap(struct cdnsp_device * pdev)1823 static void cdnsp_get_rev_cap(struct cdnsp_device *pdev)
1824 {
1825 void __iomem *reg = &pdev->cap_regs->hc_capbase;
1826
1827 reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP);
1828 pdev->rev_cap = reg;
1829
1830 pdev->rtl_revision = readl(&pdev->rev_cap->rtl_revision);
1831
1832 dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n",
1833 readl(&pdev->rev_cap->ctrl_revision),
1834 readl(&pdev->rev_cap->rtl_revision),
1835 readl(&pdev->rev_cap->ep_supported),
1836 readl(&pdev->rev_cap->rx_buff_size),
1837 readl(&pdev->rev_cap->tx_buff_size));
1838 }
1839
cdnsp_gen_setup(struct cdnsp_device * pdev)1840 static int cdnsp_gen_setup(struct cdnsp_device *pdev)
1841 {
1842 int ret;
1843 u32 reg;
1844
1845 pdev->cap_regs = pdev->regs;
1846 pdev->op_regs = pdev->regs +
1847 HC_LENGTH(readl(&pdev->cap_regs->hc_capbase));
1848 pdev->run_regs = pdev->regs +
1849 (readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK);
1850
1851 /* Cache read-only capability registers */
1852 pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1);
1853 pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase);
1854 pdev->hci_version = HC_VERSION(pdev->hcc_params);
1855 pdev->hcc_params = readl(&pdev->cap_regs->hcc_params);
1856
1857 /*
1858 * Override the APB timeout value to give the controller more time for
1859 * enabling UTMI clock and synchronizing APB and UTMI clock domains.
1860 * This fix is platform specific and is required to fixes issue with
1861 * reading incorrect value from PORTSC register after resuming
1862 * from L1 state.
1863 */
1864 cdnsp_set_apb_timeout_value(pdev);
1865
1866 cdnsp_get_rev_cap(pdev);
1867
1868 /* Make sure the Device Controller is halted. */
1869 ret = cdnsp_halt(pdev);
1870 if (ret)
1871 return ret;
1872
1873 /* Reset the internal controller memory state and registers. */
1874 ret = cdnsp_reset(pdev);
1875 if (ret)
1876 return ret;
1877
1878 /*
1879 * Set dma_mask and coherent_dma_mask to 64-bits,
1880 * if controller supports 64-bit addressing.
1881 */
1882 if (HCC_64BIT_ADDR(pdev->hcc_params) &&
1883 !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) {
1884 dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n");
1885 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64));
1886 } else {
1887 /*
1888 * This is to avoid error in cases where a 32-bit USB
1889 * controller is used on a 64-bit capable system.
1890 */
1891 ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32));
1892 if (ret)
1893 return ret;
1894
1895 dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n");
1896 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32));
1897 }
1898
1899 spin_lock_init(&pdev->lock);
1900
1901 ret = cdnsp_mem_init(pdev);
1902 if (ret)
1903 return ret;
1904
1905 /*
1906 * Software workaround for U1: after transition
1907 * to U1 the controller starts gating clock, and in some cases,
1908 * it causes that controller stack.
1909 */
1910 reg = readl(&pdev->port3x_regs->mode_2);
1911 reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN;
1912 writel(reg, &pdev->port3x_regs->mode_2);
1913
1914 return 0;
1915 }
1916
__cdnsp_gadget_init(struct cdns * cdns)1917 static int __cdnsp_gadget_init(struct cdns *cdns)
1918 {
1919 struct cdnsp_device *pdev;
1920 u32 max_speed;
1921 int ret = -ENOMEM;
1922
1923 cdns_drd_gadget_on(cdns);
1924
1925 pdev = kzalloc_obj(*pdev);
1926 if (!pdev)
1927 return -ENOMEM;
1928
1929 pm_runtime_get_sync(cdns->dev);
1930
1931 cdns->gadget_dev = pdev;
1932 pdev->dev = cdns->dev;
1933 pdev->regs = cdns->dev_regs;
1934 max_speed = usb_get_maximum_speed(cdns->dev);
1935
1936 switch (max_speed) {
1937 case USB_SPEED_FULL:
1938 case USB_SPEED_HIGH:
1939 case USB_SPEED_SUPER:
1940 case USB_SPEED_SUPER_PLUS:
1941 break;
1942 default:
1943 dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed);
1944 fallthrough;
1945 case USB_SPEED_UNKNOWN:
1946 /* Default to SSP */
1947 max_speed = USB_SPEED_SUPER_PLUS;
1948 break;
1949 }
1950
1951 pdev->gadget.ops = &cdnsp_gadget_ops;
1952 pdev->gadget.name = "cdnsp-gadget";
1953 pdev->gadget.speed = USB_SPEED_UNKNOWN;
1954 pdev->gadget.sg_supported = 1;
1955 pdev->gadget.max_speed = max_speed;
1956 pdev->gadget.lpm_capable = 1;
1957
1958 pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL);
1959 if (!pdev->setup_buf)
1960 goto free_pdev;
1961
1962 /*
1963 * Controller supports not aligned buffer but it should improve
1964 * performance.
1965 */
1966 pdev->gadget.quirk_ep_out_aligned_size = true;
1967
1968 ret = cdnsp_gen_setup(pdev);
1969 if (ret) {
1970 dev_err(pdev->dev, "Generic initialization failed %d\n", ret);
1971 goto free_setup;
1972 }
1973
1974 ret = cdnsp_gadget_init_endpoints(pdev);
1975 if (ret) {
1976 dev_err(pdev->dev, "failed to initialize endpoints\n");
1977 goto halt_pdev;
1978 }
1979
1980 ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget);
1981 if (ret) {
1982 dev_err(pdev->dev, "failed to register udc\n");
1983 goto free_endpoints;
1984 }
1985
1986 ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq,
1987 cdnsp_irq_handler,
1988 cdnsp_thread_irq_handler, IRQF_SHARED,
1989 dev_name(pdev->dev), pdev);
1990 if (ret)
1991 goto del_gadget;
1992
1993 return 0;
1994
1995 del_gadget:
1996 usb_del_gadget(&pdev->gadget);
1997 cdnsp_gadget_free_endpoints(pdev);
1998 usb_put_gadget(&pdev->gadget);
1999 goto halt_pdev;
2000 free_endpoints:
2001 cdnsp_gadget_free_endpoints(pdev);
2002 halt_pdev:
2003 cdnsp_halt(pdev);
2004 cdnsp_reset(pdev);
2005 cdnsp_mem_cleanup(pdev);
2006 free_setup:
2007 kfree(pdev->setup_buf);
2008 free_pdev:
2009 kfree(pdev);
2010
2011 return ret;
2012 }
2013
cdnsp_gadget_exit(struct cdns * cdns)2014 static void cdnsp_gadget_exit(struct cdns *cdns)
2015 {
2016 struct cdnsp_device *pdev = cdns->gadget_dev;
2017
2018 devm_free_irq(pdev->dev, cdns->dev_irq, pdev);
2019 pm_runtime_put_autosuspend(cdns->dev);
2020 usb_del_gadget(&pdev->gadget);
2021 cdnsp_gadget_free_endpoints(pdev);
2022 usb_put_gadget(&pdev->gadget);
2023 cdnsp_mem_cleanup(pdev);
2024 kfree(pdev);
2025 cdns->gadget_dev = NULL;
2026 cdns_drd_gadget_off(cdns);
2027 }
2028
cdnsp_gadget_suspend(struct cdns * cdns,bool do_wakeup)2029 static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup)
2030 {
2031 struct cdnsp_device *pdev = cdns->gadget_dev;
2032 unsigned long flags;
2033
2034 if (pdev->link_state == XDEV_U3)
2035 return 0;
2036
2037 spin_lock_irqsave(&pdev->lock, flags);
2038 cdnsp_disconnect_gadget(pdev);
2039 cdnsp_stop(pdev);
2040 spin_unlock_irqrestore(&pdev->lock, flags);
2041
2042 return 0;
2043 }
2044
cdnsp_gadget_resume(struct cdns * cdns,bool lost_power)2045 static int cdnsp_gadget_resume(struct cdns *cdns, bool lost_power)
2046 {
2047 struct cdnsp_device *pdev = cdns->gadget_dev;
2048 enum usb_device_speed max_speed;
2049 unsigned long flags;
2050 int ret;
2051
2052 if (!pdev->gadget_driver)
2053 return 0;
2054
2055 spin_lock_irqsave(&pdev->lock, flags);
2056 max_speed = pdev->gadget_driver->max_speed;
2057
2058 /* Limit speed if necessary. */
2059 max_speed = min(max_speed, pdev->gadget.max_speed);
2060
2061 ret = cdnsp_run(pdev, max_speed);
2062
2063 if (pdev->link_state == XDEV_U3)
2064 __cdnsp_gadget_wakeup(pdev);
2065
2066 spin_unlock_irqrestore(&pdev->lock, flags);
2067
2068 return ret;
2069 }
2070
2071 /**
2072 * cdnsp_gadget_init - initialize device structure
2073 * @cdns: cdnsp instance
2074 *
2075 * This function initializes the gadget.
2076 */
cdnsp_gadget_init(struct cdns * cdns)2077 int cdnsp_gadget_init(struct cdns *cdns)
2078 {
2079 struct cdns_role_driver *rdrv;
2080
2081 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2082 if (!rdrv)
2083 return -ENOMEM;
2084
2085 rdrv->start = __cdnsp_gadget_init;
2086 rdrv->stop = cdnsp_gadget_exit;
2087 rdrv->suspend = cdnsp_gadget_suspend;
2088 rdrv->resume = cdnsp_gadget_resume;
2089 rdrv->state = CDNS_ROLE_STATE_INACTIVE;
2090 rdrv->name = "gadget";
2091 cdns->roles[USB_ROLE_DEVICE] = rdrv;
2092
2093 return 0;
2094 }
2095 EXPORT_SYMBOL_GPL(cdnsp_gadget_init);
2096