xref: /linux/drivers/usb/cdns3/cdnsp-gadget.c (revision cc8b10fa70682218c2a318fc44f71f3175a23cc0)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Cadence CDNSP DRD Driver.
4  *
5  * Copyright (C) 2020 Cadence.
6  *
7  * Author: Pawel Laszczak <pawell@cadence.com>
8  *
9  */
10 
11 #include <linux/moduleparam.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/module.h>
14 #include <linux/iopoll.h>
15 #include <linux/delay.h>
16 #include <linux/log2.h>
17 #include <linux/slab.h>
18 #include <linux/string_choices.h>
19 #include <linux/pci.h>
20 #include <linux/irq.h>
21 #include <linux/dmi.h>
22 
23 #include "core.h"
24 #include "gadget-export.h"
25 #include "drd.h"
26 #include "cdnsp-gadget.h"
27 #include "cdnsp-trace.h"
28 
29 unsigned int cdnsp_port_speed(unsigned int port_status)
30 {
31 	/*Detect gadget speed based on PORTSC register*/
32 	if (DEV_SUPERSPEEDPLUS(port_status))
33 		return USB_SPEED_SUPER_PLUS;
34 	else if (DEV_SUPERSPEED(port_status))
35 		return USB_SPEED_SUPER;
36 	else if (DEV_HIGHSPEED(port_status))
37 		return USB_SPEED_HIGH;
38 	else if (DEV_FULLSPEED(port_status))
39 		return USB_SPEED_FULL;
40 
41 	/* If device is detached then speed will be USB_SPEED_UNKNOWN.*/
42 	return USB_SPEED_UNKNOWN;
43 }
44 
45 /*
46  * Given a port state, this function returns a value that would result in the
47  * port being in the same state, if the value was written to the port status
48  * control register.
49  * Save Read Only (RO) bits and save read/write bits where
50  * writing a 0 clears the bit and writing a 1 sets the bit (RWS).
51  * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect.
52  */
53 u32 cdnsp_port_state_to_neutral(u32 state)
54 {
55 	/* Save read-only status and port state. */
56 	return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS);
57 }
58 
59 /**
60  * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities
61  *                           with capability ID id.
62  * @base: PCI MMIO registers base address.
63  * @start: Address at which to start looking, (0 or HCC_PARAMS to start at
64  *         beginning of list)
65  * @id: Extended capability ID to search for.
66  *
67  * Returns the offset of the next matching extended capability structure.
68  * Some capabilities can occur several times,
69  * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all.
70  */
71 int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id)
72 {
73 	u32 offset = start;
74 	u32 next;
75 	u32 val;
76 
77 	if (!start || start == HCC_PARAMS_OFFSET) {
78 		val = readl(base + HCC_PARAMS_OFFSET);
79 		if (val == ~0)
80 			return 0;
81 
82 		offset = HCC_EXT_CAPS(val) << 2;
83 		if (!offset)
84 			return 0;
85 	}
86 
87 	do {
88 		val = readl(base + offset);
89 		if (val == ~0)
90 			return 0;
91 
92 		if (EXT_CAPS_ID(val) == id && offset != start)
93 			return offset;
94 
95 		next = EXT_CAPS_NEXT(val);
96 		offset += next << 2;
97 	} while (next);
98 
99 	return 0;
100 }
101 
102 void cdnsp_set_link_state(struct cdnsp_device *pdev,
103 			  __le32 __iomem *port_regs,
104 			  u32 link_state)
105 {
106 	int port_num = 0xFF;
107 	u32 temp;
108 
109 	temp = readl(port_regs);
110 	temp = cdnsp_port_state_to_neutral(temp);
111 	temp |= PORT_WKCONN_E | PORT_WKDISC_E;
112 	writel(temp, port_regs);
113 
114 	temp &= ~PORT_PLS_MASK;
115 	temp |= PORT_LINK_STROBE | link_state;
116 
117 	if (pdev->active_port)
118 		port_num = pdev->active_port->port_num;
119 
120 	trace_cdnsp_handle_port_status(port_num, readl(port_regs));
121 	writel(temp, port_regs);
122 	trace_cdnsp_link_state_changed(port_num, readl(port_regs));
123 }
124 
125 static void cdnsp_disable_port(struct cdnsp_device *pdev,
126 			       __le32 __iomem *port_regs)
127 {
128 	u32 temp = cdnsp_port_state_to_neutral(readl(port_regs));
129 
130 	writel(temp | PORT_PED, port_regs);
131 }
132 
133 static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
134 					__le32 __iomem *port_regs)
135 {
136 	u32 portsc = readl(port_regs);
137 
138 	writel(cdnsp_port_state_to_neutral(portsc) |
139 	       (portsc & PORT_CHANGE_BITS), port_regs);
140 }
141 
142 static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
143 {
144 	__le32 __iomem *reg;
145 	void __iomem *base;
146 	u32 offset = 0;
147 
148 	base = &pdev->cap_regs->hc_capbase;
149 	offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
150 	reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
151 
152 	bit = readl(reg) | bit;
153 	writel(bit, reg);
154 }
155 
156 static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
157 {
158 	__le32 __iomem *reg;
159 	void __iomem *base;
160 	u32 offset = 0;
161 
162 	base = &pdev->cap_regs->hc_capbase;
163 	offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
164 	reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
165 
166 	bit = readl(reg) & ~bit;
167 	writel(bit, reg);
168 }
169 
170 /*
171  * Disable interrupts and begin the controller halting process.
172  */
173 static void cdnsp_quiesce(struct cdnsp_device *pdev)
174 {
175 	u32 halted;
176 	u32 mask;
177 	u32 cmd;
178 
179 	mask = ~(u32)(CDNSP_IRQS);
180 
181 	halted = readl(&pdev->op_regs->status) & STS_HALT;
182 	if (!halted)
183 		mask &= ~(CMD_R_S | CMD_DEVEN);
184 
185 	cmd = readl(&pdev->op_regs->command);
186 	cmd &= mask;
187 	writel(cmd, &pdev->op_regs->command);
188 }
189 
190 /*
191  * Force controller into halt state.
192  *
193  * Disable any IRQs and clear the run/stop bit.
194  * Controller will complete any current and actively pipelined transactions, and
195  * should halt within 16 ms of the run/stop bit being cleared.
196  * Read controller Halted bit in the status register to see when the
197  * controller is finished.
198  */
199 int cdnsp_halt(struct cdnsp_device *pdev)
200 {
201 	int ret;
202 	u32 val;
203 
204 	cdnsp_quiesce(pdev);
205 
206 	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val,
207 					val & STS_HALT, 1,
208 					CDNSP_MAX_HALT_USEC);
209 	if (ret) {
210 		dev_err(pdev->dev, "ERROR: Device halt failed\n");
211 		return ret;
212 	}
213 
214 	pdev->cdnsp_state |= CDNSP_STATE_HALTED;
215 
216 	return 0;
217 }
218 
219 /*
220  * device controller died, register read returns 0xffffffff, or command never
221  * ends.
222  */
223 void cdnsp_died(struct cdnsp_device *pdev)
224 {
225 	dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n");
226 	pdev->cdnsp_state |= CDNSP_STATE_DYING;
227 	cdnsp_halt(pdev);
228 }
229 
230 /*
231  * Set the run bit and wait for the device to be running.
232  */
233 static int cdnsp_start(struct cdnsp_device *pdev)
234 {
235 	u32 temp;
236 	int ret;
237 
238 	temp = readl(&pdev->op_regs->command);
239 	temp |= (CMD_R_S | CMD_DEVEN);
240 	writel(temp, &pdev->op_regs->command);
241 
242 	pdev->cdnsp_state = 0;
243 
244 	/*
245 	 * Wait for the STS_HALT Status bit to be 0 to indicate the device is
246 	 * running.
247 	 */
248 	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
249 					!(temp & STS_HALT), 1,
250 					CDNSP_MAX_HALT_USEC);
251 	if (ret) {
252 		pdev->cdnsp_state = CDNSP_STATE_DYING;
253 		dev_err(pdev->dev, "ERROR: Controller run failed\n");
254 	}
255 
256 	return ret;
257 }
258 
259 /*
260  * Reset a halted controller.
261  *
262  * This resets pipelines, timers, counters, state machines, etc.
263  * Transactions will be terminated immediately, and operational registers
264  * will be set to their defaults.
265  */
266 int cdnsp_reset(struct cdnsp_device *pdev)
267 {
268 	u32 command;
269 	u32 temp;
270 	int ret;
271 
272 	temp = readl(&pdev->op_regs->status);
273 
274 	if (temp == ~(u32)0) {
275 		dev_err(pdev->dev, "Device not accessible, reset failed.\n");
276 		return -ENODEV;
277 	}
278 
279 	if ((temp & STS_HALT) == 0) {
280 		dev_err(pdev->dev, "Controller not halted, aborting reset.\n");
281 		return -EINVAL;
282 	}
283 
284 	command = readl(&pdev->op_regs->command);
285 	command |= CMD_RESET;
286 	writel(command, &pdev->op_regs->command);
287 
288 	ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp,
289 					!(temp & CMD_RESET), 1,
290 					10 * 1000);
291 	if (ret) {
292 		dev_err(pdev->dev, "ERROR: Controller reset failed\n");
293 		return ret;
294 	}
295 
296 	/*
297 	 * CDNSP cannot write any doorbells or operational registers other
298 	 * than status until the "Controller Not Ready" flag is cleared.
299 	 */
300 	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
301 					!(temp & STS_CNR), 1,
302 					10 * 1000);
303 
304 	if (ret) {
305 		dev_err(pdev->dev, "ERROR: Controller not ready to work\n");
306 		return ret;
307 	}
308 
309 	dev_dbg(pdev->dev, "Controller ready to work");
310 
311 	return ret;
312 }
313 
314 /*
315  * cdnsp_get_endpoint_index - Find the index for an endpoint given its
316  * descriptor.Use the return value to right shift 1 for the bitmask.
317  *
318  * Index = (epnum * 2) + direction - 1,
319  * where direction = 0 for OUT, 1 for IN.
320  * For control endpoints, the IN index is used (OUT index is unused), so
321  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
322  */
323 static unsigned int
324 	cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc)
325 {
326 	unsigned int index = (unsigned int)usb_endpoint_num(desc);
327 
328 	if (usb_endpoint_xfer_control(desc))
329 		return index * 2;
330 
331 	return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
332 }
333 
334 /*
335  * Find the flag for this endpoint (for use in the control context). Use the
336  * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is
337  * bit 1, etc.
338  */
339 static unsigned int
340 	cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc)
341 {
342 	return 1 << (cdnsp_get_endpoint_index(desc) + 1);
343 }
344 
345 int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
346 {
347 	struct cdnsp_device *pdev = pep->pdev;
348 	struct usb_request *request;
349 	int ret;
350 
351 	if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {
352 		trace_cdnsp_request_enqueue_busy(preq);
353 		return -EBUSY;
354 	}
355 
356 	request = &preq->request;
357 	request->actual = 0;
358 	request->status = -EINPROGRESS;
359 	preq->direction = pep->direction;
360 	preq->epnum = pep->number;
361 	preq->td.drbl = 0;
362 
363 	ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);
364 	if (ret) {
365 		trace_cdnsp_request_enqueue_error(preq);
366 		return ret;
367 	}
368 
369 	list_add_tail(&preq->list, &pep->pending_list);
370 
371 	trace_cdnsp_request_enqueue(preq);
372 
373 	switch (usb_endpoint_type(pep->endpoint.desc)) {
374 	case USB_ENDPOINT_XFER_CONTROL:
375 		ret = cdnsp_queue_ctrl_tx(pdev, preq);
376 		break;
377 	case USB_ENDPOINT_XFER_BULK:
378 	case USB_ENDPOINT_XFER_INT:
379 		ret = cdnsp_queue_bulk_tx(pdev, preq);
380 		break;
381 	case USB_ENDPOINT_XFER_ISOC:
382 		ret = cdnsp_queue_isoc_tx(pdev, preq);
383 	}
384 
385 	if (ret)
386 		goto unmap;
387 
388 	return 0;
389 
390 unmap:
391 	usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
392 					pep->direction);
393 	list_del(&preq->list);
394 	trace_cdnsp_request_enqueue_error(preq);
395 
396 	return ret;
397 }
398 
399 /*
400  * Remove the request's TD from the endpoint ring. This may cause the
401  * controller to stop USB transfers, potentially stopping in the middle of a
402  * TRB buffer. The controller should pick up where it left off in the TD,
403  * unless a Set Transfer Ring Dequeue Pointer is issued.
404  *
405  * The TRBs that make up the buffers for the canceled request will be "removed"
406  * from the ring. Since the ring is a contiguous structure, they can't be
407  * physically removed. Instead, there are two options:
408  *
409  *  1) If the controller is in the middle of processing the request to be
410  *     canceled, we simply move the ring's dequeue pointer past those TRBs
411  *     using the Set Transfer Ring Dequeue Pointer command. This will be
412  *     the common case, when drivers timeout on the last submitted request
413  *     and attempt to cancel.
414  *
415  *  2) If the controller is in the middle of a different TD, we turn the TRBs
416  *     into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained.
417  *     The controller will need to invalidate the any TRBs it has cached after
418  *     the stop endpoint command.
419  *
420  *  3) The TD may have completed by the time the Stop Endpoint Command
421  *     completes, so software needs to handle that case too.
422  *
423  */
424 int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
425 {
426 	struct cdnsp_device *pdev = pep->pdev;
427 	int ret_stop = 0;
428 	int ret_rem;
429 
430 	trace_cdnsp_request_dequeue(preq);
431 
432 	if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)
433 		ret_stop = cdnsp_cmd_stop_ep(pdev, pep);
434 
435 	ret_rem = cdnsp_remove_request(pdev, preq, pep);
436 
437 	return ret_rem ? ret_rem : ret_stop;
438 }
439 
440 static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev)
441 {
442 	struct cdnsp_input_control_ctx *ctrl_ctx;
443 	struct cdnsp_slot_ctx *slot_ctx;
444 	struct cdnsp_ep_ctx *ep_ctx;
445 	int i;
446 
447 	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
448 
449 	/*
450 	 * When a device's add flag and drop flag are zero, any subsequent
451 	 * configure endpoint command will leave that endpoint's state
452 	 * untouched. Make sure we don't leave any old state in the input
453 	 * endpoint contexts.
454 	 */
455 	ctrl_ctx->drop_flags = 0;
456 	ctrl_ctx->add_flags = 0;
457 	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
458 	slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
459 
460 	/* Endpoint 0 is always valid */
461 	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
462 	for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) {
463 		ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i);
464 		ep_ctx->ep_info = 0;
465 		ep_ctx->ep_info2 = 0;
466 		ep_ctx->deq = 0;
467 		ep_ctx->tx_info = 0;
468 	}
469 }
470 
471 /* Issue a configure endpoint command and wait for it to finish. */
472 static int cdnsp_configure_endpoint(struct cdnsp_device *pdev)
473 {
474 	int ret;
475 
476 	cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma);
477 	cdnsp_ring_cmd_db(pdev);
478 	ret = cdnsp_wait_for_cmd_compl(pdev);
479 	if (ret) {
480 		dev_err(pdev->dev,
481 			"ERR: unexpected command completion code 0x%x.\n", ret);
482 		return -EINVAL;
483 	}
484 
485 	return ret;
486 }
487 
488 static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev,
489 				       struct cdnsp_ep *pep)
490 {
491 	struct cdnsp_segment *segment;
492 	union cdnsp_trb *event;
493 	u32 cycle_state;
494 	u32  data;
495 
496 	event = pdev->event_ring->dequeue;
497 	segment = pdev->event_ring->deq_seg;
498 	cycle_state = pdev->event_ring->cycle_state;
499 
500 	while (1) {
501 		data = le32_to_cpu(event->trans_event.flags);
502 
503 		/* Check the owner of the TRB. */
504 		if ((data & TRB_CYCLE) != cycle_state)
505 			break;
506 
507 		if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER &&
508 		    TRB_TO_EP_ID(data) == (pep->idx + 1)) {
509 			data |= TRB_EVENT_INVALIDATE;
510 			event->trans_event.flags = cpu_to_le32(data);
511 		}
512 
513 		if (cdnsp_last_trb_on_seg(segment, event)) {
514 			cycle_state ^= 1;
515 			segment = pdev->event_ring->deq_seg->next;
516 			event = segment->trbs;
517 		} else {
518 			event++;
519 		}
520 	}
521 }
522 
523 int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev)
524 {
525 	struct cdnsp_segment *event_deq_seg;
526 	union cdnsp_trb *cmd_trb;
527 	dma_addr_t cmd_deq_dma;
528 	union cdnsp_trb *event;
529 	u32 cycle_state;
530 	int ret, val;
531 	u64 cmd_dma;
532 	u32  flags;
533 
534 	cmd_trb = pdev->cmd.command_trb;
535 	pdev->cmd.status = 0;
536 
537 	trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic);
538 
539 	ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val,
540 					!CMD_RING_BUSY(val), 1,
541 					CDNSP_CMD_TIMEOUT);
542 	if (ret) {
543 		dev_err(pdev->dev, "ERR: Timeout while waiting for command\n");
544 		trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic);
545 		pdev->cdnsp_state = CDNSP_STATE_DYING;
546 		return -ETIMEDOUT;
547 	}
548 
549 	event = pdev->event_ring->dequeue;
550 	event_deq_seg = pdev->event_ring->deq_seg;
551 	cycle_state = pdev->event_ring->cycle_state;
552 
553 	cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb);
554 	if (!cmd_deq_dma)
555 		return -EINVAL;
556 
557 	while (1) {
558 		flags = le32_to_cpu(event->event_cmd.flags);
559 
560 		/* Check the owner of the TRB. */
561 		if ((flags & TRB_CYCLE) != cycle_state)
562 			return -EINVAL;
563 
564 		cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb);
565 
566 		/*
567 		 * Check whether the completion event is for last queued
568 		 * command.
569 		 */
570 		if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION ||
571 		    cmd_dma != (u64)cmd_deq_dma) {
572 			if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
573 				event++;
574 				continue;
575 			}
576 
577 			if (cdnsp_last_trb_on_ring(pdev->event_ring,
578 						   event_deq_seg, event))
579 				cycle_state ^= 1;
580 
581 			event_deq_seg = event_deq_seg->next;
582 			event = event_deq_seg->trbs;
583 			continue;
584 		}
585 
586 		trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic);
587 
588 		pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status));
589 		if (pdev->cmd.status == COMP_SUCCESS)
590 			return 0;
591 
592 		return -pdev->cmd.status;
593 	}
594 }
595 
596 int cdnsp_halt_endpoint(struct cdnsp_device *pdev,
597 			struct cdnsp_ep *pep,
598 			int value)
599 {
600 	int ret;
601 
602 	trace_cdnsp_ep_halt(value ? "Set" : "Clear");
603 
604 	ret = cdnsp_cmd_stop_ep(pdev, pep);
605 	if (ret)
606 		return ret;
607 
608 	if (value) {
609 		if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {
610 			cdnsp_queue_halt_endpoint(pdev, pep->idx);
611 			cdnsp_ring_cmd_db(pdev);
612 			ret = cdnsp_wait_for_cmd_compl(pdev);
613 		}
614 
615 		pep->ep_state |= EP_HALTED;
616 	} else {
617 		cdnsp_queue_reset_ep(pdev, pep->idx);
618 		cdnsp_ring_cmd_db(pdev);
619 		ret = cdnsp_wait_for_cmd_compl(pdev);
620 		trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);
621 
622 		if (ret)
623 			return ret;
624 
625 		pep->ep_state &= ~EP_HALTED;
626 
627 		if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))
628 			cdnsp_ring_doorbell_for_active_rings(pdev, pep);
629 
630 		pep->ep_state &= ~EP_WEDGE;
631 	}
632 
633 	return 0;
634 }
635 
636 static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev,
637 					  struct cdnsp_ep *pep)
638 {
639 	struct cdnsp_input_control_ctx *ctrl_ctx;
640 	struct cdnsp_slot_ctx *slot_ctx;
641 	int ret = 0;
642 	u32 ep_sts;
643 	int i;
644 
645 	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
646 
647 	/* Don't issue the command if there's no endpoints to update. */
648 	if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0)
649 		return 0;
650 
651 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
652 	ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
653 	ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
654 
655 	/* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
656 	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
657 	for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) {
658 		__le32 le32 = cpu_to_le32(BIT(i));
659 
660 		if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) ||
661 		    (ctrl_ctx->add_flags & le32) || i == 1) {
662 			slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
663 			slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
664 			break;
665 		}
666 	}
667 
668 	ep_sts = GET_EP_CTX_STATE(pep->out_ctx);
669 
670 	if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) &&
671 	     ep_sts == EP_STATE_DISABLED) ||
672 	    (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags))
673 		ret = cdnsp_configure_endpoint(pdev);
674 
675 	trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx));
676 	trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);
677 
678 	cdnsp_zero_in_ctx(pdev);
679 
680 	return ret;
681 }
682 
683 /*
684  * This submits a Reset Device Command, which will set the device state to 0,
685  * set the device address to 0, and disable all the endpoints except the default
686  * control endpoint. The USB core should come back and call
687  * cdnsp_setup_device(), and then re-set up the configuration.
688  */
689 int cdnsp_reset_device(struct cdnsp_device *pdev)
690 {
691 	struct cdnsp_slot_ctx *slot_ctx;
692 	int slot_state;
693 	int ret, i;
694 
695 	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
696 	slot_ctx->dev_info = 0;
697 	pdev->device_address = 0;
698 
699 	/* If device is not setup, there is no point in resetting it. */
700 	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
701 	slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
702 	trace_cdnsp_reset_device(slot_ctx);
703 
704 	if (slot_state <= SLOT_STATE_DEFAULT &&
705 	    pdev->eps[0].ep_state & EP_HALTED) {
706 		cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0);
707 	}
708 
709 	/*
710 	 * During Reset Device command controller shall transition the
711 	 * endpoint ep0 to the Running State.
712 	 */
713 	pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED);
714 	pdev->eps[0].ep_state |= EP_ENABLED;
715 
716 	if (slot_state <= SLOT_STATE_DEFAULT)
717 		return 0;
718 
719 	cdnsp_queue_reset_device(pdev);
720 	cdnsp_ring_cmd_db(pdev);
721 	ret = cdnsp_wait_for_cmd_compl(pdev);
722 
723 	/*
724 	 * After Reset Device command all not default endpoints
725 	 * are in Disabled state.
726 	 */
727 	for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i)
728 		pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED;
729 
730 	trace_cdnsp_handle_cmd_reset_dev(slot_ctx);
731 
732 	if (ret)
733 		dev_err(pdev->dev, "Reset device failed with error code %d",
734 			ret);
735 
736 	return ret;
737 }
738 
739 /*
740  * Sets the MaxPStreams field and the Linear Stream Array field.
741  * Sets the dequeue pointer to the stream context array.
742  */
743 static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev,
744 					     struct cdnsp_ep_ctx *ep_ctx,
745 					     struct cdnsp_stream_info *stream_info)
746 {
747 	u32 max_primary_streams;
748 
749 	/* MaxPStreams is the number of stream context array entries, not the
750 	 * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
751 	 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
752 	 */
753 	max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
754 	ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
755 	ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
756 				       | EP_HAS_LSA);
757 	ep_ctx->deq  = cpu_to_le64(stream_info->ctx_array_dma);
758 }
759 
760 /*
761  * The drivers use this function to prepare a bulk endpoints to use streams.
762  *
763  * Don't allow the call to succeed if endpoint only supports one stream
764  * (which means it doesn't support streams at all).
765  */
766 int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
767 {
768 	unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);
769 	unsigned int num_stream_ctxs;
770 	int ret;
771 
772 	if (num_streams ==  0)
773 		return 0;
774 
775 	if (num_streams > STREAM_NUM_STREAMS)
776 		return -EINVAL;
777 
778 	/*
779 	 * Add two to the number of streams requested to account for
780 	 * stream 0 that is reserved for controller usage and one additional
781 	 * for TASK SET FULL response.
782 	 */
783 	num_streams += 2;
784 
785 	/* The stream context array size must be a power of two */
786 	num_stream_ctxs = roundup_pow_of_two(num_streams);
787 
788 	trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);
789 
790 	ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);
791 	if (ret)
792 		return ret;
793 
794 	cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);
795 
796 	pep->ep_state |= EP_HAS_STREAMS;
797 	pep->stream_info.td_count = 0;
798 	pep->stream_info.first_prime_det = 0;
799 
800 	/* Subtract 1 for stream 0, which drivers can't use. */
801 	return num_streams - 1;
802 }
803 
804 int cdnsp_disable_slot(struct cdnsp_device *pdev)
805 {
806 	int ret;
807 
808 	cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT);
809 	cdnsp_ring_cmd_db(pdev);
810 	ret = cdnsp_wait_for_cmd_compl(pdev);
811 
812 	pdev->slot_id = 0;
813 	pdev->active_port = NULL;
814 
815 	trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
816 
817 	memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE);
818 	memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE);
819 
820 	return ret;
821 }
822 
823 int cdnsp_enable_slot(struct cdnsp_device *pdev)
824 {
825 	struct cdnsp_slot_ctx *slot_ctx;
826 	int slot_state;
827 	int ret;
828 
829 	/* If device is not setup, there is no point in resetting it */
830 	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
831 	slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
832 
833 	if (slot_state != SLOT_STATE_DISABLED)
834 		return 0;
835 
836 	cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT);
837 	cdnsp_ring_cmd_db(pdev);
838 	ret = cdnsp_wait_for_cmd_compl(pdev);
839 	if (ret)
840 		goto show_trace;
841 
842 	pdev->slot_id = 1;
843 
844 show_trace:
845 	trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
846 
847 	return ret;
848 }
849 
850 /*
851  * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY
852  * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS.
853  */
854 int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup)
855 {
856 	struct cdnsp_input_control_ctx *ctrl_ctx;
857 	struct cdnsp_slot_ctx *slot_ctx;
858 	int dev_state = 0;
859 	int ret;
860 
861 	if (!pdev->slot_id) {
862 		trace_cdnsp_slot_id("incorrect");
863 		return -EINVAL;
864 	}
865 
866 	if (!pdev->active_port->port_num)
867 		return -EINVAL;
868 
869 	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
870 	dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
871 
872 	if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) {
873 		trace_cdnsp_slot_already_in_default(slot_ctx);
874 		return 0;
875 	}
876 
877 	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
878 	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
879 
880 	if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) {
881 		ret = cdnsp_setup_addressable_priv_dev(pdev);
882 		if (ret)
883 			return ret;
884 	}
885 
886 	cdnsp_copy_ep0_dequeue_into_input_ctx(pdev);
887 
888 	ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
889 	ctrl_ctx->drop_flags = 0;
890 
891 	trace_cdnsp_setup_device_slot(slot_ctx);
892 
893 	cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup);
894 	cdnsp_ring_cmd_db(pdev);
895 	ret = cdnsp_wait_for_cmd_compl(pdev);
896 
897 	trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx));
898 
899 	/* Zero the input context control for later use. */
900 	ctrl_ctx->add_flags = 0;
901 	ctrl_ctx->drop_flags = 0;
902 
903 	return ret;
904 }
905 
906 void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
907 				 struct usb_request *req,
908 				 int enable)
909 {
910 	if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable)
911 		return;
912 
913 	trace_cdnsp_lpm(enable);
914 
915 	if (enable)
916 		writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE,
917 		       &pdev->active_port->regs->portpmsc);
918 	else
919 		writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc);
920 }
921 
922 static int cdnsp_get_frame(struct cdnsp_device *pdev)
923 {
924 	return readl(&pdev->run_regs->microframe_index) >> 3;
925 }
926 
927 static int cdnsp_gadget_ep_enable(struct usb_ep *ep,
928 				  const struct usb_endpoint_descriptor *desc)
929 {
930 	struct cdnsp_input_control_ctx *ctrl_ctx;
931 	struct cdnsp_device *pdev;
932 	struct cdnsp_ep *pep;
933 	unsigned long flags;
934 	u32 added_ctxs;
935 	int ret;
936 
937 	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
938 	    !desc->wMaxPacketSize)
939 		return -EINVAL;
940 
941 	pep = to_cdnsp_ep(ep);
942 	pdev = pep->pdev;
943 	pep->ep_state &= ~EP_UNCONFIGURED;
944 
945 	if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
946 			  "%s is already enabled\n", pep->name))
947 		return 0;
948 
949 	spin_lock_irqsave(&pdev->lock, flags);
950 
951 	added_ctxs = cdnsp_get_endpoint_flag(desc);
952 	if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
953 		dev_err(pdev->dev, "ERROR: Bad endpoint number\n");
954 		ret = -EINVAL;
955 		goto unlock;
956 	}
957 
958 	pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
959 
960 	if (pdev->gadget.speed == USB_SPEED_FULL) {
961 		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT)
962 			pep->interval = desc->bInterval << 3;
963 		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
964 			pep->interval = BIT(desc->bInterval - 1) << 3;
965 	}
966 
967 	if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) {
968 		if (pep->interval > BIT(12)) {
969 			dev_err(pdev->dev, "bInterval %d not supported\n",
970 				desc->bInterval);
971 			ret = -EINVAL;
972 			goto unlock;
973 		}
974 		cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
975 	}
976 
977 	ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);
978 	if (ret)
979 		goto unlock;
980 
981 	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
982 	ctrl_ctx->add_flags = cpu_to_le32(added_ctxs);
983 	ctrl_ctx->drop_flags = 0;
984 
985 	ret = cdnsp_update_eps_configuration(pdev, pep);
986 	if (ret) {
987 		cdnsp_free_endpoint_rings(pdev, pep);
988 		goto unlock;
989 	}
990 
991 	pep->ep_state |= EP_ENABLED;
992 	pep->ep_state &= ~EP_STOPPED;
993 
994 unlock:
995 	trace_cdnsp_ep_enable_end(pep, 0);
996 	spin_unlock_irqrestore(&pdev->lock, flags);
997 
998 	return ret;
999 }
1000 
1001 static int cdnsp_gadget_ep_disable(struct usb_ep *ep)
1002 {
1003 	struct cdnsp_input_control_ctx *ctrl_ctx;
1004 	struct cdnsp_request *preq;
1005 	struct cdnsp_device *pdev;
1006 	struct cdnsp_ep *pep;
1007 	unsigned long flags;
1008 	u32 drop_flag;
1009 	int ret = 0;
1010 
1011 	if (!ep)
1012 		return -EINVAL;
1013 
1014 	pep = to_cdnsp_ep(ep);
1015 	pdev = pep->pdev;
1016 
1017 	spin_lock_irqsave(&pdev->lock, flags);
1018 
1019 	if (!(pep->ep_state & EP_ENABLED)) {
1020 		dev_err(pdev->dev, "%s is already disabled\n", pep->name);
1021 		ret = -EINVAL;
1022 		goto finish;
1023 	}
1024 
1025 	pep->ep_state |= EP_DIS_IN_RROGRESS;
1026 
1027 	/* Endpoint was unconfigured by Reset Device command. */
1028 	if (!(pep->ep_state & EP_UNCONFIGURED))
1029 		cdnsp_cmd_stop_ep(pdev, pep);
1030 
1031 	/* Remove all queued USB requests. */
1032 	while (!list_empty(&pep->pending_list)) {
1033 		preq = next_request(&pep->pending_list);
1034 		cdnsp_ep_dequeue(pep, preq);
1035 	}
1036 
1037 	cdnsp_invalidate_ep_events(pdev, pep);
1038 
1039 	pep->ep_state &= ~EP_DIS_IN_RROGRESS;
1040 	drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);
1041 	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1042 	ctrl_ctx->drop_flags = cpu_to_le32(drop_flag);
1043 	ctrl_ctx->add_flags = 0;
1044 
1045 	cdnsp_endpoint_zero(pdev, pep);
1046 
1047 	if (!(pep->ep_state & EP_UNCONFIGURED))
1048 		ret = cdnsp_update_eps_configuration(pdev, pep);
1049 
1050 	cdnsp_free_endpoint_rings(pdev, pep);
1051 
1052 	pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);
1053 	pep->ep_state |= EP_STOPPED;
1054 
1055 finish:
1056 	trace_cdnsp_ep_disable_end(pep, 0);
1057 	spin_unlock_irqrestore(&pdev->lock, flags);
1058 
1059 	return ret;
1060 }
1061 
1062 static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep,
1063 							 gfp_t gfp_flags)
1064 {
1065 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1066 	struct cdnsp_request *preq;
1067 
1068 	preq = kzalloc(sizeof(*preq), gfp_flags);
1069 	if (!preq)
1070 		return NULL;
1071 
1072 	preq->epnum = pep->number;
1073 	preq->pep = pep;
1074 
1075 	trace_cdnsp_alloc_request(preq);
1076 
1077 	return &preq->request;
1078 }
1079 
1080 static void cdnsp_gadget_ep_free_request(struct usb_ep *ep,
1081 					 struct usb_request *request)
1082 {
1083 	struct cdnsp_request *preq = to_cdnsp_request(request);
1084 
1085 	trace_cdnsp_free_request(preq);
1086 	kfree(preq);
1087 }
1088 
1089 static int cdnsp_gadget_ep_queue(struct usb_ep *ep,
1090 				 struct usb_request *request,
1091 				 gfp_t gfp_flags)
1092 {
1093 	struct cdnsp_request *preq;
1094 	struct cdnsp_device *pdev;
1095 	struct cdnsp_ep *pep;
1096 	unsigned long flags;
1097 	int ret;
1098 
1099 	if (!request || !ep)
1100 		return -EINVAL;
1101 
1102 	pep = to_cdnsp_ep(ep);
1103 	pdev = pep->pdev;
1104 
1105 	if (!(pep->ep_state & EP_ENABLED)) {
1106 		dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1107 			pep->name);
1108 		return -EINVAL;
1109 	}
1110 
1111 	preq = to_cdnsp_request(request);
1112 	spin_lock_irqsave(&pdev->lock, flags);
1113 	ret = cdnsp_ep_enqueue(pep, preq);
1114 	spin_unlock_irqrestore(&pdev->lock, flags);
1115 
1116 	return ret;
1117 }
1118 
1119 static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep,
1120 				   struct usb_request *request)
1121 {
1122 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1123 	struct cdnsp_device *pdev = pep->pdev;
1124 	unsigned long flags;
1125 	int ret;
1126 
1127 	if (request->status != -EINPROGRESS)
1128 		return 0;
1129 
1130 	if (!pep->endpoint.desc) {
1131 		dev_err(pdev->dev,
1132 			"%s: can't dequeue to disabled endpoint\n",
1133 			pep->name);
1134 		return -ESHUTDOWN;
1135 	}
1136 
1137 	/* Requests has been dequeued during disabling endpoint. */
1138 	if (!(pep->ep_state & EP_ENABLED))
1139 		return 0;
1140 
1141 	spin_lock_irqsave(&pdev->lock, flags);
1142 	ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));
1143 	spin_unlock_irqrestore(&pdev->lock, flags);
1144 
1145 	return ret;
1146 }
1147 
1148 static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value)
1149 {
1150 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1151 	struct cdnsp_device *pdev = pep->pdev;
1152 	struct cdnsp_request *preq;
1153 	unsigned long flags;
1154 	int ret;
1155 
1156 	spin_lock_irqsave(&pdev->lock, flags);
1157 
1158 	preq = next_request(&pep->pending_list);
1159 	if (value) {
1160 		if (preq) {
1161 			trace_cdnsp_ep_busy_try_halt_again(pep, 0);
1162 			ret = -EAGAIN;
1163 			goto done;
1164 		}
1165 	}
1166 
1167 	ret = cdnsp_halt_endpoint(pdev, pep, value);
1168 
1169 done:
1170 	spin_unlock_irqrestore(&pdev->lock, flags);
1171 	return ret;
1172 }
1173 
1174 static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep)
1175 {
1176 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1177 	struct cdnsp_device *pdev = pep->pdev;
1178 	unsigned long flags;
1179 	int ret;
1180 
1181 	spin_lock_irqsave(&pdev->lock, flags);
1182 	pep->ep_state |= EP_WEDGE;
1183 	ret = cdnsp_halt_endpoint(pdev, pep, 1);
1184 	spin_unlock_irqrestore(&pdev->lock, flags);
1185 
1186 	return ret;
1187 }
1188 
1189 static const struct usb_ep_ops cdnsp_gadget_ep0_ops = {
1190 	.enable		= cdnsp_gadget_ep_enable,
1191 	.disable	= cdnsp_gadget_ep_disable,
1192 	.alloc_request	= cdnsp_gadget_ep_alloc_request,
1193 	.free_request	= cdnsp_gadget_ep_free_request,
1194 	.queue		= cdnsp_gadget_ep_queue,
1195 	.dequeue	= cdnsp_gadget_ep_dequeue,
1196 	.set_halt	= cdnsp_gadget_ep_set_halt,
1197 	.set_wedge	= cdnsp_gadget_ep_set_wedge,
1198 };
1199 
1200 static const struct usb_ep_ops cdnsp_gadget_ep_ops = {
1201 	.enable		= cdnsp_gadget_ep_enable,
1202 	.disable	= cdnsp_gadget_ep_disable,
1203 	.alloc_request	= cdnsp_gadget_ep_alloc_request,
1204 	.free_request	= cdnsp_gadget_ep_free_request,
1205 	.queue		= cdnsp_gadget_ep_queue,
1206 	.dequeue	= cdnsp_gadget_ep_dequeue,
1207 	.set_halt	= cdnsp_gadget_ep_set_halt,
1208 	.set_wedge	= cdnsp_gadget_ep_set_wedge,
1209 };
1210 
1211 void cdnsp_gadget_giveback(struct cdnsp_ep *pep,
1212 			   struct cdnsp_request *preq,
1213 			   int status)
1214 {
1215 	struct cdnsp_device *pdev = pep->pdev;
1216 
1217 	list_del(&preq->list);
1218 
1219 	if (preq->request.status == -EINPROGRESS)
1220 		preq->request.status = status;
1221 
1222 	usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
1223 					preq->direction);
1224 
1225 	trace_cdnsp_request_giveback(preq);
1226 
1227 	if (preq != &pdev->ep0_preq) {
1228 		spin_unlock(&pdev->lock);
1229 		usb_gadget_giveback_request(&pep->endpoint, &preq->request);
1230 		spin_lock(&pdev->lock);
1231 	}
1232 }
1233 
1234 static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = {
1235 	.bLength =		USB_DT_ENDPOINT_SIZE,
1236 	.bDescriptorType =	USB_DT_ENDPOINT,
1237 	.bmAttributes =		USB_ENDPOINT_XFER_CONTROL,
1238 };
1239 
1240 static int cdnsp_run(struct cdnsp_device *pdev,
1241 		     enum usb_device_speed speed)
1242 {
1243 	u32 fs_speed = 0;
1244 	u32 temp;
1245 	int ret;
1246 
1247 	temp = readl(&pdev->ir_set->irq_control);
1248 	temp &= ~IMOD_INTERVAL_MASK;
1249 	temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK);
1250 	writel(temp, &pdev->ir_set->irq_control);
1251 
1252 	temp = readl(&pdev->port3x_regs->mode_addr);
1253 
1254 	switch (speed) {
1255 	case USB_SPEED_SUPER_PLUS:
1256 		temp |= CFG_3XPORT_SSP_SUPPORT;
1257 		break;
1258 	case USB_SPEED_SUPER:
1259 		temp &= ~CFG_3XPORT_SSP_SUPPORT;
1260 		break;
1261 	case USB_SPEED_HIGH:
1262 		break;
1263 	case USB_SPEED_FULL:
1264 		fs_speed = PORT_REG6_FORCE_FS;
1265 		break;
1266 	default:
1267 		dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
1268 			speed);
1269 		fallthrough;
1270 	case USB_SPEED_UNKNOWN:
1271 		/* Default to superspeed. */
1272 		speed = USB_SPEED_SUPER;
1273 		break;
1274 	}
1275 
1276 	if (speed >= USB_SPEED_SUPER) {
1277 		writel(temp, &pdev->port3x_regs->mode_addr);
1278 		cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
1279 				     XDEV_RXDETECT);
1280 	} else {
1281 		cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1282 	}
1283 
1284 	cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
1285 			     XDEV_RXDETECT);
1286 
1287 	cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1288 
1289 	writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
1290 
1291 	ret = cdnsp_start(pdev);
1292 	if (ret) {
1293 		ret = -ENODEV;
1294 		goto err;
1295 	}
1296 
1297 	temp = readl(&pdev->op_regs->command);
1298 	temp |= (CMD_INTE);
1299 	writel(temp, &pdev->op_regs->command);
1300 
1301 	temp = readl(&pdev->ir_set->irq_pending);
1302 	writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending);
1303 
1304 	trace_cdnsp_init("Controller ready to work");
1305 	return 0;
1306 err:
1307 	cdnsp_halt(pdev);
1308 	return ret;
1309 }
1310 
1311 static int cdnsp_gadget_udc_start(struct usb_gadget *g,
1312 				  struct usb_gadget_driver *driver)
1313 {
1314 	enum usb_device_speed max_speed = driver->max_speed;
1315 	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1316 	unsigned long flags;
1317 	int ret;
1318 
1319 	spin_lock_irqsave(&pdev->lock, flags);
1320 	pdev->gadget_driver = driver;
1321 
1322 	/* limit speed if necessary */
1323 	max_speed = min(driver->max_speed, g->max_speed);
1324 	ret = cdnsp_run(pdev, max_speed);
1325 
1326 	spin_unlock_irqrestore(&pdev->lock, flags);
1327 
1328 	return ret;
1329 }
1330 
1331 /*
1332  * Update Event Ring Dequeue Pointer:
1333  * - When all events have finished
1334  * - To avoid "Event Ring Full Error" condition
1335  */
1336 void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev,
1337 			       union cdnsp_trb *event_ring_deq,
1338 			       u8 clear_ehb)
1339 {
1340 	u64 temp_64;
1341 	dma_addr_t deq;
1342 
1343 	temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1344 
1345 	/* If necessary, update the HW's version of the event ring deq ptr. */
1346 	if (event_ring_deq != pdev->event_ring->dequeue) {
1347 		deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1348 					    pdev->event_ring->dequeue);
1349 		temp_64 &= ERST_PTR_MASK;
1350 		temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK);
1351 	}
1352 
1353 	/* Clear the event handler busy flag (RW1C). */
1354 	if (clear_ehb)
1355 		temp_64 |= ERST_EHB;
1356 	else
1357 		temp_64 &= ~ERST_EHB;
1358 
1359 	cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue);
1360 }
1361 
1362 static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev)
1363 {
1364 	struct cdnsp_segment *seg;
1365 	u64 val_64;
1366 	int i;
1367 
1368 	cdnsp_initialize_ring_info(pdev->cmd_ring);
1369 
1370 	seg = pdev->cmd_ring->first_seg;
1371 	for (i = 0; i < pdev->cmd_ring->num_segs; i++) {
1372 		memset(seg->trbs, 0,
1373 		       sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1));
1374 		seg = seg->next;
1375 	}
1376 
1377 	/* Set the address in the Command Ring Control register. */
1378 	val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1379 	val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1380 		 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1381 		 pdev->cmd_ring->cycle_state;
1382 	cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1383 }
1384 
1385 static void cdnsp_consume_all_events(struct cdnsp_device *pdev)
1386 {
1387 	struct cdnsp_segment *event_deq_seg;
1388 	union cdnsp_trb *event_ring_deq;
1389 	union cdnsp_trb *event;
1390 	u32 cycle_bit;
1391 
1392 	event_ring_deq = pdev->event_ring->dequeue;
1393 	event_deq_seg = pdev->event_ring->deq_seg;
1394 	event = pdev->event_ring->dequeue;
1395 
1396 	/* Update ring dequeue pointer. */
1397 	while (1) {
1398 		cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE);
1399 
1400 		/* Does the controller or driver own the TRB? */
1401 		if (cycle_bit != pdev->event_ring->cycle_state)
1402 			break;
1403 
1404 		cdnsp_inc_deq(pdev, pdev->event_ring);
1405 
1406 		if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
1407 			event++;
1408 			continue;
1409 		}
1410 
1411 		if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg,
1412 					   event))
1413 			cycle_bit ^= 1;
1414 
1415 		event_deq_seg = event_deq_seg->next;
1416 		event = event_deq_seg->trbs;
1417 	}
1418 
1419 	cdnsp_update_erst_dequeue(pdev,  event_ring_deq, 1);
1420 }
1421 
1422 static void cdnsp_stop(struct cdnsp_device *pdev)
1423 {
1424 	u32 temp;
1425 
1426 	/* Remove internally queued request for ep0. */
1427 	if (!list_empty(&pdev->eps[0].pending_list)) {
1428 		struct cdnsp_request *req;
1429 
1430 		req = next_request(&pdev->eps[0].pending_list);
1431 		if (req == &pdev->ep0_preq)
1432 			cdnsp_ep_dequeue(&pdev->eps[0], req);
1433 	}
1434 
1435 	cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc);
1436 	cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1437 	cdnsp_disable_slot(pdev);
1438 	cdnsp_halt(pdev);
1439 
1440 	temp = readl(&pdev->op_regs->status);
1441 	writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status);
1442 	temp = readl(&pdev->ir_set->irq_pending);
1443 	writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
1444 
1445 	cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc);
1446 	cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc);
1447 
1448 	/* Clear interrupt line */
1449 	temp = readl(&pdev->ir_set->irq_pending);
1450 	temp |= IMAN_IP;
1451 	writel(temp, &pdev->ir_set->irq_pending);
1452 
1453 	cdnsp_consume_all_events(pdev);
1454 	cdnsp_clear_cmd_ring(pdev);
1455 
1456 	trace_cdnsp_exit("Controller stopped.");
1457 }
1458 
1459 /*
1460  * Stop controller.
1461  * This function is called by the gadget core when the driver is removed.
1462  * Disable slot, disable IRQs, and quiesce the controller.
1463  */
1464 static int cdnsp_gadget_udc_stop(struct usb_gadget *g)
1465 {
1466 	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1467 	unsigned long flags;
1468 
1469 	spin_lock_irqsave(&pdev->lock, flags);
1470 	cdnsp_stop(pdev);
1471 	pdev->gadget_driver = NULL;
1472 	spin_unlock_irqrestore(&pdev->lock, flags);
1473 
1474 	return 0;
1475 }
1476 
1477 static int cdnsp_gadget_get_frame(struct usb_gadget *g)
1478 {
1479 	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1480 
1481 	return cdnsp_get_frame(pdev);
1482 }
1483 
1484 static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev)
1485 {
1486 	struct cdnsp_port_regs __iomem *port_regs;
1487 	u32 portpm, portsc;
1488 
1489 	port_regs = pdev->active_port->regs;
1490 	portsc = readl(&port_regs->portsc) & PORT_PLS_MASK;
1491 
1492 	/* Remote wakeup feature is not enabled by host. */
1493 	if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) {
1494 		portpm = readl(&port_regs->portpmsc);
1495 
1496 		if (!(portpm & PORT_RWE))
1497 			return;
1498 	}
1499 
1500 	if (portsc == XDEV_U3 && !pdev->may_wakeup)
1501 		return;
1502 
1503 	cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0);
1504 
1505 	pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING;
1506 }
1507 
1508 static int cdnsp_gadget_wakeup(struct usb_gadget *g)
1509 {
1510 	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1511 	unsigned long flags;
1512 
1513 	spin_lock_irqsave(&pdev->lock, flags);
1514 	__cdnsp_gadget_wakeup(pdev);
1515 	spin_unlock_irqrestore(&pdev->lock, flags);
1516 
1517 	return 0;
1518 }
1519 
1520 static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g,
1521 					int is_selfpowered)
1522 {
1523 	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1524 	unsigned long flags;
1525 
1526 	spin_lock_irqsave(&pdev->lock, flags);
1527 	g->is_selfpowered = !!is_selfpowered;
1528 	spin_unlock_irqrestore(&pdev->lock, flags);
1529 
1530 	return 0;
1531 }
1532 
1533 static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
1534 {
1535 	struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
1536 	struct cdns *cdns = dev_get_drvdata(pdev->dev);
1537 	unsigned long flags;
1538 
1539 	trace_cdnsp_pullup(is_on);
1540 
1541 	/*
1542 	 * Disable events handling while controller is being
1543 	 * enabled/disabled.
1544 	 */
1545 	disable_irq(cdns->dev_irq);
1546 	spin_lock_irqsave(&pdev->lock, flags);
1547 
1548 	if (!is_on) {
1549 		cdnsp_reset_device(pdev);
1550 		cdns_clear_vbus(cdns);
1551 	} else {
1552 		cdns_set_vbus(cdns);
1553 	}
1554 
1555 	spin_unlock_irqrestore(&pdev->lock, flags);
1556 	enable_irq(cdns->dev_irq);
1557 
1558 	return 0;
1559 }
1560 
1561 static const struct usb_gadget_ops cdnsp_gadget_ops = {
1562 	.get_frame		= cdnsp_gadget_get_frame,
1563 	.wakeup			= cdnsp_gadget_wakeup,
1564 	.set_selfpowered	= cdnsp_gadget_set_selfpowered,
1565 	.pullup			= cdnsp_gadget_pullup,
1566 	.udc_start		= cdnsp_gadget_udc_start,
1567 	.udc_stop		= cdnsp_gadget_udc_stop,
1568 };
1569 
1570 static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev,
1571 				   struct cdnsp_ep *pep)
1572 {
1573 	void __iomem *reg = &pdev->cap_regs->hc_capbase;
1574 	int endpoints;
1575 
1576 	reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID);
1577 
1578 	if (!pep->direction) {
1579 		pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);
1580 		pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);
1581 		pep->buffering = (pep->buffering + 1) / 2;
1582 		pep->buffering_period = (pep->buffering_period + 1) / 2;
1583 		return;
1584 	}
1585 
1586 	endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2;
1587 
1588 	/* Set to XBUF_TX_TAG_MASK_0 register. */
1589 	reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32);
1590 	/* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */
1591 	reg += pep->number * sizeof(u32) * 2;
1592 
1593 	pep->buffering = (readl(reg) + 1) / 2;
1594 	pep->buffering_period = pep->buffering;
1595 }
1596 
1597 static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev)
1598 {
1599 	int max_streams = HCC_MAX_PSA(pdev->hcc_params);
1600 	struct cdnsp_ep *pep;
1601 	int i;
1602 
1603 	INIT_LIST_HEAD(&pdev->gadget.ep_list);
1604 
1605 	if (max_streams < STREAM_LOG_STREAMS) {
1606 		dev_err(pdev->dev, "Stream size %d not supported\n",
1607 			max_streams);
1608 		return -EINVAL;
1609 	}
1610 
1611 	max_streams = STREAM_LOG_STREAMS;
1612 
1613 	for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1614 		bool direction = !(i & 1); /* Start from OUT endpoint. */
1615 		u8 epnum = ((i + 1) >> 1);
1616 
1617 		if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction))
1618 			continue;
1619 
1620 		pep = &pdev->eps[i];
1621 		pep->pdev = pdev;
1622 		pep->number = epnum;
1623 		pep->direction = direction; /* 0 for OUT, 1 for IN. */
1624 
1625 		/*
1626 		 * Ep0 is bidirectional, so ep0in and ep0out are represented by
1627 		 * pdev->eps[0]
1628 		 */
1629 		if (epnum == 0) {
1630 			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1631 				 epnum, "BiDir");
1632 
1633 			pep->idx = 0;
1634 			usb_ep_set_maxpacket_limit(&pep->endpoint, 512);
1635 			pep->endpoint.maxburst = 1;
1636 			pep->endpoint.ops = &cdnsp_gadget_ep0_ops;
1637 			pep->endpoint.desc = &cdnsp_gadget_ep0_desc;
1638 			pep->endpoint.comp_desc = NULL;
1639 			pep->endpoint.caps.type_control = true;
1640 			pep->endpoint.caps.dir_in = true;
1641 			pep->endpoint.caps.dir_out = true;
1642 
1643 			pdev->ep0_preq.epnum = pep->number;
1644 			pdev->ep0_preq.pep = pep;
1645 			pdev->gadget.ep0 = &pep->endpoint;
1646 		} else {
1647 			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1648 				 epnum, (pep->direction) ? "in" : "out");
1649 
1650 			pep->idx =  (epnum * 2 + (direction ? 1 : 0)) - 1;
1651 			usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
1652 
1653 			pep->endpoint.max_streams = max_streams;
1654 			pep->endpoint.ops = &cdnsp_gadget_ep_ops;
1655 			list_add_tail(&pep->endpoint.ep_list,
1656 				      &pdev->gadget.ep_list);
1657 
1658 			pep->endpoint.caps.type_iso = true;
1659 			pep->endpoint.caps.type_bulk = true;
1660 			pep->endpoint.caps.type_int = true;
1661 
1662 			pep->endpoint.caps.dir_in = direction;
1663 			pep->endpoint.caps.dir_out = !direction;
1664 		}
1665 
1666 		pep->endpoint.name = pep->name;
1667 		pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);
1668 		pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);
1669 		cdnsp_get_ep_buffering(pdev, pep);
1670 
1671 		dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: "
1672 			"CTRL: %s, INT: %s, BULK: %s, ISOC %s, "
1673 			"SupDir IN: %s, OUT: %s\n",
1674 			pep->name, 1024,
1675 			str_yes_no(pep->endpoint.caps.type_control),
1676 			str_yes_no(pep->endpoint.caps.type_int),
1677 			str_yes_no(pep->endpoint.caps.type_bulk),
1678 			str_yes_no(pep->endpoint.caps.type_iso),
1679 			str_yes_no(pep->endpoint.caps.dir_in),
1680 			str_yes_no(pep->endpoint.caps.dir_out));
1681 
1682 		INIT_LIST_HEAD(&pep->pending_list);
1683 	}
1684 
1685 	return 0;
1686 }
1687 
1688 static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev)
1689 {
1690 	struct cdnsp_ep *pep;
1691 	int i;
1692 
1693 	for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1694 		pep = &pdev->eps[i];
1695 		if (pep->number != 0 && pep->out_ctx)
1696 			list_del(&pep->endpoint.ep_list);
1697 	}
1698 }
1699 
1700 void cdnsp_disconnect_gadget(struct cdnsp_device *pdev)
1701 {
1702 	pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING;
1703 
1704 	if (pdev->gadget_driver && pdev->gadget_driver->disconnect) {
1705 		spin_unlock(&pdev->lock);
1706 		pdev->gadget_driver->disconnect(&pdev->gadget);
1707 		spin_lock(&pdev->lock);
1708 	}
1709 
1710 	pdev->gadget.speed = USB_SPEED_UNKNOWN;
1711 	usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
1712 
1713 	pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING;
1714 }
1715 
1716 void cdnsp_suspend_gadget(struct cdnsp_device *pdev)
1717 {
1718 	if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1719 		spin_unlock(&pdev->lock);
1720 		pdev->gadget_driver->suspend(&pdev->gadget);
1721 		spin_lock(&pdev->lock);
1722 	}
1723 }
1724 
1725 void cdnsp_resume_gadget(struct cdnsp_device *pdev)
1726 {
1727 	if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1728 		spin_unlock(&pdev->lock);
1729 		pdev->gadget_driver->resume(&pdev->gadget);
1730 		spin_lock(&pdev->lock);
1731 	}
1732 }
1733 
1734 void cdnsp_irq_reset(struct cdnsp_device *pdev)
1735 {
1736 	struct cdnsp_port_regs __iomem *port_regs;
1737 
1738 	cdnsp_reset_device(pdev);
1739 
1740 	port_regs = pdev->active_port->regs;
1741 	pdev->gadget.speed = cdnsp_port_speed(readl(port_regs));
1742 
1743 	spin_unlock(&pdev->lock);
1744 	usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver);
1745 	spin_lock(&pdev->lock);
1746 
1747 	switch (pdev->gadget.speed) {
1748 	case USB_SPEED_SUPER_PLUS:
1749 	case USB_SPEED_SUPER:
1750 		cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1751 		pdev->gadget.ep0->maxpacket = 512;
1752 		break;
1753 	case USB_SPEED_HIGH:
1754 	case USB_SPEED_FULL:
1755 		cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1756 		pdev->gadget.ep0->maxpacket = 64;
1757 		break;
1758 	default:
1759 		/* Low speed is not supported. */
1760 		dev_err(pdev->dev, "Unknown device speed\n");
1761 		break;
1762 	}
1763 
1764 	cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1765 	cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY);
1766 	usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT);
1767 }
1768 
1769 static void cdnsp_get_rev_cap(struct cdnsp_device *pdev)
1770 {
1771 	void __iomem *reg = &pdev->cap_regs->hc_capbase;
1772 
1773 	reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP);
1774 	pdev->rev_cap  = reg;
1775 
1776 	dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n",
1777 		 readl(&pdev->rev_cap->ctrl_revision),
1778 		 readl(&pdev->rev_cap->rtl_revision),
1779 		 readl(&pdev->rev_cap->ep_supported),
1780 		 readl(&pdev->rev_cap->rx_buff_size),
1781 		 readl(&pdev->rev_cap->tx_buff_size));
1782 }
1783 
1784 static int cdnsp_gen_setup(struct cdnsp_device *pdev)
1785 {
1786 	int ret;
1787 	u32 reg;
1788 
1789 	pdev->cap_regs = pdev->regs;
1790 	pdev->op_regs = pdev->regs +
1791 		HC_LENGTH(readl(&pdev->cap_regs->hc_capbase));
1792 	pdev->run_regs = pdev->regs +
1793 		(readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK);
1794 
1795 	/* Cache read-only capability registers */
1796 	pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1);
1797 	pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase);
1798 	pdev->hci_version = HC_VERSION(pdev->hcc_params);
1799 	pdev->hcc_params = readl(&pdev->cap_regs->hcc_params);
1800 
1801 	cdnsp_get_rev_cap(pdev);
1802 
1803 	/* Make sure the Device Controller is halted. */
1804 	ret = cdnsp_halt(pdev);
1805 	if (ret)
1806 		return ret;
1807 
1808 	/* Reset the internal controller memory state and registers. */
1809 	ret = cdnsp_reset(pdev);
1810 	if (ret)
1811 		return ret;
1812 
1813 	/*
1814 	 * Set dma_mask and coherent_dma_mask to 64-bits,
1815 	 * if controller supports 64-bit addressing.
1816 	 */
1817 	if (HCC_64BIT_ADDR(pdev->hcc_params) &&
1818 	    !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) {
1819 		dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n");
1820 		dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64));
1821 	} else {
1822 		/*
1823 		 * This is to avoid error in cases where a 32-bit USB
1824 		 * controller is used on a 64-bit capable system.
1825 		 */
1826 		ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32));
1827 		if (ret)
1828 			return ret;
1829 
1830 		dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n");
1831 		dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32));
1832 	}
1833 
1834 	spin_lock_init(&pdev->lock);
1835 
1836 	ret = cdnsp_mem_init(pdev);
1837 	if (ret)
1838 		return ret;
1839 
1840 	/*
1841 	 * Software workaround for U1: after transition
1842 	 * to U1 the controller starts gating clock, and in some cases,
1843 	 * it causes that controller stack.
1844 	 */
1845 	reg = readl(&pdev->port3x_regs->mode_2);
1846 	reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN;
1847 	writel(reg, &pdev->port3x_regs->mode_2);
1848 
1849 	return 0;
1850 }
1851 
1852 static int __cdnsp_gadget_init(struct cdns *cdns)
1853 {
1854 	struct cdnsp_device *pdev;
1855 	u32 max_speed;
1856 	int ret = -ENOMEM;
1857 
1858 	cdns_drd_gadget_on(cdns);
1859 
1860 	pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
1861 	if (!pdev)
1862 		return -ENOMEM;
1863 
1864 	pm_runtime_get_sync(cdns->dev);
1865 
1866 	cdns->gadget_dev = pdev;
1867 	pdev->dev = cdns->dev;
1868 	pdev->regs = cdns->dev_regs;
1869 	max_speed = usb_get_maximum_speed(cdns->dev);
1870 
1871 	switch (max_speed) {
1872 	case USB_SPEED_FULL:
1873 	case USB_SPEED_HIGH:
1874 	case USB_SPEED_SUPER:
1875 	case USB_SPEED_SUPER_PLUS:
1876 		break;
1877 	default:
1878 		dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed);
1879 		fallthrough;
1880 	case USB_SPEED_UNKNOWN:
1881 		/* Default to SSP */
1882 		max_speed = USB_SPEED_SUPER_PLUS;
1883 		break;
1884 	}
1885 
1886 	pdev->gadget.ops = &cdnsp_gadget_ops;
1887 	pdev->gadget.name = "cdnsp-gadget";
1888 	pdev->gadget.speed = USB_SPEED_UNKNOWN;
1889 	pdev->gadget.sg_supported = 1;
1890 	pdev->gadget.max_speed = max_speed;
1891 	pdev->gadget.lpm_capable = 1;
1892 
1893 	pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL);
1894 	if (!pdev->setup_buf)
1895 		goto free_pdev;
1896 
1897 	/*
1898 	 * Controller supports not aligned buffer but it should improve
1899 	 * performance.
1900 	 */
1901 	pdev->gadget.quirk_ep_out_aligned_size = true;
1902 
1903 	ret = cdnsp_gen_setup(pdev);
1904 	if (ret) {
1905 		dev_err(pdev->dev, "Generic initialization failed %d\n", ret);
1906 		goto free_setup;
1907 	}
1908 
1909 	ret = cdnsp_gadget_init_endpoints(pdev);
1910 	if (ret) {
1911 		dev_err(pdev->dev, "failed to initialize endpoints\n");
1912 		goto halt_pdev;
1913 	}
1914 
1915 	ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget);
1916 	if (ret) {
1917 		dev_err(pdev->dev, "failed to register udc\n");
1918 		goto free_endpoints;
1919 	}
1920 
1921 	ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq,
1922 					cdnsp_irq_handler,
1923 					cdnsp_thread_irq_handler, IRQF_SHARED,
1924 					dev_name(pdev->dev), pdev);
1925 	if (ret)
1926 		goto del_gadget;
1927 
1928 	return 0;
1929 
1930 del_gadget:
1931 	usb_del_gadget_udc(&pdev->gadget);
1932 free_endpoints:
1933 	cdnsp_gadget_free_endpoints(pdev);
1934 halt_pdev:
1935 	cdnsp_halt(pdev);
1936 	cdnsp_reset(pdev);
1937 	cdnsp_mem_cleanup(pdev);
1938 free_setup:
1939 	kfree(pdev->setup_buf);
1940 free_pdev:
1941 	kfree(pdev);
1942 
1943 	return ret;
1944 }
1945 
1946 static void cdnsp_gadget_exit(struct cdns *cdns)
1947 {
1948 	struct cdnsp_device *pdev = cdns->gadget_dev;
1949 
1950 	devm_free_irq(pdev->dev, cdns->dev_irq, pdev);
1951 	pm_runtime_mark_last_busy(cdns->dev);
1952 	pm_runtime_put_autosuspend(cdns->dev);
1953 	usb_del_gadget_udc(&pdev->gadget);
1954 	cdnsp_gadget_free_endpoints(pdev);
1955 	cdnsp_mem_cleanup(pdev);
1956 	kfree(pdev);
1957 	cdns->gadget_dev = NULL;
1958 	cdns_drd_gadget_off(cdns);
1959 }
1960 
1961 static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup)
1962 {
1963 	struct cdnsp_device *pdev = cdns->gadget_dev;
1964 	unsigned long flags;
1965 
1966 	if (pdev->link_state == XDEV_U3)
1967 		return 0;
1968 
1969 	spin_lock_irqsave(&pdev->lock, flags);
1970 	cdnsp_disconnect_gadget(pdev);
1971 	cdnsp_stop(pdev);
1972 	spin_unlock_irqrestore(&pdev->lock, flags);
1973 
1974 	return 0;
1975 }
1976 
1977 static int cdnsp_gadget_resume(struct cdns *cdns, bool hibernated)
1978 {
1979 	struct cdnsp_device *pdev = cdns->gadget_dev;
1980 	enum usb_device_speed max_speed;
1981 	unsigned long flags;
1982 	int ret;
1983 
1984 	if (!pdev->gadget_driver)
1985 		return 0;
1986 
1987 	spin_lock_irqsave(&pdev->lock, flags);
1988 	max_speed = pdev->gadget_driver->max_speed;
1989 
1990 	/* Limit speed if necessary. */
1991 	max_speed = min(max_speed, pdev->gadget.max_speed);
1992 
1993 	ret = cdnsp_run(pdev, max_speed);
1994 
1995 	if (pdev->link_state == XDEV_U3)
1996 		__cdnsp_gadget_wakeup(pdev);
1997 
1998 	spin_unlock_irqrestore(&pdev->lock, flags);
1999 
2000 	return ret;
2001 }
2002 
2003 /**
2004  * cdnsp_gadget_init - initialize device structure
2005  * @cdns: cdnsp instance
2006  *
2007  * This function initializes the gadget.
2008  */
2009 int cdnsp_gadget_init(struct cdns *cdns)
2010 {
2011 	struct cdns_role_driver *rdrv;
2012 
2013 	rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2014 	if (!rdrv)
2015 		return -ENOMEM;
2016 
2017 	rdrv->start	= __cdnsp_gadget_init;
2018 	rdrv->stop	= cdnsp_gadget_exit;
2019 	rdrv->suspend	= cdnsp_gadget_suspend;
2020 	rdrv->resume	= cdnsp_gadget_resume;
2021 	rdrv->state	= CDNS_ROLE_STATE_INACTIVE;
2022 	rdrv->name	= "gadget";
2023 	cdns->roles[USB_ROLE_DEVICE] = rdrv;
2024 
2025 	return 0;
2026 }
2027