xref: /linux/drivers/usb/dwc2/hcd_ddma.c (revision b82779648dfd3814df4e381f086326ec70fd791f)
1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /*
3  * hcd_ddma.c - DesignWare HS OTG Controller descriptor DMA routines
4  *
5  * Copyright (C) 2004-2013 Synopsys, Inc.
6  */
7 
8 /*
9  * This file contains the Descriptor DMA implementation for Host mode
10  */
11 #include <linux/kernel.h>
12 #include <linux/module.h>
13 #include <linux/spinlock.h>
14 #include <linux/interrupt.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/io.h>
17 #include <linux/slab.h>
18 #include <linux/usb.h>
19 
20 #include <linux/usb/hcd.h>
21 #include <linux/usb/ch11.h>
22 
23 #include "core.h"
24 #include "hcd.h"
25 
dwc2_frame_list_idx(u16 frame)26 static u16 dwc2_frame_list_idx(u16 frame)
27 {
28 	return frame & (FRLISTEN_64_SIZE - 1);
29 }
30 
dwc2_desclist_idx_inc(u16 idx,u16 inc,u8 speed)31 static u16 dwc2_desclist_idx_inc(u16 idx, u16 inc, u8 speed)
32 {
33 	return (idx + inc) &
34 		((speed == USB_SPEED_HIGH ? MAX_DMA_DESC_NUM_HS_ISOC :
35 		  MAX_DMA_DESC_NUM_GENERIC) - 1);
36 }
37 
dwc2_desclist_idx_dec(u16 idx,u16 inc,u8 speed)38 static u16 dwc2_desclist_idx_dec(u16 idx, u16 inc, u8 speed)
39 {
40 	return (idx - inc) &
41 		((speed == USB_SPEED_HIGH ? MAX_DMA_DESC_NUM_HS_ISOC :
42 		  MAX_DMA_DESC_NUM_GENERIC) - 1);
43 }
44 
dwc2_max_desc_num(struct dwc2_qh * qh)45 static u16 dwc2_max_desc_num(struct dwc2_qh *qh)
46 {
47 	return (qh->ep_type == USB_ENDPOINT_XFER_ISOC &&
48 		qh->dev_speed == USB_SPEED_HIGH) ?
49 		MAX_DMA_DESC_NUM_HS_ISOC : MAX_DMA_DESC_NUM_GENERIC;
50 }
51 
dwc2_frame_incr_val(struct dwc2_qh * qh)52 static u16 dwc2_frame_incr_val(struct dwc2_qh *qh)
53 {
54 	return qh->dev_speed == USB_SPEED_HIGH ?
55 	       (qh->host_interval + 8 - 1) / 8 : qh->host_interval;
56 }
57 
dwc2_desc_list_alloc(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,gfp_t flags)58 static int dwc2_desc_list_alloc(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
59 				gfp_t flags)
60 {
61 	struct kmem_cache *desc_cache;
62 
63 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC &&
64 	    qh->dev_speed == USB_SPEED_HIGH)
65 		desc_cache = hsotg->desc_hsisoc_cache;
66 	else
67 		desc_cache = hsotg->desc_gen_cache;
68 
69 	qh->desc_list_sz = sizeof(struct dwc2_dma_desc) *
70 						dwc2_max_desc_num(qh);
71 
72 	qh->desc_list = kmem_cache_zalloc(desc_cache, flags | GFP_DMA);
73 	if (!qh->desc_list)
74 		return -ENOMEM;
75 
76 	qh->desc_list_dma = dma_map_single(hsotg->dev, qh->desc_list,
77 					   qh->desc_list_sz,
78 					   DMA_TO_DEVICE);
79 
80 	qh->n_bytes = kcalloc(dwc2_max_desc_num(qh), sizeof(u32), flags);
81 	if (!qh->n_bytes) {
82 		dma_unmap_single(hsotg->dev, qh->desc_list_dma,
83 				 qh->desc_list_sz,
84 				 DMA_FROM_DEVICE);
85 		kmem_cache_free(desc_cache, qh->desc_list);
86 		qh->desc_list = NULL;
87 		return -ENOMEM;
88 	}
89 
90 	return 0;
91 }
92 
dwc2_desc_list_free(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)93 static void dwc2_desc_list_free(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
94 {
95 	struct kmem_cache *desc_cache;
96 
97 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC &&
98 	    qh->dev_speed == USB_SPEED_HIGH)
99 		desc_cache = hsotg->desc_hsisoc_cache;
100 	else
101 		desc_cache = hsotg->desc_gen_cache;
102 
103 	if (qh->desc_list) {
104 		dma_unmap_single(hsotg->dev, qh->desc_list_dma,
105 				 qh->desc_list_sz, DMA_FROM_DEVICE);
106 		kmem_cache_free(desc_cache, qh->desc_list);
107 		qh->desc_list = NULL;
108 	}
109 
110 	kfree(qh->n_bytes);
111 	qh->n_bytes = NULL;
112 }
113 
dwc2_frame_list_alloc(struct dwc2_hsotg * hsotg,gfp_t mem_flags)114 static int dwc2_frame_list_alloc(struct dwc2_hsotg *hsotg, gfp_t mem_flags)
115 {
116 	if (hsotg->frame_list)
117 		return 0;
118 
119 	hsotg->frame_list_sz = 4 * FRLISTEN_64_SIZE;
120 	hsotg->frame_list = kzalloc(hsotg->frame_list_sz, GFP_ATOMIC | GFP_DMA);
121 	if (!hsotg->frame_list)
122 		return -ENOMEM;
123 
124 	hsotg->frame_list_dma = dma_map_single(hsotg->dev, hsotg->frame_list,
125 					       hsotg->frame_list_sz,
126 					       DMA_TO_DEVICE);
127 
128 	return 0;
129 }
130 
dwc2_frame_list_free(struct dwc2_hsotg * hsotg)131 static void dwc2_frame_list_free(struct dwc2_hsotg *hsotg)
132 {
133 	unsigned long flags;
134 
135 	spin_lock_irqsave(&hsotg->lock, flags);
136 
137 	if (!hsotg->frame_list) {
138 		spin_unlock_irqrestore(&hsotg->lock, flags);
139 		return;
140 	}
141 
142 	dma_unmap_single(hsotg->dev, hsotg->frame_list_dma,
143 			 hsotg->frame_list_sz, DMA_FROM_DEVICE);
144 
145 	kfree(hsotg->frame_list);
146 	hsotg->frame_list = NULL;
147 
148 	spin_unlock_irqrestore(&hsotg->lock, flags);
149 }
150 
dwc2_per_sched_enable(struct dwc2_hsotg * hsotg,u32 fr_list_en)151 static void dwc2_per_sched_enable(struct dwc2_hsotg *hsotg, u32 fr_list_en)
152 {
153 	u32 hcfg;
154 	unsigned long flags;
155 
156 	spin_lock_irqsave(&hsotg->lock, flags);
157 
158 	hcfg = dwc2_readl(hsotg, HCFG);
159 	if (hcfg & HCFG_PERSCHEDENA) {
160 		/* already enabled */
161 		spin_unlock_irqrestore(&hsotg->lock, flags);
162 		return;
163 	}
164 
165 	dwc2_writel(hsotg, hsotg->frame_list_dma, HFLBADDR);
166 
167 	hcfg &= ~HCFG_FRLISTEN_MASK;
168 	hcfg |= fr_list_en | HCFG_PERSCHEDENA;
169 	dev_vdbg(hsotg->dev, "Enabling Periodic schedule\n");
170 	dwc2_writel(hsotg, hcfg, HCFG);
171 
172 	spin_unlock_irqrestore(&hsotg->lock, flags);
173 }
174 
dwc2_per_sched_disable(struct dwc2_hsotg * hsotg)175 static void dwc2_per_sched_disable(struct dwc2_hsotg *hsotg)
176 {
177 	u32 hcfg;
178 	unsigned long flags;
179 
180 	spin_lock_irqsave(&hsotg->lock, flags);
181 
182 	hcfg = dwc2_readl(hsotg, HCFG);
183 	if (!(hcfg & HCFG_PERSCHEDENA)) {
184 		/* already disabled */
185 		spin_unlock_irqrestore(&hsotg->lock, flags);
186 		return;
187 	}
188 
189 	hcfg &= ~HCFG_PERSCHEDENA;
190 	dev_vdbg(hsotg->dev, "Disabling Periodic schedule\n");
191 	dwc2_writel(hsotg, hcfg, HCFG);
192 
193 	spin_unlock_irqrestore(&hsotg->lock, flags);
194 }
195 
196 /*
197  * Activates/Deactivates FrameList entries for the channel based on endpoint
198  * servicing period
199  */
dwc2_update_frame_list(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,int enable)200 static void dwc2_update_frame_list(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
201 				   int enable)
202 {
203 	struct dwc2_host_chan *chan;
204 	u16 i, j, inc;
205 
206 	if (!hsotg) {
207 		pr_err("hsotg = %p\n", hsotg);
208 		return;
209 	}
210 
211 	if (!qh->channel) {
212 		dev_err(hsotg->dev, "qh->channel = %p\n", qh->channel);
213 		return;
214 	}
215 
216 	if (!hsotg->frame_list) {
217 		dev_err(hsotg->dev, "hsotg->frame_list = %p\n",
218 			hsotg->frame_list);
219 		return;
220 	}
221 
222 	chan = qh->channel;
223 	inc = dwc2_frame_incr_val(qh);
224 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC)
225 		i = dwc2_frame_list_idx(qh->next_active_frame);
226 	else
227 		i = 0;
228 
229 	j = i;
230 	do {
231 		if (enable)
232 			hsotg->frame_list[j] |= 1 << chan->hc_num;
233 		else
234 			hsotg->frame_list[j] &= ~(1 << chan->hc_num);
235 		j = (j + inc) & (FRLISTEN_64_SIZE - 1);
236 	} while (j != i);
237 
238 	/*
239 	 * Sync frame list since controller will access it if periodic
240 	 * channel is currently enabled.
241 	 */
242 	dma_sync_single_for_device(hsotg->dev,
243 				   hsotg->frame_list_dma,
244 				   hsotg->frame_list_sz,
245 				   DMA_TO_DEVICE);
246 
247 	if (!enable)
248 		return;
249 
250 	chan->schinfo = 0;
251 	if (chan->speed == USB_SPEED_HIGH && qh->host_interval) {
252 		j = 1;
253 		/* TODO - check this */
254 		inc = (8 + qh->host_interval - 1) / qh->host_interval;
255 		for (i = 0; i < inc; i++) {
256 			chan->schinfo |= j;
257 			j = j << qh->host_interval;
258 		}
259 	} else {
260 		chan->schinfo = 0xff;
261 	}
262 }
263 
dwc2_release_channel_ddma(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)264 static void dwc2_release_channel_ddma(struct dwc2_hsotg *hsotg,
265 				      struct dwc2_qh *qh)
266 {
267 	struct dwc2_host_chan *chan = qh->channel;
268 
269 	if (dwc2_qh_is_non_per(qh)) {
270 		if (hsotg->params.uframe_sched)
271 			hsotg->available_host_channels++;
272 		else
273 			hsotg->non_periodic_channels--;
274 	} else {
275 		dwc2_update_frame_list(hsotg, qh, 0);
276 		hsotg->available_host_channels++;
277 	}
278 
279 	/*
280 	 * The condition is added to prevent double cleanup try in case of
281 	 * device disconnect. See channel cleanup in dwc2_hcd_disconnect().
282 	 */
283 	if (chan->qh) {
284 		if (!list_empty(&chan->hc_list_entry))
285 			list_del(&chan->hc_list_entry);
286 		dwc2_hc_cleanup(hsotg, chan);
287 		list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
288 		chan->qh = NULL;
289 	}
290 
291 	qh->channel = NULL;
292 	qh->ntd = 0;
293 
294 	if (qh->desc_list)
295 		memset(qh->desc_list, 0, sizeof(struct dwc2_dma_desc) *
296 		       dwc2_max_desc_num(qh));
297 }
298 
299 /**
300  * dwc2_hcd_qh_init_ddma() - Initializes a QH structure's Descriptor DMA
301  * related members
302  *
303  * @hsotg: The HCD state structure for the DWC OTG controller
304  * @qh:    The QH to init
305  * @mem_flags: Indicates the type of memory allocation
306  *
307  * Return: 0 if successful, negative error code otherwise
308  *
309  * Allocates memory for the descriptor list. For the first periodic QH,
310  * allocates memory for the FrameList and enables periodic scheduling.
311  */
dwc2_hcd_qh_init_ddma(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,gfp_t mem_flags)312 int dwc2_hcd_qh_init_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
313 			  gfp_t mem_flags)
314 {
315 	int retval;
316 
317 	if (qh->do_split) {
318 		dev_err(hsotg->dev,
319 			"SPLIT Transfers are not supported in Descriptor DMA mode.\n");
320 		retval = -EINVAL;
321 		goto err0;
322 	}
323 
324 	retval = dwc2_desc_list_alloc(hsotg, qh, mem_flags);
325 	if (retval)
326 		goto err0;
327 
328 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC ||
329 	    qh->ep_type == USB_ENDPOINT_XFER_INT) {
330 		if (!hsotg->frame_list) {
331 			retval = dwc2_frame_list_alloc(hsotg, mem_flags);
332 			if (retval)
333 				goto err1;
334 			/* Enable periodic schedule on first periodic QH */
335 			dwc2_per_sched_enable(hsotg, HCFG_FRLISTEN_64);
336 		}
337 	}
338 
339 	qh->ntd = 0;
340 	return 0;
341 
342 err1:
343 	dwc2_desc_list_free(hsotg, qh);
344 err0:
345 	return retval;
346 }
347 
348 /**
349  * dwc2_hcd_qh_free_ddma() - Frees a QH structure's Descriptor DMA related
350  * members
351  *
352  * @hsotg: The HCD state structure for the DWC OTG controller
353  * @qh:    The QH to free
354  *
355  * Frees descriptor list memory associated with the QH. If QH is periodic and
356  * the last, frees FrameList memory and disables periodic scheduling.
357  */
dwc2_hcd_qh_free_ddma(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)358 void dwc2_hcd_qh_free_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
359 {
360 	unsigned long flags;
361 
362 	dwc2_desc_list_free(hsotg, qh);
363 
364 	/*
365 	 * Channel still assigned due to some reasons.
366 	 * Seen on Isoc URB dequeue. Channel halted but no subsequent
367 	 * ChHalted interrupt to release the channel. Afterwards
368 	 * when it comes here from endpoint disable routine
369 	 * channel remains assigned.
370 	 */
371 	spin_lock_irqsave(&hsotg->lock, flags);
372 	if (qh->channel)
373 		dwc2_release_channel_ddma(hsotg, qh);
374 	spin_unlock_irqrestore(&hsotg->lock, flags);
375 
376 	if ((qh->ep_type == USB_ENDPOINT_XFER_ISOC ||
377 	     qh->ep_type == USB_ENDPOINT_XFER_INT) &&
378 	    (hsotg->params.uframe_sched ||
379 	     !hsotg->periodic_channels) && hsotg->frame_list) {
380 		dwc2_per_sched_disable(hsotg);
381 		dwc2_frame_list_free(hsotg);
382 	}
383 }
384 
dwc2_frame_to_desc_idx(struct dwc2_qh * qh,u16 frame_idx)385 static u8 dwc2_frame_to_desc_idx(struct dwc2_qh *qh, u16 frame_idx)
386 {
387 	if (qh->dev_speed == USB_SPEED_HIGH)
388 		/* Descriptor set (8 descriptors) index which is 8-aligned */
389 		return (frame_idx & ((MAX_DMA_DESC_NUM_HS_ISOC / 8) - 1)) * 8;
390 	else
391 		return frame_idx & (MAX_DMA_DESC_NUM_GENERIC - 1);
392 }
393 
394 /*
395  * Determine starting frame for Isochronous transfer.
396  * Few frames skipped to prevent race condition with HC.
397  */
dwc2_calc_starting_frame(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,u16 * skip_frames)398 static u16 dwc2_calc_starting_frame(struct dwc2_hsotg *hsotg,
399 				    struct dwc2_qh *qh, u16 *skip_frames)
400 {
401 	u16 frame;
402 
403 	hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
404 
405 	/*
406 	 * next_active_frame is always frame number (not uFrame) both in FS
407 	 * and HS!
408 	 */
409 
410 	/*
411 	 * skip_frames is used to limit activated descriptors number
412 	 * to avoid the situation when HC services the last activated
413 	 * descriptor firstly.
414 	 * Example for FS:
415 	 * Current frame is 1, scheduled frame is 3. Since HC always fetches
416 	 * the descriptor corresponding to curr_frame+1, the descriptor
417 	 * corresponding to frame 2 will be fetched. If the number of
418 	 * descriptors is max=64 (or greather) the list will be fully programmed
419 	 * with Active descriptors and it is possible case (rare) that the
420 	 * latest descriptor(considering rollback) corresponding to frame 2 will
421 	 * be serviced first. HS case is more probable because, in fact, up to
422 	 * 11 uframes (16 in the code) may be skipped.
423 	 */
424 	if (qh->dev_speed == USB_SPEED_HIGH) {
425 		/*
426 		 * Consider uframe counter also, to start xfer asap. If half of
427 		 * the frame elapsed skip 2 frames otherwise just 1 frame.
428 		 * Starting descriptor index must be 8-aligned, so if the
429 		 * current frame is near to complete the next one is skipped as
430 		 * well.
431 		 */
432 		if (dwc2_micro_frame_num(hsotg->frame_number) >= 5) {
433 			*skip_frames = 2 * 8;
434 			frame = dwc2_frame_num_inc(hsotg->frame_number,
435 						   *skip_frames);
436 		} else {
437 			*skip_frames = 1 * 8;
438 			frame = dwc2_frame_num_inc(hsotg->frame_number,
439 						   *skip_frames);
440 		}
441 
442 		frame = dwc2_full_frame_num(frame);
443 	} else {
444 		/*
445 		 * Two frames are skipped for FS - the current and the next.
446 		 * But for descriptor programming, 1 frame (descriptor) is
447 		 * enough, see example above.
448 		 */
449 		*skip_frames = 1;
450 		frame = dwc2_frame_num_inc(hsotg->frame_number, 2);
451 	}
452 
453 	return frame;
454 }
455 
456 /*
457  * Calculate initial descriptor index for isochronous transfer based on
458  * scheduled frame
459  */
dwc2_recalc_initial_desc_idx(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)460 static u16 dwc2_recalc_initial_desc_idx(struct dwc2_hsotg *hsotg,
461 					struct dwc2_qh *qh)
462 {
463 	u16 frame, fr_idx, fr_idx_tmp, skip_frames;
464 
465 	/*
466 	 * With current ISOC processing algorithm the channel is being released
467 	 * when no more QTDs in the list (qh->ntd == 0). Thus this function is
468 	 * called only when qh->ntd == 0 and qh->channel == 0.
469 	 *
470 	 * So qh->channel != NULL branch is not used and just not removed from
471 	 * the source file. It is required for another possible approach which
472 	 * is, do not disable and release the channel when ISOC session
473 	 * completed, just move QH to inactive schedule until new QTD arrives.
474 	 * On new QTD, the QH moved back to 'ready' schedule, starting frame and
475 	 * therefore starting desc_index are recalculated. In this case channel
476 	 * is released only on ep_disable.
477 	 */
478 
479 	/*
480 	 * Calculate starting descriptor index. For INTERRUPT endpoint it is
481 	 * always 0.
482 	 */
483 	if (qh->channel) {
484 		frame = dwc2_calc_starting_frame(hsotg, qh, &skip_frames);
485 		/*
486 		 * Calculate initial descriptor index based on FrameList current
487 		 * bitmap and servicing period
488 		 */
489 		fr_idx_tmp = dwc2_frame_list_idx(frame);
490 		fr_idx = (FRLISTEN_64_SIZE +
491 			  dwc2_frame_list_idx(qh->next_active_frame) -
492 			  fr_idx_tmp) % dwc2_frame_incr_val(qh);
493 		fr_idx = (fr_idx + fr_idx_tmp) % FRLISTEN_64_SIZE;
494 	} else {
495 		qh->next_active_frame = dwc2_calc_starting_frame(hsotg, qh,
496 							   &skip_frames);
497 		fr_idx = dwc2_frame_list_idx(qh->next_active_frame);
498 	}
499 
500 	qh->td_first = qh->td_last = dwc2_frame_to_desc_idx(qh, fr_idx);
501 
502 	return skip_frames;
503 }
504 
505 #define ISOC_URB_GIVEBACK_ASAP
506 
507 #define MAX_ISOC_XFER_SIZE_FS	1023
508 #define MAX_ISOC_XFER_SIZE_HS	3072
509 #define DESCNUM_THRESHOLD	4
510 
dwc2_fill_host_isoc_dma_desc(struct dwc2_hsotg * hsotg,struct dwc2_qtd * qtd,struct dwc2_qh * qh,u32 max_xfer_size,u16 idx)511 static void dwc2_fill_host_isoc_dma_desc(struct dwc2_hsotg *hsotg,
512 					 struct dwc2_qtd *qtd,
513 					 struct dwc2_qh *qh, u32 max_xfer_size,
514 					 u16 idx)
515 {
516 	struct dwc2_dma_desc *dma_desc = &qh->desc_list[idx];
517 	struct dwc2_hcd_iso_packet_desc *frame_desc;
518 
519 	memset(dma_desc, 0, sizeof(*dma_desc));
520 	frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index_last];
521 
522 	if (frame_desc->length > max_xfer_size)
523 		qh->n_bytes[idx] = max_xfer_size;
524 	else
525 		qh->n_bytes[idx] = frame_desc->length;
526 
527 	dma_desc->buf = (u32)(qtd->urb->dma + frame_desc->offset);
528 	dma_desc->status = qh->n_bytes[idx] << HOST_DMA_ISOC_NBYTES_SHIFT &
529 			   HOST_DMA_ISOC_NBYTES_MASK;
530 
531 	/* Set active bit */
532 	dma_desc->status |= HOST_DMA_A;
533 
534 	qh->ntd++;
535 	qtd->isoc_frame_index_last++;
536 
537 #ifdef ISOC_URB_GIVEBACK_ASAP
538 	/* Set IOC for each descriptor corresponding to last frame of URB */
539 	if (qtd->isoc_frame_index_last == qtd->urb->packet_count)
540 		dma_desc->status |= HOST_DMA_IOC;
541 #endif
542 
543 	dma_sync_single_for_device(hsotg->dev,
544 				   qh->desc_list_dma +
545 			(idx * sizeof(struct dwc2_dma_desc)),
546 			sizeof(struct dwc2_dma_desc),
547 			DMA_TO_DEVICE);
548 }
549 
dwc2_init_isoc_dma_desc(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh,u16 skip_frames)550 static void dwc2_init_isoc_dma_desc(struct dwc2_hsotg *hsotg,
551 				    struct dwc2_qh *qh, u16 skip_frames)
552 {
553 	struct dwc2_qtd *qtd;
554 	u32 max_xfer_size;
555 	u16 idx, inc, n_desc = 0, ntd_max = 0;
556 	u16 cur_idx;
557 	u16 next_idx;
558 
559 	idx = qh->td_last;
560 	inc = qh->host_interval;
561 	hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
562 	cur_idx = idx;
563 	next_idx = dwc2_desclist_idx_inc(qh->td_last, inc, qh->dev_speed);
564 
565 	/*
566 	 * Ensure current frame number didn't overstep last scheduled
567 	 * descriptor. If it happens, the only way to recover is to move
568 	 * qh->td_last to current frame number + 1.
569 	 * So that next isoc descriptor will be scheduled on frame number + 1
570 	 * and not on a past frame.
571 	 */
572 	if (dwc2_frame_idx_num_gt(cur_idx, next_idx) || (cur_idx == next_idx)) {
573 		if (inc < 32) {
574 			dev_vdbg(hsotg->dev,
575 				 "current frame number overstep last descriptor\n");
576 			qh->td_last = dwc2_desclist_idx_inc(cur_idx, inc,
577 							    qh->dev_speed);
578 			idx = qh->td_last;
579 		}
580 	}
581 
582 	if (qh->host_interval) {
583 		ntd_max = (dwc2_max_desc_num(qh) + qh->host_interval - 1) /
584 				qh->host_interval;
585 		if (skip_frames && !qh->channel)
586 			ntd_max -= skip_frames / qh->host_interval;
587 	}
588 
589 	max_xfer_size = qh->dev_speed == USB_SPEED_HIGH ?
590 			MAX_ISOC_XFER_SIZE_HS : MAX_ISOC_XFER_SIZE_FS;
591 
592 	list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry) {
593 		if (qtd->in_process &&
594 		    qtd->isoc_frame_index_last ==
595 		    qtd->urb->packet_count)
596 			continue;
597 
598 		qtd->isoc_td_first = idx;
599 		while (qh->ntd < ntd_max && qtd->isoc_frame_index_last <
600 						qtd->urb->packet_count) {
601 			dwc2_fill_host_isoc_dma_desc(hsotg, qtd, qh,
602 						     max_xfer_size, idx);
603 			idx = dwc2_desclist_idx_inc(idx, inc, qh->dev_speed);
604 			n_desc++;
605 		}
606 		qtd->isoc_td_last = idx;
607 		qtd->in_process = 1;
608 	}
609 
610 	qh->td_last = idx;
611 
612 #ifdef ISOC_URB_GIVEBACK_ASAP
613 	/* Set IOC for last descriptor if descriptor list is full */
614 	if (qh->ntd == ntd_max) {
615 		idx = dwc2_desclist_idx_dec(qh->td_last, inc, qh->dev_speed);
616 		qh->desc_list[idx].status |= HOST_DMA_IOC;
617 		dma_sync_single_for_device(hsotg->dev,
618 					   qh->desc_list_dma + (idx *
619 					   sizeof(struct dwc2_dma_desc)),
620 					   sizeof(struct dwc2_dma_desc),
621 					   DMA_TO_DEVICE);
622 	}
623 #else
624 	/*
625 	 * Set IOC bit only for one descriptor. Always try to be ahead of HW
626 	 * processing, i.e. on IOC generation driver activates next descriptor
627 	 * but core continues to process descriptors following the one with IOC
628 	 * set.
629 	 */
630 
631 	if (n_desc > DESCNUM_THRESHOLD)
632 		/*
633 		 * Move IOC "up". Required even if there is only one QTD
634 		 * in the list, because QTDs might continue to be queued,
635 		 * but during the activation it was only one queued.
636 		 * Actually more than one QTD might be in the list if this
637 		 * function called from XferCompletion - QTDs was queued during
638 		 * HW processing of the previous descriptor chunk.
639 		 */
640 		idx = dwc2_desclist_idx_dec(idx, inc * ((qh->ntd + 1) / 2),
641 					    qh->dev_speed);
642 	else
643 		/*
644 		 * Set the IOC for the latest descriptor if either number of
645 		 * descriptors is not greater than threshold or no more new
646 		 * descriptors activated
647 		 */
648 		idx = dwc2_desclist_idx_dec(qh->td_last, inc, qh->dev_speed);
649 
650 	qh->desc_list[idx].status |= HOST_DMA_IOC;
651 	dma_sync_single_for_device(hsotg->dev,
652 				   qh->desc_list_dma +
653 				   (idx * sizeof(struct dwc2_dma_desc)),
654 				   sizeof(struct dwc2_dma_desc),
655 				   DMA_TO_DEVICE);
656 #endif
657 }
658 
dwc2_fill_host_dma_desc(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,struct dwc2_qtd * qtd,struct dwc2_qh * qh,int n_desc)659 static void dwc2_fill_host_dma_desc(struct dwc2_hsotg *hsotg,
660 				    struct dwc2_host_chan *chan,
661 				    struct dwc2_qtd *qtd, struct dwc2_qh *qh,
662 				    int n_desc)
663 {
664 	struct dwc2_dma_desc *dma_desc = &qh->desc_list[n_desc];
665 	int len = chan->xfer_len;
666 
667 	if (len > HOST_DMA_NBYTES_LIMIT - (chan->max_packet - 1))
668 		len = HOST_DMA_NBYTES_LIMIT - (chan->max_packet - 1);
669 
670 	if (chan->ep_is_in) {
671 		int num_packets;
672 
673 		if (len > 0 && chan->max_packet)
674 			num_packets = (len + chan->max_packet - 1)
675 					/ chan->max_packet;
676 		else
677 			/* Need 1 packet for transfer length of 0 */
678 			num_packets = 1;
679 
680 		/* Always program an integral # of packets for IN transfers */
681 		len = num_packets * chan->max_packet;
682 	}
683 
684 	dma_desc->status = len << HOST_DMA_NBYTES_SHIFT & HOST_DMA_NBYTES_MASK;
685 	qh->n_bytes[n_desc] = len;
686 
687 	if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL &&
688 	    qtd->control_phase == DWC2_CONTROL_SETUP)
689 		dma_desc->status |= HOST_DMA_SUP;
690 
691 	dma_desc->buf = (u32)chan->xfer_dma;
692 
693 	dma_sync_single_for_device(hsotg->dev,
694 				   qh->desc_list_dma +
695 				   (n_desc * sizeof(struct dwc2_dma_desc)),
696 				   sizeof(struct dwc2_dma_desc),
697 				   DMA_TO_DEVICE);
698 
699 	/*
700 	 * Last (or only) descriptor of IN transfer with actual size less
701 	 * than MaxPacket
702 	 */
703 	if (len > chan->xfer_len) {
704 		chan->xfer_len = 0;
705 	} else {
706 		chan->xfer_dma += len;
707 		chan->xfer_len -= len;
708 	}
709 }
710 
dwc2_init_non_isoc_dma_desc(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)711 static void dwc2_init_non_isoc_dma_desc(struct dwc2_hsotg *hsotg,
712 					struct dwc2_qh *qh)
713 {
714 	struct dwc2_qtd *qtd;
715 	struct dwc2_host_chan *chan = qh->channel;
716 	int n_desc = 0;
717 
718 	dev_vdbg(hsotg->dev, "%s(): qh=%p dma=%08lx len=%d\n", __func__, qh,
719 		 (unsigned long)chan->xfer_dma, chan->xfer_len);
720 
721 	/*
722 	 * Start with chan->xfer_dma initialized in assign_and_init_hc(), then
723 	 * if SG transfer consists of multiple URBs, this pointer is re-assigned
724 	 * to the buffer of the currently processed QTD. For non-SG request
725 	 * there is always one QTD active.
726 	 */
727 
728 	list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry) {
729 		dev_vdbg(hsotg->dev, "qtd=%p\n", qtd);
730 
731 		if (n_desc) {
732 			/* SG request - more than 1 QTD */
733 			chan->xfer_dma = qtd->urb->dma +
734 					qtd->urb->actual_length;
735 			chan->xfer_len = qtd->urb->length -
736 					qtd->urb->actual_length;
737 			dev_vdbg(hsotg->dev, "buf=%08lx len=%d\n",
738 				 (unsigned long)chan->xfer_dma, chan->xfer_len);
739 		}
740 
741 		qtd->n_desc = 0;
742 		do {
743 			if (n_desc > 1) {
744 				qh->desc_list[n_desc - 1].status |= HOST_DMA_A;
745 				dev_vdbg(hsotg->dev,
746 					 "set A bit in desc %d (%p)\n",
747 					 n_desc - 1,
748 					 &qh->desc_list[n_desc - 1]);
749 				dma_sync_single_for_device(hsotg->dev,
750 							   qh->desc_list_dma +
751 					((n_desc - 1) *
752 					sizeof(struct dwc2_dma_desc)),
753 					sizeof(struct dwc2_dma_desc),
754 					DMA_TO_DEVICE);
755 			}
756 			dwc2_fill_host_dma_desc(hsotg, chan, qtd, qh, n_desc);
757 			dev_vdbg(hsotg->dev,
758 				 "desc %d (%p) buf=%08x status=%08x\n",
759 				 n_desc, &qh->desc_list[n_desc],
760 				 qh->desc_list[n_desc].buf,
761 				 qh->desc_list[n_desc].status);
762 			qtd->n_desc++;
763 			n_desc++;
764 		} while (chan->xfer_len > 0 &&
765 			 n_desc != MAX_DMA_DESC_NUM_GENERIC);
766 
767 		dev_vdbg(hsotg->dev, "n_desc=%d\n", n_desc);
768 		qtd->in_process = 1;
769 		if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL)
770 			break;
771 		if (n_desc == MAX_DMA_DESC_NUM_GENERIC)
772 			break;
773 	}
774 
775 	if (n_desc) {
776 		qh->desc_list[n_desc - 1].status |=
777 				HOST_DMA_IOC | HOST_DMA_EOL | HOST_DMA_A;
778 		dev_vdbg(hsotg->dev, "set IOC/EOL/A bits in desc %d (%p)\n",
779 			 n_desc - 1, &qh->desc_list[n_desc - 1]);
780 		dma_sync_single_for_device(hsotg->dev,
781 					   qh->desc_list_dma + (n_desc - 1) *
782 					   sizeof(struct dwc2_dma_desc),
783 					   sizeof(struct dwc2_dma_desc),
784 					   DMA_TO_DEVICE);
785 		if (n_desc > 1) {
786 			qh->desc_list[0].status |= HOST_DMA_A;
787 			dev_vdbg(hsotg->dev, "set A bit in desc 0 (%p)\n",
788 				 &qh->desc_list[0]);
789 			dma_sync_single_for_device(hsotg->dev,
790 						   qh->desc_list_dma,
791 					sizeof(struct dwc2_dma_desc),
792 					DMA_TO_DEVICE);
793 		}
794 		chan->ntd = n_desc;
795 	}
796 }
797 
798 /**
799  * dwc2_hcd_start_xfer_ddma() - Starts a transfer in Descriptor DMA mode
800  *
801  * @hsotg: The HCD state structure for the DWC OTG controller
802  * @qh:    The QH to init
803  *
804  * Return: 0 if successful, negative error code otherwise
805  *
806  * For Control and Bulk endpoints, initializes descriptor list and starts the
807  * transfer. For Interrupt and Isochronous endpoints, initializes descriptor
808  * list then updates FrameList, marking appropriate entries as active.
809  *
810  * For Isochronous endpoints the starting descriptor index is calculated based
811  * on the scheduled frame, but only on the first transfer descriptor within a
812  * session. Then the transfer is started via enabling the channel.
813  *
814  * For Isochronous endpoints the channel is not halted on XferComplete
815  * interrupt so remains assigned to the endpoint(QH) until session is done.
816  */
dwc2_hcd_start_xfer_ddma(struct dwc2_hsotg * hsotg,struct dwc2_qh * qh)817 void dwc2_hcd_start_xfer_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)
818 {
819 	/* Channel is already assigned */
820 	struct dwc2_host_chan *chan = qh->channel;
821 	u16 skip_frames = 0;
822 
823 	switch (chan->ep_type) {
824 	case USB_ENDPOINT_XFER_CONTROL:
825 	case USB_ENDPOINT_XFER_BULK:
826 		dwc2_init_non_isoc_dma_desc(hsotg, qh);
827 		dwc2_hc_start_transfer_ddma(hsotg, chan);
828 		break;
829 	case USB_ENDPOINT_XFER_INT:
830 		dwc2_init_non_isoc_dma_desc(hsotg, qh);
831 		dwc2_update_frame_list(hsotg, qh, 1);
832 		dwc2_hc_start_transfer_ddma(hsotg, chan);
833 		break;
834 	case USB_ENDPOINT_XFER_ISOC:
835 		if (!qh->ntd)
836 			skip_frames = dwc2_recalc_initial_desc_idx(hsotg, qh);
837 		dwc2_init_isoc_dma_desc(hsotg, qh, skip_frames);
838 
839 		if (!chan->xfer_started) {
840 			dwc2_update_frame_list(hsotg, qh, 1);
841 
842 			/*
843 			 * Always set to max, instead of actual size. Otherwise
844 			 * ntd will be changed with channel being enabled. Not
845 			 * recommended.
846 			 */
847 			chan->ntd = dwc2_max_desc_num(qh);
848 
849 			/* Enable channel only once for ISOC */
850 			dwc2_hc_start_transfer_ddma(hsotg, chan);
851 		}
852 
853 		break;
854 	default:
855 		break;
856 	}
857 }
858 
859 #define DWC2_CMPL_DONE		1
860 #define DWC2_CMPL_STOP		2
861 
dwc2_cmpl_host_isoc_dma_desc(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,struct dwc2_qtd * qtd,struct dwc2_qh * qh,u16 idx)862 static int dwc2_cmpl_host_isoc_dma_desc(struct dwc2_hsotg *hsotg,
863 					struct dwc2_host_chan *chan,
864 					struct dwc2_qtd *qtd,
865 					struct dwc2_qh *qh, u16 idx)
866 {
867 	struct dwc2_dma_desc *dma_desc;
868 	struct dwc2_hcd_iso_packet_desc *frame_desc;
869 	u16 frame_desc_idx;
870 	struct urb *usb_urb;
871 	u16 remain = 0;
872 	int rc = 0;
873 
874 	if (!qtd->urb)
875 		return -EINVAL;
876 
877 	usb_urb = qtd->urb->priv;
878 
879 	dma_sync_single_for_cpu(hsotg->dev, qh->desc_list_dma + (idx *
880 				sizeof(struct dwc2_dma_desc)),
881 				sizeof(struct dwc2_dma_desc),
882 				DMA_FROM_DEVICE);
883 
884 	dma_desc = &qh->desc_list[idx];
885 	frame_desc_idx = (idx - qtd->isoc_td_first) & (usb_urb->number_of_packets - 1);
886 
887 	frame_desc = &qtd->urb->iso_descs[frame_desc_idx];
888 	if (idx == qtd->isoc_td_first)
889 		usb_urb->start_frame = dwc2_hcd_get_frame_number(hsotg);
890 	dma_desc->buf = (u32)(qtd->urb->dma + frame_desc->offset);
891 	if (chan->ep_is_in)
892 		remain = (dma_desc->status & HOST_DMA_ISOC_NBYTES_MASK) >>
893 			 HOST_DMA_ISOC_NBYTES_SHIFT;
894 
895 	if ((dma_desc->status & HOST_DMA_STS_MASK) == HOST_DMA_STS_PKTERR) {
896 		/*
897 		 * XactError, or unable to complete all the transactions
898 		 * in the scheduled micro-frame/frame, both indicated by
899 		 * HOST_DMA_STS_PKTERR
900 		 */
901 		qtd->urb->error_count++;
902 		frame_desc->actual_length = qh->n_bytes[idx] - remain;
903 		frame_desc->status = -EPROTO;
904 	} else {
905 		/* Success */
906 		frame_desc->actual_length = qh->n_bytes[idx] - remain;
907 		frame_desc->status = 0;
908 	}
909 
910 	if (++qtd->isoc_frame_index == usb_urb->number_of_packets) {
911 		/*
912 		 * urb->status is not used for isoc transfers here. The
913 		 * individual frame_desc status are used instead.
914 		 */
915 		dwc2_host_complete(hsotg, qtd, 0);
916 		dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
917 
918 		/*
919 		 * This check is necessary because urb_dequeue can be called
920 		 * from urb complete callback (sound driver for example). All
921 		 * pending URBs are dequeued there, so no need for further
922 		 * processing.
923 		 */
924 		if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE)
925 			return -1;
926 		rc = DWC2_CMPL_DONE;
927 	}
928 
929 	qh->ntd--;
930 
931 	/* Stop if IOC requested descriptor reached */
932 	if (dma_desc->status & HOST_DMA_IOC)
933 		rc = DWC2_CMPL_STOP;
934 
935 	return rc;
936 }
937 
dwc2_complete_isoc_xfer_ddma(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,enum dwc2_halt_status halt_status)938 static void dwc2_complete_isoc_xfer_ddma(struct dwc2_hsotg *hsotg,
939 					 struct dwc2_host_chan *chan,
940 					 enum dwc2_halt_status halt_status)
941 {
942 	struct dwc2_hcd_iso_packet_desc *frame_desc;
943 	struct dwc2_qtd *qtd, *qtd_tmp;
944 	struct dwc2_qh *qh;
945 	u16 idx;
946 	int rc;
947 
948 	qh = chan->qh;
949 	idx = qh->td_first;
950 
951 	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
952 		list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry)
953 			qtd->in_process = 0;
954 		return;
955 	}
956 
957 	if (halt_status == DWC2_HC_XFER_AHB_ERR ||
958 	    halt_status == DWC2_HC_XFER_BABBLE_ERR) {
959 		/*
960 		 * Channel is halted in these error cases, considered as serious
961 		 * issues.
962 		 * Complete all URBs marking all frames as failed, irrespective
963 		 * whether some of the descriptors (frames) succeeded or not.
964 		 * Pass error code to completion routine as well, to update
965 		 * urb->status, some of class drivers might use it to stop
966 		 * queing transfer requests.
967 		 */
968 		int err = halt_status == DWC2_HC_XFER_AHB_ERR ?
969 			  -EIO : -EOVERFLOW;
970 
971 		list_for_each_entry_safe(qtd, qtd_tmp, &qh->qtd_list,
972 					 qtd_list_entry) {
973 			if (qtd->urb) {
974 				for (idx = 0; idx < qtd->urb->packet_count;
975 				     idx++) {
976 					frame_desc = &qtd->urb->iso_descs[idx];
977 					frame_desc->status = err;
978 				}
979 
980 				dwc2_host_complete(hsotg, qtd, err);
981 			}
982 
983 			dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
984 		}
985 
986 		return;
987 	}
988 
989 	list_for_each_entry_safe(qtd, qtd_tmp, &qh->qtd_list, qtd_list_entry) {
990 		if (!qtd->in_process)
991 			break;
992 
993 		/*
994 		 * Ensure idx corresponds to descriptor where first urb of this
995 		 * qtd was added. In fact, during isoc desc init, dwc2 may skip
996 		 * an index if current frame number is already over this index.
997 		 */
998 		if (idx != qtd->isoc_td_first) {
999 			dev_vdbg(hsotg->dev,
1000 				 "try to complete %d instead of %d\n",
1001 				 idx, qtd->isoc_td_first);
1002 			idx = qtd->isoc_td_first;
1003 		}
1004 
1005 		do {
1006 			struct dwc2_qtd *qtd_next;
1007 			u16 cur_idx;
1008 
1009 			rc = dwc2_cmpl_host_isoc_dma_desc(hsotg, chan, qtd, qh,
1010 							  idx);
1011 			if (rc < 0)
1012 				return;
1013 			idx = dwc2_desclist_idx_inc(idx, qh->host_interval,
1014 						    chan->speed);
1015 			if (rc == 0)
1016 				continue;
1017 
1018 			if (rc == DWC2_CMPL_DONE || rc == DWC2_CMPL_STOP)
1019 				goto stop_scan;
1020 
1021 			/* rc == DWC2_CMPL_STOP */
1022 
1023 			if (qh->host_interval >= 32)
1024 				goto stop_scan;
1025 
1026 			qh->td_first = idx;
1027 			cur_idx = dwc2_frame_list_idx(hsotg->frame_number);
1028 			qtd_next = list_first_entry(&qh->qtd_list,
1029 						    struct dwc2_qtd,
1030 						    qtd_list_entry);
1031 			if (dwc2_frame_idx_num_gt(cur_idx,
1032 						  qtd_next->isoc_td_last))
1033 				break;
1034 
1035 			goto stop_scan;
1036 
1037 		} while (idx != qh->td_first);
1038 	}
1039 
1040 stop_scan:
1041 	qh->td_first = idx;
1042 }
1043 
dwc2_update_non_isoc_urb_state_ddma(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,struct dwc2_qtd * qtd,struct dwc2_dma_desc * dma_desc,enum dwc2_halt_status halt_status,u32 n_bytes,int * xfer_done)1044 static int dwc2_update_non_isoc_urb_state_ddma(struct dwc2_hsotg *hsotg,
1045 					       struct dwc2_host_chan *chan,
1046 					struct dwc2_qtd *qtd,
1047 					struct dwc2_dma_desc *dma_desc,
1048 					enum dwc2_halt_status halt_status,
1049 					u32 n_bytes, int *xfer_done)
1050 {
1051 	struct dwc2_hcd_urb *urb = qtd->urb;
1052 	u16 remain = 0;
1053 
1054 	if (chan->ep_is_in)
1055 		remain = (dma_desc->status & HOST_DMA_NBYTES_MASK) >>
1056 			 HOST_DMA_NBYTES_SHIFT;
1057 
1058 	dev_vdbg(hsotg->dev, "remain=%d dwc2_urb=%p\n", remain, urb);
1059 
1060 	if (halt_status == DWC2_HC_XFER_AHB_ERR) {
1061 		dev_err(hsotg->dev, "EIO\n");
1062 		urb->status = -EIO;
1063 		return 1;
1064 	}
1065 
1066 	if ((dma_desc->status & HOST_DMA_STS_MASK) == HOST_DMA_STS_PKTERR) {
1067 		switch (halt_status) {
1068 		case DWC2_HC_XFER_STALL:
1069 			dev_vdbg(hsotg->dev, "Stall\n");
1070 			urb->status = -EPIPE;
1071 			break;
1072 		case DWC2_HC_XFER_BABBLE_ERR:
1073 			dev_err(hsotg->dev, "Babble\n");
1074 			urb->status = -EOVERFLOW;
1075 			break;
1076 		case DWC2_HC_XFER_XACT_ERR:
1077 			dev_err(hsotg->dev, "XactErr\n");
1078 			urb->status = -EPROTO;
1079 			break;
1080 		default:
1081 			dev_err(hsotg->dev,
1082 				"%s: Unhandled descriptor error status (%d)\n",
1083 				__func__, halt_status);
1084 			break;
1085 		}
1086 		return 1;
1087 	}
1088 
1089 	if (dma_desc->status & HOST_DMA_A) {
1090 		dev_vdbg(hsotg->dev,
1091 			 "Active descriptor encountered on channel %d\n",
1092 			 chan->hc_num);
1093 		return 0;
1094 	}
1095 
1096 	if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL) {
1097 		if (qtd->control_phase == DWC2_CONTROL_DATA) {
1098 			urb->actual_length += n_bytes - remain;
1099 			if (remain || urb->actual_length >= urb->length) {
1100 				/*
1101 				 * For Control Data stage do not set urb->status
1102 				 * to 0, to prevent URB callback. Set it when
1103 				 * Status phase is done. See below.
1104 				 */
1105 				*xfer_done = 1;
1106 			}
1107 		} else if (qtd->control_phase == DWC2_CONTROL_STATUS) {
1108 			urb->status = 0;
1109 			*xfer_done = 1;
1110 		}
1111 		/* No handling for SETUP stage */
1112 	} else {
1113 		/* BULK and INTR */
1114 		urb->actual_length += n_bytes - remain;
1115 		dev_vdbg(hsotg->dev, "length=%d actual=%d\n", urb->length,
1116 			 urb->actual_length);
1117 		if (remain || urb->actual_length >= urb->length) {
1118 			urb->status = 0;
1119 			*xfer_done = 1;
1120 		}
1121 	}
1122 
1123 	return 0;
1124 }
1125 
dwc2_process_non_isoc_desc(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,struct dwc2_qtd * qtd,int desc_num,enum dwc2_halt_status halt_status,int * xfer_done)1126 static int dwc2_process_non_isoc_desc(struct dwc2_hsotg *hsotg,
1127 				      struct dwc2_host_chan *chan,
1128 				      int chnum, struct dwc2_qtd *qtd,
1129 				      int desc_num,
1130 				      enum dwc2_halt_status halt_status,
1131 				      int *xfer_done)
1132 {
1133 	struct dwc2_qh *qh = chan->qh;
1134 	struct dwc2_hcd_urb *urb = qtd->urb;
1135 	struct dwc2_dma_desc *dma_desc;
1136 	u32 n_bytes;
1137 	int failed;
1138 
1139 	dev_vdbg(hsotg->dev, "%s()\n", __func__);
1140 
1141 	if (!urb)
1142 		return -EINVAL;
1143 
1144 	dma_sync_single_for_cpu(hsotg->dev,
1145 				qh->desc_list_dma + (desc_num *
1146 				sizeof(struct dwc2_dma_desc)),
1147 				sizeof(struct dwc2_dma_desc),
1148 				DMA_FROM_DEVICE);
1149 
1150 	dma_desc = &qh->desc_list[desc_num];
1151 	n_bytes = qh->n_bytes[desc_num];
1152 	dev_vdbg(hsotg->dev,
1153 		 "qtd=%p dwc2_urb=%p desc_num=%d desc=%p n_bytes=%d\n",
1154 		 qtd, urb, desc_num, dma_desc, n_bytes);
1155 	failed = dwc2_update_non_isoc_urb_state_ddma(hsotg, chan, qtd, dma_desc,
1156 						     halt_status, n_bytes,
1157 						     xfer_done);
1158 	if (failed || (*xfer_done && urb->status != -EINPROGRESS)) {
1159 		dwc2_host_complete(hsotg, qtd, urb->status);
1160 		dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
1161 		dev_vdbg(hsotg->dev, "failed=%1x xfer_done=%1x\n",
1162 			 failed, *xfer_done);
1163 		return failed;
1164 	}
1165 
1166 	if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL) {
1167 		switch (qtd->control_phase) {
1168 		case DWC2_CONTROL_SETUP:
1169 			if (urb->length > 0)
1170 				qtd->control_phase = DWC2_CONTROL_DATA;
1171 			else
1172 				qtd->control_phase = DWC2_CONTROL_STATUS;
1173 			dev_vdbg(hsotg->dev,
1174 				 "  Control setup transaction done\n");
1175 			break;
1176 		case DWC2_CONTROL_DATA:
1177 			if (*xfer_done) {
1178 				qtd->control_phase = DWC2_CONTROL_STATUS;
1179 				dev_vdbg(hsotg->dev,
1180 					 "  Control data transfer done\n");
1181 			} else if (desc_num + 1 == qtd->n_desc) {
1182 				/*
1183 				 * Last descriptor for Control data stage which
1184 				 * is not completed yet
1185 				 */
1186 				dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1187 							  qtd);
1188 			}
1189 			break;
1190 		default:
1191 			break;
1192 		}
1193 	}
1194 
1195 	return 0;
1196 }
1197 
dwc2_complete_non_isoc_xfer_ddma(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,enum dwc2_halt_status halt_status)1198 static void dwc2_complete_non_isoc_xfer_ddma(struct dwc2_hsotg *hsotg,
1199 					     struct dwc2_host_chan *chan,
1200 					     int chnum,
1201 					     enum dwc2_halt_status halt_status)
1202 {
1203 	struct list_head *qtd_item, *qtd_tmp;
1204 	struct dwc2_qh *qh = chan->qh;
1205 	struct dwc2_qtd *qtd = NULL;
1206 	int xfer_done;
1207 	int desc_num = 0;
1208 
1209 	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
1210 		list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry)
1211 			qtd->in_process = 0;
1212 		return;
1213 	}
1214 
1215 	list_for_each_safe(qtd_item, qtd_tmp, &qh->qtd_list) {
1216 		int i;
1217 		int qtd_desc_count;
1218 
1219 		qtd = list_entry(qtd_item, struct dwc2_qtd, qtd_list_entry);
1220 		xfer_done = 0;
1221 		qtd_desc_count = qtd->n_desc;
1222 
1223 		for (i = 0; i < qtd_desc_count; i++) {
1224 			if (dwc2_process_non_isoc_desc(hsotg, chan, chnum, qtd,
1225 						       desc_num, halt_status,
1226 						       &xfer_done)) {
1227 				qtd = NULL;
1228 				goto stop_scan;
1229 			}
1230 
1231 			desc_num++;
1232 		}
1233 	}
1234 
1235 stop_scan:
1236 	if (qh->ep_type != USB_ENDPOINT_XFER_CONTROL) {
1237 		/*
1238 		 * Resetting the data toggle for bulk and interrupt endpoints
1239 		 * in case of stall. See handle_hc_stall_intr().
1240 		 */
1241 		if (halt_status == DWC2_HC_XFER_STALL)
1242 			qh->data_toggle = DWC2_HC_PID_DATA0;
1243 		else
1244 			dwc2_hcd_save_data_toggle(hsotg, chan, chnum, NULL);
1245 	}
1246 
1247 	if (halt_status == DWC2_HC_XFER_COMPLETE) {
1248 		if (chan->hcint & HCINTMSK_NYET) {
1249 			/*
1250 			 * Got a NYET on the last transaction of the transfer.
1251 			 * It means that the endpoint should be in the PING
1252 			 * state at the beginning of the next transfer.
1253 			 */
1254 			qh->ping_state = 1;
1255 		}
1256 	}
1257 }
1258 
1259 /**
1260  * dwc2_hcd_complete_xfer_ddma() - Scans the descriptor list, updates URB's
1261  * status and calls completion routine for the URB if it's done. Called from
1262  * interrupt handlers.
1263  *
1264  * @hsotg:       The HCD state structure for the DWC OTG controller
1265  * @chan:        Host channel the transfer is completed on
1266  * @chnum:       Index of Host channel registers
1267  * @halt_status: Reason the channel is being halted or just XferComplete
1268  *               for isochronous transfers
1269  *
1270  * Releases the channel to be used by other transfers.
1271  * In case of Isochronous endpoint the channel is not halted until the end of
1272  * the session, i.e. QTD list is empty.
1273  * If periodic channel released the FrameList is updated accordingly.
1274  * Calls transaction selection routines to activate pending transfers.
1275  */
dwc2_hcd_complete_xfer_ddma(struct dwc2_hsotg * hsotg,struct dwc2_host_chan * chan,int chnum,enum dwc2_halt_status halt_status)1276 void dwc2_hcd_complete_xfer_ddma(struct dwc2_hsotg *hsotg,
1277 				 struct dwc2_host_chan *chan, int chnum,
1278 				 enum dwc2_halt_status halt_status)
1279 {
1280 	struct dwc2_qh *qh = chan->qh;
1281 	int continue_isoc_xfer = 0;
1282 	enum dwc2_transaction_type tr_type;
1283 
1284 	if (chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1285 		dwc2_complete_isoc_xfer_ddma(hsotg, chan, halt_status);
1286 
1287 		/* Release the channel if halted or session completed */
1288 		if (halt_status != DWC2_HC_XFER_COMPLETE ||
1289 		    list_empty(&qh->qtd_list)) {
1290 			struct dwc2_qtd *qtd, *qtd_tmp;
1291 
1292 			/*
1293 			 * Kill all remainings QTDs since channel has been
1294 			 * halted.
1295 			 */
1296 			list_for_each_entry_safe(qtd, qtd_tmp,
1297 						 &qh->qtd_list,
1298 						 qtd_list_entry) {
1299 				dwc2_host_complete(hsotg, qtd,
1300 						   -ECONNRESET);
1301 				dwc2_hcd_qtd_unlink_and_free(hsotg,
1302 							     qtd, qh);
1303 			}
1304 
1305 			/* Halt the channel if session completed */
1306 			if (halt_status == DWC2_HC_XFER_COMPLETE)
1307 				dwc2_hc_halt(hsotg, chan, halt_status);
1308 			dwc2_release_channel_ddma(hsotg, qh);
1309 			dwc2_hcd_qh_unlink(hsotg, qh);
1310 		} else {
1311 			/* Keep in assigned schedule to continue transfer */
1312 			list_move_tail(&qh->qh_list_entry,
1313 				       &hsotg->periodic_sched_assigned);
1314 			/*
1315 			 * If channel has been halted during giveback of urb
1316 			 * then prevent any new scheduling.
1317 			 */
1318 			if (!chan->halt_status)
1319 				continue_isoc_xfer = 1;
1320 		}
1321 		/*
1322 		 * Todo: Consider the case when period exceeds FrameList size.
1323 		 * Frame Rollover interrupt should be used.
1324 		 */
1325 	} else {
1326 		/*
1327 		 * Scan descriptor list to complete the URB(s), then release
1328 		 * the channel
1329 		 */
1330 		dwc2_complete_non_isoc_xfer_ddma(hsotg, chan, chnum,
1331 						 halt_status);
1332 		dwc2_release_channel_ddma(hsotg, qh);
1333 		dwc2_hcd_qh_unlink(hsotg, qh);
1334 
1335 		if (!list_empty(&qh->qtd_list)) {
1336 			/*
1337 			 * Add back to inactive non-periodic schedule on normal
1338 			 * completion
1339 			 */
1340 			dwc2_hcd_qh_add(hsotg, qh);
1341 		}
1342 	}
1343 
1344 	tr_type = dwc2_hcd_select_transactions(hsotg);
1345 	if (tr_type != DWC2_TRANSACTION_NONE || continue_isoc_xfer) {
1346 		if (continue_isoc_xfer) {
1347 			if (tr_type == DWC2_TRANSACTION_NONE)
1348 				tr_type = DWC2_TRANSACTION_PERIODIC;
1349 			else if (tr_type == DWC2_TRANSACTION_NON_PERIODIC)
1350 				tr_type = DWC2_TRANSACTION_ALL;
1351 		}
1352 		dwc2_hcd_queue_transactions(hsotg, tr_type);
1353 	}
1354 }
1355