369 		struct cdnsp_ring *ep_ring;  in cdnsp_ring_doorbell_for_active_rings()  local
374 		ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id);  in cdnsp_ring_doorbell_for_active_rings()
375 		if (!ep_ring)  in cdnsp_ring_doorbell_for_active_rings()
378 		if (!ep_ring->stream_active || ep_ring->stream_rejected)  in cdnsp_ring_doorbell_for_active_rings()
381 		list_for_each_entry_safe(td, td_temp, &ep_ring->td_list,  in cdnsp_ring_doorbell_for_active_rings()
439 	struct cdnsp_ring *ep_ring;  in cdnsp_find_new_dequeue_state()  local
444 	ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id);  in cdnsp_find_new_dequeue_state()
445 	if (!ep_ring)  in cdnsp_find_new_dequeue_state()
453 	new_seg = ep_ring->deq_seg;  in cdnsp_find_new_dequeue_state()
454 	new_deq = ep_ring->dequeue;  in cdnsp_find_new_dequeue_state()
480 		cdnsp_next_trb(pdev, ep_ring, &new_seg, &new_deq);  in cdnsp_find_new_dequeue_state()
505 			     struct cdnsp_ring *ep_ring,  in cdnsp_td_to_noop()  argument
522 		cdnsp_next_trb(pdev, ep_ring, &seg, &trb);  in cdnsp_td_to_noop()
639 	struct cdnsp_ring *ep_ring;  in cdnsp_cmd_set_deq()  local
658 	ep_ring = cdnsp_get_transfer_ring(pdev, pep, deq_state->stream_id);  in cdnsp_cmd_set_deq()
660 	if (cdnsp_trb_is_link(ep_ring->dequeue)) {  in cdnsp_cmd_set_deq()
661 		ep_ring->deq_seg = ep_ring->deq_seg->next;  in cdnsp_cmd_set_deq()
662 		ep_ring->dequeue = ep_ring->deq_seg->trbs;  in cdnsp_cmd_set_deq()
665 	while (ep_ring->dequeue != deq_state->new_deq_ptr) {  in cdnsp_cmd_set_deq()
666 		ep_ring->num_trbs_free++;  in cdnsp_cmd_set_deq()
667 		ep_ring->dequeue++;  in cdnsp_cmd_set_deq()
669 		if (cdnsp_trb_is_link(ep_ring->dequeue)) {  in cdnsp_cmd_set_deq()
670 			if (ep_ring->dequeue == deq_state->new_deq_ptr)  in cdnsp_cmd_set_deq()
673 			ep_ring->deq_seg = ep_ring->deq_seg->next;  in cdnsp_cmd_set_deq()
674 			ep_ring->dequeue = ep_ring->deq_seg->trbs;  in cdnsp_cmd_set_deq()
697 	struct cdnsp_ring *ep_ring;  in cdnsp_remove_request()  local
709 	ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);  in cdnsp_remove_request()
727 		cdnsp_td_to_noop(pdev, ep_ring, cur_td, false);  in cdnsp_remove_request()
734 	ep_ring->num_tds--;  in cdnsp_remove_request()
747 	cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, cur_td);  in cdnsp_remove_request()
896 			     struct cdnsp_ring *ep_ring,  in cdnsp_td_cleanup()  argument
902 	cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, td);  in cdnsp_td_cleanup()
914 	ep_ring->num_tds--;  in cdnsp_td_cleanup()
926 	struct cdnsp_ring *ep_ring;  in cdnsp_finish_td()  local
929 	ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));  in cdnsp_finish_td()
944 	while (ep_ring->dequeue != td->last_trb)  in cdnsp_finish_td()
945 		cdnsp_inc_deq(pdev, ep_ring);  in cdnsp_finish_td()
947 	cdnsp_inc_deq(pdev, ep_ring);  in cdnsp_finish_td()
949 	cdnsp_td_cleanup(pdev, td, ep_ring, status);  in cdnsp_finish_td()
1004 	struct cdnsp_ring *ep_ring;  in cdnsp_process_ctrl_td()  local
1009 	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));  in cdnsp_process_ctrl_td()
1028 		td = list_entry(ep_ring->td_list.next, struct cdnsp_td,  in cdnsp_process_ctrl_td()
1032 		cdnsp_giveback_first_trb(pdev, pep, 0, ep_ring->cycle_state,  in cdnsp_process_ctrl_td()
1055 	struct cdnsp_ring *ep_ring;  in cdnsp_process_isoc_td()  local
1059 	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));  in cdnsp_process_isoc_td()
1098 		td_length = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb);  in cdnsp_process_isoc_td()
1115 	struct cdnsp_ring *ep_ring;  in cdnsp_skip_isoc_td()  local
1117 	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));  in cdnsp_skip_isoc_td()
1122 	while (ep_ring->dequeue != td->last_trb)  in cdnsp_skip_isoc_td()
1123 		cdnsp_inc_deq(pdev, ep_ring);  in cdnsp_skip_isoc_td()
1125 	cdnsp_inc_deq(pdev, ep_ring);  in cdnsp_skip_isoc_td()
1127 	cdnsp_td_cleanup(pdev, td, ep_ring, &status);  in cdnsp_skip_isoc_td()
1141 	struct cdnsp_ring *ep_ring;  in cdnsp_process_bulk_intr_td()  local
1144 	ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));  in cdnsp_process_bulk_intr_td()
1168 		ep_trb_len = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb) +  in cdnsp_process_bulk_intr_td()
1182 	struct cdnsp_ring *ep_ring;  in cdnsp_handle_tx_nrdy()  local
1203 			ep_ring = pep->stream_info.stream_rings[cur_stream];  in cdnsp_handle_tx_nrdy()
1204 			ep_ring->stream_active = 1;  in cdnsp_handle_tx_nrdy()
1205 			ep_ring->stream_rejected = 0;  in cdnsp_handle_tx_nrdy()
1213 		ep_ring = pep->stream_info.stream_rings[dev_sid];  in cdnsp_handle_tx_nrdy()
1214 		ep_ring->stream_active = 0;  in cdnsp_handle_tx_nrdy()
1215 		ep_ring->stream_rejected = 1;  in cdnsp_handle_tx_nrdy()
1217 		list_for_each_entry_safe(td, td_temp, &ep_ring->td_list,  in cdnsp_handle_tx_nrdy()
1236 	struct cdnsp_ring *ep_ring;  in cdnsp_handle_tx_event()  local
1252 	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));  in cdnsp_handle_tx_event()
1268 	if (!ep_ring) {  in cdnsp_handle_tx_event()
1311 		if (list_empty(&ep_ring->td_list)) {  in cdnsp_handle_tx_event()
1321 			      ep_ring->last_td_was_short))  in cdnsp_handle_tx_event()
1322 				trace_cdnsp_trb_without_td(ep_ring,  in cdnsp_handle_tx_event()
1333 		td = list_entry(ep_ring->td_list.next, struct cdnsp_td,  in cdnsp_handle_tx_event()
1337 		ep_seg = cdnsp_trb_in_td(pdev, ep_ring->deq_seg,  in cdnsp_handle_tx_event()
1338 					 ep_ring->dequeue, td->last_trb,  in cdnsp_handle_tx_event()
1347 			trace_cdnsp_handle_transfer(ep_ring,  in cdnsp_handle_tx_event()
1385 			ep_ring->last_td_was_short = true;  in cdnsp_handle_tx_event()
1387 			ep_ring->last_td_was_short = false;  in cdnsp_handle_tx_event()
1629 			      struct cdnsp_ring *ep_ring,  in cdnsp_prepare_ring()  argument
1648 		if (cdnsp_room_on_ring(pdev, ep_ring, num_trbs))  in cdnsp_prepare_ring()
1653 		num_trbs_needed = num_trbs - ep_ring->num_trbs_free;  in cdnsp_prepare_ring()
1654 		if (cdnsp_ring_expansion(pdev, ep_ring, num_trbs_needed,  in cdnsp_prepare_ring()
1661 	while (cdnsp_trb_is_link(ep_ring->enqueue)) {  in cdnsp_prepare_ring()
1662 		ep_ring->enqueue->link.control |= cpu_to_le32(TRB_CHAIN);  in cdnsp_prepare_ring()
1665 		ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);  in cdnsp_prepare_ring()
1668 		if (cdnsp_link_trb_toggles_cycle(ep_ring->enqueue))  in cdnsp_prepare_ring()
1669 			ep_ring->cycle_state ^= 1;  in cdnsp_prepare_ring()
1670 		ep_ring->enq_seg = ep_ring->enq_seg->next;  in cdnsp_prepare_ring()
1671 		ep_ring->enqueue = ep_ring->enq_seg->trbs;  in cdnsp_prepare_ring()
1680 	struct cdnsp_ring *ep_ring;  in cdnsp_prepare_transfer()  local
1683 	ep_ring = cdnsp_get_transfer_ring(pdev, preq->pep,  in cdnsp_prepare_transfer()
1685 	if (!ep_ring)  in cdnsp_prepare_transfer()
1688 	ret = cdnsp_prepare_ring(pdev, ep_ring,  in cdnsp_prepare_transfer()
1698 	list_add_tail(&preq->td.td_list, &ep_ring->td_list);  in cdnsp_prepare_transfer()
1699 	ep_ring->num_tds++;  in cdnsp_prepare_transfer()
1702 	preq->td.start_seg = ep_ring->enq_seg;  in cdnsp_prepare_transfer()
1703 	preq->td.first_trb = ep_ring->enqueue;  in cdnsp_prepare_transfer()
2049 	struct cdnsp_ring *ep_ring;  in cdnsp_queue_ctrl_tx()  local
2054 	ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);  in cdnsp_queue_ctrl_tx()
2055 	if (!ep_ring)  in cdnsp_queue_ctrl_tx()
2088 		cdnsp_queue_trb(pdev, ep_ring, true,  in cdnsp_queue_ctrl_tx()
2091 				field | ep_ring->cycle_state |  in cdnsp_queue_ctrl_tx()
2101 			cdnsp_queue_trb(pdev, ep_ring, true,  in cdnsp_queue_ctrl_tx()
2104 					field | ep_ring->cycle_state |  in cdnsp_queue_ctrl_tx()
2113 	preq->td.last_trb = ep_ring->enqueue;  in cdnsp_queue_ctrl_tx()
2117 		field = ep_ring->cycle_state;  in cdnsp_queue_ctrl_tx()
2119 		field = (ep_ring->cycle_state ^ 1);  in cdnsp_queue_ctrl_tx()
2131 	cdnsp_queue_trb(pdev, ep_ring, false, 0, 0, TRB_INTR_TARGET(0),  in cdnsp_queue_ctrl_tx()
2229 	struct cdnsp_ring *ep_ring;  in cdnsp_queue_isoc_tx()  local
2238 	ep_ring = preq->pep->ring;  in cdnsp_queue_isoc_tx()
2258 	start_trb = &ep_ring->enqueue->generic;  in cdnsp_queue_isoc_tx()
2259 	start_cycle = ep_ring->cycle_state;  in cdnsp_queue_isoc_tx()
2305 			field = TRB_TYPE(TRB_NORMAL) | ep_ring->cycle_state;  in cdnsp_queue_isoc_tx()
2321 			preq->td.last_trb = ep_ring->enqueue;  in cdnsp_queue_isoc_tx()
2325 		cdnsp_queue_trb(pdev, ep_ring, more_trbs_coming,  in cdnsp_queue_isoc_tx()
2364 	ep_ring->num_tds--;  in cdnsp_queue_isoc_tx()
2373 	preq->td.last_trb = ep_ring->enqueue;  in cdnsp_queue_isoc_tx()
2375 	cdnsp_td_to_noop(pdev, ep_ring, &preq->td, true);  in cdnsp_queue_isoc_tx()
2378 	ep_ring->enqueue = preq->td.first_trb;  in cdnsp_queue_isoc_tx()
2379 	ep_ring->enq_seg = preq->td.start_seg;  in cdnsp_queue_isoc_tx()
2380 	ep_ring->cycle_state = start_cycle;  in cdnsp_queue_isoc_tx()