30 static int tsi721_submit_sg(struct tsi721_tx_desc *desc);
333 tsi721_desc_fill_init(struct tsi721_tx_desc *desc,  in tsi721_desc_fill_init()  argument
344 				      (desc->rtype << 19) | desc->destid);  in tsi721_desc_fill_init()
345 	bd_ptr->bcount = cpu_to_le32(((desc->rio_addr & 0x3) << 30) |  in tsi721_desc_fill_init()
347 	rio_addr = (desc->rio_addr >> 2) |  in tsi721_desc_fill_init()
348 				((u64)(desc->rio_addr_u & 0x3) << 62);  in tsi721_desc_fill_init()
375 			      struct tsi721_tx_desc *desc)  in tsi721_dma_tx_err()  argument
377 	struct dma_async_tx_descriptor *txd = &desc->txd;  in tsi721_dma_tx_err()
381 	list_move(&desc->desc_node, &bdma_chan->free_list);  in tsi721_dma_tx_err()
411 static int tsi721_submit_sg(struct tsi721_tx_desc *desc)  in tsi721_submit_sg()  argument
413 	struct dma_chan *dchan = desc->txd.chan;  in tsi721_submit_sg()
437 	rio_addr = desc->rio_addr;  in tsi721_submit_sg()
455 	for_each_sg(desc->sg, sg, desc->sg_len, i) {  in tsi721_submit_sg()
458 			bdma_chan->id, i, desc->sg_len,  in tsi721_submit_sg()
484 		desc->rio_addr = rio_addr;  in tsi721_submit_sg()
490 			desc->sg = sg;  in tsi721_submit_sg()
491 			desc->sg_len -= i;  in tsi721_submit_sg()
496 		err = tsi721_desc_fill_init(desc, bd_ptr, sg, sys_size);  in tsi721_submit_sg()
503 			  bdma_chan->id, bd_ptr, desc->destid, desc->rio_addr);  in tsi721_submit_sg()
521 			desc->sg_len = 0;  in tsi721_submit_sg()
535 				struct tsi721_tx_desc *desc)  in tsi721_advance_work()  argument
548 	if (!desc && !bdma_chan->active_tx && !list_empty(&bdma_chan->queue)) {  in tsi721_advance_work()
549 		desc = list_first_entry(&bdma_chan->queue,  in tsi721_advance_work()
551 		list_del_init((&desc->desc_node));  in tsi721_advance_work()
552 		bdma_chan->active_tx = desc;  in tsi721_advance_work()
555 	if (desc) {  in tsi721_advance_work()
556 		err = tsi721_submit_sg(desc);  in tsi721_advance_work()
560 			tsi721_dma_tx_err(bdma_chan, desc);  in tsi721_advance_work()
584 		struct tsi721_tx_desc *desc;  in tsi721_dma_tasklet()  local
586 		desc = bdma_chan->active_tx;  in tsi721_dma_tasklet()
590 			bdma_chan->id, dmac_sts, desc->destid, desc->rio_addr);  in tsi721_dma_tasklet()
641 		desc = bdma_chan->active_tx;  in tsi721_dma_tasklet()
642 		desc->status = DMA_ERROR;  in tsi721_dma_tasklet()
643 		dma_cookie_complete(&desc->txd);  in tsi721_dma_tasklet()
644 		list_add(&desc->desc_node, &bdma_chan->free_list);  in tsi721_dma_tasklet()
658 		struct tsi721_tx_desc *desc;  in tsi721_dma_tasklet()  local
662 		desc = bdma_chan->active_tx;  in tsi721_dma_tasklet()
664 		if (desc->sg_len == 0) {  in tsi721_dma_tasklet()
668 			desc->status = DMA_COMPLETE;  in tsi721_dma_tasklet()
669 			dma_cookie_complete(&desc->txd);  in tsi721_dma_tasklet()
670 			if (desc->txd.flags & DMA_PREP_INTERRUPT) {  in tsi721_dma_tasklet()
671 				callback = desc->txd.callback;  in tsi721_dma_tasklet()
672 				param = desc->txd.callback_param;  in tsi721_dma_tasklet()
674 			list_add(&desc->desc_node, &bdma_chan->free_list);  in tsi721_dma_tasklet()
695 	struct tsi721_tx_desc *desc = to_tsi721_desc(txd);  in tsi721_tx_submit()  local
700 	if (!list_empty(&desc->desc_node)) {  in tsi721_tx_submit()
715 	desc->status = DMA_IN_PROGRESS;  in tsi721_tx_submit()
716 	list_add_tail(&desc->desc_node, &bdma_chan->queue);  in tsi721_tx_submit()
726 	struct tsi721_tx_desc *desc;  in tsi721_alloc_chan_resources()  local
742 	desc = kcalloc(dma_txqueue_sz, sizeof(struct tsi721_tx_desc),  in tsi721_alloc_chan_resources()
744 	if (!desc) {  in tsi721_alloc_chan_resources()
749 	bdma_chan->tx_desc = desc;  in tsi721_alloc_chan_resources()
752 		dma_async_tx_descriptor_init(&desc[i].txd, dchan);  in tsi721_alloc_chan_resources()
753 		desc[i].txd.tx_submit = tsi721_tx_submit;  in tsi721_alloc_chan_resources()
754 		desc[i].txd.flags = DMA_CTRL_ACK;  in tsi721_alloc_chan_resources()
755 		list_add(&desc[i].desc_node, &bdma_chan->free_list);  in tsi721_alloc_chan_resources()
832 	struct tsi721_tx_desc *desc;  in tsi721_prep_rio_sg()  local
871 		desc = list_first_entry(&bdma_chan->free_list,  in tsi721_prep_rio_sg()
873 		list_del_init(&desc->desc_node);  in tsi721_prep_rio_sg()
874 		desc->destid = rext->destid;  in tsi721_prep_rio_sg()
875 		desc->rio_addr = rext->rio_addr;  in tsi721_prep_rio_sg()
876 		desc->rio_addr_u = 0;  in tsi721_prep_rio_sg()
877 		desc->rtype = rtype;  in tsi721_prep_rio_sg()
878 		desc->sg_len	= sg_len;  in tsi721_prep_rio_sg()
879 		desc->sg	= sgl;  in tsi721_prep_rio_sg()
880 		txd		= &desc->txd;  in tsi721_prep_rio_sg()
898 	struct tsi721_tx_desc *desc, *_d;  in tsi721_terminate_all()  local
926 	list_for_each_entry_safe(desc, _d, &list, desc_node)  in tsi721_terminate_all()
927 		tsi721_dma_tx_err(bdma_chan, desc);  in tsi721_terminate_all()