219 static inline bool imxdma_chan_is_doing_cyclic(struct imxdma_channel *imxdmac)  in imxdma_chan_is_doing_cyclic()  argument
223 	if (!list_empty(&imxdmac->ld_active)) {  in imxdma_chan_is_doing_cyclic()
224 		desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc,  in imxdma_chan_is_doing_cyclic()
245 static int imxdma_hw_chain(struct imxdma_channel *imxdmac)  in imxdma_hw_chain()  argument
247 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_hw_chain()
250 		return imxdmac->hw_chaining;  in imxdma_hw_chain()
260 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_sg_next()  local
261 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_sg_next()
271 				 DMA_DAR(imxdmac->channel));  in imxdma_sg_next()
274 				 DMA_SAR(imxdmac->channel));  in imxdma_sg_next()
276 	imx_dmav1_writel(imxdma, now, DMA_CNTR(imxdmac->channel));  in imxdma_sg_next()
279 		"size 0x%08x\n", __func__, imxdmac->channel,  in imxdma_sg_next()
280 		 imx_dmav1_readl(imxdma, DMA_DAR(imxdmac->channel)),  in imxdma_sg_next()
281 		 imx_dmav1_readl(imxdma, DMA_SAR(imxdmac->channel)),  in imxdma_sg_next()
282 		 imx_dmav1_readl(imxdma, DMA_CNTR(imxdmac->channel)));  in imxdma_sg_next()
287 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_enable_hw()  local
288 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_enable_hw()
289 	int channel = imxdmac->channel;  in imxdma_enable_hw()
303 			d->sg && imxdma_hw_chain(imxdmac)) {  in imxdma_enable_hw()
317 static void imxdma_disable_hw(struct imxdma_channel *imxdmac)  in imxdma_disable_hw()  argument
319 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_disable_hw()
320 	int channel = imxdmac->channel;  in imxdma_disable_hw()
325 	if (imxdma_hw_chain(imxdmac))  in imxdma_disable_hw()
326 		del_timer(&imxdmac->watchdog);  in imxdma_disable_hw()
339 	struct imxdma_channel *imxdmac = from_timer(imxdmac, t, watchdog);  in imxdma_watchdog()  local
340 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_watchdog()
341 	int channel = imxdmac->channel;  in imxdma_watchdog()
346 	tasklet_schedule(&imxdmac->dma_tasklet);  in imxdma_watchdog()
348 		imxdmac->channel);  in imxdma_watchdog()
404 static void dma_irq_handle_channel(struct imxdma_channel *imxdmac)  in dma_irq_handle_channel()  argument
406 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in dma_irq_handle_channel()
407 	int chno = imxdmac->channel;  in dma_irq_handle_channel()
412 	if (list_empty(&imxdmac->ld_active)) {  in dma_irq_handle_channel()
417 	desc = list_first_entry(&imxdmac->ld_active,  in dma_irq_handle_channel()
431 			if (imxdma_hw_chain(imxdmac)) {  in dma_irq_handle_channel()
435 				mod_timer(&imxdmac->watchdog,  in dma_irq_handle_channel()
448 			if (imxdma_chan_is_doing_cyclic(imxdmac))  in dma_irq_handle_channel()
450 				tasklet_schedule(&imxdmac->dma_tasklet);  in dma_irq_handle_channel()
455 		if (imxdma_hw_chain(imxdmac)) {  in dma_irq_handle_channel()
456 			del_timer(&imxdmac->watchdog);  in dma_irq_handle_channel()
464 	tasklet_schedule(&imxdmac->dma_tasklet);  in dma_irq_handle_channel()
490 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_xfer_desc()  local
491 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_xfer_desc()
516 		imxdmac->slot_2d = slot;  in imxdma_xfer_desc()
517 		imxdmac->enabled_2d = true;  in imxdma_xfer_desc()
538 		imx_dmav1_writel(imxdma, d->src, DMA_SAR(imxdmac->channel));  in imxdma_xfer_desc()
539 		imx_dmav1_writel(imxdma, d->dest, DMA_DAR(imxdmac->channel));  in imxdma_xfer_desc()
541 			 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
543 		imx_dmav1_writel(imxdma, d->len, DMA_CNTR(imxdmac->channel));  in imxdma_xfer_desc()
547 			__func__, imxdmac->channel,  in imxdma_xfer_desc()
556 			imx_dmav1_writel(imxdma, imxdmac->per_address,  in imxdma_xfer_desc()
557 					 DMA_SAR(imxdmac->channel));  in imxdma_xfer_desc()
558 			imx_dmav1_writel(imxdma, imxdmac->ccr_from_device,  in imxdma_xfer_desc()
559 					 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
563 				__func__, imxdmac->channel,  in imxdma_xfer_desc()
565 				(unsigned long long)imxdmac->per_address);  in imxdma_xfer_desc()
567 			imx_dmav1_writel(imxdma, imxdmac->per_address,  in imxdma_xfer_desc()
568 					 DMA_DAR(imxdmac->channel));  in imxdma_xfer_desc()
569 			imx_dmav1_writel(imxdma, imxdmac->ccr_to_device,  in imxdma_xfer_desc()
570 					 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
574 				__func__, imxdmac->channel,  in imxdma_xfer_desc()
576 				(unsigned long long)imxdmac->per_address);  in imxdma_xfer_desc()
579 				__func__, imxdmac->channel);  in imxdma_xfer_desc()
595 	struct imxdma_channel *imxdmac = from_tasklet(imxdmac, t, dma_tasklet);  in imxdma_tasklet()  local
596 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_tasklet()
602 	if (list_empty(&imxdmac->ld_active)) {  in imxdma_tasklet()
607 	desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc, node);  in imxdma_tasklet()
613 	if (imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_tasklet()
619 	if (imxdmac->enabled_2d) {  in imxdma_tasklet()
620 		imxdma->slots_2d[imxdmac->slot_2d].count--;  in imxdma_tasklet()
621 		imxdmac->enabled_2d = false;  in imxdma_tasklet()
624 	list_move_tail(imxdmac->ld_active.next, &imxdmac->ld_free);  in imxdma_tasklet()
626 	if (!list_empty(&imxdmac->ld_queue)) {  in imxdma_tasklet()
627 		next_desc = list_first_entry(&imxdmac->ld_queue,  in imxdma_tasklet()
629 		list_move_tail(imxdmac->ld_queue.next, &imxdmac->ld_active);  in imxdma_tasklet()
632 				 __func__, imxdmac->channel);  in imxdma_tasklet()
642 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_terminate_all()  local
643 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_terminate_all()
646 	imxdma_disable_hw(imxdmac);  in imxdma_terminate_all()
649 	list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);  in imxdma_terminate_all()
650 	list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);  in imxdma_terminate_all()
659 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_config_write()  local
660 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_config_write()
664 		imxdmac->per_address = dmaengine_cfg->src_addr;  in imxdma_config_write()
665 		imxdmac->watermark_level = dmaengine_cfg->src_maxburst;  in imxdma_config_write()
666 		imxdmac->word_size = dmaengine_cfg->src_addr_width;  in imxdma_config_write()
668 		imxdmac->per_address = dmaengine_cfg->dst_addr;  in imxdma_config_write()
669 		imxdmac->watermark_level = dmaengine_cfg->dst_maxburst;  in imxdma_config_write()
670 		imxdmac->word_size = dmaengine_cfg->dst_addr_width;  in imxdma_config_write()
673 	switch (imxdmac->word_size) {  in imxdma_config_write()
686 	imxdmac->hw_chaining = 0;  in imxdma_config_write()
688 	imxdmac->ccr_from_device = (mode | IMX_DMA_TYPE_FIFO) |  in imxdma_config_write()
691 	imxdmac->ccr_to_device =  in imxdma_config_write()
694 	imx_dmav1_writel(imxdma, imxdmac->dma_request,  in imxdma_config_write()
695 			 DMA_RSSR(imxdmac->channel));  in imxdma_config_write()
698 	imx_dmav1_writel(imxdma, imxdmac->watermark_level *  in imxdma_config_write()
699 			 imxdmac->word_size, DMA_BLR(imxdmac->channel));  in imxdma_config_write()
707 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_config()  local
709 	memcpy(&imxdmac->config, dmaengine_cfg, sizeof(*dmaengine_cfg));  in imxdma_config()
723 	struct imxdma_channel *imxdmac = to_imxdma_chan(tx->chan);  in imxdma_tx_submit()  local
724 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_tx_submit()
729 	list_move_tail(imxdmac->ld_free.next, &imxdmac->ld_queue);  in imxdma_tx_submit()
738 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_alloc_chan_resources()  local
742 		imxdmac->dma_request = data->dma_request;  in imxdma_alloc_chan_resources()
744 	while (imxdmac->descs_allocated < IMXDMA_MAX_CHAN_DESCRIPTORS) {  in imxdma_alloc_chan_resources()
756 		list_add_tail(&desc->node, &imxdmac->ld_free);  in imxdma_alloc_chan_resources()
757 		imxdmac->descs_allocated++;  in imxdma_alloc_chan_resources()
760 	if (!imxdmac->descs_allocated)  in imxdma_alloc_chan_resources()
763 	return imxdmac->descs_allocated;  in imxdma_alloc_chan_resources()
768 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_free_chan_resources()  local
769 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_free_chan_resources()
775 	imxdma_disable_hw(imxdmac);  in imxdma_free_chan_resources()
776 	list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);  in imxdma_free_chan_resources()
777 	list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);  in imxdma_free_chan_resources()
781 	list_for_each_entry_safe(desc, _desc, &imxdmac->ld_free, node) {  in imxdma_free_chan_resources()
783 		imxdmac->descs_allocated--;  in imxdma_free_chan_resources()
785 	INIT_LIST_HEAD(&imxdmac->ld_free);  in imxdma_free_chan_resources()
787 	kfree(imxdmac->sg_list);  in imxdma_free_chan_resources()
788 	imxdmac->sg_list = NULL;  in imxdma_free_chan_resources()
796 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_slave_sg()  local
801 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_slave_sg()
802 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_slave_sg()
805 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_slave_sg()
811 	imxdma_config_write(chan, &imxdmac->config, direction);  in imxdma_prep_slave_sg()
813 	switch (imxdmac->word_size) {  in imxdma_prep_slave_sg()
834 		desc->src = imxdmac->per_address;  in imxdma_prep_slave_sg()
836 		desc->dest = imxdmac->per_address;  in imxdma_prep_slave_sg()
849 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_cyclic()  local
850 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_cyclic()
856 			__func__, imxdmac->channel, buf_len, period_len);  in imxdma_prep_dma_cyclic()
858 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_cyclic()
859 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_cyclic()
862 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_cyclic()
864 	kfree(imxdmac->sg_list);  in imxdma_prep_dma_cyclic()
866 	imxdmac->sg_list = kcalloc(periods + 1,  in imxdma_prep_dma_cyclic()
868 	if (!imxdmac->sg_list)  in imxdma_prep_dma_cyclic()
871 	sg_init_table(imxdmac->sg_list, periods);  in imxdma_prep_dma_cyclic()
874 		sg_assign_page(&imxdmac->sg_list[i], NULL);  in imxdma_prep_dma_cyclic()
875 		imxdmac->sg_list[i].offset = 0;  in imxdma_prep_dma_cyclic()
876 		imxdmac->sg_list[i].dma_address = dma_addr;  in imxdma_prep_dma_cyclic()
877 		sg_dma_len(&imxdmac->sg_list[i]) = period_len;  in imxdma_prep_dma_cyclic()
882 	sg_chain(imxdmac->sg_list, periods + 1, imxdmac->sg_list);  in imxdma_prep_dma_cyclic()
885 	desc->sg = imxdmac->sg_list;  in imxdma_prep_dma_cyclic()
890 		desc->src = imxdmac->per_address;  in imxdma_prep_dma_cyclic()
892 		desc->dest = imxdmac->per_address;  in imxdma_prep_dma_cyclic()
897 	imxdma_config_write(chan, &imxdmac->config, direction);  in imxdma_prep_dma_cyclic()
906 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_memcpy()  local
907 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_memcpy()
911 		__func__, imxdmac->channel, (unsigned long long)src,  in imxdma_prep_dma_memcpy()
914 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_memcpy()
915 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_memcpy()
918 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_memcpy()
937 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_interleaved()  local
938 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_interleaved()
943 		imxdmac->channel, (unsigned long long)xt->src_start,  in imxdma_prep_dma_interleaved()
948 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_interleaved()
949 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_interleaved()
955 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_interleaved()
979 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_issue_pending()  local
980 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_issue_pending()
985 	if (list_empty(&imxdmac->ld_active) &&  in imxdma_issue_pending()
986 	    !list_empty(&imxdmac->ld_queue)) {  in imxdma_issue_pending()
987 		desc = list_first_entry(&imxdmac->ld_queue,  in imxdma_issue_pending()
993 				 __func__, imxdmac->channel);  in imxdma_issue_pending()
995 			list_move_tail(imxdmac->ld_queue.next,  in imxdma_issue_pending()
996 				       &imxdmac->ld_active);  in imxdma_issue_pending()
1121 		struct imxdma_channel *imxdmac = &imxdma->channel[i];  in imxdma_probe()  local
1133 			imxdmac->irq = irq + i;  in imxdma_probe()
1134 			timer_setup(&imxdmac->watchdog, imxdma_watchdog, 0);  in imxdma_probe()
1137 		imxdmac->imxdma = imxdma;  in imxdma_probe()
1139 		INIT_LIST_HEAD(&imxdmac->ld_queue);  in imxdma_probe()
1140 		INIT_LIST_HEAD(&imxdmac->ld_free);  in imxdma_probe()
1141 		INIT_LIST_HEAD(&imxdmac->ld_active);  in imxdma_probe()
1143 		tasklet_setup(&imxdmac->dma_tasklet, imxdma_tasklet);  in imxdma_probe()
1144 		imxdmac->chan.device = &imxdma->dma_device;  in imxdma_probe()
1145 		dma_cookie_init(&imxdmac->chan);  in imxdma_probe()
1146 		imxdmac->channel = i;  in imxdma_probe()
1149 		list_add_tail(&imxdmac->chan.device_node,  in imxdma_probe()
1207 		struct imxdma_channel *imxdmac = &imxdma->channel[i];  in imxdma_free_irq()  local
1210 			disable_irq(imxdmac->irq);  in imxdma_free_irq()
1212 		tasklet_kill(&imxdmac->dma_tasklet);  in imxdma_free_irq()