1559 	struct dma_pl330_chan *pch;  in dma_pl330_rqcb()  local
1565 	pch = desc->pchan;  in dma_pl330_rqcb()
1568 	if (!pch)  in dma_pl330_rqcb()
1571 	spin_lock_irqsave(&pch->lock, flags);  in dma_pl330_rqcb()
1575 	spin_unlock_irqrestore(&pch->lock, flags);  in dma_pl330_rqcb()
1577 	tasklet_schedule(&pch->task);  in dma_pl330_rqcb()
2041 static inline void fill_queue(struct dma_pl330_chan *pch)  in fill_queue()  argument
2046 	list_for_each_entry(desc, &pch->work_list, node) {  in fill_queue()
2052 		ret = pl330_submit_req(pch->thread, desc);  in fill_queue()
2061 			dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n",  in fill_queue()
2063 			tasklet_schedule(&pch->task);  in fill_queue()
2070 	struct dma_pl330_chan *pch = from_tasklet(pch, t, task);  in pl330_tasklet()  local
2075 	spin_lock_irqsave(&pch->lock, flags);  in pl330_tasklet()
2078 	list_for_each_entry_safe(desc, _dt, &pch->work_list, node)  in pl330_tasklet()
2080 			if (!pch->cyclic)  in pl330_tasklet()
2082 			list_move_tail(&desc->node, &pch->completed_list);  in pl330_tasklet()
2086 	fill_queue(pch);  in pl330_tasklet()
2088 	if (list_empty(&pch->work_list)) {  in pl330_tasklet()
2089 		spin_lock(&pch->thread->dmac->lock);  in pl330_tasklet()
2090 		_stop(pch->thread);  in pl330_tasklet()
2091 		spin_unlock(&pch->thread->dmac->lock);  in pl330_tasklet()
2093 		pch->active = false;  in pl330_tasklet()
2096 		spin_lock(&pch->thread->dmac->lock);  in pl330_tasklet()
2097 		pl330_start_thread(pch->thread);  in pl330_tasklet()
2098 		spin_unlock(&pch->thread->dmac->lock);  in pl330_tasklet()
2101 	while (!list_empty(&pch->completed_list)) {  in pl330_tasklet()
2104 		desc = list_first_entry(&pch->completed_list,  in pl330_tasklet()
2109 		if (pch->cyclic) {  in pl330_tasklet()
2111 			list_move_tail(&desc->node, &pch->work_list);  in pl330_tasklet()
2113 				pch->active = true;  in pl330_tasklet()
2114 				spin_lock(&pch->thread->dmac->lock);  in pl330_tasklet()
2115 				pl330_start_thread(pch->thread);  in pl330_tasklet()
2116 				spin_unlock(&pch->thread->dmac->lock);  in pl330_tasklet()
2121 			list_move_tail(&desc->node, &pch->dmac->desc_pool);  in pl330_tasklet()
2127 			spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tasklet()
2129 			spin_lock_irqsave(&pch->lock, flags);  in pl330_tasklet()
2132 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tasklet()
2136 		pm_runtime_put_autosuspend(pch->dmac->ddma.dev);  in pl330_tasklet()
2161 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_alloc_chan_resources()  local
2162 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_alloc_chan_resources()
2168 	pch->cyclic = false;  in pl330_alloc_chan_resources()
2170 	pch->thread = pl330_request_channel(pl330);  in pl330_alloc_chan_resources()
2171 	if (!pch->thread) {  in pl330_alloc_chan_resources()
2176 	tasklet_setup(&pch->task, pl330_tasklet);  in pl330_alloc_chan_resources()
2202 static void pl330_unprep_slave_fifo(struct dma_pl330_chan *pch)  in pl330_unprep_slave_fifo()  argument
2204 	if (pch->dir != DMA_NONE)  in pl330_unprep_slave_fifo()
2205 		dma_unmap_resource(pch->chan.device->dev, pch->fifo_dma,  in pl330_unprep_slave_fifo()
2206 				   1 << pch->burst_sz, pch->dir, 0);  in pl330_unprep_slave_fifo()
2207 	pch->dir = DMA_NONE;  in pl330_unprep_slave_fifo()
2211 static bool pl330_prep_slave_fifo(struct dma_pl330_chan *pch,  in pl330_prep_slave_fifo()  argument
2214 	struct device *dev = pch->chan.device->dev;  in pl330_prep_slave_fifo()
2218 	if (pch->dir == dma_dir)  in pl330_prep_slave_fifo()
2221 	pl330_unprep_slave_fifo(pch);  in pl330_prep_slave_fifo()
2222 	pch->fifo_dma = dma_map_resource(dev, pch->fifo_addr,  in pl330_prep_slave_fifo()
2223 					 1 << pch->burst_sz, dma_dir, 0);  in pl330_prep_slave_fifo()
2224 	if (dma_mapping_error(dev, pch->fifo_dma))  in pl330_prep_slave_fifo()
2227 	pch->dir = dma_dir;  in pl330_prep_slave_fifo()
2245 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_config_write()  local
2247 	pl330_unprep_slave_fifo(pch);  in pl330_config_write()
2250 			pch->fifo_addr = slave_config->dst_addr;  in pl330_config_write()
2252 			pch->burst_sz = __ffs(slave_config->dst_addr_width);  in pl330_config_write()
2253 		pch->burst_len = fixup_burst_len(slave_config->dst_maxburst,  in pl330_config_write()
2254 			pch->dmac->quirks);  in pl330_config_write()
2257 			pch->fifo_addr = slave_config->src_addr;  in pl330_config_write()
2259 			pch->burst_sz = __ffs(slave_config->src_addr_width);  in pl330_config_write()
2260 		pch->burst_len = fixup_burst_len(slave_config->src_maxburst,  in pl330_config_write()
2261 			pch->dmac->quirks);  in pl330_config_write()
2270 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_config()  local
2272 	memcpy(&pch->slave_config, slave_config, sizeof(*slave_config));  in pl330_config()
2279 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_terminate_all()  local
2282 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_terminate_all()
2286 	spin_lock_irqsave(&pch->lock, flags);  in pl330_terminate_all()
2289 	_stop(pch->thread);  in pl330_terminate_all()
2290 	pch->thread->req[0].desc = NULL;  in pl330_terminate_all()
2291 	pch->thread->req[1].desc = NULL;  in pl330_terminate_all()
2292 	pch->thread->req_running = -1;  in pl330_terminate_all()
2295 	power_down = pch->active;  in pl330_terminate_all()
2296 	pch->active = false;  in pl330_terminate_all()
2299 	list_for_each_entry(desc, &pch->submitted_list, node) {  in pl330_terminate_all()
2304 	list_for_each_entry(desc, &pch->work_list , node) {  in pl330_terminate_all()
2309 	list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool);  in pl330_terminate_all()
2310 	list_splice_tail_init(&pch->work_list, &pl330->desc_pool);  in pl330_terminate_all()
2311 	list_splice_tail_init(&pch->completed_list, &pl330->desc_pool);  in pl330_terminate_all()
2312 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_terminate_all()
2329 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_pause()  local
2330 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_pause()
2335 	spin_lock_irqsave(&pch->lock, flags);  in pl330_pause()
2338 	_stop(pch->thread);  in pl330_pause()
2341 	list_for_each_entry(desc, &pch->work_list, node) {  in pl330_pause()
2345 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_pause()
2353 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_free_chan_resources()  local
2354 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_free_chan_resources()
2357 	tasklet_kill(&pch->task);  in pl330_free_chan_resources()
2359 	pm_runtime_get_sync(pch->dmac->ddma.dev);  in pl330_free_chan_resources()
2362 	pl330_release_channel(pch->thread);  in pl330_free_chan_resources()
2363 	pch->thread = NULL;  in pl330_free_chan_resources()
2365 	if (pch->cyclic)  in pl330_free_chan_resources()
2366 		list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);  in pl330_free_chan_resources()
2369 	pm_runtime_put_autosuspend(pch->dmac->ddma.dev);  in pl330_free_chan_resources()
2370 	pl330_unprep_slave_fifo(pch);  in pl330_free_chan_resources()
2373 static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch,  in pl330_get_current_xferred_count()  argument
2376 	struct pl330_thread *thrd = pch->thread;  in pl330_get_current_xferred_count()
2377 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_get_current_xferred_count()
2390 	pm_runtime_mark_last_busy(pch->dmac->ddma.dev);  in pl330_get_current_xferred_count()
2407 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_tx_status()  local
2418 	spin_lock_irqsave(&pch->lock, flags);  in pl330_tx_status()
2419 	spin_lock(&pch->thread->dmac->lock);  in pl330_tx_status()
2421 	if (pch->thread->req_running != -1)  in pl330_tx_status()
2422 		running = pch->thread->req[pch->thread->req_running].desc;  in pl330_tx_status()
2424 	last_enq = pch->thread->req[pch->thread->lstenq].desc;  in pl330_tx_status()
2427 	list_for_each_entry(desc, &pch->work_list, node) {  in pl330_tx_status()
2432 				pl330_get_current_xferred_count(pch, desc);  in pl330_tx_status()
2465 	spin_unlock(&pch->thread->dmac->lock);  in pl330_tx_status()
2466 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tx_status()
2476 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_issue_pending()  local
2479 	spin_lock_irqsave(&pch->lock, flags);  in pl330_issue_pending()
2480 	if (list_empty(&pch->work_list)) {  in pl330_issue_pending()
2486 		WARN_ON(list_empty(&pch->submitted_list));  in pl330_issue_pending()
2487 		pch->active = true;  in pl330_issue_pending()
2488 		pm_runtime_get_sync(pch->dmac->ddma.dev);  in pl330_issue_pending()
2490 	list_splice_tail_init(&pch->submitted_list, &pch->work_list);  in pl330_issue_pending()
2491 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_issue_pending()
2493 	pl330_tasklet(&pch->task);  in pl330_issue_pending()
2504 	struct dma_pl330_chan *pch = to_pchan(tx->chan);  in pl330_tx_submit()  local
2508 	spin_lock_irqsave(&pch->lock, flags);  in pl330_tx_submit()
2513 		if (pch->cyclic) {  in pl330_tx_submit()
2521 		list_move_tail(&desc->node, &pch->submitted_list);  in pl330_tx_submit()
2526 	list_add_tail(&last->node, &pch->submitted_list);  in pl330_tx_submit()
2527 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tx_submit()
2590 static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)  in pl330_get_desc()  argument
2592 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_get_desc()
2593 	u8 *peri_id = pch->chan.private;  in pl330_get_desc()
2612 	desc->pchan = pch;  in pl330_get_desc()
2616 	desc->peri = peri_id ? pch->chan.chan_id : 0;  in pl330_get_desc()
2617 	desc->rqcfg.pcfg = &pch->dmac->pcfg;  in pl330_get_desc()
2619 	dma_async_tx_descriptor_init(&desc->txd, &pch->chan);  in pl330_get_desc()
2633 __pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,  in __pl330_prep_dma_memcpy()  argument
2636 	struct dma_pl330_desc *desc = pl330_get_desc(pch);  in __pl330_prep_dma_memcpy()
2639 		dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",  in __pl330_prep_dma_memcpy()
2662 	struct dma_pl330_chan *pch = desc->pchan;  in get_burst_len()  local
2663 	struct pl330_dmac *pl330 = pch->dmac;  in get_burst_len()
2683 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_prep_dma_cyclic()  local
2684 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_prep_dma_cyclic()
2693 		dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n",  in pl330_prep_dma_cyclic()
2698 	pl330_config_write(chan, &pch->slave_config, direction);  in pl330_prep_dma_cyclic()
2700 	if (!pl330_prep_slave_fifo(pch, direction))  in pl330_prep_dma_cyclic()
2704 		desc = pl330_get_desc(pch);  in pl330_prep_dma_cyclic()
2708 			dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",  in pl330_prep_dma_cyclic()
2734 			dst = pch->fifo_dma;  in pl330_prep_dma_cyclic()
2739 			src = pch->fifo_dma;  in pl330_prep_dma_cyclic()
2747 		desc->rqcfg.brst_size = pch->burst_sz;  in pl330_prep_dma_cyclic()
2748 		desc->rqcfg.brst_len = pch->burst_len;  in pl330_prep_dma_cyclic()
2763 	pch->cyclic = true;  in pl330_prep_dma_cyclic()
2773 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_prep_dma_memcpy()  local
2777 	if (unlikely(!pch || !len))  in pl330_prep_dma_memcpy()
2780 	pl330 = pch->dmac;  in pl330_prep_dma_memcpy()
2782 	desc = __pl330_prep_dma_memcpy(pch, dst, src, len);  in pl330_prep_dma_memcpy()
2846 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_prep_slave_sg()  local
2850 	if (unlikely(!pch || !sgl || !sg_len))  in pl330_prep_slave_sg()
2853 	pl330_config_write(chan, &pch->slave_config, direction);  in pl330_prep_slave_sg()
2855 	if (!pl330_prep_slave_fifo(pch, direction))  in pl330_prep_slave_sg()
2862 		desc = pl330_get_desc(pch);  in pl330_prep_slave_sg()
2864 			struct pl330_dmac *pl330 = pch->dmac;  in pl330_prep_slave_sg()
2866 			dev_err(pch->dmac->ddma.dev,  in pl330_prep_slave_sg()
2882 			fill_px(&desc->px, pch->fifo_dma, sg_dma_address(sg),  in pl330_prep_slave_sg()
2887 			fill_px(&desc->px, sg_dma_address(sg), pch->fifo_dma,  in pl330_prep_slave_sg()
2891 		desc->rqcfg.brst_size = pch->burst_sz;  in pl330_prep_slave_sg()
2892 		desc->rqcfg.brst_len = pch->burst_len;  in pl330_prep_slave_sg()
2933 			struct dma_pl330_chan *pch = &pl330->peripherals[pr];  in pl330_debugfs_show()  local
2935 			if (!pch->thread || thrd->id != pch->thread->id)  in pl330_debugfs_show()
3004 	struct dma_pl330_chan *pch, *_p;  in pl330_probe()  local
3095 	pl330->peripherals = kzalloc_objs(*pch, num_chan);  in pl330_probe()
3102 		pch = &pl330->peripherals[i];  in pl330_probe()
3104 		pch->chan.private = adev->dev.of_node;  in pl330_probe()
3105 		INIT_LIST_HEAD(&pch->submitted_list);  in pl330_probe()
3106 		INIT_LIST_HEAD(&pch->work_list);  in pl330_probe()
3107 		INIT_LIST_HEAD(&pch->completed_list);  in pl330_probe()
3108 		spin_lock_init(&pch->lock);  in pl330_probe()
3109 		pch->thread = NULL;  in pl330_probe()
3110 		pch->chan.device = pd;  in pl330_probe()
3111 		pch->dmac = pl330;  in pl330_probe()
3112 		pch->dir = DMA_NONE;  in pl330_probe()
3115 		list_add_tail(&pch->chan.device_node, &pd->channels);  in pl330_probe()
3178 	list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,  in pl330_probe()
3182 		list_del(&pch->chan.device_node);  in pl330_probe()
3185 		if (pch->thread) {  in pl330_probe()
3186 			pl330_terminate_all(&pch->chan);  in pl330_probe()
3187 			pl330_free_chan_resources(&pch->chan);  in pl330_probe()
3204 	struct dma_pl330_chan *pch, *_p;  in pl330_remove()  local
3221 	list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,  in pl330_remove()
3225 		list_del(&pch->chan.device_node);  in pl330_remove()
3228 		if (pch->thread) {  in pl330_remove()
3229 			pl330_terminate_all(&pch->chan);  in pl330_remove()
3230 			pl330_free_chan_resources(&pch->chan);  in pl330_remove()