xref: /linux/drivers/dma/dmaengine.c (revision ae3690eb2a994dd74a43639d6bf800b51f18b48c)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
4  */
5 
6 /*
7  * This code implements the DMA subsystem. It provides a HW-neutral interface
8  * for other kernel code to use asynchronous memory copy capabilities,
9  * if present, and allows different HW DMA drivers to register as providing
10  * this capability.
11  *
12  * Due to the fact we are accelerating what is already a relatively fast
13  * operation, the code goes to great lengths to avoid additional overhead,
14  * such as locking.
15  *
16  * LOCKING:
17  *
18  * The subsystem keeps a global list of dma_device structs it is protected by a
19  * mutex, dma_list_mutex.
20  *
21  * A subsystem can get access to a channel by calling dmaengine_get() followed
22  * by dma_find_channel(), or if it has need for an exclusive channel it can call
23  * dma_request_channel().  Once a channel is allocated a reference is taken
24  * against its corresponding driver to disable removal.
25  *
26  * Each device has a channels list, which runs unlocked but is never modified
27  * once the device is registered, it's just setup by the driver.
28  *
29  * See Documentation/driver-api/dmaengine for more details
30  */
31 
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33 
34 #include <linux/acpi.h>
35 #include <linux/acpi_dma.h>
36 #include <linux/device.h>
37 #include <linux/dma-mapping.h>
38 #include <linux/dmaengine.h>
39 #include <linux/hardirq.h>
40 #include <linux/idr.h>
41 #include <linux/init.h>
42 #include <linux/jiffies.h>
43 #include <linux/mempool.h>
44 #include <linux/mm.h>
45 #include <linux/module.h>
46 #include <linux/mutex.h>
47 #include <linux/numa.h>
48 #include <linux/of.h>
49 #include <linux/of_dma.h>
50 #include <linux/percpu.h>
51 #include <linux/platform_device.h>
52 #include <linux/property.h>
53 #include <linux/rculist.h>
54 #include <linux/rcupdate.h>
55 #include <linux/slab.h>
56 #include <linux/spinlock.h>
57 
58 #include "dmaengine.h"
59 
60 static DEFINE_MUTEX(dma_list_mutex);
61 static DEFINE_IDA(dma_ida);
62 static LIST_HEAD(dma_device_list);
63 static long dmaengine_ref_count;
64 
65 /* --- debugfs implementation --- */
66 #ifdef CONFIG_DEBUG_FS
67 #include <linux/debugfs.h>
68 
69 static struct dentry *rootdir;
70 
71 static void dmaengine_debug_register(struct dma_device *dma_dev)
72 {
73 	dma_dev->dbg_dev_root = debugfs_create_dir(dev_name(dma_dev->dev),
74 						   rootdir);
75 	if (IS_ERR(dma_dev->dbg_dev_root))
76 		dma_dev->dbg_dev_root = NULL;
77 }
78 
79 static void dmaengine_debug_unregister(struct dma_device *dma_dev)
80 {
81 	debugfs_remove_recursive(dma_dev->dbg_dev_root);
82 	dma_dev->dbg_dev_root = NULL;
83 }
84 
85 static void dmaengine_dbg_summary_show(struct seq_file *s,
86 				       struct dma_device *dma_dev)
87 {
88 	struct dma_chan *chan;
89 
90 	list_for_each_entry(chan, &dma_dev->channels, device_node) {
91 		if (chan->client_count) {
92 			seq_printf(s, " %-13s| %s", dma_chan_name(chan),
93 				   chan->dbg_client_name ?: "in-use");
94 
95 			if (chan->router)
96 				seq_printf(s, " (via router: %s)\n",
97 					dev_name(chan->router->dev));
98 			else
99 				seq_puts(s, "\n");
100 		}
101 	}
102 }
103 
104 static int dmaengine_summary_show(struct seq_file *s, void *data)
105 {
106 	struct dma_device *dma_dev = NULL;
107 
108 	mutex_lock(&dma_list_mutex);
109 	list_for_each_entry(dma_dev, &dma_device_list, global_node) {
110 		seq_printf(s, "dma%d (%s): number of channels: %u\n",
111 			   dma_dev->dev_id, dev_name(dma_dev->dev),
112 			   dma_dev->chancnt);
113 
114 		if (dma_dev->dbg_summary_show)
115 			dma_dev->dbg_summary_show(s, dma_dev);
116 		else
117 			dmaengine_dbg_summary_show(s, dma_dev);
118 
119 		if (!list_is_last(&dma_dev->global_node, &dma_device_list))
120 			seq_puts(s, "\n");
121 	}
122 	mutex_unlock(&dma_list_mutex);
123 
124 	return 0;
125 }
126 DEFINE_SHOW_ATTRIBUTE(dmaengine_summary);
127 
128 static void __init dmaengine_debugfs_init(void)
129 {
130 	rootdir = debugfs_create_dir("dmaengine", NULL);
131 
132 	/* /sys/kernel/debug/dmaengine/summary */
133 	debugfs_create_file("summary", 0444, rootdir, NULL,
134 			    &dmaengine_summary_fops);
135 }
136 #else
137 static inline void dmaengine_debugfs_init(void) { }
138 static inline int dmaengine_debug_register(struct dma_device *dma_dev)
139 {
140 	return 0;
141 }
142 
143 static inline void dmaengine_debug_unregister(struct dma_device *dma_dev) { }
144 #endif	/* DEBUG_FS */
145 
146 /* --- sysfs implementation --- */
147 
148 #define DMA_SLAVE_NAME	"slave"
149 
150 /**
151  * dev_to_dma_chan - convert a device pointer to its sysfs container object
152  * @dev:	device node
153  *
154  * Must be called under dma_list_mutex.
155  */
156 static struct dma_chan *dev_to_dma_chan(struct device *dev)
157 {
158 	struct dma_chan_dev *chan_dev;
159 
160 	chan_dev = container_of(dev, typeof(*chan_dev), device);
161 	return chan_dev->chan;
162 }
163 
164 static ssize_t memcpy_count_show(struct device *dev,
165 				 struct device_attribute *attr, char *buf)
166 {
167 	struct dma_chan *chan;
168 	unsigned long count = 0;
169 	int i;
170 	int err;
171 
172 	mutex_lock(&dma_list_mutex);
173 	chan = dev_to_dma_chan(dev);
174 	if (chan) {
175 		for_each_possible_cpu(i)
176 			count += per_cpu_ptr(chan->local, i)->memcpy_count;
177 		err = sysfs_emit(buf, "%lu\n", count);
178 	} else
179 		err = -ENODEV;
180 	mutex_unlock(&dma_list_mutex);
181 
182 	return err;
183 }
184 static DEVICE_ATTR_RO(memcpy_count);
185 
186 static ssize_t bytes_transferred_show(struct device *dev,
187 				      struct device_attribute *attr, char *buf)
188 {
189 	struct dma_chan *chan;
190 	unsigned long count = 0;
191 	int i;
192 	int err;
193 
194 	mutex_lock(&dma_list_mutex);
195 	chan = dev_to_dma_chan(dev);
196 	if (chan) {
197 		for_each_possible_cpu(i)
198 			count += per_cpu_ptr(chan->local, i)->bytes_transferred;
199 		err = sysfs_emit(buf, "%lu\n", count);
200 	} else
201 		err = -ENODEV;
202 	mutex_unlock(&dma_list_mutex);
203 
204 	return err;
205 }
206 static DEVICE_ATTR_RO(bytes_transferred);
207 
208 static ssize_t in_use_show(struct device *dev, struct device_attribute *attr,
209 			   char *buf)
210 {
211 	struct dma_chan *chan;
212 	int err;
213 
214 	mutex_lock(&dma_list_mutex);
215 	chan = dev_to_dma_chan(dev);
216 	if (chan)
217 		err = sysfs_emit(buf, "%d\n", chan->client_count);
218 	else
219 		err = -ENODEV;
220 	mutex_unlock(&dma_list_mutex);
221 
222 	return err;
223 }
224 static DEVICE_ATTR_RO(in_use);
225 
226 static struct attribute *dma_dev_attrs[] = {
227 	&dev_attr_memcpy_count.attr,
228 	&dev_attr_bytes_transferred.attr,
229 	&dev_attr_in_use.attr,
230 	NULL,
231 };
232 ATTRIBUTE_GROUPS(dma_dev);
233 
234 static void chan_dev_release(struct device *dev)
235 {
236 	struct dma_chan_dev *chan_dev;
237 
238 	chan_dev = container_of(dev, typeof(*chan_dev), device);
239 	kfree(chan_dev);
240 }
241 
242 static struct class dma_devclass = {
243 	.name		= "dma",
244 	.dev_groups	= dma_dev_groups,
245 	.dev_release	= chan_dev_release,
246 };
247 
248 /* --- client and device registration --- */
249 
250 /* enable iteration over all operation types */
251 static dma_cap_mask_t dma_cap_mask_all;
252 
253 /**
254  * struct dma_chan_tbl_ent - tracks channel allocations per core/operation
255  * @chan:	associated channel for this entry
256  */
257 struct dma_chan_tbl_ent {
258 	struct dma_chan *chan;
259 };
260 
261 /* percpu lookup table for memory-to-memory offload providers */
262 static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END];
263 
264 static int __init dma_channel_table_init(void)
265 {
266 	enum dma_transaction_type cap;
267 	int err = 0;
268 
269 	bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);
270 
271 	/* 'interrupt', 'private', and 'slave' are channel capabilities,
272 	 * but are not associated with an operation so they do not need
273 	 * an entry in the channel_table
274 	 */
275 	clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);
276 	clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits);
277 	clear_bit(DMA_SLAVE, dma_cap_mask_all.bits);
278 
279 	for_each_dma_cap_mask(cap, dma_cap_mask_all) {
280 		channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent);
281 		if (!channel_table[cap]) {
282 			err = -ENOMEM;
283 			break;
284 		}
285 	}
286 
287 	if (err) {
288 		pr_err("dmaengine dma_channel_table_init failure: %d\n", err);
289 		for_each_dma_cap_mask(cap, dma_cap_mask_all)
290 			free_percpu(channel_table[cap]);
291 	}
292 
293 	return err;
294 }
295 arch_initcall(dma_channel_table_init);
296 
297 /**
298  * dma_chan_is_local - checks if the channel is in the same NUMA-node as the CPU
299  * @chan:	DMA channel to test
300  * @cpu:	CPU index which the channel should be close to
301  *
302  * Returns true if the channel is in the same NUMA-node as the CPU.
303  */
304 static bool dma_chan_is_local(struct dma_chan *chan, int cpu)
305 {
306 	int node = dev_to_node(chan->device->dev);
307 	return node == NUMA_NO_NODE ||
308 		cpumask_test_cpu(cpu, cpumask_of_node(node));
309 }
310 
311 /**
312  * min_chan - finds the channel with min count and in the same NUMA-node as the CPU
313  * @cap:	capability to match
314  * @cpu:	CPU index which the channel should be close to
315  *
316  * If some channels are close to the given CPU, the one with the lowest
317  * reference count is returned. Otherwise, CPU is ignored and only the
318  * reference count is taken into account.
319  *
320  * Must be called under dma_list_mutex.
321  */
322 static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu)
323 {
324 	struct dma_device *device;
325 	struct dma_chan *chan;
326 	struct dma_chan *min = NULL;
327 	struct dma_chan *localmin = NULL;
328 
329 	list_for_each_entry(device, &dma_device_list, global_node) {
330 		if (!dma_has_cap(cap, device->cap_mask) ||
331 		    dma_has_cap(DMA_PRIVATE, device->cap_mask))
332 			continue;
333 		list_for_each_entry(chan, &device->channels, device_node) {
334 			if (!chan->client_count)
335 				continue;
336 			if (!min || chan->table_count < min->table_count)
337 				min = chan;
338 
339 			if (dma_chan_is_local(chan, cpu))
340 				if (!localmin ||
341 				    chan->table_count < localmin->table_count)
342 					localmin = chan;
343 		}
344 	}
345 
346 	chan = localmin ? localmin : min;
347 
348 	if (chan)
349 		chan->table_count++;
350 
351 	return chan;
352 }
353 
354 /**
355  * dma_channel_rebalance - redistribute the available channels
356  *
357  * Optimize for CPU isolation (each CPU gets a dedicated channel for an
358  * operation type) in the SMP case, and operation isolation (avoid
359  * multi-tasking channels) in the non-SMP case.
360  *
361  * Must be called under dma_list_mutex.
362  */
363 static void dma_channel_rebalance(void)
364 {
365 	struct dma_chan *chan;
366 	struct dma_device *device;
367 	int cpu;
368 	int cap;
369 
370 	/* undo the last distribution */
371 	for_each_dma_cap_mask(cap, dma_cap_mask_all)
372 		for_each_possible_cpu(cpu)
373 			per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;
374 
375 	list_for_each_entry(device, &dma_device_list, global_node) {
376 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
377 			continue;
378 		list_for_each_entry(chan, &device->channels, device_node)
379 			chan->table_count = 0;
380 	}
381 
382 	/* don't populate the channel_table if no clients are available */
383 	if (!dmaengine_ref_count)
384 		return;
385 
386 	/* redistribute available channels */
387 	for_each_dma_cap_mask(cap, dma_cap_mask_all)
388 		for_each_online_cpu(cpu) {
389 			chan = min_chan(cap, cpu);
390 			per_cpu_ptr(channel_table[cap], cpu)->chan = chan;
391 		}
392 }
393 
394 static int dma_device_satisfies_mask(struct dma_device *device,
395 				     const dma_cap_mask_t *want)
396 {
397 	dma_cap_mask_t has;
398 
399 	bitmap_and(has.bits, want->bits, device->cap_mask.bits,
400 		DMA_TX_TYPE_END);
401 	return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
402 }
403 
404 static struct module *dma_chan_to_owner(struct dma_chan *chan)
405 {
406 	return chan->device->owner;
407 }
408 
409 /**
410  * balance_ref_count - catch up the channel reference count
411  * @chan:	channel to balance ->client_count versus dmaengine_ref_count
412  *
413  * Must be called under dma_list_mutex.
414  */
415 static void balance_ref_count(struct dma_chan *chan)
416 {
417 	struct module *owner = dma_chan_to_owner(chan);
418 
419 	while (chan->client_count < dmaengine_ref_count) {
420 		__module_get(owner);
421 		chan->client_count++;
422 	}
423 }
424 
425 static void dma_device_release(struct kref *ref)
426 {
427 	struct dma_device *device = container_of(ref, struct dma_device, ref);
428 
429 	list_del_rcu(&device->global_node);
430 	dma_channel_rebalance();
431 	synchronize_rcu();
432 
433 	if (device->device_release)
434 		device->device_release(device);
435 }
436 
437 static int __must_check dma_device_get(struct dma_device *device)
438 {
439 	lockdep_assert_held(&dma_list_mutex);
440 	return kref_get_unless_zero(&device->ref);
441 }
442 
443 static void dma_device_put(struct dma_device *device)
444 {
445 	lockdep_assert_held(&dma_list_mutex);
446 	kref_put(&device->ref, dma_device_release);
447 }
448 
449 /**
450  * dma_chan_get - try to grab a DMA channel's parent driver module
451  * @chan:	channel to grab
452  *
453  * Must be called under dma_list_mutex.
454  */
455 static int dma_chan_get(struct dma_chan *chan)
456 {
457 	struct module *owner = dma_chan_to_owner(chan);
458 	int ret;
459 
460 	/* The channel is already in use, update client count */
461 	if (chan->client_count) {
462 		__module_get(owner);
463 		chan->client_count++;
464 		return 0;
465 	}
466 
467 	if (!try_module_get(owner))
468 		return -ENODEV;
469 
470 	if (!dma_device_get(chan->device)) {
471 		ret = -ENODEV;
472 		goto module_put_out;
473 	}
474 
475 	/* allocate upon first client reference */
476 	if (chan->device->device_alloc_chan_resources) {
477 		ret = chan->device->device_alloc_chan_resources(chan);
478 		if (ret < 0)
479 			goto err_out;
480 	}
481 
482 	chan->client_count++;
483 
484 	if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))
485 		balance_ref_count(chan);
486 
487 	return 0;
488 
489 err_out:
490 	dma_device_put(chan->device);
491 module_put_out:
492 	module_put(owner);
493 	return ret;
494 }
495 
496 /**
497  * dma_chan_put - drop a reference to a DMA channel's parent driver module
498  * @chan:	channel to release
499  *
500  * Must be called under dma_list_mutex.
501  */
502 static void dma_chan_put(struct dma_chan *chan)
503 {
504 	struct module *owner;
505 
506 	/* This channel is not in use, bail out */
507 	if (!chan->client_count)
508 		return;
509 
510 	owner = dma_chan_to_owner(chan);
511 	chan->client_count--;
512 
513 	/* This channel is not in use anymore, free it */
514 	if (!chan->client_count && chan->device->device_free_chan_resources) {
515 		/* Make sure all operations have completed */
516 		dmaengine_synchronize(chan);
517 		chan->device->device_free_chan_resources(chan);
518 	}
519 
520 	/* If the channel is used via a DMA request router, free the mapping */
521 	if (chan->router && chan->router->route_free) {
522 		chan->router->route_free(chan->router->dev, chan->route_data);
523 		chan->router = NULL;
524 		chan->route_data = NULL;
525 	}
526 
527 	/* This channel is not in use anymore, drop the device ref */
528 	if (!chan->client_count)
529 		dma_device_put(chan->device);
530 	module_put(owner);
531 }
532 
533 enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
534 {
535 	enum dma_status status;
536 	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
537 
538 	dma_async_issue_pending(chan);
539 	do {
540 		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
541 		if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
542 			dev_err(chan->device->dev, "%s: timeout!\n", __func__);
543 			return DMA_ERROR;
544 		}
545 		if (status != DMA_IN_PROGRESS)
546 			break;
547 		cpu_relax();
548 	} while (1);
549 
550 	return status;
551 }
552 EXPORT_SYMBOL(dma_sync_wait);
553 
554 /**
555  * dma_find_channel - find a channel to carry out the operation
556  * @tx_type:	transaction type
557  */
558 struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
559 {
560 	return this_cpu_read(channel_table[tx_type]->chan);
561 }
562 EXPORT_SYMBOL(dma_find_channel);
563 
564 /**
565  * dma_issue_pending_all - flush all pending operations across all channels
566  */
567 void dma_issue_pending_all(void)
568 {
569 	struct dma_device *device;
570 	struct dma_chan *chan;
571 
572 	rcu_read_lock();
573 	list_for_each_entry_rcu(device, &dma_device_list, global_node) {
574 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
575 			continue;
576 		list_for_each_entry(chan, &device->channels, device_node)
577 			if (chan->client_count)
578 				device->device_issue_pending(chan);
579 	}
580 	rcu_read_unlock();
581 }
582 EXPORT_SYMBOL(dma_issue_pending_all);
583 
584 int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps)
585 {
586 	struct dma_device *device;
587 
588 	if (!chan || !caps)
589 		return -EINVAL;
590 
591 	device = chan->device;
592 
593 	/* check if the channel supports slave transactions */
594 	if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||
595 	      test_bit(DMA_CYCLIC, device->cap_mask.bits)))
596 		return -ENXIO;
597 
598 	/*
599 	 * Check whether it reports it uses the generic slave
600 	 * capabilities, if not, that means it doesn't support any
601 	 * kind of slave capabilities reporting.
602 	 */
603 	if (!device->directions)
604 		return -ENXIO;
605 
606 	caps->src_addr_widths = device->src_addr_widths;
607 	caps->dst_addr_widths = device->dst_addr_widths;
608 	caps->directions = device->directions;
609 	caps->min_burst = device->min_burst;
610 	caps->max_burst = device->max_burst;
611 	caps->max_sg_burst = device->max_sg_burst;
612 	caps->residue_granularity = device->residue_granularity;
613 	caps->descriptor_reuse = device->descriptor_reuse;
614 	caps->cmd_pause = !!device->device_pause;
615 	caps->cmd_resume = !!device->device_resume;
616 	caps->cmd_terminate = !!device->device_terminate_all;
617 
618 	/*
619 	 * DMA engine device might be configured with non-uniformly
620 	 * distributed slave capabilities per device channels. In this
621 	 * case the corresponding driver may provide the device_caps
622 	 * callback to override the generic capabilities with
623 	 * channel-specific ones.
624 	 */
625 	if (device->device_caps)
626 		device->device_caps(chan, caps);
627 
628 	return 0;
629 }
630 EXPORT_SYMBOL_GPL(dma_get_slave_caps);
631 
632 static struct dma_chan *private_candidate(const dma_cap_mask_t *mask,
633 					  struct dma_device *dev,
634 					  dma_filter_fn fn, void *fn_param)
635 {
636 	struct dma_chan *chan;
637 
638 	if (mask && !dma_device_satisfies_mask(dev, mask)) {
639 		dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__);
640 		return NULL;
641 	}
642 	/* devices with multiple channels need special handling as we need to
643 	 * ensure that all channels are either private or public.
644 	 */
645 	if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask))
646 		list_for_each_entry(chan, &dev->channels, device_node) {
647 			/* some channels are already publicly allocated */
648 			if (chan->client_count)
649 				return NULL;
650 		}
651 
652 	list_for_each_entry(chan, &dev->channels, device_node) {
653 		if (chan->client_count) {
654 			dev_dbg(dev->dev, "%s: %s busy\n",
655 				 __func__, dma_chan_name(chan));
656 			continue;
657 		}
658 		if (fn && !fn(chan, fn_param)) {
659 			dev_dbg(dev->dev, "%s: %s filter said false\n",
660 				 __func__, dma_chan_name(chan));
661 			continue;
662 		}
663 		return chan;
664 	}
665 
666 	return NULL;
667 }
668 
669 static struct dma_chan *find_candidate(struct dma_device *device,
670 				       const dma_cap_mask_t *mask,
671 				       dma_filter_fn fn, void *fn_param)
672 {
673 	struct dma_chan *chan = private_candidate(mask, device, fn, fn_param);
674 	int err;
675 
676 	if (chan) {
677 		/* Found a suitable channel, try to grab, prep, and return it.
678 		 * We first set DMA_PRIVATE to disable balance_ref_count as this
679 		 * channel will not be published in the general-purpose
680 		 * allocator
681 		 */
682 		dma_cap_set(DMA_PRIVATE, device->cap_mask);
683 		device->privatecnt++;
684 		err = dma_chan_get(chan);
685 
686 		if (err) {
687 			if (err == -ENODEV) {
688 				dev_dbg(device->dev, "%s: %s module removed\n",
689 					__func__, dma_chan_name(chan));
690 				list_del_rcu(&device->global_node);
691 			} else
692 				dev_dbg(device->dev,
693 					"%s: failed to get %s: (%d)\n",
694 					 __func__, dma_chan_name(chan), err);
695 
696 			if (--device->privatecnt == 0)
697 				dma_cap_clear(DMA_PRIVATE, device->cap_mask);
698 
699 			chan = ERR_PTR(err);
700 		}
701 	}
702 
703 	return chan ? chan : ERR_PTR(-EPROBE_DEFER);
704 }
705 
706 /**
707  * dma_get_slave_channel - try to get specific channel exclusively
708  * @chan:	target channel
709  */
710 struct dma_chan *dma_get_slave_channel(struct dma_chan *chan)
711 {
712 	/* lock against __dma_request_channel */
713 	mutex_lock(&dma_list_mutex);
714 
715 	if (chan->client_count == 0) {
716 		struct dma_device *device = chan->device;
717 		int err;
718 
719 		dma_cap_set(DMA_PRIVATE, device->cap_mask);
720 		device->privatecnt++;
721 		err = dma_chan_get(chan);
722 		if (err) {
723 			dev_dbg(chan->device->dev,
724 				"%s: failed to get %s: (%d)\n",
725 				__func__, dma_chan_name(chan), err);
726 			chan = NULL;
727 			if (--device->privatecnt == 0)
728 				dma_cap_clear(DMA_PRIVATE, device->cap_mask);
729 		}
730 	} else
731 		chan = NULL;
732 
733 	mutex_unlock(&dma_list_mutex);
734 
735 
736 	return chan;
737 }
738 EXPORT_SYMBOL_GPL(dma_get_slave_channel);
739 
740 struct dma_chan *dma_get_any_slave_channel(struct dma_device *device)
741 {
742 	dma_cap_mask_t mask;
743 	struct dma_chan *chan;
744 
745 	dma_cap_zero(mask);
746 	dma_cap_set(DMA_SLAVE, mask);
747 
748 	/* lock against __dma_request_channel */
749 	mutex_lock(&dma_list_mutex);
750 
751 	chan = find_candidate(device, &mask, NULL, NULL);
752 
753 	mutex_unlock(&dma_list_mutex);
754 
755 	return IS_ERR(chan) ? NULL : chan;
756 }
757 EXPORT_SYMBOL_GPL(dma_get_any_slave_channel);
758 
759 /**
760  * __dma_request_channel - try to allocate an exclusive channel
761  * @mask:	capabilities that the channel must satisfy
762  * @fn:		optional callback to disposition available channels
763  * @fn_param:	opaque parameter to pass to dma_filter_fn()
764  * @np:		device node to look for DMA channels
765  *
766  * Returns pointer to appropriate DMA channel on success or NULL.
767  */
768 struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask,
769 				       dma_filter_fn fn, void *fn_param,
770 				       struct device_node *np)
771 {
772 	struct dma_device *device, *_d;
773 	struct dma_chan *chan = NULL;
774 
775 	/* Find a channel */
776 	mutex_lock(&dma_list_mutex);
777 	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
778 		/* Finds a DMA controller with matching device node */
779 		if (np && !device_match_of_node(device->dev, np))
780 			continue;
781 
782 		chan = find_candidate(device, mask, fn, fn_param);
783 		if (!IS_ERR(chan))
784 			break;
785 
786 		chan = NULL;
787 	}
788 	mutex_unlock(&dma_list_mutex);
789 
790 	pr_debug("%s: %s (%s)\n",
791 		 __func__,
792 		 chan ? "success" : "fail",
793 		 chan ? dma_chan_name(chan) : NULL);
794 
795 	return chan;
796 }
797 EXPORT_SYMBOL_GPL(__dma_request_channel);
798 
799 static const struct dma_slave_map *dma_filter_match(struct dma_device *device,
800 						    const char *name,
801 						    struct device *dev)
802 {
803 	int i;
804 
805 	if (!device->filter.mapcnt)
806 		return NULL;
807 
808 	for (i = 0; i < device->filter.mapcnt; i++) {
809 		const struct dma_slave_map *map = &device->filter.map[i];
810 
811 		if (!strcmp(map->devname, dev_name(dev)) &&
812 		    !strcmp(map->slave, name))
813 			return map;
814 	}
815 
816 	return NULL;
817 }
818 
819 /**
820  * dma_request_chan - try to allocate an exclusive slave channel
821  * @dev:	pointer to client device structure
822  * @name:	slave channel name
823  *
824  * Returns pointer to appropriate DMA channel on success or an error pointer.
825  */
826 struct dma_chan *dma_request_chan(struct device *dev, const char *name)
827 {
828 	struct fwnode_handle *fwnode;
829 	struct dma_device *d, *_d;
830 	struct dma_chan *chan = NULL;
831 
832 	if (WARN_ON(!dev || !name))
833 		return ERR_PTR(-EINVAL);
834 
835 	fwnode = dev_fwnode(dev);
836 
837 	if (is_of_node(fwnode))
838 		chan = of_dma_request_slave_channel(to_of_node(fwnode), name);
839 	else if (is_acpi_device_node(fwnode))
840 		chan = acpi_dma_request_slave_chan_by_name(dev, name);
841 
842 	if (PTR_ERR(chan) == -EPROBE_DEFER)
843 		return chan;
844 
845 	if (!IS_ERR_OR_NULL(chan))
846 		goto found;
847 
848 	/* Try to find the channel via the DMA filter map(s) */
849 	mutex_lock(&dma_list_mutex);
850 	list_for_each_entry_safe(d, _d, &dma_device_list, global_node) {
851 		dma_cap_mask_t mask;
852 		const struct dma_slave_map *map = dma_filter_match(d, name, dev);
853 
854 		if (!map)
855 			continue;
856 
857 		dma_cap_zero(mask);
858 		dma_cap_set(DMA_SLAVE, mask);
859 
860 		chan = find_candidate(d, &mask, d->filter.fn, map->param);
861 		if (!IS_ERR(chan))
862 			break;
863 	}
864 	mutex_unlock(&dma_list_mutex);
865 
866 	if (IS_ERR(chan))
867 		return chan;
868 	if (!chan)
869 		return ERR_PTR(-EPROBE_DEFER);
870 
871 found:
872 #ifdef CONFIG_DEBUG_FS
873 	chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev), name);
874 	/* No functional issue if it fails, users are supposed to test before use */
875 #endif
876 
877 	chan->name = kasprintf(GFP_KERNEL, "dma:%s", name);
878 	if (!chan->name)
879 		return chan;
880 	chan->slave = dev;
881 
882 	if (sysfs_create_link(&dmaengine_chan_dev(chan)->kobj, &dev->kobj,
883 			      DMA_SLAVE_NAME))
884 		dev_warn(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME);
885 	if (sysfs_create_link(&dev->kobj, &dmaengine_chan_dev(chan)->kobj, chan->name))
886 		dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name);
887 
888 	return chan;
889 }
890 EXPORT_SYMBOL_GPL(dma_request_chan);
891 
892 /**
893  * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities
894  * @mask:	capabilities that the channel must satisfy
895  *
896  * Returns pointer to appropriate DMA channel on success or an error pointer.
897  */
898 struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask)
899 {
900 	struct dma_chan *chan;
901 
902 	if (!mask)
903 		return ERR_PTR(-ENODEV);
904 
905 	chan = __dma_request_channel(mask, NULL, NULL, NULL);
906 	if (!chan) {
907 		mutex_lock(&dma_list_mutex);
908 		if (list_empty(&dma_device_list))
909 			chan = ERR_PTR(-EPROBE_DEFER);
910 		else
911 			chan = ERR_PTR(-ENODEV);
912 		mutex_unlock(&dma_list_mutex);
913 	}
914 
915 	return chan;
916 }
917 EXPORT_SYMBOL_GPL(dma_request_chan_by_mask);
918 
919 void dma_release_channel(struct dma_chan *chan)
920 {
921 	mutex_lock(&dma_list_mutex);
922 	WARN_ONCE(chan->client_count != 1,
923 		  "chan reference count %d != 1\n", chan->client_count);
924 	/* drop PRIVATE cap enabled by __dma_request_channel() */
925 	if (--chan->device->privatecnt == 0)
926 		dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);
927 
928 	dma_chan_put(chan);
929 
930 	if (chan->slave) {
931 		sysfs_remove_link(&dmaengine_chan_dev(chan)->kobj, DMA_SLAVE_NAME);
932 		sysfs_remove_link(&chan->slave->kobj, chan->name);
933 		kfree(chan->name);
934 		chan->name = NULL;
935 		chan->slave = NULL;
936 	}
937 
938 #ifdef CONFIG_DEBUG_FS
939 	kfree(chan->dbg_client_name);
940 	chan->dbg_client_name = NULL;
941 #endif
942 	mutex_unlock(&dma_list_mutex);
943 }
944 EXPORT_SYMBOL_GPL(dma_release_channel);
945 
946 static void dmaenginem_release_channel(void *chan)
947 {
948 	dma_release_channel(chan);
949 }
950 
951 /**
952  * devm_dma_request_chan - try to allocate an exclusive slave channel
953  * @dev:	pointer to client device structure
954  * @name:	slave channel name
955  *
956  * Returns pointer to appropriate DMA channel on success or an error pointer.
957  *
958  * The operation is managed and will be undone on driver detach.
959  */
960 
961 struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name)
962 {
963 	struct dma_chan *chan;
964 	int ret;
965 
966 	chan = dma_request_chan(dev, name);
967 	if (IS_ERR(chan))
968 		return chan;
969 
970 	ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan);
971 	if (ret)
972 		return ERR_PTR(ret);
973 
974 	return chan;
975 }
976 EXPORT_SYMBOL_GPL(devm_dma_request_chan);
977 
978 /**
979  * dmaengine_get - register interest in dma_channels
980  */
981 void dmaengine_get(void)
982 {
983 	struct dma_device *device, *_d;
984 	struct dma_chan *chan;
985 	int err;
986 
987 	mutex_lock(&dma_list_mutex);
988 	dmaengine_ref_count++;
989 
990 	/* try to grab channels */
991 	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
992 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
993 			continue;
994 		list_for_each_entry(chan, &device->channels, device_node) {
995 			err = dma_chan_get(chan);
996 			if (err == -ENODEV) {
997 				/* module removed before we could use it */
998 				list_del_rcu(&device->global_node);
999 				break;
1000 			} else if (err)
1001 				dev_dbg(chan->device->dev,
1002 					"%s: failed to get %s: (%d)\n",
1003 					__func__, dma_chan_name(chan), err);
1004 		}
1005 	}
1006 
1007 	/* if this is the first reference and there were channels
1008 	 * waiting we need to rebalance to get those channels
1009 	 * incorporated into the channel table
1010 	 */
1011 	if (dmaengine_ref_count == 1)
1012 		dma_channel_rebalance();
1013 	mutex_unlock(&dma_list_mutex);
1014 }
1015 EXPORT_SYMBOL(dmaengine_get);
1016 
1017 /**
1018  * dmaengine_put - let DMA drivers be removed when ref_count == 0
1019  */
1020 void dmaengine_put(void)
1021 {
1022 	struct dma_device *device, *_d;
1023 	struct dma_chan *chan;
1024 
1025 	mutex_lock(&dma_list_mutex);
1026 	dmaengine_ref_count--;
1027 	BUG_ON(dmaengine_ref_count < 0);
1028 	/* drop channel references */
1029 	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {
1030 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
1031 			continue;
1032 		list_for_each_entry(chan, &device->channels, device_node)
1033 			dma_chan_put(chan);
1034 	}
1035 	mutex_unlock(&dma_list_mutex);
1036 }
1037 EXPORT_SYMBOL(dmaengine_put);
1038 
1039 static bool device_has_all_tx_types(struct dma_device *device)
1040 {
1041 	/* A device that satisfies this test has channels that will never cause
1042 	 * an async_tx channel switch event as all possible operation types can
1043 	 * be handled.
1044 	 */
1045 	#ifdef CONFIG_ASYNC_TX_DMA
1046 	if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))
1047 		return false;
1048 	#endif
1049 
1050 	#if IS_ENABLED(CONFIG_ASYNC_MEMCPY)
1051 	if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
1052 		return false;
1053 	#endif
1054 
1055 	#if IS_ENABLED(CONFIG_ASYNC_XOR)
1056 	if (!dma_has_cap(DMA_XOR, device->cap_mask))
1057 		return false;
1058 
1059 	#ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA
1060 	if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))
1061 		return false;
1062 	#endif
1063 	#endif
1064 
1065 	#if IS_ENABLED(CONFIG_ASYNC_PQ)
1066 	if (!dma_has_cap(DMA_PQ, device->cap_mask))
1067 		return false;
1068 
1069 	#ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA
1070 	if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))
1071 		return false;
1072 	#endif
1073 	#endif
1074 
1075 	return true;
1076 }
1077 
1078 static int get_dma_id(struct dma_device *device)
1079 {
1080 	int rc = ida_alloc(&dma_ida, GFP_KERNEL);
1081 
1082 	if (rc < 0)
1083 		return rc;
1084 	device->dev_id = rc;
1085 	return 0;
1086 }
1087 
1088 static int __dma_async_device_channel_register(struct dma_device *device,
1089 					       struct dma_chan *chan,
1090 					       const char *name)
1091 {
1092 	unsigned int id;
1093 	int rc;
1094 
1095 	chan->local = alloc_percpu(typeof(*chan->local));
1096 	if (!chan->local)
1097 		return -ENOMEM;
1098 	chan->chan_dev = kzalloc_obj(*chan->chan_dev);
1099 	if (!chan->chan_dev) {
1100 		rc = -ENOMEM;
1101 		goto err_free_local;
1102 	}
1103 
1104 	if (chan->chan_id & DMA_CHAN_ID_STATIC) {
1105 		id = chan->chan_id & ~DMA_CHAN_ID_STATIC;
1106 		chan->chan_id = ida_alloc_range(&device->chan_ida, id, id,
1107 						GFP_KERNEL);
1108 	} else {
1109 		chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL);
1110 	}
1111 	if (chan->chan_id < 0) {
1112 		pr_err("%s: unable to alloc ida for chan: %d\n",
1113 		       __func__, chan->chan_id);
1114 		rc = chan->chan_id;
1115 		goto err_free_dev;
1116 	}
1117 
1118 	dmaengine_chan_dev(chan)->class = &dma_devclass;
1119 	dmaengine_chan_dev(chan)->parent = device->dev;
1120 	chan->chan_dev->chan = chan;
1121 	chan->chan_dev->dev_id = device->dev_id;
1122 	spin_lock_init(&chan->lock);
1123 
1124 	if (!name)
1125 		dev_set_name(dmaengine_chan_dev(chan), "dma%dchan%d", device->dev_id, chan->chan_id);
1126 	else
1127 		dev_set_name(dmaengine_chan_dev(chan), "%s", name);
1128 	rc = device_register(dmaengine_chan_dev(chan));
1129 	if (rc)
1130 		goto err_out_ida;
1131 	chan->client_count = 0;
1132 	device->chancnt++;
1133 
1134 	return 0;
1135 
1136  err_out_ida:
1137 	ida_free(&device->chan_ida, chan->chan_id);
1138  err_free_dev:
1139 	kfree(chan->chan_dev);
1140  err_free_local:
1141 	free_percpu(chan->local);
1142 	chan->local = NULL;
1143 	return rc;
1144 }
1145 
1146 int dma_async_device_channel_register(struct dma_device *device,
1147 				      struct dma_chan *chan,
1148 				      const char *name)
1149 {
1150 	int rc;
1151 
1152 	rc = __dma_async_device_channel_register(device, chan, name);
1153 	if (rc < 0)
1154 		return rc;
1155 
1156 	dma_channel_rebalance();
1157 	return 0;
1158 }
1159 EXPORT_SYMBOL_GPL(dma_async_device_channel_register);
1160 
1161 static void __dma_async_device_channel_unregister(struct dma_device *device,
1162 						  struct dma_chan *chan)
1163 {
1164 	if (chan->local == NULL)
1165 		return;
1166 
1167 	WARN_ONCE(!device->device_release && chan->client_count,
1168 		  "%s called while %d clients hold a reference\n",
1169 		  __func__, chan->client_count);
1170 	mutex_lock(&dma_list_mutex);
1171 	device->chancnt--;
1172 	chan->chan_dev->chan = NULL;
1173 	mutex_unlock(&dma_list_mutex);
1174 	ida_free(&device->chan_ida, chan->chan_id);
1175 	device_unregister(dmaengine_chan_dev(chan));
1176 	free_percpu(chan->local);
1177 }
1178 
1179 void dma_async_device_channel_unregister(struct dma_device *device,
1180 					 struct dma_chan *chan)
1181 {
1182 	__dma_async_device_channel_unregister(device, chan);
1183 	dma_channel_rebalance();
1184 }
1185 EXPORT_SYMBOL_GPL(dma_async_device_channel_unregister);
1186 
1187 /**
1188  * dma_async_device_register - registers DMA devices found
1189  * @device:	pointer to &struct dma_device
1190  *
1191  * After calling this routine the structure should not be freed except in the
1192  * device_release() callback which will be called after
1193  * dma_async_device_unregister() is called and no further references are taken.
1194  */
1195 int dma_async_device_register(struct dma_device *device)
1196 {
1197 	int rc;
1198 	struct dma_chan* chan;
1199 
1200 	if (!device)
1201 		return -ENODEV;
1202 
1203 	/* validate device routines */
1204 	if (!device->dev) {
1205 		pr_err("DMAdevice must have dev\n");
1206 		return -EIO;
1207 	}
1208 
1209 	device->owner = device->dev->driver->owner;
1210 
1211 #define CHECK_CAP(_name, _type)								\
1212 {											\
1213 	if (dma_has_cap(_type, device->cap_mask) && !device->device_prep_##_name) {	\
1214 		dev_err(device->dev,							\
1215 			"Device claims capability %s, but op is not defined\n",		\
1216 			__stringify(_type));						\
1217 		return -EIO;								\
1218 	}										\
1219 }
1220 
1221 	CHECK_CAP(dma_memcpy,      DMA_MEMCPY);
1222 	CHECK_CAP(dma_xor,         DMA_XOR);
1223 	CHECK_CAP(dma_xor_val,     DMA_XOR_VAL);
1224 	CHECK_CAP(dma_pq,          DMA_PQ);
1225 	CHECK_CAP(dma_pq_val,      DMA_PQ_VAL);
1226 	CHECK_CAP(dma_memset,      DMA_MEMSET);
1227 	CHECK_CAP(dma_interrupt,   DMA_INTERRUPT);
1228 	CHECK_CAP(dma_cyclic,      DMA_CYCLIC);
1229 	CHECK_CAP(interleaved_dma, DMA_INTERLEAVE);
1230 
1231 #undef CHECK_CAP
1232 
1233 	if (!device->device_tx_status) {
1234 		dev_err(device->dev, "Device tx_status is not defined\n");
1235 		return -EIO;
1236 	}
1237 
1238 
1239 	if (!device->device_issue_pending) {
1240 		dev_err(device->dev, "Device issue_pending is not defined\n");
1241 		return -EIO;
1242 	}
1243 
1244 	if (!device->device_release)
1245 		dev_dbg(device->dev,
1246 			 "WARN: Device release is not defined so it is not safe to unbind this driver while in use\n");
1247 
1248 	kref_init(&device->ref);
1249 
1250 	/* note: this only matters in the
1251 	 * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case
1252 	 */
1253 	if (device_has_all_tx_types(device))
1254 		dma_cap_set(DMA_ASYNC_TX, device->cap_mask);
1255 
1256 	rc = get_dma_id(device);
1257 	if (rc != 0)
1258 		return rc;
1259 
1260 	ida_init(&device->chan_ida);
1261 
1262 	/* represent channels in sysfs. Probably want devs too */
1263 	list_for_each_entry(chan, &device->channels, device_node) {
1264 		rc = __dma_async_device_channel_register(device, chan, NULL);
1265 		if (rc < 0)
1266 			goto err_out;
1267 	}
1268 
1269 	mutex_lock(&dma_list_mutex);
1270 	/* take references on public channels */
1271 	if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))
1272 		list_for_each_entry(chan, &device->channels, device_node) {
1273 			/* if clients are already waiting for channels we need
1274 			 * to take references on their behalf
1275 			 */
1276 			if (dma_chan_get(chan) == -ENODEV) {
1277 				/* note we can only get here for the first
1278 				 * channel as the remaining channels are
1279 				 * guaranteed to get a reference
1280 				 */
1281 				rc = -ENODEV;
1282 				mutex_unlock(&dma_list_mutex);
1283 				goto err_out;
1284 			}
1285 		}
1286 	list_add_tail_rcu(&device->global_node, &dma_device_list);
1287 	if (dma_has_cap(DMA_PRIVATE, device->cap_mask))
1288 		device->privatecnt++;	/* Always private */
1289 	dma_channel_rebalance();
1290 	mutex_unlock(&dma_list_mutex);
1291 
1292 	dmaengine_debug_register(device);
1293 
1294 	return 0;
1295 
1296 err_out:
1297 	/* if we never registered a channel just release the idr */
1298 	if (!device->chancnt) {
1299 		ida_free(&dma_ida, device->dev_id);
1300 		return rc;
1301 	}
1302 
1303 	list_for_each_entry(chan, &device->channels, device_node) {
1304 		if (chan->local == NULL)
1305 			continue;
1306 		mutex_lock(&dma_list_mutex);
1307 		chan->chan_dev->chan = NULL;
1308 		mutex_unlock(&dma_list_mutex);
1309 		device_unregister(dmaengine_chan_dev(chan));
1310 		free_percpu(chan->local);
1311 	}
1312 	return rc;
1313 }
1314 EXPORT_SYMBOL(dma_async_device_register);
1315 
1316 /**
1317  * dma_async_device_unregister - unregister a DMA device
1318  * @device:	pointer to &struct dma_device
1319  *
1320  * This routine is called by dma driver exit routines, dmaengine holds module
1321  * references to prevent it being called while channels are in use.
1322  */
1323 void dma_async_device_unregister(struct dma_device *device)
1324 {
1325 	struct dma_chan *chan, *n;
1326 
1327 	dmaengine_debug_unregister(device);
1328 
1329 	list_for_each_entry_safe(chan, n, &device->channels, device_node)
1330 		__dma_async_device_channel_unregister(device, chan);
1331 
1332 	mutex_lock(&dma_list_mutex);
1333 	/*
1334 	 * setting DMA_PRIVATE ensures the device being torn down will not
1335 	 * be used in the channel_table
1336 	 */
1337 	dma_cap_set(DMA_PRIVATE, device->cap_mask);
1338 	dma_channel_rebalance();
1339 	ida_free(&dma_ida, device->dev_id);
1340 	dma_device_put(device);
1341 	mutex_unlock(&dma_list_mutex);
1342 }
1343 EXPORT_SYMBOL(dma_async_device_unregister);
1344 
1345 static void dmaenginem_async_device_unregister(void *device)
1346 {
1347 	dma_async_device_unregister(device);
1348 }
1349 
1350 /**
1351  * dmaenginem_async_device_register - registers DMA devices found
1352  * @device:	pointer to &struct dma_device
1353  *
1354  * The operation is managed and will be undone on driver detach.
1355  */
1356 int dmaenginem_async_device_register(struct dma_device *device)
1357 {
1358 	int ret;
1359 
1360 	ret = dma_async_device_register(device);
1361 	if (ret)
1362 		return ret;
1363 
1364 	return devm_add_action_or_reset(device->dev, dmaenginem_async_device_unregister, device);
1365 }
1366 EXPORT_SYMBOL(dmaenginem_async_device_register);
1367 
1368 struct dmaengine_unmap_pool {
1369 	struct kmem_cache *cache;
1370 	const char *name;
1371 	mempool_t *pool;
1372 	size_t size;
1373 };
1374 
1375 #define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) }
1376 static struct dmaengine_unmap_pool unmap_pool[] = {
1377 	__UNMAP_POOL(2),
1378 	#if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
1379 	__UNMAP_POOL(16),
1380 	__UNMAP_POOL(128),
1381 	__UNMAP_POOL(256),
1382 	#endif
1383 };
1384 
1385 static struct dmaengine_unmap_pool *__get_unmap_pool(int nr)
1386 {
1387 	int order = get_count_order(nr);
1388 
1389 	switch (order) {
1390 	case 0 ... 1:
1391 		return &unmap_pool[0];
1392 #if IS_ENABLED(CONFIG_DMA_ENGINE_RAID)
1393 	case 2 ... 4:
1394 		return &unmap_pool[1];
1395 	case 5 ... 7:
1396 		return &unmap_pool[2];
1397 	case 8:
1398 		return &unmap_pool[3];
1399 #endif
1400 	default:
1401 		BUG();
1402 		return NULL;
1403 	}
1404 }
1405 
1406 static void dmaengine_unmap(struct kref *kref)
1407 {
1408 	struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref);
1409 	struct device *dev = unmap->dev;
1410 	int cnt, i;
1411 
1412 	cnt = unmap->to_cnt;
1413 	for (i = 0; i < cnt; i++)
1414 		dma_unmap_page(dev, unmap->addr[i], unmap->len,
1415 			       DMA_TO_DEVICE);
1416 	cnt += unmap->from_cnt;
1417 	for (; i < cnt; i++)
1418 		dma_unmap_page(dev, unmap->addr[i], unmap->len,
1419 			       DMA_FROM_DEVICE);
1420 	cnt += unmap->bidi_cnt;
1421 	for (; i < cnt; i++) {
1422 		if (unmap->addr[i] == 0)
1423 			continue;
1424 		dma_unmap_page(dev, unmap->addr[i], unmap->len,
1425 			       DMA_BIDIRECTIONAL);
1426 	}
1427 	cnt = unmap->map_cnt;
1428 	mempool_free(unmap, __get_unmap_pool(cnt)->pool);
1429 }
1430 
1431 void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap)
1432 {
1433 	if (unmap)
1434 		kref_put(&unmap->kref, dmaengine_unmap);
1435 }
1436 EXPORT_SYMBOL_GPL(dmaengine_unmap_put);
1437 
1438 static void dmaengine_destroy_unmap_pool(void)
1439 {
1440 	int i;
1441 
1442 	for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
1443 		struct dmaengine_unmap_pool *p = &unmap_pool[i];
1444 
1445 		mempool_destroy(p->pool);
1446 		p->pool = NULL;
1447 		kmem_cache_destroy(p->cache);
1448 		p->cache = NULL;
1449 	}
1450 }
1451 
1452 static int __init dmaengine_init_unmap_pool(void)
1453 {
1454 	int i;
1455 
1456 	for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) {
1457 		struct dmaengine_unmap_pool *p = &unmap_pool[i];
1458 		size_t size;
1459 
1460 		size = sizeof(struct dmaengine_unmap_data) +
1461 		       sizeof(dma_addr_t) * p->size;
1462 
1463 		p->cache = kmem_cache_create(p->name, size, 0,
1464 					     SLAB_HWCACHE_ALIGN, NULL);
1465 		if (!p->cache)
1466 			break;
1467 		p->pool = mempool_create_slab_pool(1, p->cache);
1468 		if (!p->pool)
1469 			break;
1470 	}
1471 
1472 	if (i == ARRAY_SIZE(unmap_pool))
1473 		return 0;
1474 
1475 	dmaengine_destroy_unmap_pool();
1476 	return -ENOMEM;
1477 }
1478 
1479 struct dmaengine_unmap_data *
1480 dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)
1481 {
1482 	struct dmaengine_unmap_data *unmap;
1483 
1484 	unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags);
1485 	if (!unmap)
1486 		return NULL;
1487 
1488 	memset(unmap, 0, sizeof(*unmap));
1489 	kref_init(&unmap->kref);
1490 	unmap->dev = dev;
1491 	unmap->map_cnt = nr;
1492 
1493 	return unmap;
1494 }
1495 EXPORT_SYMBOL(dmaengine_get_unmap_data);
1496 
1497 void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
1498 	struct dma_chan *chan)
1499 {
1500 	tx->chan = chan;
1501 	#ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
1502 	spin_lock_init(&tx->lock);
1503 	#endif
1504 }
1505 EXPORT_SYMBOL(dma_async_tx_descriptor_init);
1506 
1507 static inline int desc_check_and_set_metadata_mode(
1508 	struct dma_async_tx_descriptor *desc, enum dma_desc_metadata_mode mode)
1509 {
1510 	/* Make sure that the metadata mode is not mixed */
1511 	if (!desc->desc_metadata_mode) {
1512 		if (dmaengine_is_metadata_mode_supported(desc->chan, mode))
1513 			desc->desc_metadata_mode = mode;
1514 		else
1515 			return -ENOTSUPP;
1516 	} else if (desc->desc_metadata_mode != mode) {
1517 		return -EINVAL;
1518 	}
1519 
1520 	return 0;
1521 }
1522 
1523 int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc,
1524 				   void *data, size_t len)
1525 {
1526 	int ret;
1527 
1528 	if (!desc)
1529 		return -EINVAL;
1530 
1531 	ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_CLIENT);
1532 	if (ret)
1533 		return ret;
1534 
1535 	if (!desc->metadata_ops || !desc->metadata_ops->attach)
1536 		return -ENOTSUPP;
1537 
1538 	return desc->metadata_ops->attach(desc, data, len);
1539 }
1540 EXPORT_SYMBOL_GPL(dmaengine_desc_attach_metadata);
1541 
1542 void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc,
1543 				      size_t *payload_len, size_t *max_len)
1544 {
1545 	int ret;
1546 
1547 	if (!desc)
1548 		return ERR_PTR(-EINVAL);
1549 
1550 	ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
1551 	if (ret)
1552 		return ERR_PTR(ret);
1553 
1554 	if (!desc->metadata_ops || !desc->metadata_ops->get_ptr)
1555 		return ERR_PTR(-ENOTSUPP);
1556 
1557 	return desc->metadata_ops->get_ptr(desc, payload_len, max_len);
1558 }
1559 EXPORT_SYMBOL_GPL(dmaengine_desc_get_metadata_ptr);
1560 
1561 int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc,
1562 				    size_t payload_len)
1563 {
1564 	int ret;
1565 
1566 	if (!desc)
1567 		return -EINVAL;
1568 
1569 	ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE);
1570 	if (ret)
1571 		return ret;
1572 
1573 	if (!desc->metadata_ops || !desc->metadata_ops->set_len)
1574 		return -ENOTSUPP;
1575 
1576 	return desc->metadata_ops->set_len(desc, payload_len);
1577 }
1578 EXPORT_SYMBOL_GPL(dmaengine_desc_set_metadata_len);
1579 
1580 /**
1581  * dma_wait_for_async_tx - spin wait for a transaction to complete
1582  * @tx:		in-flight transaction to wait on
1583  */
1584 enum dma_status
1585 dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
1586 {
1587 	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
1588 
1589 	if (!tx)
1590 		return DMA_COMPLETE;
1591 
1592 	while (tx->cookie == -EBUSY) {
1593 		if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
1594 			dev_err(tx->chan->device->dev,
1595 				"%s timeout waiting for descriptor submission\n",
1596 				__func__);
1597 			return DMA_ERROR;
1598 		}
1599 		cpu_relax();
1600 	}
1601 	return dma_sync_wait(tx->chan, tx->cookie);
1602 }
1603 EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
1604 
1605 /**
1606  * dma_run_dependencies - process dependent operations on the target channel
1607  * @tx:		transaction with dependencies
1608  *
1609  * Helper routine for DMA drivers to process (start) dependent operations
1610  * on their target channel.
1611  */
1612 void dma_run_dependencies(struct dma_async_tx_descriptor *tx)
1613 {
1614 	struct dma_async_tx_descriptor *dep = txd_next(tx);
1615 	struct dma_async_tx_descriptor *dep_next;
1616 	struct dma_chan *chan;
1617 
1618 	if (!dep)
1619 		return;
1620 
1621 	/* we'll submit tx->next now, so clear the link */
1622 	txd_clear_next(tx);
1623 	chan = dep->chan;
1624 
1625 	/* keep submitting up until a channel switch is detected
1626 	 * in that case we will be called again as a result of
1627 	 * processing the interrupt from async_tx_channel_switch
1628 	 */
1629 	for (; dep; dep = dep_next) {
1630 		txd_lock(dep);
1631 		txd_clear_parent(dep);
1632 		dep_next = txd_next(dep);
1633 		if (dep_next && dep_next->chan == chan)
1634 			txd_clear_next(dep); /* ->next will be submitted */
1635 		else
1636 			dep_next = NULL; /* submit current dep and terminate */
1637 		txd_unlock(dep);
1638 
1639 		dep->tx_submit(dep);
1640 	}
1641 
1642 	chan->device->device_issue_pending(chan);
1643 }
1644 EXPORT_SYMBOL_GPL(dma_run_dependencies);
1645 
1646 static int __init dma_bus_init(void)
1647 {
1648 	int err = dmaengine_init_unmap_pool();
1649 
1650 	if (err)
1651 		return err;
1652 
1653 	err = class_register(&dma_devclass);
1654 	if (!err)
1655 		dmaengine_debugfs_init();
1656 
1657 	return err;
1658 }
1659 arch_initcall(dma_bus_init);
1660