26  * Each device has a channels list, which runs unlocked but is never modified
27  * once the device is registered, it's just setup by the driver.
39 #include <linux/device.h>
151  * dev_to_dma_chan - convert a device pointer to its sysfs container object
152  * @dev:	device node
156 static struct dma_chan *dev_to_dma_chan(struct device *dev)  in dev_to_dma_chan()
160 	chan_dev = container_of(dev, typeof(*chan_dev), device);  in dev_to_dma_chan()
164 static ssize_t memcpy_count_show(struct device *dev,  in memcpy_count_show()
186 static ssize_t bytes_transferred_show(struct device *dev,  in bytes_transferred_show()
208 static ssize_t in_use_show(struct device *dev, struct device_attribute *attr,  in in_use_show()
234 static void chan_dev_release(struct device *dev)  in chan_dev_release()
238 	chan_dev = container_of(dev, typeof(*chan_dev), device);  in chan_dev_release()
248 /* --- client and device registration --- */
306 	int node = dev_to_node(chan->device->dev);  in dma_chan_is_local()
324 	struct dma_device *device;  in min_chan()  local
329 	list_for_each_entry(device, &dma_device_list, global_node) {  in min_chan()
330 		if (!dma_has_cap(cap, device->cap_mask) ||  in min_chan()
331 		    dma_has_cap(DMA_PRIVATE, device->cap_mask))  in min_chan()
333 		list_for_each_entry(chan, &device->channels, device_node) {  in min_chan()
366 	struct dma_device *device;  in dma_channel_rebalance()  local
375 	list_for_each_entry(device, &dma_device_list, global_node) {  in dma_channel_rebalance()
376 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dma_channel_rebalance()
378 		list_for_each_entry(chan, &device->channels, device_node)  in dma_channel_rebalance()
394 static int dma_device_satisfies_mask(struct dma_device *device,  in dma_device_satisfies_mask()  argument
399 	bitmap_and(has.bits, want->bits, device->cap_mask.bits,  in dma_device_satisfies_mask()
406 	return chan->device->owner;  in dma_chan_to_owner()
427 	struct dma_device *device = container_of(ref, struct dma_device, ref);  in dma_device_release()  local
429 	list_del_rcu(&device->global_node);  in dma_device_release()
432 	if (device->device_release)  in dma_device_release()
433 		device->device_release(device);  in dma_device_release()
436 static void dma_device_put(struct dma_device *device)  in dma_device_put()  argument
439 	kref_put(&device->ref, dma_device_release);  in dma_device_put()
463 	ret = kref_get_unless_zero(&chan->device->ref);  in dma_chan_get()
470 	if (chan->device->device_alloc_chan_resources) {  in dma_chan_get()
471 		ret = chan->device->device_alloc_chan_resources(chan);  in dma_chan_get()
478 	if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask))  in dma_chan_get()
484 	dma_device_put(chan->device);  in dma_chan_get()
505 	if (!chan->client_count && chan->device->device_free_chan_resources) {  in dma_chan_put()
508 		chan->device->device_free_chan_resources(chan);  in dma_chan_put()
518 	dma_device_put(chan->device);  in dma_chan_put()
531 			dev_err(chan->device->dev, "%s: timeout!\n", __func__);  in dma_sync_wait()
558 	struct dma_device *device;  in dma_issue_pending_all()  local
562 	list_for_each_entry_rcu(device, &dma_device_list, global_node) {  in dma_issue_pending_all()
563 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dma_issue_pending_all()
565 		list_for_each_entry(chan, &device->channels, device_node)  in dma_issue_pending_all()
567 				device->device_issue_pending(chan);  in dma_issue_pending_all()
575 	struct dma_device *device;  in dma_get_slave_caps()  local
580 	device = chan->device;  in dma_get_slave_caps()
583 	if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) ||  in dma_get_slave_caps()
584 	      test_bit(DMA_CYCLIC, device->cap_mask.bits)))  in dma_get_slave_caps()
592 	if (!device->directions)  in dma_get_slave_caps()
595 	caps->src_addr_widths = device->src_addr_widths;  in dma_get_slave_caps()
596 	caps->dst_addr_widths = device->dst_addr_widths;  in dma_get_slave_caps()
597 	caps->directions = device->directions;  in dma_get_slave_caps()
598 	caps->min_burst = device->min_burst;  in dma_get_slave_caps()
599 	caps->max_burst = device->max_burst;  in dma_get_slave_caps()
600 	caps->max_sg_burst = device->max_sg_burst;  in dma_get_slave_caps()
601 	caps->residue_granularity = device->residue_granularity;  in dma_get_slave_caps()
602 	caps->descriptor_reuse = device->descriptor_reuse;  in dma_get_slave_caps()
603 	caps->cmd_pause = !!device->device_pause;  in dma_get_slave_caps()
604 	caps->cmd_resume = !!device->device_resume;  in dma_get_slave_caps()
605 	caps->cmd_terminate = !!device->device_terminate_all;  in dma_get_slave_caps()
608 	 * DMA engine device might be configured with non-uniformly  in dma_get_slave_caps()
609 	 * distributed slave capabilities per device channels. In this  in dma_get_slave_caps()
614 	if (device->device_caps)  in dma_get_slave_caps()
615 		device->device_caps(chan, caps);  in dma_get_slave_caps()
658 static struct dma_chan *find_candidate(struct dma_device *device,  in find_candidate()  argument
662 	struct dma_chan *chan = private_candidate(mask, device, fn, fn_param);  in find_candidate()
671 		dma_cap_set(DMA_PRIVATE, device->cap_mask);  in find_candidate()
672 		device->privatecnt++;  in find_candidate()
677 				dev_dbg(device->dev, "%s: %s module removed\n",  in find_candidate()
679 				list_del_rcu(&device->global_node);  in find_candidate()
681 				dev_dbg(device->dev,  in find_candidate()
685 			if (--device->privatecnt == 0)  in find_candidate()
686 				dma_cap_clear(DMA_PRIVATE, device->cap_mask);  in find_candidate()
705 		struct dma_device *device = chan->device;  in dma_get_slave_channel()  local
708 		dma_cap_set(DMA_PRIVATE, device->cap_mask);  in dma_get_slave_channel()
709 		device->privatecnt++;  in dma_get_slave_channel()
712 			dev_dbg(chan->device->dev,  in dma_get_slave_channel()
716 			if (--device->privatecnt == 0)  in dma_get_slave_channel()
717 				dma_cap_clear(DMA_PRIVATE, device->cap_mask);  in dma_get_slave_channel()
729 struct dma_chan *dma_get_any_slave_channel(struct dma_device *device)  in dma_get_any_slave_channel()  argument
740 	chan = find_candidate(device, &mask, NULL, NULL);  in dma_get_any_slave_channel()
753  * @np:		device node to look for DMA channels
761 	struct dma_device *device, *_d;  in __dma_request_channel()  local
766 	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {  in __dma_request_channel()
767 		/* Finds a DMA controller with matching device node */  in __dma_request_channel()
768 		if (np && device->dev->of_node && np != device->dev->of_node)  in __dma_request_channel()
771 		chan = find_candidate(device, mask, fn, fn_param);  in __dma_request_channel()
788 static const struct dma_slave_map *dma_filter_match(struct dma_device *device,  in dma_filter_match()  argument
790 						    struct device *dev)  in dma_filter_match()
794 	if (!device->filter.mapcnt)  in dma_filter_match()
797 	for (i = 0; i < device->filter.mapcnt; i++) {  in dma_filter_match()
798 		const struct dma_slave_map *map = &device->filter.map[i];  in dma_filter_match()
810  * @dev:	pointer to client device structure
815 struct dma_chan *dma_request_chan(struct device *dev, const char *name)  in dma_request_chan()
866 	if (sysfs_create_link(&chan->dev->device.kobj, &dev->kobj,  in dma_request_chan()
869 	if (sysfs_create_link(&dev->kobj, &chan->dev->device.kobj, chan->name))  in dma_request_chan()
910 	if (--chan->device->privatecnt == 0)  in dma_release_channel()
911 		dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask);  in dma_release_channel()
914 		sysfs_remove_link(&chan->dev->device.kobj, DMA_SLAVE_NAME);  in dma_release_channel()
936  * @dev:	pointer to client device structure
944 struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name)  in devm_dma_request_chan()
964 	struct dma_device *device, *_d;  in dmaengine_get()  local
972 	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {  in dmaengine_get()
973 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dmaengine_get()
975 		list_for_each_entry(chan, &device->channels, device_node) {  in dmaengine_get()
979 				list_del_rcu(&device->global_node);  in dmaengine_get()
982 				dev_dbg(chan->device->dev,  in dmaengine_get()
1003 	struct dma_device *device, *_d;  in dmaengine_put()  local
1010 	list_for_each_entry_safe(device, _d, &dma_device_list, global_node) {  in dmaengine_put()
1011 		if (dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dmaengine_put()
1013 		list_for_each_entry(chan, &device->channels, device_node)  in dmaengine_put()
1020 static bool device_has_all_tx_types(struct dma_device *device)  in device_has_all_tx_types()  argument
1022 	/* A device that satisfies this test has channels that will never cause  in device_has_all_tx_types()
1027 	if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))  in device_has_all_tx_types()
1032 	if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))  in device_has_all_tx_types()
1037 	if (!dma_has_cap(DMA_XOR, device->cap_mask))  in device_has_all_tx_types()
1041 	if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask))  in device_has_all_tx_types()
1047 	if (!dma_has_cap(DMA_PQ, device->cap_mask))  in device_has_all_tx_types()
1051 	if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask))  in device_has_all_tx_types()
1059 static int get_dma_id(struct dma_device *device)  in get_dma_id()  argument
1065 	device->dev_id = rc;  in get_dma_id()
1069 static int __dma_async_device_channel_register(struct dma_device *device,  in __dma_async_device_channel_register()  argument
1088 	chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL);  in __dma_async_device_channel_register()
1096 	chan->dev->device.class = &dma_devclass;  in __dma_async_device_channel_register()
1097 	chan->dev->device.parent = device->dev;  in __dma_async_device_channel_register()
1099 	chan->dev->dev_id = device->dev_id;  in __dma_async_device_channel_register()
1101 		dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id);  in __dma_async_device_channel_register()
1103 		dev_set_name(&chan->dev->device, "%s", name);  in __dma_async_device_channel_register()
1104 	rc = device_register(&chan->dev->device);  in __dma_async_device_channel_register()
1108 	device->chancnt++;  in __dma_async_device_channel_register()
1113 	ida_free(&device->chan_ida, chan->chan_id);  in __dma_async_device_channel_register()
1122 int dma_async_device_channel_register(struct dma_device *device,  in dma_async_device_channel_register()  argument
1128 	rc = __dma_async_device_channel_register(device, chan, name);  in dma_async_device_channel_register()
1137 static void __dma_async_device_channel_unregister(struct dma_device *device,  in __dma_async_device_channel_unregister()  argument
1143 	WARN_ONCE(!device->device_release && chan->client_count,  in __dma_async_device_channel_unregister()
1147 	device->chancnt--;  in __dma_async_device_channel_unregister()
1150 	ida_free(&device->chan_ida, chan->chan_id);  in __dma_async_device_channel_unregister()
1151 	device_unregister(&chan->dev->device);  in __dma_async_device_channel_unregister()
1155 void dma_async_device_channel_unregister(struct dma_device *device,  in dma_async_device_channel_unregister()  argument
1158 	__dma_async_device_channel_unregister(device, chan);  in dma_async_device_channel_unregister()
1165  * @device:	pointer to &struct dma_device
1171 int dma_async_device_register(struct dma_device *device)  in dma_async_device_register()  argument
1176 	if (!device)  in dma_async_device_register()
1179 	/* validate device routines */  in dma_async_device_register()
1180 	if (!device->dev) {  in dma_async_device_register()
1185 	device->owner = device->dev->driver->owner;  in dma_async_device_register()
1189 	if (dma_has_cap(_type, device->cap_mask) && !device->device_prep_##_name) {	\  in dma_async_device_register()
1190 		dev_err(device->dev,							\  in dma_async_device_register()
1191 			"Device claims capability %s, but op is not defined\n",		\  in dma_async_device_register()
1209 	if (!device->device_tx_status) {  in dma_async_device_register()
1210 		dev_err(device->dev, "Device tx_status is not defined\n");  in dma_async_device_register()
1215 	if (!device->device_issue_pending) {  in dma_async_device_register()
1216 		dev_err(device->dev, "Device issue_pending is not defined\n");  in dma_async_device_register()
1220 	if (!device->device_release)  in dma_async_device_register()
1221 		dev_dbg(device->dev,  in dma_async_device_register()
1222 			 "WARN: Device release is not defined so it is not safe to unbind this driver while in use\n");  in dma_async_device_register()
1224 	kref_init(&device->ref);  in dma_async_device_register()
1229 	if (device_has_all_tx_types(device))  in dma_async_device_register()
1230 		dma_cap_set(DMA_ASYNC_TX, device->cap_mask);  in dma_async_device_register()
1232 	rc = get_dma_id(device);  in dma_async_device_register()
1236 	ida_init(&device->chan_ida);  in dma_async_device_register()
1239 	list_for_each_entry(chan, &device->channels, device_node) {  in dma_async_device_register()
1240 		rc = __dma_async_device_channel_register(device, chan, NULL);  in dma_async_device_register()
1247 	if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dma_async_device_register()
1248 		list_for_each_entry(chan, &device->channels, device_node) {  in dma_async_device_register()
1262 	list_add_tail_rcu(&device->global_node, &dma_device_list);  in dma_async_device_register()
1263 	if (dma_has_cap(DMA_PRIVATE, device->cap_mask))  in dma_async_device_register()
1264 		device->privatecnt++;	/* Always private */  in dma_async_device_register()
1268 	dmaengine_debug_register(device);  in dma_async_device_register()
1274 	if (!device->chancnt) {  in dma_async_device_register()
1275 		ida_free(&dma_ida, device->dev_id);  in dma_async_device_register()
1279 	list_for_each_entry(chan, &device->channels, device_node) {  in dma_async_device_register()
1285 		device_unregister(&chan->dev->device);  in dma_async_device_register()
1293  * dma_async_device_unregister - unregister a DMA device
1294  * @device:	pointer to &struct dma_device
1299 void dma_async_device_unregister(struct dma_device *device)  in dma_async_device_unregister()  argument
1303 	dmaengine_debug_unregister(device);  in dma_async_device_unregister()
1305 	list_for_each_entry_safe(chan, n, &device->channels, device_node)  in dma_async_device_unregister()
1306 		__dma_async_device_channel_unregister(device, chan);  in dma_async_device_unregister()
1310 	 * setting DMA_PRIVATE ensures the device being torn down will not  in dma_async_device_unregister()
1313 	dma_cap_set(DMA_PRIVATE, device->cap_mask);  in dma_async_device_unregister()
1315 	ida_free(&dma_ida, device->dev_id);  in dma_async_device_unregister()
1316 	dma_device_put(device);  in dma_async_device_unregister()
1321 static void dmaenginem_async_device_unregister(void *device)  in dmaenginem_async_device_unregister()  argument
1323 	dma_async_device_unregister(device);  in dmaenginem_async_device_unregister()
1328  * @device:	pointer to &struct dma_device
1332 int dmaenginem_async_device_register(struct dma_device *device)  in dmaenginem_async_device_register()  argument
1336 	ret = dma_async_device_register(device);  in dmaenginem_async_device_register()
1340 	return devm_add_action_or_reset(device->dev, dmaenginem_async_device_unregister, device);  in dmaenginem_async_device_register()
1385 	struct device *dev = unmap->dev;  in dmaengine_unmap()
1456 dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags)  in dmaengine_get_unmap_data()
1570 			dev_err(tx->chan->device->dev,  in dma_wait_for_async_tx()
1618 	chan->device->device_issue_pending(chan);  in dma_run_dependencies()