1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. 4 */ 5 6 /* 7 * This code implements the DMA subsystem. It provides a HW-neutral interface 8 * for other kernel code to use asynchronous memory copy capabilities, 9 * if present, and allows different HW DMA drivers to register as providing 10 * this capability. 11 * 12 * Due to the fact we are accelerating what is already a relatively fast 13 * operation, the code goes to great lengths to avoid additional overhead, 14 * such as locking. 15 * 16 * LOCKING: 17 * 18 * The subsystem keeps a global list of dma_device structs it is protected by a 19 * mutex, dma_list_mutex. 20 * 21 * A subsystem can get access to a channel by calling dmaengine_get() followed 22 * by dma_find_channel(), or if it has need for an exclusive channel it can call 23 * dma_request_channel(). Once a channel is allocated a reference is taken 24 * against its corresponding driver to disable removal. 25 * 26 * Each device has a channels list, which runs unlocked but is never modified 27 * once the device is registered, it's just setup by the driver. 28 * 29 * See Documentation/driver-api/dmaengine for more details 30 */ 31 32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 33 34 #include <linux/acpi.h> 35 #include <linux/acpi_dma.h> 36 #include <linux/device.h> 37 #include <linux/dma-mapping.h> 38 #include <linux/dmaengine.h> 39 #include <linux/hardirq.h> 40 #include <linux/idr.h> 41 #include <linux/init.h> 42 #include <linux/jiffies.h> 43 #include <linux/mempool.h> 44 #include <linux/mm.h> 45 #include <linux/module.h> 46 #include <linux/mutex.h> 47 #include <linux/numa.h> 48 #include <linux/of.h> 49 #include <linux/of_dma.h> 50 #include <linux/percpu.h> 51 #include <linux/platform_device.h> 52 #include <linux/property.h> 53 #include <linux/rculist.h> 54 #include <linux/rcupdate.h> 55 #include <linux/slab.h> 56 #include <linux/spinlock.h> 57 58 #include "dmaengine.h" 59 60 static DEFINE_MUTEX(dma_list_mutex); 61 static DEFINE_IDA(dma_ida); 62 static LIST_HEAD(dma_device_list); 63 static long dmaengine_ref_count; 64 65 /* --- debugfs implementation --- */ 66 #ifdef CONFIG_DEBUG_FS 67 #include <linux/debugfs.h> 68 69 static struct dentry *rootdir; 70 71 static void dmaengine_debug_register(struct dma_device *dma_dev) 72 { 73 dma_dev->dbg_dev_root = debugfs_create_dir(dev_name(dma_dev->dev), 74 rootdir); 75 if (IS_ERR(dma_dev->dbg_dev_root)) 76 dma_dev->dbg_dev_root = NULL; 77 } 78 79 static void dmaengine_debug_unregister(struct dma_device *dma_dev) 80 { 81 debugfs_remove_recursive(dma_dev->dbg_dev_root); 82 dma_dev->dbg_dev_root = NULL; 83 } 84 85 static void dmaengine_dbg_summary_show(struct seq_file *s, 86 struct dma_device *dma_dev) 87 { 88 struct dma_chan *chan; 89 90 list_for_each_entry(chan, &dma_dev->channels, device_node) { 91 if (chan->client_count) { 92 seq_printf(s, " %-13s| %s", dma_chan_name(chan), 93 chan->dbg_client_name ?: "in-use"); 94 95 if (chan->router) 96 seq_printf(s, " (via router: %s)\n", 97 dev_name(chan->router->dev)); 98 else 99 seq_puts(s, "\n"); 100 } 101 } 102 } 103 104 static int dmaengine_summary_show(struct seq_file *s, void *data) 105 { 106 struct dma_device *dma_dev = NULL; 107 108 mutex_lock(&dma_list_mutex); 109 list_for_each_entry(dma_dev, &dma_device_list, global_node) { 110 seq_printf(s, "dma%d (%s): number of channels: %u\n", 111 dma_dev->dev_id, dev_name(dma_dev->dev), 112 dma_dev->chancnt); 113 114 if (dma_dev->dbg_summary_show) 115 dma_dev->dbg_summary_show(s, dma_dev); 116 else 117 dmaengine_dbg_summary_show(s, dma_dev); 118 119 if (!list_is_last(&dma_dev->global_node, &dma_device_list)) 120 seq_puts(s, "\n"); 121 } 122 mutex_unlock(&dma_list_mutex); 123 124 return 0; 125 } 126 DEFINE_SHOW_ATTRIBUTE(dmaengine_summary); 127 128 static void __init dmaengine_debugfs_init(void) 129 { 130 rootdir = debugfs_create_dir("dmaengine", NULL); 131 132 /* /sys/kernel/debug/dmaengine/summary */ 133 debugfs_create_file("summary", 0444, rootdir, NULL, 134 &dmaengine_summary_fops); 135 } 136 #else 137 static inline void dmaengine_debugfs_init(void) { } 138 static inline int dmaengine_debug_register(struct dma_device *dma_dev) 139 { 140 return 0; 141 } 142 143 static inline void dmaengine_debug_unregister(struct dma_device *dma_dev) { } 144 #endif /* DEBUG_FS */ 145 146 /* --- sysfs implementation --- */ 147 148 #define DMA_SLAVE_NAME "slave" 149 150 /** 151 * dev_to_dma_chan - convert a device pointer to its sysfs container object 152 * @dev: device node 153 * 154 * Must be called under dma_list_mutex. 155 */ 156 static struct dma_chan *dev_to_dma_chan(struct device *dev) 157 { 158 struct dma_chan_dev *chan_dev; 159 160 chan_dev = container_of(dev, typeof(*chan_dev), device); 161 return chan_dev->chan; 162 } 163 164 static ssize_t memcpy_count_show(struct device *dev, 165 struct device_attribute *attr, char *buf) 166 { 167 struct dma_chan *chan; 168 unsigned long count = 0; 169 int i; 170 int err; 171 172 mutex_lock(&dma_list_mutex); 173 chan = dev_to_dma_chan(dev); 174 if (chan) { 175 for_each_possible_cpu(i) 176 count += per_cpu_ptr(chan->local, i)->memcpy_count; 177 err = sysfs_emit(buf, "%lu\n", count); 178 } else 179 err = -ENODEV; 180 mutex_unlock(&dma_list_mutex); 181 182 return err; 183 } 184 static DEVICE_ATTR_RO(memcpy_count); 185 186 static ssize_t bytes_transferred_show(struct device *dev, 187 struct device_attribute *attr, char *buf) 188 { 189 struct dma_chan *chan; 190 unsigned long count = 0; 191 int i; 192 int err; 193 194 mutex_lock(&dma_list_mutex); 195 chan = dev_to_dma_chan(dev); 196 if (chan) { 197 for_each_possible_cpu(i) 198 count += per_cpu_ptr(chan->local, i)->bytes_transferred; 199 err = sysfs_emit(buf, "%lu\n", count); 200 } else 201 err = -ENODEV; 202 mutex_unlock(&dma_list_mutex); 203 204 return err; 205 } 206 static DEVICE_ATTR_RO(bytes_transferred); 207 208 static ssize_t in_use_show(struct device *dev, struct device_attribute *attr, 209 char *buf) 210 { 211 struct dma_chan *chan; 212 int err; 213 214 mutex_lock(&dma_list_mutex); 215 chan = dev_to_dma_chan(dev); 216 if (chan) 217 err = sysfs_emit(buf, "%d\n", chan->client_count); 218 else 219 err = -ENODEV; 220 mutex_unlock(&dma_list_mutex); 221 222 return err; 223 } 224 static DEVICE_ATTR_RO(in_use); 225 226 static struct attribute *dma_dev_attrs[] = { 227 &dev_attr_memcpy_count.attr, 228 &dev_attr_bytes_transferred.attr, 229 &dev_attr_in_use.attr, 230 NULL, 231 }; 232 ATTRIBUTE_GROUPS(dma_dev); 233 234 static void chan_dev_release(struct device *dev) 235 { 236 struct dma_chan_dev *chan_dev; 237 238 chan_dev = container_of(dev, typeof(*chan_dev), device); 239 kfree(chan_dev); 240 } 241 242 static struct class dma_devclass = { 243 .name = "dma", 244 .dev_groups = dma_dev_groups, 245 .dev_release = chan_dev_release, 246 }; 247 248 /* --- client and device registration --- */ 249 250 /* enable iteration over all operation types */ 251 static dma_cap_mask_t dma_cap_mask_all; 252 253 /** 254 * struct dma_chan_tbl_ent - tracks channel allocations per core/operation 255 * @chan: associated channel for this entry 256 */ 257 struct dma_chan_tbl_ent { 258 struct dma_chan *chan; 259 }; 260 261 /* percpu lookup table for memory-to-memory offload providers */ 262 static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END]; 263 264 static int __init dma_channel_table_init(void) 265 { 266 enum dma_transaction_type cap; 267 int err = 0; 268 269 bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END); 270 271 /* 'interrupt', 'private', and 'slave' are channel capabilities, 272 * but are not associated with an operation so they do not need 273 * an entry in the channel_table 274 */ 275 clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits); 276 clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits); 277 clear_bit(DMA_SLAVE, dma_cap_mask_all.bits); 278 279 for_each_dma_cap_mask(cap, dma_cap_mask_all) { 280 channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent); 281 if (!channel_table[cap]) { 282 err = -ENOMEM; 283 break; 284 } 285 } 286 287 if (err) { 288 pr_err("dmaengine dma_channel_table_init failure: %d\n", err); 289 for_each_dma_cap_mask(cap, dma_cap_mask_all) 290 free_percpu(channel_table[cap]); 291 } 292 293 return err; 294 } 295 arch_initcall(dma_channel_table_init); 296 297 /** 298 * dma_chan_is_local - checks if the channel is in the same NUMA-node as the CPU 299 * @chan: DMA channel to test 300 * @cpu: CPU index which the channel should be close to 301 * 302 * Returns true if the channel is in the same NUMA-node as the CPU. 303 */ 304 static bool dma_chan_is_local(struct dma_chan *chan, int cpu) 305 { 306 int node = dev_to_node(chan->device->dev); 307 return node == NUMA_NO_NODE || 308 cpumask_test_cpu(cpu, cpumask_of_node(node)); 309 } 310 311 /** 312 * min_chan - finds the channel with min count and in the same NUMA-node as the CPU 313 * @cap: capability to match 314 * @cpu: CPU index which the channel should be close to 315 * 316 * If some channels are close to the given CPU, the one with the lowest 317 * reference count is returned. Otherwise, CPU is ignored and only the 318 * reference count is taken into account. 319 * 320 * Must be called under dma_list_mutex. 321 */ 322 static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu) 323 { 324 struct dma_device *device; 325 struct dma_chan *chan; 326 struct dma_chan *min = NULL; 327 struct dma_chan *localmin = NULL; 328 329 list_for_each_entry(device, &dma_device_list, global_node) { 330 if (!dma_has_cap(cap, device->cap_mask) || 331 dma_has_cap(DMA_PRIVATE, device->cap_mask)) 332 continue; 333 list_for_each_entry(chan, &device->channels, device_node) { 334 if (!chan->client_count) 335 continue; 336 if (!min || chan->table_count < min->table_count) 337 min = chan; 338 339 if (dma_chan_is_local(chan, cpu)) 340 if (!localmin || 341 chan->table_count < localmin->table_count) 342 localmin = chan; 343 } 344 } 345 346 chan = localmin ? localmin : min; 347 348 if (chan) 349 chan->table_count++; 350 351 return chan; 352 } 353 354 /** 355 * dma_channel_rebalance - redistribute the available channels 356 * 357 * Optimize for CPU isolation (each CPU gets a dedicated channel for an 358 * operation type) in the SMP case, and operation isolation (avoid 359 * multi-tasking channels) in the non-SMP case. 360 * 361 * Must be called under dma_list_mutex. 362 */ 363 static void dma_channel_rebalance(void) 364 { 365 struct dma_chan *chan; 366 struct dma_device *device; 367 int cpu; 368 int cap; 369 370 /* undo the last distribution */ 371 for_each_dma_cap_mask(cap, dma_cap_mask_all) 372 for_each_possible_cpu(cpu) 373 per_cpu_ptr(channel_table[cap], cpu)->chan = NULL; 374 375 list_for_each_entry(device, &dma_device_list, global_node) { 376 if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 377 continue; 378 list_for_each_entry(chan, &device->channels, device_node) 379 chan->table_count = 0; 380 } 381 382 /* don't populate the channel_table if no clients are available */ 383 if (!dmaengine_ref_count) 384 return; 385 386 /* redistribute available channels */ 387 for_each_dma_cap_mask(cap, dma_cap_mask_all) 388 for_each_online_cpu(cpu) { 389 chan = min_chan(cap, cpu); 390 per_cpu_ptr(channel_table[cap], cpu)->chan = chan; 391 } 392 } 393 394 static int dma_device_satisfies_mask(struct dma_device *device, 395 const dma_cap_mask_t *want) 396 { 397 dma_cap_mask_t has; 398 399 bitmap_and(has.bits, want->bits, device->cap_mask.bits, 400 DMA_TX_TYPE_END); 401 return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END); 402 } 403 404 static struct module *dma_chan_to_owner(struct dma_chan *chan) 405 { 406 return chan->device->owner; 407 } 408 409 /** 410 * balance_ref_count - catch up the channel reference count 411 * @chan: channel to balance ->client_count versus dmaengine_ref_count 412 * 413 * Must be called under dma_list_mutex. 414 */ 415 static void balance_ref_count(struct dma_chan *chan) 416 { 417 struct module *owner = dma_chan_to_owner(chan); 418 419 while (chan->client_count < dmaengine_ref_count) { 420 __module_get(owner); 421 chan->client_count++; 422 } 423 } 424 425 static void dma_device_release(struct kref *ref) 426 { 427 struct dma_device *device = container_of(ref, struct dma_device, ref); 428 429 list_del_rcu(&device->global_node); 430 dma_channel_rebalance(); 431 synchronize_rcu(); 432 433 if (device->device_release) 434 device->device_release(device); 435 } 436 437 static int __must_check dma_device_get(struct dma_device *device) 438 { 439 lockdep_assert_held(&dma_list_mutex); 440 return kref_get_unless_zero(&device->ref); 441 } 442 443 static void dma_device_put(struct dma_device *device) 444 { 445 lockdep_assert_held(&dma_list_mutex); 446 kref_put(&device->ref, dma_device_release); 447 } 448 449 /** 450 * dma_chan_get - try to grab a DMA channel's parent driver module 451 * @chan: channel to grab 452 * 453 * Must be called under dma_list_mutex. 454 */ 455 static int dma_chan_get(struct dma_chan *chan) 456 { 457 struct module *owner = dma_chan_to_owner(chan); 458 int ret; 459 460 /* The channel is already in use, update client count */ 461 if (chan->client_count) { 462 __module_get(owner); 463 chan->client_count++; 464 return 0; 465 } 466 467 if (!try_module_get(owner)) 468 return -ENODEV; 469 470 if (!dma_device_get(chan->device)) { 471 ret = -ENODEV; 472 goto module_put_out; 473 } 474 475 /* allocate upon first client reference */ 476 if (chan->device->device_alloc_chan_resources) { 477 ret = chan->device->device_alloc_chan_resources(chan); 478 if (ret < 0) 479 goto err_out; 480 } 481 482 chan->client_count++; 483 484 if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask)) 485 balance_ref_count(chan); 486 487 return 0; 488 489 err_out: 490 dma_device_put(chan->device); 491 module_put_out: 492 module_put(owner); 493 return ret; 494 } 495 496 /** 497 * dma_chan_put - drop a reference to a DMA channel's parent driver module 498 * @chan: channel to release 499 * 500 * Must be called under dma_list_mutex. 501 */ 502 static void dma_chan_put(struct dma_chan *chan) 503 { 504 struct module *owner; 505 506 /* This channel is not in use, bail out */ 507 if (!chan->client_count) 508 return; 509 510 owner = dma_chan_to_owner(chan); 511 chan->client_count--; 512 513 /* This channel is not in use anymore, free it */ 514 if (!chan->client_count && chan->device->device_free_chan_resources) { 515 /* Make sure all operations have completed */ 516 dmaengine_synchronize(chan); 517 chan->device->device_free_chan_resources(chan); 518 } 519 520 /* If the channel is used via a DMA request router, free the mapping */ 521 if (chan->router && chan->router->route_free) { 522 chan->router->route_free(chan->router->dev, chan->route_data); 523 chan->router = NULL; 524 chan->route_data = NULL; 525 } 526 527 /* This channel is not in use anymore, drop the device ref */ 528 if (!chan->client_count) 529 dma_device_put(chan->device); 530 module_put(owner); 531 } 532 533 enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie) 534 { 535 enum dma_status status; 536 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); 537 538 dma_async_issue_pending(chan); 539 do { 540 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 541 if (time_after_eq(jiffies, dma_sync_wait_timeout)) { 542 dev_err(chan->device->dev, "%s: timeout!\n", __func__); 543 return DMA_ERROR; 544 } 545 if (status != DMA_IN_PROGRESS) 546 break; 547 cpu_relax(); 548 } while (1); 549 550 return status; 551 } 552 EXPORT_SYMBOL(dma_sync_wait); 553 554 /** 555 * dma_find_channel - find a channel to carry out the operation 556 * @tx_type: transaction type 557 */ 558 struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type) 559 { 560 return this_cpu_read(channel_table[tx_type]->chan); 561 } 562 EXPORT_SYMBOL(dma_find_channel); 563 564 /** 565 * dma_issue_pending_all - flush all pending operations across all channels 566 */ 567 void dma_issue_pending_all(void) 568 { 569 struct dma_device *device; 570 struct dma_chan *chan; 571 572 rcu_read_lock(); 573 list_for_each_entry_rcu(device, &dma_device_list, global_node) { 574 if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 575 continue; 576 list_for_each_entry(chan, &device->channels, device_node) 577 if (chan->client_count) 578 device->device_issue_pending(chan); 579 } 580 rcu_read_unlock(); 581 } 582 EXPORT_SYMBOL(dma_issue_pending_all); 583 584 int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps) 585 { 586 struct dma_device *device; 587 588 if (!chan || !caps) 589 return -EINVAL; 590 591 device = chan->device; 592 593 /* check if the channel supports slave transactions */ 594 if (!(test_bit(DMA_SLAVE, device->cap_mask.bits) || 595 test_bit(DMA_CYCLIC, device->cap_mask.bits))) 596 return -ENXIO; 597 598 /* 599 * Check whether it reports it uses the generic slave 600 * capabilities, if not, that means it doesn't support any 601 * kind of slave capabilities reporting. 602 */ 603 if (!device->directions) 604 return -ENXIO; 605 606 caps->src_addr_widths = device->src_addr_widths; 607 caps->dst_addr_widths = device->dst_addr_widths; 608 caps->directions = device->directions; 609 caps->min_burst = device->min_burst; 610 caps->max_burst = device->max_burst; 611 caps->max_sg_burst = device->max_sg_burst; 612 caps->residue_granularity = device->residue_granularity; 613 caps->descriptor_reuse = device->descriptor_reuse; 614 caps->cmd_pause = !!device->device_pause; 615 caps->cmd_resume = !!device->device_resume; 616 caps->cmd_terminate = !!device->device_terminate_all; 617 618 /* 619 * DMA engine device might be configured with non-uniformly 620 * distributed slave capabilities per device channels. In this 621 * case the corresponding driver may provide the device_caps 622 * callback to override the generic capabilities with 623 * channel-specific ones. 624 */ 625 if (device->device_caps) 626 device->device_caps(chan, caps); 627 628 return 0; 629 } 630 EXPORT_SYMBOL_GPL(dma_get_slave_caps); 631 632 static struct dma_chan *private_candidate(const dma_cap_mask_t *mask, 633 struct dma_device *dev, 634 dma_filter_fn fn, void *fn_param) 635 { 636 struct dma_chan *chan; 637 638 if (mask && !dma_device_satisfies_mask(dev, mask)) { 639 dev_dbg(dev->dev, "%s: wrong capabilities\n", __func__); 640 return NULL; 641 } 642 /* devices with multiple channels need special handling as we need to 643 * ensure that all channels are either private or public. 644 */ 645 if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask)) 646 list_for_each_entry(chan, &dev->channels, device_node) { 647 /* some channels are already publicly allocated */ 648 if (chan->client_count) 649 return NULL; 650 } 651 652 list_for_each_entry(chan, &dev->channels, device_node) { 653 if (chan->client_count) { 654 dev_dbg(dev->dev, "%s: %s busy\n", 655 __func__, dma_chan_name(chan)); 656 continue; 657 } 658 if (fn && !fn(chan, fn_param)) { 659 dev_dbg(dev->dev, "%s: %s filter said false\n", 660 __func__, dma_chan_name(chan)); 661 continue; 662 } 663 return chan; 664 } 665 666 return NULL; 667 } 668 669 static struct dma_chan *find_candidate(struct dma_device *device, 670 const dma_cap_mask_t *mask, 671 dma_filter_fn fn, void *fn_param) 672 { 673 struct dma_chan *chan = private_candidate(mask, device, fn, fn_param); 674 int err; 675 676 if (chan) { 677 /* Found a suitable channel, try to grab, prep, and return it. 678 * We first set DMA_PRIVATE to disable balance_ref_count as this 679 * channel will not be published in the general-purpose 680 * allocator 681 */ 682 dma_cap_set(DMA_PRIVATE, device->cap_mask); 683 device->privatecnt++; 684 err = dma_chan_get(chan); 685 686 if (err) { 687 if (err == -ENODEV) { 688 dev_dbg(device->dev, "%s: %s module removed\n", 689 __func__, dma_chan_name(chan)); 690 list_del_rcu(&device->global_node); 691 } else 692 dev_dbg(device->dev, 693 "%s: failed to get %s: (%d)\n", 694 __func__, dma_chan_name(chan), err); 695 696 if (--device->privatecnt == 0) 697 dma_cap_clear(DMA_PRIVATE, device->cap_mask); 698 699 chan = ERR_PTR(err); 700 } 701 } 702 703 return chan ? chan : ERR_PTR(-EPROBE_DEFER); 704 } 705 706 /** 707 * dma_get_slave_channel - try to get specific channel exclusively 708 * @chan: target channel 709 */ 710 struct dma_chan *dma_get_slave_channel(struct dma_chan *chan) 711 { 712 /* lock against __dma_request_channel */ 713 mutex_lock(&dma_list_mutex); 714 715 if (chan->client_count == 0) { 716 struct dma_device *device = chan->device; 717 int err; 718 719 dma_cap_set(DMA_PRIVATE, device->cap_mask); 720 device->privatecnt++; 721 err = dma_chan_get(chan); 722 if (err) { 723 dev_dbg(chan->device->dev, 724 "%s: failed to get %s: (%d)\n", 725 __func__, dma_chan_name(chan), err); 726 chan = NULL; 727 if (--device->privatecnt == 0) 728 dma_cap_clear(DMA_PRIVATE, device->cap_mask); 729 } 730 } else 731 chan = NULL; 732 733 mutex_unlock(&dma_list_mutex); 734 735 736 return chan; 737 } 738 EXPORT_SYMBOL_GPL(dma_get_slave_channel); 739 740 struct dma_chan *dma_get_any_slave_channel(struct dma_device *device) 741 { 742 dma_cap_mask_t mask; 743 struct dma_chan *chan; 744 745 dma_cap_zero(mask); 746 dma_cap_set(DMA_SLAVE, mask); 747 748 /* lock against __dma_request_channel */ 749 mutex_lock(&dma_list_mutex); 750 751 chan = find_candidate(device, &mask, NULL, NULL); 752 753 mutex_unlock(&dma_list_mutex); 754 755 return IS_ERR(chan) ? NULL : chan; 756 } 757 EXPORT_SYMBOL_GPL(dma_get_any_slave_channel); 758 759 /** 760 * __dma_request_channel - try to allocate an exclusive channel 761 * @mask: capabilities that the channel must satisfy 762 * @fn: optional callback to disposition available channels 763 * @fn_param: opaque parameter to pass to dma_filter_fn() 764 * @np: device node to look for DMA channels 765 * 766 * Returns pointer to appropriate DMA channel on success or NULL. 767 */ 768 struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, 769 dma_filter_fn fn, void *fn_param, 770 struct device_node *np) 771 { 772 struct dma_device *device, *_d; 773 struct dma_chan *chan = NULL; 774 775 /* Find a channel */ 776 mutex_lock(&dma_list_mutex); 777 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { 778 /* Finds a DMA controller with matching device node */ 779 if (np && !device_match_of_node(device->dev, np)) 780 continue; 781 782 chan = find_candidate(device, mask, fn, fn_param); 783 if (!IS_ERR(chan)) 784 break; 785 786 chan = NULL; 787 } 788 mutex_unlock(&dma_list_mutex); 789 790 pr_debug("%s: %s (%s)\n", 791 __func__, 792 chan ? "success" : "fail", 793 chan ? dma_chan_name(chan) : NULL); 794 795 return chan; 796 } 797 EXPORT_SYMBOL_GPL(__dma_request_channel); 798 799 static const struct dma_slave_map *dma_filter_match(struct dma_device *device, 800 const char *name, 801 struct device *dev) 802 { 803 int i; 804 805 if (!device->filter.mapcnt) 806 return NULL; 807 808 for (i = 0; i < device->filter.mapcnt; i++) { 809 const struct dma_slave_map *map = &device->filter.map[i]; 810 811 if (!strcmp(map->devname, dev_name(dev)) && 812 !strcmp(map->slave, name)) 813 return map; 814 } 815 816 return NULL; 817 } 818 819 /** 820 * dma_request_chan - try to allocate an exclusive slave channel 821 * @dev: pointer to client device structure 822 * @name: slave channel name 823 * 824 * Returns pointer to appropriate DMA channel on success or an error pointer. 825 */ 826 struct dma_chan *dma_request_chan(struct device *dev, const char *name) 827 { 828 struct fwnode_handle *fwnode; 829 struct dma_device *d, *_d; 830 struct dma_chan *chan = NULL; 831 832 if (WARN_ON(!dev || !name)) 833 return ERR_PTR(-EINVAL); 834 835 fwnode = dev_fwnode(dev); 836 837 if (is_of_node(fwnode)) 838 chan = of_dma_request_slave_channel(to_of_node(fwnode), name); 839 else if (is_acpi_device_node(fwnode)) 840 chan = acpi_dma_request_slave_chan_by_name(dev, name); 841 842 if (PTR_ERR(chan) == -EPROBE_DEFER) 843 return chan; 844 845 if (!IS_ERR_OR_NULL(chan)) 846 goto found; 847 848 /* Try to find the channel via the DMA filter map(s) */ 849 mutex_lock(&dma_list_mutex); 850 list_for_each_entry_safe(d, _d, &dma_device_list, global_node) { 851 dma_cap_mask_t mask; 852 const struct dma_slave_map *map = dma_filter_match(d, name, dev); 853 854 if (!map) 855 continue; 856 857 dma_cap_zero(mask); 858 dma_cap_set(DMA_SLAVE, mask); 859 860 chan = find_candidate(d, &mask, d->filter.fn, map->param); 861 if (!IS_ERR(chan)) 862 break; 863 } 864 mutex_unlock(&dma_list_mutex); 865 866 if (IS_ERR(chan)) 867 return chan; 868 if (!chan) 869 return ERR_PTR(-EPROBE_DEFER); 870 871 found: 872 #ifdef CONFIG_DEBUG_FS 873 chan->dbg_client_name = kasprintf(GFP_KERNEL, "%s:%s", dev_name(dev), name); 874 /* No functional issue if it fails, users are supposed to test before use */ 875 #endif 876 877 chan->name = kasprintf(GFP_KERNEL, "dma:%s", name); 878 if (!chan->name) 879 return chan; 880 chan->slave = dev; 881 882 if (sysfs_create_link(&dmaengine_chan_dev(chan)->kobj, &dev->kobj, 883 DMA_SLAVE_NAME)) 884 dev_warn(dev, "Cannot create DMA %s symlink\n", DMA_SLAVE_NAME); 885 if (sysfs_create_link(&dev->kobj, &dmaengine_chan_dev(chan)->kobj, chan->name)) 886 dev_warn(dev, "Cannot create DMA %s symlink\n", chan->name); 887 888 return chan; 889 } 890 EXPORT_SYMBOL_GPL(dma_request_chan); 891 892 /** 893 * dma_request_chan_by_mask - allocate a channel satisfying certain capabilities 894 * @mask: capabilities that the channel must satisfy 895 * 896 * Returns pointer to appropriate DMA channel on success or an error pointer. 897 */ 898 struct dma_chan *dma_request_chan_by_mask(const dma_cap_mask_t *mask) 899 { 900 struct dma_chan *chan; 901 902 if (!mask) 903 return ERR_PTR(-ENODEV); 904 905 chan = __dma_request_channel(mask, NULL, NULL, NULL); 906 if (!chan) { 907 mutex_lock(&dma_list_mutex); 908 if (list_empty(&dma_device_list)) 909 chan = ERR_PTR(-EPROBE_DEFER); 910 else 911 chan = ERR_PTR(-ENODEV); 912 mutex_unlock(&dma_list_mutex); 913 } 914 915 return chan; 916 } 917 EXPORT_SYMBOL_GPL(dma_request_chan_by_mask); 918 919 void dma_release_channel(struct dma_chan *chan) 920 { 921 mutex_lock(&dma_list_mutex); 922 WARN_ONCE(chan->client_count != 1, 923 "chan reference count %d != 1\n", chan->client_count); 924 /* drop PRIVATE cap enabled by __dma_request_channel() */ 925 if (--chan->device->privatecnt == 0) 926 dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask); 927 928 dma_chan_put(chan); 929 930 if (chan->slave) { 931 sysfs_remove_link(&dmaengine_chan_dev(chan)->kobj, DMA_SLAVE_NAME); 932 sysfs_remove_link(&chan->slave->kobj, chan->name); 933 kfree(chan->name); 934 chan->name = NULL; 935 chan->slave = NULL; 936 } 937 938 #ifdef CONFIG_DEBUG_FS 939 kfree(chan->dbg_client_name); 940 chan->dbg_client_name = NULL; 941 #endif 942 mutex_unlock(&dma_list_mutex); 943 } 944 EXPORT_SYMBOL_GPL(dma_release_channel); 945 946 static void dmaenginem_release_channel(void *chan) 947 { 948 dma_release_channel(chan); 949 } 950 951 /** 952 * devm_dma_request_chan - try to allocate an exclusive slave channel 953 * @dev: pointer to client device structure 954 * @name: slave channel name 955 * 956 * Returns pointer to appropriate DMA channel on success or an error pointer. 957 * 958 * The operation is managed and will be undone on driver detach. 959 */ 960 961 struct dma_chan *devm_dma_request_chan(struct device *dev, const char *name) 962 { 963 struct dma_chan *chan; 964 int ret; 965 966 chan = dma_request_chan(dev, name); 967 if (IS_ERR(chan)) 968 return chan; 969 970 ret = devm_add_action_or_reset(dev, dmaenginem_release_channel, chan); 971 if (ret) 972 return ERR_PTR(ret); 973 974 return chan; 975 } 976 EXPORT_SYMBOL_GPL(devm_dma_request_chan); 977 978 /** 979 * dmaengine_get - register interest in dma_channels 980 */ 981 void dmaengine_get(void) 982 { 983 struct dma_device *device, *_d; 984 struct dma_chan *chan; 985 int err; 986 987 mutex_lock(&dma_list_mutex); 988 dmaengine_ref_count++; 989 990 /* try to grab channels */ 991 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { 992 if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 993 continue; 994 list_for_each_entry(chan, &device->channels, device_node) { 995 err = dma_chan_get(chan); 996 if (err == -ENODEV) { 997 /* module removed before we could use it */ 998 list_del_rcu(&device->global_node); 999 break; 1000 } else if (err) 1001 dev_dbg(chan->device->dev, 1002 "%s: failed to get %s: (%d)\n", 1003 __func__, dma_chan_name(chan), err); 1004 } 1005 } 1006 1007 /* if this is the first reference and there were channels 1008 * waiting we need to rebalance to get those channels 1009 * incorporated into the channel table 1010 */ 1011 if (dmaengine_ref_count == 1) 1012 dma_channel_rebalance(); 1013 mutex_unlock(&dma_list_mutex); 1014 } 1015 EXPORT_SYMBOL(dmaengine_get); 1016 1017 /** 1018 * dmaengine_put - let DMA drivers be removed when ref_count == 0 1019 */ 1020 void dmaengine_put(void) 1021 { 1022 struct dma_device *device, *_d; 1023 struct dma_chan *chan; 1024 1025 mutex_lock(&dma_list_mutex); 1026 dmaengine_ref_count--; 1027 BUG_ON(dmaengine_ref_count < 0); 1028 /* drop channel references */ 1029 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { 1030 if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 1031 continue; 1032 list_for_each_entry(chan, &device->channels, device_node) 1033 dma_chan_put(chan); 1034 } 1035 mutex_unlock(&dma_list_mutex); 1036 } 1037 EXPORT_SYMBOL(dmaengine_put); 1038 1039 static bool device_has_all_tx_types(struct dma_device *device) 1040 { 1041 /* A device that satisfies this test has channels that will never cause 1042 * an async_tx channel switch event as all possible operation types can 1043 * be handled. 1044 */ 1045 #ifdef CONFIG_ASYNC_TX_DMA 1046 if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask)) 1047 return false; 1048 #endif 1049 1050 #if IS_ENABLED(CONFIG_ASYNC_MEMCPY) 1051 if (!dma_has_cap(DMA_MEMCPY, device->cap_mask)) 1052 return false; 1053 #endif 1054 1055 #if IS_ENABLED(CONFIG_ASYNC_XOR) 1056 if (!dma_has_cap(DMA_XOR, device->cap_mask)) 1057 return false; 1058 1059 #ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA 1060 if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask)) 1061 return false; 1062 #endif 1063 #endif 1064 1065 #if IS_ENABLED(CONFIG_ASYNC_PQ) 1066 if (!dma_has_cap(DMA_PQ, device->cap_mask)) 1067 return false; 1068 1069 #ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA 1070 if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask)) 1071 return false; 1072 #endif 1073 #endif 1074 1075 return true; 1076 } 1077 1078 static int get_dma_id(struct dma_device *device) 1079 { 1080 int rc = ida_alloc(&dma_ida, GFP_KERNEL); 1081 1082 if (rc < 0) 1083 return rc; 1084 device->dev_id = rc; 1085 return 0; 1086 } 1087 1088 static int __dma_async_device_channel_register(struct dma_device *device, 1089 struct dma_chan *chan, 1090 const char *name) 1091 { 1092 unsigned int id; 1093 int rc; 1094 1095 chan->local = alloc_percpu(typeof(*chan->local)); 1096 if (!chan->local) 1097 return -ENOMEM; 1098 chan->chan_dev = kzalloc_obj(*chan->chan_dev); 1099 if (!chan->chan_dev) { 1100 rc = -ENOMEM; 1101 goto err_free_local; 1102 } 1103 1104 if (chan->chan_id & DMA_CHAN_ID_STATIC) { 1105 id = chan->chan_id & ~DMA_CHAN_ID_STATIC; 1106 chan->chan_id = ida_alloc_range(&device->chan_ida, id, id, 1107 GFP_KERNEL); 1108 } else { 1109 chan->chan_id = ida_alloc(&device->chan_ida, GFP_KERNEL); 1110 } 1111 if (chan->chan_id < 0) { 1112 pr_err("%s: unable to alloc ida for chan: %d\n", 1113 __func__, chan->chan_id); 1114 rc = chan->chan_id; 1115 goto err_free_dev; 1116 } 1117 1118 dmaengine_chan_dev(chan)->class = &dma_devclass; 1119 dmaengine_chan_dev(chan)->parent = device->dev; 1120 chan->chan_dev->chan = chan; 1121 chan->chan_dev->dev_id = device->dev_id; 1122 spin_lock_init(&chan->lock); 1123 1124 if (!name) 1125 dev_set_name(dmaengine_chan_dev(chan), "dma%dchan%d", device->dev_id, chan->chan_id); 1126 else 1127 dev_set_name(dmaengine_chan_dev(chan), "%s", name); 1128 rc = device_register(dmaengine_chan_dev(chan)); 1129 if (rc) 1130 goto err_out_ida; 1131 chan->client_count = 0; 1132 device->chancnt++; 1133 1134 return 0; 1135 1136 err_out_ida: 1137 ida_free(&device->chan_ida, chan->chan_id); 1138 err_free_dev: 1139 kfree(chan->chan_dev); 1140 err_free_local: 1141 free_percpu(chan->local); 1142 chan->local = NULL; 1143 return rc; 1144 } 1145 1146 int dma_async_device_channel_register(struct dma_device *device, 1147 struct dma_chan *chan, 1148 const char *name) 1149 { 1150 int rc; 1151 1152 rc = __dma_async_device_channel_register(device, chan, name); 1153 if (rc < 0) 1154 return rc; 1155 1156 dma_channel_rebalance(); 1157 return 0; 1158 } 1159 EXPORT_SYMBOL_GPL(dma_async_device_channel_register); 1160 1161 static void __dma_async_device_channel_unregister(struct dma_device *device, 1162 struct dma_chan *chan) 1163 { 1164 if (chan->local == NULL) 1165 return; 1166 1167 WARN_ONCE(!device->device_release && chan->client_count, 1168 "%s called while %d clients hold a reference\n", 1169 __func__, chan->client_count); 1170 mutex_lock(&dma_list_mutex); 1171 device->chancnt--; 1172 chan->chan_dev->chan = NULL; 1173 mutex_unlock(&dma_list_mutex); 1174 ida_free(&device->chan_ida, chan->chan_id); 1175 device_unregister(dmaengine_chan_dev(chan)); 1176 free_percpu(chan->local); 1177 } 1178 1179 void dma_async_device_channel_unregister(struct dma_device *device, 1180 struct dma_chan *chan) 1181 { 1182 __dma_async_device_channel_unregister(device, chan); 1183 dma_channel_rebalance(); 1184 } 1185 EXPORT_SYMBOL_GPL(dma_async_device_channel_unregister); 1186 1187 /** 1188 * dma_async_device_register - registers DMA devices found 1189 * @device: pointer to &struct dma_device 1190 * 1191 * After calling this routine the structure should not be freed except in the 1192 * device_release() callback which will be called after 1193 * dma_async_device_unregister() is called and no further references are taken. 1194 */ 1195 int dma_async_device_register(struct dma_device *device) 1196 { 1197 int rc; 1198 struct dma_chan* chan; 1199 1200 if (!device) 1201 return -ENODEV; 1202 1203 /* validate device routines */ 1204 if (!device->dev) { 1205 pr_err("DMAdevice must have dev\n"); 1206 return -EIO; 1207 } 1208 1209 device->owner = device->dev->driver->owner; 1210 1211 #define CHECK_CAP(_name, _type) \ 1212 { \ 1213 if (dma_has_cap(_type, device->cap_mask) && !device->device_prep_##_name) { \ 1214 dev_err(device->dev, \ 1215 "Device claims capability %s, but op is not defined\n", \ 1216 __stringify(_type)); \ 1217 return -EIO; \ 1218 } \ 1219 } 1220 1221 CHECK_CAP(dma_memcpy, DMA_MEMCPY); 1222 CHECK_CAP(dma_xor, DMA_XOR); 1223 CHECK_CAP(dma_xor_val, DMA_XOR_VAL); 1224 CHECK_CAP(dma_pq, DMA_PQ); 1225 CHECK_CAP(dma_pq_val, DMA_PQ_VAL); 1226 CHECK_CAP(dma_memset, DMA_MEMSET); 1227 CHECK_CAP(dma_interrupt, DMA_INTERRUPT); 1228 CHECK_CAP(dma_cyclic, DMA_CYCLIC); 1229 CHECK_CAP(interleaved_dma, DMA_INTERLEAVE); 1230 1231 #undef CHECK_CAP 1232 1233 if (!device->device_tx_status) { 1234 dev_err(device->dev, "Device tx_status is not defined\n"); 1235 return -EIO; 1236 } 1237 1238 1239 if (!device->device_issue_pending) { 1240 dev_err(device->dev, "Device issue_pending is not defined\n"); 1241 return -EIO; 1242 } 1243 1244 if (!device->device_release) 1245 dev_dbg(device->dev, 1246 "WARN: Device release is not defined so it is not safe to unbind this driver while in use\n"); 1247 1248 kref_init(&device->ref); 1249 1250 /* note: this only matters in the 1251 * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case 1252 */ 1253 if (device_has_all_tx_types(device)) 1254 dma_cap_set(DMA_ASYNC_TX, device->cap_mask); 1255 1256 rc = get_dma_id(device); 1257 if (rc != 0) 1258 return rc; 1259 1260 ida_init(&device->chan_ida); 1261 1262 /* represent channels in sysfs. Probably want devs too */ 1263 list_for_each_entry(chan, &device->channels, device_node) { 1264 rc = __dma_async_device_channel_register(device, chan, NULL); 1265 if (rc < 0) 1266 goto err_out; 1267 } 1268 1269 mutex_lock(&dma_list_mutex); 1270 /* take references on public channels */ 1271 if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask)) 1272 list_for_each_entry(chan, &device->channels, device_node) { 1273 /* if clients are already waiting for channels we need 1274 * to take references on their behalf 1275 */ 1276 if (dma_chan_get(chan) == -ENODEV) { 1277 /* note we can only get here for the first 1278 * channel as the remaining channels are 1279 * guaranteed to get a reference 1280 */ 1281 rc = -ENODEV; 1282 mutex_unlock(&dma_list_mutex); 1283 goto err_out; 1284 } 1285 } 1286 list_add_tail_rcu(&device->global_node, &dma_device_list); 1287 if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 1288 device->privatecnt++; /* Always private */ 1289 dma_channel_rebalance(); 1290 mutex_unlock(&dma_list_mutex); 1291 1292 dmaengine_debug_register(device); 1293 1294 return 0; 1295 1296 err_out: 1297 /* if we never registered a channel just release the idr */ 1298 if (!device->chancnt) { 1299 ida_free(&dma_ida, device->dev_id); 1300 return rc; 1301 } 1302 1303 list_for_each_entry(chan, &device->channels, device_node) { 1304 if (chan->local == NULL) 1305 continue; 1306 mutex_lock(&dma_list_mutex); 1307 chan->chan_dev->chan = NULL; 1308 mutex_unlock(&dma_list_mutex); 1309 device_unregister(dmaengine_chan_dev(chan)); 1310 free_percpu(chan->local); 1311 } 1312 return rc; 1313 } 1314 EXPORT_SYMBOL(dma_async_device_register); 1315 1316 /** 1317 * dma_async_device_unregister - unregister a DMA device 1318 * @device: pointer to &struct dma_device 1319 * 1320 * This routine is called by dma driver exit routines, dmaengine holds module 1321 * references to prevent it being called while channels are in use. 1322 */ 1323 void dma_async_device_unregister(struct dma_device *device) 1324 { 1325 struct dma_chan *chan, *n; 1326 1327 dmaengine_debug_unregister(device); 1328 1329 list_for_each_entry_safe(chan, n, &device->channels, device_node) 1330 __dma_async_device_channel_unregister(device, chan); 1331 1332 mutex_lock(&dma_list_mutex); 1333 /* 1334 * setting DMA_PRIVATE ensures the device being torn down will not 1335 * be used in the channel_table 1336 */ 1337 dma_cap_set(DMA_PRIVATE, device->cap_mask); 1338 dma_channel_rebalance(); 1339 ida_free(&dma_ida, device->dev_id); 1340 dma_device_put(device); 1341 mutex_unlock(&dma_list_mutex); 1342 } 1343 EXPORT_SYMBOL(dma_async_device_unregister); 1344 1345 static void dmaenginem_async_device_unregister(void *device) 1346 { 1347 dma_async_device_unregister(device); 1348 } 1349 1350 /** 1351 * dmaenginem_async_device_register - registers DMA devices found 1352 * @device: pointer to &struct dma_device 1353 * 1354 * The operation is managed and will be undone on driver detach. 1355 */ 1356 int dmaenginem_async_device_register(struct dma_device *device) 1357 { 1358 int ret; 1359 1360 ret = dma_async_device_register(device); 1361 if (ret) 1362 return ret; 1363 1364 return devm_add_action_or_reset(device->dev, dmaenginem_async_device_unregister, device); 1365 } 1366 EXPORT_SYMBOL(dmaenginem_async_device_register); 1367 1368 struct dmaengine_unmap_pool { 1369 struct kmem_cache *cache; 1370 const char *name; 1371 mempool_t *pool; 1372 size_t size; 1373 }; 1374 1375 #define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) } 1376 static struct dmaengine_unmap_pool unmap_pool[] = { 1377 __UNMAP_POOL(2), 1378 #if IS_ENABLED(CONFIG_DMA_ENGINE_RAID) 1379 __UNMAP_POOL(16), 1380 __UNMAP_POOL(128), 1381 __UNMAP_POOL(256), 1382 #endif 1383 }; 1384 1385 static struct dmaengine_unmap_pool *__get_unmap_pool(int nr) 1386 { 1387 int order = get_count_order(nr); 1388 1389 switch (order) { 1390 case 0 ... 1: 1391 return &unmap_pool[0]; 1392 #if IS_ENABLED(CONFIG_DMA_ENGINE_RAID) 1393 case 2 ... 4: 1394 return &unmap_pool[1]; 1395 case 5 ... 7: 1396 return &unmap_pool[2]; 1397 case 8: 1398 return &unmap_pool[3]; 1399 #endif 1400 default: 1401 BUG(); 1402 return NULL; 1403 } 1404 } 1405 1406 static void dmaengine_unmap(struct kref *kref) 1407 { 1408 struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref); 1409 struct device *dev = unmap->dev; 1410 int cnt, i; 1411 1412 cnt = unmap->to_cnt; 1413 for (i = 0; i < cnt; i++) 1414 dma_unmap_page(dev, unmap->addr[i], unmap->len, 1415 DMA_TO_DEVICE); 1416 cnt += unmap->from_cnt; 1417 for (; i < cnt; i++) 1418 dma_unmap_page(dev, unmap->addr[i], unmap->len, 1419 DMA_FROM_DEVICE); 1420 cnt += unmap->bidi_cnt; 1421 for (; i < cnt; i++) { 1422 if (unmap->addr[i] == 0) 1423 continue; 1424 dma_unmap_page(dev, unmap->addr[i], unmap->len, 1425 DMA_BIDIRECTIONAL); 1426 } 1427 cnt = unmap->map_cnt; 1428 mempool_free(unmap, __get_unmap_pool(cnt)->pool); 1429 } 1430 1431 void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap) 1432 { 1433 if (unmap) 1434 kref_put(&unmap->kref, dmaengine_unmap); 1435 } 1436 EXPORT_SYMBOL_GPL(dmaengine_unmap_put); 1437 1438 static void dmaengine_destroy_unmap_pool(void) 1439 { 1440 int i; 1441 1442 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) { 1443 struct dmaengine_unmap_pool *p = &unmap_pool[i]; 1444 1445 mempool_destroy(p->pool); 1446 p->pool = NULL; 1447 kmem_cache_destroy(p->cache); 1448 p->cache = NULL; 1449 } 1450 } 1451 1452 static int __init dmaengine_init_unmap_pool(void) 1453 { 1454 int i; 1455 1456 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) { 1457 struct dmaengine_unmap_pool *p = &unmap_pool[i]; 1458 size_t size; 1459 1460 size = sizeof(struct dmaengine_unmap_data) + 1461 sizeof(dma_addr_t) * p->size; 1462 1463 p->cache = kmem_cache_create(p->name, size, 0, 1464 SLAB_HWCACHE_ALIGN, NULL); 1465 if (!p->cache) 1466 break; 1467 p->pool = mempool_create_slab_pool(1, p->cache); 1468 if (!p->pool) 1469 break; 1470 } 1471 1472 if (i == ARRAY_SIZE(unmap_pool)) 1473 return 0; 1474 1475 dmaengine_destroy_unmap_pool(); 1476 return -ENOMEM; 1477 } 1478 1479 struct dmaengine_unmap_data * 1480 dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags) 1481 { 1482 struct dmaengine_unmap_data *unmap; 1483 1484 unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags); 1485 if (!unmap) 1486 return NULL; 1487 1488 memset(unmap, 0, sizeof(*unmap)); 1489 kref_init(&unmap->kref); 1490 unmap->dev = dev; 1491 unmap->map_cnt = nr; 1492 1493 return unmap; 1494 } 1495 EXPORT_SYMBOL(dmaengine_get_unmap_data); 1496 1497 void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, 1498 struct dma_chan *chan) 1499 { 1500 tx->chan = chan; 1501 #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH 1502 spin_lock_init(&tx->lock); 1503 #endif 1504 } 1505 EXPORT_SYMBOL(dma_async_tx_descriptor_init); 1506 1507 static inline int desc_check_and_set_metadata_mode( 1508 struct dma_async_tx_descriptor *desc, enum dma_desc_metadata_mode mode) 1509 { 1510 /* Make sure that the metadata mode is not mixed */ 1511 if (!desc->desc_metadata_mode) { 1512 if (dmaengine_is_metadata_mode_supported(desc->chan, mode)) 1513 desc->desc_metadata_mode = mode; 1514 else 1515 return -ENOTSUPP; 1516 } else if (desc->desc_metadata_mode != mode) { 1517 return -EINVAL; 1518 } 1519 1520 return 0; 1521 } 1522 1523 int dmaengine_desc_attach_metadata(struct dma_async_tx_descriptor *desc, 1524 void *data, size_t len) 1525 { 1526 int ret; 1527 1528 if (!desc) 1529 return -EINVAL; 1530 1531 ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_CLIENT); 1532 if (ret) 1533 return ret; 1534 1535 if (!desc->metadata_ops || !desc->metadata_ops->attach) 1536 return -ENOTSUPP; 1537 1538 return desc->metadata_ops->attach(desc, data, len); 1539 } 1540 EXPORT_SYMBOL_GPL(dmaengine_desc_attach_metadata); 1541 1542 void *dmaengine_desc_get_metadata_ptr(struct dma_async_tx_descriptor *desc, 1543 size_t *payload_len, size_t *max_len) 1544 { 1545 int ret; 1546 1547 if (!desc) 1548 return ERR_PTR(-EINVAL); 1549 1550 ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE); 1551 if (ret) 1552 return ERR_PTR(ret); 1553 1554 if (!desc->metadata_ops || !desc->metadata_ops->get_ptr) 1555 return ERR_PTR(-ENOTSUPP); 1556 1557 return desc->metadata_ops->get_ptr(desc, payload_len, max_len); 1558 } 1559 EXPORT_SYMBOL_GPL(dmaengine_desc_get_metadata_ptr); 1560 1561 int dmaengine_desc_set_metadata_len(struct dma_async_tx_descriptor *desc, 1562 size_t payload_len) 1563 { 1564 int ret; 1565 1566 if (!desc) 1567 return -EINVAL; 1568 1569 ret = desc_check_and_set_metadata_mode(desc, DESC_METADATA_ENGINE); 1570 if (ret) 1571 return ret; 1572 1573 if (!desc->metadata_ops || !desc->metadata_ops->set_len) 1574 return -ENOTSUPP; 1575 1576 return desc->metadata_ops->set_len(desc, payload_len); 1577 } 1578 EXPORT_SYMBOL_GPL(dmaengine_desc_set_metadata_len); 1579 1580 /** 1581 * dma_wait_for_async_tx - spin wait for a transaction to complete 1582 * @tx: in-flight transaction to wait on 1583 */ 1584 enum dma_status 1585 dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx) 1586 { 1587 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); 1588 1589 if (!tx) 1590 return DMA_COMPLETE; 1591 1592 while (tx->cookie == -EBUSY) { 1593 if (time_after_eq(jiffies, dma_sync_wait_timeout)) { 1594 dev_err(tx->chan->device->dev, 1595 "%s timeout waiting for descriptor submission\n", 1596 __func__); 1597 return DMA_ERROR; 1598 } 1599 cpu_relax(); 1600 } 1601 return dma_sync_wait(tx->chan, tx->cookie); 1602 } 1603 EXPORT_SYMBOL_GPL(dma_wait_for_async_tx); 1604 1605 /** 1606 * dma_run_dependencies - process dependent operations on the target channel 1607 * @tx: transaction with dependencies 1608 * 1609 * Helper routine for DMA drivers to process (start) dependent operations 1610 * on their target channel. 1611 */ 1612 void dma_run_dependencies(struct dma_async_tx_descriptor *tx) 1613 { 1614 struct dma_async_tx_descriptor *dep = txd_next(tx); 1615 struct dma_async_tx_descriptor *dep_next; 1616 struct dma_chan *chan; 1617 1618 if (!dep) 1619 return; 1620 1621 /* we'll submit tx->next now, so clear the link */ 1622 txd_clear_next(tx); 1623 chan = dep->chan; 1624 1625 /* keep submitting up until a channel switch is detected 1626 * in that case we will be called again as a result of 1627 * processing the interrupt from async_tx_channel_switch 1628 */ 1629 for (; dep; dep = dep_next) { 1630 txd_lock(dep); 1631 txd_clear_parent(dep); 1632 dep_next = txd_next(dep); 1633 if (dep_next && dep_next->chan == chan) 1634 txd_clear_next(dep); /* ->next will be submitted */ 1635 else 1636 dep_next = NULL; /* submit current dep and terminate */ 1637 txd_unlock(dep); 1638 1639 dep->tx_submit(dep); 1640 } 1641 1642 chan->device->device_issue_pending(chan); 1643 } 1644 EXPORT_SYMBOL_GPL(dma_run_dependencies); 1645 1646 static int __init dma_bus_init(void) 1647 { 1648 int err = dmaengine_init_unmap_pool(); 1649 1650 if (err) 1651 return err; 1652 1653 err = class_register(&dma_devclass); 1654 if (!err) 1655 dmaengine_debugfs_init(); 1656 1657 return err; 1658 } 1659 arch_initcall(dma_bus_init); 1660