Lines Matching +full:free +full:- +full:running
4 * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.
6 * This program is free software; you can redistribute it and/or modify
8 * only version 2 as published by the Free Software Foundation.
24 * written by Hongjun Chen <hong-jun.chen@freescale.com>.
29 * This program is free software; you can redistribute it and/or modify it
30 * under the terms of the GNU General Public License as published by the Free
46 #include <linux/dma-mapping.h>
94 INIT_LIST_HEAD(&dmadev->ddev.channels); in hidma_free()
109 struct dma_device *ddev = mchan->chan.device; in hidma_process_completed()
121 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_process_completed()
122 list_splice_tail_init(&mchan->completed, &list); in hidma_process_completed()
123 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_process_completed()
131 desc = &mdesc->desc; in hidma_process_completed()
132 last_cookie = desc->cookie; in hidma_process_completed()
134 llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); in hidma_process_completed()
136 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_process_completed()
138 mchan->last_success = last_cookie; in hidma_process_completed()
145 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_process_completed()
151 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_process_completed()
152 list_move(&mdesc->node, &mchan->free); in hidma_process_completed()
153 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_process_completed()
167 struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan); in hidma_callback()
168 struct dma_device *ddev = mchan->chan.device; in hidma_callback()
173 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_callback()
174 if (mdesc->node.next) { in hidma_callback()
176 list_move_tail(&mdesc->node, &mchan->completed); in hidma_callback()
179 /* calculate the next running descriptor */ in hidma_callback()
180 mchan->running = list_first_entry(&mchan->active, in hidma_callback()
183 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_callback()
188 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_callback()
189 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_callback()
198 mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL); in hidma_chan_init()
200 return -ENOMEM; in hidma_chan_init()
202 ddev = &dmadev->ddev; in hidma_chan_init()
203 mchan->dma_sig = dma_sig; in hidma_chan_init()
204 mchan->dmadev = dmadev; in hidma_chan_init()
205 mchan->chan.device = ddev; in hidma_chan_init()
206 dma_cookie_init(&mchan->chan); in hidma_chan_init()
208 INIT_LIST_HEAD(&mchan->free); in hidma_chan_init()
209 INIT_LIST_HEAD(&mchan->prepared); in hidma_chan_init()
210 INIT_LIST_HEAD(&mchan->active); in hidma_chan_init()
211 INIT_LIST_HEAD(&mchan->completed); in hidma_chan_init()
212 INIT_LIST_HEAD(&mchan->queued); in hidma_chan_init()
214 spin_lock_init(&mchan->lock); in hidma_chan_init()
215 list_add_tail(&mchan->chan.device_node, &ddev->channels); in hidma_chan_init()
223 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_issue_task()
224 hidma_ll_start(dmadev->lldev); in hidma_issue_task()
230 struct hidma_dev *dmadev = mchan->dmadev; in hidma_issue_pending()
235 spin_lock_irqsave(&mchan->lock, flags); in hidma_issue_pending()
236 list_for_each_entry_safe(qdesc, next, &mchan->queued, node) { in hidma_issue_pending()
237 hidma_ll_queue_request(dmadev->lldev, qdesc->tre_ch); in hidma_issue_pending()
238 list_move_tail(&qdesc->node, &mchan->active); in hidma_issue_pending()
241 if (!mchan->running) { in hidma_issue_pending()
242 struct hidma_desc *desc = list_first_entry(&mchan->active, in hidma_issue_pending()
245 mchan->running = desc; in hidma_issue_pending()
247 spin_unlock_irqrestore(&mchan->lock, flags); in hidma_issue_pending()
250 status = pm_runtime_get(dmadev->ddev.dev); in hidma_issue_pending()
252 tasklet_schedule(&dmadev->task); in hidma_issue_pending()
254 hidma_ll_start(dmadev->lldev); in hidma_issue_pending()
281 is_success = hidma_txn_is_success(cookie, mchan->last_success, in hidma_tx_status()
282 dmach->cookie); in hidma_tx_status()
286 if (mchan->paused && (ret == DMA_IN_PROGRESS)) { in hidma_tx_status()
290 spin_lock_irqsave(&mchan->lock, flags); in hidma_tx_status()
291 if (mchan->running) in hidma_tx_status()
292 runcookie = mchan->running->desc.cookie; in hidma_tx_status()
294 runcookie = -EINVAL; in hidma_tx_status()
299 spin_unlock_irqrestore(&mchan->lock, flags); in hidma_tx_status()
311 struct hidma_chan *mchan = to_hidma_chan(txd->chan); in hidma_tx_submit()
312 struct hidma_dev *dmadev = mchan->dmadev; in hidma_tx_submit()
317 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_tx_submit()
318 if (!hidma_ll_isenabled(dmadev->lldev)) { in hidma_tx_submit()
319 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_tx_submit()
320 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_tx_submit()
321 return -ENODEV; in hidma_tx_submit()
323 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_tx_submit()
324 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_tx_submit()
327 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_tx_submit()
330 list_move_tail(&mdesc->node, &mchan->queued); in hidma_tx_submit()
335 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_tx_submit()
343 struct hidma_dev *dmadev = mchan->dmadev; in hidma_alloc_chan_resources()
350 if (mchan->allocated) in hidma_alloc_chan_resources()
354 for (i = 0; i < dmadev->nr_descriptors; i++) { in hidma_alloc_chan_resources()
357 rc = -ENOMEM; in hidma_alloc_chan_resources()
360 dma_async_tx_descriptor_init(&mdesc->desc, dmach); in hidma_alloc_chan_resources()
361 mdesc->desc.tx_submit = hidma_tx_submit; in hidma_alloc_chan_resources()
363 rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig, in hidma_alloc_chan_resources()
365 &mdesc->tre_ch); in hidma_alloc_chan_resources()
367 dev_err(dmach->device->dev, in hidma_alloc_chan_resources()
372 list_add_tail(&mdesc->node, &descs); in hidma_alloc_chan_resources()
378 hidma_ll_free(dmadev->lldev, mdesc->tre_ch); in hidma_alloc_chan_resources()
384 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_alloc_chan_resources()
385 list_splice_tail_init(&descs, &mchan->free); in hidma_alloc_chan_resources()
386 mchan->allocated = true; in hidma_alloc_chan_resources()
387 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_alloc_chan_resources()
397 struct hidma_dev *mdma = mchan->dmadev; in hidma_prep_dma_memcpy()
400 /* Get free descriptor */ in hidma_prep_dma_memcpy()
401 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
402 if (!list_empty(&mchan->free)) { in hidma_prep_dma_memcpy()
403 mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); in hidma_prep_dma_memcpy()
404 list_del(&mdesc->node); in hidma_prep_dma_memcpy()
406 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
411 mdesc->desc.flags = flags; in hidma_prep_dma_memcpy()
412 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, in hidma_prep_dma_memcpy()
417 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
418 list_add_tail(&mdesc->node, &mchan->prepared); in hidma_prep_dma_memcpy()
419 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
421 return &mdesc->desc; in hidma_prep_dma_memcpy()
430 struct hidma_dev *mdma = mchan->dmadev; in hidma_prep_dma_memset()
434 /* Get free descriptor */ in hidma_prep_dma_memset()
435 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memset()
436 if (!list_empty(&mchan->free)) { in hidma_prep_dma_memset()
437 mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); in hidma_prep_dma_memset()
438 list_del(&mdesc->node); in hidma_prep_dma_memset()
440 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memset()
455 mdesc->desc.flags = flags; in hidma_prep_dma_memset()
456 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, in hidma_prep_dma_memset()
461 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memset()
462 list_add_tail(&mdesc->node, &mchan->prepared); in hidma_prep_dma_memset()
463 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memset()
465 return &mdesc->desc; in hidma_prep_dma_memset()
471 struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); in hidma_terminate_channel()
477 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_terminate_channel()
481 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_terminate_channel()
482 mchan->last_success = 0; in hidma_terminate_channel()
483 list_splice_init(&mchan->active, &list); in hidma_terminate_channel()
484 list_splice_init(&mchan->prepared, &list); in hidma_terminate_channel()
485 list_splice_init(&mchan->completed, &list); in hidma_terminate_channel()
486 list_splice_init(&mchan->queued, &list); in hidma_terminate_channel()
487 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_terminate_channel()
490 rc = hidma_ll_disable(dmadev->lldev); in hidma_terminate_channel()
492 dev_err(dmadev->ddev.dev, "channel did not pause\n"); in hidma_terminate_channel()
498 struct dma_async_tx_descriptor *txd = &mdesc->desc; in hidma_terminate_channel()
505 list_move(&mdesc->node, &mchan->free); in hidma_terminate_channel()
508 rc = hidma_ll_enable(dmadev->lldev); in hidma_terminate_channel()
510 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_terminate_channel()
511 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_terminate_channel()
518 struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); in hidma_terminate_all()
526 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_terminate_all()
527 rc = hidma_ll_setup(dmadev->lldev); in hidma_terminate_all()
528 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_terminate_all()
529 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_terminate_all()
536 struct hidma_dev *mdma = mchan->dmadev; in hidma_free_chan_resources()
541 /* terminate running transactions and free descriptors */ in hidma_free_chan_resources()
544 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_free_chan_resources()
547 list_splice_tail_init(&mchan->free, &descs); in hidma_free_chan_resources()
549 /* Free descriptors */ in hidma_free_chan_resources()
551 hidma_ll_free(mdma->lldev, mdesc->tre_ch); in hidma_free_chan_resources()
552 list_del(&mdesc->node); in hidma_free_chan_resources()
556 mchan->allocated = false; in hidma_free_chan_resources()
557 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_free_chan_resources()
566 dmadev = to_hidma_dev(mchan->chan.device); in hidma_pause()
567 if (!mchan->paused) { in hidma_pause()
568 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_pause()
569 if (hidma_ll_disable(dmadev->lldev)) in hidma_pause()
570 dev_warn(dmadev->ddev.dev, "channel did not stop\n"); in hidma_pause()
571 mchan->paused = true; in hidma_pause()
572 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_pause()
573 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_pause()
585 dmadev = to_hidma_dev(mchan->chan.device); in hidma_resume()
586 if (mchan->paused) { in hidma_resume()
587 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_resume()
588 rc = hidma_ll_enable(dmadev->lldev); in hidma_resume()
590 mchan->paused = false; in hidma_resume()
592 dev_err(dmadev->ddev.dev, in hidma_resume()
594 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_resume()
595 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_resume()
618 1 << (chirq - dmadev->msi_virqbase)); in hidma_chirq_handler_msi()
629 if (strcmp(attr->attr.name, "chid") == 0) in hidma_show_values()
630 sprintf(buf, "%d\n", mdev->chidx); in hidma_show_values()
637 device_remove_file(dev->ddev.dev, dev->chid_attrs); in hidma_sysfs_uninit()
646 attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute), in hidma_create_sysfs_entry()
651 name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL); in hidma_create_sysfs_entry()
655 attrs->attr.name = name_copy; in hidma_create_sysfs_entry()
656 attrs->attr.mode = mode; in hidma_create_sysfs_entry()
657 attrs->show = hidma_show_values; in hidma_create_sysfs_entry()
658 sysfs_attr_init(&attrs->attr); in hidma_create_sysfs_entry()
665 dev->chid_attrs = hidma_create_sysfs_entry(dev, "chid", S_IRUGO); in hidma_sysfs_init()
666 if (!dev->chid_attrs) in hidma_sysfs_init()
667 return -ENOMEM; in hidma_sysfs_init()
669 return device_create_file(dev->ddev.dev, dev->chid_attrs); in hidma_sysfs_init()
678 if (!desc->msi_index) { in hidma_write_msi_msg()
679 writel(msg->address_lo, dmadev->dev_evca + 0x118); in hidma_write_msi_msg()
680 writel(msg->address_hi, dmadev->dev_evca + 0x11C); in hidma_write_msi_msg()
681 writel(msg->data, dmadev->dev_evca + 0x120); in hidma_write_msi_msg()
689 struct device *dev = dmadev->ddev.dev; in hidma_free_msis()
695 devm_free_irq(dev, virq, &dmadev->lldev); in hidma_free_msis()
708 rc = platform_device_msi_init_and_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, in hidma_request_msi()
714 virq = msi_get_virq(&pdev->dev, i); in hidma_request_msi()
715 rc = devm_request_irq(&pdev->dev, virq, in hidma_request_msi()
717 0, "qcom-hidma-msi", in hidma_request_msi()
718 &dmadev->lldev); in hidma_request_msi()
722 dmadev->msi_virqbase = virq; in hidma_request_msi()
726 /* free allocated MSI interrupts above */ in hidma_request_msi()
727 for (--i; i >= 0; i--) { in hidma_request_msi()
728 virq = msi_get_virq(&pdev->dev, i); in hidma_request_msi()
729 devm_free_irq(&pdev->dev, virq, &dmadev->lldev); in hidma_request_msi()
731 dev_warn(&pdev->dev, in hidma_request_msi()
734 /* Add callback to free MSIs on teardown */ in hidma_request_msi()
735 hidma_ll_setup_irq(dmadev->lldev, true); in hidma_request_msi()
739 return -EINVAL; in hidma_request_msi()
762 pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT); in hidma_probe()
763 pm_runtime_use_autosuspend(&pdev->dev); in hidma_probe()
764 pm_runtime_set_active(&pdev->dev); in hidma_probe()
765 pm_runtime_enable(&pdev->dev); in hidma_probe()
789 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); in hidma_probe()
791 rc = -ENOMEM; in hidma_probe()
795 INIT_LIST_HEAD(&dmadev->ddev.channels); in hidma_probe()
796 spin_lock_init(&dmadev->lock); in hidma_probe()
797 dmadev->ddev.dev = &pdev->dev; in hidma_probe()
798 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_probe()
800 dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask); in hidma_probe()
801 dma_cap_set(DMA_MEMSET, dmadev->ddev.cap_mask); in hidma_probe()
802 if (WARN_ON(!pdev->dev.dma_mask)) { in hidma_probe()
803 rc = -ENXIO; in hidma_probe()
807 dmadev->dev_evca = evca; in hidma_probe()
808 dmadev->evca_resource = evca_resource; in hidma_probe()
809 dmadev->dev_trca = trca; in hidma_probe()
810 dmadev->trca_resource = trca_resource; in hidma_probe()
811 dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy; in hidma_probe()
812 dmadev->ddev.device_prep_dma_memset = hidma_prep_dma_memset; in hidma_probe()
813 dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources; in hidma_probe()
814 dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources; in hidma_probe()
815 dmadev->ddev.device_tx_status = hidma_tx_status; in hidma_probe()
816 dmadev->ddev.device_issue_pending = hidma_issue_pending; in hidma_probe()
817 dmadev->ddev.device_pause = hidma_pause; in hidma_probe()
818 dmadev->ddev.device_resume = hidma_resume; in hidma_probe()
819 dmadev->ddev.device_terminate_all = hidma_terminate_all; in hidma_probe()
820 dmadev->ddev.copy_align = 8; in hidma_probe()
826 msi = hidma_test_capability(&pdev->dev, HIDMA_MSI_CAP); in hidma_probe()
827 device_property_read_u32(&pdev->dev, "desc-count", in hidma_probe()
828 &dmadev->nr_descriptors); in hidma_probe()
831 dev_info(&pdev->dev, "overriding number of descriptors as %d\n", in hidma_probe()
833 dmadev->nr_descriptors = nr_desc_prm; in hidma_probe()
836 if (!dmadev->nr_descriptors) in hidma_probe()
837 dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC; in hidma_probe()
839 if (hidma_test_capability(&pdev->dev, HIDMA_IDENTITY_CAP)) in hidma_probe()
840 dmadev->chidx = readl(dmadev->dev_trca + 0x40); in hidma_probe()
842 dmadev->chidx = readl(dmadev->dev_trca + 0x28); in hidma_probe()
845 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); in hidma_probe()
847 dev_warn(&pdev->dev, "unable to set coherent mask to 64"); in hidma_probe()
851 dmadev->lldev = hidma_ll_init(dmadev->ddev.dev, in hidma_probe()
852 dmadev->nr_descriptors, dmadev->dev_trca, in hidma_probe()
853 dmadev->dev_evca, dmadev->chidx); in hidma_probe()
854 if (!dmadev->lldev) { in hidma_probe()
855 rc = -EPROBE_DEFER; in hidma_probe()
864 hidma_ll_setup_irq(dmadev->lldev, false); in hidma_probe()
865 rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler, in hidma_probe()
866 0, "qcom-hidma", dmadev->lldev); in hidma_probe()
871 INIT_LIST_HEAD(&dmadev->ddev.channels); in hidma_probe()
876 rc = dma_async_device_register(&dmadev->ddev); in hidma_probe()
880 dmadev->irq = chirq; in hidma_probe()
881 tasklet_setup(&dmadev->task, hidma_issue_task); in hidma_probe()
884 dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n"); in hidma_probe()
885 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_probe()
886 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_probe()
893 hidma_ll_uninit(dmadev->lldev); in hidma_probe()
898 pm_runtime_put_sync(&pdev->dev); in hidma_probe()
899 pm_runtime_disable(&pdev->dev); in hidma_probe()
907 dev_info(dmadev->ddev.dev, "HI-DMA engine shutdown\n"); in hidma_shutdown()
909 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_shutdown()
910 if (hidma_ll_disable(dmadev->lldev)) in hidma_shutdown()
911 dev_warn(dmadev->ddev.dev, "channel did not stop\n"); in hidma_shutdown()
912 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_shutdown()
913 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_shutdown()
921 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_remove()
922 dma_async_device_unregister(&dmadev->ddev); in hidma_remove()
923 if (!dmadev->lldev->msi_support) in hidma_remove()
924 devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev); in hidma_remove()
928 tasklet_kill(&dmadev->task); in hidma_remove()
931 hidma_ll_uninit(dmadev->lldev); in hidma_remove()
934 dev_info(&pdev->dev, "HI-DMA engine removed\n"); in hidma_remove()
935 pm_runtime_put_sync_suspend(&pdev->dev); in hidma_remove()
936 pm_runtime_disable(&pdev->dev); in hidma_remove()