1 // SPDX-License-Identifier: GPL-2.0
3  * Coherent per-device memory handling.
10 #include <linux/dma-direct.h>
11 #include <linux/dma-map-ops.h>
25 	if (dev && dev->dma_mem)  in dev_get_coherent_memory()
26 		return dev->dma_mem;  in dev_get_coherent_memory()
33 	if (mem->use_dev_dma_pfn_offset)  in dma_get_device_base()
34 		return phys_to_dma(dev, PFN_PHYS(mem->pfn_base));  in dma_get_device_base()
35 	return mem->device_base;  in dma_get_device_base()
46 		return ERR_PTR(-EINVAL);  in dma_init_coherent_memory()
50 		return ERR_PTR(-EINVAL);  in dma_init_coherent_memory()
55 	dma_mem->bitmap = bitmap_zalloc(pages, GFP_KERNEL);  in dma_init_coherent_memory()
56 	if (!dma_mem->bitmap)  in dma_init_coherent_memory()
59 	dma_mem->virt_base = mem_base;  in dma_init_coherent_memory()
60 	dma_mem->device_base = device_addr;  in dma_init_coherent_memory()
61 	dma_mem->pfn_base = PFN_DOWN(phys_addr);  in dma_init_coherent_memory()
62 	dma_mem->size = pages;  in dma_init_coherent_memory()
63 	dma_mem->use_dev_dma_pfn_offset = use_dma_pfn_offset;  in dma_init_coherent_memory()
64 	spin_lock_init(&dma_mem->spinlock);  in dma_init_coherent_memory()
72 	pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %zd MiB\n",  in dma_init_coherent_memory()
74 	return ERR_PTR(-ENOMEM);  in dma_init_coherent_memory()
82 	memunmap(mem->virt_base);  in _dma_release_coherent_memory()
83 	bitmap_free(mem->bitmap);  in _dma_release_coherent_memory()
91 		return -ENODEV;  in dma_assign_coherent_memory()
93 	if (dev->dma_mem)  in dma_assign_coherent_memory()
94 		return -EBUSY;  in dma_assign_coherent_memory()
96 	dev->dma_mem = mem;  in dma_assign_coherent_memory()
102  * is asked for coherent memory for this device.  This shall only be used
108  * device_addr is the DMA address the device needs to be programmed with to
136 		_dma_release_coherent_memory(dev->dma_mem);  in dma_release_coherent_memory()
137 		dev->dma_mem = NULL;  in dma_release_coherent_memory()
150 	spin_lock_irqsave(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
152 	if (unlikely(size > ((dma_addr_t)mem->size << PAGE_SHIFT)))  in __dma_alloc_from_coherent()
155 	pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);  in __dma_alloc_from_coherent()
160 	 * Memory was found in the coherent area.  in __dma_alloc_from_coherent()
164 	ret = mem->virt_base + ((dma_addr_t)pageno << PAGE_SHIFT);  in __dma_alloc_from_coherent()
165 	spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
169 	spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
174  * dma_alloc_from_dev_coherent() - allocate memory from device coherent pool
177  * @dma_handle:	This will be filled with the correct dma handle
181  * This function should be only called from per-arch dma_alloc_coherent()
182  * to support allocation from per-device coherent memory pools.
202 	if (mem && vaddr >= mem->virt_base && vaddr <  in __dma_release_from_coherent()
203 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {  in __dma_release_from_coherent()
204 		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;  in __dma_release_from_coherent()
207 		spin_lock_irqsave(&mem->spinlock, flags);  in __dma_release_from_coherent()
208 		bitmap_release_region(mem->bitmap, page, order);  in __dma_release_from_coherent()
209 		spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_release_from_coherent()
216  * dma_release_from_dev_coherent() - free memory to device coherent memory pool
221  * This checks whether the memory was allocated from the per-device
222  * coherent memory pool and if so, releases that memory.
237 	if (mem && vaddr >= mem->virt_base && vaddr + size <=  in __dma_mmap_from_coherent()
238 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {  in __dma_mmap_from_coherent()
239 		unsigned long off = vma->vm_pgoff;  in __dma_mmap_from_coherent()
240 		int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;  in __dma_mmap_from_coherent()
244 		*ret = -ENXIO;  in __dma_mmap_from_coherent()
245 		if (off < count && user_count <= count - off) {  in __dma_mmap_from_coherent()
246 			unsigned long pfn = mem->pfn_base + start + off;  in __dma_mmap_from_coherent()
247 			*ret = remap_pfn_range(vma, vma->vm_start, pfn,  in __dma_mmap_from_coherent()
249 					       vma->vm_page_prot);  in __dma_mmap_from_coherent()
257  * dma_mmap_from_dev_coherent() - mmap memory from the device coherent pool
264  * This checks whether the memory was allocated from the per-device
265  * coherent memory pool and if so, maps that memory to the provided vma.
267  * Returns 1 if @vaddr belongs to the device coherent pool and the caller
319 	pr_info("DMA: default coherent area is set\n");  in dma_init_global_coherent()
338 	if (!rmem->priv) {  in rmem_dma_device_init()
341 		mem = dma_init_coherent_memory(rmem->base, rmem->base,  in rmem_dma_device_init()
342 					       rmem->size, true);  in rmem_dma_device_init()
345 		rmem->priv = mem;  in rmem_dma_device_init()
347 	dma_assign_coherent_memory(dev, rmem->priv);  in rmem_dma_device_init()
355 		dev->dma_mem = NULL;  in rmem_dma_device_release()
365 	unsigned long node = rmem->fdt_node;  in rmem_dma_setup()
368 		return -EINVAL;  in rmem_dma_setup()
371 	if (!of_get_flat_dt_prop(node, "no-map", NULL)) {  in rmem_dma_setup()
372 		pr_err("Reserved memory: regions without no-map are not yet supported\n");  in rmem_dma_setup()
373 		return -EINVAL;  in rmem_dma_setup()
378 	if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) {  in rmem_dma_setup()
380 		     "Reserved memory: region for default DMA coherent area is redefined\n");  in rmem_dma_setup()
385 	rmem->ops = &rmem_dma_ops;  in rmem_dma_setup()
386 	pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n",  in rmem_dma_setup()
387 		&rmem->base, (unsigned long)rmem->size / SZ_1M);  in rmem_dma_setup()
395 		return -ENOMEM;  in dma_init_reserved_memory()
396 	return dma_init_global_coherent(dma_reserved_default_memory->base,  in dma_init_reserved_memory()
397 					dma_reserved_default_memory->size);  in dma_init_reserved_memory()
402 RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);