1 // SPDX-License-Identifier: GPL-2.0
3  * PCI Peer 2 Peer DMA support.
5  * Copyright (c) 2016-2018, Logan Gunthorpe
6  * Copyright (c) 2016-2017, Microsemi Corporation
11 #define pr_fmt(fmt) "pci-p2pdma: " fmt
13 #include <linux/dma-map-ops.h>
14 #include <linux/pci-p2pdma.h>
19 #include <linux/percpu-refcount.h>
49 	p2pdma = rcu_dereference(pdev->p2pdma);  in size_show()
50 	if (p2pdma && p2pdma->pool)  in size_show()
51 		size = gen_pool_size(p2pdma->pool);  in size_show()
66 	p2pdma = rcu_dereference(pdev->p2pdma);  in available_show()
67 	if (p2pdma && p2pdma->pool)  in available_show()
68 		avail = gen_pool_avail(p2pdma->pool);  in available_show()
83 	p2pdma = rcu_dereference(pdev->p2pdma);  in published_show()
85 		published = p2pdma->p2pmem_published;  in published_show()
96 	size_t len = vma->vm_end - vma->vm_start;  in p2pmem_alloc_mmap()
104 	if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {  in p2pmem_alloc_mmap()
107 				     current->comm);  in p2pmem_alloc_mmap()
108 		return -EINVAL;  in p2pmem_alloc_mmap()
111 	if (vma->vm_pgoff) {  in p2pmem_alloc_mmap()
113 				     "%s: fail, attempted mapping with non-zero offset\n",  in p2pmem_alloc_mmap()
114 				     current->comm);  in p2pmem_alloc_mmap()
115 		return -EINVAL;  in p2pmem_alloc_mmap()
119 	p2pdma = rcu_dereference(pdev->p2pdma);  in p2pmem_alloc_mmap()
121 		ret = -ENODEV;  in p2pmem_alloc_mmap()
125 	kaddr = (void *)gen_pool_alloc_owner(p2pdma->pool, len, (void **)&ref);  in p2pmem_alloc_mmap()
127 		ret = -ENOMEM;  in p2pmem_alloc_mmap()
137 		ret = -ENODEV;  in p2pmem_alloc_mmap()
142 	for (vaddr = vma->vm_start; vaddr < vma->vm_end; vaddr += PAGE_SIZE) {  in p2pmem_alloc_mmap()
154 			gen_pool_free(p2pdma->pool, (uintptr_t)kaddr, len);  in p2pmem_alloc_mmap()
160 		len -= PAGE_SIZE;  in p2pmem_alloc_mmap()
167 	gen_pool_free(p2pdma->pool, (uintptr_t)kaddr, len);  in p2pmem_alloc_mmap()
206 	/* safe to dereference while a reference is held to the percpu ref */  in p2pdma_page_free()
208 		rcu_dereference_protected(pgmap->provider->p2pdma, 1);  in p2pdma_page_free()
211 	gen_pool_free_owner(p2pdma->pool, (uintptr_t)page_to_virt(page),  in p2pdma_page_free()
225 	p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);  in pci_p2pdma_release()
230 	pdev->p2pdma = NULL;  in pci_p2pdma_release()
233 	gen_pool_destroy(p2pdma->pool);  in pci_p2pdma_release()
234 	sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group);  in pci_p2pdma_release()
235 	xa_destroy(&p2pdma->map_types);  in pci_p2pdma_release()
240 	int error = -ENOMEM;  in pci_p2pdma_setup()
243 	p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL);  in pci_p2pdma_setup()
245 		return -ENOMEM;  in pci_p2pdma_setup()
247 	xa_init(&p2p->map_types);  in pci_p2pdma_setup()
249 	p2p->pool = gen_pool_create(PAGE_SHIFT, dev_to_node(&pdev->dev));  in pci_p2pdma_setup()
250 	if (!p2p->pool)  in pci_p2pdma_setup()
253 	error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev);  in pci_p2pdma_setup()
257 	error = sysfs_create_group(&pdev->dev.kobj, &p2pmem_group);  in pci_p2pdma_setup()
261 	rcu_assign_pointer(pdev->p2pdma, p2p);  in pci_p2pdma_setup()
265 	gen_pool_destroy(p2p->pool);  in pci_p2pdma_setup()
267 	devm_kfree(&pdev->dev, p2p);  in pci_p2pdma_setup()
280 	sysfs_remove_file_from_group(&pdev->dev.kobj, &p2pmem_alloc_attr.attr,  in pci_p2pdma_unmap_mappings()
285  * pci_p2pdma_add_resource - add memory for use as p2p memory
292  * be used with any DMA request.
304 		return -EINVAL;  in pci_p2pdma_add_resource()
307 		return -EINVAL;  in pci_p2pdma_add_resource()
310 		size = pci_resource_len(pdev, bar) - offset;  in pci_p2pdma_add_resource()
313 		return -EINVAL;  in pci_p2pdma_add_resource()
315 	if (!pdev->p2pdma) {  in pci_p2pdma_add_resource()
321 	p2p_pgmap = devm_kzalloc(&pdev->dev, sizeof(*p2p_pgmap), GFP_KERNEL);  in pci_p2pdma_add_resource()
323 		return -ENOMEM;  in pci_p2pdma_add_resource()
325 	pgmap = &p2p_pgmap->pgmap;  in pci_p2pdma_add_resource()
326 	pgmap->range.start = pci_resource_start(pdev, bar) + offset;  in pci_p2pdma_add_resource()
327 	pgmap->range.end = pgmap->range.start + size - 1;  in pci_p2pdma_add_resource()
328 	pgmap->nr_range = 1;  in pci_p2pdma_add_resource()
329 	pgmap->type = MEMORY_DEVICE_PCI_P2PDMA;  in pci_p2pdma_add_resource()
330 	pgmap->ops = &p2pdma_pgmap_ops;  in pci_p2pdma_add_resource()
332 	p2p_pgmap->provider = pdev;  in pci_p2pdma_add_resource()
333 	p2p_pgmap->bus_offset = pci_bus_address(pdev, bar) -  in pci_p2pdma_add_resource()
336 	addr = devm_memremap_pages(&pdev->dev, pgmap);  in pci_p2pdma_add_resource()
342 	error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_unmap_mappings,  in pci_p2pdma_add_resource()
347 	p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);  in pci_p2pdma_add_resource()
348 	error = gen_pool_add_owner(p2pdma->pool, (unsigned long)addr,  in pci_p2pdma_add_resource()
350 			range_len(&pgmap->range), dev_to_node(&pdev->dev),  in pci_p2pdma_add_resource()
351 			&pgmap->ref);  in pci_p2pdma_add_resource()
355 	pci_info(pdev, "added peer-to-peer DMA memory %#llx-%#llx\n",  in pci_p2pdma_add_resource()
356 		 pgmap->range.start, pgmap->range.end);  in pci_p2pdma_add_resource()
361 	devm_memunmap_pages(&pdev->dev, pgmap);  in pci_p2pdma_add_resource()
363 	devm_kfree(&pdev->dev, p2p_pgmap);  in pci_p2pdma_add_resource()
383 		parent = get_device(dev->parent);  in find_parent_pci_dev()
401 	pos = pdev->acs_cap;  in pci_bridge_has_acs_redir()
427 	if (c->x86_vendor == X86_VENDOR_AMD && c->x86 >= 0x17)  in cpu_supports_p2pdma()
447 	/* Intel Skylake-E */
466  * This function is similar to pci_get_slot(host->bus, 0), but it does
470  * For this to be safe, the caller should hold a reference to a device on the
478 	root = list_first_entry_or_null(&host->bus->devices,  in pci_host_bridge_dev()
484 	if (root->devfn == PCI_DEVFN(0, 0))  in pci_host_bridge_dev()
503 	vendor = root->vendor;  in __host_bridge_whitelist()
504 	device = root->device;  in __host_bridge_whitelist()
506 	for (entry = pci_p2pdma_whitelist; entry->vendor; entry++) {  in __host_bridge_whitelist()
507 		if (vendor != entry->vendor || device != entry->device)  in __host_bridge_whitelist()
509 		if (entry->flags & REQ_SAME_HOST_BRIDGE && !same_host_bridge)  in __host_bridge_whitelist()
529 	struct pci_host_bridge *host_a = pci_find_host_bridge(a->bus);  in host_bridge_whitelist()
530 	struct pci_host_bridge *host_b = pci_find_host_bridge(b->bus);  in host_bridge_whitelist()
544 	return (pci_domain_nr(client->bus) << 16) | pci_dev_id(client);  in map_types_idx()
560  *     -+  Root Port
562  *       +-+ Switch Downstream Port 0
563  *       + \- Device A
564  *       \-+ Switch Downstream Port 1
565  *         \- Device B
652 		acs_list.buffer[acs_list.len-1] = 0; /* drop final semicolon */  in calc_map_type_and_dist()
664 …pci_warn(client, "cannot be used for peer-to-peer DMA as the client and provider (%s) do not share…  in calc_map_type_and_dist()
670 	p2pdma = rcu_dereference(provider->p2pdma);  in calc_map_type_and_dist()
672 		xa_store(&p2pdma->map_types, map_types_idx(client),  in calc_map_type_and_dist()
679  * pci_p2pdma_distance_many - Determine the cumulative distance between
682  * @clients: array of devices to check (NULL-terminated)
684  * @verbose: if true, print warnings for devices when we return -1
686  * Returns -1 if any of the clients are not compatible, otherwise returns a
698 	enum pci_p2pdma_map_type map;  in pci_p2pdma_distance_many()  local
705 		return -1;  in pci_p2pdma_distance_many()
712 					 "cannot be used for peer-to-peer DMA as it is not a PCI device\n");  in pci_p2pdma_distance_many()
713 			return -1;  in pci_p2pdma_distance_many()
716 		map = calc_map_type_and_dist(provider, pci_client, &distance,  in pci_p2pdma_distance_many()
721 		if (map == PCI_P2PDMA_MAP_NOT_SUPPORTED)  in pci_p2pdma_distance_many()
731 		return -1;  in pci_p2pdma_distance_many()
738  * pci_has_p2pmem - check if a given PCI device has published any p2pmem
747 	p2pdma = rcu_dereference(pdev->p2pdma);  in pci_has_p2pmem()
748 	res = p2pdma && p2pdma->p2pmem_published;  in pci_has_p2pmem()
755  * pci_p2pmem_find_many - find a peer-to-peer DMA memory device compatible with
757  * @clients: array of devices to check (NULL-terminated)
819  * pci_alloc_p2pmem - allocate peer-to-peer DMA memory
833 	 * ensure pdev->p2pdma is non-NULL for the duration of the  in pci_alloc_p2pmem()
834 	 * read-lock.  in pci_alloc_p2pmem()
837 	p2pdma = rcu_dereference(pdev->p2pdma);  in pci_alloc_p2pmem()
841 	ret = (void *)gen_pool_alloc_owner(p2pdma->pool, size, (void **) &ref);  in pci_alloc_p2pmem()
846 		gen_pool_free(p2pdma->pool, (unsigned long) ret, size);  in pci_alloc_p2pmem()
856  * pci_free_p2pmem - free peer-to-peer DMA memory
864 	struct pci_p2pdma *p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);  in pci_free_p2pmem()
866 	gen_pool_free_owner(p2pdma->pool, (uintptr_t)addr, size,  in pci_free_p2pmem()
873  * pci_p2pmem_virt_to_bus - return the PCI bus address for a given virtual
885 	p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);  in pci_p2pmem_virt_to_bus()
894 	return gen_pool_virt_to_phys(p2pdma->pool, (unsigned long)addr);  in pci_p2pmem_virt_to_bus()
899  * pci_p2pmem_alloc_sgl - allocate peer-to-peer DMA memory in a scatterlist
933  * pci_p2pmem_free_sgl - free a scatterlist allocated by pci_p2pmem_alloc_sgl()
946 		pci_free_p2pmem(pdev, sg_virt(sg), sg->length);  in pci_p2pmem_free_sgl()
953  * pci_p2pmem_publish - publish the peer-to-peer DMA memory for use by
955  * @pdev: the device with peer-to-peer DMA memory to publish
959  * peer-2-peer DMA operations. Non-published memory is reserved for
960  * exclusive use of the device driver that registers the peer-to-peer
968 	p2pdma = rcu_dereference(pdev->p2pdma);  in pci_p2pmem_publish()
970 		p2pdma->p2pmem_published = publish;  in pci_p2pmem_publish()
979 	struct pci_dev *provider = to_p2p_pgmap(pgmap)->provider;  in pci_p2pdma_map_type()
984 	if (!provider->p2pdma)  in pci_p2pdma_map_type()
993 	p2pdma = rcu_dereference(provider->p2pdma);  in pci_p2pdma_map_type()
996 		type = xa_to_value(xa_load(&p2pdma->map_types,  in pci_p2pdma_map_type()
1009 	state->pgmap = page_pgmap(page);  in __pci_p2pdma_update_state()
1010 	state->map = pci_p2pdma_map_type(state->pgmap, dev);  in __pci_p2pdma_update_state()
1011 	state->bus_off = to_p2p_pgmap(state->pgmap)->bus_offset;  in __pci_p2pdma_update_state()
1015  * pci_p2pdma_enable_store - parse a configfs/sysfs attribute store
1046 				"PCI device has no peer-to-peer memory: %s\n",  in pci_p2pdma_enable_store()
1049 			return -ENODEV;  in pci_p2pdma_enable_store()
1065 	return -ENODEV;  in pci_p2pdma_enable_store()
1070  * pci_p2pdma_enable_show - show a configfs/sysfs attribute indicating