1 // SPDX-License-Identifier: GPL-2.0
3  * PCI Peer 2 Peer DMA support.
5  * Copyright (c) 2016-2018, Logan Gunthorpe
6  * Copyright (c) 2016-2017, Microsemi Corporation
11 #define pr_fmt(fmt) "pci-p2pdma: " fmt
13 #include <linux/dma-map-ops.h>
14 #include <linux/pci-p2pdma.h>
19 #include <linux/percpu-refcount.h>
28 	struct p2pdma_provider mem[PCI_STD_NUM_BARS];  member
33 	struct p2pdma_provider *mem;  member
49 	p2pdma = rcu_dereference(pdev->p2pdma);  in size_show()
50 	if (p2pdma && p2pdma->pool)  in size_show()
51 		size = gen_pool_size(p2pdma->pool);  in size_show()
66 	p2pdma = rcu_dereference(pdev->p2pdma);  in available_show()
67 	if (p2pdma && p2pdma->pool)  in available_show()
68 		avail = gen_pool_avail(p2pdma->pool);  in available_show()
83 	p2pdma = rcu_dereference(pdev->p2pdma);  in published_show()
85 		published = p2pdma->p2pmem_published;  in published_show()
96 	size_t len = vma->vm_end - vma->vm_start;  in p2pmem_alloc_mmap()
104 	if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {  in p2pmem_alloc_mmap()
107 				     current->comm);  in p2pmem_alloc_mmap()
108 		return -EINVAL;  in p2pmem_alloc_mmap()
111 	if (vma->vm_pgoff) {  in p2pmem_alloc_mmap()
113 				     "%s: fail, attempted mapping with non-zero offset\n",  in p2pmem_alloc_mmap()
114 				     current->comm);  in p2pmem_alloc_mmap()
115 		return -EINVAL;  in p2pmem_alloc_mmap()
119 	p2pdma = rcu_dereference(pdev->p2pdma);  in p2pmem_alloc_mmap()
121 		ret = -ENODEV;  in p2pmem_alloc_mmap()
125 	kaddr = (void *)gen_pool_alloc_owner(p2pdma->pool, len, (void **)&ref);  in p2pmem_alloc_mmap()
127 		ret = -ENOMEM;  in p2pmem_alloc_mmap()
137 		ret = -ENODEV;  in p2pmem_alloc_mmap()
142 	for (vaddr = vma->vm_start; vaddr < vma->vm_end; vaddr += PAGE_SIZE) {  in p2pmem_alloc_mmap()
154 			gen_pool_free(p2pdma->pool, (uintptr_t)kaddr, len);  in p2pmem_alloc_mmap()
160 		len -= PAGE_SIZE;  in p2pmem_alloc_mmap()
167 	gen_pool_free(p2pdma->pool, (uintptr_t)kaddr, len);  in p2pmem_alloc_mmap()
205 	struct page *page = &folio->page;  in p2pdma_folio_free()
209 		to_pci_dev(pgmap->mem->owner)->p2pdma, 1);  in p2pdma_folio_free()
212 	gen_pool_free_owner(p2pdma->pool, (uintptr_t)page_to_virt(page),  in p2pdma_folio_free()
226 	p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);  in pci_p2pdma_release()
231 	pdev->p2pdma = NULL;  in pci_p2pdma_release()
232 	if (p2pdma->pool)  in pci_p2pdma_release()
234 	xa_destroy(&p2pdma->map_types);  in pci_p2pdma_release()
236 	if (!p2pdma->pool)  in pci_p2pdma_release()
239 	gen_pool_destroy(p2pdma->pool);  in pci_p2pdma_release()
240 	sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group);  in pci_p2pdma_release()
244  * pcim_p2pdma_init - Initialise peer-to-peer DMA providers
247  * This function initializes the peer-to-peer DMA infrastructure
257 	p2p = rcu_dereference_protected(pdev->p2pdma, 1);  in pcim_p2pdma_init()
261 	p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL);  in pcim_p2pdma_init()
263 		return -ENOMEM;  in pcim_p2pdma_init()
265 	xa_init(&p2p->map_types);  in pcim_p2pdma_init()
268 	 * correspond to MMIO regions. Skip non-memory resources (e.g. I/O  in pcim_p2pdma_init()
269 	 * port BARs) since they cannot be used for peer-to-peer (P2P)  in pcim_p2pdma_init()
276 		p2p->mem[i].owner = &pdev->dev;  in pcim_p2pdma_init()
277 		p2p->mem[i].bus_offset =  in pcim_p2pdma_init()
278 			pci_bus_address(pdev, i) - pci_resource_start(pdev, i);  in pcim_p2pdma_init()
281 	ret = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev);  in pcim_p2pdma_init()
285 	rcu_assign_pointer(pdev->p2pdma, p2p);  in pcim_p2pdma_init()
289 	devm_kfree(&pdev->dev, p2p);  in pcim_p2pdma_init()
295  * pcim_p2pdma_provider - Get peer-to-peer DMA provider
299  * This function gets peer-to-peer DMA provider for a PCI device. The lifetime
302  * provider, and any DMA mappings created for any MMIO, are all cleaned up
306  * to notify, invalidate and revoke the MMIO's DMA must be in place to use this
316 	p2p = rcu_dereference_protected(pdev->p2pdma, 1);  in pcim_p2pdma_provider()
321 	return &p2p->mem[bar];  in pcim_p2pdma_provider()
330 	p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);  in pci_p2pdma_setup_pool()
331 	if (p2pdma->pool)  in pci_p2pdma_setup_pool()
335 	p2pdma->pool = gen_pool_create(PAGE_SHIFT, dev_to_node(&pdev->dev));  in pci_p2pdma_setup_pool()
336 	if (!p2pdma->pool)  in pci_p2pdma_setup_pool()
337 		return -ENOMEM;  in pci_p2pdma_setup_pool()
339 	ret = sysfs_create_group(&pdev->dev.kobj, &p2pmem_group);  in pci_p2pdma_setup_pool()
346 	gen_pool_destroy(p2pdma->pool);  in pci_p2pdma_setup_pool()
347 	p2pdma->pool = NULL;  in pci_p2pdma_setup_pool()
360 	sysfs_remove_file_from_group(&p2p_pgmap->mem->owner->kobj,  in pci_p2pdma_unmap_mappings()
366  * pci_p2pdma_add_resource - add memory for use as p2p memory
373  * be used with any DMA request.
379 	struct p2pdma_provider *mem;  in pci_p2pdma_add_resource()  local
386 		return -EINVAL;  in pci_p2pdma_add_resource()
389 		return -EINVAL;  in pci_p2pdma_add_resource()
392 		size = pci_resource_len(pdev, bar) - offset;  in pci_p2pdma_add_resource()
395 		return -EINVAL;  in pci_p2pdma_add_resource()
405 	mem = pcim_p2pdma_provider(pdev, bar);  in pci_p2pdma_add_resource()
410 	if (WARN_ON(!mem))  in pci_p2pdma_add_resource()
411 		return -EINVAL;  in pci_p2pdma_add_resource()
413 	p2p_pgmap = devm_kzalloc(&pdev->dev, sizeof(*p2p_pgmap), GFP_KERNEL);  in pci_p2pdma_add_resource()
415 		return -ENOMEM;  in pci_p2pdma_add_resource()
417 	pgmap = &p2p_pgmap->pgmap;  in pci_p2pdma_add_resource()
418 	pgmap->range.start = pci_resource_start(pdev, bar) + offset;  in pci_p2pdma_add_resource()
419 	pgmap->range.end = pgmap->range.start + size - 1;  in pci_p2pdma_add_resource()
420 	pgmap->nr_range = 1;  in pci_p2pdma_add_resource()
421 	pgmap->type = MEMORY_DEVICE_PCI_P2PDMA;  in pci_p2pdma_add_resource()
422 	pgmap->ops = &p2pdma_pgmap_ops;  in pci_p2pdma_add_resource()
423 	p2p_pgmap->mem = mem;  in pci_p2pdma_add_resource()
425 	addr = devm_memremap_pages(&pdev->dev, pgmap);  in pci_p2pdma_add_resource()
431 	error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_unmap_mappings,  in pci_p2pdma_add_resource()
436 	p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);  in pci_p2pdma_add_resource()
437 	error = gen_pool_add_owner(p2pdma->pool, (unsigned long)addr,  in pci_p2pdma_add_resource()
439 			range_len(&pgmap->range), dev_to_node(&pdev->dev),  in pci_p2pdma_add_resource()
440 			&pgmap->ref);  in pci_p2pdma_add_resource()
444 	pci_info(pdev, "added peer-to-peer DMA memory %#llx-%#llx\n",  in pci_p2pdma_add_resource()
445 		 pgmap->range.start, pgmap->range.end);  in pci_p2pdma_add_resource()
450 	devm_memunmap_pages(&pdev->dev, pgmap);  in pci_p2pdma_add_resource()
452 	devm_kfree(&pdev->dev, p2p_pgmap);  in pci_p2pdma_add_resource()
472 		parent = get_device(dev->parent);  in find_parent_pci_dev()
490 	pos = pdev->acs_cap;  in pci_bridge_has_acs_redir()
516 	if (c->x86_vendor == X86_VENDOR_AMD && c->x86 >= 0x17)  in cpu_supports_p2pdma()
536 	/* Intel Skylake-E */
555  * This function is similar to pci_get_slot(host->bus, 0), but it does
567 	root = list_first_entry_or_null(&host->bus->devices,  in pci_host_bridge_dev()
573 	if (root->devfn == PCI_DEVFN(0, 0))  in pci_host_bridge_dev()
592 	vendor = root->vendor;  in __host_bridge_whitelist()
593 	device = root->device;  in __host_bridge_whitelist()
595 	for (entry = pci_p2pdma_whitelist; entry->vendor; entry++) {  in __host_bridge_whitelist()
596 		if (vendor != entry->vendor || device != entry->device)  in __host_bridge_whitelist()
598 		if (entry->flags & REQ_SAME_HOST_BRIDGE && !same_host_bridge)  in __host_bridge_whitelist()
618 	struct pci_host_bridge *host_a = pci_find_host_bridge(a->bus);  in host_bridge_whitelist()
619 	struct pci_host_bridge *host_b = pci_find_host_bridge(b->bus);  in host_bridge_whitelist()
633 	return (pci_domain_nr(client->bus) << 16) | pci_dev_id(client);  in map_types_idx()
649  *     -+  Root Port
651  *       +-+ Switch Downstream Port 0
652  *       + \- Device A
653  *       \-+ Switch Downstream Port 1
654  *         \- Device B
741 		acs_list.buffer[acs_list.len-1] = 0; /* drop final semicolon */  in calc_map_type_and_dist()
753 …pci_warn(client, "cannot be used for peer-to-peer DMA as the client and provider (%s) do not share…  in calc_map_type_and_dist()
759 	p2pdma = rcu_dereference(provider->p2pdma);  in calc_map_type_and_dist()
761 		xa_store(&p2pdma->map_types, map_types_idx(client),  in calc_map_type_and_dist()
768  * pci_p2pdma_distance_many - Determine the cumulative distance between
771  * @clients: array of devices to check (NULL-terminated)
773  * @verbose: if true, print warnings for devices when we return -1
775  * Returns -1 if any of the clients are not compatible, otherwise returns a
794 		return -1;  in pci_p2pdma_distance_many()
801 					 "cannot be used for peer-to-peer DMA as it is not a PCI device\n");  in pci_p2pdma_distance_many()
802 			return -1;  in pci_p2pdma_distance_many()
820 		return -1;  in pci_p2pdma_distance_many()
827  * pci_has_p2pmem - check if a given PCI device has published any p2pmem
836 	p2pdma = rcu_dereference(pdev->p2pdma);  in pci_has_p2pmem()
837 	res = p2pdma && p2pdma->p2pmem_published;  in pci_has_p2pmem()
844  * pci_p2pmem_find_many - find a peer-to-peer DMA memory device compatible with
846  * @clients: array of devices to check (NULL-terminated)
908  * pci_alloc_p2pmem - allocate peer-to-peer DMA memory
922 	 * ensure pdev->p2pdma is non-NULL for the duration of the  in pci_alloc_p2pmem()
923 	 * read-lock.  in pci_alloc_p2pmem()
926 	p2pdma = rcu_dereference(pdev->p2pdma);  in pci_alloc_p2pmem()
930 	ret = (void *)gen_pool_alloc_owner(p2pdma->pool, size, (void **) &ref);  in pci_alloc_p2pmem()
935 		gen_pool_free(p2pdma->pool, (unsigned long) ret, size);  in pci_alloc_p2pmem()
945  * pci_free_p2pmem - free peer-to-peer DMA memory
953 	struct pci_p2pdma *p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);  in pci_free_p2pmem()
955 	gen_pool_free_owner(p2pdma->pool, (uintptr_t)addr, size,  in pci_free_p2pmem()
962  * pci_p2pmem_virt_to_bus - return the PCI bus address for a given virtual
974 	p2pdma = rcu_dereference_protected(pdev->p2pdma, 1);  in pci_p2pmem_virt_to_bus()
983 	return gen_pool_virt_to_phys(p2pdma->pool, (unsigned long)addr);  in pci_p2pmem_virt_to_bus()
988  * pci_p2pmem_alloc_sgl - allocate peer-to-peer DMA memory in a scatterlist
1022  * pci_p2pmem_free_sgl - free a scatterlist allocated by pci_p2pmem_alloc_sgl()
1035 		pci_free_p2pmem(pdev, sg_virt(sg), sg->length);  in pci_p2pmem_free_sgl()
1042  * pci_p2pmem_publish - publish the peer-to-peer DMA memory for use by
1044  * @pdev: the device with peer-to-peer DMA memory to publish
1048  * peer-2-peer DMA operations. Non-published memory is reserved for
1049  * exclusive use of the device driver that registers the peer-to-peer
1057 	p2pdma = rcu_dereference(pdev->p2pdma);  in pci_p2pmem_publish()
1059 		p2pdma->p2pmem_published = publish;  in pci_p2pmem_publish()
1065  * pci_p2pdma_map_type - Determine the mapping type for P2PDMA transfers
1069  * Determines how peer-to-peer DMA transfers should be mapped between
1078 	struct pci_dev *pdev = to_pci_dev(provider->owner);  in pci_p2pdma_map_type()
1083 	if (!pdev->p2pdma)  in pci_p2pdma_map_type()
1092 	p2pdma = rcu_dereference(pdev->p2pdma);  in pci_p2pdma_map_type()
1095 		type = xa_to_value(xa_load(&p2pdma->map_types,  in pci_p2pdma_map_type()
1110 	if (state->mem == p2p_pgmap->mem)  in __pci_p2pdma_update_state()
1113 	state->mem = p2p_pgmap->mem;  in __pci_p2pdma_update_state()
1114 	state->map = pci_p2pdma_map_type(p2p_pgmap->mem, dev);  in __pci_p2pdma_update_state()
1118  * pci_p2pdma_enable_store - parse a configfs/sysfs attribute store
1149 				"PCI device has no peer-to-peer memory: %s\n",  in pci_p2pdma_enable_store()
1152 			return -ENODEV;  in pci_p2pdma_enable_store()
1168 	return -ENODEV;  in pci_p2pdma_enable_store()
1173  * pci_p2pdma_enable_show - show a configfs/sysfs attribute indicating