1 // SPDX-License-Identifier: GPL-2.0-only
5  * Use the integrated AGP GART in the Hammer northbridge as an IOMMU for PCI.
6  * This allows to use PCI devices that only support 32bit addresses on systems
9  * See Documentation/core-api/dma-api-howto.rst for the interface specification.
23 #include <linux/pci.h>
29 #include <linux/iommu-helper.h>
34 #include <linux/dma-direct.h>
35 #include <linux/dma-map-ops.h>
38 #include <asm/iommu.h>
52  * If this is disabled the IOMMU will use an optimized flushing strategy
55  * to trigger bugs with some popular PCI cards, in particular 3ware (but
103 	if (offset == -1) {  in alloc_iommu()
109 	if (offset != -1) {  in alloc_iommu()
150 /* Debugging aid for drivers that don't free their IOMMU tables */
167 	 * Ran out of IOMMU space for this operation. This is very bad.  in iommu_full()
176 	dev_err(dev, "PCI-DMA: Out of IOMMU space for %lu bytes\n", size);  in iommu_full()
194 /* Map a single continuous physical area into the IOMMU.
195  * Caller needs to check if the iommu is needed and flush.
208 	if (iommu_page == -1) {  in dma_map_area()
224 /* Map a single area into the IOMMU */
266 	iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT;  in gart_unmap_phys()
284 		if (!s->dma_length || !s->length)  in gart_unmap_sg()
286 		gart_unmap_phys(dev, s->dma_address, s->dma_length, dir, 0);  in gart_unmap_sg()
304 		if (nonforced_iommu(dev, addr, s->length)) {  in dma_map_sg_nonforce()
305 			addr = dma_map_area(dev, addr, s->length, dir, 0);  in dma_map_sg_nonforce()
314 		s->dma_address = addr;  in dma_map_sg_nonforce()
315 		s->dma_length = s->length;  in dma_map_sg_nonforce()
332 	if (iommu_start == -1)  in __dma_map_cont()
333 		return -ENOMEM;  in __dma_map_cont()
337 		unsigned long phys_addr = s->dma_address;  in __dma_map_cont()
339 		BUG_ON(s != start && s->offset);  in __dma_map_cont()
341 			sout->dma_address = iommu_bus_base;  in __dma_map_cont()
342 			sout->dma_address += iommu_page*PAGE_SIZE + s->offset;  in __dma_map_cont()
343 			sout->dma_length = s->length;  in __dma_map_cont()
345 			sout->dma_length += s->length;  in __dma_map_cont()
349 		pages = iommu_num_pages(s->offset, s->length, PAGE_SIZE);  in __dma_map_cont()
350 		while (pages--) {  in __dma_map_cont()
356 	BUG_ON(iommu_page - iommu_start != pages);  in __dma_map_cont()
367 		sout->dma_address = start->dma_address;  in dma_map_cont()
368 		sout->dma_length = start->length;  in dma_map_cont()
388 		return -EINVAL;  in gart_map_sg()
401 		s->dma_address = addr;  in gart_map_sg()
402 		BUG_ON(s->length == 0);  in gart_map_sg()
404 		nextneed = need_iommu(dev, addr, s->length);  in gart_map_sg()
413 			if (!iommu_merge || !nextneed || !need || s->offset ||  in gart_map_sg()
414 			    (s->length + seg_size > max_seg_size) ||  in gart_map_sg()
415 			    (ps->offset + ps->length) % PAGE_SIZE) {  in gart_map_sg()
416 				ret = dma_map_cont(dev, start_sg, i - start,  in gart_map_sg()
430 		seg_size += s->length;  in gart_map_sg()
432 		pages += iommu_num_pages(s->offset, s->length, PAGE_SIZE);  in gart_map_sg()
435 	ret = dma_map_cont(dev, start_sg, i - start, sgmap, pages, need);  in gart_map_sg()
442 		sgmap->dma_length = 0;  in gart_map_sg()
472 	    !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24))  in gart_alloc_coherent()
476 			DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);  in gart_alloc_coherent()
508 	iommu_size -= round_up(a, PMD_SIZE) - a;  in check_iommu_size()
511 		pr_warn("PCI-DMA: Warning: Small IOMMU %luMB."  in check_iommu_size()
547 		struct pci_dev *dev = node_to_amd_nb(i)->misc;  in enable_gart_translations()
552 	/* Flush the GART-TLB to remove stale entries */  in enable_gart_translations()
581 	pr_info("PCI-DMA: Restoring GART aperture settings\n");  in gart_fixup_northbridges()
584 		struct pci_dev *dev = node_to_amd_nb(i)->misc;  in gart_fixup_northbridges()
588 		 * step.  Restore the pre-suspend aperture settings.  in gart_fixup_northbridges()
597 	pr_info("PCI-DMA: Resuming GART IOMMU\n");  in gart_resume()
625 	pr_info("PCI-DMA: Disabling AGP.\n");  in init_amd_gatt()
627 	aper_size = aper_base = info->aper_size = 0;  in init_amd_gatt()
630 		dev = node_to_amd_nb(i)->misc;  in init_amd_gatt()
645 	info->aper_base = aper_base;  in init_amd_gatt()
646 	info->aper_size = aper_size >> 20;  in init_amd_gatt()
662 	pr_info("PCI-DMA: aperture base @ %x size %u KB\n",  in init_amd_gatt()
669 	pr_warn("PCI-DMA: More than 4GB of RAM and no IOMMU - falling back to iommu=soft.\n");  in init_amd_gatt()
670 	return -1;  in init_amd_gatt()
703 		dev = node_to_amd_nb(i)->misc;  in gart_iommu_shutdown()
726 	/* Makefile puts PCI initialization via subsys_initcall first. */  in gart_iommu_init()
738 			pr_warn("More than 4GB of memory but GART IOMMU not available.\n");  in gart_iommu_init()
739 			pr_warn("falling back to iommu=soft.\n");  in gart_iommu_init()
754 	pr_info("PCI-DMA: using GART IOMMU.\n");  in gart_iommu_init()
761 		panic("Cannot allocate iommu bitmap\n");  in gart_iommu_init()
763 	pr_info("PCI-DMA: Reserving %luMB of IOMMU area in the AGP aperture\n",  in gart_iommu_init()
767 	iommu_start		= aper_size - iommu_size;  in gart_iommu_init()
772 	 * Unmap the IOMMU part of the GART. The alias of the page is  in gart_iommu_init()
777 	 * the backing memory. The GART address is only used by PCI  in gart_iommu_init()
785 	 * is remapped to UC later on, we might surprise the PCI devices  in gart_iommu_init()
787 	 * do an explicit, full-scale wbinvd() _after_ having marked all  in gart_iommu_init()
788 	 * the pages as Not-Present:  in gart_iommu_init()
808 		panic("Cannot allocate iommu scratch page");  in gart_iommu_init()
833 	/* duplicated from pci-dma.c */  in gart_parse_options()