1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2024, Google LLC. 4 * Pasha Tatashin <pasha.tatashin@soleen.com> 5 */ 6 #include "iommu-pages.h" 7 #include <linux/dma-mapping.h> 8 #include <linux/gfp.h> 9 #include <linux/mm.h> 10 11 #define IOPTDESC_MATCH(pg_elm, elm) \ 12 static_assert(offsetof(struct page, pg_elm) == \ 13 offsetof(struct ioptdesc, elm)) 14 IOPTDESC_MATCH(flags, __page_flags); 15 IOPTDESC_MATCH(lru, iopt_freelist_elm); /* Ensure bit 0 is clear */ 16 IOPTDESC_MATCH(mapping, __page_mapping); 17 IOPTDESC_MATCH(private, _private); 18 IOPTDESC_MATCH(page_type, __page_type); 19 IOPTDESC_MATCH(_refcount, __page_refcount); 20 #ifdef CONFIG_MEMCG 21 IOPTDESC_MATCH(memcg_data, memcg_data); 22 #endif 23 #undef IOPTDESC_MATCH 24 static_assert(sizeof(struct ioptdesc) <= sizeof(struct page)); 25 26 static inline size_t ioptdesc_mem_size(struct ioptdesc *desc) 27 { 28 return 1UL << (folio_order(ioptdesc_folio(desc)) + PAGE_SHIFT); 29 } 30 31 /** 32 * iommu_alloc_pages_node_sz - Allocate a zeroed page of a given size from 33 * specific NUMA node 34 * @nid: memory NUMA node id 35 * @gfp: buddy allocator flags 36 * @size: Memory size to allocate, rounded up to a power of 2 37 * 38 * Returns the virtual address of the allocated page. The page must be freed 39 * either by calling iommu_free_pages() or via iommu_put_pages_list(). The 40 * returned allocation is round_up_pow_two(size) big, and is physically aligned 41 * to its size. 42 */ 43 void *iommu_alloc_pages_node_sz(int nid, gfp_t gfp, size_t size) 44 { 45 struct ioptdesc *iopt; 46 unsigned long pgcnt; 47 struct folio *folio; 48 unsigned int order; 49 50 /* This uses page_address() on the memory. */ 51 if (WARN_ON(gfp & __GFP_HIGHMEM)) 52 return NULL; 53 54 /* 55 * Currently sub page allocations result in a full page being returned. 56 */ 57 order = get_order(size); 58 59 /* 60 * __folio_alloc_node() does not handle NUMA_NO_NODE like 61 * alloc_pages_node() did. 62 */ 63 if (nid == NUMA_NO_NODE) 64 nid = numa_mem_id(); 65 66 folio = __folio_alloc_node(gfp | __GFP_ZERO, order, nid); 67 if (unlikely(!folio)) 68 return NULL; 69 70 iopt = folio_ioptdesc(folio); 71 iopt->incoherent = false; 72 73 /* 74 * All page allocations that should be reported to as "iommu-pagetables" 75 * to userspace must use one of the functions below. This includes 76 * allocations of page-tables and other per-iommu_domain configuration 77 * structures. 78 * 79 * This is necessary for the proper accounting as IOMMU state can be 80 * rather large, i.e. multiple gigabytes in size. 81 */ 82 pgcnt = 1UL << order; 83 mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, pgcnt); 84 lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, pgcnt); 85 86 return folio_address(folio); 87 } 88 EXPORT_SYMBOL_GPL(iommu_alloc_pages_node_sz); 89 90 static void __iommu_free_desc(struct ioptdesc *iopt) 91 { 92 struct folio *folio = ioptdesc_folio(iopt); 93 const unsigned long pgcnt = folio_nr_pages(folio); 94 95 if (IOMMU_PAGES_USE_DMA_API) 96 WARN_ON_ONCE(iopt->incoherent); 97 98 mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, -pgcnt); 99 lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, -pgcnt); 100 folio_put(folio); 101 } 102 103 /** 104 * iommu_free_pages - free pages 105 * @virt: virtual address of the page to be freed. 106 * 107 * The page must have have been allocated by iommu_alloc_pages_node_sz() 108 */ 109 void iommu_free_pages(void *virt) 110 { 111 if (!virt) 112 return; 113 __iommu_free_desc(virt_to_ioptdesc(virt)); 114 } 115 EXPORT_SYMBOL_GPL(iommu_free_pages); 116 117 /** 118 * iommu_put_pages_list - free a list of pages. 119 * @list: The list of pages to be freed 120 * 121 * Frees a list of pages allocated by iommu_alloc_pages_node_sz(). On return the 122 * passed list is invalid, the caller must use IOMMU_PAGES_LIST_INIT to reinit 123 * the list if it expects to use it again. 124 */ 125 void iommu_put_pages_list(struct iommu_pages_list *list) 126 { 127 struct ioptdesc *iopt, *tmp; 128 129 list_for_each_entry_safe(iopt, tmp, &list->pages, iopt_freelist_elm) 130 __iommu_free_desc(iopt); 131 } 132 EXPORT_SYMBOL_GPL(iommu_put_pages_list); 133 134 /** 135 * iommu_pages_start_incoherent - Setup the page for cache incoherent operation 136 * @virt: The page to setup 137 * @dma_dev: The iommu device 138 * 139 * For incoherent memory this will use the DMA API to manage the cache flushing 140 * on some arches. This is a lot of complexity compared to just calling 141 * arch_sync_dma_for_device(), but it is what the existing ARM iommu drivers 142 * have been doing. The DMA API requires keeping track of the DMA map and 143 * freeing it when required. This keeps track of the dma map inside the ioptdesc 144 * so that error paths are simple for the caller. 145 */ 146 int iommu_pages_start_incoherent(void *virt, struct device *dma_dev) 147 { 148 struct ioptdesc *iopt = virt_to_ioptdesc(virt); 149 dma_addr_t dma; 150 151 if (WARN_ON(iopt->incoherent)) 152 return -EINVAL; 153 154 if (!IOMMU_PAGES_USE_DMA_API) { 155 iommu_pages_flush_incoherent(dma_dev, virt, 0, 156 ioptdesc_mem_size(iopt)); 157 } else { 158 dma = dma_map_single(dma_dev, virt, ioptdesc_mem_size(iopt), 159 DMA_TO_DEVICE); 160 if (dma_mapping_error(dma_dev, dma)) 161 return -EINVAL; 162 163 /* 164 * The DMA API is not allowed to do anything other than DMA 165 * direct. It would be nice to also check 166 * dev_is_dma_coherent(dma_dev)); 167 */ 168 if (WARN_ON(dma != virt_to_phys(virt))) { 169 dma_unmap_single(dma_dev, dma, ioptdesc_mem_size(iopt), 170 DMA_TO_DEVICE); 171 return -EOPNOTSUPP; 172 } 173 } 174 175 iopt->incoherent = 1; 176 return 0; 177 } 178 EXPORT_SYMBOL_GPL(iommu_pages_start_incoherent); 179 180 /** 181 * iommu_pages_start_incoherent_list - Make a list of pages incoherent 182 * @list: The list of pages to setup 183 * @dma_dev: The iommu device 184 * 185 * Perform iommu_pages_start_incoherent() across all of list. 186 * 187 * If this fails the caller must call iommu_pages_stop_incoherent_list(). 188 */ 189 int iommu_pages_start_incoherent_list(struct iommu_pages_list *list, 190 struct device *dma_dev) 191 { 192 struct ioptdesc *cur; 193 int ret; 194 195 list_for_each_entry(cur, &list->pages, iopt_freelist_elm) { 196 if (WARN_ON(cur->incoherent)) 197 continue; 198 199 ret = iommu_pages_start_incoherent( 200 folio_address(ioptdesc_folio(cur)), dma_dev); 201 if (ret) 202 return ret; 203 } 204 return 0; 205 } 206 EXPORT_SYMBOL_GPL(iommu_pages_start_incoherent_list); 207 208 /** 209 * iommu_pages_stop_incoherent_list - Undo incoherence across a list 210 * @list: The list of pages to release 211 * @dma_dev: The iommu device 212 * 213 * Revert iommu_pages_start_incoherent() across all of the list. Pages that did 214 * not call or succeed iommu_pages_start_incoherent() will be ignored. 215 */ 216 #if IOMMU_PAGES_USE_DMA_API 217 void iommu_pages_stop_incoherent_list(struct iommu_pages_list *list, 218 struct device *dma_dev) 219 { 220 struct ioptdesc *cur; 221 222 list_for_each_entry(cur, &list->pages, iopt_freelist_elm) { 223 struct folio *folio = ioptdesc_folio(cur); 224 225 if (!cur->incoherent) 226 continue; 227 dma_unmap_single(dma_dev, virt_to_phys(folio_address(folio)), 228 ioptdesc_mem_size(cur), DMA_TO_DEVICE); 229 cur->incoherent = 0; 230 } 231 } 232 EXPORT_SYMBOL_GPL(iommu_pages_stop_incoherent_list); 233 234 /** 235 * iommu_pages_free_incoherent - Free an incoherent page 236 * @virt: virtual address of the page to be freed. 237 * @dma_dev: The iommu device 238 * 239 * If the page is incoherent it made coherent again then freed. 240 */ 241 void iommu_pages_free_incoherent(void *virt, struct device *dma_dev) 242 { 243 struct ioptdesc *iopt = virt_to_ioptdesc(virt); 244 245 if (iopt->incoherent) { 246 dma_unmap_single(dma_dev, virt_to_phys(virt), 247 ioptdesc_mem_size(iopt), DMA_TO_DEVICE); 248 iopt->incoherent = 0; 249 } 250 __iommu_free_desc(iopt); 251 } 252 EXPORT_SYMBOL_GPL(iommu_pages_free_incoherent); 253 #endif 254