1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2024, Google LLC.
4 * Pasha Tatashin <pasha.tatashin@soleen.com>
5 */
6 #include "iommu-pages.h"
7 #include <linux/dma-mapping.h>
8 #include <linux/gfp.h>
9 #include <linux/mm.h>
10
11 #define IOPTDESC_MATCH(pg_elm, elm) \
12 static_assert(offsetof(struct page, pg_elm) == \
13 offsetof(struct ioptdesc, elm))
14 IOPTDESC_MATCH(flags, __page_flags);
15 IOPTDESC_MATCH(lru, iopt_freelist_elm); /* Ensure bit 0 is clear */
16 IOPTDESC_MATCH(mapping, __page_mapping);
17 IOPTDESC_MATCH(private, _private);
18 IOPTDESC_MATCH(page_type, __page_type);
19 IOPTDESC_MATCH(_refcount, __page_refcount);
20 #ifdef CONFIG_MEMCG
21 IOPTDESC_MATCH(memcg_data, memcg_data);
22 #endif
23 #undef IOPTDESC_MATCH
24 static_assert(sizeof(struct ioptdesc) <= sizeof(struct page));
25
ioptdesc_mem_size(struct ioptdesc * desc)26 static inline size_t ioptdesc_mem_size(struct ioptdesc *desc)
27 {
28 return 1UL << (folio_order(ioptdesc_folio(desc)) + PAGE_SHIFT);
29 }
30
31 /**
32 * iommu_alloc_pages_node_sz - Allocate a zeroed page of a given size from
33 * specific NUMA node
34 * @nid: memory NUMA node id
35 * @gfp: buddy allocator flags
36 * @size: Memory size to allocate, rounded up to a power of 2
37 *
38 * Returns the virtual address of the allocated page. The page must be freed
39 * either by calling iommu_free_pages() or via iommu_put_pages_list(). The
40 * returned allocation is round_up_pow_two(size) big, and is physically aligned
41 * to its size.
42 */
iommu_alloc_pages_node_sz(int nid,gfp_t gfp,size_t size)43 void *iommu_alloc_pages_node_sz(int nid, gfp_t gfp, size_t size)
44 {
45 struct ioptdesc *iopt;
46 unsigned long pgcnt;
47 struct folio *folio;
48 unsigned int order;
49
50 /* This uses page_address() on the memory. */
51 if (WARN_ON(gfp & __GFP_HIGHMEM))
52 return NULL;
53
54 /*
55 * Currently sub page allocations result in a full page being returned.
56 */
57 order = get_order(size);
58
59 /*
60 * __folio_alloc_node() does not handle NUMA_NO_NODE like
61 * alloc_pages_node() did.
62 */
63 if (nid == NUMA_NO_NODE)
64 nid = numa_mem_id();
65
66 folio = __folio_alloc_node(gfp | __GFP_ZERO, order, nid);
67 if (unlikely(!folio))
68 return NULL;
69
70 iopt = folio_ioptdesc(folio);
71 iopt->incoherent = false;
72
73 /*
74 * All page allocations that should be reported to as "iommu-pagetables"
75 * to userspace must use one of the functions below. This includes
76 * allocations of page-tables and other per-iommu_domain configuration
77 * structures.
78 *
79 * This is necessary for the proper accounting as IOMMU state can be
80 * rather large, i.e. multiple gigabytes in size.
81 */
82 pgcnt = 1UL << order;
83 mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, pgcnt);
84 lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, pgcnt);
85
86 return folio_address(folio);
87 }
88 EXPORT_SYMBOL_GPL(iommu_alloc_pages_node_sz);
89
__iommu_free_desc(struct ioptdesc * iopt)90 static void __iommu_free_desc(struct ioptdesc *iopt)
91 {
92 struct folio *folio = ioptdesc_folio(iopt);
93 const unsigned long pgcnt = folio_nr_pages(folio);
94
95 if (IOMMU_PAGES_USE_DMA_API)
96 WARN_ON_ONCE(iopt->incoherent);
97
98 mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, -pgcnt);
99 lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, -pgcnt);
100 folio_put(folio);
101 }
102
103 /**
104 * iommu_free_pages - free pages
105 * @virt: virtual address of the page to be freed.
106 *
107 * The page must have have been allocated by iommu_alloc_pages_node_sz()
108 */
iommu_free_pages(void * virt)109 void iommu_free_pages(void *virt)
110 {
111 if (!virt)
112 return;
113 __iommu_free_desc(virt_to_ioptdesc(virt));
114 }
115 EXPORT_SYMBOL_GPL(iommu_free_pages);
116
117 /**
118 * iommu_put_pages_list - free a list of pages.
119 * @list: The list of pages to be freed
120 *
121 * Frees a list of pages allocated by iommu_alloc_pages_node_sz(). On return the
122 * passed list is invalid, the caller must use IOMMU_PAGES_LIST_INIT to reinit
123 * the list if it expects to use it again.
124 */
iommu_put_pages_list(struct iommu_pages_list * list)125 void iommu_put_pages_list(struct iommu_pages_list *list)
126 {
127 struct ioptdesc *iopt, *tmp;
128
129 list_for_each_entry_safe(iopt, tmp, &list->pages, iopt_freelist_elm)
130 __iommu_free_desc(iopt);
131 }
132 EXPORT_SYMBOL_GPL(iommu_put_pages_list);
133
134 /**
135 * iommu_pages_start_incoherent - Setup the page for cache incoherent operation
136 * @virt: The page to setup
137 * @dma_dev: The iommu device
138 *
139 * For incoherent memory this will use the DMA API to manage the cache flushing
140 * on some arches. This is a lot of complexity compared to just calling
141 * arch_sync_dma_for_device(), but it is what the existing ARM iommu drivers
142 * have been doing. The DMA API requires keeping track of the DMA map and
143 * freeing it when required. This keeps track of the dma map inside the ioptdesc
144 * so that error paths are simple for the caller.
145 */
iommu_pages_start_incoherent(void * virt,struct device * dma_dev)146 int iommu_pages_start_incoherent(void *virt, struct device *dma_dev)
147 {
148 struct ioptdesc *iopt = virt_to_ioptdesc(virt);
149 dma_addr_t dma;
150
151 if (WARN_ON(iopt->incoherent))
152 return -EINVAL;
153
154 if (!IOMMU_PAGES_USE_DMA_API) {
155 iommu_pages_flush_incoherent(dma_dev, virt, 0,
156 ioptdesc_mem_size(iopt));
157 } else {
158 dma = dma_map_single(dma_dev, virt, ioptdesc_mem_size(iopt),
159 DMA_TO_DEVICE);
160 if (dma_mapping_error(dma_dev, dma))
161 return -EINVAL;
162
163 /*
164 * The DMA API is not allowed to do anything other than DMA
165 * direct. It would be nice to also check
166 * dev_is_dma_coherent(dma_dev));
167 */
168 if (WARN_ON(dma != virt_to_phys(virt))) {
169 dma_unmap_single(dma_dev, dma, ioptdesc_mem_size(iopt),
170 DMA_TO_DEVICE);
171 return -EOPNOTSUPP;
172 }
173 }
174
175 iopt->incoherent = 1;
176 return 0;
177 }
178 EXPORT_SYMBOL_GPL(iommu_pages_start_incoherent);
179
180 /**
181 * iommu_pages_start_incoherent_list - Make a list of pages incoherent
182 * @list: The list of pages to setup
183 * @dma_dev: The iommu device
184 *
185 * Perform iommu_pages_start_incoherent() across all of list.
186 *
187 * If this fails the caller must call iommu_pages_stop_incoherent_list().
188 */
iommu_pages_start_incoherent_list(struct iommu_pages_list * list,struct device * dma_dev)189 int iommu_pages_start_incoherent_list(struct iommu_pages_list *list,
190 struct device *dma_dev)
191 {
192 struct ioptdesc *cur;
193 int ret;
194
195 list_for_each_entry(cur, &list->pages, iopt_freelist_elm) {
196 if (WARN_ON(cur->incoherent))
197 continue;
198
199 ret = iommu_pages_start_incoherent(
200 folio_address(ioptdesc_folio(cur)), dma_dev);
201 if (ret)
202 return ret;
203 }
204 return 0;
205 }
206 EXPORT_SYMBOL_GPL(iommu_pages_start_incoherent_list);
207
208 /**
209 * iommu_pages_stop_incoherent_list - Undo incoherence across a list
210 * @list: The list of pages to release
211 * @dma_dev: The iommu device
212 *
213 * Revert iommu_pages_start_incoherent() across all of the list. Pages that did
214 * not call or succeed iommu_pages_start_incoherent() will be ignored.
215 */
216 #if IOMMU_PAGES_USE_DMA_API
iommu_pages_stop_incoherent_list(struct iommu_pages_list * list,struct device * dma_dev)217 void iommu_pages_stop_incoherent_list(struct iommu_pages_list *list,
218 struct device *dma_dev)
219 {
220 struct ioptdesc *cur;
221
222 list_for_each_entry(cur, &list->pages, iopt_freelist_elm) {
223 struct folio *folio = ioptdesc_folio(cur);
224
225 if (!cur->incoherent)
226 continue;
227 dma_unmap_single(dma_dev, virt_to_phys(folio_address(folio)),
228 ioptdesc_mem_size(cur), DMA_TO_DEVICE);
229 cur->incoherent = 0;
230 }
231 }
232 EXPORT_SYMBOL_GPL(iommu_pages_stop_incoherent_list);
233
234 /**
235 * iommu_pages_free_incoherent - Free an incoherent page
236 * @virt: virtual address of the page to be freed.
237 * @dma_dev: The iommu device
238 *
239 * If the page is incoherent it made coherent again then freed.
240 */
iommu_pages_free_incoherent(void * virt,struct device * dma_dev)241 void iommu_pages_free_incoherent(void *virt, struct device *dma_dev)
242 {
243 struct ioptdesc *iopt = virt_to_ioptdesc(virt);
244
245 if (iopt->incoherent) {
246 dma_unmap_single(dma_dev, virt_to_phys(virt),
247 ioptdesc_mem_size(iopt), DMA_TO_DEVICE);
248 iopt->incoherent = 0;
249 }
250 __iommu_free_desc(iopt);
251 }
252 EXPORT_SYMBOL_GPL(iommu_pages_free_incoherent);
253 #endif
254