1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __LINUX_SWIOTLB_H
3 #define __LINUX_SWIOTLB_H
4
5 #include <linux/device.h>
6 #include <linux/dma-direction.h>
7 #include <linux/init.h>
8 #include <linux/types.h>
9 #include <linux/limits.h>
10 #include <linux/spinlock.h>
11 #include <linux/workqueue.h>
12
13 struct device;
14 struct page;
15 struct scatterlist;
16
17 #define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */
18 #define SWIOTLB_ANY (1 << 1) /* allow any memory for the buffer */
19
20 /*
21 * Maximum allowable number of contiguous slabs to map,
22 * must be a power of 2. What is the appropriate value ?
23 * The complexity of {map,unmap}_single is linearly dependent on this value.
24 */
25 #define IO_TLB_SEGSIZE 128
26
27 /*
28 * log of the size of each IO TLB slab. The number of slabs is command line
29 * controllable.
30 */
31 #define IO_TLB_SHIFT 11
32 #define IO_TLB_SIZE (1 << IO_TLB_SHIFT)
33
34 /* compile-time default; overridable via CONFIG_SWIOTLB_DEFAULT_SIZE_MB */
35 #ifdef CONFIG_SWIOTLB
36 #define IO_TLB_DEFAULT_SIZE ((unsigned long)CONFIG_SWIOTLB_DEFAULT_SIZE_MB << 20)
37 #else
38 #define IO_TLB_DEFAULT_SIZE (64UL << 20)
39 #endif
40
41 unsigned long swiotlb_size_or_default(void);
42 void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
43 int (*remap)(void *tlb, unsigned long nslabs));
44 int swiotlb_init_late(size_t size, gfp_t gfp_mask,
45 int (*remap)(void *tlb, unsigned long nslabs));
46 extern void __init swiotlb_update_mem_attributes(void);
47
48 #ifdef CONFIG_SWIOTLB
49
50 /**
51 * struct io_tlb_pool - IO TLB memory pool descriptor
52 * @start: The start address of the swiotlb memory pool. Used to do a quick
53 * range check to see if the memory was in fact allocated by this
54 * API.
55 * @end: The end address of the swiotlb memory pool. Used to do a quick
56 * range check to see if the memory was in fact allocated by this
57 * API.
58 * @vaddr: The vaddr of the swiotlb memory pool. The swiotlb memory pool
59 * may be remapped in the memory encrypted case and store virtual
60 * address for bounce buffer operation.
61 * @nslabs: The number of IO TLB slots between @start and @end. For the
62 * default swiotlb, this can be adjusted with a boot parameter,
63 * see setup_io_tlb_npages().
64 * @late_alloc: %true if allocated using the page allocator.
65 * @nareas: Number of areas in the pool.
66 * @area_nslabs: Number of slots in each area.
67 * @areas: Array of memory area descriptors.
68 * @slots: Array of slot descriptors.
69 * @node: Member of the IO TLB memory pool list.
70 * @dyn_free: RCU work item used to free the pool from process context.
71 * @transient: %true if transient memory pool.
72 * @cc_shared: %true if the pool memory is shared for confidential computing.
73 */
74 struct io_tlb_pool {
75 phys_addr_t start;
76 phys_addr_t end;
77 void *vaddr;
78 unsigned long nslabs;
79 bool late_alloc;
80 unsigned int nareas;
81 unsigned int area_nslabs;
82 struct io_tlb_area *areas;
83 struct io_tlb_slot *slots;
84 #ifdef CONFIG_SWIOTLB_DYNAMIC
85 struct list_head node;
86 struct rcu_work dyn_free;
87 bool transient;
88 bool cc_shared;
89 #endif
90 };
91
92 /**
93 * struct io_tlb_mem - Software IO TLB allocator
94 * @defpool: Default (initial) IO TLB memory pool descriptor.
95 * @pool: IO TLB memory pool descriptor (if not dynamic).
96 * @nslabs: Total number of IO TLB slabs in all pools.
97 * @debugfs: The dentry to debugfs.
98 * @force_bounce: %true if swiotlb bouncing is forced
99 * @for_alloc: %true if the pool is used for memory allocation
100 * @cc_shared: %true if the pool memory is shared for confidential computing.
101 * @can_grow: %true if more pools can be allocated dynamically.
102 * @phys_limit: Maximum allowed physical address.
103 * @lock: Lock to synchronize changes to the list.
104 * @pools: List of IO TLB memory pool descriptors (if dynamic).
105 * @dyn_alloc: Dynamic IO TLB pool allocation work.
106 * @total_used: The total number of slots in the pool that are currently used
107 * across all areas. Used only for calculating used_hiwater via boot
108 * parameter swiotlb=track_hiwater and exposed via debugfs.
109 * @used_hiwater: The high water mark for total_used. Can be enabled at boot
110 * time via swiotlb=track_hiwater and exposed via debugfs.
111 * @transient_nslabs: The total number of slots in all transient pools that
112 * are currently used across all areas.
113 */
114 struct io_tlb_mem {
115 struct io_tlb_pool defpool;
116 unsigned long nslabs;
117 struct dentry *debugfs;
118 bool force_bounce;
119 bool for_alloc;
120 bool cc_shared;
121 #ifdef CONFIG_SWIOTLB_DYNAMIC
122 bool can_grow;
123 u64 phys_limit;
124 spinlock_t lock;
125 struct list_head pools;
126 struct work_struct dyn_alloc;
127 #endif
128 #ifdef CONFIG_DEBUG_FS
129 atomic_long_t total_used;
130 atomic_long_t used_hiwater;
131 atomic_long_t transient_nslabs;
132 #endif
133 };
134
135 struct io_tlb_pool *__swiotlb_find_pool(struct device *dev, phys_addr_t paddr);
136
137 /**
138 * swiotlb_find_pool() - find swiotlb pool to which a physical address belongs
139 * @dev: Device which has mapped the buffer.
140 * @paddr: Physical address within the DMA buffer.
141 *
142 * Find the swiotlb pool that @paddr points into.
143 *
144 * Return:
145 * * pool address if @paddr points into a bounce buffer
146 * * NULL if @paddr does not point into a bounce buffer. As such, this function
147 * can be used to determine if @paddr denotes a swiotlb bounce buffer.
148 */
swiotlb_find_pool(struct device * dev,phys_addr_t paddr)149 static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
150 phys_addr_t paddr)
151 {
152 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
153
154 if (!mem)
155 return NULL;
156
157 #ifdef CONFIG_SWIOTLB_DYNAMIC
158 /*
159 * All SWIOTLB buffer addresses must have been returned by
160 * swiotlb_tbl_map_single() and passed to a device driver.
161 * If a SWIOTLB address is checked on another CPU, then it was
162 * presumably loaded by the device driver from an unspecified private
163 * data structure. Make sure that this load is ordered before reading
164 * dev->dma_uses_io_tlb here and mem->pools in __swiotlb_find_pool().
165 *
166 * This barrier pairs with smp_mb() in swiotlb_find_slots().
167 */
168 smp_rmb();
169 if (READ_ONCE(dev->dma_uses_io_tlb))
170 return __swiotlb_find_pool(dev, paddr);
171 #else
172 if (paddr >= mem->defpool.start && paddr < mem->defpool.end)
173 return &mem->defpool;
174 #endif
175
176 return NULL;
177 }
178
is_swiotlb_force_bounce(struct device * dev)179 static inline bool is_swiotlb_force_bounce(struct device *dev)
180 {
181 struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
182
183 return mem && mem->force_bounce;
184 }
185
186 void swiotlb_init(bool addressing_limited, unsigned int flags);
187 void __init swiotlb_exit(void);
188 void swiotlb_dev_init(struct device *dev);
189 size_t swiotlb_max_mapping_size(struct device *dev);
190 bool is_swiotlb_allocated(void);
191 bool is_swiotlb_active(struct device *dev);
192 void __init swiotlb_adjust_size(unsigned long size);
193 phys_addr_t default_swiotlb_base(void);
194 phys_addr_t default_swiotlb_limit(void);
195 #else
swiotlb_init(bool addressing_limited,unsigned int flags)196 static inline void swiotlb_init(bool addressing_limited, unsigned int flags)
197 {
198 }
199
swiotlb_dev_init(struct device * dev)200 static inline void swiotlb_dev_init(struct device *dev)
201 {
202 }
203
swiotlb_find_pool(struct device * dev,phys_addr_t paddr)204 static inline struct io_tlb_pool *swiotlb_find_pool(struct device *dev,
205 phys_addr_t paddr)
206 {
207 return NULL;
208 }
is_swiotlb_force_bounce(struct device * dev)209 static inline bool is_swiotlb_force_bounce(struct device *dev)
210 {
211 return false;
212 }
swiotlb_exit(void)213 static inline void swiotlb_exit(void)
214 {
215 }
swiotlb_max_mapping_size(struct device * dev)216 static inline size_t swiotlb_max_mapping_size(struct device *dev)
217 {
218 return SIZE_MAX;
219 }
220
is_swiotlb_allocated(void)221 static inline bool is_swiotlb_allocated(void)
222 {
223 return false;
224 }
225
is_swiotlb_active(struct device * dev)226 static inline bool is_swiotlb_active(struct device *dev)
227 {
228 return false;
229 }
230
swiotlb_adjust_size(unsigned long size)231 static inline void swiotlb_adjust_size(unsigned long size)
232 {
233 }
234
default_swiotlb_base(void)235 static inline phys_addr_t default_swiotlb_base(void)
236 {
237 return 0;
238 }
239
default_swiotlb_limit(void)240 static inline phys_addr_t default_swiotlb_limit(void)
241 {
242 return 0;
243 }
244 #endif /* CONFIG_SWIOTLB */
245
246 phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys,
247 size_t mapping_size, unsigned int alloc_aligned_mask,
248 enum dma_data_direction dir, unsigned long *attrs);
249 dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys,
250 size_t size, enum dma_data_direction dir, unsigned long attrs);
251
252 void __swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
253 size_t mapping_size, enum dma_data_direction dir,
254 unsigned long attrs, struct io_tlb_pool *pool);
swiotlb_tbl_unmap_single(struct device * dev,phys_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)255 static inline void swiotlb_tbl_unmap_single(struct device *dev,
256 phys_addr_t addr, size_t size, enum dma_data_direction dir,
257 unsigned long attrs)
258 {
259 struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
260
261 if (unlikely(pool))
262 __swiotlb_tbl_unmap_single(dev, addr, size, dir, attrs, pool);
263 }
264
265 void __swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
266 size_t size, enum dma_data_direction dir,
267 struct io_tlb_pool *pool);
swiotlb_sync_single_for_device(struct device * dev,phys_addr_t addr,size_t size,enum dma_data_direction dir)268 static inline void swiotlb_sync_single_for_device(struct device *dev,
269 phys_addr_t addr, size_t size, enum dma_data_direction dir)
270 {
271 struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
272
273 if (unlikely(pool))
274 __swiotlb_sync_single_for_device(dev, addr, size, dir, pool);
275 }
276
277 void __swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
278 size_t size, enum dma_data_direction dir,
279 struct io_tlb_pool *pool);
swiotlb_sync_single_for_cpu(struct device * dev,phys_addr_t addr,size_t size,enum dma_data_direction dir)280 static inline void swiotlb_sync_single_for_cpu(struct device *dev,
281 phys_addr_t addr, size_t size, enum dma_data_direction dir)
282 {
283 struct io_tlb_pool *pool = swiotlb_find_pool(dev, addr);
284
285 if (unlikely(pool))
286 __swiotlb_sync_single_for_cpu(dev, addr, size, dir, pool);
287 }
288
289 extern void swiotlb_print_info(void);
290
291 #ifdef CONFIG_DMA_RESTRICTED_POOL
292 struct page *swiotlb_alloc(struct device *dev, size_t size,
293 unsigned long attrs);
294 bool swiotlb_free(struct device *dev, struct page *page, size_t size);
295 void swiotlb_free_from_pool(struct device *dev,
296 phys_addr_t tlb_addr, struct io_tlb_pool *pool);
297
is_swiotlb_for_alloc(struct device * dev)298 static inline bool is_swiotlb_for_alloc(struct device *dev)
299 {
300 return dev->dma_io_tlb_mem->for_alloc;
301 }
302 #else
swiotlb_alloc(struct device * dev,size_t size,unsigned long attrs)303 static inline struct page *swiotlb_alloc(struct device *dev, size_t size,
304 unsigned long attrs)
305 {
306 return NULL;
307 }
swiotlb_free(struct device * dev,struct page * page,size_t size)308 static inline bool swiotlb_free(struct device *dev, struct page *page,
309 size_t size)
310 {
311 return false;
312 }
swiotlb_free_from_pool(struct device * dev,phys_addr_t tlb_addr,struct io_tlb_pool * pool)313 static inline void swiotlb_free_from_pool(struct device *dev,
314 phys_addr_t tlb_addr, struct io_tlb_pool *pool)
315 {
316 }
is_swiotlb_for_alloc(struct device * dev)317 static inline bool is_swiotlb_for_alloc(struct device *dev)
318 {
319 return false;
320 }
321 #endif /* CONFIG_DMA_RESTRICTED_POOL */
322
323 #endif /* __LINUX_SWIOTLB_H */
324