xref: /freebsd/sys/x86/iommu/intel_dmar.h (revision 8d20be1e22095c27faf8fe8b2f0d089739cc742e)
1 /*-
2  * Copyright (c) 2013 The FreeBSD Foundation
3  * All rights reserved.
4  *
5  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
6  * under sponsorship from the FreeBSD Foundation.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 #ifndef __X86_IOMMU_INTEL_DMAR_H
33 #define	__X86_IOMMU_INTEL_DMAR_H
34 
35 /* Host or physical memory address, after translation. */
36 typedef uint64_t dmar_haddr_t;
37 /* Guest or bus address, before translation. */
38 typedef uint64_t dmar_gaddr_t;
39 
40 struct dmar_map_entry {
41 	dmar_gaddr_t start;
42 	dmar_gaddr_t end;
43 	dmar_gaddr_t free_after;	/* Free space after the entry */
44 	dmar_gaddr_t free_down;		/* Max free space below the
45 					   current R/B tree node */
46 	u_int flags;
47 	TAILQ_ENTRY(dmar_map_entry) dmamap_link; /* Link for dmamap entries */
48 	RB_ENTRY(dmar_map_entry) rb_entry;	 /* Links for ctx entries */
49 	TAILQ_ENTRY(dmar_map_entry) unroll_link; /* Link for unroll after
50 						    dmamap_load failure */
51 };
52 
53 RB_HEAD(dmar_gas_entries_tree, dmar_map_entry);
54 RB_PROTOTYPE(dmar_gas_entries_tree, dmar_map_entry, rb_entry,
55     dmar_gas_cmp_entries);
56 
57 #define	DMAR_MAP_ENTRY_PLACE	0x0001	/* Fake entry */
58 #define	DMAR_MAP_ENTRY_RMRR	0x0002	/* Permanent, not linked by
59 					   dmamap_link */
60 #define	DMAR_MAP_ENTRY_MAP	0x0004	/* Busdma created, linked by
61 					   dmamap_link */
62 #define	DMAR_MAP_ENTRY_UNMAPPED	0x0010	/* No backing pages */
63 #define	DMAR_MAP_ENTRY_READ	0x1000	/* Read permitted */
64 #define	DMAR_MAP_ENTRY_WRITE	0x2000	/* Write permitted */
65 #define	DMAR_MAP_ENTRY_SNOOP	0x4000	/* Snoop */
66 #define	DMAR_MAP_ENTRY_TM	0x8000	/* Transient */
67 
68 struct dmar_ctx {
69 	int bus;	/* pci bus/slot/func */
70 	int slot;
71 	int func;
72 	int domain;	/* DID */
73 	int mgaw;	/* Real max address width */
74 	int agaw;	/* Adjusted guest address width */
75 	int pglvl;	/* The pagelevel */
76 	int awlvl;	/* The pagelevel as the bitmask, to set in
77 			   context entry */
78 	dmar_gaddr_t end;/* Highest address + 1 in the guest AS */
79 	u_int refs;	/* References to the context, from tags */
80 	struct dmar_unit *dmar;
81 	struct bus_dma_tag_dmar ctx_tag; /* Root tag */
82 	struct mtx lock;
83 	LIST_ENTRY(dmar_ctx) link;	/* Member in the dmar list */
84 	vm_object_t pgtbl_obj;		/* Page table pages */
85 	u_int flags;			/* Protected by dmar lock */
86 	uint64_t last_fault_rec[2];	/* Last fault reported */
87 	u_int entries_cnt;
88 	u_long loads;
89 	u_long unloads;
90 	struct dmar_gas_entries_tree rb_root;
91 	struct dmar_map_entries_tailq unload_entries; /* Entries to unload */
92 	struct dmar_map_entry *first_place, *last_place;
93 	struct task unload_task;
94 };
95 
96 /* struct dmar_ctx flags */
97 #define	DMAR_CTX_FAULTED	0x0001	/* Fault was reported,
98 					   last_fault_rec is valid */
99 #define	DMAR_CTX_IDMAP		0x0002	/* Context uses identity page table */
100 #define	DMAR_CTX_RMRR		0x0004	/* Context contains RMRR entry,
101 					   cannot be turned off */
102 #define	DMAR_CTX_DISABLED	0x0008	/* Device is disabled, the
103 					   ephemeral reference is kept
104 					   to prevent context destruction */
105 
106 #define	DMAR_CTX_PGLOCK(ctx)	VM_OBJECT_WLOCK((ctx)->pgtbl_obj)
107 #define	DMAR_CTX_PGTRYLOCK(ctx)	VM_OBJECT_TRYWLOCK((ctx)->pgtbl_obj)
108 #define	DMAR_CTX_PGUNLOCK(ctx)	VM_OBJECT_WUNLOCK((ctx)->pgtbl_obj)
109 #define	DMAR_CTX_ASSERT_PGLOCKED(ctx) \
110 	VM_OBJECT_ASSERT_WLOCKED((ctx)->pgtbl_obj)
111 
112 #define	DMAR_CTX_LOCK(ctx)	mtx_lock(&(ctx)->lock)
113 #define	DMAR_CTX_UNLOCK(ctx)	mtx_unlock(&(ctx)->lock)
114 #define	DMAR_CTX_ASSERT_LOCKED(ctx) mtx_assert(&(ctx)->lock, MA_OWNED)
115 
116 struct dmar_unit {
117 	device_t dev;
118 	int unit;
119 	uint16_t segment;
120 	uint64_t base;
121 
122 	/* Resources */
123 	int reg_rid;
124 	struct resource *regs;
125 	int irq;
126 	int irq_rid;
127 	struct resource *irq_res;
128 	void *intr_handle;
129 
130 	/* Hardware registers cache */
131 	uint32_t hw_ver;
132 	uint64_t hw_cap;
133 	uint64_t hw_ecap;
134 	uint32_t hw_gcmd;
135 
136 	/* Data for being a dmar */
137 	struct mtx lock;
138 	LIST_HEAD(, dmar_ctx) contexts;
139 	struct unrhdr *domids;
140 	vm_object_t ctx_obj;
141 	u_int barrier_flags;
142 
143 	/* Fault handler data */
144 	struct mtx fault_lock;
145 	uint64_t *fault_log;
146 	int fault_log_head;
147 	int fault_log_tail;
148 	int fault_log_size;
149 	struct task fault_task;
150 	struct taskqueue *fault_taskqueue;
151 
152 	/* Busdma delayed map load */
153 	struct task dmamap_load_task;
154 	TAILQ_HEAD(, bus_dmamap_dmar) delayed_maps;
155 	struct taskqueue *delayed_taskqueue;
156 };
157 
158 #define	DMAR_LOCK(dmar)		mtx_lock(&(dmar)->lock)
159 #define	DMAR_UNLOCK(dmar)	mtx_unlock(&(dmar)->lock)
160 #define	DMAR_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->lock, MA_OWNED)
161 
162 #define	DMAR_FAULT_LOCK(dmar)	mtx_lock_spin(&(dmar)->fault_lock)
163 #define	DMAR_FAULT_UNLOCK(dmar)	mtx_unlock_spin(&(dmar)->fault_lock)
164 #define	DMAR_FAULT_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->fault_lock, MA_OWNED)
165 
166 #define	DMAR_IS_COHERENT(dmar)	(((dmar)->hw_ecap & DMAR_ECAP_C) != 0)
167 
168 /* Barrier ids */
169 #define	DMAR_BARRIER_RMRR	0
170 #define	DMAR_BARRIER_USEQ	1
171 
172 struct dmar_unit *dmar_find(device_t dev);
173 
174 u_int dmar_nd2mask(u_int nd);
175 bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl);
176 int ctx_set_agaw(struct dmar_ctx *ctx, int mgaw);
177 int dmar_maxaddr2mgaw(struct dmar_unit* unit, dmar_gaddr_t maxaddr,
178     bool allow_less);
179 vm_pindex_t pglvl_max_pages(int pglvl);
180 int ctx_is_sp_lvl(struct dmar_ctx *ctx, int lvl);
181 dmar_gaddr_t pglvl_page_size(int total_pglvl, int lvl);
182 dmar_gaddr_t ctx_page_size(struct dmar_ctx *ctx, int lvl);
183 struct vm_page *dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags);
184 void dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags);
185 void *dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags,
186     struct sf_buf **sf);
187 void dmar_unmap_pgtbl(struct sf_buf *sf, bool coherent);
188 int dmar_load_root_entry_ptr(struct dmar_unit *unit);
189 int dmar_inv_ctx_glob(struct dmar_unit *unit);
190 int dmar_inv_iotlb_glob(struct dmar_unit *unit);
191 int dmar_flush_write_bufs(struct dmar_unit *unit);
192 int dmar_enable_translation(struct dmar_unit *unit);
193 int dmar_disable_translation(struct dmar_unit *unit);
194 void dmar_enable_intr(struct dmar_unit *unit);
195 void dmar_disable_intr(struct dmar_unit *unit);
196 bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id);
197 void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id);
198 
199 int dmar_intr(void *arg);
200 int dmar_init_fault_log(struct dmar_unit *unit);
201 void dmar_fini_fault_log(struct dmar_unit *unit);
202 
203 vm_object_t ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr);
204 void put_idmap_pgtbl(vm_object_t obj);
205 int ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
206     vm_page_t *ma, uint64_t pflags, int flags);
207 int ctx_unmap_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
208     int flags);
209 int ctx_alloc_pgtbl(struct dmar_ctx *ctx);
210 void ctx_free_pgtbl(struct dmar_ctx *ctx);
211 
212 struct dmar_ctx *dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev,
213     bool rmrr);
214 struct dmar_ctx *dmar_get_ctx(struct dmar_unit *dmar, device_t dev,
215     bool id_mapped, bool rmrr_init);
216 void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx);
217 void dmar_free_ctx(struct dmar_ctx *ctx);
218 struct dmar_ctx *dmar_find_ctx_locked(struct dmar_unit *dmar, int bus,
219     int slot, int func);
220 void dmar_ctx_unload(struct dmar_ctx *ctx,
221     struct dmar_map_entries_tailq *entries, bool cansleep);
222 
223 int dmar_init_busdma(struct dmar_unit *unit);
224 void dmar_fini_busdma(struct dmar_unit *unit);
225 
226 void dmar_gas_init_ctx(struct dmar_ctx *ctx);
227 void dmar_gas_fini_ctx(struct dmar_ctx *ctx);
228 struct dmar_map_entry *dmar_gas_alloc_entry(struct dmar_ctx *ctx, u_int flags);
229 void dmar_gas_free_entry(struct dmar_ctx *ctx, struct dmar_map_entry *entry);
230 void dmar_gas_free_space(struct dmar_ctx *ctx, struct dmar_map_entry *entry);
231 int dmar_gas_map(struct dmar_ctx *ctx, const struct bus_dma_tag_common *common,
232     dmar_gaddr_t size, u_int eflags, u_int flags, vm_page_t *ma,
233     struct dmar_map_entry **res);
234 int dmar_gas_map_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
235     u_int eflags, u_int flags, vm_page_t *ma);
236 int dmar_gas_reserve_region(struct dmar_ctx *ctx, dmar_gaddr_t start,
237     dmar_gaddr_t end);
238 
239 void dmar_ctx_parse_rmrr(struct dmar_ctx *ctx, device_t dev,
240     struct dmar_map_entries_tailq *rmrr_entries);
241 int dmar_instantiate_rmrr_ctxs(struct dmar_unit *dmar);
242 
243 void dmar_quirks_post_ident(struct dmar_unit *dmar);
244 void dmar_quirks_pre_use(struct dmar_unit *dmar);
245 
246 #define	DMAR_GM_CANWAIT	0x0001
247 #define	DMAR_GM_CANSPLIT 0x0002
248 
249 #define	DMAR_PGF_WAITOK	0x0001
250 #define	DMAR_PGF_ZERO	0x0002
251 #define	DMAR_PGF_ALLOC	0x0004
252 #define	DMAR_PGF_NOALLOC 0x0008
253 #define	DMAR_PGF_OBJL	0x0010
254 
255 extern dmar_haddr_t dmar_high;
256 extern int haw;
257 extern int dmar_tbl_pagecnt;
258 extern int dmar_match_verbose;
259 extern int dmar_check_free;
260 
261 static inline uint32_t
262 dmar_read4(const struct dmar_unit *unit, int reg)
263 {
264 
265 	return (bus_read_4(unit->regs, reg));
266 }
267 
268 static inline uint64_t
269 dmar_read8(const struct dmar_unit *unit, int reg)
270 {
271 #ifdef __i386__
272 	uint32_t high, low;
273 
274 	low = bus_read_4(unit->regs, reg);
275 	high = bus_read_4(unit->regs, reg + 4);
276 	return (low | ((uint64_t)high << 32));
277 #else
278 	return (bus_read_8(unit->regs, reg));
279 #endif
280 }
281 
282 static inline void
283 dmar_write4(const struct dmar_unit *unit, int reg, uint32_t val)
284 {
285 
286 	KASSERT(reg != DMAR_GCMD_REG || (val & DMAR_GCMD_TE) ==
287 	    (unit->hw_gcmd & DMAR_GCMD_TE),
288 	    ("dmar%d clearing TE 0x%08x 0x%08x", unit->unit,
289 	    unit->hw_gcmd, val));
290 	bus_write_4(unit->regs, reg, val);
291 }
292 
293 static inline void
294 dmar_write8(const struct dmar_unit *unit, int reg, uint64_t val)
295 {
296 
297 	KASSERT(reg != DMAR_GCMD_REG, ("8byte GCMD write"));
298 #ifdef __i386__
299 	uint32_t high, low;
300 
301 	low = val;
302 	high = val >> 32;
303 	bus_write_4(unit->regs, reg, low);
304 	bus_write_4(unit->regs, reg + 4, high);
305 #else
306 	bus_write_8(unit->regs, reg, val);
307 #endif
308 }
309 
310 /*
311  * dmar_pte_store and dmar_pte_clear ensure that on i386, 32bit writes
312  * are issued in the correct order.  For store, the lower word,
313  * containing the P or R and W bits, is set only after the high word
314  * is written.  For clear, the P bit is cleared first, then the high
315  * word is cleared.
316  */
317 static inline void
318 dmar_pte_store(volatile uint64_t *dst, uint64_t val)
319 {
320 
321 	KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx",
322 	    dst, (uintmax_t)*dst, (uintmax_t)val));
323 #ifdef __i386__
324 	volatile uint32_t *p;
325 	uint32_t hi, lo;
326 
327 	hi = val >> 32;
328 	lo = val;
329 	p = (volatile uint32_t *)dst;
330 	*(p + 1) = hi;
331 	*p = lo;
332 #else
333 	*dst = val;
334 #endif
335 }
336 
337 static inline void
338 dmar_pte_clear(volatile uint64_t *dst)
339 {
340 #ifdef __i386__
341 	volatile uint32_t *p;
342 
343 	p = (volatile uint32_t *)dst;
344 	*p = 0;
345 	*(p + 1) = 0;
346 #else
347 	*dst = 0;
348 #endif
349 }
350 
351 static inline bool
352 dmar_test_boundary(dmar_gaddr_t start, dmar_gaddr_t size,
353     dmar_gaddr_t boundary)
354 {
355 
356 	if (boundary == 0)
357 		return (true);
358 	return (start + size <= ((start + boundary) & ~(boundary - 1)));
359 }
360 
361 #ifdef INVARIANTS
362 #define	TD_PREP_PINNED_ASSERT						\
363 	int old_td_pinned;						\
364 	old_td_pinned = curthread->td_pinned
365 #define	TD_PINNED_ASSERT						\
366 	KASSERT(curthread->td_pinned == old_td_pinned,			\
367 	    ("pin count leak: %d %d %s:%d", curthread->td_pinned,	\
368 	    old_td_pinned, __FILE__, __LINE__))
369 #else
370 #define	TD_PREP_PINNED_ASSERT
371 #define	TD_PINNED_ASSERT
372 #endif
373 
374 #endif
375