62 amdiommu_enable_cmdbuf(struct amdiommu_unit *unit)  in amdiommu_enable_cmdbuf()  argument
64 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_enable_cmdbuf()
66 	unit->hw_ctrl |= AMDIOMMU_CTRL_CMDBUF_EN;  in amdiommu_enable_cmdbuf()
67 	amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);  in amdiommu_enable_cmdbuf()
71 amdiommu_disable_cmdbuf(struct amdiommu_unit *unit)  in amdiommu_disable_cmdbuf()  argument
73 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_disable_cmdbuf()
75 	unit->hw_ctrl &= ~AMDIOMMU_CTRL_CMDBUF_EN;  in amdiommu_disable_cmdbuf()
76 	amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);  in amdiommu_disable_cmdbuf()
83 	struct amdiommu_unit *unit;  in amdiommu_enable_qi_intr()  local
85 	unit = IOMMU2AMD(iommu);  in amdiommu_enable_qi_intr()
86 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_enable_qi_intr()
87 	unit->hw_ctrl |= AMDIOMMU_CTRL_COMWINT_EN;  in amdiommu_enable_qi_intr()
88 	amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);  in amdiommu_enable_qi_intr()
89 	amdiommu_write8(unit, AMDIOMMU_CMDEV_STATUS,  in amdiommu_enable_qi_intr()
96 	struct amdiommu_unit *unit;  in amdiommu_disable_qi_intr()  local
98 	unit = IOMMU2AMD(iommu);  in amdiommu_disable_qi_intr()
99 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_disable_qi_intr()
100 	unit->hw_ctrl &= ~AMDIOMMU_CTRL_COMWINT_EN;  in amdiommu_disable_qi_intr()
101 	amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);  in amdiommu_disable_qi_intr()
107 	struct amdiommu_unit *unit;  in amdiommu_cmd_advance_tail()  local
109 	unit = IOMMU2AMD(iommu);  in amdiommu_cmd_advance_tail()
110 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_cmd_advance_tail()
111 	amdiommu_write8(unit, AMDIOMMU_CMDBUF_TAIL, unit->x86c.inv_queue_tail);  in amdiommu_cmd_advance_tail()
117 	struct amdiommu_unit *unit;  in amdiommu_cmd_ensure()  local
121 	unit = IOMMU2AMD(iommu);  in amdiommu_cmd_ensure()
122 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_cmd_ensure()
125 		if (bytes <= unit->x86c.inv_queue_avail)  in amdiommu_cmd_ensure()
128 		head = amdiommu_read8(unit, AMDIOMMU_CMDBUF_HEAD);  in amdiommu_cmd_ensure()
130 		unit->x86c.inv_queue_avail = head - unit->x86c.inv_queue_tail -  in amdiommu_cmd_ensure()
132 		if (head <= unit->x86c.inv_queue_tail)  in amdiommu_cmd_ensure()
133 			unit->x86c.inv_queue_avail += unit->x86c.inv_queue_size;  in amdiommu_cmd_ensure()
134 		if (bytes <= unit->x86c.inv_queue_avail)  in amdiommu_cmd_ensure()
148 		unit->x86c.inv_queue_full++;  in amdiommu_cmd_ensure()
151 	unit->x86c.inv_queue_avail -= bytes;  in amdiommu_cmd_ensure()
155 amdiommu_cmd_emit(struct amdiommu_unit *unit, const struct  in amdiommu_cmd_emit()  argument
158 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_cmd_emit()
160 	memcpy(unit->x86c.inv_queue + unit->x86c.inv_queue_tail, cmd,  in amdiommu_cmd_emit()
162 	unit->x86c.inv_queue_tail += AMDIOMMU_CMD_SZ;  in amdiommu_cmd_emit()
163 	KASSERT(unit->x86c.inv_queue_tail <= unit->x86c.inv_queue_size,  in amdiommu_cmd_emit()
164 	    ("tail overflow 0x%x 0x%jx", unit->x86c.inv_queue_tail,  in amdiommu_cmd_emit()
165 	    (uintmax_t)unit->x86c.inv_queue_size));  in amdiommu_cmd_emit()
166 	unit->x86c.inv_queue_tail &= unit->x86c.inv_queue_size - 1;  in amdiommu_cmd_emit()
173 	struct amdiommu_unit *unit;  in amdiommu_cmd_emit_wait_descr()  local
176 	unit = IOMMU2AMD(iommu);  in amdiommu_cmd_emit_wait_descr()
177 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_cmd_emit_wait_descr()
185 		x = unit->x86c.inv_waitd_seq_hw_phys;  in amdiommu_cmd_emit_wait_descr()
188 		x = unit->x86c.inv_waitd_seq_hw_phys >> 32;  in amdiommu_cmd_emit_wait_descr()
196 	amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c);  in amdiommu_cmd_emit_wait_descr()
204 	struct amdiommu_unit *unit;  in amdiommu_qi_invalidate_emit()  local
209 	unit = domain->unit;  in amdiommu_qi_invalidate_emit()
210 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_qi_invalidate_emit()
217 		amdiommu_cmd_ensure(AMD2IOMMU(unit), 1);  in amdiommu_qi_invalidate_emit()
221 		amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c);  in amdiommu_qi_invalidate_emit()
223 	iommu_qi_emit_wait_seq(AMD2IOMMU(unit), pseq, emit_wait);  in amdiommu_qi_invalidate_emit()
229 	struct amdiommu_unit *unit;  in amdiommu_qi_invalidate_all_pages_locked_nowait()  local
232 	unit = domain->unit;  in amdiommu_qi_invalidate_all_pages_locked_nowait()
233 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_qi_invalidate_all_pages_locked_nowait()
246 	amdiommu_cmd_ensure(AMD2IOMMU(unit), 1);  in amdiommu_qi_invalidate_all_pages_locked_nowait()
247 	amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c);  in amdiommu_qi_invalidate_all_pages_locked_nowait()
283 amdiommu_qi_invalidate_ir_locked_nowait(struct amdiommu_unit *unit,  in amdiommu_qi_invalidate_ir_locked_nowait()  argument
288 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_qi_invalidate_ir_locked_nowait()
290 	amdiommu_cmd_ensure(AMD2IOMMU(unit), 1);  in amdiommu_qi_invalidate_ir_locked_nowait()
294 	amdiommu_cmd_emit(unit, (struct amdiommu_cmd_generic *)&c);  in amdiommu_qi_invalidate_ir_locked_nowait()
298 amdiommu_qi_invalidate_ir_locked(struct amdiommu_unit *unit, uint16_t devid)  in amdiommu_qi_invalidate_ir_locked()  argument
300 	amdiommu_qi_invalidate_ir_locked_nowait(unit, devid);  in amdiommu_qi_invalidate_ir_locked()
301 	amdiommu_qi_invalidate_wait_sync(AMD2IOMMU(unit));  in amdiommu_qi_invalidate_ir_locked()
307 	struct amdiommu_unit *unit;  in amdiommu_qi_task()  local
309 	unit = IOMMU2AMD(arg);  in amdiommu_qi_task()
310 	iommu_qi_drain_tlb_flush(AMD2IOMMU(unit));  in amdiommu_qi_task()
312 	AMDIOMMU_LOCK(unit);  in amdiommu_qi_task()
313 	if (unit->x86c.inv_seq_waiters > 0)  in amdiommu_qi_task()
314 		wakeup(&unit->x86c.inv_seq_waiters);  in amdiommu_qi_task()
315 	AMDIOMMU_UNLOCK(unit);  in amdiommu_qi_task()
319 amdiommu_init_cmd(struct amdiommu_unit *unit)  in amdiommu_init_cmd()  argument
323 	unit->x86c.qi_buf_maxsz = ilog2(AMDIOMMU_CMDBUF_MAX / PAGE_SIZE);  in amdiommu_init_cmd()
324 	unit->x86c.qi_cmd_sz = AMDIOMMU_CMD_SZ;  in amdiommu_init_cmd()
325 	iommu_qi_common_init(AMD2IOMMU(unit), amdiommu_qi_task);  in amdiommu_init_cmd()
331 	rv = pmap_kextract((uintptr_t)unit->x86c.inv_queue);  in amdiommu_init_cmd()
337 	qi_sz = ilog2(unit->x86c.inv_queue_size / PAGE_SIZE) + 8;  in amdiommu_init_cmd()
340 	AMDIOMMU_LOCK(unit);  in amdiommu_init_cmd()
341 	amdiommu_write8(unit, AMDIOMMU_CMDBUF_BASE, rv);  in amdiommu_init_cmd()
342 	amdiommu_enable_cmdbuf(unit);  in amdiommu_init_cmd()
343 	amdiommu_enable_qi_intr(AMD2IOMMU(unit));  in amdiommu_init_cmd()
344 	AMDIOMMU_UNLOCK(unit);  in amdiommu_init_cmd()
357 amdiommu_fini_cmd(struct amdiommu_unit *unit)  in amdiommu_fini_cmd()  argument
359 	iommu_qi_common_fini(AMD2IOMMU(unit), amdiommu_fini_cmd_helper);  in amdiommu_fini_cmd()