Lines Matching refs:unit
62 dmar_enable_qi(struct dmar_unit *unit) in dmar_enable_qi() argument
66 DMAR_ASSERT_LOCKED(unit); in dmar_enable_qi()
67 unit->hw_gcmd |= DMAR_GCMD_QIE; in dmar_enable_qi()
68 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); in dmar_enable_qi()
69 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) in dmar_enable_qi()
75 dmar_disable_qi(struct dmar_unit *unit) in dmar_disable_qi() argument
79 DMAR_ASSERT_LOCKED(unit); in dmar_disable_qi()
80 unit->hw_gcmd &= ~DMAR_GCMD_QIE; in dmar_disable_qi()
81 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); in dmar_disable_qi()
82 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) in dmar_disable_qi()
90 struct dmar_unit *unit; in dmar_qi_advance_tail() local
92 unit = IOMMU2DMAR(iommu); in dmar_qi_advance_tail()
93 DMAR_ASSERT_LOCKED(unit); in dmar_qi_advance_tail()
94 dmar_write4(unit, DMAR_IQT_REG, unit->x86c.inv_queue_tail); in dmar_qi_advance_tail()
100 struct dmar_unit *unit; in dmar_qi_ensure() local
104 unit = IOMMU2DMAR(iommu); in dmar_qi_ensure()
105 DMAR_ASSERT_LOCKED(unit); in dmar_qi_ensure()
108 if (bytes <= unit->x86c.inv_queue_avail) in dmar_qi_ensure()
111 head = dmar_read4(unit, DMAR_IQH_REG); in dmar_qi_ensure()
113 unit->x86c.inv_queue_avail = head - unit->x86c.inv_queue_tail - in dmar_qi_ensure()
115 if (head <= unit->x86c.inv_queue_tail) in dmar_qi_ensure()
116 unit->x86c.inv_queue_avail += unit->x86c.inv_queue_size; in dmar_qi_ensure()
117 if (bytes <= unit->x86c.inv_queue_avail) in dmar_qi_ensure()
130 dmar_qi_advance_tail(DMAR2IOMMU(unit)); in dmar_qi_ensure()
131 unit->x86c.inv_queue_full++; in dmar_qi_ensure()
134 unit->x86c.inv_queue_avail -= bytes; in dmar_qi_ensure()
138 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2) in dmar_qi_emit() argument
141 DMAR_ASSERT_LOCKED(unit); in dmar_qi_emit()
143 atomic_store_64((uint64_t *)(unit->x86c.inv_queue + in dmar_qi_emit()
144 unit->x86c.inv_queue_tail), data1); in dmar_qi_emit()
146 *(volatile uint64_t *)(unit->x86c.inv_queue + in dmar_qi_emit()
147 unit->x86c.inv_queue_tail) = data1; in dmar_qi_emit()
149 unit->x86c.inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; in dmar_qi_emit()
150 KASSERT(unit->x86c.inv_queue_tail <= unit->x86c.inv_queue_size, in dmar_qi_emit()
151 ("tail overflow 0x%x 0x%jx", unit->x86c.inv_queue_tail, in dmar_qi_emit()
152 (uintmax_t)unit->x86c.inv_queue_size)); in dmar_qi_emit()
153 unit->x86c.inv_queue_tail &= unit->x86c.inv_queue_size - 1; in dmar_qi_emit()
155 atomic_store_64((uint64_t *)(unit->x86c.inv_queue + in dmar_qi_emit()
156 unit->x86c.inv_queue_tail), data2); in dmar_qi_emit()
158 *(volatile uint64_t *)(unit->x86c.inv_queue + in dmar_qi_emit()
159 unit->x86c.inv_queue_tail) = data2; in dmar_qi_emit()
161 unit->x86c.inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; in dmar_qi_emit()
162 KASSERT(unit->x86c.inv_queue_tail <= unit->x86c.inv_queue_size, in dmar_qi_emit()
163 ("tail overflow 0x%x 0x%jx", unit->x86c.inv_queue_tail, in dmar_qi_emit()
164 (uintmax_t)unit->x86c.inv_queue_size)); in dmar_qi_emit()
165 unit->x86c.inv_queue_tail &= unit->x86c.inv_queue_size - 1; in dmar_qi_emit()
172 struct dmar_unit *unit; in dmar_qi_emit_wait_descr() local
174 unit = IOMMU2DMAR(iommu); in dmar_qi_emit_wait_descr()
175 DMAR_ASSERT_LOCKED(unit); in dmar_qi_emit_wait_descr()
176 dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID | in dmar_qi_emit_wait_descr()
181 memw ? unit->x86c.inv_waitd_seq_hw_phys : 0); in dmar_qi_emit_wait_descr()
188 struct dmar_unit *unit; in dmar_qi_invalidate_emit() local
194 unit = domain->dmar; in dmar_qi_invalidate_emit()
195 DMAR_ASSERT_LOCKED(unit); in dmar_qi_invalidate_emit()
197 am = calc_am(unit, base, size, &isize); in dmar_qi_invalidate_emit()
198 dmar_qi_ensure(DMAR2IOMMU(unit), 1); in dmar_qi_invalidate_emit()
199 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | in dmar_qi_invalidate_emit()
205 iommu_qi_emit_wait_seq(DMAR2IOMMU(unit), pseq, emit_wait); in dmar_qi_invalidate_emit()
209 dmar_qi_invalidate_glob_impl(struct dmar_unit *unit, uint64_t data1) in dmar_qi_invalidate_glob_impl() argument
213 DMAR_ASSERT_LOCKED(unit); in dmar_qi_invalidate_glob_impl()
214 dmar_qi_ensure(DMAR2IOMMU(unit), 2); in dmar_qi_invalidate_glob_impl()
215 dmar_qi_emit(unit, data1, 0); in dmar_qi_invalidate_glob_impl()
216 iommu_qi_emit_wait_seq(DMAR2IOMMU(unit), &gseq, true); in dmar_qi_invalidate_glob_impl()
218 unit->x86c.inv_seq_waiters++; in dmar_qi_invalidate_glob_impl()
219 dmar_qi_advance_tail(DMAR2IOMMU(unit)); in dmar_qi_invalidate_glob_impl()
220 iommu_qi_wait_for_seq(DMAR2IOMMU(unit), &gseq, false); in dmar_qi_invalidate_glob_impl()
224 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) in dmar_qi_invalidate_ctx_glob_locked() argument
226 dmar_qi_invalidate_glob_impl(unit, DMAR_IQ_DESCR_CTX_INV | in dmar_qi_invalidate_ctx_glob_locked()
231 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) in dmar_qi_invalidate_iotlb_glob_locked() argument
233 dmar_qi_invalidate_glob_impl(unit, DMAR_IQ_DESCR_IOTLB_INV | in dmar_qi_invalidate_iotlb_glob_locked()
239 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) in dmar_qi_invalidate_iec_glob() argument
241 dmar_qi_invalidate_glob_impl(unit, DMAR_IQ_DESCR_IEC_INV); in dmar_qi_invalidate_iec_glob()
245 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) in dmar_qi_invalidate_iec() argument
250 DMAR_ASSERT_LOCKED(unit); in dmar_qi_invalidate_iec()
251 KASSERT(start < unit->irte_cnt && start < start + cnt && in dmar_qi_invalidate_iec()
252 start + cnt <= unit->irte_cnt, in dmar_qi_invalidate_iec()
253 ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); in dmar_qi_invalidate_iec()
257 dmar_qi_ensure(DMAR2IOMMU(unit), 1); in dmar_qi_invalidate_iec()
258 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV | in dmar_qi_invalidate_iec()
262 dmar_qi_ensure(DMAR2IOMMU(unit), 1); in dmar_qi_invalidate_iec()
263 iommu_qi_emit_wait_seq(DMAR2IOMMU(unit), &gseq, true); in dmar_qi_invalidate_iec()
269 unit->x86c.inv_seq_waiters++; in dmar_qi_invalidate_iec()
271 dmar_qi_advance_tail(DMAR2IOMMU(unit)); in dmar_qi_invalidate_iec()
288 iommu_qi_wait_for_seq(DMAR2IOMMU(unit), &gseq, true); in dmar_qi_invalidate_iec()
294 struct dmar_unit *unit; in dmar_qi_intr() local
296 unit = IOMMU2DMAR((struct iommu_unit *)arg); in dmar_qi_intr()
297 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", in dmar_qi_intr()
298 unit->iommu.unit)); in dmar_qi_intr()
299 taskqueue_enqueue(unit->x86c.qi_taskqueue, &unit->x86c.qi_task); in dmar_qi_intr()
306 struct dmar_unit *unit; in dmar_qi_task() local
309 unit = IOMMU2DMAR(arg); in dmar_qi_task()
310 iommu_qi_drain_tlb_flush(DMAR2IOMMU(unit)); in dmar_qi_task()
316 ics = dmar_read4(unit, DMAR_ICS_REG); in dmar_qi_task()
319 dmar_write4(unit, DMAR_ICS_REG, ics); in dmar_qi_task()
327 iommu_qi_drain_tlb_flush(DMAR2IOMMU(unit)); in dmar_qi_task()
330 if (unit->x86c.inv_seq_waiters > 0) { in dmar_qi_task()
335 DMAR_LOCK(unit); in dmar_qi_task()
336 wakeup(&unit->x86c.inv_seq_waiters); in dmar_qi_task()
337 DMAR_UNLOCK(unit); in dmar_qi_task()
342 dmar_init_qi(struct dmar_unit *unit) in dmar_init_qi() argument
348 if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0) in dmar_init_qi()
350 unit->qi_enabled = 1; in dmar_init_qi()
351 TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled); in dmar_init_qi()
352 if (!unit->qi_enabled) in dmar_init_qi()
355 unit->x86c.qi_buf_maxsz = DMAR_IQA_QS_MAX; in dmar_init_qi()
356 unit->x86c.qi_cmd_sz = DMAR_IQ_DESCR_SZ; in dmar_init_qi()
357 iommu_qi_common_init(DMAR2IOMMU(unit), dmar_qi_task); in dmar_init_qi()
363 qi_sz = ilog2(unit->x86c.inv_queue_size / PAGE_SIZE); in dmar_init_qi()
365 DMAR_LOCK(unit); in dmar_init_qi()
366 dmar_write8(unit, DMAR_IQT_REG, 0); in dmar_init_qi()
367 iqa = pmap_kextract((uintptr_t)unit->x86c.inv_queue); in dmar_init_qi()
369 dmar_write8(unit, DMAR_IQA_REG, iqa); in dmar_init_qi()
370 dmar_enable_qi(unit); in dmar_init_qi()
371 ics = dmar_read4(unit, DMAR_ICS_REG); in dmar_init_qi()
374 dmar_write4(unit, DMAR_ICS_REG, ics); in dmar_init_qi()
376 dmar_enable_qi_intr(DMAR2IOMMU(unit)); in dmar_init_qi()
377 DMAR_UNLOCK(unit); in dmar_init_qi()
390 dmar_fini_qi(struct dmar_unit *unit) in dmar_fini_qi() argument
392 if (!unit->qi_enabled) in dmar_fini_qi()
394 iommu_qi_common_fini(DMAR2IOMMU(unit), dmar_fini_qi_helper); in dmar_fini_qi()
395 unit->qi_enabled = 0; in dmar_fini_qi()
401 struct dmar_unit *unit; in dmar_enable_qi_intr() local
404 unit = IOMMU2DMAR(iommu); in dmar_enable_qi_intr()
405 DMAR_ASSERT_LOCKED(unit); in dmar_enable_qi_intr()
406 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", in dmar_enable_qi_intr()
407 unit->iommu.unit)); in dmar_enable_qi_intr()
408 iectl = dmar_read4(unit, DMAR_IECTL_REG); in dmar_enable_qi_intr()
410 dmar_write4(unit, DMAR_IECTL_REG, iectl); in dmar_enable_qi_intr()
416 struct dmar_unit *unit; in dmar_disable_qi_intr() local
419 unit = IOMMU2DMAR(iommu); in dmar_disable_qi_intr()
420 DMAR_ASSERT_LOCKED(unit); in dmar_disable_qi_intr()
421 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", in dmar_disable_qi_intr()
422 unit->iommu.unit)); in dmar_disable_qi_intr()
423 iectl = dmar_read4(unit, DMAR_IECTL_REG); in dmar_disable_qi_intr()
424 dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM); in dmar_disable_qi_intr()