73 amdiommu_event_rearm_intr(struct amdiommu_unit *unit)  in amdiommu_event_rearm_intr()  argument
75 	amdiommu_write8(unit, AMDIOMMU_CMDEV_STATUS,  in amdiommu_event_rearm_intr()
80 amdiommu_event_log_inc_head(struct amdiommu_unit *unit)  in amdiommu_event_log_inc_head()  argument
82 	unit->event_log_head++;  in amdiommu_event_log_inc_head()
83 	if (unit->event_log_head >= unit->event_log_size)  in amdiommu_event_log_inc_head()
84 		unit->event_log_head = 0;  in amdiommu_event_log_inc_head()
88 amdiommu_event_log_print(struct amdiommu_unit *unit,  in amdiommu_event_log_print()  argument
92 	    unit->iommu.unit, evp->code, evp->w0, evp->ww1, evp->w2, evp->w3);  in amdiommu_event_log_print()
96 	AMDIOMMU_ASSERT_LOCKED(unit);  in amdiommu_event_log_print()
105 		dte = &unit->dev_tbl[ev_dte_p->devid];  in amdiommu_event_log_print()
121 		ctx = amdiommu_find_ctx_locked(unit, ev_iopf_p->devid);  in amdiommu_event_log_print()
139 amdiommu_event_log_tail(struct amdiommu_unit *unit)  in amdiommu_event_log_tail()  argument
141 	return (amdiommu_read8(unit, AMDIOMMU_EVNTLOG_TAIL) >>  in amdiommu_event_log_tail()
155 amdiommu_event_copy_log_hasspace(struct amdiommu_unit *unit)  in amdiommu_event_copy_log_hasspace()  argument
157 	return (unit->event_copy_tail != amdiommu_event_copy_log_inc(  in amdiommu_event_copy_log_hasspace()
158 	    unit->event_copy_head));  in amdiommu_event_copy_log_hasspace()
162 amdiommu_event_intr(struct amdiommu_unit *unit, uint64_t status)  in amdiommu_event_intr()  argument
170 	hw_tail1 = amdiommu_event_log_tail(unit);  in amdiommu_event_intr()
173 		for (; hw_tail != unit->event_log_head;  in amdiommu_event_intr()
174 		     amdiommu_event_log_inc_head(unit)) {  in amdiommu_event_intr()
175 			evp = &unit->event_log[unit->event_log_head];  in amdiommu_event_intr()
176 			mtx_lock_spin(&unit->event_lock);  in amdiommu_event_intr()
177 			if (amdiommu_event_copy_log_hasspace(unit)) {  in amdiommu_event_intr()
178 				unit->event_copy_log[unit->event_copy_head] =  in amdiommu_event_intr()
180 				unit->event_copy_head =  in amdiommu_event_intr()
181 				    amdiommu_event_copy_log_inc(unit->  in amdiommu_event_intr()
185 				amdiommu_event_log_print(unit, evp, false);  in amdiommu_event_intr()
187 			mtx_unlock_spin(&unit->event_lock);  in amdiommu_event_intr()
189 		amdiommu_write8(unit, AMDIOMMU_EVNTLOG_HEAD,  in amdiommu_event_intr()
190 		    unit->event_log_head << AMDIOMMU_EV_SZ_SHIFT);  in amdiommu_event_intr()
191 		hw_tail1 = amdiommu_event_log_tail(unit);  in amdiommu_event_intr()
193 	amdiommu_event_rearm_intr(unit);  in amdiommu_event_intr()
196 		taskqueue_enqueue(unit->event_taskqueue, &unit->event_task);  in amdiommu_event_intr()
202 	struct amdiommu_unit *unit;  in amdiommu_event_task()  local
206 	unit = arg;  in amdiommu_event_task()
207 	AMDIOMMU_LOCK(unit);  in amdiommu_event_task()
209 	if ((unit->efr & AMDIOMMU_EFR_HWEV_SUP) != 0) {  in amdiommu_event_task()
210 		hwev_status = amdiommu_read8(unit, AMDIOMMU_HWEV_STATUS);  in amdiommu_event_task()
212 			*(uint64_t *)&hwev = amdiommu_read8(unit,  in amdiommu_event_task()
214 			*((uint64_t *)&hwev + 1) = amdiommu_read8(unit,  in amdiommu_event_task()
216 			printf("amdiommu%d: hw event%s\n", unit->iommu.unit,  in amdiommu_event_task()
219 			amdiommu_event_log_print(unit, &hwev, true);  in amdiommu_event_task()
220 			amdiommu_write8(unit, AMDIOMMU_HWEV_STATUS,  in amdiommu_event_task()
225 	status = amdiommu_read8(unit, AMDIOMMU_CMDEV_STATUS);  in amdiommu_event_task()
227 		printf("amdiommu%d: event log overflow\n", unit->iommu.unit);  in amdiommu_event_task()
231 			status = amdiommu_read8(unit, AMDIOMMU_CMDEV_STATUS);  in amdiommu_event_task()
234 		unit->hw_ctrl &= ~AMDIOMMU_CTRL_EVNTLOG_EN;  in amdiommu_event_task()
235 		amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);  in amdiommu_event_task()
237 		unit->event_log_head = 0;  in amdiommu_event_task()
238 		amdiommu_write8(unit, AMDIOMMU_EVNTLOG_HEAD, 0);  in amdiommu_event_task()
240 		amdiommu_write8(unit, AMDIOMMU_CMDEV_STATUS,  in amdiommu_event_task()
243 		unit->hw_ctrl |= AMDIOMMU_CTRL_EVNTLOG_EN;  in amdiommu_event_task()
244 		amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);  in amdiommu_event_task()
246 		amdiommu_event_rearm_intr(unit);  in amdiommu_event_task()
249 	mtx_lock_spin(&unit->event_lock);  in amdiommu_event_task()
250 	while (unit->event_copy_head != unit->event_copy_tail) {  in amdiommu_event_task()
251 		mtx_unlock_spin(&unit->event_lock);  in amdiommu_event_task()
252 		amdiommu_event_log_print(unit, &unit->event_copy_log[  in amdiommu_event_task()
253 		    unit->event_copy_tail], true);  in amdiommu_event_task()
254 		mtx_lock_spin(&unit->event_lock);  in amdiommu_event_task()
255 		unit->event_copy_tail = amdiommu_event_copy_log_inc(unit->  in amdiommu_event_task()
258 	mtx_unlock_spin(&unit->event_lock);  in amdiommu_event_task()
260 	AMDIOMMU_UNLOCK(unit);  in amdiommu_event_task()
264 amdiommu_init_event(struct amdiommu_unit *unit)  in amdiommu_init_event()  argument
268 	mtx_init(&unit->event_lock, "amdevl", NULL, MTX_SPIN);  in amdiommu_init_event()
271 	unit->event_log_size = AMDIOMMU_EVNTLOG_MIN;  in amdiommu_init_event()
272 	TUNABLE_INT_FETCH("hw.amdiommu.event_log_size", &unit->event_log_size);  in amdiommu_init_event()
273 	if (unit->event_log_size < AMDIOMMU_EVNTLOG_MIN ||  in amdiommu_init_event()
274 	    unit->event_log_size > AMDIOMMU_EVNTLOG_MAX ||  in amdiommu_init_event()
275 	    !powerof2(unit->event_log_size))  in amdiommu_init_event()
277 	unit->event_log = kmem_alloc_contig(AMDIOMMU_EV_SZ *  in amdiommu_init_event()
278 	    unit->event_log_size, M_WAITOK | M_ZERO, 0, ~0ull, PAGE_SIZE,  in amdiommu_init_event()
281 	TASK_INIT(&unit->event_task, 0, amdiommu_event_task, unit);  in amdiommu_init_event()
282 	unit->event_taskqueue = taskqueue_create_fast("amdiommuff", M_WAITOK,  in amdiommu_init_event()
283 	    taskqueue_thread_enqueue, &unit->event_taskqueue);  in amdiommu_init_event()
284 	taskqueue_start_threads(&unit->event_taskqueue, 1, PI_AV,  in amdiommu_init_event()
285 	    "amdiommu%d event taskq", unit->iommu.unit);  in amdiommu_init_event()
287 	base_reg = pmap_kextract((vm_offset_t)unit->event_log) |  in amdiommu_init_event()
288 	    (((uint64_t)0x8 + ilog2(unit->event_log_size /  in amdiommu_init_event()
290 	AMDIOMMU_LOCK(unit);  in amdiommu_init_event()
295 	amdiommu_event_rearm_intr(unit);  in amdiommu_init_event()
296 	amdiommu_write8(unit, AMDIOMMU_EVNTLOG_BASE, base_reg);  in amdiommu_init_event()
297 	unit->hw_ctrl |= AMDIOMMU_CTRL_EVNTLOG_EN | AMDIOMMU_CTRL_EVENTINT_EN;  in amdiommu_init_event()
298 	amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);  in amdiommu_init_event()
299 	AMDIOMMU_UNLOCK(unit);  in amdiommu_init_event()
305 amdiommu_fini_event(struct amdiommu_unit *unit)  in amdiommu_fini_event()  argument
307 	AMDIOMMU_LOCK(unit);  in amdiommu_fini_event()
308 	unit->hw_ctrl &= ~(AMDIOMMU_CTRL_EVNTLOG_EN |  in amdiommu_fini_event()
310 	amdiommu_write8(unit, AMDIOMMU_CTRL, unit->hw_ctrl);  in amdiommu_fini_event()
311 	amdiommu_write8(unit, AMDIOMMU_EVNTLOG_BASE, 0);  in amdiommu_fini_event()
312 	AMDIOMMU_UNLOCK(unit);  in amdiommu_fini_event()
314 	taskqueue_drain(unit->event_taskqueue, &unit->event_task);  in amdiommu_fini_event()
315 	taskqueue_free(unit->event_taskqueue);  in amdiommu_fini_event()
316 	unit->event_taskqueue = NULL;  in amdiommu_fini_event()
318 	kmem_free(unit->event_log, unit->event_log_size * AMDIOMMU_EV_SZ);  in amdiommu_fini_event()
319 	unit->event_log = NULL;  in amdiommu_fini_event()
320 	unit->event_log_head = unit->event_log_tail = 0;  in amdiommu_fini_event()
322 	mtx_destroy(&unit->event_lock);  in amdiommu_fini_event()