343 	struct x86_unit_common *x86c;  in iommu_qi_seq_processed()  local
346 	x86c = IOMMU2X86C(unit);  in iommu_qi_seq_processed()
347 	gen = x86c->inv_waitd_gen;  in iommu_qi_seq_processed()
349 	    atomic_load_64(&x86c->inv_waitd_seq_hw)));  in iommu_qi_seq_processed()
356 	struct x86_unit_common *x86c;  in iommu_qi_emit_wait_seq()  local
362 	x86c = IOMMU2X86C(unit);  in iommu_qi_emit_wait_seq()
364 	if (x86c->inv_waitd_seq == 0xffffffff) {  in iommu_qi_emit_wait_seq()
365 		gsec.gen = x86c->inv_waitd_gen;  in iommu_qi_emit_wait_seq()
366 		gsec.seq = x86c->inv_waitd_seq;  in iommu_qi_emit_wait_seq()
373 		x86c->inv_waitd_gen++;  in iommu_qi_emit_wait_seq()
374 		x86c->inv_waitd_seq = 1;  in iommu_qi_emit_wait_seq()
376 	seq = x86c->inv_waitd_seq++;  in iommu_qi_emit_wait_seq()
377 	pseq->gen = x86c->inv_waitd_gen;  in iommu_qi_emit_wait_seq()
393 	struct x86_unit_common *x86c;  in iommu_qi_wait_for_seq()  local
396 	x86c = IOMMU2X86C(unit);  in iommu_qi_wait_for_seq()
398 	KASSERT(x86c->inv_seq_waiters > 0, ("%s: no waiters", __func__));  in iommu_qi_wait_for_seq()
403 			msleep(&x86c->inv_seq_waiters, &unit->lock, 0,  in iommu_qi_wait_for_seq()
407 	x86c->inv_seq_waiters--;  in iommu_qi_wait_for_seq()
418 	struct x86_unit_common *x86c;  in iommu_qi_invalidate_locked()  local
421 	x86c = IOMMU2X86C(unit);  in iommu_qi_invalidate_locked()
441 	atomic_store_rel_ptr((uintptr_t *)&x86c->tlb_flush_tail->  in iommu_qi_invalidate_locked()
443 	x86c->tlb_flush_tail = entry;  in iommu_qi_invalidate_locked()
474 	struct x86_unit_common *x86c;  in iommu_qi_drain_tlb_flush()  local
477 	x86c = IOMMU2X86C(unit);  in iommu_qi_drain_tlb_flush()
478 	for (head = x86c->tlb_flush_head;; head = entry) {  in iommu_qi_drain_tlb_flush()
484 		x86c->tlb_flush_head = entry;  in iommu_qi_drain_tlb_flush()
496 	struct x86_unit_common *x86c;  in iommu_qi_common_init()  local
499 	x86c = IOMMU2X86C(unit);  in iommu_qi_common_init()
501 	x86c->tlb_flush_head = x86c->tlb_flush_tail =  in iommu_qi_common_init()
503 	TASK_INIT(&x86c->qi_task, 0, qi_task, unit);  in iommu_qi_common_init()
504 	x86c->qi_taskqueue = taskqueue_create_fast("iommuqf", M_WAITOK,  in iommu_qi_common_init()
505 	    taskqueue_thread_enqueue, &x86c->qi_taskqueue);  in iommu_qi_common_init()
506 	taskqueue_start_threads(&x86c->qi_taskqueue, 1, PI_AV,  in iommu_qi_common_init()
509 	x86c->inv_waitd_gen = 0;  in iommu_qi_common_init()
510 	x86c->inv_waitd_seq = 1;  in iommu_qi_common_init()
514 	if (qi_sz > x86c->qi_buf_maxsz)  in iommu_qi_common_init()
515 		qi_sz = x86c->qi_buf_maxsz;  in iommu_qi_common_init()
516 	x86c->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;  in iommu_qi_common_init()
518 	x86c->inv_queue_avail = x86c->inv_queue_size -  in iommu_qi_common_init()
519 	    x86c->qi_cmd_sz;  in iommu_qi_common_init()
525 	x86c->inv_queue = kmem_alloc_contig(x86c->inv_queue_size,  in iommu_qi_common_init()
528 	x86c->inv_waitd_seq_hw_phys = pmap_kextract(  in iommu_qi_common_init()
529 	    (vm_offset_t)&x86c->inv_waitd_seq_hw);  in iommu_qi_common_init()
536 	struct x86_unit_common *x86c;  in iommu_qi_common_fini()  local
539 	x86c = IOMMU2X86C(unit);  in iommu_qi_common_fini()
541 	taskqueue_drain(x86c->qi_taskqueue, &x86c->qi_task);  in iommu_qi_common_fini()
542 	taskqueue_free(x86c->qi_taskqueue);  in iommu_qi_common_fini()
543 	x86c->qi_taskqueue = NULL;  in iommu_qi_common_fini()
550 	x86c->inv_seq_waiters++;  in iommu_qi_common_fini()
555 	KASSERT(x86c->inv_seq_waiters == 0,  in iommu_qi_common_fini()
559 	kmem_free(x86c->inv_queue, x86c->inv_queue_size);  in iommu_qi_common_fini()
560 	x86c->inv_queue = NULL;  in iommu_qi_common_fini()
561 	x86c->inv_queue_size = 0;  in iommu_qi_common_fini()