Lines Matching +full:iommu +full:- +full:ctx
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
66 #include <dev/iommu/busdma_iommu.h>
67 #include <dev/iommu/iommu.h>
68 #include <x86/iommu/x86_iommu.h>
69 #include <x86/iommu/iommu_intrmap.h>
101 if (zeroed && (m->flags & PG_ZERO) == 0) in iommu_pgalloc()
127 SLIST_INSERT_HEAD(&entry->pgtbl_free, m, plinks.s.ss); in iommu_pgfree()
164 iommu_pgfree(obj, m->pindex, flags | IOMMU_PGF_OBJL, in iommu_map_pgtbl()
194 "Count of pages used for IOMMU pagetables");
268 x86_iommu->domain_unload_entry(entry, free, cansleep); in iommu_domain_unload_entry()
275 x86_iommu->domain_unload(iodom, entries, cansleep); in iommu_domain_unload()
279 iommu_get_ctx(struct iommu_unit *iommu, device_t dev, uint16_t rid, in iommu_get_ctx() argument
282 return (x86_iommu->get_ctx(iommu, dev, rid, id_mapped, rmrr_init)); in iommu_get_ctx()
286 iommu_free_ctx_locked(struct iommu_unit *iommu, struct iommu_ctx *context) in iommu_free_ctx_locked() argument
288 x86_iommu->free_ctx_locked(iommu, context); in iommu_free_ctx_locked()
294 return (x86_iommu->find(dev, verbose)); in iommu_find()
300 return (x86_iommu->alloc_msi_intr(src, cookies, count)); in iommu_alloc_msi_intr()
307 return (x86_iommu->map_msi_intr(src, cpu, vector, cookie, in iommu_map_msi_intr()
314 return (x86_iommu->unmap_msi_intr(src, cookie)); in iommu_unmap_msi_intr()
321 return (x86_iommu->map_ioapic_intr(ioapic_id, cpu, vector, edge, in iommu_map_ioapic_intr()
328 return (x86_iommu->unmap_ioapic_intr(ioapic_id, cookie)); in iommu_unmap_ioapic_intr()
334 x86_iommu->unit_pre_instantiate_ctx(unit); in iommu_unit_pre_instantiate_ctx()
337 #define IOMMU2X86C(iommu) (x86_iommu->get_x86_common(iommu)) argument
347 gen = x86c->inv_waitd_gen; in iommu_qi_seq_processed()
348 return (pseq->gen < gen || (pseq->gen == gen && pseq->seq <= in iommu_qi_seq_processed()
349 atomic_load_64(&x86c->inv_waitd_seq_hw))); in iommu_qi_seq_processed()
364 if (x86c->inv_waitd_seq == 0xffffffff) { in iommu_qi_emit_wait_seq()
365 gsec.gen = x86c->inv_waitd_gen; in iommu_qi_emit_wait_seq()
366 gsec.seq = x86c->inv_waitd_seq; in iommu_qi_emit_wait_seq()
367 x86_iommu->qi_ensure(unit, 1); in iommu_qi_emit_wait_seq()
368 x86_iommu->qi_emit_wait_descr(unit, gsec.seq, false, in iommu_qi_emit_wait_seq()
370 x86_iommu->qi_advance_tail(unit); in iommu_qi_emit_wait_seq()
373 x86c->inv_waitd_gen++; in iommu_qi_emit_wait_seq()
374 x86c->inv_waitd_seq = 1; in iommu_qi_emit_wait_seq()
376 seq = x86c->inv_waitd_seq++; in iommu_qi_emit_wait_seq()
377 pseq->gen = x86c->inv_waitd_gen; in iommu_qi_emit_wait_seq()
378 pseq->seq = seq; in iommu_qi_emit_wait_seq()
380 x86_iommu->qi_ensure(unit, 1); in iommu_qi_emit_wait_seq()
381 x86_iommu->qi_emit_wait_descr(unit, seq, true, true, false); in iommu_qi_emit_wait_seq()
398 KASSERT(x86c->inv_seq_waiters > 0, ("%s: no waiters", __func__)); in iommu_qi_wait_for_seq()
403 msleep(&x86c->inv_seq_waiters, &unit->lock, 0, in iommu_qi_wait_for_seq()
407 x86c->inv_seq_waiters--; in iommu_qi_wait_for_seq()
420 unit = domain->iommu; in iommu_qi_invalidate_locked()
424 x86_iommu->qi_invalidate_emit(domain, entry->start, entry->end - in iommu_qi_invalidate_locked()
425 entry->start, &entry->gseq, emit_wait); in iommu_qi_invalidate_locked()
440 entry->tlb_flush_next = NULL; in iommu_qi_invalidate_locked()
441 atomic_store_rel_ptr((uintptr_t *)&x86c->tlb_flush_tail-> in iommu_qi_invalidate_locked()
443 x86c->tlb_flush_tail = entry; in iommu_qi_invalidate_locked()
445 x86_iommu->qi_advance_tail(unit); in iommu_qi_invalidate_locked()
455 unit = domain->iommu; in iommu_qi_invalidate_sync()
457 x86_iommu->qi_invalidate_emit(domain, base, size, &gseq, true); in iommu_qi_invalidate_sync()
464 IOMMU2X86C(unit)->inv_seq_waiters++; in iommu_qi_invalidate_sync()
466 x86_iommu->qi_advance_tail(unit); in iommu_qi_invalidate_sync()
478 for (head = x86c->tlb_flush_head;; head = entry) { in iommu_qi_drain_tlb_flush()
480 atomic_load_acq_ptr((uintptr_t *)&head->tlb_flush_next); in iommu_qi_drain_tlb_flush()
482 !iommu_qi_seq_processed(unit, &entry->gseq)) in iommu_qi_drain_tlb_flush()
484 x86c->tlb_flush_head = entry; in iommu_qi_drain_tlb_flush()
486 if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) in iommu_qi_drain_tlb_flush()
501 x86c->tlb_flush_head = x86c->tlb_flush_tail = in iommu_qi_common_init()
503 TASK_INIT(&x86c->qi_task, 0, qi_task, unit); in iommu_qi_common_init()
504 x86c->qi_taskqueue = taskqueue_create_fast("iommuqf", M_WAITOK, in iommu_qi_common_init()
505 taskqueue_thread_enqueue, &x86c->qi_taskqueue); in iommu_qi_common_init()
506 taskqueue_start_threads(&x86c->qi_taskqueue, 1, PI_AV, in iommu_qi_common_init()
507 "iommu%d qi taskq", unit->unit); in iommu_qi_common_init()
509 x86c->inv_waitd_gen = 0; in iommu_qi_common_init()
510 x86c->inv_waitd_seq = 1; in iommu_qi_common_init()
513 TUNABLE_INT_FETCH("hw.iommu.qi_size", &qi_sz); in iommu_qi_common_init()
514 if (qi_sz > x86c->qi_buf_maxsz) in iommu_qi_common_init()
515 qi_sz = x86c->qi_buf_maxsz; in iommu_qi_common_init()
516 x86c->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE; in iommu_qi_common_init()
518 x86c->inv_queue_avail = x86c->inv_queue_size - in iommu_qi_common_init()
519 x86c->qi_cmd_sz; in iommu_qi_common_init()
525 x86c->inv_queue = kmem_alloc_contig(x86c->inv_queue_size, in iommu_qi_common_init()
528 x86c->inv_waitd_seq_hw_phys = pmap_kextract( in iommu_qi_common_init()
529 (vm_offset_t)&x86c->inv_waitd_seq_hw); in iommu_qi_common_init()
541 taskqueue_drain(x86c->qi_taskqueue, &x86c->qi_task); in iommu_qi_common_fini()
542 taskqueue_free(x86c->qi_taskqueue); in iommu_qi_common_fini()
543 x86c->qi_taskqueue = NULL; in iommu_qi_common_fini()
547 x86_iommu->qi_ensure(unit, 1); in iommu_qi_common_fini()
550 x86c->inv_seq_waiters++; in iommu_qi_common_fini()
551 x86_iommu->qi_advance_tail(unit); in iommu_qi_common_fini()
555 KASSERT(x86c->inv_seq_waiters == 0, in iommu_qi_common_fini()
556 ("iommu%d: waiters on disabled queue", unit->unit)); in iommu_qi_common_fini()
559 kmem_free(x86c->inv_queue, x86c->inv_queue_size); in iommu_qi_common_fini()
560 x86c->inv_queue = NULL; in iommu_qi_common_fini()
561 x86c->inv_queue_size = 0; in iommu_qi_common_fini()
575 dev = unit->dev; in iommu_alloc_irq()
576 dmd = &IOMMU2X86C(unit)->intrs[idx]; in iommu_alloc_irq()
578 error = PCIB_ALLOC_MSIX(pcib, dev, &dmd->irq); in iommu_alloc_irq()
581 dmd->name, error); in iommu_alloc_irq()
584 error = bus_set_resource(dev, SYS_RES_IRQ, dmd->irq_rid, in iommu_alloc_irq()
585 dmd->irq, 1); in iommu_alloc_irq()
588 dmd->name, error); in iommu_alloc_irq()
591 dmd->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, in iommu_alloc_irq()
592 &dmd->irq_rid, RF_ACTIVE); in iommu_alloc_irq()
593 if (dmd->irq_res == NULL) { in iommu_alloc_irq()
595 "cannot allocate resource for %s interrupt\n", dmd->name); in iommu_alloc_irq()
599 error = bus_setup_intr(dev, dmd->irq_res, INTR_TYPE_MISC, in iommu_alloc_irq()
600 dmd->handler, NULL, unit, &dmd->intr_handle); in iommu_alloc_irq()
603 dmd->name, error); in iommu_alloc_irq()
606 bus_describe_intr(dev, dmd->irq_res, dmd->intr_handle, "%s", dmd->name); in iommu_alloc_irq()
607 error = PCIB_MAP_MSI(pcib, dev, dmd->irq, &msi_addr, &msi_data); in iommu_alloc_irq()
610 dmd->name, error); in iommu_alloc_irq()
614 dmd->msi_data = msi_data; in iommu_alloc_irq()
615 dmd->msi_addr = msi_addr; in iommu_alloc_irq()
620 bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle); in iommu_alloc_irq()
622 bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res); in iommu_alloc_irq()
624 bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid); in iommu_alloc_irq()
626 PCIB_RELEASE_MSIX(pcib, dev, dmd->irq); in iommu_alloc_irq()
627 dmd->irq = -1; in iommu_alloc_irq()
640 dmd = &IOMMU2X86C(unit)->intrs[idx]; in iommu_release_intr()
641 if (dmd->handler == NULL || dmd->irq == -1) in iommu_release_intr()
643 dev = unit->dev; in iommu_release_intr()
645 bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle); in iommu_release_intr()
646 bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res); in iommu_release_intr()
647 bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid); in iommu_release_intr()
649 dev, dmd->irq); in iommu_release_intr()
650 dmd->irq = -1; in iommu_release_intr()
654 iommu_device_tag_init(struct iommu_ctx *ctx, device_t dev) in iommu_device_tag_init() argument
658 maxaddr = MIN(ctx->domain->end, BUS_SPACE_MAXADDR); in iommu_device_tag_init()
659 ctx->tag->common.impl = &bus_dma_iommu_impl; in iommu_device_tag_init()
660 ctx->tag->common.boundary = 0; in iommu_device_tag_init()
661 ctx->tag->common.lowaddr = maxaddr; in iommu_device_tag_init()
662 ctx->tag->common.highaddr = maxaddr; in iommu_device_tag_init()
663 ctx->tag->common.maxsize = maxaddr; in iommu_device_tag_init()
664 ctx->tag->common.nsegments = BUS_SPACE_UNRESTRICTED; in iommu_device_tag_init()
665 ctx->tag->common.maxsegsz = maxaddr; in iommu_device_tag_init()
666 ctx->tag->ctx = ctx; in iommu_device_tag_init()
667 ctx->tag->owner = dev; in iommu_device_tag_init()
673 if ((entry->flags & IOMMU_MAP_ENTRY_RMRR) != 0) in iommu_domain_free_entry()
680 entry->flags = 0; in iommu_domain_free_entry()
691 base >>= IOMMU_PAGE_SHIFT + (pglvl - lvl - 1) * in pglvl_pgtbl_pte_off()
727 for (res = 0, i = pglvl; i > 0; i--) { in pglvl_max_pages()
750 rlvl = total_pglvl - lvl - 1; in pglvl_page_size()
756 iommu_device_set_iommu_prop(device_t dev, device_t iommu) in iommu_device_set_iommu_prop() argument
764 device_set_prop(dev, DEV_PROP_NAME_IOMMU, iommu, NULL, NULL); in iommu_device_set_iommu_prop()
779 entry->start, entry->end, entry->first, entry->last, in iommu_db_print_domain_entry()
780 entry->free_down, entry->flags); in iommu_db_print_domain_entry()
786 db_printf("%jx ", l->start); in iommu_db_print_domain_entry()
792 db_printf("%jx", r->start); in iommu_db_print_domain_entry()
797 iommu_db_print_ctx(struct iommu_ctx *ctx) in iommu_db_print_ctx() argument
801 ctx, pci_get_bus(ctx->tag->owner), in iommu_db_print_ctx()
802 pci_get_slot(ctx->tag->owner), in iommu_db_print_ctx()
803 pci_get_function(ctx->tag->owner), ctx->refs, in iommu_db_print_ctx()
804 ctx->flags, ctx->loads, ctx->unloads); in iommu_db_print_ctx()
810 struct iommu_ctx *ctx; in iommu_db_domain_print_contexts() local
812 if (LIST_EMPTY(&iodom->contexts)) in iommu_db_domain_print_contexts()
816 LIST_FOREACH(ctx, &iodom->contexts, link) in iommu_db_domain_print_contexts()
817 iommu_db_print_ctx(ctx); in iommu_db_domain_print_contexts()
826 RB_FOREACH(entry, iommu_gas_entries_tree, &iodom->rb_root) { in iommu_db_domain_print_mappings()
834 TAILQ_FOREACH(entry, &iodom->unload_entries, dmamap_link) { in iommu_db_domain_print_mappings()