Lines Matching +full:iommu +full:- +full:ctx
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
55 #include "io/iommu.h"
63 #define MOD_DEC(a, s, m) (((a) - (s)) % ((m) * (s)))
127 return (pci_cfgregread(softc->pci_seg, PCI_RID2BUS(softc->pci_rid), in amdvi_pci_read()
128 PCI_RID2SLOT(softc->pci_rid), PCI_RID2FUNC(softc->pci_rid), in amdvi_pci_read()
136 * If ATS is absent or disabled, return (-1), otherwise ATS
144 int qlen = -1; in amdvi_find_ats_qlen()
150 return (-1); in amdvi_find_ats_qlen()
158 printf("AMD-Vi: PCI device %d.%d.%d ATS %s qlen=%d\n", in amdvi_find_ats_qlen()
162 qlen = (cap & PCIM_ATS_EN) ? qlen : -1; in amdvi_find_ats_qlen()
183 cfg = softc->dev_cfg; in amdvi_dev_support_iotlb()
186 for (i = 0; i < softc->dev_cfg_cnt; i++) { in amdvi_dev_support_iotlb()
187 if ((cfg->start_id <= devid) && (cfg->end_id >= devid)) { in amdvi_dev_support_iotlb()
188 ivhd_ats = cfg->enable_ats; in amdvi_dev_support_iotlb()
196 device_printf(softc->dev, in amdvi_dev_support_iotlb()
205 /* Enable IOTLB support for IOMMU if its supported. */
210 softc->iotlb = false; in amdvi_hw_enable_iotlb()
214 supported = (softc->ivhd_flag & IVHD_FLAG_IOTLB) ? true : false; in amdvi_hw_enable_iotlb()
216 if (softc->pci_cap & AMDVI_PCI_CAP_IOTLB) { in amdvi_hw_enable_iotlb()
218 device_printf(softc->dev, "IOTLB disabled by BIOS.\n"); in amdvi_hw_enable_iotlb()
221 device_printf(softc->dev, "IOTLB disabled by user.\n"); in amdvi_hw_enable_iotlb()
227 softc->iotlb = supported; in amdvi_hw_enable_iotlb()
235 struct amdvi_ctrl *ctrl = softc->ctrl; in amdvi_init_cmd()
237 ctrl->cmd.len = 8; /* Use 256 command buffer entries. */ in amdvi_init_cmd()
238 softc->cmd_max = 1 << ctrl->cmd.len; in amdvi_init_cmd()
240 softc->cmd = malloc(sizeof(struct amdvi_cmd) * in amdvi_init_cmd()
241 softc->cmd_max, M_AMDVI, M_WAITOK | M_ZERO); in amdvi_init_cmd()
243 if ((uintptr_t)softc->cmd & PAGE_MASK) in amdvi_init_cmd()
246 ctrl->cmd.base = vtophys(softc->cmd) / PAGE_SIZE; in amdvi_init_cmd()
248 * XXX: Reset the h/w pointers in case IOMMU is restarting, in amdvi_init_cmd()
251 ctrl->cmd_tail = 0; in amdvi_init_cmd()
252 ctrl->cmd_head = 0; in amdvi_init_cmd()
260 * of AMD IOMMU spec ver 2.0.
270 KASSERT(softc->cmd != NULL, ("cmd is NULL")); in amdvi_get_cmd_tail()
272 ctrl = softc->ctrl; in amdvi_get_cmd_tail()
275 tail = (struct amdvi_cmd *)((uint8_t *)softc->cmd + in amdvi_get_cmd_tail()
276 ctrl->cmd_tail); in amdvi_get_cmd_tail()
291 KASSERT(softc->cmd != NULL, ("cmd is NULL")); in amdvi_update_cmd_tail()
293 ctrl = softc->ctrl; in amdvi_update_cmd_tail()
296 ctrl->cmd_tail = MOD_INC(ctrl->cmd_tail, size, softc->cmd_max); in amdvi_update_cmd_tail()
297 softc->total_cmd++; in amdvi_update_cmd_tail()
300 device_printf(softc->dev, "cmd_tail: %s Tail:0x%x, Head:0x%x.\n", in amdvi_update_cmd_tail()
301 ctrl->cmd_tail, in amdvi_update_cmd_tail()
302 ctrl->cmd_head); in amdvi_update_cmd_tail()
308 * Various commands supported by IOMMU.
321 pa = vtophys(&softc->cmp_data); in amdvi_cmd_cmp()
322 cmd->opcode = AMDVI_CMP_WAIT_OPCODE; in amdvi_cmd_cmp()
323 cmd->word0 = (pa & 0xFFFFFFF8) | AMDVI_CMP_WAIT_STORE; in amdvi_cmd_cmp()
324 cmd->word1 = (pa >> 32) & 0xFFFFF; in amdvi_cmd_cmp()
325 cmd->addr = data; in amdvi_cmd_cmp()
338 cmd->opcode = AMDVI_INVD_DTE_OPCODE; in amdvi_cmd_inv_dte()
339 cmd->word0 = devid; in amdvi_cmd_inv_dte()
342 device_printf(softc->dev, "Invalidated DTE:0x%x\n", devid); in amdvi_cmd_inv_dte()
346 /* Invalidate IOMMU page, use for invalidation of domain. */
357 cmd->opcode = AMDVI_INVD_PAGE_OPCODE; in amdvi_cmd_inv_iommu_pages()
358 cmd->word1 = domain_id; in amdvi_cmd_inv_iommu_pages()
362 cmd->addr = addr; in amdvi_cmd_inv_iommu_pages()
363 cmd->addr |= pde ? AMDVI_INVD_PAGE_PDE : 0; in amdvi_cmd_inv_iommu_pages()
364 cmd->addr |= page ? AMDVI_INVD_PAGE_S : 0; in amdvi_cmd_inv_iommu_pages()
377 if (!softc->iotlb) in amdvi_cmd_inv_iotlb()
389 device_printf(softc->dev, "Invalidate IOTLB devID 0x%x" in amdvi_cmd_inv_iotlb()
392 cmd->opcode = AMDVI_INVD_IOTLB_OPCODE; in amdvi_cmd_inv_iotlb()
393 cmd->word0 = devid; in amdvi_cmd_inv_iotlb()
394 cmd->word1 = qlen; in amdvi_cmd_inv_iotlb()
395 cmd->addr = AMDVI_INVD_IOTLB_ALL_ADDR | in amdvi_cmd_inv_iotlb()
410 cmd->opcode = AMDVI_INVD_INTR_OPCODE; in amdvi_cmd_inv_intr_map()
411 cmd->word0 = devid; in amdvi_cmd_inv_intr_map()
414 device_printf(softc->dev, "Invalidate INTR map of devID 0x%x\n", devid); in amdvi_cmd_inv_intr_map()
429 * See section 3.3.3 of IOMMU spec rev 2.0, software note in amdvi_inv_domain()
436 device_printf(softc->dev, "Invalidate domain:0x%x\n", domain_id); in amdvi_inv_domain()
445 struct amdvi_ctrl *ctrl = softc->ctrl; in amdvi_cmp_wait()
452 read = &softc->cmp_data; in amdvi_cmp_wait()
459 status = (VERIFY == softc->cmp_data) ? true : false; in amdvi_cmp_wait()
463 device_printf(softc->dev, "CMD completion DONE Tail:0x%x, " in amdvi_cmp_wait()
464 "Head:0x%x, loop:%d.\n", ctrl->cmd_tail, in amdvi_cmp_wait()
465 ctrl->cmd_head, loop); in amdvi_cmp_wait()
478 ctrl = softc->ctrl; in amdvi_wait()
481 if ((ctrl->control & AMDVI_CTRL_EN) == 0) in amdvi_wait()
489 device_printf(softc->dev, "Error: completion failed" in amdvi_wait()
491 ctrl->cmd_tail, ctrl->cmd_head); in amdvi_wait()
503 ctrl = softc->ctrl; in amdvi_dump_cmds()
504 device_printf(softc->dev, "Dump last %d command(s):\n", count); in amdvi_dump_cmds()
509 off = MOD_DEC(ctrl->cmd_head, sizeof(struct amdvi_cmd), in amdvi_dump_cmds()
510 softc->cmd_max); in amdvi_dump_cmds()
511 for (i = 0; off != ctrl->cmd_tail && i < count; i++) { in amdvi_dump_cmds()
512 cmd = (struct amdvi_cmd *)((uint8_t *)softc->cmd + off); in amdvi_dump_cmds()
514 " 0x%x 0x%lx\n", i, off, cmd->opcode, in amdvi_dump_cmds()
515 cmd->word0, cmd->word1, cmd->addr); in amdvi_dump_cmds()
516 off = MOD_INC(off, sizeof(struct amdvi_cmd), softc->cmd_max); in amdvi_dump_cmds()
525 ctrl = softc->ctrl; in amdvi_init_event()
526 ctrl->event.len = 8; in amdvi_init_event()
527 softc->event_max = 1 << ctrl->event.len; in amdvi_init_event()
528 softc->event = malloc(sizeof(struct amdvi_event) * in amdvi_init_event()
529 softc->event_max, M_AMDVI, M_WAITOK | M_ZERO); in amdvi_init_event()
530 if ((uintptr_t)softc->event & PAGE_MASK) { in amdvi_init_event()
531 device_printf(softc->dev, "Event buffer not aligned on page."); in amdvi_init_event()
534 ctrl->event.base = vtophys(softc->event) / PAGE_SIZE; in amdvi_init_event()
537 ctrl->evt_head = 0; in amdvi_init_event()
538 ctrl->evt_tail = 0; in amdvi_init_event()
562 /* See section 2.5.4 of AMD IOMMU spec ver 2.62.*/
634 switch (evt->opcode) { in amdvi_decode_evt()
636 amdvi_decode_inv_dte_evt(evt->devid, evt->pasid_domid, in amdvi_decode_evt()
637 evt->addr, evt->flag); in amdvi_decode_evt()
641 amdvi_decode_pf_evt(evt->devid, evt->pasid_domid, in amdvi_decode_evt()
642 evt->addr, evt->flag); in amdvi_decode_evt()
646 amdvi_decode_dte_hwerr_evt(evt->devid, evt->pasid_domid, in amdvi_decode_evt()
647 evt->addr, evt->flag); in amdvi_decode_evt()
651 amdvi_decode_page_hwerr_evt(evt->devid, evt->pasid_domid, in amdvi_decode_evt()
652 evt->addr, evt->flag); in amdvi_decode_evt()
658 printf("\t[%s EVT]\n", (evt->opcode == AMDVI_EVENT_ILLEGAL_CMD) ? in amdvi_decode_evt()
660 cmd = (struct amdvi_cmd *)PHYS_TO_DMAP(evt->addr); in amdvi_decode_evt()
662 cmd->opcode, cmd->word0, cmd->word1, cmd->addr); in amdvi_decode_evt()
667 evt->devid, evt->addr); in amdvi_decode_evt()
672 evt->devid, evt->addr, evt->flag >> 9, in amdvi_decode_evt()
673 (evt->flag >> 8) & 1); in amdvi_decode_evt()
678 printf("AMD-Vi: v2 events.\n"); in amdvi_decode_evt()
682 printf("Unsupported AMD-Vi event:%d\n", evt->opcode); in amdvi_decode_evt()
693 ctrl = softc->ctrl; in amdvi_print_events()
695 for (i = 0; i < softc->event_max; i++) { in amdvi_print_events()
696 event = &softc->event[ctrl->evt_head / size]; in amdvi_print_events()
697 if (!event->opcode) in amdvi_print_events()
699 device_printf(softc->dev, "\t[Event%d: Head:0x%x Tail:0x%x]\n", in amdvi_print_events()
700 i, ctrl->evt_head, ctrl->evt_tail); in amdvi_print_events()
702 ctrl->evt_head = MOD_INC(ctrl->evt_head, size, in amdvi_print_events()
703 softc->event_max); in amdvi_print_events()
712 ctrl = softc->ctrl; in amdvi_init_dte()
713 ctrl->dte.base = vtophys(amdvi_dte) / PAGE_SIZE; in amdvi_init_dte()
714 ctrl->dte.size = 0x1FF; /* 2MB device table. */ in amdvi_init_dte()
720 * Not all capabilities of IOMMU are available in ACPI IVHD flag
730 off = softc->cap_off; in amdvi_print_pci_cap()
733 * Section 3.7.1 of IOMMU sepc rev 2.0. in amdvi_print_pci_cap()
740 ("Not a IOMMU capability 0x%x@0x%x", cap, off)); in amdvi_print_pci_cap()
742 softc->pci_cap = cap >> 24; in amdvi_print_pci_cap()
743 device_printf(softc->dev, "PCI cap 0x%x@0x%x feature:%b\n", in amdvi_print_pci_cap()
744 cap, off, softc->pci_cap, in amdvi_print_pci_cap()
757 ctrl = softc->ctrl; in amdvi_event_intr()
758 device_printf(softc->dev, "EVT INTR %ld Status:0x%x" in amdvi_event_intr()
759 " EVT Head:0x%x Tail:0x%x]\n", softc->event_intr_cnt++, in amdvi_event_intr()
760 ctrl->status, ctrl->evt_head, ctrl->evt_tail); in amdvi_event_intr()
762 softc->total_cmd, ctrl->cmd_tail, ctrl->cmd_head); in amdvi_event_intr()
765 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; in amdvi_event_intr()
776 mmio_dev = softc->pci_dev; in amdvi_free_evt_intr_res()
788 dev = softc->dev; in amdvi_alloc_intr_resources()
789 mmio_dev = softc->pci_dev; in amdvi_alloc_intr_resources()
792 ctrl = softc->ctrl; in amdvi_alloc_intr_resources()
793 ctrl->status &= AMDVI_STATUS_EV_OF | AMDVI_STATUS_EV_INTR; in amdvi_alloc_intr_resources()
808 cfg = softc->dev_cfg; in amdvi_print_dev_cap()
809 for (i = 0; i < softc->dev_cfg_cnt; i++) { in amdvi_print_dev_cap()
810 device_printf(softc->dev, "device [0x%x - 0x%x] " in amdvi_print_dev_cap()
811 "config:%b%s\n", cfg->start_id, cfg->end_id, in amdvi_print_dev_cap()
812 cfg->data, in amdvi_print_dev_cap()
815 cfg->enable_ats ? "ATS enabled" : ""); in amdvi_print_dev_cap()
831 result = softc->ctrl->cmd_head; in amdvi_handle_sysctl()
836 result = softc->ctrl->cmd_tail; in amdvi_handle_sysctl()
841 result = softc->ctrl->evt_head; in amdvi_handle_sysctl()
846 result = softc->ctrl->evt_tail; in amdvi_handle_sysctl()
852 device_printf(softc->dev, "Unknown sysctl:%d\n", type); in amdvi_handle_sysctl()
862 struct sysctl_ctx_list *ctx; in amdvi_add_sysctl() local
865 dev = softc->dev; in amdvi_add_sysctl()
866 ctx = device_get_sysctl_ctx(dev); in amdvi_add_sysctl()
869 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "event_intr_count", CTLFLAG_RD, in amdvi_add_sysctl()
870 &softc->event_intr_cnt, "Event interrupt count"); in amdvi_add_sysctl()
871 SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "command_count", CTLFLAG_RD, in amdvi_add_sysctl()
872 &softc->total_cmd, "Command submitted count"); in amdvi_add_sysctl()
873 SYSCTL_ADD_U16(ctx, child, OID_AUTO, "pci_rid", CTLFLAG_RD, in amdvi_add_sysctl()
874 &softc->pci_rid, 0, "IOMMU RID"); in amdvi_add_sysctl()
875 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_head", in amdvi_add_sysctl()
878 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "command_tail", in amdvi_add_sysctl()
881 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_head", in amdvi_add_sysctl()
884 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "event_tail", in amdvi_add_sysctl()
895 dev = softc->dev; in amdvi_setup_hw()
929 dev = softc->dev; in amdvi_teardown_hw()
936 if (softc->cmd) in amdvi_teardown_hw()
937 free(softc->cmd, M_AMDVI); in amdvi_teardown_hw()
939 if (softc->event) in amdvi_teardown_hw()
940 free(softc->event, M_AMDVI); in amdvi_teardown_hw()
953 printf("bhyve: Found %d AMD-Vi/IOMMU device(s), " in amdvi_init()
954 "use hw.vmm.amdvi.enable=1 to enable pass-through.\n", in amdvi_init()
996 if (create && ((softc->pci_cap & AMDVI_PCI_CAP_NPCACHE) == 0)) in amdvi_do_inv_domain()
1010 dom->id = amdvi_domainId(); in amdvi_create_domain()
1011 //dom->maxaddr = maxaddr; in amdvi_create_domain()
1013 printf("Created domain #%d\n", dom->id); in amdvi_create_domain()
1018 if (dom->id || amdvi_host_ptp) in amdvi_create_domain()
1019 dom->ptp = malloc(PAGE_SIZE, M_AMDVI, M_WAITOK | M_ZERO); in amdvi_create_domain()
1021 dom->ptp_level = amdvi_ptp_level; in amdvi_create_domain()
1023 amdvi_do_inv_domain(dom->id, true); in amdvi_create_domain()
1040 /* XXX: Add super-page or PTE mapping > 4KB. */ in amdvi_free_ptp()
1042 /* Super-page mapping. */ in amdvi_free_ptp()
1048 & AMDVI_PT_MASK), level - 1); in amdvi_free_ptp()
1062 printf("Destroying domain %d\n", domain->id); in amdvi_destroy_domain()
1064 if (domain->ptp) in amdvi_destroy_domain()
1065 amdvi_free_ptp(domain->ptp, domain->ptp_level); in amdvi_destroy_domain()
1067 amdvi_do_inv_domain(domain->id, false); in amdvi_destroy_domain()
1079 const int PT_INDEX_MASK = (1 << PT_SHIFT) - 1; /* Based on PT_SHIFT */ in amdvi_set_pt()
1084 if (hpa & (pg_size - 1)) { in amdvi_set_pt()
1088 if (gpa & (pg_size - 1)) { in amdvi_set_pt()
1100 ((level - 1) << AMDVI_PD_LEVEL_SHIFT); in amdvi_set_pt()
1110 shift -= PT_SHIFT; in amdvi_set_pt()
1111 level--; in amdvi_set_pt()
1138 level = domain->ptp_level; in amdvi_update_mapping()
1141 ptp = domain->ptp; in amdvi_update_mapping()
1166 if (domain->id && !domain->ptp) { in amdvi_create_mapping()
1168 return (-1); in amdvi_create_mapping()
1172 * If host domain is created w/o page table, skip IOMMU page in amdvi_create_mapping()
1173 * table set-up. in amdvi_create_mapping()
1175 if (domain->ptp) in amdvi_create_mapping()
1188 * If host domain is created w/o page table, skip IOMMU page in amdvi_remove_mapping()
1189 * table set-up. in amdvi_remove_mapping()
1191 if (domain->ptp) in amdvi_remove_mapping()
1205 for (j = 0; j < softc->dev_cfg_cnt; j++) in amdvi_find_iommu()
1206 if ((devid >= softc->dev_cfg[j].start_id) && in amdvi_find_iommu()
1207 (devid <= softc->dev_cfg[j].end_id)) in amdvi_find_iommu()
1215 * Set-up device table entry.
1216 * IOMMU spec Rev 2.0, section 3.2.2.2, some of the fields must
1231 /* If IOMMU and device support IOTLB, enable it. */ in amdvi_set_dte()
1232 if (amdvi_dev_support_iotlb(softc, devid) && softc->iotlb) in amdvi_set_dte()
1233 temp->iotlb_enable = 1; in amdvi_set_dte()
1237 temp->sup_second_io_fault = 1; in amdvi_set_dte()
1238 temp->sup_all_io_fault = amdvi_disable_io_fault; in amdvi_set_dte()
1240 temp->dt_valid = 1; in amdvi_set_dte()
1241 temp->domain_id = domain->id; in amdvi_set_dte()
1244 if (domain->ptp) { in amdvi_set_dte()
1245 temp->pt_base = vtophys(domain->ptp) >> 12; in amdvi_set_dte()
1246 temp->pt_level = amdvi_ptp_level; in amdvi_set_dte()
1252 temp->pt_valid = 1; in amdvi_set_dte()
1253 temp->read_allow = 1; in amdvi_set_dte()
1254 temp->write_allow = 1; in amdvi_set_dte()
1281 RID2PCI_STR(devid), domain->id); in amdvi_add_device()
1299 devid, domain->id); in amdvi_remove_device()
1319 ctrl = softc->ctrl; in amdvi_enable()
1328 if (softc->ivhd_flag & IVHD_FLAG_COH) in amdvi_enable()
1330 if (softc->ivhd_flag & IVHD_FLAG_HTT) in amdvi_enable()
1332 if (softc->ivhd_flag & IVHD_FLAG_RPPW) in amdvi_enable()
1334 if (softc->ivhd_flag & IVHD_FLAG_PPW) in amdvi_enable()
1336 if (softc->ivhd_flag & IVHD_FLAG_ISOC) in amdvi_enable()
1339 ctrl->control = val; in amdvi_enable()
1353 ctrl = softc->ctrl; in amdvi_disable()
1356 ctrl->control = 0; in amdvi_disable()
1367 amdvi_do_inv_domain(domain->id, false); in amdvi_invalidate_tlb()