110 SYSCTL_NODE(_hw_vmm, OID_AUTO, ppt, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
153 	struct pptdev *ppt;  in ppt_attach()  local
157 	ppt = device_get_softc(dev);  in ppt_attach()
168 	TAILQ_INSERT_TAIL(&pptdev_list, ppt, next);  in ppt_attach()
169 	ppt->dev = dev;  in ppt_attach()
180 	struct pptdev *ppt;  in ppt_detach()  local
183 	ppt = device_get_softc(dev);  in ppt_detach()
185 	if (ppt->vm != NULL)  in ppt_detach()
196 	TAILQ_REMOVE(&pptdev_list, ppt, next);  in ppt_detach()
209 DEFINE_CLASS_0(ppt, ppt_driver, ppt_methods, sizeof(struct pptdev));
210 DRIVER_MODULE(ppt, pci, ppt_driver, NULL, NULL);
216 	struct pptdev *ppt;  in ppt_find()  local
219 	TAILQ_FOREACH(ppt, &pptdev_list, next) {  in ppt_find()
220 		dev = ppt->dev;  in ppt_find()
228 	if (ppt == NULL)  in ppt_find()
230 	if (ppt->vm != vm)		/* Make sure we own this device */  in ppt_find()
232 	*pptp = ppt;  in ppt_find()
237 ppt_unmap_all_mmio(struct vm *vm, struct pptdev *ppt)  in ppt_unmap_all_mmio()  argument
243 		seg = &ppt->mmio[i];  in ppt_unmap_all_mmio()
252 ppt_teardown_msi(struct pptdev *ppt)  in ppt_teardown_msi()  argument
258 	if (ppt->msi.num_msgs == 0)  in ppt_teardown_msi()
261 	for (i = 0; i < ppt->msi.num_msgs; i++) {  in ppt_teardown_msi()
262 		rid = ppt->msi.startrid + i;  in ppt_teardown_msi()
263 		res = ppt->msi.res[i];  in ppt_teardown_msi()
264 		cookie = ppt->msi.cookie[i];  in ppt_teardown_msi()
267 			bus_teardown_intr(ppt->dev, res, cookie);  in ppt_teardown_msi()
270 			bus_release_resource(ppt->dev, SYS_RES_IRQ, rid, res);  in ppt_teardown_msi()
272 		ppt->msi.res[i] = NULL;  in ppt_teardown_msi()
273 		ppt->msi.cookie[i] = NULL;  in ppt_teardown_msi()
276 	if (ppt->msi.startrid == 1)  in ppt_teardown_msi()
277 		pci_release_msi(ppt->dev);  in ppt_teardown_msi()
279 	ppt->msi.num_msgs = 0;  in ppt_teardown_msi()
283 ppt_teardown_msix_intr(struct pptdev *ppt, int idx)  in ppt_teardown_msix_intr()  argument
289 	rid = ppt->msix.startrid + idx;  in ppt_teardown_msix_intr()
290 	res = ppt->msix.res[idx];  in ppt_teardown_msix_intr()
291 	cookie = ppt->msix.cookie[idx];  in ppt_teardown_msix_intr()
294 		bus_teardown_intr(ppt->dev, res, cookie);  in ppt_teardown_msix_intr()
297 		bus_release_resource(ppt->dev, SYS_RES_IRQ, rid, res);  in ppt_teardown_msix_intr()
299 	ppt->msix.res[idx] = NULL;  in ppt_teardown_msix_intr()
300 	ppt->msix.cookie[idx] = NULL;  in ppt_teardown_msix_intr()
304 ppt_teardown_msix(struct pptdev *ppt)  in ppt_teardown_msix()  argument
308 	if (ppt->msix.num_msgs == 0)  in ppt_teardown_msix()
311 	for (i = 0; i < ppt->msix.num_msgs; i++)  in ppt_teardown_msix()
312 		ppt_teardown_msix_intr(ppt, i);  in ppt_teardown_msix()
314 	free(ppt->msix.res, M_PPTMSIX);  in ppt_teardown_msix()
315 	free(ppt->msix.cookie, M_PPTMSIX);  in ppt_teardown_msix()
316 	free(ppt->msix.arg, M_PPTMSIX);  in ppt_teardown_msix()
318 	pci_release_msi(ppt->dev);  in ppt_teardown_msix()
320 	if (ppt->msix.msix_table_res) {  in ppt_teardown_msix()
321 		bus_release_resource(ppt->dev, SYS_RES_MEMORY,  in ppt_teardown_msix()
322 				     ppt->msix.msix_table_rid,  in ppt_teardown_msix()
323 				     ppt->msix.msix_table_res);  in ppt_teardown_msix()
324 		ppt->msix.msix_table_res = NULL;  in ppt_teardown_msix()
325 		ppt->msix.msix_table_rid = 0;  in ppt_teardown_msix()
327 	if (ppt->msix.msix_pba_res) {  in ppt_teardown_msix()
328 		bus_release_resource(ppt->dev, SYS_RES_MEMORY,  in ppt_teardown_msix()
329 				     ppt->msix.msix_pba_rid,  in ppt_teardown_msix()
330 				     ppt->msix.msix_pba_res);  in ppt_teardown_msix()
331 		ppt->msix.msix_pba_res = NULL;  in ppt_teardown_msix()
332 		ppt->msix.msix_pba_rid = 0;  in ppt_teardown_msix()
335 	ppt->msix.num_msgs = 0;  in ppt_teardown_msix()
348 	struct pptdev *ppt;  in ppt_assigned_devices()  local
352 	TAILQ_FOREACH(ppt, &pptdev_list, next) {  in ppt_assigned_devices()
353 		if (ppt->vm == vm)  in ppt_assigned_devices()
363 	struct pptdev *ppt;  in ppt_is_mmio()  local
366 	TAILQ_FOREACH(ppt, &pptdev_list, next) {  in ppt_is_mmio()
367 		if (ppt->vm != vm)  in ppt_is_mmio()
371 			seg = &ppt->mmio[i];  in ppt_is_mmio()
394 ppt_bar_enables(struct pptdev *ppt)  in ppt_bar_enables()  argument
400 	for (pm = pci_first_bar(ppt->dev); pm != NULL; pm = pci_next_bar(pm)) {  in ppt_bar_enables()
412 	struct pptdev *ppt;  in ppt_assign_device()  local
417 	error = ppt_find(NULL, bus, slot, func, &ppt);  in ppt_assign_device()
421 	pci_save_state(ppt->dev);  in ppt_assign_device()
422 	ppt_pci_reset(ppt->dev);  in ppt_assign_device()
423 	pci_restore_state(ppt->dev);  in ppt_assign_device()
424 	error = iommu_add_device(vm_iommu_domain(vm), ppt->dev,  in ppt_assign_device()
425 	    pci_get_rid(ppt->dev));  in ppt_assign_device()
428 	ppt->vm = vm;  in ppt_assign_device()
429 	cmd = pci_read_config(ppt->dev, PCIR_COMMAND, 2);  in ppt_assign_device()
430 	cmd |= PCIM_CMD_BUSMASTEREN | ppt_bar_enables(ppt);  in ppt_assign_device()
431 	pci_write_config(ppt->dev, PCIR_COMMAND, cmd, 2);  in ppt_assign_device()
438 	struct pptdev *ppt;  in ppt_unassign_device()  local
442 	error = ppt_find(vm, bus, slot, func, &ppt);  in ppt_unassign_device()
446 	cmd = pci_read_config(ppt->dev, PCIR_COMMAND, 2);  in ppt_unassign_device()
448 	pci_write_config(ppt->dev, PCIR_COMMAND, cmd, 2);  in ppt_unassign_device()
449 	pci_save_state(ppt->dev);  in ppt_unassign_device()
450 	ppt_pci_reset(ppt->dev);  in ppt_unassign_device()
451 	pci_restore_state(ppt->dev);  in ppt_unassign_device()
452 	ppt_unmap_all_mmio(vm, ppt);  in ppt_unassign_device()
453 	ppt_teardown_msi(ppt);  in ppt_unassign_device()
454 	ppt_teardown_msix(ppt);  in ppt_unassign_device()
455 	error = iommu_remove_device(vm_iommu_domain(vm), ppt->dev,  in ppt_unassign_device()
456 	    pci_get_rid(ppt->dev));  in ppt_unassign_device()
457 	ppt->vm = NULL;  in ppt_unassign_device()
464 	struct pptdev *ppt;  in ppt_unassign_all()  local
468 	TAILQ_FOREACH(ppt, &pptdev_list, next) {  in ppt_unassign_all()
469 		if (ppt->vm == vm) {  in ppt_unassign_all()
470 			dev = ppt->dev;  in ppt_unassign_all()
482 ppt_valid_bar_mapping(struct pptdev *ppt, vm_paddr_t hpa, size_t len)  in ppt_valid_bar_mapping()  argument
487 	for (pm = pci_first_bar(ppt->dev); pm != NULL; pm = pci_next_bar(pm)) {  in ppt_valid_bar_mapping()
504 	struct pptdev *ppt;  in ppt_map_mmio()  local
510 	error = ppt_find(vm, bus, slot, func, &ppt);  in ppt_map_mmio()
514 	if (!ppt_valid_bar_mapping(ppt, hpa, len))  in ppt_map_mmio()
518 		seg = &ppt->mmio[i];  in ppt_map_mmio()
537 	struct pptdev *ppt;  in ppt_unmap_mmio()  local
539 	error = ppt_find(vm, bus, slot, func, &ppt);  in ppt_unmap_mmio()
544 		seg = &ppt->mmio[i];  in ppt_unmap_mmio()
560 	struct pptdev *ppt;  in pptintr()  local
564 	ppt = pptarg->pptdev;  in pptintr()
566 	if (ppt->vm != NULL)  in pptintr()
567 		lapic_intr_msi(ppt->vm, pptarg->addr, pptarg->msg_data);  in pptintr()
579 	if (ppt->msi.startrid == 0)  in pptintr()
591 	struct pptdev *ppt;  in ppt_setup_msi()  local
596 	error = ppt_find(vm, bus, slot, func, &ppt);  in ppt_setup_msi()
601 	if (ppt->msix.num_msgs != 0 && numvec != 0)  in ppt_setup_msi()
605 	ppt_teardown_msi(ppt);  in ppt_setup_msi()
611 	msi_count = pci_msi_count(ppt->dev);  in ppt_setup_msi()
632 		error = pci_alloc_msi(ppt->dev, &tmp);  in ppt_setup_msi()
636 			pci_release_msi(ppt->dev);  in ppt_setup_msi()
643 	ppt->msi.startrid = startrid;  in ppt_setup_msi()
649 		ppt->msi.num_msgs = i + 1;  in ppt_setup_msi()
650 		ppt->msi.cookie[i] = NULL;  in ppt_setup_msi()
653 		ppt->msi.res[i] = bus_alloc_resource_any(ppt->dev, SYS_RES_IRQ,  in ppt_setup_msi()
655 		if (ppt->msi.res[i] == NULL)  in ppt_setup_msi()
658 		ppt->msi.arg[i].pptdev = ppt;  in ppt_setup_msi()
659 		ppt->msi.arg[i].addr = addr;  in ppt_setup_msi()
660 		ppt->msi.arg[i].msg_data = msg + i;  in ppt_setup_msi()
662 		error = bus_setup_intr(ppt->dev, ppt->msi.res[i],  in ppt_setup_msi()
664 				       pptintr, NULL, &ppt->msi.arg[i],  in ppt_setup_msi()
665 				       &ppt->msi.cookie[i]);  in ppt_setup_msi()
671 		ppt_teardown_msi(ppt);  in ppt_setup_msi()
682 	struct pptdev *ppt;  in ppt_setup_msix()  local
687 	error = ppt_find(vm, bus, slot, func, &ppt);  in ppt_setup_msix()
692 	if (ppt->msi.num_msgs != 0)  in ppt_setup_msix()
695 	dinfo = device_get_ivars(ppt->dev);  in ppt_setup_msix()
705 	if (ppt->msix.num_msgs == 0) {  in ppt_setup_msix()
706 		numvec = pci_msix_count(ppt->dev);  in ppt_setup_msix()
710 		ppt->msix.startrid = 1;  in ppt_setup_msix()
711 		ppt->msix.num_msgs = numvec;  in ppt_setup_msix()
713 		res_size = numvec * sizeof(ppt->msix.res[0]);  in ppt_setup_msix()
714 		cookie_size = numvec * sizeof(ppt->msix.cookie[0]);  in ppt_setup_msix()
715 		arg_size = numvec * sizeof(ppt->msix.arg[0]);  in ppt_setup_msix()
717 		ppt->msix.res = malloc(res_size, M_PPTMSIX, M_WAITOK | M_ZERO);  in ppt_setup_msix()
718 		ppt->msix.cookie = malloc(cookie_size, M_PPTMSIX,  in ppt_setup_msix()
720 		ppt->msix.arg = malloc(arg_size, M_PPTMSIX, M_WAITOK | M_ZERO);  in ppt_setup_msix()
723 		ppt->msix.msix_table_res = bus_alloc_resource_any(ppt->dev,  in ppt_setup_msix()
726 		if (ppt->msix.msix_table_res == NULL) {  in ppt_setup_msix()
727 			ppt_teardown_msix(ppt);  in ppt_setup_msix()
730 		ppt->msix.msix_table_rid = rid;  in ppt_setup_msix()
735 			ppt->msix.msix_pba_res = bus_alloc_resource_any(  in ppt_setup_msix()
736 			    ppt->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);  in ppt_setup_msix()
738 			if (ppt->msix.msix_pba_res == NULL) {  in ppt_setup_msix()
739 				ppt_teardown_msix(ppt);  in ppt_setup_msix()
742 			ppt->msix.msix_pba_rid = rid;  in ppt_setup_msix()
746 		error = pci_alloc_msix(ppt->dev, &alloced);  in ppt_setup_msix()
748 			ppt_teardown_msix(ppt);  in ppt_setup_msix()
753 	if (idx >= ppt->msix.num_msgs)  in ppt_setup_msix()
758 		ppt_teardown_msix_intr(ppt, idx);  in ppt_setup_msix()
761 		ppt->msix.cookie[idx] = NULL;  in ppt_setup_msix()
762 		rid = ppt->msix.startrid + idx;  in ppt_setup_msix()
763 		ppt->msix.res[idx] = bus_alloc_resource_any(ppt->dev, SYS_RES_IRQ,  in ppt_setup_msix()
765 		if (ppt->msix.res[idx] == NULL)  in ppt_setup_msix()
768 		ppt->msix.arg[idx].pptdev = ppt;  in ppt_setup_msix()
769 		ppt->msix.arg[idx].addr = addr;  in ppt_setup_msix()
770 		ppt->msix.arg[idx].msg_data = msg;  in ppt_setup_msix()
773 		error = bus_setup_intr(ppt->dev, ppt->msix.res[idx],  in ppt_setup_msix()
775 				       pptintr, NULL, &ppt->msix.arg[idx],  in ppt_setup_msix()
776 				       &ppt->msix.cookie[idx]);  in ppt_setup_msix()
779 			bus_release_resource(ppt->dev, SYS_RES_IRQ, rid, ppt->msix.res[idx]);  in ppt_setup_msix()
780 			ppt->msix.cookie[idx] = NULL;  in ppt_setup_msix()
781 			ppt->msix.res[idx] = NULL;  in ppt_setup_msix()
786 		ppt_teardown_msix_intr(ppt, idx);  in ppt_setup_msix()
795 	struct pptdev *ppt;  in ppt_disable_msix()  local
798 	error = ppt_find(vm, bus, slot, func, &ppt);  in ppt_disable_msix()
802 	ppt_teardown_msix(ppt);  in ppt_disable_msix()