84 static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid);
86 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid);
154 	struct dev_table_entry *ptr = &dev_table[dev_data->devid];  in update_dte256()
162 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
167 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
175 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
184 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
197 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
202 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
232 	ptr = &dev_table[dev_data->devid];  in get_dte256()
300 	return p1->devid;  in get_acpihid_device_id()
336 		u32 devid = get_acpihid_device_id(dev, NULL);  in get_device_segment()  local
338 		seg = PCI_SBDF_TO_SEGID(devid);  in get_device_segment()
345 void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid)  in amd_iommu_set_rlookup_table()  argument
349 	pci_seg->rlookup_table[devid] = iommu;  in amd_iommu_set_rlookup_table()
352 static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid)  in __rlookup_amd_iommu()  argument
360 		if (devid > pci_seg->last_bdf)  in __rlookup_amd_iommu()
362 		return pci_seg->rlookup_table[devid];  in __rlookup_amd_iommu()
370 	int devid = get_device_sbdf_id(dev);  in rlookup_amd_iommu()  local
372 	if (devid < 0)  in rlookup_amd_iommu()
374 	return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid));  in rlookup_amd_iommu()
377 static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid)  in alloc_dev_data()  argument
388 	dev_data->devid = devid;  in alloc_dev_data()
395 struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid)  in search_dev_data()  argument
406 		if (dev_data->devid == devid)  in search_dev_data()
419 	u16 devid = pci_dev_id(pdev);  in clone_alias()  local
422 	if (devid == alias)  in clone_alias()
432 		pr_err("%s : Failed to get dev_data for 0x%x\n", __func__, devid);  in clone_alias()
492 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid)  in find_dev_data()  argument
496 	dev_data = search_dev_data(iommu, devid);  in find_dev_data()
499 		dev_data = alloc_dev_data(iommu, devid);  in find_dev_data()
516 	int devid;  in acpihid_device_group()  local
518 	devid = get_acpihid_device_id(dev, &entry);  in acpihid_device_group()
519 	if (devid < 0)  in acpihid_device_group()
520 		return ERR_PTR(devid);  in acpihid_device_group()
523 		if ((devid == p->devid) && p->group)  in acpihid_device_group()
682 	int devid, sbdf;  in check_device()  local
690 	devid = PCI_SBDF_TO_DEVID(sbdf);  in check_device()
698 	if (devid > pci_seg->last_bdf)  in check_device()
707 	int devid, sbdf;  in iommu_init_device()  local
716 	devid = PCI_SBDF_TO_DEVID(sbdf);  in iommu_init_device()
717 	dev_data = find_dev_data(iommu, devid);  in iommu_init_device()
748 	int devid, sbdf;  in iommu_ignore_device()  local
754 	devid = PCI_SBDF_TO_DEVID(sbdf);  in iommu_ignore_device()
755 	pci_seg->rlookup_table[devid] = NULL;  in iommu_ignore_device()
756 	memset(&dev_table[devid], 0, sizeof(struct dev_table_entry));  in iommu_ignore_device()
768 static void dump_dte_entry(struct amd_iommu *iommu, u16 devid)  in dump_dte_entry()  argument
772 	struct iommu_dev_data *dev_data = find_dev_data(iommu, devid);  in dump_dte_entry()
792 	int devid, vmg_tag, flags;  in amd_iommu_report_rmp_hw_error()  local
796 	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;  in amd_iommu_report_rmp_hw_error()
801 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),  in amd_iommu_report_rmp_hw_error()
802 					   devid & 0xff);  in amd_iommu_report_rmp_hw_error()
813 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in amd_iommu_report_rmp_hw_error()
824 	int devid, flags_rmp, vmg_tag, flags;  in amd_iommu_report_rmp_fault()  local
828 	devid     = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;  in amd_iommu_report_rmp_fault()
834 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),  in amd_iommu_report_rmp_fault()
835 					   devid & 0xff);  in amd_iommu_report_rmp_fault()
846 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in amd_iommu_report_rmp_fault()
861 					u16 devid, u16 domain_id,  in amd_iommu_report_page_fault()  argument
867 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),  in amd_iommu_report_page_fault()
868 					   devid & 0xff);  in amd_iommu_report_page_fault()
883 						   iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),  in amd_iommu_report_page_fault()
884 						   PCI_FUNC(devid), domain_id);  in amd_iommu_report_page_fault()
902 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in amd_iommu_report_page_fault()
912 				     u16 devid, u64 address, int flags)  in amd_iommu_report_ppr_err()  argument
922 			    iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),  in amd_iommu_report_ppr_err()
923 			    PCI_FUNC(devid), pasid, address, flags, tag);  in amd_iommu_report_ppr_err()
929 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),  in amd_iommu_report_ppr_err()
930 					   devid & 0xff);  in amd_iommu_report_ppr_err()
948 	int type, devid, flags;  in iommu_print_event()  local
956 	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;  in iommu_print_event()
974 		amd_iommu_report_page_fault(iommu, devid, pasid, address, flags);  in iommu_print_event()
981 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
984 		dump_dte_entry(iommu, devid);  in iommu_print_event()
989 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
994 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
1007 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
1012 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
1022 		amd_iommu_report_ppr_err(iommu, event, devid, address, flags);  in iommu_print_event()
1101 			pr_debug("%s: devid=%#x, ga_tag=%#x\n",  in iommu_poll_ga_log()
1248 			 iommu->pci_seg->id, PCI_BUS_NUM(iommu->devid),  in wait_on_sem()
1249 			 PCI_SLOT(iommu->devid), PCI_FUNC(iommu->devid));  in wait_on_sem()
1292 static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)  in build_inv_dte()  argument
1295 	cmd->data[0] = devid;  in build_inv_dte()
1355 static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,  in build_inv_iotlb_pages()  argument
1363 	cmd->data[0]  = devid;  in build_inv_iotlb_pages()
1365 	cmd->data[1]  = devid;  in build_inv_iotlb_pages()
1377 static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid,  in build_complete_ppr()  argument
1382 	cmd->data[0]  = devid;  in build_complete_ppr()
1399 static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)  in build_inv_irt()  argument
1402 	cmd->data[0] = devid;  in build_inv_irt()
1527 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)  in iommu_flush_dte()  argument
1531 	build_inv_dte(&cmd, devid);  in iommu_flush_dte()
1536 static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid)  in iommu_flush_dte_sync()  argument
1540 	ret = iommu_flush_dte(iommu, devid);  in iommu_flush_dte_sync()
1547 	u32 devid;  in amd_iommu_flush_dte_all()  local
1550 	for (devid = 0; devid <= last_bdf; ++devid)  in amd_iommu_flush_dte_all()
1551 		iommu_flush_dte(iommu, devid);  in amd_iommu_flush_dte_all()
1605 				 iommu->devid, gdom_info->hdom_id);  in iommu_flush_pages_v1_hdom_ids()
1625 static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)  in iommu_flush_irt()  argument
1629 	build_inv_irt(&cmd, devid);  in iommu_flush_irt()
1636 	u32 devid;  in amd_iommu_flush_irt_all()  local
1642 	for (devid = 0; devid <= last_bdf; devid++)  in amd_iommu_flush_irt_all()
1643 		iommu_flush_irt(iommu, devid);  in amd_iommu_flush_irt_all()
1669 	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address,  in device_flush_iotlb()
1700 		ret = iommu_flush_dte(iommu, dev_data->devid);  in device_flush_dte()
1705 	alias = pci_seg->alias_table[dev_data->devid];  in device_flush_dte()
1706 	if (alias != dev_data->devid) {  in device_flush_dte()
1893 	build_complete_ppr(&cmd, dev_data->devid, pasid, status, tag, gn);  in __amd_iommu_complete_ppr()
2207 	struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];  in set_dte_entry()
3078 		dte = &get_dev_table(iommu)[dev_data->devid];  in amd_iommu_set_dirty_tracking()
3106 	int devid, sbdf;  in amd_iommu_get_resv_regions()  local
3112 	devid = PCI_SBDF_TO_DEVID(sbdf);  in amd_iommu_get_resv_regions()
3120 		if (devid < entry->devid_start || devid > entry->devid_end)  in amd_iommu_get_resv_regions()
3229 static int iommu_flush_dev_irt(struct pci_dev *unused, u16 devid, void *data)  in iommu_flush_dev_irt()  argument
3235 	build_inv_irt(&cmd, devid);  in iommu_flush_dev_irt()
3240 static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)  in iommu_flush_irt_and_complete()  argument
3247 	struct iommu_dev_data *dev_data = search_dev_data(iommu, devid);  in iommu_flush_irt_and_complete()
3262 		ret = iommu_flush_dev_irt(NULL, devid, iommu);  in iommu_flush_irt_and_complete()
3285 static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,  in set_dte_irq_entry()  argument
3289 	struct dev_table_entry *dte = &get_dev_table(iommu)[devid];  in set_dte_irq_entry()
3290 	struct iommu_dev_data *dev_data = search_dev_data(iommu, devid);  in set_dte_irq_entry()
3307 static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid)  in get_irq_table()  argument
3312 	if (WARN_ONCE(!pci_seg->rlookup_table[devid],  in get_irq_table()
3313 		      "%s: no iommu for devid %x:%x\n",  in get_irq_table()
3314 		      __func__, pci_seg->id, devid))  in get_irq_table()
3317 	table = pci_seg->irq_lookup_table[devid];  in get_irq_table()
3318 	if (WARN_ONCE(!table, "%s: no table for devid %x:%x\n",  in get_irq_table()
3319 		      __func__, pci_seg->id, devid))  in get_irq_table()
3344 static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid,  in set_remap_table_entry()  argument
3349 	pci_seg->irq_lookup_table[devid] = table;  in set_remap_table_entry()
3350 	set_dte_irq_entry(iommu, devid, table);  in set_remap_table_entry()
3351 	iommu_flush_dte(iommu, devid);  in set_remap_table_entry()
3381 					       u16 devid, struct pci_dev *pdev,  in alloc_irq_table()  argument
3394 	table = pci_seg->irq_lookup_table[devid];  in alloc_irq_table()
3398 	alias = pci_seg->alias_table[devid];  in alloc_irq_table()
3401 		set_remap_table_entry(iommu, devid, table);  in alloc_irq_table()
3413 	table = pci_seg->irq_lookup_table[devid];  in alloc_irq_table()
3419 		set_remap_table_entry(iommu, devid, table);  in alloc_irq_table()
3430 		set_remap_table_entry(iommu, devid, table);  in alloc_irq_table()
3432 	if (devid != alias)  in alloc_irq_table()
3448 static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,  in alloc_irq_index()  argument
3456 	table = alloc_irq_table(iommu, devid, pdev, max_irqs);  in alloc_irq_index()
3495 static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,  in __modify_irte_ga()  argument
3503 	table = get_irq_table(iommu, devid);  in __modify_irte_ga()
3526 static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,  in modify_irte_ga()  argument
3531 	ret = __modify_irte_ga(iommu, devid, index, irte);  in modify_irte_ga()
3535 	iommu_flush_irt_and_complete(iommu, devid);  in modify_irte_ga()
3541 		       u16 devid, int index, union irte *irte)  in modify_irte()  argument
3546 	table = get_irq_table(iommu, devid);  in modify_irte()
3554 	iommu_flush_irt_and_complete(iommu, devid);  in modify_irte()
3559 static void free_irte(struct amd_iommu *iommu, u16 devid, int index)  in free_irte()  argument
3564 	table = get_irq_table(iommu, devid);  in free_irte()
3572 	iommu_flush_irt_and_complete(iommu, devid);  in free_irte()
3577 			 u8 vector, u32 dest_apicid, int devid)  in irte_prepare()  argument
3591 			    u8 vector, u32 dest_apicid, int devid)  in irte_ga_prepare()  argument
3605 static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)  in irte_activate()  argument
3610 	modify_irte(iommu, devid, index, irte);  in irte_activate()
3613 static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)  in irte_ga_activate()  argument
3618 	modify_irte_ga(iommu, devid, index, irte);  in irte_ga_activate()
3621 static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)  in irte_deactivate()  argument
3626 	modify_irte(iommu, devid, index, irte);  in irte_deactivate()
3629 static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)  in irte_ga_deactivate()  argument
3634 	modify_irte_ga(iommu, devid, index, irte);  in irte_ga_deactivate()
3637 static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,  in irte_set_affinity()  argument
3644 	modify_irte(iommu, devid, index, irte);  in irte_set_affinity()
3647 static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,  in irte_ga_set_affinity()  argument
3658 		modify_irte_ga(iommu, devid, index, irte);  in irte_ga_set_affinity()
3712 		return get_ioapic_devid(info->devid);  in get_devid()
3714 		return get_hpet_devid(info->devid);  in get_devid()
3751 				       int devid, int index, int sub_handle)  in irq_remapping_prepare_irte()  argument
3759 	data->irq_2_irte.devid = devid;  in irq_remapping_prepare_irte()
3763 				 irq_cfg->dest_apicid, devid);  in irq_remapping_prepare_irte()
3809 	int i, ret, devid, seg, sbdf;  in irq_remapping_alloc()  local
3822 	devid = PCI_SBDF_TO_DEVID(sbdf);  in irq_remapping_alloc()
3823 	iommu = __rlookup_amd_iommu(seg, devid);  in irq_remapping_alloc()
3827 	dev_data = search_dev_data(iommu, devid);  in irq_remapping_alloc()
3837 		table = alloc_irq_table(iommu, devid, NULL, max_irqs);  in irq_remapping_alloc()
3857 		index = alloc_irq_index(iommu, devid, nr_irqs, align,  in irq_remapping_alloc()
3861 		index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL,  in irq_remapping_alloc()
3894 		irq_data->hwirq = (devid << 16) + i;  in irq_remapping_alloc()
3897 		irq_remapping_prepare_irte(data, cfg, info, devid, index, i);  in irq_remapping_alloc()
3909 		free_irte(iommu, devid, index + i);  in irq_remapping_alloc()
3928 			free_irte(data->iommu, irte_info->devid, irte_info->index);  in irq_remapping_free()
3952 	iommu->irte_ops->activate(iommu, data->entry, irte_info->devid,  in irq_remapping_activate()
3966 		iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid,  in irq_remapping_deactivate()
3974 	int devid = -1;  in irq_remapping_select()  local
3977 		devid = get_ioapic_devid(fwspec->param[0]);  in irq_remapping_select()
3979 		devid = get_hpet_devid(fwspec->param[0]);  in irq_remapping_select()
3981 	if (devid < 0)  in irq_remapping_select()
3983 	iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff));  in irq_remapping_select()
4045 	return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,  in amd_iommu_update_ga()
4075 	return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,  in amd_iommu_activate_guest_mode()
4107 	return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,  in amd_iommu_deactivate_guest_mode()
4126 	dev_data = search_dev_data(ir_data->iommu, irte_info->devid);  in amd_ir_set_vcpu_affinity()
4166 	iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid,  in amd_ir_update_irte()