76 			  struct iommu_dev_data *dev_data,
79 static int device_flush_dte(struct iommu_dev_data *dev_data);
146 static void update_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data,  in update_dte256()  argument
151 	struct dev_table_entry *ptr = &dev_table[dev_data->devid];  in update_dte256()
153 	spin_lock_irqsave(&dev_data->dte_lock, flags);  in update_dte256()
159 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
164 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
172 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
181 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
194 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
199 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
209 	spin_unlock_irqrestore(&dev_data->dte_lock, flags);  in update_dte256()
213 			     struct iommu_dev_data *dev_data,  in amd_iommu_update_dte()  argument
216 	update_dte256(iommu, dev_data, new);  in amd_iommu_update_dte()
217 	clone_aliases(iommu, dev_data->dev);  in amd_iommu_update_dte()
218 	device_flush_dte(dev_data);  in amd_iommu_update_dte()
222 static void get_dte256(struct amd_iommu *iommu, struct iommu_dev_data *dev_data,  in get_dte256()  argument
229 	ptr = &dev_table[dev_data->devid];  in get_dte256()
231 	spin_lock_irqsave(&dev_data->dte_lock, flags);  in get_dte256()
234 	spin_unlock_irqrestore(&dev_data->dte_lock, flags);  in get_dte256()
372 	struct iommu_dev_data *dev_data;  in alloc_dev_data()  local
375 	dev_data = kzalloc_obj(*dev_data);  in alloc_dev_data()
376 	if (!dev_data)  in alloc_dev_data()
379 	mutex_init(&dev_data->mutex);  in alloc_dev_data()
380 	spin_lock_init(&dev_data->dte_lock);  in alloc_dev_data()
381 	dev_data->devid = devid;  in alloc_dev_data()
382 	ratelimit_default_init(&dev_data->rs);  in alloc_dev_data()
384 	llist_add(&dev_data->dev_data_list, &pci_seg->dev_data_list);  in alloc_dev_data()
385 	return dev_data;  in alloc_dev_data()
390 	struct iommu_dev_data *dev_data;  in search_dev_data()  local
398 	llist_for_each_entry(dev_data, node, dev_data_list) {  in search_dev_data()
399 		if (dev_data->devid == devid)  in search_dev_data()
400 			return dev_data;  in search_dev_data()
410 	struct iommu_dev_data *dev_data, *alias_data;  in clone_alias()  local
422 	dev_data = dev_iommu_priv_get(&pdev->dev);  in clone_alias()
423 	if (!dev_data) {  in clone_alias()
428 	get_dte256(iommu, dev_data, &new);  in clone_alias()
486 	struct iommu_dev_data *dev_data;  in find_dev_data()  local
488 	dev_data = search_dev_data(iommu, devid);  in find_dev_data()
490 	if (dev_data == NULL) {  in find_dev_data()
491 		dev_data = alloc_dev_data(iommu, devid);  in find_dev_data()
492 		if (!dev_data)  in find_dev_data()
496 			dev_data->defer_attach = true;  in find_dev_data()
499 	return dev_data;  in find_dev_data()
527 static inline bool pdev_pasid_supported(struct iommu_dev_data *dev_data)  in pdev_pasid_supported()  argument
529 	return (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP);  in pdev_pasid_supported()
559 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);  in pdev_enable_cap_ats()  local
562 	if (dev_data->ats_enabled)  in pdev_enable_cap_ats()
566 	    (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) {  in pdev_enable_cap_ats()
569 			dev_data->ats_enabled = 1;  in pdev_enable_cap_ats()
570 			dev_data->ats_qdep    = pci_ats_queue_depth(pdev);  in pdev_enable_cap_ats()
579 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);  in pdev_disable_cap_ats()  local
581 	if (dev_data->ats_enabled) {  in pdev_disable_cap_ats()
583 		dev_data->ats_enabled = 0;  in pdev_disable_cap_ats()
589 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);  in pdev_enable_cap_pri()  local
592 	if (dev_data->pri_enabled)  in pdev_enable_cap_pri()
595 	if (!dev_data->ats_enabled)  in pdev_enable_cap_pri()
598 	if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) {  in pdev_enable_cap_pri()
604 			dev_data->pri_enabled = 1;  in pdev_enable_cap_pri()
605 			dev_data->pri_tlp     = pci_prg_resp_pasid_required(pdev);  in pdev_enable_cap_pri()
616 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);  in pdev_disable_cap_pri()  local
618 	if (dev_data->pri_enabled) {  in pdev_disable_cap_pri()
620 		dev_data->pri_enabled = 0;  in pdev_disable_cap_pri()
626 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);  in pdev_enable_cap_pasid()  local
629 	if (dev_data->pasid_enabled)  in pdev_enable_cap_pasid()
632 	if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) {  in pdev_enable_cap_pasid()
636 			dev_data->pasid_enabled = 1;  in pdev_enable_cap_pasid()
644 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev);  in pdev_disable_cap_pasid()  local
646 	if (dev_data->pasid_enabled) {  in pdev_disable_cap_pasid()
648 		dev_data->pasid_enabled = 0;  in pdev_disable_cap_pasid()
698 	struct iommu_dev_data *dev_data;  in iommu_init_device()  local
709 	dev_data = find_dev_data(iommu, devid);  in iommu_init_device()
710 	if (!dev_data)  in iommu_init_device()
713 	dev_data->dev = dev;  in iommu_init_device()
719 	dev_iommu_priv_set(dev, dev_data);  in iommu_init_device()
730 		dev_data->flags = pdev_get_caps(to_pci_dev(dev));  in iommu_init_device()
764 	struct iommu_dev_data *dev_data = find_dev_data(iommu, devid);  in dump_dte_entry()  local
766 	get_dte256(iommu, dev_data, &dte);  in dump_dte_entry()
783 	struct iommu_dev_data *dev_data = NULL;  in amd_iommu_report_rmp_hw_error()  local
796 		dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_report_rmp_hw_error()
798 	if (dev_data) {  in amd_iommu_report_rmp_hw_error()
799 		if (__ratelimit(&dev_data->rs)) {  in amd_iommu_report_rmp_hw_error()
815 	struct iommu_dev_data *dev_data = NULL;  in amd_iommu_report_rmp_fault()  local
829 		dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_report_rmp_fault()
831 	if (dev_data) {  in amd_iommu_report_rmp_fault()
832 		if (__ratelimit(&dev_data->rs)) {  in amd_iommu_report_rmp_fault()
856 	struct iommu_dev_data *dev_data = NULL;  in amd_iommu_report_page_fault()  local
862 		dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_report_page_fault()
864 	if (dev_data) {  in amd_iommu_report_page_fault()
872 			if (dev_data->domain == NULL) {  in amd_iommu_report_page_fault()
880 			if (!report_iommu_fault(&dev_data->domain->domain,  in amd_iommu_report_page_fault()
888 		if (__ratelimit(&dev_data->rs)) {  in amd_iommu_report_page_fault()
1620 static int device_flush_iotlb(struct iommu_dev_data *dev_data, u64 address,  in device_flush_iotlb()  argument
1623 	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);  in device_flush_iotlb()
1625 	int qdep = dev_data->ats_qdep;  in device_flush_iotlb()
1627 	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address,  in device_flush_iotlb()
1643 static int device_flush_dte(struct iommu_dev_data *dev_data)  in device_flush_dte()  argument
1645 	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);  in device_flush_dte()
1651 	if (dev_is_pci(dev_data->dev))  in device_flush_dte()
1652 		pdev = to_pci_dev(dev_data->dev);  in device_flush_dte()
1658 		ret = iommu_flush_dte(iommu, dev_data->devid);  in device_flush_dte()
1663 	alias = pci_seg->alias_table[dev_data->devid];  in device_flush_dte()
1664 	if (alias != dev_data->devid) {  in device_flush_dte()
1670 	if (dev_data->ats_enabled) {  in device_flush_dte()
1672 		ret = device_flush_iotlb(dev_data, 0, ~0UL,  in device_flush_dte()
1682 	struct iommu_dev_data *dev_data;  in domain_flush_pages_v2()  local
1687 	list_for_each_entry(dev_data, &pdom->dev_list, list) {  in domain_flush_pages_v2()
1688 		struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);  in domain_flush_pages_v2()
1689 		u16 domid = dev_data->gcr3_info.domid;  in domain_flush_pages_v2()
1742 	struct iommu_dev_data *dev_data;  in __domain_flush_pages()  local
1756 	list_for_each_entry(dev_data, &domain->dev_list, list) {  in __domain_flush_pages()
1758 		if (!dev_data->ats_enabled)  in __domain_flush_pages()
1761 		ret |= device_flush_iotlb(dev_data, address, size, pasid, gn);  in __domain_flush_pages()
1826 void amd_iommu_dev_flush_pasid_pages(struct iommu_dev_data *dev_data,  in amd_iommu_dev_flush_pasid_pages()  argument
1830 	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);  in amd_iommu_dev_flush_pasid_pages()
1833 			      dev_data->gcr3_info.domid, pasid, true);  in amd_iommu_dev_flush_pasid_pages()
1836 	if (dev_data->ats_enabled)  in amd_iommu_dev_flush_pasid_pages()
1837 		device_flush_iotlb(dev_data, address, size, pasid, true);  in amd_iommu_dev_flush_pasid_pages()
1842 static void dev_flush_pasid_all(struct iommu_dev_data *dev_data,  in dev_flush_pasid_all()  argument
1845 	amd_iommu_dev_flush_pasid_pages(dev_data, pasid, 0,  in dev_flush_pasid_all()
1851 	struct iommu_dev_data *dev_data;  in amd_iommu_complete_ppr()  local
1855 	dev_data = dev_iommu_priv_get(dev);  in amd_iommu_complete_ppr()
1858 	build_complete_ppr(&cmd, dev_data->devid, pasid, status,  in amd_iommu_complete_ppr()
1859 			   tag, dev_data->pri_tlp);  in amd_iommu_complete_ppr()
2022 static int update_gcr3(struct iommu_dev_data *dev_data,  in update_gcr3()  argument
2025 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;  in update_gcr3()
2037 	dev_flush_pasid_all(dev_data, pasid);  in update_gcr3()
2041 int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid,  in amd_iommu_set_gcr3()  argument
2044 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;  in amd_iommu_set_gcr3()
2047 	iommu_group_mutex_assert(dev_data->dev);  in amd_iommu_set_gcr3()
2049 	ret = update_gcr3(dev_data, pasid, gcr3, true);  in amd_iommu_set_gcr3()
2057 int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid)  in amd_iommu_clear_gcr3()  argument
2059 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;  in amd_iommu_clear_gcr3()
2062 	iommu_group_mutex_assert(dev_data->dev);  in amd_iommu_clear_gcr3()
2064 	ret = update_gcr3(dev_data, pasid, 0, false);  in amd_iommu_clear_gcr3()
2076 static void set_dte_gcr3_table(struct iommu_dev_data *dev_data,  in set_dte_gcr3_table()  argument
2079 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;  in set_dte_gcr3_table()
2083 			(dev_data->ppr ? DTE_FLAG_PPR : 0) |  in set_dte_gcr3_table()
2084 			(pdom_is_v2_pgtbl_mode(dev_data->domain) ?  DTE_FLAG_GIOV : 0) |  in set_dte_gcr3_table()
2090 	new->data[1] |= FIELD_PREP(DTE_DOMID_MASK, dev_data->gcr3_info.domid) |  in set_dte_gcr3_table()
2092 			(dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0) |  in set_dte_gcr3_table()
2102 void amd_iommu_set_dte_v1(struct iommu_dev_data *dev_data,  in amd_iommu_set_dte_v1()  argument
2117 			(dev_data->ats_enabled ? DTE_FLAG_IOTLB : 0);  in amd_iommu_set_dte_v1()
2120 static void set_dte_v1(struct iommu_dev_data *dev_data,  in set_dte_v1()  argument
2141 	amd_iommu_set_dte_v1(dev_data, domain, domid, &pt_info, new);  in set_dte_v1()
2144 static void set_dte_passthrough(struct iommu_dev_data *dev_data,  in set_dte_passthrough()  argument
2151 			(dev_data->ats_enabled) ? DTE_FLAG_IOTLB : 0;  in set_dte_passthrough()
2155 			  struct iommu_dev_data *dev_data,  in set_dte_entry()  argument
2160 	struct protection_domain *domain = dev_data->domain;  in set_dte_entry()
2161 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;  in set_dte_entry()
2162 	struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];  in set_dte_entry()
2164 	amd_iommu_make_clear_dte(dev_data, &new);  in set_dte_entry()
2168 		set_dte_gcr3_table(dev_data, &new);  in set_dte_entry()
2170 		set_dte_passthrough(dev_data, domain, &new);  in set_dte_entry()
2173 		set_dte_v1(dev_data, domain, domain->id, top_paddr, top_level, &new);  in set_dte_entry()
2177 	amd_iommu_update_dte(iommu, dev_data, &new);  in set_dte_entry()
2192 static void clear_dte_entry(struct amd_iommu *iommu, struct iommu_dev_data *dev_data)  in clear_dte_entry()  argument
2196 	amd_iommu_make_clear_dte(dev_data, &new);  in clear_dte_entry()
2197 	amd_iommu_update_dte(iommu, dev_data, &new);  in clear_dte_entry()
2201 static void dev_update_dte(struct iommu_dev_data *dev_data, bool set)  in dev_update_dte()  argument
2203 	struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev);  in dev_update_dte()
2206 		set_dte_entry(iommu, dev_data, 0, 0);  in dev_update_dte()
2208 		clear_dte_entry(iommu, dev_data);  in dev_update_dte()
2215 static int init_gcr3_table(struct iommu_dev_data *dev_data,  in init_gcr3_table()  argument
2218 	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);  in init_gcr3_table()
2219 	int max_pasids = dev_data->max_pasids;  in init_gcr3_table()
2227 	if (pdom_is_in_pt_mode(pdom) && !pdev_pasid_supported(dev_data))  in init_gcr3_table()
2234 	ret = setup_gcr3_table(&dev_data->gcr3_info, iommu,  in init_gcr3_table()
2244 	ret = update_gcr3(dev_data, 0, __sme_set(pt_info.gcr3_pt), true);  in init_gcr3_table()
2246 		free_gcr3_table(&dev_data->gcr3_info);  in init_gcr3_table()
2251 static void destroy_gcr3_table(struct iommu_dev_data *dev_data,  in destroy_gcr3_table()  argument
2254 	struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info;  in destroy_gcr3_table()
2257 		update_gcr3(dev_data, 0, 0, false);  in destroy_gcr3_table()
2332 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);  in attach_device()  local
2333 	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);  in attach_device()
2338 	mutex_lock(&dev_data->mutex);  in attach_device()
2340 	if (dev_data->domain != NULL) {  in attach_device()
2352 		ret = init_gcr3_table(dev_data, domain);  in attach_device()
2359 	pdev = dev_is_pci(dev_data->dev) ? to_pci_dev(dev_data->dev) : NULL;  in attach_device()
2368 		if (amd_iommu_iopf_add_device(iommu, dev_data))  in attach_device()
2375 	dev_data->domain = domain;  in attach_device()
2377 	list_add(&dev_data->list, &domain->dev_list);  in attach_device()
2381 	dev_update_dte(dev_data, true);  in attach_device()
2384 	mutex_unlock(&dev_data->mutex);  in attach_device()
2394 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);  in detach_device()  local
2395 	struct amd_iommu *iommu = get_amd_iommu_from_dev_data(dev_data);  in detach_device()
2396 	struct protection_domain *domain = dev_data->domain;  in detach_device()
2399 	mutex_lock(&dev_data->mutex);  in detach_device()
2407 	if (WARN_ON(!dev_data->domain))  in detach_device()
2411 	if (dev_data->ppr) {  in detach_device()
2413 		amd_iommu_iopf_remove_device(iommu, dev_data);  in detach_device()
2420 	dev_update_dte(dev_data, false);  in detach_device()
2425 	list_del(&dev_data->list);  in detach_device()
2430 		destroy_gcr3_table(dev_data, domain);  in detach_device()
2433 	dev_data->domain = NULL;  in detach_device()
2439 	mutex_unlock(&dev_data->mutex);  in detach_device()
2446 	struct iommu_dev_data *dev_data;  in amd_iommu_probe_device()  local
2478 	dev_data = dev_iommu_priv_get(dev);  in amd_iommu_probe_device()
2480 	    pdev_pasid_supported(dev_data)) {  in amd_iommu_probe_device()
2481 		dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids,  in amd_iommu_probe_device()
2495 		dev_data->max_irqs = MAX_IRQS_PER_TABLE_2K;  in amd_iommu_probe_device()
2497 		dev_data->max_irqs = MAX_IRQS_PER_TABLE_512;  in amd_iommu_probe_device()
2508 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);  in amd_iommu_release_device()  local
2510 	WARN_ON(dev_data->domain);  in amd_iommu_release_device()
2590 	struct iommu_dev_data *dev_data;  in amd_iommu_change_top()  local
2595 	list_for_each_entry(dev_data, &pdom->dev_list, list) {  in amd_iommu_change_top()
2596 		struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev);  in amd_iommu_change_top()
2599 		set_dte_entry(iommu, dev_data, top_paddr, top_level);  in amd_iommu_change_top()
2600 		clone_aliases(iommu, dev_data->dev);  in amd_iommu_change_top()
2603 	list_for_each_entry(dev_data, &pdom->dev_list, list)  in amd_iommu_change_top()
2604 		device_flush_dte(dev_data);  in amd_iommu_change_top()
2881 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);  in blocked_domain_attach_device()  local
2883 	if (dev_data->domain)  in blocked_domain_attach_device()
2887 	mutex_lock(&dev_data->mutex);  in blocked_domain_attach_device()
2888 	dev_update_dte(dev_data, false);  in blocked_domain_attach_device()
2889 	mutex_unlock(&dev_data->mutex);  in blocked_domain_attach_device()
2945 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);  in amd_iommu_attach_device()  local
2954 	if (dev_data->domain == domain)  in amd_iommu_attach_device()
2957 	dev_data->defer_attach = false;  in amd_iommu_attach_device()
2966 	if (dev_data->domain)  in amd_iommu_attach_device()
2974 			dev_data->use_vapic = 1;  in amd_iommu_attach_device()
2976 			dev_data->use_vapic = 0;  in amd_iommu_attach_device()
3013 	struct iommu_dev_data *dev_data;  in amd_iommu_set_dirty_tracking()  local
3025 	list_for_each_entry(dev_data, &pdomain->dev_list, list) {  in amd_iommu_set_dirty_tracking()
3026 		spin_lock(&dev_data->dte_lock);  in amd_iommu_set_dirty_tracking()
3027 		iommu = get_amd_iommu_from_dev_data(dev_data);  in amd_iommu_set_dirty_tracking()
3028 		dte = &get_dev_table(iommu)[dev_data->devid];  in amd_iommu_set_dirty_tracking()
3032 		spin_unlock(&dev_data->dte_lock);  in amd_iommu_set_dirty_tracking()
3035 		device_flush_dte(dev_data);  in amd_iommu_set_dirty_tracking()
3113 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);  in amd_iommu_is_attach_deferred()  local
3115 	return dev_data->defer_attach;  in amd_iommu_is_attach_deferred()
3120 	struct iommu_dev_data *dev_data;  in amd_iommu_def_domain_type()  local
3122 	dev_data = dev_iommu_priv_get(dev);  in amd_iommu_def_domain_type()
3123 	if (!dev_data)  in amd_iommu_def_domain_type()
3137 	if (pdev_pasid_supported(dev_data) &&  in amd_iommu_def_domain_type()
3213 static inline u8 iommu_get_int_tablen(struct iommu_dev_data *dev_data)  in iommu_get_int_tablen()  argument
3215 	if (dev_data && dev_data->max_irqs == MAX_IRQS_PER_TABLE_2K)  in iommu_get_int_tablen()
3225 	struct iommu_dev_data *dev_data = search_dev_data(iommu, devid);  in set_dte_irq_entry()  local
3227 	if (dev_data)  in set_dte_irq_entry()
3228 		spin_lock(&dev_data->dte_lock);  in set_dte_irq_entry()
3234 	new |= iommu_get_int_tablen(dev_data);  in set_dte_irq_entry()
3238 	if (dev_data)  in set_dte_irq_entry()
3239 		spin_unlock(&dev_data->dte_lock);  in set_dte_irq_entry()
3742 	struct iommu_dev_data *dev_data;  in irq_remapping_alloc()  local
3762 	dev_data = search_dev_data(iommu, devid);  in irq_remapping_alloc()
3763 	max_irqs = dev_data ? dev_data->max_irqs : MAX_IRQS_PER_TABLE_512;  in irq_remapping_alloc()
4056 	struct iommu_dev_data *dev_data;  in amd_ir_set_vcpu_affinity()  local
4064 	dev_data = search_dev_data(ir_data->iommu, irte_info->devid);  in amd_ir_set_vcpu_affinity()
4070 	if (!dev_data || !dev_data->use_vapic)  in amd_ir_set_vcpu_affinity()