1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2024 Advanced Micro Devices, Inc. 4 */ 5 6 #define pr_fmt(fmt) "AMD-Vi: " fmt 7 #define dev_fmt(fmt) pr_fmt(fmt) 8 9 #include <linux/iommu.h> 10 #include <linux/mm_types.h> 11 12 #include "amd_iommu.h" 13 14 static inline bool is_pasid_enabled(struct iommu_dev_data *dev_data) 15 { 16 if (dev_data->pasid_enabled && dev_data->max_pasids && 17 dev_data->gcr3_info.gcr3_tbl != NULL) 18 return true; 19 20 return false; 21 } 22 23 static inline bool is_pasid_valid(struct iommu_dev_data *dev_data, 24 ioasid_t pasid) 25 { 26 if (pasid > 0 && pasid < dev_data->max_pasids) 27 return true; 28 29 return false; 30 } 31 32 static void remove_dev_pasid(struct pdom_dev_data *pdom_dev_data) 33 { 34 /* Update GCR3 table and flush IOTLB */ 35 amd_iommu_clear_gcr3(pdom_dev_data->dev_data, pdom_dev_data->pasid); 36 37 list_del(&pdom_dev_data->list); 38 kfree(pdom_dev_data); 39 } 40 41 /* Clear PASID from device GCR3 table and remove pdom_dev_data from list */ 42 static void remove_pdom_dev_pasid(struct protection_domain *pdom, 43 struct device *dev, ioasid_t pasid) 44 { 45 struct pdom_dev_data *pdom_dev_data; 46 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 47 48 lockdep_assert_held(&pdom->lock); 49 50 for_each_pdom_dev_data(pdom_dev_data, pdom) { 51 if (pdom_dev_data->dev_data == dev_data && 52 pdom_dev_data->pasid == pasid) { 53 remove_dev_pasid(pdom_dev_data); 54 break; 55 } 56 } 57 } 58 59 static void sva_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, 60 struct mm_struct *mm, 61 unsigned long start, unsigned long end) 62 { 63 struct pdom_dev_data *pdom_dev_data; 64 struct protection_domain *sva_pdom; 65 unsigned long flags; 66 67 sva_pdom = container_of(mn, struct protection_domain, mn); 68 69 spin_lock_irqsave(&sva_pdom->lock, flags); 70 71 for_each_pdom_dev_data(pdom_dev_data, sva_pdom) { 72 amd_iommu_dev_flush_pasid_pages(pdom_dev_data->dev_data, 73 pdom_dev_data->pasid, 74 start, end - start); 75 } 76 77 spin_unlock_irqrestore(&sva_pdom->lock, flags); 78 } 79 80 static void sva_mn_release(struct mmu_notifier *mn, struct mm_struct *mm) 81 { 82 struct pdom_dev_data *pdom_dev_data, *next; 83 struct protection_domain *sva_pdom; 84 unsigned long flags; 85 86 sva_pdom = container_of(mn, struct protection_domain, mn); 87 88 spin_lock_irqsave(&sva_pdom->lock, flags); 89 90 /* Assume dev_data_list contains same PASID with different devices */ 91 for_each_pdom_dev_data_safe(pdom_dev_data, next, sva_pdom) 92 remove_dev_pasid(pdom_dev_data); 93 94 spin_unlock_irqrestore(&sva_pdom->lock, flags); 95 } 96 97 static const struct mmu_notifier_ops sva_mn = { 98 .arch_invalidate_secondary_tlbs = sva_arch_invalidate_secondary_tlbs, 99 .release = sva_mn_release, 100 }; 101 102 int iommu_sva_set_dev_pasid(struct iommu_domain *domain, 103 struct device *dev, ioasid_t pasid, 104 struct iommu_domain *old) 105 { 106 struct pdom_dev_data *pdom_dev_data; 107 struct protection_domain *sva_pdom = to_pdomain(domain); 108 struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev); 109 unsigned long flags; 110 int ret = -EINVAL; 111 112 if (old) 113 return -EOPNOTSUPP; 114 115 /* PASID zero is used for requests from the I/O device without PASID */ 116 if (!is_pasid_valid(dev_data, pasid)) 117 return ret; 118 119 /* Make sure PASID is enabled */ 120 if (!is_pasid_enabled(dev_data)) 121 return ret; 122 123 /* Add PASID to protection domain pasid list */ 124 pdom_dev_data = kzalloc(sizeof(*pdom_dev_data), GFP_KERNEL); 125 if (pdom_dev_data == NULL) 126 return ret; 127 128 pdom_dev_data->pasid = pasid; 129 pdom_dev_data->dev_data = dev_data; 130 131 spin_lock_irqsave(&sva_pdom->lock, flags); 132 133 /* Setup GCR3 table */ 134 ret = amd_iommu_set_gcr3(dev_data, pasid, 135 iommu_virt_to_phys(domain->mm->pgd)); 136 if (ret) { 137 kfree(pdom_dev_data); 138 goto out_unlock; 139 } 140 141 list_add(&pdom_dev_data->list, &sva_pdom->dev_data_list); 142 143 out_unlock: 144 spin_unlock_irqrestore(&sva_pdom->lock, flags); 145 return ret; 146 } 147 148 void amd_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid, 149 struct iommu_domain *domain) 150 { 151 struct protection_domain *sva_pdom; 152 unsigned long flags; 153 154 if (!is_pasid_valid(dev_iommu_priv_get(dev), pasid)) 155 return; 156 157 sva_pdom = to_pdomain(domain); 158 159 spin_lock_irqsave(&sva_pdom->lock, flags); 160 161 /* Remove PASID from dev_data_list */ 162 remove_pdom_dev_pasid(sva_pdom, dev, pasid); 163 164 spin_unlock_irqrestore(&sva_pdom->lock, flags); 165 } 166 167 static void iommu_sva_domain_free(struct iommu_domain *domain) 168 { 169 struct protection_domain *sva_pdom = to_pdomain(domain); 170 171 if (sva_pdom->mn.ops) 172 mmu_notifier_unregister(&sva_pdom->mn, domain->mm); 173 174 amd_iommu_domain_free(domain); 175 } 176 177 static const struct iommu_domain_ops amd_sva_domain_ops = { 178 .set_dev_pasid = iommu_sva_set_dev_pasid, 179 .free = iommu_sva_domain_free 180 }; 181 182 struct iommu_domain *amd_iommu_domain_alloc_sva(struct device *dev, 183 struct mm_struct *mm) 184 { 185 struct protection_domain *pdom; 186 int ret; 187 188 pdom = protection_domain_alloc(IOMMU_DOMAIN_SVA, dev_to_node(dev)); 189 if (!pdom) 190 return ERR_PTR(-ENOMEM); 191 192 pdom->domain.ops = &amd_sva_domain_ops; 193 pdom->mn.ops = &sva_mn; 194 195 ret = mmu_notifier_register(&pdom->mn, mm); 196 if (ret) { 197 protection_domain_free(pdom); 198 return ERR_PTR(ret); 199 } 200 201 return &pdom->domain; 202 } 203