1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Helpers for IOMMU drivers implementing SVA 4 */ 5 #include <linux/mmu_context.h> 6 #include <linux/mmu_notifier.h> 7 #include <linux/mutex.h> 8 #include <linux/sched/mm.h> 9 #include <linux/iommu.h> 10 11 #include "iommu-priv.h" 12 13 static DEFINE_MUTEX(iommu_sva_lock); 14 static bool iommu_sva_present; 15 static LIST_HEAD(iommu_sva_mms); 16 static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, 17 struct mm_struct *mm); 18 19 /* Allocate a PASID for the mm within range (inclusive) */ 20 static struct iommu_mm_data *iommu_alloc_mm_data(struct mm_struct *mm, struct device *dev) 21 { 22 struct iommu_mm_data *iommu_mm; 23 ioasid_t pasid; 24 25 lockdep_assert_held(&iommu_sva_lock); 26 27 if (!arch_pgtable_dma_compat(mm)) 28 return ERR_PTR(-EBUSY); 29 30 iommu_mm = mm->iommu_mm; 31 /* Is a PASID already associated with this mm? */ 32 if (iommu_mm) { 33 if (iommu_mm->pasid >= dev->iommu->max_pasids) 34 return ERR_PTR(-EOVERFLOW); 35 return iommu_mm; 36 } 37 38 iommu_mm = kzalloc(sizeof(struct iommu_mm_data), GFP_KERNEL); 39 if (!iommu_mm) 40 return ERR_PTR(-ENOMEM); 41 42 pasid = iommu_alloc_global_pasid(dev); 43 if (pasid == IOMMU_PASID_INVALID) { 44 kfree(iommu_mm); 45 return ERR_PTR(-ENOSPC); 46 } 47 iommu_mm->pasid = pasid; 48 iommu_mm->mm = mm; 49 INIT_LIST_HEAD(&iommu_mm->sva_domains); 50 /* 51 * Make sure the write to mm->iommu_mm is not reordered in front of 52 * initialization to iommu_mm fields. If it does, readers may see a 53 * valid iommu_mm with uninitialized values. 54 */ 55 smp_store_release(&mm->iommu_mm, iommu_mm); 56 return iommu_mm; 57 } 58 59 /** 60 * iommu_sva_bind_device() - Bind a process address space to a device 61 * @dev: the device 62 * @mm: the mm to bind, caller must hold a reference to mm_users 63 * 64 * Create a bond between device and address space, allowing the device to 65 * access the mm using the PASID returned by iommu_sva_get_pasid(). If a 66 * bond already exists between @device and @mm, an additional internal 67 * reference is taken. Caller must call iommu_sva_unbind_device() 68 * to release each reference. 69 * 70 * On error, returns an ERR_PTR value. 71 */ 72 struct iommu_sva *iommu_sva_bind_device(struct device *dev, struct mm_struct *mm) 73 { 74 struct iommu_group *group = dev->iommu_group; 75 struct iommu_attach_handle *attach_handle; 76 struct iommu_mm_data *iommu_mm; 77 struct iommu_domain *domain; 78 struct iommu_sva *handle; 79 int ret; 80 81 if (!group) 82 return ERR_PTR(-ENODEV); 83 84 mutex_lock(&iommu_sva_lock); 85 86 /* Allocate mm->pasid if necessary. */ 87 iommu_mm = iommu_alloc_mm_data(mm, dev); 88 if (IS_ERR(iommu_mm)) { 89 ret = PTR_ERR(iommu_mm); 90 goto out_unlock; 91 } 92 93 /* A bond already exists, just take a reference`. */ 94 attach_handle = iommu_attach_handle_get(group, iommu_mm->pasid, IOMMU_DOMAIN_SVA); 95 if (!IS_ERR(attach_handle)) { 96 handle = container_of(attach_handle, struct iommu_sva, handle); 97 if (attach_handle->domain->mm != mm) { 98 ret = -EBUSY; 99 goto out_unlock; 100 } 101 refcount_inc(&handle->users); 102 mutex_unlock(&iommu_sva_lock); 103 return handle; 104 } 105 106 if (PTR_ERR(attach_handle) != -ENOENT) { 107 ret = PTR_ERR(attach_handle); 108 goto out_unlock; 109 } 110 111 handle = kzalloc(sizeof(*handle), GFP_KERNEL); 112 if (!handle) { 113 ret = -ENOMEM; 114 goto out_unlock; 115 } 116 117 /* Search for an existing domain. */ 118 list_for_each_entry(domain, &mm->iommu_mm->sva_domains, next) { 119 ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid, 120 &handle->handle); 121 if (!ret) { 122 domain->users++; 123 goto out; 124 } 125 } 126 127 /* Allocate a new domain and set it on device pasid. */ 128 domain = iommu_sva_domain_alloc(dev, mm); 129 if (IS_ERR(domain)) { 130 ret = PTR_ERR(domain); 131 goto out_free_handle; 132 } 133 134 ret = iommu_attach_device_pasid(domain, dev, iommu_mm->pasid, 135 &handle->handle); 136 if (ret) 137 goto out_free_domain; 138 domain->users = 1; 139 140 if (list_empty(&iommu_mm->sva_domains)) { 141 if (list_empty(&iommu_sva_mms)) 142 iommu_sva_present = true; 143 list_add(&iommu_mm->mm_list_elm, &iommu_sva_mms); 144 } 145 list_add(&domain->next, &iommu_mm->sva_domains); 146 out: 147 refcount_set(&handle->users, 1); 148 mutex_unlock(&iommu_sva_lock); 149 handle->dev = dev; 150 return handle; 151 152 out_free_domain: 153 iommu_domain_free(domain); 154 out_free_handle: 155 kfree(handle); 156 out_unlock: 157 mutex_unlock(&iommu_sva_lock); 158 return ERR_PTR(ret); 159 } 160 EXPORT_SYMBOL_GPL(iommu_sva_bind_device); 161 162 /** 163 * iommu_sva_unbind_device() - Remove a bond created with iommu_sva_bind_device 164 * @handle: the handle returned by iommu_sva_bind_device() 165 * 166 * Put reference to a bond between device and address space. The device should 167 * not be issuing any more transaction for this PASID. All outstanding page 168 * requests for this PASID must have been flushed to the IOMMU. 169 */ 170 void iommu_sva_unbind_device(struct iommu_sva *handle) 171 { 172 struct iommu_domain *domain = handle->handle.domain; 173 struct iommu_mm_data *iommu_mm = domain->mm->iommu_mm; 174 struct device *dev = handle->dev; 175 176 mutex_lock(&iommu_sva_lock); 177 if (!refcount_dec_and_test(&handle->users)) { 178 mutex_unlock(&iommu_sva_lock); 179 return; 180 } 181 182 iommu_detach_device_pasid(domain, dev, iommu_mm->pasid); 183 if (--domain->users == 0) { 184 list_del(&domain->next); 185 iommu_domain_free(domain); 186 } 187 188 if (list_empty(&iommu_mm->sva_domains)) { 189 list_del(&iommu_mm->mm_list_elm); 190 if (list_empty(&iommu_sva_mms)) 191 iommu_sva_present = false; 192 } 193 194 mutex_unlock(&iommu_sva_lock); 195 kfree(handle); 196 } 197 EXPORT_SYMBOL_GPL(iommu_sva_unbind_device); 198 199 u32 iommu_sva_get_pasid(struct iommu_sva *handle) 200 { 201 struct iommu_domain *domain = handle->handle.domain; 202 203 return mm_get_enqcmd_pasid(domain->mm); 204 } 205 EXPORT_SYMBOL_GPL(iommu_sva_get_pasid); 206 207 void mm_pasid_drop(struct mm_struct *mm) 208 { 209 struct iommu_mm_data *iommu_mm = mm->iommu_mm; 210 211 if (!iommu_mm) 212 return; 213 214 iommu_free_global_pasid(iommu_mm->pasid); 215 kfree(iommu_mm); 216 } 217 218 /* 219 * I/O page fault handler for SVA 220 */ 221 static enum iommu_page_response_code 222 iommu_sva_handle_mm(struct iommu_fault *fault, struct mm_struct *mm) 223 { 224 vm_fault_t ret; 225 struct vm_area_struct *vma; 226 unsigned int access_flags = 0; 227 unsigned int fault_flags = FAULT_FLAG_REMOTE; 228 struct iommu_fault_page_request *prm = &fault->prm; 229 enum iommu_page_response_code status = IOMMU_PAGE_RESP_INVALID; 230 231 if (!(prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID)) 232 return status; 233 234 if (!mmget_not_zero(mm)) 235 return status; 236 237 mmap_read_lock(mm); 238 239 vma = vma_lookup(mm, prm->addr); 240 if (!vma) 241 /* Unmapped area */ 242 goto out_put_mm; 243 244 if (prm->perm & IOMMU_FAULT_PERM_READ) 245 access_flags |= VM_READ; 246 247 if (prm->perm & IOMMU_FAULT_PERM_WRITE) { 248 access_flags |= VM_WRITE; 249 fault_flags |= FAULT_FLAG_WRITE; 250 } 251 252 if (prm->perm & IOMMU_FAULT_PERM_EXEC) { 253 access_flags |= VM_EXEC; 254 fault_flags |= FAULT_FLAG_INSTRUCTION; 255 } 256 257 if (!(prm->perm & IOMMU_FAULT_PERM_PRIV)) 258 fault_flags |= FAULT_FLAG_USER; 259 260 if (access_flags & ~vma->vm_flags) 261 /* Access fault */ 262 goto out_put_mm; 263 264 ret = handle_mm_fault(vma, prm->addr, fault_flags, NULL); 265 status = ret & VM_FAULT_ERROR ? IOMMU_PAGE_RESP_INVALID : 266 IOMMU_PAGE_RESP_SUCCESS; 267 268 out_put_mm: 269 mmap_read_unlock(mm); 270 mmput(mm); 271 272 return status; 273 } 274 275 static void iommu_sva_handle_iopf(struct work_struct *work) 276 { 277 struct iopf_fault *iopf; 278 struct iopf_group *group; 279 enum iommu_page_response_code status = IOMMU_PAGE_RESP_SUCCESS; 280 281 group = container_of(work, struct iopf_group, work); 282 list_for_each_entry(iopf, &group->faults, list) { 283 /* 284 * For the moment, errors are sticky: don't handle subsequent 285 * faults in the group if there is an error. 286 */ 287 if (status != IOMMU_PAGE_RESP_SUCCESS) 288 break; 289 290 status = iommu_sva_handle_mm(&iopf->fault, 291 group->attach_handle->domain->mm); 292 } 293 294 iopf_group_response(group, status); 295 iopf_free_group(group); 296 } 297 298 static int iommu_sva_iopf_handler(struct iopf_group *group) 299 { 300 struct iommu_fault_param *fault_param = group->fault_param; 301 302 INIT_WORK(&group->work, iommu_sva_handle_iopf); 303 if (!queue_work(fault_param->queue->wq, &group->work)) 304 return -EBUSY; 305 306 return 0; 307 } 308 309 static struct iommu_domain *iommu_sva_domain_alloc(struct device *dev, 310 struct mm_struct *mm) 311 { 312 const struct iommu_ops *ops = dev_iommu_ops(dev); 313 struct iommu_domain *domain; 314 315 if (!ops->domain_alloc_sva) 316 return ERR_PTR(-EOPNOTSUPP); 317 318 domain = ops->domain_alloc_sva(dev, mm); 319 if (IS_ERR(domain)) 320 return domain; 321 322 domain->type = IOMMU_DOMAIN_SVA; 323 domain->cookie_type = IOMMU_COOKIE_SVA; 324 mmgrab(mm); 325 domain->mm = mm; 326 domain->owner = ops; 327 domain->iopf_handler = iommu_sva_iopf_handler; 328 329 return domain; 330 } 331 332 void iommu_sva_invalidate_kva_range(unsigned long start, unsigned long end) 333 { 334 struct iommu_mm_data *iommu_mm; 335 336 guard(mutex)(&iommu_sva_lock); 337 if (!iommu_sva_present) 338 return; 339 340 list_for_each_entry(iommu_mm, &iommu_sva_mms, mm_list_elm) 341 mmu_notifier_arch_invalidate_secondary_tlbs(iommu_mm->mm, start, end); 342 } 343