1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (C) 2024 Intel Corporation 3 */ 4 #define pr_fmt(fmt) "iommufd: " fmt 5 6 #include <linux/anon_inodes.h> 7 #include <linux/file.h> 8 #include <linux/fs.h> 9 #include <linux/iommufd.h> 10 #include <linux/module.h> 11 #include <linux/mutex.h> 12 #include <linux/pci.h> 13 #include <linux/poll.h> 14 #include <uapi/linux/iommufd.h> 15 16 #include "../iommu-priv.h" 17 #include "iommufd_private.h" 18 19 static int iommufd_fault_iopf_enable(struct iommufd_device *idev) 20 { 21 struct device *dev = idev->dev; 22 int ret; 23 24 /* 25 * Once we turn on PCI/PRI support for VF, the response failure code 26 * should not be forwarded to the hardware due to PRI being a shared 27 * resource between PF and VFs. There is no coordination for this 28 * shared capability. This waits for a vPRI reset to recover. 29 */ 30 if (dev_is_pci(dev) && to_pci_dev(dev)->is_virtfn) 31 return -EINVAL; 32 33 mutex_lock(&idev->iopf_lock); 34 /* Device iopf has already been on. */ 35 if (++idev->iopf_enabled > 1) { 36 mutex_unlock(&idev->iopf_lock); 37 return 0; 38 } 39 40 ret = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_IOPF); 41 if (ret) 42 --idev->iopf_enabled; 43 mutex_unlock(&idev->iopf_lock); 44 45 return ret; 46 } 47 48 static void iommufd_fault_iopf_disable(struct iommufd_device *idev) 49 { 50 mutex_lock(&idev->iopf_lock); 51 if (!WARN_ON(idev->iopf_enabled == 0)) { 52 if (--idev->iopf_enabled == 0) 53 iommu_dev_disable_feature(idev->dev, IOMMU_DEV_FEAT_IOPF); 54 } 55 mutex_unlock(&idev->iopf_lock); 56 } 57 58 static int __fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt, 59 struct iommufd_device *idev) 60 { 61 struct iommufd_attach_handle *handle; 62 int ret; 63 64 handle = kzalloc(sizeof(*handle), GFP_KERNEL); 65 if (!handle) 66 return -ENOMEM; 67 68 handle->idev = idev; 69 ret = iommu_attach_group_handle(hwpt->domain, idev->igroup->group, 70 &handle->handle); 71 if (ret) 72 kfree(handle); 73 74 return ret; 75 } 76 77 int iommufd_fault_domain_attach_dev(struct iommufd_hw_pagetable *hwpt, 78 struct iommufd_device *idev) 79 { 80 int ret; 81 82 if (!hwpt->fault) 83 return -EINVAL; 84 85 ret = iommufd_fault_iopf_enable(idev); 86 if (ret) 87 return ret; 88 89 ret = __fault_domain_attach_dev(hwpt, idev); 90 if (ret) 91 iommufd_fault_iopf_disable(idev); 92 93 return ret; 94 } 95 96 static void iommufd_auto_response_faults(struct iommufd_hw_pagetable *hwpt, 97 struct iommufd_attach_handle *handle) 98 { 99 struct iommufd_fault *fault = hwpt->fault; 100 struct iopf_group *group, *next; 101 unsigned long index; 102 103 if (!fault) 104 return; 105 106 mutex_lock(&fault->mutex); 107 list_for_each_entry_safe(group, next, &fault->deliver, node) { 108 if (group->attach_handle != &handle->handle) 109 continue; 110 list_del(&group->node); 111 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID); 112 iopf_free_group(group); 113 } 114 115 xa_for_each(&fault->response, index, group) { 116 if (group->attach_handle != &handle->handle) 117 continue; 118 xa_erase(&fault->response, index); 119 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID); 120 iopf_free_group(group); 121 } 122 mutex_unlock(&fault->mutex); 123 } 124 125 static struct iommufd_attach_handle * 126 iommufd_device_get_attach_handle(struct iommufd_device *idev) 127 { 128 struct iommu_attach_handle *handle; 129 130 handle = iommu_attach_handle_get(idev->igroup->group, IOMMU_NO_PASID, 0); 131 if (IS_ERR(handle)) 132 return NULL; 133 134 return to_iommufd_handle(handle); 135 } 136 137 void iommufd_fault_domain_detach_dev(struct iommufd_hw_pagetable *hwpt, 138 struct iommufd_device *idev) 139 { 140 struct iommufd_attach_handle *handle; 141 142 handle = iommufd_device_get_attach_handle(idev); 143 iommu_detach_group_handle(hwpt->domain, idev->igroup->group); 144 iommufd_auto_response_faults(hwpt, handle); 145 iommufd_fault_iopf_disable(idev); 146 kfree(handle); 147 } 148 149 static int __fault_domain_replace_dev(struct iommufd_device *idev, 150 struct iommufd_hw_pagetable *hwpt, 151 struct iommufd_hw_pagetable *old) 152 { 153 struct iommufd_attach_handle *handle, *curr = NULL; 154 int ret; 155 156 if (old->fault) 157 curr = iommufd_device_get_attach_handle(idev); 158 159 if (hwpt->fault) { 160 handle = kzalloc(sizeof(*handle), GFP_KERNEL); 161 if (!handle) 162 return -ENOMEM; 163 164 handle->idev = idev; 165 ret = iommu_replace_group_handle(idev->igroup->group, 166 hwpt->domain, &handle->handle); 167 } else { 168 ret = iommu_replace_group_handle(idev->igroup->group, 169 hwpt->domain, NULL); 170 } 171 172 if (!ret && curr) { 173 iommufd_auto_response_faults(old, curr); 174 kfree(curr); 175 } 176 177 return ret; 178 } 179 180 int iommufd_fault_domain_replace_dev(struct iommufd_device *idev, 181 struct iommufd_hw_pagetable *hwpt, 182 struct iommufd_hw_pagetable *old) 183 { 184 bool iopf_off = !hwpt->fault && old->fault; 185 bool iopf_on = hwpt->fault && !old->fault; 186 int ret; 187 188 if (iopf_on) { 189 ret = iommufd_fault_iopf_enable(idev); 190 if (ret) 191 return ret; 192 } 193 194 ret = __fault_domain_replace_dev(idev, hwpt, old); 195 if (ret) { 196 if (iopf_on) 197 iommufd_fault_iopf_disable(idev); 198 return ret; 199 } 200 201 if (iopf_off) 202 iommufd_fault_iopf_disable(idev); 203 204 return 0; 205 } 206 207 void iommufd_fault_destroy(struct iommufd_object *obj) 208 { 209 struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj); 210 struct iopf_group *group, *next; 211 212 /* 213 * The iommufd object's reference count is zero at this point. 214 * We can be confident that no other threads are currently 215 * accessing this pointer. Therefore, acquiring the mutex here 216 * is unnecessary. 217 */ 218 list_for_each_entry_safe(group, next, &fault->deliver, node) { 219 list_del(&group->node); 220 iopf_group_response(group, IOMMU_PAGE_RESP_INVALID); 221 iopf_free_group(group); 222 } 223 } 224 225 static void iommufd_compose_fault_message(struct iommu_fault *fault, 226 struct iommu_hwpt_pgfault *hwpt_fault, 227 struct iommufd_device *idev, 228 u32 cookie) 229 { 230 hwpt_fault->flags = fault->prm.flags; 231 hwpt_fault->dev_id = idev->obj.id; 232 hwpt_fault->pasid = fault->prm.pasid; 233 hwpt_fault->grpid = fault->prm.grpid; 234 hwpt_fault->perm = fault->prm.perm; 235 hwpt_fault->addr = fault->prm.addr; 236 hwpt_fault->length = 0; 237 hwpt_fault->cookie = cookie; 238 } 239 240 static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf, 241 size_t count, loff_t *ppos) 242 { 243 size_t fault_size = sizeof(struct iommu_hwpt_pgfault); 244 struct iommufd_fault *fault = filep->private_data; 245 struct iommu_hwpt_pgfault data; 246 struct iommufd_device *idev; 247 struct iopf_group *group; 248 struct iopf_fault *iopf; 249 size_t done = 0; 250 int rc = 0; 251 252 if (*ppos || count % fault_size) 253 return -ESPIPE; 254 255 mutex_lock(&fault->mutex); 256 while (!list_empty(&fault->deliver) && count > done) { 257 group = list_first_entry(&fault->deliver, 258 struct iopf_group, node); 259 260 if (group->fault_count * fault_size > count - done) 261 break; 262 263 rc = xa_alloc(&fault->response, &group->cookie, group, 264 xa_limit_32b, GFP_KERNEL); 265 if (rc) 266 break; 267 268 idev = to_iommufd_handle(group->attach_handle)->idev; 269 list_for_each_entry(iopf, &group->faults, list) { 270 iommufd_compose_fault_message(&iopf->fault, 271 &data, idev, 272 group->cookie); 273 if (copy_to_user(buf + done, &data, fault_size)) { 274 xa_erase(&fault->response, group->cookie); 275 rc = -EFAULT; 276 break; 277 } 278 done += fault_size; 279 } 280 281 list_del(&group->node); 282 } 283 mutex_unlock(&fault->mutex); 284 285 return done == 0 ? rc : done; 286 } 287 288 static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *buf, 289 size_t count, loff_t *ppos) 290 { 291 size_t response_size = sizeof(struct iommu_hwpt_page_response); 292 struct iommufd_fault *fault = filep->private_data; 293 struct iommu_hwpt_page_response response; 294 struct iopf_group *group; 295 size_t done = 0; 296 int rc = 0; 297 298 if (*ppos || count % response_size) 299 return -ESPIPE; 300 301 mutex_lock(&fault->mutex); 302 while (count > done) { 303 rc = copy_from_user(&response, buf + done, response_size); 304 if (rc) 305 break; 306 307 static_assert((int)IOMMUFD_PAGE_RESP_SUCCESS == 308 (int)IOMMU_PAGE_RESP_SUCCESS); 309 static_assert((int)IOMMUFD_PAGE_RESP_INVALID == 310 (int)IOMMU_PAGE_RESP_INVALID); 311 if (response.code != IOMMUFD_PAGE_RESP_SUCCESS && 312 response.code != IOMMUFD_PAGE_RESP_INVALID) { 313 rc = -EINVAL; 314 break; 315 } 316 317 group = xa_erase(&fault->response, response.cookie); 318 if (!group) { 319 rc = -EINVAL; 320 break; 321 } 322 323 iopf_group_response(group, response.code); 324 iopf_free_group(group); 325 done += response_size; 326 } 327 mutex_unlock(&fault->mutex); 328 329 return done == 0 ? rc : done; 330 } 331 332 static __poll_t iommufd_fault_fops_poll(struct file *filep, 333 struct poll_table_struct *wait) 334 { 335 struct iommufd_fault *fault = filep->private_data; 336 __poll_t pollflags = EPOLLOUT; 337 338 poll_wait(filep, &fault->wait_queue, wait); 339 mutex_lock(&fault->mutex); 340 if (!list_empty(&fault->deliver)) 341 pollflags |= EPOLLIN | EPOLLRDNORM; 342 mutex_unlock(&fault->mutex); 343 344 return pollflags; 345 } 346 347 static int iommufd_fault_fops_release(struct inode *inode, struct file *filep) 348 { 349 struct iommufd_fault *fault = filep->private_data; 350 351 refcount_dec(&fault->obj.users); 352 iommufd_ctx_put(fault->ictx); 353 return 0; 354 } 355 356 static const struct file_operations iommufd_fault_fops = { 357 .owner = THIS_MODULE, 358 .open = nonseekable_open, 359 .read = iommufd_fault_fops_read, 360 .write = iommufd_fault_fops_write, 361 .poll = iommufd_fault_fops_poll, 362 .release = iommufd_fault_fops_release, 363 }; 364 365 int iommufd_fault_alloc(struct iommufd_ucmd *ucmd) 366 { 367 struct iommu_fault_alloc *cmd = ucmd->cmd; 368 struct iommufd_fault *fault; 369 struct file *filep; 370 int fdno; 371 int rc; 372 373 if (cmd->flags) 374 return -EOPNOTSUPP; 375 376 fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT); 377 if (IS_ERR(fault)) 378 return PTR_ERR(fault); 379 380 fault->ictx = ucmd->ictx; 381 INIT_LIST_HEAD(&fault->deliver); 382 xa_init_flags(&fault->response, XA_FLAGS_ALLOC1); 383 mutex_init(&fault->mutex); 384 init_waitqueue_head(&fault->wait_queue); 385 386 filep = anon_inode_getfile("[iommufd-pgfault]", &iommufd_fault_fops, 387 fault, O_RDWR); 388 if (IS_ERR(filep)) { 389 rc = PTR_ERR(filep); 390 goto out_abort; 391 } 392 393 refcount_inc(&fault->obj.users); 394 iommufd_ctx_get(fault->ictx); 395 fault->filep = filep; 396 397 fdno = get_unused_fd_flags(O_CLOEXEC); 398 if (fdno < 0) { 399 rc = fdno; 400 goto out_fput; 401 } 402 403 cmd->out_fault_id = fault->obj.id; 404 cmd->out_fault_fd = fdno; 405 406 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 407 if (rc) 408 goto out_put_fdno; 409 iommufd_object_finalize(ucmd->ictx, &fault->obj); 410 411 fd_install(fdno, fault->filep); 412 413 return 0; 414 out_put_fdno: 415 put_unused_fd(fdno); 416 out_fput: 417 fput(filep); 418 refcount_dec(&fault->obj.users); 419 iommufd_ctx_put(fault->ictx); 420 out_abort: 421 iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj); 422 423 return rc; 424 } 425 426 int iommufd_fault_iopf_handler(struct iopf_group *group) 427 { 428 struct iommufd_hw_pagetable *hwpt; 429 struct iommufd_fault *fault; 430 431 hwpt = group->attach_handle->domain->fault_data; 432 fault = hwpt->fault; 433 434 mutex_lock(&fault->mutex); 435 list_add_tail(&group->node, &fault->deliver); 436 mutex_unlock(&fault->mutex); 437 438 wake_up_interruptible(&fault->wait_queue); 439 440 return 0; 441 } 442