1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/cred.h> 3 #include <linux/device.h> 4 #include <linux/dma-buf.h> 5 #include <linux/dma-resv.h> 6 #include <linux/highmem.h> 7 #include <linux/init.h> 8 #include <linux/kernel.h> 9 #include <linux/memfd.h> 10 #include <linux/miscdevice.h> 11 #include <linux/module.h> 12 #include <linux/shmem_fs.h> 13 #include <linux/hugetlb.h> 14 #include <linux/slab.h> 15 #include <linux/udmabuf.h> 16 #include <linux/vmalloc.h> 17 #include <linux/iosys-map.h> 18 19 static int list_limit = 1024; 20 module_param(list_limit, int, 0644); 21 MODULE_PARM_DESC(list_limit, "udmabuf_create_list->count limit. Default is 1024."); 22 23 static int size_limit_mb = 64; 24 module_param(size_limit_mb, int, 0644); 25 MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is 64."); 26 27 struct udmabuf { 28 pgoff_t pagecount; 29 struct page **pages; 30 31 /** 32 * Unlike pages, pinned_folios is only used for unpin. 33 * So, nr_pinned is not the same to pagecount, the pinned_folios 34 * only set each folio which already pinned when udmabuf_create. 35 * Note that, since a folio may be pinned multiple times, each folio 36 * can be added to pinned_folios multiple times, depending on how many 37 * times the folio has been pinned when create. 38 */ 39 pgoff_t nr_pinned; 40 struct folio **pinned_folios; 41 42 struct sg_table *sg; 43 enum dma_data_direction sg_dir; 44 struct miscdevice *device; 45 }; 46 47 static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) 48 { 49 struct vm_area_struct *vma = vmf->vma; 50 struct udmabuf *ubuf = vma->vm_private_data; 51 pgoff_t pgoff = vmf->pgoff; 52 unsigned long addr, pfn; 53 vm_fault_t ret; 54 55 if (pgoff >= ubuf->pagecount) 56 return VM_FAULT_SIGBUS; 57 58 pfn = page_to_pfn(ubuf->pages[pgoff]); 59 60 ret = vmf_insert_pfn(vma, vmf->address, pfn); 61 if (ret & VM_FAULT_ERROR) 62 return ret; 63 64 /* pre fault */ 65 pgoff = vma->vm_pgoff; 66 addr = vma->vm_start; 67 68 for (; addr < vma->vm_end; pgoff++, addr += PAGE_SIZE) { 69 if (addr == vmf->address) 70 continue; 71 72 if (WARN_ON(pgoff >= ubuf->pagecount)) 73 break; 74 75 pfn = page_to_pfn(ubuf->pages[pgoff]); 76 77 /** 78 * If the below vmf_insert_pfn() fails, we do not return an 79 * error here during this pre-fault step. However, an error 80 * will be returned if the failure occurs when the addr is 81 * truly accessed. 82 */ 83 if (vmf_insert_pfn(vma, addr, pfn) & VM_FAULT_ERROR) 84 break; 85 } 86 87 return ret; 88 } 89 90 static const struct vm_operations_struct udmabuf_vm_ops = { 91 .fault = udmabuf_vm_fault, 92 }; 93 94 static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma) 95 { 96 struct udmabuf *ubuf = buf->priv; 97 98 if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) 99 return -EINVAL; 100 101 vma->vm_ops = &udmabuf_vm_ops; 102 vma->vm_private_data = ubuf; 103 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); 104 return 0; 105 } 106 107 static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) 108 { 109 struct udmabuf *ubuf = buf->priv; 110 void *vaddr; 111 112 dma_resv_assert_held(buf->resv); 113 114 vaddr = vm_map_ram(ubuf->pages, ubuf->pagecount, -1); 115 if (!vaddr) 116 return -EINVAL; 117 118 iosys_map_set_vaddr(map, vaddr); 119 return 0; 120 } 121 122 static void vunmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) 123 { 124 struct udmabuf *ubuf = buf->priv; 125 126 dma_resv_assert_held(buf->resv); 127 128 vm_unmap_ram(map->vaddr, ubuf->pagecount); 129 } 130 131 static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, 132 enum dma_data_direction direction) 133 { 134 struct udmabuf *ubuf = buf->priv; 135 struct sg_table *sg; 136 int ret; 137 138 sg = kzalloc_obj(*sg); 139 if (!sg) 140 return ERR_PTR(-ENOMEM); 141 142 ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->pagecount, 0, 143 ubuf->pagecount << PAGE_SHIFT, 144 GFP_KERNEL); 145 if (ret < 0) 146 goto err_alloc; 147 148 ret = dma_map_sgtable(dev, sg, direction, DMA_ATTR_SKIP_CPU_SYNC); 149 if (ret < 0) 150 goto err_map; 151 return sg; 152 153 err_map: 154 sg_free_table(sg); 155 err_alloc: 156 kfree(sg); 157 return ERR_PTR(ret); 158 } 159 160 static void put_sg_table(struct device *dev, struct sg_table *sg, 161 enum dma_data_direction direction) 162 { 163 dma_unmap_sgtable(dev, sg, direction, DMA_ATTR_SKIP_CPU_SYNC); 164 sg_free_table(sg); 165 kfree(sg); 166 } 167 168 static struct sg_table *map_udmabuf(struct dma_buf_attachment *at, 169 enum dma_data_direction direction) 170 { 171 return get_sg_table(at->dev, at->dmabuf, direction); 172 } 173 174 static void unmap_udmabuf(struct dma_buf_attachment *at, 175 struct sg_table *sg, 176 enum dma_data_direction direction) 177 { 178 return put_sg_table(at->dev, sg, direction); 179 } 180 181 static void unpin_all_folios(struct udmabuf *ubuf) 182 { 183 pgoff_t i; 184 185 for (i = 0; i < ubuf->nr_pinned; ++i) 186 unpin_folio(ubuf->pinned_folios[i]); 187 188 kvfree(ubuf->pinned_folios); 189 } 190 191 static __always_inline int init_udmabuf(struct udmabuf *ubuf, pgoff_t pgcnt) 192 { 193 ubuf->pages = kvmalloc_objs(*ubuf->pages, pgcnt); 194 if (!ubuf->pages) 195 return -ENOMEM; 196 197 ubuf->pinned_folios = kvmalloc_objs(*ubuf->pinned_folios, pgcnt); 198 if (!ubuf->pinned_folios) 199 return -ENOMEM; 200 201 return 0; 202 } 203 204 static __always_inline void deinit_udmabuf(struct udmabuf *ubuf) 205 { 206 unpin_all_folios(ubuf); 207 kvfree(ubuf->pages); 208 } 209 210 static void release_udmabuf(struct dma_buf *buf) 211 { 212 struct udmabuf *ubuf = buf->priv; 213 struct device *dev = ubuf->device->this_device; 214 215 if (ubuf->sg) 216 put_sg_table(dev, ubuf->sg, ubuf->sg_dir); 217 218 deinit_udmabuf(ubuf); 219 kfree(ubuf); 220 } 221 222 static int begin_cpu_udmabuf(struct dma_buf *buf, 223 enum dma_data_direction direction) 224 { 225 struct udmabuf *ubuf = buf->priv; 226 struct device *dev = ubuf->device->this_device; 227 228 if (!ubuf->sg) { 229 ubuf->sg = get_sg_table(dev, buf, direction); 230 if (IS_ERR(ubuf->sg)) { 231 int ret; 232 233 ret = PTR_ERR(ubuf->sg); 234 ubuf->sg = NULL; 235 return ret; 236 } else { 237 ubuf->sg_dir = direction; 238 } 239 } 240 241 dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction); 242 return 0; 243 } 244 245 static int end_cpu_udmabuf(struct dma_buf *buf, 246 enum dma_data_direction direction) 247 { 248 struct udmabuf *ubuf = buf->priv; 249 struct device *dev = ubuf->device->this_device; 250 251 if (!ubuf->sg) 252 return -EINVAL; 253 254 dma_sync_sgtable_for_device(dev, ubuf->sg, direction); 255 return 0; 256 } 257 258 static const struct dma_buf_ops udmabuf_ops = { 259 .map_dma_buf = map_udmabuf, 260 .unmap_dma_buf = unmap_udmabuf, 261 .release = release_udmabuf, 262 .mmap = mmap_udmabuf, 263 .vmap = vmap_udmabuf, 264 .vunmap = vunmap_udmabuf, 265 .begin_cpu_access = begin_cpu_udmabuf, 266 .end_cpu_access = end_cpu_udmabuf, 267 }; 268 269 #define SEALS_WANTED (F_SEAL_SHRINK) 270 #define SEALS_DENIED (F_SEAL_WRITE|F_SEAL_FUTURE_WRITE) 271 272 static int check_memfd_seals(struct file *memfd) 273 { 274 int seals; 275 276 if (!shmem_file(memfd) && !is_file_hugepages(memfd)) 277 return -EBADFD; 278 279 seals = memfd_fcntl(memfd, F_GET_SEALS, 0); 280 if (seals == -EINVAL) 281 return -EBADFD; 282 283 if ((seals & SEALS_WANTED) != SEALS_WANTED || 284 (seals & SEALS_DENIED) != 0) 285 return -EINVAL; 286 287 return 0; 288 } 289 290 static struct dma_buf *export_udmabuf(struct udmabuf *ubuf, 291 struct miscdevice *device) 292 { 293 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 294 295 ubuf->device = device; 296 exp_info.ops = &udmabuf_ops; 297 exp_info.size = ubuf->pagecount << PAGE_SHIFT; 298 exp_info.priv = ubuf; 299 exp_info.flags = O_RDWR; 300 301 return dma_buf_export(&exp_info); 302 } 303 304 static long udmabuf_pin_folios(struct udmabuf *ubuf, struct file *memfd, 305 loff_t start, loff_t size, struct folio **folios) 306 { 307 pgoff_t nr_pinned = ubuf->nr_pinned; 308 pgoff_t upgcnt = ubuf->pagecount; 309 u32 cur_folio, cur_pgcnt; 310 pgoff_t pgoff, pgcnt; 311 long nr_folios; 312 loff_t end; 313 314 pgcnt = size >> PAGE_SHIFT; 315 end = start + (pgcnt << PAGE_SHIFT) - 1; 316 nr_folios = memfd_pin_folios(memfd, start, end, folios, pgcnt, &pgoff); 317 if (nr_folios <= 0) 318 return nr_folios ? nr_folios : -EINVAL; 319 320 cur_pgcnt = 0; 321 for (cur_folio = 0; cur_folio < nr_folios; ++cur_folio) { 322 pgoff_t subpgoff = pgoff; 323 size_t fsize = folio_size(folios[cur_folio]); 324 325 ubuf->pinned_folios[nr_pinned++] = folios[cur_folio]; 326 327 for (; subpgoff < fsize; subpgoff += PAGE_SIZE) { 328 ubuf->pages[upgcnt] = folio_page(folios[cur_folio], 329 subpgoff >> PAGE_SHIFT); 330 ++upgcnt; 331 332 if (++cur_pgcnt >= pgcnt) 333 goto end; 334 } 335 336 /** 337 * In a given range, only the first subpage of the first folio 338 * has an offset, that is returned by memfd_pin_folios(). 339 * The first subpages of other folios (in the range) have an 340 * offset of 0. 341 */ 342 pgoff = 0; 343 } 344 end: 345 ubuf->pagecount = upgcnt; 346 ubuf->nr_pinned = nr_pinned; 347 return 0; 348 } 349 350 static long udmabuf_create(struct miscdevice *device, 351 struct udmabuf_create_list *head, 352 struct udmabuf_create_item *list) 353 { 354 unsigned long max_nr_folios = 0; 355 struct folio **folios = NULL; 356 pgoff_t pgcnt = 0, pglimit; 357 struct udmabuf *ubuf; 358 struct dma_buf *dmabuf; 359 long ret = -EINVAL; 360 u32 i, flags; 361 362 ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL); 363 if (!ubuf) 364 return -ENOMEM; 365 366 pglimit = ((u64)size_limit_mb * 1024 * 1024) >> PAGE_SHIFT; 367 for (i = 0; i < head->count; i++) { 368 pgoff_t subpgcnt; 369 370 if (!PAGE_ALIGNED(list[i].offset)) 371 goto err_noinit; 372 if (!PAGE_ALIGNED(list[i].size)) 373 goto err_noinit; 374 375 subpgcnt = list[i].size >> PAGE_SHIFT; 376 pgcnt += subpgcnt; 377 if (pgcnt > pglimit) 378 goto err_noinit; 379 380 max_nr_folios = max_t(unsigned long, subpgcnt, max_nr_folios); 381 } 382 383 if (!pgcnt) 384 goto err_noinit; 385 386 ret = init_udmabuf(ubuf, pgcnt); 387 if (ret) 388 goto err; 389 390 folios = kvmalloc_array(max_nr_folios, sizeof(*folios), GFP_KERNEL); 391 if (!folios) { 392 ret = -ENOMEM; 393 goto err; 394 } 395 396 for (i = 0; i < head->count; i++) { 397 struct file *memfd = fget(list[i].memfd); 398 399 if (!memfd) { 400 ret = -EBADFD; 401 goto err; 402 } 403 404 /* 405 * Take the inode lock to protect against concurrent 406 * memfd_add_seals(), which takes this lock in write mode. 407 */ 408 inode_lock_shared(file_inode(memfd)); 409 ret = check_memfd_seals(memfd); 410 if (ret) 411 goto out_unlock; 412 413 ret = udmabuf_pin_folios(ubuf, memfd, list[i].offset, 414 list[i].size, folios); 415 out_unlock: 416 inode_unlock_shared(file_inode(memfd)); 417 fput(memfd); 418 if (ret) 419 goto err; 420 } 421 422 flags = head->flags & UDMABUF_FLAGS_CLOEXEC ? O_CLOEXEC : 0; 423 dmabuf = export_udmabuf(ubuf, device); 424 if (IS_ERR(dmabuf)) { 425 ret = PTR_ERR(dmabuf); 426 goto err; 427 } 428 /* 429 * Ownership of ubuf is held by the dmabuf from here. 430 * If the following dma_buf_fd() fails, dma_buf_put() cleans up both the 431 * dmabuf and the ubuf (through udmabuf_ops.release). 432 */ 433 434 ret = dma_buf_fd(dmabuf, flags); 435 if (ret < 0) 436 dma_buf_put(dmabuf); 437 438 kvfree(folios); 439 return ret; 440 441 err: 442 deinit_udmabuf(ubuf); 443 err_noinit: 444 kfree(ubuf); 445 kvfree(folios); 446 return ret; 447 } 448 449 static long udmabuf_ioctl_create(struct file *filp, unsigned long arg) 450 { 451 struct udmabuf_create create; 452 struct udmabuf_create_list head; 453 struct udmabuf_create_item list; 454 455 if (copy_from_user(&create, (void __user *)arg, 456 sizeof(create))) 457 return -EFAULT; 458 459 head.flags = create.flags; 460 head.count = 1; 461 list.memfd = create.memfd; 462 list.offset = create.offset; 463 list.size = create.size; 464 465 return udmabuf_create(filp->private_data, &head, &list); 466 } 467 468 static long udmabuf_ioctl_create_list(struct file *filp, unsigned long arg) 469 { 470 struct udmabuf_create_list head; 471 struct udmabuf_create_item *list; 472 int ret = -EINVAL; 473 u32 lsize; 474 475 if (copy_from_user(&head, (void __user *)arg, sizeof(head))) 476 return -EFAULT; 477 if (head.count > list_limit) 478 return -EINVAL; 479 lsize = sizeof(struct udmabuf_create_item) * head.count; 480 list = memdup_user((void __user *)(arg + sizeof(head)), lsize); 481 if (IS_ERR(list)) 482 return PTR_ERR(list); 483 484 ret = udmabuf_create(filp->private_data, &head, list); 485 kfree(list); 486 return ret; 487 } 488 489 static long udmabuf_ioctl(struct file *filp, unsigned int ioctl, 490 unsigned long arg) 491 { 492 long ret; 493 494 switch (ioctl) { 495 case UDMABUF_CREATE: 496 ret = udmabuf_ioctl_create(filp, arg); 497 break; 498 case UDMABUF_CREATE_LIST: 499 ret = udmabuf_ioctl_create_list(filp, arg); 500 break; 501 default: 502 ret = -ENOTTY; 503 break; 504 } 505 return ret; 506 } 507 508 static const struct file_operations udmabuf_fops = { 509 .owner = THIS_MODULE, 510 .unlocked_ioctl = udmabuf_ioctl, 511 #ifdef CONFIG_COMPAT 512 .compat_ioctl = udmabuf_ioctl, 513 #endif 514 }; 515 516 static struct miscdevice udmabuf_misc = { 517 .minor = MISC_DYNAMIC_MINOR, 518 .name = "udmabuf", 519 .fops = &udmabuf_fops, 520 }; 521 522 static int __init udmabuf_dev_init(void) 523 { 524 int ret; 525 526 ret = misc_register(&udmabuf_misc); 527 if (ret < 0) { 528 pr_err("Could not initialize udmabuf device\n"); 529 return ret; 530 } 531 532 ret = dma_coerce_mask_and_coherent(udmabuf_misc.this_device, 533 DMA_BIT_MASK(64)); 534 if (ret < 0) { 535 pr_err("Could not setup DMA mask for udmabuf device\n"); 536 misc_deregister(&udmabuf_misc); 537 return ret; 538 } 539 540 return 0; 541 } 542 543 static void __exit udmabuf_dev_exit(void) 544 { 545 misc_deregister(&udmabuf_misc); 546 } 547 548 module_init(udmabuf_dev_init) 549 module_exit(udmabuf_dev_exit) 550 551 MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>"); 552