1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/cred.h> 3 #include <linux/device.h> 4 #include <linux/dma-buf.h> 5 #include <linux/dma-resv.h> 6 #include <linux/highmem.h> 7 #include <linux/init.h> 8 #include <linux/kernel.h> 9 #include <linux/memfd.h> 10 #include <linux/miscdevice.h> 11 #include <linux/module.h> 12 #include <linux/shmem_fs.h> 13 #include <linux/hugetlb.h> 14 #include <linux/slab.h> 15 #include <linux/udmabuf.h> 16 #include <linux/vmalloc.h> 17 #include <linux/iosys-map.h> 18 19 static int list_limit = 1024; 20 module_param(list_limit, int, 0644); 21 MODULE_PARM_DESC(list_limit, "udmabuf_create_list->count limit. Default is 1024."); 22 23 static int size_limit_mb = 64; 24 module_param(size_limit_mb, int, 0644); 25 MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is 64."); 26 27 struct udmabuf { 28 pgoff_t pagecount; 29 struct folio **folios; 30 31 /** 32 * Unlike folios, pinned_folios is only used for unpin. 33 * So, nr_pinned is not the same to pagecount, the pinned_folios 34 * only set each folio which already pinned when udmabuf_create. 35 * Note that, since a folio may be pinned multiple times, each folio 36 * can be added to pinned_folios multiple times, depending on how many 37 * times the folio has been pinned when create. 38 */ 39 pgoff_t nr_pinned; 40 struct folio **pinned_folios; 41 42 struct sg_table *sg; 43 struct miscdevice *device; 44 pgoff_t *offsets; 45 }; 46 47 static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf) 48 { 49 struct vm_area_struct *vma = vmf->vma; 50 struct udmabuf *ubuf = vma->vm_private_data; 51 pgoff_t pgoff = vmf->pgoff; 52 unsigned long addr, pfn; 53 vm_fault_t ret; 54 55 if (pgoff >= ubuf->pagecount) 56 return VM_FAULT_SIGBUS; 57 58 pfn = folio_pfn(ubuf->folios[pgoff]); 59 pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT; 60 61 ret = vmf_insert_pfn(vma, vmf->address, pfn); 62 if (ret & VM_FAULT_ERROR) 63 return ret; 64 65 /* pre fault */ 66 pgoff = vma->vm_pgoff; 67 addr = vma->vm_start; 68 69 for (; addr < vma->vm_end; pgoff++, addr += PAGE_SIZE) { 70 if (addr == vmf->address) 71 continue; 72 73 if (WARN_ON(pgoff >= ubuf->pagecount)) 74 break; 75 76 pfn = folio_pfn(ubuf->folios[pgoff]); 77 pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT; 78 79 /** 80 * If the below vmf_insert_pfn() fails, we do not return an 81 * error here during this pre-fault step. However, an error 82 * will be returned if the failure occurs when the addr is 83 * truly accessed. 84 */ 85 if (vmf_insert_pfn(vma, addr, pfn) & VM_FAULT_ERROR) 86 break; 87 } 88 89 return ret; 90 } 91 92 static const struct vm_operations_struct udmabuf_vm_ops = { 93 .fault = udmabuf_vm_fault, 94 }; 95 96 static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma) 97 { 98 struct udmabuf *ubuf = buf->priv; 99 100 if ((vma->vm_flags & (VM_SHARED | VM_MAYSHARE)) == 0) 101 return -EINVAL; 102 103 vma->vm_ops = &udmabuf_vm_ops; 104 vma->vm_private_data = ubuf; 105 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); 106 return 0; 107 } 108 109 static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) 110 { 111 struct udmabuf *ubuf = buf->priv; 112 struct page **pages; 113 void *vaddr; 114 pgoff_t pg; 115 116 dma_resv_assert_held(buf->resv); 117 118 pages = kvmalloc_array(ubuf->pagecount, sizeof(*pages), GFP_KERNEL); 119 if (!pages) 120 return -ENOMEM; 121 122 for (pg = 0; pg < ubuf->pagecount; pg++) 123 pages[pg] = folio_page(ubuf->folios[pg], 124 ubuf->offsets[pg] >> PAGE_SHIFT); 125 126 vaddr = vm_map_ram(pages, ubuf->pagecount, -1); 127 kvfree(pages); 128 if (!vaddr) 129 return -EINVAL; 130 131 iosys_map_set_vaddr(map, vaddr); 132 return 0; 133 } 134 135 static void vunmap_udmabuf(struct dma_buf *buf, struct iosys_map *map) 136 { 137 struct udmabuf *ubuf = buf->priv; 138 139 dma_resv_assert_held(buf->resv); 140 141 vm_unmap_ram(map->vaddr, ubuf->pagecount); 142 } 143 144 static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf, 145 enum dma_data_direction direction) 146 { 147 struct udmabuf *ubuf = buf->priv; 148 struct sg_table *sg; 149 struct scatterlist *sgl; 150 unsigned int i = 0; 151 int ret; 152 153 sg = kzalloc(sizeof(*sg), GFP_KERNEL); 154 if (!sg) 155 return ERR_PTR(-ENOMEM); 156 157 ret = sg_alloc_table(sg, ubuf->pagecount, GFP_KERNEL); 158 if (ret < 0) 159 goto err_alloc; 160 161 for_each_sg(sg->sgl, sgl, ubuf->pagecount, i) 162 sg_set_folio(sgl, ubuf->folios[i], PAGE_SIZE, 163 ubuf->offsets[i]); 164 165 ret = dma_map_sgtable(dev, sg, direction, 0); 166 if (ret < 0) 167 goto err_map; 168 return sg; 169 170 err_map: 171 sg_free_table(sg); 172 err_alloc: 173 kfree(sg); 174 return ERR_PTR(ret); 175 } 176 177 static void put_sg_table(struct device *dev, struct sg_table *sg, 178 enum dma_data_direction direction) 179 { 180 dma_unmap_sgtable(dev, sg, direction, 0); 181 sg_free_table(sg); 182 kfree(sg); 183 } 184 185 static struct sg_table *map_udmabuf(struct dma_buf_attachment *at, 186 enum dma_data_direction direction) 187 { 188 return get_sg_table(at->dev, at->dmabuf, direction); 189 } 190 191 static void unmap_udmabuf(struct dma_buf_attachment *at, 192 struct sg_table *sg, 193 enum dma_data_direction direction) 194 { 195 return put_sg_table(at->dev, sg, direction); 196 } 197 198 static void unpin_all_folios(struct udmabuf *ubuf) 199 { 200 pgoff_t i; 201 202 for (i = 0; i < ubuf->nr_pinned; ++i) 203 unpin_folio(ubuf->pinned_folios[i]); 204 205 kvfree(ubuf->pinned_folios); 206 } 207 208 static __always_inline int init_udmabuf(struct udmabuf *ubuf, pgoff_t pgcnt) 209 { 210 ubuf->folios = kvmalloc_array(pgcnt, sizeof(*ubuf->folios), GFP_KERNEL); 211 if (!ubuf->folios) 212 return -ENOMEM; 213 214 ubuf->offsets = kvcalloc(pgcnt, sizeof(*ubuf->offsets), GFP_KERNEL); 215 if (!ubuf->offsets) 216 return -ENOMEM; 217 218 ubuf->pinned_folios = kvmalloc_array(pgcnt, 219 sizeof(*ubuf->pinned_folios), 220 GFP_KERNEL); 221 if (!ubuf->pinned_folios) 222 return -ENOMEM; 223 224 return 0; 225 } 226 227 static __always_inline void deinit_udmabuf(struct udmabuf *ubuf) 228 { 229 unpin_all_folios(ubuf); 230 kvfree(ubuf->offsets); 231 kvfree(ubuf->folios); 232 } 233 234 static void release_udmabuf(struct dma_buf *buf) 235 { 236 struct udmabuf *ubuf = buf->priv; 237 struct device *dev = ubuf->device->this_device; 238 239 if (ubuf->sg) 240 put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL); 241 242 deinit_udmabuf(ubuf); 243 kfree(ubuf); 244 } 245 246 static int begin_cpu_udmabuf(struct dma_buf *buf, 247 enum dma_data_direction direction) 248 { 249 struct udmabuf *ubuf = buf->priv; 250 struct device *dev = ubuf->device->this_device; 251 int ret = 0; 252 253 if (!ubuf->sg) { 254 ubuf->sg = get_sg_table(dev, buf, direction); 255 if (IS_ERR(ubuf->sg)) { 256 ret = PTR_ERR(ubuf->sg); 257 ubuf->sg = NULL; 258 } 259 } else { 260 dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction); 261 } 262 263 return ret; 264 } 265 266 static int end_cpu_udmabuf(struct dma_buf *buf, 267 enum dma_data_direction direction) 268 { 269 struct udmabuf *ubuf = buf->priv; 270 struct device *dev = ubuf->device->this_device; 271 272 if (!ubuf->sg) 273 return -EINVAL; 274 275 dma_sync_sgtable_for_device(dev, ubuf->sg, direction); 276 return 0; 277 } 278 279 static const struct dma_buf_ops udmabuf_ops = { 280 .map_dma_buf = map_udmabuf, 281 .unmap_dma_buf = unmap_udmabuf, 282 .release = release_udmabuf, 283 .mmap = mmap_udmabuf, 284 .vmap = vmap_udmabuf, 285 .vunmap = vunmap_udmabuf, 286 .begin_cpu_access = begin_cpu_udmabuf, 287 .end_cpu_access = end_cpu_udmabuf, 288 }; 289 290 #define SEALS_WANTED (F_SEAL_SHRINK) 291 #define SEALS_DENIED (F_SEAL_WRITE|F_SEAL_FUTURE_WRITE) 292 293 static int check_memfd_seals(struct file *memfd) 294 { 295 int seals; 296 297 if (!shmem_file(memfd) && !is_file_hugepages(memfd)) 298 return -EBADFD; 299 300 seals = memfd_fcntl(memfd, F_GET_SEALS, 0); 301 if (seals == -EINVAL) 302 return -EBADFD; 303 304 if ((seals & SEALS_WANTED) != SEALS_WANTED || 305 (seals & SEALS_DENIED) != 0) 306 return -EINVAL; 307 308 return 0; 309 } 310 311 static struct dma_buf *export_udmabuf(struct udmabuf *ubuf, 312 struct miscdevice *device) 313 { 314 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 315 316 ubuf->device = device; 317 exp_info.ops = &udmabuf_ops; 318 exp_info.size = ubuf->pagecount << PAGE_SHIFT; 319 exp_info.priv = ubuf; 320 exp_info.flags = O_RDWR; 321 322 return dma_buf_export(&exp_info); 323 } 324 325 static long udmabuf_pin_folios(struct udmabuf *ubuf, struct file *memfd, 326 loff_t start, loff_t size, struct folio **folios) 327 { 328 pgoff_t nr_pinned = ubuf->nr_pinned; 329 pgoff_t upgcnt = ubuf->pagecount; 330 u32 cur_folio, cur_pgcnt; 331 pgoff_t pgoff, pgcnt; 332 long nr_folios; 333 loff_t end; 334 335 pgcnt = size >> PAGE_SHIFT; 336 end = start + (pgcnt << PAGE_SHIFT) - 1; 337 nr_folios = memfd_pin_folios(memfd, start, end, folios, pgcnt, &pgoff); 338 if (nr_folios <= 0) 339 return nr_folios ? nr_folios : -EINVAL; 340 341 cur_pgcnt = 0; 342 for (cur_folio = 0; cur_folio < nr_folios; ++cur_folio) { 343 pgoff_t subpgoff = pgoff; 344 size_t fsize = folio_size(folios[cur_folio]); 345 346 ubuf->pinned_folios[nr_pinned++] = folios[cur_folio]; 347 348 for (; subpgoff < fsize; subpgoff += PAGE_SIZE) { 349 ubuf->folios[upgcnt] = folios[cur_folio]; 350 ubuf->offsets[upgcnt] = subpgoff; 351 ++upgcnt; 352 353 if (++cur_pgcnt >= pgcnt) 354 goto end; 355 } 356 357 /** 358 * In a given range, only the first subpage of the first folio 359 * has an offset, that is returned by memfd_pin_folios(). 360 * The first subpages of other folios (in the range) have an 361 * offset of 0. 362 */ 363 pgoff = 0; 364 } 365 end: 366 ubuf->pagecount = upgcnt; 367 ubuf->nr_pinned = nr_pinned; 368 return 0; 369 } 370 371 static long udmabuf_create(struct miscdevice *device, 372 struct udmabuf_create_list *head, 373 struct udmabuf_create_item *list) 374 { 375 unsigned long max_nr_folios = 0; 376 struct folio **folios = NULL; 377 pgoff_t pgcnt = 0, pglimit; 378 struct udmabuf *ubuf; 379 struct dma_buf *dmabuf; 380 long ret = -EINVAL; 381 u32 i, flags; 382 383 ubuf = kzalloc(sizeof(*ubuf), GFP_KERNEL); 384 if (!ubuf) 385 return -ENOMEM; 386 387 pglimit = ((u64)size_limit_mb * 1024 * 1024) >> PAGE_SHIFT; 388 for (i = 0; i < head->count; i++) { 389 pgoff_t subpgcnt; 390 391 if (!PAGE_ALIGNED(list[i].offset)) 392 goto err_noinit; 393 if (!PAGE_ALIGNED(list[i].size)) 394 goto err_noinit; 395 396 subpgcnt = list[i].size >> PAGE_SHIFT; 397 pgcnt += subpgcnt; 398 if (pgcnt > pglimit) 399 goto err_noinit; 400 401 max_nr_folios = max_t(unsigned long, subpgcnt, max_nr_folios); 402 } 403 404 if (!pgcnt) 405 goto err_noinit; 406 407 ret = init_udmabuf(ubuf, pgcnt); 408 if (ret) 409 goto err; 410 411 folios = kvmalloc_array(max_nr_folios, sizeof(*folios), GFP_KERNEL); 412 if (!folios) { 413 ret = -ENOMEM; 414 goto err; 415 } 416 417 for (i = 0; i < head->count; i++) { 418 struct file *memfd = fget(list[i].memfd); 419 420 if (!memfd) { 421 ret = -EBADFD; 422 goto err; 423 } 424 425 /* 426 * Take the inode lock to protect against concurrent 427 * memfd_add_seals(), which takes this lock in write mode. 428 */ 429 inode_lock_shared(file_inode(memfd)); 430 ret = check_memfd_seals(memfd); 431 if (ret) 432 goto out_unlock; 433 434 ret = udmabuf_pin_folios(ubuf, memfd, list[i].offset, 435 list[i].size, folios); 436 out_unlock: 437 inode_unlock_shared(file_inode(memfd)); 438 fput(memfd); 439 if (ret) 440 goto err; 441 } 442 443 flags = head->flags & UDMABUF_FLAGS_CLOEXEC ? O_CLOEXEC : 0; 444 dmabuf = export_udmabuf(ubuf, device); 445 if (IS_ERR(dmabuf)) { 446 ret = PTR_ERR(dmabuf); 447 goto err; 448 } 449 /* 450 * Ownership of ubuf is held by the dmabuf from here. 451 * If the following dma_buf_fd() fails, dma_buf_put() cleans up both the 452 * dmabuf and the ubuf (through udmabuf_ops.release). 453 */ 454 455 ret = dma_buf_fd(dmabuf, flags); 456 if (ret < 0) 457 dma_buf_put(dmabuf); 458 459 kvfree(folios); 460 return ret; 461 462 err: 463 deinit_udmabuf(ubuf); 464 err_noinit: 465 kfree(ubuf); 466 kvfree(folios); 467 return ret; 468 } 469 470 static long udmabuf_ioctl_create(struct file *filp, unsigned long arg) 471 { 472 struct udmabuf_create create; 473 struct udmabuf_create_list head; 474 struct udmabuf_create_item list; 475 476 if (copy_from_user(&create, (void __user *)arg, 477 sizeof(create))) 478 return -EFAULT; 479 480 head.flags = create.flags; 481 head.count = 1; 482 list.memfd = create.memfd; 483 list.offset = create.offset; 484 list.size = create.size; 485 486 return udmabuf_create(filp->private_data, &head, &list); 487 } 488 489 static long udmabuf_ioctl_create_list(struct file *filp, unsigned long arg) 490 { 491 struct udmabuf_create_list head; 492 struct udmabuf_create_item *list; 493 int ret = -EINVAL; 494 u32 lsize; 495 496 if (copy_from_user(&head, (void __user *)arg, sizeof(head))) 497 return -EFAULT; 498 if (head.count > list_limit) 499 return -EINVAL; 500 lsize = sizeof(struct udmabuf_create_item) * head.count; 501 list = memdup_user((void __user *)(arg + sizeof(head)), lsize); 502 if (IS_ERR(list)) 503 return PTR_ERR(list); 504 505 ret = udmabuf_create(filp->private_data, &head, list); 506 kfree(list); 507 return ret; 508 } 509 510 static long udmabuf_ioctl(struct file *filp, unsigned int ioctl, 511 unsigned long arg) 512 { 513 long ret; 514 515 switch (ioctl) { 516 case UDMABUF_CREATE: 517 ret = udmabuf_ioctl_create(filp, arg); 518 break; 519 case UDMABUF_CREATE_LIST: 520 ret = udmabuf_ioctl_create_list(filp, arg); 521 break; 522 default: 523 ret = -ENOTTY; 524 break; 525 } 526 return ret; 527 } 528 529 static const struct file_operations udmabuf_fops = { 530 .owner = THIS_MODULE, 531 .unlocked_ioctl = udmabuf_ioctl, 532 #ifdef CONFIG_COMPAT 533 .compat_ioctl = udmabuf_ioctl, 534 #endif 535 }; 536 537 static struct miscdevice udmabuf_misc = { 538 .minor = MISC_DYNAMIC_MINOR, 539 .name = "udmabuf", 540 .fops = &udmabuf_fops, 541 }; 542 543 static int __init udmabuf_dev_init(void) 544 { 545 int ret; 546 547 ret = misc_register(&udmabuf_misc); 548 if (ret < 0) { 549 pr_err("Could not initialize udmabuf device\n"); 550 return ret; 551 } 552 553 ret = dma_coerce_mask_and_coherent(udmabuf_misc.this_device, 554 DMA_BIT_MASK(64)); 555 if (ret < 0) { 556 pr_err("Could not setup DMA mask for udmabuf device\n"); 557 misc_deregister(&udmabuf_misc); 558 return ret; 559 } 560 561 return 0; 562 } 563 564 static void __exit udmabuf_dev_exit(void) 565 { 566 misc_deregister(&udmabuf_misc); 567 } 568 569 module_init(udmabuf_dev_init) 570 module_exit(udmabuf_dev_exit) 571 572 MODULE_AUTHOR("Gerd Hoffmann <kraxel@redhat.com>"); 573