1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Xen dma-buf functionality for gntdev. 5 * 6 * DMA buffer implementation is based on drivers/gpu/drm/drm_prime.c. 7 * 8 * Copyright (c) 2018 Oleksandr Andrushchenko, EPAM Systems Inc. 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/errno.h> 13 #include <linux/dma-buf.h> 14 #include <linux/slab.h> 15 #include <linux/types.h> 16 #include <linux/uaccess.h> 17 18 #include <xen/xen.h> 19 #include <xen/grant_table.h> 20 21 #include "gntdev-common.h" 22 #include "gntdev-dmabuf.h" 23 24 #ifndef GRANT_INVALID_REF 25 /* 26 * Note on usage of grant reference 0 as invalid grant reference: 27 * grant reference 0 is valid, but never exposed to a driver, 28 * because of the fact it is already in use/reserved by the PV console. 29 */ 30 #define GRANT_INVALID_REF 0 31 #endif 32 33 struct gntdev_dmabuf { 34 struct gntdev_dmabuf_priv *priv; 35 struct dma_buf *dmabuf; 36 struct list_head next; 37 int fd; 38 39 union { 40 struct { 41 /* Exported buffers are reference counted. */ 42 struct kref refcount; 43 44 struct gntdev_priv *priv; 45 struct gntdev_grant_map *map; 46 } exp; 47 struct { 48 /* Granted references of the imported buffer. */ 49 grant_ref_t *refs; 50 /* Scatter-gather table of the imported buffer. */ 51 struct sg_table *sgt; 52 /* dma-buf attachment of the imported buffer. */ 53 struct dma_buf_attachment *attach; 54 } imp; 55 } u; 56 57 /* Number of pages this buffer has. */ 58 int nr_pages; 59 /* Pages of this buffer. */ 60 struct page **pages; 61 }; 62 63 struct gntdev_dmabuf_wait_obj { 64 struct list_head next; 65 struct gntdev_dmabuf *gntdev_dmabuf; 66 struct completion completion; 67 }; 68 69 struct gntdev_dmabuf_attachment { 70 struct sg_table *sgt; 71 enum dma_data_direction dir; 72 }; 73 74 struct gntdev_dmabuf_priv { 75 /* List of exported DMA buffers. */ 76 struct list_head exp_list; 77 /* List of wait objects. */ 78 struct list_head exp_wait_list; 79 /* List of imported DMA buffers. */ 80 struct list_head imp_list; 81 /* This is the lock which protects dma_buf_xxx lists. */ 82 struct mutex lock; 83 /* 84 * We reference this file while exporting dma-bufs, so 85 * the grant device context is not destroyed while there are 86 * external users alive. 87 */ 88 struct file *filp; 89 }; 90 91 /* DMA buffer export support. */ 92 93 /* Implementation of wait for exported DMA buffer to be released. */ 94 95 static void dmabuf_exp_release(struct kref *kref); 96 97 static struct gntdev_dmabuf_wait_obj * 98 dmabuf_exp_wait_obj_new(struct gntdev_dmabuf_priv *priv, 99 struct gntdev_dmabuf *gntdev_dmabuf) 100 { 101 struct gntdev_dmabuf_wait_obj *obj; 102 103 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 104 if (!obj) 105 return ERR_PTR(-ENOMEM); 106 107 init_completion(&obj->completion); 108 obj->gntdev_dmabuf = gntdev_dmabuf; 109 110 mutex_lock(&priv->lock); 111 list_add(&obj->next, &priv->exp_wait_list); 112 /* Put our reference and wait for gntdev_dmabuf's release to fire. */ 113 kref_put(&gntdev_dmabuf->u.exp.refcount, dmabuf_exp_release); 114 mutex_unlock(&priv->lock); 115 return obj; 116 } 117 118 static void dmabuf_exp_wait_obj_free(struct gntdev_dmabuf_priv *priv, 119 struct gntdev_dmabuf_wait_obj *obj) 120 { 121 mutex_lock(&priv->lock); 122 list_del(&obj->next); 123 mutex_unlock(&priv->lock); 124 kfree(obj); 125 } 126 127 static int dmabuf_exp_wait_obj_wait(struct gntdev_dmabuf_wait_obj *obj, 128 u32 wait_to_ms) 129 { 130 if (wait_for_completion_timeout(&obj->completion, 131 msecs_to_jiffies(wait_to_ms)) <= 0) 132 return -ETIMEDOUT; 133 134 return 0; 135 } 136 137 static void dmabuf_exp_wait_obj_signal(struct gntdev_dmabuf_priv *priv, 138 struct gntdev_dmabuf *gntdev_dmabuf) 139 { 140 struct gntdev_dmabuf_wait_obj *obj; 141 142 list_for_each_entry(obj, &priv->exp_wait_list, next) 143 if (obj->gntdev_dmabuf == gntdev_dmabuf) { 144 pr_debug("Found gntdev_dmabuf in the wait list, wake\n"); 145 complete_all(&obj->completion); 146 break; 147 } 148 } 149 150 static struct gntdev_dmabuf * 151 dmabuf_exp_wait_obj_get_dmabuf(struct gntdev_dmabuf_priv *priv, int fd) 152 { 153 struct gntdev_dmabuf *gntdev_dmabuf, *ret = ERR_PTR(-ENOENT); 154 155 mutex_lock(&priv->lock); 156 list_for_each_entry(gntdev_dmabuf, &priv->exp_list, next) 157 if (gntdev_dmabuf->fd == fd) { 158 pr_debug("Found gntdev_dmabuf in the wait list\n"); 159 kref_get(&gntdev_dmabuf->u.exp.refcount); 160 ret = gntdev_dmabuf; 161 break; 162 } 163 mutex_unlock(&priv->lock); 164 return ret; 165 } 166 167 static int dmabuf_exp_wait_released(struct gntdev_dmabuf_priv *priv, int fd, 168 int wait_to_ms) 169 { 170 struct gntdev_dmabuf *gntdev_dmabuf; 171 struct gntdev_dmabuf_wait_obj *obj; 172 int ret; 173 174 pr_debug("Will wait for dma-buf with fd %d\n", fd); 175 /* 176 * Try to find the DMA buffer: if not found means that 177 * either the buffer has already been released or file descriptor 178 * provided is wrong. 179 */ 180 gntdev_dmabuf = dmabuf_exp_wait_obj_get_dmabuf(priv, fd); 181 if (IS_ERR(gntdev_dmabuf)) 182 return PTR_ERR(gntdev_dmabuf); 183 184 /* 185 * gntdev_dmabuf still exists and is reference count locked by us now, 186 * so prepare to wait: allocate wait object and add it to the wait list, 187 * so we can find it on release. 188 */ 189 obj = dmabuf_exp_wait_obj_new(priv, gntdev_dmabuf); 190 if (IS_ERR(obj)) 191 return PTR_ERR(obj); 192 193 ret = dmabuf_exp_wait_obj_wait(obj, wait_to_ms); 194 dmabuf_exp_wait_obj_free(priv, obj); 195 return ret; 196 } 197 198 /* DMA buffer export support. */ 199 200 static struct sg_table * 201 dmabuf_pages_to_sgt(struct page **pages, unsigned int nr_pages) 202 { 203 struct sg_table *sgt; 204 int ret; 205 206 sgt = kmalloc(sizeof(*sgt), GFP_KERNEL); 207 if (!sgt) { 208 ret = -ENOMEM; 209 goto out; 210 } 211 212 ret = sg_alloc_table_from_pages(sgt, pages, nr_pages, 0, 213 nr_pages << PAGE_SHIFT, 214 GFP_KERNEL); 215 if (ret) 216 goto out; 217 218 return sgt; 219 220 out: 221 kfree(sgt); 222 return ERR_PTR(ret); 223 } 224 225 static int dmabuf_exp_ops_attach(struct dma_buf *dma_buf, 226 struct dma_buf_attachment *attach) 227 { 228 struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach; 229 230 gntdev_dmabuf_attach = kzalloc(sizeof(*gntdev_dmabuf_attach), 231 GFP_KERNEL); 232 if (!gntdev_dmabuf_attach) 233 return -ENOMEM; 234 235 gntdev_dmabuf_attach->dir = DMA_NONE; 236 attach->priv = gntdev_dmabuf_attach; 237 return 0; 238 } 239 240 static void dmabuf_exp_ops_detach(struct dma_buf *dma_buf, 241 struct dma_buf_attachment *attach) 242 { 243 struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach = attach->priv; 244 245 if (gntdev_dmabuf_attach) { 246 struct sg_table *sgt = gntdev_dmabuf_attach->sgt; 247 248 if (sgt) { 249 if (gntdev_dmabuf_attach->dir != DMA_NONE) 250 dma_unmap_sg_attrs(attach->dev, sgt->sgl, 251 sgt->nents, 252 gntdev_dmabuf_attach->dir, 253 DMA_ATTR_SKIP_CPU_SYNC); 254 sg_free_table(sgt); 255 } 256 257 kfree(sgt); 258 kfree(gntdev_dmabuf_attach); 259 attach->priv = NULL; 260 } 261 } 262 263 static struct sg_table * 264 dmabuf_exp_ops_map_dma_buf(struct dma_buf_attachment *attach, 265 enum dma_data_direction dir) 266 { 267 struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach = attach->priv; 268 struct gntdev_dmabuf *gntdev_dmabuf = attach->dmabuf->priv; 269 struct sg_table *sgt; 270 271 pr_debug("Mapping %d pages for dev %p\n", gntdev_dmabuf->nr_pages, 272 attach->dev); 273 274 if (dir == DMA_NONE || !gntdev_dmabuf_attach) 275 return ERR_PTR(-EINVAL); 276 277 /* Return the cached mapping when possible. */ 278 if (gntdev_dmabuf_attach->dir == dir) 279 return gntdev_dmabuf_attach->sgt; 280 281 /* 282 * Two mappings with different directions for the same attachment are 283 * not allowed. 284 */ 285 if (gntdev_dmabuf_attach->dir != DMA_NONE) 286 return ERR_PTR(-EBUSY); 287 288 sgt = dmabuf_pages_to_sgt(gntdev_dmabuf->pages, 289 gntdev_dmabuf->nr_pages); 290 if (!IS_ERR(sgt)) { 291 if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir, 292 DMA_ATTR_SKIP_CPU_SYNC)) { 293 sg_free_table(sgt); 294 kfree(sgt); 295 sgt = ERR_PTR(-ENOMEM); 296 } else { 297 gntdev_dmabuf_attach->sgt = sgt; 298 gntdev_dmabuf_attach->dir = dir; 299 } 300 } 301 if (IS_ERR(sgt)) 302 pr_debug("Failed to map sg table for dev %p\n", attach->dev); 303 return sgt; 304 } 305 306 static void dmabuf_exp_ops_unmap_dma_buf(struct dma_buf_attachment *attach, 307 struct sg_table *sgt, 308 enum dma_data_direction dir) 309 { 310 /* Not implemented. The unmap is done at dmabuf_exp_ops_detach(). */ 311 } 312 313 static void dmabuf_exp_release(struct kref *kref) 314 { 315 struct gntdev_dmabuf *gntdev_dmabuf = 316 container_of(kref, struct gntdev_dmabuf, u.exp.refcount); 317 318 dmabuf_exp_wait_obj_signal(gntdev_dmabuf->priv, gntdev_dmabuf); 319 list_del(&gntdev_dmabuf->next); 320 fput(gntdev_dmabuf->priv->filp); 321 kfree(gntdev_dmabuf); 322 } 323 324 static void dmabuf_exp_remove_map(struct gntdev_priv *priv, 325 struct gntdev_grant_map *map) 326 { 327 mutex_lock(&priv->lock); 328 list_del(&map->next); 329 gntdev_put_map(NULL /* already removed */, map); 330 mutex_unlock(&priv->lock); 331 } 332 333 static void dmabuf_exp_ops_release(struct dma_buf *dma_buf) 334 { 335 struct gntdev_dmabuf *gntdev_dmabuf = dma_buf->priv; 336 struct gntdev_dmabuf_priv *priv = gntdev_dmabuf->priv; 337 338 dmabuf_exp_remove_map(gntdev_dmabuf->u.exp.priv, 339 gntdev_dmabuf->u.exp.map); 340 mutex_lock(&priv->lock); 341 kref_put(&gntdev_dmabuf->u.exp.refcount, dmabuf_exp_release); 342 mutex_unlock(&priv->lock); 343 } 344 345 static const struct dma_buf_ops dmabuf_exp_ops = { 346 .attach = dmabuf_exp_ops_attach, 347 .detach = dmabuf_exp_ops_detach, 348 .map_dma_buf = dmabuf_exp_ops_map_dma_buf, 349 .unmap_dma_buf = dmabuf_exp_ops_unmap_dma_buf, 350 .release = dmabuf_exp_ops_release, 351 }; 352 353 struct gntdev_dmabuf_export_args { 354 struct gntdev_priv *priv; 355 struct gntdev_grant_map *map; 356 struct gntdev_dmabuf_priv *dmabuf_priv; 357 struct device *dev; 358 int count; 359 struct page **pages; 360 u32 fd; 361 }; 362 363 static int dmabuf_exp_from_pages(struct gntdev_dmabuf_export_args *args) 364 { 365 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 366 struct gntdev_dmabuf *gntdev_dmabuf; 367 int ret; 368 369 gntdev_dmabuf = kzalloc(sizeof(*gntdev_dmabuf), GFP_KERNEL); 370 if (!gntdev_dmabuf) 371 return -ENOMEM; 372 373 kref_init(&gntdev_dmabuf->u.exp.refcount); 374 375 gntdev_dmabuf->priv = args->dmabuf_priv; 376 gntdev_dmabuf->nr_pages = args->count; 377 gntdev_dmabuf->pages = args->pages; 378 gntdev_dmabuf->u.exp.priv = args->priv; 379 gntdev_dmabuf->u.exp.map = args->map; 380 381 exp_info.exp_name = KBUILD_MODNAME; 382 if (args->dev->driver && args->dev->driver->owner) 383 exp_info.owner = args->dev->driver->owner; 384 else 385 exp_info.owner = THIS_MODULE; 386 exp_info.ops = &dmabuf_exp_ops; 387 exp_info.size = args->count << PAGE_SHIFT; 388 exp_info.flags = O_RDWR; 389 exp_info.priv = gntdev_dmabuf; 390 391 gntdev_dmabuf->dmabuf = dma_buf_export(&exp_info); 392 if (IS_ERR(gntdev_dmabuf->dmabuf)) { 393 ret = PTR_ERR(gntdev_dmabuf->dmabuf); 394 gntdev_dmabuf->dmabuf = NULL; 395 goto fail; 396 } 397 398 ret = dma_buf_fd(gntdev_dmabuf->dmabuf, O_CLOEXEC); 399 if (ret < 0) 400 goto fail; 401 402 gntdev_dmabuf->fd = ret; 403 args->fd = ret; 404 405 pr_debug("Exporting DMA buffer with fd %d\n", ret); 406 407 mutex_lock(&args->dmabuf_priv->lock); 408 list_add(&gntdev_dmabuf->next, &args->dmabuf_priv->exp_list); 409 mutex_unlock(&args->dmabuf_priv->lock); 410 get_file(gntdev_dmabuf->priv->filp); 411 return 0; 412 413 fail: 414 if (gntdev_dmabuf->dmabuf) 415 dma_buf_put(gntdev_dmabuf->dmabuf); 416 kfree(gntdev_dmabuf); 417 return ret; 418 } 419 420 static struct gntdev_grant_map * 421 dmabuf_exp_alloc_backing_storage(struct gntdev_priv *priv, int dmabuf_flags, 422 int count) 423 { 424 struct gntdev_grant_map *map; 425 426 if (unlikely(count <= 0)) 427 return ERR_PTR(-EINVAL); 428 429 if ((dmabuf_flags & GNTDEV_DMA_FLAG_WC) && 430 (dmabuf_flags & GNTDEV_DMA_FLAG_COHERENT)) { 431 pr_debug("Wrong dma-buf flags: 0x%x\n", dmabuf_flags); 432 return ERR_PTR(-EINVAL); 433 } 434 435 map = gntdev_alloc_map(priv, count, dmabuf_flags); 436 if (!map) 437 return ERR_PTR(-ENOMEM); 438 439 if (unlikely(gntdev_account_mapped_pages(count))) { 440 pr_debug("can't map %d pages: over limit\n", count); 441 gntdev_put_map(NULL, map); 442 return ERR_PTR(-ENOMEM); 443 } 444 return map; 445 } 446 447 static int dmabuf_exp_from_refs(struct gntdev_priv *priv, int flags, 448 int count, u32 domid, u32 *refs, u32 *fd) 449 { 450 struct gntdev_grant_map *map; 451 struct gntdev_dmabuf_export_args args; 452 int i, ret; 453 454 map = dmabuf_exp_alloc_backing_storage(priv, flags, count); 455 if (IS_ERR(map)) 456 return PTR_ERR(map); 457 458 for (i = 0; i < count; i++) { 459 map->grants[i].domid = domid; 460 map->grants[i].ref = refs[i]; 461 } 462 463 mutex_lock(&priv->lock); 464 gntdev_add_map(priv, map); 465 mutex_unlock(&priv->lock); 466 467 map->flags |= GNTMAP_host_map; 468 #if defined(CONFIG_X86) 469 map->flags |= GNTMAP_device_map; 470 #endif 471 472 ret = gntdev_map_grant_pages(map); 473 if (ret < 0) 474 goto out; 475 476 args.priv = priv; 477 args.map = map; 478 args.dev = priv->dma_dev; 479 args.dmabuf_priv = priv->dmabuf_priv; 480 args.count = map->count; 481 args.pages = map->pages; 482 args.fd = -1; /* Shut up unnecessary gcc warning for i386 */ 483 484 ret = dmabuf_exp_from_pages(&args); 485 if (ret < 0) 486 goto out; 487 488 *fd = args.fd; 489 return 0; 490 491 out: 492 dmabuf_exp_remove_map(priv, map); 493 return ret; 494 } 495 496 /* DMA buffer import support. */ 497 498 static int 499 dmabuf_imp_grant_foreign_access(struct page **pages, u32 *refs, 500 int count, int domid) 501 { 502 grant_ref_t priv_gref_head; 503 int i, ret; 504 505 ret = gnttab_alloc_grant_references(count, &priv_gref_head); 506 if (ret < 0) { 507 pr_debug("Cannot allocate grant references, ret %d\n", ret); 508 return ret; 509 } 510 511 for (i = 0; i < count; i++) { 512 int cur_ref; 513 514 cur_ref = gnttab_claim_grant_reference(&priv_gref_head); 515 if (cur_ref < 0) { 516 ret = cur_ref; 517 pr_debug("Cannot claim grant reference, ret %d\n", ret); 518 goto out; 519 } 520 521 gnttab_grant_foreign_access_ref(cur_ref, domid, 522 xen_page_to_gfn(pages[i]), 0); 523 refs[i] = cur_ref; 524 } 525 526 return 0; 527 528 out: 529 gnttab_free_grant_references(priv_gref_head); 530 return ret; 531 } 532 533 static void dmabuf_imp_end_foreign_access(u32 *refs, int count) 534 { 535 int i; 536 537 for (i = 0; i < count; i++) 538 if (refs[i] != GRANT_INVALID_REF) 539 gnttab_end_foreign_access(refs[i], 0, 0UL); 540 } 541 542 static void dmabuf_imp_free_storage(struct gntdev_dmabuf *gntdev_dmabuf) 543 { 544 kfree(gntdev_dmabuf->pages); 545 kfree(gntdev_dmabuf->u.imp.refs); 546 kfree(gntdev_dmabuf); 547 } 548 549 static struct gntdev_dmabuf *dmabuf_imp_alloc_storage(int count) 550 { 551 struct gntdev_dmabuf *gntdev_dmabuf; 552 int i; 553 554 gntdev_dmabuf = kzalloc(sizeof(*gntdev_dmabuf), GFP_KERNEL); 555 if (!gntdev_dmabuf) 556 goto fail_no_free; 557 558 gntdev_dmabuf->u.imp.refs = kcalloc(count, 559 sizeof(gntdev_dmabuf->u.imp.refs[0]), 560 GFP_KERNEL); 561 if (!gntdev_dmabuf->u.imp.refs) 562 goto fail; 563 564 gntdev_dmabuf->pages = kcalloc(count, 565 sizeof(gntdev_dmabuf->pages[0]), 566 GFP_KERNEL); 567 if (!gntdev_dmabuf->pages) 568 goto fail; 569 570 gntdev_dmabuf->nr_pages = count; 571 572 for (i = 0; i < count; i++) 573 gntdev_dmabuf->u.imp.refs[i] = GRANT_INVALID_REF; 574 575 return gntdev_dmabuf; 576 577 fail: 578 dmabuf_imp_free_storage(gntdev_dmabuf); 579 fail_no_free: 580 return ERR_PTR(-ENOMEM); 581 } 582 583 static struct gntdev_dmabuf * 584 dmabuf_imp_to_refs(struct gntdev_dmabuf_priv *priv, struct device *dev, 585 int fd, int count, int domid) 586 { 587 struct gntdev_dmabuf *gntdev_dmabuf, *ret; 588 struct dma_buf *dma_buf; 589 struct dma_buf_attachment *attach; 590 struct sg_table *sgt; 591 struct sg_page_iter sg_iter; 592 int i; 593 594 dma_buf = dma_buf_get(fd); 595 if (IS_ERR(dma_buf)) 596 return ERR_CAST(dma_buf); 597 598 gntdev_dmabuf = dmabuf_imp_alloc_storage(count); 599 if (IS_ERR(gntdev_dmabuf)) { 600 ret = gntdev_dmabuf; 601 goto fail_put; 602 } 603 604 gntdev_dmabuf->priv = priv; 605 gntdev_dmabuf->fd = fd; 606 607 attach = dma_buf_attach(dma_buf, dev); 608 if (IS_ERR(attach)) { 609 ret = ERR_CAST(attach); 610 goto fail_free_obj; 611 } 612 613 gntdev_dmabuf->u.imp.attach = attach; 614 615 sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); 616 if (IS_ERR(sgt)) { 617 ret = ERR_CAST(sgt); 618 goto fail_detach; 619 } 620 621 /* Check number of pages that imported buffer has. */ 622 if (attach->dmabuf->size != gntdev_dmabuf->nr_pages << PAGE_SHIFT) { 623 ret = ERR_PTR(-EINVAL); 624 pr_debug("DMA buffer has %zu pages, user-space expects %d\n", 625 attach->dmabuf->size, gntdev_dmabuf->nr_pages); 626 goto fail_unmap; 627 } 628 629 gntdev_dmabuf->u.imp.sgt = sgt; 630 631 /* Now convert sgt to array of pages and check for page validity. */ 632 i = 0; 633 for_each_sg_page(sgt->sgl, &sg_iter, sgt->nents, 0) { 634 struct page *page = sg_page_iter_page(&sg_iter); 635 /* 636 * Check if page is valid: this can happen if we are given 637 * a page from VRAM or other resources which are not backed 638 * by a struct page. 639 */ 640 if (!pfn_valid(page_to_pfn(page))) { 641 ret = ERR_PTR(-EINVAL); 642 goto fail_unmap; 643 } 644 645 gntdev_dmabuf->pages[i++] = page; 646 } 647 648 ret = ERR_PTR(dmabuf_imp_grant_foreign_access(gntdev_dmabuf->pages, 649 gntdev_dmabuf->u.imp.refs, 650 count, domid)); 651 if (IS_ERR(ret)) 652 goto fail_end_access; 653 654 pr_debug("Imported DMA buffer with fd %d\n", fd); 655 656 mutex_lock(&priv->lock); 657 list_add(&gntdev_dmabuf->next, &priv->imp_list); 658 mutex_unlock(&priv->lock); 659 660 return gntdev_dmabuf; 661 662 fail_end_access: 663 dmabuf_imp_end_foreign_access(gntdev_dmabuf->u.imp.refs, count); 664 fail_unmap: 665 dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL); 666 fail_detach: 667 dma_buf_detach(dma_buf, attach); 668 fail_free_obj: 669 dmabuf_imp_free_storage(gntdev_dmabuf); 670 fail_put: 671 dma_buf_put(dma_buf); 672 return ret; 673 } 674 675 /* 676 * Find the hyper dma-buf by its file descriptor and remove 677 * it from the buffer's list. 678 */ 679 static struct gntdev_dmabuf * 680 dmabuf_imp_find_unlink(struct gntdev_dmabuf_priv *priv, int fd) 681 { 682 struct gntdev_dmabuf *q, *gntdev_dmabuf, *ret = ERR_PTR(-ENOENT); 683 684 mutex_lock(&priv->lock); 685 list_for_each_entry_safe(gntdev_dmabuf, q, &priv->imp_list, next) { 686 if (gntdev_dmabuf->fd == fd) { 687 pr_debug("Found gntdev_dmabuf in the import list\n"); 688 ret = gntdev_dmabuf; 689 list_del(&gntdev_dmabuf->next); 690 break; 691 } 692 } 693 mutex_unlock(&priv->lock); 694 return ret; 695 } 696 697 static int dmabuf_imp_release(struct gntdev_dmabuf_priv *priv, u32 fd) 698 { 699 struct gntdev_dmabuf *gntdev_dmabuf; 700 struct dma_buf_attachment *attach; 701 struct dma_buf *dma_buf; 702 703 gntdev_dmabuf = dmabuf_imp_find_unlink(priv, fd); 704 if (IS_ERR(gntdev_dmabuf)) 705 return PTR_ERR(gntdev_dmabuf); 706 707 pr_debug("Releasing DMA buffer with fd %d\n", fd); 708 709 dmabuf_imp_end_foreign_access(gntdev_dmabuf->u.imp.refs, 710 gntdev_dmabuf->nr_pages); 711 712 attach = gntdev_dmabuf->u.imp.attach; 713 714 if (gntdev_dmabuf->u.imp.sgt) 715 dma_buf_unmap_attachment(attach, gntdev_dmabuf->u.imp.sgt, 716 DMA_BIDIRECTIONAL); 717 dma_buf = attach->dmabuf; 718 dma_buf_detach(attach->dmabuf, attach); 719 dma_buf_put(dma_buf); 720 721 dmabuf_imp_free_storage(gntdev_dmabuf); 722 return 0; 723 } 724 725 static void dmabuf_imp_release_all(struct gntdev_dmabuf_priv *priv) 726 { 727 struct gntdev_dmabuf *q, *gntdev_dmabuf; 728 729 list_for_each_entry_safe(gntdev_dmabuf, q, &priv->imp_list, next) 730 dmabuf_imp_release(priv, gntdev_dmabuf->fd); 731 } 732 733 /* DMA buffer IOCTL support. */ 734 735 long gntdev_ioctl_dmabuf_exp_from_refs(struct gntdev_priv *priv, int use_ptemod, 736 struct ioctl_gntdev_dmabuf_exp_from_refs __user *u) 737 { 738 struct ioctl_gntdev_dmabuf_exp_from_refs op; 739 u32 *refs; 740 long ret; 741 742 if (use_ptemod) { 743 pr_debug("Cannot provide dma-buf: use_ptemode %d\n", 744 use_ptemod); 745 return -EINVAL; 746 } 747 748 if (copy_from_user(&op, u, sizeof(op)) != 0) 749 return -EFAULT; 750 751 if (unlikely(op.count <= 0)) 752 return -EINVAL; 753 754 refs = kcalloc(op.count, sizeof(*refs), GFP_KERNEL); 755 if (!refs) 756 return -ENOMEM; 757 758 if (copy_from_user(refs, u->refs, sizeof(*refs) * op.count) != 0) { 759 ret = -EFAULT; 760 goto out; 761 } 762 763 ret = dmabuf_exp_from_refs(priv, op.flags, op.count, 764 op.domid, refs, &op.fd); 765 if (ret) 766 goto out; 767 768 if (copy_to_user(u, &op, sizeof(op)) != 0) 769 ret = -EFAULT; 770 771 out: 772 kfree(refs); 773 return ret; 774 } 775 776 long gntdev_ioctl_dmabuf_exp_wait_released(struct gntdev_priv *priv, 777 struct ioctl_gntdev_dmabuf_exp_wait_released __user *u) 778 { 779 struct ioctl_gntdev_dmabuf_exp_wait_released op; 780 781 if (copy_from_user(&op, u, sizeof(op)) != 0) 782 return -EFAULT; 783 784 return dmabuf_exp_wait_released(priv->dmabuf_priv, op.fd, 785 op.wait_to_ms); 786 } 787 788 long gntdev_ioctl_dmabuf_imp_to_refs(struct gntdev_priv *priv, 789 struct ioctl_gntdev_dmabuf_imp_to_refs __user *u) 790 { 791 struct ioctl_gntdev_dmabuf_imp_to_refs op; 792 struct gntdev_dmabuf *gntdev_dmabuf; 793 long ret; 794 795 if (copy_from_user(&op, u, sizeof(op)) != 0) 796 return -EFAULT; 797 798 if (unlikely(op.count <= 0)) 799 return -EINVAL; 800 801 gntdev_dmabuf = dmabuf_imp_to_refs(priv->dmabuf_priv, 802 priv->dma_dev, op.fd, 803 op.count, op.domid); 804 if (IS_ERR(gntdev_dmabuf)) 805 return PTR_ERR(gntdev_dmabuf); 806 807 if (copy_to_user(u->refs, gntdev_dmabuf->u.imp.refs, 808 sizeof(*u->refs) * op.count) != 0) { 809 ret = -EFAULT; 810 goto out_release; 811 } 812 return 0; 813 814 out_release: 815 dmabuf_imp_release(priv->dmabuf_priv, op.fd); 816 return ret; 817 } 818 819 long gntdev_ioctl_dmabuf_imp_release(struct gntdev_priv *priv, 820 struct ioctl_gntdev_dmabuf_imp_release __user *u) 821 { 822 struct ioctl_gntdev_dmabuf_imp_release op; 823 824 if (copy_from_user(&op, u, sizeof(op)) != 0) 825 return -EFAULT; 826 827 return dmabuf_imp_release(priv->dmabuf_priv, op.fd); 828 } 829 830 struct gntdev_dmabuf_priv *gntdev_dmabuf_init(struct file *filp) 831 { 832 struct gntdev_dmabuf_priv *priv; 833 834 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 835 if (!priv) 836 return ERR_PTR(-ENOMEM); 837 838 mutex_init(&priv->lock); 839 INIT_LIST_HEAD(&priv->exp_list); 840 INIT_LIST_HEAD(&priv->exp_wait_list); 841 INIT_LIST_HEAD(&priv->imp_list); 842 843 priv->filp = filp; 844 845 return priv; 846 } 847 848 void gntdev_dmabuf_fini(struct gntdev_dmabuf_priv *priv) 849 { 850 dmabuf_imp_release_all(priv); 851 kfree(priv); 852 } 853