1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_cache.h> 8 #include <drm/drm_device.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/gpu_scheduler.h> 12 #include <linux/dma-buf.h> 13 #include <linux/dma-direct.h> 14 #include <linux/iosys-map.h> 15 #include <linux/pagemap.h> 16 #include <linux/vmalloc.h> 17 18 #include "amdxdna_ctx.h" 19 #include "amdxdna_gem.h" 20 #include "amdxdna_pci_drv.h" 21 #include "amdxdna_ubuf.h" 22 23 #define XDNA_MAX_CMD_BO_SIZE SZ_32K 24 25 MODULE_IMPORT_NS("DMA_BUF"); 26 27 static int 28 amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo) 29 { 30 struct amdxdna_client *client = abo->client; 31 struct amdxdna_dev *xdna = client->xdna; 32 struct amdxdna_mem *mem = &abo->mem; 33 struct amdxdna_gem_obj *heap; 34 u64 offset; 35 u32 align; 36 int ret; 37 38 mutex_lock(&client->mm_lock); 39 40 heap = client->dev_heap; 41 if (!heap) { 42 ret = -EINVAL; 43 goto unlock_out; 44 } 45 46 if (heap->mem.userptr == AMDXDNA_INVALID_ADDR) { 47 XDNA_ERR(xdna, "Invalid dev heap userptr"); 48 ret = -EINVAL; 49 goto unlock_out; 50 } 51 52 if (mem->size == 0 || mem->size > heap->mem.size) { 53 XDNA_ERR(xdna, "Invalid dev bo size 0x%lx, limit 0x%lx", 54 mem->size, heap->mem.size); 55 ret = -EINVAL; 56 goto unlock_out; 57 } 58 59 align = 1 << max(PAGE_SHIFT, xdna->dev_info->dev_mem_buf_shift); 60 ret = drm_mm_insert_node_generic(&heap->mm, &abo->mm_node, 61 mem->size, align, 62 0, DRM_MM_INSERT_BEST); 63 if (ret) { 64 XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret); 65 goto unlock_out; 66 } 67 68 mem->dev_addr = abo->mm_node.start; 69 offset = mem->dev_addr - heap->mem.dev_addr; 70 mem->userptr = heap->mem.userptr + offset; 71 mem->kva = heap->mem.kva + offset; 72 73 drm_gem_object_get(to_gobj(heap)); 74 75 unlock_out: 76 mutex_unlock(&client->mm_lock); 77 78 return ret; 79 } 80 81 static void 82 amdxdna_gem_destroy_obj(struct amdxdna_gem_obj *abo) 83 { 84 mutex_destroy(&abo->lock); 85 kfree(abo); 86 } 87 88 static void 89 amdxdna_gem_heap_free(struct amdxdna_gem_obj *abo) 90 { 91 struct amdxdna_gem_obj *heap; 92 93 mutex_lock(&abo->client->mm_lock); 94 95 drm_mm_remove_node(&abo->mm_node); 96 97 heap = abo->client->dev_heap; 98 drm_gem_object_put(to_gobj(heap)); 99 100 mutex_unlock(&abo->client->mm_lock); 101 } 102 103 static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni, 104 const struct mmu_notifier_range *range, 105 unsigned long cur_seq) 106 { 107 struct amdxdna_umap *mapp = container_of(mni, struct amdxdna_umap, notifier); 108 struct amdxdna_gem_obj *abo = mapp->abo; 109 struct amdxdna_dev *xdna; 110 111 xdna = to_xdna_dev(to_gobj(abo)->dev); 112 XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d", 113 mapp->vma->vm_start, mapp->vma->vm_end, abo->type); 114 115 if (!mmu_notifier_range_blockable(range)) 116 return false; 117 118 down_write(&xdna->notifier_lock); 119 abo->mem.map_invalid = true; 120 mapp->invalid = true; 121 mmu_interval_set_seq(&mapp->notifier, cur_seq); 122 up_write(&xdna->notifier_lock); 123 124 xdna->dev_info->ops->hmm_invalidate(abo, cur_seq); 125 126 if (range->event == MMU_NOTIFY_UNMAP) { 127 down_write(&xdna->notifier_lock); 128 if (!mapp->unmapped) { 129 queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work); 130 mapp->unmapped = true; 131 } 132 up_write(&xdna->notifier_lock); 133 } 134 135 return true; 136 } 137 138 static const struct mmu_interval_notifier_ops amdxdna_hmm_ops = { 139 .invalidate = amdxdna_hmm_invalidate, 140 }; 141 142 static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo, 143 struct vm_area_struct *vma) 144 { 145 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 146 struct amdxdna_umap *mapp; 147 148 down_read(&xdna->notifier_lock); 149 list_for_each_entry(mapp, &abo->mem.umap_list, node) { 150 if (!vma || mapp->vma == vma) { 151 if (!mapp->unmapped) { 152 queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work); 153 mapp->unmapped = true; 154 } 155 if (vma) 156 break; 157 } 158 } 159 up_read(&xdna->notifier_lock); 160 } 161 162 static void amdxdna_umap_release(struct kref *ref) 163 { 164 struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt); 165 struct vm_area_struct *vma = mapp->vma; 166 struct amdxdna_dev *xdna; 167 168 mmu_interval_notifier_remove(&mapp->notifier); 169 if (is_import_bo(mapp->abo) && vma->vm_file && vma->vm_file->f_mapping) 170 mapping_clear_unevictable(vma->vm_file->f_mapping); 171 172 xdna = to_xdna_dev(to_gobj(mapp->abo)->dev); 173 down_write(&xdna->notifier_lock); 174 list_del(&mapp->node); 175 up_write(&xdna->notifier_lock); 176 177 kvfree(mapp->range.hmm_pfns); 178 kfree(mapp); 179 } 180 181 void amdxdna_umap_put(struct amdxdna_umap *mapp) 182 { 183 kref_put(&mapp->refcnt, amdxdna_umap_release); 184 } 185 186 static void amdxdna_hmm_unreg_work(struct work_struct *work) 187 { 188 struct amdxdna_umap *mapp = container_of(work, struct amdxdna_umap, 189 hmm_unreg_work); 190 191 amdxdna_umap_put(mapp); 192 } 193 194 static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, 195 struct vm_area_struct *vma) 196 { 197 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 198 unsigned long len = vma->vm_end - vma->vm_start; 199 unsigned long addr = vma->vm_start; 200 struct amdxdna_umap *mapp; 201 u32 nr_pages; 202 int ret; 203 204 if (!xdna->dev_info->ops->hmm_invalidate) 205 return 0; 206 207 mapp = kzalloc(sizeof(*mapp), GFP_KERNEL); 208 if (!mapp) 209 return -ENOMEM; 210 211 nr_pages = (PAGE_ALIGN(addr + len) - (addr & PAGE_MASK)) >> PAGE_SHIFT; 212 mapp->range.hmm_pfns = kvcalloc(nr_pages, sizeof(*mapp->range.hmm_pfns), 213 GFP_KERNEL); 214 if (!mapp->range.hmm_pfns) { 215 ret = -ENOMEM; 216 goto free_map; 217 } 218 219 ret = mmu_interval_notifier_insert_locked(&mapp->notifier, 220 current->mm, 221 addr, 222 len, 223 &amdxdna_hmm_ops); 224 if (ret) { 225 XDNA_ERR(xdna, "Insert mmu notifier failed, ret %d", ret); 226 goto free_pfns; 227 } 228 229 mapp->range.notifier = &mapp->notifier; 230 mapp->range.start = vma->vm_start; 231 mapp->range.end = vma->vm_end; 232 mapp->range.default_flags = HMM_PFN_REQ_FAULT; 233 mapp->vma = vma; 234 mapp->abo = abo; 235 kref_init(&mapp->refcnt); 236 237 if (abo->mem.userptr == AMDXDNA_INVALID_ADDR) 238 abo->mem.userptr = addr; 239 INIT_WORK(&mapp->hmm_unreg_work, amdxdna_hmm_unreg_work); 240 if (is_import_bo(abo) && vma->vm_file && vma->vm_file->f_mapping) 241 mapping_set_unevictable(vma->vm_file->f_mapping); 242 243 down_write(&xdna->notifier_lock); 244 list_add_tail(&mapp->node, &abo->mem.umap_list); 245 up_write(&xdna->notifier_lock); 246 247 return 0; 248 249 free_pfns: 250 kvfree(mapp->range.hmm_pfns); 251 free_map: 252 kfree(mapp); 253 return ret; 254 } 255 256 static void amdxdna_gem_dev_obj_free(struct drm_gem_object *gobj) 257 { 258 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); 259 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 260 261 XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, abo->mem.dev_addr); 262 if (abo->pinned) 263 amdxdna_gem_unpin(abo); 264 265 amdxdna_gem_heap_free(abo); 266 drm_gem_object_release(gobj); 267 amdxdna_gem_destroy_obj(abo); 268 } 269 270 static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo, 271 struct vm_area_struct *vma) 272 { 273 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 274 unsigned long num_pages = vma_pages(vma); 275 unsigned long offset = 0; 276 int ret; 277 278 if (!is_import_bo(abo)) { 279 ret = drm_gem_shmem_mmap(&abo->base, vma); 280 if (ret) { 281 XDNA_ERR(xdna, "Failed shmem mmap %d", ret); 282 return ret; 283 } 284 285 /* The buffer is based on memory pages. Fix the flag. */ 286 vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP); 287 ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages, 288 &num_pages); 289 if (ret) { 290 XDNA_ERR(xdna, "Failed insert pages %d", ret); 291 vma->vm_ops->close(vma); 292 return ret; 293 } 294 295 return 0; 296 } 297 298 vma->vm_private_data = NULL; 299 vma->vm_ops = NULL; 300 ret = dma_buf_mmap(abo->dma_buf, vma, 0); 301 if (ret) { 302 XDNA_ERR(xdna, "Failed to mmap dma buf %d", ret); 303 return ret; 304 } 305 306 do { 307 vm_fault_t fault_ret; 308 309 fault_ret = handle_mm_fault(vma, vma->vm_start + offset, 310 FAULT_FLAG_WRITE, NULL); 311 if (fault_ret & VM_FAULT_ERROR) { 312 vma->vm_ops->close(vma); 313 XDNA_ERR(xdna, "Fault in page failed"); 314 return -EFAULT; 315 } 316 317 offset += PAGE_SIZE; 318 } while (--num_pages); 319 320 /* Drop the reference drm_gem_mmap_obj() acquired.*/ 321 drm_gem_object_put(to_gobj(abo)); 322 323 return 0; 324 } 325 326 static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj, 327 struct vm_area_struct *vma) 328 { 329 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); 330 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 331 int ret; 332 333 ret = amdxdna_hmm_register(abo, vma); 334 if (ret) 335 return ret; 336 337 ret = amdxdna_insert_pages(abo, vma); 338 if (ret) { 339 XDNA_ERR(xdna, "Failed insert pages, ret %d", ret); 340 goto hmm_unreg; 341 } 342 343 XDNA_DBG(xdna, "BO map_offset 0x%llx type %d userptr 0x%lx size 0x%lx", 344 drm_vma_node_offset_addr(&gobj->vma_node), abo->type, 345 vma->vm_start, gobj->size); 346 return 0; 347 348 hmm_unreg: 349 amdxdna_hmm_unregister(abo, vma); 350 return ret; 351 } 352 353 static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) 354 { 355 struct drm_gem_object *gobj = dma_buf->priv; 356 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 357 unsigned long num_pages = vma_pages(vma); 358 int ret; 359 360 vma->vm_ops = &drm_gem_shmem_vm_ops; 361 vma->vm_private_data = gobj; 362 363 drm_gem_object_get(gobj); 364 ret = drm_gem_shmem_mmap(&abo->base, vma); 365 if (ret) 366 goto put_obj; 367 368 /* The buffer is based on memory pages. Fix the flag. */ 369 vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP); 370 ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages, 371 &num_pages); 372 if (ret) 373 goto close_vma; 374 375 return 0; 376 377 close_vma: 378 vma->vm_ops->close(vma); 379 put_obj: 380 drm_gem_object_put(gobj); 381 return ret; 382 } 383 384 static const struct dma_buf_ops amdxdna_dmabuf_ops = { 385 .attach = drm_gem_map_attach, 386 .detach = drm_gem_map_detach, 387 .map_dma_buf = drm_gem_map_dma_buf, 388 .unmap_dma_buf = drm_gem_unmap_dma_buf, 389 .release = drm_gem_dmabuf_release, 390 .mmap = amdxdna_gem_dmabuf_mmap, 391 .vmap = drm_gem_dmabuf_vmap, 392 .vunmap = drm_gem_dmabuf_vunmap, 393 }; 394 395 static int amdxdna_gem_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map) 396 { 397 struct amdxdna_gem_obj *abo = to_xdna_obj(obj); 398 399 iosys_map_clear(map); 400 401 dma_resv_assert_held(obj->resv); 402 403 if (is_import_bo(abo)) 404 dma_buf_vmap(abo->dma_buf, map); 405 else 406 drm_gem_shmem_object_vmap(obj, map); 407 408 if (!map->vaddr) 409 return -ENOMEM; 410 411 return 0; 412 } 413 414 static void amdxdna_gem_obj_vunmap(struct drm_gem_object *obj, struct iosys_map *map) 415 { 416 struct amdxdna_gem_obj *abo = to_xdna_obj(obj); 417 418 dma_resv_assert_held(obj->resv); 419 420 if (is_import_bo(abo)) 421 dma_buf_vunmap(abo->dma_buf, map); 422 else 423 drm_gem_shmem_object_vunmap(obj, map); 424 } 425 426 static struct dma_buf *amdxdna_gem_prime_export(struct drm_gem_object *gobj, int flags) 427 { 428 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 429 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 430 431 if (abo->dma_buf) { 432 get_dma_buf(abo->dma_buf); 433 return abo->dma_buf; 434 } 435 436 exp_info.ops = &amdxdna_dmabuf_ops; 437 exp_info.size = gobj->size; 438 exp_info.flags = flags; 439 exp_info.priv = gobj; 440 exp_info.resv = gobj->resv; 441 442 return drm_gem_dmabuf_export(gobj->dev, &exp_info); 443 } 444 445 static void amdxdna_imported_obj_free(struct amdxdna_gem_obj *abo) 446 { 447 dma_buf_unmap_attachment_unlocked(abo->attach, abo->base.sgt, DMA_BIDIRECTIONAL); 448 dma_buf_detach(abo->dma_buf, abo->attach); 449 dma_buf_put(abo->dma_buf); 450 drm_gem_object_release(to_gobj(abo)); 451 kfree(abo); 452 } 453 454 static void amdxdna_gem_obj_free(struct drm_gem_object *gobj) 455 { 456 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); 457 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 458 struct iosys_map map = IOSYS_MAP_INIT_VADDR(abo->mem.kva); 459 460 XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, abo->mem.dev_addr); 461 462 amdxdna_hmm_unregister(abo, NULL); 463 flush_workqueue(xdna->notifier_wq); 464 465 if (abo->pinned) 466 amdxdna_gem_unpin(abo); 467 468 if (abo->type == AMDXDNA_BO_DEV_HEAP) 469 drm_mm_takedown(&abo->mm); 470 471 drm_gem_vunmap(gobj, &map); 472 mutex_destroy(&abo->lock); 473 474 if (is_import_bo(abo)) { 475 amdxdna_imported_obj_free(abo); 476 return; 477 } 478 479 drm_gem_shmem_free(&abo->base); 480 } 481 482 static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = { 483 .free = amdxdna_gem_dev_obj_free, 484 }; 485 486 static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = { 487 .free = amdxdna_gem_obj_free, 488 .print_info = drm_gem_shmem_object_print_info, 489 .pin = drm_gem_shmem_object_pin, 490 .unpin = drm_gem_shmem_object_unpin, 491 .get_sg_table = drm_gem_shmem_object_get_sg_table, 492 .vmap = amdxdna_gem_obj_vmap, 493 .vunmap = amdxdna_gem_obj_vunmap, 494 .mmap = amdxdna_gem_obj_mmap, 495 .vm_ops = &drm_gem_shmem_vm_ops, 496 .export = amdxdna_gem_prime_export, 497 }; 498 499 static struct amdxdna_gem_obj * 500 amdxdna_gem_create_obj(struct drm_device *dev, size_t size) 501 { 502 struct amdxdna_gem_obj *abo; 503 504 abo = kzalloc(sizeof(*abo), GFP_KERNEL); 505 if (!abo) 506 return ERR_PTR(-ENOMEM); 507 508 abo->pinned = false; 509 abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE; 510 mutex_init(&abo->lock); 511 512 abo->mem.userptr = AMDXDNA_INVALID_ADDR; 513 abo->mem.dev_addr = AMDXDNA_INVALID_ADDR; 514 abo->mem.size = size; 515 INIT_LIST_HEAD(&abo->mem.umap_list); 516 517 return abo; 518 } 519 520 /* For drm_driver->gem_create_object callback */ 521 struct drm_gem_object * 522 amdxdna_gem_create_object_cb(struct drm_device *dev, size_t size) 523 { 524 struct amdxdna_gem_obj *abo; 525 526 abo = amdxdna_gem_create_obj(dev, size); 527 if (IS_ERR(abo)) 528 return ERR_CAST(abo); 529 530 to_gobj(abo)->funcs = &amdxdna_gem_shmem_funcs; 531 532 return to_gobj(abo); 533 } 534 535 static struct amdxdna_gem_obj * 536 amdxdna_gem_create_shmem_object(struct drm_device *dev, size_t size) 537 { 538 struct drm_gem_shmem_object *shmem = drm_gem_shmem_create(dev, size); 539 540 if (IS_ERR(shmem)) 541 return ERR_CAST(shmem); 542 543 shmem->map_wc = false; 544 return to_xdna_obj(&shmem->base); 545 } 546 547 static struct amdxdna_gem_obj * 548 amdxdna_gem_create_ubuf_object(struct drm_device *dev, struct amdxdna_drm_create_bo *args) 549 { 550 struct amdxdna_dev *xdna = to_xdna_dev(dev); 551 enum amdxdna_ubuf_flag flags = 0; 552 struct amdxdna_drm_va_tbl va_tbl; 553 struct drm_gem_object *gobj; 554 struct dma_buf *dma_buf; 555 556 if (copy_from_user(&va_tbl, u64_to_user_ptr(args->vaddr), sizeof(va_tbl))) { 557 XDNA_DBG(xdna, "Access va table failed"); 558 return ERR_PTR(-EINVAL); 559 } 560 561 if (va_tbl.num_entries) { 562 if (args->type == AMDXDNA_BO_CMD) 563 flags |= AMDXDNA_UBUF_FLAG_MAP_DMA; 564 565 dma_buf = amdxdna_get_ubuf(dev, flags, va_tbl.num_entries, 566 u64_to_user_ptr(args->vaddr + sizeof(va_tbl))); 567 } else { 568 dma_buf = dma_buf_get(va_tbl.dmabuf_fd); 569 } 570 571 if (IS_ERR(dma_buf)) 572 return ERR_CAST(dma_buf); 573 574 gobj = amdxdna_gem_prime_import(dev, dma_buf); 575 if (IS_ERR(gobj)) { 576 dma_buf_put(dma_buf); 577 return ERR_CAST(gobj); 578 } 579 580 dma_buf_put(dma_buf); 581 582 return to_xdna_obj(gobj); 583 } 584 585 static struct amdxdna_gem_obj * 586 amdxdna_gem_create_object(struct drm_device *dev, 587 struct amdxdna_drm_create_bo *args) 588 { 589 size_t aligned_sz = PAGE_ALIGN(args->size); 590 591 if (args->vaddr) 592 return amdxdna_gem_create_ubuf_object(dev, args); 593 594 return amdxdna_gem_create_shmem_object(dev, aligned_sz); 595 } 596 597 struct drm_gem_object * 598 amdxdna_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf) 599 { 600 struct dma_buf_attachment *attach; 601 struct amdxdna_gem_obj *abo; 602 struct drm_gem_object *gobj; 603 struct sg_table *sgt; 604 int ret; 605 606 get_dma_buf(dma_buf); 607 608 attach = dma_buf_attach(dma_buf, dev->dev); 609 if (IS_ERR(attach)) { 610 ret = PTR_ERR(attach); 611 goto put_buf; 612 } 613 614 sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL); 615 if (IS_ERR(sgt)) { 616 ret = PTR_ERR(sgt); 617 goto fail_detach; 618 } 619 620 gobj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); 621 if (IS_ERR(gobj)) { 622 ret = PTR_ERR(gobj); 623 goto fail_unmap; 624 } 625 626 abo = to_xdna_obj(gobj); 627 abo->attach = attach; 628 abo->dma_buf = dma_buf; 629 630 return gobj; 631 632 fail_unmap: 633 dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL); 634 fail_detach: 635 dma_buf_detach(dma_buf, attach); 636 put_buf: 637 dma_buf_put(dma_buf); 638 639 return ERR_PTR(ret); 640 } 641 642 static struct amdxdna_gem_obj * 643 amdxdna_drm_alloc_shmem(struct drm_device *dev, 644 struct amdxdna_drm_create_bo *args, 645 struct drm_file *filp) 646 { 647 struct amdxdna_client *client = filp->driver_priv; 648 struct amdxdna_gem_obj *abo; 649 650 abo = amdxdna_gem_create_object(dev, args); 651 if (IS_ERR(abo)) 652 return ERR_CAST(abo); 653 654 abo->client = client; 655 abo->type = AMDXDNA_BO_SHMEM; 656 657 return abo; 658 } 659 660 static struct amdxdna_gem_obj * 661 amdxdna_drm_create_dev_heap(struct drm_device *dev, 662 struct amdxdna_drm_create_bo *args, 663 struct drm_file *filp) 664 { 665 struct amdxdna_client *client = filp->driver_priv; 666 struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); 667 struct amdxdna_dev *xdna = to_xdna_dev(dev); 668 struct amdxdna_gem_obj *abo; 669 int ret; 670 671 if (args->size > xdna->dev_info->dev_mem_size) { 672 XDNA_DBG(xdna, "Invalid dev heap size 0x%llx, limit 0x%lx", 673 args->size, xdna->dev_info->dev_mem_size); 674 return ERR_PTR(-EINVAL); 675 } 676 677 mutex_lock(&client->mm_lock); 678 if (client->dev_heap) { 679 XDNA_DBG(client->xdna, "dev heap is already created"); 680 ret = -EBUSY; 681 goto mm_unlock; 682 } 683 684 abo = amdxdna_gem_create_object(dev, args); 685 if (IS_ERR(abo)) { 686 ret = PTR_ERR(abo); 687 goto mm_unlock; 688 } 689 690 abo->type = AMDXDNA_BO_DEV_HEAP; 691 abo->client = client; 692 abo->mem.dev_addr = client->xdna->dev_info->dev_mem_base; 693 drm_mm_init(&abo->mm, abo->mem.dev_addr, abo->mem.size); 694 695 ret = drm_gem_vmap(to_gobj(abo), &map); 696 if (ret) { 697 XDNA_ERR(xdna, "Vmap heap bo failed, ret %d", ret); 698 goto release_obj; 699 } 700 abo->mem.kva = map.vaddr; 701 702 client->dev_heap = abo; 703 drm_gem_object_get(to_gobj(abo)); 704 mutex_unlock(&client->mm_lock); 705 706 return abo; 707 708 release_obj: 709 drm_gem_object_put(to_gobj(abo)); 710 mm_unlock: 711 mutex_unlock(&client->mm_lock); 712 return ERR_PTR(ret); 713 } 714 715 struct amdxdna_gem_obj * 716 amdxdna_drm_alloc_dev_bo(struct drm_device *dev, 717 struct amdxdna_drm_create_bo *args, 718 struct drm_file *filp) 719 { 720 struct amdxdna_client *client = filp->driver_priv; 721 struct amdxdna_dev *xdna = to_xdna_dev(dev); 722 size_t aligned_sz = PAGE_ALIGN(args->size); 723 struct amdxdna_gem_obj *abo; 724 int ret; 725 726 abo = amdxdna_gem_create_obj(&xdna->ddev, aligned_sz); 727 if (IS_ERR(abo)) 728 return abo; 729 730 to_gobj(abo)->funcs = &amdxdna_gem_dev_obj_funcs; 731 abo->type = AMDXDNA_BO_DEV; 732 abo->client = client; 733 734 ret = amdxdna_gem_heap_alloc(abo); 735 if (ret) { 736 XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret); 737 amdxdna_gem_destroy_obj(abo); 738 return ERR_PTR(ret); 739 } 740 741 drm_gem_private_object_init(&xdna->ddev, to_gobj(abo), aligned_sz); 742 743 return abo; 744 } 745 746 static struct amdxdna_gem_obj * 747 amdxdna_drm_create_cmd_bo(struct drm_device *dev, 748 struct amdxdna_drm_create_bo *args, 749 struct drm_file *filp) 750 { 751 struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); 752 struct amdxdna_dev *xdna = to_xdna_dev(dev); 753 struct amdxdna_gem_obj *abo; 754 int ret; 755 756 if (args->size > XDNA_MAX_CMD_BO_SIZE) { 757 XDNA_ERR(xdna, "Command bo size 0x%llx too large", args->size); 758 return ERR_PTR(-EINVAL); 759 } 760 761 if (args->size < sizeof(struct amdxdna_cmd)) { 762 XDNA_DBG(xdna, "Command BO size 0x%llx too small", args->size); 763 return ERR_PTR(-EINVAL); 764 } 765 766 abo = amdxdna_gem_create_object(dev, args); 767 if (IS_ERR(abo)) 768 return ERR_CAST(abo); 769 770 abo->type = AMDXDNA_BO_CMD; 771 abo->client = filp->driver_priv; 772 773 ret = drm_gem_vmap(to_gobj(abo), &map); 774 if (ret) { 775 XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret); 776 goto release_obj; 777 } 778 abo->mem.kva = map.vaddr; 779 780 return abo; 781 782 release_obj: 783 drm_gem_object_put(to_gobj(abo)); 784 return ERR_PTR(ret); 785 } 786 787 int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 788 { 789 struct amdxdna_dev *xdna = to_xdna_dev(dev); 790 struct amdxdna_drm_create_bo *args = data; 791 struct amdxdna_gem_obj *abo; 792 int ret; 793 794 if (args->flags) 795 return -EINVAL; 796 797 XDNA_DBG(xdna, "BO arg type %d vaddr 0x%llx size 0x%llx flags 0x%llx", 798 args->type, args->vaddr, args->size, args->flags); 799 switch (args->type) { 800 case AMDXDNA_BO_SHMEM: 801 abo = amdxdna_drm_alloc_shmem(dev, args, filp); 802 break; 803 case AMDXDNA_BO_DEV_HEAP: 804 abo = amdxdna_drm_create_dev_heap(dev, args, filp); 805 break; 806 case AMDXDNA_BO_DEV: 807 abo = amdxdna_drm_alloc_dev_bo(dev, args, filp); 808 break; 809 case AMDXDNA_BO_CMD: 810 abo = amdxdna_drm_create_cmd_bo(dev, args, filp); 811 break; 812 default: 813 return -EINVAL; 814 } 815 if (IS_ERR(abo)) 816 return PTR_ERR(abo); 817 818 /* ready to publish object to userspace */ 819 ret = drm_gem_handle_create(filp, to_gobj(abo), &args->handle); 820 if (ret) { 821 XDNA_ERR(xdna, "Create handle failed"); 822 goto put_obj; 823 } 824 825 XDNA_DBG(xdna, "BO hdl %d type %d userptr 0x%llx xdna_addr 0x%llx size 0x%lx", 826 args->handle, args->type, abo->mem.userptr, 827 abo->mem.dev_addr, abo->mem.size); 828 put_obj: 829 /* Dereference object reference. Handle holds it now. */ 830 drm_gem_object_put(to_gobj(abo)); 831 return ret; 832 } 833 834 int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo) 835 { 836 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 837 int ret; 838 839 if (abo->type == AMDXDNA_BO_DEV) 840 abo = abo->client->dev_heap; 841 842 if (is_import_bo(abo)) 843 return 0; 844 845 ret = drm_gem_shmem_pin(&abo->base); 846 847 XDNA_DBG(xdna, "BO type %d ret %d", abo->type, ret); 848 return ret; 849 } 850 851 int amdxdna_gem_pin(struct amdxdna_gem_obj *abo) 852 { 853 int ret; 854 855 mutex_lock(&abo->lock); 856 ret = amdxdna_gem_pin_nolock(abo); 857 mutex_unlock(&abo->lock); 858 859 return ret; 860 } 861 862 void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo) 863 { 864 if (abo->type == AMDXDNA_BO_DEV) 865 abo = abo->client->dev_heap; 866 867 if (is_import_bo(abo)) 868 return; 869 870 mutex_lock(&abo->lock); 871 drm_gem_shmem_unpin(&abo->base); 872 mutex_unlock(&abo->lock); 873 } 874 875 struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, 876 u32 bo_hdl, u8 bo_type) 877 { 878 struct amdxdna_dev *xdna = client->xdna; 879 struct amdxdna_gem_obj *abo; 880 struct drm_gem_object *gobj; 881 882 gobj = drm_gem_object_lookup(client->filp, bo_hdl); 883 if (!gobj) { 884 XDNA_DBG(xdna, "Can not find bo %d", bo_hdl); 885 return NULL; 886 } 887 888 abo = to_xdna_obj(gobj); 889 if (bo_type == AMDXDNA_BO_INVALID || abo->type == bo_type) 890 return abo; 891 892 drm_gem_object_put(gobj); 893 return NULL; 894 } 895 896 int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 897 { 898 struct amdxdna_drm_get_bo_info *args = data; 899 struct amdxdna_dev *xdna = to_xdna_dev(dev); 900 struct amdxdna_gem_obj *abo; 901 struct drm_gem_object *gobj; 902 int ret = 0; 903 904 if (args->ext || args->ext_flags || args->pad) 905 return -EINVAL; 906 907 gobj = drm_gem_object_lookup(filp, args->handle); 908 if (!gobj) { 909 XDNA_DBG(xdna, "Lookup GEM object %d failed", args->handle); 910 return -ENOENT; 911 } 912 913 abo = to_xdna_obj(gobj); 914 args->vaddr = abo->mem.userptr; 915 args->xdna_addr = abo->mem.dev_addr; 916 917 if (abo->type != AMDXDNA_BO_DEV) 918 args->map_offset = drm_vma_node_offset_addr(&gobj->vma_node); 919 else 920 args->map_offset = AMDXDNA_INVALID_ADDR; 921 922 XDNA_DBG(xdna, "BO hdl %d map_offset 0x%llx vaddr 0x%llx xdna_addr 0x%llx", 923 args->handle, args->map_offset, args->vaddr, args->xdna_addr); 924 925 drm_gem_object_put(gobj); 926 return ret; 927 } 928 929 /* 930 * The sync bo ioctl is to make sure the CPU cache is in sync with memory. 931 * This is required because NPU is not cache coherent device. CPU cache 932 * flushing/invalidation is expensive so it is best to handle this outside 933 * of the command submission path. This ioctl allows explicit cache 934 * flushing/invalidation outside of the critical path. 935 */ 936 int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev, 937 void *data, struct drm_file *filp) 938 { 939 struct amdxdna_dev *xdna = to_xdna_dev(dev); 940 struct amdxdna_drm_sync_bo *args = data; 941 struct amdxdna_gem_obj *abo; 942 struct drm_gem_object *gobj; 943 int ret; 944 945 gobj = drm_gem_object_lookup(filp, args->handle); 946 if (!gobj) { 947 XDNA_ERR(xdna, "Lookup GEM object failed"); 948 return -ENOENT; 949 } 950 abo = to_xdna_obj(gobj); 951 952 ret = amdxdna_gem_pin(abo); 953 if (ret) { 954 XDNA_ERR(xdna, "Pin BO %d failed, ret %d", args->handle, ret); 955 goto put_obj; 956 } 957 958 if (is_import_bo(abo)) 959 drm_clflush_sg(abo->base.sgt); 960 else if (abo->mem.kva) 961 drm_clflush_virt_range(abo->mem.kva + args->offset, args->size); 962 else if (abo->base.pages) 963 drm_clflush_pages(abo->base.pages, gobj->size >> PAGE_SHIFT); 964 else 965 drm_WARN(&xdna->ddev, 1, "Can not get flush memory"); 966 967 amdxdna_gem_unpin(abo); 968 969 XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n", 970 args->handle, args->offset, args->size); 971 972 put_obj: 973 drm_gem_object_put(gobj); 974 return ret; 975 } 976