1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_cache.h> 8 #include <drm/drm_device.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/gpu_scheduler.h> 12 #include <linux/dma-buf.h> 13 #include <linux/dma-direct.h> 14 #include <linux/iosys-map.h> 15 #include <linux/pagemap.h> 16 #include <linux/vmalloc.h> 17 18 #include "amdxdna_ctx.h" 19 #include "amdxdna_gem.h" 20 #include "amdxdna_pci_drv.h" 21 22 #define XDNA_MAX_CMD_BO_SIZE SZ_32K 23 24 MODULE_IMPORT_NS("DMA_BUF"); 25 26 static int 27 amdxdna_gem_insert_node_locked(struct amdxdna_gem_obj *abo, bool use_vmap) 28 { 29 struct amdxdna_client *client = abo->client; 30 struct amdxdna_dev *xdna = client->xdna; 31 struct amdxdna_mem *mem = &abo->mem; 32 u64 offset; 33 u32 align; 34 int ret; 35 36 align = 1 << max(PAGE_SHIFT, xdna->dev_info->dev_mem_buf_shift); 37 ret = drm_mm_insert_node_generic(&abo->dev_heap->mm, &abo->mm_node, 38 mem->size, align, 39 0, DRM_MM_INSERT_BEST); 40 if (ret) { 41 XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret); 42 return ret; 43 } 44 45 mem->dev_addr = abo->mm_node.start; 46 offset = mem->dev_addr - abo->dev_heap->mem.dev_addr; 47 mem->userptr = abo->dev_heap->mem.userptr + offset; 48 mem->pages = &abo->dev_heap->base.pages[offset >> PAGE_SHIFT]; 49 mem->nr_pages = mem->size >> PAGE_SHIFT; 50 51 if (use_vmap) { 52 mem->kva = vmap(mem->pages, mem->nr_pages, VM_MAP, PAGE_KERNEL); 53 if (!mem->kva) { 54 XDNA_ERR(xdna, "Failed to vmap"); 55 drm_mm_remove_node(&abo->mm_node); 56 return -EFAULT; 57 } 58 } 59 60 return 0; 61 } 62 63 static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni, 64 const struct mmu_notifier_range *range, 65 unsigned long cur_seq) 66 { 67 struct amdxdna_umap *mapp = container_of(mni, struct amdxdna_umap, notifier); 68 struct amdxdna_gem_obj *abo = mapp->abo; 69 struct amdxdna_dev *xdna; 70 71 xdna = to_xdna_dev(to_gobj(abo)->dev); 72 XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d", 73 mapp->vma->vm_start, mapp->vma->vm_end, abo->type); 74 75 if (!mmu_notifier_range_blockable(range)) 76 return false; 77 78 down_write(&xdna->notifier_lock); 79 abo->mem.map_invalid = true; 80 mapp->invalid = true; 81 mmu_interval_set_seq(&mapp->notifier, cur_seq); 82 up_write(&xdna->notifier_lock); 83 84 xdna->dev_info->ops->hmm_invalidate(abo, cur_seq); 85 86 if (range->event == MMU_NOTIFY_UNMAP) { 87 down_write(&xdna->notifier_lock); 88 if (!mapp->unmapped) { 89 queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work); 90 mapp->unmapped = true; 91 } 92 up_write(&xdna->notifier_lock); 93 } 94 95 return true; 96 } 97 98 static const struct mmu_interval_notifier_ops amdxdna_hmm_ops = { 99 .invalidate = amdxdna_hmm_invalidate, 100 }; 101 102 static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo, 103 struct vm_area_struct *vma) 104 { 105 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 106 struct amdxdna_umap *mapp; 107 108 down_read(&xdna->notifier_lock); 109 list_for_each_entry(mapp, &abo->mem.umap_list, node) { 110 if (!vma || mapp->vma == vma) { 111 if (!mapp->unmapped) { 112 queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work); 113 mapp->unmapped = true; 114 } 115 if (vma) 116 break; 117 } 118 } 119 up_read(&xdna->notifier_lock); 120 } 121 122 static void amdxdna_umap_release(struct kref *ref) 123 { 124 struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt); 125 struct vm_area_struct *vma = mapp->vma; 126 struct amdxdna_dev *xdna; 127 128 mmu_interval_notifier_remove(&mapp->notifier); 129 if (is_import_bo(mapp->abo) && vma->vm_file && vma->vm_file->f_mapping) 130 mapping_clear_unevictable(vma->vm_file->f_mapping); 131 132 xdna = to_xdna_dev(to_gobj(mapp->abo)->dev); 133 down_write(&xdna->notifier_lock); 134 list_del(&mapp->node); 135 up_write(&xdna->notifier_lock); 136 137 kvfree(mapp->range.hmm_pfns); 138 kfree(mapp); 139 } 140 141 void amdxdna_umap_put(struct amdxdna_umap *mapp) 142 { 143 kref_put(&mapp->refcnt, amdxdna_umap_release); 144 } 145 146 static void amdxdna_hmm_unreg_work(struct work_struct *work) 147 { 148 struct amdxdna_umap *mapp = container_of(work, struct amdxdna_umap, 149 hmm_unreg_work); 150 151 amdxdna_umap_put(mapp); 152 } 153 154 static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, 155 struct vm_area_struct *vma) 156 { 157 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 158 unsigned long len = vma->vm_end - vma->vm_start; 159 unsigned long addr = vma->vm_start; 160 struct amdxdna_umap *mapp; 161 u32 nr_pages; 162 int ret; 163 164 if (!xdna->dev_info->ops->hmm_invalidate) 165 return 0; 166 167 mapp = kzalloc(sizeof(*mapp), GFP_KERNEL); 168 if (!mapp) 169 return -ENOMEM; 170 171 nr_pages = (PAGE_ALIGN(addr + len) - (addr & PAGE_MASK)) >> PAGE_SHIFT; 172 mapp->range.hmm_pfns = kvcalloc(nr_pages, sizeof(*mapp->range.hmm_pfns), 173 GFP_KERNEL); 174 if (!mapp->range.hmm_pfns) { 175 ret = -ENOMEM; 176 goto free_map; 177 } 178 179 ret = mmu_interval_notifier_insert_locked(&mapp->notifier, 180 current->mm, 181 addr, 182 len, 183 &amdxdna_hmm_ops); 184 if (ret) { 185 XDNA_ERR(xdna, "Insert mmu notifier failed, ret %d", ret); 186 goto free_pfns; 187 } 188 189 mapp->range.notifier = &mapp->notifier; 190 mapp->range.start = vma->vm_start; 191 mapp->range.end = vma->vm_end; 192 mapp->range.default_flags = HMM_PFN_REQ_FAULT; 193 mapp->vma = vma; 194 mapp->abo = abo; 195 kref_init(&mapp->refcnt); 196 197 if (abo->mem.userptr == AMDXDNA_INVALID_ADDR) 198 abo->mem.userptr = addr; 199 INIT_WORK(&mapp->hmm_unreg_work, amdxdna_hmm_unreg_work); 200 if (is_import_bo(abo) && vma->vm_file && vma->vm_file->f_mapping) 201 mapping_set_unevictable(vma->vm_file->f_mapping); 202 203 down_write(&xdna->notifier_lock); 204 list_add_tail(&mapp->node, &abo->mem.umap_list); 205 up_write(&xdna->notifier_lock); 206 207 return 0; 208 209 free_pfns: 210 kvfree(mapp->range.hmm_pfns); 211 free_map: 212 kfree(mapp); 213 return ret; 214 } 215 216 static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo, 217 struct vm_area_struct *vma) 218 { 219 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 220 unsigned long num_pages = vma_pages(vma); 221 unsigned long offset = 0; 222 int ret; 223 224 if (!is_import_bo(abo)) { 225 ret = drm_gem_shmem_mmap(&abo->base, vma); 226 if (ret) { 227 XDNA_ERR(xdna, "Failed shmem mmap %d", ret); 228 return ret; 229 } 230 231 /* The buffer is based on memory pages. Fix the flag. */ 232 vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP); 233 ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages, 234 &num_pages); 235 if (ret) { 236 XDNA_ERR(xdna, "Failed insert pages %d", ret); 237 vma->vm_ops->close(vma); 238 return ret; 239 } 240 241 return 0; 242 } 243 244 vma->vm_private_data = NULL; 245 vma->vm_ops = NULL; 246 ret = dma_buf_mmap(to_gobj(abo)->dma_buf, vma, 0); 247 if (ret) { 248 XDNA_ERR(xdna, "Failed to mmap dma buf %d", ret); 249 return ret; 250 } 251 252 do { 253 vm_fault_t fault_ret; 254 255 fault_ret = handle_mm_fault(vma, vma->vm_start + offset, 256 FAULT_FLAG_WRITE, NULL); 257 if (fault_ret & VM_FAULT_ERROR) { 258 vma->vm_ops->close(vma); 259 XDNA_ERR(xdna, "Fault in page failed"); 260 return -EFAULT; 261 } 262 263 offset += PAGE_SIZE; 264 } while (--num_pages); 265 266 /* Drop the reference drm_gem_mmap_obj() acquired.*/ 267 drm_gem_object_put(to_gobj(abo)); 268 269 return 0; 270 } 271 272 static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj, 273 struct vm_area_struct *vma) 274 { 275 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); 276 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 277 int ret; 278 279 ret = amdxdna_hmm_register(abo, vma); 280 if (ret) 281 return ret; 282 283 ret = amdxdna_insert_pages(abo, vma); 284 if (ret) { 285 XDNA_ERR(xdna, "Failed insert pages, ret %d", ret); 286 goto hmm_unreg; 287 } 288 289 XDNA_DBG(xdna, "BO map_offset 0x%llx type %d userptr 0x%lx size 0x%lx", 290 drm_vma_node_offset_addr(&gobj->vma_node), abo->type, 291 vma->vm_start, gobj->size); 292 return 0; 293 294 hmm_unreg: 295 amdxdna_hmm_unregister(abo, vma); 296 return ret; 297 } 298 299 static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) 300 { 301 struct drm_gem_object *gobj = dma_buf->priv; 302 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 303 unsigned long num_pages = vma_pages(vma); 304 int ret; 305 306 vma->vm_ops = &drm_gem_shmem_vm_ops; 307 vma->vm_private_data = gobj; 308 309 drm_gem_object_get(gobj); 310 ret = drm_gem_shmem_mmap(&abo->base, vma); 311 if (ret) 312 goto put_obj; 313 314 /* The buffer is based on memory pages. Fix the flag. */ 315 vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP); 316 ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages, 317 &num_pages); 318 if (ret) 319 goto close_vma; 320 321 return 0; 322 323 close_vma: 324 vma->vm_ops->close(vma); 325 put_obj: 326 drm_gem_object_put(gobj); 327 return ret; 328 } 329 330 static const struct dma_buf_ops amdxdna_dmabuf_ops = { 331 .attach = drm_gem_map_attach, 332 .detach = drm_gem_map_detach, 333 .map_dma_buf = drm_gem_map_dma_buf, 334 .unmap_dma_buf = drm_gem_unmap_dma_buf, 335 .release = drm_gem_dmabuf_release, 336 .mmap = amdxdna_gem_dmabuf_mmap, 337 .vmap = drm_gem_dmabuf_vmap, 338 .vunmap = drm_gem_dmabuf_vunmap, 339 }; 340 341 static struct dma_buf *amdxdna_gem_prime_export(struct drm_gem_object *gobj, int flags) 342 { 343 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 344 345 exp_info.ops = &amdxdna_dmabuf_ops; 346 exp_info.size = gobj->size; 347 exp_info.flags = flags; 348 exp_info.priv = gobj; 349 exp_info.resv = gobj->resv; 350 351 return drm_gem_dmabuf_export(gobj->dev, &exp_info); 352 } 353 354 static void amdxdna_imported_obj_free(struct amdxdna_gem_obj *abo) 355 { 356 dma_buf_unmap_attachment_unlocked(abo->attach, abo->base.sgt, DMA_BIDIRECTIONAL); 357 dma_buf_detach(abo->dma_buf, abo->attach); 358 dma_buf_put(abo->dma_buf); 359 drm_gem_object_release(to_gobj(abo)); 360 kfree(abo); 361 } 362 363 static void amdxdna_gem_obj_free(struct drm_gem_object *gobj) 364 { 365 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); 366 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 367 struct iosys_map map = IOSYS_MAP_INIT_VADDR(abo->mem.kva); 368 369 XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, abo->mem.dev_addr); 370 371 amdxdna_hmm_unregister(abo, NULL); 372 flush_workqueue(xdna->notifier_wq); 373 374 if (abo->pinned) 375 amdxdna_gem_unpin(abo); 376 377 if (abo->type == AMDXDNA_BO_DEV) { 378 mutex_lock(&abo->client->mm_lock); 379 drm_mm_remove_node(&abo->mm_node); 380 mutex_unlock(&abo->client->mm_lock); 381 382 vunmap(abo->mem.kva); 383 drm_gem_object_put(to_gobj(abo->dev_heap)); 384 drm_gem_object_release(gobj); 385 mutex_destroy(&abo->lock); 386 kfree(abo); 387 return; 388 } 389 390 if (abo->type == AMDXDNA_BO_DEV_HEAP) 391 drm_mm_takedown(&abo->mm); 392 393 drm_gem_vunmap(gobj, &map); 394 mutex_destroy(&abo->lock); 395 396 if (is_import_bo(abo)) { 397 amdxdna_imported_obj_free(abo); 398 return; 399 } 400 401 drm_gem_shmem_free(&abo->base); 402 } 403 404 static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = { 405 .free = amdxdna_gem_obj_free, 406 }; 407 408 static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = { 409 .free = amdxdna_gem_obj_free, 410 .print_info = drm_gem_shmem_object_print_info, 411 .pin = drm_gem_shmem_object_pin, 412 .unpin = drm_gem_shmem_object_unpin, 413 .get_sg_table = drm_gem_shmem_object_get_sg_table, 414 .vmap = drm_gem_shmem_object_vmap, 415 .vunmap = drm_gem_shmem_object_vunmap, 416 .mmap = amdxdna_gem_obj_mmap, 417 .vm_ops = &drm_gem_shmem_vm_ops, 418 .export = amdxdna_gem_prime_export, 419 }; 420 421 static struct amdxdna_gem_obj * 422 amdxdna_gem_create_obj(struct drm_device *dev, size_t size) 423 { 424 struct amdxdna_gem_obj *abo; 425 426 abo = kzalloc(sizeof(*abo), GFP_KERNEL); 427 if (!abo) 428 return ERR_PTR(-ENOMEM); 429 430 abo->pinned = false; 431 abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE; 432 mutex_init(&abo->lock); 433 434 abo->mem.userptr = AMDXDNA_INVALID_ADDR; 435 abo->mem.dev_addr = AMDXDNA_INVALID_ADDR; 436 abo->mem.size = size; 437 INIT_LIST_HEAD(&abo->mem.umap_list); 438 439 return abo; 440 } 441 442 /* For drm_driver->gem_create_object callback */ 443 struct drm_gem_object * 444 amdxdna_gem_create_object_cb(struct drm_device *dev, size_t size) 445 { 446 struct amdxdna_gem_obj *abo; 447 448 abo = amdxdna_gem_create_obj(dev, size); 449 if (IS_ERR(abo)) 450 return ERR_CAST(abo); 451 452 to_gobj(abo)->funcs = &amdxdna_gem_shmem_funcs; 453 454 return to_gobj(abo); 455 } 456 457 struct drm_gem_object * 458 amdxdna_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf) 459 { 460 struct dma_buf_attachment *attach; 461 struct amdxdna_gem_obj *abo; 462 struct drm_gem_object *gobj; 463 struct sg_table *sgt; 464 int ret; 465 466 get_dma_buf(dma_buf); 467 468 attach = dma_buf_attach(dma_buf, dev->dev); 469 if (IS_ERR(attach)) { 470 ret = PTR_ERR(attach); 471 goto put_buf; 472 } 473 474 sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL); 475 if (IS_ERR(sgt)) { 476 ret = PTR_ERR(sgt); 477 goto fail_detach; 478 } 479 480 gobj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); 481 if (IS_ERR(gobj)) { 482 ret = PTR_ERR(gobj); 483 goto fail_unmap; 484 } 485 486 abo = to_xdna_obj(gobj); 487 abo->attach = attach; 488 abo->dma_buf = dma_buf; 489 490 return gobj; 491 492 fail_unmap: 493 dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL); 494 fail_detach: 495 dma_buf_detach(dma_buf, attach); 496 put_buf: 497 dma_buf_put(dma_buf); 498 499 return ERR_PTR(ret); 500 } 501 502 static struct amdxdna_gem_obj * 503 amdxdna_drm_alloc_shmem(struct drm_device *dev, 504 struct amdxdna_drm_create_bo *args, 505 struct drm_file *filp) 506 { 507 struct amdxdna_client *client = filp->driver_priv; 508 struct drm_gem_shmem_object *shmem; 509 struct amdxdna_gem_obj *abo; 510 511 shmem = drm_gem_shmem_create(dev, args->size); 512 if (IS_ERR(shmem)) 513 return ERR_CAST(shmem); 514 515 shmem->map_wc = false; 516 517 abo = to_xdna_obj(&shmem->base); 518 abo->client = client; 519 abo->type = AMDXDNA_BO_SHMEM; 520 521 return abo; 522 } 523 524 static struct amdxdna_gem_obj * 525 amdxdna_drm_create_dev_heap(struct drm_device *dev, 526 struct amdxdna_drm_create_bo *args, 527 struct drm_file *filp) 528 { 529 struct amdxdna_client *client = filp->driver_priv; 530 struct amdxdna_dev *xdna = to_xdna_dev(dev); 531 struct drm_gem_shmem_object *shmem; 532 struct amdxdna_gem_obj *abo; 533 int ret; 534 535 if (args->size > xdna->dev_info->dev_mem_size) { 536 XDNA_DBG(xdna, "Invalid dev heap size 0x%llx, limit 0x%lx", 537 args->size, xdna->dev_info->dev_mem_size); 538 return ERR_PTR(-EINVAL); 539 } 540 541 mutex_lock(&client->mm_lock); 542 if (client->dev_heap) { 543 XDNA_DBG(client->xdna, "dev heap is already created"); 544 ret = -EBUSY; 545 goto mm_unlock; 546 } 547 548 shmem = drm_gem_shmem_create(dev, args->size); 549 if (IS_ERR(shmem)) { 550 ret = PTR_ERR(shmem); 551 goto mm_unlock; 552 } 553 554 shmem->map_wc = false; 555 abo = to_xdna_obj(&shmem->base); 556 557 abo->type = AMDXDNA_BO_DEV_HEAP; 558 abo->client = client; 559 abo->mem.dev_addr = client->xdna->dev_info->dev_mem_base; 560 drm_mm_init(&abo->mm, abo->mem.dev_addr, abo->mem.size); 561 562 client->dev_heap = abo; 563 drm_gem_object_get(to_gobj(abo)); 564 mutex_unlock(&client->mm_lock); 565 566 return abo; 567 568 mm_unlock: 569 mutex_unlock(&client->mm_lock); 570 return ERR_PTR(ret); 571 } 572 573 struct amdxdna_gem_obj * 574 amdxdna_drm_alloc_dev_bo(struct drm_device *dev, 575 struct amdxdna_drm_create_bo *args, 576 struct drm_file *filp, bool use_vmap) 577 { 578 struct amdxdna_client *client = filp->driver_priv; 579 struct amdxdna_dev *xdna = to_xdna_dev(dev); 580 size_t aligned_sz = PAGE_ALIGN(args->size); 581 struct amdxdna_gem_obj *abo, *heap; 582 int ret; 583 584 mutex_lock(&client->mm_lock); 585 heap = client->dev_heap; 586 if (!heap) { 587 ret = -EINVAL; 588 goto mm_unlock; 589 } 590 591 if (heap->mem.userptr == AMDXDNA_INVALID_ADDR) { 592 XDNA_ERR(xdna, "Invalid dev heap userptr"); 593 ret = -EINVAL; 594 goto mm_unlock; 595 } 596 597 if (args->size > heap->mem.size) { 598 XDNA_ERR(xdna, "Invalid dev bo size 0x%llx, limit 0x%lx", 599 args->size, heap->mem.size); 600 ret = -EINVAL; 601 goto mm_unlock; 602 } 603 604 abo = amdxdna_gem_create_obj(&xdna->ddev, aligned_sz); 605 if (IS_ERR(abo)) { 606 ret = PTR_ERR(abo); 607 goto mm_unlock; 608 } 609 to_gobj(abo)->funcs = &amdxdna_gem_dev_obj_funcs; 610 abo->type = AMDXDNA_BO_DEV; 611 abo->client = client; 612 abo->dev_heap = heap; 613 ret = amdxdna_gem_insert_node_locked(abo, use_vmap); 614 if (ret) { 615 XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret); 616 goto mm_unlock; 617 } 618 619 drm_gem_object_get(to_gobj(heap)); 620 drm_gem_private_object_init(&xdna->ddev, to_gobj(abo), aligned_sz); 621 622 mutex_unlock(&client->mm_lock); 623 return abo; 624 625 mm_unlock: 626 mutex_unlock(&client->mm_lock); 627 return ERR_PTR(ret); 628 } 629 630 static struct amdxdna_gem_obj * 631 amdxdna_drm_create_cmd_bo(struct drm_device *dev, 632 struct amdxdna_drm_create_bo *args, 633 struct drm_file *filp) 634 { 635 struct amdxdna_dev *xdna = to_xdna_dev(dev); 636 struct drm_gem_shmem_object *shmem; 637 struct amdxdna_gem_obj *abo; 638 struct iosys_map map; 639 int ret; 640 641 if (args->size > XDNA_MAX_CMD_BO_SIZE) { 642 XDNA_ERR(xdna, "Command bo size 0x%llx too large", args->size); 643 return ERR_PTR(-EINVAL); 644 } 645 646 if (args->size < sizeof(struct amdxdna_cmd)) { 647 XDNA_DBG(xdna, "Command BO size 0x%llx too small", args->size); 648 return ERR_PTR(-EINVAL); 649 } 650 651 shmem = drm_gem_shmem_create(dev, args->size); 652 if (IS_ERR(shmem)) 653 return ERR_CAST(shmem); 654 655 shmem->map_wc = false; 656 abo = to_xdna_obj(&shmem->base); 657 658 abo->type = AMDXDNA_BO_CMD; 659 abo->client = filp->driver_priv; 660 661 ret = drm_gem_vmap(to_gobj(abo), &map); 662 if (ret) { 663 XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret); 664 goto release_obj; 665 } 666 abo->mem.kva = map.vaddr; 667 668 return abo; 669 670 release_obj: 671 drm_gem_shmem_free(shmem); 672 return ERR_PTR(ret); 673 } 674 675 int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 676 { 677 struct amdxdna_dev *xdna = to_xdna_dev(dev); 678 struct amdxdna_drm_create_bo *args = data; 679 struct amdxdna_gem_obj *abo; 680 int ret; 681 682 if (args->flags || args->vaddr || !args->size) 683 return -EINVAL; 684 685 XDNA_DBG(xdna, "BO arg type %d vaddr 0x%llx size 0x%llx flags 0x%llx", 686 args->type, args->vaddr, args->size, args->flags); 687 switch (args->type) { 688 case AMDXDNA_BO_SHMEM: 689 abo = amdxdna_drm_alloc_shmem(dev, args, filp); 690 break; 691 case AMDXDNA_BO_DEV_HEAP: 692 abo = amdxdna_drm_create_dev_heap(dev, args, filp); 693 break; 694 case AMDXDNA_BO_DEV: 695 abo = amdxdna_drm_alloc_dev_bo(dev, args, filp, false); 696 break; 697 case AMDXDNA_BO_CMD: 698 abo = amdxdna_drm_create_cmd_bo(dev, args, filp); 699 break; 700 default: 701 return -EINVAL; 702 } 703 if (IS_ERR(abo)) 704 return PTR_ERR(abo); 705 706 /* ready to publish object to userspace */ 707 ret = drm_gem_handle_create(filp, to_gobj(abo), &args->handle); 708 if (ret) { 709 XDNA_ERR(xdna, "Create handle failed"); 710 goto put_obj; 711 } 712 713 XDNA_DBG(xdna, "BO hdl %d type %d userptr 0x%llx xdna_addr 0x%llx size 0x%lx", 714 args->handle, args->type, abo->mem.userptr, 715 abo->mem.dev_addr, abo->mem.size); 716 put_obj: 717 /* Dereference object reference. Handle holds it now. */ 718 drm_gem_object_put(to_gobj(abo)); 719 return ret; 720 } 721 722 int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo) 723 { 724 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 725 int ret; 726 727 if (is_import_bo(abo)) 728 return 0; 729 730 switch (abo->type) { 731 case AMDXDNA_BO_SHMEM: 732 case AMDXDNA_BO_DEV_HEAP: 733 ret = drm_gem_shmem_pin(&abo->base); 734 break; 735 case AMDXDNA_BO_DEV: 736 ret = drm_gem_shmem_pin(&abo->dev_heap->base); 737 break; 738 default: 739 ret = -EOPNOTSUPP; 740 } 741 742 XDNA_DBG(xdna, "BO type %d ret %d", abo->type, ret); 743 return ret; 744 } 745 746 int amdxdna_gem_pin(struct amdxdna_gem_obj *abo) 747 { 748 int ret; 749 750 if (abo->type == AMDXDNA_BO_DEV) 751 abo = abo->dev_heap; 752 753 mutex_lock(&abo->lock); 754 ret = amdxdna_gem_pin_nolock(abo); 755 mutex_unlock(&abo->lock); 756 757 return ret; 758 } 759 760 void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo) 761 { 762 if (is_import_bo(abo)) 763 return; 764 765 if (abo->type == AMDXDNA_BO_DEV) 766 abo = abo->dev_heap; 767 768 mutex_lock(&abo->lock); 769 drm_gem_shmem_unpin(&abo->base); 770 mutex_unlock(&abo->lock); 771 } 772 773 struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, 774 u32 bo_hdl, u8 bo_type) 775 { 776 struct amdxdna_dev *xdna = client->xdna; 777 struct amdxdna_gem_obj *abo; 778 struct drm_gem_object *gobj; 779 780 gobj = drm_gem_object_lookup(client->filp, bo_hdl); 781 if (!gobj) { 782 XDNA_DBG(xdna, "Can not find bo %d", bo_hdl); 783 return NULL; 784 } 785 786 abo = to_xdna_obj(gobj); 787 if (bo_type == AMDXDNA_BO_INVALID || abo->type == bo_type) 788 return abo; 789 790 drm_gem_object_put(gobj); 791 return NULL; 792 } 793 794 int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 795 { 796 struct amdxdna_drm_get_bo_info *args = data; 797 struct amdxdna_dev *xdna = to_xdna_dev(dev); 798 struct amdxdna_gem_obj *abo; 799 struct drm_gem_object *gobj; 800 int ret = 0; 801 802 if (args->ext || args->ext_flags || args->pad) 803 return -EINVAL; 804 805 gobj = drm_gem_object_lookup(filp, args->handle); 806 if (!gobj) { 807 XDNA_DBG(xdna, "Lookup GEM object %d failed", args->handle); 808 return -ENOENT; 809 } 810 811 abo = to_xdna_obj(gobj); 812 args->vaddr = abo->mem.userptr; 813 args->xdna_addr = abo->mem.dev_addr; 814 815 if (abo->type != AMDXDNA_BO_DEV) 816 args->map_offset = drm_vma_node_offset_addr(&gobj->vma_node); 817 else 818 args->map_offset = AMDXDNA_INVALID_ADDR; 819 820 XDNA_DBG(xdna, "BO hdl %d map_offset 0x%llx vaddr 0x%llx xdna_addr 0x%llx", 821 args->handle, args->map_offset, args->vaddr, args->xdna_addr); 822 823 drm_gem_object_put(gobj); 824 return ret; 825 } 826 827 /* 828 * The sync bo ioctl is to make sure the CPU cache is in sync with memory. 829 * This is required because NPU is not cache coherent device. CPU cache 830 * flushing/invalidation is expensive so it is best to handle this outside 831 * of the command submission path. This ioctl allows explicit cache 832 * flushing/invalidation outside of the critical path. 833 */ 834 int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev, 835 void *data, struct drm_file *filp) 836 { 837 struct amdxdna_dev *xdna = to_xdna_dev(dev); 838 struct amdxdna_drm_sync_bo *args = data; 839 struct amdxdna_gem_obj *abo; 840 struct drm_gem_object *gobj; 841 int ret; 842 843 gobj = drm_gem_object_lookup(filp, args->handle); 844 if (!gobj) { 845 XDNA_ERR(xdna, "Lookup GEM object failed"); 846 return -ENOENT; 847 } 848 abo = to_xdna_obj(gobj); 849 850 ret = amdxdna_gem_pin(abo); 851 if (ret) { 852 XDNA_ERR(xdna, "Pin BO %d failed, ret %d", args->handle, ret); 853 goto put_obj; 854 } 855 856 if (is_import_bo(abo)) 857 drm_clflush_sg(abo->base.sgt); 858 else if (abo->type == AMDXDNA_BO_DEV) 859 drm_clflush_pages(abo->mem.pages, abo->mem.nr_pages); 860 else 861 drm_clflush_pages(abo->base.pages, gobj->size >> PAGE_SHIFT); 862 863 amdxdna_gem_unpin(abo); 864 865 XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n", 866 args->handle, args->offset, args->size); 867 868 put_obj: 869 drm_gem_object_put(gobj); 870 return ret; 871 } 872