1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_cache.h> 8 #include <drm/drm_device.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/gpu_scheduler.h> 12 #include <linux/dma-buf.h> 13 #include <linux/dma-direct.h> 14 #include <linux/iosys-map.h> 15 #include <linux/pagemap.h> 16 #include <linux/vmalloc.h> 17 18 #include "amdxdna_ctx.h" 19 #include "amdxdna_gem.h" 20 #include "amdxdna_pci_drv.h" 21 22 #define XDNA_MAX_CMD_BO_SIZE SZ_32K 23 24 MODULE_IMPORT_NS("DMA_BUF"); 25 26 static int 27 amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo) 28 { 29 struct amdxdna_client *client = abo->client; 30 struct amdxdna_dev *xdna = client->xdna; 31 struct amdxdna_mem *mem = &abo->mem; 32 struct amdxdna_gem_obj *heap; 33 u64 offset; 34 u32 align; 35 int ret; 36 37 mutex_lock(&client->mm_lock); 38 39 heap = client->dev_heap; 40 if (!heap) { 41 ret = -EINVAL; 42 goto unlock_out; 43 } 44 45 if (heap->mem.userptr == AMDXDNA_INVALID_ADDR) { 46 XDNA_ERR(xdna, "Invalid dev heap userptr"); 47 ret = -EINVAL; 48 goto unlock_out; 49 } 50 51 if (mem->size == 0 || mem->size > heap->mem.size) { 52 XDNA_ERR(xdna, "Invalid dev bo size 0x%lx, limit 0x%lx", 53 mem->size, heap->mem.size); 54 ret = -EINVAL; 55 goto unlock_out; 56 } 57 58 align = 1 << max(PAGE_SHIFT, xdna->dev_info->dev_mem_buf_shift); 59 ret = drm_mm_insert_node_generic(&heap->mm, &abo->mm_node, 60 mem->size, align, 61 0, DRM_MM_INSERT_BEST); 62 if (ret) { 63 XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret); 64 goto unlock_out; 65 } 66 67 mem->dev_addr = abo->mm_node.start; 68 offset = mem->dev_addr - heap->mem.dev_addr; 69 mem->userptr = heap->mem.userptr + offset; 70 mem->kva = heap->mem.kva + offset; 71 72 drm_gem_object_get(to_gobj(heap)); 73 74 unlock_out: 75 mutex_unlock(&client->mm_lock); 76 77 return ret; 78 } 79 80 static void 81 amdxdna_gem_destroy_obj(struct amdxdna_gem_obj *abo) 82 { 83 mutex_destroy(&abo->lock); 84 kfree(abo); 85 } 86 87 static void 88 amdxdna_gem_heap_free(struct amdxdna_gem_obj *abo) 89 { 90 struct amdxdna_gem_obj *heap; 91 92 mutex_lock(&abo->client->mm_lock); 93 94 drm_mm_remove_node(&abo->mm_node); 95 96 heap = abo->client->dev_heap; 97 drm_gem_object_put(to_gobj(heap)); 98 99 mutex_unlock(&abo->client->mm_lock); 100 } 101 102 static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni, 103 const struct mmu_notifier_range *range, 104 unsigned long cur_seq) 105 { 106 struct amdxdna_umap *mapp = container_of(mni, struct amdxdna_umap, notifier); 107 struct amdxdna_gem_obj *abo = mapp->abo; 108 struct amdxdna_dev *xdna; 109 110 xdna = to_xdna_dev(to_gobj(abo)->dev); 111 XDNA_DBG(xdna, "Invalidating range 0x%lx, 0x%lx, type %d", 112 mapp->vma->vm_start, mapp->vma->vm_end, abo->type); 113 114 if (!mmu_notifier_range_blockable(range)) 115 return false; 116 117 down_write(&xdna->notifier_lock); 118 abo->mem.map_invalid = true; 119 mapp->invalid = true; 120 mmu_interval_set_seq(&mapp->notifier, cur_seq); 121 up_write(&xdna->notifier_lock); 122 123 xdna->dev_info->ops->hmm_invalidate(abo, cur_seq); 124 125 if (range->event == MMU_NOTIFY_UNMAP) { 126 down_write(&xdna->notifier_lock); 127 if (!mapp->unmapped) { 128 queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work); 129 mapp->unmapped = true; 130 } 131 up_write(&xdna->notifier_lock); 132 } 133 134 return true; 135 } 136 137 static const struct mmu_interval_notifier_ops amdxdna_hmm_ops = { 138 .invalidate = amdxdna_hmm_invalidate, 139 }; 140 141 static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo, 142 struct vm_area_struct *vma) 143 { 144 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 145 struct amdxdna_umap *mapp; 146 147 down_read(&xdna->notifier_lock); 148 list_for_each_entry(mapp, &abo->mem.umap_list, node) { 149 if (!vma || mapp->vma == vma) { 150 if (!mapp->unmapped) { 151 queue_work(xdna->notifier_wq, &mapp->hmm_unreg_work); 152 mapp->unmapped = true; 153 } 154 if (vma) 155 break; 156 } 157 } 158 up_read(&xdna->notifier_lock); 159 } 160 161 static void amdxdna_umap_release(struct kref *ref) 162 { 163 struct amdxdna_umap *mapp = container_of(ref, struct amdxdna_umap, refcnt); 164 struct vm_area_struct *vma = mapp->vma; 165 struct amdxdna_dev *xdna; 166 167 mmu_interval_notifier_remove(&mapp->notifier); 168 if (is_import_bo(mapp->abo) && vma->vm_file && vma->vm_file->f_mapping) 169 mapping_clear_unevictable(vma->vm_file->f_mapping); 170 171 xdna = to_xdna_dev(to_gobj(mapp->abo)->dev); 172 down_write(&xdna->notifier_lock); 173 list_del(&mapp->node); 174 up_write(&xdna->notifier_lock); 175 176 kvfree(mapp->range.hmm_pfns); 177 kfree(mapp); 178 } 179 180 void amdxdna_umap_put(struct amdxdna_umap *mapp) 181 { 182 kref_put(&mapp->refcnt, amdxdna_umap_release); 183 } 184 185 static void amdxdna_hmm_unreg_work(struct work_struct *work) 186 { 187 struct amdxdna_umap *mapp = container_of(work, struct amdxdna_umap, 188 hmm_unreg_work); 189 190 amdxdna_umap_put(mapp); 191 } 192 193 static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, 194 struct vm_area_struct *vma) 195 { 196 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 197 unsigned long len = vma->vm_end - vma->vm_start; 198 unsigned long addr = vma->vm_start; 199 struct amdxdna_umap *mapp; 200 u32 nr_pages; 201 int ret; 202 203 if (!xdna->dev_info->ops->hmm_invalidate) 204 return 0; 205 206 mapp = kzalloc(sizeof(*mapp), GFP_KERNEL); 207 if (!mapp) 208 return -ENOMEM; 209 210 nr_pages = (PAGE_ALIGN(addr + len) - (addr & PAGE_MASK)) >> PAGE_SHIFT; 211 mapp->range.hmm_pfns = kvcalloc(nr_pages, sizeof(*mapp->range.hmm_pfns), 212 GFP_KERNEL); 213 if (!mapp->range.hmm_pfns) { 214 ret = -ENOMEM; 215 goto free_map; 216 } 217 218 ret = mmu_interval_notifier_insert_locked(&mapp->notifier, 219 current->mm, 220 addr, 221 len, 222 &amdxdna_hmm_ops); 223 if (ret) { 224 XDNA_ERR(xdna, "Insert mmu notifier failed, ret %d", ret); 225 goto free_pfns; 226 } 227 228 mapp->range.notifier = &mapp->notifier; 229 mapp->range.start = vma->vm_start; 230 mapp->range.end = vma->vm_end; 231 mapp->range.default_flags = HMM_PFN_REQ_FAULT; 232 mapp->vma = vma; 233 mapp->abo = abo; 234 kref_init(&mapp->refcnt); 235 236 if (abo->mem.userptr == AMDXDNA_INVALID_ADDR) 237 abo->mem.userptr = addr; 238 INIT_WORK(&mapp->hmm_unreg_work, amdxdna_hmm_unreg_work); 239 if (is_import_bo(abo) && vma->vm_file && vma->vm_file->f_mapping) 240 mapping_set_unevictable(vma->vm_file->f_mapping); 241 242 down_write(&xdna->notifier_lock); 243 list_add_tail(&mapp->node, &abo->mem.umap_list); 244 up_write(&xdna->notifier_lock); 245 246 return 0; 247 248 free_pfns: 249 kvfree(mapp->range.hmm_pfns); 250 free_map: 251 kfree(mapp); 252 return ret; 253 } 254 255 static void amdxdna_gem_dev_obj_free(struct drm_gem_object *gobj) 256 { 257 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); 258 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 259 260 XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, abo->mem.dev_addr); 261 if (abo->pinned) 262 amdxdna_gem_unpin(abo); 263 264 amdxdna_gem_heap_free(abo); 265 drm_gem_object_release(gobj); 266 amdxdna_gem_destroy_obj(abo); 267 } 268 269 static int amdxdna_insert_pages(struct amdxdna_gem_obj *abo, 270 struct vm_area_struct *vma) 271 { 272 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 273 unsigned long num_pages = vma_pages(vma); 274 unsigned long offset = 0; 275 int ret; 276 277 if (!is_import_bo(abo)) { 278 ret = drm_gem_shmem_mmap(&abo->base, vma); 279 if (ret) { 280 XDNA_ERR(xdna, "Failed shmem mmap %d", ret); 281 return ret; 282 } 283 284 /* The buffer is based on memory pages. Fix the flag. */ 285 vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP); 286 ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages, 287 &num_pages); 288 if (ret) { 289 XDNA_ERR(xdna, "Failed insert pages %d", ret); 290 vma->vm_ops->close(vma); 291 return ret; 292 } 293 294 return 0; 295 } 296 297 vma->vm_private_data = NULL; 298 vma->vm_ops = NULL; 299 ret = dma_buf_mmap(to_gobj(abo)->dma_buf, vma, 0); 300 if (ret) { 301 XDNA_ERR(xdna, "Failed to mmap dma buf %d", ret); 302 return ret; 303 } 304 305 do { 306 vm_fault_t fault_ret; 307 308 fault_ret = handle_mm_fault(vma, vma->vm_start + offset, 309 FAULT_FLAG_WRITE, NULL); 310 if (fault_ret & VM_FAULT_ERROR) { 311 vma->vm_ops->close(vma); 312 XDNA_ERR(xdna, "Fault in page failed"); 313 return -EFAULT; 314 } 315 316 offset += PAGE_SIZE; 317 } while (--num_pages); 318 319 /* Drop the reference drm_gem_mmap_obj() acquired.*/ 320 drm_gem_object_put(to_gobj(abo)); 321 322 return 0; 323 } 324 325 static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj, 326 struct vm_area_struct *vma) 327 { 328 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); 329 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 330 int ret; 331 332 ret = amdxdna_hmm_register(abo, vma); 333 if (ret) 334 return ret; 335 336 ret = amdxdna_insert_pages(abo, vma); 337 if (ret) { 338 XDNA_ERR(xdna, "Failed insert pages, ret %d", ret); 339 goto hmm_unreg; 340 } 341 342 XDNA_DBG(xdna, "BO map_offset 0x%llx type %d userptr 0x%lx size 0x%lx", 343 drm_vma_node_offset_addr(&gobj->vma_node), abo->type, 344 vma->vm_start, gobj->size); 345 return 0; 346 347 hmm_unreg: 348 amdxdna_hmm_unregister(abo, vma); 349 return ret; 350 } 351 352 static int amdxdna_gem_dmabuf_mmap(struct dma_buf *dma_buf, struct vm_area_struct *vma) 353 { 354 struct drm_gem_object *gobj = dma_buf->priv; 355 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 356 unsigned long num_pages = vma_pages(vma); 357 int ret; 358 359 vma->vm_ops = &drm_gem_shmem_vm_ops; 360 vma->vm_private_data = gobj; 361 362 drm_gem_object_get(gobj); 363 ret = drm_gem_shmem_mmap(&abo->base, vma); 364 if (ret) 365 goto put_obj; 366 367 /* The buffer is based on memory pages. Fix the flag. */ 368 vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP); 369 ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages, 370 &num_pages); 371 if (ret) 372 goto close_vma; 373 374 return 0; 375 376 close_vma: 377 vma->vm_ops->close(vma); 378 put_obj: 379 drm_gem_object_put(gobj); 380 return ret; 381 } 382 383 static const struct dma_buf_ops amdxdna_dmabuf_ops = { 384 .attach = drm_gem_map_attach, 385 .detach = drm_gem_map_detach, 386 .map_dma_buf = drm_gem_map_dma_buf, 387 .unmap_dma_buf = drm_gem_unmap_dma_buf, 388 .release = drm_gem_dmabuf_release, 389 .mmap = amdxdna_gem_dmabuf_mmap, 390 .vmap = drm_gem_dmabuf_vmap, 391 .vunmap = drm_gem_dmabuf_vunmap, 392 }; 393 394 static struct dma_buf *amdxdna_gem_prime_export(struct drm_gem_object *gobj, int flags) 395 { 396 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 397 398 exp_info.ops = &amdxdna_dmabuf_ops; 399 exp_info.size = gobj->size; 400 exp_info.flags = flags; 401 exp_info.priv = gobj; 402 exp_info.resv = gobj->resv; 403 404 return drm_gem_dmabuf_export(gobj->dev, &exp_info); 405 } 406 407 static void amdxdna_imported_obj_free(struct amdxdna_gem_obj *abo) 408 { 409 dma_buf_unmap_attachment_unlocked(abo->attach, abo->base.sgt, DMA_BIDIRECTIONAL); 410 dma_buf_detach(abo->dma_buf, abo->attach); 411 dma_buf_put(abo->dma_buf); 412 drm_gem_object_release(to_gobj(abo)); 413 kfree(abo); 414 } 415 416 static void amdxdna_gem_obj_free(struct drm_gem_object *gobj) 417 { 418 struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev); 419 struct amdxdna_gem_obj *abo = to_xdna_obj(gobj); 420 struct iosys_map map = IOSYS_MAP_INIT_VADDR(abo->mem.kva); 421 422 XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, abo->mem.dev_addr); 423 424 amdxdna_hmm_unregister(abo, NULL); 425 flush_workqueue(xdna->notifier_wq); 426 427 if (abo->pinned) 428 amdxdna_gem_unpin(abo); 429 430 if (abo->type == AMDXDNA_BO_DEV_HEAP) 431 drm_mm_takedown(&abo->mm); 432 433 drm_gem_vunmap(gobj, &map); 434 mutex_destroy(&abo->lock); 435 436 if (is_import_bo(abo)) { 437 amdxdna_imported_obj_free(abo); 438 return; 439 } 440 441 drm_gem_shmem_free(&abo->base); 442 } 443 444 static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = { 445 .free = amdxdna_gem_dev_obj_free, 446 }; 447 448 static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = { 449 .free = amdxdna_gem_obj_free, 450 .print_info = drm_gem_shmem_object_print_info, 451 .pin = drm_gem_shmem_object_pin, 452 .unpin = drm_gem_shmem_object_unpin, 453 .get_sg_table = drm_gem_shmem_object_get_sg_table, 454 .vmap = drm_gem_shmem_object_vmap, 455 .vunmap = drm_gem_shmem_object_vunmap, 456 .mmap = amdxdna_gem_obj_mmap, 457 .vm_ops = &drm_gem_shmem_vm_ops, 458 .export = amdxdna_gem_prime_export, 459 }; 460 461 static struct amdxdna_gem_obj * 462 amdxdna_gem_create_obj(struct drm_device *dev, size_t size) 463 { 464 struct amdxdna_gem_obj *abo; 465 466 abo = kzalloc(sizeof(*abo), GFP_KERNEL); 467 if (!abo) 468 return ERR_PTR(-ENOMEM); 469 470 abo->pinned = false; 471 abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE; 472 mutex_init(&abo->lock); 473 474 abo->mem.userptr = AMDXDNA_INVALID_ADDR; 475 abo->mem.dev_addr = AMDXDNA_INVALID_ADDR; 476 abo->mem.size = size; 477 INIT_LIST_HEAD(&abo->mem.umap_list); 478 479 return abo; 480 } 481 482 /* For drm_driver->gem_create_object callback */ 483 struct drm_gem_object * 484 amdxdna_gem_create_object_cb(struct drm_device *dev, size_t size) 485 { 486 struct amdxdna_gem_obj *abo; 487 488 abo = amdxdna_gem_create_obj(dev, size); 489 if (IS_ERR(abo)) 490 return ERR_CAST(abo); 491 492 to_gobj(abo)->funcs = &amdxdna_gem_shmem_funcs; 493 494 return to_gobj(abo); 495 } 496 497 struct drm_gem_object * 498 amdxdna_gem_prime_import(struct drm_device *dev, struct dma_buf *dma_buf) 499 { 500 struct dma_buf_attachment *attach; 501 struct amdxdna_gem_obj *abo; 502 struct drm_gem_object *gobj; 503 struct sg_table *sgt; 504 int ret; 505 506 get_dma_buf(dma_buf); 507 508 attach = dma_buf_attach(dma_buf, dev->dev); 509 if (IS_ERR(attach)) { 510 ret = PTR_ERR(attach); 511 goto put_buf; 512 } 513 514 sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL); 515 if (IS_ERR(sgt)) { 516 ret = PTR_ERR(sgt); 517 goto fail_detach; 518 } 519 520 gobj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); 521 if (IS_ERR(gobj)) { 522 ret = PTR_ERR(gobj); 523 goto fail_unmap; 524 } 525 526 abo = to_xdna_obj(gobj); 527 abo->attach = attach; 528 abo->dma_buf = dma_buf; 529 530 return gobj; 531 532 fail_unmap: 533 dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL); 534 fail_detach: 535 dma_buf_detach(dma_buf, attach); 536 put_buf: 537 dma_buf_put(dma_buf); 538 539 return ERR_PTR(ret); 540 } 541 542 static struct amdxdna_gem_obj * 543 amdxdna_drm_alloc_shmem(struct drm_device *dev, 544 struct amdxdna_drm_create_bo *args, 545 struct drm_file *filp) 546 { 547 struct amdxdna_client *client = filp->driver_priv; 548 struct drm_gem_shmem_object *shmem; 549 struct amdxdna_gem_obj *abo; 550 551 shmem = drm_gem_shmem_create(dev, args->size); 552 if (IS_ERR(shmem)) 553 return ERR_CAST(shmem); 554 555 shmem->map_wc = false; 556 557 abo = to_xdna_obj(&shmem->base); 558 abo->client = client; 559 abo->type = AMDXDNA_BO_SHMEM; 560 561 return abo; 562 } 563 564 static struct amdxdna_gem_obj * 565 amdxdna_drm_create_dev_heap(struct drm_device *dev, 566 struct amdxdna_drm_create_bo *args, 567 struct drm_file *filp) 568 { 569 struct amdxdna_client *client = filp->driver_priv; 570 struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); 571 struct amdxdna_dev *xdna = to_xdna_dev(dev); 572 struct drm_gem_shmem_object *shmem; 573 struct amdxdna_gem_obj *abo; 574 int ret; 575 576 if (args->size > xdna->dev_info->dev_mem_size) { 577 XDNA_DBG(xdna, "Invalid dev heap size 0x%llx, limit 0x%lx", 578 args->size, xdna->dev_info->dev_mem_size); 579 return ERR_PTR(-EINVAL); 580 } 581 582 mutex_lock(&client->mm_lock); 583 if (client->dev_heap) { 584 XDNA_DBG(client->xdna, "dev heap is already created"); 585 ret = -EBUSY; 586 goto mm_unlock; 587 } 588 589 shmem = drm_gem_shmem_create(dev, args->size); 590 if (IS_ERR(shmem)) { 591 ret = PTR_ERR(shmem); 592 goto mm_unlock; 593 } 594 595 shmem->map_wc = false; 596 abo = to_xdna_obj(&shmem->base); 597 abo->type = AMDXDNA_BO_DEV_HEAP; 598 abo->client = client; 599 abo->mem.dev_addr = client->xdna->dev_info->dev_mem_base; 600 drm_mm_init(&abo->mm, abo->mem.dev_addr, abo->mem.size); 601 602 ret = drm_gem_vmap(to_gobj(abo), &map); 603 if (ret) { 604 XDNA_ERR(xdna, "Vmap heap bo failed, ret %d", ret); 605 goto release_obj; 606 } 607 abo->mem.kva = map.vaddr; 608 609 client->dev_heap = abo; 610 drm_gem_object_get(to_gobj(abo)); 611 mutex_unlock(&client->mm_lock); 612 613 return abo; 614 615 release_obj: 616 drm_gem_object_put(to_gobj(abo)); 617 mm_unlock: 618 mutex_unlock(&client->mm_lock); 619 return ERR_PTR(ret); 620 } 621 622 struct amdxdna_gem_obj * 623 amdxdna_drm_alloc_dev_bo(struct drm_device *dev, 624 struct amdxdna_drm_create_bo *args, 625 struct drm_file *filp) 626 { 627 struct amdxdna_client *client = filp->driver_priv; 628 struct amdxdna_dev *xdna = to_xdna_dev(dev); 629 size_t aligned_sz = PAGE_ALIGN(args->size); 630 struct amdxdna_gem_obj *abo; 631 int ret; 632 633 abo = amdxdna_gem_create_obj(&xdna->ddev, aligned_sz); 634 if (IS_ERR(abo)) 635 return abo; 636 637 to_gobj(abo)->funcs = &amdxdna_gem_dev_obj_funcs; 638 abo->type = AMDXDNA_BO_DEV; 639 abo->client = client; 640 641 ret = amdxdna_gem_heap_alloc(abo); 642 if (ret) { 643 XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret); 644 amdxdna_gem_destroy_obj(abo); 645 return ERR_PTR(ret); 646 } 647 648 drm_gem_private_object_init(&xdna->ddev, to_gobj(abo), aligned_sz); 649 650 return abo; 651 } 652 653 static struct amdxdna_gem_obj * 654 amdxdna_drm_create_cmd_bo(struct drm_device *dev, 655 struct amdxdna_drm_create_bo *args, 656 struct drm_file *filp) 657 { 658 struct iosys_map map = IOSYS_MAP_INIT_VADDR(NULL); 659 struct amdxdna_dev *xdna = to_xdna_dev(dev); 660 struct drm_gem_shmem_object *shmem; 661 struct amdxdna_gem_obj *abo; 662 int ret; 663 664 if (args->size > XDNA_MAX_CMD_BO_SIZE) { 665 XDNA_ERR(xdna, "Command bo size 0x%llx too large", args->size); 666 return ERR_PTR(-EINVAL); 667 } 668 669 if (args->size < sizeof(struct amdxdna_cmd)) { 670 XDNA_DBG(xdna, "Command BO size 0x%llx too small", args->size); 671 return ERR_PTR(-EINVAL); 672 } 673 674 shmem = drm_gem_shmem_create(dev, args->size); 675 if (IS_ERR(shmem)) 676 return ERR_CAST(shmem); 677 678 shmem->map_wc = false; 679 abo = to_xdna_obj(&shmem->base); 680 681 abo->type = AMDXDNA_BO_CMD; 682 abo->client = filp->driver_priv; 683 684 ret = drm_gem_vmap(to_gobj(abo), &map); 685 if (ret) { 686 XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret); 687 goto release_obj; 688 } 689 abo->mem.kva = map.vaddr; 690 691 return abo; 692 693 release_obj: 694 drm_gem_shmem_free(shmem); 695 return ERR_PTR(ret); 696 } 697 698 int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 699 { 700 struct amdxdna_dev *xdna = to_xdna_dev(dev); 701 struct amdxdna_drm_create_bo *args = data; 702 struct amdxdna_gem_obj *abo; 703 int ret; 704 705 if (args->flags || args->vaddr || !args->size) 706 return -EINVAL; 707 708 XDNA_DBG(xdna, "BO arg type %d vaddr 0x%llx size 0x%llx flags 0x%llx", 709 args->type, args->vaddr, args->size, args->flags); 710 switch (args->type) { 711 case AMDXDNA_BO_SHMEM: 712 abo = amdxdna_drm_alloc_shmem(dev, args, filp); 713 break; 714 case AMDXDNA_BO_DEV_HEAP: 715 abo = amdxdna_drm_create_dev_heap(dev, args, filp); 716 break; 717 case AMDXDNA_BO_DEV: 718 abo = amdxdna_drm_alloc_dev_bo(dev, args, filp); 719 break; 720 case AMDXDNA_BO_CMD: 721 abo = amdxdna_drm_create_cmd_bo(dev, args, filp); 722 break; 723 default: 724 return -EINVAL; 725 } 726 if (IS_ERR(abo)) 727 return PTR_ERR(abo); 728 729 /* ready to publish object to userspace */ 730 ret = drm_gem_handle_create(filp, to_gobj(abo), &args->handle); 731 if (ret) { 732 XDNA_ERR(xdna, "Create handle failed"); 733 goto put_obj; 734 } 735 736 XDNA_DBG(xdna, "BO hdl %d type %d userptr 0x%llx xdna_addr 0x%llx size 0x%lx", 737 args->handle, args->type, abo->mem.userptr, 738 abo->mem.dev_addr, abo->mem.size); 739 put_obj: 740 /* Dereference object reference. Handle holds it now. */ 741 drm_gem_object_put(to_gobj(abo)); 742 return ret; 743 } 744 745 int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo) 746 { 747 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); 748 int ret; 749 750 if (abo->type == AMDXDNA_BO_DEV) 751 abo = abo->client->dev_heap; 752 753 if (is_import_bo(abo)) 754 return 0; 755 756 ret = drm_gem_shmem_pin(&abo->base); 757 758 XDNA_DBG(xdna, "BO type %d ret %d", abo->type, ret); 759 return ret; 760 } 761 762 int amdxdna_gem_pin(struct amdxdna_gem_obj *abo) 763 { 764 int ret; 765 766 mutex_lock(&abo->lock); 767 ret = amdxdna_gem_pin_nolock(abo); 768 mutex_unlock(&abo->lock); 769 770 return ret; 771 } 772 773 void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo) 774 { 775 if (abo->type == AMDXDNA_BO_DEV) 776 abo = abo->client->dev_heap; 777 778 if (is_import_bo(abo)) 779 return; 780 781 mutex_lock(&abo->lock); 782 drm_gem_shmem_unpin(&abo->base); 783 mutex_unlock(&abo->lock); 784 } 785 786 struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client, 787 u32 bo_hdl, u8 bo_type) 788 { 789 struct amdxdna_dev *xdna = client->xdna; 790 struct amdxdna_gem_obj *abo; 791 struct drm_gem_object *gobj; 792 793 gobj = drm_gem_object_lookup(client->filp, bo_hdl); 794 if (!gobj) { 795 XDNA_DBG(xdna, "Can not find bo %d", bo_hdl); 796 return NULL; 797 } 798 799 abo = to_xdna_obj(gobj); 800 if (bo_type == AMDXDNA_BO_INVALID || abo->type == bo_type) 801 return abo; 802 803 drm_gem_object_put(gobj); 804 return NULL; 805 } 806 807 int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 808 { 809 struct amdxdna_drm_get_bo_info *args = data; 810 struct amdxdna_dev *xdna = to_xdna_dev(dev); 811 struct amdxdna_gem_obj *abo; 812 struct drm_gem_object *gobj; 813 int ret = 0; 814 815 if (args->ext || args->ext_flags || args->pad) 816 return -EINVAL; 817 818 gobj = drm_gem_object_lookup(filp, args->handle); 819 if (!gobj) { 820 XDNA_DBG(xdna, "Lookup GEM object %d failed", args->handle); 821 return -ENOENT; 822 } 823 824 abo = to_xdna_obj(gobj); 825 args->vaddr = abo->mem.userptr; 826 args->xdna_addr = abo->mem.dev_addr; 827 828 if (abo->type != AMDXDNA_BO_DEV) 829 args->map_offset = drm_vma_node_offset_addr(&gobj->vma_node); 830 else 831 args->map_offset = AMDXDNA_INVALID_ADDR; 832 833 XDNA_DBG(xdna, "BO hdl %d map_offset 0x%llx vaddr 0x%llx xdna_addr 0x%llx", 834 args->handle, args->map_offset, args->vaddr, args->xdna_addr); 835 836 drm_gem_object_put(gobj); 837 return ret; 838 } 839 840 /* 841 * The sync bo ioctl is to make sure the CPU cache is in sync with memory. 842 * This is required because NPU is not cache coherent device. CPU cache 843 * flushing/invalidation is expensive so it is best to handle this outside 844 * of the command submission path. This ioctl allows explicit cache 845 * flushing/invalidation outside of the critical path. 846 */ 847 int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev, 848 void *data, struct drm_file *filp) 849 { 850 struct amdxdna_dev *xdna = to_xdna_dev(dev); 851 struct amdxdna_drm_sync_bo *args = data; 852 struct amdxdna_gem_obj *abo; 853 struct drm_gem_object *gobj; 854 int ret; 855 856 gobj = drm_gem_object_lookup(filp, args->handle); 857 if (!gobj) { 858 XDNA_ERR(xdna, "Lookup GEM object failed"); 859 return -ENOENT; 860 } 861 abo = to_xdna_obj(gobj); 862 863 ret = amdxdna_gem_pin(abo); 864 if (ret) { 865 XDNA_ERR(xdna, "Pin BO %d failed, ret %d", args->handle, ret); 866 goto put_obj; 867 } 868 869 if (is_import_bo(abo)) 870 drm_clflush_sg(abo->base.sgt); 871 else if (abo->mem.kva) 872 drm_clflush_virt_range(abo->mem.kva + args->offset, args->size); 873 else if (abo->base.pages) 874 drm_clflush_pages(abo->base.pages, gobj->size >> PAGE_SHIFT); 875 else 876 drm_WARN(&xdna->ddev, 1, "Can not get flush memory"); 877 878 amdxdna_gem_unpin(abo); 879 880 XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n", 881 args->handle, args->offset, args->size); 882 883 put_obj: 884 drm_gem_object_put(gobj); 885 return ret; 886 } 887