1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <linux/module.h> 29 #include <linux/overflow.h> 30 #include <linux/pagemap.h> 31 #include <linux/pci.h> 32 #include <linux/dma-buf.h> 33 #include <linux/dma-fence-unwrap.h> 34 #include <linux/uaccess.h> 35 #include <linux/pm_runtime.h> 36 37 #include <drm/amdgpu_drm.h> 38 #include <drm/drm_drv.h> 39 #include <drm/drm_exec.h> 40 #include <drm/drm_gem_ttm_helper.h> 41 #include <drm/ttm/ttm_tt.h> 42 #include <drm/drm_syncobj.h> 43 #include <drm/drm_timeout.h> 44 45 #include "amdgpu.h" 46 #include "amdgpu_display.h" 47 #include "amdgpu_dma_buf.h" 48 #include "amdgpu_hmm.h" 49 #include "amdgpu_xgmi.h" 50 #include "amdgpu_vm.h" 51 52 static int 53 amdgpu_gem_add_input_fence(struct drm_file *filp, 54 uint64_t syncobj_handles_array, 55 uint32_t num_syncobj_handles) 56 { 57 struct dma_fence *fence; 58 uint32_t *syncobj_handles; 59 int ret, i; 60 61 if (!num_syncobj_handles) 62 return 0; 63 64 syncobj_handles = memdup_user(u64_to_user_ptr(syncobj_handles_array), 65 size_mul(sizeof(uint32_t), num_syncobj_handles)); 66 if (IS_ERR(syncobj_handles)) 67 return PTR_ERR(syncobj_handles); 68 69 for (i = 0; i < num_syncobj_handles; i++) { 70 71 if (!syncobj_handles[i]) { 72 ret = -EINVAL; 73 goto free_memdup; 74 } 75 76 ret = drm_syncobj_find_fence(filp, syncobj_handles[i], 0, 0, &fence); 77 if (ret) 78 goto free_memdup; 79 80 dma_fence_wait(fence, false); 81 82 /* TODO: optimize async handling */ 83 dma_fence_put(fence); 84 } 85 86 free_memdup: 87 kfree(syncobj_handles); 88 return ret; 89 } 90 91 static int 92 amdgpu_gem_update_timeline_node(struct drm_file *filp, 93 uint32_t syncobj_handle, 94 uint64_t point, 95 struct drm_syncobj **syncobj, 96 struct dma_fence_chain **chain) 97 { 98 if (!syncobj_handle) 99 return 0; 100 101 /* Find the sync object */ 102 *syncobj = drm_syncobj_find(filp, syncobj_handle); 103 if (!*syncobj) 104 return -ENOENT; 105 106 if (!point) 107 return 0; 108 109 /* Allocate the chain node */ 110 *chain = dma_fence_chain_alloc(); 111 if (!*chain) { 112 drm_syncobj_put(*syncobj); 113 *syncobj = NULL; 114 return -ENOMEM; 115 } 116 117 return 0; 118 } 119 120 static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf) 121 { 122 struct ttm_buffer_object *bo = vmf->vma->vm_private_data; 123 struct drm_device *ddev = bo->base.dev; 124 vm_fault_t ret; 125 int idx; 126 127 ret = ttm_bo_vm_reserve(bo, vmf); 128 if (ret) 129 return ret; 130 131 if (drm_dev_enter(ddev, &idx)) { 132 pm_runtime_get_noresume(ddev->dev); 133 134 ret = amdgpu_bo_fault_reserve_notify(bo); 135 if (ret) { 136 drm_dev_exit(idx); 137 goto unlock; 138 } 139 140 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, 141 TTM_BO_VM_NUM_PREFAULT); 142 143 pm_runtime_put_noidle(ddev->dev); 144 drm_dev_exit(idx); 145 } else { 146 ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); 147 } 148 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 149 return ret; 150 151 unlock: 152 dma_resv_unlock(bo->base.resv); 153 return ret; 154 } 155 156 static const struct vm_operations_struct amdgpu_gem_vm_ops = { 157 .fault = amdgpu_gem_fault, 158 .open = ttm_bo_vm_open, 159 .close = ttm_bo_vm_close, 160 .access = ttm_bo_vm_access 161 }; 162 163 static void amdgpu_gem_object_free(struct drm_gem_object *gobj) 164 { 165 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj); 166 167 amdgpu_hmm_unregister(aobj); 168 ttm_bo_fini(&aobj->tbo); 169 } 170 171 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, 172 int alignment, u32 initial_domain, 173 u64 flags, enum ttm_bo_type type, 174 struct dma_resv *resv, 175 struct drm_gem_object **obj, int8_t xcp_id_plus1) 176 { 177 struct amdgpu_bo *bo; 178 struct amdgpu_bo_user *ubo; 179 struct amdgpu_bo_param bp; 180 int r; 181 182 memset(&bp, 0, sizeof(bp)); 183 *obj = NULL; 184 flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; 185 186 bp.size = size; 187 bp.byte_align = alignment; 188 bp.type = type; 189 bp.resv = resv; 190 bp.preferred_domain = initial_domain; 191 bp.flags = flags; 192 bp.domain = initial_domain; 193 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 194 bp.xcp_id_plus1 = xcp_id_plus1; 195 196 r = amdgpu_bo_create_user(adev, &bp, &ubo); 197 if (r) 198 return r; 199 200 bo = &ubo->bo; 201 *obj = &bo->tbo.base; 202 203 return 0; 204 } 205 206 void amdgpu_gem_force_release(struct amdgpu_device *adev) 207 { 208 struct drm_device *ddev = adev_to_drm(adev); 209 struct drm_file *file; 210 211 mutex_lock(&ddev->filelist_mutex); 212 213 list_for_each_entry(file, &ddev->filelist, lhead) { 214 struct drm_gem_object *gobj; 215 int handle; 216 217 WARN_ONCE(1, "Still active user space clients!\n"); 218 spin_lock(&file->table_lock); 219 idr_for_each_entry(&file->object_idr, gobj, handle) { 220 WARN_ONCE(1, "And also active allocations!\n"); 221 drm_gem_object_put(gobj); 222 } 223 idr_destroy(&file->object_idr); 224 spin_unlock(&file->table_lock); 225 } 226 227 mutex_unlock(&ddev->filelist_mutex); 228 } 229 230 /* 231 * Call from drm_gem_handle_create which appear in both new and open ioctl 232 * case. 233 */ 234 static int amdgpu_gem_object_open(struct drm_gem_object *obj, 235 struct drm_file *file_priv) 236 { 237 struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj); 238 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 239 struct amdgpu_fpriv *fpriv = file_priv->driver_priv; 240 struct amdgpu_vm *vm = &fpriv->vm; 241 struct amdgpu_bo_va *bo_va; 242 struct mm_struct *mm; 243 struct drm_exec exec; 244 int r = 0; 245 246 mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm); 247 if (mm && mm != current->mm) 248 return -EPERM; 249 250 if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID && 251 !amdgpu_vm_is_bo_always_valid(vm, abo)) 252 return -EPERM; 253 254 /* Validate and add eviction fence to DMABuf imports with dynamic 255 * attachment in compute VMs. Re-validation will be done by 256 * amdgpu_vm_validate. Fences are on the reservation shared with the 257 * export, which is currently required to be validated and fenced 258 * already by amdgpu_amdkfd_gpuvm_restore_process_bos. 259 * 260 * This runs before the BO is added to the VM: drm_gem_handle_create_tail() 261 * does not call the close() callback when open() fails, so nothing that 262 * would need unwinding may exist yet. 263 * 264 * Nested locking below for the case that a GEM object is opened in 265 * kfd_mem_export_dmabuf. Since the lock below is only taken for imports, 266 * but not for export, this is a different lock class that cannot lead to 267 * circular lock dependencies. 268 */ 269 if (vm->is_compute_context && vm->process_info && 270 drm_gem_is_imported(obj) && 271 dma_buf_is_dynamic(obj->import_attach->dmabuf)) { 272 mutex_lock_nested(&vm->process_info->lock, 1); 273 if (!WARN_ON(!vm->process_info->eviction_fence)) 274 r = amdgpu_amdkfd_bo_validate_and_fence(abo, 275 AMDGPU_GEM_DOMAIN_GTT, 276 &vm->process_info->eviction_fence->base); 277 mutex_unlock(&vm->process_info->lock); 278 if (r) { 279 struct amdgpu_task_info *ti = amdgpu_vm_get_task_info_vm(vm); 280 281 dev_warn(adev->dev, "validate_and_fence failed: %d\n", r); 282 if (ti) { 283 dev_warn(adev->dev, "pid %d\n", ti->task.pid); 284 amdgpu_vm_put_task_info(ti); 285 } 286 return r; 287 } 288 } 289 290 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); 291 drm_exec_until_all_locked(&exec) { 292 r = drm_exec_prepare_obj(&exec, &abo->tbo.base, TTM_NUM_MOVE_FENCES + 1); 293 drm_exec_retry_on_contention(&exec); 294 if (unlikely(r)) 295 goto out_unlock; 296 297 r = amdgpu_vm_lock_pd(vm, &exec, 0); 298 drm_exec_retry_on_contention(&exec); 299 if (unlikely(r)) 300 goto out_unlock; 301 } 302 303 amdgpu_vm_bo_update_shared(abo); 304 bo_va = amdgpu_vm_bo_find(vm, abo); 305 if (!bo_va) { 306 bo_va = amdgpu_vm_bo_add(adev, vm, abo); 307 if (!bo_va) { 308 r = -ENOMEM; 309 goto out_unlock; 310 } 311 r = amdgpu_evf_mgr_attach_fence(&fpriv->evf_mgr, abo); 312 if (r) { 313 amdgpu_vm_bo_del(adev, bo_va); 314 goto out_unlock; 315 } 316 } else { 317 ++bo_va->ref_count; 318 } 319 320 drm_exec_fini(&exec); 321 return 0; 322 323 out_unlock: 324 drm_exec_fini(&exec); 325 return r; 326 } 327 328 static void amdgpu_gem_object_close(struct drm_gem_object *obj, 329 struct drm_file *file_priv) 330 { 331 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 332 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 333 struct amdgpu_fpriv *fpriv = file_priv->driver_priv; 334 struct amdgpu_vm *vm = &fpriv->vm; 335 336 struct dma_fence *fence = NULL; 337 struct amdgpu_bo_va *bo_va; 338 struct drm_exec exec; 339 long r; 340 341 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); 342 drm_exec_until_all_locked(&exec) { 343 r = drm_exec_prepare_obj(&exec, &bo->tbo.base, 1); 344 drm_exec_retry_on_contention(&exec); 345 if (unlikely(r)) 346 goto out_unlock; 347 348 r = amdgpu_vm_lock_pd(vm, &exec, 0); 349 drm_exec_retry_on_contention(&exec); 350 if (unlikely(r)) 351 goto out_unlock; 352 } 353 354 bo_va = amdgpu_vm_bo_find(vm, bo); 355 /* the fence is attached once per bo_va, so keep it for other handles */ 356 if (bo_va && --bo_va->ref_count) 357 goto out_unlock; 358 359 if (!amdgpu_vm_is_bo_always_valid(vm, bo)) 360 amdgpu_evf_mgr_detach_fence(&fpriv->evf_mgr, bo); 361 if (!bo_va) 362 goto out_unlock; 363 364 amdgpu_vm_bo_del(adev, bo_va); 365 amdgpu_vm_bo_update_shared(bo); 366 if (!amdgpu_vm_ready(vm)) 367 goto out_unlock; 368 369 r = amdgpu_vm_clear_freed(adev, vm, &fence); 370 if (unlikely(r < 0) && !drm_dev_is_unplugged(adev_to_drm(adev))) 371 dev_err(adev->dev, "failed to clear page " 372 "tables on GEM object close (%ld)\n", r); 373 if (r || !fence) 374 goto out_unlock; 375 376 amdgpu_bo_fence(bo, fence, true); 377 dma_fence_put(fence); 378 379 out_unlock: 380 if (r && !drm_dev_is_unplugged(adev_to_drm(adev))) 381 dev_err(adev->dev, "leaking bo va (%ld)\n", r); 382 drm_exec_fini(&exec); 383 } 384 385 static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 386 { 387 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 388 389 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) 390 return -EPERM; 391 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 392 return -EPERM; 393 394 /* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings 395 * for debugger access to invisible VRAM. Should have used MAP_SHARED 396 * instead. Clearing VM_MAYWRITE prevents the mapping from ever 397 * becoming writable and makes vma_is_cow_mapping(vma) false. 398 */ 399 if (vma_is_cow_mapping(vma) && 400 !(vma->vm_flags & VM_ACCESS_FLAGS)) 401 vm_flags_clear(vma, VM_MAYWRITE); 402 403 return drm_gem_ttm_mmap(obj, vma); 404 } 405 406 static void amdgpu_gem_object_handle_free(struct drm_gem_object *gobj) 407 { 408 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj); 409 410 amdgpu_ualink_revoke_exported_memory(aobj); 411 412 } 413 414 const struct drm_gem_object_funcs amdgpu_gem_object_funcs = { 415 .free = amdgpu_gem_object_free, 416 .open = amdgpu_gem_object_open, 417 .close = amdgpu_gem_object_close, 418 .export = amdgpu_gem_prime_export, 419 .vmap = drm_gem_ttm_vmap, 420 .vunmap = drm_gem_ttm_vunmap, 421 .mmap = amdgpu_gem_object_mmap, 422 .vm_ops = &amdgpu_gem_vm_ops, 423 .handle_free = amdgpu_gem_object_handle_free 424 }; 425 426 static bool amdgpu_gem_are_domains_valid(u32 domains) 427 { 428 u32 normal = AMDGPU_GEM_DOMAIN_CPU | 429 AMDGPU_GEM_DOMAIN_GTT | 430 AMDGPU_GEM_DOMAIN_VRAM; 431 /* Treat all non CPU/GTT/VRAM domains as special domains. */ 432 u32 special = AMDGPU_GEM_DOMAIN_MASK & ~normal; 433 u32 normal_mask = domains & normal; 434 u32 special_mask = domains & special; 435 436 if (!special_mask) 437 return true; 438 439 if (normal_mask) 440 return false; 441 442 return !(special_mask & (special_mask - 1)); 443 } 444 445 /* 446 * GEM ioctls. 447 */ 448 int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, 449 struct drm_file *filp) 450 { 451 struct amdgpu_device *adev = drm_to_adev(dev); 452 struct amdgpu_fpriv *fpriv = filp->driver_priv; 453 struct amdgpu_vm *vm = &fpriv->vm; 454 union drm_amdgpu_gem_create *args = data; 455 uint64_t flags = args->in.domain_flags; 456 uint64_t size = args->in.bo_size; 457 struct dma_resv *resv = NULL; 458 struct drm_gem_object *gobj; 459 uint32_t handle, initial_domain; 460 int r; 461 462 PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock); 463 464 r = PM_RUNTIME_ACQUIRE_ERR(&lock); 465 if (r) 466 return r; 467 468 /* reject invalid gem flags */ 469 if (flags & ~AMDGPU_GEM_CREATE_SETTABLE_MASK) 470 return -EINVAL; 471 472 /* reject invalid gem domains */ 473 if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK) 474 return -EINVAL; 475 if (!amdgpu_gem_are_domains_valid(args->in.domains)) 476 return -EINVAL; 477 478 if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) { 479 DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n"); 480 return -EINVAL; 481 } 482 483 /* always clear VRAM */ 484 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED; 485 486 /* create a gem object to contain this object in */ 487 if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS | 488 AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { 489 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { 490 /* if gds bo is created from user space, it must be 491 * passed to bo list 492 */ 493 DRM_ERROR("GDS bo cannot be per-vm-bo\n"); 494 return -EINVAL; 495 } 496 flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 497 } 498 499 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { 500 r = amdgpu_bo_reserve(vm->root.bo, false); 501 if (r) 502 return r; 503 504 resv = vm->root.bo->tbo.base.resv; 505 } 506 507 initial_domain = (u32)(0xffffffff & args->in.domains); 508 retry: 509 r = amdgpu_gem_object_create(adev, size, args->in.alignment, 510 initial_domain, 511 flags, ttm_bo_type_device, resv, &gobj, fpriv->xcp_id + 1); 512 if (r && r != -ERESTARTSYS) { 513 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { 514 flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 515 goto retry; 516 } 517 518 if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { 519 initial_domain |= AMDGPU_GEM_DOMAIN_GTT; 520 goto retry; 521 } 522 DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n", 523 size, initial_domain, args->in.alignment, r); 524 } 525 526 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { 527 if (!r) { 528 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); 529 530 abo->parent = amdgpu_bo_ref(vm->root.bo); 531 } 532 amdgpu_bo_unreserve(vm->root.bo); 533 } 534 if (r) 535 return r; 536 537 r = drm_gem_handle_create(filp, gobj, &handle); 538 /* drop reference from allocate - handle holds it now */ 539 drm_gem_object_put(gobj); 540 if (r) 541 return r; 542 543 memset(args, 0, sizeof(*args)); 544 args->out.handle = handle; 545 return 0; 546 } 547 548 int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, 549 struct drm_file *filp) 550 { 551 struct ttm_operation_ctx ctx = { true, false }; 552 struct amdgpu_device *adev = drm_to_adev(dev); 553 struct drm_amdgpu_gem_userptr *args = data; 554 struct amdgpu_fpriv *fpriv = filp->driver_priv; 555 struct drm_gem_object *gobj; 556 struct amdgpu_hmm_range *range; 557 struct amdgpu_bo *bo; 558 uint32_t handle; 559 int r; 560 561 args->addr = untagged_addr(args->addr); 562 563 if (offset_in_page(args->addr | args->size)) 564 return -EINVAL; 565 566 if (!access_ok((void __user *)(uintptr_t)args->addr, args->size)) 567 return -EFAULT; 568 569 /* reject unknown flag values */ 570 if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY | 571 AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE | 572 AMDGPU_GEM_USERPTR_REGISTER)) 573 return -EINVAL; 574 575 if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) && 576 !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) { 577 578 /* if we want to write to it we must install a MMU notifier */ 579 return -EACCES; 580 } 581 582 /* create a gem object to contain this object in */ 583 r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU, 584 0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1); 585 if (r) 586 return r; 587 588 bo = gem_to_amdgpu_bo(gobj); 589 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; 590 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; 591 bo->parent = amdgpu_bo_ref(fpriv->vm.root.bo); 592 r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags); 593 if (r) 594 goto release_object; 595 596 r = amdgpu_hmm_register(bo, args->addr); 597 if (r) 598 goto release_object; 599 600 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) { 601 range = amdgpu_hmm_range_alloc(NULL); 602 if (unlikely(!range)) { 603 r = -ENOMEM; 604 goto release_object; 605 } 606 r = amdgpu_ttm_tt_get_user_pages(bo, range); 607 if (r) { 608 amdgpu_hmm_range_free(range); 609 goto release_object; 610 } 611 612 r = pm_runtime_resume_and_get(adev->dev); 613 if (r < 0) 614 goto user_pages_done; 615 616 r = amdgpu_bo_reserve(bo, true); 617 if (r) 618 goto user_pages_done; 619 620 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range); 621 622 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 623 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 624 amdgpu_bo_unreserve(bo); 625 626 pm_runtime_put_autosuspend(adev->dev); 627 628 if (r) 629 goto user_pages_done; 630 } 631 632 r = drm_gem_handle_create(filp, gobj, &handle); 633 if (r) 634 goto user_pages_done; 635 636 args->handle = handle; 637 638 user_pages_done: 639 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) 640 amdgpu_hmm_range_free(range); 641 release_object: 642 drm_gem_object_put(gobj); 643 644 return r; 645 } 646 647 int amdgpu_mode_dumb_mmap(struct drm_file *filp, 648 struct drm_device *dev, 649 uint32_t handle, uint64_t *offset_p) 650 { 651 struct drm_gem_object *gobj; 652 struct amdgpu_bo *robj; 653 654 gobj = drm_gem_object_lookup(filp, handle); 655 if (!gobj) 656 return -ENOENT; 657 658 robj = gem_to_amdgpu_bo(gobj); 659 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) || 660 (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) { 661 drm_gem_object_put(gobj); 662 return -EPERM; 663 } 664 *offset_p = amdgpu_bo_mmap_offset(robj); 665 drm_gem_object_put(gobj); 666 return 0; 667 } 668 669 int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data, 670 struct drm_file *filp) 671 { 672 union drm_amdgpu_gem_mmap *args = data; 673 uint32_t handle = args->in.handle; 674 675 memset(args, 0, sizeof(*args)); 676 return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr); 677 } 678 679 /** 680 * amdgpu_gem_timeout - calculate jiffies timeout from absolute value 681 * 682 * @timeout_ns: timeout in ns 683 * 684 * Calculate the timeout in jiffies from an absolute timeout in ns. 685 */ 686 unsigned long amdgpu_gem_timeout(uint64_t timeout_ns) 687 { 688 /* Map anything that doesn't fit in a s64 to an infinite wait */ 689 if (timeout_ns > S64_MAX) 690 return MAX_SCHEDULE_TIMEOUT; 691 692 return drm_timeout_abs_to_jiffies(timeout_ns); 693 } 694 695 int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data, 696 struct drm_file *filp) 697 { 698 union drm_amdgpu_gem_wait_idle *args = data; 699 struct drm_gem_object *gobj; 700 struct amdgpu_bo *robj; 701 uint32_t handle = args->in.handle; 702 unsigned long timeout = amdgpu_gem_timeout(args->in.timeout); 703 int r = 0; 704 long ret; 705 706 gobj = drm_gem_object_lookup(filp, handle); 707 if (!gobj) 708 return -ENOENT; 709 710 robj = gem_to_amdgpu_bo(gobj); 711 ret = dma_resv_wait_timeout(robj->tbo.base.resv, DMA_RESV_USAGE_READ, 712 true, timeout); 713 714 /* ret == 0 means not signaled, 715 * ret > 0 means signaled 716 * ret < 0 means interrupted before timeout 717 */ 718 if (ret >= 0) { 719 memset(args, 0, sizeof(*args)); 720 args->out.status = (ret == 0); 721 } else 722 r = ret; 723 724 drm_gem_object_put(gobj); 725 return r; 726 } 727 728 int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data, 729 struct drm_file *filp) 730 { 731 struct drm_amdgpu_gem_metadata *args = data; 732 struct drm_gem_object *gobj; 733 struct amdgpu_bo *robj; 734 int r = -1; 735 736 DRM_DEBUG("%d\n", args->handle); 737 gobj = drm_gem_object_lookup(filp, args->handle); 738 if (gobj == NULL) 739 return -ENOENT; 740 robj = gem_to_amdgpu_bo(gobj); 741 742 r = amdgpu_bo_reserve(robj, false); 743 if (unlikely(r != 0)) 744 goto out; 745 746 /* Reject MMIO_REMAP BOs at IOCTL level: metadata/tiling does not apply. */ 747 if (robj->tbo.resource && 748 robj->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) { 749 DRM_WARN("metadata ioctl on MMIO_REMAP BO (handle %d)\n", 750 args->handle); 751 r = -EINVAL; 752 goto unreserve; 753 } 754 755 if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) { 756 amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info); 757 r = amdgpu_bo_get_metadata(robj, args->data.data, 758 sizeof(args->data.data), 759 &args->data.data_size_bytes, 760 &args->data.flags); 761 } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) { 762 if (args->data.data_size_bytes > sizeof(args->data.data)) { 763 r = -EINVAL; 764 goto unreserve; 765 } 766 r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info); 767 if (!r) 768 r = amdgpu_bo_set_metadata(robj, args->data.data, 769 args->data.data_size_bytes, 770 args->data.flags); 771 } 772 773 unreserve: 774 amdgpu_bo_unreserve(robj); 775 out: 776 drm_gem_object_put(gobj); 777 return r; 778 } 779 780 /** 781 * amdgpu_gem_va_update_vm -update the bo_va in its VM 782 * 783 * @adev: amdgpu_device pointer 784 * @vm: vm to update 785 * @bo_va: bo_va to update 786 * @operation: map, unmap or clear 787 * 788 * Update the bo_va directly after setting its address. Errors are not 789 * vital here, so they are not reported back to userspace. 790 * 791 * Returns resulting fence if freed BO(s) got cleared from the PT. 792 * otherwise stub fence in case of error. 793 */ 794 static struct dma_fence * 795 amdgpu_gem_va_update_vm(struct amdgpu_device *adev, 796 struct amdgpu_vm *vm, 797 struct amdgpu_bo_va *bo_va, 798 uint32_t operation) 799 { 800 struct dma_fence *fence; 801 int r = 0; 802 803 /* If the VM is not ready return only a stub. */ 804 if (!amdgpu_vm_ready(vm)) 805 return dma_fence_get_stub(); 806 807 808 /* 809 * First clean up any freed mappings in the VM. 810 * 811 * amdgpu_vm_clear_freed() may replace @fence with a new fence if it 812 * schedules GPU work. If nothing needs clearing, @fence can remain as 813 * the original vm->last_update. 814 */ 815 r = amdgpu_vm_clear_freed(adev, vm, &vm->last_update); 816 if (r) 817 goto error; 818 819 /* For MAP/REPLACE we also need to update the BO mappings. */ 820 if (operation == AMDGPU_VA_OP_MAP || 821 operation == AMDGPU_VA_OP_REPLACE) { 822 r = amdgpu_vm_bo_update(adev, bo_va, false); 823 if (r) 824 goto error; 825 } 826 827 /* Always update PDEs after we touched the mappings. */ 828 r = amdgpu_vm_update_pdes(adev, vm, false); 829 if (r) 830 goto error; 831 832 if ((operation == AMDGPU_VA_OP_MAP || 833 operation == AMDGPU_VA_OP_REPLACE) && 834 !amdgpu_vm_is_bo_always_valid(vm, bo_va->base.bo)) { 835 836 /* 837 * For MAP/REPLACE of non per-VM BOs we need to sync to both the 838 * bo_va->last_pt_update and vm->last_update or otherwise we 839 * potentially miss the PDE updates. 840 */ 841 fence = dma_fence_unwrap_merge(vm->last_update, 842 bo_va->last_pt_update); 843 if (!fence) { 844 /* As fallback in OOM situations */ 845 dma_fence_wait(vm->last_update, false); 846 dma_fence_wait(bo_va->last_pt_update, false); 847 fence = dma_fence_get_stub(); 848 } 849 } else { 850 fence = dma_fence_get(vm->last_update); 851 } 852 853 return fence; 854 855 error: 856 if (r && r != -ERESTARTSYS) 857 DRM_ERROR("Couldn't update BO_VA (%d)\n", r); 858 859 return dma_fence_get(vm->last_update); 860 } 861 862 int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, 863 struct drm_file *filp) 864 { 865 const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE | 866 AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE | 867 AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK | 868 AMDGPU_VM_PAGE_NOALLOC; 869 const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE | 870 AMDGPU_VM_PAGE_PRT; 871 872 struct drm_amdgpu_gem_va *args = data; 873 struct drm_gem_object *gobj; 874 struct amdgpu_device *adev = drm_to_adev(dev); 875 struct amdgpu_fpriv *fpriv = filp->driver_priv; 876 struct amdgpu_bo *abo; 877 struct amdgpu_bo_va *bo_va; 878 struct drm_syncobj *timeline_syncobj = NULL; 879 struct dma_fence_chain *timeline_chain = NULL; 880 struct drm_exec exec; 881 uint64_t vm_size, tmp; 882 int r = 0; 883 884 /* Validate virtual address range against reserved regions. */ 885 if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) { 886 dev_dbg(dev->dev, 887 "va_address 0x%llx is in reserved area 0x%llx\n", 888 args->va_address, AMDGPU_VA_RESERVED_BOTTOM); 889 return -EINVAL; 890 } 891 892 if (args->va_address >= AMDGPU_GMC_HOLE_START && 893 args->va_address < AMDGPU_GMC_HOLE_END) { 894 dev_dbg(dev->dev, 895 "va_address 0x%llx is in VA hole 0x%llx-0x%llx\n", 896 args->va_address, AMDGPU_GMC_HOLE_START, 897 AMDGPU_GMC_HOLE_END); 898 return -EINVAL; 899 } 900 901 args->va_address &= AMDGPU_GMC_HOLE_MASK; 902 903 vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE; 904 vm_size -= AMDGPU_VA_RESERVED_TOP; 905 if (check_add_overflow(args->va_address, args->map_size, &tmp) || tmp > vm_size) { 906 dev_dbg(dev->dev, 907 "va_address 0x%llx is in top reserved area 0x%llx\n", 908 args->va_address + args->map_size, vm_size); 909 return -EINVAL; 910 } 911 912 if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) { 913 dev_dbg(dev->dev, "invalid flags combination 0x%08X\n", 914 args->flags); 915 return -EINVAL; 916 } 917 918 /* Validate operation type. */ 919 switch (args->operation) { 920 case AMDGPU_VA_OP_MAP: 921 case AMDGPU_VA_OP_UNMAP: 922 case AMDGPU_VA_OP_CLEAR: 923 case AMDGPU_VA_OP_REPLACE: 924 break; 925 default: 926 dev_dbg(dev->dev, "unsupported operation %d\n", 927 args->operation); 928 return -EINVAL; 929 } 930 931 if (args->flags & AMDGPU_VM_DELAY_UPDATE && 932 args->vm_timeline_syncobj_out) 933 return -EINVAL; 934 935 if ((args->operation != AMDGPU_VA_OP_CLEAR) && 936 !(args->flags & AMDGPU_VM_PAGE_PRT)) { 937 gobj = drm_gem_object_lookup(filp, args->handle); 938 if (gobj == NULL) 939 return -ENOENT; 940 abo = gem_to_amdgpu_bo(gobj); 941 } else { 942 gobj = NULL; 943 abo = NULL; 944 } 945 946 PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock); 947 948 r = PM_RUNTIME_ACQUIRE_ERR(&lock); 949 if (r) 950 goto error_put_gobj; 951 952 /* Add input syncobj fences (if any) for synchronization. */ 953 r = amdgpu_gem_add_input_fence(filp, 954 args->input_fence_syncobj_handles, 955 args->num_syncobj_handles); 956 if (r) 957 goto error_put_gobj; 958 959 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | 960 DRM_EXEC_IGNORE_DUPLICATES, 0); 961 drm_exec_until_all_locked(&exec) { 962 if (gobj) { 963 r = drm_exec_lock_obj(&exec, gobj); 964 drm_exec_retry_on_contention(&exec); 965 if (unlikely(r)) 966 goto error; 967 } 968 969 r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 2); 970 drm_exec_retry_on_contention(&exec); 971 if (unlikely(r)) 972 goto error; 973 } 974 975 /* Resolve the BO-VA mapping for this VM/BO combination. */ 976 if (abo) { 977 bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo); 978 if (!bo_va) { 979 r = -ENOENT; 980 goto error; 981 } 982 } else if (args->operation != AMDGPU_VA_OP_CLEAR) { 983 bo_va = fpriv->prt_va; 984 } else { 985 bo_va = NULL; 986 } 987 988 /* 989 * Prepare the timeline syncobj node if the user requested a VM 990 * timeline update. This only allocates/looks up the syncobj and 991 * chain node; the actual fence is attached later. 992 */ 993 r = amdgpu_gem_update_timeline_node(filp, 994 args->vm_timeline_syncobj_out, 995 args->vm_timeline_point, 996 &timeline_syncobj, 997 &timeline_chain); 998 if (r) 999 goto error; 1000 1001 switch (args->operation) { 1002 case AMDGPU_VA_OP_MAP: 1003 r = amdgpu_vm_bo_map(adev, bo_va, args->va_address, 1004 args->offset_in_bo, args->map_size, 1005 args->flags); 1006 break; 1007 case AMDGPU_VA_OP_UNMAP: 1008 r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address); 1009 break; 1010 1011 case AMDGPU_VA_OP_CLEAR: 1012 r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm, 1013 args->va_address, 1014 args->map_size); 1015 break; 1016 case AMDGPU_VA_OP_REPLACE: 1017 r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address, 1018 args->offset_in_bo, args->map_size, 1019 args->flags); 1020 break; 1021 default: 1022 break; 1023 } 1024 1025 /* 1026 * Once the VA operation is done, update the VM and obtain the fence 1027 * that represents the last relevant update for this mapping. This 1028 * fence can then be exported to the user-visible VM timeline. 1029 */ 1030 if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && 1031 (!adev->debug_vm || timeline_syncobj)) { 1032 struct dma_fence *fence; 1033 1034 fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, 1035 args->operation); 1036 if (timeline_syncobj) { 1037 if (!args->vm_timeline_point) { 1038 /* Replace the existing fence when no point is given. */ 1039 drm_syncobj_replace_fence(timeline_syncobj, 1040 fence); 1041 } else { 1042 /* Attach the last-update fence at a specific point. */ 1043 drm_syncobj_add_point(timeline_syncobj, 1044 timeline_chain, 1045 fence, 1046 args->vm_timeline_point); 1047 timeline_chain = NULL; 1048 } 1049 } 1050 dma_fence_put(fence); 1051 1052 } 1053 1054 error: 1055 dma_fence_chain_free(timeline_chain); 1056 if (timeline_syncobj) 1057 drm_syncobj_put(timeline_syncobj); 1058 drm_exec_fini(&exec); 1059 error_put_gobj: 1060 drm_gem_object_put(gobj); 1061 return r; 1062 } 1063 1064 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, 1065 struct drm_file *filp) 1066 { 1067 struct drm_amdgpu_gem_op *args = data; 1068 struct drm_gem_object *gobj; 1069 struct amdgpu_vm_bo_base *base; 1070 struct amdgpu_bo *robj; 1071 struct drm_exec exec; 1072 struct amdgpu_fpriv *fpriv = filp->driver_priv; 1073 int r; 1074 1075 if (args->padding) 1076 return -EINVAL; 1077 1078 gobj = drm_gem_object_lookup(filp, args->handle); 1079 if (!gobj) 1080 return -ENOENT; 1081 1082 robj = gem_to_amdgpu_bo(gobj); 1083 1084 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT | 1085 DRM_EXEC_IGNORE_DUPLICATES, 0); 1086 drm_exec_until_all_locked(&exec) { 1087 r = drm_exec_lock_obj(&exec, gobj); 1088 drm_exec_retry_on_contention(&exec); 1089 if (r) 1090 goto out_exec; 1091 1092 if (args->op == AMDGPU_GEM_OP_GET_MAPPING_INFO) { 1093 r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 0); 1094 drm_exec_retry_on_contention(&exec); 1095 if (r) 1096 goto out_exec; 1097 } 1098 } 1099 1100 switch (args->op) { 1101 case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: { 1102 struct drm_amdgpu_gem_create_in info; 1103 void __user *out = u64_to_user_ptr(args->value); 1104 1105 info.bo_size = robj->tbo.base.size; 1106 info.alignment = robj->tbo.page_alignment << PAGE_SHIFT; 1107 info.domains = robj->preferred_domains; 1108 info.domain_flags = robj->flags; 1109 drm_exec_fini(&exec); 1110 if (copy_to_user(out, &info, sizeof(info))) 1111 r = -EFAULT; 1112 break; 1113 } 1114 case AMDGPU_GEM_OP_SET_PLACEMENT: 1115 if (drm_gem_is_imported(&robj->tbo.base) && 1116 args->value & AMDGPU_GEM_DOMAIN_VRAM) { 1117 r = -EINVAL; 1118 goto out_exec; 1119 } 1120 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) { 1121 r = -EPERM; 1122 goto out_exec; 1123 } 1124 for (base = robj->vm_bo; base; base = base->next) 1125 if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev), 1126 amdgpu_ttm_adev(base->vm->root.bo->tbo.bdev))) { 1127 r = -EINVAL; 1128 goto out_exec; 1129 } 1130 1131 1132 robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM | 1133 AMDGPU_GEM_DOMAIN_GTT | 1134 AMDGPU_GEM_DOMAIN_CPU); 1135 robj->allowed_domains = robj->preferred_domains; 1136 if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) 1137 robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 1138 1139 if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) 1140 amdgpu_vm_bo_invalidate(robj, true); 1141 drm_exec_fini(&exec); 1142 break; 1143 case AMDGPU_GEM_OP_GET_MAPPING_INFO: { 1144 struct amdgpu_bo_va *bo_va = amdgpu_vm_bo_find(&fpriv->vm, robj); 1145 struct drm_amdgpu_gem_vm_entry *vm_entries; 1146 struct amdgpu_bo_va_mapping *mapping; 1147 int num_mappings = 0; 1148 /* 1149 * num_entries is set as an input to the size of the user-allocated array of 1150 * drm_amdgpu_gem_vm_entry stored at args->value. 1151 * num_entries is sent back as output as the number of mappings the bo has. 1152 * If that number is larger than the size of the array, the ioctl must 1153 * be retried. 1154 */ 1155 if (!bo_va) { 1156 r = -ENOENT; 1157 goto out_exec; 1158 } 1159 1160 if (args->num_entries > INT_MAX / sizeof(*vm_entries)) { 1161 r = -EINVAL; 1162 goto out_exec; 1163 } 1164 1165 vm_entries = kvcalloc(args->num_entries, sizeof(*vm_entries), GFP_KERNEL); 1166 if (!vm_entries) { 1167 r = -ENOMEM; 1168 goto out_exec; 1169 } 1170 1171 amdgpu_vm_bo_va_for_each_valid_mapping(bo_va, mapping) { 1172 if (num_mappings < args->num_entries) { 1173 vm_entries[num_mappings].addr = mapping->start * AMDGPU_GPU_PAGE_SIZE; 1174 vm_entries[num_mappings].size = (mapping->last - mapping->start + 1) * AMDGPU_GPU_PAGE_SIZE; 1175 vm_entries[num_mappings].offset = mapping->offset; 1176 vm_entries[num_mappings].flags = mapping->flags; 1177 } 1178 num_mappings += 1; 1179 } 1180 1181 amdgpu_vm_bo_va_for_each_invalid_mapping(bo_va, mapping) { 1182 if (num_mappings < args->num_entries) { 1183 vm_entries[num_mappings].addr = mapping->start * AMDGPU_GPU_PAGE_SIZE; 1184 vm_entries[num_mappings].size = (mapping->last - mapping->start + 1) * AMDGPU_GPU_PAGE_SIZE; 1185 vm_entries[num_mappings].offset = mapping->offset; 1186 vm_entries[num_mappings].flags = mapping->flags; 1187 } 1188 num_mappings += 1; 1189 } 1190 1191 drm_exec_fini(&exec); 1192 1193 if (num_mappings > 0 && num_mappings <= args->num_entries) 1194 if (copy_to_user(u64_to_user_ptr(args->value), vm_entries, num_mappings * sizeof(*vm_entries))) 1195 r = -EFAULT; 1196 1197 args->num_entries = num_mappings; 1198 1199 kvfree(vm_entries); 1200 break; 1201 } 1202 default: 1203 drm_exec_fini(&exec); 1204 r = -EINVAL; 1205 } 1206 1207 drm_gem_object_put(gobj); 1208 return r; 1209 out_exec: 1210 drm_exec_fini(&exec); 1211 drm_gem_object_put(gobj); 1212 return r; 1213 } 1214 1215 /** 1216 * amdgpu_gem_list_handles_ioctl - get information about a process' buffer objects 1217 * 1218 * @dev: drm device pointer 1219 * @data: drm_amdgpu_gem_list_handles 1220 * @filp: drm file pointer 1221 * 1222 * num_entries is set as an input to the size of the entries array. 1223 * num_entries is sent back as output as the number of bos in the process. 1224 * If that number is larger than the size of the array, the ioctl must 1225 * be retried. 1226 * 1227 * Returns: 1228 * 0 for success, -errno for errors. 1229 */ 1230 int amdgpu_gem_list_handles_ioctl(struct drm_device *dev, void *data, 1231 struct drm_file *filp) 1232 { 1233 struct drm_amdgpu_gem_list_handles *args = data; 1234 struct drm_amdgpu_gem_list_handles_entry *bo_entries; 1235 struct drm_gem_object *gobj; 1236 int id, ret = 0; 1237 int bo_index = 0; 1238 int num_bos = 0; 1239 1240 spin_lock(&filp->table_lock); 1241 idr_for_each_entry(&filp->object_idr, gobj, id) 1242 num_bos += 1; 1243 spin_unlock(&filp->table_lock); 1244 1245 if (args->num_entries < num_bos) { 1246 args->num_entries = num_bos; 1247 return 0; 1248 } 1249 1250 if (num_bos == 0) { 1251 args->num_entries = 0; 1252 return 0; 1253 } 1254 1255 bo_entries = kvzalloc_objs(*bo_entries, num_bos); 1256 if (!bo_entries) 1257 return -ENOMEM; 1258 1259 spin_lock(&filp->table_lock); 1260 idr_for_each_entry(&filp->object_idr, gobj, id) { 1261 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj); 1262 struct drm_amdgpu_gem_list_handles_entry *bo_entry; 1263 1264 if (bo_index >= num_bos) { 1265 ret = -EAGAIN; 1266 break; 1267 } 1268 1269 bo_entry = &bo_entries[bo_index]; 1270 1271 bo_entry->size = amdgpu_bo_size(bo); 1272 bo_entry->alloc_flags = bo->flags & AMDGPU_GEM_CREATE_SETTABLE_MASK; 1273 bo_entry->preferred_domains = bo->preferred_domains; 1274 bo_entry->gem_handle = id; 1275 bo_entry->alignment = (u64)bo->tbo.page_alignment << PAGE_SHIFT; 1276 1277 if (bo->tbo.base.import_attach) 1278 bo_entry->flags |= AMDGPU_GEM_LIST_HANDLES_FLAG_IS_IMPORT; 1279 1280 bo_index += 1; 1281 } 1282 spin_unlock(&filp->table_lock); 1283 1284 args->num_entries = bo_index; 1285 1286 if (!ret) 1287 if (copy_to_user(u64_to_user_ptr(args->entries), bo_entries, num_bos * sizeof(*bo_entries))) 1288 ret = -EFAULT; 1289 1290 kvfree(bo_entries); 1291 1292 return ret; 1293 } 1294 1295 static unsigned int amdgpu_gem_align_pitch(struct amdgpu_device *adev, 1296 unsigned int width, 1297 unsigned int cpp, 1298 bool tiled) 1299 { 1300 unsigned int aligned = width; 1301 unsigned int pitch_mask = 0; 1302 unsigned int pitch; 1303 1304 switch (cpp) { 1305 case 1: 1306 pitch_mask = 255; 1307 break; 1308 case 2: 1309 pitch_mask = 127; 1310 break; 1311 case 3: 1312 case 4: 1313 pitch_mask = 63; 1314 break; 1315 } 1316 1317 if (check_add_overflow(aligned, pitch_mask, &aligned)) 1318 return 0; 1319 aligned &= ~pitch_mask; 1320 if (check_mul_overflow(aligned, cpp, &pitch)) 1321 return 0; 1322 return pitch; 1323 } 1324 1325 int amdgpu_mode_dumb_create(struct drm_file *file_priv, 1326 struct drm_device *dev, 1327 struct drm_mode_create_dumb *args) 1328 { 1329 struct amdgpu_device *adev = drm_to_adev(dev); 1330 struct amdgpu_fpriv *fpriv = file_priv->driver_priv; 1331 struct drm_gem_object *gobj; 1332 uint32_t handle; 1333 u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 1334 AMDGPU_GEM_CREATE_CPU_GTT_USWC | 1335 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1336 u32 domain; 1337 int r; 1338 1339 /* 1340 * The buffer returned from this function should be cleared, but 1341 * it can only be done if the ring is enabled or we'll fail to 1342 * create the buffer. 1343 */ 1344 if (adev->mman.buffer_funcs_enabled) 1345 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED; 1346 1347 args->pitch = amdgpu_gem_align_pitch(adev, args->width, 1348 DIV_ROUND_UP(args->bpp, 8), 0); 1349 if (!args->pitch) 1350 return -EINVAL; 1351 args->size = (u64)args->pitch * args->height; 1352 args->size = ALIGN(args->size, PAGE_SIZE); 1353 if (!args->size) 1354 return -EINVAL; 1355 domain = amdgpu_bo_get_preferred_domain(adev, 1356 amdgpu_display_supported_domains(adev, flags)); 1357 r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags, 1358 ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1); 1359 if (r) 1360 return -ENOMEM; 1361 1362 r = drm_gem_handle_create(file_priv, gobj, &handle); 1363 /* drop reference from allocate - handle holds it now */ 1364 drm_gem_object_put(gobj); 1365 if (r) 1366 return r; 1367 1368 args->handle = handle; 1369 return 0; 1370 } 1371 1372 #if defined(CONFIG_DEBUG_FS) 1373 static int amdgpu_debugfs_gem_info_show(struct seq_file *m, void *unused) 1374 { 1375 struct amdgpu_device *adev = m->private; 1376 struct drm_device *dev = adev_to_drm(adev); 1377 struct drm_file *file; 1378 int r; 1379 1380 r = mutex_lock_interruptible(&dev->filelist_mutex); 1381 if (r) 1382 return r; 1383 1384 list_for_each_entry(file, &dev->filelist, lhead) { 1385 struct task_struct *task; 1386 struct drm_gem_object *gobj; 1387 struct pid *pid; 1388 int id; 1389 1390 /* 1391 * Although we have a valid reference on file->pid, that does 1392 * not guarantee that the task_struct who called get_pid() is 1393 * still alive (e.g. get_pid(current) => fork() => exit()). 1394 * Therefore, we need to protect this ->comm access using RCU. 1395 */ 1396 rcu_read_lock(); 1397 pid = rcu_dereference(file->pid); 1398 task = pid_task(pid, PIDTYPE_TGID); 1399 seq_printf(m, "pid %8d command %s:\n", pid_nr(pid), 1400 task ? task->comm : "<unknown>"); 1401 rcu_read_unlock(); 1402 1403 spin_lock(&file->table_lock); 1404 idr_for_each_entry(&file->object_idr, gobj, id) { 1405 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj); 1406 1407 amdgpu_bo_print_info(id, bo, m); 1408 } 1409 spin_unlock(&file->table_lock); 1410 } 1411 1412 mutex_unlock(&dev->filelist_mutex); 1413 return 0; 1414 } 1415 1416 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_gem_info); 1417 1418 #endif 1419 1420 void amdgpu_debugfs_gem_init(struct amdgpu_device *adev) 1421 { 1422 #if defined(CONFIG_DEBUG_FS) 1423 struct drm_minor *minor = adev_to_drm(adev)->primary; 1424 struct dentry *root = minor->debugfs_root; 1425 1426 debugfs_create_file("amdgpu_gem_info", 0444, root, adev, 1427 &amdgpu_debugfs_gem_info_fops); 1428 #endif 1429 } 1430