1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <linux/slab.h> 34 #include <linux/dma-buf.h> 35 #include <linux/export.h> 36 37 #include <drm/drm_drv.h> 38 #include <drm/amdgpu_drm.h> 39 #include <drm/drm_cache.h> 40 #include "amdgpu.h" 41 #include "amdgpu_trace.h" 42 #include "amdgpu_amdkfd.h" 43 #include "amdgpu_vram_mgr.h" 44 #include "amdgpu_vm.h" 45 #include "amdgpu_dma_buf.h" 46 #include "kfd_svm.h" 47 48 /** 49 * DOC: amdgpu_object 50 * 51 * This defines the interfaces to operate on an &amdgpu_bo buffer object which 52 * represents memory used by driver (VRAM, system memory, etc.). The driver 53 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces 54 * to create/destroy/set buffer object which are then managed by the kernel TTM 55 * memory manager. 56 * The interfaces are also used internally by kernel clients, including gfx, 57 * uvd, etc. for kernel managed allocations used by the GPU. 58 * 59 */ 60 61 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) 62 { 63 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 64 65 amdgpu_bo_kunmap(bo); 66 67 if (drm_gem_is_imported(&bo->tbo.base)) 68 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); 69 drm_gem_object_release(&bo->tbo.base); 70 amdgpu_bo_unref(&bo->parent); 71 kvfree(bo); 72 } 73 74 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo) 75 { 76 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 77 struct amdgpu_bo_user *ubo; 78 79 ubo = to_amdgpu_bo_user(bo); 80 kfree(ubo->metadata); 81 amdgpu_bo_destroy(tbo); 82 } 83 84 /** 85 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo 86 * @bo: buffer object to be checked 87 * 88 * Uses destroy function associated with the object to determine if this is 89 * an &amdgpu_bo. 90 * 91 * Returns: 92 * true if the object belongs to &amdgpu_bo, false if not. 93 */ 94 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) 95 { 96 if (bo->destroy == &amdgpu_bo_destroy || 97 bo->destroy == &amdgpu_bo_user_destroy || 98 bo->destroy == &svm_range_bo_destroy) 99 return true; 100 101 return false; 102 } 103 104 /** 105 * amdgpu_bo_placement_from_domain - set buffer's placement 106 * @abo: &amdgpu_bo buffer object whose placement is to be set 107 * @domain: requested domain 108 * 109 * Sets buffer's placement according to requested domain and the buffer's 110 * flags. 111 */ 112 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) 113 { 114 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 115 struct ttm_placement *placement = &abo->placement; 116 struct ttm_place *places = abo->placements; 117 u64 flags = abo->flags; 118 u32 c = 0; 119 120 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 121 unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 122 int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); 123 124 if (adev->gmc.mem_partitions && mem_id >= 0) { 125 places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn; 126 /* 127 * memory partition range lpfn is inclusive start + size - 1 128 * TTM place lpfn is exclusive start + size 129 */ 130 places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1; 131 } else { 132 places[c].fpfn = 0; 133 places[c].lpfn = 0; 134 } 135 places[c].mem_type = TTM_PL_VRAM; 136 places[c].flags = 0; 137 138 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 139 places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn); 140 else 141 places[c].flags |= TTM_PL_FLAG_TOPDOWN; 142 143 if (abo->tbo.type == ttm_bo_type_kernel && 144 flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) 145 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; 146 147 c++; 148 } 149 150 if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) { 151 places[c].fpfn = 0; 152 places[c].lpfn = 0; 153 places[c].mem_type = AMDGPU_PL_DOORBELL; 154 places[c].flags = 0; 155 c++; 156 } 157 158 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 159 places[c].fpfn = 0; 160 places[c].lpfn = 0; 161 places[c].mem_type = 162 abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ? 163 AMDGPU_PL_PREEMPT : TTM_PL_TT; 164 places[c].flags = 0; 165 /* 166 * When GTT is just an alternative to VRAM make sure that we 167 * only use it as fallback and still try to fill up VRAM first. 168 */ 169 if (abo->tbo.resource && !(adev->flags & AMD_IS_APU) && 170 domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) 171 places[c].flags |= TTM_PL_FLAG_FALLBACK; 172 c++; 173 } 174 175 if (domain & AMDGPU_GEM_DOMAIN_CPU) { 176 places[c].fpfn = 0; 177 places[c].lpfn = 0; 178 places[c].mem_type = TTM_PL_SYSTEM; 179 places[c].flags = 0; 180 c++; 181 } 182 183 if (domain & AMDGPU_GEM_DOMAIN_GDS) { 184 places[c].fpfn = 0; 185 places[c].lpfn = 0; 186 places[c].mem_type = AMDGPU_PL_GDS; 187 places[c].flags = 0; 188 c++; 189 } 190 191 if (domain & AMDGPU_GEM_DOMAIN_GWS) { 192 places[c].fpfn = 0; 193 places[c].lpfn = 0; 194 places[c].mem_type = AMDGPU_PL_GWS; 195 places[c].flags = 0; 196 c++; 197 } 198 199 if (domain & AMDGPU_GEM_DOMAIN_OA) { 200 places[c].fpfn = 0; 201 places[c].lpfn = 0; 202 places[c].mem_type = AMDGPU_PL_OA; 203 places[c].flags = 0; 204 c++; 205 } 206 207 if (!c) { 208 places[c].fpfn = 0; 209 places[c].lpfn = 0; 210 places[c].mem_type = TTM_PL_SYSTEM; 211 places[c].flags = 0; 212 c++; 213 } 214 215 BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS); 216 217 placement->num_placement = c; 218 placement->placement = places; 219 } 220 221 /** 222 * amdgpu_bo_create_reserved - create reserved BO for kernel use 223 * 224 * @adev: amdgpu device object 225 * @size: size for the new BO 226 * @align: alignment for the new BO 227 * @domain: where to place it 228 * @bo_ptr: used to initialize BOs in structures 229 * @gpu_addr: GPU addr of the pinned BO 230 * @cpu_addr: optional CPU address mapping 231 * 232 * Allocates and pins a BO for kernel internal use, and returns it still 233 * reserved. 234 * 235 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 236 * 237 * Returns: 238 * 0 on success, negative error code otherwise. 239 */ 240 int amdgpu_bo_create_reserved(struct amdgpu_device *adev, 241 unsigned long size, int align, 242 u32 domain, struct amdgpu_bo **bo_ptr, 243 u64 *gpu_addr, void **cpu_addr) 244 { 245 struct amdgpu_bo_param bp; 246 bool free = false; 247 int r; 248 249 if (!size) { 250 amdgpu_bo_unref(bo_ptr); 251 return 0; 252 } 253 254 memset(&bp, 0, sizeof(bp)); 255 bp.size = size; 256 bp.byte_align = align; 257 bp.domain = domain; 258 bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED 259 : AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 260 bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 261 bp.type = ttm_bo_type_kernel; 262 bp.resv = NULL; 263 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 264 265 if (!*bo_ptr) { 266 r = amdgpu_bo_create(adev, &bp, bo_ptr); 267 if (r) { 268 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", 269 r); 270 return r; 271 } 272 free = true; 273 } 274 275 r = amdgpu_bo_reserve(*bo_ptr, false); 276 if (r) { 277 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); 278 goto error_free; 279 } 280 281 if (free) { 282 r = amdgpu_bo_pin(*bo_ptr, domain); 283 if (r) { 284 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); 285 goto error_unreserve; 286 } 287 } 288 289 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); 290 if (r) { 291 dev_err(adev->dev, "%p bind failed\n", *bo_ptr); 292 goto error_unpin; 293 } 294 295 if (gpu_addr) 296 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); 297 298 if (cpu_addr) { 299 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 300 if (r) { 301 dev_err(adev->dev, "(%d) kernel bo map failed\n", r); 302 goto error_unpin; 303 } 304 } 305 306 return 0; 307 308 error_unpin: 309 if (free) 310 amdgpu_bo_unpin(*bo_ptr); 311 error_unreserve: 312 amdgpu_bo_unreserve(*bo_ptr); 313 314 error_free: 315 if (free) 316 amdgpu_bo_unref(bo_ptr); 317 318 return r; 319 } 320 321 /** 322 * amdgpu_bo_create_kernel - create BO for kernel use 323 * 324 * @adev: amdgpu device object 325 * @size: size for the new BO 326 * @align: alignment for the new BO 327 * @domain: where to place it 328 * @bo_ptr: used to initialize BOs in structures 329 * @gpu_addr: GPU addr of the pinned BO 330 * @cpu_addr: optional CPU address mapping 331 * 332 * Allocates and pins a BO for kernel internal use. 333 * 334 * This function is exported to allow the V4L2 isp device 335 * external to drm device to create and access the kernel BO. 336 * 337 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 338 * 339 * Returns: 340 * 0 on success, negative error code otherwise. 341 */ 342 int amdgpu_bo_create_kernel(struct amdgpu_device *adev, 343 unsigned long size, int align, 344 u32 domain, struct amdgpu_bo **bo_ptr, 345 u64 *gpu_addr, void **cpu_addr) 346 { 347 int r; 348 349 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, 350 gpu_addr, cpu_addr); 351 352 if (r) 353 return r; 354 355 if (*bo_ptr) 356 amdgpu_bo_unreserve(*bo_ptr); 357 358 return 0; 359 } 360 361 /** 362 * amdgpu_bo_create_isp_user - create user BO for isp 363 * 364 * @adev: amdgpu device object 365 * @dma_buf: DMABUF handle for isp buffer 366 * @domain: where to place it 367 * @bo: used to initialize BOs in structures 368 * @gpu_addr: GPU addr of the pinned BO 369 * 370 * Imports isp DMABUF to allocate and pin a user BO for isp internal use. It does 371 * GART alloc to generate gpu_addr for BO to make it accessible through the 372 * GART aperture for ISP HW. 373 * 374 * This function is exported to allow the V4L2 isp device external to drm device 375 * to create and access the isp user BO. 376 * 377 * Returns: 378 * 0 on success, negative error code otherwise. 379 */ 380 int amdgpu_bo_create_isp_user(struct amdgpu_device *adev, 381 struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo, 382 u64 *gpu_addr) 383 384 { 385 struct drm_gem_object *gem_obj; 386 int r; 387 388 gem_obj = amdgpu_gem_prime_import(&adev->ddev, dma_buf); 389 *bo = gem_to_amdgpu_bo(gem_obj); 390 if (!(*bo)) { 391 dev_err(adev->dev, "failed to get valid isp user bo\n"); 392 return -EINVAL; 393 } 394 395 r = amdgpu_bo_reserve(*bo, false); 396 if (r) { 397 dev_err(adev->dev, "(%d) failed to reserve isp user bo\n", r); 398 return r; 399 } 400 401 r = amdgpu_bo_pin(*bo, domain); 402 if (r) { 403 dev_err(adev->dev, "(%d) isp user bo pin failed\n", r); 404 goto error_unreserve; 405 } 406 407 r = amdgpu_ttm_alloc_gart(&(*bo)->tbo); 408 if (r) { 409 dev_err(adev->dev, "%p bind failed\n", *bo); 410 goto error_unpin; 411 } 412 413 if (!WARN_ON(!gpu_addr)) 414 *gpu_addr = amdgpu_bo_gpu_offset(*bo); 415 416 amdgpu_bo_unreserve(*bo); 417 418 return 0; 419 420 error_unpin: 421 amdgpu_bo_unpin(*bo); 422 error_unreserve: 423 amdgpu_bo_unreserve(*bo); 424 amdgpu_bo_unref(bo); 425 426 return r; 427 } 428 429 /** 430 * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location 431 * 432 * @adev: amdgpu device object 433 * @offset: offset of the BO 434 * @size: size of the BO 435 * @bo_ptr: used to initialize BOs in structures 436 * @cpu_addr: optional CPU address mapping 437 * 438 * Creates a kernel BO at a specific offset in VRAM. 439 * 440 * Returns: 441 * 0 on success, negative error code otherwise. 442 */ 443 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, 444 uint64_t offset, uint64_t size, 445 struct amdgpu_bo **bo_ptr, void **cpu_addr) 446 { 447 struct ttm_operation_ctx ctx = { false, false }; 448 unsigned int i; 449 int r; 450 451 offset &= PAGE_MASK; 452 size = ALIGN(size, PAGE_SIZE); 453 454 r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, 455 AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL, 456 cpu_addr); 457 if (r) 458 return r; 459 460 if ((*bo_ptr) == NULL) 461 return 0; 462 463 /* 464 * Remove the original mem node and create a new one at the request 465 * position. 466 */ 467 if (cpu_addr) 468 amdgpu_bo_kunmap(*bo_ptr); 469 470 ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource); 471 472 for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) { 473 (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT; 474 (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; 475 } 476 r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement, 477 &(*bo_ptr)->tbo.resource, &ctx); 478 if (r) 479 goto error; 480 481 if (cpu_addr) { 482 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 483 if (r) 484 goto error; 485 } 486 487 amdgpu_bo_unreserve(*bo_ptr); 488 return 0; 489 490 error: 491 amdgpu_bo_unreserve(*bo_ptr); 492 amdgpu_bo_unref(bo_ptr); 493 return r; 494 } 495 496 /** 497 * amdgpu_bo_free_kernel - free BO for kernel use 498 * 499 * @bo: amdgpu BO to free 500 * @gpu_addr: pointer to where the BO's GPU memory space address was stored 501 * @cpu_addr: pointer to where the BO's CPU memory space address was stored 502 * 503 * unmaps and unpin a BO for kernel internal use. 504 * 505 * This function is exported to allow the V4L2 isp device 506 * external to drm device to free the kernel BO. 507 */ 508 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, 509 void **cpu_addr) 510 { 511 if (*bo == NULL) 512 return; 513 514 WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); 515 516 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { 517 if (cpu_addr) 518 amdgpu_bo_kunmap(*bo); 519 520 amdgpu_bo_unpin(*bo); 521 amdgpu_bo_unreserve(*bo); 522 } 523 amdgpu_bo_unref(bo); 524 525 if (gpu_addr) 526 *gpu_addr = 0; 527 528 if (cpu_addr) 529 *cpu_addr = NULL; 530 } 531 532 /** 533 * amdgpu_bo_free_isp_user - free BO for isp use 534 * 535 * @bo: amdgpu isp user BO to free 536 * 537 * unpin and unref BO for isp internal use. 538 * 539 * This function is exported to allow the V4L2 isp device 540 * external to drm device to free the isp user BO. 541 */ 542 void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo) 543 { 544 if (bo == NULL) 545 return; 546 547 if (amdgpu_bo_reserve(bo, true) == 0) { 548 amdgpu_bo_unpin(bo); 549 amdgpu_bo_unreserve(bo); 550 } 551 amdgpu_bo_unref(&bo); 552 } 553 554 /* Validate bo size is bit bigger than the request domain */ 555 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, 556 unsigned long size, u32 domain) 557 { 558 struct ttm_resource_manager *man = NULL; 559 560 /* 561 * If GTT is part of requested domains the check must succeed to 562 * allow fall back to GTT. 563 */ 564 if (domain & AMDGPU_GEM_DOMAIN_GTT) 565 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); 566 else if (domain & AMDGPU_GEM_DOMAIN_VRAM) 567 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 568 else 569 return true; 570 571 if (!man) { 572 if (domain & AMDGPU_GEM_DOMAIN_GTT) 573 WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized"); 574 return false; 575 } 576 577 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */ 578 if (size < man->size) 579 return true; 580 581 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size); 582 return false; 583 } 584 585 bool amdgpu_bo_support_uswc(u64 bo_flags) 586 { 587 588 #ifdef CONFIG_X86_32 589 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit 590 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 591 */ 592 return false; 593 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) 594 /* Don't try to enable write-combining when it can't work, or things 595 * may be slow 596 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 597 */ 598 599 #ifndef CONFIG_COMPILE_TEST 600 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ 601 thanks to write-combining 602 #endif 603 604 if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 605 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " 606 "better performance thanks to write-combining\n"); 607 return false; 608 #else 609 /* For architectures that don't support WC memory, 610 * mask out the WC flag from the BO 611 */ 612 if (!drm_arch_can_wc_memory()) 613 return false; 614 615 return true; 616 #endif 617 } 618 619 /** 620 * amdgpu_bo_create - create an &amdgpu_bo buffer object 621 * @adev: amdgpu device object 622 * @bp: parameters to be used for the buffer object 623 * @bo_ptr: pointer to the buffer object pointer 624 * 625 * Creates an &amdgpu_bo buffer object. 626 * 627 * Returns: 628 * 0 for success or a negative error code on failure. 629 */ 630 int amdgpu_bo_create(struct amdgpu_device *adev, 631 struct amdgpu_bo_param *bp, 632 struct amdgpu_bo **bo_ptr) 633 { 634 struct ttm_operation_ctx ctx = { 635 .interruptible = (bp->type != ttm_bo_type_kernel), 636 .no_wait_gpu = bp->no_wait_gpu, 637 /* We opt to avoid OOM on system pages allocations */ 638 .gfp_retry_mayfail = true, 639 .allow_res_evict = bp->type != ttm_bo_type_kernel, 640 .resv = bp->resv 641 }; 642 struct amdgpu_bo *bo; 643 unsigned long page_align, size = bp->size; 644 int r; 645 646 /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ 647 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { 648 /* GWS and OA don't need any alignment. */ 649 page_align = bp->byte_align; 650 size <<= PAGE_SHIFT; 651 652 } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { 653 /* Both size and alignment must be a multiple of 4. */ 654 page_align = ALIGN(bp->byte_align, 4); 655 size = ALIGN(size, 4) << PAGE_SHIFT; 656 } else { 657 /* Memory should be aligned at least to a page size. */ 658 page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; 659 size = ALIGN(size, PAGE_SIZE); 660 } 661 662 if (!amdgpu_bo_validate_size(adev, size, bp->domain)) 663 return -ENOMEM; 664 665 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo)); 666 667 *bo_ptr = NULL; 668 bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL); 669 if (bo == NULL) 670 return -ENOMEM; 671 drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); 672 bo->tbo.base.funcs = &amdgpu_gem_object_funcs; 673 bo->vm_bo = NULL; 674 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : 675 bp->domain; 676 bo->allowed_domains = bo->preferred_domains; 677 if (bp->type != ttm_bo_type_kernel && 678 !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) && 679 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) 680 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 681 682 bo->flags = bp->flags; 683 684 if (adev->gmc.mem_partitions) 685 /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */ 686 bo->xcp_id = bp->xcp_id_plus1 - 1; 687 else 688 /* For GPUs without spatial partitioning */ 689 bo->xcp_id = 0; 690 691 if (!amdgpu_bo_support_uswc(bo->flags)) 692 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 693 694 bo->tbo.bdev = &adev->mman.bdev; 695 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | 696 AMDGPU_GEM_DOMAIN_GDS)) 697 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 698 else 699 amdgpu_bo_placement_from_domain(bo, bp->domain); 700 if (bp->type == ttm_bo_type_kernel) 701 bo->tbo.priority = 2; 702 else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE)) 703 bo->tbo.priority = 1; 704 705 if (!bp->destroy) 706 bp->destroy = &amdgpu_bo_destroy; 707 708 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type, 709 &bo->placement, page_align, &ctx, NULL, 710 bp->resv, bp->destroy); 711 if (unlikely(r != 0)) 712 return r; 713 714 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 715 amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 716 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 717 ctx.bytes_moved); 718 else 719 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); 720 721 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && 722 bo->tbo.resource->mem_type == TTM_PL_VRAM) { 723 struct dma_fence *fence; 724 725 r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev), 726 bo, bo->tbo.base.resv, &fence, 727 true, AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER); 728 if (unlikely(r)) 729 goto fail_unreserve; 730 731 if (fence) { 732 dma_resv_add_fence(bo->tbo.base.resv, fence, 733 DMA_RESV_USAGE_KERNEL); 734 dma_fence_put(fence); 735 } 736 } 737 if (!bp->resv) 738 amdgpu_bo_unreserve(bo); 739 *bo_ptr = bo; 740 741 trace_amdgpu_bo_create(bo); 742 743 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ 744 if (bp->type == ttm_bo_type_device) 745 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 746 747 return 0; 748 749 fail_unreserve: 750 if (!bp->resv) 751 dma_resv_unlock(bo->tbo.base.resv); 752 amdgpu_bo_unref(&bo); 753 return r; 754 } 755 756 /** 757 * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object 758 * @adev: amdgpu device object 759 * @bp: parameters to be used for the buffer object 760 * @ubo_ptr: pointer to the buffer object pointer 761 * 762 * Create a BO to be used by user application; 763 * 764 * Returns: 765 * 0 for success or a negative error code on failure. 766 */ 767 768 int amdgpu_bo_create_user(struct amdgpu_device *adev, 769 struct amdgpu_bo_param *bp, 770 struct amdgpu_bo_user **ubo_ptr) 771 { 772 struct amdgpu_bo *bo_ptr; 773 int r; 774 775 bp->bo_ptr_size = sizeof(struct amdgpu_bo_user); 776 bp->destroy = &amdgpu_bo_user_destroy; 777 r = amdgpu_bo_create(adev, bp, &bo_ptr); 778 if (r) 779 return r; 780 781 *ubo_ptr = to_amdgpu_bo_user(bo_ptr); 782 return r; 783 } 784 785 /** 786 * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object 787 * @adev: amdgpu device object 788 * @bp: parameters to be used for the buffer object 789 * @vmbo_ptr: pointer to the buffer object pointer 790 * 791 * Create a BO to be for GPUVM. 792 * 793 * Returns: 794 * 0 for success or a negative error code on failure. 795 */ 796 797 int amdgpu_bo_create_vm(struct amdgpu_device *adev, 798 struct amdgpu_bo_param *bp, 799 struct amdgpu_bo_vm **vmbo_ptr) 800 { 801 struct amdgpu_bo *bo_ptr; 802 int r; 803 804 /* bo_ptr_size will be determined by the caller and it depends on 805 * num of amdgpu_vm_pt entries. 806 */ 807 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm)); 808 r = amdgpu_bo_create(adev, bp, &bo_ptr); 809 if (r) 810 return r; 811 812 *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr); 813 return r; 814 } 815 816 /** 817 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object 818 * @bo: &amdgpu_bo buffer object to be mapped 819 * @ptr: kernel virtual address to be returned 820 * 821 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls 822 * amdgpu_bo_kptr() to get the kernel virtual address. 823 * 824 * Returns: 825 * 0 for success or a negative error code on failure. 826 */ 827 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) 828 { 829 void *kptr; 830 long r; 831 832 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 833 return -EPERM; 834 835 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL, 836 false, MAX_SCHEDULE_TIMEOUT); 837 if (r < 0) 838 return r; 839 840 kptr = amdgpu_bo_kptr(bo); 841 if (kptr) { 842 if (ptr) 843 *ptr = kptr; 844 return 0; 845 } 846 847 r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap); 848 if (r) 849 return r; 850 851 if (ptr) 852 *ptr = amdgpu_bo_kptr(bo); 853 854 return 0; 855 } 856 857 /** 858 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object 859 * @bo: &amdgpu_bo buffer object 860 * 861 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address 862 * 863 * Returns: 864 * the virtual address of a buffer object area. 865 */ 866 void *amdgpu_bo_kptr(struct amdgpu_bo *bo) 867 { 868 bool is_iomem; 869 870 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 871 } 872 873 /** 874 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object 875 * @bo: &amdgpu_bo buffer object to be unmapped 876 * 877 * Unmaps a kernel map set up by amdgpu_bo_kmap(). 878 */ 879 void amdgpu_bo_kunmap(struct amdgpu_bo *bo) 880 { 881 if (bo->kmap.bo) 882 ttm_bo_kunmap(&bo->kmap); 883 } 884 885 /** 886 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object 887 * @bo: &amdgpu_bo buffer object 888 * 889 * References the contained &ttm_buffer_object. 890 * 891 * Returns: 892 * a refcounted pointer to the &amdgpu_bo buffer object. 893 */ 894 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) 895 { 896 if (bo == NULL) 897 return NULL; 898 899 drm_gem_object_get(&bo->tbo.base); 900 return bo; 901 } 902 903 /** 904 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object 905 * @bo: &amdgpu_bo buffer object 906 * 907 * Unreferences the contained &ttm_buffer_object and clear the pointer 908 */ 909 void amdgpu_bo_unref(struct amdgpu_bo **bo) 910 { 911 if ((*bo) == NULL) 912 return; 913 914 drm_gem_object_put(&(*bo)->tbo.base); 915 *bo = NULL; 916 } 917 918 /** 919 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object 920 * @bo: &amdgpu_bo buffer object to be pinned 921 * @domain: domain to be pinned to 922 * 923 * Pins the buffer object according to requested domain. If the memory is 924 * unbound gart memory, binds the pages into gart table. Adjusts pin_count and 925 * pin_size accordingly. 926 * 927 * Pinning means to lock pages in memory along with keeping them at a fixed 928 * offset. It is required when a buffer can not be moved, for example, when 929 * a display buffer is being scanned out. 930 * 931 * Returns: 932 * 0 for success or a negative error code on failure. 933 */ 934 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) 935 { 936 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 937 struct ttm_operation_ctx ctx = { false, false }; 938 int r, i; 939 940 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) 941 return -EPERM; 942 943 /* Check domain to be pinned to against preferred domains */ 944 if (bo->preferred_domains & domain) 945 domain = bo->preferred_domains & domain; 946 947 /* A shared bo cannot be migrated to VRAM */ 948 if (drm_gem_is_imported(&bo->tbo.base)) { 949 if (domain & AMDGPU_GEM_DOMAIN_GTT) 950 domain = AMDGPU_GEM_DOMAIN_GTT; 951 else 952 return -EINVAL; 953 } 954 955 if (bo->tbo.pin_count) { 956 uint32_t mem_type = bo->tbo.resource->mem_type; 957 uint32_t mem_flags = bo->tbo.resource->placement; 958 959 if (!(domain & amdgpu_mem_type_to_domain(mem_type))) 960 return -EINVAL; 961 962 if ((mem_type == TTM_PL_VRAM) && 963 (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && 964 !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) 965 return -EINVAL; 966 967 ttm_bo_pin(&bo->tbo); 968 return 0; 969 } 970 971 /* This assumes only APU display buffers are pinned with (VRAM|GTT). 972 * See function amdgpu_display_supported_domains() 973 */ 974 domain = amdgpu_bo_get_preferred_domain(adev, domain); 975 976 if (drm_gem_is_imported(&bo->tbo.base)) 977 dma_buf_pin(bo->tbo.base.import_attach); 978 979 /* force to pin into visible video ram */ 980 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) 981 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 982 amdgpu_bo_placement_from_domain(bo, domain); 983 for (i = 0; i < bo->placement.num_placement; i++) { 984 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && 985 bo->placements[i].mem_type == TTM_PL_VRAM) 986 bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS; 987 } 988 989 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 990 if (unlikely(r)) { 991 dev_err(adev->dev, "%p pin failed\n", bo); 992 goto error; 993 } 994 995 ttm_bo_pin(&bo->tbo); 996 997 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 998 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); 999 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), 1000 &adev->visible_pin_size); 1001 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 1002 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); 1003 } 1004 1005 error: 1006 return r; 1007 } 1008 1009 /** 1010 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object 1011 * @bo: &amdgpu_bo buffer object to be unpinned 1012 * 1013 * Decreases the pin_count, and clears the flags if pin_count reaches 0. 1014 * Changes placement and pin size accordingly. 1015 * 1016 * Returns: 1017 * 0 for success or a negative error code on failure. 1018 */ 1019 void amdgpu_bo_unpin(struct amdgpu_bo *bo) 1020 { 1021 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1022 1023 ttm_bo_unpin(&bo->tbo); 1024 if (bo->tbo.pin_count) 1025 return; 1026 1027 if (drm_gem_is_imported(&bo->tbo.base)) 1028 dma_buf_unpin(bo->tbo.base.import_attach); 1029 1030 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 1031 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); 1032 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), 1033 &adev->visible_pin_size); 1034 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 1035 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); 1036 } 1037 1038 } 1039 1040 static const char * const amdgpu_vram_names[] = { 1041 "UNKNOWN", 1042 "GDDR1", 1043 "DDR2", 1044 "GDDR3", 1045 "GDDR4", 1046 "GDDR5", 1047 "HBM", 1048 "DDR3", 1049 "DDR4", 1050 "GDDR6", 1051 "DDR5", 1052 "LPDDR4", 1053 "LPDDR5", 1054 "HBM3E", 1055 "HBM4" 1056 }; 1057 1058 /** 1059 * amdgpu_bo_init - initialize memory manager 1060 * @adev: amdgpu device object 1061 * 1062 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. 1063 * 1064 * Returns: 1065 * 0 for success or a negative error code on failure. 1066 */ 1067 int amdgpu_bo_init(struct amdgpu_device *adev) 1068 { 1069 /* On A+A platform, VRAM can be mapped as WB */ 1070 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 1071 /* reserve PAT memory space to WC for VRAM */ 1072 int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base, 1073 adev->gmc.aper_size); 1074 1075 if (r) { 1076 DRM_ERROR("Unable to set WC memtype for the aperture base\n"); 1077 return r; 1078 } 1079 1080 /* Add an MTRR for the VRAM */ 1081 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, 1082 adev->gmc.aper_size); 1083 } 1084 1085 drm_info(adev_to_drm(adev), "Detected VRAM RAM=%lluM, BAR=%lluM\n", 1086 adev->gmc.mc_vram_size >> 20, 1087 (unsigned long long)adev->gmc.aper_size >> 20); 1088 drm_info(adev_to_drm(adev), "RAM width %dbits %s\n", 1089 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); 1090 return amdgpu_ttm_init(adev); 1091 } 1092 1093 /** 1094 * amdgpu_bo_fini - tear down memory manager 1095 * @adev: amdgpu device object 1096 * 1097 * Reverses amdgpu_bo_init() to tear down memory manager. 1098 */ 1099 void amdgpu_bo_fini(struct amdgpu_device *adev) 1100 { 1101 int idx; 1102 1103 amdgpu_ttm_fini(adev); 1104 1105 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 1106 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 1107 arch_phys_wc_del(adev->gmc.vram_mtrr); 1108 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 1109 } 1110 drm_dev_exit(idx); 1111 } 1112 } 1113 1114 /** 1115 * amdgpu_bo_set_tiling_flags - set tiling flags 1116 * @bo: &amdgpu_bo buffer object 1117 * @tiling_flags: new flags 1118 * 1119 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or 1120 * kernel driver to set the tiling flags on a buffer. 1121 * 1122 * Returns: 1123 * 0 for success or a negative error code on failure. 1124 */ 1125 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) 1126 { 1127 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1128 struct amdgpu_bo_user *ubo; 1129 1130 /* MMIO_REMAP is BAR I/O space; tiling should never be used here. */ 1131 WARN_ON_ONCE(bo->tbo.resource && 1132 bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP); 1133 1134 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1135 if (adev->family <= AMDGPU_FAMILY_CZ && 1136 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) 1137 return -EINVAL; 1138 1139 ubo = to_amdgpu_bo_user(bo); 1140 ubo->tiling_flags = tiling_flags; 1141 return 0; 1142 } 1143 1144 /** 1145 * amdgpu_bo_get_tiling_flags - get tiling flags 1146 * @bo: &amdgpu_bo buffer object 1147 * @tiling_flags: returned flags 1148 * 1149 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to 1150 * set the tiling flags on a buffer. 1151 */ 1152 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) 1153 { 1154 struct amdgpu_bo_user *ubo; 1155 1156 /* 1157 * MMIO_REMAP BOs are not real VRAM/GTT memory but a fixed BAR I/O window. 1158 * They should never go through GEM tiling helpers. 1159 */ 1160 WARN_ON_ONCE(bo->tbo.resource && 1161 bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP); 1162 1163 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1164 dma_resv_assert_held(bo->tbo.base.resv); 1165 ubo = to_amdgpu_bo_user(bo); 1166 1167 if (tiling_flags) 1168 *tiling_flags = ubo->tiling_flags; 1169 } 1170 1171 /** 1172 * amdgpu_bo_set_metadata - set metadata 1173 * @bo: &amdgpu_bo buffer object 1174 * @metadata: new metadata 1175 * @metadata_size: size of the new metadata 1176 * @flags: flags of the new metadata 1177 * 1178 * Sets buffer object's metadata, its size and flags. 1179 * Used via GEM ioctl. 1180 * 1181 * Returns: 1182 * 0 for success or a negative error code on failure. 1183 */ 1184 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata, 1185 u32 metadata_size, uint64_t flags) 1186 { 1187 struct amdgpu_bo_user *ubo; 1188 void *buffer; 1189 1190 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1191 ubo = to_amdgpu_bo_user(bo); 1192 if (!metadata_size) { 1193 if (ubo->metadata_size) { 1194 kfree(ubo->metadata); 1195 ubo->metadata = NULL; 1196 ubo->metadata_size = 0; 1197 } 1198 return 0; 1199 } 1200 1201 if (metadata == NULL) 1202 return -EINVAL; 1203 1204 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); 1205 if (buffer == NULL) 1206 return -ENOMEM; 1207 1208 kfree(ubo->metadata); 1209 ubo->metadata_flags = flags; 1210 ubo->metadata = buffer; 1211 ubo->metadata_size = metadata_size; 1212 1213 return 0; 1214 } 1215 1216 /** 1217 * amdgpu_bo_get_metadata - get metadata 1218 * @bo: &amdgpu_bo buffer object 1219 * @buffer: returned metadata 1220 * @buffer_size: size of the buffer 1221 * @metadata_size: size of the returned metadata 1222 * @flags: flags of the returned metadata 1223 * 1224 * Gets buffer object's metadata, its size and flags. buffer_size shall not be 1225 * less than metadata_size. 1226 * Used via GEM ioctl. 1227 * 1228 * Returns: 1229 * 0 for success or a negative error code on failure. 1230 */ 1231 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, 1232 size_t buffer_size, uint32_t *metadata_size, 1233 uint64_t *flags) 1234 { 1235 struct amdgpu_bo_user *ubo; 1236 1237 if (!buffer && !metadata_size) 1238 return -EINVAL; 1239 1240 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1241 ubo = to_amdgpu_bo_user(bo); 1242 if (metadata_size) 1243 *metadata_size = ubo->metadata_size; 1244 1245 if (buffer) { 1246 if (buffer_size < ubo->metadata_size) 1247 return -EINVAL; 1248 1249 if (ubo->metadata_size) 1250 memcpy(buffer, ubo->metadata, ubo->metadata_size); 1251 } 1252 1253 if (flags) 1254 *flags = ubo->metadata_flags; 1255 1256 return 0; 1257 } 1258 1259 /** 1260 * amdgpu_bo_move_notify - notification about a memory move 1261 * @bo: pointer to a buffer object 1262 * @evict: if this move is evicting the buffer from the graphics address space 1263 * @new_mem: new resource for backing the BO 1264 * 1265 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs 1266 * bookkeeping. 1267 * TTM driver callback which is called when ttm moves a buffer. 1268 */ 1269 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, 1270 bool evict, 1271 struct ttm_resource *new_mem) 1272 { 1273 struct ttm_resource *old_mem = bo->resource; 1274 struct amdgpu_bo *abo; 1275 1276 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1277 return; 1278 1279 abo = ttm_to_amdgpu_bo(bo); 1280 amdgpu_vm_bo_move(abo, new_mem, evict); 1281 1282 amdgpu_bo_kunmap(abo); 1283 1284 if (abo->tbo.base.dma_buf && !drm_gem_is_imported(&abo->tbo.base) && 1285 old_mem && old_mem->mem_type != TTM_PL_SYSTEM) 1286 dma_buf_invalidate_mappings(abo->tbo.base.dma_buf); 1287 1288 /* move_notify is called before move happens */ 1289 trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1, 1290 old_mem ? old_mem->mem_type : -1); 1291 } 1292 1293 /** 1294 * amdgpu_bo_release_notify - notification about a BO being released 1295 * @bo: pointer to a buffer object 1296 * 1297 * Wipes VRAM buffers whose contents should not be leaked before the 1298 * memory is released. 1299 */ 1300 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) 1301 { 1302 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1303 struct dma_fence *fence = NULL; 1304 struct amdgpu_bo *abo; 1305 int r; 1306 1307 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1308 return; 1309 1310 abo = ttm_to_amdgpu_bo(bo); 1311 1312 WARN_ON(abo->vm_bo); 1313 1314 if (abo->kfd_bo) 1315 amdgpu_amdkfd_release_notify(abo); 1316 1317 /* 1318 * We lock the private dma_resv object here and since the BO is about to 1319 * be released nobody else should have a pointer to it. 1320 * So when this locking here fails something is wrong with the reference 1321 * counting. 1322 */ 1323 if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv))) 1324 return; 1325 1326 amdgpu_amdkfd_remove_all_eviction_fences(abo); 1327 1328 if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM || 1329 !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) || 1330 adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev))) 1331 goto out; 1332 1333 r = dma_resv_reserve_fences(&bo->base._resv, 1); 1334 if (r) 1335 goto out; 1336 1337 r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev), 1338 abo, &bo->base._resv, &fence, 1339 false, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE); 1340 if (WARN_ON(r)) 1341 goto out; 1342 1343 amdgpu_vram_mgr_set_cleared(bo->resource); 1344 dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL); 1345 dma_fence_put(fence); 1346 1347 out: 1348 dma_resv_unlock(&bo->base._resv); 1349 } 1350 1351 /** 1352 * amdgpu_bo_fault_reserve_notify - notification about a memory fault 1353 * @bo: pointer to a buffer object 1354 * 1355 * Notifies the driver we are taking a fault on this BO and have reserved it, 1356 * also performs bookkeeping. 1357 * TTM driver callback for dealing with vm faults. 1358 * 1359 * Returns: 1360 * 0 for success or a negative error code on failure. 1361 */ 1362 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 1363 { 1364 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1365 struct ttm_operation_ctx ctx = { false, false }; 1366 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1367 int r; 1368 1369 /* Remember that this BO was accessed by the CPU */ 1370 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 1371 1372 if (amdgpu_res_cpu_visible(adev, bo->resource)) 1373 return 0; 1374 1375 /* Can't move a pinned BO to visible VRAM */ 1376 if (abo->tbo.pin_count > 0) 1377 return VM_FAULT_SIGBUS; 1378 1379 /* hurrah the memory is not visible ! */ 1380 atomic64_inc(&adev->num_vram_cpu_page_faults); 1381 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 1382 AMDGPU_GEM_DOMAIN_GTT); 1383 1384 /* Avoid costly evictions; only set GTT as a busy placement */ 1385 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; 1386 1387 r = ttm_bo_validate(bo, &abo->placement, &ctx); 1388 if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) 1389 return VM_FAULT_NOPAGE; 1390 else if (unlikely(r)) 1391 return VM_FAULT_SIGBUS; 1392 1393 /* this should never happen */ 1394 if (bo->resource->mem_type == TTM_PL_VRAM && 1395 !amdgpu_res_cpu_visible(adev, bo->resource)) 1396 return VM_FAULT_SIGBUS; 1397 1398 ttm_bo_move_to_lru_tail_unlocked(bo); 1399 return 0; 1400 } 1401 1402 /** 1403 * amdgpu_bo_fence - add fence to buffer object 1404 * 1405 * @bo: buffer object in question 1406 * @fence: fence to add 1407 * @shared: true if fence should be added shared 1408 * 1409 */ 1410 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, 1411 bool shared) 1412 { 1413 struct dma_resv *resv = bo->tbo.base.resv; 1414 int r; 1415 1416 r = dma_resv_reserve_fences(resv, 1); 1417 if (r) { 1418 /* As last resort on OOM we block for the fence */ 1419 dma_fence_wait(fence, false); 1420 return; 1421 } 1422 1423 dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ : 1424 DMA_RESV_USAGE_WRITE); 1425 } 1426 1427 /** 1428 * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences 1429 * 1430 * @adev: amdgpu device pointer 1431 * @resv: reservation object to sync to 1432 * @sync_mode: synchronization mode 1433 * @owner: fence owner 1434 * @intr: Whether the wait is interruptible 1435 * 1436 * Extract the fences from the reservation object and waits for them to finish. 1437 * 1438 * Returns: 1439 * 0 on success, errno otherwise. 1440 */ 1441 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, 1442 enum amdgpu_sync_mode sync_mode, void *owner, 1443 bool intr) 1444 { 1445 struct amdgpu_sync sync; 1446 int r; 1447 1448 amdgpu_sync_create(&sync); 1449 amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner); 1450 r = amdgpu_sync_wait(&sync, intr); 1451 amdgpu_sync_free(&sync); 1452 return r; 1453 } 1454 1455 /** 1456 * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv 1457 * @bo: buffer object to wait for 1458 * @owner: fence owner 1459 * @intr: Whether the wait is interruptible 1460 * 1461 * Wrapper to wait for fences in a BO. 1462 * Returns: 1463 * 0 on success, errno otherwise. 1464 */ 1465 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) 1466 { 1467 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1468 1469 return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, 1470 AMDGPU_SYNC_NE_OWNER, owner, intr); 1471 } 1472 1473 /** 1474 * amdgpu_bo_gpu_offset - return GPU offset of bo 1475 * @bo: amdgpu object for which we query the offset 1476 * 1477 * Note: object should either be pinned or reserved when calling this 1478 * function, it might be useful to add check for this for debugging. 1479 * 1480 * Returns: 1481 * current GPU offset of the object. 1482 */ 1483 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) 1484 { 1485 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM); 1486 WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) && 1487 !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel); 1488 WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET); 1489 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM && 1490 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); 1491 1492 return amdgpu_bo_gpu_offset_no_check(bo); 1493 } 1494 1495 /** 1496 * amdgpu_bo_fb_aper_addr - return FB aperture GPU offset of the VRAM bo 1497 * @bo: amdgpu VRAM buffer object for which we query the offset 1498 * 1499 * Returns: 1500 * current FB aperture GPU offset of the object. 1501 */ 1502 u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo) 1503 { 1504 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1505 uint64_t offset, fb_base; 1506 1507 WARN_ON_ONCE(bo->tbo.resource->mem_type != TTM_PL_VRAM); 1508 1509 fb_base = adev->gmc.fb_start; 1510 fb_base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1511 offset = (bo->tbo.resource->start << PAGE_SHIFT) + fb_base; 1512 return amdgpu_gmc_sign_extend(offset); 1513 } 1514 1515 /** 1516 * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo 1517 * @bo: amdgpu object for which we query the offset 1518 * 1519 * Returns: 1520 * current GPU offset of the object without raising warnings. 1521 */ 1522 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) 1523 { 1524 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1525 uint64_t offset = AMDGPU_BO_INVALID_OFFSET; 1526 1527 if (bo->tbo.resource->mem_type == TTM_PL_TT) 1528 offset = amdgpu_gmc_agp_addr(&bo->tbo); 1529 1530 if (offset == AMDGPU_BO_INVALID_OFFSET) 1531 offset = (bo->tbo.resource->start << PAGE_SHIFT) + 1532 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type); 1533 1534 return amdgpu_gmc_sign_extend(offset); 1535 } 1536 1537 /** 1538 * amdgpu_bo_mem_stats_placement - bo placement for memory accounting 1539 * @bo: the buffer object we should look at 1540 * 1541 * BO can have multiple preferred placements, to avoid double counting we want 1542 * to file it under a single placement for memory stats. 1543 * Luckily, if we take the highest set bit in preferred_domains the result is 1544 * quite sensible. 1545 * 1546 * Returns: 1547 * Which of the placements should the BO be accounted under. 1548 */ 1549 uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo) 1550 { 1551 u32 domain; 1552 1553 /* 1554 * MMIO_REMAP is internal now, so it no longer maps from a userspace 1555 * domain bit. Keep fdinfo/mem-stats visibility by checking the actual 1556 * TTM placement. 1557 */ 1558 if (bo->tbo.resource && bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) 1559 return AMDGPU_PL_MMIO_REMAP; 1560 1561 domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK; 1562 if (!domain) 1563 return TTM_PL_SYSTEM; 1564 1565 switch (rounddown_pow_of_two(domain)) { 1566 case AMDGPU_GEM_DOMAIN_CPU: 1567 return TTM_PL_SYSTEM; 1568 case AMDGPU_GEM_DOMAIN_GTT: 1569 return TTM_PL_TT; 1570 case AMDGPU_GEM_DOMAIN_VRAM: 1571 return TTM_PL_VRAM; 1572 case AMDGPU_GEM_DOMAIN_GDS: 1573 return AMDGPU_PL_GDS; 1574 case AMDGPU_GEM_DOMAIN_GWS: 1575 return AMDGPU_PL_GWS; 1576 case AMDGPU_GEM_DOMAIN_OA: 1577 return AMDGPU_PL_OA; 1578 case AMDGPU_GEM_DOMAIN_DOORBELL: 1579 return AMDGPU_PL_DOORBELL; 1580 default: 1581 return TTM_PL_SYSTEM; 1582 } 1583 } 1584 1585 /** 1586 * amdgpu_bo_get_preferred_domain - get preferred domain 1587 * @adev: amdgpu device object 1588 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` 1589 * 1590 * Returns: 1591 * Which of the allowed domains is preferred for allocating the BO. 1592 */ 1593 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, 1594 uint32_t domain) 1595 { 1596 if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && 1597 ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) { 1598 domain = AMDGPU_GEM_DOMAIN_VRAM; 1599 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) 1600 domain = AMDGPU_GEM_DOMAIN_GTT; 1601 } 1602 return domain; 1603 } 1604 1605 #if defined(CONFIG_DEBUG_FS) 1606 #define amdgpu_bo_print_flag(m, bo, flag) \ 1607 do { \ 1608 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ 1609 seq_printf((m), " " #flag); \ 1610 } \ 1611 } while (0) 1612 1613 /** 1614 * amdgpu_bo_print_info - print BO info in debugfs file 1615 * 1616 * @id: Index or Id of the BO 1617 * @bo: Requested BO for printing info 1618 * @m: debugfs file 1619 * 1620 * Print BO information in debugfs file 1621 * 1622 * Returns: 1623 * Size of the BO in bytes. 1624 */ 1625 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m) 1626 { 1627 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1628 struct dma_buf_attachment *attachment; 1629 struct dma_buf *dma_buf; 1630 const char *placement; 1631 unsigned int pin_count; 1632 u64 size; 1633 1634 if (dma_resv_trylock(bo->tbo.base.resv)) { 1635 if (!bo->tbo.resource) { 1636 placement = "NONE"; 1637 } else { 1638 switch (bo->tbo.resource->mem_type) { 1639 case TTM_PL_VRAM: 1640 if (amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 1641 placement = "VRAM VISIBLE"; 1642 else 1643 placement = "VRAM"; 1644 break; 1645 case TTM_PL_TT: 1646 placement = "GTT"; 1647 break; 1648 case AMDGPU_PL_GDS: 1649 placement = "GDS"; 1650 break; 1651 case AMDGPU_PL_GWS: 1652 placement = "GWS"; 1653 break; 1654 case AMDGPU_PL_OA: 1655 placement = "OA"; 1656 break; 1657 case AMDGPU_PL_PREEMPT: 1658 placement = "PREEMPTIBLE"; 1659 break; 1660 case AMDGPU_PL_DOORBELL: 1661 placement = "DOORBELL"; 1662 break; 1663 case AMDGPU_PL_MMIO_REMAP: 1664 placement = "MMIO REMAP"; 1665 break; 1666 case TTM_PL_SYSTEM: 1667 default: 1668 placement = "CPU"; 1669 break; 1670 } 1671 } 1672 dma_resv_unlock(bo->tbo.base.resv); 1673 } else { 1674 placement = "UNKNOWN"; 1675 } 1676 1677 size = amdgpu_bo_size(bo); 1678 seq_printf(m, "\t\t0x%08x: %12lld byte %s", 1679 id, size, placement); 1680 1681 pin_count = READ_ONCE(bo->tbo.pin_count); 1682 if (pin_count) 1683 seq_printf(m, " pin count %d", pin_count); 1684 1685 dma_buf = READ_ONCE(bo->tbo.base.dma_buf); 1686 attachment = READ_ONCE(bo->tbo.base.import_attach); 1687 1688 if (attachment) 1689 seq_printf(m, " imported from ino:%llu", file_inode(dma_buf->file)->i_ino); 1690 else if (dma_buf) 1691 seq_printf(m, " exported as ino:%llu", file_inode(dma_buf->file)->i_ino); 1692 1693 amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); 1694 amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS); 1695 amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC); 1696 amdgpu_bo_print_flag(m, bo, VRAM_CLEARED); 1697 amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS); 1698 amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID); 1699 amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC); 1700 /* Add the gem obj resv fence dump*/ 1701 if (dma_resv_trylock(bo->tbo.base.resv)) { 1702 dma_resv_describe(bo->tbo.base.resv, m); 1703 dma_resv_unlock(bo->tbo.base.resv); 1704 } else { 1705 seq_puts(m, "\n"); 1706 } 1707 1708 return size; 1709 } 1710 #endif 1711