1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <linux/slab.h> 34 #include <linux/dma-buf.h> 35 #include <linux/export.h> 36 37 #include <drm/drm_drv.h> 38 #include <drm/amdgpu_drm.h> 39 #include <drm/drm_cache.h> 40 #include "amdgpu.h" 41 #include "amdgpu_trace.h" 42 #include "amdgpu_amdkfd.h" 43 #include "amdgpu_vram_mgr.h" 44 #include "amdgpu_vm.h" 45 #include "amdgpu_dma_buf.h" 46 47 /** 48 * DOC: amdgpu_object 49 * 50 * This defines the interfaces to operate on an &amdgpu_bo buffer object which 51 * represents memory used by driver (VRAM, system memory, etc.). The driver 52 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces 53 * to create/destroy/set buffer object which are then managed by the kernel TTM 54 * memory manager. 55 * The interfaces are also used internally by kernel clients, including gfx, 56 * uvd, etc. for kernel managed allocations used by the GPU. 57 * 58 */ 59 60 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) 61 { 62 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 63 64 amdgpu_bo_kunmap(bo); 65 66 if (drm_gem_is_imported(&bo->tbo.base)) 67 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); 68 drm_gem_object_release(&bo->tbo.base); 69 amdgpu_bo_unref(&bo->parent); 70 kvfree(bo); 71 } 72 73 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo) 74 { 75 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 76 struct amdgpu_bo_user *ubo; 77 78 ubo = to_amdgpu_bo_user(bo); 79 kfree(ubo->metadata); 80 amdgpu_bo_destroy(tbo); 81 } 82 83 /** 84 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo 85 * @bo: buffer object to be checked 86 * 87 * Uses destroy function associated with the object to determine if this is 88 * an &amdgpu_bo. 89 * 90 * Returns: 91 * true if the object belongs to &amdgpu_bo, false if not. 92 */ 93 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) 94 { 95 if (bo->destroy == &amdgpu_bo_destroy || 96 bo->destroy == &amdgpu_bo_user_destroy) 97 return true; 98 99 return false; 100 } 101 102 /** 103 * amdgpu_bo_placement_from_domain - set buffer's placement 104 * @abo: &amdgpu_bo buffer object whose placement is to be set 105 * @domain: requested domain 106 * 107 * Sets buffer's placement according to requested domain and the buffer's 108 * flags. 109 */ 110 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) 111 { 112 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 113 struct ttm_placement *placement = &abo->placement; 114 struct ttm_place *places = abo->placements; 115 u64 flags = abo->flags; 116 u32 c = 0; 117 118 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 119 unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 120 int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); 121 122 if (adev->gmc.mem_partitions && mem_id >= 0) { 123 places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn; 124 /* 125 * memory partition range lpfn is inclusive start + size - 1 126 * TTM place lpfn is exclusive start + size 127 */ 128 places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1; 129 } else { 130 places[c].fpfn = 0; 131 places[c].lpfn = 0; 132 } 133 places[c].mem_type = TTM_PL_VRAM; 134 places[c].flags = 0; 135 136 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 137 places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn); 138 else 139 places[c].flags |= TTM_PL_FLAG_TOPDOWN; 140 141 if (abo->tbo.type == ttm_bo_type_kernel && 142 flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) 143 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; 144 145 c++; 146 } 147 148 if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) { 149 places[c].fpfn = 0; 150 places[c].lpfn = 0; 151 places[c].mem_type = AMDGPU_PL_DOORBELL; 152 places[c].flags = 0; 153 c++; 154 } 155 156 if (domain & AMDGPU_GEM_DOMAIN_MMIO_REMAP) { 157 places[c].fpfn = 0; 158 places[c].lpfn = 0; 159 places[c].mem_type = AMDGPU_PL_MMIO_REMAP; 160 places[c].flags = 0; 161 c++; 162 } 163 164 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 165 places[c].fpfn = 0; 166 places[c].lpfn = 0; 167 places[c].mem_type = 168 abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ? 169 AMDGPU_PL_PREEMPT : TTM_PL_TT; 170 places[c].flags = 0; 171 /* 172 * When GTT is just an alternative to VRAM make sure that we 173 * only use it as fallback and still try to fill up VRAM first. 174 */ 175 if (abo->tbo.resource && !(adev->flags & AMD_IS_APU) && 176 domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) 177 places[c].flags |= TTM_PL_FLAG_FALLBACK; 178 c++; 179 } 180 181 if (domain & AMDGPU_GEM_DOMAIN_CPU) { 182 places[c].fpfn = 0; 183 places[c].lpfn = 0; 184 places[c].mem_type = TTM_PL_SYSTEM; 185 places[c].flags = 0; 186 c++; 187 } 188 189 if (domain & AMDGPU_GEM_DOMAIN_GDS) { 190 places[c].fpfn = 0; 191 places[c].lpfn = 0; 192 places[c].mem_type = AMDGPU_PL_GDS; 193 places[c].flags = 0; 194 c++; 195 } 196 197 if (domain & AMDGPU_GEM_DOMAIN_GWS) { 198 places[c].fpfn = 0; 199 places[c].lpfn = 0; 200 places[c].mem_type = AMDGPU_PL_GWS; 201 places[c].flags = 0; 202 c++; 203 } 204 205 if (domain & AMDGPU_GEM_DOMAIN_OA) { 206 places[c].fpfn = 0; 207 places[c].lpfn = 0; 208 places[c].mem_type = AMDGPU_PL_OA; 209 places[c].flags = 0; 210 c++; 211 } 212 213 if (!c) { 214 places[c].fpfn = 0; 215 places[c].lpfn = 0; 216 places[c].mem_type = TTM_PL_SYSTEM; 217 places[c].flags = 0; 218 c++; 219 } 220 221 BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS); 222 223 placement->num_placement = c; 224 placement->placement = places; 225 } 226 227 /** 228 * amdgpu_bo_create_reserved - create reserved BO for kernel use 229 * 230 * @adev: amdgpu device object 231 * @size: size for the new BO 232 * @align: alignment for the new BO 233 * @domain: where to place it 234 * @bo_ptr: used to initialize BOs in structures 235 * @gpu_addr: GPU addr of the pinned BO 236 * @cpu_addr: optional CPU address mapping 237 * 238 * Allocates and pins a BO for kernel internal use, and returns it still 239 * reserved. 240 * 241 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 242 * 243 * Returns: 244 * 0 on success, negative error code otherwise. 245 */ 246 int amdgpu_bo_create_reserved(struct amdgpu_device *adev, 247 unsigned long size, int align, 248 u32 domain, struct amdgpu_bo **bo_ptr, 249 u64 *gpu_addr, void **cpu_addr) 250 { 251 struct amdgpu_bo_param bp; 252 bool free = false; 253 int r; 254 255 if (!size) { 256 amdgpu_bo_unref(bo_ptr); 257 return 0; 258 } 259 260 memset(&bp, 0, sizeof(bp)); 261 bp.size = size; 262 bp.byte_align = align; 263 bp.domain = domain; 264 bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED 265 : AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 266 bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 267 bp.type = ttm_bo_type_kernel; 268 bp.resv = NULL; 269 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 270 271 if (!*bo_ptr) { 272 r = amdgpu_bo_create(adev, &bp, bo_ptr); 273 if (r) { 274 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", 275 r); 276 return r; 277 } 278 free = true; 279 } 280 281 r = amdgpu_bo_reserve(*bo_ptr, false); 282 if (r) { 283 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); 284 goto error_free; 285 } 286 287 r = amdgpu_bo_pin(*bo_ptr, domain); 288 if (r) { 289 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); 290 goto error_unreserve; 291 } 292 293 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); 294 if (r) { 295 dev_err(adev->dev, "%p bind failed\n", *bo_ptr); 296 goto error_unpin; 297 } 298 299 if (gpu_addr) 300 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); 301 302 if (cpu_addr) { 303 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 304 if (r) { 305 dev_err(adev->dev, "(%d) kernel bo map failed\n", r); 306 goto error_unpin; 307 } 308 } 309 310 return 0; 311 312 error_unpin: 313 amdgpu_bo_unpin(*bo_ptr); 314 error_unreserve: 315 amdgpu_bo_unreserve(*bo_ptr); 316 317 error_free: 318 if (free) 319 amdgpu_bo_unref(bo_ptr); 320 321 return r; 322 } 323 324 /** 325 * amdgpu_bo_create_kernel - create BO for kernel use 326 * 327 * @adev: amdgpu device object 328 * @size: size for the new BO 329 * @align: alignment for the new BO 330 * @domain: where to place it 331 * @bo_ptr: used to initialize BOs in structures 332 * @gpu_addr: GPU addr of the pinned BO 333 * @cpu_addr: optional CPU address mapping 334 * 335 * Allocates and pins a BO for kernel internal use. 336 * 337 * This function is exported to allow the V4L2 isp device 338 * external to drm device to create and access the kernel BO. 339 * 340 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 341 * 342 * Returns: 343 * 0 on success, negative error code otherwise. 344 */ 345 int amdgpu_bo_create_kernel(struct amdgpu_device *adev, 346 unsigned long size, int align, 347 u32 domain, struct amdgpu_bo **bo_ptr, 348 u64 *gpu_addr, void **cpu_addr) 349 { 350 int r; 351 352 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, 353 gpu_addr, cpu_addr); 354 355 if (r) 356 return r; 357 358 if (*bo_ptr) 359 amdgpu_bo_unreserve(*bo_ptr); 360 361 return 0; 362 } 363 364 /** 365 * amdgpu_bo_create_isp_user - create user BO for isp 366 * 367 * @adev: amdgpu device object 368 * @dma_buf: DMABUF handle for isp buffer 369 * @domain: where to place it 370 * @bo: used to initialize BOs in structures 371 * @gpu_addr: GPU addr of the pinned BO 372 * 373 * Imports isp DMABUF to allocate and pin a user BO for isp internal use. It does 374 * GART alloc to generate gpu_addr for BO to make it accessible through the 375 * GART aperture for ISP HW. 376 * 377 * This function is exported to allow the V4L2 isp device external to drm device 378 * to create and access the isp user BO. 379 * 380 * Returns: 381 * 0 on success, negative error code otherwise. 382 */ 383 int amdgpu_bo_create_isp_user(struct amdgpu_device *adev, 384 struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo, 385 u64 *gpu_addr) 386 387 { 388 struct drm_gem_object *gem_obj; 389 int r; 390 391 gem_obj = amdgpu_gem_prime_import(&adev->ddev, dma_buf); 392 *bo = gem_to_amdgpu_bo(gem_obj); 393 if (!(*bo)) { 394 dev_err(adev->dev, "failed to get valid isp user bo\n"); 395 return -EINVAL; 396 } 397 398 r = amdgpu_bo_reserve(*bo, false); 399 if (r) { 400 dev_err(adev->dev, "(%d) failed to reserve isp user bo\n", r); 401 return r; 402 } 403 404 r = amdgpu_bo_pin(*bo, domain); 405 if (r) { 406 dev_err(adev->dev, "(%d) isp user bo pin failed\n", r); 407 goto error_unreserve; 408 } 409 410 r = amdgpu_ttm_alloc_gart(&(*bo)->tbo); 411 if (r) { 412 dev_err(adev->dev, "%p bind failed\n", *bo); 413 goto error_unpin; 414 } 415 416 if (!WARN_ON(!gpu_addr)) 417 *gpu_addr = amdgpu_bo_gpu_offset(*bo); 418 419 amdgpu_bo_unreserve(*bo); 420 421 return 0; 422 423 error_unpin: 424 amdgpu_bo_unpin(*bo); 425 error_unreserve: 426 amdgpu_bo_unreserve(*bo); 427 amdgpu_bo_unref(bo); 428 429 return r; 430 } 431 432 /** 433 * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location 434 * 435 * @adev: amdgpu device object 436 * @offset: offset of the BO 437 * @size: size of the BO 438 * @bo_ptr: used to initialize BOs in structures 439 * @cpu_addr: optional CPU address mapping 440 * 441 * Creates a kernel BO at a specific offset in VRAM. 442 * 443 * Returns: 444 * 0 on success, negative error code otherwise. 445 */ 446 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, 447 uint64_t offset, uint64_t size, 448 struct amdgpu_bo **bo_ptr, void **cpu_addr) 449 { 450 struct ttm_operation_ctx ctx = { false, false }; 451 unsigned int i; 452 int r; 453 454 offset &= PAGE_MASK; 455 size = ALIGN(size, PAGE_SIZE); 456 457 r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, 458 AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL, 459 cpu_addr); 460 if (r) 461 return r; 462 463 if ((*bo_ptr) == NULL) 464 return 0; 465 466 /* 467 * Remove the original mem node and create a new one at the request 468 * position. 469 */ 470 if (cpu_addr) 471 amdgpu_bo_kunmap(*bo_ptr); 472 473 ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource); 474 475 for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) { 476 (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT; 477 (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; 478 } 479 r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement, 480 &(*bo_ptr)->tbo.resource, &ctx); 481 if (r) 482 goto error; 483 484 if (cpu_addr) { 485 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 486 if (r) 487 goto error; 488 } 489 490 amdgpu_bo_unreserve(*bo_ptr); 491 return 0; 492 493 error: 494 amdgpu_bo_unreserve(*bo_ptr); 495 amdgpu_bo_unref(bo_ptr); 496 return r; 497 } 498 499 /** 500 * amdgpu_bo_free_kernel - free BO for kernel use 501 * 502 * @bo: amdgpu BO to free 503 * @gpu_addr: pointer to where the BO's GPU memory space address was stored 504 * @cpu_addr: pointer to where the BO's CPU memory space address was stored 505 * 506 * unmaps and unpin a BO for kernel internal use. 507 * 508 * This function is exported to allow the V4L2 isp device 509 * external to drm device to free the kernel BO. 510 */ 511 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, 512 void **cpu_addr) 513 { 514 if (*bo == NULL) 515 return; 516 517 WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); 518 519 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { 520 if (cpu_addr) 521 amdgpu_bo_kunmap(*bo); 522 523 amdgpu_bo_unpin(*bo); 524 amdgpu_bo_unreserve(*bo); 525 } 526 amdgpu_bo_unref(bo); 527 528 if (gpu_addr) 529 *gpu_addr = 0; 530 531 if (cpu_addr) 532 *cpu_addr = NULL; 533 } 534 535 /** 536 * amdgpu_bo_free_isp_user - free BO for isp use 537 * 538 * @bo: amdgpu isp user BO to free 539 * 540 * unpin and unref BO for isp internal use. 541 * 542 * This function is exported to allow the V4L2 isp device 543 * external to drm device to free the isp user BO. 544 */ 545 void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo) 546 { 547 if (bo == NULL) 548 return; 549 550 if (amdgpu_bo_reserve(bo, true) == 0) { 551 amdgpu_bo_unpin(bo); 552 amdgpu_bo_unreserve(bo); 553 } 554 amdgpu_bo_unref(&bo); 555 } 556 557 /* Validate bo size is bit bigger than the request domain */ 558 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, 559 unsigned long size, u32 domain) 560 { 561 struct ttm_resource_manager *man = NULL; 562 563 /* 564 * If GTT is part of requested domains the check must succeed to 565 * allow fall back to GTT. 566 */ 567 if (domain & AMDGPU_GEM_DOMAIN_GTT) 568 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); 569 else if (domain & AMDGPU_GEM_DOMAIN_VRAM) 570 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 571 else 572 return true; 573 574 if (!man) { 575 if (domain & AMDGPU_GEM_DOMAIN_GTT) 576 WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized"); 577 return false; 578 } 579 580 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */ 581 if (size < man->size) 582 return true; 583 584 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size); 585 return false; 586 } 587 588 bool amdgpu_bo_support_uswc(u64 bo_flags) 589 { 590 591 #ifdef CONFIG_X86_32 592 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit 593 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 594 */ 595 return false; 596 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) 597 /* Don't try to enable write-combining when it can't work, or things 598 * may be slow 599 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 600 */ 601 602 #ifndef CONFIG_COMPILE_TEST 603 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ 604 thanks to write-combining 605 #endif 606 607 if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 608 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " 609 "better performance thanks to write-combining\n"); 610 return false; 611 #else 612 /* For architectures that don't support WC memory, 613 * mask out the WC flag from the BO 614 */ 615 if (!drm_arch_can_wc_memory()) 616 return false; 617 618 return true; 619 #endif 620 } 621 622 /** 623 * amdgpu_bo_create - create an &amdgpu_bo buffer object 624 * @adev: amdgpu device object 625 * @bp: parameters to be used for the buffer object 626 * @bo_ptr: pointer to the buffer object pointer 627 * 628 * Creates an &amdgpu_bo buffer object. 629 * 630 * Returns: 631 * 0 for success or a negative error code on failure. 632 */ 633 int amdgpu_bo_create(struct amdgpu_device *adev, 634 struct amdgpu_bo_param *bp, 635 struct amdgpu_bo **bo_ptr) 636 { 637 struct ttm_operation_ctx ctx = { 638 .interruptible = (bp->type != ttm_bo_type_kernel), 639 .no_wait_gpu = bp->no_wait_gpu, 640 /* We opt to avoid OOM on system pages allocations */ 641 .gfp_retry_mayfail = true, 642 .allow_res_evict = bp->type != ttm_bo_type_kernel, 643 .resv = bp->resv 644 }; 645 struct amdgpu_bo *bo; 646 unsigned long page_align, size = bp->size; 647 int r; 648 649 /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ 650 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { 651 /* GWS and OA don't need any alignment. */ 652 page_align = bp->byte_align; 653 size <<= PAGE_SHIFT; 654 655 } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { 656 /* Both size and alignment must be a multiple of 4. */ 657 page_align = ALIGN(bp->byte_align, 4); 658 size = ALIGN(size, 4) << PAGE_SHIFT; 659 } else { 660 /* Memory should be aligned at least to a page size. */ 661 page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; 662 size = ALIGN(size, PAGE_SIZE); 663 } 664 665 if (!amdgpu_bo_validate_size(adev, size, bp->domain)) 666 return -ENOMEM; 667 668 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo)); 669 670 *bo_ptr = NULL; 671 bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL); 672 if (bo == NULL) 673 return -ENOMEM; 674 drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); 675 bo->tbo.base.funcs = &amdgpu_gem_object_funcs; 676 bo->vm_bo = NULL; 677 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : 678 bp->domain; 679 bo->allowed_domains = bo->preferred_domains; 680 if (bp->type != ttm_bo_type_kernel && 681 !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) && 682 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) 683 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 684 685 bo->flags = bp->flags; 686 687 if (adev->gmc.mem_partitions) 688 /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */ 689 bo->xcp_id = bp->xcp_id_plus1 - 1; 690 else 691 /* For GPUs without spatial partitioning */ 692 bo->xcp_id = 0; 693 694 if (!amdgpu_bo_support_uswc(bo->flags)) 695 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 696 697 bo->tbo.bdev = &adev->mman.bdev; 698 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | 699 AMDGPU_GEM_DOMAIN_GDS)) 700 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 701 else 702 amdgpu_bo_placement_from_domain(bo, bp->domain); 703 if (bp->type == ttm_bo_type_kernel) 704 bo->tbo.priority = 2; 705 else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE)) 706 bo->tbo.priority = 1; 707 708 if (!bp->destroy) 709 bp->destroy = &amdgpu_bo_destroy; 710 711 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type, 712 &bo->placement, page_align, &ctx, NULL, 713 bp->resv, bp->destroy); 714 if (unlikely(r != 0)) 715 return r; 716 717 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 718 amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 719 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 720 ctx.bytes_moved); 721 else 722 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); 723 724 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && 725 bo->tbo.resource->mem_type == TTM_PL_VRAM) { 726 struct dma_fence *fence; 727 728 r = amdgpu_ttm_clear_buffer(bo, bo->tbo.base.resv, &fence); 729 if (unlikely(r)) 730 goto fail_unreserve; 731 732 dma_resv_add_fence(bo->tbo.base.resv, fence, 733 DMA_RESV_USAGE_KERNEL); 734 dma_fence_put(fence); 735 } 736 if (!bp->resv) 737 amdgpu_bo_unreserve(bo); 738 *bo_ptr = bo; 739 740 trace_amdgpu_bo_create(bo); 741 742 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ 743 if (bp->type == ttm_bo_type_device) 744 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 745 746 return 0; 747 748 fail_unreserve: 749 if (!bp->resv) 750 dma_resv_unlock(bo->tbo.base.resv); 751 amdgpu_bo_unref(&bo); 752 return r; 753 } 754 755 /** 756 * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object 757 * @adev: amdgpu device object 758 * @bp: parameters to be used for the buffer object 759 * @ubo_ptr: pointer to the buffer object pointer 760 * 761 * Create a BO to be used by user application; 762 * 763 * Returns: 764 * 0 for success or a negative error code on failure. 765 */ 766 767 int amdgpu_bo_create_user(struct amdgpu_device *adev, 768 struct amdgpu_bo_param *bp, 769 struct amdgpu_bo_user **ubo_ptr) 770 { 771 struct amdgpu_bo *bo_ptr; 772 int r; 773 774 bp->bo_ptr_size = sizeof(struct amdgpu_bo_user); 775 bp->destroy = &amdgpu_bo_user_destroy; 776 r = amdgpu_bo_create(adev, bp, &bo_ptr); 777 if (r) 778 return r; 779 780 *ubo_ptr = to_amdgpu_bo_user(bo_ptr); 781 return r; 782 } 783 784 /** 785 * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object 786 * @adev: amdgpu device object 787 * @bp: parameters to be used for the buffer object 788 * @vmbo_ptr: pointer to the buffer object pointer 789 * 790 * Create a BO to be for GPUVM. 791 * 792 * Returns: 793 * 0 for success or a negative error code on failure. 794 */ 795 796 int amdgpu_bo_create_vm(struct amdgpu_device *adev, 797 struct amdgpu_bo_param *bp, 798 struct amdgpu_bo_vm **vmbo_ptr) 799 { 800 struct amdgpu_bo *bo_ptr; 801 int r; 802 803 /* bo_ptr_size will be determined by the caller and it depends on 804 * num of amdgpu_vm_pt entries. 805 */ 806 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm)); 807 r = amdgpu_bo_create(adev, bp, &bo_ptr); 808 if (r) 809 return r; 810 811 *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr); 812 return r; 813 } 814 815 /** 816 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object 817 * @bo: &amdgpu_bo buffer object to be mapped 818 * @ptr: kernel virtual address to be returned 819 * 820 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls 821 * amdgpu_bo_kptr() to get the kernel virtual address. 822 * 823 * Returns: 824 * 0 for success or a negative error code on failure. 825 */ 826 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) 827 { 828 void *kptr; 829 long r; 830 831 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 832 return -EPERM; 833 834 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL, 835 false, MAX_SCHEDULE_TIMEOUT); 836 if (r < 0) 837 return r; 838 839 kptr = amdgpu_bo_kptr(bo); 840 if (kptr) { 841 if (ptr) 842 *ptr = kptr; 843 return 0; 844 } 845 846 r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap); 847 if (r) 848 return r; 849 850 if (ptr) 851 *ptr = amdgpu_bo_kptr(bo); 852 853 return 0; 854 } 855 856 /** 857 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object 858 * @bo: &amdgpu_bo buffer object 859 * 860 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address 861 * 862 * Returns: 863 * the virtual address of a buffer object area. 864 */ 865 void *amdgpu_bo_kptr(struct amdgpu_bo *bo) 866 { 867 bool is_iomem; 868 869 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 870 } 871 872 /** 873 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object 874 * @bo: &amdgpu_bo buffer object to be unmapped 875 * 876 * Unmaps a kernel map set up by amdgpu_bo_kmap(). 877 */ 878 void amdgpu_bo_kunmap(struct amdgpu_bo *bo) 879 { 880 if (bo->kmap.bo) 881 ttm_bo_kunmap(&bo->kmap); 882 } 883 884 /** 885 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object 886 * @bo: &amdgpu_bo buffer object 887 * 888 * References the contained &ttm_buffer_object. 889 * 890 * Returns: 891 * a refcounted pointer to the &amdgpu_bo buffer object. 892 */ 893 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) 894 { 895 if (bo == NULL) 896 return NULL; 897 898 drm_gem_object_get(&bo->tbo.base); 899 return bo; 900 } 901 902 /** 903 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object 904 * @bo: &amdgpu_bo buffer object 905 * 906 * Unreferences the contained &ttm_buffer_object and clear the pointer 907 */ 908 void amdgpu_bo_unref(struct amdgpu_bo **bo) 909 { 910 if ((*bo) == NULL) 911 return; 912 913 drm_gem_object_put(&(*bo)->tbo.base); 914 *bo = NULL; 915 } 916 917 /** 918 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object 919 * @bo: &amdgpu_bo buffer object to be pinned 920 * @domain: domain to be pinned to 921 * 922 * Pins the buffer object according to requested domain. If the memory is 923 * unbound gart memory, binds the pages into gart table. Adjusts pin_count and 924 * pin_size accordingly. 925 * 926 * Pinning means to lock pages in memory along with keeping them at a fixed 927 * offset. It is required when a buffer can not be moved, for example, when 928 * a display buffer is being scanned out. 929 * 930 * Returns: 931 * 0 for success or a negative error code on failure. 932 */ 933 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) 934 { 935 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 936 struct ttm_operation_ctx ctx = { false, false }; 937 int r, i; 938 939 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) 940 return -EPERM; 941 942 /* Check domain to be pinned to against preferred domains */ 943 if (bo->preferred_domains & domain) 944 domain = bo->preferred_domains & domain; 945 946 /* A shared bo cannot be migrated to VRAM */ 947 if (drm_gem_is_imported(&bo->tbo.base)) { 948 if (domain & AMDGPU_GEM_DOMAIN_GTT) 949 domain = AMDGPU_GEM_DOMAIN_GTT; 950 else 951 return -EINVAL; 952 } 953 954 if (bo->tbo.pin_count) { 955 uint32_t mem_type = bo->tbo.resource->mem_type; 956 uint32_t mem_flags = bo->tbo.resource->placement; 957 958 if (!(domain & amdgpu_mem_type_to_domain(mem_type))) 959 return -EINVAL; 960 961 if ((mem_type == TTM_PL_VRAM) && 962 (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && 963 !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) 964 return -EINVAL; 965 966 ttm_bo_pin(&bo->tbo); 967 return 0; 968 } 969 970 /* This assumes only APU display buffers are pinned with (VRAM|GTT). 971 * See function amdgpu_display_supported_domains() 972 */ 973 domain = amdgpu_bo_get_preferred_domain(adev, domain); 974 975 if (drm_gem_is_imported(&bo->tbo.base)) 976 dma_buf_pin(bo->tbo.base.import_attach); 977 978 /* force to pin into visible video ram */ 979 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) 980 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 981 amdgpu_bo_placement_from_domain(bo, domain); 982 for (i = 0; i < bo->placement.num_placement; i++) { 983 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && 984 bo->placements[i].mem_type == TTM_PL_VRAM) 985 bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS; 986 } 987 988 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 989 if (unlikely(r)) { 990 dev_err(adev->dev, "%p pin failed\n", bo); 991 goto error; 992 } 993 994 ttm_bo_pin(&bo->tbo); 995 996 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 997 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); 998 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), 999 &adev->visible_pin_size); 1000 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 1001 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); 1002 } 1003 1004 error: 1005 return r; 1006 } 1007 1008 /** 1009 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object 1010 * @bo: &amdgpu_bo buffer object to be unpinned 1011 * 1012 * Decreases the pin_count, and clears the flags if pin_count reaches 0. 1013 * Changes placement and pin size accordingly. 1014 * 1015 * Returns: 1016 * 0 for success or a negative error code on failure. 1017 */ 1018 void amdgpu_bo_unpin(struct amdgpu_bo *bo) 1019 { 1020 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1021 1022 ttm_bo_unpin(&bo->tbo); 1023 if (bo->tbo.pin_count) 1024 return; 1025 1026 if (drm_gem_is_imported(&bo->tbo.base)) 1027 dma_buf_unpin(bo->tbo.base.import_attach); 1028 1029 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 1030 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); 1031 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), 1032 &adev->visible_pin_size); 1033 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 1034 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); 1035 } 1036 1037 } 1038 1039 static const char * const amdgpu_vram_names[] = { 1040 "UNKNOWN", 1041 "GDDR1", 1042 "DDR2", 1043 "GDDR3", 1044 "GDDR4", 1045 "GDDR5", 1046 "HBM", 1047 "DDR3", 1048 "DDR4", 1049 "GDDR6", 1050 "DDR5", 1051 "LPDDR4", 1052 "LPDDR5", 1053 "HBM3E", 1054 "HBM4" 1055 }; 1056 1057 /** 1058 * amdgpu_bo_init - initialize memory manager 1059 * @adev: amdgpu device object 1060 * 1061 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. 1062 * 1063 * Returns: 1064 * 0 for success or a negative error code on failure. 1065 */ 1066 int amdgpu_bo_init(struct amdgpu_device *adev) 1067 { 1068 /* On A+A platform, VRAM can be mapped as WB */ 1069 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 1070 /* reserve PAT memory space to WC for VRAM */ 1071 int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base, 1072 adev->gmc.aper_size); 1073 1074 if (r) { 1075 DRM_ERROR("Unable to set WC memtype for the aperture base\n"); 1076 return r; 1077 } 1078 1079 /* Add an MTRR for the VRAM */ 1080 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, 1081 adev->gmc.aper_size); 1082 } 1083 1084 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", 1085 adev->gmc.mc_vram_size >> 20, 1086 (unsigned long long)adev->gmc.aper_size >> 20); 1087 DRM_INFO("RAM width %dbits %s\n", 1088 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); 1089 return amdgpu_ttm_init(adev); 1090 } 1091 1092 /** 1093 * amdgpu_bo_fini - tear down memory manager 1094 * @adev: amdgpu device object 1095 * 1096 * Reverses amdgpu_bo_init() to tear down memory manager. 1097 */ 1098 void amdgpu_bo_fini(struct amdgpu_device *adev) 1099 { 1100 int idx; 1101 1102 amdgpu_ttm_fini(adev); 1103 1104 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 1105 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 1106 arch_phys_wc_del(adev->gmc.vram_mtrr); 1107 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 1108 } 1109 drm_dev_exit(idx); 1110 } 1111 } 1112 1113 /** 1114 * amdgpu_bo_set_tiling_flags - set tiling flags 1115 * @bo: &amdgpu_bo buffer object 1116 * @tiling_flags: new flags 1117 * 1118 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or 1119 * kernel driver to set the tiling flags on a buffer. 1120 * 1121 * Returns: 1122 * 0 for success or a negative error code on failure. 1123 */ 1124 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) 1125 { 1126 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1127 struct amdgpu_bo_user *ubo; 1128 1129 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1130 if (adev->family <= AMDGPU_FAMILY_CZ && 1131 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) 1132 return -EINVAL; 1133 1134 ubo = to_amdgpu_bo_user(bo); 1135 ubo->tiling_flags = tiling_flags; 1136 return 0; 1137 } 1138 1139 /** 1140 * amdgpu_bo_get_tiling_flags - get tiling flags 1141 * @bo: &amdgpu_bo buffer object 1142 * @tiling_flags: returned flags 1143 * 1144 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to 1145 * set the tiling flags on a buffer. 1146 */ 1147 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) 1148 { 1149 struct amdgpu_bo_user *ubo; 1150 1151 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1152 dma_resv_assert_held(bo->tbo.base.resv); 1153 ubo = to_amdgpu_bo_user(bo); 1154 1155 if (tiling_flags) 1156 *tiling_flags = ubo->tiling_flags; 1157 } 1158 1159 /** 1160 * amdgpu_bo_set_metadata - set metadata 1161 * @bo: &amdgpu_bo buffer object 1162 * @metadata: new metadata 1163 * @metadata_size: size of the new metadata 1164 * @flags: flags of the new metadata 1165 * 1166 * Sets buffer object's metadata, its size and flags. 1167 * Used via GEM ioctl. 1168 * 1169 * Returns: 1170 * 0 for success or a negative error code on failure. 1171 */ 1172 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata, 1173 u32 metadata_size, uint64_t flags) 1174 { 1175 struct amdgpu_bo_user *ubo; 1176 void *buffer; 1177 1178 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1179 ubo = to_amdgpu_bo_user(bo); 1180 if (!metadata_size) { 1181 if (ubo->metadata_size) { 1182 kfree(ubo->metadata); 1183 ubo->metadata = NULL; 1184 ubo->metadata_size = 0; 1185 } 1186 return 0; 1187 } 1188 1189 if (metadata == NULL) 1190 return -EINVAL; 1191 1192 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); 1193 if (buffer == NULL) 1194 return -ENOMEM; 1195 1196 kfree(ubo->metadata); 1197 ubo->metadata_flags = flags; 1198 ubo->metadata = buffer; 1199 ubo->metadata_size = metadata_size; 1200 1201 return 0; 1202 } 1203 1204 /** 1205 * amdgpu_bo_get_metadata - get metadata 1206 * @bo: &amdgpu_bo buffer object 1207 * @buffer: returned metadata 1208 * @buffer_size: size of the buffer 1209 * @metadata_size: size of the returned metadata 1210 * @flags: flags of the returned metadata 1211 * 1212 * Gets buffer object's metadata, its size and flags. buffer_size shall not be 1213 * less than metadata_size. 1214 * Used via GEM ioctl. 1215 * 1216 * Returns: 1217 * 0 for success or a negative error code on failure. 1218 */ 1219 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, 1220 size_t buffer_size, uint32_t *metadata_size, 1221 uint64_t *flags) 1222 { 1223 struct amdgpu_bo_user *ubo; 1224 1225 if (!buffer && !metadata_size) 1226 return -EINVAL; 1227 1228 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1229 ubo = to_amdgpu_bo_user(bo); 1230 if (metadata_size) 1231 *metadata_size = ubo->metadata_size; 1232 1233 if (buffer) { 1234 if (buffer_size < ubo->metadata_size) 1235 return -EINVAL; 1236 1237 if (ubo->metadata_size) 1238 memcpy(buffer, ubo->metadata, ubo->metadata_size); 1239 } 1240 1241 if (flags) 1242 *flags = ubo->metadata_flags; 1243 1244 return 0; 1245 } 1246 1247 /** 1248 * amdgpu_bo_move_notify - notification about a memory move 1249 * @bo: pointer to a buffer object 1250 * @evict: if this move is evicting the buffer from the graphics address space 1251 * @new_mem: new resource for backing the BO 1252 * 1253 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs 1254 * bookkeeping. 1255 * TTM driver callback which is called when ttm moves a buffer. 1256 */ 1257 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, 1258 bool evict, 1259 struct ttm_resource *new_mem) 1260 { 1261 struct ttm_resource *old_mem = bo->resource; 1262 struct amdgpu_bo *abo; 1263 1264 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1265 return; 1266 1267 abo = ttm_to_amdgpu_bo(bo); 1268 amdgpu_vm_bo_move(abo, new_mem, evict); 1269 1270 amdgpu_bo_kunmap(abo); 1271 1272 if (abo->tbo.base.dma_buf && !drm_gem_is_imported(&abo->tbo.base) && 1273 old_mem && old_mem->mem_type != TTM_PL_SYSTEM) 1274 dma_buf_move_notify(abo->tbo.base.dma_buf); 1275 1276 /* move_notify is called before move happens */ 1277 trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1, 1278 old_mem ? old_mem->mem_type : -1); 1279 } 1280 1281 /** 1282 * amdgpu_bo_release_notify - notification about a BO being released 1283 * @bo: pointer to a buffer object 1284 * 1285 * Wipes VRAM buffers whose contents should not be leaked before the 1286 * memory is released. 1287 */ 1288 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) 1289 { 1290 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1291 struct dma_fence *fence = NULL; 1292 struct amdgpu_bo *abo; 1293 int r; 1294 1295 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1296 return; 1297 1298 abo = ttm_to_amdgpu_bo(bo); 1299 1300 WARN_ON(abo->vm_bo); 1301 1302 if (abo->kfd_bo) 1303 amdgpu_amdkfd_release_notify(abo); 1304 1305 /* 1306 * We lock the private dma_resv object here and since the BO is about to 1307 * be released nobody else should have a pointer to it. 1308 * So when this locking here fails something is wrong with the reference 1309 * counting. 1310 */ 1311 if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv))) 1312 return; 1313 1314 amdgpu_amdkfd_remove_all_eviction_fences(abo); 1315 1316 if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM || 1317 !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) || 1318 adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev))) 1319 goto out; 1320 1321 r = dma_resv_reserve_fences(&bo->base._resv, 1); 1322 if (r) 1323 goto out; 1324 1325 r = amdgpu_fill_buffer(abo, 0, &bo->base._resv, &fence, true, 1326 AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE); 1327 if (WARN_ON(r)) 1328 goto out; 1329 1330 amdgpu_vram_mgr_set_cleared(bo->resource); 1331 dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL); 1332 dma_fence_put(fence); 1333 1334 out: 1335 dma_resv_unlock(&bo->base._resv); 1336 } 1337 1338 /** 1339 * amdgpu_bo_fault_reserve_notify - notification about a memory fault 1340 * @bo: pointer to a buffer object 1341 * 1342 * Notifies the driver we are taking a fault on this BO and have reserved it, 1343 * also performs bookkeeping. 1344 * TTM driver callback for dealing with vm faults. 1345 * 1346 * Returns: 1347 * 0 for success or a negative error code on failure. 1348 */ 1349 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 1350 { 1351 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1352 struct ttm_operation_ctx ctx = { false, false }; 1353 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1354 int r; 1355 1356 /* Remember that this BO was accessed by the CPU */ 1357 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 1358 1359 if (amdgpu_res_cpu_visible(adev, bo->resource)) 1360 return 0; 1361 1362 /* Can't move a pinned BO to visible VRAM */ 1363 if (abo->tbo.pin_count > 0) 1364 return VM_FAULT_SIGBUS; 1365 1366 /* hurrah the memory is not visible ! */ 1367 atomic64_inc(&adev->num_vram_cpu_page_faults); 1368 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 1369 AMDGPU_GEM_DOMAIN_GTT); 1370 1371 /* Avoid costly evictions; only set GTT as a busy placement */ 1372 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; 1373 1374 r = ttm_bo_validate(bo, &abo->placement, &ctx); 1375 if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) 1376 return VM_FAULT_NOPAGE; 1377 else if (unlikely(r)) 1378 return VM_FAULT_SIGBUS; 1379 1380 /* this should never happen */ 1381 if (bo->resource->mem_type == TTM_PL_VRAM && 1382 !amdgpu_res_cpu_visible(adev, bo->resource)) 1383 return VM_FAULT_SIGBUS; 1384 1385 ttm_bo_move_to_lru_tail_unlocked(bo); 1386 return 0; 1387 } 1388 1389 /** 1390 * amdgpu_bo_fence - add fence to buffer object 1391 * 1392 * @bo: buffer object in question 1393 * @fence: fence to add 1394 * @shared: true if fence should be added shared 1395 * 1396 */ 1397 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, 1398 bool shared) 1399 { 1400 struct dma_resv *resv = bo->tbo.base.resv; 1401 int r; 1402 1403 r = dma_resv_reserve_fences(resv, 1); 1404 if (r) { 1405 /* As last resort on OOM we block for the fence */ 1406 dma_fence_wait(fence, false); 1407 return; 1408 } 1409 1410 dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ : 1411 DMA_RESV_USAGE_WRITE); 1412 } 1413 1414 /** 1415 * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences 1416 * 1417 * @adev: amdgpu device pointer 1418 * @resv: reservation object to sync to 1419 * @sync_mode: synchronization mode 1420 * @owner: fence owner 1421 * @intr: Whether the wait is interruptible 1422 * 1423 * Extract the fences from the reservation object and waits for them to finish. 1424 * 1425 * Returns: 1426 * 0 on success, errno otherwise. 1427 */ 1428 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, 1429 enum amdgpu_sync_mode sync_mode, void *owner, 1430 bool intr) 1431 { 1432 struct amdgpu_sync sync; 1433 int r; 1434 1435 amdgpu_sync_create(&sync); 1436 amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner); 1437 r = amdgpu_sync_wait(&sync, intr); 1438 amdgpu_sync_free(&sync); 1439 return r; 1440 } 1441 1442 /** 1443 * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv 1444 * @bo: buffer object to wait for 1445 * @owner: fence owner 1446 * @intr: Whether the wait is interruptible 1447 * 1448 * Wrapper to wait for fences in a BO. 1449 * Returns: 1450 * 0 on success, errno otherwise. 1451 */ 1452 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) 1453 { 1454 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1455 1456 return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, 1457 AMDGPU_SYNC_NE_OWNER, owner, intr); 1458 } 1459 1460 /** 1461 * amdgpu_bo_gpu_offset - return GPU offset of bo 1462 * @bo: amdgpu object for which we query the offset 1463 * 1464 * Note: object should either be pinned or reserved when calling this 1465 * function, it might be useful to add check for this for debugging. 1466 * 1467 * Returns: 1468 * current GPU offset of the object. 1469 */ 1470 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) 1471 { 1472 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM); 1473 WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) && 1474 !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel); 1475 WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET); 1476 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM && 1477 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); 1478 1479 return amdgpu_bo_gpu_offset_no_check(bo); 1480 } 1481 1482 /** 1483 * amdgpu_bo_fb_aper_addr - return FB aperture GPU offset of the VRAM bo 1484 * @bo: amdgpu VRAM buffer object for which we query the offset 1485 * 1486 * Returns: 1487 * current FB aperture GPU offset of the object. 1488 */ 1489 u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo) 1490 { 1491 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1492 uint64_t offset, fb_base; 1493 1494 WARN_ON_ONCE(bo->tbo.resource->mem_type != TTM_PL_VRAM); 1495 1496 fb_base = adev->gmc.fb_start; 1497 fb_base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1498 offset = (bo->tbo.resource->start << PAGE_SHIFT) + fb_base; 1499 return amdgpu_gmc_sign_extend(offset); 1500 } 1501 1502 /** 1503 * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo 1504 * @bo: amdgpu object for which we query the offset 1505 * 1506 * Returns: 1507 * current GPU offset of the object without raising warnings. 1508 */ 1509 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) 1510 { 1511 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1512 uint64_t offset = AMDGPU_BO_INVALID_OFFSET; 1513 1514 if (bo->tbo.resource->mem_type == TTM_PL_TT) 1515 offset = amdgpu_gmc_agp_addr(&bo->tbo); 1516 1517 if (offset == AMDGPU_BO_INVALID_OFFSET) 1518 offset = (bo->tbo.resource->start << PAGE_SHIFT) + 1519 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type); 1520 1521 return amdgpu_gmc_sign_extend(offset); 1522 } 1523 1524 /** 1525 * amdgpu_bo_mem_stats_placement - bo placement for memory accounting 1526 * @bo: the buffer object we should look at 1527 * 1528 * BO can have multiple preferred placements, to avoid double counting we want 1529 * to file it under a single placement for memory stats. 1530 * Luckily, if we take the highest set bit in preferred_domains the result is 1531 * quite sensible. 1532 * 1533 * Returns: 1534 * Which of the placements should the BO be accounted under. 1535 */ 1536 uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo) 1537 { 1538 uint32_t domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK; 1539 1540 if (!domain) 1541 return TTM_PL_SYSTEM; 1542 1543 switch (rounddown_pow_of_two(domain)) { 1544 case AMDGPU_GEM_DOMAIN_CPU: 1545 return TTM_PL_SYSTEM; 1546 case AMDGPU_GEM_DOMAIN_GTT: 1547 return TTM_PL_TT; 1548 case AMDGPU_GEM_DOMAIN_VRAM: 1549 return TTM_PL_VRAM; 1550 case AMDGPU_GEM_DOMAIN_GDS: 1551 return AMDGPU_PL_GDS; 1552 case AMDGPU_GEM_DOMAIN_GWS: 1553 return AMDGPU_PL_GWS; 1554 case AMDGPU_GEM_DOMAIN_OA: 1555 return AMDGPU_PL_OA; 1556 case AMDGPU_GEM_DOMAIN_DOORBELL: 1557 return AMDGPU_PL_DOORBELL; 1558 case AMDGPU_GEM_DOMAIN_MMIO_REMAP: 1559 return AMDGPU_PL_MMIO_REMAP; 1560 default: 1561 return TTM_PL_SYSTEM; 1562 } 1563 } 1564 1565 /** 1566 * amdgpu_bo_get_preferred_domain - get preferred domain 1567 * @adev: amdgpu device object 1568 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` 1569 * 1570 * Returns: 1571 * Which of the allowed domains is preferred for allocating the BO. 1572 */ 1573 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, 1574 uint32_t domain) 1575 { 1576 if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && 1577 ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) { 1578 domain = AMDGPU_GEM_DOMAIN_VRAM; 1579 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) 1580 domain = AMDGPU_GEM_DOMAIN_GTT; 1581 } 1582 return domain; 1583 } 1584 1585 #if defined(CONFIG_DEBUG_FS) 1586 #define amdgpu_bo_print_flag(m, bo, flag) \ 1587 do { \ 1588 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ 1589 seq_printf((m), " " #flag); \ 1590 } \ 1591 } while (0) 1592 1593 /** 1594 * amdgpu_bo_print_info - print BO info in debugfs file 1595 * 1596 * @id: Index or Id of the BO 1597 * @bo: Requested BO for printing info 1598 * @m: debugfs file 1599 * 1600 * Print BO information in debugfs file 1601 * 1602 * Returns: 1603 * Size of the BO in bytes. 1604 */ 1605 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m) 1606 { 1607 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1608 struct dma_buf_attachment *attachment; 1609 struct dma_buf *dma_buf; 1610 const char *placement; 1611 unsigned int pin_count; 1612 u64 size; 1613 1614 if (dma_resv_trylock(bo->tbo.base.resv)) { 1615 if (!bo->tbo.resource) { 1616 placement = "NONE"; 1617 } else { 1618 switch (bo->tbo.resource->mem_type) { 1619 case TTM_PL_VRAM: 1620 if (amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 1621 placement = "VRAM VISIBLE"; 1622 else 1623 placement = "VRAM"; 1624 break; 1625 case TTM_PL_TT: 1626 placement = "GTT"; 1627 break; 1628 case AMDGPU_PL_GDS: 1629 placement = "GDS"; 1630 break; 1631 case AMDGPU_PL_GWS: 1632 placement = "GWS"; 1633 break; 1634 case AMDGPU_PL_OA: 1635 placement = "OA"; 1636 break; 1637 case AMDGPU_PL_PREEMPT: 1638 placement = "PREEMPTIBLE"; 1639 break; 1640 case AMDGPU_PL_DOORBELL: 1641 placement = "DOORBELL"; 1642 break; 1643 case AMDGPU_PL_MMIO_REMAP: 1644 placement = "MMIO REMAP"; 1645 break; 1646 case TTM_PL_SYSTEM: 1647 default: 1648 placement = "CPU"; 1649 break; 1650 } 1651 } 1652 dma_resv_unlock(bo->tbo.base.resv); 1653 } else { 1654 placement = "UNKNOWN"; 1655 } 1656 1657 size = amdgpu_bo_size(bo); 1658 seq_printf(m, "\t\t0x%08x: %12lld byte %s", 1659 id, size, placement); 1660 1661 pin_count = READ_ONCE(bo->tbo.pin_count); 1662 if (pin_count) 1663 seq_printf(m, " pin count %d", pin_count); 1664 1665 dma_buf = READ_ONCE(bo->tbo.base.dma_buf); 1666 attachment = READ_ONCE(bo->tbo.base.import_attach); 1667 1668 if (attachment) 1669 seq_printf(m, " imported from ino:%lu", file_inode(dma_buf->file)->i_ino); 1670 else if (dma_buf) 1671 seq_printf(m, " exported as ino:%lu", file_inode(dma_buf->file)->i_ino); 1672 1673 amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); 1674 amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS); 1675 amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC); 1676 amdgpu_bo_print_flag(m, bo, VRAM_CLEARED); 1677 amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS); 1678 amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID); 1679 amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC); 1680 /* Add the gem obj resv fence dump*/ 1681 if (dma_resv_trylock(bo->tbo.base.resv)) { 1682 dma_resv_describe(bo->tbo.base.resv, m); 1683 dma_resv_unlock(bo->tbo.base.resv); 1684 } 1685 seq_puts(m, "\n"); 1686 1687 return size; 1688 } 1689 #endif 1690