1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <linux/slab.h> 34 #include <linux/dma-buf.h> 35 #include <linux/export.h> 36 37 #include <drm/drm_drv.h> 38 #include <drm/amdgpu_drm.h> 39 #include <drm/drm_cache.h> 40 #include "amdgpu.h" 41 #include "amdgpu_trace.h" 42 #include "amdgpu_amdkfd.h" 43 #include "amdgpu_vram_mgr.h" 44 #include "amdgpu_vm.h" 45 #include "amdgpu_dma_buf.h" 46 47 /** 48 * DOC: amdgpu_object 49 * 50 * This defines the interfaces to operate on an &amdgpu_bo buffer object which 51 * represents memory used by driver (VRAM, system memory, etc.). The driver 52 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces 53 * to create/destroy/set buffer object which are then managed by the kernel TTM 54 * memory manager. 55 * The interfaces are also used internally by kernel clients, including gfx, 56 * uvd, etc. for kernel managed allocations used by the GPU. 57 * 58 */ 59 60 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) 61 { 62 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 63 64 amdgpu_bo_kunmap(bo); 65 66 if (drm_gem_is_imported(&bo->tbo.base)) 67 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); 68 drm_gem_object_release(&bo->tbo.base); 69 amdgpu_bo_unref(&bo->parent); 70 kvfree(bo); 71 } 72 73 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo) 74 { 75 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 76 struct amdgpu_bo_user *ubo; 77 78 ubo = to_amdgpu_bo_user(bo); 79 kfree(ubo->metadata); 80 amdgpu_bo_destroy(tbo); 81 } 82 83 /** 84 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo 85 * @bo: buffer object to be checked 86 * 87 * Uses destroy function associated with the object to determine if this is 88 * an &amdgpu_bo. 89 * 90 * Returns: 91 * true if the object belongs to &amdgpu_bo, false if not. 92 */ 93 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) 94 { 95 if (bo->destroy == &amdgpu_bo_destroy || 96 bo->destroy == &amdgpu_bo_user_destroy) 97 return true; 98 99 return false; 100 } 101 102 /** 103 * amdgpu_bo_placement_from_domain - set buffer's placement 104 * @abo: &amdgpu_bo buffer object whose placement is to be set 105 * @domain: requested domain 106 * 107 * Sets buffer's placement according to requested domain and the buffer's 108 * flags. 109 */ 110 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) 111 { 112 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 113 struct ttm_placement *placement = &abo->placement; 114 struct ttm_place *places = abo->placements; 115 u64 flags = abo->flags; 116 u32 c = 0; 117 118 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 119 unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 120 int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); 121 122 if (adev->gmc.mem_partitions && mem_id >= 0) { 123 places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn; 124 /* 125 * memory partition range lpfn is inclusive start + size - 1 126 * TTM place lpfn is exclusive start + size 127 */ 128 places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1; 129 } else { 130 places[c].fpfn = 0; 131 places[c].lpfn = 0; 132 } 133 places[c].mem_type = TTM_PL_VRAM; 134 places[c].flags = 0; 135 136 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 137 places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn); 138 else 139 places[c].flags |= TTM_PL_FLAG_TOPDOWN; 140 141 if (abo->tbo.type == ttm_bo_type_kernel && 142 flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) 143 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; 144 145 c++; 146 } 147 148 if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) { 149 places[c].fpfn = 0; 150 places[c].lpfn = 0; 151 places[c].mem_type = AMDGPU_PL_DOORBELL; 152 places[c].flags = 0; 153 c++; 154 } 155 156 if (domain & AMDGPU_GEM_DOMAIN_MMIO_REMAP) { 157 places[c].fpfn = 0; 158 places[c].lpfn = 0; 159 places[c].mem_type = AMDGPU_PL_MMIO_REMAP; 160 places[c].flags = 0; 161 c++; 162 } 163 164 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 165 places[c].fpfn = 0; 166 places[c].lpfn = 0; 167 places[c].mem_type = 168 abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ? 169 AMDGPU_PL_PREEMPT : TTM_PL_TT; 170 places[c].flags = 0; 171 /* 172 * When GTT is just an alternative to VRAM make sure that we 173 * only use it as fallback and still try to fill up VRAM first. 174 */ 175 if (abo->tbo.resource && !(adev->flags & AMD_IS_APU) && 176 domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) 177 places[c].flags |= TTM_PL_FLAG_FALLBACK; 178 c++; 179 } 180 181 if (domain & AMDGPU_GEM_DOMAIN_CPU) { 182 places[c].fpfn = 0; 183 places[c].lpfn = 0; 184 places[c].mem_type = TTM_PL_SYSTEM; 185 places[c].flags = 0; 186 c++; 187 } 188 189 if (domain & AMDGPU_GEM_DOMAIN_GDS) { 190 places[c].fpfn = 0; 191 places[c].lpfn = 0; 192 places[c].mem_type = AMDGPU_PL_GDS; 193 places[c].flags = 0; 194 c++; 195 } 196 197 if (domain & AMDGPU_GEM_DOMAIN_GWS) { 198 places[c].fpfn = 0; 199 places[c].lpfn = 0; 200 places[c].mem_type = AMDGPU_PL_GWS; 201 places[c].flags = 0; 202 c++; 203 } 204 205 if (domain & AMDGPU_GEM_DOMAIN_OA) { 206 places[c].fpfn = 0; 207 places[c].lpfn = 0; 208 places[c].mem_type = AMDGPU_PL_OA; 209 places[c].flags = 0; 210 c++; 211 } 212 213 if (!c) { 214 places[c].fpfn = 0; 215 places[c].lpfn = 0; 216 places[c].mem_type = TTM_PL_SYSTEM; 217 places[c].flags = 0; 218 c++; 219 } 220 221 BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS); 222 223 placement->num_placement = c; 224 placement->placement = places; 225 } 226 227 /** 228 * amdgpu_bo_create_reserved - create reserved BO for kernel use 229 * 230 * @adev: amdgpu device object 231 * @size: size for the new BO 232 * @align: alignment for the new BO 233 * @domain: where to place it 234 * @bo_ptr: used to initialize BOs in structures 235 * @gpu_addr: GPU addr of the pinned BO 236 * @cpu_addr: optional CPU address mapping 237 * 238 * Allocates and pins a BO for kernel internal use, and returns it still 239 * reserved. 240 * 241 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 242 * 243 * Returns: 244 * 0 on success, negative error code otherwise. 245 */ 246 int amdgpu_bo_create_reserved(struct amdgpu_device *adev, 247 unsigned long size, int align, 248 u32 domain, struct amdgpu_bo **bo_ptr, 249 u64 *gpu_addr, void **cpu_addr) 250 { 251 struct amdgpu_bo_param bp; 252 bool free = false; 253 int r; 254 255 if (!size) { 256 amdgpu_bo_unref(bo_ptr); 257 return 0; 258 } 259 260 memset(&bp, 0, sizeof(bp)); 261 bp.size = size; 262 bp.byte_align = align; 263 bp.domain = domain; 264 bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED 265 : AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 266 bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 267 bp.type = ttm_bo_type_kernel; 268 bp.resv = NULL; 269 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 270 271 if (!*bo_ptr) { 272 r = amdgpu_bo_create(adev, &bp, bo_ptr); 273 if (r) { 274 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", 275 r); 276 return r; 277 } 278 free = true; 279 } 280 281 r = amdgpu_bo_reserve(*bo_ptr, false); 282 if (r) { 283 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); 284 goto error_free; 285 } 286 287 r = amdgpu_bo_pin(*bo_ptr, domain); 288 if (r) { 289 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); 290 goto error_unreserve; 291 } 292 293 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); 294 if (r) { 295 dev_err(adev->dev, "%p bind failed\n", *bo_ptr); 296 goto error_unpin; 297 } 298 299 if (gpu_addr) 300 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); 301 302 if (cpu_addr) { 303 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 304 if (r) { 305 dev_err(adev->dev, "(%d) kernel bo map failed\n", r); 306 goto error_unpin; 307 } 308 } 309 310 return 0; 311 312 error_unpin: 313 amdgpu_bo_unpin(*bo_ptr); 314 error_unreserve: 315 amdgpu_bo_unreserve(*bo_ptr); 316 317 error_free: 318 if (free) 319 amdgpu_bo_unref(bo_ptr); 320 321 return r; 322 } 323 324 /** 325 * amdgpu_bo_create_kernel - create BO for kernel use 326 * 327 * @adev: amdgpu device object 328 * @size: size for the new BO 329 * @align: alignment for the new BO 330 * @domain: where to place it 331 * @bo_ptr: used to initialize BOs in structures 332 * @gpu_addr: GPU addr of the pinned BO 333 * @cpu_addr: optional CPU address mapping 334 * 335 * Allocates and pins a BO for kernel internal use. 336 * 337 * This function is exported to allow the V4L2 isp device 338 * external to drm device to create and access the kernel BO. 339 * 340 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 341 * 342 * Returns: 343 * 0 on success, negative error code otherwise. 344 */ 345 int amdgpu_bo_create_kernel(struct amdgpu_device *adev, 346 unsigned long size, int align, 347 u32 domain, struct amdgpu_bo **bo_ptr, 348 u64 *gpu_addr, void **cpu_addr) 349 { 350 int r; 351 352 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, 353 gpu_addr, cpu_addr); 354 355 if (r) 356 return r; 357 358 if (*bo_ptr) 359 amdgpu_bo_unreserve(*bo_ptr); 360 361 return 0; 362 } 363 364 /** 365 * amdgpu_bo_create_isp_user - create user BO for isp 366 * 367 * @adev: amdgpu device object 368 * @dma_buf: DMABUF handle for isp buffer 369 * @domain: where to place it 370 * @bo: used to initialize BOs in structures 371 * @gpu_addr: GPU addr of the pinned BO 372 * 373 * Imports isp DMABUF to allocate and pin a user BO for isp internal use. It does 374 * GART alloc to generate gpu_addr for BO to make it accessible through the 375 * GART aperture for ISP HW. 376 * 377 * This function is exported to allow the V4L2 isp device external to drm device 378 * to create and access the isp user BO. 379 * 380 * Returns: 381 * 0 on success, negative error code otherwise. 382 */ 383 int amdgpu_bo_create_isp_user(struct amdgpu_device *adev, 384 struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo, 385 u64 *gpu_addr) 386 387 { 388 struct drm_gem_object *gem_obj; 389 int r; 390 391 gem_obj = amdgpu_gem_prime_import(&adev->ddev, dma_buf); 392 *bo = gem_to_amdgpu_bo(gem_obj); 393 if (!(*bo)) { 394 dev_err(adev->dev, "failed to get valid isp user bo\n"); 395 return -EINVAL; 396 } 397 398 r = amdgpu_bo_reserve(*bo, false); 399 if (r) { 400 dev_err(adev->dev, "(%d) failed to reserve isp user bo\n", r); 401 return r; 402 } 403 404 r = amdgpu_bo_pin(*bo, domain); 405 if (r) { 406 dev_err(adev->dev, "(%d) isp user bo pin failed\n", r); 407 goto error_unreserve; 408 } 409 410 r = amdgpu_ttm_alloc_gart(&(*bo)->tbo); 411 if (r) { 412 dev_err(adev->dev, "%p bind failed\n", *bo); 413 goto error_unpin; 414 } 415 416 if (!WARN_ON(!gpu_addr)) 417 *gpu_addr = amdgpu_bo_gpu_offset(*bo); 418 419 amdgpu_bo_unreserve(*bo); 420 421 return 0; 422 423 error_unpin: 424 amdgpu_bo_unpin(*bo); 425 error_unreserve: 426 amdgpu_bo_unreserve(*bo); 427 amdgpu_bo_unref(bo); 428 429 return r; 430 } 431 432 /** 433 * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location 434 * 435 * @adev: amdgpu device object 436 * @offset: offset of the BO 437 * @size: size of the BO 438 * @bo_ptr: used to initialize BOs in structures 439 * @cpu_addr: optional CPU address mapping 440 * 441 * Creates a kernel BO at a specific offset in VRAM. 442 * 443 * Returns: 444 * 0 on success, negative error code otherwise. 445 */ 446 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, 447 uint64_t offset, uint64_t size, 448 struct amdgpu_bo **bo_ptr, void **cpu_addr) 449 { 450 struct ttm_operation_ctx ctx = { false, false }; 451 unsigned int i; 452 int r; 453 454 offset &= PAGE_MASK; 455 size = ALIGN(size, PAGE_SIZE); 456 457 r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, 458 AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL, 459 cpu_addr); 460 if (r) 461 return r; 462 463 if ((*bo_ptr) == NULL) 464 return 0; 465 466 /* 467 * Remove the original mem node and create a new one at the request 468 * position. 469 */ 470 if (cpu_addr) 471 amdgpu_bo_kunmap(*bo_ptr); 472 473 ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource); 474 475 for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) { 476 (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT; 477 (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; 478 } 479 r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement, 480 &(*bo_ptr)->tbo.resource, &ctx); 481 if (r) 482 goto error; 483 484 if (cpu_addr) { 485 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 486 if (r) 487 goto error; 488 } 489 490 amdgpu_bo_unreserve(*bo_ptr); 491 return 0; 492 493 error: 494 amdgpu_bo_unreserve(*bo_ptr); 495 amdgpu_bo_unref(bo_ptr); 496 return r; 497 } 498 499 /** 500 * amdgpu_bo_free_kernel - free BO for kernel use 501 * 502 * @bo: amdgpu BO to free 503 * @gpu_addr: pointer to where the BO's GPU memory space address was stored 504 * @cpu_addr: pointer to where the BO's CPU memory space address was stored 505 * 506 * unmaps and unpin a BO for kernel internal use. 507 * 508 * This function is exported to allow the V4L2 isp device 509 * external to drm device to free the kernel BO. 510 */ 511 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, 512 void **cpu_addr) 513 { 514 if (*bo == NULL) 515 return; 516 517 WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); 518 519 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { 520 if (cpu_addr) 521 amdgpu_bo_kunmap(*bo); 522 523 amdgpu_bo_unpin(*bo); 524 amdgpu_bo_unreserve(*bo); 525 } 526 amdgpu_bo_unref(bo); 527 528 if (gpu_addr) 529 *gpu_addr = 0; 530 531 if (cpu_addr) 532 *cpu_addr = NULL; 533 } 534 535 /** 536 * amdgpu_bo_free_isp_user - free BO for isp use 537 * 538 * @bo: amdgpu isp user BO to free 539 * 540 * unpin and unref BO for isp internal use. 541 * 542 * This function is exported to allow the V4L2 isp device 543 * external to drm device to free the isp user BO. 544 */ 545 void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo) 546 { 547 if (bo == NULL) 548 return; 549 550 if (amdgpu_bo_reserve(bo, true) == 0) { 551 amdgpu_bo_unpin(bo); 552 amdgpu_bo_unreserve(bo); 553 } 554 amdgpu_bo_unref(&bo); 555 } 556 557 /* Validate bo size is bit bigger than the request domain */ 558 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, 559 unsigned long size, u32 domain) 560 { 561 struct ttm_resource_manager *man = NULL; 562 563 /* 564 * If GTT is part of requested domains the check must succeed to 565 * allow fall back to GTT. 566 */ 567 if (domain & AMDGPU_GEM_DOMAIN_GTT) 568 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); 569 else if (domain & AMDGPU_GEM_DOMAIN_VRAM) 570 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 571 else 572 return true; 573 574 if (!man) { 575 if (domain & AMDGPU_GEM_DOMAIN_GTT) 576 WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized"); 577 return false; 578 } 579 580 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */ 581 if (size < man->size) 582 return true; 583 584 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size); 585 return false; 586 } 587 588 bool amdgpu_bo_support_uswc(u64 bo_flags) 589 { 590 591 #ifdef CONFIG_X86_32 592 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit 593 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 594 */ 595 return false; 596 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) 597 /* Don't try to enable write-combining when it can't work, or things 598 * may be slow 599 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 600 */ 601 602 #ifndef CONFIG_COMPILE_TEST 603 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ 604 thanks to write-combining 605 #endif 606 607 if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 608 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " 609 "better performance thanks to write-combining\n"); 610 return false; 611 #else 612 /* For architectures that don't support WC memory, 613 * mask out the WC flag from the BO 614 */ 615 if (!drm_arch_can_wc_memory()) 616 return false; 617 618 return true; 619 #endif 620 } 621 622 /** 623 * amdgpu_bo_create - create an &amdgpu_bo buffer object 624 * @adev: amdgpu device object 625 * @bp: parameters to be used for the buffer object 626 * @bo_ptr: pointer to the buffer object pointer 627 * 628 * Creates an &amdgpu_bo buffer object. 629 * 630 * Returns: 631 * 0 for success or a negative error code on failure. 632 */ 633 int amdgpu_bo_create(struct amdgpu_device *adev, 634 struct amdgpu_bo_param *bp, 635 struct amdgpu_bo **bo_ptr) 636 { 637 struct ttm_operation_ctx ctx = { 638 .interruptible = (bp->type != ttm_bo_type_kernel), 639 .no_wait_gpu = bp->no_wait_gpu, 640 /* We opt to avoid OOM on system pages allocations */ 641 .gfp_retry_mayfail = true, 642 .allow_res_evict = bp->type != ttm_bo_type_kernel, 643 .resv = bp->resv 644 }; 645 struct amdgpu_bo *bo; 646 unsigned long page_align, size = bp->size; 647 int r; 648 649 /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ 650 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { 651 /* GWS and OA don't need any alignment. */ 652 page_align = bp->byte_align; 653 size <<= PAGE_SHIFT; 654 655 } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { 656 /* Both size and alignment must be a multiple of 4. */ 657 page_align = ALIGN(bp->byte_align, 4); 658 size = ALIGN(size, 4) << PAGE_SHIFT; 659 } else { 660 /* Memory should be aligned at least to a page size. */ 661 page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; 662 size = ALIGN(size, PAGE_SIZE); 663 } 664 665 if (!amdgpu_bo_validate_size(adev, size, bp->domain)) 666 return -ENOMEM; 667 668 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo)); 669 670 *bo_ptr = NULL; 671 bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL); 672 if (bo == NULL) 673 return -ENOMEM; 674 drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); 675 bo->tbo.base.funcs = &amdgpu_gem_object_funcs; 676 bo->vm_bo = NULL; 677 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : 678 bp->domain; 679 bo->allowed_domains = bo->preferred_domains; 680 if (bp->type != ttm_bo_type_kernel && 681 !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) && 682 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) 683 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 684 685 bo->flags = bp->flags; 686 687 if (adev->gmc.mem_partitions) 688 /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */ 689 bo->xcp_id = bp->xcp_id_plus1 - 1; 690 else 691 /* For GPUs without spatial partitioning */ 692 bo->xcp_id = 0; 693 694 if (!amdgpu_bo_support_uswc(bo->flags)) 695 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 696 697 bo->tbo.bdev = &adev->mman.bdev; 698 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | 699 AMDGPU_GEM_DOMAIN_GDS)) 700 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 701 else 702 amdgpu_bo_placement_from_domain(bo, bp->domain); 703 if (bp->type == ttm_bo_type_kernel) 704 bo->tbo.priority = 2; 705 else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE)) 706 bo->tbo.priority = 1; 707 708 if (!bp->destroy) 709 bp->destroy = &amdgpu_bo_destroy; 710 711 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type, 712 &bo->placement, page_align, &ctx, NULL, 713 bp->resv, bp->destroy); 714 if (unlikely(r != 0)) 715 return r; 716 717 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 718 amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 719 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 720 ctx.bytes_moved); 721 else 722 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); 723 724 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && 725 bo->tbo.resource->mem_type == TTM_PL_VRAM) { 726 struct dma_fence *fence; 727 728 r = amdgpu_ttm_clear_buffer(bo, bo->tbo.base.resv, &fence); 729 if (unlikely(r)) 730 goto fail_unreserve; 731 732 dma_resv_add_fence(bo->tbo.base.resv, fence, 733 DMA_RESV_USAGE_KERNEL); 734 dma_fence_put(fence); 735 } 736 if (!bp->resv) 737 amdgpu_bo_unreserve(bo); 738 *bo_ptr = bo; 739 740 trace_amdgpu_bo_create(bo); 741 742 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ 743 if (bp->type == ttm_bo_type_device) 744 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 745 746 return 0; 747 748 fail_unreserve: 749 if (!bp->resv) 750 dma_resv_unlock(bo->tbo.base.resv); 751 amdgpu_bo_unref(&bo); 752 return r; 753 } 754 755 /** 756 * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object 757 * @adev: amdgpu device object 758 * @bp: parameters to be used for the buffer object 759 * @ubo_ptr: pointer to the buffer object pointer 760 * 761 * Create a BO to be used by user application; 762 * 763 * Returns: 764 * 0 for success or a negative error code on failure. 765 */ 766 767 int amdgpu_bo_create_user(struct amdgpu_device *adev, 768 struct amdgpu_bo_param *bp, 769 struct amdgpu_bo_user **ubo_ptr) 770 { 771 struct amdgpu_bo *bo_ptr; 772 int r; 773 774 bp->bo_ptr_size = sizeof(struct amdgpu_bo_user); 775 bp->destroy = &amdgpu_bo_user_destroy; 776 r = amdgpu_bo_create(adev, bp, &bo_ptr); 777 if (r) 778 return r; 779 780 *ubo_ptr = to_amdgpu_bo_user(bo_ptr); 781 return r; 782 } 783 784 /** 785 * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object 786 * @adev: amdgpu device object 787 * @bp: parameters to be used for the buffer object 788 * @vmbo_ptr: pointer to the buffer object pointer 789 * 790 * Create a BO to be for GPUVM. 791 * 792 * Returns: 793 * 0 for success or a negative error code on failure. 794 */ 795 796 int amdgpu_bo_create_vm(struct amdgpu_device *adev, 797 struct amdgpu_bo_param *bp, 798 struct amdgpu_bo_vm **vmbo_ptr) 799 { 800 struct amdgpu_bo *bo_ptr; 801 int r; 802 803 /* bo_ptr_size will be determined by the caller and it depends on 804 * num of amdgpu_vm_pt entries. 805 */ 806 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm)); 807 r = amdgpu_bo_create(adev, bp, &bo_ptr); 808 if (r) 809 return r; 810 811 *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr); 812 return r; 813 } 814 815 /** 816 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object 817 * @bo: &amdgpu_bo buffer object to be mapped 818 * @ptr: kernel virtual address to be returned 819 * 820 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls 821 * amdgpu_bo_kptr() to get the kernel virtual address. 822 * 823 * Returns: 824 * 0 for success or a negative error code on failure. 825 */ 826 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) 827 { 828 void *kptr; 829 long r; 830 831 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 832 return -EPERM; 833 834 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL, 835 false, MAX_SCHEDULE_TIMEOUT); 836 if (r < 0) 837 return r; 838 839 kptr = amdgpu_bo_kptr(bo); 840 if (kptr) { 841 if (ptr) 842 *ptr = kptr; 843 return 0; 844 } 845 846 r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap); 847 if (r) 848 return r; 849 850 if (ptr) 851 *ptr = amdgpu_bo_kptr(bo); 852 853 return 0; 854 } 855 856 /** 857 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object 858 * @bo: &amdgpu_bo buffer object 859 * 860 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address 861 * 862 * Returns: 863 * the virtual address of a buffer object area. 864 */ 865 void *amdgpu_bo_kptr(struct amdgpu_bo *bo) 866 { 867 bool is_iomem; 868 869 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 870 } 871 872 /** 873 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object 874 * @bo: &amdgpu_bo buffer object to be unmapped 875 * 876 * Unmaps a kernel map set up by amdgpu_bo_kmap(). 877 */ 878 void amdgpu_bo_kunmap(struct amdgpu_bo *bo) 879 { 880 if (bo->kmap.bo) 881 ttm_bo_kunmap(&bo->kmap); 882 } 883 884 /** 885 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object 886 * @bo: &amdgpu_bo buffer object 887 * 888 * References the contained &ttm_buffer_object. 889 * 890 * Returns: 891 * a refcounted pointer to the &amdgpu_bo buffer object. 892 */ 893 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) 894 { 895 if (bo == NULL) 896 return NULL; 897 898 drm_gem_object_get(&bo->tbo.base); 899 return bo; 900 } 901 902 /** 903 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object 904 * @bo: &amdgpu_bo buffer object 905 * 906 * Unreferences the contained &ttm_buffer_object and clear the pointer 907 */ 908 void amdgpu_bo_unref(struct amdgpu_bo **bo) 909 { 910 if ((*bo) == NULL) 911 return; 912 913 drm_gem_object_put(&(*bo)->tbo.base); 914 *bo = NULL; 915 } 916 917 /** 918 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object 919 * @bo: &amdgpu_bo buffer object to be pinned 920 * @domain: domain to be pinned to 921 * 922 * Pins the buffer object according to requested domain. If the memory is 923 * unbound gart memory, binds the pages into gart table. Adjusts pin_count and 924 * pin_size accordingly. 925 * 926 * Pinning means to lock pages in memory along with keeping them at a fixed 927 * offset. It is required when a buffer can not be moved, for example, when 928 * a display buffer is being scanned out. 929 * 930 * Returns: 931 * 0 for success or a negative error code on failure. 932 */ 933 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) 934 { 935 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 936 struct ttm_operation_ctx ctx = { false, false }; 937 int r, i; 938 939 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) 940 return -EPERM; 941 942 /* Check domain to be pinned to against preferred domains */ 943 if (bo->preferred_domains & domain) 944 domain = bo->preferred_domains & domain; 945 946 /* A shared bo cannot be migrated to VRAM */ 947 if (drm_gem_is_imported(&bo->tbo.base)) { 948 if (domain & AMDGPU_GEM_DOMAIN_GTT) 949 domain = AMDGPU_GEM_DOMAIN_GTT; 950 else 951 return -EINVAL; 952 } 953 954 if (bo->tbo.pin_count) { 955 uint32_t mem_type = bo->tbo.resource->mem_type; 956 uint32_t mem_flags = bo->tbo.resource->placement; 957 958 if (!(domain & amdgpu_mem_type_to_domain(mem_type))) 959 return -EINVAL; 960 961 if ((mem_type == TTM_PL_VRAM) && 962 (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && 963 !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) 964 return -EINVAL; 965 966 ttm_bo_pin(&bo->tbo); 967 return 0; 968 } 969 970 /* This assumes only APU display buffers are pinned with (VRAM|GTT). 971 * See function amdgpu_display_supported_domains() 972 */ 973 domain = amdgpu_bo_get_preferred_domain(adev, domain); 974 975 if (drm_gem_is_imported(&bo->tbo.base)) 976 dma_buf_pin(bo->tbo.base.import_attach); 977 978 /* force to pin into visible video ram */ 979 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) 980 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 981 amdgpu_bo_placement_from_domain(bo, domain); 982 for (i = 0; i < bo->placement.num_placement; i++) { 983 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && 984 bo->placements[i].mem_type == TTM_PL_VRAM) 985 bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS; 986 } 987 988 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 989 if (unlikely(r)) { 990 dev_err(adev->dev, "%p pin failed\n", bo); 991 goto error; 992 } 993 994 ttm_bo_pin(&bo->tbo); 995 996 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 997 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); 998 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), 999 &adev->visible_pin_size); 1000 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 1001 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); 1002 } 1003 1004 error: 1005 return r; 1006 } 1007 1008 /** 1009 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object 1010 * @bo: &amdgpu_bo buffer object to be unpinned 1011 * 1012 * Decreases the pin_count, and clears the flags if pin_count reaches 0. 1013 * Changes placement and pin size accordingly. 1014 * 1015 * Returns: 1016 * 0 for success or a negative error code on failure. 1017 */ 1018 void amdgpu_bo_unpin(struct amdgpu_bo *bo) 1019 { 1020 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1021 1022 ttm_bo_unpin(&bo->tbo); 1023 if (bo->tbo.pin_count) 1024 return; 1025 1026 if (drm_gem_is_imported(&bo->tbo.base)) 1027 dma_buf_unpin(bo->tbo.base.import_attach); 1028 1029 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 1030 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); 1031 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), 1032 &adev->visible_pin_size); 1033 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 1034 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); 1035 } 1036 1037 } 1038 1039 static const char * const amdgpu_vram_names[] = { 1040 "UNKNOWN", 1041 "GDDR1", 1042 "DDR2", 1043 "GDDR3", 1044 "GDDR4", 1045 "GDDR5", 1046 "HBM", 1047 "DDR3", 1048 "DDR4", 1049 "GDDR6", 1050 "DDR5", 1051 "LPDDR4", 1052 "LPDDR5", 1053 "HBM3E", 1054 "HBM4" 1055 }; 1056 1057 /** 1058 * amdgpu_bo_init - initialize memory manager 1059 * @adev: amdgpu device object 1060 * 1061 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. 1062 * 1063 * Returns: 1064 * 0 for success or a negative error code on failure. 1065 */ 1066 int amdgpu_bo_init(struct amdgpu_device *adev) 1067 { 1068 /* On A+A platform, VRAM can be mapped as WB */ 1069 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 1070 /* reserve PAT memory space to WC for VRAM */ 1071 int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base, 1072 adev->gmc.aper_size); 1073 1074 if (r) { 1075 DRM_ERROR("Unable to set WC memtype for the aperture base\n"); 1076 return r; 1077 } 1078 1079 /* Add an MTRR for the VRAM */ 1080 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, 1081 adev->gmc.aper_size); 1082 } 1083 1084 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", 1085 adev->gmc.mc_vram_size >> 20, 1086 (unsigned long long)adev->gmc.aper_size >> 20); 1087 DRM_INFO("RAM width %dbits %s\n", 1088 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); 1089 return amdgpu_ttm_init(adev); 1090 } 1091 1092 /** 1093 * amdgpu_bo_fini - tear down memory manager 1094 * @adev: amdgpu device object 1095 * 1096 * Reverses amdgpu_bo_init() to tear down memory manager. 1097 */ 1098 void amdgpu_bo_fini(struct amdgpu_device *adev) 1099 { 1100 int idx; 1101 1102 amdgpu_ttm_fini(adev); 1103 1104 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 1105 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 1106 arch_phys_wc_del(adev->gmc.vram_mtrr); 1107 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 1108 } 1109 drm_dev_exit(idx); 1110 } 1111 } 1112 1113 /** 1114 * amdgpu_bo_set_tiling_flags - set tiling flags 1115 * @bo: &amdgpu_bo buffer object 1116 * @tiling_flags: new flags 1117 * 1118 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or 1119 * kernel driver to set the tiling flags on a buffer. 1120 * 1121 * Returns: 1122 * 0 for success or a negative error code on failure. 1123 */ 1124 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) 1125 { 1126 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1127 struct amdgpu_bo_user *ubo; 1128 1129 /* MMIO_REMAP is BAR I/O space; tiling should never be used here. */ 1130 WARN_ON_ONCE(bo->tbo.resource && 1131 bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP); 1132 1133 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1134 if (adev->family <= AMDGPU_FAMILY_CZ && 1135 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) 1136 return -EINVAL; 1137 1138 ubo = to_amdgpu_bo_user(bo); 1139 ubo->tiling_flags = tiling_flags; 1140 return 0; 1141 } 1142 1143 /** 1144 * amdgpu_bo_get_tiling_flags - get tiling flags 1145 * @bo: &amdgpu_bo buffer object 1146 * @tiling_flags: returned flags 1147 * 1148 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to 1149 * set the tiling flags on a buffer. 1150 */ 1151 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) 1152 { 1153 struct amdgpu_bo_user *ubo; 1154 1155 /* 1156 * MMIO_REMAP BOs are not real VRAM/GTT memory but a fixed BAR I/O window. 1157 * They should never go through GEM tiling helpers. 1158 */ 1159 WARN_ON_ONCE(bo->tbo.resource && 1160 bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP); 1161 1162 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1163 dma_resv_assert_held(bo->tbo.base.resv); 1164 ubo = to_amdgpu_bo_user(bo); 1165 1166 if (tiling_flags) 1167 *tiling_flags = ubo->tiling_flags; 1168 } 1169 1170 /** 1171 * amdgpu_bo_set_metadata - set metadata 1172 * @bo: &amdgpu_bo buffer object 1173 * @metadata: new metadata 1174 * @metadata_size: size of the new metadata 1175 * @flags: flags of the new metadata 1176 * 1177 * Sets buffer object's metadata, its size and flags. 1178 * Used via GEM ioctl. 1179 * 1180 * Returns: 1181 * 0 for success or a negative error code on failure. 1182 */ 1183 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata, 1184 u32 metadata_size, uint64_t flags) 1185 { 1186 struct amdgpu_bo_user *ubo; 1187 void *buffer; 1188 1189 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1190 ubo = to_amdgpu_bo_user(bo); 1191 if (!metadata_size) { 1192 if (ubo->metadata_size) { 1193 kfree(ubo->metadata); 1194 ubo->metadata = NULL; 1195 ubo->metadata_size = 0; 1196 } 1197 return 0; 1198 } 1199 1200 if (metadata == NULL) 1201 return -EINVAL; 1202 1203 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); 1204 if (buffer == NULL) 1205 return -ENOMEM; 1206 1207 kfree(ubo->metadata); 1208 ubo->metadata_flags = flags; 1209 ubo->metadata = buffer; 1210 ubo->metadata_size = metadata_size; 1211 1212 return 0; 1213 } 1214 1215 /** 1216 * amdgpu_bo_get_metadata - get metadata 1217 * @bo: &amdgpu_bo buffer object 1218 * @buffer: returned metadata 1219 * @buffer_size: size of the buffer 1220 * @metadata_size: size of the returned metadata 1221 * @flags: flags of the returned metadata 1222 * 1223 * Gets buffer object's metadata, its size and flags. buffer_size shall not be 1224 * less than metadata_size. 1225 * Used via GEM ioctl. 1226 * 1227 * Returns: 1228 * 0 for success or a negative error code on failure. 1229 */ 1230 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, 1231 size_t buffer_size, uint32_t *metadata_size, 1232 uint64_t *flags) 1233 { 1234 struct amdgpu_bo_user *ubo; 1235 1236 if (!buffer && !metadata_size) 1237 return -EINVAL; 1238 1239 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1240 ubo = to_amdgpu_bo_user(bo); 1241 if (metadata_size) 1242 *metadata_size = ubo->metadata_size; 1243 1244 if (buffer) { 1245 if (buffer_size < ubo->metadata_size) 1246 return -EINVAL; 1247 1248 if (ubo->metadata_size) 1249 memcpy(buffer, ubo->metadata, ubo->metadata_size); 1250 } 1251 1252 if (flags) 1253 *flags = ubo->metadata_flags; 1254 1255 return 0; 1256 } 1257 1258 /** 1259 * amdgpu_bo_move_notify - notification about a memory move 1260 * @bo: pointer to a buffer object 1261 * @evict: if this move is evicting the buffer from the graphics address space 1262 * @new_mem: new resource for backing the BO 1263 * 1264 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs 1265 * bookkeeping. 1266 * TTM driver callback which is called when ttm moves a buffer. 1267 */ 1268 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, 1269 bool evict, 1270 struct ttm_resource *new_mem) 1271 { 1272 struct ttm_resource *old_mem = bo->resource; 1273 struct amdgpu_bo *abo; 1274 1275 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1276 return; 1277 1278 abo = ttm_to_amdgpu_bo(bo); 1279 amdgpu_vm_bo_move(abo, new_mem, evict); 1280 1281 amdgpu_bo_kunmap(abo); 1282 1283 if (abo->tbo.base.dma_buf && !drm_gem_is_imported(&abo->tbo.base) && 1284 old_mem && old_mem->mem_type != TTM_PL_SYSTEM) 1285 dma_buf_move_notify(abo->tbo.base.dma_buf); 1286 1287 /* move_notify is called before move happens */ 1288 trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1, 1289 old_mem ? old_mem->mem_type : -1); 1290 } 1291 1292 /** 1293 * amdgpu_bo_release_notify - notification about a BO being released 1294 * @bo: pointer to a buffer object 1295 * 1296 * Wipes VRAM buffers whose contents should not be leaked before the 1297 * memory is released. 1298 */ 1299 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) 1300 { 1301 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1302 struct dma_fence *fence = NULL; 1303 struct amdgpu_bo *abo; 1304 int r; 1305 1306 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1307 return; 1308 1309 abo = ttm_to_amdgpu_bo(bo); 1310 1311 WARN_ON(abo->vm_bo); 1312 1313 if (abo->kfd_bo) 1314 amdgpu_amdkfd_release_notify(abo); 1315 1316 /* 1317 * We lock the private dma_resv object here and since the BO is about to 1318 * be released nobody else should have a pointer to it. 1319 * So when this locking here fails something is wrong with the reference 1320 * counting. 1321 */ 1322 if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv))) 1323 return; 1324 1325 amdgpu_amdkfd_remove_all_eviction_fences(abo); 1326 1327 if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM || 1328 !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) || 1329 adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev))) 1330 goto out; 1331 1332 r = dma_resv_reserve_fences(&bo->base._resv, 1); 1333 if (r) 1334 goto out; 1335 1336 r = amdgpu_fill_buffer(abo, 0, &bo->base._resv, &fence, true, 1337 AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE); 1338 if (WARN_ON(r)) 1339 goto out; 1340 1341 amdgpu_vram_mgr_set_cleared(bo->resource); 1342 dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL); 1343 dma_fence_put(fence); 1344 1345 out: 1346 dma_resv_unlock(&bo->base._resv); 1347 } 1348 1349 /** 1350 * amdgpu_bo_fault_reserve_notify - notification about a memory fault 1351 * @bo: pointer to a buffer object 1352 * 1353 * Notifies the driver we are taking a fault on this BO and have reserved it, 1354 * also performs bookkeeping. 1355 * TTM driver callback for dealing with vm faults. 1356 * 1357 * Returns: 1358 * 0 for success or a negative error code on failure. 1359 */ 1360 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 1361 { 1362 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1363 struct ttm_operation_ctx ctx = { false, false }; 1364 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1365 int r; 1366 1367 /* Remember that this BO was accessed by the CPU */ 1368 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 1369 1370 if (amdgpu_res_cpu_visible(adev, bo->resource)) 1371 return 0; 1372 1373 /* Can't move a pinned BO to visible VRAM */ 1374 if (abo->tbo.pin_count > 0) 1375 return VM_FAULT_SIGBUS; 1376 1377 /* hurrah the memory is not visible ! */ 1378 atomic64_inc(&adev->num_vram_cpu_page_faults); 1379 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 1380 AMDGPU_GEM_DOMAIN_GTT); 1381 1382 /* Avoid costly evictions; only set GTT as a busy placement */ 1383 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; 1384 1385 r = ttm_bo_validate(bo, &abo->placement, &ctx); 1386 if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) 1387 return VM_FAULT_NOPAGE; 1388 else if (unlikely(r)) 1389 return VM_FAULT_SIGBUS; 1390 1391 /* this should never happen */ 1392 if (bo->resource->mem_type == TTM_PL_VRAM && 1393 !amdgpu_res_cpu_visible(adev, bo->resource)) 1394 return VM_FAULT_SIGBUS; 1395 1396 ttm_bo_move_to_lru_tail_unlocked(bo); 1397 return 0; 1398 } 1399 1400 /** 1401 * amdgpu_bo_fence - add fence to buffer object 1402 * 1403 * @bo: buffer object in question 1404 * @fence: fence to add 1405 * @shared: true if fence should be added shared 1406 * 1407 */ 1408 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, 1409 bool shared) 1410 { 1411 struct dma_resv *resv = bo->tbo.base.resv; 1412 int r; 1413 1414 r = dma_resv_reserve_fences(resv, 1); 1415 if (r) { 1416 /* As last resort on OOM we block for the fence */ 1417 dma_fence_wait(fence, false); 1418 return; 1419 } 1420 1421 dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ : 1422 DMA_RESV_USAGE_WRITE); 1423 } 1424 1425 /** 1426 * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences 1427 * 1428 * @adev: amdgpu device pointer 1429 * @resv: reservation object to sync to 1430 * @sync_mode: synchronization mode 1431 * @owner: fence owner 1432 * @intr: Whether the wait is interruptible 1433 * 1434 * Extract the fences from the reservation object and waits for them to finish. 1435 * 1436 * Returns: 1437 * 0 on success, errno otherwise. 1438 */ 1439 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, 1440 enum amdgpu_sync_mode sync_mode, void *owner, 1441 bool intr) 1442 { 1443 struct amdgpu_sync sync; 1444 int r; 1445 1446 amdgpu_sync_create(&sync); 1447 amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner); 1448 r = amdgpu_sync_wait(&sync, intr); 1449 amdgpu_sync_free(&sync); 1450 return r; 1451 } 1452 1453 /** 1454 * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv 1455 * @bo: buffer object to wait for 1456 * @owner: fence owner 1457 * @intr: Whether the wait is interruptible 1458 * 1459 * Wrapper to wait for fences in a BO. 1460 * Returns: 1461 * 0 on success, errno otherwise. 1462 */ 1463 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) 1464 { 1465 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1466 1467 return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, 1468 AMDGPU_SYNC_NE_OWNER, owner, intr); 1469 } 1470 1471 /** 1472 * amdgpu_bo_gpu_offset - return GPU offset of bo 1473 * @bo: amdgpu object for which we query the offset 1474 * 1475 * Note: object should either be pinned or reserved when calling this 1476 * function, it might be useful to add check for this for debugging. 1477 * 1478 * Returns: 1479 * current GPU offset of the object. 1480 */ 1481 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) 1482 { 1483 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM); 1484 WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) && 1485 !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel); 1486 WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET); 1487 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM && 1488 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); 1489 1490 return amdgpu_bo_gpu_offset_no_check(bo); 1491 } 1492 1493 /** 1494 * amdgpu_bo_fb_aper_addr - return FB aperture GPU offset of the VRAM bo 1495 * @bo: amdgpu VRAM buffer object for which we query the offset 1496 * 1497 * Returns: 1498 * current FB aperture GPU offset of the object. 1499 */ 1500 u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo) 1501 { 1502 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1503 uint64_t offset, fb_base; 1504 1505 WARN_ON_ONCE(bo->tbo.resource->mem_type != TTM_PL_VRAM); 1506 1507 fb_base = adev->gmc.fb_start; 1508 fb_base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1509 offset = (bo->tbo.resource->start << PAGE_SHIFT) + fb_base; 1510 return amdgpu_gmc_sign_extend(offset); 1511 } 1512 1513 /** 1514 * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo 1515 * @bo: amdgpu object for which we query the offset 1516 * 1517 * Returns: 1518 * current GPU offset of the object without raising warnings. 1519 */ 1520 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) 1521 { 1522 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1523 uint64_t offset = AMDGPU_BO_INVALID_OFFSET; 1524 1525 if (bo->tbo.resource->mem_type == TTM_PL_TT) 1526 offset = amdgpu_gmc_agp_addr(&bo->tbo); 1527 1528 if (offset == AMDGPU_BO_INVALID_OFFSET) 1529 offset = (bo->tbo.resource->start << PAGE_SHIFT) + 1530 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type); 1531 1532 return amdgpu_gmc_sign_extend(offset); 1533 } 1534 1535 /** 1536 * amdgpu_bo_mem_stats_placement - bo placement for memory accounting 1537 * @bo: the buffer object we should look at 1538 * 1539 * BO can have multiple preferred placements, to avoid double counting we want 1540 * to file it under a single placement for memory stats. 1541 * Luckily, if we take the highest set bit in preferred_domains the result is 1542 * quite sensible. 1543 * 1544 * Returns: 1545 * Which of the placements should the BO be accounted under. 1546 */ 1547 uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo) 1548 { 1549 uint32_t domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK; 1550 1551 if (!domain) 1552 return TTM_PL_SYSTEM; 1553 1554 switch (rounddown_pow_of_two(domain)) { 1555 case AMDGPU_GEM_DOMAIN_CPU: 1556 return TTM_PL_SYSTEM; 1557 case AMDGPU_GEM_DOMAIN_GTT: 1558 return TTM_PL_TT; 1559 case AMDGPU_GEM_DOMAIN_VRAM: 1560 return TTM_PL_VRAM; 1561 case AMDGPU_GEM_DOMAIN_GDS: 1562 return AMDGPU_PL_GDS; 1563 case AMDGPU_GEM_DOMAIN_GWS: 1564 return AMDGPU_PL_GWS; 1565 case AMDGPU_GEM_DOMAIN_OA: 1566 return AMDGPU_PL_OA; 1567 case AMDGPU_GEM_DOMAIN_DOORBELL: 1568 return AMDGPU_PL_DOORBELL; 1569 case AMDGPU_GEM_DOMAIN_MMIO_REMAP: 1570 return AMDGPU_PL_MMIO_REMAP; 1571 default: 1572 return TTM_PL_SYSTEM; 1573 } 1574 } 1575 1576 /** 1577 * amdgpu_bo_get_preferred_domain - get preferred domain 1578 * @adev: amdgpu device object 1579 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` 1580 * 1581 * Returns: 1582 * Which of the allowed domains is preferred for allocating the BO. 1583 */ 1584 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, 1585 uint32_t domain) 1586 { 1587 if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && 1588 ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) { 1589 domain = AMDGPU_GEM_DOMAIN_VRAM; 1590 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) 1591 domain = AMDGPU_GEM_DOMAIN_GTT; 1592 } 1593 return domain; 1594 } 1595 1596 #if defined(CONFIG_DEBUG_FS) 1597 #define amdgpu_bo_print_flag(m, bo, flag) \ 1598 do { \ 1599 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ 1600 seq_printf((m), " " #flag); \ 1601 } \ 1602 } while (0) 1603 1604 /** 1605 * amdgpu_bo_print_info - print BO info in debugfs file 1606 * 1607 * @id: Index or Id of the BO 1608 * @bo: Requested BO for printing info 1609 * @m: debugfs file 1610 * 1611 * Print BO information in debugfs file 1612 * 1613 * Returns: 1614 * Size of the BO in bytes. 1615 */ 1616 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m) 1617 { 1618 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1619 struct dma_buf_attachment *attachment; 1620 struct dma_buf *dma_buf; 1621 const char *placement; 1622 unsigned int pin_count; 1623 u64 size; 1624 1625 if (dma_resv_trylock(bo->tbo.base.resv)) { 1626 if (!bo->tbo.resource) { 1627 placement = "NONE"; 1628 } else { 1629 switch (bo->tbo.resource->mem_type) { 1630 case TTM_PL_VRAM: 1631 if (amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 1632 placement = "VRAM VISIBLE"; 1633 else 1634 placement = "VRAM"; 1635 break; 1636 case TTM_PL_TT: 1637 placement = "GTT"; 1638 break; 1639 case AMDGPU_PL_GDS: 1640 placement = "GDS"; 1641 break; 1642 case AMDGPU_PL_GWS: 1643 placement = "GWS"; 1644 break; 1645 case AMDGPU_PL_OA: 1646 placement = "OA"; 1647 break; 1648 case AMDGPU_PL_PREEMPT: 1649 placement = "PREEMPTIBLE"; 1650 break; 1651 case AMDGPU_PL_DOORBELL: 1652 placement = "DOORBELL"; 1653 break; 1654 case AMDGPU_PL_MMIO_REMAP: 1655 placement = "MMIO REMAP"; 1656 break; 1657 case TTM_PL_SYSTEM: 1658 default: 1659 placement = "CPU"; 1660 break; 1661 } 1662 } 1663 dma_resv_unlock(bo->tbo.base.resv); 1664 } else { 1665 placement = "UNKNOWN"; 1666 } 1667 1668 size = amdgpu_bo_size(bo); 1669 seq_printf(m, "\t\t0x%08x: %12lld byte %s", 1670 id, size, placement); 1671 1672 pin_count = READ_ONCE(bo->tbo.pin_count); 1673 if (pin_count) 1674 seq_printf(m, " pin count %d", pin_count); 1675 1676 dma_buf = READ_ONCE(bo->tbo.base.dma_buf); 1677 attachment = READ_ONCE(bo->tbo.base.import_attach); 1678 1679 if (attachment) 1680 seq_printf(m, " imported from ino:%lu", file_inode(dma_buf->file)->i_ino); 1681 else if (dma_buf) 1682 seq_printf(m, " exported as ino:%lu", file_inode(dma_buf->file)->i_ino); 1683 1684 amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); 1685 amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS); 1686 amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC); 1687 amdgpu_bo_print_flag(m, bo, VRAM_CLEARED); 1688 amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS); 1689 amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID); 1690 amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC); 1691 /* Add the gem obj resv fence dump*/ 1692 if (dma_resv_trylock(bo->tbo.base.resv)) { 1693 dma_resv_describe(bo->tbo.base.resv, m); 1694 dma_resv_unlock(bo->tbo.base.resv); 1695 } 1696 seq_puts(m, "\n"); 1697 1698 return size; 1699 } 1700 #endif 1701