1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <linux/slab.h> 34 #include <linux/dma-buf.h> 35 #include <linux/export.h> 36 37 #include <drm/drm_drv.h> 38 #include <drm/amdgpu_drm.h> 39 #include <drm/drm_cache.h> 40 #include "amdgpu.h" 41 #include "amdgpu_trace.h" 42 #include "amdgpu_amdkfd.h" 43 #include "amdgpu_vram_mgr.h" 44 #include "amdgpu_vm.h" 45 #include "amdgpu_dma_buf.h" 46 #include "kfd_svm.h" 47 48 /** 49 * DOC: amdgpu_object 50 * 51 * This defines the interfaces to operate on an &amdgpu_bo buffer object which 52 * represents memory used by driver (VRAM, system memory, etc.). The driver 53 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces 54 * to create/destroy/set buffer object which are then managed by the kernel TTM 55 * memory manager. 56 * The interfaces are also used internally by kernel clients, including gfx, 57 * uvd, etc. for kernel managed allocations used by the GPU. 58 * 59 */ 60 61 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) 62 { 63 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 64 65 amdgpu_bo_kunmap(bo); 66 67 if (drm_gem_is_imported(&bo->tbo.base)) 68 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); 69 drm_gem_object_release(&bo->tbo.base); 70 amdgpu_bo_unref(&bo->parent); 71 kvfree(bo); 72 } 73 74 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo) 75 { 76 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 77 struct amdgpu_bo_user *ubo; 78 79 ubo = to_amdgpu_bo_user(bo); 80 kfree(ubo->metadata); 81 amdgpu_bo_destroy(tbo); 82 } 83 84 /** 85 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo 86 * @bo: buffer object to be checked 87 * 88 * Uses destroy function associated with the object to determine if this is 89 * an &amdgpu_bo. 90 * 91 * Returns: 92 * true if the object belongs to &amdgpu_bo, false if not. 93 */ 94 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) 95 { 96 if (bo->destroy == &amdgpu_bo_destroy || 97 bo->destroy == &amdgpu_bo_user_destroy || 98 bo->destroy == &svm_range_bo_destroy) 99 return true; 100 101 return false; 102 } 103 104 /** 105 * amdgpu_bo_placement_from_domain - set buffer's placement 106 * @abo: &amdgpu_bo buffer object whose placement is to be set 107 * @domain: requested domain 108 * 109 * Sets buffer's placement according to requested domain and the buffer's 110 * flags. 111 */ 112 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) 113 { 114 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 115 struct ttm_placement *placement = &abo->placement; 116 struct ttm_place *places = abo->placements; 117 u64 flags = abo->flags; 118 u32 c = 0; 119 120 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 121 unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 122 int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); 123 124 if (adev->gmc.mem_partitions && mem_id >= 0) { 125 places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn; 126 /* 127 * memory partition range lpfn is inclusive start + size - 1 128 * TTM place lpfn is exclusive start + size 129 */ 130 places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1; 131 } else { 132 places[c].fpfn = 0; 133 places[c].lpfn = 0; 134 } 135 places[c].mem_type = TTM_PL_VRAM; 136 places[c].flags = 0; 137 138 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 139 places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn); 140 else 141 places[c].flags |= TTM_PL_FLAG_TOPDOWN; 142 143 if (abo->tbo.type == ttm_bo_type_kernel && 144 flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) 145 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; 146 147 c++; 148 } 149 150 if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) { 151 places[c].fpfn = 0; 152 places[c].lpfn = 0; 153 places[c].mem_type = AMDGPU_PL_DOORBELL; 154 places[c].flags = 0; 155 c++; 156 } 157 158 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 159 places[c].fpfn = 0; 160 places[c].lpfn = 0; 161 places[c].mem_type = 162 abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ? 163 AMDGPU_PL_PREEMPT : TTM_PL_TT; 164 places[c].flags = 0; 165 /* 166 * When GTT is just an alternative to VRAM make sure that we 167 * only use it as fallback and still try to fill up VRAM first. 168 */ 169 if (abo->tbo.resource && !(adev->flags & AMD_IS_APU) && 170 domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) 171 places[c].flags |= TTM_PL_FLAG_FALLBACK; 172 c++; 173 } 174 175 if (domain & AMDGPU_GEM_DOMAIN_CPU) { 176 places[c].fpfn = 0; 177 places[c].lpfn = 0; 178 places[c].mem_type = TTM_PL_SYSTEM; 179 places[c].flags = 0; 180 c++; 181 } 182 183 if (domain & AMDGPU_GEM_DOMAIN_GDS) { 184 places[c].fpfn = 0; 185 places[c].lpfn = 0; 186 places[c].mem_type = AMDGPU_PL_GDS; 187 places[c].flags = 0; 188 c++; 189 } 190 191 if (domain & AMDGPU_GEM_DOMAIN_GWS) { 192 places[c].fpfn = 0; 193 places[c].lpfn = 0; 194 places[c].mem_type = AMDGPU_PL_GWS; 195 places[c].flags = 0; 196 c++; 197 } 198 199 if (domain & AMDGPU_GEM_DOMAIN_OA) { 200 places[c].fpfn = 0; 201 places[c].lpfn = 0; 202 places[c].mem_type = AMDGPU_PL_OA; 203 places[c].flags = 0; 204 c++; 205 } 206 207 if (domain & AMDGPU_GEM_DOMAIN_NPA) { 208 places[c].fpfn = 0; 209 places[c].lpfn = 0; 210 places[c].mem_type = AMDGPU_PL_NPA; 211 places[c].flags = 0; 212 c++; 213 } 214 215 if (!c) { 216 places[c].fpfn = 0; 217 places[c].lpfn = 0; 218 places[c].mem_type = TTM_PL_SYSTEM; 219 places[c].flags = 0; 220 c++; 221 } 222 223 BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS); 224 225 placement->num_placement = c; 226 placement->placement = places; 227 } 228 229 /** 230 * amdgpu_bo_create_reserved - create reserved BO for kernel use 231 * 232 * @adev: amdgpu device object 233 * @size: size for the new BO 234 * @align: alignment for the new BO 235 * @domain: where to place it 236 * @bo_ptr: used to initialize BOs in structures 237 * @gpu_addr: GPU addr of the pinned BO 238 * @cpu_addr: optional CPU address mapping 239 * 240 * Allocates and pins a BO for kernel internal use, and returns it still 241 * reserved. 242 * 243 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 244 * 245 * Returns: 246 * 0 on success, negative error code otherwise. 247 */ 248 int amdgpu_bo_create_reserved(struct amdgpu_device *adev, 249 unsigned long size, int align, 250 u32 domain, struct amdgpu_bo **bo_ptr, 251 u64 *gpu_addr, void **cpu_addr) 252 { 253 struct amdgpu_bo_param bp; 254 bool free = false; 255 int r; 256 257 if (!size) { 258 amdgpu_bo_unref(bo_ptr); 259 return 0; 260 } 261 262 memset(&bp, 0, sizeof(bp)); 263 bp.size = size; 264 bp.byte_align = align; 265 bp.domain = domain; 266 bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED 267 : AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 268 bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 269 bp.type = ttm_bo_type_kernel; 270 bp.resv = NULL; 271 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 272 273 if (!*bo_ptr) { 274 r = amdgpu_bo_create(adev, &bp, bo_ptr); 275 if (r) { 276 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", 277 r); 278 return r; 279 } 280 free = true; 281 } 282 283 r = amdgpu_bo_reserve(*bo_ptr, false); 284 if (r) { 285 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); 286 goto error_free; 287 } 288 289 if (free) { 290 r = amdgpu_bo_pin(*bo_ptr, domain); 291 if (r) { 292 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); 293 goto error_unreserve; 294 } 295 } 296 297 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); 298 if (r) { 299 dev_err(adev->dev, "%p bind failed\n", *bo_ptr); 300 goto error_unpin; 301 } 302 303 if (gpu_addr) 304 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); 305 306 if (cpu_addr) { 307 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 308 if (r) { 309 dev_err(adev->dev, "(%d) kernel bo map failed\n", r); 310 goto error_unpin; 311 } 312 } 313 314 return 0; 315 316 error_unpin: 317 if (free) 318 amdgpu_bo_unpin(*bo_ptr); 319 error_unreserve: 320 amdgpu_bo_unreserve(*bo_ptr); 321 322 error_free: 323 if (free) 324 amdgpu_bo_unref(bo_ptr); 325 326 return r; 327 } 328 329 /** 330 * amdgpu_bo_create_kernel - create BO for kernel use 331 * 332 * @adev: amdgpu device object 333 * @size: size for the new BO 334 * @align: alignment for the new BO 335 * @domain: where to place it 336 * @bo_ptr: used to initialize BOs in structures 337 * @gpu_addr: GPU addr of the pinned BO 338 * @cpu_addr: optional CPU address mapping 339 * 340 * Allocates and pins a BO for kernel internal use. 341 * 342 * This function is exported to allow the V4L2 isp device 343 * external to drm device to create and access the kernel BO. 344 * 345 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 346 * 347 * Returns: 348 * 0 on success, negative error code otherwise. 349 */ 350 int amdgpu_bo_create_kernel(struct amdgpu_device *adev, 351 unsigned long size, int align, 352 u32 domain, struct amdgpu_bo **bo_ptr, 353 u64 *gpu_addr, void **cpu_addr) 354 { 355 int r; 356 357 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, 358 gpu_addr, cpu_addr); 359 360 if (r) 361 return r; 362 363 if (*bo_ptr) 364 amdgpu_bo_unreserve(*bo_ptr); 365 366 return 0; 367 } 368 369 /** 370 * amdgpu_bo_create_isp_user - create user BO for isp 371 * 372 * @adev: amdgpu device object 373 * @dma_buf: DMABUF handle for isp buffer 374 * @domain: where to place it 375 * @bo: used to initialize BOs in structures 376 * @gpu_addr: GPU addr of the pinned BO 377 * 378 * Imports isp DMABUF to allocate and pin a user BO for isp internal use. It does 379 * GART alloc to generate gpu_addr for BO to make it accessible through the 380 * GART aperture for ISP HW. 381 * 382 * This function is exported to allow the V4L2 isp device external to drm device 383 * to create and access the isp user BO. 384 * 385 * Returns: 386 * 0 on success, negative error code otherwise. 387 */ 388 int amdgpu_bo_create_isp_user(struct amdgpu_device *adev, 389 struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo, 390 u64 *gpu_addr) 391 392 { 393 struct drm_gem_object *gem_obj; 394 int r; 395 396 gem_obj = amdgpu_gem_prime_import(&adev->ddev, dma_buf); 397 *bo = gem_to_amdgpu_bo(gem_obj); 398 if (!(*bo)) { 399 dev_err(adev->dev, "failed to get valid isp user bo\n"); 400 return -EINVAL; 401 } 402 403 r = amdgpu_bo_reserve(*bo, false); 404 if (r) { 405 dev_err(adev->dev, "(%d) failed to reserve isp user bo\n", r); 406 return r; 407 } 408 409 r = amdgpu_bo_pin(*bo, domain); 410 if (r) { 411 dev_err(adev->dev, "(%d) isp user bo pin failed\n", r); 412 goto error_unreserve; 413 } 414 415 r = amdgpu_ttm_alloc_gart(&(*bo)->tbo); 416 if (r) { 417 dev_err(adev->dev, "%p bind failed\n", *bo); 418 goto error_unpin; 419 } 420 421 if (!WARN_ON(!gpu_addr)) 422 *gpu_addr = amdgpu_bo_gpu_offset(*bo); 423 424 amdgpu_bo_unreserve(*bo); 425 426 return 0; 427 428 error_unpin: 429 amdgpu_bo_unpin(*bo); 430 error_unreserve: 431 amdgpu_bo_unreserve(*bo); 432 amdgpu_bo_unref(bo); 433 434 return r; 435 } 436 437 /** 438 * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location 439 * 440 * @adev: amdgpu device object 441 * @offset: offset of the BO 442 * @size: size of the BO 443 * @bo_ptr: used to initialize BOs in structures 444 * @cpu_addr: optional CPU address mapping 445 * 446 * Creates a kernel BO at a specific offset in VRAM. 447 * 448 * Returns: 449 * 0 on success, negative error code otherwise. 450 */ 451 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, 452 uint64_t offset, uint64_t size, 453 struct amdgpu_bo **bo_ptr, void **cpu_addr) 454 { 455 struct ttm_operation_ctx ctx = { false, false }; 456 unsigned int i; 457 int r; 458 459 offset &= PAGE_MASK; 460 size = ALIGN(size, PAGE_SIZE); 461 462 r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, 463 AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL, 464 cpu_addr); 465 if (r) 466 return r; 467 468 if ((*bo_ptr) == NULL) 469 return 0; 470 471 /* 472 * Remove the original mem node and create a new one at the request 473 * position. 474 */ 475 if (cpu_addr) 476 amdgpu_bo_kunmap(*bo_ptr); 477 478 ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource); 479 480 for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) { 481 (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT; 482 (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; 483 } 484 r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement, 485 &(*bo_ptr)->tbo.resource, &ctx); 486 if (r) 487 goto error; 488 489 if (cpu_addr) { 490 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 491 if (r) 492 goto error; 493 } 494 495 amdgpu_bo_unreserve(*bo_ptr); 496 return 0; 497 498 error: 499 amdgpu_bo_unreserve(*bo_ptr); 500 amdgpu_bo_unref(bo_ptr); 501 return r; 502 } 503 504 /** 505 * amdgpu_bo_free_kernel - free BO for kernel use 506 * 507 * @bo: amdgpu BO to free 508 * @gpu_addr: pointer to where the BO's GPU memory space address was stored 509 * @cpu_addr: pointer to where the BO's CPU memory space address was stored 510 * 511 * unmaps and unpin a BO for kernel internal use. 512 * 513 * This function is exported to allow the V4L2 isp device 514 * external to drm device to free the kernel BO. 515 */ 516 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, 517 void **cpu_addr) 518 { 519 if (*bo == NULL) 520 return; 521 522 WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); 523 524 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { 525 if (cpu_addr) 526 amdgpu_bo_kunmap(*bo); 527 528 amdgpu_bo_unpin(*bo); 529 amdgpu_bo_unreserve(*bo); 530 } 531 amdgpu_bo_unref(bo); 532 533 if (gpu_addr) 534 *gpu_addr = 0; 535 536 if (cpu_addr) 537 *cpu_addr = NULL; 538 } 539 540 /** 541 * amdgpu_bo_free_isp_user - free BO for isp use 542 * 543 * @bo: amdgpu isp user BO to free 544 * 545 * unpin and unref BO for isp internal use. 546 * 547 * This function is exported to allow the V4L2 isp device 548 * external to drm device to free the isp user BO. 549 */ 550 void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo) 551 { 552 if (bo == NULL) 553 return; 554 555 if (amdgpu_bo_reserve(bo, true) == 0) { 556 amdgpu_bo_unpin(bo); 557 amdgpu_bo_unreserve(bo); 558 } 559 amdgpu_bo_unref(&bo); 560 } 561 562 /* Validate bo size is bit bigger than the request domain */ 563 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, 564 unsigned long size, u32 domain) 565 { 566 struct ttm_resource_manager *man = NULL; 567 568 /* 569 * If GTT is part of requested domains the check must succeed to 570 * allow fall back to GTT. 571 */ 572 if (domain & AMDGPU_GEM_DOMAIN_GTT) 573 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); 574 else if (domain & AMDGPU_GEM_DOMAIN_VRAM) 575 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 576 else 577 return true; 578 579 if (!man) { 580 if (domain & AMDGPU_GEM_DOMAIN_GTT) 581 WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized"); 582 return false; 583 } 584 585 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */ 586 if (size < man->size) 587 return true; 588 589 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size); 590 return false; 591 } 592 593 bool amdgpu_bo_support_uswc(u64 bo_flags) 594 { 595 596 #ifdef CONFIG_X86_32 597 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit 598 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 599 */ 600 return false; 601 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) 602 /* Don't try to enable write-combining when it can't work, or things 603 * may be slow 604 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 605 */ 606 607 #ifndef CONFIG_COMPILE_TEST 608 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ 609 thanks to write-combining 610 #endif 611 612 if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 613 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " 614 "better performance thanks to write-combining\n"); 615 return false; 616 #else 617 /* For architectures that don't support WC memory, 618 * mask out the WC flag from the BO 619 */ 620 if (!drm_arch_can_wc_memory()) 621 return false; 622 623 return true; 624 #endif 625 } 626 627 /** 628 * amdgpu_bo_create - create an &amdgpu_bo buffer object 629 * @adev: amdgpu device object 630 * @bp: parameters to be used for the buffer object 631 * @bo_ptr: pointer to the buffer object pointer 632 * 633 * Creates an &amdgpu_bo buffer object. 634 * 635 * Returns: 636 * 0 for success or a negative error code on failure. 637 */ 638 int amdgpu_bo_create(struct amdgpu_device *adev, 639 struct amdgpu_bo_param *bp, 640 struct amdgpu_bo **bo_ptr) 641 { 642 struct ttm_operation_ctx ctx = { 643 .interruptible = (bp->type != ttm_bo_type_kernel), 644 .no_wait_gpu = bp->no_wait_gpu, 645 /* We opt to avoid OOM on system pages allocations */ 646 .gfp_retry_mayfail = true, 647 .allow_res_evict = bp->type != ttm_bo_type_kernel, 648 .resv = bp->resv 649 }; 650 struct amdgpu_bo *bo; 651 unsigned long page_align, size = bp->size; 652 int r; 653 654 /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ 655 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { 656 /* GWS and OA don't need any alignment. */ 657 page_align = bp->byte_align; 658 size <<= PAGE_SHIFT; 659 660 } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { 661 /* Both size and alignment must be a multiple of 4. */ 662 page_align = ALIGN(bp->byte_align, 4); 663 size = ALIGN(size, 4) << PAGE_SHIFT; 664 } else { 665 /* Memory should be aligned at least to a page size. */ 666 page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; 667 size = ALIGN(size, PAGE_SIZE); 668 } 669 670 if (!amdgpu_bo_validate_size(adev, size, bp->domain)) 671 return -ENOMEM; 672 673 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo)); 674 675 *bo_ptr = NULL; 676 bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL); 677 if (bo == NULL) 678 return -ENOMEM; 679 drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); 680 bo->tbo.base.funcs = &amdgpu_gem_object_funcs; 681 bo->vm_bo = NULL; 682 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : 683 bp->domain; 684 bo->allowed_domains = bo->preferred_domains; 685 if (bp->type != ttm_bo_type_kernel && 686 !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) && 687 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) 688 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 689 690 bo->flags = bp->flags; 691 692 if (adev->gmc.mem_partitions) 693 /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */ 694 bo->xcp_id = bp->xcp_id_plus1 - 1; 695 else 696 /* For GPUs without spatial partitioning */ 697 bo->xcp_id = 0; 698 699 if (!amdgpu_bo_support_uswc(bo->flags)) 700 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 701 702 bo->tbo.bdev = &adev->mman.bdev; 703 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | 704 AMDGPU_GEM_DOMAIN_GDS)) 705 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 706 else 707 amdgpu_bo_placement_from_domain(bo, bp->domain); 708 if (bp->type == ttm_bo_type_kernel) 709 bo->tbo.priority = 2; 710 else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE)) 711 bo->tbo.priority = 1; 712 713 if (!bp->destroy) 714 bp->destroy = &amdgpu_bo_destroy; 715 716 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type, 717 &bo->placement, page_align, &ctx, NULL, 718 bp->resv, bp->destroy); 719 if (unlikely(r != 0)) 720 return r; 721 722 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 723 amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 724 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 725 ctx.bytes_moved); 726 else 727 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); 728 729 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && 730 bo->tbo.resource->mem_type == TTM_PL_VRAM) { 731 struct dma_fence *fence; 732 733 r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev), 734 bo, bo->tbo.base.resv, &fence, 735 true, AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER); 736 if (unlikely(r)) 737 goto fail_unreserve; 738 739 if (fence) { 740 dma_resv_add_fence(bo->tbo.base.resv, fence, 741 DMA_RESV_USAGE_KERNEL); 742 dma_fence_put(fence); 743 } 744 } 745 if (!bp->resv) 746 amdgpu_bo_unreserve(bo); 747 *bo_ptr = bo; 748 749 trace_amdgpu_bo_create(bo); 750 751 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ 752 if (bp->type == ttm_bo_type_device) 753 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 754 755 return 0; 756 757 fail_unreserve: 758 if (!bp->resv) 759 dma_resv_unlock(bo->tbo.base.resv); 760 amdgpu_bo_unref(&bo); 761 return r; 762 } 763 764 /** 765 * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object 766 * @adev: amdgpu device object 767 * @bp: parameters to be used for the buffer object 768 * @ubo_ptr: pointer to the buffer object pointer 769 * 770 * Create a BO to be used by user application; 771 * 772 * Returns: 773 * 0 for success or a negative error code on failure. 774 */ 775 776 int amdgpu_bo_create_user(struct amdgpu_device *adev, 777 struct amdgpu_bo_param *bp, 778 struct amdgpu_bo_user **ubo_ptr) 779 { 780 struct amdgpu_bo *bo_ptr; 781 int r; 782 783 bp->bo_ptr_size = sizeof(struct amdgpu_bo_user); 784 bp->destroy = &amdgpu_bo_user_destroy; 785 r = amdgpu_bo_create(adev, bp, &bo_ptr); 786 if (r) 787 return r; 788 789 *ubo_ptr = to_amdgpu_bo_user(bo_ptr); 790 return r; 791 } 792 793 /** 794 * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object 795 * @adev: amdgpu device object 796 * @bp: parameters to be used for the buffer object 797 * @vmbo_ptr: pointer to the buffer object pointer 798 * 799 * Create a BO to be for GPUVM. 800 * 801 * Returns: 802 * 0 for success or a negative error code on failure. 803 */ 804 805 int amdgpu_bo_create_vm(struct amdgpu_device *adev, 806 struct amdgpu_bo_param *bp, 807 struct amdgpu_bo_vm **vmbo_ptr) 808 { 809 struct amdgpu_bo *bo_ptr; 810 int r; 811 812 /* bo_ptr_size will be determined by the caller and it depends on 813 * num of amdgpu_vm_pt entries. 814 */ 815 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm)); 816 r = amdgpu_bo_create(adev, bp, &bo_ptr); 817 if (r) 818 return r; 819 820 *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr); 821 return r; 822 } 823 824 /** 825 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object 826 * @bo: &amdgpu_bo buffer object to be mapped 827 * @ptr: kernel virtual address to be returned 828 * 829 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls 830 * amdgpu_bo_kptr() to get the kernel virtual address. 831 * 832 * Returns: 833 * 0 for success or a negative error code on failure. 834 */ 835 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) 836 { 837 void *kptr; 838 long r; 839 840 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 841 return -EPERM; 842 843 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL, 844 false, MAX_SCHEDULE_TIMEOUT); 845 if (r < 0) 846 return r; 847 848 kptr = amdgpu_bo_kptr(bo); 849 if (kptr) { 850 if (ptr) 851 *ptr = kptr; 852 return 0; 853 } 854 855 r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap); 856 if (r) 857 return r; 858 859 if (ptr) 860 *ptr = amdgpu_bo_kptr(bo); 861 862 return 0; 863 } 864 865 /** 866 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object 867 * @bo: &amdgpu_bo buffer object 868 * 869 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address 870 * 871 * Returns: 872 * the virtual address of a buffer object area. 873 */ 874 void *amdgpu_bo_kptr(struct amdgpu_bo *bo) 875 { 876 bool is_iomem; 877 878 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 879 } 880 881 /** 882 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object 883 * @bo: &amdgpu_bo buffer object to be unmapped 884 * 885 * Unmaps a kernel map set up by amdgpu_bo_kmap(). 886 */ 887 void amdgpu_bo_kunmap(struct amdgpu_bo *bo) 888 { 889 if (bo->kmap.bo) 890 ttm_bo_kunmap(&bo->kmap); 891 } 892 893 /** 894 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object 895 * @bo: &amdgpu_bo buffer object 896 * 897 * References the contained &ttm_buffer_object. 898 * 899 * Returns: 900 * a refcounted pointer to the &amdgpu_bo buffer object. 901 */ 902 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) 903 { 904 if (bo == NULL) 905 return NULL; 906 907 drm_gem_object_get(&bo->tbo.base); 908 return bo; 909 } 910 911 /** 912 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object 913 * @bo: &amdgpu_bo buffer object 914 * 915 * Unreferences the contained &ttm_buffer_object and clear the pointer 916 */ 917 void amdgpu_bo_unref(struct amdgpu_bo **bo) 918 { 919 if ((*bo) == NULL) 920 return; 921 922 drm_gem_object_put(&(*bo)->tbo.base); 923 *bo = NULL; 924 } 925 926 /** 927 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object 928 * @bo: &amdgpu_bo buffer object to be pinned 929 * @domain: domain to be pinned to 930 * 931 * Pins the buffer object according to requested domain. If the memory is 932 * unbound gart memory, binds the pages into gart table. Adjusts pin_count and 933 * pin_size accordingly. 934 * 935 * Pinning means to lock pages in memory along with keeping them at a fixed 936 * offset. It is required when a buffer can not be moved, for example, when 937 * a display buffer is being scanned out. 938 * 939 * Returns: 940 * 0 for success or a negative error code on failure. 941 */ 942 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) 943 { 944 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 945 struct ttm_operation_ctx ctx = { false, false }; 946 int r, i; 947 948 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) 949 return -EPERM; 950 951 /* Check domain to be pinned to against preferred domains */ 952 if (bo->preferred_domains & domain) 953 domain = bo->preferred_domains & domain; 954 955 /* A shared bo cannot be migrated to VRAM */ 956 if (drm_gem_is_imported(&bo->tbo.base)) { 957 if (domain & AMDGPU_GEM_DOMAIN_GTT) 958 domain = AMDGPU_GEM_DOMAIN_GTT; 959 else 960 return -EINVAL; 961 } 962 963 if (bo->tbo.pin_count) { 964 uint32_t mem_type = bo->tbo.resource->mem_type; 965 uint32_t mem_flags = bo->tbo.resource->placement; 966 967 if (!(domain & amdgpu_mem_type_to_domain(mem_type))) 968 return -EINVAL; 969 970 if ((mem_type == TTM_PL_VRAM) && 971 (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && 972 !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) 973 return -EINVAL; 974 975 ttm_bo_pin(&bo->tbo); 976 return 0; 977 } 978 979 /* This assumes only APU display buffers are pinned with (VRAM|GTT). 980 * See function amdgpu_display_supported_domains() 981 */ 982 domain = amdgpu_bo_get_preferred_domain(adev, domain); 983 984 if (drm_gem_is_imported(&bo->tbo.base)) 985 dma_buf_pin(bo->tbo.base.import_attach); 986 987 /* force to pin into visible video ram */ 988 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) 989 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 990 amdgpu_bo_placement_from_domain(bo, domain); 991 for (i = 0; i < bo->placement.num_placement; i++) { 992 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && 993 bo->placements[i].mem_type == TTM_PL_VRAM) 994 bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS; 995 } 996 997 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 998 if (unlikely(r)) { 999 dev_err(adev->dev, "%p pin failed\n", bo); 1000 goto error; 1001 } 1002 1003 ttm_bo_pin(&bo->tbo); 1004 1005 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 1006 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); 1007 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), 1008 &adev->visible_pin_size); 1009 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 1010 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); 1011 } 1012 1013 error: 1014 return r; 1015 } 1016 1017 /** 1018 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object 1019 * @bo: &amdgpu_bo buffer object to be unpinned 1020 * 1021 * Decreases the pin_count, and clears the flags if pin_count reaches 0. 1022 * Changes placement and pin size accordingly. 1023 * 1024 * Returns: 1025 * 0 for success or a negative error code on failure. 1026 */ 1027 void amdgpu_bo_unpin(struct amdgpu_bo *bo) 1028 { 1029 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1030 1031 ttm_bo_unpin(&bo->tbo); 1032 if (bo->tbo.pin_count) 1033 return; 1034 1035 if (drm_gem_is_imported(&bo->tbo.base)) 1036 dma_buf_unpin(bo->tbo.base.import_attach); 1037 1038 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 1039 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); 1040 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), 1041 &adev->visible_pin_size); 1042 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 1043 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); 1044 } 1045 1046 } 1047 1048 static const char * const amdgpu_vram_names[] = { 1049 "UNKNOWN", 1050 "GDDR1", 1051 "DDR2", 1052 "GDDR3", 1053 "GDDR4", 1054 "GDDR5", 1055 "HBM", 1056 "DDR3", 1057 "DDR4", 1058 "GDDR6", 1059 "DDR5", 1060 "LPDDR4", 1061 "LPDDR5", 1062 "HBM3E", 1063 "HBM4", 1064 "GDDR7" 1065 }; 1066 1067 /** 1068 * amdgpu_bo_init - initialize memory manager 1069 * @adev: amdgpu device object 1070 * 1071 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. 1072 * 1073 * Returns: 1074 * 0 for success or a negative error code on failure. 1075 */ 1076 int amdgpu_bo_init(struct amdgpu_device *adev) 1077 { 1078 /* On A+A platform, VRAM can be mapped as WB */ 1079 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 1080 /* reserve PAT memory space to WC for VRAM */ 1081 int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base, 1082 adev->gmc.aper_size); 1083 1084 if (r) { 1085 DRM_ERROR("Unable to set WC memtype for the aperture base\n"); 1086 return r; 1087 } 1088 1089 /* Add an MTRR for the VRAM */ 1090 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, 1091 adev->gmc.aper_size); 1092 } 1093 1094 drm_info(adev_to_drm(adev), "Detected VRAM RAM=%lluM, BAR=%lluM\n", 1095 adev->gmc.mc_vram_size >> 20, 1096 (unsigned long long)adev->gmc.aper_size >> 20); 1097 drm_info(adev_to_drm(adev), "RAM width %dbits %s\n", 1098 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); 1099 return amdgpu_ttm_init(adev); 1100 } 1101 1102 /** 1103 * amdgpu_bo_fini - tear down memory manager 1104 * @adev: amdgpu device object 1105 * 1106 * Reverses amdgpu_bo_init() to tear down memory manager. 1107 */ 1108 void amdgpu_bo_fini(struct amdgpu_device *adev) 1109 { 1110 int idx; 1111 1112 amdgpu_ttm_fini(adev); 1113 1114 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 1115 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 1116 arch_phys_wc_del(adev->gmc.vram_mtrr); 1117 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 1118 } 1119 drm_dev_exit(idx); 1120 } 1121 } 1122 1123 /** 1124 * amdgpu_bo_set_tiling_flags - set tiling flags 1125 * @bo: &amdgpu_bo buffer object 1126 * @tiling_flags: new flags 1127 * 1128 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or 1129 * kernel driver to set the tiling flags on a buffer. 1130 * 1131 * Returns: 1132 * 0 for success or a negative error code on failure. 1133 */ 1134 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) 1135 { 1136 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1137 struct amdgpu_bo_user *ubo; 1138 1139 /* MMIO_REMAP is BAR I/O space; tiling should never be used here. */ 1140 WARN_ON_ONCE(bo->tbo.resource && 1141 bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP); 1142 1143 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1144 if (adev->family <= AMDGPU_FAMILY_CZ && 1145 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) 1146 return -EINVAL; 1147 1148 ubo = to_amdgpu_bo_user(bo); 1149 ubo->tiling_flags = tiling_flags; 1150 return 0; 1151 } 1152 1153 /** 1154 * amdgpu_bo_get_tiling_flags - get tiling flags 1155 * @bo: &amdgpu_bo buffer object 1156 * @tiling_flags: returned flags 1157 * 1158 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to 1159 * set the tiling flags on a buffer. 1160 */ 1161 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) 1162 { 1163 struct amdgpu_bo_user *ubo; 1164 1165 /* 1166 * MMIO_REMAP BOs are not real VRAM/GTT memory but a fixed BAR I/O window. 1167 * They should never go through GEM tiling helpers. 1168 */ 1169 WARN_ON_ONCE(bo->tbo.resource && 1170 bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP); 1171 1172 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1173 dma_resv_assert_held(bo->tbo.base.resv); 1174 ubo = to_amdgpu_bo_user(bo); 1175 1176 if (tiling_flags) 1177 *tiling_flags = ubo->tiling_flags; 1178 } 1179 1180 /** 1181 * amdgpu_bo_set_metadata - set metadata 1182 * @bo: &amdgpu_bo buffer object 1183 * @metadata: new metadata 1184 * @metadata_size: size of the new metadata 1185 * @flags: flags of the new metadata 1186 * 1187 * Sets buffer object's metadata, its size and flags. 1188 * Used via GEM ioctl. 1189 * 1190 * Returns: 1191 * 0 for success or a negative error code on failure. 1192 */ 1193 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata, 1194 u32 metadata_size, uint64_t flags) 1195 { 1196 struct amdgpu_bo_user *ubo; 1197 void *buffer; 1198 1199 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1200 ubo = to_amdgpu_bo_user(bo); 1201 if (!metadata_size) { 1202 if (ubo->metadata_size) { 1203 kfree(ubo->metadata); 1204 ubo->metadata = NULL; 1205 ubo->metadata_size = 0; 1206 } 1207 return 0; 1208 } 1209 1210 if (metadata == NULL) 1211 return -EINVAL; 1212 1213 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); 1214 if (buffer == NULL) 1215 return -ENOMEM; 1216 1217 kfree(ubo->metadata); 1218 ubo->metadata_flags = flags; 1219 ubo->metadata = buffer; 1220 ubo->metadata_size = metadata_size; 1221 1222 return 0; 1223 } 1224 1225 /** 1226 * amdgpu_bo_get_metadata - get metadata 1227 * @bo: &amdgpu_bo buffer object 1228 * @buffer: returned metadata 1229 * @buffer_size: size of the buffer 1230 * @metadata_size: size of the returned metadata 1231 * @flags: flags of the returned metadata 1232 * 1233 * Gets buffer object's metadata, its size and flags. buffer_size shall not be 1234 * less than metadata_size. 1235 * Used via GEM ioctl. 1236 * 1237 * Returns: 1238 * 0 for success or a negative error code on failure. 1239 */ 1240 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, 1241 size_t buffer_size, uint32_t *metadata_size, 1242 uint64_t *flags) 1243 { 1244 struct amdgpu_bo_user *ubo; 1245 1246 if (!buffer && !metadata_size) 1247 return -EINVAL; 1248 1249 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1250 ubo = to_amdgpu_bo_user(bo); 1251 if (metadata_size) 1252 *metadata_size = ubo->metadata_size; 1253 1254 if (buffer) { 1255 if (buffer_size < ubo->metadata_size) 1256 return -EINVAL; 1257 1258 if (ubo->metadata_size) 1259 memcpy(buffer, ubo->metadata, ubo->metadata_size); 1260 } 1261 1262 if (flags) 1263 *flags = ubo->metadata_flags; 1264 1265 return 0; 1266 } 1267 1268 /** 1269 * amdgpu_bo_move_notify - notification about a memory move 1270 * @bo: pointer to a buffer object 1271 * @evict: if this move is evicting the buffer from the graphics address space 1272 * @new_mem: new resource for backing the BO 1273 * 1274 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs 1275 * bookkeeping. 1276 * TTM driver callback which is called when ttm moves a buffer. 1277 */ 1278 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, 1279 bool evict, 1280 struct ttm_resource *new_mem) 1281 { 1282 struct ttm_resource *old_mem = bo->resource; 1283 struct amdgpu_bo *abo; 1284 1285 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1286 return; 1287 1288 abo = ttm_to_amdgpu_bo(bo); 1289 amdgpu_vm_bo_move(abo, new_mem, evict); 1290 1291 amdgpu_bo_kunmap(abo); 1292 1293 if (abo->tbo.base.dma_buf && !drm_gem_is_imported(&abo->tbo.base) && 1294 old_mem && old_mem->mem_type != TTM_PL_SYSTEM) 1295 dma_buf_invalidate_mappings(abo->tbo.base.dma_buf); 1296 1297 /* move_notify is called before move happens */ 1298 trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1, 1299 old_mem ? old_mem->mem_type : -1); 1300 } 1301 1302 /** 1303 * amdgpu_bo_release_notify - notification about a BO being released 1304 * @bo: pointer to a buffer object 1305 * 1306 * Wipes VRAM buffers whose contents should not be leaked before the 1307 * memory is released. 1308 */ 1309 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) 1310 { 1311 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1312 struct dma_fence *fence = NULL; 1313 struct amdgpu_bo *abo; 1314 int r; 1315 1316 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1317 return; 1318 1319 abo = ttm_to_amdgpu_bo(bo); 1320 1321 WARN_ON(abo->vm_bo); 1322 1323 if (abo->kfd_bo) 1324 amdgpu_amdkfd_release_notify(abo); 1325 1326 /* 1327 * We lock the private dma_resv object here and since the BO is about to 1328 * be released nobody else should have a pointer to it. 1329 * So when this locking here fails something is wrong with the reference 1330 * counting. 1331 */ 1332 if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv))) 1333 return; 1334 1335 amdgpu_amdkfd_remove_all_eviction_fences(abo); 1336 1337 if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM || 1338 !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) || 1339 adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev))) 1340 goto out; 1341 1342 r = dma_resv_reserve_fences(&bo->base._resv, 1); 1343 if (r) 1344 goto out; 1345 1346 r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev), 1347 abo, &bo->base._resv, &fence, 1348 false, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE); 1349 if (WARN_ON(r)) 1350 goto out; 1351 1352 amdgpu_vram_mgr_set_cleared(bo->resource); 1353 dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL); 1354 dma_fence_put(fence); 1355 1356 out: 1357 dma_resv_unlock(&bo->base._resv); 1358 } 1359 1360 /** 1361 * amdgpu_bo_fault_reserve_notify - notification about a memory fault 1362 * @bo: pointer to a buffer object 1363 * 1364 * Notifies the driver we are taking a fault on this BO and have reserved it, 1365 * also performs bookkeeping. 1366 * TTM driver callback for dealing with vm faults. 1367 * 1368 * Returns: 1369 * 0 for success or a negative error code on failure. 1370 */ 1371 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 1372 { 1373 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1374 struct ttm_operation_ctx ctx = { false, false }; 1375 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1376 int r; 1377 1378 /* Remember that this BO was accessed by the CPU */ 1379 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 1380 1381 if (amdgpu_res_cpu_visible(adev, bo->resource)) 1382 return 0; 1383 1384 /* Can't move a pinned BO to visible VRAM */ 1385 if (abo->tbo.pin_count > 0) 1386 return VM_FAULT_SIGBUS; 1387 1388 /* hurrah the memory is not visible ! */ 1389 atomic64_inc(&adev->num_vram_cpu_page_faults); 1390 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 1391 AMDGPU_GEM_DOMAIN_GTT); 1392 1393 /* Avoid costly evictions; only set GTT as a busy placement */ 1394 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; 1395 1396 r = ttm_bo_validate(bo, &abo->placement, &ctx); 1397 if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) 1398 return VM_FAULT_NOPAGE; 1399 else if (unlikely(r)) 1400 return VM_FAULT_SIGBUS; 1401 1402 /* this should never happen */ 1403 if (bo->resource->mem_type == TTM_PL_VRAM && 1404 !amdgpu_res_cpu_visible(adev, bo->resource)) 1405 return VM_FAULT_SIGBUS; 1406 1407 ttm_bo_move_to_lru_tail_unlocked(bo); 1408 return 0; 1409 } 1410 1411 /** 1412 * amdgpu_bo_fence - add fence to buffer object 1413 * 1414 * @bo: buffer object in question 1415 * @fence: fence to add 1416 * @shared: true if fence should be added shared 1417 * 1418 */ 1419 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, 1420 bool shared) 1421 { 1422 struct dma_resv *resv = bo->tbo.base.resv; 1423 int r; 1424 1425 r = dma_resv_reserve_fences(resv, 1); 1426 if (r) { 1427 /* As last resort on OOM we block for the fence */ 1428 dma_fence_wait(fence, false); 1429 return; 1430 } 1431 1432 dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ : 1433 DMA_RESV_USAGE_WRITE); 1434 } 1435 1436 /** 1437 * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences 1438 * 1439 * @adev: amdgpu device pointer 1440 * @resv: reservation object to sync to 1441 * @sync_mode: synchronization mode 1442 * @owner: fence owner 1443 * @intr: Whether the wait is interruptible 1444 * 1445 * Extract the fences from the reservation object and waits for them to finish. 1446 * 1447 * Returns: 1448 * 0 on success, errno otherwise. 1449 */ 1450 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, 1451 enum amdgpu_sync_mode sync_mode, void *owner, 1452 bool intr) 1453 { 1454 struct amdgpu_sync sync; 1455 int r; 1456 1457 amdgpu_sync_create(&sync); 1458 amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner); 1459 r = amdgpu_sync_wait(&sync, intr); 1460 amdgpu_sync_free(&sync); 1461 return r; 1462 } 1463 1464 /** 1465 * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv 1466 * @bo: buffer object to wait for 1467 * @owner: fence owner 1468 * @intr: Whether the wait is interruptible 1469 * 1470 * Wrapper to wait for fences in a BO. 1471 * Returns: 1472 * 0 on success, errno otherwise. 1473 */ 1474 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) 1475 { 1476 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1477 1478 return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, 1479 AMDGPU_SYNC_NE_OWNER, owner, intr); 1480 } 1481 1482 /** 1483 * amdgpu_bo_gpu_offset - return GPU offset of bo 1484 * @bo: amdgpu object for which we query the offset 1485 * 1486 * Note: object should either be pinned or reserved when calling this 1487 * function, it might be useful to add check for this for debugging. 1488 * 1489 * Returns: 1490 * current GPU offset of the object. 1491 */ 1492 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) 1493 { 1494 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM); 1495 WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) && 1496 !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel); 1497 WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET); 1498 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM && 1499 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); 1500 1501 return amdgpu_bo_gpu_offset_no_check(bo); 1502 } 1503 1504 /** 1505 * amdgpu_bo_fb_aper_addr - return FB aperture GPU offset of the VRAM bo 1506 * @bo: amdgpu VRAM buffer object for which we query the offset 1507 * 1508 * Returns: 1509 * current FB aperture GPU offset of the object. 1510 */ 1511 u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo) 1512 { 1513 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1514 uint64_t offset, fb_base; 1515 1516 WARN_ON_ONCE(bo->tbo.resource->mem_type != TTM_PL_VRAM); 1517 1518 fb_base = adev->gmc.fb_start; 1519 fb_base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1520 offset = (bo->tbo.resource->start << PAGE_SHIFT) + fb_base; 1521 return amdgpu_gmc_sign_extend(offset); 1522 } 1523 1524 /** 1525 * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo 1526 * @bo: amdgpu object for which we query the offset 1527 * 1528 * Returns: 1529 * current GPU offset of the object without raising warnings. 1530 */ 1531 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) 1532 { 1533 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1534 uint64_t offset = AMDGPU_BO_INVALID_OFFSET; 1535 1536 if (bo->tbo.resource->mem_type == TTM_PL_TT) 1537 offset = amdgpu_gmc_agp_addr(&bo->tbo); 1538 1539 if (offset == AMDGPU_BO_INVALID_OFFSET) 1540 offset = (bo->tbo.resource->start << PAGE_SHIFT) + 1541 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type); 1542 1543 return amdgpu_gmc_sign_extend(offset); 1544 } 1545 1546 /** 1547 * amdgpu_bo_mem_stats_placement - bo placement for memory accounting 1548 * @bo: the buffer object we should look at 1549 * 1550 * BO can have multiple preferred placements, to avoid double counting we want 1551 * to file it under a single placement for memory stats. 1552 * Luckily, if we take the highest set bit in preferred_domains the result is 1553 * quite sensible. 1554 * 1555 * Returns: 1556 * Which of the placements should the BO be accounted under. 1557 */ 1558 uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo) 1559 { 1560 u32 domain; 1561 1562 /* 1563 * MMIO_REMAP is internal now, so it no longer maps from a userspace 1564 * domain bit. Keep fdinfo/mem-stats visibility by checking the actual 1565 * TTM placement. 1566 */ 1567 if (bo->tbo.resource && bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) 1568 return AMDGPU_PL_MMIO_REMAP; 1569 1570 domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK; 1571 if (!domain) 1572 return TTM_PL_SYSTEM; 1573 1574 switch (rounddown_pow_of_two(domain)) { 1575 case AMDGPU_GEM_DOMAIN_CPU: 1576 return TTM_PL_SYSTEM; 1577 case AMDGPU_GEM_DOMAIN_GTT: 1578 return TTM_PL_TT; 1579 case AMDGPU_GEM_DOMAIN_VRAM: 1580 return TTM_PL_VRAM; 1581 case AMDGPU_GEM_DOMAIN_GDS: 1582 return AMDGPU_PL_GDS; 1583 case AMDGPU_GEM_DOMAIN_GWS: 1584 return AMDGPU_PL_GWS; 1585 case AMDGPU_GEM_DOMAIN_OA: 1586 return AMDGPU_PL_OA; 1587 case AMDGPU_GEM_DOMAIN_DOORBELL: 1588 return AMDGPU_PL_DOORBELL; 1589 default: 1590 return TTM_PL_SYSTEM; 1591 } 1592 } 1593 1594 /** 1595 * amdgpu_bo_get_preferred_domain - get preferred domain 1596 * @adev: amdgpu device object 1597 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` 1598 * 1599 * Returns: 1600 * Which of the allowed domains is preferred for allocating the BO. 1601 */ 1602 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, 1603 uint32_t domain) 1604 { 1605 if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && 1606 ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) { 1607 domain = AMDGPU_GEM_DOMAIN_VRAM; 1608 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) 1609 domain = AMDGPU_GEM_DOMAIN_GTT; 1610 } 1611 return domain; 1612 } 1613 1614 #if defined(CONFIG_DEBUG_FS) 1615 #define amdgpu_bo_print_flag(m, bo, flag) \ 1616 do { \ 1617 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ 1618 seq_printf((m), " " #flag); \ 1619 } \ 1620 } while (0) 1621 1622 /** 1623 * amdgpu_bo_print_info - print BO info in debugfs file 1624 * 1625 * @id: Index or Id of the BO 1626 * @bo: Requested BO for printing info 1627 * @m: debugfs file 1628 * 1629 * Print BO information in debugfs file 1630 * 1631 * Returns: 1632 * Size of the BO in bytes. 1633 */ 1634 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m) 1635 { 1636 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1637 struct dma_buf_attachment *attachment; 1638 struct dma_buf *dma_buf; 1639 const char *placement; 1640 unsigned int pin_count; 1641 u64 size; 1642 1643 if (dma_resv_trylock(bo->tbo.base.resv)) { 1644 if (!bo->tbo.resource) { 1645 placement = "NONE"; 1646 } else { 1647 switch (bo->tbo.resource->mem_type) { 1648 case TTM_PL_VRAM: 1649 if (amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 1650 placement = "VRAM VISIBLE"; 1651 else 1652 placement = "VRAM"; 1653 break; 1654 case TTM_PL_TT: 1655 placement = "GTT"; 1656 break; 1657 case AMDGPU_PL_GDS: 1658 placement = "GDS"; 1659 break; 1660 case AMDGPU_PL_GWS: 1661 placement = "GWS"; 1662 break; 1663 case AMDGPU_PL_OA: 1664 placement = "OA"; 1665 break; 1666 case AMDGPU_PL_PREEMPT: 1667 placement = "PREEMPTIBLE"; 1668 break; 1669 case AMDGPU_PL_DOORBELL: 1670 placement = "DOORBELL"; 1671 break; 1672 case AMDGPU_PL_MMIO_REMAP: 1673 placement = "MMIO REMAP"; 1674 break; 1675 case AMDGPU_PL_NPA: 1676 placement = "NPA"; 1677 break; 1678 case TTM_PL_SYSTEM: 1679 default: 1680 placement = "CPU"; 1681 break; 1682 } 1683 } 1684 dma_resv_unlock(bo->tbo.base.resv); 1685 } else { 1686 placement = "UNKNOWN"; 1687 } 1688 1689 size = amdgpu_bo_size(bo); 1690 seq_printf(m, "\t\t0x%08x: %12lld byte %s", 1691 id, size, placement); 1692 1693 pin_count = READ_ONCE(bo->tbo.pin_count); 1694 if (pin_count) 1695 seq_printf(m, " pin count %d", pin_count); 1696 1697 dma_buf = READ_ONCE(bo->tbo.base.dma_buf); 1698 attachment = READ_ONCE(bo->tbo.base.import_attach); 1699 1700 if (attachment) 1701 seq_printf(m, " imported from ino:%llu", file_inode(dma_buf->file)->i_ino); 1702 else if (dma_buf) 1703 seq_printf(m, " exported as ino:%llu", file_inode(dma_buf->file)->i_ino); 1704 1705 amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); 1706 amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS); 1707 amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC); 1708 amdgpu_bo_print_flag(m, bo, VRAM_CLEARED); 1709 amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS); 1710 amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID); 1711 amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC); 1712 /* Add the gem obj resv fence dump*/ 1713 if (dma_resv_trylock(bo->tbo.base.resv)) { 1714 dma_resv_describe(bo->tbo.base.resv, m); 1715 dma_resv_unlock(bo->tbo.base.resv); 1716 } 1717 seq_puts(m, "\n"); 1718 1719 return size; 1720 } 1721 #endif 1722