1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <linux/slab.h> 34 #include <linux/dma-buf.h> 35 #include <linux/export.h> 36 37 #include <drm/drm_drv.h> 38 #include <drm/amdgpu_drm.h> 39 #include <drm/drm_cache.h> 40 #include "amdgpu.h" 41 #include "amdgpu_trace.h" 42 #include "amdgpu_amdkfd.h" 43 #include "amdgpu_vram_mgr.h" 44 #include "amdgpu_vm.h" 45 #include "amdgpu_dma_buf.h" 46 #include "kfd_svm.h" 47 48 /** 49 * DOC: amdgpu_object 50 * 51 * This defines the interfaces to operate on an &amdgpu_bo buffer object which 52 * represents memory used by driver (VRAM, system memory, etc.). The driver 53 * provides DRM/GEM APIs to userspace. DRM/GEM APIs then use these interfaces 54 * to create/destroy/set buffer object which are then managed by the kernel TTM 55 * memory manager. 56 * The interfaces are also used internally by kernel clients, including gfx, 57 * uvd, etc. for kernel managed allocations used by the GPU. 58 * 59 */ 60 61 static void amdgpu_bo_destroy(struct ttm_buffer_object *tbo) 62 { 63 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 64 65 amdgpu_bo_kunmap(bo); 66 67 if (drm_gem_is_imported(&bo->tbo.base)) 68 drm_prime_gem_destroy(&bo->tbo.base, bo->tbo.sg); 69 drm_gem_object_release(&bo->tbo.base); 70 amdgpu_bo_unref(&bo->parent); 71 kvfree(bo); 72 } 73 74 static void amdgpu_bo_user_destroy(struct ttm_buffer_object *tbo) 75 { 76 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 77 struct amdgpu_bo_user *ubo; 78 79 ubo = to_amdgpu_bo_user(bo); 80 kfree(ubo->metadata); 81 amdgpu_bo_destroy(tbo); 82 } 83 84 /** 85 * amdgpu_bo_is_amdgpu_bo - check if the buffer object is an &amdgpu_bo 86 * @bo: buffer object to be checked 87 * 88 * Uses destroy function associated with the object to determine if this is 89 * an &amdgpu_bo. 90 * 91 * Returns: 92 * true if the object belongs to &amdgpu_bo, false if not. 93 */ 94 bool amdgpu_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) 95 { 96 if (bo->destroy == &amdgpu_bo_destroy || 97 bo->destroy == &amdgpu_bo_user_destroy || 98 bo->destroy == &svm_range_bo_destroy) 99 return true; 100 101 return false; 102 } 103 104 /** 105 * amdgpu_bo_placement_from_domain - set buffer's placement 106 * @abo: &amdgpu_bo buffer object whose placement is to be set 107 * @domain: requested domain 108 * 109 * Sets buffer's placement according to requested domain and the buffer's 110 * flags. 111 */ 112 void amdgpu_bo_placement_from_domain(struct amdgpu_bo *abo, u32 domain) 113 { 114 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 115 struct ttm_placement *placement = &abo->placement; 116 struct ttm_place *places = abo->placements; 117 u64 flags = abo->flags; 118 u32 c = 0; 119 120 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 121 unsigned int visible_pfn = adev->gmc.visible_vram_size >> PAGE_SHIFT; 122 int8_t mem_id = KFD_XCP_MEM_ID(adev, abo->xcp_id); 123 124 if (adev->gmc.mem_partitions && mem_id >= 0) { 125 places[c].fpfn = adev->gmc.mem_partitions[mem_id].range.fpfn; 126 /* 127 * memory partition range lpfn is inclusive start + size - 1 128 * TTM place lpfn is exclusive start + size 129 */ 130 places[c].lpfn = adev->gmc.mem_partitions[mem_id].range.lpfn + 1; 131 } else { 132 places[c].fpfn = 0; 133 places[c].lpfn = 0; 134 } 135 places[c].mem_type = TTM_PL_VRAM; 136 places[c].flags = 0; 137 138 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 139 places[c].lpfn = min_not_zero(places[c].lpfn, visible_pfn); 140 else 141 places[c].flags |= TTM_PL_FLAG_TOPDOWN; 142 143 if (abo->tbo.type == ttm_bo_type_kernel && 144 flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) 145 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; 146 147 c++; 148 } 149 150 if (domain & AMDGPU_GEM_DOMAIN_DOORBELL) { 151 places[c].fpfn = 0; 152 places[c].lpfn = 0; 153 places[c].mem_type = AMDGPU_PL_DOORBELL; 154 places[c].flags = 0; 155 c++; 156 } 157 158 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 159 places[c].fpfn = 0; 160 places[c].lpfn = 0; 161 places[c].mem_type = 162 abo->flags & AMDGPU_GEM_CREATE_PREEMPTIBLE ? 163 AMDGPU_PL_PREEMPT : TTM_PL_TT; 164 places[c].flags = 0; 165 /* 166 * When GTT is just an alternative to VRAM make sure that we 167 * only use it as fallback and still try to fill up VRAM first. 168 */ 169 if (abo->tbo.resource && !(adev->flags & AMD_IS_APU) && 170 domain & abo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) 171 places[c].flags |= TTM_PL_FLAG_FALLBACK; 172 c++; 173 } 174 175 if (domain & AMDGPU_GEM_DOMAIN_CPU) { 176 places[c].fpfn = 0; 177 places[c].lpfn = 0; 178 places[c].mem_type = TTM_PL_SYSTEM; 179 places[c].flags = 0; 180 c++; 181 } 182 183 if (domain & AMDGPU_GEM_DOMAIN_GDS) { 184 places[c].fpfn = 0; 185 places[c].lpfn = 0; 186 places[c].mem_type = AMDGPU_PL_GDS; 187 places[c].flags = 0; 188 c++; 189 } 190 191 if (domain & AMDGPU_GEM_DOMAIN_GWS) { 192 places[c].fpfn = 0; 193 places[c].lpfn = 0; 194 places[c].mem_type = AMDGPU_PL_GWS; 195 places[c].flags = 0; 196 c++; 197 } 198 199 if (domain & AMDGPU_GEM_DOMAIN_OA) { 200 places[c].fpfn = 0; 201 places[c].lpfn = 0; 202 places[c].mem_type = AMDGPU_PL_OA; 203 places[c].flags = 0; 204 c++; 205 } 206 207 if (domain & AMDGPU_GEM_DOMAIN_NPA) { 208 places[c].fpfn = 0; 209 places[c].lpfn = 0; 210 places[c].mem_type = AMDGPU_PL_NPA; 211 places[c].flags = 0; 212 c++; 213 } 214 215 if (!c) { 216 places[c].fpfn = 0; 217 places[c].lpfn = 0; 218 places[c].mem_type = TTM_PL_SYSTEM; 219 places[c].flags = 0; 220 c++; 221 } 222 223 BUG_ON(c > AMDGPU_BO_MAX_PLACEMENTS); 224 225 placement->num_placement = c; 226 placement->placement = places; 227 } 228 229 /** 230 * amdgpu_bo_create_reserved - create reserved BO for kernel use 231 * 232 * @adev: amdgpu device object 233 * @size: size for the new BO 234 * @align: alignment for the new BO 235 * @domain: where to place it 236 * @bo_ptr: used to initialize BOs in structures 237 * @gpu_addr: GPU addr of the pinned BO 238 * @cpu_addr: optional CPU address mapping 239 * 240 * Allocates and pins a BO for kernel internal use, and returns it still 241 * reserved. 242 * 243 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 244 * 245 * Returns: 246 * 0 on success, negative error code otherwise. 247 */ 248 int amdgpu_bo_create_reserved(struct amdgpu_device *adev, 249 unsigned long size, int align, 250 u32 domain, struct amdgpu_bo **bo_ptr, 251 u64 *gpu_addr, void **cpu_addr) 252 { 253 struct amdgpu_bo_param bp; 254 bool free = false; 255 int r; 256 257 if (!size) { 258 amdgpu_bo_unref(bo_ptr); 259 return 0; 260 } 261 262 memset(&bp, 0, sizeof(bp)); 263 bp.size = size; 264 bp.byte_align = align; 265 bp.domain = domain; 266 bp.flags = cpu_addr ? AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED 267 : AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 268 bp.flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 269 bp.type = ttm_bo_type_kernel; 270 bp.resv = NULL; 271 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 272 273 if (!*bo_ptr) { 274 r = amdgpu_bo_create(adev, &bp, bo_ptr); 275 if (r) { 276 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", 277 r); 278 return r; 279 } 280 free = true; 281 } 282 283 r = amdgpu_bo_reserve(*bo_ptr, false); 284 if (r) { 285 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); 286 goto error_free; 287 } 288 289 if (free) { 290 r = amdgpu_bo_pin(*bo_ptr, domain); 291 if (r) { 292 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); 293 goto error_unreserve; 294 } 295 } 296 297 r = amdgpu_ttm_alloc_gart(&(*bo_ptr)->tbo); 298 if (r) { 299 dev_err(adev->dev, "%p bind failed\n", *bo_ptr); 300 goto error_unpin; 301 } 302 303 if (gpu_addr) 304 *gpu_addr = amdgpu_bo_gpu_offset(*bo_ptr); 305 306 if (cpu_addr) { 307 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 308 if (r) { 309 dev_err(adev->dev, "(%d) kernel bo map failed\n", r); 310 goto error_unpin; 311 } 312 } 313 314 return 0; 315 316 error_unpin: 317 if (free) 318 amdgpu_bo_unpin(*bo_ptr); 319 error_unreserve: 320 amdgpu_bo_unreserve(*bo_ptr); 321 322 error_free: 323 if (free) 324 amdgpu_bo_unref(bo_ptr); 325 326 return r; 327 } 328 329 /** 330 * amdgpu_bo_create_kernel - create BO for kernel use 331 * 332 * @adev: amdgpu device object 333 * @size: size for the new BO 334 * @align: alignment for the new BO 335 * @domain: where to place it 336 * @bo_ptr: used to initialize BOs in structures 337 * @gpu_addr: GPU addr of the pinned BO 338 * @cpu_addr: optional CPU address mapping 339 * 340 * Allocates and pins a BO for kernel internal use. 341 * 342 * This function is exported to allow the V4L2 isp device 343 * external to drm device to create and access the kernel BO. 344 * 345 * Note: For bo_ptr new BO is only created if bo_ptr points to NULL. 346 * 347 * Returns: 348 * 0 on success, negative error code otherwise. 349 */ 350 int amdgpu_bo_create_kernel(struct amdgpu_device *adev, 351 unsigned long size, int align, 352 u32 domain, struct amdgpu_bo **bo_ptr, 353 u64 *gpu_addr, void **cpu_addr) 354 { 355 int r; 356 357 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, 358 gpu_addr, cpu_addr); 359 360 if (r) 361 return r; 362 363 if (*bo_ptr) 364 amdgpu_bo_unreserve(*bo_ptr); 365 366 return 0; 367 } 368 369 /** 370 * amdgpu_bo_create_isp_user - create user BO for isp 371 * 372 * @adev: amdgpu device object 373 * @dma_buf: DMABUF handle for isp buffer 374 * @domain: where to place it 375 * @bo: used to initialize BOs in structures 376 * @gpu_addr: GPU addr of the pinned BO 377 * 378 * Imports isp DMABUF to allocate and pin a user BO for isp internal use. It does 379 * GART alloc to generate gpu_addr for BO to make it accessible through the 380 * GART aperture for ISP HW. 381 * 382 * This function is exported to allow the V4L2 isp device external to drm device 383 * to create and access the isp user BO. 384 * 385 * Returns: 386 * 0 on success, negative error code otherwise. 387 */ 388 int amdgpu_bo_create_isp_user(struct amdgpu_device *adev, 389 struct dma_buf *dma_buf, u32 domain, struct amdgpu_bo **bo, 390 u64 *gpu_addr) 391 392 { 393 struct drm_gem_object *gem_obj; 394 int r; 395 396 gem_obj = amdgpu_gem_prime_import(&adev->ddev, dma_buf); 397 *bo = gem_to_amdgpu_bo(gem_obj); 398 if (!(*bo)) { 399 dev_err(adev->dev, "failed to get valid isp user bo\n"); 400 return -EINVAL; 401 } 402 403 r = amdgpu_bo_reserve(*bo, false); 404 if (r) { 405 dev_err(adev->dev, "(%d) failed to reserve isp user bo\n", r); 406 return r; 407 } 408 409 r = amdgpu_bo_pin(*bo, domain); 410 if (r) { 411 dev_err(adev->dev, "(%d) isp user bo pin failed\n", r); 412 goto error_unreserve; 413 } 414 415 r = amdgpu_ttm_alloc_gart(&(*bo)->tbo); 416 if (r) { 417 dev_err(adev->dev, "%p bind failed\n", *bo); 418 goto error_unpin; 419 } 420 421 if (!WARN_ON(!gpu_addr)) 422 *gpu_addr = amdgpu_bo_gpu_offset(*bo); 423 424 amdgpu_bo_unreserve(*bo); 425 426 return 0; 427 428 error_unpin: 429 amdgpu_bo_unpin(*bo); 430 error_unreserve: 431 amdgpu_bo_unreserve(*bo); 432 amdgpu_bo_unref(bo); 433 434 return r; 435 } 436 437 /** 438 * amdgpu_bo_create_kernel_at - create BO for kernel use at specific location 439 * 440 * @adev: amdgpu device object 441 * @offset: offset of the BO 442 * @size: size of the BO 443 * @bo_ptr: used to initialize BOs in structures 444 * @cpu_addr: optional CPU address mapping 445 * 446 * Creates a kernel BO at a specific offset in VRAM. 447 * 448 * Returns: 449 * 0 on success, negative error code otherwise. 450 */ 451 int amdgpu_bo_create_kernel_at(struct amdgpu_device *adev, 452 uint64_t offset, uint64_t size, 453 struct amdgpu_bo **bo_ptr, void **cpu_addr) 454 { 455 struct ttm_operation_ctx ctx = { false, false }; 456 unsigned int i; 457 int r; 458 459 offset &= PAGE_MASK; 460 size = ALIGN(size, PAGE_SIZE); 461 462 r = amdgpu_bo_create_reserved(adev, size, PAGE_SIZE, 463 AMDGPU_GEM_DOMAIN_VRAM, bo_ptr, NULL, 464 cpu_addr); 465 if (r) 466 return r; 467 468 if ((*bo_ptr) == NULL) 469 return 0; 470 471 /* 472 * Remove the original mem node and create a new one at the request 473 * position. 474 */ 475 if (cpu_addr) 476 amdgpu_bo_kunmap(*bo_ptr); 477 478 ttm_resource_free(&(*bo_ptr)->tbo, &(*bo_ptr)->tbo.resource); 479 480 for (i = 0; i < (*bo_ptr)->placement.num_placement; ++i) { 481 (*bo_ptr)->placements[i].fpfn = offset >> PAGE_SHIFT; 482 (*bo_ptr)->placements[i].lpfn = (offset + size) >> PAGE_SHIFT; 483 } 484 r = ttm_bo_mem_space(&(*bo_ptr)->tbo, &(*bo_ptr)->placement, 485 &(*bo_ptr)->tbo.resource, &ctx); 486 if (r) 487 goto error; 488 489 if (cpu_addr) { 490 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 491 if (r) 492 goto error; 493 } 494 495 amdgpu_bo_unreserve(*bo_ptr); 496 return 0; 497 498 error: 499 amdgpu_bo_unreserve(*bo_ptr); 500 amdgpu_bo_unref(bo_ptr); 501 return r; 502 } 503 504 /** 505 * amdgpu_bo_free_kernel - free BO for kernel use 506 * 507 * @bo: amdgpu BO to free 508 * @gpu_addr: pointer to where the BO's GPU memory space address was stored 509 * @cpu_addr: pointer to where the BO's CPU memory space address was stored 510 * 511 * unmaps and unpin a BO for kernel internal use. 512 * 513 * This function is exported to allow the V4L2 isp device 514 * external to drm device to free the kernel BO. 515 */ 516 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, 517 void **cpu_addr) 518 { 519 if (*bo == NULL) 520 return; 521 522 WARN_ON(amdgpu_ttm_adev((*bo)->tbo.bdev)->in_suspend); 523 524 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { 525 if (cpu_addr) 526 amdgpu_bo_kunmap(*bo); 527 528 amdgpu_bo_unpin(*bo); 529 amdgpu_bo_unreserve(*bo); 530 } 531 amdgpu_bo_unref(bo); 532 533 if (gpu_addr) 534 *gpu_addr = 0; 535 536 if (cpu_addr) 537 *cpu_addr = NULL; 538 } 539 540 /** 541 * amdgpu_bo_free_isp_user - free BO for isp use 542 * 543 * @bo: amdgpu isp user BO to free 544 * 545 * unpin and unref BO for isp internal use. 546 * 547 * This function is exported to allow the V4L2 isp device 548 * external to drm device to free the isp user BO. 549 */ 550 void amdgpu_bo_free_isp_user(struct amdgpu_bo *bo) 551 { 552 if (bo == NULL) 553 return; 554 555 if (amdgpu_bo_reserve(bo, true) == 0) { 556 amdgpu_bo_unpin(bo); 557 amdgpu_bo_unreserve(bo); 558 } 559 amdgpu_bo_unref(&bo); 560 } 561 562 /* Validate bo size is bit bigger than the request domain */ 563 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, 564 unsigned long size, u32 domain) 565 { 566 struct ttm_resource_manager *man = NULL; 567 568 /* 569 * If GTT is part of requested domains the check must succeed to 570 * allow fall back to GTT. 571 */ 572 if (domain & AMDGPU_GEM_DOMAIN_GTT) 573 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT); 574 else if (domain & AMDGPU_GEM_DOMAIN_VRAM) 575 man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM); 576 else 577 return true; 578 579 if (!man) { 580 if (domain & AMDGPU_GEM_DOMAIN_GTT) 581 WARN_ON_ONCE("GTT domain requested but GTT mem manager uninitialized"); 582 return false; 583 } 584 585 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU, _DOMAIN_DOORBELL */ 586 if (size < man->size) 587 return true; 588 589 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, man->size); 590 return false; 591 } 592 593 bool amdgpu_bo_support_uswc(u64 bo_flags) 594 { 595 596 #ifdef CONFIG_X86_32 597 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit 598 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 599 */ 600 return false; 601 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) 602 /* Don't try to enable write-combining when it can't work, or things 603 * may be slow 604 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 605 */ 606 607 #ifndef CONFIG_COMPILE_TEST 608 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ 609 thanks to write-combining 610 #endif 611 612 if (bo_flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 613 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " 614 "better performance thanks to write-combining\n"); 615 return false; 616 #else 617 /* For architectures that don't support WC memory, 618 * mask out the WC flag from the BO 619 */ 620 if (!drm_arch_can_wc_memory()) 621 return false; 622 623 return true; 624 #endif 625 } 626 627 /** 628 * amdgpu_bo_create - create an &amdgpu_bo buffer object 629 * @adev: amdgpu device object 630 * @bp: parameters to be used for the buffer object 631 * @bo_ptr: pointer to the buffer object pointer 632 * 633 * Creates an &amdgpu_bo buffer object. 634 * 635 * Returns: 636 * 0 for success or a negative error code on failure. 637 */ 638 int amdgpu_bo_create(struct amdgpu_device *adev, 639 struct amdgpu_bo_param *bp, 640 struct amdgpu_bo **bo_ptr) 641 { 642 struct ttm_operation_ctx ctx = { 643 .interruptible = (bp->type != ttm_bo_type_kernel), 644 .no_wait_gpu = bp->no_wait_gpu, 645 /* We opt to avoid OOM on system pages allocations */ 646 .gfp_retry_mayfail = true, 647 .allow_res_evict = bp->type != ttm_bo_type_kernel, 648 .resv = bp->resv 649 }; 650 struct amdgpu_bo *bo; 651 unsigned long page_align, size = bp->size; 652 int r; 653 654 /* Note that GDS/GWS/OA allocates 1 page per byte/resource. */ 655 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { 656 /* GWS and OA don't need any alignment. */ 657 page_align = bp->byte_align; 658 size <<= PAGE_SHIFT; 659 660 } else if (bp->domain & AMDGPU_GEM_DOMAIN_GDS) { 661 /* Both size and alignment must be a multiple of 4. */ 662 page_align = ALIGN(bp->byte_align, 4); 663 size = ALIGN(size, 4) << PAGE_SHIFT; 664 } else { 665 /* Memory should be aligned at least to a page size. */ 666 page_align = ALIGN(bp->byte_align, PAGE_SIZE) >> PAGE_SHIFT; 667 size = ALIGN(size, PAGE_SIZE); 668 } 669 670 if (!amdgpu_bo_validate_size(adev, size, bp->domain)) 671 return -ENOMEM; 672 673 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo)); 674 675 *bo_ptr = NULL; 676 bo = kvzalloc(bp->bo_ptr_size, GFP_KERNEL); 677 if (bo == NULL) 678 return -ENOMEM; 679 drm_gem_private_object_init(adev_to_drm(adev), &bo->tbo.base, size); 680 bo->tbo.base.funcs = &amdgpu_gem_object_funcs; 681 bo->vm_bo = NULL; 682 bo->preferred_domains = bp->preferred_domain ? bp->preferred_domain : 683 bp->domain; 684 bo->allowed_domains = bo->preferred_domains; 685 if (bp->type != ttm_bo_type_kernel && 686 !(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE) && 687 bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) 688 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 689 690 bo->flags = bp->flags; 691 692 if (adev->gmc.mem_partitions) 693 /* For GPUs with spatial partitioning, bo->xcp_id=-1 means any partition */ 694 bo->xcp_id = bp->xcp_id_plus1 - 1; 695 else 696 /* For GPUs without spatial partitioning */ 697 bo->xcp_id = 0; 698 699 if (!amdgpu_bo_support_uswc(bo->flags)) 700 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 701 702 bo->tbo.bdev = &adev->mman.bdev; 703 if (bp->domain & (AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA | 704 AMDGPU_GEM_DOMAIN_GDS)) 705 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 706 else 707 amdgpu_bo_placement_from_domain(bo, bp->domain); 708 if (bp->type == ttm_bo_type_kernel) 709 bo->tbo.priority = 2; 710 else if (!(bp->flags & AMDGPU_GEM_CREATE_DISCARDABLE)) 711 bo->tbo.priority = 1; 712 713 if (!bp->destroy) 714 bp->destroy = &amdgpu_bo_destroy; 715 716 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type, 717 &bo->placement, page_align, &ctx, NULL, 718 bp->resv, bp->destroy); 719 if (unlikely(r != 0)) 720 return r; 721 722 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 723 amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 724 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 725 ctx.bytes_moved); 726 else 727 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); 728 729 if (bp->flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && 730 bo->tbo.resource->mem_type == TTM_PL_VRAM) { 731 struct dma_fence *fence; 732 733 r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev), 734 bo, bo->tbo.base.resv, &fence, 735 true, AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER); 736 if (unlikely(r)) 737 goto fail_unreserve; 738 739 if (fence) { 740 dma_resv_add_fence(bo->tbo.base.resv, fence, 741 DMA_RESV_USAGE_KERNEL); 742 dma_fence_put(fence); 743 } 744 } 745 if (!bp->resv) 746 amdgpu_bo_unreserve(bo); 747 *bo_ptr = bo; 748 749 trace_amdgpu_bo_create(bo); 750 751 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ 752 if (bp->type == ttm_bo_type_device) 753 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 754 755 return 0; 756 757 fail_unreserve: 758 if (!bp->resv) 759 dma_resv_unlock(bo->tbo.base.resv); 760 amdgpu_bo_unref(&bo); 761 return r; 762 } 763 764 /** 765 * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object 766 * @adev: amdgpu device object 767 * @bp: parameters to be used for the buffer object 768 * @ubo_ptr: pointer to the buffer object pointer 769 * 770 * Create a BO to be used by user application; 771 * 772 * Returns: 773 * 0 for success or a negative error code on failure. 774 */ 775 776 int amdgpu_bo_create_user(struct amdgpu_device *adev, 777 struct amdgpu_bo_param *bp, 778 struct amdgpu_bo_user **ubo_ptr) 779 { 780 struct amdgpu_bo *bo_ptr; 781 int r; 782 783 bp->bo_ptr_size = sizeof(struct amdgpu_bo_user); 784 bp->destroy = &amdgpu_bo_user_destroy; 785 r = amdgpu_bo_create(adev, bp, &bo_ptr); 786 if (r) 787 return r; 788 789 *ubo_ptr = to_amdgpu_bo_user(bo_ptr); 790 return r; 791 } 792 793 /** 794 * amdgpu_bo_create_vm - create an &amdgpu_bo_vm buffer object 795 * @adev: amdgpu device object 796 * @bp: parameters to be used for the buffer object 797 * @vmbo_ptr: pointer to the buffer object pointer 798 * 799 * Create a BO to be for GPUVM. 800 * 801 * Returns: 802 * 0 for success or a negative error code on failure. 803 */ 804 805 int amdgpu_bo_create_vm(struct amdgpu_device *adev, 806 struct amdgpu_bo_param *bp, 807 struct amdgpu_bo_vm **vmbo_ptr) 808 { 809 struct amdgpu_bo *bo_ptr; 810 int r; 811 812 /* bo_ptr_size will be determined by the caller and it depends on 813 * num of amdgpu_vm_pt entries. 814 */ 815 BUG_ON(bp->bo_ptr_size < sizeof(struct amdgpu_bo_vm)); 816 r = amdgpu_bo_create(adev, bp, &bo_ptr); 817 if (r) 818 return r; 819 820 *vmbo_ptr = to_amdgpu_bo_vm(bo_ptr); 821 return r; 822 } 823 824 /** 825 * amdgpu_bo_kmap - map an &amdgpu_bo buffer object 826 * @bo: &amdgpu_bo buffer object to be mapped 827 * @ptr: kernel virtual address to be returned 828 * 829 * Calls ttm_bo_kmap() to set up the kernel virtual mapping; calls 830 * amdgpu_bo_kptr() to get the kernel virtual address. 831 * 832 * Returns: 833 * 0 for success or a negative error code on failure. 834 */ 835 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) 836 { 837 void *kptr; 838 long r; 839 840 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 841 return -EPERM; 842 843 r = dma_resv_wait_timeout(bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL, 844 false, MAX_SCHEDULE_TIMEOUT); 845 if (r < 0) 846 return r; 847 848 kptr = amdgpu_bo_kptr(bo); 849 if (kptr) { 850 if (ptr) 851 *ptr = kptr; 852 return 0; 853 } 854 855 r = ttm_bo_kmap(&bo->tbo, 0, PFN_UP(bo->tbo.base.size), &bo->kmap); 856 if (r) 857 return r; 858 859 if (ptr) 860 *ptr = amdgpu_bo_kptr(bo); 861 862 return 0; 863 } 864 865 /** 866 * amdgpu_bo_kptr - returns a kernel virtual address of the buffer object 867 * @bo: &amdgpu_bo buffer object 868 * 869 * Calls ttm_kmap_obj_virtual() to get the kernel virtual address 870 * 871 * Returns: 872 * the virtual address of a buffer object area. 873 */ 874 void *amdgpu_bo_kptr(struct amdgpu_bo *bo) 875 { 876 bool is_iomem; 877 878 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 879 } 880 881 /** 882 * amdgpu_bo_kunmap - unmap an &amdgpu_bo buffer object 883 * @bo: &amdgpu_bo buffer object to be unmapped 884 * 885 * Unmaps a kernel map set up by amdgpu_bo_kmap(). 886 */ 887 void amdgpu_bo_kunmap(struct amdgpu_bo *bo) 888 { 889 if (bo->kmap.bo) 890 ttm_bo_kunmap(&bo->kmap); 891 } 892 893 /** 894 * amdgpu_bo_ref - reference an &amdgpu_bo buffer object 895 * @bo: &amdgpu_bo buffer object 896 * 897 * References the contained &ttm_buffer_object. 898 * 899 * Returns: 900 * a refcounted pointer to the &amdgpu_bo buffer object. 901 */ 902 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) 903 { 904 if (bo == NULL) 905 return NULL; 906 907 drm_gem_object_get(&bo->tbo.base); 908 return bo; 909 } 910 911 /** 912 * amdgpu_bo_unref - unreference an &amdgpu_bo buffer object 913 * @bo: &amdgpu_bo buffer object 914 * 915 * Unreferences the contained &ttm_buffer_object and clear the pointer 916 */ 917 void amdgpu_bo_unref(struct amdgpu_bo **bo) 918 { 919 if ((*bo) == NULL) 920 return; 921 922 drm_gem_object_put(&(*bo)->tbo.base); 923 *bo = NULL; 924 } 925 926 /* Map a BO onto vram_pin_size_xcp[]. bo->xcp_id < 0 means "any partition": 927 * such pins are not owned by a single partition and go to 928 * vram_pin_size_common, which every partition subtracts. 929 */ 930 static int amdgpu_bo_get_xcp(struct amdgpu_bo *bo) 931 { 932 s8 xcp = bo->xcp_id; 933 934 if (xcp >= 0 && xcp < MAX_XCP) 935 return xcp; 936 return -1; 937 } 938 939 /** 940 * amdgpu_bo_pin - pin an &amdgpu_bo buffer object 941 * @bo: &amdgpu_bo buffer object to be pinned 942 * @domain: domain to be pinned to 943 * 944 * Pins the buffer object according to requested domain. If the memory is 945 * unbound gart memory, binds the pages into gart table. Adjusts pin_count and 946 * pin_size accordingly. 947 * 948 * Pinning means to lock pages in memory along with keeping them at a fixed 949 * offset. It is required when a buffer can not be moved, for example, when 950 * a display buffer is being scanned out. 951 * 952 * Returns: 953 * 0 for success or a negative error code on failure. 954 */ 955 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain) 956 { 957 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 958 struct ttm_operation_ctx ctx = { false, false }; 959 int r, i; 960 961 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) 962 return -EPERM; 963 964 /* Check domain to be pinned to against preferred domains */ 965 if (bo->preferred_domains & domain) 966 domain = bo->preferred_domains & domain; 967 968 /* A shared bo cannot be migrated to VRAM */ 969 if (drm_gem_is_imported(&bo->tbo.base)) { 970 if (domain & AMDGPU_GEM_DOMAIN_GTT) 971 domain = AMDGPU_GEM_DOMAIN_GTT; 972 else 973 return -EINVAL; 974 } 975 976 if (bo->tbo.pin_count) { 977 uint32_t mem_type = bo->tbo.resource->mem_type; 978 uint32_t mem_flags = bo->tbo.resource->placement; 979 980 if (!(domain & amdgpu_mem_type_to_domain(mem_type))) 981 return -EINVAL; 982 983 if ((mem_type == TTM_PL_VRAM) && 984 (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) && 985 !(mem_flags & TTM_PL_FLAG_CONTIGUOUS)) 986 return -EINVAL; 987 988 ttm_bo_pin(&bo->tbo); 989 return 0; 990 } 991 992 /* This assumes only APU display buffers are pinned with (VRAM|GTT). 993 * See function amdgpu_display_supported_domains() 994 */ 995 domain = amdgpu_bo_get_preferred_domain(adev, domain); 996 997 if (drm_gem_is_imported(&bo->tbo.base)) 998 dma_buf_pin(bo->tbo.base.import_attach); 999 1000 /* force to pin into visible video ram */ 1001 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) 1002 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 1003 amdgpu_bo_placement_from_domain(bo, domain); 1004 for (i = 0; i < bo->placement.num_placement; i++) { 1005 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS && 1006 bo->placements[i].mem_type == TTM_PL_VRAM) 1007 bo->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS; 1008 } 1009 1010 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 1011 if (unlikely(r)) { 1012 dev_err(adev->dev, "%p pin failed\n", bo); 1013 goto error; 1014 } 1015 1016 ttm_bo_pin(&bo->tbo); 1017 1018 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 1019 int xcp = amdgpu_bo_get_xcp(bo); 1020 1021 atomic64_add(amdgpu_bo_size(bo), &adev->vram_pin_size); 1022 if (xcp >= 0) { 1023 atomic64_add(amdgpu_bo_size(bo), 1024 &adev->vram_pin_size_xcp[xcp]); 1025 } else { 1026 atomic64_add(amdgpu_bo_size(bo), 1027 &adev->vram_pin_size_common); 1028 } 1029 amdgpu_amdkfd_account_vram_pin(bo, xcp); 1030 atomic64_add(amdgpu_vram_mgr_bo_visible_size(bo), 1031 &adev->visible_pin_size); 1032 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 1033 atomic64_add(amdgpu_bo_size(bo), &adev->gart_pin_size); 1034 } 1035 1036 error: 1037 return r; 1038 } 1039 1040 /** 1041 * amdgpu_bo_unpin - unpin an &amdgpu_bo buffer object 1042 * @bo: &amdgpu_bo buffer object to be unpinned 1043 * 1044 * Decreases the pin_count, and clears the flags if pin_count reaches 0. 1045 * Changes placement and pin size accordingly. 1046 * 1047 * Returns: 1048 * 0 for success or a negative error code on failure. 1049 */ 1050 void amdgpu_bo_unpin(struct amdgpu_bo *bo) 1051 { 1052 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1053 1054 ttm_bo_unpin(&bo->tbo); 1055 if (bo->tbo.pin_count) 1056 return; 1057 1058 if (drm_gem_is_imported(&bo->tbo.base)) 1059 dma_buf_unpin(bo->tbo.base.import_attach); 1060 1061 if (bo->tbo.resource->mem_type == TTM_PL_VRAM) { 1062 int xcp = amdgpu_bo_get_xcp(bo); 1063 1064 atomic64_sub(amdgpu_bo_size(bo), &adev->vram_pin_size); 1065 if (xcp >= 0) { 1066 atomic64_sub(amdgpu_bo_size(bo), 1067 &adev->vram_pin_size_xcp[xcp]); 1068 } else { 1069 atomic64_sub(amdgpu_bo_size(bo), 1070 &adev->vram_pin_size_common); 1071 } 1072 amdgpu_amdkfd_account_vram_unpin(bo, xcp); 1073 atomic64_sub(amdgpu_vram_mgr_bo_visible_size(bo), 1074 &adev->visible_pin_size); 1075 } else if (bo->tbo.resource->mem_type == TTM_PL_TT) { 1076 atomic64_sub(amdgpu_bo_size(bo), &adev->gart_pin_size); 1077 } 1078 1079 } 1080 1081 static const char * const amdgpu_vram_names[] = { 1082 "UNKNOWN", 1083 "GDDR1", 1084 "DDR2", 1085 "GDDR3", 1086 "GDDR4", 1087 "GDDR5", 1088 "HBM", 1089 "DDR3", 1090 "DDR4", 1091 "GDDR6", 1092 "DDR5", 1093 "LPDDR4", 1094 "LPDDR5", 1095 "HBM3E", 1096 "HBM4", 1097 "GDDR7" 1098 }; 1099 1100 /** 1101 * amdgpu_bo_init - initialize memory manager 1102 * @adev: amdgpu device object 1103 * 1104 * Calls amdgpu_ttm_init() to initialize amdgpu memory manager. 1105 * 1106 * Returns: 1107 * 0 for success or a negative error code on failure. 1108 */ 1109 int amdgpu_bo_init(struct amdgpu_device *adev) 1110 { 1111 /* On A+A platform, VRAM can be mapped as WB */ 1112 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 1113 /* reserve PAT memory space to WC for VRAM */ 1114 int r = arch_io_reserve_memtype_wc(adev->gmc.aper_base, 1115 adev->gmc.aper_size); 1116 1117 if (r) { 1118 DRM_ERROR("Unable to set WC memtype for the aperture base\n"); 1119 return r; 1120 } 1121 1122 /* Add an MTRR for the VRAM */ 1123 adev->gmc.vram_mtrr = arch_phys_wc_add(adev->gmc.aper_base, 1124 adev->gmc.aper_size); 1125 } 1126 1127 drm_info(adev_to_drm(adev), "Detected VRAM RAM=%lluM, BAR=%lluM\n", 1128 adev->gmc.mc_vram_size >> 20, 1129 (unsigned long long)adev->gmc.aper_size >> 20); 1130 drm_info(adev_to_drm(adev), "RAM width %dbits %s\n", 1131 adev->gmc.vram_width, amdgpu_vram_names[adev->gmc.vram_type]); 1132 return amdgpu_ttm_init(adev); 1133 } 1134 1135 /** 1136 * amdgpu_bo_fini - tear down memory manager 1137 * @adev: amdgpu device object 1138 * 1139 * Reverses amdgpu_bo_init() to tear down memory manager. 1140 */ 1141 void amdgpu_bo_fini(struct amdgpu_device *adev) 1142 { 1143 int idx; 1144 1145 amdgpu_ttm_fini(adev); 1146 1147 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 1148 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) { 1149 arch_phys_wc_del(adev->gmc.vram_mtrr); 1150 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 1151 } 1152 drm_dev_exit(idx); 1153 } 1154 } 1155 1156 /** 1157 * amdgpu_bo_set_tiling_flags - set tiling flags 1158 * @bo: &amdgpu_bo buffer object 1159 * @tiling_flags: new flags 1160 * 1161 * Sets buffer object's tiling flags with the new one. Used by GEM ioctl or 1162 * kernel driver to set the tiling flags on a buffer. 1163 * 1164 * Returns: 1165 * 0 for success or a negative error code on failure. 1166 */ 1167 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) 1168 { 1169 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1170 struct amdgpu_bo_user *ubo; 1171 1172 /* MMIO_REMAP is BAR I/O space; tiling should never be used here. */ 1173 WARN_ON_ONCE(bo->tbo.resource && 1174 bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP); 1175 1176 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1177 if (adev->family <= AMDGPU_FAMILY_CZ && 1178 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) 1179 return -EINVAL; 1180 1181 ubo = to_amdgpu_bo_user(bo); 1182 ubo->tiling_flags = tiling_flags; 1183 return 0; 1184 } 1185 1186 /** 1187 * amdgpu_bo_get_tiling_flags - get tiling flags 1188 * @bo: &amdgpu_bo buffer object 1189 * @tiling_flags: returned flags 1190 * 1191 * Gets buffer object's tiling flags. Used by GEM ioctl or kernel driver to 1192 * set the tiling flags on a buffer. 1193 */ 1194 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) 1195 { 1196 struct amdgpu_bo_user *ubo; 1197 1198 /* 1199 * MMIO_REMAP BOs are not real VRAM/GTT memory but a fixed BAR I/O window. 1200 * They should never go through GEM tiling helpers. 1201 */ 1202 WARN_ON_ONCE(bo->tbo.resource && 1203 bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP); 1204 1205 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1206 dma_resv_assert_held(bo->tbo.base.resv); 1207 ubo = to_amdgpu_bo_user(bo); 1208 1209 if (tiling_flags) 1210 *tiling_flags = ubo->tiling_flags; 1211 } 1212 1213 /** 1214 * amdgpu_bo_set_metadata - set metadata 1215 * @bo: &amdgpu_bo buffer object 1216 * @metadata: new metadata 1217 * @metadata_size: size of the new metadata 1218 * @flags: flags of the new metadata 1219 * 1220 * Sets buffer object's metadata, its size and flags. 1221 * Used via GEM ioctl. 1222 * 1223 * Returns: 1224 * 0 for success or a negative error code on failure. 1225 */ 1226 int amdgpu_bo_set_metadata(struct amdgpu_bo *bo, void *metadata, 1227 u32 metadata_size, uint64_t flags) 1228 { 1229 struct amdgpu_bo_user *ubo; 1230 void *buffer; 1231 1232 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1233 ubo = to_amdgpu_bo_user(bo); 1234 if (!metadata_size) { 1235 if (ubo->metadata_size) { 1236 kfree(ubo->metadata); 1237 ubo->metadata = NULL; 1238 ubo->metadata_size = 0; 1239 } 1240 return 0; 1241 } 1242 1243 if (metadata == NULL) 1244 return -EINVAL; 1245 1246 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); 1247 if (buffer == NULL) 1248 return -ENOMEM; 1249 1250 kfree(ubo->metadata); 1251 ubo->metadata_flags = flags; 1252 ubo->metadata = buffer; 1253 ubo->metadata_size = metadata_size; 1254 1255 return 0; 1256 } 1257 1258 /** 1259 * amdgpu_bo_get_metadata - get metadata 1260 * @bo: &amdgpu_bo buffer object 1261 * @buffer: returned metadata 1262 * @buffer_size: size of the buffer 1263 * @metadata_size: size of the returned metadata 1264 * @flags: flags of the returned metadata 1265 * 1266 * Gets buffer object's metadata, its size and flags. buffer_size shall not be 1267 * less than metadata_size. 1268 * Used via GEM ioctl. 1269 * 1270 * Returns: 1271 * 0 for success or a negative error code on failure. 1272 */ 1273 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, 1274 size_t buffer_size, uint32_t *metadata_size, 1275 uint64_t *flags) 1276 { 1277 struct amdgpu_bo_user *ubo; 1278 1279 if (!buffer && !metadata_size) 1280 return -EINVAL; 1281 1282 BUG_ON(bo->tbo.type == ttm_bo_type_kernel); 1283 ubo = to_amdgpu_bo_user(bo); 1284 if (metadata_size) 1285 *metadata_size = ubo->metadata_size; 1286 1287 if (buffer) { 1288 if (buffer_size < ubo->metadata_size) 1289 return -EINVAL; 1290 1291 if (ubo->metadata_size) 1292 memcpy(buffer, ubo->metadata, ubo->metadata_size); 1293 } 1294 1295 if (flags) 1296 *flags = ubo->metadata_flags; 1297 1298 return 0; 1299 } 1300 1301 /** 1302 * amdgpu_bo_move_notify - notification about a memory move 1303 * @bo: pointer to a buffer object 1304 * @evict: if this move is evicting the buffer from the graphics address space 1305 * @new_mem: new resource for backing the BO 1306 * 1307 * Marks the corresponding &amdgpu_bo buffer object as invalid, also performs 1308 * bookkeeping. 1309 * TTM driver callback which is called when ttm moves a buffer. 1310 */ 1311 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, 1312 bool evict, 1313 struct ttm_resource *new_mem) 1314 { 1315 struct ttm_resource *old_mem = bo->resource; 1316 struct amdgpu_bo *abo; 1317 1318 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1319 return; 1320 1321 abo = ttm_to_amdgpu_bo(bo); 1322 amdgpu_vm_bo_move(abo, new_mem, evict); 1323 1324 amdgpu_bo_kunmap(abo); 1325 1326 if (abo->tbo.base.dma_buf && !drm_gem_is_imported(&abo->tbo.base) && 1327 old_mem && old_mem->mem_type != TTM_PL_SYSTEM) 1328 dma_buf_invalidate_mappings(abo->tbo.base.dma_buf); 1329 1330 /* move_notify is called before move happens */ 1331 trace_amdgpu_bo_move(abo, new_mem ? new_mem->mem_type : -1, 1332 old_mem ? old_mem->mem_type : -1); 1333 } 1334 1335 /** 1336 * amdgpu_bo_release_notify - notification about a BO being released 1337 * @bo: pointer to a buffer object 1338 * 1339 * Wipes VRAM buffers whose contents should not be leaked before the 1340 * memory is released. 1341 */ 1342 void amdgpu_bo_release_notify(struct ttm_buffer_object *bo) 1343 { 1344 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1345 struct dma_fence *fence = NULL; 1346 struct amdgpu_bo *abo; 1347 int r; 1348 1349 if (!amdgpu_bo_is_amdgpu_bo(bo)) 1350 return; 1351 1352 abo = ttm_to_amdgpu_bo(bo); 1353 1354 WARN_ON(abo->vm_bo); 1355 1356 if (abo->kfd_bo) 1357 amdgpu_amdkfd_release_notify(abo); 1358 1359 /* 1360 * We lock the private dma_resv object here and since the BO is about to 1361 * be released nobody else should have a pointer to it. 1362 * So when this locking here fails something is wrong with the reference 1363 * counting. 1364 */ 1365 if (WARN_ON_ONCE(!dma_resv_trylock(&bo->base._resv))) 1366 return; 1367 1368 amdgpu_amdkfd_remove_all_eviction_fences(abo); 1369 1370 if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM || 1371 !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) || 1372 adev->in_suspend || drm_dev_is_unplugged(adev_to_drm(adev))) 1373 goto out; 1374 1375 r = dma_resv_reserve_fences(&bo->base._resv, 1); 1376 if (r) 1377 goto out; 1378 1379 r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev), 1380 abo, &bo->base._resv, &fence, 1381 false, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE); 1382 if (WARN_ON(r)) 1383 goto out; 1384 1385 amdgpu_vram_mgr_set_cleared(bo->resource); 1386 dma_resv_add_fence(&bo->base._resv, fence, DMA_RESV_USAGE_KERNEL); 1387 dma_fence_put(fence); 1388 1389 out: 1390 dma_resv_unlock(&bo->base._resv); 1391 } 1392 1393 /** 1394 * amdgpu_bo_fault_reserve_notify - notification about a memory fault 1395 * @bo: pointer to a buffer object 1396 * 1397 * Notifies the driver we are taking a fault on this BO and have reserved it, 1398 * also performs bookkeeping. 1399 * TTM driver callback for dealing with vm faults. 1400 * 1401 * Returns: 1402 * 0 for success or a negative error code on failure. 1403 */ 1404 vm_fault_t amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 1405 { 1406 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 1407 struct ttm_operation_ctx ctx = { false, false }; 1408 struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo); 1409 int r; 1410 1411 /* Remember that this BO was accessed by the CPU */ 1412 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 1413 1414 if (amdgpu_res_cpu_visible(adev, bo->resource)) 1415 return 0; 1416 1417 /* Can't move a pinned BO to visible VRAM */ 1418 if (abo->tbo.pin_count > 0) 1419 return VM_FAULT_SIGBUS; 1420 1421 /* hurrah the memory is not visible ! */ 1422 atomic64_inc(&adev->num_vram_cpu_page_faults); 1423 amdgpu_bo_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 1424 AMDGPU_GEM_DOMAIN_GTT); 1425 1426 /* Avoid costly evictions; only set GTT as a busy placement */ 1427 abo->placements[0].flags |= TTM_PL_FLAG_DESIRED; 1428 1429 r = ttm_bo_validate(bo, &abo->placement, &ctx); 1430 if (unlikely(r == -EBUSY || r == -ERESTARTSYS)) 1431 return VM_FAULT_NOPAGE; 1432 else if (unlikely(r)) 1433 return VM_FAULT_SIGBUS; 1434 1435 /* this should never happen */ 1436 if (bo->resource->mem_type == TTM_PL_VRAM && 1437 !amdgpu_res_cpu_visible(adev, bo->resource)) 1438 return VM_FAULT_SIGBUS; 1439 1440 ttm_bo_move_to_lru_tail_unlocked(bo); 1441 return 0; 1442 } 1443 1444 /** 1445 * amdgpu_bo_fence - add fence to buffer object 1446 * 1447 * @bo: buffer object in question 1448 * @fence: fence to add 1449 * @shared: true if fence should be added shared 1450 * 1451 */ 1452 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, 1453 bool shared) 1454 { 1455 struct dma_resv *resv = bo->tbo.base.resv; 1456 int r; 1457 1458 r = dma_resv_reserve_fences(resv, 1); 1459 if (r) { 1460 /* As last resort on OOM we block for the fence */ 1461 dma_fence_wait(fence, false); 1462 return; 1463 } 1464 1465 dma_resv_add_fence(resv, fence, shared ? DMA_RESV_USAGE_READ : 1466 DMA_RESV_USAGE_WRITE); 1467 } 1468 1469 /** 1470 * amdgpu_bo_sync_wait_resv - Wait for BO reservation fences 1471 * 1472 * @adev: amdgpu device pointer 1473 * @resv: reservation object to sync to 1474 * @sync_mode: synchronization mode 1475 * @owner: fence owner 1476 * @intr: Whether the wait is interruptible 1477 * 1478 * Extract the fences from the reservation object and waits for them to finish. 1479 * 1480 * Returns: 1481 * 0 on success, errno otherwise. 1482 */ 1483 int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev, struct dma_resv *resv, 1484 enum amdgpu_sync_mode sync_mode, void *owner, 1485 bool intr) 1486 { 1487 struct amdgpu_sync sync; 1488 int r; 1489 1490 amdgpu_sync_create(&sync); 1491 amdgpu_sync_resv(adev, &sync, resv, sync_mode, owner); 1492 r = amdgpu_sync_wait(&sync, intr); 1493 amdgpu_sync_free(&sync); 1494 return r; 1495 } 1496 1497 /** 1498 * amdgpu_bo_sync_wait - Wrapper for amdgpu_bo_sync_wait_resv 1499 * @bo: buffer object to wait for 1500 * @owner: fence owner 1501 * @intr: Whether the wait is interruptible 1502 * 1503 * Wrapper to wait for fences in a BO. 1504 * Returns: 1505 * 0 on success, errno otherwise. 1506 */ 1507 int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr) 1508 { 1509 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1510 1511 return amdgpu_bo_sync_wait_resv(adev, bo->tbo.base.resv, 1512 AMDGPU_SYNC_NE_OWNER, owner, intr); 1513 } 1514 1515 /** 1516 * amdgpu_bo_gpu_offset - return GPU offset of bo 1517 * @bo: amdgpu object for which we query the offset 1518 * 1519 * Note: object should either be pinned or reserved when calling this 1520 * function, it might be useful to add check for this for debugging. 1521 * 1522 * Returns: 1523 * current GPU offset of the object. 1524 */ 1525 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) 1526 { 1527 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_SYSTEM); 1528 WARN_ON_ONCE(!dma_resv_is_locked(bo->tbo.base.resv) && 1529 !bo->tbo.pin_count && bo->tbo.type != ttm_bo_type_kernel); 1530 WARN_ON_ONCE(bo->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET); 1531 WARN_ON_ONCE(bo->tbo.resource->mem_type == TTM_PL_VRAM && 1532 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); 1533 1534 return amdgpu_bo_gpu_offset_no_check(bo); 1535 } 1536 1537 /** 1538 * amdgpu_bo_fb_aper_addr - return FB aperture GPU offset of the VRAM bo 1539 * @bo: amdgpu VRAM buffer object for which we query the offset 1540 * 1541 * Returns: 1542 * current FB aperture GPU offset of the object. 1543 */ 1544 u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo) 1545 { 1546 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1547 uint64_t offset, fb_base; 1548 1549 WARN_ON_ONCE(bo->tbo.resource->mem_type != TTM_PL_VRAM); 1550 1551 fb_base = adev->gmc.fb_start; 1552 fb_base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1553 offset = (bo->tbo.resource->start << PAGE_SHIFT) + fb_base; 1554 return amdgpu_gmc_sign_extend(offset); 1555 } 1556 1557 /** 1558 * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo 1559 * @bo: amdgpu object for which we query the offset 1560 * 1561 * Returns: 1562 * current GPU offset of the object without raising warnings. 1563 */ 1564 u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo) 1565 { 1566 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1567 uint64_t offset = AMDGPU_BO_INVALID_OFFSET; 1568 1569 if (bo->tbo.resource->mem_type == TTM_PL_TT) 1570 offset = amdgpu_gmc_agp_addr(&bo->tbo); 1571 1572 if (offset == AMDGPU_BO_INVALID_OFFSET) 1573 offset = (bo->tbo.resource->start << PAGE_SHIFT) + 1574 amdgpu_ttm_domain_start(adev, bo->tbo.resource->mem_type); 1575 1576 return amdgpu_gmc_sign_extend(offset); 1577 } 1578 1579 /** 1580 * amdgpu_bo_mem_stats_placement - bo placement for memory accounting 1581 * @bo: the buffer object we should look at 1582 * 1583 * BO can have multiple preferred placements, to avoid double counting we want 1584 * to file it under a single placement for memory stats. 1585 * Luckily, if we take the highest set bit in preferred_domains the result is 1586 * quite sensible. 1587 * 1588 * Returns: 1589 * Which of the placements should the BO be accounted under. 1590 */ 1591 uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo) 1592 { 1593 u32 domain; 1594 1595 /* 1596 * MMIO_REMAP is internal now, so it no longer maps from a userspace 1597 * domain bit. Keep fdinfo/mem-stats visibility by checking the actual 1598 * TTM placement. 1599 */ 1600 if (bo->tbo.resource && bo->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) 1601 return AMDGPU_PL_MMIO_REMAP; 1602 1603 domain = bo->preferred_domains & AMDGPU_GEM_DOMAIN_MASK; 1604 if (!domain) 1605 return TTM_PL_SYSTEM; 1606 1607 switch (rounddown_pow_of_two(domain)) { 1608 case AMDGPU_GEM_DOMAIN_CPU: 1609 return TTM_PL_SYSTEM; 1610 case AMDGPU_GEM_DOMAIN_GTT: 1611 return TTM_PL_TT; 1612 case AMDGPU_GEM_DOMAIN_VRAM: 1613 return TTM_PL_VRAM; 1614 case AMDGPU_GEM_DOMAIN_GDS: 1615 return AMDGPU_PL_GDS; 1616 case AMDGPU_GEM_DOMAIN_GWS: 1617 return AMDGPU_PL_GWS; 1618 case AMDGPU_GEM_DOMAIN_OA: 1619 return AMDGPU_PL_OA; 1620 case AMDGPU_GEM_DOMAIN_DOORBELL: 1621 return AMDGPU_PL_DOORBELL; 1622 default: 1623 return TTM_PL_SYSTEM; 1624 } 1625 } 1626 1627 /** 1628 * amdgpu_bo_get_preferred_domain - get preferred domain 1629 * @adev: amdgpu device object 1630 * @domain: allowed :ref:`memory domains <amdgpu_memory_domains>` 1631 * 1632 * Returns: 1633 * Which of the allowed domains is preferred for allocating the BO. 1634 */ 1635 uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, 1636 uint32_t domain) 1637 { 1638 if ((domain == (AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT)) && 1639 ((adev->asic_type == CHIP_CARRIZO) || (adev->asic_type == CHIP_STONEY))) { 1640 domain = AMDGPU_GEM_DOMAIN_VRAM; 1641 if (adev->gmc.real_vram_size <= AMDGPU_SG_THRESHOLD) 1642 domain = AMDGPU_GEM_DOMAIN_GTT; 1643 } 1644 return domain; 1645 } 1646 1647 #if defined(CONFIG_DEBUG_FS) 1648 #define amdgpu_bo_print_flag(m, bo, flag) \ 1649 do { \ 1650 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ 1651 seq_printf((m), " " #flag); \ 1652 } \ 1653 } while (0) 1654 1655 /** 1656 * amdgpu_bo_print_info - print BO info in debugfs file 1657 * 1658 * @id: Index or Id of the BO 1659 * @bo: Requested BO for printing info 1660 * @m: debugfs file 1661 * 1662 * Print BO information in debugfs file 1663 * 1664 * Returns: 1665 * Size of the BO in bytes. 1666 */ 1667 u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m) 1668 { 1669 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 1670 struct dma_buf_attachment *attachment; 1671 struct dma_buf *dma_buf; 1672 const char *placement; 1673 unsigned int pin_count; 1674 u64 size; 1675 1676 if (dma_resv_trylock(bo->tbo.base.resv)) { 1677 if (!bo->tbo.resource) { 1678 placement = "NONE"; 1679 } else { 1680 switch (bo->tbo.resource->mem_type) { 1681 case TTM_PL_VRAM: 1682 if (amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 1683 placement = "VRAM VISIBLE"; 1684 else 1685 placement = "VRAM"; 1686 break; 1687 case TTM_PL_TT: 1688 placement = "GTT"; 1689 break; 1690 case AMDGPU_PL_GDS: 1691 placement = "GDS"; 1692 break; 1693 case AMDGPU_PL_GWS: 1694 placement = "GWS"; 1695 break; 1696 case AMDGPU_PL_OA: 1697 placement = "OA"; 1698 break; 1699 case AMDGPU_PL_PREEMPT: 1700 placement = "PREEMPTIBLE"; 1701 break; 1702 case AMDGPU_PL_DOORBELL: 1703 placement = "DOORBELL"; 1704 break; 1705 case AMDGPU_PL_MMIO_REMAP: 1706 placement = "MMIO REMAP"; 1707 break; 1708 case AMDGPU_PL_NPA: 1709 placement = "NPA"; 1710 break; 1711 case TTM_PL_SYSTEM: 1712 default: 1713 placement = "CPU"; 1714 break; 1715 } 1716 } 1717 dma_resv_unlock(bo->tbo.base.resv); 1718 } else { 1719 placement = "UNKNOWN"; 1720 } 1721 1722 size = amdgpu_bo_size(bo); 1723 seq_printf(m, "\t\t0x%08x: %12lld byte %s", 1724 id, size, placement); 1725 1726 pin_count = READ_ONCE(bo->tbo.pin_count); 1727 if (pin_count) 1728 seq_printf(m, " pin count %d", pin_count); 1729 1730 dma_buf = READ_ONCE(bo->tbo.base.dma_buf); 1731 attachment = READ_ONCE(bo->tbo.base.import_attach); 1732 1733 if (attachment) 1734 seq_printf(m, " imported from ino:%llu", file_inode(dma_buf->file)->i_ino); 1735 else if (dma_buf) 1736 seq_printf(m, " exported as ino:%llu", file_inode(dma_buf->file)->i_ino); 1737 1738 amdgpu_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); 1739 amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS); 1740 amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC); 1741 amdgpu_bo_print_flag(m, bo, VRAM_CLEARED); 1742 amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS); 1743 amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID); 1744 amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC); 1745 /* Add the gem obj resv fence dump*/ 1746 if (dma_resv_trylock(bo->tbo.base.resv)) { 1747 dma_resv_describe(bo->tbo.base.resv, m); 1748 dma_resv_unlock(bo->tbo.base.resv); 1749 } 1750 seq_puts(m, "\n"); 1751 1752 return size; 1753 } 1754 #endif 1755