1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <linux/slab.h> 34 #include <drm/drmP.h> 35 #include <drm/amdgpu_drm.h> 36 #include <drm/drm_cache.h> 37 #include "amdgpu.h" 38 #include "amdgpu_trace.h" 39 40 static bool amdgpu_need_backup(struct amdgpu_device *adev) 41 { 42 if (adev->flags & AMD_IS_APU) 43 return false; 44 45 if (amdgpu_gpu_recovery == 0 || 46 (amdgpu_gpu_recovery == -1 && !amdgpu_sriov_vf(adev))) 47 return false; 48 49 return true; 50 } 51 52 static void amdgpu_ttm_bo_destroy(struct ttm_buffer_object *tbo) 53 { 54 struct amdgpu_device *adev = amdgpu_ttm_adev(tbo->bdev); 55 struct amdgpu_bo *bo = ttm_to_amdgpu_bo(tbo); 56 57 amdgpu_bo_kunmap(bo); 58 59 if (bo->gem_base.import_attach) 60 drm_prime_gem_destroy(&bo->gem_base, bo->tbo.sg); 61 drm_gem_object_release(&bo->gem_base); 62 amdgpu_bo_unref(&bo->parent); 63 if (!list_empty(&bo->shadow_list)) { 64 mutex_lock(&adev->shadow_list_lock); 65 list_del_init(&bo->shadow_list); 66 mutex_unlock(&adev->shadow_list_lock); 67 } 68 kfree(bo->metadata); 69 kfree(bo); 70 } 71 72 bool amdgpu_ttm_bo_is_amdgpu_bo(struct ttm_buffer_object *bo) 73 { 74 if (bo->destroy == &amdgpu_ttm_bo_destroy) 75 return true; 76 return false; 77 } 78 79 void amdgpu_ttm_placement_from_domain(struct amdgpu_bo *abo, u32 domain) 80 { 81 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 82 struct ttm_placement *placement = &abo->placement; 83 struct ttm_place *places = abo->placements; 84 u64 flags = abo->flags; 85 u32 c = 0; 86 87 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 88 unsigned visible_pfn = adev->mc.visible_vram_size >> PAGE_SHIFT; 89 90 places[c].fpfn = 0; 91 places[c].lpfn = 0; 92 places[c].flags = TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED | 93 TTM_PL_FLAG_VRAM; 94 95 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) 96 places[c].lpfn = visible_pfn; 97 else 98 places[c].flags |= TTM_PL_FLAG_TOPDOWN; 99 100 if (flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) 101 places[c].flags |= TTM_PL_FLAG_CONTIGUOUS; 102 c++; 103 } 104 105 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 106 places[c].fpfn = 0; 107 if (flags & AMDGPU_GEM_CREATE_SHADOW) 108 places[c].lpfn = adev->mc.gart_size >> PAGE_SHIFT; 109 else 110 places[c].lpfn = 0; 111 places[c].flags = TTM_PL_FLAG_TT; 112 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 113 places[c].flags |= TTM_PL_FLAG_WC | 114 TTM_PL_FLAG_UNCACHED; 115 else 116 places[c].flags |= TTM_PL_FLAG_CACHED; 117 c++; 118 } 119 120 if (domain & AMDGPU_GEM_DOMAIN_CPU) { 121 places[c].fpfn = 0; 122 places[c].lpfn = 0; 123 places[c].flags = TTM_PL_FLAG_SYSTEM; 124 if (flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 125 places[c].flags |= TTM_PL_FLAG_WC | 126 TTM_PL_FLAG_UNCACHED; 127 else 128 places[c].flags |= TTM_PL_FLAG_CACHED; 129 c++; 130 } 131 132 if (domain & AMDGPU_GEM_DOMAIN_GDS) { 133 places[c].fpfn = 0; 134 places[c].lpfn = 0; 135 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GDS; 136 c++; 137 } 138 139 if (domain & AMDGPU_GEM_DOMAIN_GWS) { 140 places[c].fpfn = 0; 141 places[c].lpfn = 0; 142 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_GWS; 143 c++; 144 } 145 146 if (domain & AMDGPU_GEM_DOMAIN_OA) { 147 places[c].fpfn = 0; 148 places[c].lpfn = 0; 149 places[c].flags = TTM_PL_FLAG_UNCACHED | AMDGPU_PL_FLAG_OA; 150 c++; 151 } 152 153 if (!c) { 154 places[c].fpfn = 0; 155 places[c].lpfn = 0; 156 places[c].flags = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM; 157 c++; 158 } 159 160 placement->num_placement = c; 161 placement->placement = places; 162 163 placement->num_busy_placement = c; 164 placement->busy_placement = places; 165 } 166 167 /** 168 * amdgpu_bo_create_reserved - create reserved BO for kernel use 169 * 170 * @adev: amdgpu device object 171 * @size: size for the new BO 172 * @align: alignment for the new BO 173 * @domain: where to place it 174 * @bo_ptr: resulting BO 175 * @gpu_addr: GPU addr of the pinned BO 176 * @cpu_addr: optional CPU address mapping 177 * 178 * Allocates and pins a BO for kernel internal use, and returns it still 179 * reserved. 180 * 181 * Returns 0 on success, negative error code otherwise. 182 */ 183 int amdgpu_bo_create_reserved(struct amdgpu_device *adev, 184 unsigned long size, int align, 185 u32 domain, struct amdgpu_bo **bo_ptr, 186 u64 *gpu_addr, void **cpu_addr) 187 { 188 bool free = false; 189 int r; 190 191 if (!*bo_ptr) { 192 r = amdgpu_bo_create(adev, size, align, true, domain, 193 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 194 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, 195 NULL, NULL, 0, bo_ptr); 196 if (r) { 197 dev_err(adev->dev, "(%d) failed to allocate kernel bo\n", 198 r); 199 return r; 200 } 201 free = true; 202 } 203 204 r = amdgpu_bo_reserve(*bo_ptr, false); 205 if (r) { 206 dev_err(adev->dev, "(%d) failed to reserve kernel bo\n", r); 207 goto error_free; 208 } 209 210 r = amdgpu_bo_pin(*bo_ptr, domain, gpu_addr); 211 if (r) { 212 dev_err(adev->dev, "(%d) kernel bo pin failed\n", r); 213 goto error_unreserve; 214 } 215 216 if (cpu_addr) { 217 r = amdgpu_bo_kmap(*bo_ptr, cpu_addr); 218 if (r) { 219 dev_err(adev->dev, "(%d) kernel bo map failed\n", r); 220 goto error_unreserve; 221 } 222 } 223 224 return 0; 225 226 error_unreserve: 227 amdgpu_bo_unreserve(*bo_ptr); 228 229 error_free: 230 if (free) 231 amdgpu_bo_unref(bo_ptr); 232 233 return r; 234 } 235 236 /** 237 * amdgpu_bo_create_kernel - create BO for kernel use 238 * 239 * @adev: amdgpu device object 240 * @size: size for the new BO 241 * @align: alignment for the new BO 242 * @domain: where to place it 243 * @bo_ptr: resulting BO 244 * @gpu_addr: GPU addr of the pinned BO 245 * @cpu_addr: optional CPU address mapping 246 * 247 * Allocates and pins a BO for kernel internal use. 248 * 249 * Returns 0 on success, negative error code otherwise. 250 */ 251 int amdgpu_bo_create_kernel(struct amdgpu_device *adev, 252 unsigned long size, int align, 253 u32 domain, struct amdgpu_bo **bo_ptr, 254 u64 *gpu_addr, void **cpu_addr) 255 { 256 int r; 257 258 r = amdgpu_bo_create_reserved(adev, size, align, domain, bo_ptr, 259 gpu_addr, cpu_addr); 260 261 if (r) 262 return r; 263 264 amdgpu_bo_unreserve(*bo_ptr); 265 266 return 0; 267 } 268 269 /** 270 * amdgpu_bo_free_kernel - free BO for kernel use 271 * 272 * @bo: amdgpu BO to free 273 * 274 * unmaps and unpin a BO for kernel internal use. 275 */ 276 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr, 277 void **cpu_addr) 278 { 279 if (*bo == NULL) 280 return; 281 282 if (likely(amdgpu_bo_reserve(*bo, true) == 0)) { 283 if (cpu_addr) 284 amdgpu_bo_kunmap(*bo); 285 286 amdgpu_bo_unpin(*bo); 287 amdgpu_bo_unreserve(*bo); 288 } 289 amdgpu_bo_unref(bo); 290 291 if (gpu_addr) 292 *gpu_addr = 0; 293 294 if (cpu_addr) 295 *cpu_addr = NULL; 296 } 297 298 /* Validate bo size is bit bigger then the request domain */ 299 static bool amdgpu_bo_validate_size(struct amdgpu_device *adev, 300 unsigned long size, u32 domain) 301 { 302 struct ttm_mem_type_manager *man = NULL; 303 304 /* 305 * If GTT is part of requested domains the check must succeed to 306 * allow fall back to GTT 307 */ 308 if (domain & AMDGPU_GEM_DOMAIN_GTT) { 309 man = &adev->mman.bdev.man[TTM_PL_TT]; 310 311 if (size < (man->size << PAGE_SHIFT)) 312 return true; 313 else 314 goto fail; 315 } 316 317 if (domain & AMDGPU_GEM_DOMAIN_VRAM) { 318 man = &adev->mman.bdev.man[TTM_PL_VRAM]; 319 320 if (size < (man->size << PAGE_SHIFT)) 321 return true; 322 else 323 goto fail; 324 } 325 326 327 /* TODO add more domains checks, such as AMDGPU_GEM_DOMAIN_CPU */ 328 return true; 329 330 fail: 331 DRM_DEBUG("BO size %lu > total memory in domain: %llu\n", size, 332 man->size << PAGE_SHIFT); 333 return false; 334 } 335 336 static int amdgpu_bo_do_create(struct amdgpu_device *adev, 337 unsigned long size, int byte_align, 338 bool kernel, u32 domain, u64 flags, 339 struct sg_table *sg, 340 struct reservation_object *resv, 341 uint64_t init_value, 342 struct amdgpu_bo **bo_ptr) 343 { 344 struct ttm_operation_ctx ctx = { 345 .interruptible = !kernel, 346 .no_wait_gpu = false, 347 .allow_reserved_eviction = true, 348 .resv = resv 349 }; 350 struct amdgpu_bo *bo; 351 enum ttm_bo_type type; 352 unsigned long page_align; 353 size_t acc_size; 354 int r; 355 356 page_align = roundup(byte_align, PAGE_SIZE) >> PAGE_SHIFT; 357 size = ALIGN(size, PAGE_SIZE); 358 359 if (!amdgpu_bo_validate_size(adev, size, domain)) 360 return -ENOMEM; 361 362 if (kernel) { 363 type = ttm_bo_type_kernel; 364 } else if (sg) { 365 type = ttm_bo_type_sg; 366 } else { 367 type = ttm_bo_type_device; 368 } 369 *bo_ptr = NULL; 370 371 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size, 372 sizeof(struct amdgpu_bo)); 373 374 bo = kzalloc(sizeof(struct amdgpu_bo), GFP_KERNEL); 375 if (bo == NULL) 376 return -ENOMEM; 377 r = drm_gem_object_init(adev->ddev, &bo->gem_base, size); 378 if (unlikely(r)) { 379 kfree(bo); 380 return r; 381 } 382 INIT_LIST_HEAD(&bo->shadow_list); 383 INIT_LIST_HEAD(&bo->va); 384 bo->preferred_domains = domain & (AMDGPU_GEM_DOMAIN_VRAM | 385 AMDGPU_GEM_DOMAIN_GTT | 386 AMDGPU_GEM_DOMAIN_CPU | 387 AMDGPU_GEM_DOMAIN_GDS | 388 AMDGPU_GEM_DOMAIN_GWS | 389 AMDGPU_GEM_DOMAIN_OA); 390 bo->allowed_domains = bo->preferred_domains; 391 if (!kernel && bo->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) 392 bo->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 393 394 bo->flags = flags; 395 396 #ifdef CONFIG_X86_32 397 /* XXX: Write-combined CPU mappings of GTT seem broken on 32-bit 398 * See https://bugs.freedesktop.org/show_bug.cgi?id=84627 399 */ 400 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 401 #elif defined(CONFIG_X86) && !defined(CONFIG_X86_PAT) 402 /* Don't try to enable write-combining when it can't work, or things 403 * may be slow 404 * See https://bugs.freedesktop.org/show_bug.cgi?id=88758 405 */ 406 407 #ifndef CONFIG_COMPILE_TEST 408 #warning Please enable CONFIG_MTRR and CONFIG_X86_PAT for better performance \ 409 thanks to write-combining 410 #endif 411 412 if (bo->flags & AMDGPU_GEM_CREATE_CPU_GTT_USWC) 413 DRM_INFO_ONCE("Please enable CONFIG_MTRR and CONFIG_X86_PAT for " 414 "better performance thanks to write-combining\n"); 415 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 416 #else 417 /* For architectures that don't support WC memory, 418 * mask out the WC flag from the BO 419 */ 420 if (!drm_arch_can_wc_memory()) 421 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_GTT_USWC; 422 #endif 423 424 bo->tbo.bdev = &adev->mman.bdev; 425 amdgpu_ttm_placement_from_domain(bo, domain); 426 427 r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, type, 428 &bo->placement, page_align, &ctx, NULL, 429 acc_size, sg, resv, &amdgpu_ttm_bo_destroy); 430 if (unlikely(r != 0)) 431 return r; 432 433 if (adev->mc.visible_vram_size < adev->mc.real_vram_size && 434 bo->tbo.mem.mem_type == TTM_PL_VRAM && 435 bo->tbo.mem.start < adev->mc.visible_vram_size >> PAGE_SHIFT) 436 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 437 ctx.bytes_moved); 438 else 439 amdgpu_cs_report_moved_bytes(adev, ctx.bytes_moved, 0); 440 441 if (kernel) 442 bo->tbo.priority = 1; 443 444 if (flags & AMDGPU_GEM_CREATE_VRAM_CLEARED && 445 bo->tbo.mem.placement & TTM_PL_FLAG_VRAM) { 446 struct dma_fence *fence; 447 448 r = amdgpu_fill_buffer(bo, init_value, bo->tbo.resv, &fence); 449 if (unlikely(r)) 450 goto fail_unreserve; 451 452 amdgpu_bo_fence(bo, fence, false); 453 dma_fence_put(bo->tbo.moving); 454 bo->tbo.moving = dma_fence_get(fence); 455 dma_fence_put(fence); 456 } 457 if (!resv) 458 amdgpu_bo_unreserve(bo); 459 *bo_ptr = bo; 460 461 trace_amdgpu_bo_create(bo); 462 463 /* Treat CPU_ACCESS_REQUIRED only as a hint if given by UMD */ 464 if (type == ttm_bo_type_device) 465 bo->flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 466 467 return 0; 468 469 fail_unreserve: 470 if (!resv) 471 ww_mutex_unlock(&bo->tbo.resv->lock); 472 amdgpu_bo_unref(&bo); 473 return r; 474 } 475 476 static int amdgpu_bo_create_shadow(struct amdgpu_device *adev, 477 unsigned long size, int byte_align, 478 struct amdgpu_bo *bo) 479 { 480 int r; 481 482 if (bo->shadow) 483 return 0; 484 485 r = amdgpu_bo_do_create(adev, size, byte_align, true, 486 AMDGPU_GEM_DOMAIN_GTT, 487 AMDGPU_GEM_CREATE_CPU_GTT_USWC | 488 AMDGPU_GEM_CREATE_SHADOW, 489 NULL, bo->tbo.resv, 0, 490 &bo->shadow); 491 if (!r) { 492 bo->shadow->parent = amdgpu_bo_ref(bo); 493 mutex_lock(&adev->shadow_list_lock); 494 list_add_tail(&bo->shadow_list, &adev->shadow_list); 495 mutex_unlock(&adev->shadow_list_lock); 496 } 497 498 return r; 499 } 500 501 /* init_value will only take effect when flags contains 502 * AMDGPU_GEM_CREATE_VRAM_CLEARED. 503 */ 504 int amdgpu_bo_create(struct amdgpu_device *adev, 505 unsigned long size, int byte_align, 506 bool kernel, u32 domain, u64 flags, 507 struct sg_table *sg, 508 struct reservation_object *resv, 509 uint64_t init_value, 510 struct amdgpu_bo **bo_ptr) 511 { 512 uint64_t parent_flags = flags & ~AMDGPU_GEM_CREATE_SHADOW; 513 int r; 514 515 r = amdgpu_bo_do_create(adev, size, byte_align, kernel, domain, 516 parent_flags, sg, resv, init_value, bo_ptr); 517 if (r) 518 return r; 519 520 if ((flags & AMDGPU_GEM_CREATE_SHADOW) && amdgpu_need_backup(adev)) { 521 if (!resv) 522 WARN_ON(reservation_object_lock((*bo_ptr)->tbo.resv, 523 NULL)); 524 525 r = amdgpu_bo_create_shadow(adev, size, byte_align, (*bo_ptr)); 526 527 if (!resv) 528 reservation_object_unlock((*bo_ptr)->tbo.resv); 529 530 if (r) 531 amdgpu_bo_unref(bo_ptr); 532 } 533 534 return r; 535 } 536 537 int amdgpu_bo_backup_to_shadow(struct amdgpu_device *adev, 538 struct amdgpu_ring *ring, 539 struct amdgpu_bo *bo, 540 struct reservation_object *resv, 541 struct dma_fence **fence, 542 bool direct) 543 544 { 545 struct amdgpu_bo *shadow = bo->shadow; 546 uint64_t bo_addr, shadow_addr; 547 int r; 548 549 if (!shadow) 550 return -EINVAL; 551 552 bo_addr = amdgpu_bo_gpu_offset(bo); 553 shadow_addr = amdgpu_bo_gpu_offset(bo->shadow); 554 555 r = reservation_object_reserve_shared(bo->tbo.resv); 556 if (r) 557 goto err; 558 559 r = amdgpu_copy_buffer(ring, bo_addr, shadow_addr, 560 amdgpu_bo_size(bo), resv, fence, 561 direct, false); 562 if (!r) 563 amdgpu_bo_fence(bo, *fence, true); 564 565 err: 566 return r; 567 } 568 569 int amdgpu_bo_validate(struct amdgpu_bo *bo) 570 { 571 struct ttm_operation_ctx ctx = { false, false }; 572 uint32_t domain; 573 int r; 574 575 if (bo->pin_count) 576 return 0; 577 578 domain = bo->preferred_domains; 579 580 retry: 581 amdgpu_ttm_placement_from_domain(bo, domain); 582 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 583 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { 584 domain = bo->allowed_domains; 585 goto retry; 586 } 587 588 return r; 589 } 590 591 int amdgpu_bo_restore_from_shadow(struct amdgpu_device *adev, 592 struct amdgpu_ring *ring, 593 struct amdgpu_bo *bo, 594 struct reservation_object *resv, 595 struct dma_fence **fence, 596 bool direct) 597 598 { 599 struct amdgpu_bo *shadow = bo->shadow; 600 uint64_t bo_addr, shadow_addr; 601 int r; 602 603 if (!shadow) 604 return -EINVAL; 605 606 bo_addr = amdgpu_bo_gpu_offset(bo); 607 shadow_addr = amdgpu_bo_gpu_offset(bo->shadow); 608 609 r = reservation_object_reserve_shared(bo->tbo.resv); 610 if (r) 611 goto err; 612 613 r = amdgpu_copy_buffer(ring, shadow_addr, bo_addr, 614 amdgpu_bo_size(bo), resv, fence, 615 direct, false); 616 if (!r) 617 amdgpu_bo_fence(bo, *fence, true); 618 619 err: 620 return r; 621 } 622 623 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr) 624 { 625 void *kptr; 626 long r; 627 628 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 629 return -EPERM; 630 631 kptr = amdgpu_bo_kptr(bo); 632 if (kptr) { 633 if (ptr) 634 *ptr = kptr; 635 return 0; 636 } 637 638 r = reservation_object_wait_timeout_rcu(bo->tbo.resv, false, false, 639 MAX_SCHEDULE_TIMEOUT); 640 if (r < 0) 641 return r; 642 643 r = ttm_bo_kmap(&bo->tbo, 0, bo->tbo.num_pages, &bo->kmap); 644 if (r) 645 return r; 646 647 if (ptr) 648 *ptr = amdgpu_bo_kptr(bo); 649 650 return 0; 651 } 652 653 void *amdgpu_bo_kptr(struct amdgpu_bo *bo) 654 { 655 bool is_iomem; 656 657 return ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); 658 } 659 660 void amdgpu_bo_kunmap(struct amdgpu_bo *bo) 661 { 662 if (bo->kmap.bo) 663 ttm_bo_kunmap(&bo->kmap); 664 } 665 666 struct amdgpu_bo *amdgpu_bo_ref(struct amdgpu_bo *bo) 667 { 668 if (bo == NULL) 669 return NULL; 670 671 ttm_bo_reference(&bo->tbo); 672 return bo; 673 } 674 675 void amdgpu_bo_unref(struct amdgpu_bo **bo) 676 { 677 struct ttm_buffer_object *tbo; 678 679 if ((*bo) == NULL) 680 return; 681 682 tbo = &((*bo)->tbo); 683 ttm_bo_unref(&tbo); 684 if (tbo == NULL) 685 *bo = NULL; 686 } 687 688 int amdgpu_bo_pin_restricted(struct amdgpu_bo *bo, u32 domain, 689 u64 min_offset, u64 max_offset, 690 u64 *gpu_addr) 691 { 692 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 693 struct ttm_operation_ctx ctx = { false, false }; 694 int r, i; 695 696 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) 697 return -EPERM; 698 699 if (WARN_ON_ONCE(min_offset > max_offset)) 700 return -EINVAL; 701 702 /* A shared bo cannot be migrated to VRAM */ 703 if (bo->prime_shared_count && (domain == AMDGPU_GEM_DOMAIN_VRAM)) 704 return -EINVAL; 705 706 if (bo->pin_count) { 707 uint32_t mem_type = bo->tbo.mem.mem_type; 708 709 if (!(domain & amdgpu_mem_type_to_domain(mem_type))) 710 return -EINVAL; 711 712 bo->pin_count++; 713 if (gpu_addr) 714 *gpu_addr = amdgpu_bo_gpu_offset(bo); 715 716 if (max_offset != 0) { 717 u64 domain_start = bo->tbo.bdev->man[mem_type].gpu_offset; 718 WARN_ON_ONCE(max_offset < 719 (amdgpu_bo_gpu_offset(bo) - domain_start)); 720 } 721 722 return 0; 723 } 724 725 bo->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 726 /* force to pin into visible video ram */ 727 if (!(bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) 728 bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 729 amdgpu_ttm_placement_from_domain(bo, domain); 730 for (i = 0; i < bo->placement.num_placement; i++) { 731 unsigned fpfn, lpfn; 732 733 fpfn = min_offset >> PAGE_SHIFT; 734 lpfn = max_offset >> PAGE_SHIFT; 735 736 if (fpfn > bo->placements[i].fpfn) 737 bo->placements[i].fpfn = fpfn; 738 if (!bo->placements[i].lpfn || 739 (lpfn && lpfn < bo->placements[i].lpfn)) 740 bo->placements[i].lpfn = lpfn; 741 bo->placements[i].flags |= TTM_PL_FLAG_NO_EVICT; 742 } 743 744 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 745 if (unlikely(r)) { 746 dev_err(adev->dev, "%p pin failed\n", bo); 747 goto error; 748 } 749 750 r = amdgpu_ttm_alloc_gart(&bo->tbo); 751 if (unlikely(r)) { 752 dev_err(adev->dev, "%p bind failed\n", bo); 753 goto error; 754 } 755 756 bo->pin_count = 1; 757 if (gpu_addr != NULL) 758 *gpu_addr = amdgpu_bo_gpu_offset(bo); 759 760 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); 761 if (domain == AMDGPU_GEM_DOMAIN_VRAM) { 762 adev->vram_pin_size += amdgpu_bo_size(bo); 763 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 764 adev->invisible_pin_size += amdgpu_bo_size(bo); 765 } else if (domain == AMDGPU_GEM_DOMAIN_GTT) { 766 adev->gart_pin_size += amdgpu_bo_size(bo); 767 } 768 769 error: 770 return r; 771 } 772 773 int amdgpu_bo_pin(struct amdgpu_bo *bo, u32 domain, u64 *gpu_addr) 774 { 775 return amdgpu_bo_pin_restricted(bo, domain, 0, 0, gpu_addr); 776 } 777 778 int amdgpu_bo_unpin(struct amdgpu_bo *bo) 779 { 780 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 781 struct ttm_operation_ctx ctx = { false, false }; 782 int r, i; 783 784 if (!bo->pin_count) { 785 dev_warn(adev->dev, "%p unpin not necessary\n", bo); 786 return 0; 787 } 788 bo->pin_count--; 789 if (bo->pin_count) 790 return 0; 791 for (i = 0; i < bo->placement.num_placement; i++) { 792 bo->placements[i].lpfn = 0; 793 bo->placements[i].flags &= ~TTM_PL_FLAG_NO_EVICT; 794 } 795 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 796 if (unlikely(r)) { 797 dev_err(adev->dev, "%p validate failed for unpin\n", bo); 798 goto error; 799 } 800 801 if (bo->tbo.mem.mem_type == TTM_PL_VRAM) { 802 adev->vram_pin_size -= amdgpu_bo_size(bo); 803 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS) 804 adev->invisible_pin_size -= amdgpu_bo_size(bo); 805 } else if (bo->tbo.mem.mem_type == TTM_PL_TT) { 806 adev->gart_pin_size -= amdgpu_bo_size(bo); 807 } 808 809 error: 810 return r; 811 } 812 813 int amdgpu_bo_evict_vram(struct amdgpu_device *adev) 814 { 815 /* late 2.6.33 fix IGP hibernate - we need pm ops to do this correct */ 816 if (0 && (adev->flags & AMD_IS_APU)) { 817 /* Useless to evict on IGP chips */ 818 return 0; 819 } 820 return ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_VRAM); 821 } 822 823 static const char *amdgpu_vram_names[] = { 824 "UNKNOWN", 825 "GDDR1", 826 "DDR2", 827 "GDDR3", 828 "GDDR4", 829 "GDDR5", 830 "HBM", 831 "DDR3" 832 }; 833 834 int amdgpu_bo_init(struct amdgpu_device *adev) 835 { 836 /* reserve PAT memory space to WC for VRAM */ 837 arch_io_reserve_memtype_wc(adev->mc.aper_base, 838 adev->mc.aper_size); 839 840 /* Add an MTRR for the VRAM */ 841 adev->mc.vram_mtrr = arch_phys_wc_add(adev->mc.aper_base, 842 adev->mc.aper_size); 843 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", 844 adev->mc.mc_vram_size >> 20, 845 (unsigned long long)adev->mc.aper_size >> 20); 846 DRM_INFO("RAM width %dbits %s\n", 847 adev->mc.vram_width, amdgpu_vram_names[adev->mc.vram_type]); 848 return amdgpu_ttm_init(adev); 849 } 850 851 void amdgpu_bo_fini(struct amdgpu_device *adev) 852 { 853 amdgpu_ttm_fini(adev); 854 arch_phys_wc_del(adev->mc.vram_mtrr); 855 arch_io_free_memtype_wc(adev->mc.aper_base, adev->mc.aper_size); 856 } 857 858 int amdgpu_bo_fbdev_mmap(struct amdgpu_bo *bo, 859 struct vm_area_struct *vma) 860 { 861 return ttm_fbdev_mmap(vma, &bo->tbo); 862 } 863 864 int amdgpu_bo_set_tiling_flags(struct amdgpu_bo *bo, u64 tiling_flags) 865 { 866 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 867 868 if (adev->family <= AMDGPU_FAMILY_CZ && 869 AMDGPU_TILING_GET(tiling_flags, TILE_SPLIT) > 6) 870 return -EINVAL; 871 872 bo->tiling_flags = tiling_flags; 873 return 0; 874 } 875 876 void amdgpu_bo_get_tiling_flags(struct amdgpu_bo *bo, u64 *tiling_flags) 877 { 878 lockdep_assert_held(&bo->tbo.resv->lock.base); 879 880 if (tiling_flags) 881 *tiling_flags = bo->tiling_flags; 882 } 883 884 int amdgpu_bo_set_metadata (struct amdgpu_bo *bo, void *metadata, 885 uint32_t metadata_size, uint64_t flags) 886 { 887 void *buffer; 888 889 if (!metadata_size) { 890 if (bo->metadata_size) { 891 kfree(bo->metadata); 892 bo->metadata = NULL; 893 bo->metadata_size = 0; 894 } 895 return 0; 896 } 897 898 if (metadata == NULL) 899 return -EINVAL; 900 901 buffer = kmemdup(metadata, metadata_size, GFP_KERNEL); 902 if (buffer == NULL) 903 return -ENOMEM; 904 905 kfree(bo->metadata); 906 bo->metadata_flags = flags; 907 bo->metadata = buffer; 908 bo->metadata_size = metadata_size; 909 910 return 0; 911 } 912 913 int amdgpu_bo_get_metadata(struct amdgpu_bo *bo, void *buffer, 914 size_t buffer_size, uint32_t *metadata_size, 915 uint64_t *flags) 916 { 917 if (!buffer && !metadata_size) 918 return -EINVAL; 919 920 if (buffer) { 921 if (buffer_size < bo->metadata_size) 922 return -EINVAL; 923 924 if (bo->metadata_size) 925 memcpy(buffer, bo->metadata, bo->metadata_size); 926 } 927 928 if (metadata_size) 929 *metadata_size = bo->metadata_size; 930 if (flags) 931 *flags = bo->metadata_flags; 932 933 return 0; 934 } 935 936 void amdgpu_bo_move_notify(struct ttm_buffer_object *bo, 937 bool evict, 938 struct ttm_mem_reg *new_mem) 939 { 940 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 941 struct amdgpu_bo *abo; 942 struct ttm_mem_reg *old_mem = &bo->mem; 943 944 if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) 945 return; 946 947 abo = ttm_to_amdgpu_bo(bo); 948 amdgpu_vm_bo_invalidate(adev, abo, evict); 949 950 amdgpu_bo_kunmap(abo); 951 952 /* remember the eviction */ 953 if (evict) 954 atomic64_inc(&adev->num_evictions); 955 956 /* update statistics */ 957 if (!new_mem) 958 return; 959 960 /* move_notify is called before move happens */ 961 trace_amdgpu_ttm_bo_move(abo, new_mem->mem_type, old_mem->mem_type); 962 } 963 964 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 965 { 966 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 967 struct ttm_operation_ctx ctx = { false, false }; 968 struct amdgpu_bo *abo; 969 unsigned long offset, size; 970 int r; 971 972 if (!amdgpu_ttm_bo_is_amdgpu_bo(bo)) 973 return 0; 974 975 abo = ttm_to_amdgpu_bo(bo); 976 977 /* Remember that this BO was accessed by the CPU */ 978 abo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 979 980 if (bo->mem.mem_type != TTM_PL_VRAM) 981 return 0; 982 983 size = bo->mem.num_pages << PAGE_SHIFT; 984 offset = bo->mem.start << PAGE_SHIFT; 985 if ((offset + size) <= adev->mc.visible_vram_size) 986 return 0; 987 988 /* Can't move a pinned BO to visible VRAM */ 989 if (abo->pin_count > 0) 990 return -EINVAL; 991 992 /* hurrah the memory is not visible ! */ 993 atomic64_inc(&adev->num_vram_cpu_page_faults); 994 amdgpu_ttm_placement_from_domain(abo, AMDGPU_GEM_DOMAIN_VRAM | 995 AMDGPU_GEM_DOMAIN_GTT); 996 997 /* Avoid costly evictions; only set GTT as a busy placement */ 998 abo->placement.num_busy_placement = 1; 999 abo->placement.busy_placement = &abo->placements[1]; 1000 1001 r = ttm_bo_validate(bo, &abo->placement, &ctx); 1002 if (unlikely(r != 0)) 1003 return r; 1004 1005 offset = bo->mem.start << PAGE_SHIFT; 1006 /* this should never happen */ 1007 if (bo->mem.mem_type == TTM_PL_VRAM && 1008 (offset + size) > adev->mc.visible_vram_size) 1009 return -EINVAL; 1010 1011 return 0; 1012 } 1013 1014 /** 1015 * amdgpu_bo_fence - add fence to buffer object 1016 * 1017 * @bo: buffer object in question 1018 * @fence: fence to add 1019 * @shared: true if fence should be added shared 1020 * 1021 */ 1022 void amdgpu_bo_fence(struct amdgpu_bo *bo, struct dma_fence *fence, 1023 bool shared) 1024 { 1025 struct reservation_object *resv = bo->tbo.resv; 1026 1027 if (shared) 1028 reservation_object_add_shared_fence(resv, fence); 1029 else 1030 reservation_object_add_excl_fence(resv, fence); 1031 } 1032 1033 /** 1034 * amdgpu_bo_gpu_offset - return GPU offset of bo 1035 * @bo: amdgpu object for which we query the offset 1036 * 1037 * Returns current GPU offset of the object. 1038 * 1039 * Note: object should either be pinned or reserved when calling this 1040 * function, it might be useful to add check for this for debugging. 1041 */ 1042 u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo) 1043 { 1044 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_SYSTEM); 1045 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_TT && 1046 !amdgpu_gtt_mgr_has_gart_addr(&bo->tbo.mem)); 1047 WARN_ON_ONCE(!ww_mutex_is_locked(&bo->tbo.resv->lock) && 1048 !bo->pin_count); 1049 WARN_ON_ONCE(bo->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET); 1050 WARN_ON_ONCE(bo->tbo.mem.mem_type == TTM_PL_VRAM && 1051 !(bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS)); 1052 1053 return bo->tbo.offset; 1054 } 1055