1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #include <linux/dma-mapping.h> 26 #include <drm/ttm/ttm_range_manager.h> 27 28 #include "amdgpu.h" 29 #include "amdgpu_vm.h" 30 #include "amdgpu_res_cursor.h" 31 #include "amdgpu_atomfirmware.h" 32 #include "atom.h" 33 34 struct amdgpu_vram_reservation { 35 u64 start; 36 u64 size; 37 struct list_head allocated; 38 struct list_head blocks; 39 }; 40 41 static inline struct amdgpu_vram_mgr * 42 to_vram_mgr(struct ttm_resource_manager *man) 43 { 44 return container_of(man, struct amdgpu_vram_mgr, manager); 45 } 46 47 static inline struct amdgpu_device * 48 to_amdgpu_device(struct amdgpu_vram_mgr *mgr) 49 { 50 return container_of(mgr, struct amdgpu_device, mman.vram_mgr); 51 } 52 53 static inline struct drm_buddy_block * 54 amdgpu_vram_mgr_first_block(struct list_head *list) 55 { 56 return list_first_entry_or_null(list, struct drm_buddy_block, link); 57 } 58 59 static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head) 60 { 61 struct drm_buddy_block *block; 62 u64 start, size; 63 64 block = amdgpu_vram_mgr_first_block(head); 65 if (!block) 66 return false; 67 68 while (head != block->link.next) { 69 start = amdgpu_vram_mgr_block_start(block); 70 size = amdgpu_vram_mgr_block_size(block); 71 72 block = list_entry(block->link.next, struct drm_buddy_block, link); 73 if (start + size != amdgpu_vram_mgr_block_start(block)) 74 return false; 75 } 76 77 return true; 78 } 79 80 81 82 /** 83 * DOC: mem_info_vram_total 84 * 85 * The amdgpu driver provides a sysfs API for reporting current total VRAM 86 * available on the device 87 * The file mem_info_vram_total is used for this and returns the total 88 * amount of VRAM in bytes 89 */ 90 static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev, 91 struct device_attribute *attr, char *buf) 92 { 93 struct drm_device *ddev = dev_get_drvdata(dev); 94 struct amdgpu_device *adev = drm_to_adev(ddev); 95 96 return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size); 97 } 98 99 /** 100 * DOC: mem_info_vis_vram_total 101 * 102 * The amdgpu driver provides a sysfs API for reporting current total 103 * visible VRAM available on the device 104 * The file mem_info_vis_vram_total is used for this and returns the total 105 * amount of visible VRAM in bytes 106 */ 107 static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, 108 struct device_attribute *attr, char *buf) 109 { 110 struct drm_device *ddev = dev_get_drvdata(dev); 111 struct amdgpu_device *adev = drm_to_adev(ddev); 112 113 return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size); 114 } 115 116 /** 117 * DOC: mem_info_vram_used 118 * 119 * The amdgpu driver provides a sysfs API for reporting current total VRAM 120 * available on the device 121 * The file mem_info_vram_used is used for this and returns the total 122 * amount of currently used VRAM in bytes 123 */ 124 static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, 125 struct device_attribute *attr, 126 char *buf) 127 { 128 struct drm_device *ddev = dev_get_drvdata(dev); 129 struct amdgpu_device *adev = drm_to_adev(ddev); 130 struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager; 131 132 return sysfs_emit(buf, "%llu\n", ttm_resource_manager_usage(man)); 133 } 134 135 /** 136 * DOC: mem_info_vis_vram_used 137 * 138 * The amdgpu driver provides a sysfs API for reporting current total of 139 * used visible VRAM 140 * The file mem_info_vis_vram_used is used for this and returns the total 141 * amount of currently used visible VRAM in bytes 142 */ 143 static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev, 144 struct device_attribute *attr, 145 char *buf) 146 { 147 struct drm_device *ddev = dev_get_drvdata(dev); 148 struct amdgpu_device *adev = drm_to_adev(ddev); 149 150 return sysfs_emit(buf, "%llu\n", 151 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr)); 152 } 153 154 /** 155 * DOC: mem_info_vram_vendor 156 * 157 * The amdgpu driver provides a sysfs API for reporting the vendor of the 158 * installed VRAM 159 * The file mem_info_vram_vendor is used for this and returns the name of the 160 * vendor. 161 */ 162 static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev, 163 struct device_attribute *attr, 164 char *buf) 165 { 166 struct drm_device *ddev = dev_get_drvdata(dev); 167 struct amdgpu_device *adev = drm_to_adev(ddev); 168 169 switch (adev->gmc.vram_vendor) { 170 case SAMSUNG: 171 return sysfs_emit(buf, "samsung\n"); 172 case INFINEON: 173 return sysfs_emit(buf, "infineon\n"); 174 case ELPIDA: 175 return sysfs_emit(buf, "elpida\n"); 176 case ETRON: 177 return sysfs_emit(buf, "etron\n"); 178 case NANYA: 179 return sysfs_emit(buf, "nanya\n"); 180 case HYNIX: 181 return sysfs_emit(buf, "hynix\n"); 182 case MOSEL: 183 return sysfs_emit(buf, "mosel\n"); 184 case WINBOND: 185 return sysfs_emit(buf, "winbond\n"); 186 case ESMT: 187 return sysfs_emit(buf, "esmt\n"); 188 case MICRON: 189 return sysfs_emit(buf, "micron\n"); 190 default: 191 return sysfs_emit(buf, "unknown\n"); 192 } 193 } 194 195 static DEVICE_ATTR(mem_info_vram_total, S_IRUGO, 196 amdgpu_mem_info_vram_total_show, NULL); 197 static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO, 198 amdgpu_mem_info_vis_vram_total_show,NULL); 199 static DEVICE_ATTR(mem_info_vram_used, S_IRUGO, 200 amdgpu_mem_info_vram_used_show, NULL); 201 static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO, 202 amdgpu_mem_info_vis_vram_used_show, NULL); 203 static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO, 204 amdgpu_mem_info_vram_vendor, NULL); 205 206 static struct attribute *amdgpu_vram_mgr_attributes[] = { 207 &dev_attr_mem_info_vram_total.attr, 208 &dev_attr_mem_info_vis_vram_total.attr, 209 &dev_attr_mem_info_vram_used.attr, 210 &dev_attr_mem_info_vis_vram_used.attr, 211 &dev_attr_mem_info_vram_vendor.attr, 212 NULL 213 }; 214 215 const struct attribute_group amdgpu_vram_mgr_attr_group = { 216 .attrs = amdgpu_vram_mgr_attributes 217 }; 218 219 /** 220 * amdgpu_vram_mgr_vis_size - Calculate visible block size 221 * 222 * @adev: amdgpu_device pointer 223 * @block: DRM BUDDY block structure 224 * 225 * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM 226 */ 227 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev, 228 struct drm_buddy_block *block) 229 { 230 u64 start = amdgpu_vram_mgr_block_start(block); 231 u64 end = start + amdgpu_vram_mgr_block_size(block); 232 233 if (start >= adev->gmc.visible_vram_size) 234 return 0; 235 236 return (end > adev->gmc.visible_vram_size ? 237 adev->gmc.visible_vram_size : end) - start; 238 } 239 240 /** 241 * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size 242 * 243 * @bo: &amdgpu_bo buffer object (must be in VRAM) 244 * 245 * Returns: 246 * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM. 247 */ 248 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) 249 { 250 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 251 struct ttm_resource *res = bo->tbo.resource; 252 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); 253 struct drm_buddy_block *block; 254 u64 usage = 0; 255 256 if (amdgpu_gmc_vram_full_visible(&adev->gmc)) 257 return amdgpu_bo_size(bo); 258 259 if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT) 260 return 0; 261 262 list_for_each_entry(block, &vres->blocks, link) 263 usage += amdgpu_vram_mgr_vis_size(adev, block); 264 265 return usage; 266 } 267 268 /* Commit the reservation of VRAM pages */ 269 static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man) 270 { 271 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 272 struct amdgpu_device *adev = to_amdgpu_device(mgr); 273 struct drm_buddy *mm = &mgr->mm; 274 struct amdgpu_vram_reservation *rsv, *temp; 275 struct drm_buddy_block *block; 276 uint64_t vis_usage; 277 278 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) { 279 if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size, 280 rsv->size, mm->chunk_size, &rsv->allocated, 281 DRM_BUDDY_RANGE_ALLOCATION)) 282 continue; 283 284 block = amdgpu_vram_mgr_first_block(&rsv->allocated); 285 if (!block) 286 continue; 287 288 dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n", 289 rsv->start, rsv->size); 290 291 vis_usage = amdgpu_vram_mgr_vis_size(adev, block); 292 atomic64_add(vis_usage, &mgr->vis_usage); 293 spin_lock(&man->bdev->lru_lock); 294 man->usage += rsv->size; 295 spin_unlock(&man->bdev->lru_lock); 296 list_move(&rsv->blocks, &mgr->reserved_pages); 297 } 298 } 299 300 /** 301 * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM 302 * 303 * @mgr: amdgpu_vram_mgr pointer 304 * @start: start address of the range in VRAM 305 * @size: size of the range 306 * 307 * Reserve memory from start address with the specified size in VRAM 308 */ 309 int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr, 310 uint64_t start, uint64_t size) 311 { 312 struct amdgpu_vram_reservation *rsv; 313 314 rsv = kzalloc(sizeof(*rsv), GFP_KERNEL); 315 if (!rsv) 316 return -ENOMEM; 317 318 INIT_LIST_HEAD(&rsv->allocated); 319 INIT_LIST_HEAD(&rsv->blocks); 320 321 rsv->start = start; 322 rsv->size = size; 323 324 mutex_lock(&mgr->lock); 325 list_add_tail(&rsv->blocks, &mgr->reservations_pending); 326 amdgpu_vram_mgr_do_reserve(&mgr->manager); 327 mutex_unlock(&mgr->lock); 328 329 return 0; 330 } 331 332 /** 333 * amdgpu_vram_mgr_query_page_status - query the reservation status 334 * 335 * @mgr: amdgpu_vram_mgr pointer 336 * @start: start address of a page in VRAM 337 * 338 * Returns: 339 * -EBUSY: the page is still hold and in pending list 340 * 0: the page has been reserved 341 * -ENOENT: the input page is not a reservation 342 */ 343 int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr, 344 uint64_t start) 345 { 346 struct amdgpu_vram_reservation *rsv; 347 int ret; 348 349 mutex_lock(&mgr->lock); 350 351 list_for_each_entry(rsv, &mgr->reservations_pending, blocks) { 352 if (rsv->start <= start && 353 (start < (rsv->start + rsv->size))) { 354 ret = -EBUSY; 355 goto out; 356 } 357 } 358 359 list_for_each_entry(rsv, &mgr->reserved_pages, blocks) { 360 if (rsv->start <= start && 361 (start < (rsv->start + rsv->size))) { 362 ret = 0; 363 goto out; 364 } 365 } 366 367 ret = -ENOENT; 368 out: 369 mutex_unlock(&mgr->lock); 370 return ret; 371 } 372 373 /** 374 * amdgpu_vram_mgr_new - allocate new ranges 375 * 376 * @man: TTM memory type manager 377 * @tbo: TTM BO we need this range for 378 * @place: placement flags and restrictions 379 * @res: the resulting mem object 380 * 381 * Allocate VRAM for the given BO. 382 */ 383 static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, 384 struct ttm_buffer_object *tbo, 385 const struct ttm_place *place, 386 struct ttm_resource **res) 387 { 388 u64 vis_usage = 0, max_bytes, cur_size, min_block_size; 389 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 390 struct amdgpu_device *adev = to_amdgpu_device(mgr); 391 struct amdgpu_vram_mgr_resource *vres; 392 u64 size, remaining_size, lpfn, fpfn; 393 struct drm_buddy *mm = &mgr->mm; 394 struct drm_buddy_block *block; 395 unsigned long pages_per_block; 396 int r; 397 398 lpfn = (u64)place->lpfn << PAGE_SHIFT; 399 if (!lpfn) 400 lpfn = man->size; 401 402 fpfn = (u64)place->fpfn << PAGE_SHIFT; 403 404 max_bytes = adev->gmc.mc_vram_size; 405 if (tbo->type != ttm_bo_type_kernel) 406 max_bytes -= AMDGPU_VM_RESERVED_VRAM; 407 408 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 409 pages_per_block = ~0ul; 410 } else { 411 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 412 pages_per_block = HPAGE_PMD_NR; 413 #else 414 /* default to 2MB */ 415 pages_per_block = 2UL << (20UL - PAGE_SHIFT); 416 #endif 417 pages_per_block = max_t(uint32_t, pages_per_block, 418 tbo->page_alignment); 419 } 420 421 vres = kzalloc(sizeof(*vres), GFP_KERNEL); 422 if (!vres) 423 return -ENOMEM; 424 425 ttm_resource_init(tbo, place, &vres->base); 426 427 /* bail out quickly if there's likely not enough VRAM for this BO */ 428 if (ttm_resource_manager_usage(man) > max_bytes) { 429 r = -ENOSPC; 430 goto error_fini; 431 } 432 433 INIT_LIST_HEAD(&vres->blocks); 434 435 if (place->flags & TTM_PL_FLAG_TOPDOWN) 436 vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; 437 438 if (fpfn || lpfn != mgr->mm.size) 439 /* Allocate blocks in desired range */ 440 vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; 441 442 remaining_size = (u64)vres->base.size; 443 444 mutex_lock(&mgr->lock); 445 while (remaining_size) { 446 if (tbo->page_alignment) 447 min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT; 448 else 449 min_block_size = mgr->default_page_size; 450 451 BUG_ON(min_block_size < mm->chunk_size); 452 453 /* Limit maximum size to 2GiB due to SG table limitations */ 454 size = min(remaining_size, 2ULL << 30); 455 456 if ((size >= (u64)pages_per_block << PAGE_SHIFT) && 457 !(size & (((u64)pages_per_block << PAGE_SHIFT) - 1))) 458 min_block_size = (u64)pages_per_block << PAGE_SHIFT; 459 460 cur_size = size; 461 462 if (fpfn + size != (u64)place->lpfn << PAGE_SHIFT) { 463 /* 464 * Except for actual range allocation, modify the size and 465 * min_block_size conforming to continuous flag enablement 466 */ 467 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 468 size = roundup_pow_of_two(size); 469 min_block_size = size; 470 /* 471 * Modify the size value if size is not 472 * aligned with min_block_size 473 */ 474 } else if (!IS_ALIGNED(size, min_block_size)) { 475 size = round_up(size, min_block_size); 476 } 477 } 478 479 r = drm_buddy_alloc_blocks(mm, fpfn, 480 lpfn, 481 size, 482 min_block_size, 483 &vres->blocks, 484 vres->flags); 485 if (unlikely(r)) 486 goto error_free_blocks; 487 488 if (size > remaining_size) 489 remaining_size = 0; 490 else 491 remaining_size -= size; 492 } 493 mutex_unlock(&mgr->lock); 494 495 if (cur_size != size) { 496 struct drm_buddy_block *block; 497 struct list_head *trim_list; 498 u64 original_size; 499 LIST_HEAD(temp); 500 501 trim_list = &vres->blocks; 502 original_size = (u64)vres->base.size; 503 504 /* 505 * If size value is rounded up to min_block_size, trim the last 506 * block to the required size 507 */ 508 if (!list_is_singular(&vres->blocks)) { 509 block = list_last_entry(&vres->blocks, typeof(*block), link); 510 list_move_tail(&block->link, &temp); 511 trim_list = &temp; 512 /* 513 * Compute the original_size value by subtracting the 514 * last block size with (aligned size - original size) 515 */ 516 original_size = amdgpu_vram_mgr_block_size(block) - (size - cur_size); 517 } 518 519 mutex_lock(&mgr->lock); 520 drm_buddy_block_trim(mm, 521 original_size, 522 trim_list); 523 mutex_unlock(&mgr->lock); 524 525 if (!list_empty(&temp)) 526 list_splice_tail(trim_list, &vres->blocks); 527 } 528 529 vres->base.start = 0; 530 list_for_each_entry(block, &vres->blocks, link) { 531 unsigned long start; 532 533 start = amdgpu_vram_mgr_block_start(block) + 534 amdgpu_vram_mgr_block_size(block); 535 start >>= PAGE_SHIFT; 536 537 if (start > PFN_UP(vres->base.size)) 538 start -= PFN_UP(vres->base.size); 539 else 540 start = 0; 541 vres->base.start = max(vres->base.start, start); 542 543 vis_usage += amdgpu_vram_mgr_vis_size(adev, block); 544 } 545 546 if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks)) 547 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; 548 549 if (adev->gmc.xgmi.connected_to_cpu) 550 vres->base.bus.caching = ttm_cached; 551 else 552 vres->base.bus.caching = ttm_write_combined; 553 554 atomic64_add(vis_usage, &mgr->vis_usage); 555 *res = &vres->base; 556 return 0; 557 558 error_free_blocks: 559 drm_buddy_free_list(mm, &vres->blocks); 560 mutex_unlock(&mgr->lock); 561 error_fini: 562 ttm_resource_fini(man, &vres->base); 563 kfree(vres); 564 565 return r; 566 } 567 568 /** 569 * amdgpu_vram_mgr_del - free ranges 570 * 571 * @man: TTM memory type manager 572 * @res: TTM memory object 573 * 574 * Free the allocated VRAM again. 575 */ 576 static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man, 577 struct ttm_resource *res) 578 { 579 struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res); 580 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 581 struct amdgpu_device *adev = to_amdgpu_device(mgr); 582 struct drm_buddy *mm = &mgr->mm; 583 struct drm_buddy_block *block; 584 uint64_t vis_usage = 0; 585 586 mutex_lock(&mgr->lock); 587 list_for_each_entry(block, &vres->blocks, link) 588 vis_usage += amdgpu_vram_mgr_vis_size(adev, block); 589 590 amdgpu_vram_mgr_do_reserve(man); 591 592 drm_buddy_free_list(mm, &vres->blocks); 593 mutex_unlock(&mgr->lock); 594 595 atomic64_sub(vis_usage, &mgr->vis_usage); 596 597 ttm_resource_fini(man, res); 598 kfree(vres); 599 } 600 601 /** 602 * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table 603 * 604 * @adev: amdgpu device pointer 605 * @res: TTM memory object 606 * @offset: byte offset from the base of VRAM BO 607 * @length: number of bytes to export in sg_table 608 * @dev: the other device 609 * @dir: dma direction 610 * @sgt: resulting sg table 611 * 612 * Allocate and fill a sg table from a VRAM allocation. 613 */ 614 int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev, 615 struct ttm_resource *res, 616 u64 offset, u64 length, 617 struct device *dev, 618 enum dma_data_direction dir, 619 struct sg_table **sgt) 620 { 621 struct amdgpu_res_cursor cursor; 622 struct scatterlist *sg; 623 int num_entries = 0; 624 int i, r; 625 626 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL); 627 if (!*sgt) 628 return -ENOMEM; 629 630 /* Determine the number of DRM_BUDDY blocks to export */ 631 amdgpu_res_first(res, offset, length, &cursor); 632 while (cursor.remaining) { 633 num_entries++; 634 amdgpu_res_next(&cursor, cursor.size); 635 } 636 637 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); 638 if (r) 639 goto error_free; 640 641 /* Initialize scatterlist nodes of sg_table */ 642 for_each_sgtable_sg((*sgt), sg, i) 643 sg->length = 0; 644 645 /* 646 * Walk down DRM_BUDDY blocks to populate scatterlist nodes 647 * @note: Use iterator api to get first the DRM_BUDDY block 648 * and the number of bytes from it. Access the following 649 * DRM_BUDDY block(s) if more buffer needs to exported 650 */ 651 amdgpu_res_first(res, offset, length, &cursor); 652 for_each_sgtable_sg((*sgt), sg, i) { 653 phys_addr_t phys = cursor.start + adev->gmc.aper_base; 654 size_t size = cursor.size; 655 dma_addr_t addr; 656 657 addr = dma_map_resource(dev, phys, size, dir, 658 DMA_ATTR_SKIP_CPU_SYNC); 659 r = dma_mapping_error(dev, addr); 660 if (r) 661 goto error_unmap; 662 663 sg_set_page(sg, NULL, size, 0); 664 sg_dma_address(sg) = addr; 665 sg_dma_len(sg) = size; 666 667 amdgpu_res_next(&cursor, cursor.size); 668 } 669 670 return 0; 671 672 error_unmap: 673 for_each_sgtable_sg((*sgt), sg, i) { 674 if (!sg->length) 675 continue; 676 677 dma_unmap_resource(dev, sg->dma_address, 678 sg->length, dir, 679 DMA_ATTR_SKIP_CPU_SYNC); 680 } 681 sg_free_table(*sgt); 682 683 error_free: 684 kfree(*sgt); 685 return r; 686 } 687 688 /** 689 * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table 690 * 691 * @dev: device pointer 692 * @dir: data direction of resource to unmap 693 * @sgt: sg table to free 694 * 695 * Free a previously allocate sg table. 696 */ 697 void amdgpu_vram_mgr_free_sgt(struct device *dev, 698 enum dma_data_direction dir, 699 struct sg_table *sgt) 700 { 701 struct scatterlist *sg; 702 int i; 703 704 for_each_sgtable_sg(sgt, sg, i) 705 dma_unmap_resource(dev, sg->dma_address, 706 sg->length, dir, 707 DMA_ATTR_SKIP_CPU_SYNC); 708 sg_free_table(sgt); 709 kfree(sgt); 710 } 711 712 /** 713 * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part 714 * 715 * @mgr: amdgpu_vram_mgr pointer 716 * 717 * Returns how many bytes are used in the visible part of VRAM 718 */ 719 uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr) 720 { 721 return atomic64_read(&mgr->vis_usage); 722 } 723 724 /** 725 * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection 726 * 727 * @man: TTM memory type manager 728 * @res: The resource to test 729 * @place: The place to test against 730 * @size: Size of the new allocation 731 * 732 * Test each drm buddy block for intersection for eviction decision. 733 */ 734 static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man, 735 struct ttm_resource *res, 736 const struct ttm_place *place, 737 size_t size) 738 { 739 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); 740 struct drm_buddy_block *block; 741 742 /* Check each drm buddy block individually */ 743 list_for_each_entry(block, &mgr->blocks, link) { 744 unsigned long fpfn = 745 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; 746 unsigned long lpfn = fpfn + 747 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT); 748 749 if (place->fpfn < lpfn && 750 (!place->lpfn || place->lpfn > fpfn)) 751 return true; 752 } 753 754 return false; 755 } 756 757 /** 758 * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility 759 * 760 * @man: TTM memory type manager 761 * @res: The resource to test 762 * @place: The place to test against 763 * @size: Size of the new allocation 764 * 765 * Test each drm buddy block for placement compatibility. 766 */ 767 static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man, 768 struct ttm_resource *res, 769 const struct ttm_place *place, 770 size_t size) 771 { 772 struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res); 773 struct drm_buddy_block *block; 774 775 /* Check each drm buddy block individually */ 776 list_for_each_entry(block, &mgr->blocks, link) { 777 unsigned long fpfn = 778 amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT; 779 unsigned long lpfn = fpfn + 780 (amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT); 781 782 if (fpfn < place->fpfn || 783 (place->lpfn && lpfn > place->lpfn)) 784 return false; 785 } 786 787 return true; 788 } 789 790 /** 791 * amdgpu_vram_mgr_debug - dump VRAM table 792 * 793 * @man: TTM memory type manager 794 * @printer: DRM printer to use 795 * 796 * Dump the table content using printk. 797 */ 798 static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man, 799 struct drm_printer *printer) 800 { 801 struct amdgpu_vram_mgr *mgr = to_vram_mgr(man); 802 struct drm_buddy *mm = &mgr->mm; 803 struct drm_buddy_block *block; 804 805 drm_printf(printer, " vis usage:%llu\n", 806 amdgpu_vram_mgr_vis_usage(mgr)); 807 808 mutex_lock(&mgr->lock); 809 drm_printf(printer, "default_page_size: %lluKiB\n", 810 mgr->default_page_size >> 10); 811 812 drm_buddy_print(mm, printer); 813 814 drm_printf(printer, "reserved:\n"); 815 list_for_each_entry(block, &mgr->reserved_pages, link) 816 drm_buddy_block_print(mm, block, printer); 817 mutex_unlock(&mgr->lock); 818 } 819 820 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { 821 .alloc = amdgpu_vram_mgr_new, 822 .free = amdgpu_vram_mgr_del, 823 .intersects = amdgpu_vram_mgr_intersects, 824 .compatible = amdgpu_vram_mgr_compatible, 825 .debug = amdgpu_vram_mgr_debug 826 }; 827 828 /** 829 * amdgpu_vram_mgr_init - init VRAM manager and DRM MM 830 * 831 * @adev: amdgpu_device pointer 832 * 833 * Allocate and initialize the VRAM manager. 834 */ 835 int amdgpu_vram_mgr_init(struct amdgpu_device *adev) 836 { 837 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 838 struct ttm_resource_manager *man = &mgr->manager; 839 int err; 840 841 ttm_resource_manager_init(man, &adev->mman.bdev, 842 adev->gmc.real_vram_size); 843 844 man->func = &amdgpu_vram_mgr_func; 845 846 err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE); 847 if (err) 848 return err; 849 850 mutex_init(&mgr->lock); 851 INIT_LIST_HEAD(&mgr->reservations_pending); 852 INIT_LIST_HEAD(&mgr->reserved_pages); 853 mgr->default_page_size = PAGE_SIZE; 854 855 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager); 856 ttm_resource_manager_set_used(man, true); 857 return 0; 858 } 859 860 /** 861 * amdgpu_vram_mgr_fini - free and destroy VRAM manager 862 * 863 * @adev: amdgpu_device pointer 864 * 865 * Destroy and free the VRAM manager, returns -EBUSY if ranges are still 866 * allocated inside it. 867 */ 868 void amdgpu_vram_mgr_fini(struct amdgpu_device *adev) 869 { 870 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr; 871 struct ttm_resource_manager *man = &mgr->manager; 872 int ret; 873 struct amdgpu_vram_reservation *rsv, *temp; 874 875 ttm_resource_manager_set_used(man, false); 876 877 ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man); 878 if (ret) 879 return; 880 881 mutex_lock(&mgr->lock); 882 list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) 883 kfree(rsv); 884 885 list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) { 886 drm_buddy_free_list(&mgr->mm, &rsv->allocated); 887 kfree(rsv); 888 } 889 drm_buddy_fini(&mgr->mm); 890 mutex_unlock(&mgr->lock); 891 892 ttm_resource_manager_cleanup(man); 893 ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL); 894 } 895