1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <linux/bug.h> 7 #include <linux/export.h> 8 #include <linux/kmemleak.h> 9 #include <linux/module.h> 10 #include <linux/sizes.h> 11 12 #include <linux/gpu_buddy.h> 13 14 /** 15 * gpu_buddy_assert - assert a condition in the buddy allocator 16 * @condition: condition expected to be true 17 * 18 * When CONFIG_KUNIT is enabled, evaluates @condition and, if false, triggers 19 * a WARN_ON() and also calls kunit_fail_current_test() so that any running 20 * kunit test is properly marked as failed. The stringified condition is 21 * included in the failure message for easy identification. 22 * 23 * When CONFIG_KUNIT is not enabled, this reduces to WARN_ON() so production 24 * builds retain the same warning semantics as before. 25 */ 26 #if IS_ENABLED(CONFIG_KUNIT) 27 #include <kunit/test-bug.h> 28 #define gpu_buddy_assert(condition) do { \ 29 if (WARN_ON(!(condition))) \ 30 kunit_fail_current_test("gpu_buddy_assert(" #condition ")"); \ 31 } while (0) 32 #else 33 #define gpu_buddy_assert(condition) WARN_ON(!(condition)) 34 #endif 35 36 static struct kmem_cache *slab_blocks; 37 38 static unsigned int 39 gpu_buddy_block_state(struct gpu_buddy_block *block) 40 { 41 return block->header & GPU_BUDDY_HEADER_STATE; 42 } 43 44 static bool 45 gpu_buddy_block_is_allocated(struct gpu_buddy_block *block) 46 { 47 return gpu_buddy_block_state(block) == GPU_BUDDY_ALLOCATED; 48 } 49 50 static bool 51 gpu_buddy_block_is_split(struct gpu_buddy_block *block) 52 { 53 return gpu_buddy_block_state(block) == GPU_BUDDY_SPLIT; 54 } 55 56 static unsigned int gpu_buddy_block_offset_alignment(struct gpu_buddy_block *block) 57 { 58 u64 offset = gpu_buddy_block_offset(block); 59 60 if (!offset) 61 /* 62 * __ffs64(0) is undefined; offset 0 is maximally aligned, so return 63 * a value greater than any possible alignment. 64 */ 65 return 64 + 1; 66 67 return __ffs64(offset); 68 } 69 70 RB_DECLARE_CALLBACKS_MAX(static, gpu_buddy_augment_cb, 71 struct gpu_buddy_block, rb, 72 unsigned int, subtree_max_alignment, 73 gpu_buddy_block_offset_alignment); 74 75 static struct gpu_buddy_block *gpu_block_alloc(struct gpu_buddy *mm, 76 struct gpu_buddy_block *parent, 77 unsigned int order, 78 u64 offset) 79 { 80 struct gpu_buddy_block *block; 81 82 BUG_ON(order > GPU_BUDDY_MAX_ORDER); 83 84 block = kmem_cache_zalloc(slab_blocks, GFP_KERNEL); 85 if (!block) 86 return NULL; 87 88 block->header = offset; 89 block->header |= order; 90 block->parent = parent; 91 92 RB_CLEAR_NODE(&block->rb); 93 94 BUG_ON(block->header & GPU_BUDDY_HEADER_UNUSED); 95 return block; 96 } 97 98 static void gpu_block_free(struct gpu_buddy *mm, 99 struct gpu_buddy_block *block) 100 { 101 kmem_cache_free(slab_blocks, block); 102 } 103 104 static enum gpu_buddy_free_tree 105 get_block_tree(struct gpu_buddy_block *block) 106 { 107 return gpu_buddy_block_is_clear(block) ? 108 GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE; 109 } 110 111 static struct gpu_buddy_block * 112 rbtree_get_free_block(const struct rb_node *node) 113 { 114 return node ? rb_entry(node, struct gpu_buddy_block, rb) : NULL; 115 } 116 117 static struct gpu_buddy_block * 118 rbtree_last_free_block(struct rb_root *root) 119 { 120 return rbtree_get_free_block(rb_last(root)); 121 } 122 123 static bool rbtree_is_empty(struct rb_root *root) 124 { 125 return RB_EMPTY_ROOT(root); 126 } 127 128 static void rbtree_insert(struct gpu_buddy *mm, 129 struct gpu_buddy_block *block, 130 enum gpu_buddy_free_tree tree) 131 { 132 struct rb_node **link, *parent = NULL; 133 unsigned int block_alignment, order; 134 struct gpu_buddy_block *node; 135 struct rb_root *root; 136 137 order = gpu_buddy_block_order(block); 138 block_alignment = gpu_buddy_block_offset_alignment(block); 139 140 root = &mm->free_trees[tree][order]; 141 link = &root->rb_node; 142 143 while (*link) { 144 parent = *link; 145 node = rbtree_get_free_block(parent); 146 /* 147 * Manual augmentation update during insertion traversal. Required 148 * because rb_insert_augmented() only calls rotate callback during 149 * rotations. This ensures all ancestors on the insertion path have 150 * correct subtree_max_alignment values. 151 */ 152 if (node->subtree_max_alignment < block_alignment) 153 node->subtree_max_alignment = block_alignment; 154 155 if (gpu_buddy_block_offset(block) < gpu_buddy_block_offset(node)) 156 link = &parent->rb_left; 157 else 158 link = &parent->rb_right; 159 } 160 161 block->subtree_max_alignment = block_alignment; 162 rb_link_node(&block->rb, parent, link); 163 rb_insert_augmented(&block->rb, root, &gpu_buddy_augment_cb); 164 } 165 166 static void rbtree_remove(struct gpu_buddy *mm, 167 struct gpu_buddy_block *block) 168 { 169 unsigned int order = gpu_buddy_block_order(block); 170 enum gpu_buddy_free_tree tree; 171 struct rb_root *root; 172 173 tree = get_block_tree(block); 174 root = &mm->free_trees[tree][order]; 175 176 rb_erase_augmented(&block->rb, root, &gpu_buddy_augment_cb); 177 RB_CLEAR_NODE(&block->rb); 178 } 179 180 static void clear_reset(struct gpu_buddy_block *block) 181 { 182 block->header &= ~GPU_BUDDY_HEADER_CLEAR; 183 } 184 185 static void mark_cleared(struct gpu_buddy_block *block) 186 { 187 block->header |= GPU_BUDDY_HEADER_CLEAR; 188 } 189 190 static void mark_allocated(struct gpu_buddy *mm, 191 struct gpu_buddy_block *block) 192 { 193 block->header &= ~GPU_BUDDY_HEADER_STATE; 194 block->header |= GPU_BUDDY_ALLOCATED; 195 196 mm->free_scoreboard[gpu_buddy_block_order(block)]--; 197 mm->used_scoreboard[gpu_buddy_block_order(block)]++; 198 199 rbtree_remove(mm, block); 200 } 201 202 static void mark_free(struct gpu_buddy *mm, 203 struct gpu_buddy_block *block) 204 { 205 enum gpu_buddy_free_tree tree; 206 207 if (gpu_buddy_block_is_allocated(block)) 208 mm->used_scoreboard[gpu_buddy_block_order(block)]--; 209 210 block->header &= ~GPU_BUDDY_HEADER_STATE; 211 block->header |= GPU_BUDDY_FREE; 212 213 mm->free_scoreboard[gpu_buddy_block_order(block)]++; 214 215 tree = get_block_tree(block); 216 rbtree_insert(mm, block, tree); 217 } 218 219 static void mark_split(struct gpu_buddy *mm, 220 struct gpu_buddy_block *block) 221 { 222 block->header &= ~GPU_BUDDY_HEADER_STATE; 223 block->header |= GPU_BUDDY_SPLIT; 224 225 mm->free_scoreboard[gpu_buddy_block_order(block)]--; 226 227 rbtree_remove(mm, block); 228 } 229 230 static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2) 231 { 232 return s1 <= e2 && e1 >= s2; 233 } 234 235 static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2) 236 { 237 return s1 <= s2 && e1 >= e2; 238 } 239 240 static struct gpu_buddy_block * 241 __get_buddy(struct gpu_buddy_block *block) 242 { 243 struct gpu_buddy_block *parent; 244 245 parent = block->parent; 246 if (!parent) 247 return NULL; 248 249 if (parent->left == block) 250 return parent->right; 251 252 return parent->left; 253 } 254 255 static unsigned int __gpu_buddy_free(struct gpu_buddy *mm, 256 struct gpu_buddy_block *block, 257 bool force_merge) 258 { 259 struct gpu_buddy_block *parent; 260 unsigned int order; 261 262 while ((parent = block->parent)) { 263 struct gpu_buddy_block *buddy; 264 265 buddy = __get_buddy(block); 266 267 if (!gpu_buddy_block_is_free(buddy)) 268 break; 269 270 if (!force_merge) { 271 /* 272 * Check the block and its buddy clear state and exit 273 * the loop if they both have the dissimilar state. 274 */ 275 if (gpu_buddy_block_is_clear(block) != 276 gpu_buddy_block_is_clear(buddy)) 277 break; 278 279 if (gpu_buddy_block_is_clear(block)) 280 mark_cleared(parent); 281 } 282 283 rbtree_remove(mm, buddy); 284 mm->free_scoreboard[gpu_buddy_block_order(buddy)]--; 285 if (force_merge && gpu_buddy_block_is_clear(buddy)) 286 mm->clear_avail -= gpu_buddy_block_size(mm, buddy); 287 288 if (gpu_buddy_block_is_allocated(block)) 289 mm->used_scoreboard[gpu_buddy_block_order(block)]--; 290 291 gpu_block_free(mm, block); 292 gpu_block_free(mm, buddy); 293 294 block = parent; 295 } 296 297 order = gpu_buddy_block_order(block); 298 mark_free(mm, block); 299 300 return order; 301 } 302 303 static int __force_merge(struct gpu_buddy *mm, 304 u64 start, 305 u64 end, 306 unsigned int min_order) 307 { 308 unsigned int tree, order; 309 int i; 310 311 if (!min_order) 312 return -ENOMEM; 313 314 if (min_order > mm->max_order) 315 return -EINVAL; 316 317 for_each_free_tree(tree) { 318 for (i = min_order - 1; i >= 0; i--) { 319 struct rb_node *iter = rb_last(&mm->free_trees[tree][i]); 320 321 while (iter) { 322 struct gpu_buddy_block *block, *buddy; 323 u64 block_start, block_end; 324 325 block = rbtree_get_free_block(iter); 326 iter = rb_prev(iter); 327 328 if (!block || !block->parent) 329 continue; 330 331 block_start = gpu_buddy_block_offset(block); 332 block_end = block_start + gpu_buddy_block_size(mm, block) - 1; 333 334 if (!contains(start, end, block_start, block_end)) 335 continue; 336 337 buddy = __get_buddy(block); 338 if (!gpu_buddy_block_is_free(buddy)) 339 continue; 340 341 gpu_buddy_assert(gpu_buddy_block_is_clear(block) != 342 gpu_buddy_block_is_clear(buddy)); 343 344 /* 345 * Advance to the next node when the current node is the buddy, 346 * as freeing the block will also remove its buddy from the tree. 347 */ 348 if (iter == &buddy->rb) 349 iter = rb_prev(iter); 350 351 rbtree_remove(mm, block); 352 mm->free_scoreboard[gpu_buddy_block_order(block)]--; 353 if (gpu_buddy_block_is_clear(block)) 354 mm->clear_avail -= gpu_buddy_block_size(mm, block); 355 356 order = __gpu_buddy_free(mm, block, true); 357 if (order >= min_order) 358 return 0; 359 } 360 } 361 } 362 363 return -ENOMEM; 364 } 365 366 /** 367 * gpu_buddy_init - init memory manager 368 * 369 * @mm: GPU buddy manager to initialize 370 * @size: size in bytes to manage 371 * @chunk_size: minimum page size in bytes for our allocations 372 * 373 * Initializes the memory manager and its resources. 374 * 375 * Returns: 376 * 0 on success, error code on failure. 377 */ 378 int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size) 379 { 380 unsigned int i, j, root_count = 0; 381 u64 offset = 0; 382 383 if (size < chunk_size) 384 return -EINVAL; 385 386 if (chunk_size < SZ_4K) 387 return -EINVAL; 388 389 if (!is_power_of_2(chunk_size)) 390 return -EINVAL; 391 392 size = round_down(size, chunk_size); 393 394 mm->size = size; 395 mm->avail = size; 396 mm->clear_avail = 0; 397 mm->chunk_size = chunk_size; 398 mm->max_order = ilog2(size) - ilog2(chunk_size); 399 400 BUG_ON(mm->max_order > GPU_BUDDY_MAX_ORDER); 401 402 mm->free_scoreboard = kcalloc(mm->max_order + 1, 403 sizeof(*mm->free_scoreboard), 404 GFP_KERNEL); 405 if (!mm->free_scoreboard) 406 return -ENOMEM; 407 408 mm->used_scoreboard = kcalloc(mm->max_order + 1, 409 sizeof(*mm->used_scoreboard), 410 GFP_KERNEL); 411 if (!mm->used_scoreboard) 412 goto out_free_free_scoreboard; 413 414 mm->free_trees = kmalloc_array(GPU_BUDDY_MAX_FREE_TREES, 415 sizeof(*mm->free_trees), 416 GFP_KERNEL); 417 if (!mm->free_trees) 418 goto out_free_used_scoreboard; 419 420 for_each_free_tree(i) { 421 mm->free_trees[i] = kmalloc_array(mm->max_order + 1, 422 sizeof(struct rb_root), 423 GFP_KERNEL); 424 if (!mm->free_trees[i]) 425 goto out_free_tree; 426 427 for (j = 0; j <= mm->max_order; ++j) 428 mm->free_trees[i][j] = RB_ROOT; 429 } 430 431 mm->n_roots = hweight64(size); 432 433 mm->roots = kmalloc_array(mm->n_roots, 434 sizeof(struct gpu_buddy_block *), 435 GFP_KERNEL); 436 if (!mm->roots) 437 goto out_free_tree; 438 439 /* 440 * Split into power-of-two blocks, in case we are given a size that is 441 * not itself a power-of-two. 442 */ 443 do { 444 struct gpu_buddy_block *root; 445 unsigned int order; 446 u64 root_size; 447 448 order = ilog2(size) - ilog2(chunk_size); 449 root_size = chunk_size << order; 450 451 root = gpu_block_alloc(mm, NULL, order, offset); 452 if (!root) 453 goto out_free_roots; 454 455 mark_free(mm, root); 456 457 BUG_ON(root_count > mm->max_order); 458 BUG_ON(gpu_buddy_block_size(mm, root) < chunk_size); 459 460 mm->roots[root_count] = root; 461 462 offset += root_size; 463 size -= root_size; 464 root_count++; 465 } while (size); 466 467 #ifdef CONFIG_LOCKDEP 468 mm->lock_dep_map = NULL; 469 #endif 470 return 0; 471 472 out_free_roots: 473 while (root_count--) 474 gpu_block_free(mm, mm->roots[root_count]); 475 kfree(mm->roots); 476 out_free_tree: 477 while (i--) 478 kfree(mm->free_trees[i]); 479 kfree(mm->free_trees); 480 out_free_used_scoreboard: 481 kfree(mm->used_scoreboard); 482 out_free_free_scoreboard: 483 kfree(mm->free_scoreboard); 484 return -ENOMEM; 485 } 486 EXPORT_SYMBOL(gpu_buddy_init); 487 488 /** 489 * gpu_buddy_fini - tear down the memory manager 490 * 491 * @mm: GPU buddy manager to free 492 * 493 * Cleanup memory manager resources and the freetree 494 */ 495 void gpu_buddy_fini(struct gpu_buddy *mm) 496 { 497 u64 root_size, size, start; 498 unsigned int order; 499 int i; 500 501 size = mm->size; 502 503 for (i = 0; i < mm->n_roots; ++i) { 504 order = ilog2(size) - ilog2(mm->chunk_size); 505 start = gpu_buddy_block_offset(mm->roots[i]); 506 __force_merge(mm, start, start + size, order); 507 508 gpu_buddy_assert(gpu_buddy_block_is_free(mm->roots[i])); 509 510 gpu_block_free(mm, mm->roots[i]); 511 512 root_size = mm->chunk_size << order; 513 size -= root_size; 514 } 515 516 gpu_buddy_assert(mm->avail == mm->size); 517 518 for (i = 0; i <= mm->max_order; ++i) 519 gpu_buddy_assert(!mm->used_scoreboard[i]); 520 521 for_each_free_tree(i) 522 kfree(mm->free_trees[i]); 523 kfree(mm->free_trees); 524 kfree(mm->roots); 525 kfree(mm->free_scoreboard); 526 kfree(mm->used_scoreboard); 527 } 528 EXPORT_SYMBOL(gpu_buddy_fini); 529 530 static int split_block(struct gpu_buddy *mm, 531 struct gpu_buddy_block *block) 532 { 533 unsigned int block_order = gpu_buddy_block_order(block) - 1; 534 u64 offset = gpu_buddy_block_offset(block); 535 536 BUG_ON(!gpu_buddy_block_is_free(block)); 537 BUG_ON(!gpu_buddy_block_order(block)); 538 539 block->left = gpu_block_alloc(mm, block, block_order, offset); 540 if (!block->left) 541 return -ENOMEM; 542 543 block->right = gpu_block_alloc(mm, block, block_order, 544 offset + (mm->chunk_size << block_order)); 545 if (!block->right) { 546 gpu_block_free(mm, block->left); 547 return -ENOMEM; 548 } 549 550 mark_split(mm, block); 551 552 if (gpu_buddy_block_is_clear(block)) { 553 mark_cleared(block->left); 554 mark_cleared(block->right); 555 clear_reset(block); 556 } 557 558 mark_free(mm, block->left); 559 mark_free(mm, block->right); 560 561 return 0; 562 } 563 564 /** 565 * gpu_buddy_reset_clear - reset blocks clear state 566 * 567 * @mm: GPU buddy manager 568 * @is_clear: blocks clear state 569 * 570 * Reset the clear state based on @is_clear value for each block 571 * in the freetree. 572 */ 573 void gpu_buddy_reset_clear(struct gpu_buddy *mm, bool is_clear) 574 { 575 enum gpu_buddy_free_tree src_tree, dst_tree; 576 u64 root_size, size, start; 577 unsigned int order; 578 int i; 579 580 gpu_buddy_driver_lock_held(mm); 581 size = mm->size; 582 for (i = 0; i < mm->n_roots; ++i) { 583 order = ilog2(size) - ilog2(mm->chunk_size); 584 start = gpu_buddy_block_offset(mm->roots[i]); 585 __force_merge(mm, start, start + size, order); 586 587 root_size = mm->chunk_size << order; 588 size -= root_size; 589 } 590 591 src_tree = is_clear ? GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE; 592 dst_tree = is_clear ? GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE; 593 594 for (i = 0; i <= mm->max_order; ++i) { 595 struct rb_root *root = &mm->free_trees[src_tree][i]; 596 struct gpu_buddy_block *block, *tmp; 597 598 rbtree_postorder_for_each_entry_safe(block, tmp, root, rb) { 599 rbtree_remove(mm, block); 600 if (is_clear) { 601 mark_cleared(block); 602 mm->clear_avail += gpu_buddy_block_size(mm, block); 603 } else { 604 clear_reset(block); 605 mm->clear_avail -= gpu_buddy_block_size(mm, block); 606 } 607 608 rbtree_insert(mm, block, dst_tree); 609 } 610 } 611 } 612 EXPORT_SYMBOL(gpu_buddy_reset_clear); 613 614 /** 615 * gpu_buddy_free_block - free a block 616 * 617 * @mm: GPU buddy manager 618 * @block: block to be freed 619 */ 620 void gpu_buddy_free_block(struct gpu_buddy *mm, 621 struct gpu_buddy_block *block) 622 { 623 gpu_buddy_driver_lock_held(mm); 624 BUG_ON(!gpu_buddy_block_is_allocated(block)); 625 mm->avail += gpu_buddy_block_size(mm, block); 626 if (gpu_buddy_block_is_clear(block)) 627 mm->clear_avail += gpu_buddy_block_size(mm, block); 628 629 __gpu_buddy_free(mm, block, false); 630 } 631 EXPORT_SYMBOL(gpu_buddy_free_block); 632 633 static void __gpu_buddy_free_list(struct gpu_buddy *mm, 634 struct list_head *objects, 635 bool mark_clear, 636 bool mark_dirty) 637 { 638 struct gpu_buddy_block *block, *on; 639 640 gpu_buddy_assert(!(mark_dirty && mark_clear)); 641 642 list_for_each_entry_safe(block, on, objects, link) { 643 if (mark_clear) 644 mark_cleared(block); 645 else if (mark_dirty) 646 clear_reset(block); 647 gpu_buddy_free_block(mm, block); 648 cond_resched(); 649 } 650 INIT_LIST_HEAD(objects); 651 } 652 653 static void gpu_buddy_free_list_internal(struct gpu_buddy *mm, 654 struct list_head *objects) 655 { 656 /* 657 * Don't touch the clear/dirty bit, since allocation is still internal 658 * at this point. For example we might have just failed part of the 659 * allocation. 660 */ 661 __gpu_buddy_free_list(mm, objects, false, false); 662 } 663 664 /** 665 * gpu_buddy_free_list - free blocks 666 * 667 * @mm: GPU buddy manager 668 * @objects: input list head to free blocks 669 * @flags: optional flags like GPU_BUDDY_CLEARED 670 */ 671 void gpu_buddy_free_list(struct gpu_buddy *mm, 672 struct list_head *objects, 673 unsigned int flags) 674 { 675 bool mark_clear = flags & GPU_BUDDY_CLEARED; 676 677 gpu_buddy_driver_lock_held(mm); 678 __gpu_buddy_free_list(mm, objects, mark_clear, !mark_clear); 679 } 680 EXPORT_SYMBOL(gpu_buddy_free_list); 681 682 static bool block_incompatible(struct gpu_buddy_block *block, unsigned int flags) 683 { 684 bool needs_clear = flags & GPU_BUDDY_CLEAR_ALLOCATION; 685 686 return needs_clear != gpu_buddy_block_is_clear(block); 687 } 688 689 static void __gpu_buddy_undo_splits(struct gpu_buddy *mm, 690 struct gpu_buddy_block *block) 691 { 692 struct gpu_buddy_block *buddy = __get_buddy(block); 693 694 if (buddy && 695 (gpu_buddy_block_is_free(block) && 696 gpu_buddy_block_is_free(buddy))) { 697 rbtree_remove(mm, block); 698 mm->free_scoreboard[gpu_buddy_block_order(block)]--; 699 __gpu_buddy_free(mm, block, false); 700 } 701 } 702 703 static struct gpu_buddy_block * 704 __alloc_range_bias(struct gpu_buddy *mm, 705 u64 start, u64 end, 706 unsigned int order, 707 unsigned long flags, 708 bool fallback) 709 { 710 u64 req_size = mm->chunk_size << order; 711 struct gpu_buddy_block *block; 712 LIST_HEAD(dfs); 713 int err; 714 int i; 715 716 end = end - 1; 717 718 for (i = 0; i < mm->n_roots; ++i) 719 list_add_tail(&mm->roots[i]->tmp_link, &dfs); 720 721 do { 722 u64 block_start; 723 u64 block_end; 724 725 block = list_first_entry_or_null(&dfs, 726 struct gpu_buddy_block, 727 tmp_link); 728 if (!block) 729 break; 730 731 list_del(&block->tmp_link); 732 733 if (gpu_buddy_block_order(block) < order) 734 continue; 735 736 block_start = gpu_buddy_block_offset(block); 737 block_end = block_start + gpu_buddy_block_size(mm, block) - 1; 738 739 if (!overlaps(start, end, block_start, block_end)) 740 continue; 741 742 if (gpu_buddy_block_is_allocated(block)) 743 continue; 744 745 if (block_start < start || block_end > end) { 746 u64 adjusted_start = max(block_start, start); 747 u64 adjusted_end = min(block_end, end); 748 749 if (round_down(adjusted_end + 1, req_size) <= 750 round_up(adjusted_start, req_size)) 751 continue; 752 } 753 754 if (!fallback && block_incompatible(block, flags)) 755 continue; 756 757 if (contains(start, end, block_start, block_end) && 758 order == gpu_buddy_block_order(block)) { 759 /* 760 * Find the free block within the range. 761 */ 762 if (gpu_buddy_block_is_free(block)) 763 return block; 764 765 continue; 766 } 767 768 if (!gpu_buddy_block_is_split(block)) { 769 err = split_block(mm, block); 770 if (unlikely(err)) 771 goto err_undo; 772 } 773 774 list_add(&block->right->tmp_link, &dfs); 775 list_add(&block->left->tmp_link, &dfs); 776 } while (1); 777 778 return ERR_PTR(-ENOSPC); 779 780 err_undo: 781 /* 782 * We really don't want to leave around a bunch of split blocks, since 783 * bigger is better, so make sure we merge everything back before we 784 * free the allocated blocks. 785 */ 786 __gpu_buddy_undo_splits(mm, block); 787 return ERR_PTR(err); 788 } 789 790 static struct gpu_buddy_block * 791 __gpu_buddy_alloc_range_bias(struct gpu_buddy *mm, 792 u64 start, u64 end, 793 unsigned int order, 794 unsigned long flags) 795 { 796 struct gpu_buddy_block *block; 797 bool fallback = false; 798 799 block = __alloc_range_bias(mm, start, end, order, 800 flags, fallback); 801 if (IS_ERR(block)) 802 return __alloc_range_bias(mm, start, end, order, 803 flags, !fallback); 804 805 return block; 806 } 807 808 static struct gpu_buddy_block * 809 get_maxblock(struct gpu_buddy *mm, 810 unsigned int order, 811 enum gpu_buddy_free_tree tree) 812 { 813 struct gpu_buddy_block *max_block = NULL, *block = NULL; 814 struct rb_root *root; 815 unsigned int i; 816 817 for (i = order; i <= mm->max_order; ++i) { 818 root = &mm->free_trees[tree][i]; 819 block = rbtree_last_free_block(root); 820 if (!block) 821 continue; 822 823 if (!max_block) { 824 max_block = block; 825 continue; 826 } 827 828 if (gpu_buddy_block_offset(block) > 829 gpu_buddy_block_offset(max_block)) { 830 max_block = block; 831 } 832 } 833 834 return max_block; 835 } 836 837 static struct gpu_buddy_block * 838 alloc_from_freetree(struct gpu_buddy *mm, 839 unsigned int order, 840 unsigned long flags) 841 { 842 struct gpu_buddy_block *block = NULL; 843 struct rb_root *root; 844 enum gpu_buddy_free_tree tree; 845 unsigned int tmp; 846 int err; 847 848 tree = (flags & GPU_BUDDY_CLEAR_ALLOCATION) ? 849 GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE; 850 851 if (flags & GPU_BUDDY_TOPDOWN_ALLOCATION) { 852 block = get_maxblock(mm, order, tree); 853 if (block) 854 /* Store the obtained block order */ 855 tmp = gpu_buddy_block_order(block); 856 } else { 857 for (tmp = order; tmp <= mm->max_order; ++tmp) { 858 /* Get RB tree root for this order and tree */ 859 root = &mm->free_trees[tree][tmp]; 860 block = rbtree_last_free_block(root); 861 if (block) 862 break; 863 } 864 } 865 866 if (!block) { 867 /* Try allocating from the other tree */ 868 tree = (tree == GPU_BUDDY_CLEAR_TREE) ? 869 GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE; 870 871 for (tmp = order; tmp <= mm->max_order; ++tmp) { 872 root = &mm->free_trees[tree][tmp]; 873 block = rbtree_last_free_block(root); 874 if (block) 875 break; 876 } 877 878 if (!block) 879 return ERR_PTR(-ENOSPC); 880 } 881 882 BUG_ON(!gpu_buddy_block_is_free(block)); 883 884 while (tmp != order) { 885 err = split_block(mm, block); 886 if (unlikely(err)) 887 goto err_undo; 888 889 block = block->right; 890 tmp--; 891 } 892 return block; 893 894 err_undo: 895 __gpu_buddy_undo_splits(mm, block); 896 return ERR_PTR(err); 897 } 898 899 static bool 900 gpu_buddy_can_offset_align(u64 size, u64 min_block_size) 901 { 902 return size < min_block_size && is_power_of_2(size); 903 } 904 905 static bool gpu_buddy_subtree_can_satisfy(struct rb_node *node, 906 unsigned int alignment) 907 { 908 struct gpu_buddy_block *block; 909 910 block = rbtree_get_free_block(node); 911 return block->subtree_max_alignment >= alignment; 912 } 913 914 static struct gpu_buddy_block * 915 gpu_buddy_find_block_aligned(struct gpu_buddy *mm, 916 enum gpu_buddy_free_tree tree, 917 unsigned int order, 918 unsigned int alignment, 919 unsigned long flags) 920 { 921 struct rb_root *root = &mm->free_trees[tree][order]; 922 struct rb_node *rb = root->rb_node; 923 924 while (rb) { 925 struct gpu_buddy_block *block = rbtree_get_free_block(rb); 926 struct rb_node *left_node = rb->rb_left, *right_node = rb->rb_right; 927 928 if (right_node) { 929 if (gpu_buddy_subtree_can_satisfy(right_node, alignment)) { 930 rb = right_node; 931 continue; 932 } 933 } 934 935 if (gpu_buddy_block_offset_alignment(block) >= alignment) 936 return block; 937 938 if (left_node) { 939 if (gpu_buddy_subtree_can_satisfy(left_node, alignment)) { 940 rb = left_node; 941 continue; 942 } 943 } 944 945 break; 946 } 947 948 return NULL; 949 } 950 951 static struct gpu_buddy_block * 952 gpu_buddy_offset_aligned_allocation(struct gpu_buddy *mm, 953 u64 size, 954 u64 min_block_size, 955 unsigned long flags) 956 { 957 struct gpu_buddy_block *block = NULL; 958 unsigned int order, tmp, alignment; 959 enum gpu_buddy_free_tree tree; 960 unsigned long pages; 961 int err; 962 963 alignment = ilog2(min_block_size); 964 pages = size >> ilog2(mm->chunk_size); 965 order = fls(pages) - 1; 966 967 tree = (flags & GPU_BUDDY_CLEAR_ALLOCATION) ? 968 GPU_BUDDY_CLEAR_TREE : GPU_BUDDY_DIRTY_TREE; 969 970 for (tmp = order; tmp <= mm->max_order; ++tmp) { 971 block = gpu_buddy_find_block_aligned(mm, tree, tmp, 972 alignment, flags); 973 if (!block) { 974 tree = (tree == GPU_BUDDY_CLEAR_TREE) ? 975 GPU_BUDDY_DIRTY_TREE : GPU_BUDDY_CLEAR_TREE; 976 block = gpu_buddy_find_block_aligned(mm, tree, tmp, 977 alignment, flags); 978 } 979 980 if (block) 981 break; 982 } 983 984 if (!block) 985 return ERR_PTR(-ENOSPC); 986 987 while (gpu_buddy_block_order(block) > order) { 988 struct gpu_buddy_block *left, *right; 989 990 err = split_block(mm, block); 991 if (unlikely(err)) 992 goto err_undo; 993 994 left = block->left; 995 right = block->right; 996 997 if (gpu_buddy_block_offset_alignment(right) >= alignment) 998 block = right; 999 else 1000 block = left; 1001 } 1002 1003 return block; 1004 1005 err_undo: 1006 /* 1007 * We really don't want to leave around a bunch of split blocks, since 1008 * bigger is better, so make sure we merge everything back before we 1009 * free the allocated blocks. 1010 */ 1011 __gpu_buddy_undo_splits(mm, block); 1012 return ERR_PTR(err); 1013 } 1014 1015 static int __alloc_range(struct gpu_buddy *mm, 1016 struct list_head *dfs, 1017 u64 start, u64 size, 1018 struct list_head *blocks, 1019 u64 *total_allocated_on_err) 1020 { 1021 struct gpu_buddy_block *block; 1022 u64 total_allocated = 0; 1023 LIST_HEAD(allocated); 1024 u64 end; 1025 int err; 1026 1027 end = start + size - 1; 1028 1029 do { 1030 u64 block_start; 1031 u64 block_end; 1032 1033 block = list_first_entry_or_null(dfs, 1034 struct gpu_buddy_block, 1035 tmp_link); 1036 if (!block) 1037 break; 1038 1039 list_del(&block->tmp_link); 1040 1041 block_start = gpu_buddy_block_offset(block); 1042 block_end = block_start + gpu_buddy_block_size(mm, block) - 1; 1043 1044 if (!overlaps(start, end, block_start, block_end)) 1045 continue; 1046 1047 if (gpu_buddy_block_is_allocated(block)) { 1048 err = -ENOSPC; 1049 goto err_free; 1050 } 1051 1052 if (contains(start, end, block_start, block_end)) { 1053 if (gpu_buddy_block_is_free(block)) { 1054 mark_allocated(mm, block); 1055 total_allocated += gpu_buddy_block_size(mm, block); 1056 mm->avail -= gpu_buddy_block_size(mm, block); 1057 if (gpu_buddy_block_is_clear(block)) 1058 mm->clear_avail -= gpu_buddy_block_size(mm, block); 1059 list_add_tail(&block->link, &allocated); 1060 continue; 1061 } else if (!mm->clear_avail) { 1062 err = -ENOSPC; 1063 goto err_free; 1064 } 1065 } 1066 1067 if (!gpu_buddy_block_is_split(block)) { 1068 err = split_block(mm, block); 1069 if (unlikely(err)) 1070 goto err_undo; 1071 } 1072 1073 list_add(&block->right->tmp_link, dfs); 1074 list_add(&block->left->tmp_link, dfs); 1075 } while (1); 1076 1077 if (total_allocated < size) { 1078 err = -ENOSPC; 1079 goto err_free; 1080 } 1081 1082 list_splice_tail(&allocated, blocks); 1083 1084 return 0; 1085 1086 err_undo: 1087 /* 1088 * We really don't want to leave around a bunch of split blocks, since 1089 * bigger is better, so make sure we merge everything back before we 1090 * free the allocated blocks. 1091 */ 1092 __gpu_buddy_undo_splits(mm, block); 1093 1094 err_free: 1095 if (err == -ENOSPC && total_allocated_on_err) { 1096 list_splice_tail(&allocated, blocks); 1097 *total_allocated_on_err = total_allocated; 1098 } else { 1099 gpu_buddy_free_list_internal(mm, &allocated); 1100 } 1101 1102 return err; 1103 } 1104 1105 static int __gpu_buddy_alloc_range(struct gpu_buddy *mm, 1106 u64 start, 1107 u64 size, 1108 u64 *total_allocated_on_err, 1109 struct list_head *blocks) 1110 { 1111 LIST_HEAD(dfs); 1112 int i; 1113 1114 for (i = 0; i < mm->n_roots; ++i) 1115 list_add_tail(&mm->roots[i]->tmp_link, &dfs); 1116 1117 return __alloc_range(mm, &dfs, start, size, 1118 blocks, total_allocated_on_err); 1119 } 1120 1121 static int __alloc_contig_try_harder(struct gpu_buddy *mm, 1122 u64 size, 1123 u64 min_block_size, 1124 struct list_head *blocks) 1125 { 1126 u64 rhs_offset, lhs_offset, lhs_size, filled; 1127 struct gpu_buddy_block *block; 1128 unsigned int tree, order; 1129 LIST_HEAD(blocks_lhs); 1130 unsigned long pages; 1131 u64 modify_size; 1132 int err; 1133 1134 modify_size = rounddown_pow_of_two(size); 1135 pages = modify_size >> ilog2(mm->chunk_size); 1136 order = fls(pages) - 1; 1137 if (order == 0) 1138 return -ENOSPC; 1139 1140 for_each_free_tree(tree) { 1141 struct rb_root *root; 1142 struct rb_node *iter; 1143 1144 root = &mm->free_trees[tree][order]; 1145 if (rbtree_is_empty(root)) 1146 continue; 1147 1148 iter = rb_last(root); 1149 while (iter) { 1150 block = rbtree_get_free_block(iter); 1151 1152 /* Allocate blocks traversing RHS */ 1153 rhs_offset = gpu_buddy_block_offset(block); 1154 err = __gpu_buddy_alloc_range(mm, rhs_offset, size, 1155 &filled, blocks); 1156 if (!err || err != -ENOSPC) 1157 return err; 1158 1159 lhs_size = max((size - filled), min_block_size); 1160 if (!IS_ALIGNED(lhs_size, min_block_size)) 1161 lhs_size = round_up(lhs_size, min_block_size); 1162 1163 /* Allocate blocks traversing LHS */ 1164 lhs_offset = gpu_buddy_block_offset(block) - lhs_size; 1165 err = __gpu_buddy_alloc_range(mm, lhs_offset, lhs_size, 1166 NULL, &blocks_lhs); 1167 if (!err) { 1168 list_splice(&blocks_lhs, blocks); 1169 return 0; 1170 } else if (err != -ENOSPC) { 1171 gpu_buddy_free_list_internal(mm, blocks); 1172 return err; 1173 } 1174 /* Free blocks for the next iteration */ 1175 gpu_buddy_free_list_internal(mm, blocks); 1176 1177 iter = rb_prev(iter); 1178 } 1179 } 1180 1181 return -ENOSPC; 1182 } 1183 1184 /** 1185 * gpu_buddy_block_trim - free unused pages 1186 * 1187 * @mm: GPU buddy manager 1188 * @start: start address to begin the trimming. 1189 * @new_size: original size requested 1190 * @blocks: Input and output list of allocated blocks. 1191 * MUST contain single block as input to be trimmed. 1192 * On success will contain the newly allocated blocks 1193 * making up the @new_size. Blocks always appear in 1194 * ascending order 1195 * 1196 * For contiguous allocation, we round up the size to the nearest 1197 * power of two value, drivers consume *actual* size, so remaining 1198 * portions are unused and can be optionally freed with this function 1199 * 1200 * Returns: 1201 * 0 on success, error code on failure. 1202 */ 1203 int gpu_buddy_block_trim(struct gpu_buddy *mm, 1204 u64 *start, 1205 u64 new_size, 1206 struct list_head *blocks) 1207 { 1208 struct gpu_buddy_block *parent; 1209 struct gpu_buddy_block *block; 1210 u64 block_start, block_end; 1211 LIST_HEAD(dfs); 1212 u64 new_start; 1213 int err; 1214 1215 gpu_buddy_driver_lock_held(mm); 1216 1217 if (!list_is_singular(blocks)) 1218 return -EINVAL; 1219 1220 block = list_first_entry(blocks, 1221 struct gpu_buddy_block, 1222 link); 1223 1224 block_start = gpu_buddy_block_offset(block); 1225 block_end = block_start + gpu_buddy_block_size(mm, block); 1226 1227 if (WARN_ON(!gpu_buddy_block_is_allocated(block))) 1228 return -EINVAL; 1229 1230 if (new_size > gpu_buddy_block_size(mm, block)) 1231 return -EINVAL; 1232 1233 if (!new_size || !IS_ALIGNED(new_size, mm->chunk_size)) 1234 return -EINVAL; 1235 1236 if (new_size == gpu_buddy_block_size(mm, block)) 1237 return 0; 1238 1239 new_start = block_start; 1240 if (start) { 1241 new_start = *start; 1242 1243 if (new_start < block_start) 1244 return -EINVAL; 1245 1246 if (!IS_ALIGNED(new_start, mm->chunk_size)) 1247 return -EINVAL; 1248 1249 if (range_overflows(new_start, new_size, block_end)) 1250 return -EINVAL; 1251 } 1252 1253 list_del(&block->link); 1254 mark_free(mm, block); 1255 mm->avail += gpu_buddy_block_size(mm, block); 1256 if (gpu_buddy_block_is_clear(block)) 1257 mm->clear_avail += gpu_buddy_block_size(mm, block); 1258 1259 /* Prevent recursively freeing this node */ 1260 parent = block->parent; 1261 block->parent = NULL; 1262 1263 list_add(&block->tmp_link, &dfs); 1264 err = __alloc_range(mm, &dfs, new_start, new_size, blocks, NULL); 1265 if (err) { 1266 mark_allocated(mm, block); 1267 mm->avail -= gpu_buddy_block_size(mm, block); 1268 if (gpu_buddy_block_is_clear(block)) 1269 mm->clear_avail -= gpu_buddy_block_size(mm, block); 1270 list_add(&block->link, blocks); 1271 } 1272 1273 block->parent = parent; 1274 return err; 1275 } 1276 EXPORT_SYMBOL(gpu_buddy_block_trim); 1277 1278 static struct gpu_buddy_block * 1279 __gpu_buddy_alloc_blocks(struct gpu_buddy *mm, 1280 u64 start, u64 end, 1281 u64 size, u64 min_block_size, 1282 unsigned int order, 1283 unsigned long flags) 1284 { 1285 if (flags & GPU_BUDDY_RANGE_ALLOCATION) 1286 /* Allocate traversing within the range */ 1287 return __gpu_buddy_alloc_range_bias(mm, start, end, 1288 order, flags); 1289 else if (size < min_block_size) 1290 /* Allocate from an offset-aligned region without size rounding */ 1291 return gpu_buddy_offset_aligned_allocation(mm, size, 1292 min_block_size, 1293 flags); 1294 else 1295 /* Allocate from freetree */ 1296 return alloc_from_freetree(mm, order, flags); 1297 } 1298 1299 /** 1300 * gpu_buddy_alloc_blocks - allocate power-of-two blocks 1301 * 1302 * @mm: GPU buddy manager to allocate from 1303 * @start: start of the allowed range for this block 1304 * @end: end of the allowed range for this block 1305 * @size: size of the allocation in bytes 1306 * @min_block_size: alignment of the allocation 1307 * @blocks: output list head to add allocated blocks 1308 * @flags: GPU_BUDDY_*_ALLOCATION flags 1309 * 1310 * alloc_range_bias() called on range limitations, which traverses 1311 * the tree and returns the desired block. 1312 * 1313 * alloc_from_freetree() called when *no* range restrictions 1314 * are enforced, which picks the block from the freetree. 1315 * 1316 * Returns: 1317 * 0 on success, error code on failure. 1318 */ 1319 int gpu_buddy_alloc_blocks(struct gpu_buddy *mm, 1320 u64 start, u64 end, u64 size, 1321 u64 min_block_size, 1322 struct list_head *blocks, 1323 unsigned long flags) 1324 { 1325 struct gpu_buddy_block *block = NULL; 1326 u64 original_size, original_min_size; 1327 unsigned int min_order, order; 1328 LIST_HEAD(allocated); 1329 unsigned long pages; 1330 int err; 1331 1332 gpu_buddy_driver_lock_held(mm); 1333 1334 if (size < mm->chunk_size) 1335 return -EINVAL; 1336 1337 if (min_block_size < mm->chunk_size) 1338 return -EINVAL; 1339 1340 if (!is_power_of_2(min_block_size)) 1341 return -EINVAL; 1342 1343 if (!IS_ALIGNED(start | end | size, mm->chunk_size)) 1344 return -EINVAL; 1345 1346 if (end > mm->size) 1347 return -EINVAL; 1348 1349 if (range_overflows(start, size, mm->size)) 1350 return -EINVAL; 1351 1352 /* Actual range allocation */ 1353 if (start + size == end) { 1354 if (!IS_ALIGNED(start | end, min_block_size)) 1355 return -EINVAL; 1356 1357 return __gpu_buddy_alloc_range(mm, start, size, NULL, blocks); 1358 } 1359 1360 original_size = size; 1361 original_min_size = min_block_size; 1362 1363 /* Roundup the size to power of 2 */ 1364 if (flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION) { 1365 size = roundup_pow_of_two(size); 1366 min_block_size = size; 1367 /* 1368 * Normalize the requested size to min_block_size for regular allocations. 1369 * Offset-aligned allocations intentionally skip size rounding. 1370 */ 1371 } else if (!gpu_buddy_can_offset_align(size, min_block_size)) { 1372 size = round_up(size, min_block_size); 1373 } 1374 1375 pages = size >> ilog2(mm->chunk_size); 1376 order = fls(pages) - 1; 1377 min_order = ilog2(min_block_size) - ilog2(mm->chunk_size); 1378 1379 if (order > mm->max_order || size > mm->size) { 1380 if ((flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION) && 1381 !(flags & GPU_BUDDY_RANGE_ALLOCATION)) 1382 return __alloc_contig_try_harder(mm, original_size, 1383 original_min_size, blocks); 1384 1385 return -EINVAL; 1386 } 1387 1388 do { 1389 order = min(order, (unsigned int)fls(pages) - 1); 1390 BUG_ON(order > mm->max_order); 1391 /* 1392 * Regular allocations must not allocate blocks smaller than min_block_size. 1393 * Offset-aligned allocations deliberately bypass this constraint. 1394 */ 1395 BUG_ON(size >= min_block_size && order < min_order); 1396 1397 do { 1398 unsigned int fallback_order; 1399 1400 block = __gpu_buddy_alloc_blocks(mm, start, 1401 end, 1402 size, 1403 min_block_size, 1404 order, 1405 flags); 1406 if (!IS_ERR(block)) 1407 break; 1408 1409 if (size < min_block_size) { 1410 fallback_order = order; 1411 } else if (order == min_order) { 1412 fallback_order = min_order; 1413 } else { 1414 order--; 1415 continue; 1416 } 1417 1418 /* Try allocation through force merge method */ 1419 if (mm->clear_avail && 1420 !__force_merge(mm, start, end, fallback_order)) { 1421 block = __gpu_buddy_alloc_blocks(mm, start, 1422 end, 1423 size, 1424 min_block_size, 1425 fallback_order, 1426 flags); 1427 if (!IS_ERR(block)) { 1428 order = fallback_order; 1429 break; 1430 } 1431 } 1432 1433 /* 1434 * Try contiguous block allocation through 1435 * try harder method. 1436 */ 1437 if (flags & GPU_BUDDY_CONTIGUOUS_ALLOCATION && 1438 !(flags & GPU_BUDDY_RANGE_ALLOCATION)) 1439 return __alloc_contig_try_harder(mm, 1440 original_size, 1441 original_min_size, 1442 blocks); 1443 err = -ENOSPC; 1444 goto err_free; 1445 } while (1); 1446 1447 mark_allocated(mm, block); 1448 mm->avail -= gpu_buddy_block_size(mm, block); 1449 if (gpu_buddy_block_is_clear(block)) 1450 mm->clear_avail -= gpu_buddy_block_size(mm, block); 1451 kmemleak_update_trace(block); 1452 list_add_tail(&block->link, &allocated); 1453 1454 pages -= BIT(order); 1455 1456 if (!pages) 1457 break; 1458 } while (1); 1459 1460 /* Trim the allocated block to the required size */ 1461 if (!(flags & GPU_BUDDY_TRIM_DISABLE) && 1462 original_size != size) { 1463 struct list_head *trim_list; 1464 LIST_HEAD(temp); 1465 u64 trim_size; 1466 1467 trim_list = &allocated; 1468 trim_size = original_size; 1469 1470 if (!list_is_singular(&allocated)) { 1471 block = list_last_entry(&allocated, typeof(*block), link); 1472 list_move(&block->link, &temp); 1473 trim_list = &temp; 1474 trim_size = gpu_buddy_block_size(mm, block) - 1475 (size - original_size); 1476 } 1477 1478 gpu_buddy_block_trim(mm, 1479 NULL, 1480 trim_size, 1481 trim_list); 1482 1483 if (!list_empty(&temp)) 1484 list_splice_tail(trim_list, &allocated); 1485 } 1486 1487 list_splice_tail(&allocated, blocks); 1488 return 0; 1489 1490 err_free: 1491 gpu_buddy_free_list_internal(mm, &allocated); 1492 return err; 1493 } 1494 EXPORT_SYMBOL(gpu_buddy_alloc_blocks); 1495 1496 /** 1497 * gpu_buddy_block_print - print block information 1498 * 1499 * @mm: GPU buddy manager 1500 * @block: GPU buddy block 1501 */ 1502 void gpu_buddy_block_print(struct gpu_buddy *mm, 1503 struct gpu_buddy_block *block) 1504 { 1505 u64 start = gpu_buddy_block_offset(block); 1506 u64 size = gpu_buddy_block_size(mm, block); 1507 1508 pr_info("%#018llx-%#018llx: %llu\n", start, start + size, size); 1509 } 1510 EXPORT_SYMBOL(gpu_buddy_block_print); 1511 1512 /** 1513 * gpu_buddy_print - print allocator state 1514 * 1515 * @mm: GPU buddy manager 1516 * @p: GPU printer to use 1517 */ 1518 void gpu_buddy_print(struct gpu_buddy *mm) 1519 { 1520 int order; 1521 1522 gpu_buddy_driver_lock_held(mm); 1523 pr_info("chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB, clear_free: %lluMiB\n", 1524 mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20); 1525 1526 for (order = mm->max_order; order >= 0; order--) { 1527 u64 free_count = mm->free_scoreboard[order]; 1528 u64 used_count = mm->used_scoreboard[order]; 1529 u64 block_size = mm->chunk_size << order; 1530 u64 free = free_count * block_size; 1531 u64 used = used_count * block_size; 1532 1533 if (block_size < SZ_1M) 1534 pr_info("order-%2d free: %8llu KiB, used: %8llu KiB, free_blocks: %llu, used_blocks: %llu\n", 1535 order, free >> 10, used >> 10, free_count, used_count); 1536 else 1537 pr_info("order-%2d free: %8llu MiB, used: %8llu MiB, free_blocks: %llu, used_blocks: %llu\n", 1538 order, free >> 20, used >> 20, free_count, used_count); 1539 } 1540 } 1541 EXPORT_SYMBOL(gpu_buddy_print); 1542 1543 static void gpu_buddy_module_exit(void) 1544 { 1545 kmem_cache_destroy(slab_blocks); 1546 } 1547 1548 static int __init gpu_buddy_module_init(void) 1549 { 1550 slab_blocks = KMEM_CACHE(gpu_buddy_block, 0); 1551 if (!slab_blocks) 1552 return -ENOMEM; 1553 1554 return 0; 1555 } 1556 1557 module_init(gpu_buddy_module_init); 1558 module_exit(gpu_buddy_module_exit); 1559 1560 MODULE_DESCRIPTION("GPU Buddy Allocator"); 1561 MODULE_LICENSE("Dual MIT/GPL"); 1562