1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2020 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 * Authors: Christian König 24 */ 25 26 /* Pooling of allocated pages is necessary because changing the caching 27 * attributes on x86 of the linear mapping requires a costly cross CPU TLB 28 * invalidate for those addresses. 29 * 30 * Additional to that allocations from the DMA coherent API are pooled as well 31 * cause they are rather slow compared to alloc_pages+map. 32 */ 33 34 #include <linux/export.h> 35 #include <linux/module.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/debugfs.h> 38 #include <linux/highmem.h> 39 #include <linux/sched/mm.h> 40 41 #ifdef CONFIG_X86 42 #include <asm/set_memory.h> 43 #endif 44 45 #include <drm/ttm/ttm_backup.h> 46 #include <drm/ttm/ttm_pool.h> 47 #include <drm/ttm/ttm_tt.h> 48 #include <drm/ttm/ttm_bo.h> 49 50 #include "ttm_module.h" 51 #include "ttm_pool_internal.h" 52 53 #ifdef CONFIG_FAULT_INJECTION 54 #include <linux/fault-inject.h> 55 static DECLARE_FAULT_ATTR(backup_fault_inject); 56 57 /* 58 * Exposed to ttm_backup.c so a mid-compound subpage can be made to fail 59 * with -ENOMEM, exercising the reactive split-and-retry fallback in 60 * ttm_pool_backup() for high-order backups. 61 */ 62 bool ttm_backup_fault_inject_folio(void) 63 { 64 return should_fail(&backup_fault_inject, 1); 65 } 66 #else 67 #define should_fail(...) false 68 69 bool ttm_backup_fault_inject_folio(void) 70 { 71 return false; 72 } 73 #endif 74 75 /** 76 * struct ttm_pool_dma - Helper object for coherent DMA mappings 77 * 78 * @addr: original DMA address returned for the mapping 79 * @vaddr: original vaddr return for the mapping and order in the lower bits 80 */ 81 struct ttm_pool_dma { 82 dma_addr_t addr; 83 unsigned long vaddr; 84 }; 85 86 /** 87 * struct ttm_pool_alloc_state - Current state of the tt page allocation process 88 * @pages: Pointer to the next tt page pointer to populate. 89 * @caching_divide: Pointer to the first page pointer whose page has a staged but 90 * not committed caching transition from write-back to @tt_caching. 91 * @dma_addr: Pointer to the next tt dma_address entry to populate if any. 92 * @remaining_pages: Remaining pages to populate. 93 * @tt_caching: The requested cpu-caching for the pages allocated. 94 */ 95 struct ttm_pool_alloc_state { 96 struct page **pages; 97 struct page **caching_divide; 98 dma_addr_t *dma_addr; 99 pgoff_t remaining_pages; 100 enum ttm_caching tt_caching; 101 }; 102 103 /** 104 * struct ttm_pool_tt_restore - State representing restore from backup 105 * @pool: The pool used for page allocation while restoring. 106 * @snapshot_alloc: A snapshot of the most recent struct ttm_pool_alloc_state. 107 * @alloced_page: Pointer to the page most recently allocated from a pool or system. 108 * @first_dma: The dma address corresponding to @alloced_page if dma_mapping 109 * is requested. 110 * @alloced_pages: The number of allocated pages present in the struct ttm_tt 111 * page vector from this restore session. 112 * @restored_pages: The number of 4K pages restored for @alloced_page (which 113 * is typically a multi-order page). 114 * @page_caching: The struct ttm_tt requested caching 115 * @order: The order of @alloced_page. 116 * 117 * Recovery from backup might fail when we've recovered less than the 118 * full ttm_tt. In order not to loose any data (yet), keep information 119 * around that allows us to restart a failed ttm backup recovery. 120 */ 121 struct ttm_pool_tt_restore { 122 struct ttm_pool *pool; 123 struct ttm_pool_alloc_state snapshot_alloc; 124 struct page *alloced_page; 125 dma_addr_t first_dma; 126 pgoff_t alloced_pages; 127 pgoff_t restored_pages; 128 enum ttm_caching page_caching; 129 unsigned int order; 130 }; 131 132 static unsigned long page_pool_size; 133 134 MODULE_PARM_DESC(page_pool_size, "Number of pages in the WC/UC/DMA pool per NUMA node"); 135 module_param(page_pool_size, ulong, 0644); 136 137 static unsigned long pool_node_limit[MAX_NUMNODES]; 138 static atomic_long_t allocated_pages[MAX_NUMNODES]; 139 140 static struct ttm_pool_type global_write_combined[NR_PAGE_ORDERS]; 141 static struct ttm_pool_type global_uncached[NR_PAGE_ORDERS]; 142 143 static struct ttm_pool_type global_dma32_write_combined[NR_PAGE_ORDERS]; 144 static struct ttm_pool_type global_dma32_uncached[NR_PAGE_ORDERS]; 145 146 static spinlock_t shrinker_lock; 147 static struct list_head shrinker_list; 148 static struct shrinker *mm_shrinker; 149 static DECLARE_RWSEM(pool_shrink_rwsem); 150 151 static int ttm_pool_nid(struct ttm_pool *pool) 152 { 153 int nid = NUMA_NO_NODE; 154 if (pool) 155 nid = pool->nid; 156 if (nid == NUMA_NO_NODE) 157 nid = numa_node_id(); 158 return nid; 159 } 160 161 /* Allocate pages of size 1 << order with the given gfp_flags */ 162 static struct page *ttm_pool_alloc_page(struct ttm_pool *pool, gfp_t gfp_flags, 163 unsigned int order) 164 { 165 const unsigned int beneficial_order = ttm_pool_beneficial_order(pool); 166 unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; 167 struct ttm_pool_dma *dma; 168 struct page *p; 169 void *vaddr; 170 171 /* Don't set the __GFP_COMP flag for higher order allocations. 172 * Mapping pages directly into an userspace process and calling 173 * put_page() on a TTM allocated page is illegal. 174 */ 175 if (order) 176 gfp_flags |= __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | 177 __GFP_THISNODE; 178 179 /* 180 * Do not add latency to the allocation path for allocations orders 181 * device tolds us do not bring them additional performance gains. 182 */ 183 if (beneficial_order && order > beneficial_order) 184 gfp_flags &= ~__GFP_DIRECT_RECLAIM; 185 186 if (!ttm_pool_uses_dma_alloc(pool)) { 187 p = alloc_pages_node(pool->nid, gfp_flags, order); 188 if (p) { 189 p->private = order; 190 mod_lruvec_page_state(p, NR_GPU_ACTIVE, 1 << order); 191 } 192 return p; 193 } 194 195 dma = kmalloc_obj(*dma); 196 if (!dma) 197 return NULL; 198 199 if (order) 200 attr |= DMA_ATTR_NO_WARN; 201 202 vaddr = dma_alloc_attrs(pool->dev, (1ULL << order) * PAGE_SIZE, 203 &dma->addr, gfp_flags, attr); 204 if (!vaddr) 205 goto error_free; 206 207 /* TODO: This is an illegal abuse of the DMA API, but we need to rework 208 * TTM page fault handling and extend the DMA API to clean this up. 209 */ 210 if (is_vmalloc_addr(vaddr)) 211 p = vmalloc_to_page(vaddr); 212 else 213 p = virt_to_page(vaddr); 214 215 dma->vaddr = (unsigned long)vaddr | order; 216 p->private = (unsigned long)dma; 217 return p; 218 219 error_free: 220 kfree(dma); 221 return NULL; 222 } 223 224 static void __free_pages_gpu_account(struct page *p, unsigned int order, 225 bool reclaim) 226 { 227 mod_lruvec_page_state(p, reclaim ? NR_GPU_RECLAIM : NR_GPU_ACTIVE, 228 -(1 << order)); 229 __free_pages(p, order); 230 } 231 232 /* Reset the caching and pages of size 1 << order */ 233 static void ttm_pool_free_page(struct ttm_pool *pool, enum ttm_caching caching, 234 unsigned int order, struct page *p, bool reclaim) 235 { 236 unsigned long attr = DMA_ATTR_FORCE_CONTIGUOUS; 237 struct ttm_pool_dma *dma; 238 void *vaddr; 239 240 #ifdef CONFIG_X86 241 /* We don't care that set_pages_wb is inefficient here. This is only 242 * used when we have to shrink and CPU overhead is irrelevant then. 243 */ 244 if (caching != ttm_cached && !PageHighMem(p)) 245 set_pages_wb(p, 1 << order); 246 #endif 247 248 if (!pool || !ttm_pool_uses_dma_alloc(pool)) { 249 __free_pages_gpu_account(p, order, reclaim); 250 return; 251 } 252 253 if (order) 254 attr |= DMA_ATTR_NO_WARN; 255 256 dma = (void *)p->private; 257 vaddr = (void *)(dma->vaddr & PAGE_MASK); 258 dma_free_attrs(pool->dev, (1UL << order) * PAGE_SIZE, vaddr, dma->addr, 259 attr); 260 kfree(dma); 261 } 262 263 /* Apply any cpu-caching deferred during page allocation */ 264 static int ttm_pool_apply_caching(struct ttm_pool_alloc_state *alloc) 265 { 266 #ifdef CONFIG_X86 267 unsigned int num_pages = alloc->pages - alloc->caching_divide; 268 269 if (!num_pages) 270 return 0; 271 272 switch (alloc->tt_caching) { 273 case ttm_cached: 274 break; 275 case ttm_write_combined: 276 return set_pages_array_wc(alloc->caching_divide, num_pages); 277 case ttm_uncached: 278 return set_pages_array_uc(alloc->caching_divide, num_pages); 279 } 280 #endif 281 alloc->caching_divide = alloc->pages; 282 return 0; 283 } 284 285 /* DMA Map pages of 1 << order size and return the resulting dma_address. */ 286 static int ttm_pool_map(struct ttm_pool *pool, unsigned int order, 287 struct page *p, dma_addr_t *dma_addr) 288 { 289 dma_addr_t addr; 290 291 if (ttm_pool_uses_dma_alloc(pool)) { 292 struct ttm_pool_dma *dma = (void *)p->private; 293 294 addr = dma->addr; 295 } else { 296 size_t size = (1ULL << order) * PAGE_SIZE; 297 298 addr = dma_map_page(pool->dev, p, 0, size, DMA_BIDIRECTIONAL); 299 if (dma_mapping_error(pool->dev, addr)) 300 return -EFAULT; 301 } 302 303 *dma_addr = addr; 304 305 return 0; 306 } 307 308 /* Unmap pages of 1 << order size */ 309 static void ttm_pool_unmap(struct ttm_pool *pool, dma_addr_t dma_addr, 310 unsigned int num_pages) 311 { 312 /* Unmapped while freeing the page */ 313 if (ttm_pool_uses_dma_alloc(pool)) 314 return; 315 316 dma_unmap_page(pool->dev, dma_addr, (long)num_pages << PAGE_SHIFT, 317 DMA_BIDIRECTIONAL); 318 } 319 320 /* Give pages into a specific pool_type */ 321 static void ttm_pool_type_give(struct ttm_pool_type *pt, struct page *p) 322 { 323 unsigned int i, num_pages = 1 << pt->order; 324 int nid = page_to_nid(p); 325 326 for (i = 0; i < num_pages; ++i) { 327 if (PageHighMem(p)) 328 clear_highpage(p + i); 329 else 330 clear_page(page_address(p + i)); 331 } 332 333 INIT_LIST_HEAD(&p->lru); 334 rcu_read_lock(); 335 list_lru_add(&pt->pages, &p->lru, nid, NULL); 336 rcu_read_unlock(); 337 338 atomic_long_add(num_pages, &allocated_pages[nid]); 339 mod_lruvec_page_state(p, NR_GPU_ACTIVE, -num_pages); 340 mod_lruvec_page_state(p, NR_GPU_RECLAIM, num_pages); 341 } 342 343 static enum lru_status take_one_from_lru(struct list_head *item, 344 struct list_lru_one *list, 345 void *cb_arg) 346 { 347 struct page **out_page = cb_arg; 348 struct page *p = container_of(item, struct page, lru); 349 list_lru_isolate(list, item); 350 351 *out_page = p; 352 return LRU_REMOVED; 353 } 354 355 /* Take pages from a specific pool_type, return NULL when nothing available */ 356 static struct page *ttm_pool_type_take(struct ttm_pool_type *pt, int nid) 357 { 358 int ret; 359 struct page *p = NULL; 360 unsigned long nr_to_walk = 1; 361 362 ret = list_lru_walk_node(&pt->pages, nid, take_one_from_lru, (void *)&p, &nr_to_walk); 363 if (ret == 1 && p) { 364 atomic_long_sub(1 << pt->order, &allocated_pages[nid]); 365 mod_lruvec_page_state(p, NR_GPU_ACTIVE, (1 << pt->order)); 366 mod_lruvec_page_state(p, NR_GPU_RECLAIM, -(1 << pt->order)); 367 } 368 return p; 369 } 370 371 /* Initialize and add a pool type to the global shrinker list */ 372 static void ttm_pool_type_init(struct ttm_pool_type *pt, struct ttm_pool *pool, 373 enum ttm_caching caching, unsigned int order) 374 { 375 pt->pool = pool; 376 pt->caching = caching; 377 pt->order = order; 378 list_lru_init(&pt->pages); 379 380 spin_lock(&shrinker_lock); 381 list_add_tail(&pt->shrinker_list, &shrinker_list); 382 spin_unlock(&shrinker_lock); 383 } 384 385 static enum lru_status pool_move_to_dispose_list(struct list_head *item, 386 struct list_lru_one *list, 387 void *cb_arg) 388 { 389 struct list_head *dispose = cb_arg; 390 391 list_lru_isolate_move(list, item, dispose); 392 393 return LRU_REMOVED; 394 } 395 396 static void ttm_pool_dispose_list(struct ttm_pool_type *pt, 397 struct list_head *dispose) 398 { 399 while (!list_empty(dispose)) { 400 struct page *p; 401 p = list_first_entry(dispose, struct page, lru); 402 list_del_init(&p->lru); 403 atomic_long_sub(1 << pt->order, &allocated_pages[page_to_nid(p)]); 404 ttm_pool_free_page(pt->pool, pt->caching, pt->order, p, true); 405 } 406 } 407 408 /* Remove a pool_type from the global shrinker list and free all pages */ 409 static void ttm_pool_type_fini(struct ttm_pool_type *pt) 410 { 411 LIST_HEAD(dispose); 412 413 spin_lock(&shrinker_lock); 414 list_del(&pt->shrinker_list); 415 spin_unlock(&shrinker_lock); 416 417 list_lru_walk(&pt->pages, pool_move_to_dispose_list, &dispose, LONG_MAX); 418 ttm_pool_dispose_list(pt, &dispose); 419 } 420 421 /* Return the pool_type to use for the given caching and order */ 422 static struct ttm_pool_type *ttm_pool_select_type(struct ttm_pool *pool, 423 enum ttm_caching caching, 424 unsigned int order) 425 { 426 if (ttm_pool_uses_dma_alloc(pool)) 427 return &pool->caching[caching].orders[order]; 428 429 #ifdef CONFIG_X86 430 switch (caching) { 431 case ttm_write_combined: 432 if (ttm_pool_uses_dma32(pool)) 433 return &global_dma32_write_combined[order]; 434 435 return &global_write_combined[order]; 436 case ttm_uncached: 437 if (ttm_pool_uses_dma32(pool)) 438 return &global_dma32_uncached[order]; 439 440 return &global_uncached[order]; 441 default: 442 break; 443 } 444 #endif 445 446 return NULL; 447 } 448 449 /* Free pages using the per-node shrinker list */ 450 static unsigned int ttm_pool_shrink(int nid, unsigned long num_to_free) 451 { 452 LIST_HEAD(dispose); 453 struct ttm_pool_type *pt; 454 unsigned int num_pages; 455 456 down_read(&pool_shrink_rwsem); 457 spin_lock(&shrinker_lock); 458 pt = list_first_entry(&shrinker_list, typeof(*pt), shrinker_list); 459 list_move_tail(&pt->shrinker_list, &shrinker_list); 460 spin_unlock(&shrinker_lock); 461 462 num_pages = list_lru_walk_node(&pt->pages, nid, pool_move_to_dispose_list, &dispose, &num_to_free); 463 num_pages *= 1 << pt->order; 464 465 ttm_pool_dispose_list(pt, &dispose); 466 up_read(&pool_shrink_rwsem); 467 468 return num_pages; 469 } 470 471 /* Return the allocation order based for a page */ 472 static unsigned int ttm_pool_page_order(struct ttm_pool *pool, struct page *p) 473 { 474 if (ttm_pool_uses_dma_alloc(pool)) { 475 struct ttm_pool_dma *dma = (void *)p->private; 476 477 return dma->vaddr & ~PAGE_MASK; 478 } 479 480 return p->private; 481 } 482 483 /* 484 * Split larger pages so that we can free each PAGE_SIZE page as soon 485 * as it has been backed up, in order to avoid memory pressure during 486 * reclaim. 487 */ 488 static void ttm_pool_split_for_swap(struct ttm_pool *pool, struct page *p) 489 { 490 unsigned int order = ttm_pool_page_order(pool, p); 491 pgoff_t nr; 492 493 if (!order) 494 return; 495 496 split_page(p, order); 497 nr = 1UL << order; 498 while (nr--) 499 (p++)->private = 0; 500 } 501 502 /** 503 * DOC: Partial backup and restoration of a struct ttm_tt. 504 * 505 * Swapout using ttm_backup_backup_folio() and swapin using 506 * ttm_backup_copy_page() may fail. 507 * The former most likely due to lack of swap-space or memory, the latter due 508 * to lack of memory or because of signal interruption during waits. 509 * 510 * Backup failure is easily handled by using a ttm_tt pages vector that holds 511 * both backup handles and page pointers. This has to be taken into account when 512 * restoring such a ttm_tt from backup, and when freeing it while backed up. 513 * When restoring, for simplicity, new pages are actually allocated from the 514 * pool and the contents of any old pages are copied in and then the old pages 515 * are released. 516 * 517 * For restoration failures, the struct ttm_pool_tt_restore holds sufficient state 518 * to be able to resume an interrupted restore, and that structure is freed once 519 * the restoration is complete. If the struct ttm_tt is destroyed while there 520 * is a valid struct ttm_pool_tt_restore attached, that is also properly taken 521 * care of. 522 */ 523 524 /* Is restore ongoing for the currently allocated page? */ 525 static bool ttm_pool_restore_valid(const struct ttm_pool_tt_restore *restore) 526 { 527 return restore && restore->restored_pages < (1 << restore->order); 528 } 529 530 /* DMA unmap and free a multi-order page, either to the relevant pool or to system. */ 531 static pgoff_t ttm_pool_unmap_and_free(struct ttm_pool *pool, struct page *page, 532 const dma_addr_t *dma_addr, enum ttm_caching caching) 533 { 534 struct ttm_pool_type *pt = NULL; 535 unsigned int order; 536 pgoff_t nr; 537 538 if (pool) { 539 order = ttm_pool_page_order(pool, page); 540 nr = (1UL << order); 541 if (dma_addr) 542 ttm_pool_unmap(pool, *dma_addr, nr); 543 544 pt = ttm_pool_select_type(pool, caching, order); 545 } else { 546 order = page->private; 547 nr = (1UL << order); 548 } 549 550 if (pt) 551 ttm_pool_type_give(pt, page); 552 else 553 ttm_pool_free_page(pool, caching, order, page, false); 554 555 return nr; 556 } 557 558 /* Populate the page-array using the most recent allocated multi-order page. */ 559 static void ttm_pool_allocated_page_commit(struct page *allocated, 560 dma_addr_t first_dma, 561 struct ttm_pool_alloc_state *alloc, 562 pgoff_t nr) 563 { 564 pgoff_t i; 565 566 for (i = 0; i < nr; ++i) 567 *alloc->pages++ = allocated++; 568 569 alloc->remaining_pages -= nr; 570 571 if (!alloc->dma_addr) 572 return; 573 574 for (i = 0; i < nr; ++i) { 575 *alloc->dma_addr++ = first_dma; 576 first_dma += PAGE_SIZE; 577 } 578 } 579 580 /* 581 * When restoring, restore backed-up content to the newly allocated page and 582 * if successful, populate the page-table and dma-address arrays. 583 */ 584 static int ttm_pool_restore_commit(struct ttm_pool_tt_restore *restore, 585 struct file *backup, 586 const struct ttm_operation_ctx *ctx, 587 struct ttm_pool_alloc_state *alloc) 588 589 { 590 pgoff_t i, nr = 1UL << restore->order; 591 struct page **first_page = alloc->pages; 592 struct page *p; 593 int ret = 0; 594 595 for (i = restore->restored_pages; i < nr; ++i) { 596 p = first_page[i]; 597 if (ttm_backup_page_ptr_is_handle(p)) { 598 unsigned long handle = ttm_backup_page_ptr_to_handle(p); 599 gfp_t additional_gfp = ctx->gfp_retry_mayfail ? 600 __GFP_RETRY_MAYFAIL | __GFP_NOWARN : 0; 601 602 if (IS_ENABLED(CONFIG_FAULT_INJECTION) && ctx->interruptible && 603 should_fail(&backup_fault_inject, 1)) { 604 ret = -EINTR; 605 break; 606 } 607 608 if (handle == 0) { 609 restore->restored_pages++; 610 continue; 611 } 612 613 ret = ttm_backup_copy_page(backup, restore->alloced_page + i, 614 handle, ctx->interruptible, 615 additional_gfp); 616 if (ret) 617 break; 618 619 ttm_backup_drop(backup, handle); 620 } else if (p) { 621 /* 622 * We could probably avoid splitting the old page 623 * using clever logic, but ATM we don't care, as 624 * we prioritize releasing memory ASAP. Note that 625 * here, the old retained page is always write-back 626 * cached. 627 */ 628 ttm_pool_split_for_swap(restore->pool, p); 629 copy_highpage(restore->alloced_page + i, p); 630 __free_pages_gpu_account(p, 0, false); 631 } 632 633 restore->restored_pages++; 634 first_page[i] = ttm_backup_handle_to_page_ptr(0); 635 } 636 637 if (ret) { 638 if (!restore->restored_pages) { 639 dma_addr_t *dma_addr = alloc->dma_addr ? &restore->first_dma : NULL; 640 641 ttm_pool_unmap_and_free(restore->pool, restore->alloced_page, 642 dma_addr, restore->page_caching); 643 restore->restored_pages = nr; 644 } 645 return ret; 646 } 647 648 ttm_pool_allocated_page_commit(restore->alloced_page, restore->first_dma, 649 alloc, nr); 650 if (restore->page_caching == alloc->tt_caching || PageHighMem(restore->alloced_page)) 651 alloc->caching_divide = alloc->pages; 652 restore->snapshot_alloc = *alloc; 653 restore->alloced_pages += nr; 654 655 return 0; 656 } 657 658 /* If restoring, save information needed for ttm_pool_restore_commit(). */ 659 static void 660 ttm_pool_page_allocated_restore(struct ttm_pool *pool, unsigned int order, 661 struct page *p, 662 enum ttm_caching page_caching, 663 dma_addr_t first_dma, 664 struct ttm_pool_tt_restore *restore, 665 const struct ttm_pool_alloc_state *alloc) 666 { 667 restore->pool = pool; 668 restore->order = order; 669 restore->restored_pages = 0; 670 restore->page_caching = page_caching; 671 restore->first_dma = first_dma; 672 restore->alloced_page = p; 673 restore->snapshot_alloc = *alloc; 674 } 675 676 /* 677 * Called when we got a page, either from a pool or newly allocated. 678 * if needed, dma map the page and populate the dma address array. 679 * Populate the page address array. 680 * If the caching is consistent, update any deferred caching. Otherwise 681 * stage this page for an upcoming deferred caching update. 682 */ 683 static int ttm_pool_page_allocated(struct ttm_pool *pool, unsigned int order, 684 struct page *p, enum ttm_caching page_caching, 685 struct ttm_pool_alloc_state *alloc, 686 struct ttm_pool_tt_restore *restore) 687 { 688 bool caching_consistent; 689 dma_addr_t first_dma; 690 int r = 0; 691 692 caching_consistent = (page_caching == alloc->tt_caching) || PageHighMem(p); 693 694 if (caching_consistent) { 695 r = ttm_pool_apply_caching(alloc); 696 if (r) 697 return r; 698 } 699 700 if (alloc->dma_addr) { 701 r = ttm_pool_map(pool, order, p, &first_dma); 702 if (r) 703 return r; 704 } 705 706 if (restore) { 707 ttm_pool_page_allocated_restore(pool, order, p, page_caching, 708 first_dma, restore, alloc); 709 } else { 710 ttm_pool_allocated_page_commit(p, first_dma, alloc, 1UL << order); 711 712 if (caching_consistent) 713 alloc->caching_divide = alloc->pages; 714 } 715 716 return 0; 717 } 718 719 /** 720 * ttm_pool_free_range() - Free a range of TTM pages 721 * @pool: The pool used for allocating. 722 * @tt: The struct ttm_tt holding the page pointers. 723 * @caching: The page caching mode used by the range. 724 * @start_page: index for first page to free. 725 * @end_page: index for last page to free + 1. 726 * 727 * During allocation the ttm_tt page-vector may be populated with ranges of 728 * pages with different attributes if allocation hit an error without being 729 * able to completely fulfill the allocation. This function can be used 730 * to free these individual ranges. 731 */ 732 static void ttm_pool_free_range(struct ttm_pool *pool, struct ttm_tt *tt, 733 enum ttm_caching caching, 734 pgoff_t start_page, pgoff_t end_page) 735 { 736 struct page **pages = &tt->pages[start_page]; 737 struct file *backup = tt->backup; 738 pgoff_t i, nr; 739 740 for (i = start_page; i < end_page; i += nr, pages += nr) { 741 struct page *p = *pages; 742 743 nr = 1; 744 if (ttm_backup_page_ptr_is_handle(p)) { 745 unsigned long handle = ttm_backup_page_ptr_to_handle(p); 746 747 if (handle != 0) 748 ttm_backup_drop(backup, handle); 749 } else if (p) { 750 dma_addr_t *dma_addr = tt->dma_address ? 751 tt->dma_address + i : NULL; 752 753 nr = ttm_pool_unmap_and_free(pool, p, dma_addr, caching); 754 } 755 } 756 } 757 758 static void ttm_pool_alloc_state_init(const struct ttm_tt *tt, 759 struct ttm_pool_alloc_state *alloc) 760 { 761 alloc->pages = tt->pages; 762 alloc->caching_divide = tt->pages; 763 alloc->dma_addr = tt->dma_address; 764 alloc->remaining_pages = tt->num_pages; 765 alloc->tt_caching = tt->caching; 766 } 767 768 /* 769 * Find a suitable allocation order based on highest desired order 770 * and number of remaining pages 771 */ 772 static unsigned int ttm_pool_alloc_find_order(unsigned int highest, 773 const struct ttm_pool_alloc_state *alloc) 774 { 775 return min_t(unsigned int, highest, __fls(alloc->remaining_pages)); 776 } 777 778 static int __ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, 779 const struct ttm_operation_ctx *ctx, 780 struct ttm_pool_alloc_state *alloc, 781 struct ttm_pool_tt_restore *restore) 782 { 783 enum ttm_caching page_caching; 784 gfp_t gfp_flags = GFP_USER; 785 pgoff_t caching_divide; 786 unsigned int order; 787 bool allow_pools; 788 struct page *p; 789 int r; 790 791 WARN_ON(!alloc->remaining_pages || ttm_tt_is_populated(tt)); 792 WARN_ON(alloc->dma_addr && !pool->dev); 793 794 if (tt->page_flags & TTM_TT_FLAG_ZERO_ALLOC) 795 gfp_flags |= __GFP_ZERO; 796 797 if (ctx->gfp_retry_mayfail) 798 gfp_flags |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN; 799 800 if (ttm_pool_uses_dma32(pool)) 801 gfp_flags |= GFP_DMA32; 802 else 803 gfp_flags |= GFP_HIGHUSER; 804 805 page_caching = tt->caching; 806 allow_pools = true; 807 for (order = ttm_pool_alloc_find_order(MAX_PAGE_ORDER, alloc); 808 alloc->remaining_pages; 809 order = ttm_pool_alloc_find_order(order, alloc)) { 810 struct ttm_pool_type *pt; 811 812 /* First, try to allocate a page from a pool if one exists. */ 813 p = NULL; 814 pt = ttm_pool_select_type(pool, page_caching, order); 815 if (pt && allow_pools) 816 p = ttm_pool_type_take(pt, ttm_pool_nid(pool)); 817 818 /* 819 * If that fails or previously failed, allocate from system. 820 * Note that this also disallows additional pool allocations using 821 * write-back cached pools of the same order. Consider removing 822 * that behaviour. 823 */ 824 if (!p) { 825 page_caching = ttm_cached; 826 allow_pools = false; 827 p = ttm_pool_alloc_page(pool, gfp_flags, order); 828 } 829 /* If that fails, lower the order if possible and retry. */ 830 if (!p) { 831 if (order) { 832 --order; 833 page_caching = tt->caching; 834 allow_pools = true; 835 continue; 836 } 837 r = -ENOMEM; 838 goto error_free_all; 839 } 840 r = ttm_pool_page_allocated(pool, order, p, page_caching, alloc, 841 restore); 842 if (r) 843 goto error_free_page; 844 845 if (ttm_pool_restore_valid(restore)) { 846 r = ttm_pool_restore_commit(restore, tt->backup, ctx, alloc); 847 if (r) 848 goto error_free_all; 849 } 850 } 851 852 r = ttm_pool_apply_caching(alloc); 853 if (r) 854 goto error_free_all; 855 856 kfree(tt->restore); 857 tt->restore = NULL; 858 859 return 0; 860 861 error_free_page: 862 ttm_pool_free_page(pool, page_caching, order, p, false); 863 864 error_free_all: 865 if (tt->restore) 866 return r; 867 868 caching_divide = alloc->caching_divide - tt->pages; 869 ttm_pool_free_range(pool, tt, tt->caching, 0, caching_divide); 870 ttm_pool_free_range(pool, tt, ttm_cached, caching_divide, 871 tt->num_pages - alloc->remaining_pages); 872 873 return r; 874 } 875 876 /** 877 * ttm_pool_alloc - Fill a ttm_tt object 878 * 879 * @pool: ttm_pool to use 880 * @tt: ttm_tt object to fill 881 * @ctx: operation context 882 * 883 * Fill the ttm_tt object with pages and also make sure to DMA map them when 884 * necessary. 885 * 886 * Returns: 0 on successe, negative error code otherwise. 887 */ 888 int ttm_pool_alloc(struct ttm_pool *pool, struct ttm_tt *tt, 889 struct ttm_operation_ctx *ctx) 890 { 891 struct ttm_pool_alloc_state alloc; 892 893 if (WARN_ON(ttm_tt_is_backed_up(tt))) 894 return -EINVAL; 895 896 ttm_pool_alloc_state_init(tt, &alloc); 897 898 return __ttm_pool_alloc(pool, tt, ctx, &alloc, NULL); 899 } 900 EXPORT_SYMBOL(ttm_pool_alloc); 901 902 /** 903 * ttm_pool_restore_and_alloc - Fill a ttm_tt, restoring previously backed-up 904 * content. 905 * 906 * @pool: ttm_pool to use 907 * @tt: ttm_tt object to fill 908 * @ctx: operation context 909 * 910 * Fill the ttm_tt object with pages and also make sure to DMA map them when 911 * necessary. Read in backed-up content. 912 * 913 * Returns: 0 on successe, negative error code otherwise. 914 */ 915 int ttm_pool_restore_and_alloc(struct ttm_pool *pool, struct ttm_tt *tt, 916 const struct ttm_operation_ctx *ctx) 917 { 918 struct ttm_pool_tt_restore *restore = tt->restore; 919 struct ttm_pool_alloc_state alloc; 920 921 if (WARN_ON(!ttm_tt_is_backed_up(tt))) 922 return -EINVAL; 923 924 if (!restore) { 925 gfp_t gfp = GFP_KERNEL | __GFP_NOWARN; 926 927 ttm_pool_alloc_state_init(tt, &alloc); 928 if (ctx->gfp_retry_mayfail) 929 gfp |= __GFP_RETRY_MAYFAIL; 930 931 restore = kzalloc_obj(*restore, gfp); 932 if (!restore) 933 return -ENOMEM; 934 935 restore->snapshot_alloc = alloc; 936 restore->pool = pool; 937 restore->restored_pages = 1; 938 939 tt->restore = restore; 940 } else { 941 alloc = restore->snapshot_alloc; 942 if (ttm_pool_restore_valid(restore)) { 943 int ret = ttm_pool_restore_commit(restore, tt->backup, 944 ctx, &alloc); 945 946 if (ret) 947 return ret; 948 } 949 if (!alloc.remaining_pages) 950 return 0; 951 } 952 953 return __ttm_pool_alloc(pool, tt, ctx, &alloc, restore); 954 } 955 956 /** 957 * ttm_pool_free - Free the backing pages from a ttm_tt object 958 * 959 * @pool: Pool to give pages back to. 960 * @tt: ttm_tt object to unpopulate 961 * 962 * Give the packing pages back to a pool or free them 963 */ 964 void ttm_pool_free(struct ttm_pool *pool, struct ttm_tt *tt) 965 { 966 int nid = ttm_pool_nid(pool); 967 968 ttm_pool_free_range(pool, tt, tt->caching, 0, tt->num_pages); 969 970 while (atomic_long_read(&allocated_pages[nid]) > pool_node_limit[nid]) { 971 unsigned long diff = atomic_long_read(&allocated_pages[nid]) - pool_node_limit[nid]; 972 ttm_pool_shrink(nid, diff); 973 } 974 } 975 EXPORT_SYMBOL(ttm_pool_free); 976 977 /** 978 * ttm_pool_drop_backed_up() - Release content of a swapped-out struct ttm_tt 979 * @tt: The struct ttm_tt. 980 * 981 * Release handles with associated content or any remaining pages of 982 * a backed-up struct ttm_tt. 983 */ 984 void ttm_pool_drop_backed_up(struct ttm_tt *tt) 985 { 986 struct ttm_pool_tt_restore *restore; 987 pgoff_t start_page = 0; 988 989 WARN_ON(!ttm_tt_is_backed_up(tt)); 990 991 restore = tt->restore; 992 993 /* 994 * Unmap and free any uncommitted restore page. 995 * any tt page-array backup entries already read back has 996 * been cleared already 997 */ 998 if (ttm_pool_restore_valid(restore)) { 999 dma_addr_t *dma_addr = tt->dma_address ? &restore->first_dma : NULL; 1000 1001 ttm_pool_unmap_and_free(restore->pool, restore->alloced_page, 1002 dma_addr, restore->page_caching); 1003 restore->restored_pages = 1UL << restore->order; 1004 } 1005 1006 /* 1007 * If a restore is ongoing, part of the tt pages may have a 1008 * caching different than writeback. 1009 */ 1010 if (restore) { 1011 pgoff_t mid = restore->snapshot_alloc.caching_divide - tt->pages; 1012 1013 start_page = restore->alloced_pages; 1014 WARN_ON(mid > start_page); 1015 /* Pages that might be dma-mapped and non-cached */ 1016 ttm_pool_free_range(restore->pool, tt, tt->caching, 1017 0, mid); 1018 /* Pages that might be dma-mapped but cached */ 1019 ttm_pool_free_range(restore->pool, tt, ttm_cached, 1020 mid, restore->alloced_pages); 1021 kfree(restore); 1022 tt->restore = NULL; 1023 } 1024 1025 ttm_pool_free_range(NULL, tt, ttm_cached, start_page, tt->num_pages); 1026 } 1027 1028 /** 1029 * ttm_pool_backup() - Back up or purge a struct ttm_tt 1030 * @pool: The pool used when allocating the struct ttm_tt. 1031 * @tt: The struct ttm_tt. 1032 * @flags: Flags to govern the backup behaviour. 1033 * 1034 * Back up or purge a struct ttm_tt. If @purge is true, then 1035 * all pages will be freed directly to the system rather than to the pool 1036 * they were allocated from, making the function behave similarly to 1037 * ttm_pool_free(). If @purge is false the pages will be backed up instead, 1038 * exchanged for handles. 1039 * A subsequent call to ttm_pool_restore_and_alloc() will then read back the content and 1040 * a subsequent call to ttm_pool_drop_backed_up() will drop it. 1041 * If backup of a page fails for whatever reason, @ttm will still be 1042 * partially backed up, retaining those pages for which backup fails. 1043 * In that case, this function can be retried, possibly after freeing up 1044 * memory resources. 1045 * 1046 * Return: Number of pages actually backed up or freed, or negative 1047 * error code on error. 1048 */ 1049 long ttm_pool_backup(struct ttm_pool *pool, struct ttm_tt *tt, 1050 const struct ttm_backup_flags *flags) 1051 { 1052 struct file *backup = tt->backup; 1053 struct page *page; 1054 gfp_t alloc_gfp; 1055 gfp_t gfp; 1056 int ret = 0; 1057 pgoff_t shrunken = 0; 1058 pgoff_t i, j, num_pages, npages; 1059 pgoff_t nr_backed; 1060 1061 if (WARN_ON(ttm_tt_is_backed_up(tt))) 1062 return -EINVAL; 1063 1064 if ((!ttm_backup_bytes_avail() && !flags->purge) || 1065 ttm_pool_uses_dma_alloc(pool) || ttm_tt_is_backed_up(tt)) 1066 return -EBUSY; 1067 1068 #ifdef CONFIG_X86 1069 /* Anything returned to the system needs to be cached. Walk allocations 1070 * skipping NULL pages and issue set_pages_array_wb() per contiguous run. 1071 */ 1072 if (tt->caching != ttm_cached) { 1073 pgoff_t run_start = 0, run_count = 0; 1074 1075 for (i = 0; i < tt->num_pages; i += num_pages) { 1076 page = tt->pages[i]; 1077 if (unlikely(!page || ttm_backup_page_ptr_is_handle(page))) { 1078 if (run_count) { 1079 set_pages_array_wb(&tt->pages[run_start], 1080 run_count); 1081 run_count = 0; 1082 } 1083 num_pages = 1; 1084 continue; 1085 } 1086 num_pages = 1UL << ttm_pool_page_order(pool, page); 1087 if (!run_count) 1088 run_start = i; 1089 run_count += num_pages; 1090 } 1091 if (run_count) 1092 set_pages_array_wb(&tt->pages[run_start], run_count); 1093 } 1094 #endif 1095 1096 if (tt->dma_address || flags->purge) { 1097 for (i = 0; i < tt->num_pages; i += num_pages) { 1098 unsigned int order; 1099 1100 page = tt->pages[i]; 1101 if (unlikely(!page || ttm_backup_page_ptr_is_handle(page))) { 1102 num_pages = 1; 1103 continue; 1104 } 1105 1106 order = ttm_pool_page_order(pool, page); 1107 num_pages = 1UL << order; 1108 if (tt->dma_address) 1109 ttm_pool_unmap(pool, tt->dma_address[i], 1110 num_pages); 1111 if (flags->purge) { 1112 shrunken += num_pages; 1113 page->private = 0; 1114 __free_pages_gpu_account(page, order, false); 1115 memset(tt->pages + i, 0, 1116 num_pages * sizeof(*tt->pages)); 1117 } 1118 } 1119 } 1120 1121 if (flags->purge) 1122 return shrunken; 1123 1124 if (ttm_pool_uses_dma32(pool)) 1125 gfp = GFP_DMA32; 1126 else 1127 gfp = GFP_HIGHUSER; 1128 1129 alloc_gfp = GFP_KERNEL | __GFP_HIGH | __GFP_NOWARN | __GFP_RETRY_MAYFAIL; 1130 1131 num_pages = tt->num_pages; 1132 1133 /* Pretend doing fault injection by shrinking only half of the pages. */ 1134 if (IS_ENABLED(CONFIG_FAULT_INJECTION) && should_fail(&backup_fault_inject, 1)) 1135 num_pages = DIV_ROUND_UP(num_pages, 2); 1136 1137 for (i = 0; i < num_pages; i += npages) { 1138 unsigned int order; 1139 s64 handle; 1140 1141 npages = 1; 1142 page = tt->pages[i]; 1143 if (unlikely(!page)) 1144 continue; 1145 1146 /* Already-handled entry from a previous attempt. */ 1147 if (unlikely(ttm_backup_page_ptr_is_handle(page))) 1148 continue; 1149 1150 order = ttm_pool_page_order(pool, page); 1151 npages = 1UL << order; 1152 1153 /* 1154 * We don't allow dipping kernel reserves for high order backup 1155 */ 1156 if (order) 1157 alloc_gfp |= __GFP_NOMEMALLOC; 1158 else 1159 alloc_gfp &= ~__GFP_NOMEMALLOC; 1160 1161 /* 1162 * Back up the compound atomically at its native order. If 1163 * fault injection truncated num_pages mid-compound, skip 1164 * the partial tail rather than splitting. 1165 */ 1166 if (unlikely(i + npages > num_pages)) 1167 break; 1168 1169 handle = ttm_backup_backup_folio(backup, page_folio(page), 1170 order, flags->writeback, i, 1171 gfp, alloc_gfp, 1172 &nr_backed); 1173 /* 1174 * Zero progress on this compound (whether order 0 or a 1175 * high-order compound that failed before backing up even 1176 * its first subpage) is unrecoverable: bail out rather than 1177 * looping forever with npages == nr_backed == 0 below. 1178 */ 1179 if (unlikely(handle < 0 && !nr_backed)) { 1180 ret = handle; 1181 break; 1182 } 1183 1184 for (j = 0; j < nr_backed; j++) 1185 tt->pages[i + j] = ttm_backup_handle_to_page_ptr(handle + j); 1186 1187 shrunken += nr_backed; 1188 1189 if (unlikely(nr_backed < npages)) { 1190 /* 1191 * Partial OOM backup: split the compound and free the 1192 * subpages whose content is now in shmem. Continue the 1193 * loop from the first un-backed order-0 page. 1194 */ 1195 ttm_pool_split_for_swap(pool, page); 1196 for (j = 0; j < nr_backed; j++) 1197 __free_pages_gpu_account(page + j, 0, false); 1198 npages = nr_backed; 1199 continue; 1200 } 1201 1202 /* Fully backed up: free at native order. */ 1203 page->private = 0; 1204 __free_pages_gpu_account(page, order, false); 1205 } 1206 1207 return shrunken ? shrunken : ret; 1208 } 1209 1210 /** 1211 * ttm_pool_init - Initialize a pool 1212 * 1213 * @pool: the pool to initialize 1214 * @dev: device for DMA allocations and mappings 1215 * @nid: NUMA node to use for allocations 1216 * @alloc_flags: TTM_ALLOCATION_POOL_* flags 1217 * 1218 * Initialize the pool and its pool types. 1219 */ 1220 void ttm_pool_init(struct ttm_pool *pool, struct device *dev, 1221 int nid, unsigned int alloc_flags) 1222 { 1223 unsigned int i, j; 1224 1225 WARN_ON(!dev && ttm_pool_uses_dma_alloc(pool)); 1226 1227 pool->dev = dev; 1228 pool->nid = nid; 1229 pool->alloc_flags = alloc_flags; 1230 1231 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { 1232 for (j = 0; j < NR_PAGE_ORDERS; ++j) { 1233 struct ttm_pool_type *pt; 1234 1235 /* Initialize only pool types which are actually used */ 1236 pt = ttm_pool_select_type(pool, i, j); 1237 if (pt != &pool->caching[i].orders[j]) 1238 continue; 1239 1240 ttm_pool_type_init(pt, pool, i, j); 1241 } 1242 } 1243 } 1244 EXPORT_SYMBOL(ttm_pool_init); 1245 1246 /** 1247 * ttm_pool_synchronize_shrinkers - Wait for all running shrinkers to complete. 1248 * 1249 * This is useful to guarantee that all shrinker invocations have seen an 1250 * update, before freeing memory, similar to rcu. 1251 */ 1252 static void ttm_pool_synchronize_shrinkers(void) 1253 { 1254 down_write(&pool_shrink_rwsem); 1255 up_write(&pool_shrink_rwsem); 1256 } 1257 1258 /** 1259 * ttm_pool_fini - Cleanup a pool 1260 * 1261 * @pool: the pool to clean up 1262 * 1263 * Free all pages in the pool and unregister the types from the global 1264 * shrinker. 1265 */ 1266 void ttm_pool_fini(struct ttm_pool *pool) 1267 { 1268 unsigned int i, j; 1269 1270 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { 1271 for (j = 0; j < NR_PAGE_ORDERS; ++j) { 1272 struct ttm_pool_type *pt; 1273 1274 pt = ttm_pool_select_type(pool, i, j); 1275 if (pt != &pool->caching[i].orders[j]) 1276 continue; 1277 1278 ttm_pool_type_fini(pt); 1279 } 1280 } 1281 1282 /* We removed the pool types from the LRU, but we need to also make sure 1283 * that no shrinker is concurrently freeing pages from the pool. 1284 */ 1285 ttm_pool_synchronize_shrinkers(); 1286 } 1287 EXPORT_SYMBOL(ttm_pool_fini); 1288 1289 /* Free average pool number of pages. */ 1290 #define TTM_SHRINKER_BATCH ((1 << (MAX_PAGE_ORDER / 2)) * NR_PAGE_ORDERS) 1291 1292 static unsigned long ttm_pool_shrinker_scan(struct shrinker *shrink, 1293 struct shrink_control *sc) 1294 { 1295 unsigned long num_freed = 0; 1296 1297 do 1298 num_freed += ttm_pool_shrink(sc->nid, sc->nr_to_scan); 1299 while (num_freed < sc->nr_to_scan && 1300 atomic_long_read(&allocated_pages[sc->nid])); 1301 1302 sc->nr_scanned = num_freed; 1303 1304 return num_freed ?: SHRINK_STOP; 1305 } 1306 1307 /* Return the number of pages available or SHRINK_EMPTY if we have none */ 1308 static unsigned long ttm_pool_shrinker_count(struct shrinker *shrink, 1309 struct shrink_control *sc) 1310 { 1311 unsigned long num_pages = atomic_long_read(&allocated_pages[sc->nid]); 1312 1313 return num_pages ? num_pages : SHRINK_EMPTY; 1314 } 1315 1316 #ifdef CONFIG_DEBUG_FS 1317 /* Count the number of pages available in a pool_type */ 1318 static unsigned int ttm_pool_type_count(struct ttm_pool_type *pt) 1319 { 1320 return list_lru_count(&pt->pages); 1321 } 1322 1323 /* Print a nice header for the order */ 1324 static void ttm_pool_debugfs_header(struct seq_file *m) 1325 { 1326 unsigned int i; 1327 1328 seq_puts(m, "\t "); 1329 for (i = 0; i < NR_PAGE_ORDERS; ++i) 1330 seq_printf(m, " ---%2u---", i); 1331 seq_puts(m, "\n"); 1332 } 1333 1334 /* Dump information about the different pool types */ 1335 static void ttm_pool_debugfs_orders(struct ttm_pool_type *pt, 1336 struct seq_file *m) 1337 { 1338 unsigned int i; 1339 1340 for (i = 0; i < NR_PAGE_ORDERS; ++i) 1341 seq_printf(m, " %8u", ttm_pool_type_count(&pt[i])); 1342 seq_puts(m, "\n"); 1343 } 1344 1345 /* Dump the total amount of allocated pages */ 1346 static void ttm_pool_debugfs_footer(struct seq_file *m) 1347 { 1348 int nid; 1349 1350 for_each_node(nid) { 1351 seq_printf(m, "\ntotal node%d\t: %8lu of %8lu\n", nid, 1352 atomic_long_read(&allocated_pages[nid]), pool_node_limit[nid]); 1353 } 1354 } 1355 1356 /* Dump the information for the global pools */ 1357 static int ttm_pool_debugfs_globals_show(struct seq_file *m, void *data) 1358 { 1359 ttm_pool_debugfs_header(m); 1360 1361 spin_lock(&shrinker_lock); 1362 seq_puts(m, "wc\t:"); 1363 ttm_pool_debugfs_orders(global_write_combined, m); 1364 seq_puts(m, "uc\t:"); 1365 ttm_pool_debugfs_orders(global_uncached, m); 1366 seq_puts(m, "wc 32\t:"); 1367 ttm_pool_debugfs_orders(global_dma32_write_combined, m); 1368 seq_puts(m, "uc 32\t:"); 1369 ttm_pool_debugfs_orders(global_dma32_uncached, m); 1370 spin_unlock(&shrinker_lock); 1371 1372 ttm_pool_debugfs_footer(m); 1373 1374 return 0; 1375 } 1376 DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_globals); 1377 1378 /** 1379 * ttm_pool_debugfs - Debugfs dump function for a pool 1380 * 1381 * @pool: the pool to dump the information for 1382 * @m: seq_file to dump to 1383 * 1384 * Make a debugfs dump with the per pool and global information. 1385 */ 1386 int ttm_pool_debugfs(struct ttm_pool *pool, struct seq_file *m) 1387 { 1388 unsigned int i; 1389 1390 if (!ttm_pool_uses_dma_alloc(pool)) { 1391 seq_puts(m, "unused\n"); 1392 return 0; 1393 } 1394 1395 ttm_pool_debugfs_header(m); 1396 1397 spin_lock(&shrinker_lock); 1398 for (i = 0; i < TTM_NUM_CACHING_TYPES; ++i) { 1399 if (!ttm_pool_select_type(pool, i, 0)) 1400 continue; 1401 seq_puts(m, "DMA "); 1402 switch (i) { 1403 case ttm_cached: 1404 seq_puts(m, "\t:"); 1405 break; 1406 case ttm_write_combined: 1407 seq_puts(m, "wc\t:"); 1408 break; 1409 case ttm_uncached: 1410 seq_puts(m, "uc\t:"); 1411 break; 1412 } 1413 ttm_pool_debugfs_orders(pool->caching[i].orders, m); 1414 } 1415 spin_unlock(&shrinker_lock); 1416 1417 ttm_pool_debugfs_footer(m); 1418 return 0; 1419 } 1420 EXPORT_SYMBOL(ttm_pool_debugfs); 1421 1422 /* Test the shrinker functions and dump the result */ 1423 static int ttm_pool_debugfs_shrink_show(struct seq_file *m, void *data) 1424 { 1425 struct shrink_control sc = { 1426 .gfp_mask = GFP_NOFS, 1427 .nr_to_scan = TTM_SHRINKER_BATCH, 1428 }; 1429 unsigned long count; 1430 int nid; 1431 1432 fs_reclaim_acquire(GFP_KERNEL); 1433 for_each_node(nid) { 1434 sc.nid = nid; 1435 count = ttm_pool_shrinker_count(mm_shrinker, &sc); 1436 seq_printf(m, "%d: %lu/%lu\n", nid, count, 1437 ttm_pool_shrinker_scan(mm_shrinker, &sc)); 1438 } 1439 fs_reclaim_release(GFP_KERNEL); 1440 1441 return 0; 1442 } 1443 DEFINE_SHOW_ATTRIBUTE(ttm_pool_debugfs_shrink); 1444 1445 #endif 1446 1447 static inline u64 ttm_get_node_memory_size(int nid) 1448 { 1449 /* 1450 * This is directly using si_meminfo_node implementation as the 1451 * function is not exported. 1452 */ 1453 int zone_type; 1454 u64 managed_pages = 0; 1455 1456 pg_data_t *pgdat = NODE_DATA(nid); 1457 1458 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) 1459 managed_pages += 1460 zone_managed_pages(&pgdat->node_zones[zone_type]); 1461 return managed_pages * PAGE_SIZE; 1462 } 1463 1464 /** 1465 * ttm_pool_mgr_init - Initialize globals 1466 * 1467 * @num_pages: default number of pages 1468 * 1469 * Initialize the global locks and lists for the MM shrinker. 1470 */ 1471 int ttm_pool_mgr_init(unsigned long num_pages) 1472 { 1473 unsigned int i; 1474 1475 int nid; 1476 for_each_node(nid) { 1477 if (!page_pool_size) { 1478 u64 node_size = ttm_get_node_memory_size(nid); 1479 pool_node_limit[nid] = (node_size >> PAGE_SHIFT) / 2; 1480 } else { 1481 pool_node_limit[nid] = page_pool_size; 1482 } 1483 } 1484 1485 spin_lock_init(&shrinker_lock); 1486 INIT_LIST_HEAD(&shrinker_list); 1487 1488 for (i = 0; i < NR_PAGE_ORDERS; ++i) { 1489 ttm_pool_type_init(&global_write_combined[i], NULL, 1490 ttm_write_combined, i); 1491 ttm_pool_type_init(&global_uncached[i], NULL, ttm_uncached, i); 1492 1493 ttm_pool_type_init(&global_dma32_write_combined[i], NULL, 1494 ttm_write_combined, i); 1495 ttm_pool_type_init(&global_dma32_uncached[i], NULL, 1496 ttm_uncached, i); 1497 } 1498 1499 #ifdef CONFIG_DEBUG_FS 1500 debugfs_create_file("page_pool", 0444, ttm_debugfs_root, NULL, 1501 &ttm_pool_debugfs_globals_fops); 1502 debugfs_create_file("page_pool_shrink", 0400, ttm_debugfs_root, NULL, 1503 &ttm_pool_debugfs_shrink_fops); 1504 #ifdef CONFIG_FAULT_INJECTION 1505 fault_create_debugfs_attr("backup_fault_inject", ttm_debugfs_root, 1506 &backup_fault_inject); 1507 #endif 1508 #endif 1509 1510 mm_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE, "drm-ttm_pool"); 1511 if (!mm_shrinker) 1512 return -ENOMEM; 1513 1514 mm_shrinker->count_objects = ttm_pool_shrinker_count; 1515 mm_shrinker->scan_objects = ttm_pool_shrinker_scan; 1516 mm_shrinker->batch = TTM_SHRINKER_BATCH; 1517 mm_shrinker->seeks = 1; 1518 1519 shrinker_register(mm_shrinker); 1520 1521 return 0; 1522 } 1523 1524 /** 1525 * ttm_pool_mgr_fini - Finalize globals 1526 * 1527 * Cleanup the global pools and unregister the MM shrinker. 1528 */ 1529 void ttm_pool_mgr_fini(void) 1530 { 1531 unsigned int i; 1532 1533 for (i = 0; i < NR_PAGE_ORDERS; ++i) { 1534 ttm_pool_type_fini(&global_write_combined[i]); 1535 ttm_pool_type_fini(&global_uncached[i]); 1536 1537 ttm_pool_type_fini(&global_dma32_write_combined[i]); 1538 ttm_pool_type_fini(&global_dma32_uncached[i]); 1539 } 1540 1541 shrinker_free(mm_shrinker); 1542 WARN_ON(!list_empty(&shrinker_list)); 1543 } 1544