1 // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 /* 3 * Copyright © 2024-2025 Intel Corporation 4 */ 5 6 #include <linux/dma-fence.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/migrate.h> 9 #include <linux/pagemap.h> 10 #include <drm/drm_drv.h> 11 #include <drm/drm_pagemap.h> 12 #include <drm/drm_pagemap_util.h> 13 #include <drm/drm_print.h> 14 15 /** 16 * DOC: Overview 17 * 18 * The DRM pagemap layer is intended to augment the dev_pagemap functionality by 19 * providing a way to populate a struct mm_struct virtual range with device 20 * private pages and to provide helpers to abstract device memory allocations, 21 * to migrate memory back and forth between device memory and system RAM and 22 * to handle access (and in the future migration) between devices implementing 23 * a fast interconnect that is not necessarily visible to the rest of the 24 * system. 25 * 26 * Typically the DRM pagemap receives requests from one or more DRM GPU SVM 27 * instances to populate struct mm_struct virtual ranges with memory, and the 28 * migration is best effort only and may thus fail. The implementation should 29 * also handle device unbinding by blocking (return an -ENODEV) error for new 30 * population requests and after that migrate all device pages to system ram. 31 */ 32 33 /** 34 * DOC: Migration 35 * 36 * Migration granularity typically follows the GPU SVM range requests, but 37 * if there are clashes, due to races or due to the fact that multiple GPU 38 * SVM instances have different views of the ranges used, and because of that 39 * parts of a requested range is already present in the requested device memory, 40 * the implementation has a variety of options. It can fail and it can choose 41 * to populate only the part of the range that isn't already in device memory, 42 * and it can evict the range to system before trying to migrate. Ideally an 43 * implementation would just try to migrate the missing part of the range and 44 * allocate just enough memory to do so. 45 * 46 * When migrating to system memory as a response to a cpu fault or a device 47 * memory eviction request, currently a full device memory allocation is 48 * migrated back to system. Moving forward this might need improvement for 49 * situations where a single page needs bouncing between system memory and 50 * device memory due to, for example, atomic operations. 51 * 52 * Key DRM pagemap components: 53 * 54 * - Device Memory Allocations: 55 * Embedded structure containing enough information for the drm_pagemap to 56 * migrate to / from device memory. 57 * 58 * - Device Memory Operations: 59 * Define the interface for driver-specific device memory operations 60 * release memory, populate pfns, and copy to / from device memory. 61 */ 62 63 /** 64 * struct drm_pagemap_zdd - GPU SVM zone device data 65 * 66 * @refcount: Reference count for the zdd 67 * @devmem_allocation: device memory allocation 68 * @dpagemap: Refcounted pointer to the underlying struct drm_pagemap. 69 * 70 * This structure serves as a generic wrapper installed in 71 * page->zone_device_data. It provides infrastructure for looking up a device 72 * memory allocation upon CPU page fault and asynchronously releasing device 73 * memory once the CPU has no page references. Asynchronous release is useful 74 * because CPU page references can be dropped in IRQ contexts, while releasing 75 * device memory likely requires sleeping locks. 76 */ 77 struct drm_pagemap_zdd { 78 struct kref refcount; 79 struct drm_pagemap_devmem *devmem_allocation; 80 struct drm_pagemap *dpagemap; 81 }; 82 83 /** 84 * drm_pagemap_zdd_alloc() - Allocate a zdd structure. 85 * @dpagemap: Pointer to the underlying struct drm_pagemap. 86 * 87 * This function allocates and initializes a new zdd structure. It sets up the 88 * reference count and initializes the destroy work. 89 * 90 * Return: Pointer to the allocated zdd on success, ERR_PTR() on failure. 91 */ 92 static struct drm_pagemap_zdd * 93 drm_pagemap_zdd_alloc(struct drm_pagemap *dpagemap) 94 { 95 struct drm_pagemap_zdd *zdd; 96 97 zdd = kmalloc_obj(*zdd); 98 if (!zdd) 99 return NULL; 100 101 kref_init(&zdd->refcount); 102 zdd->devmem_allocation = NULL; 103 zdd->dpagemap = drm_pagemap_get(dpagemap); 104 105 return zdd; 106 } 107 108 /** 109 * drm_pagemap_zdd_get() - Get a reference to a zdd structure. 110 * @zdd: Pointer to the zdd structure. 111 * 112 * This function increments the reference count of the provided zdd structure. 113 * 114 * Return: Pointer to the zdd structure. 115 */ 116 static struct drm_pagemap_zdd *drm_pagemap_zdd_get(struct drm_pagemap_zdd *zdd) 117 { 118 kref_get(&zdd->refcount); 119 return zdd; 120 } 121 122 /** 123 * drm_pagemap_zdd_destroy() - Destroy a zdd structure. 124 * @ref: Pointer to the reference count structure. 125 * 126 * This function queues the destroy_work of the zdd for asynchronous destruction. 127 */ 128 static void drm_pagemap_zdd_destroy(struct kref *ref) 129 { 130 struct drm_pagemap_zdd *zdd = 131 container_of(ref, struct drm_pagemap_zdd, refcount); 132 struct drm_pagemap_devmem *devmem = zdd->devmem_allocation; 133 struct drm_pagemap *dpagemap = zdd->dpagemap; 134 135 if (devmem) { 136 complete_all(&devmem->detached); 137 if (devmem->ops->devmem_release) 138 devmem->ops->devmem_release(devmem); 139 } 140 kfree(zdd); 141 drm_pagemap_put(dpagemap); 142 } 143 144 /** 145 * drm_pagemap_zdd_put() - Put a zdd reference. 146 * @zdd: Pointer to the zdd structure. 147 * 148 * This function decrements the reference count of the provided zdd structure 149 * and schedules its destruction if the count drops to zero. 150 */ 151 static void drm_pagemap_zdd_put(struct drm_pagemap_zdd *zdd) 152 { 153 kref_put(&zdd->refcount, drm_pagemap_zdd_destroy); 154 } 155 156 /** 157 * drm_pagemap_migration_unlock_put_folio() - Put a migration folio 158 * @folio: Pointer to the folio to put 159 * 160 * This function unlocks and puts a folio. 161 */ 162 static void drm_pagemap_migration_unlock_put_folio(struct folio *folio) 163 { 164 folio_unlock(folio); 165 folio_put(folio); 166 } 167 168 /** 169 * drm_pagemap_migration_unlock_put_pages() - Put migration pages 170 * @npages: Number of pages 171 * @migrate_pfn: Array of migrate page frame numbers 172 * 173 * This function unlocks and puts an array of pages. 174 */ 175 static void drm_pagemap_migration_unlock_put_pages(unsigned long npages, 176 unsigned long *migrate_pfn) 177 { 178 unsigned long i; 179 180 for (i = 0; i < npages;) { 181 struct page *page; 182 struct folio *folio; 183 unsigned int order = 0; 184 185 if (!migrate_pfn[i]) 186 goto next; 187 188 page = migrate_pfn_to_page(migrate_pfn[i]); 189 folio = page_folio(page); 190 order = folio_order(folio); 191 192 drm_pagemap_migration_unlock_put_folio(folio); 193 migrate_pfn[i] = 0; 194 195 next: 196 i += NR_PAGES(order); 197 } 198 } 199 200 /** 201 * drm_pagemap_get_devmem_page() - Get a reference to a device memory page 202 * @page: Pointer to the page 203 * @order: Order 204 * @zdd: Pointer to the GPU SVM zone device data 205 * 206 * This function associates the given page with the specified GPU SVM zone 207 * device data and initializes it for zone device usage. 208 */ 209 static void drm_pagemap_get_devmem_page(struct page *page, 210 unsigned int order, 211 struct drm_pagemap_zdd *zdd) 212 { 213 zone_device_folio_init((struct folio *)page, zdd->dpagemap->pagemap, 214 order); 215 folio_set_zone_device_data(page_folio(page), drm_pagemap_zdd_get(zdd)); 216 } 217 218 /** 219 * drm_pagemap_migrate_map_device_private_pages() - Map device private migration 220 * pages for GPU SVM migration 221 * @dev: The device performing the migration. 222 * @local_dpagemap: The drm_pagemap local to the migrating device. 223 * @pagemap_addr: Array to store DMA information corresponding to mapped pages. 224 * @migrate_pfn: Array of page frame numbers of system pages or peer pages to map. 225 * @npages: Number of system pages or peer pages to map. 226 * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL) 227 * @mdetails: Details governing the migration behaviour. 228 * 229 * This function maps pages of memory for migration usage in GPU SVM. It 230 * iterates over each page frame number provided in @migrate_pfn, maps the 231 * corresponding page, and stores the DMA address in the provided @dma_addr 232 * array. 233 * 234 * Returns: 0 on success, -EFAULT if an error occurs during mapping. 235 */ 236 static int 237 drm_pagemap_migrate_map_device_private_pages(struct device *dev, 238 struct drm_pagemap *local_dpagemap, 239 struct drm_pagemap_addr *pagemap_addr, 240 unsigned long *migrate_pfn, 241 unsigned long npages, 242 enum dma_data_direction dir, 243 const struct drm_pagemap_migrate_details *mdetails) 244 { 245 unsigned long num_peer_pages = 0, num_local_pages = 0, i; 246 247 for (i = 0; i < npages;) { 248 struct page *page = migrate_pfn_to_page(migrate_pfn[i]); 249 struct drm_pagemap_zdd *zdd; 250 struct drm_pagemap *dpagemap; 251 struct drm_pagemap_addr addr; 252 struct folio *folio; 253 unsigned int order = 0; 254 255 if (!page) 256 goto next; 257 258 WARN_ON_ONCE(!is_device_private_page(page)); 259 folio = page_folio(page); 260 order = folio_order(folio); 261 262 zdd = drm_pagemap_page_zone_device_data(page); 263 dpagemap = zdd->dpagemap; 264 265 if (dpagemap == local_dpagemap) { 266 if (!mdetails->can_migrate_same_pagemap) 267 goto next; 268 269 num_local_pages += NR_PAGES(order); 270 } else { 271 num_peer_pages += NR_PAGES(order); 272 } 273 274 addr = dpagemap->ops->device_map(dpagemap, dev, page, order, dir); 275 if (dma_mapping_error(dev, addr.addr)) 276 return -EFAULT; 277 278 pagemap_addr[i] = addr; 279 280 next: 281 i += NR_PAGES(order); 282 } 283 284 if (num_peer_pages) 285 drm_dbg(local_dpagemap->drm, "Migrating %lu peer pages over interconnect.\n", 286 num_peer_pages); 287 if (num_local_pages) 288 drm_dbg(local_dpagemap->drm, "Migrating %lu local pages over interconnect.\n", 289 num_local_pages); 290 291 return 0; 292 } 293 294 /** 295 * struct drm_pagemap_iova_state - DRM pagemap IOVA state 296 * @dma_state: DMA IOVA state. 297 * @offset: Current offset in IOVA. 298 * 299 * This structure acts as an iterator for packing all IOVA addresses within a 300 * contiguous range. 301 */ 302 struct drm_pagemap_iova_state { 303 struct dma_iova_state dma_state; 304 unsigned long offset; 305 }; 306 307 /** 308 * drm_pagemap_migrate_map_system_pages() - Map system or device coherent 309 * migration pages for GPU SVM migration 310 * @dev: The device performing the migration. 311 * @pagemap_addr: Array to store DMA information corresponding to mapped pages. 312 * @migrate_pfn: Array of page frame numbers of system pages or peer pages to map. 313 * @npages: Number of system or device coherent pages to map. 314 * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL) 315 * @state: DMA IOVA state for mapping. 316 * 317 * This function maps pages of memory for migration usage in GPU SVM. It 318 * iterates over each page frame number provided in @migrate_pfn, maps the 319 * corresponding page, and stores the DMA address in the provided @dma_addr 320 * array. 321 * 322 * Returns: 0 on success, negative error code on failure. 323 */ 324 static int 325 drm_pagemap_migrate_map_system_pages(struct device *dev, 326 struct drm_pagemap_addr *pagemap_addr, 327 unsigned long *migrate_pfn, 328 unsigned long npages, 329 enum dma_data_direction dir, 330 struct drm_pagemap_iova_state *state) 331 { 332 unsigned long i; 333 bool try_alloc = false; 334 335 for (i = 0; i < npages;) { 336 struct page *page = migrate_pfn_to_page(migrate_pfn[i]); 337 dma_addr_t dma_addr; 338 struct folio *folio; 339 unsigned int order = 0; 340 341 if (!page) 342 goto next; 343 344 WARN_ON_ONCE(is_device_private_page(page)); 345 folio = page_folio(page); 346 order = folio_order(folio); 347 348 if (!try_alloc) { 349 dma_iova_try_alloc(dev, &state->dma_state, 350 0, npages * PAGE_SIZE); 351 try_alloc = true; 352 } 353 354 if (dma_use_iova(&state->dma_state)) { 355 int err = dma_iova_link(dev, &state->dma_state, 356 page_to_phys(page), 357 state->offset, page_size(page), 358 dir, 0); 359 if (err) 360 return err; 361 362 dma_addr = state->dma_state.addr + state->offset; 363 state->offset += page_size(page); 364 } else { 365 dma_addr = dma_map_page(dev, page, 0, page_size(page), 366 dir); 367 if (dma_mapping_error(dev, dma_addr)) 368 return -EFAULT; 369 } 370 371 pagemap_addr[i] = 372 drm_pagemap_addr_encode(dma_addr, 373 DRM_INTERCONNECT_SYSTEM, 374 order, dir); 375 376 next: 377 i += NR_PAGES(order); 378 } 379 380 if (dma_use_iova(&state->dma_state)) 381 return dma_iova_sync(dev, &state->dma_state, 0, state->offset); 382 383 return 0; 384 } 385 386 /** 387 * drm_pagemap_migrate_populate_src_pages() - Populate the source page array 388 * @pages: Array of source pages to populate 389 * @src_mpfn: Source array of migrate PFNs 390 * @dst_mpfn: Destination array of migrate PFNs 391 * @npages: Number of pages in the arrays 392 * 393 * Populate @pages with the device pages the copy callback is to read from. 394 * 395 * Entries are normally only populated at the head of each source folio, with 396 * the copy callback deriving the rest of the folio from the order recorded in 397 * the corresponding drm_pagemap_addr. That does not work where 398 * drm_pagemap_migrate_populate_ram_pfn() had to demote a higher-order source 399 * folio to order-0 destination folios: the drm_pagemap_addr entries are then 400 * per-page, and the copy callback needs a source page for each of them. 401 * Populate every entry for those ranges. 402 * 403 * Note that the source folio itself is only split later, by 404 * migrate_vma_pages() / migrate_device_pages(), so its order cannot be used to 405 * detect the demotion - the destination has to be inspected instead. 406 */ 407 static void drm_pagemap_migrate_populate_src_pages(struct page **pages, 408 unsigned long *src_mpfn, 409 unsigned long *dst_mpfn, 410 unsigned long npages) 411 { 412 unsigned long i; 413 414 for (i = 0; i < npages;) { 415 struct page *page = migrate_pfn_to_page(src_mpfn[i]); 416 unsigned int order = 0; 417 unsigned long j, nr; 418 419 if (!page) { 420 i++; 421 continue; 422 } 423 424 order = folio_order(page_folio(page)); 425 nr = NR_PAGES(order); 426 427 if (order && !(dst_mpfn[i] & MIGRATE_PFN_COMPOUND)) { 428 for (j = 0; j < nr && i + j < npages; j++) 429 pages[i + j] = folio_page(page_folio(page), j); 430 } else { 431 pages[i] = page; 432 } 433 434 i += nr; 435 } 436 } 437 438 /** 439 * drm_pagemap_migrate_unmap_pages() - Unmap pages previously mapped for GPU SVM migration 440 * @dev: The device for which the pages were mapped 441 * @migrate_pfn: Array of migrate pfns set up for the mapped pages. Used to 442 * determine the drm_pagemap of a peer device private page. 443 * @pagemap_addr: Array of DMA information corresponding to mapped pages 444 * @npages: Number of pages to unmap 445 * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL) 446 * @state: DMA IOVA state for mapping. 447 * 448 * This function unmaps previously mapped pages of memory for GPU Shared Virtual 449 * Memory (SVM). It iterates over each DMA address provided in @pagemap_addr, 450 * checks if it's valid and not already unmapped, and unmaps the corresponding 451 * page. 452 */ 453 static void drm_pagemap_migrate_unmap_pages(struct device *dev, 454 struct drm_pagemap_addr *pagemap_addr, 455 unsigned long *migrate_pfn, 456 unsigned long npages, 457 enum dma_data_direction dir, 458 struct drm_pagemap_iova_state *state) 459 { 460 unsigned long i; 461 462 if (state && dma_use_iova(&state->dma_state)) { 463 dma_iova_destroy(dev, &state->dma_state, state->offset, dir, 0); 464 return; 465 } 466 467 for (i = 0; i < npages;) { 468 struct page *page = migrate_pfn_to_page(migrate_pfn[i]); 469 470 if (!page || !pagemap_addr[i].addr || dma_mapping_error(dev, pagemap_addr[i].addr)) 471 goto next; 472 473 if (is_zone_device_page(page)) { 474 struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page); 475 struct drm_pagemap *dpagemap = zdd->dpagemap; 476 477 dpagemap->ops->device_unmap(dpagemap, dev, &pagemap_addr[i]); 478 } else { 479 dma_unmap_page(dev, pagemap_addr[i].addr, 480 PAGE_SIZE << pagemap_addr[i].order, dir); 481 } 482 483 next: 484 i += NR_PAGES(pagemap_addr[i].order); 485 } 486 } 487 488 static unsigned long 489 npages_in_range(unsigned long start, unsigned long end) 490 { 491 return (end - start) >> PAGE_SHIFT; 492 } 493 494 static int 495 drm_pagemap_migrate_remote_to_local(struct drm_pagemap_devmem *devmem, 496 struct device *remote_device, 497 struct drm_pagemap *remote_dpagemap, 498 unsigned long local_pfns[], 499 struct page *remote_pages[], 500 struct drm_pagemap_addr pagemap_addr[], 501 unsigned long npages, 502 const struct drm_pagemap_devmem_ops *ops, 503 const struct drm_pagemap_migrate_details *mdetails) 504 505 { 506 int err = drm_pagemap_migrate_map_device_private_pages(remote_device, 507 remote_dpagemap, 508 pagemap_addr, 509 local_pfns, 510 npages, 511 DMA_FROM_DEVICE, 512 mdetails); 513 514 if (err) 515 goto out; 516 517 err = ops->copy_to_ram(remote_pages, pagemap_addr, npages, 518 devmem->pre_migrate_fence); 519 out: 520 drm_pagemap_migrate_unmap_pages(remote_device, pagemap_addr, local_pfns, 521 npages, DMA_FROM_DEVICE, NULL); 522 return err; 523 } 524 525 static int 526 drm_pagemap_migrate_sys_to_dev(struct drm_pagemap_devmem *devmem, 527 unsigned long sys_pfns[], 528 struct page *local_pages[], 529 struct drm_pagemap_addr pagemap_addr[], 530 unsigned long npages, 531 const struct drm_pagemap_devmem_ops *ops, 532 struct drm_pagemap_iova_state *state) 533 { 534 int err = drm_pagemap_migrate_map_system_pages(devmem->dev, 535 pagemap_addr, sys_pfns, 536 npages, DMA_TO_DEVICE, 537 state); 538 539 if (err) 540 goto out; 541 542 err = ops->copy_to_devmem(local_pages, pagemap_addr, npages, 543 devmem->pre_migrate_fence); 544 out: 545 drm_pagemap_migrate_unmap_pages(devmem->dev, pagemap_addr, sys_pfns, npages, 546 DMA_TO_DEVICE, state); 547 return err; 548 } 549 550 /** 551 * struct migrate_range_loc - Cursor into the loop over migrate_pfns for migrating to 552 * device. 553 * @start: The current loop index. 554 * @device: migrating device. 555 * @dpagemap: Pointer to struct drm_pagemap used by the migrating device. 556 * @ops: The copy ops to be used for the migrating device. 557 */ 558 struct migrate_range_loc { 559 unsigned long start; 560 struct device *device; 561 struct drm_pagemap *dpagemap; 562 const struct drm_pagemap_devmem_ops *ops; 563 }; 564 565 static int drm_pagemap_migrate_range(struct drm_pagemap_devmem *devmem, 566 unsigned long src_pfns[], 567 unsigned long dst_pfns[], 568 struct page *pages[], 569 struct drm_pagemap_addr pagemap_addr[], 570 struct migrate_range_loc *last, 571 const struct migrate_range_loc *cur, 572 const struct drm_pagemap_migrate_details *mdetails) 573 { 574 struct drm_pagemap_iova_state state = {}; 575 int ret = 0; 576 577 if (cur->start == 0) 578 goto out; 579 580 if (cur->start <= last->start) 581 return 0; 582 583 if (cur->dpagemap == last->dpagemap && cur->ops == last->ops) 584 return 0; 585 586 if (last->dpagemap) 587 ret = drm_pagemap_migrate_remote_to_local(devmem, 588 last->device, 589 last->dpagemap, 590 &dst_pfns[last->start], 591 &pages[last->start], 592 &pagemap_addr[last->start], 593 cur->start - last->start, 594 last->ops, mdetails); 595 596 else 597 ret = drm_pagemap_migrate_sys_to_dev(devmem, 598 &src_pfns[last->start], 599 &pages[last->start], 600 &pagemap_addr[last->start], 601 cur->start - last->start, 602 last->ops, &state); 603 604 out: 605 *last = *cur; 606 return ret; 607 } 608 609 /** 610 * drm_pagemap_cpages() - Count collected pages 611 * @migrate_pfn: Array of migrate_pfn entries to account 612 * @npages: Number of entries in @migrate_pfn 613 * 614 * Compute the total number of minimum-sized pages represented by the 615 * collected entries in @migrate_pfn. The total is derived from the 616 * order encoded in each entry. 617 * 618 * Return: Total number of minimum-sized pages. 619 */ 620 static int drm_pagemap_cpages(unsigned long *migrate_pfn, unsigned long npages) 621 { 622 unsigned long i, cpages = 0; 623 624 for (i = 0; i < npages;) { 625 struct page *page = migrate_pfn_to_page(migrate_pfn[i]); 626 struct folio *folio; 627 unsigned int order = 0; 628 629 if (page) { 630 folio = page_folio(page); 631 order = folio_order(folio); 632 cpages += NR_PAGES(order); 633 } else if (migrate_pfn[i] & MIGRATE_PFN_COMPOUND) { 634 order = HPAGE_PMD_ORDER; 635 cpages += NR_PAGES(order); 636 } 637 638 i += NR_PAGES(order); 639 } 640 641 return cpages; 642 } 643 644 /** 645 * drm_pagemap_migrate_to_devmem() - Migrate a struct mm_struct range to device memory 646 * @devmem_allocation: The device memory allocation to migrate to. 647 * The caller should hold a reference to the device memory allocation, 648 * and the reference is consumed by this function even if it returns with 649 * an error. 650 * @mm: Pointer to the struct mm_struct. 651 * @start: Start of the virtual address range to migrate. 652 * @end: End of the virtual address range to migrate. 653 * @mdetails: Details to govern the migration. 654 * 655 * This function migrates the specified virtual address range to device memory. 656 * It performs the necessary setup and invokes the driver-specific operations for 657 * migration to device memory. Expected to be called while holding the mmap lock in 658 * at least read mode. 659 * 660 * Note: The @timeslice_ms parameter can typically be used to force data to 661 * remain in pagemap pages long enough for a GPU to perform a task and to prevent 662 * a migration livelock. One alternative would be for the GPU driver to block 663 * in a mmu_notifier for the specified amount of time, but adding the 664 * functionality to the pagemap is likely nicer to the system as a whole. 665 * 666 * Return: %0 on success, negative error code on failure. 667 */ 668 int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, 669 struct mm_struct *mm, 670 unsigned long start, unsigned long end, 671 const struct drm_pagemap_migrate_details *mdetails) 672 { 673 const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops; 674 struct drm_pagemap *dpagemap = devmem_allocation->dpagemap; 675 struct dev_pagemap *pagemap = dpagemap->pagemap; 676 struct migrate_vma migrate = { 677 .start = start, 678 .end = end, 679 .pgmap_owner = pagemap->owner, 680 .flags = MIGRATE_VMA_SELECT_SYSTEM | MIGRATE_VMA_SELECT_DEVICE_COHERENT | 681 MIGRATE_VMA_SELECT_DEVICE_PRIVATE | MIGRATE_VMA_SELECT_COMPOUND, 682 }; 683 unsigned long i, npages = npages_in_range(start, end); 684 unsigned long own_pages = 0, migrated_pages = 0; 685 struct migrate_range_loc cur, last = {.device = dpagemap->drm->dev, .ops = ops}; 686 struct vm_area_struct *vas; 687 struct drm_pagemap_zdd *zdd = NULL; 688 struct page **pages; 689 struct drm_pagemap_addr *pagemap_addr; 690 void *buf; 691 int err; 692 693 mmap_assert_locked(mm); 694 695 if (!ops->populate_devmem_pfn || !ops->copy_to_devmem || 696 !ops->copy_to_ram) 697 return -EOPNOTSUPP; 698 699 vas = vma_lookup(mm, start); 700 if (!vas) { 701 err = -ENOENT; 702 goto err_out; 703 } 704 705 if (end > vas->vm_end || start < vas->vm_start) { 706 err = -EINVAL; 707 goto err_out; 708 } 709 710 if (!vma_is_anonymous(vas)) { 711 err = -EBUSY; 712 goto err_out; 713 } 714 715 buf = kvcalloc(npages, 2 * sizeof(*migrate.src) + sizeof(*pagemap_addr) + 716 sizeof(*pages), GFP_KERNEL); 717 if (!buf) { 718 err = -ENOMEM; 719 goto err_out; 720 } 721 pagemap_addr = buf + (2 * sizeof(*migrate.src) * npages); 722 pages = buf + (2 * sizeof(*migrate.src) + sizeof(*pagemap_addr)) * npages; 723 724 zdd = drm_pagemap_zdd_alloc(dpagemap); 725 if (!zdd) { 726 err = -ENOMEM; 727 kvfree(buf); 728 goto err_out; 729 } 730 zdd->devmem_allocation = devmem_allocation; /* Owns ref */ 731 732 migrate.vma = vas; 733 migrate.src = buf; 734 migrate.dst = migrate.src + npages; 735 736 err = migrate_vma_setup(&migrate); 737 if (err) 738 goto err_free; 739 740 if (!migrate.cpages) { 741 /* No pages to migrate. Raced or unknown device pages. */ 742 err = -EBUSY; 743 goto err_free; 744 } 745 746 if (migrate.cpages != npages && 747 drm_pagemap_cpages(migrate.src, npages) != npages) { 748 /* 749 * Some pages to migrate. But we want to migrate all or 750 * nothing. Raced or unknown device pages. 751 */ 752 err = -EBUSY; 753 goto err_aborted_migration; 754 } 755 756 /* Count device-private pages to migrate */ 757 for (i = 0; i < npages;) { 758 struct page *src_page = migrate_pfn_to_page(migrate.src[i]); 759 unsigned long nr_pages = src_page ? NR_PAGES(folio_order(page_folio(src_page))) : 1; 760 761 if (src_page && is_zone_device_page(src_page)) { 762 if (page_pgmap(src_page) == pagemap) 763 own_pages += nr_pages; 764 } 765 766 i += nr_pages; 767 } 768 769 drm_dbg(dpagemap->drm, "Total pages %lu; Own pages: %lu.\n", 770 npages, own_pages); 771 if (own_pages == npages) { 772 err = 0; 773 drm_dbg(dpagemap->drm, "Migration wasn't necessary.\n"); 774 goto err_aborted_migration; 775 } else if (own_pages && !mdetails->can_migrate_same_pagemap) { 776 err = -EBUSY; 777 drm_dbg(dpagemap->drm, "Migration aborted due to fragmentation.\n"); 778 goto err_aborted_migration; 779 } 780 781 err = ops->populate_devmem_pfn(devmem_allocation, npages, migrate.dst); 782 if (err) { 783 npages = 0; 784 goto err_finalize; 785 } 786 787 own_pages = 0; 788 789 for (i = 0; i < npages;) { 790 unsigned long j; 791 struct page *page = pfn_to_page(migrate.dst[i]); 792 struct page *src_page = migrate_pfn_to_page(migrate.src[i]); 793 unsigned int order = 0; 794 795 cur.start = i; 796 pages[i] = NULL; 797 if (src_page && is_device_private_page(src_page)) { 798 struct drm_pagemap_zdd *src_zdd = 799 drm_pagemap_page_zone_device_data(src_page); 800 801 if (page_pgmap(src_page) == pagemap && 802 !mdetails->can_migrate_same_pagemap) { 803 migrate.dst[i] = 0; 804 own_pages++; 805 goto next; 806 } 807 cur.dpagemap = src_zdd->dpagemap; 808 cur.ops = src_zdd->devmem_allocation->ops; 809 cur.device = cur.dpagemap->drm->dev; 810 pages[i] = src_page; 811 } 812 if (!pages[i]) { 813 cur.dpagemap = NULL; 814 cur.ops = ops; 815 cur.device = dpagemap->drm->dev; 816 pages[i] = page; 817 } 818 migrate.dst[i] = migrate_pfn(migrate.dst[i]); 819 820 if (migrate.src[i] & MIGRATE_PFN_COMPOUND) { 821 drm_WARN_ONCE(dpagemap->drm, src_page && 822 folio_order(page_folio(src_page)) != HPAGE_PMD_ORDER, 823 "Unexpected folio order\n"); 824 825 order = HPAGE_PMD_ORDER; 826 migrate.dst[i] |= MIGRATE_PFN_COMPOUND; 827 828 for (j = 1; j < NR_PAGES(order) && i + j < npages; j++) 829 migrate.dst[i + j] = 0; 830 } 831 832 drm_pagemap_get_devmem_page(page, order, zdd); 833 834 /* If we switched the migrating drm_pagemap, migrate previous pages now */ 835 err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst, 836 pages, pagemap_addr, &last, &cur, 837 mdetails); 838 if (err) { 839 npages = i + 1; 840 goto err_finalize; 841 } 842 843 next: 844 i += NR_PAGES(order); 845 } 846 847 cur.start = npages; 848 cur.ops = NULL; /* Force migration */ 849 err = drm_pagemap_migrate_range(devmem_allocation, migrate.src, migrate.dst, 850 pages, pagemap_addr, &last, &cur, mdetails); 851 if (err) 852 goto err_finalize; 853 854 drm_WARN_ON(dpagemap->drm, !!own_pages); 855 856 dma_fence_put(devmem_allocation->pre_migrate_fence); 857 devmem_allocation->pre_migrate_fence = NULL; 858 859 /* Upon success bind devmem allocation to range and zdd */ 860 devmem_allocation->timeslice_expiration = get_jiffies_64() + 861 msecs_to_jiffies(mdetails->timeslice_ms); 862 863 err_finalize: 864 if (err) { 865 drm_pagemap_migration_unlock_put_pages(npages, migrate.dst); 866 for (i = npages; i < npages_in_range(start, end); ++i) 867 migrate.dst[i] = 0; 868 } 869 err_aborted_migration: 870 migrate_vma_pages(&migrate); 871 872 for (i = 0; !err && i < npages;) { 873 struct page *page = migrate_pfn_to_page(migrate.src[i]); 874 unsigned long nr_pages = page ? NR_PAGES(folio_order(page_folio(page))) : 1; 875 876 if (migrate.src[i] & MIGRATE_PFN_MIGRATE) 877 migrated_pages += nr_pages; 878 879 i += nr_pages; 880 } 881 882 if (!err && migrated_pages < npages - own_pages) { 883 drm_dbg(dpagemap->drm, "Raced while finalizing migration.\n"); 884 err = -EBUSY; 885 } 886 887 migrate_vma_finalize(&migrate); 888 err_free: 889 drm_pagemap_zdd_put(zdd); 890 kvfree(buf); 891 return err; 892 893 err_out: 894 devmem_allocation->ops->devmem_release(devmem_allocation); 895 return err; 896 } 897 EXPORT_SYMBOL_GPL(drm_pagemap_migrate_to_devmem); 898 899 /** 900 * drm_pagemap_migrate_populate_ram_pfn() - Populate RAM PFNs for a VM area 901 * @vas: Pointer to the VM area structure, can be NULL 902 * @fault_page: Fault page 903 * @npages: Number of pages to populate 904 * @mpages: Number of pages to migrate 905 * @src_mpfn: Source array of migrate PFNs 906 * @mpfn: Array of migrate PFNs to populate 907 * @addr: Start address for PFN allocation 908 * 909 * This function populates the RAM migrate page frame numbers (PFNs) for the 910 * specified VM area structure. It allocates and locks pages in the VM area for 911 * RAM usage. If vas is non-NULL use alloc_page_vma for allocation, if NULL use 912 * alloc_page for allocation. 913 * 914 * Return: 0 on success, negative error code on failure. 915 */ 916 static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas, 917 struct page *fault_page, 918 unsigned long npages, 919 unsigned long *mpages, 920 unsigned long *src_mpfn, 921 unsigned long *mpfn, 922 unsigned long addr) 923 { 924 unsigned long i; 925 926 for (i = 0; i < npages;) { 927 struct page *page = NULL, *src_page; 928 struct folio *folio; 929 unsigned int order = 0; 930 gfp_t gfp = GFP_HIGHUSER; 931 932 if (!(src_mpfn[i] & MIGRATE_PFN_MIGRATE)) 933 goto next; 934 935 src_page = migrate_pfn_to_page(src_mpfn[i]); 936 if (!src_page) 937 goto next; 938 939 if (fault_page) { 940 if (drm_pagemap_page_zone_device_data(src_page) != 941 drm_pagemap_page_zone_device_data(fault_page)) 942 goto next; 943 } 944 945 order = folio_order(page_folio(src_page)); 946 947 /* 948 * A large source folio is always collected whole, at its head 949 * page, PMD aligned and flagged MIGRATE_PFN_COMPOUND: anything 950 * else is split before it reaches us, either by 951 * migrate_vma_collect_pmd() or, for the eviction path, by 952 * migrate_device_pfns(). Both the order-0 fallback below and 953 * drm_pagemap_migrate_populate_src_pages() rely on that, as 954 * they index the folio from @i. 955 */ 956 WARN_ON_ONCE(order && 957 (src_page != folio_page(page_folio(src_page), 0) || 958 !(src_mpfn[i] & MIGRATE_PFN_COMPOUND))); 959 960 if (order) 961 gfp |= __GFP_NOWARN; 962 963 if (vas) 964 folio = vma_alloc_folio(gfp, order, vas, addr); 965 else 966 folio = folio_alloc(gfp, order); 967 968 if (!folio && order) { 969 /* 970 * Higher-order allocation failed, fall back to 971 * order-0 allocations for the entire range covered 972 * by the original higher-order allocation, without 973 * setting MIGRATE_PFN_COMPOUND, until we move past 974 * that range. 975 */ 976 unsigned long nr = NR_PAGES(order); 977 unsigned long j; 978 979 gfp &= ~__GFP_NOWARN; 980 for (j = 0; j < nr && i < npages; j++, i++, addr += PAGE_SIZE) { 981 folio = vas ? 982 vma_alloc_folio(gfp, 0, vas, addr) : 983 folio_alloc(gfp, 0); 984 if (!folio) 985 goto free_pages; 986 987 page = folio_page(folio, 0); 988 mpfn[i] = migrate_pfn(page_to_pfn(page)); 989 } 990 continue; 991 } 992 993 if (!folio) 994 goto free_pages; 995 996 page = folio_page(folio, 0); 997 mpfn[i] = migrate_pfn(page_to_pfn(page)); 998 999 if (order) 1000 mpfn[i] |= MIGRATE_PFN_COMPOUND; 1001 next: 1002 if (page) 1003 addr += page_size(page); 1004 else 1005 addr += PAGE_SIZE; 1006 1007 i += NR_PAGES(order); 1008 } 1009 1010 for (i = 0; i < npages;) { 1011 struct page *page = migrate_pfn_to_page(mpfn[i]); 1012 unsigned int order = 0; 1013 1014 if (!page) 1015 goto next_lock; 1016 1017 WARN_ON_ONCE(!folio_trylock(page_folio(page))); 1018 1019 order = folio_order(page_folio(page)); 1020 *mpages += NR_PAGES(order); 1021 1022 next_lock: 1023 i += NR_PAGES(order); 1024 } 1025 1026 return 0; 1027 1028 free_pages: 1029 for (i = 0; i < npages;) { 1030 struct page *page = migrate_pfn_to_page(mpfn[i]); 1031 unsigned int order = 0; 1032 1033 if (!page) 1034 goto next_put; 1035 1036 order = folio_order(page_folio(page)); 1037 1038 put_page(page); 1039 mpfn[i] = 0; 1040 1041 next_put: 1042 i += NR_PAGES(order); 1043 } 1044 return -ENOMEM; 1045 } 1046 1047 static void drm_pagemap_dev_unhold_work(struct work_struct *work); 1048 static LLIST_HEAD(drm_pagemap_unhold_list); 1049 static DECLARE_WORK(drm_pagemap_work, drm_pagemap_dev_unhold_work); 1050 1051 /** 1052 * struct drm_pagemap_dev_hold - Struct to aid in drm_device release. 1053 * @link: Link into drm_pagemap_unhold_list for deferred reference releases. 1054 * @drm: drm device to put. 1055 * 1056 * When a struct drm_pagemap is released, we also need to release the 1057 * reference it holds on the drm device. However, typically that needs 1058 * to be done separately from a system-wide workqueue. 1059 * Each time a struct drm_pagemap is initialized 1060 * (or re-initialized if cached) therefore allocate a separate 1061 * drm_pagemap_dev_hold item, from which we put the drm device and 1062 * associated module. 1063 */ 1064 struct drm_pagemap_dev_hold { 1065 struct llist_node link; 1066 struct drm_device *drm; 1067 }; 1068 1069 static void drm_pagemap_release(struct kref *ref) 1070 { 1071 struct drm_pagemap *dpagemap = container_of(ref, typeof(*dpagemap), ref); 1072 struct drm_pagemap_dev_hold *dev_hold = dpagemap->dev_hold; 1073 1074 /* 1075 * We know the pagemap provider is alive at this point, since 1076 * the struct drm_pagemap_dev_hold holds a reference to the 1077 * pagemap provider drm_device and its module. 1078 */ 1079 dpagemap->dev_hold = NULL; 1080 drm_pagemap_shrinker_add(dpagemap); 1081 llist_add(&dev_hold->link, &drm_pagemap_unhold_list); 1082 schedule_work(&drm_pagemap_work); 1083 /* 1084 * Here, either the provider device is still alive, since if called from 1085 * page_free(), the caller is holding a reference on the dev_pagemap, 1086 * or if called from drm_pagemap_put(), the direct caller is still alive. 1087 * This ensures we can't race with THIS module unload. 1088 */ 1089 } 1090 1091 static void drm_pagemap_dev_unhold_work(struct work_struct *work) 1092 { 1093 struct llist_node *node = llist_del_all(&drm_pagemap_unhold_list); 1094 struct drm_pagemap_dev_hold *dev_hold, *next; 1095 1096 /* 1097 * Deferred release of drm_pagemap provider device and module. 1098 * THIS module is kept alive during the release by the 1099 * flush_work() in the drm_pagemap_exit() function. 1100 */ 1101 llist_for_each_entry_safe(dev_hold, next, node, link) { 1102 struct drm_device *drm = dev_hold->drm; 1103 struct module *module = drm->driver->fops->owner; 1104 1105 drm_dbg(drm, "Releasing reference on provider device and module.\n"); 1106 drm_dev_put(drm); 1107 module_put(module); 1108 kfree(dev_hold); 1109 } 1110 } 1111 1112 static struct drm_pagemap_dev_hold * 1113 drm_pagemap_dev_hold(struct drm_pagemap *dpagemap) 1114 { 1115 struct drm_pagemap_dev_hold *dev_hold; 1116 struct drm_device *drm = dpagemap->drm; 1117 1118 dev_hold = kzalloc_obj(*dev_hold); 1119 if (!dev_hold) 1120 return ERR_PTR(-ENOMEM); 1121 1122 init_llist_node(&dev_hold->link); 1123 dev_hold->drm = drm; 1124 (void)try_module_get(drm->driver->fops->owner); 1125 drm_dev_get(drm); 1126 1127 return dev_hold; 1128 } 1129 1130 /** 1131 * drm_pagemap_reinit() - Reinitialize a drm_pagemap 1132 * @dpagemap: The drm_pagemap to reinitialize 1133 * 1134 * Reinitialize a drm_pagemap, for which drm_pagemap_release 1135 * has already been called. This interface is intended for the 1136 * situation where the driver caches a destroyed drm_pagemap. 1137 * 1138 * Return: 0 on success, negative error code on failure. 1139 */ 1140 int drm_pagemap_reinit(struct drm_pagemap *dpagemap) 1141 { 1142 dpagemap->dev_hold = drm_pagemap_dev_hold(dpagemap); 1143 if (IS_ERR(dpagemap->dev_hold)) 1144 return PTR_ERR(dpagemap->dev_hold); 1145 1146 kref_init(&dpagemap->ref); 1147 return 0; 1148 } 1149 EXPORT_SYMBOL(drm_pagemap_reinit); 1150 1151 /** 1152 * drm_pagemap_init() - Initialize a pre-allocated drm_pagemap 1153 * @dpagemap: The drm_pagemap to initialize. 1154 * @pagemap: The associated dev_pagemap providing the device 1155 * private pages. 1156 * @drm: The drm device. The drm_pagemap holds a reference on the 1157 * drm_device and the module owning the drm_device until 1158 * drm_pagemap_release(). This facilitates drm_pagemap exporting. 1159 * @ops: The drm_pagemap ops. 1160 * 1161 * Initialize and take an initial reference on a drm_pagemap. 1162 * After successful return, use drm_pagemap_put() to destroy. 1163 * 1164 ** Return: 0 on success, negative error code on error. 1165 */ 1166 int drm_pagemap_init(struct drm_pagemap *dpagemap, 1167 struct dev_pagemap *pagemap, 1168 struct drm_device *drm, 1169 const struct drm_pagemap_ops *ops) 1170 { 1171 kref_init(&dpagemap->ref); 1172 dpagemap->ops = ops; 1173 dpagemap->pagemap = pagemap; 1174 dpagemap->drm = drm; 1175 dpagemap->cache = NULL; 1176 INIT_LIST_HEAD(&dpagemap->shrink_link); 1177 1178 return drm_pagemap_reinit(dpagemap); 1179 } 1180 EXPORT_SYMBOL(drm_pagemap_init); 1181 1182 /** 1183 * drm_pagemap_put() - Put a struct drm_pagemap reference 1184 * @dpagemap: Pointer to a struct drm_pagemap object. 1185 * 1186 * Puts a struct drm_pagemap reference and frees the drm_pagemap object 1187 * if the refount reaches zero. 1188 */ 1189 void drm_pagemap_put(struct drm_pagemap *dpagemap) 1190 { 1191 if (likely(dpagemap)) { 1192 drm_pagemap_shrinker_might_lock(dpagemap); 1193 kref_put(&dpagemap->ref, drm_pagemap_release); 1194 } 1195 } 1196 EXPORT_SYMBOL(drm_pagemap_put); 1197 1198 /** 1199 * drm_pagemap_page_get_flags() - Read flags from a device-private folio 1200 * @page: Pointer to a page of the device-private folio 1201 * 1202 * Return: The DRM_PAGEMAP_ZDD_FLAG_* bits encoded in zone_device_data. 1203 */ 1204 static unsigned long drm_pagemap_page_get_flags(struct page *page) 1205 { 1206 struct folio *folio = page_folio(page); 1207 1208 return (unsigned long)folio_zone_device_data(folio) & 1209 DRM_PAGEMAP_ZDD_FLAG_MASK; 1210 } 1211 1212 /** 1213 * drm_pagemap_page_set_flags() - Set flags on a device-private folio 1214 * @page: Pointer to a page of the device-private folio 1215 * @flags: DRM_PAGEMAP_ZDD_FLAG_* bits to set 1216 * 1217 * Preserve any flags already encoded alongside the ZDD pointer. 1218 */ 1219 static void drm_pagemap_page_set_flags(struct page *page, 1220 unsigned long flags) 1221 { 1222 struct folio *folio = page_folio(page); 1223 unsigned long old; 1224 1225 if (WARN_ON_ONCE(flags & ~DRM_PAGEMAP_ZDD_FLAG_MASK)) 1226 return; 1227 1228 old = (unsigned long)folio_zone_device_data(folio); 1229 folio_set_zone_device_data(folio, (void *)(old | flags)); 1230 } 1231 1232 /** 1233 * drm_pagemap_retire_migrated_pages() - Record migrated device-private folios 1234 * @src_pfns: source array after migrate_vma_pages() or migrate_device_pages() 1235 * @npages: number of entries in @src_pfns 1236 * 1237 * Flag device-private folios successfully migrated to RAM before finalize 1238 * unlocks the sources. The migrated state is stored in the physical folio, so 1239 * it survives later folio splits and subsequent migrations can skip it. 1240 */ 1241 static void drm_pagemap_retire_migrated_pages(unsigned long *src_pfns, 1242 unsigned long npages) 1243 { 1244 unsigned long i = 0; 1245 1246 while (i < npages) { 1247 struct page *page = migrate_pfn_to_page(src_pfns[i]); 1248 unsigned long nr = 1; 1249 1250 if (!page) { 1251 i++; 1252 continue; 1253 } 1254 1255 if (src_pfns[i] & MIGRATE_PFN_COMPOUND) 1256 nr = folio_nr_pages(page_folio(page)); 1257 1258 if ((src_pfns[i] & MIGRATE_PFN_MIGRATE) && 1259 is_device_private_page(page)) 1260 drm_pagemap_page_set_flags(page, 1261 DRM_PAGEMAP_ZDD_FLAG_MIGRATED); 1262 1263 i += nr; 1264 } 1265 } 1266 1267 /** 1268 * drm_pagemap_skip_retired_pages() - Skip retired device-private folios 1269 * @src_pfns: MIGRATE_PFN-encoded source array 1270 * @npages: number of entries in @src_pfns 1271 * 1272 * Skip source folios already migrated to RAM, identified by the migrated flag 1273 * stored in the physical folio's zone_device_data. 1274 */ 1275 static void drm_pagemap_skip_retired_pages(unsigned long *src_pfns, 1276 unsigned long npages) 1277 { 1278 unsigned long i = 0; 1279 1280 while (i < npages) { 1281 struct page *page = migrate_pfn_to_page(src_pfns[i]); 1282 unsigned long nr = 1; 1283 1284 if (!page) { 1285 i++; 1286 continue; 1287 } 1288 1289 if (src_pfns[i] & MIGRATE_PFN_COMPOUND) 1290 nr = folio_nr_pages(page_folio(page)); 1291 1292 if ((src_pfns[i] & MIGRATE_PFN_MIGRATE) && 1293 is_device_private_page(page) && 1294 (drm_pagemap_page_get_flags(page) & 1295 DRM_PAGEMAP_ZDD_FLAG_MIGRATED)) 1296 src_pfns[i] &= ~MIGRATE_PFN_MIGRATE; 1297 1298 i += nr; 1299 } 1300 } 1301 1302 /** 1303 * drm_pagemap_evict_to_ram() - Evict GPU SVM range to RAM 1304 * @devmem_allocation: Pointer to the device memory allocation 1305 * 1306 * Similar to __drm_pagemap_migrate_to_ram(), but uses the 1307 * migrate_device_* helpers and does not require the mmap lock. 1308 * Device-private PFNs already migrated to RAM by either path are skipped. 1309 * 1310 * Return: 0 on success, negative error code on failure. 1311 */ 1312 int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation) 1313 { 1314 const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops; 1315 struct drm_pagemap_iova_state state = {}; 1316 unsigned long npages, mpages; 1317 struct page **pages; 1318 unsigned long *src, *dst; 1319 struct drm_pagemap_addr *pagemap_addr; 1320 void *buf; 1321 int err = 0; 1322 unsigned int retry_count = 2; 1323 1324 npages = devmem_allocation->size >> PAGE_SHIFT; 1325 1326 retry: 1327 mpages = 0; 1328 if (!mmget_not_zero(devmem_allocation->mm)) 1329 return -EFAULT; 1330 1331 buf = kvcalloc(npages, 2 * sizeof(*src) + sizeof(*pagemap_addr) + 1332 sizeof(*pages), GFP_KERNEL); 1333 if (!buf) { 1334 err = -ENOMEM; 1335 goto err_out; 1336 } 1337 src = buf; 1338 dst = buf + (sizeof(*src) * npages); 1339 pagemap_addr = buf + (2 * sizeof(*src) * npages); 1340 pages = buf + (2 * sizeof(*src) + sizeof(*pagemap_addr)) * npages; 1341 1342 err = ops->populate_devmem_pfn(devmem_allocation, npages, src); 1343 if (err) 1344 goto err_free; 1345 1346 err = migrate_device_pfns(src, npages); 1347 if (err) 1348 goto err_free; 1349 1350 drm_pagemap_skip_retired_pages(src, npages); 1351 1352 err = drm_pagemap_migrate_populate_ram_pfn(NULL, NULL, npages, &mpages, 1353 src, dst, 0); 1354 if (err || !mpages) 1355 goto err_finalize; 1356 1357 err = drm_pagemap_migrate_map_system_pages(devmem_allocation->dev, 1358 pagemap_addr, 1359 dst, npages, 1360 DMA_FROM_DEVICE, &state); 1361 if (err) 1362 goto err_finalize; 1363 1364 drm_pagemap_migrate_populate_src_pages(pages, src, dst, npages); 1365 1366 err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL); 1367 if (err) 1368 goto err_finalize; 1369 1370 err_finalize: 1371 drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, dst, npages, 1372 DMA_FROM_DEVICE, &state); 1373 if (err) 1374 drm_pagemap_migration_unlock_put_pages(npages, dst); 1375 migrate_device_pages(src, dst, npages); 1376 drm_pagemap_retire_migrated_pages(src, npages); 1377 migrate_device_finalize(src, dst, npages); 1378 1379 err_free: 1380 kvfree(buf); 1381 err_out: 1382 mmput_async(devmem_allocation->mm); 1383 1384 if (completion_done(&devmem_allocation->detached)) 1385 return 0; 1386 1387 if (retry_count--) { 1388 cond_resched(); 1389 state = (struct drm_pagemap_iova_state){}; 1390 goto retry; 1391 } 1392 1393 return err ?: -EBUSY; 1394 } 1395 EXPORT_SYMBOL_GPL(drm_pagemap_evict_to_ram); 1396 1397 /** 1398 * __drm_pagemap_migrate_to_ram() - Migrate GPU SVM range to RAM (internal) 1399 * @vas: Pointer to the VM area structure 1400 * @page: Pointer to the page for fault handling. 1401 * @fault_addr: Fault address 1402 * @size: Size of migration 1403 * 1404 * This internal function performs the migration of the specified GPU SVM range 1405 * to RAM. It sets up the migration, populates + dma maps RAM PFNs, and 1406 * invokes the driver-specific operations for migration to RAM. 1407 * 1408 * Return: 0 on success, negative error code on failure. 1409 */ 1410 static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas, 1411 struct page *page, 1412 unsigned long fault_addr, 1413 unsigned long size) 1414 { 1415 struct migrate_vma migrate = { 1416 .vma = vas, 1417 .pgmap_owner = page_pgmap(page)->owner, 1418 .flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE | 1419 MIGRATE_VMA_SELECT_DEVICE_COHERENT | 1420 MIGRATE_VMA_SELECT_COMPOUND, 1421 .fault_page = page, 1422 }; 1423 struct drm_pagemap_iova_state state = {}; 1424 struct drm_pagemap_zdd *zdd; 1425 const struct drm_pagemap_devmem_ops *ops; 1426 struct device *dev = NULL; 1427 unsigned long npages, mpages = 0; 1428 struct page **pages; 1429 struct drm_pagemap_addr *pagemap_addr; 1430 unsigned long start, end; 1431 void *buf; 1432 int err = 0; 1433 1434 zdd = drm_pagemap_page_zone_device_data(page); 1435 if (time_before64(get_jiffies_64(), zdd->devmem_allocation->timeslice_expiration)) 1436 return 0; 1437 1438 start = ALIGN_DOWN(fault_addr, size); 1439 end = ALIGN(fault_addr + 1, size); 1440 1441 /* Corner where VMA area struct has been partially unmapped */ 1442 if (start < vas->vm_start) 1443 start = vas->vm_start; 1444 if (end > vas->vm_end) 1445 end = vas->vm_end; 1446 1447 migrate.start = start; 1448 migrate.end = end; 1449 npages = npages_in_range(start, end); 1450 1451 buf = kvcalloc(npages, 2 * sizeof(*migrate.src) + sizeof(*pagemap_addr) + 1452 sizeof(*pages), GFP_KERNEL); 1453 if (!buf) { 1454 err = -ENOMEM; 1455 goto err_out; 1456 } 1457 pagemap_addr = buf + (2 * sizeof(*migrate.src) * npages); 1458 pages = buf + (2 * sizeof(*migrate.src) + sizeof(*pagemap_addr)) * npages; 1459 1460 migrate.vma = vas; 1461 migrate.src = buf; 1462 migrate.dst = migrate.src + npages; 1463 1464 err = migrate_vma_setup(&migrate); 1465 if (err) 1466 goto err_free; 1467 1468 /* Raced with another CPU fault, nothing to do */ 1469 if (!migrate.cpages) 1470 goto err_free; 1471 1472 drm_pagemap_skip_retired_pages(migrate.src, npages); 1473 1474 ops = zdd->devmem_allocation->ops; 1475 dev = zdd->devmem_allocation->dev; 1476 1477 err = drm_pagemap_migrate_populate_ram_pfn(vas, page, npages, &mpages, 1478 migrate.src, migrate.dst, 1479 start); 1480 if (err || !mpages) 1481 goto err_finalize; 1482 1483 err = drm_pagemap_migrate_map_system_pages(dev, pagemap_addr, 1484 migrate.dst, npages, 1485 DMA_FROM_DEVICE, &state); 1486 if (err) 1487 goto err_finalize; 1488 1489 drm_pagemap_migrate_populate_src_pages(pages, migrate.src, migrate.dst, 1490 npages); 1491 1492 err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL); 1493 if (err) 1494 goto err_finalize; 1495 1496 err_finalize: 1497 if (dev) 1498 drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, migrate.dst, 1499 npages, DMA_FROM_DEVICE, 1500 &state); 1501 if (err) 1502 drm_pagemap_migration_unlock_put_pages(npages, migrate.dst); 1503 migrate_vma_pages(&migrate); 1504 drm_pagemap_retire_migrated_pages(migrate.src, npages); 1505 migrate_vma_finalize(&migrate); 1506 err_free: 1507 kvfree(buf); 1508 err_out: 1509 1510 return err; 1511 } 1512 1513 /** 1514 * drm_pagemap_folio_free() - Put GPU SVM zone device data associated with a folio 1515 * @folio: Pointer to the folio 1516 * 1517 * This function is a callback used to put the GPU SVM zone device data 1518 * associated with a page when it is being released. 1519 */ 1520 static void drm_pagemap_folio_free(struct folio *folio) 1521 { 1522 struct page *page = folio_page(folio, 0); 1523 1524 drm_pagemap_zdd_put(drm_pagemap_page_zone_device_data(page)); 1525 } 1526 1527 /** 1528 * drm_pagemap_migrate_to_ram() - Migrate a virtual range to RAM (page fault handler) 1529 * @vmf: Pointer to the fault information structure 1530 * 1531 * This function is a page fault handler used to migrate a virtual range 1532 * to ram. The device memory allocation in which the device page is found is 1533 * migrated in its entirety. 1534 * 1535 * Returns: 1536 * VM_FAULT_SIGBUS on failure, 0 on success. 1537 */ 1538 static vm_fault_t drm_pagemap_migrate_to_ram(struct vm_fault *vmf) 1539 { 1540 struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(vmf->page); 1541 int err; 1542 1543 err = __drm_pagemap_migrate_to_ram(vmf->vma, 1544 vmf->page, vmf->address, 1545 zdd->devmem_allocation->size); 1546 1547 return err ? VM_FAULT_SIGBUS : 0; 1548 } 1549 1550 static void drm_pagemap_folio_split(struct folio *orig_folio, struct folio *new_folio) 1551 { 1552 struct drm_pagemap_zdd *zdd; 1553 unsigned long orig_data, new_data; 1554 1555 if (!new_folio) 1556 return; 1557 1558 new_folio->pgmap = orig_folio->pgmap; 1559 1560 orig_data = (unsigned long)folio_zone_device_data(orig_folio); 1561 zdd = (struct drm_pagemap_zdd *)(orig_data & ~DRM_PAGEMAP_ZDD_FLAG_MASK); 1562 1563 new_data = (unsigned long)drm_pagemap_zdd_get(zdd); 1564 new_data |= orig_data & DRM_PAGEMAP_ZDD_FLAG_MASK; 1565 folio_set_zone_device_data(new_folio, (void *)new_data); 1566 } 1567 1568 static const struct dev_pagemap_ops drm_pagemap_pagemap_ops = { 1569 .folio_free = drm_pagemap_folio_free, 1570 .migrate_to_ram = drm_pagemap_migrate_to_ram, 1571 .folio_split = drm_pagemap_folio_split, 1572 }; 1573 1574 /** 1575 * drm_pagemap_pagemap_ops_get() - Retrieve GPU SVM device page map operations 1576 * 1577 * Returns: 1578 * Pointer to the GPU SVM device page map operations structure. 1579 */ 1580 const struct dev_pagemap_ops *drm_pagemap_pagemap_ops_get(void) 1581 { 1582 return &drm_pagemap_pagemap_ops; 1583 } 1584 EXPORT_SYMBOL_GPL(drm_pagemap_pagemap_ops_get); 1585 1586 /** 1587 * drm_pagemap_devmem_init() - Initialize a drm_pagemap device memory allocation 1588 * 1589 * @devmem_allocation: The struct drm_pagemap_devmem to initialize. 1590 * @dev: Pointer to the device structure which device memory allocation belongs to 1591 * @mm: Pointer to the mm_struct for the address space 1592 * @ops: Pointer to the operations structure for GPU SVM device memory 1593 * @dpagemap: The struct drm_pagemap we're allocating from. 1594 * @size: Size of device memory allocation 1595 * @pre_migrate_fence: Fence to wait for or pipeline behind before migration starts. 1596 * (May be NULL). 1597 */ 1598 void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation, 1599 struct device *dev, struct mm_struct *mm, 1600 const struct drm_pagemap_devmem_ops *ops, 1601 struct drm_pagemap *dpagemap, size_t size, 1602 struct dma_fence *pre_migrate_fence) 1603 { 1604 init_completion(&devmem_allocation->detached); 1605 devmem_allocation->dev = dev; 1606 devmem_allocation->mm = mm; 1607 devmem_allocation->ops = ops; 1608 devmem_allocation->dpagemap = dpagemap; 1609 devmem_allocation->size = size; 1610 devmem_allocation->pre_migrate_fence = pre_migrate_fence; 1611 } 1612 EXPORT_SYMBOL_GPL(drm_pagemap_devmem_init); 1613 1614 /** 1615 * drm_pagemap_page_to_dpagemap() - Return a pointer the drm_pagemap of a page 1616 * @page: The struct page. 1617 * 1618 * Return: A pointer to the struct drm_pagemap of a device private page that 1619 * was populated from the struct drm_pagemap. If the page was *not* populated 1620 * from a struct drm_pagemap, the result is undefined and the function call 1621 * may result in dereferencing and invalid address. 1622 */ 1623 struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page) 1624 { 1625 struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page); 1626 1627 return zdd->devmem_allocation->dpagemap; 1628 } 1629 EXPORT_SYMBOL_GPL(drm_pagemap_page_to_dpagemap); 1630 1631 /** 1632 * drm_pagemap_populate_mm() - Populate a virtual range with device memory pages 1633 * @dpagemap: Pointer to the drm_pagemap managing the device memory 1634 * @start: Start of the virtual range to populate. 1635 * @end: End of the virtual range to populate. 1636 * @mm: Pointer to the virtual address space. 1637 * @timeslice_ms: The time requested for the migrated pagemap pages to 1638 * be present in @mm before being allowed to be migrated back. 1639 * 1640 * Attempt to populate a virtual range with device memory pages, 1641 * clearing them or migrating data from the existing pages if necessary. 1642 * The function is best effort only, and implementations may vary 1643 * in how hard they try to satisfy the request. 1644 * 1645 * Return: %0 on success, negative error code on error. If the hardware 1646 * device was removed / unbound the function will return %-ENODEV. 1647 */ 1648 int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, 1649 unsigned long start, unsigned long end, 1650 struct mm_struct *mm, 1651 unsigned long timeslice_ms) 1652 { 1653 int err; 1654 1655 if (!mmget_not_zero(mm)) 1656 return -EFAULT; 1657 mmap_read_lock(mm); 1658 err = dpagemap->ops->populate_mm(dpagemap, start, end, mm, 1659 timeslice_ms); 1660 mmap_read_unlock(mm); 1661 mmput(mm); 1662 1663 return err; 1664 } 1665 EXPORT_SYMBOL(drm_pagemap_populate_mm); 1666 1667 void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim) 1668 { 1669 if (dpagemap->ops->destroy) 1670 dpagemap->ops->destroy(dpagemap, is_atomic_or_reclaim); 1671 else 1672 kfree(dpagemap); 1673 } 1674 1675 static void drm_pagemap_exit(void) 1676 { 1677 flush_work(&drm_pagemap_work); 1678 if (WARN_ON(!llist_empty(&drm_pagemap_unhold_list))) 1679 disable_work_sync(&drm_pagemap_work); 1680 } 1681 module_exit(drm_pagemap_exit); 1682