1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
2 /*
3 * Copyright © 2024-2025 Intel Corporation
4 */
5
6 #include <linux/dma-mapping.h>
7 #include <linux/migrate.h>
8 #include <linux/pagemap.h>
9 #include <drm/drm_drv.h>
10 #include <drm/drm_pagemap.h>
11
12 /**
13 * DOC: Overview
14 *
15 * The DRM pagemap layer is intended to augment the dev_pagemap functionality by
16 * providing a way to populate a struct mm_struct virtual range with device
17 * private pages and to provide helpers to abstract device memory allocations,
18 * to migrate memory back and forth between device memory and system RAM and
19 * to handle access (and in the future migration) between devices implementing
20 * a fast interconnect that is not necessarily visible to the rest of the
21 * system.
22 *
23 * Typically the DRM pagemap receives requests from one or more DRM GPU SVM
24 * instances to populate struct mm_struct virtual ranges with memory, and the
25 * migration is best effort only and may thus fail. The implementation should
26 * also handle device unbinding by blocking (return an -ENODEV) error for new
27 * population requests and after that migrate all device pages to system ram.
28 */
29
30 /**
31 * DOC: Migration
32 *
33 * Migration granularity typically follows the GPU SVM range requests, but
34 * if there are clashes, due to races or due to the fact that multiple GPU
35 * SVM instances have different views of the ranges used, and because of that
36 * parts of a requested range is already present in the requested device memory,
37 * the implementation has a variety of options. It can fail and it can choose
38 * to populate only the part of the range that isn't already in device memory,
39 * and it can evict the range to system before trying to migrate. Ideally an
40 * implementation would just try to migrate the missing part of the range and
41 * allocate just enough memory to do so.
42 *
43 * When migrating to system memory as a response to a cpu fault or a device
44 * memory eviction request, currently a full device memory allocation is
45 * migrated back to system. Moving forward this might need improvement for
46 * situations where a single page needs bouncing between system memory and
47 * device memory due to, for example, atomic operations.
48 *
49 * Key DRM pagemap components:
50 *
51 * - Device Memory Allocations:
52 * Embedded structure containing enough information for the drm_pagemap to
53 * migrate to / from device memory.
54 *
55 * - Device Memory Operations:
56 * Define the interface for driver-specific device memory operations
57 * release memory, populate pfns, and copy to / from device memory.
58 */
59
60 /**
61 * struct drm_pagemap_zdd - GPU SVM zone device data
62 *
63 * @refcount: Reference count for the zdd
64 * @devmem_allocation: device memory allocation
65 * @device_private_page_owner: Device private pages owner
66 *
67 * This structure serves as a generic wrapper installed in
68 * page->zone_device_data. It provides infrastructure for looking up a device
69 * memory allocation upon CPU page fault and asynchronously releasing device
70 * memory once the CPU has no page references. Asynchronous release is useful
71 * because CPU page references can be dropped in IRQ contexts, while releasing
72 * device memory likely requires sleeping locks.
73 */
74 struct drm_pagemap_zdd {
75 struct kref refcount;
76 struct drm_pagemap_devmem *devmem_allocation;
77 void *device_private_page_owner;
78 };
79
80 /**
81 * drm_pagemap_zdd_alloc() - Allocate a zdd structure.
82 * @device_private_page_owner: Device private pages owner
83 *
84 * This function allocates and initializes a new zdd structure. It sets up the
85 * reference count and initializes the destroy work.
86 *
87 * Return: Pointer to the allocated zdd on success, ERR_PTR() on failure.
88 */
89 static struct drm_pagemap_zdd *
drm_pagemap_zdd_alloc(void * device_private_page_owner)90 drm_pagemap_zdd_alloc(void *device_private_page_owner)
91 {
92 struct drm_pagemap_zdd *zdd;
93
94 zdd = kmalloc(sizeof(*zdd), GFP_KERNEL);
95 if (!zdd)
96 return NULL;
97
98 kref_init(&zdd->refcount);
99 zdd->devmem_allocation = NULL;
100 zdd->device_private_page_owner = device_private_page_owner;
101
102 return zdd;
103 }
104
105 /**
106 * drm_pagemap_zdd_get() - Get a reference to a zdd structure.
107 * @zdd: Pointer to the zdd structure.
108 *
109 * This function increments the reference count of the provided zdd structure.
110 *
111 * Return: Pointer to the zdd structure.
112 */
drm_pagemap_zdd_get(struct drm_pagemap_zdd * zdd)113 static struct drm_pagemap_zdd *drm_pagemap_zdd_get(struct drm_pagemap_zdd *zdd)
114 {
115 kref_get(&zdd->refcount);
116 return zdd;
117 }
118
119 /**
120 * drm_pagemap_zdd_destroy() - Destroy a zdd structure.
121 * @ref: Pointer to the reference count structure.
122 *
123 * This function queues the destroy_work of the zdd for asynchronous destruction.
124 */
drm_pagemap_zdd_destroy(struct kref * ref)125 static void drm_pagemap_zdd_destroy(struct kref *ref)
126 {
127 struct drm_pagemap_zdd *zdd =
128 container_of(ref, struct drm_pagemap_zdd, refcount);
129 struct drm_pagemap_devmem *devmem = zdd->devmem_allocation;
130
131 if (devmem) {
132 complete_all(&devmem->detached);
133 if (devmem->ops->devmem_release)
134 devmem->ops->devmem_release(devmem);
135 }
136 kfree(zdd);
137 }
138
139 /**
140 * drm_pagemap_zdd_put() - Put a zdd reference.
141 * @zdd: Pointer to the zdd structure.
142 *
143 * This function decrements the reference count of the provided zdd structure
144 * and schedules its destruction if the count drops to zero.
145 */
drm_pagemap_zdd_put(struct drm_pagemap_zdd * zdd)146 static void drm_pagemap_zdd_put(struct drm_pagemap_zdd *zdd)
147 {
148 kref_put(&zdd->refcount, drm_pagemap_zdd_destroy);
149 }
150
151 /**
152 * drm_pagemap_migration_unlock_put_page() - Put a migration page
153 * @page: Pointer to the page to put
154 *
155 * This function unlocks and puts a page.
156 */
drm_pagemap_migration_unlock_put_page(struct page * page)157 static void drm_pagemap_migration_unlock_put_page(struct page *page)
158 {
159 unlock_page(page);
160 put_page(page);
161 }
162
163 /**
164 * drm_pagemap_migration_unlock_put_pages() - Put migration pages
165 * @npages: Number of pages
166 * @migrate_pfn: Array of migrate page frame numbers
167 *
168 * This function unlocks and puts an array of pages.
169 */
drm_pagemap_migration_unlock_put_pages(unsigned long npages,unsigned long * migrate_pfn)170 static void drm_pagemap_migration_unlock_put_pages(unsigned long npages,
171 unsigned long *migrate_pfn)
172 {
173 unsigned long i;
174
175 for (i = 0; i < npages; ++i) {
176 struct page *page;
177
178 if (!migrate_pfn[i])
179 continue;
180
181 page = migrate_pfn_to_page(migrate_pfn[i]);
182 drm_pagemap_migration_unlock_put_page(page);
183 migrate_pfn[i] = 0;
184 }
185 }
186
187 /**
188 * drm_pagemap_get_devmem_page() - Get a reference to a device memory page
189 * @page: Pointer to the page
190 * @zdd: Pointer to the GPU SVM zone device data
191 *
192 * This function associates the given page with the specified GPU SVM zone
193 * device data and initializes it for zone device usage.
194 */
drm_pagemap_get_devmem_page(struct page * page,struct drm_pagemap_zdd * zdd)195 static void drm_pagemap_get_devmem_page(struct page *page,
196 struct drm_pagemap_zdd *zdd)
197 {
198 page->zone_device_data = drm_pagemap_zdd_get(zdd);
199 zone_device_page_init(page, 0);
200 }
201
202 /**
203 * drm_pagemap_migrate_map_pages() - Map migration pages for GPU SVM migration
204 * @dev: The device for which the pages are being mapped
205 * @pagemap_addr: Array to store DMA information corresponding to mapped pages
206 * @migrate_pfn: Array of migrate page frame numbers to map
207 * @npages: Number of pages to map
208 * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL)
209 *
210 * This function maps pages of memory for migration usage in GPU SVM. It
211 * iterates over each page frame number provided in @migrate_pfn, maps the
212 * corresponding page, and stores the DMA address in the provided @dma_addr
213 * array.
214 *
215 * Returns: 0 on success, -EFAULT if an error occurs during mapping.
216 */
drm_pagemap_migrate_map_pages(struct device * dev,struct drm_pagemap_addr * pagemap_addr,unsigned long * migrate_pfn,unsigned long npages,enum dma_data_direction dir)217 static int drm_pagemap_migrate_map_pages(struct device *dev,
218 struct drm_pagemap_addr *pagemap_addr,
219 unsigned long *migrate_pfn,
220 unsigned long npages,
221 enum dma_data_direction dir)
222 {
223 unsigned long i;
224
225 for (i = 0; i < npages;) {
226 struct page *page = migrate_pfn_to_page(migrate_pfn[i]);
227 dma_addr_t dma_addr;
228 struct folio *folio;
229 unsigned int order = 0;
230
231 if (!page)
232 goto next;
233
234 if (WARN_ON_ONCE(is_zone_device_page(page)))
235 return -EFAULT;
236
237 folio = page_folio(page);
238 order = folio_order(folio);
239
240 dma_addr = dma_map_page(dev, page, 0, page_size(page), dir);
241 if (dma_mapping_error(dev, dma_addr))
242 return -EFAULT;
243
244 pagemap_addr[i] =
245 drm_pagemap_addr_encode(dma_addr,
246 DRM_INTERCONNECT_SYSTEM,
247 order, dir);
248
249 next:
250 i += NR_PAGES(order);
251 }
252
253 return 0;
254 }
255
256 /**
257 * drm_pagemap_migrate_unmap_pages() - Unmap pages previously mapped for GPU SVM migration
258 * @dev: The device for which the pages were mapped
259 * @pagemap_addr: Array of DMA information corresponding to mapped pages
260 * @npages: Number of pages to unmap
261 * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL)
262 *
263 * This function unmaps previously mapped pages of memory for GPU Shared Virtual
264 * Memory (SVM). It iterates over each DMA address provided in @dma_addr, checks
265 * if it's valid and not already unmapped, and unmaps the corresponding page.
266 */
drm_pagemap_migrate_unmap_pages(struct device * dev,struct drm_pagemap_addr * pagemap_addr,unsigned long npages,enum dma_data_direction dir)267 static void drm_pagemap_migrate_unmap_pages(struct device *dev,
268 struct drm_pagemap_addr *pagemap_addr,
269 unsigned long npages,
270 enum dma_data_direction dir)
271 {
272 unsigned long i;
273
274 for (i = 0; i < npages;) {
275 if (!pagemap_addr[i].addr || dma_mapping_error(dev, pagemap_addr[i].addr))
276 goto next;
277
278 dma_unmap_page(dev, pagemap_addr[i].addr, PAGE_SIZE << pagemap_addr[i].order, dir);
279
280 next:
281 i += NR_PAGES(pagemap_addr[i].order);
282 }
283 }
284
285 static unsigned long
npages_in_range(unsigned long start,unsigned long end)286 npages_in_range(unsigned long start, unsigned long end)
287 {
288 return (end - start) >> PAGE_SHIFT;
289 }
290
291 /**
292 * drm_pagemap_migrate_to_devmem() - Migrate a struct mm_struct range to device memory
293 * @devmem_allocation: The device memory allocation to migrate to.
294 * The caller should hold a reference to the device memory allocation,
295 * and the reference is consumed by this function unless it returns with
296 * an error.
297 * @mm: Pointer to the struct mm_struct.
298 * @start: Start of the virtual address range to migrate.
299 * @end: End of the virtual address range to migrate.
300 * @timeslice_ms: The time requested for the migrated pagemap pages to
301 * be present in @mm before being allowed to be migrated back.
302 * @pgmap_owner: Not used currently, since only system memory is considered.
303 *
304 * This function migrates the specified virtual address range to device memory.
305 * It performs the necessary setup and invokes the driver-specific operations for
306 * migration to device memory. Expected to be called while holding the mmap lock in
307 * at least read mode.
308 *
309 * Note: The @timeslice_ms parameter can typically be used to force data to
310 * remain in pagemap pages long enough for a GPU to perform a task and to prevent
311 * a migration livelock. One alternative would be for the GPU driver to block
312 * in a mmu_notifier for the specified amount of time, but adding the
313 * functionality to the pagemap is likely nicer to the system as a whole.
314 *
315 * Return: %0 on success, negative error code on failure.
316 */
drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem * devmem_allocation,struct mm_struct * mm,unsigned long start,unsigned long end,unsigned long timeslice_ms,void * pgmap_owner)317 int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation,
318 struct mm_struct *mm,
319 unsigned long start, unsigned long end,
320 unsigned long timeslice_ms,
321 void *pgmap_owner)
322 {
323 const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops;
324 struct migrate_vma migrate = {
325 .start = start,
326 .end = end,
327 .pgmap_owner = pgmap_owner,
328 .flags = MIGRATE_VMA_SELECT_SYSTEM,
329 };
330 unsigned long i, npages = npages_in_range(start, end);
331 struct vm_area_struct *vas;
332 struct drm_pagemap_zdd *zdd = NULL;
333 struct page **pages;
334 struct drm_pagemap_addr *pagemap_addr;
335 void *buf;
336 int err;
337
338 mmap_assert_locked(mm);
339
340 if (!ops->populate_devmem_pfn || !ops->copy_to_devmem ||
341 !ops->copy_to_ram)
342 return -EOPNOTSUPP;
343
344 vas = vma_lookup(mm, start);
345 if (!vas) {
346 err = -ENOENT;
347 goto err_out;
348 }
349
350 if (end > vas->vm_end || start < vas->vm_start) {
351 err = -EINVAL;
352 goto err_out;
353 }
354
355 if (!vma_is_anonymous(vas)) {
356 err = -EBUSY;
357 goto err_out;
358 }
359
360 buf = kvcalloc(npages, 2 * sizeof(*migrate.src) + sizeof(*pagemap_addr) +
361 sizeof(*pages), GFP_KERNEL);
362 if (!buf) {
363 err = -ENOMEM;
364 goto err_out;
365 }
366 pagemap_addr = buf + (2 * sizeof(*migrate.src) * npages);
367 pages = buf + (2 * sizeof(*migrate.src) + sizeof(*pagemap_addr)) * npages;
368
369 zdd = drm_pagemap_zdd_alloc(pgmap_owner);
370 if (!zdd) {
371 err = -ENOMEM;
372 goto err_free;
373 }
374
375 migrate.vma = vas;
376 migrate.src = buf;
377 migrate.dst = migrate.src + npages;
378
379 err = migrate_vma_setup(&migrate);
380 if (err)
381 goto err_free;
382
383 if (!migrate.cpages) {
384 err = -EFAULT;
385 goto err_free;
386 }
387
388 if (migrate.cpages != npages) {
389 err = -EBUSY;
390 goto err_finalize;
391 }
392
393 err = ops->populate_devmem_pfn(devmem_allocation, npages, migrate.dst);
394 if (err)
395 goto err_finalize;
396
397 err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, pagemap_addr,
398 migrate.src, npages, DMA_TO_DEVICE);
399
400 if (err)
401 goto err_finalize;
402
403 for (i = 0; i < npages; ++i) {
404 struct page *page = pfn_to_page(migrate.dst[i]);
405
406 pages[i] = page;
407 migrate.dst[i] = migrate_pfn(migrate.dst[i]);
408 drm_pagemap_get_devmem_page(page, zdd);
409 }
410
411 err = ops->copy_to_devmem(pages, pagemap_addr, npages);
412 if (err)
413 goto err_finalize;
414
415 /* Upon success bind devmem allocation to range and zdd */
416 devmem_allocation->timeslice_expiration = get_jiffies_64() +
417 msecs_to_jiffies(timeslice_ms);
418 zdd->devmem_allocation = devmem_allocation; /* Owns ref */
419
420 err_finalize:
421 if (err)
422 drm_pagemap_migration_unlock_put_pages(npages, migrate.dst);
423 migrate_vma_pages(&migrate);
424 migrate_vma_finalize(&migrate);
425 drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, npages,
426 DMA_TO_DEVICE);
427 err_free:
428 if (zdd)
429 drm_pagemap_zdd_put(zdd);
430 kvfree(buf);
431 err_out:
432 return err;
433 }
434 EXPORT_SYMBOL_GPL(drm_pagemap_migrate_to_devmem);
435
436 /**
437 * drm_pagemap_migrate_populate_ram_pfn() - Populate RAM PFNs for a VM area
438 * @vas: Pointer to the VM area structure, can be NULL
439 * @fault_page: Fault page
440 * @npages: Number of pages to populate
441 * @mpages: Number of pages to migrate
442 * @src_mpfn: Source array of migrate PFNs
443 * @mpfn: Array of migrate PFNs to populate
444 * @addr: Start address for PFN allocation
445 *
446 * This function populates the RAM migrate page frame numbers (PFNs) for the
447 * specified VM area structure. It allocates and locks pages in the VM area for
448 * RAM usage. If vas is non-NULL use alloc_page_vma for allocation, if NULL use
449 * alloc_page for allocation.
450 *
451 * Return: 0 on success, negative error code on failure.
452 */
drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct * vas,struct page * fault_page,unsigned long npages,unsigned long * mpages,unsigned long * src_mpfn,unsigned long * mpfn,unsigned long addr)453 static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
454 struct page *fault_page,
455 unsigned long npages,
456 unsigned long *mpages,
457 unsigned long *src_mpfn,
458 unsigned long *mpfn,
459 unsigned long addr)
460 {
461 unsigned long i;
462
463 for (i = 0; i < npages;) {
464 struct page *page = NULL, *src_page;
465 struct folio *folio;
466 unsigned int order = 0;
467
468 if (!(src_mpfn[i] & MIGRATE_PFN_MIGRATE))
469 goto next;
470
471 src_page = migrate_pfn_to_page(src_mpfn[i]);
472 if (!src_page)
473 goto next;
474
475 if (fault_page) {
476 if (src_page->zone_device_data !=
477 fault_page->zone_device_data)
478 goto next;
479 }
480
481 order = folio_order(page_folio(src_page));
482
483 /* TODO: Support fallback to single pages if THP allocation fails */
484 if (vas)
485 folio = vma_alloc_folio(GFP_HIGHUSER, order, vas, addr);
486 else
487 folio = folio_alloc(GFP_HIGHUSER, order);
488
489 if (!folio)
490 goto free_pages;
491
492 page = folio_page(folio, 0);
493 mpfn[i] = migrate_pfn(page_to_pfn(page));
494
495 next:
496 if (page)
497 addr += page_size(page);
498 else
499 addr += PAGE_SIZE;
500
501 i += NR_PAGES(order);
502 }
503
504 for (i = 0; i < npages;) {
505 struct page *page = migrate_pfn_to_page(mpfn[i]);
506 unsigned int order = 0;
507
508 if (!page)
509 goto next_lock;
510
511 WARN_ON_ONCE(!folio_trylock(page_folio(page)));
512
513 order = folio_order(page_folio(page));
514 *mpages += NR_PAGES(order);
515
516 next_lock:
517 i += NR_PAGES(order);
518 }
519
520 return 0;
521
522 free_pages:
523 for (i = 0; i < npages;) {
524 struct page *page = migrate_pfn_to_page(mpfn[i]);
525 unsigned int order = 0;
526
527 if (!page)
528 goto next_put;
529
530 put_page(page);
531 mpfn[i] = 0;
532
533 order = folio_order(page_folio(page));
534
535 next_put:
536 i += NR_PAGES(order);
537 }
538 return -ENOMEM;
539 }
540
541 /**
542 * drm_pagemap_evict_to_ram() - Evict GPU SVM range to RAM
543 * @devmem_allocation: Pointer to the device memory allocation
544 *
545 * Similar to __drm_pagemap_migrate_to_ram but does not require mmap lock and
546 * migration done via migrate_device_* functions.
547 *
548 * Return: 0 on success, negative error code on failure.
549 */
drm_pagemap_evict_to_ram(struct drm_pagemap_devmem * devmem_allocation)550 int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation)
551 {
552 const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops;
553 unsigned long npages, mpages = 0;
554 struct page **pages;
555 unsigned long *src, *dst;
556 struct drm_pagemap_addr *pagemap_addr;
557 void *buf;
558 int i, err = 0;
559 unsigned int retry_count = 2;
560
561 npages = devmem_allocation->size >> PAGE_SHIFT;
562
563 retry:
564 if (!mmget_not_zero(devmem_allocation->mm))
565 return -EFAULT;
566
567 buf = kvcalloc(npages, 2 * sizeof(*src) + sizeof(*pagemap_addr) +
568 sizeof(*pages), GFP_KERNEL);
569 if (!buf) {
570 err = -ENOMEM;
571 goto err_out;
572 }
573 src = buf;
574 dst = buf + (sizeof(*src) * npages);
575 pagemap_addr = buf + (2 * sizeof(*src) * npages);
576 pages = buf + (2 * sizeof(*src) + sizeof(*pagemap_addr)) * npages;
577
578 err = ops->populate_devmem_pfn(devmem_allocation, npages, src);
579 if (err)
580 goto err_free;
581
582 err = migrate_device_pfns(src, npages);
583 if (err)
584 goto err_free;
585
586 err = drm_pagemap_migrate_populate_ram_pfn(NULL, NULL, npages, &mpages,
587 src, dst, 0);
588 if (err || !mpages)
589 goto err_finalize;
590
591 err = drm_pagemap_migrate_map_pages(devmem_allocation->dev, pagemap_addr,
592 dst, npages, DMA_FROM_DEVICE);
593 if (err)
594 goto err_finalize;
595
596 for (i = 0; i < npages; ++i)
597 pages[i] = migrate_pfn_to_page(src[i]);
598
599 err = ops->copy_to_ram(pages, pagemap_addr, npages);
600 if (err)
601 goto err_finalize;
602
603 err_finalize:
604 if (err)
605 drm_pagemap_migration_unlock_put_pages(npages, dst);
606 migrate_device_pages(src, dst, npages);
607 migrate_device_finalize(src, dst, npages);
608 drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, npages,
609 DMA_FROM_DEVICE);
610 err_free:
611 kvfree(buf);
612 err_out:
613 mmput_async(devmem_allocation->mm);
614
615 if (completion_done(&devmem_allocation->detached))
616 return 0;
617
618 if (retry_count--) {
619 cond_resched();
620 goto retry;
621 }
622
623 return err ?: -EBUSY;
624 }
625 EXPORT_SYMBOL_GPL(drm_pagemap_evict_to_ram);
626
627 /**
628 * __drm_pagemap_migrate_to_ram() - Migrate GPU SVM range to RAM (internal)
629 * @vas: Pointer to the VM area structure
630 * @device_private_page_owner: Device private pages owner
631 * @page: Pointer to the page for fault handling (can be NULL)
632 * @fault_addr: Fault address
633 * @size: Size of migration
634 *
635 * This internal function performs the migration of the specified GPU SVM range
636 * to RAM. It sets up the migration, populates + dma maps RAM PFNs, and
637 * invokes the driver-specific operations for migration to RAM.
638 *
639 * Return: 0 on success, negative error code on failure.
640 */
__drm_pagemap_migrate_to_ram(struct vm_area_struct * vas,void * device_private_page_owner,struct page * page,unsigned long fault_addr,unsigned long size)641 static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
642 void *device_private_page_owner,
643 struct page *page,
644 unsigned long fault_addr,
645 unsigned long size)
646 {
647 struct migrate_vma migrate = {
648 .vma = vas,
649 .pgmap_owner = device_private_page_owner,
650 .flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE |
651 MIGRATE_VMA_SELECT_DEVICE_COHERENT,
652 .fault_page = page,
653 };
654 struct drm_pagemap_zdd *zdd;
655 const struct drm_pagemap_devmem_ops *ops;
656 struct device *dev = NULL;
657 unsigned long npages, mpages = 0;
658 struct page **pages;
659 struct drm_pagemap_addr *pagemap_addr;
660 unsigned long start, end;
661 void *buf;
662 int i, err = 0;
663
664 if (page) {
665 zdd = page->zone_device_data;
666 if (time_before64(get_jiffies_64(),
667 zdd->devmem_allocation->timeslice_expiration))
668 return 0;
669 }
670
671 start = ALIGN_DOWN(fault_addr, size);
672 end = ALIGN(fault_addr + 1, size);
673
674 /* Corner where VMA area struct has been partially unmapped */
675 if (start < vas->vm_start)
676 start = vas->vm_start;
677 if (end > vas->vm_end)
678 end = vas->vm_end;
679
680 migrate.start = start;
681 migrate.end = end;
682 npages = npages_in_range(start, end);
683
684 buf = kvcalloc(npages, 2 * sizeof(*migrate.src) + sizeof(*pagemap_addr) +
685 sizeof(*pages), GFP_KERNEL);
686 if (!buf) {
687 err = -ENOMEM;
688 goto err_out;
689 }
690 pagemap_addr = buf + (2 * sizeof(*migrate.src) * npages);
691 pages = buf + (2 * sizeof(*migrate.src) + sizeof(*pagemap_addr)) * npages;
692
693 migrate.vma = vas;
694 migrate.src = buf;
695 migrate.dst = migrate.src + npages;
696
697 err = migrate_vma_setup(&migrate);
698 if (err)
699 goto err_free;
700
701 /* Raced with another CPU fault, nothing to do */
702 if (!migrate.cpages)
703 goto err_free;
704
705 if (!page) {
706 for (i = 0; i < npages; ++i) {
707 if (!(migrate.src[i] & MIGRATE_PFN_MIGRATE))
708 continue;
709
710 page = migrate_pfn_to_page(migrate.src[i]);
711 break;
712 }
713
714 if (!page)
715 goto err_finalize;
716 }
717 zdd = page->zone_device_data;
718 ops = zdd->devmem_allocation->ops;
719 dev = zdd->devmem_allocation->dev;
720
721 err = drm_pagemap_migrate_populate_ram_pfn(vas, page, npages, &mpages,
722 migrate.src, migrate.dst,
723 start);
724 if (err)
725 goto err_finalize;
726
727 err = drm_pagemap_migrate_map_pages(dev, pagemap_addr, migrate.dst, npages,
728 DMA_FROM_DEVICE);
729 if (err)
730 goto err_finalize;
731
732 for (i = 0; i < npages; ++i)
733 pages[i] = migrate_pfn_to_page(migrate.src[i]);
734
735 err = ops->copy_to_ram(pages, pagemap_addr, npages);
736 if (err)
737 goto err_finalize;
738
739 err_finalize:
740 if (err)
741 drm_pagemap_migration_unlock_put_pages(npages, migrate.dst);
742 migrate_vma_pages(&migrate);
743 migrate_vma_finalize(&migrate);
744 if (dev)
745 drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, npages,
746 DMA_FROM_DEVICE);
747 err_free:
748 kvfree(buf);
749 err_out:
750
751 return err;
752 }
753
754 /**
755 * drm_pagemap_folio_free() - Put GPU SVM zone device data associated with a folio
756 * @folio: Pointer to the folio
757 *
758 * This function is a callback used to put the GPU SVM zone device data
759 * associated with a page when it is being released.
760 */
drm_pagemap_folio_free(struct folio * folio)761 static void drm_pagemap_folio_free(struct folio *folio)
762 {
763 drm_pagemap_zdd_put(folio->page.zone_device_data);
764 }
765
766 /**
767 * drm_pagemap_migrate_to_ram() - Migrate a virtual range to RAM (page fault handler)
768 * @vmf: Pointer to the fault information structure
769 *
770 * This function is a page fault handler used to migrate a virtual range
771 * to ram. The device memory allocation in which the device page is found is
772 * migrated in its entirety.
773 *
774 * Returns:
775 * VM_FAULT_SIGBUS on failure, 0 on success.
776 */
drm_pagemap_migrate_to_ram(struct vm_fault * vmf)777 static vm_fault_t drm_pagemap_migrate_to_ram(struct vm_fault *vmf)
778 {
779 struct drm_pagemap_zdd *zdd = vmf->page->zone_device_data;
780 int err;
781
782 err = __drm_pagemap_migrate_to_ram(vmf->vma,
783 zdd->device_private_page_owner,
784 vmf->page, vmf->address,
785 zdd->devmem_allocation->size);
786
787 return err ? VM_FAULT_SIGBUS : 0;
788 }
789
790 static const struct dev_pagemap_ops drm_pagemap_pagemap_ops = {
791 .folio_free = drm_pagemap_folio_free,
792 .migrate_to_ram = drm_pagemap_migrate_to_ram,
793 };
794
795 /**
796 * drm_pagemap_pagemap_ops_get() - Retrieve GPU SVM device page map operations
797 *
798 * Returns:
799 * Pointer to the GPU SVM device page map operations structure.
800 */
drm_pagemap_pagemap_ops_get(void)801 const struct dev_pagemap_ops *drm_pagemap_pagemap_ops_get(void)
802 {
803 return &drm_pagemap_pagemap_ops;
804 }
805 EXPORT_SYMBOL_GPL(drm_pagemap_pagemap_ops_get);
806
807 /**
808 * drm_pagemap_devmem_init() - Initialize a drm_pagemap device memory allocation
809 *
810 * @devmem_allocation: The struct drm_pagemap_devmem to initialize.
811 * @dev: Pointer to the device structure which device memory allocation belongs to
812 * @mm: Pointer to the mm_struct for the address space
813 * @ops: Pointer to the operations structure for GPU SVM device memory
814 * @dpagemap: The struct drm_pagemap we're allocating from.
815 * @size: Size of device memory allocation
816 */
drm_pagemap_devmem_init(struct drm_pagemap_devmem * devmem_allocation,struct device * dev,struct mm_struct * mm,const struct drm_pagemap_devmem_ops * ops,struct drm_pagemap * dpagemap,size_t size)817 void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation,
818 struct device *dev, struct mm_struct *mm,
819 const struct drm_pagemap_devmem_ops *ops,
820 struct drm_pagemap *dpagemap, size_t size)
821 {
822 init_completion(&devmem_allocation->detached);
823 devmem_allocation->dev = dev;
824 devmem_allocation->mm = mm;
825 devmem_allocation->ops = ops;
826 devmem_allocation->dpagemap = dpagemap;
827 devmem_allocation->size = size;
828 }
829 EXPORT_SYMBOL_GPL(drm_pagemap_devmem_init);
830
831 /**
832 * drm_pagemap_page_to_dpagemap() - Return a pointer the drm_pagemap of a page
833 * @page: The struct page.
834 *
835 * Return: A pointer to the struct drm_pagemap of a device private page that
836 * was populated from the struct drm_pagemap. If the page was *not* populated
837 * from a struct drm_pagemap, the result is undefined and the function call
838 * may result in dereferencing and invalid address.
839 */
drm_pagemap_page_to_dpagemap(struct page * page)840 struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page)
841 {
842 struct drm_pagemap_zdd *zdd = page->zone_device_data;
843
844 return zdd->devmem_allocation->dpagemap;
845 }
846 EXPORT_SYMBOL_GPL(drm_pagemap_page_to_dpagemap);
847
848 /**
849 * drm_pagemap_populate_mm() - Populate a virtual range with device memory pages
850 * @dpagemap: Pointer to the drm_pagemap managing the device memory
851 * @start: Start of the virtual range to populate.
852 * @end: End of the virtual range to populate.
853 * @mm: Pointer to the virtual address space.
854 * @timeslice_ms: The time requested for the migrated pagemap pages to
855 * be present in @mm before being allowed to be migrated back.
856 *
857 * Attempt to populate a virtual range with device memory pages,
858 * clearing them or migrating data from the existing pages if necessary.
859 * The function is best effort only, and implementations may vary
860 * in how hard they try to satisfy the request.
861 *
862 * Return: %0 on success, negative error code on error. If the hardware
863 * device was removed / unbound the function will return %-ENODEV.
864 */
drm_pagemap_populate_mm(struct drm_pagemap * dpagemap,unsigned long start,unsigned long end,struct mm_struct * mm,unsigned long timeslice_ms)865 int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
866 unsigned long start, unsigned long end,
867 struct mm_struct *mm,
868 unsigned long timeslice_ms)
869 {
870 int err;
871
872 if (!mmget_not_zero(mm))
873 return -EFAULT;
874 mmap_read_lock(mm);
875 err = dpagemap->ops->populate_mm(dpagemap, start, end, mm,
876 timeslice_ms);
877 mmap_read_unlock(mm);
878 mmput(mm);
879
880 return err;
881 }
882 EXPORT_SYMBOL(drm_pagemap_populate_mm);
883