1 // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 * 5 * Authors: 6 * Matthew Brost <matthew.brost@intel.com> 7 */ 8 9 #include <linux/dma-mapping.h> 10 #include <linux/export.h> 11 #include <linux/hmm.h> 12 #include <linux/memremap.h> 13 #include <linux/mm_types.h> 14 #include <linux/mm.h> 15 #include <linux/slab.h> 16 17 #include <drm/drm_device.h> 18 #include <drm/drm_gpusvm.h> 19 #include <drm/drm_pagemap.h> 20 #include <drm/drm_print.h> 21 22 /** 23 * DOC: Overview 24 * 25 * GPU Shared Virtual Memory (GPU SVM) layer for the Direct Rendering Manager (DRM) 26 * is a component of the DRM framework designed to manage shared virtual memory 27 * between the CPU and GPU. It enables efficient data exchange and processing 28 * for GPU-accelerated applications by allowing memory sharing and 29 * synchronization between the CPU's and GPU's virtual address spaces. 30 * 31 * Key GPU SVM Components: 32 * 33 * - Notifiers: 34 * Used for tracking memory intervals and notifying the GPU of changes, 35 * notifiers are sized based on a GPU SVM initialization parameter, with a 36 * recommendation of 512M or larger. They maintain a Red-BlacK tree and a 37 * list of ranges that fall within the notifier interval. Notifiers are 38 * tracked within a GPU SVM Red-BlacK tree and list and are dynamically 39 * inserted or removed as ranges within the interval are created or 40 * destroyed. 41 * - Ranges: 42 * Represent memory ranges mapped in a DRM device and managed by GPU SVM. 43 * They are sized based on an array of chunk sizes, which is a GPU SVM 44 * initialization parameter, and the CPU address space. Upon GPU fault, 45 * the largest aligned chunk that fits within the faulting CPU address 46 * space is chosen for the range size. Ranges are expected to be 47 * dynamically allocated on GPU fault and removed on an MMU notifier UNMAP 48 * event. As mentioned above, ranges are tracked in a notifier's Red-Black 49 * tree. 50 * 51 * - Pages: 52 * struct drm_gpusvm_pages holds the DMA mapping state for a range of 53 * CPU virtual addresses: the DMA mapped device addresses, 54 * the device private pagemap, the IOVA state, the per mapping 55 * notifier sequence number, and the drm_device that owns the DMA 56 * mappings. 57 * A driver embeds one or more struct drm_gpusvm_pages alongside its 58 * struct drm_gpusvm_range, choosing one of two layouts: 59 * 60 * 1:1 - one drm_gpusvm_pages per range (one drm_device). Simplest 61 * layout; to mirror a VA range on several devices a driver uses a 62 * separate range (and notifier) per device, so the HMM fault is taken 63 * once per device. 64 * 65 * N:1 - one drm_gpusvm_pages per drm_device, all sharing one range and 66 * notifier; only the per-device DMA mapping differs. The instances must 67 * sit in contiguous memory so a single drm_gpusvm_range_set_unmapped() 68 * can mark them all. A driver can keep one instance inline for the single 69 * device case and switch to a heap array only when more devices join, 70 * e.g.: 71 * 72 * .. code-block:: c 73 * 74 * struct driver_range { 75 * struct drm_gpusvm_range base; 76 * unsigned int num_pages; // 1: inline_pages, >1: pages[] 77 * union { 78 * struct drm_gpusvm_pages inline_pages; 79 * struct drm_gpusvm_pages *pages; 80 * }; 81 * }; 82 * 83 * static struct drm_gpusvm_pages * 84 * driver_pages(struct driver_range *drange) 85 * { 86 * return drange->num_pages == 1 ? &drange->inline_pages : 87 * drange->pages; 88 * } 89 * 90 * In the N:1 case the driver allocates the pages array with a zeroing 91 * allocator (e.g. kcalloc(num_pages, ...)), initialises each entry with 92 * drm_gpusvm_init_pages(), and frees each entry with 93 * drm_gpusvm_free_pages() plus the array itself, from its range free 94 * callback. Each drm_gpusvm_pages is mapped independently by their own 95 * drm_device. 96 * Each drm_gpusvm_pages must be zero-initialised and initialised with 97 * drm_gpusvm_init_pages(), called once per entry. 98 * 99 * The 1:1 examples below pass @num_pages == 1 and &drange->pages. In the 100 * N:1 case the driver instead passes the whole array and its count, so a 101 * single call faults the CPU range once and DMA maps it for every owning 102 * drm_device, e.g.: 103 * 104 * .. code-block:: c 105 * 106 * // GPU fault handler: one fault, one DMA mapping per device 107 * err = drm_gpusvm_get_pages(gpusvm, driver_pages(drange), 108 * drange->num_pages, gpusvm->mm, 109 * &range->notifier->notifier, 110 * drm_gpusvm_range_start(range), 111 * drm_gpusvm_range_end(range), &ctx); 112 * 113 * // Notifier callback: mark every instance unmapped in one call 114 * drm_gpusvm_range_set_unmapped(range, driver_pages(drange), 115 * drange->num_pages, mmu_range); 116 * 117 * The unmap and free paths stay per-instance: iterate @num_pages over 118 * driver_pages(drange) and call drm_gpusvm_unmap_pages() / 119 * drm_gpusvm_free_pages() for each entry. 120 * 121 * - Operations: 122 * Define the interface for driver-specific GPU SVM operations such as 123 * range allocation, notifier allocation, and invalidations. 124 * 125 * - Device Memory Allocations: 126 * Embedded structure containing enough information for GPU SVM to migrate 127 * to / from device memory. 128 * 129 * - Device Memory Operations: 130 * Define the interface for driver-specific device memory operations 131 * release memory, populate pfns, and copy to / from device memory. 132 * 133 * This layer provides interfaces for allocating, mapping, migrating, and 134 * releasing memory ranges between the CPU and GPU. It handles all core memory 135 * management interactions (DMA mapping, HMM, and migration) and provides 136 * driver-specific virtual functions (vfuncs). This infrastructure is sufficient 137 * to build the expected driver components for an SVM implementation as detailed 138 * below. 139 * 140 * Expected Driver Components: 141 * 142 * - GPU page fault handler: 143 * Used to create ranges and notifiers based on the fault address, 144 * optionally migrate the range to device memory, and create GPU bindings. 145 * 146 * - Garbage collector: 147 * Used to unmap and destroy GPU bindings for ranges. Ranges are expected 148 * to be added to the garbage collector upon a MMU_NOTIFY_UNMAP event in 149 * notifier callback. 150 * 151 * - Notifier callback: 152 * Used to invalidate and DMA unmap GPU bindings for ranges. 153 */ 154 155 /** 156 * DOC: Locking 157 * 158 * GPU SVM handles locking for core MM interactions, i.e., it locks/unlocks the 159 * mmap lock as needed. 160 * 161 * GPU SVM introduces a global notifier lock, which safeguards the notifier's 162 * range RB tree and list, as well as the range's DMA mappings and sequence 163 * number. GPU SVM manages all necessary locking and unlocking operations, 164 * except for the recheck range's pages being valid 165 * (drm_gpusvm_pages_valid) when the driver is committing GPU bindings. 166 * This lock corresponds to the ``driver->update`` lock mentioned in 167 * Documentation/mm/hmm.rst. Future revisions may transition from a GPU SVM 168 * global lock to a per-notifier lock if finer-grained locking is deemed 169 * necessary. 170 * 171 * In addition to the locking mentioned above, the driver should implement a 172 * lock to safeguard core GPU SVM function calls that modify state, such as 173 * drm_gpusvm_range_find_or_insert and drm_gpusvm_range_remove. This lock is 174 * denoted as 'driver_svm_lock' in code examples. Finer grained driver side 175 * locking should also be possible for concurrent GPU fault processing within a 176 * single GPU SVM. The 'driver_svm_lock' can be via drm_gpusvm_driver_set_lock 177 * to add annotations to GPU SVM. 178 */ 179 180 /** 181 * DOC: Partial Unmapping of Ranges 182 * 183 * Partial unmapping of ranges (e.g., 1M out of 2M is unmapped by CPU resulting 184 * in MMU_NOTIFY_UNMAP event) presents several challenges, with the main one 185 * being that a subset of the range still has CPU and GPU mappings. If the 186 * backing store for the range is in device memory, a subset of the backing 187 * store has references. One option would be to split the range and device 188 * memory backing store, but the implementation for this would be quite 189 * complicated. Given that partial unmappings are rare and driver-defined range 190 * sizes are relatively small, GPU SVM does not support splitting of ranges. 191 * 192 * With no support for range splitting, upon partial unmapping of a range, the 193 * driver is expected to invalidate and destroy the entire range. If the range 194 * has device memory as its backing, the driver is also expected to migrate any 195 * remaining pages back to RAM. 196 */ 197 198 /** 199 * DOC: Examples 200 * 201 * This section provides three examples of how to build the expected driver 202 * components: the GPU page fault handler, the garbage collector, and the 203 * notifier callback. 204 * 205 * The generic code provided does not include logic for complex migration 206 * policies, optimized invalidations, fined grained driver locking, or other 207 * potentially required driver locking (e.g., DMA-resv locks). 208 * 209 * 1) GPU page fault handler 210 * 211 * .. code-block:: c 212 * 213 * struct driver_range { 214 * struct drm_gpusvm_range base; 215 * struct drm_gpusvm_pages pages; 216 * }; 217 * 218 * int driver_bind_range(struct drm_gpusvm *gpusvm, struct driver_range *drange) 219 * { 220 * int err = 0; 221 * 222 * driver_alloc_and_setup_memory_for_bind(gpusvm, drange); 223 * 224 * drm_gpusvm_notifier_lock(gpusvm); 225 * if (drm_gpusvm_pages_valid(gpusvm, &drange->pages)) 226 * driver_commit_bind(gpusvm, drange); 227 * else 228 * err = -EAGAIN; 229 * drm_gpusvm_notifier_unlock(gpusvm); 230 * 231 * return err; 232 * } 233 * 234 * int driver_gpu_fault(struct drm_gpusvm *gpusvm, unsigned long fault_addr, 235 * unsigned long gpuva_start, unsigned long gpuva_end) 236 * { 237 * struct drm_gpusvm_ctx ctx = {}; 238 * struct driver_range *drange; 239 * struct drm_gpusvm_range *range; 240 * int err; 241 * 242 * driver_svm_lock(); 243 * retry: 244 * // Always process UNMAPs first so view of GPU SVM ranges is current 245 * driver_garbage_collector(gpusvm); 246 * 247 * range = drm_gpusvm_range_find_or_insert(gpusvm, fault_addr, 248 * gpuva_start, gpuva_end, 249 * &ctx); 250 * if (IS_ERR(range)) { 251 * err = PTR_ERR(range); 252 * goto unlock; 253 * } 254 * drange = container_of(range, struct driver_range, base); 255 * 256 * if (driver_migration_policy(range)) { 257 * err = drm_pagemap_populate_mm(driver_choose_drm_pagemap(), 258 * gpuva_start, gpuva_end, gpusvm->mm, 259 * ctx->timeslice_ms); 260 * if (err) // CPU mappings may have changed 261 * goto retry; 262 * } 263 * 264 * err = drm_gpusvm_get_pages(gpusvm, &drange->pages, 1, 265 * gpusvm->mm, &range->notifier->notifier, 266 * drm_gpusvm_range_start(range), 267 * drm_gpusvm_range_end(range), &ctx); 268 * if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) { // CPU mappings changed 269 * if (err == -EOPNOTSUPP) 270 * drm_gpusvm_range_evict(gpusvm, range); 271 * goto retry; 272 * } else if (err) { 273 * goto unlock; 274 * } 275 * 276 * err = driver_bind_range(gpusvm, drange); 277 * if (err == -EAGAIN) // CPU mappings changed 278 * goto retry 279 * 280 * unlock: 281 * driver_svm_unlock(); 282 * return err; 283 * } 284 * 285 * 2) Garbage Collector 286 * 287 * .. code-block:: c 288 * 289 * // The driver owns the drm_gpusvm_pages lifecycle. ops->range_free is 290 * // the final fallback: drm_gpusvm_free_pages() unmaps any 291 * // lingering DMA mapping and a no-op if already unmapped and frees the 292 * // dma_addr array. The normal flow is to DMA unmap before 293 * // drm_gpusvm_range_remove() (before the range leaves the tree). 294 * void driver_range_free(struct drm_gpusvm_range *range) 295 * { 296 * struct driver_range *drange = 297 * container_of(range, struct driver_range, base); 298 * 299 * drm_gpusvm_free_pages(range->gpusvm, &drange->pages, 300 * drm_gpusvm_range_size(range) >> PAGE_SHIFT); 301 * kfree(drange); 302 * } 303 * 304 * void __driver_garbage_collector(struct drm_gpusvm *gpusvm, 305 * struct drm_gpusvm_range *range) 306 * { 307 * assert_driver_svm_locked(gpusvm); 308 * 309 * // Partial unmap, migrate any remaining device memory pages back to RAM 310 * if (range->flags.partial_unmap) 311 * drm_gpusvm_range_evict(gpusvm, range); 312 * 313 * driver_unbind_range(range); 314 * // The pages must be DMA unmapped before drm_gpusvm_range_remove() 315 * // , so a range is never off the MMU interval tree while still DMA 316 * // mapped as the original drmsvm design flow. Otherwise a concurrent CPU 317 * // munmap's notifier could miss this range and free pages still mapped 318 * // for device DMA. This is the normal unmap point. 319 * drm_gpusvm_unmap_pages(gpusvm, &drange->pages, 320 * drm_gpusvm_range_size(range) >> PAGE_SHIFT, 321 * &(struct drm_gpusvm_ctx){ .in_notifier = false }); 322 * drm_gpusvm_range_remove(gpusvm, range); 323 * } 324 * 325 * void driver_garbage_collector(struct drm_gpusvm *gpusvm) 326 * { 327 * assert_driver_svm_locked(gpusvm); 328 * 329 * for_each_range_in_garbage_collector(gpusvm, range) 330 * __driver_garbage_collector(gpusvm, range); 331 * } 332 * 333 * 3) Notifier callback 334 * 335 * .. code-block:: c 336 * 337 * void driver_invalidation(struct drm_gpusvm *gpusvm, 338 * struct drm_gpusvm_notifier *notifier, 339 * const struct mmu_notifier_range *mmu_range) 340 * { 341 * struct drm_gpusvm_ctx ctx = { .in_notifier = true, }; 342 * struct drm_gpusvm_range *range = NULL; 343 * struct driver_range *drange; 344 * 345 * driver_invalidate_device_pages(gpusvm, mmu_range->start, mmu_range->end); 346 * 347 * drm_gpusvm_for_each_range(range, notifier, mmu_range->start, 348 * mmu_range->end) { 349 * drange = container_of(range, struct driver_range, base); 350 * 351 * drm_gpusvm_unmap_pages(gpusvm, &drange->pages, 352 * drm_gpusvm_range_size(range) >> PAGE_SHIFT, 353 * &ctx); 354 * 355 * if (mmu_range->event != MMU_NOTIFY_UNMAP) 356 * continue; 357 * 358 * drm_gpusvm_range_set_unmapped(range, &drange->pages, 1, mmu_range); 359 * driver_garbage_collector_add(gpusvm, range); 360 * } 361 * } 362 */ 363 364 /** 365 * npages_in_range() - Calculate the number of pages in a given range 366 * @start: The start address of the range 367 * @end: The end address of the range 368 * 369 * This macro calculates the number of pages in a given memory range, 370 * specified by the start and end addresses. It divides the difference 371 * between the end and start addresses by the page size (PAGE_SIZE) to 372 * determine the number of pages in the range. 373 * 374 * Return: The number of pages in the specified range. 375 */ 376 static unsigned long 377 npages_in_range(unsigned long start, unsigned long end) 378 { 379 return (end - start) >> PAGE_SHIFT; 380 } 381 382 /** 383 * drm_gpusvm_notifier_find() - Find GPU SVM notifier from GPU SVM 384 * @gpusvm: Pointer to the GPU SVM structure. 385 * @start: Start address of the notifier 386 * @end: End address of the notifier 387 * 388 * Return: A pointer to the drm_gpusvm_notifier if found or NULL 389 */ 390 struct drm_gpusvm_notifier * 391 drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start, 392 unsigned long end) 393 { 394 struct interval_tree_node *itree; 395 396 itree = interval_tree_iter_first(&gpusvm->root, start, end - 1); 397 398 if (itree) 399 return container_of(itree, struct drm_gpusvm_notifier, itree); 400 else 401 return NULL; 402 } 403 EXPORT_SYMBOL_GPL(drm_gpusvm_notifier_find); 404 405 /** 406 * drm_gpusvm_range_find() - Find GPU SVM range from GPU SVM notifier 407 * @notifier: Pointer to the GPU SVM notifier structure. 408 * @start: Start address of the range 409 * @end: End address of the range 410 * 411 * Return: A pointer to the drm_gpusvm_range if found or NULL 412 */ 413 struct drm_gpusvm_range * 414 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start, 415 unsigned long end) 416 { 417 struct interval_tree_node *itree; 418 419 itree = interval_tree_iter_first(¬ifier->root, start, end - 1); 420 421 if (itree) 422 return container_of(itree, struct drm_gpusvm_range, itree); 423 else 424 return NULL; 425 } 426 EXPORT_SYMBOL_GPL(drm_gpusvm_range_find); 427 428 /** 429 * drm_gpusvm_notifier_invalidate() - Invalidate a GPU SVM notifier. 430 * @mni: Pointer to the mmu_interval_notifier structure. 431 * @mmu_range: Pointer to the mmu_notifier_range structure. 432 * @cur_seq: Current sequence number. 433 * 434 * This function serves as a generic MMU notifier for GPU SVM. It sets the MMU 435 * notifier sequence number and calls the driver invalidate vfunc under 436 * gpusvm->notifier_lock. 437 * 438 * Return: true if the operation succeeds, false otherwise. 439 */ 440 static bool 441 drm_gpusvm_notifier_invalidate(struct mmu_interval_notifier *mni, 442 const struct mmu_notifier_range *mmu_range, 443 unsigned long cur_seq) 444 { 445 struct drm_gpusvm_notifier *notifier = 446 container_of(mni, typeof(*notifier), notifier); 447 struct drm_gpusvm *gpusvm = notifier->gpusvm; 448 449 if (!mmu_notifier_range_blockable(mmu_range)) 450 return false; 451 452 down_write(&gpusvm->notifier_lock); 453 mmu_interval_set_seq(mni, cur_seq); 454 gpusvm->ops->invalidate(gpusvm, notifier, mmu_range); 455 up_write(&gpusvm->notifier_lock); 456 457 return true; 458 } 459 460 /* 461 * drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM 462 */ 463 static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = { 464 .invalidate = drm_gpusvm_notifier_invalidate, 465 }; 466 467 /** 468 * drm_gpusvm_init() - Initialize the GPU SVM. 469 * @gpusvm: Pointer to the GPU SVM structure. 470 * @name: Name of the GPU SVM. 471 * @mm: Pointer to the mm_struct for the address space. 472 * @mm_start: Start address of GPU SVM. 473 * @mm_range: Range of the GPU SVM. 474 * @notifier_size: Size of individual notifiers. 475 * @ops: Pointer to the operations structure for GPU SVM. 476 * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation. 477 * Entries should be powers of 2 in descending order with last 478 * entry being SZ_4K. 479 * @num_chunks: Number of chunks. 480 * 481 * This function initializes the GPU SVM. 482 * 483 * Note: If only using the simple drm_gpusvm_pages API (get/unmap/free), 484 * then only @gpusvm and @name are expected. The @drm drm_device for dma 485 * mappings is bound per-pages via drm_gpusvm_init_pages() before the first 486 * drm_gpusvm_get_pages() call. However, the same base 487 * @gpusvm can also be used with both modes together in which case the full 488 * setup is needed, where the core drm_gpusvm_pages API will simply never use 489 * the other fields. 490 * 491 * Return: 0 on success, a negative error code on failure. 492 */ 493 int drm_gpusvm_init(struct drm_gpusvm *gpusvm, 494 const char *name, 495 struct mm_struct *mm, 496 unsigned long mm_start, unsigned long mm_range, 497 unsigned long notifier_size, 498 const struct drm_gpusvm_ops *ops, 499 const unsigned long *chunk_sizes, int num_chunks) 500 { 501 if (mm) { 502 if (!ops->invalidate || !num_chunks) 503 return -EINVAL; 504 mmgrab(mm); 505 } else { 506 /* No full SVM mode, only core drm_gpusvm_pages API. */ 507 if (ops || num_chunks || mm_range || notifier_size) 508 return -EINVAL; 509 } 510 511 gpusvm->name = name; 512 gpusvm->mm = mm; 513 gpusvm->mm_start = mm_start; 514 gpusvm->mm_range = mm_range; 515 gpusvm->notifier_size = notifier_size; 516 gpusvm->ops = ops; 517 gpusvm->chunk_sizes = chunk_sizes; 518 gpusvm->num_chunks = num_chunks; 519 520 gpusvm->root = RB_ROOT_CACHED; 521 INIT_LIST_HEAD(&gpusvm->notifier_list); 522 523 init_rwsem(&gpusvm->notifier_lock); 524 525 fs_reclaim_acquire(GFP_KERNEL); 526 might_lock(&gpusvm->notifier_lock); 527 fs_reclaim_release(GFP_KERNEL); 528 529 #ifdef CONFIG_LOCKDEP 530 gpusvm->lock_dep_map = NULL; 531 #endif 532 533 return 0; 534 } 535 EXPORT_SYMBOL_GPL(drm_gpusvm_init); 536 537 /** 538 * to_drm_gpusvm_notifier() - retrieve the container struct for a given rbtree node 539 * @node: a pointer to the rbtree node embedded within a drm_gpusvm_notifier struct 540 * 541 * Return: A pointer to the containing drm_gpusvm_notifier structure. 542 */ 543 static struct drm_gpusvm_notifier *to_drm_gpusvm_notifier(struct rb_node *node) 544 { 545 return container_of(node, struct drm_gpusvm_notifier, itree.rb); 546 } 547 548 /** 549 * drm_gpusvm_notifier_insert() - Insert GPU SVM notifier 550 * @gpusvm: Pointer to the GPU SVM structure 551 * @notifier: Pointer to the GPU SVM notifier structure 552 * 553 * This function inserts the GPU SVM notifier into the GPU SVM RB tree and list. 554 */ 555 static void drm_gpusvm_notifier_insert(struct drm_gpusvm *gpusvm, 556 struct drm_gpusvm_notifier *notifier) 557 { 558 struct rb_node *node; 559 struct list_head *head; 560 561 interval_tree_insert(¬ifier->itree, &gpusvm->root); 562 563 node = rb_prev(¬ifier->itree.rb); 564 if (node) 565 head = &(to_drm_gpusvm_notifier(node))->entry; 566 else 567 head = &gpusvm->notifier_list; 568 569 list_add(¬ifier->entry, head); 570 } 571 572 /** 573 * drm_gpusvm_notifier_remove() - Remove GPU SVM notifier 574 * @gpusvm: Pointer to the GPU SVM tructure 575 * @notifier: Pointer to the GPU SVM notifier structure 576 * 577 * This function removes the GPU SVM notifier from the GPU SVM RB tree and list. 578 */ 579 static void drm_gpusvm_notifier_remove(struct drm_gpusvm *gpusvm, 580 struct drm_gpusvm_notifier *notifier) 581 { 582 interval_tree_remove(¬ifier->itree, &gpusvm->root); 583 list_del(¬ifier->entry); 584 } 585 586 /** 587 * drm_gpusvm_fini() - Finalize the GPU SVM. 588 * @gpusvm: Pointer to the GPU SVM structure. 589 * 590 * This function finalizes the GPU SVM by cleaning up any remaining ranges and 591 * notifiers, and dropping a reference to struct MM. 592 */ 593 void drm_gpusvm_fini(struct drm_gpusvm *gpusvm) 594 { 595 struct drm_gpusvm_notifier *notifier, *next; 596 597 drm_gpusvm_for_each_notifier_safe(notifier, next, gpusvm, 0, LONG_MAX) { 598 struct drm_gpusvm_range *range, *__next; 599 600 /* 601 * Remove notifier first to avoid racing with any invalidation 602 */ 603 mmu_interval_notifier_remove(¬ifier->notifier); 604 notifier->flags.removed = true; 605 606 drm_gpusvm_for_each_range_safe(range, __next, notifier, 0, 607 LONG_MAX) 608 drm_gpusvm_range_remove(gpusvm, range); 609 } 610 611 if (gpusvm->mm) 612 mmdrop(gpusvm->mm); 613 WARN_ON(!RB_EMPTY_ROOT(&gpusvm->root.rb_root)); 614 } 615 EXPORT_SYMBOL_GPL(drm_gpusvm_fini); 616 617 /** 618 * drm_gpusvm_notifier_alloc() - Allocate GPU SVM notifier 619 * @gpusvm: Pointer to the GPU SVM structure 620 * @fault_addr: Fault address 621 * 622 * This function allocates and initializes the GPU SVM notifier structure. 623 * 624 * Return: Pointer to the allocated GPU SVM notifier on success, ERR_PTR() on failure. 625 */ 626 static struct drm_gpusvm_notifier * 627 drm_gpusvm_notifier_alloc(struct drm_gpusvm *gpusvm, unsigned long fault_addr) 628 { 629 struct drm_gpusvm_notifier *notifier; 630 631 if (gpusvm->ops->notifier_alloc) 632 notifier = gpusvm->ops->notifier_alloc(); 633 else 634 notifier = kzalloc_obj(*notifier); 635 636 if (!notifier) 637 return ERR_PTR(-ENOMEM); 638 639 notifier->gpusvm = gpusvm; 640 notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size); 641 notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1; 642 INIT_LIST_HEAD(¬ifier->entry); 643 notifier->root = RB_ROOT_CACHED; 644 INIT_LIST_HEAD(¬ifier->range_list); 645 646 return notifier; 647 } 648 649 /** 650 * drm_gpusvm_notifier_free() - Free GPU SVM notifier 651 * @gpusvm: Pointer to the GPU SVM structure 652 * @notifier: Pointer to the GPU SVM notifier structure 653 * 654 * This function frees the GPU SVM notifier structure. 655 */ 656 static void drm_gpusvm_notifier_free(struct drm_gpusvm *gpusvm, 657 struct drm_gpusvm_notifier *notifier) 658 { 659 WARN_ON(!RB_EMPTY_ROOT(¬ifier->root.rb_root)); 660 661 if (gpusvm->ops->notifier_free) 662 gpusvm->ops->notifier_free(notifier); 663 else 664 kfree(notifier); 665 } 666 667 /** 668 * to_drm_gpusvm_range() - retrieve the container struct for a given rbtree node 669 * @node: a pointer to the rbtree node embedded within a drm_gpusvm_range struct 670 * 671 * Return: A pointer to the containing drm_gpusvm_range structure. 672 */ 673 static struct drm_gpusvm_range *to_drm_gpusvm_range(struct rb_node *node) 674 { 675 return container_of(node, struct drm_gpusvm_range, itree.rb); 676 } 677 678 /** 679 * drm_gpusvm_range_insert() - Insert GPU SVM range 680 * @notifier: Pointer to the GPU SVM notifier structure 681 * @range: Pointer to the GPU SVM range structure 682 * 683 * This function inserts the GPU SVM range into the notifier RB tree and list. 684 */ 685 static void drm_gpusvm_range_insert(struct drm_gpusvm_notifier *notifier, 686 struct drm_gpusvm_range *range) 687 { 688 struct rb_node *node; 689 struct list_head *head; 690 691 drm_gpusvm_notifier_lock(notifier->gpusvm); 692 interval_tree_insert(&range->itree, ¬ifier->root); 693 694 node = rb_prev(&range->itree.rb); 695 if (node) 696 head = &(to_drm_gpusvm_range(node))->entry; 697 else 698 head = ¬ifier->range_list; 699 700 list_add(&range->entry, head); 701 drm_gpusvm_notifier_unlock(notifier->gpusvm); 702 } 703 704 /** 705 * __drm_gpusvm_range_remove() - Remove GPU SVM range 706 * @notifier: Pointer to the GPU SVM notifier structure 707 * @range: Pointer to the GPU SVM range structure 708 * 709 * This macro removes the GPU SVM range from the notifier RB tree and list. 710 */ 711 static void __drm_gpusvm_range_remove(struct drm_gpusvm_notifier *notifier, 712 struct drm_gpusvm_range *range) 713 { 714 interval_tree_remove(&range->itree, ¬ifier->root); 715 list_del(&range->entry); 716 } 717 718 /** 719 * drm_gpusvm_range_alloc() - Allocate GPU SVM range 720 * @gpusvm: Pointer to the GPU SVM structure 721 * @notifier: Pointer to the GPU SVM notifier structure 722 * @fault_addr: Fault address 723 * @chunk_size: Chunk size 724 * @migrate_devmem: Flag indicating whether to migrate device memory 725 * 726 * This function allocates and initializes the GPU SVM range structure. 727 * 728 * Return: Pointer to the allocated GPU SVM range on success, ERR_PTR() on failure. 729 */ 730 static struct drm_gpusvm_range * 731 drm_gpusvm_range_alloc(struct drm_gpusvm *gpusvm, 732 struct drm_gpusvm_notifier *notifier, 733 unsigned long fault_addr, unsigned long chunk_size, 734 bool migrate_devmem) 735 { 736 struct drm_gpusvm_range *range; 737 738 if (gpusvm->ops->range_alloc) 739 range = gpusvm->ops->range_alloc(gpusvm); 740 else 741 range = kzalloc_obj(*range); 742 743 if (!range) 744 return ERR_PTR(-ENOMEM); 745 746 kref_init(&range->refcount); 747 range->gpusvm = gpusvm; 748 range->notifier = notifier; 749 range->itree.start = ALIGN_DOWN(fault_addr, chunk_size); 750 range->itree.last = ALIGN(fault_addr + 1, chunk_size) - 1; 751 INIT_LIST_HEAD(&range->entry); 752 range->flags.migrate_devmem = migrate_devmem ? 1 : 0; 753 754 return range; 755 } 756 757 /** 758 * drm_gpusvm_hmm_pfn_to_order() - Get the largest CPU mapping order. 759 * @hmm_pfn: The current hmm_pfn. 760 * @hmm_pfn_index: Index of the @hmm_pfn within the pfn array. 761 * @npages: Number of pages within the pfn array i.e the hmm range size. 762 * 763 * To allow skipping PFNs with the same flags (like when they belong to 764 * the same huge PTE) when looping over the pfn array, take a given a hmm_pfn, 765 * and return the largest order that will fit inside the CPU PTE, but also 766 * crucially accounting for the original hmm range boundaries. 767 * 768 * Return: The largest order that will safely fit within the size of the hmm_pfn 769 * CPU PTE. 770 */ 771 static unsigned int drm_gpusvm_hmm_pfn_to_order(unsigned long hmm_pfn, 772 unsigned long hmm_pfn_index, 773 unsigned long npages) 774 { 775 unsigned long size; 776 777 size = 1UL << hmm_pfn_to_map_order(hmm_pfn); 778 size -= (hmm_pfn & ~HMM_PFN_FLAGS) & (size - 1); 779 hmm_pfn_index += size; 780 if (hmm_pfn_index > npages) 781 size -= (hmm_pfn_index - npages); 782 783 return ilog2(size); 784 } 785 786 /** 787 * drm_gpusvm_check_pages() - Check pages 788 * @gpusvm: Pointer to the GPU SVM structure 789 * @notifier: Pointer to the GPU SVM notifier structure 790 * @start: Start address 791 * @end: End address 792 * @dev_private_owner: The device private page owner 793 * 794 * Check if pages between start and end have been faulted in on the CPU. Use to 795 * prevent migration of pages without CPU backing store. 796 * 797 * Return: True if pages have been faulted into CPU, False otherwise 798 */ 799 static bool drm_gpusvm_check_pages(struct drm_gpusvm *gpusvm, 800 struct drm_gpusvm_notifier *notifier, 801 unsigned long start, unsigned long end, 802 void *dev_private_owner) 803 { 804 struct hmm_range hmm_range = { 805 .default_flags = 0, 806 .notifier = ¬ifier->notifier, 807 .start = start, 808 .end = end, 809 .dev_private_owner = dev_private_owner, 810 }; 811 unsigned long timeout = 812 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 813 unsigned long *pfns; 814 unsigned long npages = npages_in_range(start, end); 815 int err, i; 816 817 mmap_assert_locked(gpusvm->mm); 818 819 pfns = kvmalloc_objs(*pfns, npages); 820 if (!pfns) 821 return false; 822 823 hmm_range.notifier_seq = mmu_interval_read_begin(¬ifier->notifier); 824 hmm_range.hmm_pfns = pfns; 825 826 while (true) { 827 err = hmm_range_fault(&hmm_range); 828 if (err == -EBUSY) { 829 if (time_after(jiffies, timeout)) 830 break; 831 832 hmm_range.notifier_seq = 833 mmu_interval_read_begin(¬ifier->notifier); 834 continue; 835 } 836 break; 837 } 838 if (err) 839 goto err_free; 840 841 for (i = 0; i < npages;) { 842 if (!(pfns[i] & HMM_PFN_VALID)) { 843 err = -EFAULT; 844 goto err_free; 845 } 846 i += 0x1 << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); 847 } 848 849 err_free: 850 kvfree(pfns); 851 return err ? false : true; 852 } 853 854 /** 855 * drm_gpusvm_scan_mm() - Check the migration state of a drm_gpusvm_range 856 * @range: Pointer to the struct drm_gpusvm_range to check. 857 * @dev_private_owner: The struct dev_private_owner to use to determine 858 * compatible device-private pages. 859 * @pagemap: The struct dev_pagemap pointer to use for pagemap-specific 860 * checks. 861 * 862 * Scan the CPU address space corresponding to @range and return the 863 * current migration state. Note that the result may be invalid as 864 * soon as the function returns. It's an advisory check. 865 * 866 * TODO: Bail early and call hmm_range_fault() for subranges. 867 * 868 * Return: See &enum drm_gpusvm_scan_result. 869 */ 870 enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, 871 void *dev_private_owner, 872 const struct dev_pagemap *pagemap) 873 { 874 struct mmu_interval_notifier *notifier = &range->notifier->notifier; 875 unsigned long start = drm_gpusvm_range_start(range); 876 unsigned long end = drm_gpusvm_range_end(range); 877 struct hmm_range hmm_range = { 878 .default_flags = 0, 879 .notifier = notifier, 880 .start = start, 881 .end = end, 882 .dev_private_owner = dev_private_owner, 883 }; 884 unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 885 enum drm_gpusvm_scan_result state = DRM_GPUSVM_SCAN_UNPOPULATED, new_state; 886 unsigned long *pfns; 887 unsigned long npages = npages_in_range(start, end); 888 const struct dev_pagemap *other = NULL; 889 int err, i; 890 891 pfns = kvzalloc_objs(*pfns, npages); 892 if (!pfns) 893 return DRM_GPUSVM_SCAN_UNPOPULATED; 894 895 hmm_range.hmm_pfns = pfns; 896 897 retry: 898 err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout); 899 if (err) 900 goto err_free; 901 902 drm_gpusvm_notifier_lock(range->gpusvm); 903 if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) { 904 drm_gpusvm_notifier_unlock(range->gpusvm); 905 goto retry; 906 } 907 908 for (i = 0; i < npages;) { 909 struct page *page; 910 const struct dev_pagemap *cur = NULL; 911 912 if (!(pfns[i] & HMM_PFN_VALID)) { 913 state = DRM_GPUSVM_SCAN_UNPOPULATED; 914 break; 915 } 916 917 page = hmm_pfn_to_page(pfns[i]); 918 if (is_device_private_page(page) || 919 is_device_coherent_page(page)) 920 cur = page_pgmap(page); 921 922 if (cur == pagemap) { 923 new_state = DRM_GPUSVM_SCAN_EQUAL; 924 } else if (cur && (cur == other || !other)) { 925 new_state = DRM_GPUSVM_SCAN_OTHER; 926 other = cur; 927 } else if (cur) { 928 new_state = DRM_GPUSVM_SCAN_MIXED_DEVICE; 929 } else { 930 new_state = DRM_GPUSVM_SCAN_SYSTEM; 931 } 932 933 /* 934 * TODO: Could use an array for state 935 * transitions, and caller might want it 936 * to bail early for some results. 937 */ 938 if (state == DRM_GPUSVM_SCAN_UNPOPULATED) { 939 state = new_state; 940 } else if (state != new_state) { 941 if (new_state == DRM_GPUSVM_SCAN_SYSTEM || 942 state == DRM_GPUSVM_SCAN_SYSTEM) 943 state = DRM_GPUSVM_SCAN_MIXED; 944 else if (state != DRM_GPUSVM_SCAN_MIXED) 945 state = DRM_GPUSVM_SCAN_MIXED_DEVICE; 946 } 947 948 i += 1ul << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); 949 } 950 951 drm_gpusvm_notifier_unlock(range->gpusvm); 952 953 err_free: 954 kvfree(pfns); 955 return state; 956 } 957 EXPORT_SYMBOL(drm_gpusvm_scan_mm); 958 959 /** 960 * drm_gpusvm_range_chunk_size() - Determine chunk size for GPU SVM range 961 * @gpusvm: Pointer to the GPU SVM structure 962 * @notifier: Pointer to the GPU SVM notifier structure 963 * @vas: Pointer to the virtual memory area structure 964 * @fault_addr: Fault address 965 * @gpuva_start: Start address of GPUVA which mirrors CPU 966 * @gpuva_end: End address of GPUVA which mirrors CPU 967 * @check_pages_threshold: Check CPU pages for present threshold 968 * @dev_private_owner: The device private page owner 969 * 970 * This function determines the chunk size for the GPU SVM range based on the 971 * fault address, GPU SVM chunk sizes, existing GPU SVM ranges, and the virtual 972 * memory area boundaries. 973 * 974 * Return: Chunk size on success, LONG_MAX on failure. 975 */ 976 static unsigned long 977 drm_gpusvm_range_chunk_size(struct drm_gpusvm *gpusvm, 978 struct drm_gpusvm_notifier *notifier, 979 struct vm_area_struct *vas, 980 unsigned long fault_addr, 981 unsigned long gpuva_start, 982 unsigned long gpuva_end, 983 unsigned long check_pages_threshold, 984 void *dev_private_owner) 985 { 986 unsigned long start, end; 987 int i = 0; 988 989 retry: 990 for (; i < gpusvm->num_chunks; ++i) { 991 start = ALIGN_DOWN(fault_addr, gpusvm->chunk_sizes[i]); 992 end = ALIGN(fault_addr + 1, gpusvm->chunk_sizes[i]); 993 994 if (start >= vas->vm_start && end <= vas->vm_end && 995 start >= drm_gpusvm_notifier_start(notifier) && 996 end <= drm_gpusvm_notifier_end(notifier) && 997 start >= gpuva_start && end <= gpuva_end) 998 break; 999 } 1000 1001 if (i == gpusvm->num_chunks) 1002 return LONG_MAX; 1003 1004 /* 1005 * If allocation more than page, ensure not to overlap with existing 1006 * ranges. 1007 */ 1008 if (end - start != SZ_4K) { 1009 struct drm_gpusvm_range *range; 1010 1011 range = drm_gpusvm_range_find(notifier, start, end); 1012 if (range) { 1013 ++i; 1014 goto retry; 1015 } 1016 1017 /* 1018 * XXX: Only create range on pages CPU has faulted in. Without 1019 * this check, or prefault, on BMG 'xe_exec_system_allocator --r 1020 * process-many-malloc' fails. In the failure case, each process 1021 * mallocs 16k but the CPU VMA is ~128k which results in 64k SVM 1022 * ranges. When migrating the SVM ranges, some processes fail in 1023 * drm_pagemap_migrate_to_devmem with 'migrate.cpages != npages' 1024 * and then upon drm_gpusvm_get_pages device pages from 1025 * other processes are collected + faulted in which creates all 1026 * sorts of problems. Unsure exactly how this happening, also 1027 * problem goes away if 'xe_exec_system_allocator --r 1028 * process-many-malloc' mallocs at least 64k at a time. 1029 */ 1030 if (end - start <= check_pages_threshold && 1031 !drm_gpusvm_check_pages(gpusvm, notifier, start, end, dev_private_owner)) { 1032 ++i; 1033 goto retry; 1034 } 1035 } 1036 1037 return end - start; 1038 } 1039 1040 #ifdef CONFIG_LOCKDEP 1041 /** 1042 * drm_gpusvm_driver_lock_held() - Assert GPU SVM driver lock is held 1043 * @gpusvm: Pointer to the GPU SVM structure. 1044 * 1045 * Ensure driver lock is held. 1046 */ 1047 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm) 1048 { 1049 if ((gpusvm)->lock_dep_map) 1050 lockdep_assert(lock_is_held_type((gpusvm)->lock_dep_map, 0)); 1051 } 1052 #else 1053 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm) 1054 { 1055 } 1056 #endif 1057 1058 /** 1059 * drm_gpusvm_find_vma_start() - Find start address for first VMA in range 1060 * @gpusvm: Pointer to the GPU SVM structure 1061 * @start: The inclusive start user address. 1062 * @end: The exclusive end user address. 1063 * 1064 * Returns: The start address of first VMA within the provided range, 1065 * ULONG_MAX otherwise. Assumes start_addr < end_addr. 1066 */ 1067 unsigned long 1068 drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm, 1069 unsigned long start, 1070 unsigned long end) 1071 { 1072 struct mm_struct *mm = gpusvm->mm; 1073 struct vm_area_struct *vma; 1074 unsigned long addr = ULONG_MAX; 1075 1076 if (!mmget_not_zero(mm)) 1077 return addr; 1078 1079 mmap_read_lock(mm); 1080 1081 vma = find_vma_intersection(mm, start, end); 1082 if (vma) 1083 addr = vma->vm_start; 1084 1085 mmap_read_unlock(mm); 1086 mmput(mm); 1087 1088 return addr; 1089 } 1090 EXPORT_SYMBOL_GPL(drm_gpusvm_find_vma_start); 1091 1092 /** 1093 * drm_gpusvm_range_find_or_insert() - Find or insert GPU SVM range 1094 * @gpusvm: Pointer to the GPU SVM structure 1095 * @fault_addr: Fault address 1096 * @gpuva_start: Start address of GPUVA which mirrors CPU 1097 * @gpuva_end: End address of GPUVA which mirrors CPU 1098 * @ctx: GPU SVM context 1099 * 1100 * This function finds or inserts a newly allocated a GPU SVM range based on the 1101 * fault address. Caller must hold a lock to protect range lookup and insertion. 1102 * 1103 * Return: Pointer to the GPU SVM range on success, ERR_PTR() on failure. 1104 */ 1105 struct drm_gpusvm_range * 1106 drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm, 1107 unsigned long fault_addr, 1108 unsigned long gpuva_start, 1109 unsigned long gpuva_end, 1110 const struct drm_gpusvm_ctx *ctx) 1111 { 1112 struct drm_gpusvm_notifier *notifier; 1113 struct drm_gpusvm_range *range; 1114 struct mm_struct *mm = gpusvm->mm; 1115 struct vm_area_struct *vas; 1116 bool notifier_alloc = false; 1117 unsigned long chunk_size; 1118 int err; 1119 bool migrate_devmem; 1120 1121 drm_gpusvm_driver_lock_held(gpusvm); 1122 1123 if (fault_addr < gpusvm->mm_start || 1124 fault_addr > gpusvm->mm_start + gpusvm->mm_range) 1125 return ERR_PTR(-EINVAL); 1126 1127 if (!mmget_not_zero(mm)) 1128 return ERR_PTR(-EFAULT); 1129 1130 notifier = drm_gpusvm_notifier_find(gpusvm, fault_addr, fault_addr + 1); 1131 if (!notifier) { 1132 notifier = drm_gpusvm_notifier_alloc(gpusvm, fault_addr); 1133 if (IS_ERR(notifier)) { 1134 err = PTR_ERR(notifier); 1135 goto err_mmunlock; 1136 } 1137 notifier_alloc = true; 1138 err = mmu_interval_notifier_insert(¬ifier->notifier, 1139 mm, 1140 drm_gpusvm_notifier_start(notifier), 1141 drm_gpusvm_notifier_size(notifier), 1142 &drm_gpusvm_notifier_ops); 1143 if (err) 1144 goto err_notifier; 1145 } 1146 1147 mmap_read_lock(mm); 1148 1149 vas = vma_lookup(mm, fault_addr); 1150 if (!vas) { 1151 err = -ENOENT; 1152 goto err_notifier_remove; 1153 } 1154 1155 if (!ctx->read_only && !(vas->vm_flags & VM_WRITE)) { 1156 err = -EPERM; 1157 goto err_notifier_remove; 1158 } 1159 1160 if (vas->vm_flags & (VM_IO | VM_PFNMAP)) { 1161 err = -EIO; 1162 goto err_notifier_remove; 1163 } 1164 1165 range = drm_gpusvm_range_find(notifier, fault_addr, fault_addr + 1); 1166 if (range) 1167 goto out_mmunlock; 1168 /* 1169 * XXX: Short-circuiting migration based on migrate_vma_* current 1170 * limitations. If/when migrate_vma_* add more support, this logic will 1171 * have to change. 1172 */ 1173 migrate_devmem = ctx->devmem_possible && vma_is_anonymous(vas); 1174 1175 chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas, 1176 fault_addr, gpuva_start, 1177 gpuva_end, 1178 ctx->check_pages_threshold, 1179 ctx->device_private_page_owner); 1180 if (chunk_size == LONG_MAX) { 1181 err = -EINVAL; 1182 goto err_notifier_remove; 1183 } 1184 1185 range = drm_gpusvm_range_alloc(gpusvm, notifier, fault_addr, chunk_size, 1186 migrate_devmem); 1187 if (IS_ERR(range)) { 1188 err = PTR_ERR(range); 1189 goto err_notifier_remove; 1190 } 1191 1192 drm_gpusvm_range_insert(notifier, range); 1193 if (notifier_alloc) 1194 drm_gpusvm_notifier_insert(gpusvm, notifier); 1195 1196 out_mmunlock: 1197 mmap_read_unlock(mm); 1198 mmput(mm); 1199 1200 return range; 1201 1202 err_notifier_remove: 1203 mmap_read_unlock(mm); 1204 if (notifier_alloc) 1205 mmu_interval_notifier_remove(¬ifier->notifier); 1206 err_notifier: 1207 if (notifier_alloc) 1208 drm_gpusvm_notifier_free(gpusvm, notifier); 1209 err_mmunlock: 1210 mmput(mm); 1211 return ERR_PTR(err); 1212 } 1213 EXPORT_SYMBOL_GPL(drm_gpusvm_range_find_or_insert); 1214 1215 /** 1216 * __drm_gpusvm_unmap_pages() - Unmap pages associated with GPU SVM pages (internal) 1217 * @gpusvm: Pointer to the GPU SVM structure 1218 * @svm_pages: Pointer to the GPU SVM pages structure 1219 * @npages: Number of pages to unmap 1220 * 1221 * This function unmap pages associated with a GPU SVM pages struct. Assumes and 1222 * asserts correct locking is in place when called. 1223 */ 1224 static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, 1225 struct drm_gpusvm_pages *svm_pages, 1226 unsigned long npages) 1227 { 1228 struct drm_pagemap *dpagemap = svm_pages->dpagemap; 1229 struct device *dev; 1230 unsigned long i, j; 1231 1232 lockdep_assert_held(&gpusvm->notifier_lock); 1233 1234 if (!svm_pages->drm) 1235 return; 1236 1237 dev = svm_pages->drm->dev; 1238 1239 if (svm_pages->flags.has_dma_mapping) { 1240 struct drm_gpusvm_pages_flags flags = { 1241 .__flags = svm_pages->flags.__flags, 1242 }; 1243 const struct drm_pagemap_addr *addrs = 1244 drm_gpusvm_pages_first_dma(svm_pages, NULL); 1245 bool use_iova = dma_use_iova(&svm_pages->state); 1246 1247 /* 1248 * IOVA is reserved for the whole range but only the linked 1249 * system pages (state_offset bytes) need unlinking; free the 1250 * entire reservation to avoid leaking the device-page part. 1251 * On the error path state_offset is 0, so just free it. 1252 */ 1253 if (use_iova) { 1254 if (svm_pages->state_offset) 1255 dma_iova_unlink(dev, &svm_pages->state, 0, 1256 svm_pages->state_offset, 1257 addrs[0].dir, 0); 1258 dma_iova_free(dev, &svm_pages->state); 1259 } 1260 1261 /* 1262 * With IOVA and no device page the unlink above tore every 1263 * entry down, and that is also when the range may be folded 1264 * to one entry, which must not be walked per entry. dpagemap 1265 * is set before the first device_map(), so it is also right 1266 * on the error path, where the flags are not published yet. 1267 */ 1268 for (i = 0, j = 0; 1269 (!use_iova || dpagemap) && i < npages; j++) { 1270 const struct drm_pagemap_addr *addr = &addrs[j]; 1271 1272 if (addr->proto == DRM_INTERCONNECT_SYSTEM) { 1273 /* 1274 * Linked IOVA pages were already torn down by 1275 * the dma_iova_unlink()/dma_iova_free() above; 1276 * only the non-IOVA mappings need unmap here. 1277 */ 1278 if (!use_iova) 1279 dma_unmap_page(dev, 1280 addr->addr, 1281 PAGE_SIZE << addr->order, 1282 addr->dir); 1283 } else if (dpagemap && dpagemap->ops->device_unmap) 1284 dpagemap->ops->device_unmap(dpagemap, 1285 dev, addr); 1286 i += 1 << addr->order; 1287 } 1288 1289 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ 1290 flags.has_devmem_pages = false; 1291 flags.has_dma_mapping = false; 1292 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); 1293 1294 drm_pagemap_put(svm_pages->dpagemap); 1295 svm_pages->dpagemap = NULL; 1296 } 1297 } 1298 1299 /** 1300 * __drm_gpusvm_free_pages() - Free dma array associated with GPU SVM pages 1301 * @gpusvm: Pointer to the GPU SVM structure 1302 * @svm_pages: Pointer to the GPU SVM pages structure 1303 * 1304 * This function frees the dma address array associated with a GPU SVM range. 1305 */ 1306 static void __drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm, 1307 struct drm_gpusvm_pages *svm_pages) 1308 { 1309 lockdep_assert_held(&gpusvm->notifier_lock); 1310 1311 if (svm_pages->flags.inline_dma_mapping) { 1312 struct drm_gpusvm_pages_flags flags = { 1313 .__flags = svm_pages->flags.__flags, 1314 }; 1315 1316 svm_pages->inline_addr = (struct drm_pagemap_addr){}; 1317 flags.inline_dma_mapping = false; 1318 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ 1319 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); 1320 return; 1321 } 1322 1323 if (svm_pages->dma_addr) { 1324 kvfree(svm_pages->dma_addr); 1325 svm_pages->dma_addr = NULL; 1326 } 1327 } 1328 1329 /** 1330 * drm_gpusvm_free_pages() - Free dma-mapping associated with GPU SVM pages 1331 * struct 1332 * @gpusvm: Pointer to the GPU SVM structure 1333 * @svm_pages: Pointer to the GPU SVM pages structure 1334 * @npages: Number of mapped pages 1335 * 1336 * This function unmaps and frees the dma address array associated with a GPU 1337 * SVM pages struct. 1338 */ 1339 void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm, 1340 struct drm_gpusvm_pages *svm_pages, 1341 unsigned long npages) 1342 { 1343 drm_gpusvm_notifier_lock(gpusvm); 1344 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages); 1345 __drm_gpusvm_free_pages(gpusvm, svm_pages); 1346 drm_gpusvm_notifier_unlock(gpusvm); 1347 } 1348 EXPORT_SYMBOL_GPL(drm_gpusvm_free_pages); 1349 1350 /** 1351 * drm_gpusvm_range_remove() - Remove GPU SVM range 1352 * @gpusvm: Pointer to the GPU SVM structure 1353 * @range: Pointer to the GPU SVM range to be removed 1354 * 1355 * This function removes the specified GPU SVM range and also removes the parent 1356 * GPU SVM notifier if no more ranges remain in the notifier. The caller must 1357 * hold a lock to protect range and notifier removal. 1358 * 1359 * This function does not unmap or free the drm_gpusvm_pages, the driver owns 1360 * that lifecycle. The caller must DMA unmap the range's pages before calling 1361 * this function, so a range is never removed from the MMU interval tree while 1362 * still DMA mapped. Typically the driver calls drm_gpusvm_unmap_pages() first. 1363 * And the range_free callback's drm_gpusvm_free_pages() is a final fallback safe 1364 * net. 1365 */ 1366 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm, 1367 struct drm_gpusvm_range *range) 1368 { 1369 struct drm_gpusvm_notifier *notifier; 1370 1371 drm_gpusvm_driver_lock_held(gpusvm); 1372 1373 notifier = drm_gpusvm_notifier_find(gpusvm, 1374 drm_gpusvm_range_start(range), 1375 drm_gpusvm_range_start(range) + 1); 1376 if (WARN_ON_ONCE(!notifier)) 1377 return; 1378 1379 drm_gpusvm_notifier_lock(gpusvm); 1380 __drm_gpusvm_range_remove(notifier, range); 1381 drm_gpusvm_notifier_unlock(gpusvm); 1382 1383 drm_gpusvm_range_put(range); 1384 1385 if (RB_EMPTY_ROOT(¬ifier->root.rb_root)) { 1386 if (!notifier->flags.removed) 1387 mmu_interval_notifier_remove(¬ifier->notifier); 1388 drm_gpusvm_notifier_remove(gpusvm, notifier); 1389 drm_gpusvm_notifier_free(gpusvm, notifier); 1390 } 1391 } 1392 EXPORT_SYMBOL_GPL(drm_gpusvm_range_remove); 1393 1394 /** 1395 * drm_gpusvm_range_get() - Get a reference to GPU SVM range 1396 * @range: Pointer to the GPU SVM range 1397 * 1398 * This function increments the reference count of the specified GPU SVM range. 1399 * 1400 * Return: Pointer to the GPU SVM range. 1401 */ 1402 struct drm_gpusvm_range * 1403 drm_gpusvm_range_get(struct drm_gpusvm_range *range) 1404 { 1405 kref_get(&range->refcount); 1406 1407 return range; 1408 } 1409 EXPORT_SYMBOL_GPL(drm_gpusvm_range_get); 1410 1411 /** 1412 * drm_gpusvm_range_destroy() - Destroy GPU SVM range 1413 * @refcount: Pointer to the reference counter embedded in the GPU SVM range 1414 * 1415 * This function destroys the specified GPU SVM range when its reference count 1416 * reaches zero. If a custom range-free function is provided, it is invoked to 1417 * free the range; otherwise, the range is deallocated using kfree(). 1418 */ 1419 static void drm_gpusvm_range_destroy(struct kref *refcount) 1420 { 1421 struct drm_gpusvm_range *range = 1422 container_of(refcount, struct drm_gpusvm_range, refcount); 1423 struct drm_gpusvm *gpusvm = range->gpusvm; 1424 1425 if (gpusvm->ops->range_free) 1426 gpusvm->ops->range_free(range); 1427 else 1428 kfree(range); 1429 } 1430 1431 /** 1432 * drm_gpusvm_range_put() - Put a reference to GPU SVM range 1433 * @range: Pointer to the GPU SVM range 1434 * 1435 * This function decrements the reference count of the specified GPU SVM range 1436 * and frees it when the count reaches zero. 1437 */ 1438 void drm_gpusvm_range_put(struct drm_gpusvm_range *range) 1439 { 1440 kref_put(&range->refcount, drm_gpusvm_range_destroy); 1441 } 1442 EXPORT_SYMBOL_GPL(drm_gpusvm_range_put); 1443 1444 /** 1445 * drm_gpusvm_pages_valid() - GPU SVM range pages valid 1446 * @gpusvm: Pointer to the GPU SVM structure 1447 * @svm_pages: Pointer to the GPU SVM pages structure 1448 * 1449 * This function determines if a GPU SVM range pages are valid. Expected be 1450 * called holding gpusvm->notifier_lock and as the last step before committing a 1451 * GPU binding. This is akin to a notifier seqno check in the HMM documentation 1452 * but due to wider notifiers (i.e., notifiers which span multiple ranges) this 1453 * function is required for finer grained checking (i.e., per range) if pages 1454 * are valid. 1455 * 1456 * Return: True if GPU SVM range has valid pages, False otherwise 1457 */ 1458 bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm, 1459 struct drm_gpusvm_pages *svm_pages) 1460 { 1461 lockdep_assert_held(&gpusvm->notifier_lock); 1462 1463 return svm_pages->flags.has_devmem_pages || svm_pages->flags.has_dma_mapping; 1464 } 1465 EXPORT_SYMBOL_GPL(drm_gpusvm_pages_valid); 1466 1467 /** 1468 * drm_gpusvm_pages_valid_unlocked() - GPU SVM pages valid unlocked 1469 * @gpusvm: Pointer to the GPU SVM structure 1470 * @svm_pages: Array of GPU SVM pages structures 1471 * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages 1472 * 1473 * This function determines if every GPU SVM pages instance is valid, resetting 1474 * every instance which is not so that get_pages() maps it afresh. It therefore 1475 * has to walk them all. Expected be called without holding 1476 * gpusvm->notifier_lock. 1477 * 1478 * Return: True if all GPU SVM pages are valid, False otherwise 1479 */ 1480 static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm, 1481 struct drm_gpusvm_pages *svm_pages, 1482 unsigned int num_pages) 1483 { 1484 bool pages_valid = true; 1485 unsigned int p; 1486 1487 drm_gpusvm_notifier_lock(gpusvm); 1488 for (p = 0; p < num_pages; ++p) { 1489 if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p])) 1490 continue; 1491 __drm_gpusvm_free_pages(gpusvm, &svm_pages[p]); 1492 pages_valid = false; 1493 } 1494 drm_gpusvm_notifier_unlock(gpusvm); 1495 1496 return pages_valid; 1497 } 1498 1499 /** 1500 * drm_gpusvm_pages_inlinable() - Whether the dma address can be inlined 1501 * @svm_pages: The SVM pages instance that was just mapped 1502 * @nentries: Number of entries the mapping loop produced 1503 * @npages: Number of pages in the CPU range 1504 * 1505 * A THP maps as one huge page, and an IOVA reservation links every page of 1506 * the range at the next offset, so the device addresses run contiguously from 1507 * entry 0. Either way one entry describes the whole range, so the dma_addr 1508 * array can be freed and the address kept inline. 1509 * 1510 * state_offset advances only on the IOVA branch, so reaching the full range 1511 * length proves no device page was mapped in between. Only single page 1512 * entries fold, so the order kept is 0 and describes the range truthfully. 1513 * Larger chunks, several huge pages among them, stay an array that is 1514 * already short and that a consumer places with one PTE each. 1515 * 1516 * Return: True if the mapping fits in a single drm_pagemap_addr. 1517 */ 1518 static bool drm_gpusvm_pages_inlinable(struct drm_gpusvm_pages *svm_pages, 1519 unsigned long nentries, 1520 unsigned long npages) 1521 { 1522 if (nentries == 1) 1523 return true; 1524 1525 return nentries == npages && dma_use_iova(&svm_pages->state) && 1526 svm_pages->state_offset == npages * PAGE_SIZE; 1527 } 1528 1529 /** 1530 * drm_gpusvm_walk_devmem() - Invoke the devmem callback across faulted pages 1531 * @gpusvm: Pointer to the GPU SVM structure 1532 * @pfns: The already-faulted pfn array (size @npages) 1533 * @npages: Number of pages in the CPU range 1534 * @ctx: GPU SVM context, with a non-NULL &drm_gpusvm_ctx.devmem_fn 1535 * 1536 * Invoke &drm_gpusvm_ctx.devmem_fn once per contiguous run of @pfns backed by 1537 * the same device-memory allocation. Must be called under the notifier lock. 1538 */ 1539 static void drm_gpusvm_walk_devmem(struct drm_gpusvm *gpusvm, 1540 unsigned long *pfns, 1541 unsigned long npages, 1542 const struct drm_gpusvm_ctx *ctx) 1543 { 1544 struct drm_pagemap_devmem *last = NULL; 1545 unsigned int order = 0; 1546 unsigned long i; 1547 1548 lockdep_assert_held(&gpusvm->notifier_lock); 1549 1550 for (i = 0; i < npages; i += 1 << order) { 1551 struct page *page = hmm_pfn_to_page(pfns[i]); 1552 struct drm_pagemap_devmem *devmem; 1553 1554 order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); 1555 1556 if (!is_device_private_page(page) && 1557 !is_device_coherent_page(page)) { 1558 last = NULL; 1559 continue; 1560 } 1561 1562 devmem = drm_pagemap_page_to_devmem(page); 1563 if (devmem == last) 1564 continue; 1565 1566 last = devmem; 1567 ctx->devmem_fn(devmem); 1568 } 1569 } 1570 1571 /** 1572 * drm_gpusvm_dma_map_pages() - Walk and DMA map one drm_gpusvm_pages instance 1573 * @gpusvm: Pointer to the GPU SVM structure 1574 * @svm_pages: The SVM pages instance to populate with dma-addresses 1575 * @pfns: The already-faulted pfn array (size @npages) 1576 * @npages: Number of pages in the CPU range 1577 * @ctx: GPU SVM context 1578 * @dma_dir: DMA data direction for the mappings 1579 * 1580 * Walk the faulted @pfns and map them into @svm_pages for DMA access through 1581 * its owning drm_device. Must be called under the notifier lock and only for 1582 * an instance without a live mapping. On failure this unwinds the partial 1583 * mapping of this instance before returning. 1584 * 1585 * Return: 0 on success, negative error code on failure. 1586 */ 1587 static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm, 1588 struct drm_gpusvm_pages *svm_pages, 1589 unsigned long *pfns, 1590 unsigned long npages, 1591 const struct drm_gpusvm_ctx *ctx, 1592 enum dma_data_direction dma_dir) 1593 { 1594 void *zdd = NULL; 1595 unsigned long i, j; 1596 unsigned long num_dma_mapped = 0; 1597 unsigned int order = 0; 1598 int err = 0; 1599 struct dev_pagemap *pagemap = NULL; 1600 struct drm_pagemap *dpagemap; 1601 struct drm_gpusvm_pages_flags flags; 1602 struct dma_iova_state *state = &svm_pages->state; 1603 1604 lockdep_assert_held(&gpusvm->notifier_lock); 1605 1606 *state = (struct dma_iova_state){}; 1607 svm_pages->state_offset = 0; 1608 1609 flags.__flags = svm_pages->flags.__flags; 1610 1611 for (i = 0, j = 0; i < npages; ++j) { 1612 struct page *page = hmm_pfn_to_page(pfns[i]); 1613 1614 order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); 1615 if (is_device_private_page(page) || 1616 is_device_coherent_page(page)) { 1617 struct drm_pagemap_zdd *__zdd = 1618 drm_pagemap_page_zone_device_data(page); 1619 1620 if (!ctx->allow_mixed && 1621 zdd != __zdd && i > 0) { 1622 err = -EOPNOTSUPP; 1623 goto err_unmap; 1624 } 1625 zdd = __zdd; 1626 if (pagemap != page_pgmap(page)) { 1627 if (pagemap) { 1628 err = -EOPNOTSUPP; 1629 goto err_unmap; 1630 } 1631 1632 pagemap = page_pgmap(page); 1633 dpagemap = drm_pagemap_page_to_dpagemap(page); 1634 if (drm_WARN_ON(svm_pages->drm, !dpagemap)) { 1635 /* 1636 * Raced. This is not supposed to happen 1637 * since hmm_range_fault() should've migrated 1638 * this page to system. 1639 */ 1640 err = -EAGAIN; 1641 goto err_unmap; 1642 } 1643 1644 /* 1645 * Set the dpagemap as soon as the first 1646 * device page is mapped so the err_unmap path 1647 * can device_unmap() the device mappings that 1648 * have already been created. 1649 */ 1650 drm_pagemap_get(dpagemap); 1651 drm_pagemap_put(svm_pages->dpagemap); 1652 svm_pages->dpagemap = dpagemap; 1653 } 1654 svm_pages->dma_addr[j] = 1655 dpagemap->ops->device_map(dpagemap, 1656 svm_pages->drm->dev, 1657 page, order, 1658 dma_dir); 1659 if (dma_mapping_error(svm_pages->drm->dev, 1660 svm_pages->dma_addr[j].addr)) { 1661 err = -EFAULT; 1662 goto err_unmap; 1663 } 1664 } else { 1665 dma_addr_t addr; 1666 1667 if (is_zone_device_page(page) || 1668 (pagemap && !ctx->allow_mixed)) { 1669 err = -EOPNOTSUPP; 1670 goto err_unmap; 1671 } 1672 1673 if (ctx->devmem_only) { 1674 err = -EFAULT; 1675 goto err_unmap; 1676 } 1677 1678 if (!i) 1679 dma_iova_try_alloc(svm_pages->drm->dev, state, 1680 0, npages * PAGE_SIZE); 1681 1682 if (dma_use_iova(state)) { 1683 err = dma_iova_link(svm_pages->drm->dev, state, 1684 hmm_pfn_to_phys(pfns[i]), 1685 svm_pages->state_offset, 1686 PAGE_SIZE << order, 1687 dma_dir, 0); 1688 if (err) 1689 goto err_unmap; 1690 1691 addr = state->addr + svm_pages->state_offset; 1692 svm_pages->state_offset += PAGE_SIZE << order; 1693 } else { 1694 addr = dma_map_page(svm_pages->drm->dev, 1695 page, 0, 1696 PAGE_SIZE << order, 1697 dma_dir); 1698 if (dma_mapping_error(svm_pages->drm->dev, addr)) { 1699 err = -EFAULT; 1700 goto err_unmap; 1701 } 1702 } 1703 1704 svm_pages->dma_addr[j] = drm_pagemap_addr_encode 1705 (addr, DRM_INTERCONNECT_SYSTEM, order, 1706 dma_dir); 1707 } 1708 i += 1 << order; 1709 num_dma_mapped = i; 1710 flags.has_dma_mapping = true; 1711 } 1712 1713 if (dma_use_iova(state)) { 1714 err = dma_iova_sync(svm_pages->drm->dev, state, 0, 1715 svm_pages->state_offset); 1716 if (err) 1717 goto err_unmap; 1718 } 1719 1720 if (pagemap) 1721 flags.has_devmem_pages = true; 1722 1723 if (drm_gpusvm_pages_inlinable(svm_pages, j, npages)) { 1724 struct drm_pagemap_addr addr = svm_pages->dma_addr[0]; 1725 1726 kvfree(svm_pages->dma_addr); 1727 svm_pages->inline_addr = addr; 1728 flags.inline_dma_mapping = true; 1729 } 1730 1731 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ 1732 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); 1733 1734 return 0; 1735 1736 err_unmap: 1737 svm_pages->flags.has_dma_mapping = true; 1738 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped); 1739 return err; 1740 } 1741 1742 /** 1743 * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct 1744 * @gpusvm: Pointer to the GPU SVM structure 1745 * @svm_pages: Array of SVM pages instances to populate with dma addresses 1746 * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages, must not be 0 1747 * @mm: The mm corresponding to the CPU range 1748 * @notifier: The corresponding notifier for the given CPU range 1749 * @pages_start: Start CPU address for the pages 1750 * @pages_end: End CPU address for the pages (exclusive) 1751 * @ctx: GPU SVM context 1752 * 1753 * This function gets and maps pages for a CPU range and ensures they are 1754 * mapped for DMA access. The HMM fault for the CPU range is performed once, 1755 * the DMA mapping by drm_gpusvm_dma_map_pages() is then done per instance, 1756 * one per owning drm_device. The retry against notifier races is kept here 1757 * in common code so drivers never open code it. 1758 * 1759 * On error the instances mapped before the failing one stay mapped, so the 1760 * caller must unmap and free every instance regardless of the return value. 1761 * 1762 * With &drm_gpusvm_ctx.no_dma_map no mapping state is recorded, so 1763 * drm_gpusvm_pages_valid() never returns true and success is only a snapshot: 1764 * the caller must recheck mmu_interval_read_retry() against the recorded 1765 * &drm_gpusvm_pages.notifier_seq under the notifier lock, and hold it until 1766 * its work is visible to invalidation. 1767 * 1768 * Return: 0 on success, negative error code on failure. 1769 */ 1770 int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, 1771 struct drm_gpusvm_pages *svm_pages, 1772 unsigned int num_pages, 1773 struct mm_struct *mm, 1774 struct mmu_interval_notifier *notifier, 1775 unsigned long pages_start, unsigned long pages_end, 1776 const struct drm_gpusvm_ctx *ctx) 1777 { 1778 struct hmm_range hmm_range = { 1779 .default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 : 1780 HMM_PFN_REQ_WRITE), 1781 .notifier = notifier, 1782 .start = pages_start, 1783 .end = pages_end, 1784 .dev_private_owner = ctx->device_private_page_owner, 1785 }; 1786 unsigned long timeout = 1787 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1788 unsigned long remaining; 1789 unsigned long npages = npages_in_range(pages_start, pages_end); 1790 unsigned long *pfns; 1791 int err = 0; 1792 enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE : 1793 DMA_BIDIRECTIONAL; 1794 const bool map_dma = !ctx->no_dma_map; 1795 unsigned int p; 1796 1797 if (!num_pages) 1798 return -EINVAL; 1799 1800 if (ctx->no_dma_map && ctx->devmem_only) 1801 return -EINVAL; 1802 1803 if (map_dma) { 1804 for (p = 0; p < num_pages; ++p) { 1805 if (!svm_pages[p].drm) 1806 return -EINVAL; 1807 } 1808 } 1809 1810 retry: 1811 remaining = timeout - jiffies; 1812 1813 if (time_after_eq(jiffies, timeout)) 1814 return -EBUSY; 1815 1816 hmm_range.notifier_seq = mmu_interval_read_begin(notifier); 1817 1818 /* 1819 * drm_gpusvm_pages_valid_unlocked() also resets any instance whose 1820 * mapping was invalidated, so it must run on the map_dma path even when 1821 * devmem_fn suppresses the fast path; otherwise a stale inline mapping 1822 * is later mistaken for a dma_addr array by drm_gpusvm_dma_map_pages(). 1823 */ 1824 if (map_dma) { 1825 bool pages_valid = drm_gpusvm_pages_valid_unlocked(gpusvm, 1826 svm_pages, 1827 num_pages); 1828 1829 if (!ctx->devmem_fn && pages_valid) 1830 goto set_seqno; 1831 } 1832 1833 pfns = kvmalloc_objs(*pfns, npages); 1834 if (!pfns) 1835 return -ENOMEM; 1836 1837 if (!mmget_not_zero(mm)) { 1838 err = -EFAULT; 1839 goto err_free; 1840 } 1841 1842 hmm_range.hmm_pfns = pfns; 1843 err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining); 1844 mmput(mm); 1845 if (err) 1846 goto err_free; 1847 1848 if (map_dma) { 1849 for (p = 0; p < num_pages; ++p) { 1850 if (drm_gpusvm_pages_first_dma(&svm_pages[p], NULL)) 1851 continue; 1852 svm_pages[p].dma_addr = 1853 kvzalloc_objs(*svm_pages[p].dma_addr, npages); 1854 if (!svm_pages[p].dma_addr) { 1855 err = -ENOMEM; 1856 goto err_free; 1857 } 1858 } 1859 } 1860 1861 /* 1862 * Perform all dma mappings under the notifier lock to not 1863 * access freed pages. A notifier will either block on 1864 * the notifier lock or unmap dma. 1865 */ 1866 drm_gpusvm_notifier_lock(gpusvm); 1867 1868 /* 1869 * drm_gpusvm_range_set_unmapped() flags the whole array in one go under 1870 * the write lock, so any instance answers for all of them here. 1871 */ 1872 if (svm_pages[0].flags.unmapped) { 1873 drm_gpusvm_notifier_unlock(gpusvm); 1874 err = -EFAULT; 1875 goto err_free; 1876 } 1877 1878 if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) { 1879 drm_gpusvm_notifier_unlock(gpusvm); 1880 kvfree(pfns); 1881 goto retry; 1882 } 1883 1884 if (ctx->devmem_fn) 1885 drm_gpusvm_walk_devmem(gpusvm, pfns, npages, ctx); 1886 1887 if (map_dma) { 1888 for (p = 0; p < num_pages; ++p) { 1889 if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p])) 1890 continue; 1891 1892 err = drm_gpusvm_dma_map_pages(gpusvm, &svm_pages[p], pfns, 1893 npages, ctx, dma_dir); 1894 if (err) { 1895 /* 1896 * The failing instance was unwound by the 1897 * helper. Keep the ones mapped earlier: the 1898 * -EAGAIN retry reuses them, and the driver 1899 * unmaps every instance with the range on the 1900 * other error paths. 1901 */ 1902 drm_gpusvm_notifier_unlock(gpusvm); 1903 goto err_free; 1904 } 1905 } 1906 } 1907 1908 drm_gpusvm_notifier_unlock(gpusvm); 1909 kvfree(pfns); 1910 set_seqno: 1911 for (p = 0; p < num_pages; ++p) 1912 svm_pages[p].notifier_seq = hmm_range.notifier_seq; 1913 1914 return 0; 1915 1916 err_free: 1917 kvfree(pfns); 1918 if (err == -EAGAIN) 1919 goto retry; 1920 return err; 1921 } 1922 EXPORT_SYMBOL_GPL(drm_gpusvm_get_pages); 1923 1924 /** 1925 * drm_gpusvm_unmap_pages() - Unmap GPU svm pages 1926 * @gpusvm: Pointer to the GPU SVM structure 1927 * @svm_pages: Pointer to the GPU SVM pages structure 1928 * @npages: Number of pages in @svm_pages. 1929 * @ctx: GPU SVM context 1930 * 1931 * This function unmaps pages associated with a GPU SVM pages struct. If 1932 * @in_notifier is set, it is assumed that gpusvm->notifier_lock is held in 1933 * write mode; if it is clear, it acquires gpusvm->notifier_lock in read mode. 1934 * Must be called in the invalidate() callback of the corresponding notifier for 1935 * IOMMU security model. 1936 */ 1937 void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, 1938 struct drm_gpusvm_pages *svm_pages, 1939 unsigned long npages, 1940 const struct drm_gpusvm_ctx *ctx) 1941 { 1942 if (ctx->in_notifier) 1943 lockdep_assert_held_write(&gpusvm->notifier_lock); 1944 else 1945 drm_gpusvm_notifier_lock(gpusvm); 1946 1947 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages); 1948 1949 if (!ctx->in_notifier) 1950 drm_gpusvm_notifier_unlock(gpusvm); 1951 } 1952 EXPORT_SYMBOL_GPL(drm_gpusvm_unmap_pages); 1953 1954 /** 1955 * drm_gpusvm_range_evict() - Evict GPU SVM range 1956 * @gpusvm: Pointer to the GPU SVM structure 1957 * @range: Pointer to the GPU SVM range to be removed 1958 * 1959 * This function evicts the specified GPU SVM range. 1960 * 1961 * Return: 0 on success, a negative error code on failure. 1962 */ 1963 int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, 1964 struct drm_gpusvm_range *range) 1965 { 1966 struct mmu_interval_notifier *notifier = &range->notifier->notifier; 1967 struct hmm_range hmm_range = { 1968 .default_flags = HMM_PFN_REQ_FAULT, 1969 .notifier = notifier, 1970 .start = drm_gpusvm_range_start(range), 1971 .end = drm_gpusvm_range_end(range), 1972 .dev_private_owner = NULL, 1973 }; 1974 unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1975 unsigned long *pfns; 1976 unsigned long npages = npages_in_range(drm_gpusvm_range_start(range), 1977 drm_gpusvm_range_end(range)); 1978 int err = 0; 1979 struct mm_struct *mm = gpusvm->mm; 1980 1981 if (!mmget_not_zero(mm)) 1982 return -EFAULT; 1983 1984 pfns = kvmalloc_objs(*pfns, npages); 1985 if (!pfns) { 1986 mmput(mm); 1987 return -ENOMEM; 1988 } 1989 1990 hmm_range.hmm_pfns = pfns; 1991 err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout); 1992 1993 kvfree(pfns); 1994 mmput(mm); 1995 1996 return err == -EBUSY ? -ETIME : err; 1997 } 1998 EXPORT_SYMBOL_GPL(drm_gpusvm_range_evict); 1999 2000 /** 2001 * drm_gpusvm_has_mapping() - Check if GPU SVM has mapping for the given address range 2002 * @gpusvm: Pointer to the GPU SVM structure. 2003 * @start: Start address 2004 * @end: End address 2005 * 2006 * Return: True if GPU SVM has mapping, False otherwise 2007 */ 2008 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start, 2009 unsigned long end) 2010 { 2011 struct drm_gpusvm_notifier *notifier; 2012 2013 drm_gpusvm_for_each_notifier(notifier, gpusvm, start, end) { 2014 struct drm_gpusvm_range *range = NULL; 2015 2016 drm_gpusvm_for_each_range(range, notifier, start, end) 2017 return true; 2018 } 2019 2020 return false; 2021 } 2022 EXPORT_SYMBOL_GPL(drm_gpusvm_has_mapping); 2023 2024 /** 2025 * drm_gpusvm_range_set_unmapped() - Mark a GPU SVM range as unmapped 2026 * @range: Pointer to the GPU SVM range structure. 2027 * @pages: Pointer to the GPU SVM pages structure(s). 2028 * @pages_count: Number of GPU SVM pages structure(s) passed in. 2029 * @mmu_range: Pointer to the MMU notifier range structure. 2030 * 2031 * This function marks a GPU SVM range as unmapped and sets the partial_unmap flag 2032 * if the range partially falls within the provided MMU notifier range. 2033 */ 2034 void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range, 2035 struct drm_gpusvm_pages *pages, 2036 unsigned int pages_count, 2037 const struct mmu_notifier_range *mmu_range) 2038 { 2039 struct drm_gpusvm_range_flags range_flags = { 2040 .__flags = range->flags.__flags, 2041 }; 2042 unsigned int i; 2043 2044 lockdep_assert_held_write(&range->gpusvm->notifier_lock); 2045 2046 range_flags.unmapped = true; 2047 for (i = 0; i < pages_count; ++i) { 2048 struct drm_gpusvm_pages_flags flags = { 2049 .__flags = pages[i].flags.__flags, 2050 }; 2051 2052 flags.unmapped = true; 2053 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ 2054 WRITE_ONCE(pages[i].flags.__flags, flags.__flags); 2055 } 2056 if (drm_gpusvm_range_start(range) < mmu_range->start || 2057 drm_gpusvm_range_end(range) > mmu_range->end) 2058 range_flags.partial_unmap = true; 2059 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ 2060 WRITE_ONCE(range->flags.__flags, range_flags.__flags); 2061 } 2062 EXPORT_SYMBOL_GPL(drm_gpusvm_range_set_unmapped); 2063 2064 MODULE_DESCRIPTION("DRM GPUSVM"); 2065 MODULE_LICENSE("GPL"); 2066