1 // SPDX-License-Identifier: GPL-2.0-only OR MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 * 5 * Authors: 6 * Matthew Brost <matthew.brost@intel.com> 7 */ 8 9 #include <linux/dma-mapping.h> 10 #include <linux/export.h> 11 #include <linux/hmm.h> 12 #include <linux/hugetlb_inline.h> 13 #include <linux/memremap.h> 14 #include <linux/mm_types.h> 15 #include <linux/slab.h> 16 17 #include <drm/drm_device.h> 18 #include <drm/drm_gpusvm.h> 19 #include <drm/drm_pagemap.h> 20 #include <drm/drm_print.h> 21 22 /** 23 * DOC: Overview 24 * 25 * GPU Shared Virtual Memory (GPU SVM) layer for the Direct Rendering Manager (DRM) 26 * is a component of the DRM framework designed to manage shared virtual memory 27 * between the CPU and GPU. It enables efficient data exchange and processing 28 * for GPU-accelerated applications by allowing memory sharing and 29 * synchronization between the CPU's and GPU's virtual address spaces. 30 * 31 * Key GPU SVM Components: 32 * 33 * - Notifiers: 34 * Used for tracking memory intervals and notifying the GPU of changes, 35 * notifiers are sized based on a GPU SVM initialization parameter, with a 36 * recommendation of 512M or larger. They maintain a Red-BlacK tree and a 37 * list of ranges that fall within the notifier interval. Notifiers are 38 * tracked within a GPU SVM Red-BlacK tree and list and are dynamically 39 * inserted or removed as ranges within the interval are created or 40 * destroyed. 41 * - Ranges: 42 * Represent memory ranges mapped in a DRM device and managed by GPU SVM. 43 * They are sized based on an array of chunk sizes, which is a GPU SVM 44 * initialization parameter, and the CPU address space. Upon GPU fault, 45 * the largest aligned chunk that fits within the faulting CPU address 46 * space is chosen for the range size. Ranges are expected to be 47 * dynamically allocated on GPU fault and removed on an MMU notifier UNMAP 48 * event. As mentioned above, ranges are tracked in a notifier's Red-Black 49 * tree. 50 * 51 * - Pages: 52 * struct drm_gpusvm_pages holds the DMA mapping state for a range of 53 * CPU virtual addresses: the DMA mapped device addresses, 54 * the device private pagemap, the IOVA state, the per mapping 55 * notifier sequence number, and the drm_device that owns the DMA 56 * mappings. 57 * A driver embeds one or more struct drm_gpusvm_pages alongside its 58 * struct drm_gpusvm_range, choosing one of two layouts: 59 * 60 * 1:1 - one drm_gpusvm_pages per range (one drm_device). Simplest 61 * layout; to mirror a VA range on several devices a driver uses a 62 * separate range (and notifier) per device, so the HMM fault is taken 63 * once per device. 64 * 65 * N:1 - one drm_gpusvm_pages per drm_device, all sharing one range and 66 * notifier; only the per-device DMA mapping differs. The instances must 67 * sit in contiguous memory so a single drm_gpusvm_range_set_unmapped() 68 * can mark them all. A driver can keep one instance inline for the single 69 * device case and switch to a heap array only when more devices join, 70 * e.g.: 71 * 72 * .. code-block:: c 73 * 74 * struct driver_range { 75 * struct drm_gpusvm_range base; 76 * unsigned int num_pages; // 1: inline_pages, >1: pages[] 77 * union { 78 * struct drm_gpusvm_pages inline_pages; 79 * struct drm_gpusvm_pages *pages; 80 * }; 81 * }; 82 * 83 * static struct drm_gpusvm_pages * 84 * driver_pages(struct driver_range *drange) 85 * { 86 * return drange->num_pages == 1 ? &drange->inline_pages : 87 * drange->pages; 88 * } 89 * 90 * In the N:1 case the driver allocates the pages array with a zeroing 91 * allocator (e.g. kcalloc(num_pages, ...)), initialises each entry with 92 * drm_gpusvm_init_pages(), and frees each entry with 93 * drm_gpusvm_free_pages() plus the array itself, from its range free 94 * callback. Each drm_gpusvm_pages is mapped independently by their own 95 * drm_device. 96 * Each drm_gpusvm_pages must be zero-initialised and initialised with 97 * drm_gpusvm_init_pages(), called once per entry. 98 * 99 * The 1:1 examples below pass @num_pages == 1 and &drange->pages. In the 100 * N:1 case the driver instead passes the whole array and its count, so a 101 * single call faults the CPU range once and DMA maps it for every owning 102 * drm_device, e.g.: 103 * 104 * .. code-block:: c 105 * 106 * // GPU fault handler: one fault, one DMA mapping per device 107 * err = drm_gpusvm_get_pages(gpusvm, driver_pages(drange), 108 * drange->num_pages, gpusvm->mm, 109 * &range->notifier->notifier, 110 * drm_gpusvm_range_start(range), 111 * drm_gpusvm_range_end(range), &ctx); 112 * 113 * // Notifier callback: mark every instance unmapped in one call 114 * drm_gpusvm_range_set_unmapped(range, driver_pages(drange), 115 * drange->num_pages, mmu_range); 116 * 117 * The unmap and free paths stay per-instance: iterate @num_pages over 118 * driver_pages(drange) and call drm_gpusvm_unmap_pages() / 119 * drm_gpusvm_free_pages() for each entry. 120 * 121 * - Operations: 122 * Define the interface for driver-specific GPU SVM operations such as 123 * range allocation, notifier allocation, and invalidations. 124 * 125 * - Device Memory Allocations: 126 * Embedded structure containing enough information for GPU SVM to migrate 127 * to / from device memory. 128 * 129 * - Device Memory Operations: 130 * Define the interface for driver-specific device memory operations 131 * release memory, populate pfns, and copy to / from device memory. 132 * 133 * This layer provides interfaces for allocating, mapping, migrating, and 134 * releasing memory ranges between the CPU and GPU. It handles all core memory 135 * management interactions (DMA mapping, HMM, and migration) and provides 136 * driver-specific virtual functions (vfuncs). This infrastructure is sufficient 137 * to build the expected driver components for an SVM implementation as detailed 138 * below. 139 * 140 * Expected Driver Components: 141 * 142 * - GPU page fault handler: 143 * Used to create ranges and notifiers based on the fault address, 144 * optionally migrate the range to device memory, and create GPU bindings. 145 * 146 * - Garbage collector: 147 * Used to unmap and destroy GPU bindings for ranges. Ranges are expected 148 * to be added to the garbage collector upon a MMU_NOTIFY_UNMAP event in 149 * notifier callback. 150 * 151 * - Notifier callback: 152 * Used to invalidate and DMA unmap GPU bindings for ranges. 153 */ 154 155 /** 156 * DOC: Locking 157 * 158 * GPU SVM handles locking for core MM interactions, i.e., it locks/unlocks the 159 * mmap lock as needed. 160 * 161 * GPU SVM introduces a global notifier lock, which safeguards the notifier's 162 * range RB tree and list, as well as the range's DMA mappings and sequence 163 * number. GPU SVM manages all necessary locking and unlocking operations, 164 * except for the recheck range's pages being valid 165 * (drm_gpusvm_pages_valid) when the driver is committing GPU bindings. 166 * This lock corresponds to the ``driver->update`` lock mentioned in 167 * Documentation/mm/hmm.rst. Future revisions may transition from a GPU SVM 168 * global lock to a per-notifier lock if finer-grained locking is deemed 169 * necessary. 170 * 171 * In addition to the locking mentioned above, the driver should implement a 172 * lock to safeguard core GPU SVM function calls that modify state, such as 173 * drm_gpusvm_range_find_or_insert and drm_gpusvm_range_remove. This lock is 174 * denoted as 'driver_svm_lock' in code examples. Finer grained driver side 175 * locking should also be possible for concurrent GPU fault processing within a 176 * single GPU SVM. The 'driver_svm_lock' can be via drm_gpusvm_driver_set_lock 177 * to add annotations to GPU SVM. 178 */ 179 180 /** 181 * DOC: Partial Unmapping of Ranges 182 * 183 * Partial unmapping of ranges (e.g., 1M out of 2M is unmapped by CPU resulting 184 * in MMU_NOTIFY_UNMAP event) presents several challenges, with the main one 185 * being that a subset of the range still has CPU and GPU mappings. If the 186 * backing store for the range is in device memory, a subset of the backing 187 * store has references. One option would be to split the range and device 188 * memory backing store, but the implementation for this would be quite 189 * complicated. Given that partial unmappings are rare and driver-defined range 190 * sizes are relatively small, GPU SVM does not support splitting of ranges. 191 * 192 * With no support for range splitting, upon partial unmapping of a range, the 193 * driver is expected to invalidate and destroy the entire range. If the range 194 * has device memory as its backing, the driver is also expected to migrate any 195 * remaining pages back to RAM. 196 */ 197 198 /** 199 * DOC: Examples 200 * 201 * This section provides three examples of how to build the expected driver 202 * components: the GPU page fault handler, the garbage collector, and the 203 * notifier callback. 204 * 205 * The generic code provided does not include logic for complex migration 206 * policies, optimized invalidations, fined grained driver locking, or other 207 * potentially required driver locking (e.g., DMA-resv locks). 208 * 209 * 1) GPU page fault handler 210 * 211 * .. code-block:: c 212 * 213 * struct driver_range { 214 * struct drm_gpusvm_range base; 215 * struct drm_gpusvm_pages pages; 216 * }; 217 * 218 * int driver_bind_range(struct drm_gpusvm *gpusvm, struct driver_range *drange) 219 * { 220 * int err = 0; 221 * 222 * driver_alloc_and_setup_memory_for_bind(gpusvm, drange); 223 * 224 * drm_gpusvm_notifier_lock(gpusvm); 225 * if (drm_gpusvm_pages_valid(gpusvm, &drange->pages)) 226 * driver_commit_bind(gpusvm, drange); 227 * else 228 * err = -EAGAIN; 229 * drm_gpusvm_notifier_unlock(gpusvm); 230 * 231 * return err; 232 * } 233 * 234 * int driver_gpu_fault(struct drm_gpusvm *gpusvm, unsigned long fault_addr, 235 * unsigned long gpuva_start, unsigned long gpuva_end) 236 * { 237 * struct drm_gpusvm_ctx ctx = {}; 238 * struct driver_range *drange; 239 * struct drm_gpusvm_range *range; 240 * int err; 241 * 242 * driver_svm_lock(); 243 * retry: 244 * // Always process UNMAPs first so view of GPU SVM ranges is current 245 * driver_garbage_collector(gpusvm); 246 * 247 * range = drm_gpusvm_range_find_or_insert(gpusvm, fault_addr, 248 * gpuva_start, gpuva_end, 249 * &ctx); 250 * if (IS_ERR(range)) { 251 * err = PTR_ERR(range); 252 * goto unlock; 253 * } 254 * drange = container_of(range, struct driver_range, base); 255 * 256 * if (driver_migration_policy(range)) { 257 * err = drm_pagemap_populate_mm(driver_choose_drm_pagemap(), 258 * gpuva_start, gpuva_end, gpusvm->mm, 259 * ctx->timeslice_ms); 260 * if (err) // CPU mappings may have changed 261 * goto retry; 262 * } 263 * 264 * err = drm_gpusvm_get_pages(gpusvm, &drange->pages, 1, 265 * gpusvm->mm, &range->notifier->notifier, 266 * drm_gpusvm_range_start(range), 267 * drm_gpusvm_range_end(range), &ctx); 268 * if (err == -EOPNOTSUPP || err == -EFAULT || err == -EPERM) { // CPU mappings changed 269 * if (err == -EOPNOTSUPP) 270 * drm_gpusvm_range_evict(gpusvm, range); 271 * goto retry; 272 * } else if (err) { 273 * goto unlock; 274 * } 275 * 276 * err = driver_bind_range(gpusvm, drange); 277 * if (err == -EAGAIN) // CPU mappings changed 278 * goto retry 279 * 280 * unlock: 281 * driver_svm_unlock(); 282 * return err; 283 * } 284 * 285 * 2) Garbage Collector 286 * 287 * .. code-block:: c 288 * 289 * // The driver owns the drm_gpusvm_pages lifecycle. ops->range_free is 290 * // the final fallback: drm_gpusvm_free_pages() unmaps any 291 * // lingering DMA mapping and a no-op if already unmapped and frees the 292 * // dma_addr array. The normal flow is to DMA unmap before 293 * // drm_gpusvm_range_remove() (before the range leaves the tree). 294 * void driver_range_free(struct drm_gpusvm_range *range) 295 * { 296 * struct driver_range *drange = 297 * container_of(range, struct driver_range, base); 298 * 299 * drm_gpusvm_free_pages(range->gpusvm, &drange->pages, 300 * drm_gpusvm_range_size(range) >> PAGE_SHIFT); 301 * kfree(drange); 302 * } 303 * 304 * void __driver_garbage_collector(struct drm_gpusvm *gpusvm, 305 * struct drm_gpusvm_range *range) 306 * { 307 * assert_driver_svm_locked(gpusvm); 308 * 309 * // Partial unmap, migrate any remaining device memory pages back to RAM 310 * if (range->flags.partial_unmap) 311 * drm_gpusvm_range_evict(gpusvm, range); 312 * 313 * driver_unbind_range(range); 314 * // The pages must be DMA unmapped before drm_gpusvm_range_remove() 315 * // , so a range is never off the MMU interval tree while still DMA 316 * // mapped as the original drmsvm design flow. Otherwise a concurrent CPU 317 * // munmap's notifier could miss this range and free pages still mapped 318 * // for device DMA. This is the normal unmap point. 319 * drm_gpusvm_unmap_pages(gpusvm, &drange->pages, 320 * drm_gpusvm_range_size(range) >> PAGE_SHIFT, 321 * &(struct drm_gpusvm_ctx){ .in_notifier = false }); 322 * drm_gpusvm_range_remove(gpusvm, range); 323 * } 324 * 325 * void driver_garbage_collector(struct drm_gpusvm *gpusvm) 326 * { 327 * assert_driver_svm_locked(gpusvm); 328 * 329 * for_each_range_in_garbage_collector(gpusvm, range) 330 * __driver_garbage_collector(gpusvm, range); 331 * } 332 * 333 * 3) Notifier callback 334 * 335 * .. code-block:: c 336 * 337 * void driver_invalidation(struct drm_gpusvm *gpusvm, 338 * struct drm_gpusvm_notifier *notifier, 339 * const struct mmu_notifier_range *mmu_range) 340 * { 341 * struct drm_gpusvm_ctx ctx = { .in_notifier = true, }; 342 * struct drm_gpusvm_range *range = NULL; 343 * struct driver_range *drange; 344 * 345 * driver_invalidate_device_pages(gpusvm, mmu_range->start, mmu_range->end); 346 * 347 * drm_gpusvm_for_each_range(range, notifier, mmu_range->start, 348 * mmu_range->end) { 349 * drange = container_of(range, struct driver_range, base); 350 * 351 * drm_gpusvm_unmap_pages(gpusvm, &drange->pages, 352 * drm_gpusvm_range_size(range) >> PAGE_SHIFT, 353 * &ctx); 354 * 355 * if (mmu_range->event != MMU_NOTIFY_UNMAP) 356 * continue; 357 * 358 * drm_gpusvm_range_set_unmapped(range, &drange->pages, 1, mmu_range); 359 * driver_garbage_collector_add(gpusvm, range); 360 * } 361 * } 362 */ 363 364 /** 365 * npages_in_range() - Calculate the number of pages in a given range 366 * @start: The start address of the range 367 * @end: The end address of the range 368 * 369 * This macro calculates the number of pages in a given memory range, 370 * specified by the start and end addresses. It divides the difference 371 * between the end and start addresses by the page size (PAGE_SIZE) to 372 * determine the number of pages in the range. 373 * 374 * Return: The number of pages in the specified range. 375 */ 376 static unsigned long 377 npages_in_range(unsigned long start, unsigned long end) 378 { 379 return (end - start) >> PAGE_SHIFT; 380 } 381 382 /** 383 * drm_gpusvm_notifier_find() - Find GPU SVM notifier from GPU SVM 384 * @gpusvm: Pointer to the GPU SVM structure. 385 * @start: Start address of the notifier 386 * @end: End address of the notifier 387 * 388 * Return: A pointer to the drm_gpusvm_notifier if found or NULL 389 */ 390 struct drm_gpusvm_notifier * 391 drm_gpusvm_notifier_find(struct drm_gpusvm *gpusvm, unsigned long start, 392 unsigned long end) 393 { 394 struct interval_tree_node *itree; 395 396 itree = interval_tree_iter_first(&gpusvm->root, start, end - 1); 397 398 if (itree) 399 return container_of(itree, struct drm_gpusvm_notifier, itree); 400 else 401 return NULL; 402 } 403 EXPORT_SYMBOL_GPL(drm_gpusvm_notifier_find); 404 405 /** 406 * drm_gpusvm_range_find() - Find GPU SVM range from GPU SVM notifier 407 * @notifier: Pointer to the GPU SVM notifier structure. 408 * @start: Start address of the range 409 * @end: End address of the range 410 * 411 * Return: A pointer to the drm_gpusvm_range if found or NULL 412 */ 413 struct drm_gpusvm_range * 414 drm_gpusvm_range_find(struct drm_gpusvm_notifier *notifier, unsigned long start, 415 unsigned long end) 416 { 417 struct interval_tree_node *itree; 418 419 itree = interval_tree_iter_first(¬ifier->root, start, end - 1); 420 421 if (itree) 422 return container_of(itree, struct drm_gpusvm_range, itree); 423 else 424 return NULL; 425 } 426 EXPORT_SYMBOL_GPL(drm_gpusvm_range_find); 427 428 /** 429 * drm_gpusvm_notifier_invalidate() - Invalidate a GPU SVM notifier. 430 * @mni: Pointer to the mmu_interval_notifier structure. 431 * @mmu_range: Pointer to the mmu_notifier_range structure. 432 * @cur_seq: Current sequence number. 433 * 434 * This function serves as a generic MMU notifier for GPU SVM. It sets the MMU 435 * notifier sequence number and calls the driver invalidate vfunc under 436 * gpusvm->notifier_lock. 437 * 438 * Return: true if the operation succeeds, false otherwise. 439 */ 440 static bool 441 drm_gpusvm_notifier_invalidate(struct mmu_interval_notifier *mni, 442 const struct mmu_notifier_range *mmu_range, 443 unsigned long cur_seq) 444 { 445 struct drm_gpusvm_notifier *notifier = 446 container_of(mni, typeof(*notifier), notifier); 447 struct drm_gpusvm *gpusvm = notifier->gpusvm; 448 449 if (!mmu_notifier_range_blockable(mmu_range)) 450 return false; 451 452 down_write(&gpusvm->notifier_lock); 453 mmu_interval_set_seq(mni, cur_seq); 454 gpusvm->ops->invalidate(gpusvm, notifier, mmu_range); 455 up_write(&gpusvm->notifier_lock); 456 457 return true; 458 } 459 460 /* 461 * drm_gpusvm_notifier_ops - MMU interval notifier operations for GPU SVM 462 */ 463 static const struct mmu_interval_notifier_ops drm_gpusvm_notifier_ops = { 464 .invalidate = drm_gpusvm_notifier_invalidate, 465 }; 466 467 /** 468 * drm_gpusvm_init() - Initialize the GPU SVM. 469 * @gpusvm: Pointer to the GPU SVM structure. 470 * @name: Name of the GPU SVM. 471 * @mm: Pointer to the mm_struct for the address space. 472 * @mm_start: Start address of GPU SVM. 473 * @mm_range: Range of the GPU SVM. 474 * @notifier_size: Size of individual notifiers. 475 * @ops: Pointer to the operations structure for GPU SVM. 476 * @chunk_sizes: Pointer to the array of chunk sizes used in range allocation. 477 * Entries should be powers of 2 in descending order with last 478 * entry being SZ_4K. 479 * @num_chunks: Number of chunks. 480 * 481 * This function initializes the GPU SVM. 482 * 483 * Note: If only using the simple drm_gpusvm_pages API (get/unmap/free), 484 * then only @gpusvm and @name are expected. The @drm drm_device for dma 485 * mappings is bound per-pages via drm_gpusvm_init_pages() before the first 486 * drm_gpusvm_get_pages() call. However, the same base 487 * @gpusvm can also be used with both modes together in which case the full 488 * setup is needed, where the core drm_gpusvm_pages API will simply never use 489 * the other fields. 490 * 491 * Return: 0 on success, a negative error code on failure. 492 */ 493 int drm_gpusvm_init(struct drm_gpusvm *gpusvm, 494 const char *name, 495 struct mm_struct *mm, 496 unsigned long mm_start, unsigned long mm_range, 497 unsigned long notifier_size, 498 const struct drm_gpusvm_ops *ops, 499 const unsigned long *chunk_sizes, int num_chunks) 500 { 501 if (mm) { 502 if (!ops->invalidate || !num_chunks) 503 return -EINVAL; 504 mmgrab(mm); 505 } else { 506 /* No full SVM mode, only core drm_gpusvm_pages API. */ 507 if (ops || num_chunks || mm_range || notifier_size) 508 return -EINVAL; 509 } 510 511 gpusvm->name = name; 512 gpusvm->mm = mm; 513 gpusvm->mm_start = mm_start; 514 gpusvm->mm_range = mm_range; 515 gpusvm->notifier_size = notifier_size; 516 gpusvm->ops = ops; 517 gpusvm->chunk_sizes = chunk_sizes; 518 gpusvm->num_chunks = num_chunks; 519 520 gpusvm->root = RB_ROOT_CACHED; 521 INIT_LIST_HEAD(&gpusvm->notifier_list); 522 523 init_rwsem(&gpusvm->notifier_lock); 524 525 fs_reclaim_acquire(GFP_KERNEL); 526 might_lock(&gpusvm->notifier_lock); 527 fs_reclaim_release(GFP_KERNEL); 528 529 #ifdef CONFIG_LOCKDEP 530 gpusvm->lock_dep_map = NULL; 531 #endif 532 533 return 0; 534 } 535 EXPORT_SYMBOL_GPL(drm_gpusvm_init); 536 537 /** 538 * to_drm_gpusvm_notifier() - retrieve the container struct for a given rbtree node 539 * @node: a pointer to the rbtree node embedded within a drm_gpusvm_notifier struct 540 * 541 * Return: A pointer to the containing drm_gpusvm_notifier structure. 542 */ 543 static struct drm_gpusvm_notifier *to_drm_gpusvm_notifier(struct rb_node *node) 544 { 545 return container_of(node, struct drm_gpusvm_notifier, itree.rb); 546 } 547 548 /** 549 * drm_gpusvm_notifier_insert() - Insert GPU SVM notifier 550 * @gpusvm: Pointer to the GPU SVM structure 551 * @notifier: Pointer to the GPU SVM notifier structure 552 * 553 * This function inserts the GPU SVM notifier into the GPU SVM RB tree and list. 554 */ 555 static void drm_gpusvm_notifier_insert(struct drm_gpusvm *gpusvm, 556 struct drm_gpusvm_notifier *notifier) 557 { 558 struct rb_node *node; 559 struct list_head *head; 560 561 interval_tree_insert(¬ifier->itree, &gpusvm->root); 562 563 node = rb_prev(¬ifier->itree.rb); 564 if (node) 565 head = &(to_drm_gpusvm_notifier(node))->entry; 566 else 567 head = &gpusvm->notifier_list; 568 569 list_add(¬ifier->entry, head); 570 } 571 572 /** 573 * drm_gpusvm_notifier_remove() - Remove GPU SVM notifier 574 * @gpusvm: Pointer to the GPU SVM tructure 575 * @notifier: Pointer to the GPU SVM notifier structure 576 * 577 * This function removes the GPU SVM notifier from the GPU SVM RB tree and list. 578 */ 579 static void drm_gpusvm_notifier_remove(struct drm_gpusvm *gpusvm, 580 struct drm_gpusvm_notifier *notifier) 581 { 582 interval_tree_remove(¬ifier->itree, &gpusvm->root); 583 list_del(¬ifier->entry); 584 } 585 586 /** 587 * drm_gpusvm_fini() - Finalize the GPU SVM. 588 * @gpusvm: Pointer to the GPU SVM structure. 589 * 590 * This function finalizes the GPU SVM by cleaning up any remaining ranges and 591 * notifiers, and dropping a reference to struct MM. 592 */ 593 void drm_gpusvm_fini(struct drm_gpusvm *gpusvm) 594 { 595 struct drm_gpusvm_notifier *notifier, *next; 596 597 drm_gpusvm_for_each_notifier_safe(notifier, next, gpusvm, 0, LONG_MAX) { 598 struct drm_gpusvm_range *range, *__next; 599 600 /* 601 * Remove notifier first to avoid racing with any invalidation 602 */ 603 mmu_interval_notifier_remove(¬ifier->notifier); 604 notifier->flags.removed = true; 605 606 drm_gpusvm_for_each_range_safe(range, __next, notifier, 0, 607 LONG_MAX) 608 drm_gpusvm_range_remove(gpusvm, range); 609 } 610 611 if (gpusvm->mm) 612 mmdrop(gpusvm->mm); 613 WARN_ON(!RB_EMPTY_ROOT(&gpusvm->root.rb_root)); 614 } 615 EXPORT_SYMBOL_GPL(drm_gpusvm_fini); 616 617 /** 618 * drm_gpusvm_notifier_alloc() - Allocate GPU SVM notifier 619 * @gpusvm: Pointer to the GPU SVM structure 620 * @fault_addr: Fault address 621 * 622 * This function allocates and initializes the GPU SVM notifier structure. 623 * 624 * Return: Pointer to the allocated GPU SVM notifier on success, ERR_PTR() on failure. 625 */ 626 static struct drm_gpusvm_notifier * 627 drm_gpusvm_notifier_alloc(struct drm_gpusvm *gpusvm, unsigned long fault_addr) 628 { 629 struct drm_gpusvm_notifier *notifier; 630 631 if (gpusvm->ops->notifier_alloc) 632 notifier = gpusvm->ops->notifier_alloc(); 633 else 634 notifier = kzalloc_obj(*notifier); 635 636 if (!notifier) 637 return ERR_PTR(-ENOMEM); 638 639 notifier->gpusvm = gpusvm; 640 notifier->itree.start = ALIGN_DOWN(fault_addr, gpusvm->notifier_size); 641 notifier->itree.last = ALIGN(fault_addr + 1, gpusvm->notifier_size) - 1; 642 INIT_LIST_HEAD(¬ifier->entry); 643 notifier->root = RB_ROOT_CACHED; 644 INIT_LIST_HEAD(¬ifier->range_list); 645 646 return notifier; 647 } 648 649 /** 650 * drm_gpusvm_notifier_free() - Free GPU SVM notifier 651 * @gpusvm: Pointer to the GPU SVM structure 652 * @notifier: Pointer to the GPU SVM notifier structure 653 * 654 * This function frees the GPU SVM notifier structure. 655 */ 656 static void drm_gpusvm_notifier_free(struct drm_gpusvm *gpusvm, 657 struct drm_gpusvm_notifier *notifier) 658 { 659 WARN_ON(!RB_EMPTY_ROOT(¬ifier->root.rb_root)); 660 661 if (gpusvm->ops->notifier_free) 662 gpusvm->ops->notifier_free(notifier); 663 else 664 kfree(notifier); 665 } 666 667 /** 668 * to_drm_gpusvm_range() - retrieve the container struct for a given rbtree node 669 * @node: a pointer to the rbtree node embedded within a drm_gpusvm_range struct 670 * 671 * Return: A pointer to the containing drm_gpusvm_range structure. 672 */ 673 static struct drm_gpusvm_range *to_drm_gpusvm_range(struct rb_node *node) 674 { 675 return container_of(node, struct drm_gpusvm_range, itree.rb); 676 } 677 678 /** 679 * drm_gpusvm_range_insert() - Insert GPU SVM range 680 * @notifier: Pointer to the GPU SVM notifier structure 681 * @range: Pointer to the GPU SVM range structure 682 * 683 * This function inserts the GPU SVM range into the notifier RB tree and list. 684 */ 685 static void drm_gpusvm_range_insert(struct drm_gpusvm_notifier *notifier, 686 struct drm_gpusvm_range *range) 687 { 688 struct rb_node *node; 689 struct list_head *head; 690 691 drm_gpusvm_notifier_lock(notifier->gpusvm); 692 interval_tree_insert(&range->itree, ¬ifier->root); 693 694 node = rb_prev(&range->itree.rb); 695 if (node) 696 head = &(to_drm_gpusvm_range(node))->entry; 697 else 698 head = ¬ifier->range_list; 699 700 list_add(&range->entry, head); 701 drm_gpusvm_notifier_unlock(notifier->gpusvm); 702 } 703 704 /** 705 * __drm_gpusvm_range_remove() - Remove GPU SVM range 706 * @notifier: Pointer to the GPU SVM notifier structure 707 * @range: Pointer to the GPU SVM range structure 708 * 709 * This macro removes the GPU SVM range from the notifier RB tree and list. 710 */ 711 static void __drm_gpusvm_range_remove(struct drm_gpusvm_notifier *notifier, 712 struct drm_gpusvm_range *range) 713 { 714 interval_tree_remove(&range->itree, ¬ifier->root); 715 list_del(&range->entry); 716 } 717 718 /** 719 * drm_gpusvm_range_alloc() - Allocate GPU SVM range 720 * @gpusvm: Pointer to the GPU SVM structure 721 * @notifier: Pointer to the GPU SVM notifier structure 722 * @fault_addr: Fault address 723 * @chunk_size: Chunk size 724 * @migrate_devmem: Flag indicating whether to migrate device memory 725 * 726 * This function allocates and initializes the GPU SVM range structure. 727 * 728 * Return: Pointer to the allocated GPU SVM range on success, ERR_PTR() on failure. 729 */ 730 static struct drm_gpusvm_range * 731 drm_gpusvm_range_alloc(struct drm_gpusvm *gpusvm, 732 struct drm_gpusvm_notifier *notifier, 733 unsigned long fault_addr, unsigned long chunk_size, 734 bool migrate_devmem) 735 { 736 struct drm_gpusvm_range *range; 737 738 if (gpusvm->ops->range_alloc) 739 range = gpusvm->ops->range_alloc(gpusvm); 740 else 741 range = kzalloc_obj(*range); 742 743 if (!range) 744 return ERR_PTR(-ENOMEM); 745 746 kref_init(&range->refcount); 747 range->gpusvm = gpusvm; 748 range->notifier = notifier; 749 range->itree.start = ALIGN_DOWN(fault_addr, chunk_size); 750 range->itree.last = ALIGN(fault_addr + 1, chunk_size) - 1; 751 INIT_LIST_HEAD(&range->entry); 752 range->flags.migrate_devmem = migrate_devmem ? 1 : 0; 753 754 return range; 755 } 756 757 /** 758 * drm_gpusvm_hmm_pfn_to_order() - Get the largest CPU mapping order. 759 * @hmm_pfn: The current hmm_pfn. 760 * @hmm_pfn_index: Index of the @hmm_pfn within the pfn array. 761 * @npages: Number of pages within the pfn array i.e the hmm range size. 762 * 763 * To allow skipping PFNs with the same flags (like when they belong to 764 * the same huge PTE) when looping over the pfn array, take a given a hmm_pfn, 765 * and return the largest order that will fit inside the CPU PTE, but also 766 * crucially accounting for the original hmm range boundaries. 767 * 768 * Return: The largest order that will safely fit within the size of the hmm_pfn 769 * CPU PTE. 770 */ 771 static unsigned int drm_gpusvm_hmm_pfn_to_order(unsigned long hmm_pfn, 772 unsigned long hmm_pfn_index, 773 unsigned long npages) 774 { 775 unsigned long size; 776 777 size = 1UL << hmm_pfn_to_map_order(hmm_pfn); 778 size -= (hmm_pfn & ~HMM_PFN_FLAGS) & (size - 1); 779 hmm_pfn_index += size; 780 if (hmm_pfn_index > npages) 781 size -= (hmm_pfn_index - npages); 782 783 return ilog2(size); 784 } 785 786 /** 787 * drm_gpusvm_check_pages() - Check pages 788 * @gpusvm: Pointer to the GPU SVM structure 789 * @notifier: Pointer to the GPU SVM notifier structure 790 * @start: Start address 791 * @end: End address 792 * @dev_private_owner: The device private page owner 793 * 794 * Check if pages between start and end have been faulted in on the CPU. Use to 795 * prevent migration of pages without CPU backing store. 796 * 797 * Return: True if pages have been faulted into CPU, False otherwise 798 */ 799 static bool drm_gpusvm_check_pages(struct drm_gpusvm *gpusvm, 800 struct drm_gpusvm_notifier *notifier, 801 unsigned long start, unsigned long end, 802 void *dev_private_owner) 803 { 804 struct hmm_range hmm_range = { 805 .default_flags = 0, 806 .notifier = ¬ifier->notifier, 807 .start = start, 808 .end = end, 809 .dev_private_owner = dev_private_owner, 810 }; 811 unsigned long timeout = 812 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 813 unsigned long *pfns; 814 unsigned long npages = npages_in_range(start, end); 815 int err, i; 816 817 mmap_assert_locked(gpusvm->mm); 818 819 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); 820 if (!pfns) 821 return false; 822 823 hmm_range.notifier_seq = mmu_interval_read_begin(¬ifier->notifier); 824 hmm_range.hmm_pfns = pfns; 825 826 while (true) { 827 err = hmm_range_fault(&hmm_range); 828 if (err == -EBUSY) { 829 if (time_after(jiffies, timeout)) 830 break; 831 832 hmm_range.notifier_seq = 833 mmu_interval_read_begin(¬ifier->notifier); 834 continue; 835 } 836 break; 837 } 838 if (err) 839 goto err_free; 840 841 for (i = 0; i < npages;) { 842 if (!(pfns[i] & HMM_PFN_VALID)) { 843 err = -EFAULT; 844 goto err_free; 845 } 846 i += 0x1 << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); 847 } 848 849 err_free: 850 kvfree(pfns); 851 return err ? false : true; 852 } 853 854 /** 855 * drm_gpusvm_scan_mm() - Check the migration state of a drm_gpusvm_range 856 * @range: Pointer to the struct drm_gpusvm_range to check. 857 * @dev_private_owner: The struct dev_private_owner to use to determine 858 * compatible device-private pages. 859 * @pagemap: The struct dev_pagemap pointer to use for pagemap-specific 860 * checks. 861 * 862 * Scan the CPU address space corresponding to @range and return the 863 * current migration state. Note that the result may be invalid as 864 * soon as the function returns. It's an advisory check. 865 * 866 * TODO: Bail early and call hmm_range_fault() for subranges. 867 * 868 * Return: See &enum drm_gpusvm_scan_result. 869 */ 870 enum drm_gpusvm_scan_result drm_gpusvm_scan_mm(struct drm_gpusvm_range *range, 871 void *dev_private_owner, 872 const struct dev_pagemap *pagemap) 873 { 874 struct mmu_interval_notifier *notifier = &range->notifier->notifier; 875 unsigned long start = drm_gpusvm_range_start(range); 876 unsigned long end = drm_gpusvm_range_end(range); 877 struct hmm_range hmm_range = { 878 .default_flags = 0, 879 .notifier = notifier, 880 .start = start, 881 .end = end, 882 .dev_private_owner = dev_private_owner, 883 }; 884 unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 885 enum drm_gpusvm_scan_result state = DRM_GPUSVM_SCAN_UNPOPULATED, new_state; 886 unsigned long *pfns; 887 unsigned long npages = npages_in_range(start, end); 888 const struct dev_pagemap *other = NULL; 889 int err, i; 890 891 pfns = kvcalloc(npages, sizeof(*pfns), GFP_KERNEL); 892 if (!pfns) 893 return DRM_GPUSVM_SCAN_UNPOPULATED; 894 895 hmm_range.hmm_pfns = pfns; 896 897 retry: 898 err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout); 899 if (err) 900 goto err_free; 901 902 drm_gpusvm_notifier_lock(range->gpusvm); 903 if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) { 904 drm_gpusvm_notifier_unlock(range->gpusvm); 905 goto retry; 906 } 907 908 for (i = 0; i < npages;) { 909 struct page *page; 910 const struct dev_pagemap *cur = NULL; 911 912 if (!(pfns[i] & HMM_PFN_VALID)) { 913 state = DRM_GPUSVM_SCAN_UNPOPULATED; 914 break; 915 } 916 917 page = hmm_pfn_to_page(pfns[i]); 918 if (is_device_private_page(page) || 919 is_device_coherent_page(page)) 920 cur = page_pgmap(page); 921 922 if (cur == pagemap) { 923 new_state = DRM_GPUSVM_SCAN_EQUAL; 924 } else if (cur && (cur == other || !other)) { 925 new_state = DRM_GPUSVM_SCAN_OTHER; 926 other = cur; 927 } else if (cur) { 928 new_state = DRM_GPUSVM_SCAN_MIXED_DEVICE; 929 } else { 930 new_state = DRM_GPUSVM_SCAN_SYSTEM; 931 } 932 933 /* 934 * TODO: Could use an array for state 935 * transitions, and caller might want it 936 * to bail early for some results. 937 */ 938 if (state == DRM_GPUSVM_SCAN_UNPOPULATED) { 939 state = new_state; 940 } else if (state != new_state) { 941 if (new_state == DRM_GPUSVM_SCAN_SYSTEM || 942 state == DRM_GPUSVM_SCAN_SYSTEM) 943 state = DRM_GPUSVM_SCAN_MIXED; 944 else if (state != DRM_GPUSVM_SCAN_MIXED) 945 state = DRM_GPUSVM_SCAN_MIXED_DEVICE; 946 } 947 948 i += 1ul << drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); 949 } 950 951 drm_gpusvm_notifier_unlock(range->gpusvm); 952 953 err_free: 954 kvfree(pfns); 955 return state; 956 } 957 EXPORT_SYMBOL(drm_gpusvm_scan_mm); 958 959 /** 960 * drm_gpusvm_range_chunk_size() - Determine chunk size for GPU SVM range 961 * @gpusvm: Pointer to the GPU SVM structure 962 * @notifier: Pointer to the GPU SVM notifier structure 963 * @vas: Pointer to the virtual memory area structure 964 * @fault_addr: Fault address 965 * @gpuva_start: Start address of GPUVA which mirrors CPU 966 * @gpuva_end: End address of GPUVA which mirrors CPU 967 * @check_pages_threshold: Check CPU pages for present threshold 968 * @dev_private_owner: The device private page owner 969 * 970 * This function determines the chunk size for the GPU SVM range based on the 971 * fault address, GPU SVM chunk sizes, existing GPU SVM ranges, and the virtual 972 * memory area boundaries. 973 * 974 * Return: Chunk size on success, LONG_MAX on failure. 975 */ 976 static unsigned long 977 drm_gpusvm_range_chunk_size(struct drm_gpusvm *gpusvm, 978 struct drm_gpusvm_notifier *notifier, 979 struct vm_area_struct *vas, 980 unsigned long fault_addr, 981 unsigned long gpuva_start, 982 unsigned long gpuva_end, 983 unsigned long check_pages_threshold, 984 void *dev_private_owner) 985 { 986 unsigned long start, end; 987 int i = 0; 988 989 retry: 990 for (; i < gpusvm->num_chunks; ++i) { 991 start = ALIGN_DOWN(fault_addr, gpusvm->chunk_sizes[i]); 992 end = ALIGN(fault_addr + 1, gpusvm->chunk_sizes[i]); 993 994 if (start >= vas->vm_start && end <= vas->vm_end && 995 start >= drm_gpusvm_notifier_start(notifier) && 996 end <= drm_gpusvm_notifier_end(notifier) && 997 start >= gpuva_start && end <= gpuva_end) 998 break; 999 } 1000 1001 if (i == gpusvm->num_chunks) 1002 return LONG_MAX; 1003 1004 /* 1005 * If allocation more than page, ensure not to overlap with existing 1006 * ranges. 1007 */ 1008 if (end - start != SZ_4K) { 1009 struct drm_gpusvm_range *range; 1010 1011 range = drm_gpusvm_range_find(notifier, start, end); 1012 if (range) { 1013 ++i; 1014 goto retry; 1015 } 1016 1017 /* 1018 * XXX: Only create range on pages CPU has faulted in. Without 1019 * this check, or prefault, on BMG 'xe_exec_system_allocator --r 1020 * process-many-malloc' fails. In the failure case, each process 1021 * mallocs 16k but the CPU VMA is ~128k which results in 64k SVM 1022 * ranges. When migrating the SVM ranges, some processes fail in 1023 * drm_pagemap_migrate_to_devmem with 'migrate.cpages != npages' 1024 * and then upon drm_gpusvm_get_pages device pages from 1025 * other processes are collected + faulted in which creates all 1026 * sorts of problems. Unsure exactly how this happening, also 1027 * problem goes away if 'xe_exec_system_allocator --r 1028 * process-many-malloc' mallocs at least 64k at a time. 1029 */ 1030 if (end - start <= check_pages_threshold && 1031 !drm_gpusvm_check_pages(gpusvm, notifier, start, end, dev_private_owner)) { 1032 ++i; 1033 goto retry; 1034 } 1035 } 1036 1037 return end - start; 1038 } 1039 1040 #ifdef CONFIG_LOCKDEP 1041 /** 1042 * drm_gpusvm_driver_lock_held() - Assert GPU SVM driver lock is held 1043 * @gpusvm: Pointer to the GPU SVM structure. 1044 * 1045 * Ensure driver lock is held. 1046 */ 1047 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm) 1048 { 1049 if ((gpusvm)->lock_dep_map) 1050 lockdep_assert(lock_is_held_type((gpusvm)->lock_dep_map, 0)); 1051 } 1052 #else 1053 static void drm_gpusvm_driver_lock_held(struct drm_gpusvm *gpusvm) 1054 { 1055 } 1056 #endif 1057 1058 /** 1059 * drm_gpusvm_find_vma_start() - Find start address for first VMA in range 1060 * @gpusvm: Pointer to the GPU SVM structure 1061 * @start: The inclusive start user address. 1062 * @end: The exclusive end user address. 1063 * 1064 * Returns: The start address of first VMA within the provided range, 1065 * ULONG_MAX otherwise. Assumes start_addr < end_addr. 1066 */ 1067 unsigned long 1068 drm_gpusvm_find_vma_start(struct drm_gpusvm *gpusvm, 1069 unsigned long start, 1070 unsigned long end) 1071 { 1072 struct mm_struct *mm = gpusvm->mm; 1073 struct vm_area_struct *vma; 1074 unsigned long addr = ULONG_MAX; 1075 1076 if (!mmget_not_zero(mm)) 1077 return addr; 1078 1079 mmap_read_lock(mm); 1080 1081 vma = find_vma_intersection(mm, start, end); 1082 if (vma) 1083 addr = vma->vm_start; 1084 1085 mmap_read_unlock(mm); 1086 mmput(mm); 1087 1088 return addr; 1089 } 1090 EXPORT_SYMBOL_GPL(drm_gpusvm_find_vma_start); 1091 1092 /** 1093 * drm_gpusvm_range_find_or_insert() - Find or insert GPU SVM range 1094 * @gpusvm: Pointer to the GPU SVM structure 1095 * @fault_addr: Fault address 1096 * @gpuva_start: Start address of GPUVA which mirrors CPU 1097 * @gpuva_end: End address of GPUVA which mirrors CPU 1098 * @ctx: GPU SVM context 1099 * 1100 * This function finds or inserts a newly allocated a GPU SVM range based on the 1101 * fault address. Caller must hold a lock to protect range lookup and insertion. 1102 * 1103 * Return: Pointer to the GPU SVM range on success, ERR_PTR() on failure. 1104 */ 1105 struct drm_gpusvm_range * 1106 drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm, 1107 unsigned long fault_addr, 1108 unsigned long gpuva_start, 1109 unsigned long gpuva_end, 1110 const struct drm_gpusvm_ctx *ctx) 1111 { 1112 struct drm_gpusvm_notifier *notifier; 1113 struct drm_gpusvm_range *range; 1114 struct mm_struct *mm = gpusvm->mm; 1115 struct vm_area_struct *vas; 1116 bool notifier_alloc = false; 1117 unsigned long chunk_size; 1118 int err; 1119 bool migrate_devmem; 1120 1121 drm_gpusvm_driver_lock_held(gpusvm); 1122 1123 if (fault_addr < gpusvm->mm_start || 1124 fault_addr > gpusvm->mm_start + gpusvm->mm_range) 1125 return ERR_PTR(-EINVAL); 1126 1127 if (!mmget_not_zero(mm)) 1128 return ERR_PTR(-EFAULT); 1129 1130 notifier = drm_gpusvm_notifier_find(gpusvm, fault_addr, fault_addr + 1); 1131 if (!notifier) { 1132 notifier = drm_gpusvm_notifier_alloc(gpusvm, fault_addr); 1133 if (IS_ERR(notifier)) { 1134 err = PTR_ERR(notifier); 1135 goto err_mmunlock; 1136 } 1137 notifier_alloc = true; 1138 err = mmu_interval_notifier_insert(¬ifier->notifier, 1139 mm, 1140 drm_gpusvm_notifier_start(notifier), 1141 drm_gpusvm_notifier_size(notifier), 1142 &drm_gpusvm_notifier_ops); 1143 if (err) 1144 goto err_notifier; 1145 } 1146 1147 mmap_read_lock(mm); 1148 1149 vas = vma_lookup(mm, fault_addr); 1150 if (!vas) { 1151 err = -ENOENT; 1152 goto err_notifier_remove; 1153 } 1154 1155 if (!ctx->read_only && !(vas->vm_flags & VM_WRITE)) { 1156 err = -EPERM; 1157 goto err_notifier_remove; 1158 } 1159 1160 if (vas->vm_flags & (VM_IO | VM_PFNMAP)) { 1161 err = -EIO; 1162 goto err_notifier_remove; 1163 } 1164 1165 range = drm_gpusvm_range_find(notifier, fault_addr, fault_addr + 1); 1166 if (range) 1167 goto out_mmunlock; 1168 /* 1169 * XXX: Short-circuiting migration based on migrate_vma_* current 1170 * limitations. If/when migrate_vma_* add more support, this logic will 1171 * have to change. 1172 */ 1173 migrate_devmem = ctx->devmem_possible && 1174 vma_is_anonymous(vas) && !is_vm_hugetlb_page(vas); 1175 1176 chunk_size = drm_gpusvm_range_chunk_size(gpusvm, notifier, vas, 1177 fault_addr, gpuva_start, 1178 gpuva_end, 1179 ctx->check_pages_threshold, 1180 ctx->device_private_page_owner); 1181 if (chunk_size == LONG_MAX) { 1182 err = -EINVAL; 1183 goto err_notifier_remove; 1184 } 1185 1186 range = drm_gpusvm_range_alloc(gpusvm, notifier, fault_addr, chunk_size, 1187 migrate_devmem); 1188 if (IS_ERR(range)) { 1189 err = PTR_ERR(range); 1190 goto err_notifier_remove; 1191 } 1192 1193 drm_gpusvm_range_insert(notifier, range); 1194 if (notifier_alloc) 1195 drm_gpusvm_notifier_insert(gpusvm, notifier); 1196 1197 out_mmunlock: 1198 mmap_read_unlock(mm); 1199 mmput(mm); 1200 1201 return range; 1202 1203 err_notifier_remove: 1204 mmap_read_unlock(mm); 1205 if (notifier_alloc) 1206 mmu_interval_notifier_remove(¬ifier->notifier); 1207 err_notifier: 1208 if (notifier_alloc) 1209 drm_gpusvm_notifier_free(gpusvm, notifier); 1210 err_mmunlock: 1211 mmput(mm); 1212 return ERR_PTR(err); 1213 } 1214 EXPORT_SYMBOL_GPL(drm_gpusvm_range_find_or_insert); 1215 1216 /** 1217 * __drm_gpusvm_unmap_pages() - Unmap pages associated with GPU SVM pages (internal) 1218 * @gpusvm: Pointer to the GPU SVM structure 1219 * @svm_pages: Pointer to the GPU SVM pages structure 1220 * @npages: Number of pages to unmap 1221 * 1222 * This function unmap pages associated with a GPU SVM pages struct. Assumes and 1223 * asserts correct locking is in place when called. 1224 */ 1225 static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, 1226 struct drm_gpusvm_pages *svm_pages, 1227 unsigned long npages) 1228 { 1229 struct drm_pagemap *dpagemap = svm_pages->dpagemap; 1230 struct device *dev; 1231 unsigned long i, j; 1232 1233 lockdep_assert_held(&gpusvm->notifier_lock); 1234 1235 if (!svm_pages->drm) 1236 return; 1237 1238 dev = svm_pages->drm->dev; 1239 1240 if (svm_pages->flags.has_dma_mapping) { 1241 struct drm_gpusvm_pages_flags flags = { 1242 .__flags = svm_pages->flags.__flags, 1243 }; 1244 const struct drm_pagemap_addr *addrs = 1245 drm_gpusvm_pages_first_dma(svm_pages, NULL); 1246 bool use_iova = dma_use_iova(&svm_pages->state); 1247 1248 /* 1249 * IOVA is reserved for the whole range but only the linked 1250 * system pages (state_offset bytes) need unlinking; free the 1251 * entire reservation to avoid leaking the device-page part. 1252 * On the error path state_offset is 0, so just free it. 1253 */ 1254 if (use_iova) { 1255 if (svm_pages->state_offset) 1256 dma_iova_unlink(dev, &svm_pages->state, 0, 1257 svm_pages->state_offset, 1258 addrs[0].dir, 0); 1259 dma_iova_free(dev, &svm_pages->state); 1260 } 1261 1262 /* 1263 * With IOVA and no device page the unlink above tore every 1264 * entry down, and that is also when the range may be folded 1265 * to one entry, which must not be walked per entry. dpagemap 1266 * is set before the first device_map(), so it is also right 1267 * on the error path, where the flags are not published yet. 1268 */ 1269 for (i = 0, j = 0; 1270 (!use_iova || dpagemap) && i < npages; j++) { 1271 const struct drm_pagemap_addr *addr = &addrs[j]; 1272 1273 if (addr->proto == DRM_INTERCONNECT_SYSTEM) { 1274 /* 1275 * Linked IOVA pages were already torn down by 1276 * the dma_iova_unlink()/dma_iova_free() above; 1277 * only the non-IOVA mappings need unmap here. 1278 */ 1279 if (!use_iova) 1280 dma_unmap_page(dev, 1281 addr->addr, 1282 PAGE_SIZE << addr->order, 1283 addr->dir); 1284 } else if (dpagemap && dpagemap->ops->device_unmap) 1285 dpagemap->ops->device_unmap(dpagemap, 1286 dev, addr); 1287 i += 1 << addr->order; 1288 } 1289 1290 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ 1291 flags.has_devmem_pages = false; 1292 flags.has_dma_mapping = false; 1293 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); 1294 1295 drm_pagemap_put(svm_pages->dpagemap); 1296 svm_pages->dpagemap = NULL; 1297 } 1298 } 1299 1300 /** 1301 * __drm_gpusvm_free_pages() - Free dma array associated with GPU SVM pages 1302 * @gpusvm: Pointer to the GPU SVM structure 1303 * @svm_pages: Pointer to the GPU SVM pages structure 1304 * 1305 * This function frees the dma address array associated with a GPU SVM range. 1306 */ 1307 static void __drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm, 1308 struct drm_gpusvm_pages *svm_pages) 1309 { 1310 lockdep_assert_held(&gpusvm->notifier_lock); 1311 1312 if (svm_pages->flags.inline_dma_mapping) { 1313 struct drm_gpusvm_pages_flags flags = { 1314 .__flags = svm_pages->flags.__flags, 1315 }; 1316 1317 svm_pages->inline_addr = (struct drm_pagemap_addr){}; 1318 flags.inline_dma_mapping = false; 1319 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ 1320 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); 1321 return; 1322 } 1323 1324 if (svm_pages->dma_addr) { 1325 kvfree(svm_pages->dma_addr); 1326 svm_pages->dma_addr = NULL; 1327 } 1328 } 1329 1330 /** 1331 * drm_gpusvm_free_pages() - Free dma-mapping associated with GPU SVM pages 1332 * struct 1333 * @gpusvm: Pointer to the GPU SVM structure 1334 * @svm_pages: Pointer to the GPU SVM pages structure 1335 * @npages: Number of mapped pages 1336 * 1337 * This function unmaps and frees the dma address array associated with a GPU 1338 * SVM pages struct. 1339 */ 1340 void drm_gpusvm_free_pages(struct drm_gpusvm *gpusvm, 1341 struct drm_gpusvm_pages *svm_pages, 1342 unsigned long npages) 1343 { 1344 drm_gpusvm_notifier_lock(gpusvm); 1345 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages); 1346 __drm_gpusvm_free_pages(gpusvm, svm_pages); 1347 drm_gpusvm_notifier_unlock(gpusvm); 1348 } 1349 EXPORT_SYMBOL_GPL(drm_gpusvm_free_pages); 1350 1351 /** 1352 * drm_gpusvm_range_remove() - Remove GPU SVM range 1353 * @gpusvm: Pointer to the GPU SVM structure 1354 * @range: Pointer to the GPU SVM range to be removed 1355 * 1356 * This function removes the specified GPU SVM range and also removes the parent 1357 * GPU SVM notifier if no more ranges remain in the notifier. The caller must 1358 * hold a lock to protect range and notifier removal. 1359 * 1360 * This function does not unmap or free the drm_gpusvm_pages, the driver owns 1361 * that lifecycle. The caller must DMA unmap the range's pages before calling 1362 * this function, so a range is never removed from the MMU interval tree while 1363 * still DMA mapped. Typically the driver calls drm_gpusvm_unmap_pages() first. 1364 * And the range_free callback's drm_gpusvm_free_pages() is a final fallback safe 1365 * net. 1366 */ 1367 void drm_gpusvm_range_remove(struct drm_gpusvm *gpusvm, 1368 struct drm_gpusvm_range *range) 1369 { 1370 struct drm_gpusvm_notifier *notifier; 1371 1372 drm_gpusvm_driver_lock_held(gpusvm); 1373 1374 notifier = drm_gpusvm_notifier_find(gpusvm, 1375 drm_gpusvm_range_start(range), 1376 drm_gpusvm_range_start(range) + 1); 1377 if (WARN_ON_ONCE(!notifier)) 1378 return; 1379 1380 drm_gpusvm_notifier_lock(gpusvm); 1381 __drm_gpusvm_range_remove(notifier, range); 1382 drm_gpusvm_notifier_unlock(gpusvm); 1383 1384 drm_gpusvm_range_put(range); 1385 1386 if (RB_EMPTY_ROOT(¬ifier->root.rb_root)) { 1387 if (!notifier->flags.removed) 1388 mmu_interval_notifier_remove(¬ifier->notifier); 1389 drm_gpusvm_notifier_remove(gpusvm, notifier); 1390 drm_gpusvm_notifier_free(gpusvm, notifier); 1391 } 1392 } 1393 EXPORT_SYMBOL_GPL(drm_gpusvm_range_remove); 1394 1395 /** 1396 * drm_gpusvm_range_get() - Get a reference to GPU SVM range 1397 * @range: Pointer to the GPU SVM range 1398 * 1399 * This function increments the reference count of the specified GPU SVM range. 1400 * 1401 * Return: Pointer to the GPU SVM range. 1402 */ 1403 struct drm_gpusvm_range * 1404 drm_gpusvm_range_get(struct drm_gpusvm_range *range) 1405 { 1406 kref_get(&range->refcount); 1407 1408 return range; 1409 } 1410 EXPORT_SYMBOL_GPL(drm_gpusvm_range_get); 1411 1412 /** 1413 * drm_gpusvm_range_destroy() - Destroy GPU SVM range 1414 * @refcount: Pointer to the reference counter embedded in the GPU SVM range 1415 * 1416 * This function destroys the specified GPU SVM range when its reference count 1417 * reaches zero. If a custom range-free function is provided, it is invoked to 1418 * free the range; otherwise, the range is deallocated using kfree(). 1419 */ 1420 static void drm_gpusvm_range_destroy(struct kref *refcount) 1421 { 1422 struct drm_gpusvm_range *range = 1423 container_of(refcount, struct drm_gpusvm_range, refcount); 1424 struct drm_gpusvm *gpusvm = range->gpusvm; 1425 1426 if (gpusvm->ops->range_free) 1427 gpusvm->ops->range_free(range); 1428 else 1429 kfree(range); 1430 } 1431 1432 /** 1433 * drm_gpusvm_range_put() - Put a reference to GPU SVM range 1434 * @range: Pointer to the GPU SVM range 1435 * 1436 * This function decrements the reference count of the specified GPU SVM range 1437 * and frees it when the count reaches zero. 1438 */ 1439 void drm_gpusvm_range_put(struct drm_gpusvm_range *range) 1440 { 1441 kref_put(&range->refcount, drm_gpusvm_range_destroy); 1442 } 1443 EXPORT_SYMBOL_GPL(drm_gpusvm_range_put); 1444 1445 /** 1446 * drm_gpusvm_pages_valid() - GPU SVM range pages valid 1447 * @gpusvm: Pointer to the GPU SVM structure 1448 * @svm_pages: Pointer to the GPU SVM pages structure 1449 * 1450 * This function determines if a GPU SVM range pages are valid. Expected be 1451 * called holding gpusvm->notifier_lock and as the last step before committing a 1452 * GPU binding. This is akin to a notifier seqno check in the HMM documentation 1453 * but due to wider notifiers (i.e., notifiers which span multiple ranges) this 1454 * function is required for finer grained checking (i.e., per range) if pages 1455 * are valid. 1456 * 1457 * Return: True if GPU SVM range has valid pages, False otherwise 1458 */ 1459 bool drm_gpusvm_pages_valid(struct drm_gpusvm *gpusvm, 1460 struct drm_gpusvm_pages *svm_pages) 1461 { 1462 lockdep_assert_held(&gpusvm->notifier_lock); 1463 1464 return svm_pages->flags.has_devmem_pages || svm_pages->flags.has_dma_mapping; 1465 } 1466 EXPORT_SYMBOL_GPL(drm_gpusvm_pages_valid); 1467 1468 /** 1469 * drm_gpusvm_pages_valid_unlocked() - GPU SVM pages valid unlocked 1470 * @gpusvm: Pointer to the GPU SVM structure 1471 * @svm_pages: Array of GPU SVM pages structures 1472 * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages 1473 * 1474 * This function determines if every GPU SVM pages instance is valid, resetting 1475 * every instance which is not so that get_pages() maps it afresh. It therefore 1476 * has to walk them all. Expected be called without holding 1477 * gpusvm->notifier_lock. 1478 * 1479 * Return: True if all GPU SVM pages are valid, False otherwise 1480 */ 1481 static bool drm_gpusvm_pages_valid_unlocked(struct drm_gpusvm *gpusvm, 1482 struct drm_gpusvm_pages *svm_pages, 1483 unsigned int num_pages) 1484 { 1485 bool pages_valid = true; 1486 unsigned int p; 1487 1488 drm_gpusvm_notifier_lock(gpusvm); 1489 for (p = 0; p < num_pages; ++p) { 1490 if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p])) 1491 continue; 1492 __drm_gpusvm_free_pages(gpusvm, &svm_pages[p]); 1493 pages_valid = false; 1494 } 1495 drm_gpusvm_notifier_unlock(gpusvm); 1496 1497 return pages_valid; 1498 } 1499 1500 /** 1501 * drm_gpusvm_pages_inlinable() - Whether the dma address can be inlined 1502 * @svm_pages: The SVM pages instance that was just mapped 1503 * @nentries: Number of entries the mapping loop produced 1504 * @npages: Number of pages in the CPU range 1505 * 1506 * A THP maps as one huge page, and an IOVA reservation links every page of 1507 * the range at the next offset, so the device addresses run contiguously from 1508 * entry 0. Either way one entry describes the whole range, so the dma_addr 1509 * array can be freed and the address kept inline. 1510 * 1511 * state_offset advances only on the IOVA branch, so reaching the full range 1512 * length proves no device page was mapped in between. Only single page 1513 * entries fold, so the order kept is 0 and describes the range truthfully. 1514 * Larger chunks, several huge pages among them, stay an array that is 1515 * already short and that a consumer places with one PTE each. 1516 * 1517 * Return: True if the mapping fits in a single drm_pagemap_addr. 1518 */ 1519 static bool drm_gpusvm_pages_inlinable(struct drm_gpusvm_pages *svm_pages, 1520 unsigned long nentries, 1521 unsigned long npages) 1522 { 1523 if (nentries == 1) 1524 return true; 1525 1526 return nentries == npages && dma_use_iova(&svm_pages->state) && 1527 svm_pages->state_offset == npages * PAGE_SIZE; 1528 } 1529 1530 /** 1531 * drm_gpusvm_walk_devmem() - Invoke the devmem callback across faulted pages 1532 * @gpusvm: Pointer to the GPU SVM structure 1533 * @pfns: The already-faulted pfn array (size @npages) 1534 * @npages: Number of pages in the CPU range 1535 * @ctx: GPU SVM context, with a non-NULL &drm_gpusvm_ctx.devmem_fn 1536 * 1537 * Invoke &drm_gpusvm_ctx.devmem_fn once per contiguous run of @pfns backed by 1538 * the same device-memory allocation. Must be called under the notifier lock. 1539 */ 1540 static void drm_gpusvm_walk_devmem(struct drm_gpusvm *gpusvm, 1541 unsigned long *pfns, 1542 unsigned long npages, 1543 const struct drm_gpusvm_ctx *ctx) 1544 { 1545 struct drm_pagemap_devmem *last = NULL; 1546 unsigned int order = 0; 1547 unsigned long i; 1548 1549 lockdep_assert_held(&gpusvm->notifier_lock); 1550 1551 for (i = 0; i < npages; i += 1 << order) { 1552 struct page *page = hmm_pfn_to_page(pfns[i]); 1553 struct drm_pagemap_devmem *devmem; 1554 1555 order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); 1556 1557 if (!is_device_private_page(page) && 1558 !is_device_coherent_page(page)) { 1559 last = NULL; 1560 continue; 1561 } 1562 1563 devmem = drm_pagemap_page_to_devmem(page); 1564 if (devmem == last) 1565 continue; 1566 1567 last = devmem; 1568 ctx->devmem_fn(devmem); 1569 } 1570 } 1571 1572 /** 1573 * drm_gpusvm_dma_map_pages() - Walk and DMA map one drm_gpusvm_pages instance 1574 * @gpusvm: Pointer to the GPU SVM structure 1575 * @svm_pages: The SVM pages instance to populate with dma-addresses 1576 * @pfns: The already-faulted pfn array (size @npages) 1577 * @npages: Number of pages in the CPU range 1578 * @ctx: GPU SVM context 1579 * @dma_dir: DMA data direction for the mappings 1580 * 1581 * Walk the faulted @pfns and map them into @svm_pages for DMA access through 1582 * its owning drm_device. Must be called under the notifier lock and only for 1583 * an instance without a live mapping. On failure this unwinds the partial 1584 * mapping of this instance before returning. 1585 * 1586 * Return: 0 on success, negative error code on failure. 1587 */ 1588 static int drm_gpusvm_dma_map_pages(struct drm_gpusvm *gpusvm, 1589 struct drm_gpusvm_pages *svm_pages, 1590 unsigned long *pfns, 1591 unsigned long npages, 1592 const struct drm_gpusvm_ctx *ctx, 1593 enum dma_data_direction dma_dir) 1594 { 1595 void *zdd = NULL; 1596 unsigned long i, j; 1597 unsigned long num_dma_mapped = 0; 1598 unsigned int order = 0; 1599 int err = 0; 1600 struct dev_pagemap *pagemap = NULL; 1601 struct drm_pagemap *dpagemap; 1602 struct drm_gpusvm_pages_flags flags; 1603 struct dma_iova_state *state = &svm_pages->state; 1604 1605 lockdep_assert_held(&gpusvm->notifier_lock); 1606 1607 *state = (struct dma_iova_state){}; 1608 svm_pages->state_offset = 0; 1609 1610 flags.__flags = svm_pages->flags.__flags; 1611 1612 for (i = 0, j = 0; i < npages; ++j) { 1613 struct page *page = hmm_pfn_to_page(pfns[i]); 1614 1615 order = drm_gpusvm_hmm_pfn_to_order(pfns[i], i, npages); 1616 if (is_device_private_page(page) || 1617 is_device_coherent_page(page)) { 1618 struct drm_pagemap_zdd *__zdd = 1619 drm_pagemap_page_zone_device_data(page); 1620 1621 if (!ctx->allow_mixed && 1622 zdd != __zdd && i > 0) { 1623 err = -EOPNOTSUPP; 1624 goto err_unmap; 1625 } 1626 zdd = __zdd; 1627 if (pagemap != page_pgmap(page)) { 1628 if (pagemap) { 1629 err = -EOPNOTSUPP; 1630 goto err_unmap; 1631 } 1632 1633 pagemap = page_pgmap(page); 1634 dpagemap = drm_pagemap_page_to_dpagemap(page); 1635 if (drm_WARN_ON(svm_pages->drm, !dpagemap)) { 1636 /* 1637 * Raced. This is not supposed to happen 1638 * since hmm_range_fault() should've migrated 1639 * this page to system. 1640 */ 1641 err = -EAGAIN; 1642 goto err_unmap; 1643 } 1644 1645 /* 1646 * Set the dpagemap as soon as the first 1647 * device page is mapped so the err_unmap path 1648 * can device_unmap() the device mappings that 1649 * have already been created. 1650 */ 1651 drm_pagemap_get(dpagemap); 1652 drm_pagemap_put(svm_pages->dpagemap); 1653 svm_pages->dpagemap = dpagemap; 1654 } 1655 svm_pages->dma_addr[j] = 1656 dpagemap->ops->device_map(dpagemap, 1657 svm_pages->drm->dev, 1658 page, order, 1659 dma_dir); 1660 if (dma_mapping_error(svm_pages->drm->dev, 1661 svm_pages->dma_addr[j].addr)) { 1662 err = -EFAULT; 1663 goto err_unmap; 1664 } 1665 } else { 1666 dma_addr_t addr; 1667 1668 if (is_zone_device_page(page) || 1669 (pagemap && !ctx->allow_mixed)) { 1670 err = -EOPNOTSUPP; 1671 goto err_unmap; 1672 } 1673 1674 if (ctx->devmem_only) { 1675 err = -EFAULT; 1676 goto err_unmap; 1677 } 1678 1679 if (!i) 1680 dma_iova_try_alloc(svm_pages->drm->dev, state, 1681 0, npages * PAGE_SIZE); 1682 1683 if (dma_use_iova(state)) { 1684 err = dma_iova_link(svm_pages->drm->dev, state, 1685 hmm_pfn_to_phys(pfns[i]), 1686 svm_pages->state_offset, 1687 PAGE_SIZE << order, 1688 dma_dir, 0); 1689 if (err) 1690 goto err_unmap; 1691 1692 addr = state->addr + svm_pages->state_offset; 1693 svm_pages->state_offset += PAGE_SIZE << order; 1694 } else { 1695 addr = dma_map_page(svm_pages->drm->dev, 1696 page, 0, 1697 PAGE_SIZE << order, 1698 dma_dir); 1699 if (dma_mapping_error(svm_pages->drm->dev, addr)) { 1700 err = -EFAULT; 1701 goto err_unmap; 1702 } 1703 } 1704 1705 svm_pages->dma_addr[j] = drm_pagemap_addr_encode 1706 (addr, DRM_INTERCONNECT_SYSTEM, order, 1707 dma_dir); 1708 } 1709 i += 1 << order; 1710 num_dma_mapped = i; 1711 flags.has_dma_mapping = true; 1712 } 1713 1714 if (dma_use_iova(state)) { 1715 err = dma_iova_sync(svm_pages->drm->dev, state, 0, 1716 svm_pages->state_offset); 1717 if (err) 1718 goto err_unmap; 1719 } 1720 1721 if (pagemap) 1722 flags.has_devmem_pages = true; 1723 1724 if (drm_gpusvm_pages_inlinable(svm_pages, j, npages)) { 1725 struct drm_pagemap_addr addr = svm_pages->dma_addr[0]; 1726 1727 kvfree(svm_pages->dma_addr); 1728 svm_pages->inline_addr = addr; 1729 flags.inline_dma_mapping = true; 1730 } 1731 1732 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ 1733 WRITE_ONCE(svm_pages->flags.__flags, flags.__flags); 1734 1735 return 0; 1736 1737 err_unmap: 1738 svm_pages->flags.has_dma_mapping = true; 1739 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, num_dma_mapped); 1740 return err; 1741 } 1742 1743 /** 1744 * drm_gpusvm_get_pages() - Get pages and populate GPU SVM pages struct 1745 * @gpusvm: Pointer to the GPU SVM structure 1746 * @svm_pages: Array of SVM pages instances to populate with dma addresses 1747 * @num_pages: Number of drm_gpusvm_pages instances in @svm_pages, must not be 0 1748 * @mm: The mm corresponding to the CPU range 1749 * @notifier: The corresponding notifier for the given CPU range 1750 * @pages_start: Start CPU address for the pages 1751 * @pages_end: End CPU address for the pages (exclusive) 1752 * @ctx: GPU SVM context 1753 * 1754 * This function gets and maps pages for a CPU range and ensures they are 1755 * mapped for DMA access. The HMM fault for the CPU range is performed once, 1756 * the DMA mapping by drm_gpusvm_dma_map_pages() is then done per instance, 1757 * one per owning drm_device. The retry against notifier races is kept here 1758 * in common code so drivers never open code it. 1759 * 1760 * On error the instances mapped before the failing one stay mapped, so the 1761 * caller must unmap and free every instance regardless of the return value. 1762 * 1763 * With &drm_gpusvm_ctx.no_dma_map no mapping state is recorded, so 1764 * drm_gpusvm_pages_valid() never returns true and success is only a snapshot: 1765 * the caller must recheck mmu_interval_read_retry() against the recorded 1766 * &drm_gpusvm_pages.notifier_seq under the notifier lock, and hold it until 1767 * its work is visible to invalidation. 1768 * 1769 * Return: 0 on success, negative error code on failure. 1770 */ 1771 int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm, 1772 struct drm_gpusvm_pages *svm_pages, 1773 unsigned int num_pages, 1774 struct mm_struct *mm, 1775 struct mmu_interval_notifier *notifier, 1776 unsigned long pages_start, unsigned long pages_end, 1777 const struct drm_gpusvm_ctx *ctx) 1778 { 1779 struct hmm_range hmm_range = { 1780 .default_flags = HMM_PFN_REQ_FAULT | (ctx->read_only ? 0 : 1781 HMM_PFN_REQ_WRITE), 1782 .notifier = notifier, 1783 .start = pages_start, 1784 .end = pages_end, 1785 .dev_private_owner = ctx->device_private_page_owner, 1786 }; 1787 unsigned long timeout = 1788 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1789 unsigned long remaining; 1790 unsigned long npages = npages_in_range(pages_start, pages_end); 1791 unsigned long *pfns; 1792 int err = 0; 1793 enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE : 1794 DMA_BIDIRECTIONAL; 1795 const bool map_dma = !ctx->no_dma_map; 1796 unsigned int p; 1797 1798 if (!num_pages) 1799 return -EINVAL; 1800 1801 if (ctx->no_dma_map && ctx->devmem_only) 1802 return -EINVAL; 1803 1804 if (map_dma) { 1805 for (p = 0; p < num_pages; ++p) { 1806 if (!svm_pages[p].drm) 1807 return -EINVAL; 1808 } 1809 } 1810 1811 retry: 1812 remaining = timeout - jiffies; 1813 1814 if (time_after_eq(jiffies, timeout)) 1815 return -EBUSY; 1816 1817 hmm_range.notifier_seq = mmu_interval_read_begin(notifier); 1818 1819 /* 1820 * drm_gpusvm_pages_valid_unlocked() also resets any instance whose 1821 * mapping was invalidated, so it must run on the map_dma path even when 1822 * devmem_fn suppresses the fast path; otherwise a stale inline mapping 1823 * is later mistaken for a dma_addr array by drm_gpusvm_dma_map_pages(). 1824 */ 1825 if (map_dma) { 1826 bool pages_valid = drm_gpusvm_pages_valid_unlocked(gpusvm, 1827 svm_pages, 1828 num_pages); 1829 1830 if (!ctx->devmem_fn && pages_valid) 1831 goto set_seqno; 1832 } 1833 1834 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); 1835 if (!pfns) 1836 return -ENOMEM; 1837 1838 if (!mmget_not_zero(mm)) { 1839 err = -EFAULT; 1840 goto err_free; 1841 } 1842 1843 hmm_range.hmm_pfns = pfns; 1844 err = hmm_range_fault_unlocked_timeout(&hmm_range, remaining); 1845 mmput(mm); 1846 if (err) 1847 goto err_free; 1848 1849 if (map_dma) { 1850 for (p = 0; p < num_pages; ++p) { 1851 if (drm_gpusvm_pages_first_dma(&svm_pages[p], NULL)) 1852 continue; 1853 svm_pages[p].dma_addr = 1854 kvzalloc_objs(*svm_pages[p].dma_addr, npages); 1855 if (!svm_pages[p].dma_addr) { 1856 err = -ENOMEM; 1857 goto err_free; 1858 } 1859 } 1860 } 1861 1862 /* 1863 * Perform all dma mappings under the notifier lock to not 1864 * access freed pages. A notifier will either block on 1865 * the notifier lock or unmap dma. 1866 */ 1867 drm_gpusvm_notifier_lock(gpusvm); 1868 1869 /* 1870 * drm_gpusvm_range_set_unmapped() flags the whole array in one go under 1871 * the write lock, so any instance answers for all of them here. 1872 */ 1873 if (svm_pages[0].flags.unmapped) { 1874 drm_gpusvm_notifier_unlock(gpusvm); 1875 err = -EFAULT; 1876 goto err_free; 1877 } 1878 1879 if (mmu_interval_read_retry(notifier, hmm_range.notifier_seq)) { 1880 drm_gpusvm_notifier_unlock(gpusvm); 1881 kvfree(pfns); 1882 goto retry; 1883 } 1884 1885 if (ctx->devmem_fn) 1886 drm_gpusvm_walk_devmem(gpusvm, pfns, npages, ctx); 1887 1888 if (map_dma) { 1889 for (p = 0; p < num_pages; ++p) { 1890 if (drm_gpusvm_pages_valid(gpusvm, &svm_pages[p])) 1891 continue; 1892 1893 err = drm_gpusvm_dma_map_pages(gpusvm, &svm_pages[p], pfns, 1894 npages, ctx, dma_dir); 1895 if (err) { 1896 /* 1897 * The failing instance was unwound by the 1898 * helper. Keep the ones mapped earlier: the 1899 * -EAGAIN retry reuses them, and the driver 1900 * unmaps every instance with the range on the 1901 * other error paths. 1902 */ 1903 drm_gpusvm_notifier_unlock(gpusvm); 1904 goto err_free; 1905 } 1906 } 1907 } 1908 1909 drm_gpusvm_notifier_unlock(gpusvm); 1910 kvfree(pfns); 1911 set_seqno: 1912 for (p = 0; p < num_pages; ++p) 1913 svm_pages[p].notifier_seq = hmm_range.notifier_seq; 1914 1915 return 0; 1916 1917 err_free: 1918 kvfree(pfns); 1919 if (err == -EAGAIN) 1920 goto retry; 1921 return err; 1922 } 1923 EXPORT_SYMBOL_GPL(drm_gpusvm_get_pages); 1924 1925 /** 1926 * drm_gpusvm_unmap_pages() - Unmap GPU svm pages 1927 * @gpusvm: Pointer to the GPU SVM structure 1928 * @svm_pages: Pointer to the GPU SVM pages structure 1929 * @npages: Number of pages in @svm_pages. 1930 * @ctx: GPU SVM context 1931 * 1932 * This function unmaps pages associated with a GPU SVM pages struct. If 1933 * @in_notifier is set, it is assumed that gpusvm->notifier_lock is held in 1934 * write mode; if it is clear, it acquires gpusvm->notifier_lock in read mode. 1935 * Must be called in the invalidate() callback of the corresponding notifier for 1936 * IOMMU security model. 1937 */ 1938 void drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm, 1939 struct drm_gpusvm_pages *svm_pages, 1940 unsigned long npages, 1941 const struct drm_gpusvm_ctx *ctx) 1942 { 1943 if (ctx->in_notifier) 1944 lockdep_assert_held_write(&gpusvm->notifier_lock); 1945 else 1946 drm_gpusvm_notifier_lock(gpusvm); 1947 1948 __drm_gpusvm_unmap_pages(gpusvm, svm_pages, npages); 1949 1950 if (!ctx->in_notifier) 1951 drm_gpusvm_notifier_unlock(gpusvm); 1952 } 1953 EXPORT_SYMBOL_GPL(drm_gpusvm_unmap_pages); 1954 1955 /** 1956 * drm_gpusvm_range_evict() - Evict GPU SVM range 1957 * @gpusvm: Pointer to the GPU SVM structure 1958 * @range: Pointer to the GPU SVM range to be removed 1959 * 1960 * This function evicts the specified GPU SVM range. 1961 * 1962 * Return: 0 on success, a negative error code on failure. 1963 */ 1964 int drm_gpusvm_range_evict(struct drm_gpusvm *gpusvm, 1965 struct drm_gpusvm_range *range) 1966 { 1967 struct mmu_interval_notifier *notifier = &range->notifier->notifier; 1968 struct hmm_range hmm_range = { 1969 .default_flags = HMM_PFN_REQ_FAULT, 1970 .notifier = notifier, 1971 .start = drm_gpusvm_range_start(range), 1972 .end = drm_gpusvm_range_end(range), 1973 .dev_private_owner = NULL, 1974 }; 1975 unsigned long timeout = msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); 1976 unsigned long *pfns; 1977 unsigned long npages = npages_in_range(drm_gpusvm_range_start(range), 1978 drm_gpusvm_range_end(range)); 1979 int err = 0; 1980 struct mm_struct *mm = gpusvm->mm; 1981 1982 if (!mmget_not_zero(mm)) 1983 return -EFAULT; 1984 1985 pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL); 1986 if (!pfns) { 1987 mmput(mm); 1988 return -ENOMEM; 1989 } 1990 1991 hmm_range.hmm_pfns = pfns; 1992 err = hmm_range_fault_unlocked_timeout(&hmm_range, timeout); 1993 1994 kvfree(pfns); 1995 mmput(mm); 1996 1997 return err == -EBUSY ? -ETIME : err; 1998 } 1999 EXPORT_SYMBOL_GPL(drm_gpusvm_range_evict); 2000 2001 /** 2002 * drm_gpusvm_has_mapping() - Check if GPU SVM has mapping for the given address range 2003 * @gpusvm: Pointer to the GPU SVM structure. 2004 * @start: Start address 2005 * @end: End address 2006 * 2007 * Return: True if GPU SVM has mapping, False otherwise 2008 */ 2009 bool drm_gpusvm_has_mapping(struct drm_gpusvm *gpusvm, unsigned long start, 2010 unsigned long end) 2011 { 2012 struct drm_gpusvm_notifier *notifier; 2013 2014 drm_gpusvm_for_each_notifier(notifier, gpusvm, start, end) { 2015 struct drm_gpusvm_range *range = NULL; 2016 2017 drm_gpusvm_for_each_range(range, notifier, start, end) 2018 return true; 2019 } 2020 2021 return false; 2022 } 2023 EXPORT_SYMBOL_GPL(drm_gpusvm_has_mapping); 2024 2025 /** 2026 * drm_gpusvm_range_set_unmapped() - Mark a GPU SVM range as unmapped 2027 * @range: Pointer to the GPU SVM range structure. 2028 * @pages: Pointer to the GPU SVM pages structure(s). 2029 * @pages_count: Number of GPU SVM pages structure(s) passed in. 2030 * @mmu_range: Pointer to the MMU notifier range structure. 2031 * 2032 * This function marks a GPU SVM range as unmapped and sets the partial_unmap flag 2033 * if the range partially falls within the provided MMU notifier range. 2034 */ 2035 void drm_gpusvm_range_set_unmapped(struct drm_gpusvm_range *range, 2036 struct drm_gpusvm_pages *pages, 2037 unsigned int pages_count, 2038 const struct mmu_notifier_range *mmu_range) 2039 { 2040 struct drm_gpusvm_range_flags range_flags = { 2041 .__flags = range->flags.__flags, 2042 }; 2043 unsigned int i; 2044 2045 lockdep_assert_held_write(&range->gpusvm->notifier_lock); 2046 2047 range_flags.unmapped = true; 2048 for (i = 0; i < pages_count; ++i) { 2049 struct drm_gpusvm_pages_flags flags = { 2050 .__flags = pages[i].flags.__flags, 2051 }; 2052 2053 flags.unmapped = true; 2054 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ 2055 WRITE_ONCE(pages[i].flags.__flags, flags.__flags); 2056 } 2057 if (drm_gpusvm_range_start(range) < mmu_range->start || 2058 drm_gpusvm_range_end(range) > mmu_range->end) 2059 range_flags.partial_unmap = true; 2060 /* WRITE_ONCE pairs with READ_ONCE for opportunistic checks */ 2061 WRITE_ONCE(range->flags.__flags, range_flags.__flags); 2062 } 2063 EXPORT_SYMBOL_GPL(drm_gpusvm_range_set_unmapped); 2064 2065 MODULE_DESCRIPTION("DRM GPUSVM"); 2066 MODULE_LICENSE("GPL"); 2067