1 /* SPDX-License-Identifier: GPL-2.0-only OR MIT */ 2 3 #ifndef __DRM_GPUVM_H__ 4 #define __DRM_GPUVM_H__ 5 6 /* 7 * Copyright (c) 2022 Red Hat. 8 * 9 * Permission is hereby granted, free of charge, to any person obtaining a 10 * copy of this software and associated documentation files (the "Software"), 11 * to deal in the Software without restriction, including without limitation 12 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 13 * and/or sell copies of the Software, and to permit persons to whom the 14 * Software is furnished to do so, subject to the following conditions: 15 * 16 * The above copyright notice and this permission notice shall be included in 17 * all copies or substantial portions of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 23 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 24 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 25 * OTHER DEALINGS IN THE SOFTWARE. 26 */ 27 28 #include <linux/dma-resv.h> 29 #include <linux/list.h> 30 #include <linux/llist.h> 31 #include <linux/rbtree.h> 32 #include <linux/types.h> 33 34 #include <drm/drm_device.h> 35 #include <drm/drm_gem.h> 36 #include <drm/drm_exec.h> 37 38 struct drm_gpuvm; 39 struct drm_gpuvm_bo; 40 struct drm_gpuvm_ops; 41 42 /** 43 * enum drm_gpuva_flags - flags for struct drm_gpuva 44 */ 45 enum drm_gpuva_flags { 46 /** 47 * @DRM_GPUVA_INVALIDATED: 48 * 49 * Flag indicating that the &drm_gpuva's backing GEM is invalidated. 50 */ 51 DRM_GPUVA_INVALIDATED = (1 << 0), 52 53 /** 54 * @DRM_GPUVA_SPARSE: 55 * 56 * Flag indicating that the &drm_gpuva is a sparse mapping. 57 */ 58 DRM_GPUVA_SPARSE = (1 << 1), 59 60 /** 61 * @DRM_GPUVA_USERBITS: user defined bits 62 */ 63 DRM_GPUVA_USERBITS = (1 << 2), 64 }; 65 66 /** 67 * struct drm_gpuva - structure to track a GPU VA mapping 68 * 69 * This structure represents a GPU VA mapping and is associated with a 70 * &drm_gpuvm. 71 * 72 * Typically, this structure is embedded in bigger driver structures. 73 */ 74 struct drm_gpuva { 75 /** 76 * @vm: the &drm_gpuvm this object is associated with 77 */ 78 struct drm_gpuvm *vm; 79 80 /** 81 * @vm_bo: the &drm_gpuvm_bo abstraction for the mapped 82 * &drm_gem_object 83 */ 84 struct drm_gpuvm_bo *vm_bo; 85 86 /** 87 * @flags: the &drm_gpuva_flags for this mapping 88 */ 89 enum drm_gpuva_flags flags; 90 91 /** 92 * @va: structure containing the address and range of the &drm_gpuva 93 */ 94 struct { 95 /** 96 * @va.addr: the start address 97 */ 98 u64 addr; 99 100 /* 101 * @range: the range 102 */ 103 u64 range; 104 } va; 105 106 /** 107 * @gem: structure containing the &drm_gem_object and its offset 108 */ 109 struct { 110 /** 111 * @gem.offset: the offset within the &drm_gem_object 112 */ 113 u64 offset; 114 115 /** 116 * @gem.obj: the mapped &drm_gem_object 117 */ 118 struct drm_gem_object *obj; 119 120 /** 121 * @gem.entry: the &list_head to attach this object to a &drm_gpuvm_bo 122 */ 123 struct list_head entry; 124 } gem; 125 126 /** 127 * @rb: structure containing data to store &drm_gpuvas in a rb-tree 128 */ 129 struct { 130 /** 131 * @rb.node: the rb-tree node 132 */ 133 struct rb_node node; 134 135 /** 136 * @rb.entry: The &list_head to additionally connect &drm_gpuvas 137 * in the same order they appear in the interval tree. This is 138 * useful to keep iterating &drm_gpuvas from a start node found 139 * through the rb-tree while doing modifications on the rb-tree 140 * itself. 141 */ 142 struct list_head entry; 143 144 /** 145 * @rb.__subtree_last: needed by the interval tree, holding last-in-subtree 146 */ 147 u64 __subtree_last; 148 } rb; 149 }; 150 151 int drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va); 152 void drm_gpuva_remove(struct drm_gpuva *va); 153 154 void drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo); 155 void drm_gpuva_unlink(struct drm_gpuva *va); 156 void drm_gpuva_unlink_defer(struct drm_gpuva *va); 157 158 struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm, 159 u64 addr, u64 range); 160 struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuvm *gpuvm, 161 u64 addr, u64 range); 162 163 /** 164 * drm_gpuva_invalidate() - sets whether the backing GEM of this &drm_gpuva is 165 * invalidated 166 * @va: the &drm_gpuva to set the invalidate flag for 167 * @invalidate: indicates whether the &drm_gpuva is invalidated 168 */ 169 static inline void drm_gpuva_invalidate(struct drm_gpuva *va, bool invalidate) 170 { 171 if (invalidate) 172 va->flags |= DRM_GPUVA_INVALIDATED; 173 else 174 va->flags &= ~DRM_GPUVA_INVALIDATED; 175 } 176 177 /** 178 * drm_gpuva_invalidated() - indicates whether the backing BO of this &drm_gpuva 179 * is invalidated 180 * @va: the &drm_gpuva to check 181 * 182 * Returns: %true if the GPU VA is invalidated, %false otherwise 183 */ 184 static inline bool drm_gpuva_invalidated(struct drm_gpuva *va) 185 { 186 return va->flags & DRM_GPUVA_INVALIDATED; 187 } 188 189 /** 190 * enum drm_gpuvm_flags - flags for struct drm_gpuvm 191 */ 192 enum drm_gpuvm_flags { 193 /** 194 * @DRM_GPUVM_RESV_PROTECTED: GPUVM is protected externally by the 195 * GPUVM's &dma_resv lock 196 */ 197 DRM_GPUVM_RESV_PROTECTED = BIT(0), 198 199 /** 200 * @DRM_GPUVM_IMMEDIATE_MODE: use the locking scheme for GEMs designed 201 * for modifying the GPUVM during the fence signalling path 202 * 203 * When set, gpuva.lock is used to protect gpuva.list in all GEM 204 * objects associated with this GPUVM. Otherwise, the GEMs dma-resv is 205 * used. 206 */ 207 DRM_GPUVM_IMMEDIATE_MODE = BIT(1), 208 209 /** 210 * @DRM_GPUVM_USERBITS: user defined bits 211 */ 212 DRM_GPUVM_USERBITS = BIT(2), 213 }; 214 215 /** 216 * struct drm_gpuvm - DRM GPU VA Manager 217 * 218 * The DRM GPU VA Manager keeps track of a GPU's virtual address space by using 219 * &maple_tree structures. Typically, this structure is embedded in bigger 220 * driver structures. 221 * 222 * Drivers can pass addresses and ranges in an arbitrary unit, e.g. bytes or 223 * pages. 224 * 225 * There should be one manager instance per GPU virtual address space. 226 */ 227 struct drm_gpuvm { 228 /** 229 * @name: the name of the DRM GPU VA space 230 */ 231 const char *name; 232 233 /** 234 * @flags: the &drm_gpuvm_flags of this GPUVM 235 */ 236 enum drm_gpuvm_flags flags; 237 238 /** 239 * @drm: the &drm_device this VM lives in 240 */ 241 struct drm_device *drm; 242 243 /** 244 * @mm_start: start of the VA space 245 */ 246 u64 mm_start; 247 248 /** 249 * @mm_range: length of the VA space 250 */ 251 u64 mm_range; 252 253 /** 254 * @rb: structures to track &drm_gpuva entries 255 */ 256 struct { 257 /** 258 * @rb.tree: the rb-tree to track GPU VA mappings 259 */ 260 struct rb_root_cached tree; 261 262 /** 263 * @rb.list: the &list_head to track GPU VA mappings 264 */ 265 struct list_head list; 266 } rb; 267 268 /** 269 * @kref: reference count of this object 270 */ 271 struct kref kref; 272 273 /** 274 * @kernel_alloc_node: 275 * 276 * &drm_gpuva representing the address space cutout reserved for 277 * the kernel 278 */ 279 struct drm_gpuva kernel_alloc_node; 280 281 /** 282 * @ops: &drm_gpuvm_ops providing the split/merge steps to drivers 283 */ 284 const struct drm_gpuvm_ops *ops; 285 286 /** 287 * @r_obj: Resv GEM object; representing the GPUVM's common &dma_resv. 288 */ 289 struct drm_gem_object *r_obj; 290 291 /** 292 * @extobj: structure holding the extobj list 293 */ 294 struct { 295 /** 296 * @extobj.list: &list_head storing &drm_gpuvm_bos serving as 297 * external object 298 */ 299 struct list_head list; 300 301 /** 302 * @extobj.local_list: pointer to the local list temporarily 303 * storing entries from the external object list 304 */ 305 struct list_head *local_list; 306 307 /** 308 * @extobj.lock: spinlock to protect the extobj list 309 */ 310 spinlock_t lock; 311 } extobj; 312 313 /** 314 * @evict: structure holding the evict list and evict list lock 315 */ 316 struct { 317 /** 318 * @evict.list: &list_head storing &drm_gpuvm_bos currently 319 * being evicted 320 */ 321 struct list_head list; 322 323 /** 324 * @evict.local_list: pointer to the local list temporarily 325 * storing entries from the evicted object list 326 */ 327 struct list_head *local_list; 328 329 /** 330 * @evict.lock: spinlock to protect the evict list 331 */ 332 spinlock_t lock; 333 } evict; 334 335 /** 336 * @bo_defer: structure holding vm_bos that need to be destroyed 337 */ 338 struct llist_head bo_defer; 339 }; 340 341 void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, 342 enum drm_gpuvm_flags flags, 343 struct drm_device *drm, 344 struct drm_gem_object *r_obj, 345 u64 start_offset, u64 range, 346 u64 reserve_offset, u64 reserve_range, 347 const struct drm_gpuvm_ops *ops); 348 349 /** 350 * drm_gpuvm_get() - acquire a struct drm_gpuvm reference 351 * @gpuvm: the &drm_gpuvm to acquire the reference of 352 * 353 * This function acquires an additional reference to @gpuvm. It is illegal to 354 * call this without already holding a reference. No locks required. 355 * 356 * Returns: the &struct drm_gpuvm pointer 357 */ 358 static inline struct drm_gpuvm * 359 drm_gpuvm_get(struct drm_gpuvm *gpuvm) 360 { 361 kref_get(&gpuvm->kref); 362 363 return gpuvm; 364 } 365 366 void drm_gpuvm_put(struct drm_gpuvm *gpuvm); 367 368 bool drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, u64 addr, u64 range); 369 bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range); 370 371 struct drm_gem_object * 372 drm_gpuvm_resv_object_alloc(struct drm_device *drm); 373 374 /** 375 * drm_gpuvm_resv_protected() - indicates whether &DRM_GPUVM_RESV_PROTECTED is 376 * set 377 * @gpuvm: the &drm_gpuvm 378 * 379 * Returns: true if &DRM_GPUVM_RESV_PROTECTED is set, false otherwise. 380 */ 381 static inline bool 382 drm_gpuvm_resv_protected(struct drm_gpuvm *gpuvm) 383 { 384 return gpuvm->flags & DRM_GPUVM_RESV_PROTECTED; 385 } 386 387 /** 388 * drm_gpuvm_immediate_mode() - indicates whether &DRM_GPUVM_IMMEDIATE_MODE is 389 * set 390 * @gpuvm: the &drm_gpuvm 391 * 392 * Returns: true if &DRM_GPUVM_IMMEDIATE_MODE is set, false otherwise. 393 */ 394 static inline bool 395 drm_gpuvm_immediate_mode(struct drm_gpuvm *gpuvm) 396 { 397 return gpuvm->flags & DRM_GPUVM_IMMEDIATE_MODE; 398 } 399 400 /** 401 * drm_gpuvm_resv() - returns the &drm_gpuvm's &dma_resv 402 * @gpuvm__: the &drm_gpuvm 403 * 404 * Returns: a pointer to the &drm_gpuvm's shared &dma_resv 405 */ 406 #define drm_gpuvm_resv(gpuvm__) ((gpuvm__)->r_obj->resv) 407 408 /** 409 * drm_gpuvm_resv_obj() - returns the &drm_gem_object holding the &drm_gpuvm's 410 * &dma_resv 411 * @gpuvm__: the &drm_gpuvm 412 * 413 * Returns: a pointer to the &drm_gem_object holding the &drm_gpuvm's shared 414 * &dma_resv 415 */ 416 #define drm_gpuvm_resv_obj(gpuvm__) ((gpuvm__)->r_obj) 417 418 #define drm_gpuvm_resv_held(gpuvm__) \ 419 dma_resv_held(drm_gpuvm_resv(gpuvm__)) 420 421 #define drm_gpuvm_resv_assert_held(gpuvm__) \ 422 dma_resv_assert_held(drm_gpuvm_resv(gpuvm__)) 423 424 #define drm_gpuvm_resv_held(gpuvm__) \ 425 dma_resv_held(drm_gpuvm_resv(gpuvm__)) 426 427 #define drm_gpuvm_resv_assert_held(gpuvm__) \ 428 dma_resv_assert_held(drm_gpuvm_resv(gpuvm__)) 429 430 /** 431 * drm_gpuvm_is_extobj() - indicates whether the given &drm_gem_object is an 432 * external object 433 * @gpuvm: the &drm_gpuvm to check 434 * @obj: the &drm_gem_object to check 435 * 436 * Returns: true if the &drm_gem_object &dma_resv differs from the 437 * &drm_gpuvms &dma_resv, false otherwise 438 */ 439 static inline bool 440 drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, 441 struct drm_gem_object *obj) 442 { 443 return obj && obj->resv != drm_gpuvm_resv(gpuvm); 444 } 445 446 static inline struct drm_gpuva * 447 __drm_gpuva_next(struct drm_gpuva *va) 448 { 449 if (va && !list_is_last(&va->rb.entry, &va->vm->rb.list)) 450 return list_next_entry(va, rb.entry); 451 452 return NULL; 453 } 454 455 /** 456 * drm_gpuvm_for_each_va_range() - iterate over a range of &drm_gpuvas 457 * @va__: &drm_gpuva structure to assign to in each iteration step 458 * @gpuvm__: &drm_gpuvm to walk over 459 * @start__: starting offset, the first gpuva will overlap this 460 * @end__: ending offset, the last gpuva will start before this (but may 461 * overlap) 462 * 463 * This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie 464 * between @start__ and @end__. It is implemented similarly to list_for_each(), 465 * but is using the &drm_gpuvm's internal interval tree to accelerate 466 * the search for the starting &drm_gpuva, and hence isn't safe against removal 467 * of elements. It assumes that @end__ is within (or is the upper limit of) the 468 * &drm_gpuvm. This iterator does not skip over the &drm_gpuvm's 469 * @kernel_alloc_node. 470 */ 471 #define drm_gpuvm_for_each_va_range(va__, gpuvm__, start__, end__) \ 472 for (va__ = drm_gpuva_find_first((gpuvm__), (start__), (end__) - (start__)); \ 473 va__ && (va__->va.addr < (end__)); \ 474 va__ = __drm_gpuva_next(va__)) 475 476 /** 477 * drm_gpuvm_for_each_va_range_safe() - safely iterate over a range of 478 * &drm_gpuvas 479 * @va__: &drm_gpuva to assign to in each iteration step 480 * @next__: another &drm_gpuva to use as temporary storage 481 * @gpuvm__: &drm_gpuvm to walk over 482 * @start__: starting offset, the first gpuva will overlap this 483 * @end__: ending offset, the last gpuva will start before this (but may 484 * overlap) 485 * 486 * This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie 487 * between @start__ and @end__. It is implemented similarly to 488 * list_for_each_safe(), but is using the &drm_gpuvm's internal interval 489 * tree to accelerate the search for the starting &drm_gpuva, and hence is safe 490 * against removal of elements. It assumes that @end__ is within (or is the 491 * upper limit of) the &drm_gpuvm. This iterator does not skip over the 492 * &drm_gpuvm's @kernel_alloc_node. 493 */ 494 #define drm_gpuvm_for_each_va_range_safe(va__, next__, gpuvm__, start__, end__) \ 495 for (va__ = drm_gpuva_find_first((gpuvm__), (start__), (end__) - (start__)), \ 496 next__ = __drm_gpuva_next(va__); \ 497 va__ && (va__->va.addr < (end__)); \ 498 va__ = next__, next__ = __drm_gpuva_next(va__)) 499 500 /** 501 * drm_gpuvm_for_each_va() - iterate over all &drm_gpuvas 502 * @va__: &drm_gpuva to assign to in each iteration step 503 * @gpuvm__: &drm_gpuvm to walk over 504 * 505 * This iterator walks over all &drm_gpuva structures associated with the given 506 * &drm_gpuvm. 507 */ 508 #define drm_gpuvm_for_each_va(va__, gpuvm__) \ 509 list_for_each_entry(va__, &(gpuvm__)->rb.list, rb.entry) 510 511 /** 512 * drm_gpuvm_for_each_va_safe() - safely iterate over all &drm_gpuvas 513 * @va__: &drm_gpuva to assign to in each iteration step 514 * @next__: another &drm_gpuva to use as temporary storage 515 * @gpuvm__: &drm_gpuvm to walk over 516 * 517 * This iterator walks over all &drm_gpuva structures associated with the given 518 * &drm_gpuvm. It is implemented with list_for_each_entry_safe(), and 519 * hence safe against the removal of elements. 520 */ 521 #define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) \ 522 list_for_each_entry_safe(va__, next__, &(gpuvm__)->rb.list, rb.entry) 523 524 /** 525 * struct drm_gpuvm_exec - &drm_gpuvm abstraction of &drm_exec 526 * 527 * This structure should be created on the stack as &drm_exec should be. 528 * 529 * Optionally, @extra can be set in order to lock additional &drm_gem_objects. 530 */ 531 struct drm_gpuvm_exec { 532 /** 533 * @exec: the &drm_exec structure 534 */ 535 struct drm_exec exec; 536 537 /** 538 * @flags: the flags for the struct drm_exec 539 */ 540 u32 flags; 541 542 /** 543 * @vm: the &drm_gpuvm to lock its DMA reservations 544 */ 545 struct drm_gpuvm *vm; 546 547 /** 548 * @num_fences: the number of fences to reserve for the &dma_resv of the 549 * locked &drm_gem_objects 550 */ 551 unsigned int num_fences; 552 553 /** 554 * @extra: Callback and corresponding private data for the driver to 555 * lock arbitrary additional &drm_gem_objects. 556 */ 557 struct { 558 /** 559 * @extra.fn: The driver callback to lock additional 560 * &drm_gem_objects. 561 */ 562 int (*fn)(struct drm_gpuvm_exec *vm_exec); 563 564 /** 565 * @extra.priv: driver private data for the @fn callback 566 */ 567 void *priv; 568 } extra; 569 }; 570 571 int drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm, 572 struct drm_exec *exec, 573 unsigned int num_fences); 574 575 int drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, 576 struct drm_exec *exec, 577 unsigned int num_fences); 578 579 int drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, 580 struct drm_exec *exec, 581 u64 addr, u64 range, 582 unsigned int num_fences); 583 584 int drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec); 585 586 int drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec, 587 struct drm_gem_object **objs, 588 unsigned int num_objs); 589 590 int drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec, 591 u64 addr, u64 range); 592 593 /** 594 * drm_gpuvm_exec_unlock() - lock all dma-resv of all assoiciated BOs 595 * @vm_exec: the &drm_gpuvm_exec wrapper 596 * 597 * Releases all dma-resv locks of all &drm_gem_objects previously acquired 598 * through drm_gpuvm_exec_lock() or its variants. 599 * 600 * Returns: 0 on success, negative error code on failure. 601 */ 602 static inline void 603 drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec) 604 { 605 drm_exec_fini(&vm_exec->exec); 606 } 607 608 int drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec); 609 void drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm, 610 struct drm_exec *exec, 611 struct dma_fence *fence, 612 enum dma_resv_usage private_usage, 613 enum dma_resv_usage extobj_usage); 614 615 /** 616 * drm_gpuvm_exec_resv_add_fence() - add fence to private and all extobj 617 * @vm_exec: the &drm_gpuvm_exec wrapper 618 * @fence: fence to add 619 * @private_usage: private dma-resv usage 620 * @extobj_usage: extobj dma-resv usage 621 * 622 * See drm_gpuvm_resv_add_fence(). 623 */ 624 static inline void 625 drm_gpuvm_exec_resv_add_fence(struct drm_gpuvm_exec *vm_exec, 626 struct dma_fence *fence, 627 enum dma_resv_usage private_usage, 628 enum dma_resv_usage extobj_usage) 629 { 630 drm_gpuvm_resv_add_fence(vm_exec->vm, &vm_exec->exec, fence, 631 private_usage, extobj_usage); 632 } 633 634 /** 635 * drm_gpuvm_exec_validate() - validate all BOs marked as evicted 636 * @vm_exec: the &drm_gpuvm_exec wrapper 637 * 638 * See drm_gpuvm_validate(). 639 * 640 * Returns: 0 on success, negative error code on failure. 641 */ 642 static inline int 643 drm_gpuvm_exec_validate(struct drm_gpuvm_exec *vm_exec) 644 { 645 return drm_gpuvm_validate(vm_exec->vm, &vm_exec->exec); 646 } 647 648 /** 649 * struct drm_gpuvm_bo - structure representing a &drm_gpuvm and 650 * &drm_gem_object combination 651 * 652 * This structure is an abstraction representing a &drm_gpuvm and 653 * &drm_gem_object combination. It serves as an indirection to accelerate 654 * iterating all &drm_gpuvas within a &drm_gpuvm backed by the same 655 * &drm_gem_object. 656 * 657 * Furthermore it is used cache evicted GEM objects for a certain GPU-VM to 658 * accelerate validation. 659 * 660 * Typically, drivers want to create an instance of a struct drm_gpuvm_bo once 661 * a GEM object is mapped first in a GPU-VM and release the instance once the 662 * last mapping of the GEM object in this GPU-VM is unmapped. 663 */ 664 struct drm_gpuvm_bo { 665 /** 666 * @vm: The &drm_gpuvm the @obj is mapped in. This is a reference 667 * counted pointer. 668 */ 669 struct drm_gpuvm *vm; 670 671 /** 672 * @obj: The &drm_gem_object being mapped in @vm. This is a reference 673 * counted pointer. 674 */ 675 struct drm_gem_object *obj; 676 677 /** 678 * @evicted: Indicates whether the &drm_gem_object is evicted; field 679 * protected by the &drm_gem_object's dma-resv lock. 680 */ 681 bool evicted; 682 683 /** 684 * @kref: The reference count for this &drm_gpuvm_bo. 685 */ 686 struct kref kref; 687 688 /** 689 * @list: Structure containing all &list_heads. 690 */ 691 struct { 692 /** 693 * @list.gpuva: The list of linked &drm_gpuvas. 694 * 695 * It is safe to access entries from this list as long as the 696 * GEM's gpuva lock is held. See also struct drm_gem_object. 697 */ 698 struct list_head gpuva; 699 700 /** 701 * @list.entry: Structure containing all &list_heads serving as 702 * entry. 703 */ 704 struct { 705 /** 706 * @list.entry.gem: List entry to attach to the 707 * &drm_gem_objects gpuva list. 708 */ 709 struct list_head gem; 710 711 /** 712 * @list.entry.evict: List entry to attach to the 713 * &drm_gpuvms extobj list. 714 */ 715 struct list_head extobj; 716 717 /** 718 * @list.entry.evict: List entry to attach to the 719 * &drm_gpuvms evict list. 720 */ 721 struct list_head evict; 722 723 /** 724 * @list.entry.bo_defer: List entry to attach to 725 * the &drm_gpuvms bo_defer list. 726 */ 727 struct llist_node bo_defer; 728 } entry; 729 } list; 730 }; 731 732 struct drm_gpuvm_bo * 733 drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm, 734 struct drm_gem_object *obj); 735 736 struct drm_gpuvm_bo * 737 drm_gpuvm_bo_obtain_locked(struct drm_gpuvm *gpuvm, 738 struct drm_gem_object *obj); 739 struct drm_gpuvm_bo * 740 drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *vm_bo); 741 742 /** 743 * drm_gpuvm_bo_get() - acquire a struct drm_gpuvm_bo reference 744 * @vm_bo: the &drm_gpuvm_bo to acquire the reference of 745 * 746 * This function acquires an additional reference to @vm_bo. It is illegal to 747 * call this without already holding a reference. No locks required. 748 * 749 * Returns: the &struct vm_bo pointer 750 */ 751 static inline struct drm_gpuvm_bo * 752 drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) 753 { 754 kref_get(&vm_bo->kref); 755 return vm_bo; 756 } 757 758 bool drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo); 759 760 bool drm_gpuvm_bo_put_deferred(struct drm_gpuvm_bo *vm_bo); 761 void drm_gpuvm_bo_deferred_cleanup(struct drm_gpuvm *gpuvm); 762 763 struct drm_gpuvm_bo * 764 drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, 765 struct drm_gem_object *obj); 766 767 void drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict); 768 769 /** 770 * drm_gpuvm_bo_gem_evict() - add/remove all &drm_gpuvm_bo's in the list 771 * to/from the &drm_gpuvms evicted list 772 * @obj: the &drm_gem_object 773 * @evict: indicates whether @obj is evicted 774 * 775 * See drm_gpuvm_bo_evict(). 776 */ 777 static inline void 778 drm_gpuvm_bo_gem_evict(struct drm_gem_object *obj, bool evict) 779 { 780 struct drm_gpuvm_bo *vm_bo; 781 782 drm_gem_for_each_gpuvm_bo(vm_bo, obj) { 783 drm_gem_gpuva_assert_lock_held(vm_bo->vm, obj); 784 drm_gpuvm_bo_evict(vm_bo, evict); 785 } 786 } 787 788 void drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo); 789 790 /** 791 * drm_gpuvm_bo_for_each_va() - iterator to walk over a list of &drm_gpuva 792 * @va__: &drm_gpuva structure to assign to in each iteration step 793 * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with 794 * 795 * This iterator walks over all &drm_gpuva structures associated with the 796 * &drm_gpuvm_bo. 797 * 798 * The caller must hold the GEM's gpuva lock. 799 */ 800 #define drm_gpuvm_bo_for_each_va(va__, vm_bo__) \ 801 list_for_each_entry(va__, &(vm_bo)->list.gpuva, gem.entry) 802 803 /** 804 * drm_gpuvm_bo_for_each_va_safe() - iterator to safely walk over a list of 805 * &drm_gpuva 806 * @va__: &drm_gpuva structure to assign to in each iteration step 807 * @next__: &next &drm_gpuva to store the next step 808 * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with 809 * 810 * This iterator walks over all &drm_gpuva structures associated with the 811 * &drm_gpuvm_bo. It is implemented with list_for_each_entry_safe(), hence 812 * it is save against removal of elements. 813 * 814 * The caller must hold the GEM's gpuva lock. 815 */ 816 #define drm_gpuvm_bo_for_each_va_safe(va__, next__, vm_bo__) \ 817 list_for_each_entry_safe(va__, next__, &(vm_bo)->list.gpuva, gem.entry) 818 819 /** 820 * enum drm_gpuva_op_type - GPU VA operation type 821 * 822 * Operations to alter the GPU VA mappings tracked by the &drm_gpuvm. 823 */ 824 enum drm_gpuva_op_type { 825 /** 826 * @DRM_GPUVA_OP_MAP: the map op type 827 */ 828 DRM_GPUVA_OP_MAP, 829 830 /** 831 * @DRM_GPUVA_OP_REMAP: the remap op type 832 */ 833 DRM_GPUVA_OP_REMAP, 834 835 /** 836 * @DRM_GPUVA_OP_UNMAP: the unmap op type 837 */ 838 DRM_GPUVA_OP_UNMAP, 839 840 /** 841 * @DRM_GPUVA_OP_PREFETCH: the prefetch op type 842 */ 843 DRM_GPUVA_OP_PREFETCH, 844 845 /** 846 * @DRM_GPUVA_OP_DRIVER: the driver defined op type 847 */ 848 DRM_GPUVA_OP_DRIVER, 849 }; 850 851 /** 852 * struct drm_gpuva_op_map - GPU VA map operation 853 * 854 * This structure represents a single map operation generated by the 855 * DRM GPU VA manager. 856 */ 857 struct drm_gpuva_op_map { 858 /** 859 * @va: structure containing address and range of a map 860 * operation 861 */ 862 struct { 863 /** 864 * @va.addr: the base address of the new mapping 865 */ 866 u64 addr; 867 868 /** 869 * @va.range: the range of the new mapping 870 */ 871 u64 range; 872 } va; 873 874 /** 875 * @gem: structure containing the &drm_gem_object and its offset 876 */ 877 struct { 878 /** 879 * @gem.offset: the offset within the &drm_gem_object 880 */ 881 u64 offset; 882 883 /** 884 * @gem.obj: the &drm_gem_object to map 885 */ 886 struct drm_gem_object *obj; 887 } gem; 888 }; 889 890 /** 891 * struct drm_gpuva_op_unmap - GPU VA unmap operation 892 * 893 * This structure represents a single unmap operation generated by the 894 * DRM GPU VA manager. 895 */ 896 struct drm_gpuva_op_unmap { 897 /** 898 * @va: the &drm_gpuva to unmap 899 */ 900 struct drm_gpuva *va; 901 902 /** 903 * @keep: 904 * 905 * Indicates whether this &drm_gpuva is physically contiguous with the 906 * original mapping request. 907 * 908 * Optionally, if &keep is set, drivers may keep the actual page table 909 * mappings for this &drm_gpuva, adding the missing page table entries 910 * only and update the &drm_gpuvm accordingly. 911 */ 912 bool keep; 913 }; 914 915 /** 916 * struct drm_gpuva_op_remap - GPU VA remap operation 917 * 918 * This represents a single remap operation generated by the DRM GPU VA manager. 919 * 920 * A remap operation is generated when an existing GPU VA mmapping is split up 921 * by inserting a new GPU VA mapping or by partially unmapping existent 922 * mapping(s), hence it consists of a maximum of two map and one unmap 923 * operation. 924 * 925 * The @unmap operation takes care of removing the original existing mapping. 926 * @prev is used to remap the preceding part, @next the subsequent part. 927 * 928 * If either a new mapping's start address is aligned with the start address 929 * of the old mapping or the new mapping's end address is aligned with the 930 * end address of the old mapping, either @prev or @next is NULL. 931 * This will also be the case when the requested mapping begins before the 932 * old mapping's start address or stretches beyond its end address. 933 * 934 * Note, the reason for a dedicated remap operation, rather than arbitrary 935 * unmap and map operations, is to give drivers the chance of extracting driver 936 * specific data for creating the new mappings from the unmap operations's 937 * &drm_gpuva structure which typically is embedded in larger driver specific 938 * structures. 939 */ 940 struct drm_gpuva_op_remap { 941 /** 942 * @prev: the preceding part of a split mapping 943 */ 944 struct drm_gpuva_op_map *prev; 945 946 /** 947 * @next: the subsequent part of a split mapping 948 */ 949 struct drm_gpuva_op_map *next; 950 951 /** 952 * @unmap: the unmap operation for the original existing mapping 953 */ 954 struct drm_gpuva_op_unmap *unmap; 955 }; 956 957 /** 958 * struct drm_gpuva_op_prefetch - GPU VA prefetch operation 959 * 960 * This structure represents a single prefetch operation generated by the 961 * DRM GPU VA manager. 962 */ 963 struct drm_gpuva_op_prefetch { 964 /** 965 * @va: the &drm_gpuva to prefetch 966 */ 967 struct drm_gpuva *va; 968 }; 969 970 /** 971 * struct drm_gpuva_op - GPU VA operation 972 * 973 * This structure represents a single generic operation. 974 * 975 * The particular type of the operation is defined by @op. 976 */ 977 struct drm_gpuva_op { 978 /** 979 * @entry: 980 * 981 * The &list_head used to distribute instances of this struct within 982 * &drm_gpuva_ops. 983 */ 984 struct list_head entry; 985 986 /** 987 * @op: the type of the operation 988 */ 989 enum drm_gpuva_op_type op; 990 991 union { 992 /** 993 * @map: the map operation 994 */ 995 struct drm_gpuva_op_map map; 996 997 /** 998 * @remap: the remap operation 999 */ 1000 struct drm_gpuva_op_remap remap; 1001 1002 /** 1003 * @unmap: the unmap operation 1004 */ 1005 struct drm_gpuva_op_unmap unmap; 1006 1007 /** 1008 * @prefetch: the prefetch operation 1009 */ 1010 struct drm_gpuva_op_prefetch prefetch; 1011 }; 1012 }; 1013 1014 /** 1015 * struct drm_gpuva_ops - wraps a list of &drm_gpuva_op 1016 */ 1017 struct drm_gpuva_ops { 1018 /** 1019 * @list: the &list_head 1020 */ 1021 struct list_head list; 1022 }; 1023 1024 /** 1025 * drm_gpuva_for_each_op() - iterator to walk over &drm_gpuva_ops 1026 * @op: &drm_gpuva_op to assign in each iteration step 1027 * @ops: &drm_gpuva_ops to walk 1028 * 1029 * This iterator walks over all ops within a given list of operations. 1030 */ 1031 #define drm_gpuva_for_each_op(op, ops) list_for_each_entry(op, &(ops)->list, entry) 1032 1033 /** 1034 * drm_gpuva_for_each_op_safe() - iterator to safely walk over &drm_gpuva_ops 1035 * @op: &drm_gpuva_op to assign in each iteration step 1036 * @next: &next &drm_gpuva_op to store the next step 1037 * @ops: &drm_gpuva_ops to walk 1038 * 1039 * This iterator walks over all ops within a given list of operations. It is 1040 * implemented with list_for_each_safe(), so save against removal of elements. 1041 */ 1042 #define drm_gpuva_for_each_op_safe(op, next, ops) \ 1043 list_for_each_entry_safe(op, next, &(ops)->list, entry) 1044 1045 /** 1046 * drm_gpuva_for_each_op_from_reverse() - iterate backwards from the given point 1047 * @op: &drm_gpuva_op to assign in each iteration step 1048 * @ops: &drm_gpuva_ops to walk 1049 * 1050 * This iterator walks over all ops within a given list of operations beginning 1051 * from the given operation in reverse order. 1052 */ 1053 #define drm_gpuva_for_each_op_from_reverse(op, ops) \ 1054 list_for_each_entry_from_reverse(op, &(ops)->list, entry) 1055 1056 /** 1057 * drm_gpuva_for_each_op_reverse - iterator to walk over &drm_gpuva_ops in reverse 1058 * @op: &drm_gpuva_op to assign in each iteration step 1059 * @ops: &drm_gpuva_ops to walk 1060 * 1061 * This iterator walks over all ops within a given list of operations in reverse 1062 */ 1063 #define drm_gpuva_for_each_op_reverse(op, ops) \ 1064 list_for_each_entry_reverse(op, &(ops)->list, entry) 1065 1066 /** 1067 * drm_gpuva_first_op() - returns the first &drm_gpuva_op from &drm_gpuva_ops 1068 * @ops: the &drm_gpuva_ops to get the fist &drm_gpuva_op from 1069 */ 1070 #define drm_gpuva_first_op(ops) \ 1071 list_first_entry(&(ops)->list, struct drm_gpuva_op, entry) 1072 1073 /** 1074 * drm_gpuva_last_op() - returns the last &drm_gpuva_op from &drm_gpuva_ops 1075 * @ops: the &drm_gpuva_ops to get the last &drm_gpuva_op from 1076 */ 1077 #define drm_gpuva_last_op(ops) \ 1078 list_last_entry(&(ops)->list, struct drm_gpuva_op, entry) 1079 1080 /** 1081 * drm_gpuva_prev_op() - previous &drm_gpuva_op in the list 1082 * @op: the current &drm_gpuva_op 1083 */ 1084 #define drm_gpuva_prev_op(op) list_prev_entry(op, entry) 1085 1086 /** 1087 * drm_gpuva_next_op() - next &drm_gpuva_op in the list 1088 * @op: the current &drm_gpuva_op 1089 */ 1090 #define drm_gpuva_next_op(op) list_next_entry(op, entry) 1091 1092 /** 1093 * struct drm_gpuvm_map_req - arguments passed to drm_gpuvm_sm_map[_ops_create]() 1094 */ 1095 struct drm_gpuvm_map_req { 1096 /** 1097 * @map: struct drm_gpuva_op_map 1098 */ 1099 struct drm_gpuva_op_map map; 1100 }; 1101 1102 struct drm_gpuva_ops * 1103 drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm, 1104 const struct drm_gpuvm_map_req *req); 1105 struct drm_gpuva_ops * 1106 drm_gpuvm_madvise_ops_create(struct drm_gpuvm *gpuvm, 1107 const struct drm_gpuvm_map_req *req); 1108 1109 struct drm_gpuva_ops * 1110 drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm, 1111 u64 addr, u64 range); 1112 1113 struct drm_gpuva_ops * 1114 drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, 1115 u64 addr, u64 range); 1116 1117 struct drm_gpuva_ops * 1118 drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo); 1119 1120 void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, 1121 struct drm_gpuva_ops *ops); 1122 1123 static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, 1124 const struct drm_gpuva_op_map *op) 1125 { 1126 va->va.addr = op->va.addr; 1127 va->va.range = op->va.range; 1128 va->gem.obj = op->gem.obj; 1129 va->gem.offset = op->gem.offset; 1130 } 1131 1132 /** 1133 * struct drm_gpuvm_ops - callbacks for split/merge steps 1134 * 1135 * This structure defines the callbacks used by &drm_gpuvm_sm_map and 1136 * &drm_gpuvm_sm_unmap to provide the split/merge steps for map and unmap 1137 * operations to drivers. 1138 */ 1139 struct drm_gpuvm_ops { 1140 /** 1141 * @vm_free: called when the last reference of a struct drm_gpuvm is 1142 * dropped 1143 * 1144 * This callback is mandatory. 1145 */ 1146 void (*vm_free)(struct drm_gpuvm *gpuvm); 1147 1148 /** 1149 * @op_alloc: called when the &drm_gpuvm allocates 1150 * a struct drm_gpuva_op 1151 * 1152 * Some drivers may want to embed struct drm_gpuva_op into driver 1153 * specific structures. By implementing this callback drivers can 1154 * allocate memory accordingly. 1155 * 1156 * This callback is optional. 1157 */ 1158 struct drm_gpuva_op *(*op_alloc)(void); 1159 1160 /** 1161 * @op_free: called when the &drm_gpuvm frees a 1162 * struct drm_gpuva_op 1163 * 1164 * Some drivers may want to embed struct drm_gpuva_op into driver 1165 * specific structures. By implementing this callback drivers can 1166 * free the previously allocated memory accordingly. 1167 * 1168 * This callback is optional. 1169 */ 1170 void (*op_free)(struct drm_gpuva_op *op); 1171 1172 /** 1173 * @vm_bo_alloc: called when the &drm_gpuvm allocates 1174 * a struct drm_gpuvm_bo 1175 * 1176 * Some drivers may want to embed struct drm_gpuvm_bo into driver 1177 * specific structures. By implementing this callback drivers can 1178 * allocate memory accordingly. 1179 * 1180 * This callback is optional. 1181 */ 1182 struct drm_gpuvm_bo *(*vm_bo_alloc)(void); 1183 1184 /** 1185 * @vm_bo_free: called when the &drm_gpuvm frees a 1186 * struct drm_gpuvm_bo 1187 * 1188 * Some drivers may want to embed struct drm_gpuvm_bo into driver 1189 * specific structures. By implementing this callback drivers can 1190 * free the previously allocated memory accordingly. 1191 * 1192 * This callback is optional. 1193 */ 1194 void (*vm_bo_free)(struct drm_gpuvm_bo *vm_bo); 1195 1196 /** 1197 * @vm_bo_validate: called from drm_gpuvm_validate() 1198 * 1199 * Drivers receive this callback for every evicted &drm_gem_object being 1200 * mapped in the corresponding &drm_gpuvm. 1201 * 1202 * Typically, drivers would call their driver specific variant of 1203 * ttm_bo_validate() from within this callback. 1204 */ 1205 int (*vm_bo_validate)(struct drm_gpuvm_bo *vm_bo, 1206 struct drm_exec *exec); 1207 1208 /** 1209 * @sm_step_map: called from &drm_gpuvm_sm_map to finally insert the 1210 * mapping once all previous steps were completed 1211 * 1212 * The &priv pointer matches the one the driver passed to 1213 * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively. 1214 * 1215 * Can be NULL if &drm_gpuvm_sm_map is used. 1216 */ 1217 int (*sm_step_map)(struct drm_gpuva_op *op, void *priv); 1218 1219 /** 1220 * @sm_step_remap: called from &drm_gpuvm_sm_map and 1221 * &drm_gpuvm_sm_unmap to split up an existent mapping 1222 * 1223 * This callback is called when existent mapping needs to be split up. 1224 * This is the case when either a newly requested mapping overlaps or 1225 * is enclosed by an existent mapping or a partial unmap of an existent 1226 * mapping is requested. 1227 * 1228 * The &priv pointer matches the one the driver passed to 1229 * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively. 1230 * 1231 * Can be NULL if neither &drm_gpuvm_sm_map nor &drm_gpuvm_sm_unmap is 1232 * used. 1233 */ 1234 int (*sm_step_remap)(struct drm_gpuva_op *op, void *priv); 1235 1236 /** 1237 * @sm_step_unmap: called from &drm_gpuvm_sm_map and 1238 * &drm_gpuvm_sm_unmap to unmap an existing mapping 1239 * 1240 * This callback is called when existing mapping needs to be unmapped. 1241 * This is the case when either a newly requested mapping encloses an 1242 * existing mapping or an unmap of an existing mapping is requested. 1243 * 1244 * The &priv pointer matches the one the driver passed to 1245 * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively. 1246 * 1247 * Can be NULL if neither &drm_gpuvm_sm_map nor &drm_gpuvm_sm_unmap is 1248 * used. 1249 */ 1250 int (*sm_step_unmap)(struct drm_gpuva_op *op, void *priv); 1251 }; 1252 1253 int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, 1254 const struct drm_gpuvm_map_req *req); 1255 1256 int drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, 1257 u64 addr, u64 range); 1258 1259 int drm_gpuvm_sm_map_exec_lock(struct drm_gpuvm *gpuvm, 1260 struct drm_exec *exec, unsigned int num_fences, 1261 struct drm_gpuvm_map_req *req); 1262 1263 int drm_gpuvm_sm_unmap_exec_lock(struct drm_gpuvm *gpuvm, struct drm_exec *exec, 1264 u64 req_addr, u64 req_range); 1265 1266 void drm_gpuva_map(struct drm_gpuvm *gpuvm, 1267 struct drm_gpuva *va, 1268 const struct drm_gpuva_op_map *op); 1269 1270 void drm_gpuva_remap(struct drm_gpuva *prev, 1271 struct drm_gpuva *next, 1272 const struct drm_gpuva_op_remap *op); 1273 1274 void drm_gpuva_unmap(const struct drm_gpuva_op_unmap *op); 1275 1276 /** 1277 * drm_gpuva_op_remap_to_unmap_range() - Helper to get the start and range of 1278 * the unmap stage of a remap op. 1279 * @op: Remap op. 1280 * @start_addr: Output pointer for the start of the required unmap. 1281 * @range: Output pointer for the length of the required unmap. 1282 * 1283 * The given start address and range will be set such that they represent the 1284 * range of the address space that was previously covered by the mapping being 1285 * re-mapped, but is now empty. 1286 */ 1287 static inline void 1288 drm_gpuva_op_remap_to_unmap_range(const struct drm_gpuva_op_remap *op, 1289 u64 *start_addr, u64 *range) 1290 { 1291 const u64 va_start = op->prev ? 1292 op->prev->va.addr + op->prev->va.range : 1293 op->unmap->va->va.addr; 1294 const u64 va_end = op->next ? 1295 op->next->va.addr : 1296 op->unmap->va->va.addr + op->unmap->va->va.range; 1297 1298 if (start_addr) 1299 *start_addr = va_start; 1300 if (range) 1301 *range = va_end - va_start; 1302 } 1303 1304 #endif /* __DRM_GPUVM_H__ */ 1305