1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/sched/mm.h> 26 #include <linux/dma-fence-array.h> 27 #include <drm/drm_gem.h> 28 29 #include "display/intel_display.h" 30 #include "display/intel_frontbuffer.h" 31 #include "gem/i915_gem_lmem.h" 32 #include "gem/i915_gem_object_frontbuffer.h" 33 #include "gem/i915_gem_tiling.h" 34 #include "gt/intel_engine.h" 35 #include "gt/intel_engine_heartbeat.h" 36 #include "gt/intel_gt.h" 37 #include "gt/intel_gt_requests.h" 38 #include "gt/intel_tlb.h" 39 40 #include "i915_drv.h" 41 #include "i915_gem_evict.h" 42 #include "i915_sw_fence_work.h" 43 #include "i915_trace.h" 44 #include "i915_vma.h" 45 #include "i915_vma_resource.h" 46 47 static inline void assert_vma_held_evict(const struct i915_vma *vma) 48 { 49 /* 50 * We may be forced to unbind when the vm is dead, to clean it up. 51 * This is the only exception to the requirement of the object lock 52 * being held. 53 */ 54 if (kref_read(&vma->vm->ref)) 55 assert_object_held_shared(vma->obj); 56 } 57 58 static struct kmem_cache *slab_vmas; 59 60 static struct i915_vma *i915_vma_alloc(void) 61 { 62 return kmem_cache_zalloc(slab_vmas, GFP_KERNEL); 63 } 64 65 static void i915_vma_free(struct i915_vma *vma) 66 { 67 return kmem_cache_free(slab_vmas, vma); 68 } 69 70 #if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM) 71 72 #include <linux/stackdepot.h> 73 74 static void vma_print_allocator(struct i915_vma *vma, const char *reason) 75 { 76 char buf[512]; 77 78 if (!vma->node.stack) { 79 drm_dbg(vma->obj->base.dev, 80 "vma.node [%08llx + %08llx] %s: unknown owner\n", 81 vma->node.start, vma->node.size, reason); 82 return; 83 } 84 85 stack_depot_snprint(vma->node.stack, buf, sizeof(buf), 0); 86 drm_dbg(vma->obj->base.dev, 87 "vma.node [%08llx + %08llx] %s: inserted at %s\n", 88 vma->node.start, vma->node.size, reason, buf); 89 } 90 91 #else 92 93 static void vma_print_allocator(struct i915_vma *vma, const char *reason) 94 { 95 } 96 97 #endif 98 99 static inline struct i915_vma *active_to_vma(struct i915_active *ref) 100 { 101 return container_of(ref, typeof(struct i915_vma), active); 102 } 103 104 static int __i915_vma_active(struct i915_active *ref) 105 { 106 return i915_vma_tryget(active_to_vma(ref)) ? 0 : -ENOENT; 107 } 108 109 static void __i915_vma_retire(struct i915_active *ref) 110 { 111 i915_vma_put(active_to_vma(ref)); 112 } 113 114 static struct i915_vma * 115 vma_create(struct drm_i915_gem_object *obj, 116 struct i915_address_space *vm, 117 const struct i915_gtt_view *view) 118 { 119 struct i915_vma *pos = ERR_PTR(-E2BIG); 120 struct i915_vma *vma; 121 struct rb_node *rb, **p; 122 int err; 123 124 /* The aliasing_ppgtt should never be used directly! */ 125 GEM_BUG_ON(vm == &vm->gt->ggtt->alias->vm); 126 127 vma = i915_vma_alloc(); 128 if (vma == NULL) 129 return ERR_PTR(-ENOMEM); 130 131 vma->ops = &vm->vma_ops; 132 vma->obj = obj; 133 vma->size = obj->base.size; 134 vma->display_alignment = I915_GTT_MIN_ALIGNMENT; 135 136 i915_active_init(&vma->active, __i915_vma_active, __i915_vma_retire, 0); 137 138 /* Declare ourselves safe for use inside shrinkers */ 139 if (IS_ENABLED(CONFIG_LOCKDEP)) { 140 fs_reclaim_acquire(GFP_KERNEL); 141 might_lock(&vma->active.mutex); 142 fs_reclaim_release(GFP_KERNEL); 143 } 144 145 INIT_LIST_HEAD(&vma->closed_link); 146 INIT_LIST_HEAD(&vma->obj_link); 147 RB_CLEAR_NODE(&vma->obj_node); 148 149 if (view && view->type != I915_GTT_VIEW_NORMAL) { 150 vma->gtt_view = *view; 151 if (view->type == I915_GTT_VIEW_PARTIAL) { 152 GEM_BUG_ON(range_overflows_t(u64, 153 view->partial.offset, 154 view->partial.size, 155 obj->base.size >> PAGE_SHIFT)); 156 vma->size = view->partial.size; 157 vma->size <<= PAGE_SHIFT; 158 GEM_BUG_ON(vma->size > obj->base.size); 159 } else if (view->type == I915_GTT_VIEW_ROTATED) { 160 vma->size = intel_rotation_info_size(&view->rotated); 161 vma->size <<= PAGE_SHIFT; 162 } else if (view->type == I915_GTT_VIEW_REMAPPED) { 163 vma->size = intel_remapped_info_size(&view->remapped); 164 vma->size <<= PAGE_SHIFT; 165 } 166 } 167 168 if (unlikely(vma->size > vm->total)) 169 goto err_vma; 170 171 GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE)); 172 173 err = mutex_lock_interruptible(&vm->mutex); 174 if (err) { 175 pos = ERR_PTR(err); 176 goto err_vma; 177 } 178 179 vma->vm = vm; 180 list_add_tail(&vma->vm_link, &vm->unbound_list); 181 182 spin_lock(&obj->vma.lock); 183 if (i915_is_ggtt(vm)) { 184 if (unlikely(overflows_type(vma->size, u32))) 185 goto err_unlock; 186 187 vma->fence_size = i915_gem_fence_size(vm->i915, vma->size, 188 i915_gem_object_get_tiling(obj), 189 i915_gem_object_get_stride(obj)); 190 if (unlikely(vma->fence_size < vma->size || /* overflow */ 191 vma->fence_size > vm->total)) 192 goto err_unlock; 193 194 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT)); 195 196 vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size, 197 i915_gem_object_get_tiling(obj), 198 i915_gem_object_get_stride(obj)); 199 GEM_BUG_ON(!is_power_of_2(vma->fence_alignment)); 200 201 __set_bit(I915_VMA_GGTT_BIT, __i915_vma_flags(vma)); 202 } 203 204 rb = NULL; 205 p = &obj->vma.tree.rb_node; 206 while (*p) { 207 long cmp; 208 209 rb = *p; 210 pos = rb_entry(rb, struct i915_vma, obj_node); 211 212 /* 213 * If the view already exists in the tree, another thread 214 * already created a matching vma, so return the older instance 215 * and dispose of ours. 216 */ 217 cmp = i915_vma_compare(pos, vm, view); 218 if (cmp < 0) 219 p = &rb->rb_right; 220 else if (cmp > 0) 221 p = &rb->rb_left; 222 else 223 goto err_unlock; 224 } 225 rb_link_node(&vma->obj_node, rb, p); 226 rb_insert_color(&vma->obj_node, &obj->vma.tree); 227 228 if (i915_vma_is_ggtt(vma)) 229 /* 230 * We put the GGTT vma at the start of the vma-list, followed 231 * by the ppGGTT vma. This allows us to break early when 232 * iterating over only the GGTT vma for an object, see 233 * for_each_ggtt_vma() 234 */ 235 list_add(&vma->obj_link, &obj->vma.list); 236 else 237 list_add_tail(&vma->obj_link, &obj->vma.list); 238 239 spin_unlock(&obj->vma.lock); 240 mutex_unlock(&vm->mutex); 241 242 return vma; 243 244 err_unlock: 245 spin_unlock(&obj->vma.lock); 246 list_del_init(&vma->vm_link); 247 mutex_unlock(&vm->mutex); 248 err_vma: 249 i915_vma_free(vma); 250 return pos; 251 } 252 253 static struct i915_vma * 254 i915_vma_lookup(struct drm_i915_gem_object *obj, 255 struct i915_address_space *vm, 256 const struct i915_gtt_view *view) 257 { 258 struct rb_node *rb; 259 260 rb = obj->vma.tree.rb_node; 261 while (rb) { 262 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node); 263 long cmp; 264 265 cmp = i915_vma_compare(vma, vm, view); 266 if (cmp == 0) 267 return vma; 268 269 if (cmp < 0) 270 rb = rb->rb_right; 271 else 272 rb = rb->rb_left; 273 } 274 275 return NULL; 276 } 277 278 /** 279 * i915_vma_instance - return the singleton instance of the VMA 280 * @obj: parent &struct drm_i915_gem_object to be mapped 281 * @vm: address space in which the mapping is located 282 * @view: additional mapping requirements 283 * 284 * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with 285 * the same @view characteristics. If a match is not found, one is created. 286 * Once created, the VMA is kept until either the object is freed, or the 287 * address space is closed. 288 * 289 * Returns the vma, or an error pointer. 290 */ 291 struct i915_vma * 292 i915_vma_instance(struct drm_i915_gem_object *obj, 293 struct i915_address_space *vm, 294 const struct i915_gtt_view *view) 295 { 296 struct i915_vma *vma; 297 298 GEM_BUG_ON(view && !i915_is_ggtt_or_dpt(vm)); 299 GEM_BUG_ON(!kref_read(&vm->ref)); 300 301 spin_lock(&obj->vma.lock); 302 vma = i915_vma_lookup(obj, vm, view); 303 spin_unlock(&obj->vma.lock); 304 305 /* vma_create() will resolve the race if another creates the vma */ 306 if (unlikely(!vma)) 307 vma = vma_create(obj, vm, view); 308 309 GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view)); 310 return vma; 311 } 312 313 struct i915_vma_work { 314 struct dma_fence_work base; 315 struct i915_address_space *vm; 316 struct i915_vm_pt_stash stash; 317 struct i915_vma_resource *vma_res; 318 struct drm_i915_gem_object *obj; 319 struct i915_sw_dma_fence_cb cb; 320 unsigned int pat_index; 321 unsigned int flags; 322 }; 323 324 static void __vma_bind(struct dma_fence_work *work) 325 { 326 struct i915_vma_work *vw = container_of(work, typeof(*vw), base); 327 struct i915_vma_resource *vma_res = vw->vma_res; 328 329 /* 330 * We are about the bind the object, which must mean we have already 331 * signaled the work to potentially clear/move the pages underneath. If 332 * something went wrong at that stage then the object should have 333 * unknown_state set, in which case we need to skip the bind. 334 */ 335 if (i915_gem_object_has_unknown_state(vw->obj)) 336 return; 337 338 vma_res->ops->bind_vma(vma_res->vm, &vw->stash, 339 vma_res, vw->pat_index, vw->flags); 340 } 341 342 static void __vma_release(struct dma_fence_work *work) 343 { 344 struct i915_vma_work *vw = container_of(work, typeof(*vw), base); 345 346 if (vw->obj) 347 i915_gem_object_put(vw->obj); 348 349 i915_vm_free_pt_stash(vw->vm, &vw->stash); 350 if (vw->vma_res) 351 i915_vma_resource_put(vw->vma_res); 352 } 353 354 static const struct dma_fence_work_ops bind_ops = { 355 .name = "bind", 356 .work = __vma_bind, 357 .release = __vma_release, 358 }; 359 360 struct i915_vma_work *i915_vma_work(void) 361 { 362 struct i915_vma_work *vw; 363 364 vw = kzalloc(sizeof(*vw), GFP_KERNEL); 365 if (!vw) 366 return NULL; 367 368 dma_fence_work_init(&vw->base, &bind_ops); 369 vw->base.dma.error = -EAGAIN; /* disable the worker by default */ 370 371 return vw; 372 } 373 374 int i915_vma_wait_for_bind(struct i915_vma *vma) 375 { 376 int err = 0; 377 378 if (rcu_access_pointer(vma->active.excl.fence)) { 379 struct dma_fence *fence; 380 381 rcu_read_lock(); 382 fence = dma_fence_get_rcu_safe(&vma->active.excl.fence); 383 rcu_read_unlock(); 384 if (fence) { 385 err = dma_fence_wait(fence, true); 386 dma_fence_put(fence); 387 } 388 } 389 390 return err; 391 } 392 393 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 394 static int i915_vma_verify_bind_complete(struct i915_vma *vma) 395 { 396 struct dma_fence *fence = i915_active_fence_get(&vma->active.excl); 397 int err; 398 399 if (!fence) 400 return 0; 401 402 if (dma_fence_is_signaled(fence)) 403 err = fence->error; 404 else 405 err = -EBUSY; 406 407 dma_fence_put(fence); 408 409 return err; 410 } 411 #else 412 #define i915_vma_verify_bind_complete(_vma) 0 413 #endif 414 415 I915_SELFTEST_EXPORT void 416 i915_vma_resource_init_from_vma(struct i915_vma_resource *vma_res, 417 struct i915_vma *vma) 418 { 419 struct drm_i915_gem_object *obj = vma->obj; 420 421 i915_vma_resource_init(vma_res, vma->vm, vma->pages, &vma->page_sizes, 422 obj->mm.rsgt, i915_gem_object_is_readonly(obj), 423 i915_gem_object_is_lmem(obj), obj->mm.region, 424 vma->ops, vma->private, __i915_vma_offset(vma), 425 __i915_vma_size(vma), vma->size, vma->guard); 426 } 427 428 /** 429 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. 430 * @vma: VMA to map 431 * @pat_index: PAT index to set in PTE 432 * @flags: flags like global or local mapping 433 * @work: preallocated worker for allocating and binding the PTE 434 * @vma_res: pointer to a preallocated vma resource. The resource is either 435 * consumed or freed. 436 * 437 * DMA addresses are taken from the scatter-gather table of this object (or of 438 * this VMA in case of non-default GGTT views) and PTE entries set up. 439 * Note that DMA addresses are also the only part of the SG table we care about. 440 */ 441 int i915_vma_bind(struct i915_vma *vma, 442 unsigned int pat_index, 443 u32 flags, 444 struct i915_vma_work *work, 445 struct i915_vma_resource *vma_res) 446 { 447 u32 bind_flags; 448 u32 vma_flags; 449 int ret; 450 451 lockdep_assert_held(&vma->vm->mutex); 452 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 453 GEM_BUG_ON(vma->size > i915_vma_size(vma)); 454 455 if (GEM_DEBUG_WARN_ON(range_overflows(vma->node.start, 456 vma->node.size, 457 vma->vm->total))) { 458 i915_vma_resource_free(vma_res); 459 return -ENODEV; 460 } 461 462 if (GEM_DEBUG_WARN_ON(!flags)) { 463 i915_vma_resource_free(vma_res); 464 return -EINVAL; 465 } 466 467 bind_flags = flags; 468 bind_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 469 470 vma_flags = atomic_read(&vma->flags); 471 vma_flags &= I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 472 473 bind_flags &= ~vma_flags; 474 if (bind_flags == 0) { 475 i915_vma_resource_free(vma_res); 476 return 0; 477 } 478 479 GEM_BUG_ON(!atomic_read(&vma->pages_count)); 480 481 /* Wait for or await async unbinds touching our range */ 482 if (work && bind_flags & vma->vm->bind_async_flags) 483 ret = i915_vma_resource_bind_dep_await(vma->vm, 484 &work->base.chain, 485 vma->node.start, 486 vma->node.size, 487 true, 488 GFP_NOWAIT | 489 __GFP_RETRY_MAYFAIL | 490 __GFP_NOWARN); 491 else 492 ret = i915_vma_resource_bind_dep_sync(vma->vm, vma->node.start, 493 vma->node.size, true); 494 if (ret) { 495 i915_vma_resource_free(vma_res); 496 return ret; 497 } 498 499 if (vma->resource || !vma_res) { 500 /* Rebinding with an additional I915_VMA_*_BIND */ 501 GEM_WARN_ON(!vma_flags); 502 i915_vma_resource_free(vma_res); 503 } else { 504 i915_vma_resource_init_from_vma(vma_res, vma); 505 vma->resource = vma_res; 506 } 507 trace_i915_vma_bind(vma, bind_flags); 508 if (work && bind_flags & vma->vm->bind_async_flags) { 509 struct dma_fence *prev; 510 511 work->vma_res = i915_vma_resource_get(vma->resource); 512 work->pat_index = pat_index; 513 work->flags = bind_flags; 514 515 /* 516 * Note we only want to chain up to the migration fence on 517 * the pages (not the object itself). As we don't track that, 518 * yet, we have to use the exclusive fence instead. 519 * 520 * Also note that we do not want to track the async vma as 521 * part of the obj->resv->excl_fence as it only affects 522 * execution and not content or object's backing store lifetime. 523 */ 524 prev = i915_active_set_exclusive(&vma->active, &work->base.dma); 525 if (prev) { 526 __i915_sw_fence_await_dma_fence(&work->base.chain, 527 prev, 528 &work->cb); 529 dma_fence_put(prev); 530 } 531 532 work->base.dma.error = 0; /* enable the queue_work() */ 533 work->obj = i915_gem_object_get(vma->obj); 534 } else { 535 ret = i915_gem_object_wait_moving_fence(vma->obj, true); 536 if (ret) { 537 i915_vma_resource_free(vma->resource); 538 vma->resource = NULL; 539 540 return ret; 541 } 542 vma->ops->bind_vma(vma->vm, NULL, vma->resource, pat_index, 543 bind_flags); 544 } 545 546 atomic_or(bind_flags, &vma->flags); 547 return 0; 548 } 549 550 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) 551 { 552 void __iomem *ptr; 553 int err; 554 555 if (WARN_ON_ONCE(vma->obj->flags & I915_BO_ALLOC_GPU_ONLY)) 556 return IOMEM_ERR_PTR(-EINVAL); 557 558 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 559 GEM_BUG_ON(!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)); 560 GEM_BUG_ON(i915_vma_verify_bind_complete(vma)); 561 562 ptr = READ_ONCE(vma->iomap); 563 if (ptr == NULL) { 564 /* 565 * TODO: consider just using i915_gem_object_pin_map() for lmem 566 * instead, which already supports mapping non-contiguous chunks 567 * of pages, that way we can also drop the 568 * I915_BO_ALLOC_CONTIGUOUS when allocating the object. 569 */ 570 if (i915_gem_object_is_lmem(vma->obj)) { 571 ptr = i915_gem_object_lmem_io_map(vma->obj, 0, 572 vma->obj->base.size); 573 } else if (i915_vma_is_map_and_fenceable(vma)) { 574 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap, 575 i915_vma_offset(vma), 576 i915_vma_size(vma)); 577 } else { 578 ptr = (void __iomem *) 579 i915_gem_object_pin_map(vma->obj, I915_MAP_WC); 580 if (IS_ERR(ptr)) { 581 err = PTR_ERR(ptr); 582 goto err; 583 } 584 ptr = page_pack_bits(ptr, 1); 585 } 586 587 if (ptr == NULL) { 588 err = -ENOMEM; 589 goto err; 590 } 591 592 if (unlikely(cmpxchg(&vma->iomap, NULL, ptr))) { 593 if (page_unmask_bits(ptr)) 594 __i915_gem_object_release_map(vma->obj); 595 else 596 io_mapping_unmap(ptr); 597 ptr = vma->iomap; 598 } 599 } 600 601 __i915_vma_pin(vma); 602 603 err = i915_vma_pin_fence(vma); 604 if (err) 605 goto err_unpin; 606 607 i915_vma_set_ggtt_write(vma); 608 609 /* NB Access through the GTT requires the device to be awake. */ 610 return page_mask_bits(ptr); 611 612 err_unpin: 613 __i915_vma_unpin(vma); 614 err: 615 return IOMEM_ERR_PTR(err); 616 } 617 618 void i915_vma_flush_writes(struct i915_vma *vma) 619 { 620 if (i915_vma_unset_ggtt_write(vma)) 621 intel_gt_flush_ggtt_writes(vma->vm->gt); 622 } 623 624 void i915_vma_unpin_iomap(struct i915_vma *vma) 625 { 626 GEM_BUG_ON(vma->iomap == NULL); 627 628 /* XXX We keep the mapping until __i915_vma_unbind()/evict() */ 629 630 i915_vma_flush_writes(vma); 631 632 i915_vma_unpin_fence(vma); 633 i915_vma_unpin(vma); 634 } 635 636 void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags) 637 { 638 struct i915_vma *vma; 639 struct drm_i915_gem_object *obj; 640 641 vma = fetch_and_zero(p_vma); 642 if (!vma) 643 return; 644 645 obj = vma->obj; 646 GEM_BUG_ON(!obj); 647 648 i915_vma_unpin(vma); 649 650 if (flags & I915_VMA_RELEASE_MAP) 651 i915_gem_object_unpin_map(obj); 652 653 i915_gem_object_put(obj); 654 } 655 656 bool i915_vma_misplaced(const struct i915_vma *vma, 657 u64 size, u64 alignment, u64 flags) 658 { 659 if (!drm_mm_node_allocated(&vma->node)) 660 return false; 661 662 if (test_bit(I915_VMA_ERROR_BIT, __i915_vma_flags(vma))) 663 return true; 664 665 if (i915_vma_size(vma) < size) 666 return true; 667 668 GEM_BUG_ON(alignment && !is_power_of_2(alignment)); 669 if (alignment && !IS_ALIGNED(i915_vma_offset(vma), alignment)) 670 return true; 671 672 if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma)) 673 return true; 674 675 if (flags & PIN_OFFSET_BIAS && 676 i915_vma_offset(vma) < (flags & PIN_OFFSET_MASK)) 677 return true; 678 679 if (flags & PIN_OFFSET_FIXED && 680 i915_vma_offset(vma) != (flags & PIN_OFFSET_MASK)) 681 return true; 682 683 if (flags & PIN_OFFSET_GUARD && 684 vma->guard < (flags & PIN_OFFSET_MASK)) 685 return true; 686 687 return false; 688 } 689 690 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) 691 { 692 bool mappable, fenceable; 693 694 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 695 GEM_BUG_ON(!vma->fence_size); 696 697 fenceable = (i915_vma_size(vma) >= vma->fence_size && 698 IS_ALIGNED(i915_vma_offset(vma), vma->fence_alignment)); 699 700 mappable = i915_ggtt_offset(vma) + vma->fence_size <= 701 i915_vm_to_ggtt(vma->vm)->mappable_end; 702 703 if (mappable && fenceable) 704 set_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); 705 else 706 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); 707 } 708 709 bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long color) 710 { 711 struct drm_mm_node *node = &vma->node; 712 struct drm_mm_node *other; 713 714 /* 715 * On some machines we have to be careful when putting differing types 716 * of snoopable memory together to avoid the prefetcher crossing memory 717 * domains and dying. During vm initialisation, we decide whether or not 718 * these constraints apply and set the drm_mm.color_adjust 719 * appropriately. 720 */ 721 if (!i915_vm_has_cache_coloring(vma->vm)) 722 return true; 723 724 /* Only valid to be called on an already inserted vma */ 725 GEM_BUG_ON(!drm_mm_node_allocated(node)); 726 GEM_BUG_ON(list_empty(&node->node_list)); 727 728 other = list_prev_entry(node, node_list); 729 if (i915_node_color_differs(other, color) && 730 !drm_mm_hole_follows(other)) 731 return false; 732 733 other = list_next_entry(node, node_list); 734 if (i915_node_color_differs(other, color) && 735 !drm_mm_hole_follows(node)) 736 return false; 737 738 return true; 739 } 740 741 /** 742 * i915_vma_insert - finds a slot for the vma in its address space 743 * @vma: the vma 744 * @ww: An optional struct i915_gem_ww_ctx 745 * @size: requested size in bytes (can be larger than the VMA) 746 * @alignment: required alignment 747 * @flags: mask of PIN_* flags to use 748 * 749 * First we try to allocate some free space that meets the requirements for 750 * the VMA. Failiing that, if the flags permit, it will evict an old VMA, 751 * preferrably the oldest idle entry to make room for the new VMA. 752 * 753 * Returns: 754 * 0 on success, negative error code otherwise. 755 */ 756 static int 757 i915_vma_insert(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, 758 u64 size, u64 alignment, u64 flags) 759 { 760 unsigned long color, guard; 761 u64 start, end; 762 int ret; 763 764 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); 765 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 766 GEM_BUG_ON(hweight64(flags & (PIN_OFFSET_GUARD | PIN_OFFSET_FIXED | PIN_OFFSET_BIAS)) > 1); 767 768 size = max(size, vma->size); 769 alignment = max_t(typeof(alignment), alignment, vma->display_alignment); 770 if (flags & PIN_MAPPABLE) { 771 size = max_t(typeof(size), size, vma->fence_size); 772 alignment = max_t(typeof(alignment), 773 alignment, vma->fence_alignment); 774 } 775 776 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 777 GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT)); 778 GEM_BUG_ON(!is_power_of_2(alignment)); 779 780 guard = vma->guard; /* retain guard across rebinds */ 781 if (flags & PIN_OFFSET_GUARD) { 782 GEM_BUG_ON(overflows_type(flags & PIN_OFFSET_MASK, u32)); 783 guard = max_t(u32, guard, flags & PIN_OFFSET_MASK); 784 } 785 /* 786 * As we align the node upon insertion, but the hardware gets 787 * node.start + guard, the easiest way to make that work is 788 * to make the guard a multiple of the alignment size. 789 */ 790 guard = ALIGN(guard, alignment); 791 792 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; 793 GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); 794 795 end = vma->vm->total; 796 if (flags & PIN_MAPPABLE) 797 end = min_t(u64, end, i915_vm_to_ggtt(vma->vm)->mappable_end); 798 if (flags & PIN_ZONE_4G) 799 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE); 800 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); 801 802 alignment = max(alignment, i915_vm_obj_min_alignment(vma->vm, vma->obj)); 803 804 /* 805 * If binding the object/GGTT view requires more space than the entire 806 * aperture has, reject it early before evicting everything in a vain 807 * attempt to find space. 808 */ 809 if (size > end - 2 * guard) { 810 drm_dbg(vma->obj->base.dev, 811 "Attempting to bind an object larger than the aperture: request=%llu > %s aperture=%llu\n", 812 size, flags & PIN_MAPPABLE ? "mappable" : "total", end); 813 return -ENOSPC; 814 } 815 816 color = 0; 817 818 if (i915_vm_has_cache_coloring(vma->vm)) 819 color = vma->obj->pat_index; 820 821 if (flags & PIN_OFFSET_FIXED) { 822 u64 offset = flags & PIN_OFFSET_MASK; 823 if (!IS_ALIGNED(offset, alignment) || 824 range_overflows(offset, size, end)) 825 return -EINVAL; 826 /* 827 * The caller knows not of the guard added by others and 828 * requests for the offset of the start of its buffer 829 * to be fixed, which may not be the same as the position 830 * of the vma->node due to the guard pages. 831 */ 832 if (offset < guard || offset + size > end - guard) 833 return -ENOSPC; 834 835 ret = i915_gem_gtt_reserve(vma->vm, ww, &vma->node, 836 size + 2 * guard, 837 offset - guard, 838 color, flags); 839 if (ret) 840 return ret; 841 } else { 842 size += 2 * guard; 843 /* 844 * We only support huge gtt pages through the 48b PPGTT, 845 * however we also don't want to force any alignment for 846 * objects which need to be tightly packed into the low 32bits. 847 * 848 * Note that we assume that GGTT are limited to 4GiB for the 849 * forseeable future. See also i915_ggtt_offset(). 850 */ 851 if (upper_32_bits(end - 1) && 852 vma->page_sizes.sg > I915_GTT_PAGE_SIZE && 853 !HAS_64K_PAGES(vma->vm->i915)) { 854 /* 855 * We can't mix 64K and 4K PTEs in the same page-table 856 * (2M block), and so to avoid the ugliness and 857 * complexity of coloring we opt for just aligning 64K 858 * objects to 2M. 859 */ 860 u64 page_alignment = 861 rounddown_pow_of_two(vma->page_sizes.sg | 862 I915_GTT_PAGE_SIZE_2M); 863 864 /* 865 * Check we don't expand for the limited Global GTT 866 * (mappable aperture is even more precious!). This 867 * also checks that we exclude the aliasing-ppgtt. 868 */ 869 GEM_BUG_ON(i915_vma_is_ggtt(vma)); 870 871 alignment = max(alignment, page_alignment); 872 873 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K) 874 size = round_up(size, I915_GTT_PAGE_SIZE_2M); 875 } 876 877 ret = i915_gem_gtt_insert(vma->vm, ww, &vma->node, 878 size, alignment, color, 879 start, end, flags); 880 if (ret) 881 return ret; 882 883 GEM_BUG_ON(vma->node.start < start); 884 GEM_BUG_ON(vma->node.start + vma->node.size > end); 885 } 886 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 887 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, color)); 888 889 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 890 vma->guard = guard; 891 892 return 0; 893 } 894 895 static void 896 i915_vma_detach(struct i915_vma *vma) 897 { 898 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 899 GEM_BUG_ON(i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); 900 901 /* 902 * And finally now the object is completely decoupled from this 903 * vma, we can drop its hold on the backing storage and allow 904 * it to be reaped by the shrinker. 905 */ 906 list_move_tail(&vma->vm_link, &vma->vm->unbound_list); 907 } 908 909 static bool try_qad_pin(struct i915_vma *vma, unsigned int flags) 910 { 911 unsigned int bound; 912 913 bound = atomic_read(&vma->flags); 914 915 if (flags & PIN_VALIDATE) { 916 flags &= I915_VMA_BIND_MASK; 917 918 return (flags & bound) == flags; 919 } 920 921 /* with the lock mandatory for unbind, we don't race here */ 922 flags &= I915_VMA_BIND_MASK; 923 do { 924 if (unlikely(flags & ~bound)) 925 return false; 926 927 if (unlikely(bound & (I915_VMA_OVERFLOW | I915_VMA_ERROR))) 928 return false; 929 930 GEM_BUG_ON(((bound + 1) & I915_VMA_PIN_MASK) == 0); 931 } while (!atomic_try_cmpxchg(&vma->flags, &bound, bound + 1)); 932 933 return true; 934 } 935 936 static struct scatterlist * 937 rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset, 938 unsigned int width, unsigned int height, 939 unsigned int src_stride, unsigned int dst_stride, 940 struct sg_table *st, struct scatterlist *sg) 941 { 942 unsigned int column, row; 943 pgoff_t src_idx; 944 945 for (column = 0; column < width; column++) { 946 unsigned int left; 947 948 src_idx = src_stride * (height - 1) + column + offset; 949 for (row = 0; row < height; row++) { 950 st->nents++; 951 /* 952 * We don't need the pages, but need to initialize 953 * the entries so the sg list can be happily traversed. 954 * The only thing we need are DMA addresses. 955 */ 956 sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0); 957 sg_dma_address(sg) = 958 i915_gem_object_get_dma_address(obj, src_idx); 959 sg_dma_len(sg) = I915_GTT_PAGE_SIZE; 960 sg = sg_next(sg); 961 src_idx -= src_stride; 962 } 963 964 left = (dst_stride - height) * I915_GTT_PAGE_SIZE; 965 966 if (!left) 967 continue; 968 969 st->nents++; 970 971 /* 972 * The DE ignores the PTEs for the padding tiles, the sg entry 973 * here is just a conenience to indicate how many padding PTEs 974 * to insert at this spot. 975 */ 976 sg_set_page(sg, NULL, left, 0); 977 sg_dma_address(sg) = 0; 978 sg_dma_len(sg) = left; 979 sg = sg_next(sg); 980 } 981 982 return sg; 983 } 984 985 static noinline struct sg_table * 986 intel_rotate_pages(struct intel_rotation_info *rot_info, 987 struct drm_i915_gem_object *obj) 988 { 989 unsigned int size = intel_rotation_info_size(rot_info); 990 struct drm_i915_private *i915 = to_i915(obj->base.dev); 991 struct sg_table *st; 992 struct scatterlist *sg; 993 int ret = -ENOMEM; 994 int i; 995 996 /* Allocate target SG list. */ 997 st = kmalloc(sizeof(*st), GFP_KERNEL); 998 if (!st) 999 goto err_st_alloc; 1000 1001 ret = sg_alloc_table(st, size, GFP_KERNEL); 1002 if (ret) 1003 goto err_sg_alloc; 1004 1005 st->nents = 0; 1006 sg = st->sgl; 1007 1008 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) 1009 sg = rotate_pages(obj, rot_info->plane[i].offset, 1010 rot_info->plane[i].width, rot_info->plane[i].height, 1011 rot_info->plane[i].src_stride, 1012 rot_info->plane[i].dst_stride, 1013 st, sg); 1014 1015 return st; 1016 1017 err_sg_alloc: 1018 kfree(st); 1019 err_st_alloc: 1020 1021 drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n", 1022 obj->base.size, rot_info->plane[0].width, 1023 rot_info->plane[0].height, size); 1024 1025 return ERR_PTR(ret); 1026 } 1027 1028 static struct scatterlist * 1029 add_padding_pages(unsigned int count, 1030 struct sg_table *st, struct scatterlist *sg) 1031 { 1032 st->nents++; 1033 1034 /* 1035 * The DE ignores the PTEs for the padding tiles, the sg entry 1036 * here is just a convenience to indicate how many padding PTEs 1037 * to insert at this spot. 1038 */ 1039 sg_set_page(sg, NULL, count * I915_GTT_PAGE_SIZE, 0); 1040 sg_dma_address(sg) = 0; 1041 sg_dma_len(sg) = count * I915_GTT_PAGE_SIZE; 1042 sg = sg_next(sg); 1043 1044 return sg; 1045 } 1046 1047 static struct scatterlist * 1048 remap_tiled_color_plane_pages(struct drm_i915_gem_object *obj, 1049 unsigned long offset, unsigned int alignment_pad, 1050 unsigned int width, unsigned int height, 1051 unsigned int src_stride, unsigned int dst_stride, 1052 struct sg_table *st, struct scatterlist *sg, 1053 unsigned int *gtt_offset) 1054 { 1055 unsigned int row; 1056 1057 if (!width || !height) 1058 return sg; 1059 1060 if (alignment_pad) 1061 sg = add_padding_pages(alignment_pad, st, sg); 1062 1063 for (row = 0; row < height; row++) { 1064 unsigned int left = width * I915_GTT_PAGE_SIZE; 1065 1066 while (left) { 1067 dma_addr_t addr; 1068 unsigned int length; 1069 1070 /* 1071 * We don't need the pages, but need to initialize 1072 * the entries so the sg list can be happily traversed. 1073 * The only thing we need are DMA addresses. 1074 */ 1075 1076 addr = i915_gem_object_get_dma_address_len(obj, offset, &length); 1077 1078 length = min(left, length); 1079 1080 st->nents++; 1081 1082 sg_set_page(sg, NULL, length, 0); 1083 sg_dma_address(sg) = addr; 1084 sg_dma_len(sg) = length; 1085 sg = sg_next(sg); 1086 1087 offset += length / I915_GTT_PAGE_SIZE; 1088 left -= length; 1089 } 1090 1091 offset += src_stride - width; 1092 1093 left = (dst_stride - width) * I915_GTT_PAGE_SIZE; 1094 1095 if (!left) 1096 continue; 1097 1098 sg = add_padding_pages(left >> PAGE_SHIFT, st, sg); 1099 } 1100 1101 *gtt_offset += alignment_pad + dst_stride * height; 1102 1103 return sg; 1104 } 1105 1106 static struct scatterlist * 1107 remap_contiguous_pages(struct drm_i915_gem_object *obj, 1108 pgoff_t obj_offset, 1109 unsigned int count, 1110 struct sg_table *st, struct scatterlist *sg) 1111 { 1112 struct scatterlist *iter; 1113 unsigned int offset; 1114 1115 iter = i915_gem_object_get_sg_dma(obj, obj_offset, &offset); 1116 GEM_BUG_ON(!iter); 1117 1118 do { 1119 unsigned int len; 1120 1121 len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT), 1122 count << PAGE_SHIFT); 1123 sg_set_page(sg, NULL, len, 0); 1124 sg_dma_address(sg) = 1125 sg_dma_address(iter) + (offset << PAGE_SHIFT); 1126 sg_dma_len(sg) = len; 1127 1128 st->nents++; 1129 count -= len >> PAGE_SHIFT; 1130 if (count == 0) 1131 return sg; 1132 1133 sg = __sg_next(sg); 1134 iter = __sg_next(iter); 1135 offset = 0; 1136 } while (1); 1137 } 1138 1139 static struct scatterlist * 1140 remap_linear_color_plane_pages(struct drm_i915_gem_object *obj, 1141 pgoff_t obj_offset, unsigned int alignment_pad, 1142 unsigned int size, 1143 struct sg_table *st, struct scatterlist *sg, 1144 unsigned int *gtt_offset) 1145 { 1146 if (!size) 1147 return sg; 1148 1149 if (alignment_pad) 1150 sg = add_padding_pages(alignment_pad, st, sg); 1151 1152 sg = remap_contiguous_pages(obj, obj_offset, size, st, sg); 1153 sg = sg_next(sg); 1154 1155 *gtt_offset += alignment_pad + size; 1156 1157 return sg; 1158 } 1159 1160 static struct scatterlist * 1161 remap_color_plane_pages(const struct intel_remapped_info *rem_info, 1162 struct drm_i915_gem_object *obj, 1163 int color_plane, 1164 struct sg_table *st, struct scatterlist *sg, 1165 unsigned int *gtt_offset) 1166 { 1167 unsigned int alignment_pad = 0; 1168 1169 if (rem_info->plane_alignment) 1170 alignment_pad = ALIGN(*gtt_offset, rem_info->plane_alignment) - *gtt_offset; 1171 1172 if (rem_info->plane[color_plane].linear) 1173 sg = remap_linear_color_plane_pages(obj, 1174 rem_info->plane[color_plane].offset, 1175 alignment_pad, 1176 rem_info->plane[color_plane].size, 1177 st, sg, 1178 gtt_offset); 1179 1180 else 1181 sg = remap_tiled_color_plane_pages(obj, 1182 rem_info->plane[color_plane].offset, 1183 alignment_pad, 1184 rem_info->plane[color_plane].width, 1185 rem_info->plane[color_plane].height, 1186 rem_info->plane[color_plane].src_stride, 1187 rem_info->plane[color_plane].dst_stride, 1188 st, sg, 1189 gtt_offset); 1190 1191 return sg; 1192 } 1193 1194 static noinline struct sg_table * 1195 intel_remap_pages(struct intel_remapped_info *rem_info, 1196 struct drm_i915_gem_object *obj) 1197 { 1198 unsigned int size = intel_remapped_info_size(rem_info); 1199 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1200 struct sg_table *st; 1201 struct scatterlist *sg; 1202 unsigned int gtt_offset = 0; 1203 int ret = -ENOMEM; 1204 int i; 1205 1206 /* Allocate target SG list. */ 1207 st = kmalloc(sizeof(*st), GFP_KERNEL); 1208 if (!st) 1209 goto err_st_alloc; 1210 1211 ret = sg_alloc_table(st, size, GFP_KERNEL); 1212 if (ret) 1213 goto err_sg_alloc; 1214 1215 st->nents = 0; 1216 sg = st->sgl; 1217 1218 for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) 1219 sg = remap_color_plane_pages(rem_info, obj, i, st, sg, >t_offset); 1220 1221 i915_sg_trim(st); 1222 1223 return st; 1224 1225 err_sg_alloc: 1226 kfree(st); 1227 err_st_alloc: 1228 1229 drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n", 1230 obj->base.size, rem_info->plane[0].width, 1231 rem_info->plane[0].height, size); 1232 1233 return ERR_PTR(ret); 1234 } 1235 1236 static noinline struct sg_table * 1237 intel_partial_pages(const struct i915_gtt_view *view, 1238 struct drm_i915_gem_object *obj) 1239 { 1240 struct sg_table *st; 1241 struct scatterlist *sg; 1242 unsigned int count = view->partial.size; 1243 int ret = -ENOMEM; 1244 1245 st = kmalloc(sizeof(*st), GFP_KERNEL); 1246 if (!st) 1247 goto err_st_alloc; 1248 1249 ret = sg_alloc_table(st, count, GFP_KERNEL); 1250 if (ret) 1251 goto err_sg_alloc; 1252 1253 st->nents = 0; 1254 1255 sg = remap_contiguous_pages(obj, view->partial.offset, count, st, st->sgl); 1256 1257 sg_mark_end(sg); 1258 i915_sg_trim(st); /* Drop any unused tail entries. */ 1259 1260 return st; 1261 1262 err_sg_alloc: 1263 kfree(st); 1264 err_st_alloc: 1265 return ERR_PTR(ret); 1266 } 1267 1268 static int 1269 __i915_vma_get_pages(struct i915_vma *vma) 1270 { 1271 struct sg_table *pages; 1272 1273 /* 1274 * The vma->pages are only valid within the lifespan of the borrowed 1275 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so 1276 * must be the vma->pages. A simple rule is that vma->pages must only 1277 * be accessed when the obj->mm.pages are pinned. 1278 */ 1279 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj)); 1280 1281 switch (vma->gtt_view.type) { 1282 default: 1283 GEM_BUG_ON(vma->gtt_view.type); 1284 fallthrough; 1285 case I915_GTT_VIEW_NORMAL: 1286 pages = vma->obj->mm.pages; 1287 break; 1288 1289 case I915_GTT_VIEW_ROTATED: 1290 pages = 1291 intel_rotate_pages(&vma->gtt_view.rotated, vma->obj); 1292 break; 1293 1294 case I915_GTT_VIEW_REMAPPED: 1295 pages = 1296 intel_remap_pages(&vma->gtt_view.remapped, vma->obj); 1297 break; 1298 1299 case I915_GTT_VIEW_PARTIAL: 1300 pages = intel_partial_pages(&vma->gtt_view, vma->obj); 1301 break; 1302 } 1303 1304 if (IS_ERR(pages)) { 1305 drm_err(&vma->vm->i915->drm, 1306 "Failed to get pages for VMA view type %u (%ld)!\n", 1307 vma->gtt_view.type, PTR_ERR(pages)); 1308 return PTR_ERR(pages); 1309 } 1310 1311 vma->pages = pages; 1312 1313 return 0; 1314 } 1315 1316 I915_SELFTEST_EXPORT int i915_vma_get_pages(struct i915_vma *vma) 1317 { 1318 int err; 1319 1320 if (atomic_add_unless(&vma->pages_count, 1, 0)) 1321 return 0; 1322 1323 err = i915_gem_object_pin_pages(vma->obj); 1324 if (err) 1325 return err; 1326 1327 err = __i915_vma_get_pages(vma); 1328 if (err) 1329 goto err_unpin; 1330 1331 vma->page_sizes = vma->obj->mm.page_sizes; 1332 atomic_inc(&vma->pages_count); 1333 1334 return 0; 1335 1336 err_unpin: 1337 __i915_gem_object_unpin_pages(vma->obj); 1338 1339 return err; 1340 } 1341 1342 void vma_invalidate_tlb(struct i915_address_space *vm, u32 *tlb) 1343 { 1344 struct intel_gt *gt; 1345 int id; 1346 1347 if (!tlb) 1348 return; 1349 1350 /* 1351 * Before we release the pages that were bound by this vma, we 1352 * must invalidate all the TLBs that may still have a reference 1353 * back to our physical address. It only needs to be done once, 1354 * so after updating the PTE to point away from the pages, record 1355 * the most recent TLB invalidation seqno, and if we have not yet 1356 * flushed the TLBs upon release, perform a full invalidation. 1357 */ 1358 for_each_gt(gt, vm->i915, id) 1359 WRITE_ONCE(tlb[id], 1360 intel_gt_next_invalidate_tlb_full(gt)); 1361 } 1362 1363 static void __vma_put_pages(struct i915_vma *vma, unsigned int count) 1364 { 1365 /* We allocate under vma_get_pages, so beware the shrinker */ 1366 GEM_BUG_ON(atomic_read(&vma->pages_count) < count); 1367 1368 if (atomic_sub_return(count, &vma->pages_count) == 0) { 1369 if (vma->pages != vma->obj->mm.pages) { 1370 sg_free_table(vma->pages); 1371 kfree(vma->pages); 1372 } 1373 vma->pages = NULL; 1374 1375 i915_gem_object_unpin_pages(vma->obj); 1376 } 1377 } 1378 1379 I915_SELFTEST_EXPORT void i915_vma_put_pages(struct i915_vma *vma) 1380 { 1381 if (atomic_add_unless(&vma->pages_count, -1, 1)) 1382 return; 1383 1384 __vma_put_pages(vma, 1); 1385 } 1386 1387 static void vma_unbind_pages(struct i915_vma *vma) 1388 { 1389 unsigned int count; 1390 1391 lockdep_assert_held(&vma->vm->mutex); 1392 1393 /* The upper portion of pages_count is the number of bindings */ 1394 count = atomic_read(&vma->pages_count); 1395 count >>= I915_VMA_PAGES_BIAS; 1396 GEM_BUG_ON(!count); 1397 1398 __vma_put_pages(vma, count | count << I915_VMA_PAGES_BIAS); 1399 } 1400 1401 int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, 1402 u64 size, u64 alignment, u64 flags) 1403 { 1404 struct i915_vma_work *work = NULL; 1405 struct dma_fence *moving = NULL; 1406 struct i915_vma_resource *vma_res = NULL; 1407 intel_wakeref_t wakeref = 0; 1408 unsigned int bound; 1409 int err; 1410 1411 assert_vma_held(vma); 1412 GEM_BUG_ON(!ww); 1413 1414 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND); 1415 BUILD_BUG_ON(PIN_USER != I915_VMA_LOCAL_BIND); 1416 1417 GEM_BUG_ON(!(flags & (PIN_USER | PIN_GLOBAL))); 1418 1419 /* First try and grab the pin without rebinding the vma */ 1420 if (try_qad_pin(vma, flags)) 1421 return 0; 1422 1423 err = i915_vma_get_pages(vma); 1424 if (err) 1425 return err; 1426 1427 if (flags & PIN_GLOBAL) 1428 wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm); 1429 1430 if (flags & vma->vm->bind_async_flags) { 1431 /* lock VM */ 1432 err = i915_vm_lock_objects(vma->vm, ww); 1433 if (err) 1434 goto err_rpm; 1435 1436 work = i915_vma_work(); 1437 if (!work) { 1438 err = -ENOMEM; 1439 goto err_rpm; 1440 } 1441 1442 work->vm = vma->vm; 1443 1444 err = i915_gem_object_get_moving_fence(vma->obj, &moving); 1445 if (err) 1446 goto err_rpm; 1447 1448 dma_fence_work_chain(&work->base, moving); 1449 1450 /* Allocate enough page directories to used PTE */ 1451 if (vma->vm->allocate_va_range) { 1452 err = i915_vm_alloc_pt_stash(vma->vm, 1453 &work->stash, 1454 vma->size); 1455 if (err) 1456 goto err_fence; 1457 1458 err = i915_vm_map_pt_stash(vma->vm, &work->stash); 1459 if (err) 1460 goto err_fence; 1461 } 1462 } 1463 1464 vma_res = i915_vma_resource_alloc(); 1465 if (IS_ERR(vma_res)) { 1466 err = PTR_ERR(vma_res); 1467 goto err_fence; 1468 } 1469 1470 /* 1471 * Differentiate between user/kernel vma inside the aliasing-ppgtt. 1472 * 1473 * We conflate the Global GTT with the user's vma when using the 1474 * aliasing-ppgtt, but it is still vitally important to try and 1475 * keep the use cases distinct. For example, userptr objects are 1476 * not allowed inside the Global GTT as that will cause lock 1477 * inversions when we have to evict them the mmu_notifier callbacks - 1478 * but they are allowed to be part of the user ppGTT which can never 1479 * be mapped. As such we try to give the distinct users of the same 1480 * mutex, distinct lockclasses [equivalent to how we keep i915_ggtt 1481 * and i915_ppgtt separate]. 1482 * 1483 * NB this may cause us to mask real lock inversions -- while the 1484 * code is safe today, lockdep may not be able to spot future 1485 * transgressions. 1486 */ 1487 err = mutex_lock_interruptible_nested(&vma->vm->mutex, 1488 !(flags & PIN_GLOBAL)); 1489 if (err) 1490 goto err_vma_res; 1491 1492 /* No more allocations allowed now we hold vm->mutex */ 1493 1494 if (unlikely(i915_vma_is_closed(vma))) { 1495 err = -ENOENT; 1496 goto err_unlock; 1497 } 1498 1499 bound = atomic_read(&vma->flags); 1500 if (unlikely(bound & I915_VMA_ERROR)) { 1501 err = -ENOMEM; 1502 goto err_unlock; 1503 } 1504 1505 if (unlikely(!((bound + 1) & I915_VMA_PIN_MASK))) { 1506 err = -EAGAIN; /* pins are meant to be fairly temporary */ 1507 goto err_unlock; 1508 } 1509 1510 if (unlikely(!(flags & ~bound & I915_VMA_BIND_MASK))) { 1511 if (!(flags & PIN_VALIDATE)) 1512 __i915_vma_pin(vma); 1513 goto err_unlock; 1514 } 1515 1516 err = i915_active_acquire(&vma->active); 1517 if (err) 1518 goto err_unlock; 1519 1520 if (!(bound & I915_VMA_BIND_MASK)) { 1521 err = i915_vma_insert(vma, ww, size, alignment, flags); 1522 if (err) 1523 goto err_active; 1524 1525 if (i915_is_ggtt(vma->vm)) 1526 __i915_vma_set_map_and_fenceable(vma); 1527 } 1528 1529 GEM_BUG_ON(!vma->pages); 1530 err = i915_vma_bind(vma, 1531 vma->obj->pat_index, 1532 flags, work, vma_res); 1533 vma_res = NULL; 1534 if (err) 1535 goto err_remove; 1536 1537 /* There should only be at most 2 active bindings (user, global) */ 1538 GEM_BUG_ON(bound + I915_VMA_PAGES_ACTIVE < bound); 1539 atomic_add(I915_VMA_PAGES_ACTIVE, &vma->pages_count); 1540 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 1541 1542 if (!(flags & PIN_VALIDATE)) { 1543 __i915_vma_pin(vma); 1544 GEM_BUG_ON(!i915_vma_is_pinned(vma)); 1545 } 1546 GEM_BUG_ON(!i915_vma_is_bound(vma, flags)); 1547 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); 1548 1549 err_remove: 1550 if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK)) { 1551 i915_vma_detach(vma); 1552 drm_mm_remove_node(&vma->node); 1553 } 1554 err_active: 1555 i915_active_release(&vma->active); 1556 err_unlock: 1557 mutex_unlock(&vma->vm->mutex); 1558 err_vma_res: 1559 i915_vma_resource_free(vma_res); 1560 err_fence: 1561 if (work) 1562 dma_fence_work_commit_imm(&work->base); 1563 err_rpm: 1564 if (wakeref) 1565 intel_runtime_pm_put(&vma->vm->i915->runtime_pm, wakeref); 1566 1567 if (moving) 1568 dma_fence_put(moving); 1569 1570 i915_vma_put_pages(vma); 1571 return err; 1572 } 1573 1574 static void flush_idle_contexts(struct intel_gt *gt) 1575 { 1576 struct intel_engine_cs *engine; 1577 enum intel_engine_id id; 1578 1579 for_each_engine(engine, gt, id) 1580 intel_engine_flush_barriers(engine); 1581 1582 intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT); 1583 } 1584 1585 static int __i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, 1586 u32 align, unsigned int flags) 1587 { 1588 struct i915_address_space *vm = vma->vm; 1589 struct intel_gt *gt; 1590 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 1591 int err; 1592 1593 do { 1594 err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL); 1595 1596 if (err != -ENOSPC) { 1597 if (!err) { 1598 err = i915_vma_wait_for_bind(vma); 1599 if (err) 1600 i915_vma_unpin(vma); 1601 } 1602 return err; 1603 } 1604 1605 /* Unlike i915_vma_pin, we don't take no for an answer! */ 1606 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 1607 flush_idle_contexts(gt); 1608 if (mutex_lock_interruptible(&vm->mutex) == 0) { 1609 /* 1610 * We pass NULL ww here, as we don't want to unbind 1611 * locked objects when called from execbuf when pinning 1612 * is removed. This would probably regress badly. 1613 */ 1614 i915_gem_evict_vm(vm, NULL, NULL); 1615 mutex_unlock(&vm->mutex); 1616 } 1617 } while (1); 1618 } 1619 1620 int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, 1621 u32 align, unsigned int flags) 1622 { 1623 struct i915_gem_ww_ctx _ww; 1624 int err; 1625 1626 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 1627 1628 if (ww) 1629 return __i915_ggtt_pin(vma, ww, align, flags); 1630 1631 lockdep_assert_not_held(&vma->obj->base.resv->lock.base); 1632 1633 for_i915_gem_ww(&_ww, err, true) { 1634 err = i915_gem_object_lock(vma->obj, &_ww); 1635 if (!err) 1636 err = __i915_ggtt_pin(vma, &_ww, align, flags); 1637 } 1638 1639 return err; 1640 } 1641 1642 /** 1643 * i915_ggtt_clear_scanout - Clear scanout flag for all objects ggtt vmas 1644 * @obj: i915 GEM object 1645 * This function clears scanout flags for objects ggtt vmas. These flags are set 1646 * when object is pinned for display use and this function to clear them all is 1647 * targeted to be called by frontbuffer tracking code when the frontbuffer is 1648 * about to be released. 1649 */ 1650 void i915_ggtt_clear_scanout(struct drm_i915_gem_object *obj) 1651 { 1652 struct i915_vma *vma; 1653 1654 spin_lock(&obj->vma.lock); 1655 for_each_ggtt_vma(vma, obj) { 1656 i915_vma_clear_scanout(vma); 1657 vma->display_alignment = I915_GTT_MIN_ALIGNMENT; 1658 } 1659 spin_unlock(&obj->vma.lock); 1660 } 1661 1662 static void __vma_close(struct i915_vma *vma, struct intel_gt *gt) 1663 { 1664 /* 1665 * We defer actually closing, unbinding and destroying the VMA until 1666 * the next idle point, or if the object is freed in the meantime. By 1667 * postponing the unbind, we allow for it to be resurrected by the 1668 * client, avoiding the work required to rebind the VMA. This is 1669 * advantageous for DRI, where the client/server pass objects 1670 * between themselves, temporarily opening a local VMA to the 1671 * object, and then closing it again. The same object is then reused 1672 * on the next frame (or two, depending on the depth of the swap queue) 1673 * causing us to rebind the VMA once more. This ends up being a lot 1674 * of wasted work for the steady state. 1675 */ 1676 GEM_BUG_ON(i915_vma_is_closed(vma)); 1677 list_add(&vma->closed_link, >->closed_vma); 1678 } 1679 1680 void i915_vma_close(struct i915_vma *vma) 1681 { 1682 struct intel_gt *gt = vma->vm->gt; 1683 unsigned long flags; 1684 1685 if (i915_vma_is_ggtt(vma)) 1686 return; 1687 1688 GEM_BUG_ON(!atomic_read(&vma->open_count)); 1689 if (atomic_dec_and_lock_irqsave(&vma->open_count, 1690 >->closed_lock, 1691 flags)) { 1692 __vma_close(vma, gt); 1693 spin_unlock_irqrestore(>->closed_lock, flags); 1694 } 1695 } 1696 1697 static void __i915_vma_remove_closed(struct i915_vma *vma) 1698 { 1699 list_del_init(&vma->closed_link); 1700 } 1701 1702 void i915_vma_reopen(struct i915_vma *vma) 1703 { 1704 struct intel_gt *gt = vma->vm->gt; 1705 1706 spin_lock_irq(>->closed_lock); 1707 if (i915_vma_is_closed(vma)) 1708 __i915_vma_remove_closed(vma); 1709 spin_unlock_irq(>->closed_lock); 1710 } 1711 1712 static void force_unbind(struct i915_vma *vma) 1713 { 1714 if (!drm_mm_node_allocated(&vma->node)) 1715 return; 1716 1717 atomic_and(~I915_VMA_PIN_MASK, &vma->flags); 1718 WARN_ON(__i915_vma_unbind(vma)); 1719 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 1720 } 1721 1722 static void release_references(struct i915_vma *vma, struct intel_gt *gt, 1723 bool vm_ddestroy) 1724 { 1725 struct drm_i915_gem_object *obj = vma->obj; 1726 1727 GEM_BUG_ON(i915_vma_is_active(vma)); 1728 1729 spin_lock(&obj->vma.lock); 1730 list_del(&vma->obj_link); 1731 if (!RB_EMPTY_NODE(&vma->obj_node)) 1732 rb_erase(&vma->obj_node, &obj->vma.tree); 1733 1734 spin_unlock(&obj->vma.lock); 1735 1736 spin_lock_irq(>->closed_lock); 1737 __i915_vma_remove_closed(vma); 1738 spin_unlock_irq(>->closed_lock); 1739 1740 if (vm_ddestroy) 1741 i915_vm_resv_put(vma->vm); 1742 1743 /* Wait for async active retire */ 1744 i915_active_wait(&vma->active); 1745 i915_active_fini(&vma->active); 1746 GEM_WARN_ON(vma->resource); 1747 i915_vma_free(vma); 1748 } 1749 1750 /* 1751 * i915_vma_destroy_locked - Remove all weak reference to the vma and put 1752 * the initial reference. 1753 * 1754 * This function should be called when it's decided the vma isn't needed 1755 * anymore. The caller must assure that it doesn't race with another lookup 1756 * plus destroy, typically by taking an appropriate reference. 1757 * 1758 * Current callsites are 1759 * - __i915_gem_object_pages_fini() 1760 * - __i915_vm_close() - Blocks the above function by taking a reference on 1761 * the object. 1762 * - __i915_vma_parked() - Blocks the above functions by taking a reference 1763 * on the vm and a reference on the object. Also takes the object lock so 1764 * destruction from __i915_vma_parked() can be blocked by holding the 1765 * object lock. Since the object lock is only allowed from within i915 with 1766 * an object refcount, holding the object lock also implicitly blocks the 1767 * vma freeing from __i915_gem_object_pages_fini(). 1768 * 1769 * Because of locks taken during destruction, a vma is also guaranteed to 1770 * stay alive while the following locks are held if it was looked up while 1771 * holding one of the locks: 1772 * - vm->mutex 1773 * - obj->vma.lock 1774 * - gt->closed_lock 1775 */ 1776 void i915_vma_destroy_locked(struct i915_vma *vma) 1777 { 1778 lockdep_assert_held(&vma->vm->mutex); 1779 1780 force_unbind(vma); 1781 list_del_init(&vma->vm_link); 1782 release_references(vma, vma->vm->gt, false); 1783 } 1784 1785 void i915_vma_destroy(struct i915_vma *vma) 1786 { 1787 struct intel_gt *gt; 1788 bool vm_ddestroy; 1789 1790 mutex_lock(&vma->vm->mutex); 1791 force_unbind(vma); 1792 list_del_init(&vma->vm_link); 1793 vm_ddestroy = vma->vm_ddestroy; 1794 vma->vm_ddestroy = false; 1795 1796 /* vma->vm may be freed when releasing vma->vm->mutex. */ 1797 gt = vma->vm->gt; 1798 mutex_unlock(&vma->vm->mutex); 1799 release_references(vma, gt, vm_ddestroy); 1800 } 1801 1802 void i915_vma_parked(struct intel_gt *gt) 1803 { 1804 struct i915_vma *vma, *next; 1805 LIST_HEAD(closed); 1806 1807 spin_lock_irq(>->closed_lock); 1808 list_for_each_entry_safe(vma, next, >->closed_vma, closed_link) { 1809 struct drm_i915_gem_object *obj = vma->obj; 1810 struct i915_address_space *vm = vma->vm; 1811 1812 /* XXX All to avoid keeping a reference on i915_vma itself */ 1813 1814 if (!kref_get_unless_zero(&obj->base.refcount)) 1815 continue; 1816 1817 if (!i915_vm_tryget(vm)) { 1818 i915_gem_object_put(obj); 1819 continue; 1820 } 1821 1822 list_move(&vma->closed_link, &closed); 1823 } 1824 spin_unlock_irq(>->closed_lock); 1825 1826 /* As the GT is held idle, no vma can be reopened as we destroy them */ 1827 list_for_each_entry_safe(vma, next, &closed, closed_link) { 1828 struct drm_i915_gem_object *obj = vma->obj; 1829 struct i915_address_space *vm = vma->vm; 1830 1831 if (i915_gem_object_trylock(obj, NULL)) { 1832 INIT_LIST_HEAD(&vma->closed_link); 1833 i915_vma_destroy(vma); 1834 i915_gem_object_unlock(obj); 1835 } else { 1836 /* back you go.. */ 1837 spin_lock_irq(>->closed_lock); 1838 list_add(&vma->closed_link, >->closed_vma); 1839 spin_unlock_irq(>->closed_lock); 1840 } 1841 1842 i915_gem_object_put(obj); 1843 i915_vm_put(vm); 1844 } 1845 } 1846 1847 static void __i915_vma_iounmap(struct i915_vma *vma) 1848 { 1849 GEM_BUG_ON(i915_vma_is_pinned(vma)); 1850 1851 if (vma->iomap == NULL) 1852 return; 1853 1854 if (page_unmask_bits(vma->iomap)) 1855 __i915_gem_object_release_map(vma->obj); 1856 else 1857 io_mapping_unmap(vma->iomap); 1858 vma->iomap = NULL; 1859 } 1860 1861 void i915_vma_revoke_mmap(struct i915_vma *vma) 1862 { 1863 struct drm_vma_offset_node *node; 1864 u64 vma_offset; 1865 1866 if (!i915_vma_has_userfault(vma)) 1867 return; 1868 1869 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma)); 1870 GEM_BUG_ON(!vma->obj->userfault_count); 1871 1872 node = &vma->mmo->vma_node; 1873 vma_offset = vma->gtt_view.partial.offset << PAGE_SHIFT; 1874 unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping, 1875 drm_vma_node_offset_addr(node) + vma_offset, 1876 vma->size, 1877 1); 1878 1879 i915_vma_unset_userfault(vma); 1880 if (!--vma->obj->userfault_count) 1881 list_del(&vma->obj->userfault_link); 1882 } 1883 1884 static int 1885 __i915_request_await_bind(struct i915_request *rq, struct i915_vma *vma) 1886 { 1887 return __i915_request_await_exclusive(rq, &vma->active); 1888 } 1889 1890 static int __i915_vma_move_to_active(struct i915_vma *vma, struct i915_request *rq) 1891 { 1892 int err; 1893 1894 /* Wait for the vma to be bound before we start! */ 1895 err = __i915_request_await_bind(rq, vma); 1896 if (err) 1897 return err; 1898 1899 return i915_active_add_request(&vma->active, rq); 1900 } 1901 1902 int _i915_vma_move_to_active(struct i915_vma *vma, 1903 struct i915_request *rq, 1904 struct dma_fence *fence, 1905 unsigned int flags) 1906 { 1907 struct drm_i915_gem_object *obj = vma->obj; 1908 int err; 1909 1910 assert_object_held(obj); 1911 1912 GEM_BUG_ON(!vma->pages); 1913 1914 if (!(flags & __EXEC_OBJECT_NO_REQUEST_AWAIT)) { 1915 err = i915_request_await_object(rq, vma->obj, flags & EXEC_OBJECT_WRITE); 1916 if (unlikely(err)) 1917 return err; 1918 } 1919 err = __i915_vma_move_to_active(vma, rq); 1920 if (unlikely(err)) 1921 return err; 1922 1923 /* 1924 * Reserve fences slot early to prevent an allocation after preparing 1925 * the workload and associating fences with dma_resv. 1926 */ 1927 if (fence && !(flags & __EXEC_OBJECT_NO_RESERVE)) { 1928 struct dma_fence *curr; 1929 int idx; 1930 1931 dma_fence_array_for_each(curr, idx, fence) 1932 ; 1933 err = dma_resv_reserve_fences(vma->obj->base.resv, idx); 1934 if (unlikely(err)) 1935 return err; 1936 } 1937 1938 if (flags & EXEC_OBJECT_WRITE) { 1939 struct intel_frontbuffer *front; 1940 1941 front = i915_gem_object_get_frontbuffer(obj); 1942 if (unlikely(front)) { 1943 if (intel_frontbuffer_invalidate(front, ORIGIN_CS)) 1944 i915_active_add_request(&front->write, rq); 1945 intel_frontbuffer_put(front); 1946 } 1947 } 1948 1949 if (fence) { 1950 struct dma_fence *curr; 1951 enum dma_resv_usage usage; 1952 int idx; 1953 1954 if (flags & EXEC_OBJECT_WRITE) { 1955 usage = DMA_RESV_USAGE_WRITE; 1956 obj->write_domain = I915_GEM_DOMAIN_RENDER; 1957 obj->read_domains = 0; 1958 } else { 1959 usage = DMA_RESV_USAGE_READ; 1960 obj->write_domain = 0; 1961 } 1962 1963 dma_fence_array_for_each(curr, idx, fence) 1964 dma_resv_add_fence(vma->obj->base.resv, curr, usage); 1965 } 1966 1967 if (flags & EXEC_OBJECT_NEEDS_FENCE && vma->fence) 1968 i915_active_add_request(&vma->fence->active, rq); 1969 1970 obj->read_domains |= I915_GEM_GPU_DOMAINS; 1971 obj->mm.dirty = true; 1972 1973 GEM_BUG_ON(!i915_vma_is_active(vma)); 1974 return 0; 1975 } 1976 1977 struct dma_fence *__i915_vma_evict(struct i915_vma *vma, bool async) 1978 { 1979 struct i915_vma_resource *vma_res = vma->resource; 1980 struct dma_fence *unbind_fence; 1981 1982 GEM_BUG_ON(i915_vma_is_pinned(vma)); 1983 assert_vma_held_evict(vma); 1984 1985 if (i915_vma_is_map_and_fenceable(vma)) { 1986 /* Force a pagefault for domain tracking on next user access */ 1987 i915_vma_revoke_mmap(vma); 1988 1989 /* 1990 * Check that we have flushed all writes through the GGTT 1991 * before the unbind, other due to non-strict nature of those 1992 * indirect writes they may end up referencing the GGTT PTE 1993 * after the unbind. 1994 * 1995 * Note that we may be concurrently poking at the GGTT_WRITE 1996 * bit from set-domain, as we mark all GGTT vma associated 1997 * with an object. We know this is for another vma, as we 1998 * are currently unbinding this one -- so if this vma will be 1999 * reused, it will be refaulted and have its dirty bit set 2000 * before the next write. 2001 */ 2002 i915_vma_flush_writes(vma); 2003 2004 /* release the fence reg _after_ flushing */ 2005 i915_vma_revoke_fence(vma); 2006 2007 clear_bit(I915_VMA_CAN_FENCE_BIT, __i915_vma_flags(vma)); 2008 } 2009 2010 __i915_vma_iounmap(vma); 2011 2012 GEM_BUG_ON(vma->fence); 2013 GEM_BUG_ON(i915_vma_has_userfault(vma)); 2014 2015 /* Object backend must be async capable. */ 2016 GEM_WARN_ON(async && !vma->resource->bi.pages_rsgt); 2017 2018 /* If vm is not open, unbind is a nop. */ 2019 vma_res->needs_wakeref = i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND) && 2020 kref_read(&vma->vm->ref); 2021 vma_res->skip_pte_rewrite = !kref_read(&vma->vm->ref) || 2022 vma->vm->skip_pte_rewrite; 2023 trace_i915_vma_unbind(vma); 2024 2025 if (async) 2026 unbind_fence = i915_vma_resource_unbind(vma_res, 2027 vma->obj->mm.tlb); 2028 else 2029 unbind_fence = i915_vma_resource_unbind(vma_res, NULL); 2030 2031 vma->resource = NULL; 2032 2033 atomic_and(~(I915_VMA_BIND_MASK | I915_VMA_ERROR | I915_VMA_GGTT_WRITE), 2034 &vma->flags); 2035 2036 i915_vma_detach(vma); 2037 2038 if (!async) { 2039 if (unbind_fence) { 2040 dma_fence_wait(unbind_fence, false); 2041 dma_fence_put(unbind_fence); 2042 unbind_fence = NULL; 2043 } 2044 vma_invalidate_tlb(vma->vm, vma->obj->mm.tlb); 2045 } 2046 2047 /* 2048 * Binding itself may not have completed until the unbind fence signals, 2049 * so don't drop the pages until that happens, unless the resource is 2050 * async_capable. 2051 */ 2052 2053 vma_unbind_pages(vma); 2054 return unbind_fence; 2055 } 2056 2057 int __i915_vma_unbind(struct i915_vma *vma) 2058 { 2059 int ret; 2060 2061 lockdep_assert_held(&vma->vm->mutex); 2062 assert_vma_held_evict(vma); 2063 2064 if (!drm_mm_node_allocated(&vma->node)) 2065 return 0; 2066 2067 if (i915_vma_is_pinned(vma)) { 2068 vma_print_allocator(vma, "is pinned"); 2069 return -EAGAIN; 2070 } 2071 2072 /* 2073 * After confirming that no one else is pinning this vma, wait for 2074 * any laggards who may have crept in during the wait (through 2075 * a residual pin skipping the vm->mutex) to complete. 2076 */ 2077 ret = i915_vma_sync(vma); 2078 if (ret) 2079 return ret; 2080 2081 GEM_BUG_ON(i915_vma_is_active(vma)); 2082 __i915_vma_evict(vma, false); 2083 2084 drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */ 2085 return 0; 2086 } 2087 2088 static struct dma_fence *__i915_vma_unbind_async(struct i915_vma *vma) 2089 { 2090 struct dma_fence *fence; 2091 2092 lockdep_assert_held(&vma->vm->mutex); 2093 2094 if (!drm_mm_node_allocated(&vma->node)) 2095 return NULL; 2096 2097 if (i915_vma_is_pinned(vma) || 2098 &vma->obj->mm.rsgt->table != vma->resource->bi.pages) 2099 return ERR_PTR(-EAGAIN); 2100 2101 /* 2102 * We probably need to replace this with awaiting the fences of the 2103 * object's dma_resv when the vma active goes away. When doing that 2104 * we need to be careful to not add the vma_resource unbind fence 2105 * immediately to the object's dma_resv, because then unbinding 2106 * the next vma from the object, in case there are many, will 2107 * actually await the unbinding of the previous vmas, which is 2108 * undesirable. 2109 */ 2110 if (i915_sw_fence_await_active(&vma->resource->chain, &vma->active, 2111 I915_ACTIVE_AWAIT_EXCL | 2112 I915_ACTIVE_AWAIT_ACTIVE) < 0) { 2113 return ERR_PTR(-EBUSY); 2114 } 2115 2116 fence = __i915_vma_evict(vma, true); 2117 2118 drm_mm_remove_node(&vma->node); /* pairs with i915_vma_release() */ 2119 2120 return fence; 2121 } 2122 2123 int i915_vma_unbind(struct i915_vma *vma) 2124 { 2125 struct i915_address_space *vm = vma->vm; 2126 intel_wakeref_t wakeref = 0; 2127 int err; 2128 2129 assert_object_held_shared(vma->obj); 2130 2131 /* Optimistic wait before taking the mutex */ 2132 err = i915_vma_sync(vma); 2133 if (err) 2134 return err; 2135 2136 if (!drm_mm_node_allocated(&vma->node)) 2137 return 0; 2138 2139 if (i915_vma_is_pinned(vma)) { 2140 vma_print_allocator(vma, "is pinned"); 2141 return -EAGAIN; 2142 } 2143 2144 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 2145 /* XXX not always required: nop_clear_range */ 2146 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm); 2147 2148 err = mutex_lock_interruptible_nested(&vma->vm->mutex, !wakeref); 2149 if (err) 2150 goto out_rpm; 2151 2152 err = __i915_vma_unbind(vma); 2153 mutex_unlock(&vm->mutex); 2154 2155 out_rpm: 2156 if (wakeref) 2157 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref); 2158 return err; 2159 } 2160 2161 int i915_vma_unbind_async(struct i915_vma *vma, bool trylock_vm) 2162 { 2163 struct drm_i915_gem_object *obj = vma->obj; 2164 struct i915_address_space *vm = vma->vm; 2165 intel_wakeref_t wakeref = 0; 2166 struct dma_fence *fence; 2167 int err; 2168 2169 /* 2170 * We need the dma-resv lock since we add the 2171 * unbind fence to the dma-resv object. 2172 */ 2173 assert_object_held(obj); 2174 2175 if (!drm_mm_node_allocated(&vma->node)) 2176 return 0; 2177 2178 if (i915_vma_is_pinned(vma)) { 2179 vma_print_allocator(vma, "is pinned"); 2180 return -EAGAIN; 2181 } 2182 2183 if (!obj->mm.rsgt) 2184 return -EBUSY; 2185 2186 err = dma_resv_reserve_fences(obj->base.resv, 2); 2187 if (err) 2188 return -EBUSY; 2189 2190 /* 2191 * It would be great if we could grab this wakeref from the 2192 * async unbind work if needed, but we can't because it uses 2193 * kmalloc and it's in the dma-fence signalling critical path. 2194 */ 2195 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 2196 wakeref = intel_runtime_pm_get(&vm->i915->runtime_pm); 2197 2198 if (trylock_vm && !mutex_trylock(&vm->mutex)) { 2199 err = -EBUSY; 2200 goto out_rpm; 2201 } else if (!trylock_vm) { 2202 err = mutex_lock_interruptible_nested(&vm->mutex, !wakeref); 2203 if (err) 2204 goto out_rpm; 2205 } 2206 2207 fence = __i915_vma_unbind_async(vma); 2208 mutex_unlock(&vm->mutex); 2209 if (IS_ERR_OR_NULL(fence)) { 2210 err = PTR_ERR_OR_ZERO(fence); 2211 goto out_rpm; 2212 } 2213 2214 dma_resv_add_fence(obj->base.resv, fence, DMA_RESV_USAGE_READ); 2215 dma_fence_put(fence); 2216 2217 out_rpm: 2218 if (wakeref) 2219 intel_runtime_pm_put(&vm->i915->runtime_pm, wakeref); 2220 return err; 2221 } 2222 2223 int i915_vma_unbind_unlocked(struct i915_vma *vma) 2224 { 2225 int err; 2226 2227 i915_gem_object_lock(vma->obj, NULL); 2228 err = i915_vma_unbind(vma); 2229 i915_gem_object_unlock(vma->obj); 2230 2231 return err; 2232 } 2233 2234 struct i915_vma *i915_vma_make_unshrinkable(struct i915_vma *vma) 2235 { 2236 i915_gem_object_make_unshrinkable(vma->obj); 2237 return vma; 2238 } 2239 2240 void i915_vma_make_shrinkable(struct i915_vma *vma) 2241 { 2242 i915_gem_object_make_shrinkable(vma->obj); 2243 } 2244 2245 void i915_vma_make_purgeable(struct i915_vma *vma) 2246 { 2247 i915_gem_object_make_purgeable(vma->obj); 2248 } 2249 2250 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2251 #include "selftests/i915_vma.c" 2252 #endif 2253 2254 void i915_vma_module_exit(void) 2255 { 2256 kmem_cache_destroy(slab_vmas); 2257 } 2258 2259 int __init i915_vma_module_init(void) 2260 { 2261 slab_vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN); 2262 if (!slab_vmas) 2263 return -ENOMEM; 2264 2265 return 0; 2266 } 2267