1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include "i915_vma.h" 26 27 #include "i915_drv.h" 28 #include "intel_ringbuffer.h" 29 #include "intel_frontbuffer.h" 30 31 #include <drm/drm_gem.h> 32 33 static void 34 i915_vma_retire(struct i915_gem_active *active, 35 struct drm_i915_gem_request *rq) 36 { 37 const unsigned int idx = rq->engine->id; 38 struct i915_vma *vma = 39 container_of(active, struct i915_vma, last_read[idx]); 40 struct drm_i915_gem_object *obj = vma->obj; 41 42 GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx)); 43 44 i915_vma_clear_active(vma, idx); 45 if (i915_vma_is_active(vma)) 46 return; 47 48 list_move_tail(&vma->vm_link, &vma->vm->inactive_list); 49 if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma))) 50 WARN_ON(i915_vma_unbind(vma)); 51 52 GEM_BUG_ON(!i915_gem_object_is_active(obj)); 53 if (--obj->active_count) 54 return; 55 56 /* Bump our place on the bound list to keep it roughly in LRU order 57 * so that we don't steal from recently used but inactive objects 58 * (unless we are forced to ofc!) 59 */ 60 if (obj->bind_count) 61 list_move_tail(&obj->global_link, &rq->i915->mm.bound_list); 62 63 obj->mm.dirty = true; /* be paranoid */ 64 65 if (i915_gem_object_has_active_reference(obj)) { 66 i915_gem_object_clear_active_reference(obj); 67 i915_gem_object_put(obj); 68 } 69 } 70 71 static struct i915_vma * 72 __i915_vma_create(struct drm_i915_gem_object *obj, 73 struct i915_address_space *vm, 74 const struct i915_ggtt_view *view) 75 { 76 struct i915_vma *vma; 77 struct rb_node *rb, **p; 78 int i; 79 80 GEM_BUG_ON(vm->closed); 81 82 vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL); 83 if (vma == NULL) 84 return ERR_PTR(-ENOMEM); 85 86 INIT_LIST_HEAD(&vma->exec_list); 87 for (i = 0; i < ARRAY_SIZE(vma->last_read); i++) 88 init_request_active(&vma->last_read[i], i915_vma_retire); 89 init_request_active(&vma->last_fence, NULL); 90 list_add(&vma->vm_link, &vm->unbound_list); 91 vma->vm = vm; 92 vma->obj = obj; 93 vma->size = obj->base.size; 94 95 if (view) { 96 vma->ggtt_view = *view; 97 if (view->type == I915_GGTT_VIEW_PARTIAL) { 98 vma->size = view->params.partial.size; 99 vma->size <<= PAGE_SHIFT; 100 } else if (view->type == I915_GGTT_VIEW_ROTATED) { 101 vma->size = 102 intel_rotation_info_size(&view->params.rotated); 103 vma->size <<= PAGE_SHIFT; 104 } 105 } 106 107 if (i915_is_ggtt(vm)) { 108 vma->flags |= I915_VMA_GGTT; 109 list_add(&vma->obj_link, &obj->vma_list); 110 } else { 111 i915_ppgtt_get(i915_vm_to_ppgtt(vm)); 112 list_add_tail(&vma->obj_link, &obj->vma_list); 113 } 114 115 rb = NULL; 116 p = &obj->vma_tree.rb_node; 117 while (*p) { 118 struct i915_vma *pos; 119 120 rb = *p; 121 pos = rb_entry(rb, struct i915_vma, obj_node); 122 if (i915_vma_compare(pos, vm, view) < 0) 123 p = &rb->rb_right; 124 else 125 p = &rb->rb_left; 126 } 127 rb_link_node(&vma->obj_node, rb, p); 128 rb_insert_color(&vma->obj_node, &obj->vma_tree); 129 130 return vma; 131 } 132 133 struct i915_vma * 134 i915_vma_create(struct drm_i915_gem_object *obj, 135 struct i915_address_space *vm, 136 const struct i915_ggtt_view *view) 137 { 138 lockdep_assert_held(&obj->base.dev->struct_mutex); 139 GEM_BUG_ON(view && !i915_is_ggtt(vm)); 140 GEM_BUG_ON(i915_gem_obj_to_vma(obj, vm, view)); 141 142 return __i915_vma_create(obj, vm, view); 143 } 144 145 /** 146 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space. 147 * @vma: VMA to map 148 * @cache_level: mapping cache level 149 * @flags: flags like global or local mapping 150 * 151 * DMA addresses are taken from the scatter-gather table of this object (or of 152 * this VMA in case of non-default GGTT views) and PTE entries set up. 153 * Note that DMA addresses are also the only part of the SG table we care about. 154 */ 155 int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, 156 u32 flags) 157 { 158 u32 bind_flags; 159 u32 vma_flags; 160 int ret; 161 162 if (WARN_ON(flags == 0)) 163 return -EINVAL; 164 165 bind_flags = 0; 166 if (flags & PIN_GLOBAL) 167 bind_flags |= I915_VMA_GLOBAL_BIND; 168 if (flags & PIN_USER) 169 bind_flags |= I915_VMA_LOCAL_BIND; 170 171 vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); 172 if (flags & PIN_UPDATE) 173 bind_flags |= vma_flags; 174 else 175 bind_flags &= ~vma_flags; 176 if (bind_flags == 0) 177 return 0; 178 179 if (vma_flags == 0 && vma->vm->allocate_va_range) { 180 trace_i915_va_alloc(vma); 181 ret = vma->vm->allocate_va_range(vma->vm, 182 vma->node.start, 183 vma->node.size); 184 if (ret) 185 return ret; 186 } 187 188 ret = vma->vm->bind_vma(vma, cache_level, bind_flags); 189 if (ret) 190 return ret; 191 192 vma->flags |= bind_flags; 193 return 0; 194 } 195 196 void __iomem *i915_vma_pin_iomap(struct i915_vma *vma) 197 { 198 void __iomem *ptr; 199 200 /* Access through the GTT requires the device to be awake. */ 201 assert_rpm_wakelock_held(to_i915(vma->vm->dev)); 202 203 lockdep_assert_held(&vma->vm->dev->struct_mutex); 204 if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) 205 return IO_ERR_PTR(-ENODEV); 206 207 GEM_BUG_ON(!i915_vma_is_ggtt(vma)); 208 GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0); 209 210 ptr = vma->iomap; 211 if (ptr == NULL) { 212 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->mappable, 213 vma->node.start, 214 vma->node.size); 215 if (ptr == NULL) 216 return IO_ERR_PTR(-ENOMEM); 217 218 vma->iomap = ptr; 219 } 220 221 __i915_vma_pin(vma); 222 return ptr; 223 } 224 225 void i915_vma_unpin_and_release(struct i915_vma **p_vma) 226 { 227 struct i915_vma *vma; 228 struct drm_i915_gem_object *obj; 229 230 vma = fetch_and_zero(p_vma); 231 if (!vma) 232 return; 233 234 obj = vma->obj; 235 236 i915_vma_unpin(vma); 237 i915_vma_close(vma); 238 239 __i915_gem_object_release_unless_active(obj); 240 } 241 242 bool 243 i915_vma_misplaced(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) 244 { 245 if (!drm_mm_node_allocated(&vma->node)) 246 return false; 247 248 if (vma->node.size < size) 249 return true; 250 251 if (alignment && vma->node.start & (alignment - 1)) 252 return true; 253 254 if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma)) 255 return true; 256 257 if (flags & PIN_OFFSET_BIAS && 258 vma->node.start < (flags & PIN_OFFSET_MASK)) 259 return true; 260 261 if (flags & PIN_OFFSET_FIXED && 262 vma->node.start != (flags & PIN_OFFSET_MASK)) 263 return true; 264 265 return false; 266 } 267 268 void __i915_vma_set_map_and_fenceable(struct i915_vma *vma) 269 { 270 struct drm_i915_gem_object *obj = vma->obj; 271 struct drm_i915_private *dev_priv = to_i915(obj->base.dev); 272 bool mappable, fenceable; 273 u32 fence_size, fence_alignment; 274 275 fence_size = i915_gem_get_ggtt_size(dev_priv, 276 vma->size, 277 i915_gem_object_get_tiling(obj)); 278 fence_alignment = i915_gem_get_ggtt_alignment(dev_priv, 279 vma->size, 280 i915_gem_object_get_tiling(obj), 281 true); 282 283 fenceable = (vma->node.size == fence_size && 284 (vma->node.start & (fence_alignment - 1)) == 0); 285 286 mappable = (vma->node.start + fence_size <= 287 dev_priv->ggtt.mappable_end); 288 289 /* 290 * Explicitly disable for rotated VMA since the display does not 291 * need the fence and the VMA is not accessible to other users. 292 */ 293 if (mappable && fenceable && 294 vma->ggtt_view.type != I915_GGTT_VIEW_ROTATED) 295 vma->flags |= I915_VMA_CAN_FENCE; 296 else 297 vma->flags &= ~I915_VMA_CAN_FENCE; 298 } 299 300 bool i915_gem_valid_gtt_space(struct i915_vma *vma, 301 unsigned long cache_level) 302 { 303 struct drm_mm_node *gtt_space = &vma->node; 304 struct drm_mm_node *other; 305 306 /* 307 * On some machines we have to be careful when putting differing types 308 * of snoopable memory together to avoid the prefetcher crossing memory 309 * domains and dying. During vm initialisation, we decide whether or not 310 * these constraints apply and set the drm_mm.color_adjust 311 * appropriately. 312 */ 313 if (vma->vm->mm.color_adjust == NULL) 314 return true; 315 316 if (!drm_mm_node_allocated(gtt_space)) 317 return true; 318 319 if (list_empty(>t_space->node_list)) 320 return true; 321 322 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list); 323 if (other->allocated && !other->hole_follows && other->color != cache_level) 324 return false; 325 326 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list); 327 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level) 328 return false; 329 330 return true; 331 } 332 333 /** 334 * i915_vma_insert - finds a slot for the vma in its address space 335 * @vma: the vma 336 * @size: requested size in bytes (can be larger than the VMA) 337 * @alignment: required alignment 338 * @flags: mask of PIN_* flags to use 339 * 340 * First we try to allocate some free space that meets the requirements for 341 * the VMA. Failiing that, if the flags permit, it will evict an old VMA, 342 * preferrably the oldest idle entry to make room for the new VMA. 343 * 344 * Returns: 345 * 0 on success, negative error code otherwise. 346 */ 347 static int 348 i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags) 349 { 350 struct drm_i915_private *dev_priv = to_i915(vma->vm->dev); 351 struct drm_i915_gem_object *obj = vma->obj; 352 u64 start, end; 353 int ret; 354 355 GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND)); 356 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 357 358 size = max(size, vma->size); 359 if (flags & PIN_MAPPABLE) 360 size = i915_gem_get_ggtt_size(dev_priv, size, 361 i915_gem_object_get_tiling(obj)); 362 363 alignment = max(max(alignment, vma->display_alignment), 364 i915_gem_get_ggtt_alignment(dev_priv, size, 365 i915_gem_object_get_tiling(obj), 366 flags & PIN_MAPPABLE)); 367 368 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0; 369 370 end = vma->vm->total; 371 if (flags & PIN_MAPPABLE) 372 end = min_t(u64, end, dev_priv->ggtt.mappable_end); 373 if (flags & PIN_ZONE_4G) 374 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE); 375 376 /* If binding the object/GGTT view requires more space than the entire 377 * aperture has, reject it early before evicting everything in a vain 378 * attempt to find space. 379 */ 380 if (size > end) { 381 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n", 382 size, obj->base.size, 383 flags & PIN_MAPPABLE ? "mappable" : "total", 384 end); 385 return -E2BIG; 386 } 387 388 ret = i915_gem_object_pin_pages(obj); 389 if (ret) 390 return ret; 391 392 if (flags & PIN_OFFSET_FIXED) { 393 u64 offset = flags & PIN_OFFSET_MASK; 394 if (offset & (alignment - 1) || offset > end - size) { 395 ret = -EINVAL; 396 goto err_unpin; 397 } 398 399 vma->node.start = offset; 400 vma->node.size = size; 401 vma->node.color = obj->cache_level; 402 ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node); 403 if (ret) { 404 ret = i915_gem_evict_for_vma(vma); 405 if (ret == 0) 406 ret = drm_mm_reserve_node(&vma->vm->mm, &vma->node); 407 if (ret) 408 goto err_unpin; 409 } 410 } else { 411 u32 search_flag, alloc_flag; 412 413 if (flags & PIN_HIGH) { 414 search_flag = DRM_MM_SEARCH_BELOW; 415 alloc_flag = DRM_MM_CREATE_TOP; 416 } else { 417 search_flag = DRM_MM_SEARCH_DEFAULT; 418 alloc_flag = DRM_MM_CREATE_DEFAULT; 419 } 420 421 /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks, 422 * so we know that we always have a minimum alignment of 4096. 423 * The drm_mm range manager is optimised to return results 424 * with zero alignment, so where possible use the optimal 425 * path. 426 */ 427 if (alignment <= 4096) 428 alignment = 0; 429 430 search_free: 431 ret = drm_mm_insert_node_in_range_generic(&vma->vm->mm, 432 &vma->node, 433 size, alignment, 434 obj->cache_level, 435 start, end, 436 search_flag, 437 alloc_flag); 438 if (ret) { 439 ret = i915_gem_evict_something(vma->vm, size, alignment, 440 obj->cache_level, 441 start, end, 442 flags); 443 if (ret == 0) 444 goto search_free; 445 446 goto err_unpin; 447 } 448 449 GEM_BUG_ON(vma->node.start < start); 450 GEM_BUG_ON(vma->node.start + vma->node.size > end); 451 } 452 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level)); 453 454 list_move_tail(&obj->global_link, &dev_priv->mm.bound_list); 455 list_move_tail(&vma->vm_link, &vma->vm->inactive_list); 456 obj->bind_count++; 457 GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); 458 459 return 0; 460 461 err_unpin: 462 i915_gem_object_unpin_pages(obj); 463 return ret; 464 } 465 466 int __i915_vma_do_pin(struct i915_vma *vma, 467 u64 size, u64 alignment, u64 flags) 468 { 469 unsigned int bound = vma->flags; 470 int ret; 471 472 lockdep_assert_held(&vma->vm->dev->struct_mutex); 473 GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0); 474 GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma)); 475 476 if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) { 477 ret = -EBUSY; 478 goto err; 479 } 480 481 if ((bound & I915_VMA_BIND_MASK) == 0) { 482 ret = i915_vma_insert(vma, size, alignment, flags); 483 if (ret) 484 goto err; 485 } 486 487 ret = i915_vma_bind(vma, vma->obj->cache_level, flags); 488 if (ret) 489 goto err; 490 491 if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND) 492 __i915_vma_set_map_and_fenceable(vma); 493 494 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags)); 495 return 0; 496 497 err: 498 __i915_vma_unpin(vma); 499 return ret; 500 } 501 502 void i915_vma_destroy(struct i915_vma *vma) 503 { 504 GEM_BUG_ON(vma->node.allocated); 505 GEM_BUG_ON(i915_vma_is_active(vma)); 506 GEM_BUG_ON(!i915_vma_is_closed(vma)); 507 GEM_BUG_ON(vma->fence); 508 509 list_del(&vma->vm_link); 510 if (!i915_vma_is_ggtt(vma)) 511 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm)); 512 513 kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma); 514 } 515 516 void i915_vma_close(struct i915_vma *vma) 517 { 518 GEM_BUG_ON(i915_vma_is_closed(vma)); 519 vma->flags |= I915_VMA_CLOSED; 520 521 list_del(&vma->obj_link); 522 rb_erase(&vma->obj_node, &vma->obj->vma_tree); 523 524 if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma)) 525 WARN_ON(i915_vma_unbind(vma)); 526 } 527 528 static void __i915_vma_iounmap(struct i915_vma *vma) 529 { 530 GEM_BUG_ON(i915_vma_is_pinned(vma)); 531 532 if (vma->iomap == NULL) 533 return; 534 535 io_mapping_unmap(vma->iomap); 536 vma->iomap = NULL; 537 } 538 539 int i915_vma_unbind(struct i915_vma *vma) 540 { 541 struct drm_i915_gem_object *obj = vma->obj; 542 unsigned long active; 543 int ret; 544 545 lockdep_assert_held(&obj->base.dev->struct_mutex); 546 547 /* First wait upon any activity as retiring the request may 548 * have side-effects such as unpinning or even unbinding this vma. 549 */ 550 active = i915_vma_get_active(vma); 551 if (active) { 552 int idx; 553 554 /* When a closed VMA is retired, it is unbound - eek. 555 * In order to prevent it from being recursively closed, 556 * take a pin on the vma so that the second unbind is 557 * aborted. 558 * 559 * Even more scary is that the retire callback may free 560 * the object (last active vma). To prevent the explosion 561 * we defer the actual object free to a worker that can 562 * only proceed once it acquires the struct_mutex (which 563 * we currently hold, therefore it cannot free this object 564 * before we are finished). 565 */ 566 __i915_vma_pin(vma); 567 568 for_each_active(active, idx) { 569 ret = i915_gem_active_retire(&vma->last_read[idx], 570 &vma->vm->dev->struct_mutex); 571 if (ret) 572 break; 573 } 574 575 __i915_vma_unpin(vma); 576 if (ret) 577 return ret; 578 579 GEM_BUG_ON(i915_vma_is_active(vma)); 580 } 581 582 if (i915_vma_is_pinned(vma)) 583 return -EBUSY; 584 585 if (!drm_mm_node_allocated(&vma->node)) 586 goto destroy; 587 588 GEM_BUG_ON(obj->bind_count == 0); 589 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 590 591 if (i915_vma_is_map_and_fenceable(vma)) { 592 /* release the fence reg _after_ flushing */ 593 ret = i915_vma_put_fence(vma); 594 if (ret) 595 return ret; 596 597 /* Force a pagefault for domain tracking on next user access */ 598 i915_gem_release_mmap(obj); 599 600 __i915_vma_iounmap(vma); 601 vma->flags &= ~I915_VMA_CAN_FENCE; 602 } 603 604 if (likely(!vma->vm->closed)) { 605 trace_i915_vma_unbind(vma); 606 vma->vm->unbind_vma(vma); 607 } 608 vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND); 609 610 drm_mm_remove_node(&vma->node); 611 list_move_tail(&vma->vm_link, &vma->vm->unbound_list); 612 613 if (vma->pages != obj->mm.pages) { 614 GEM_BUG_ON(!vma->pages); 615 sg_free_table(vma->pages); 616 kfree(vma->pages); 617 } 618 vma->pages = NULL; 619 620 /* Since the unbound list is global, only move to that list if 621 * no more VMAs exist. */ 622 if (--obj->bind_count == 0) 623 list_move_tail(&obj->global_link, 624 &to_i915(obj->base.dev)->mm.unbound_list); 625 626 /* And finally now the object is completely decoupled from this vma, 627 * we can drop its hold on the backing storage and allow it to be 628 * reaped by the shrinker. 629 */ 630 i915_gem_object_unpin_pages(obj); 631 632 destroy: 633 if (unlikely(i915_vma_is_closed(vma))) 634 i915_vma_destroy(vma); 635 636 return 0; 637 } 638 639