1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/list_sort.h> 26 #include <linux/prime_numbers.h> 27 28 #include "gem/i915_gem_context.h" 29 #include "gem/i915_gem_internal.h" 30 #include "gem/i915_gem_lmem.h" 31 #include "gem/i915_gem_region.h" 32 #include "gem/selftests/mock_context.h" 33 #include "gt/intel_context.h" 34 #include "gt/intel_gpu_commands.h" 35 #include "gt/intel_gtt.h" 36 37 #include "i915_random.h" 38 #include "i915_selftest.h" 39 #include "i915_vma_resource.h" 40 41 #include "mock_drm.h" 42 #include "mock_gem_device.h" 43 #include "mock_gtt.h" 44 #include "igt_flush_test.h" 45 46 static void cleanup_freed_objects(struct drm_i915_private *i915) 47 { 48 i915_gem_drain_freed_objects(i915); 49 } 50 51 static void fake_free_pages(struct drm_i915_gem_object *obj, 52 struct sg_table *pages) 53 { 54 sg_free_table(pages); 55 kfree(pages); 56 } 57 58 static int fake_get_pages(struct drm_i915_gem_object *obj) 59 { 60 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 61 #define PFN_BIAS 0x1000 62 struct sg_table *pages; 63 struct scatterlist *sg; 64 typeof(obj->base.size) rem; 65 66 pages = kmalloc(sizeof(*pages), GFP); 67 if (!pages) 68 return -ENOMEM; 69 70 rem = round_up(obj->base.size, BIT(31)) >> 31; 71 /* restricted by sg_alloc_table */ 72 if (overflows_type(rem, unsigned int)) { 73 kfree(pages); 74 return -E2BIG; 75 } 76 77 if (sg_alloc_table(pages, rem, GFP)) { 78 kfree(pages); 79 return -ENOMEM; 80 } 81 82 rem = obj->base.size; 83 for (sg = pages->sgl; sg; sg = sg_next(sg)) { 84 unsigned long len = min_t(typeof(rem), rem, BIT(31)); 85 86 GEM_BUG_ON(!len); 87 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0); 88 sg_dma_address(sg) = page_to_phys(sg_page(sg)); 89 sg_dma_len(sg) = len; 90 91 rem -= len; 92 } 93 GEM_BUG_ON(rem); 94 95 __i915_gem_object_set_pages(obj, pages); 96 97 return 0; 98 #undef GFP 99 } 100 101 static void fake_put_pages(struct drm_i915_gem_object *obj, 102 struct sg_table *pages) 103 { 104 fake_free_pages(obj, pages); 105 obj->mm.dirty = false; 106 } 107 108 static const struct drm_i915_gem_object_ops fake_ops = { 109 .name = "fake-gem", 110 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 111 .get_pages = fake_get_pages, 112 .put_pages = fake_put_pages, 113 }; 114 115 static struct drm_i915_gem_object * 116 fake_dma_object(struct drm_i915_private *i915, u64 size) 117 { 118 static struct lock_class_key lock_class; 119 struct drm_i915_gem_object *obj; 120 121 GEM_BUG_ON(!size); 122 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 123 124 if (overflows_type(size, obj->base.size)) 125 return ERR_PTR(-E2BIG); 126 127 obj = i915_gem_object_alloc(); 128 if (!obj) 129 goto err; 130 131 drm_gem_private_object_init(&i915->drm, &obj->base, size); 132 i915_gem_object_init(obj, &fake_ops, &lock_class, 0); 133 134 i915_gem_object_set_volatile(obj); 135 136 obj->write_domain = I915_GEM_DOMAIN_CPU; 137 obj->read_domains = I915_GEM_DOMAIN_CPU; 138 obj->pat_index = i915_gem_get_pat_index(i915, I915_CACHE_NONE); 139 140 /* Preallocate the "backing storage" */ 141 if (i915_gem_object_pin_pages_unlocked(obj)) 142 goto err_obj; 143 144 i915_gem_object_unpin_pages(obj); 145 return obj; 146 147 err_obj: 148 i915_gem_object_put(obj); 149 err: 150 return ERR_PTR(-ENOMEM); 151 } 152 153 static int igt_ppgtt_alloc(void *arg) 154 { 155 struct drm_i915_private *dev_priv = arg; 156 struct i915_ppgtt *ppgtt; 157 struct i915_gem_ww_ctx ww; 158 u64 size, last, limit; 159 int err = 0; 160 161 /* Allocate a ppggt and try to fill the entire range */ 162 163 if (!HAS_PPGTT(dev_priv)) 164 return 0; 165 166 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); 167 if (IS_ERR(ppgtt)) 168 return PTR_ERR(ppgtt); 169 170 if (!ppgtt->vm.allocate_va_range) 171 goto ppgtt_vm_put; 172 173 /* 174 * While we only allocate the page tables here and so we could 175 * address a much larger GTT than we could actually fit into 176 * RAM, a practical limit is the amount of physical pages in the system. 177 * This should ensure that we do not run into the oomkiller during 178 * the test and take down the machine wilfully. 179 */ 180 limit = totalram_pages() << PAGE_SHIFT; 181 limit = min(ppgtt->vm.total, limit); 182 183 i915_gem_ww_ctx_init(&ww, false); 184 retry: 185 err = i915_vm_lock_objects(&ppgtt->vm, &ww); 186 if (err) 187 goto err_ppgtt_cleanup; 188 189 /* Check we can allocate the entire range */ 190 for (size = 4096; size <= limit; size <<= 2) { 191 struct i915_vm_pt_stash stash = {}; 192 193 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size); 194 if (err) 195 goto err_ppgtt_cleanup; 196 197 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 198 if (err) { 199 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 200 goto err_ppgtt_cleanup; 201 } 202 203 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size); 204 cond_resched(); 205 206 ppgtt->vm.clear_range(&ppgtt->vm, 0, size); 207 208 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 209 } 210 211 /* Check we can incrementally allocate the entire range */ 212 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) { 213 struct i915_vm_pt_stash stash = {}; 214 215 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last); 216 if (err) 217 goto err_ppgtt_cleanup; 218 219 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 220 if (err) { 221 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 222 goto err_ppgtt_cleanup; 223 } 224 225 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 226 last, size - last); 227 cond_resched(); 228 229 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 230 } 231 232 err_ppgtt_cleanup: 233 if (err == -EDEADLK) { 234 err = i915_gem_ww_ctx_backoff(&ww); 235 if (!err) 236 goto retry; 237 } 238 i915_gem_ww_ctx_fini(&ww); 239 ppgtt_vm_put: 240 i915_vm_put(&ppgtt->vm); 241 return err; 242 } 243 244 static int lowlevel_hole(struct i915_address_space *vm, 245 u64 hole_start, u64 hole_end, 246 unsigned long end_time) 247 { 248 const unsigned int min_alignment = 249 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 250 I915_RND_STATE(seed_prng); 251 struct i915_vma_resource *mock_vma_res; 252 unsigned int size; 253 254 mock_vma_res = kzalloc(sizeof(*mock_vma_res), GFP_KERNEL); 255 if (!mock_vma_res) 256 return -ENOMEM; 257 258 /* Keep creating larger objects until one cannot fit into the hole */ 259 for (size = 12; (hole_end - hole_start) >> size; size++) { 260 I915_RND_SUBSTATE(prng, seed_prng); 261 struct drm_i915_gem_object *obj; 262 unsigned int *order, count, n; 263 u64 hole_size, aligned_size; 264 265 aligned_size = max_t(u32, ilog2(min_alignment), size); 266 hole_size = (hole_end - hole_start) >> aligned_size; 267 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 268 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 269 count = hole_size >> 1; 270 if (!count) { 271 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 272 __func__, hole_start, hole_end, size, hole_size); 273 break; 274 } 275 276 do { 277 order = i915_random_order(count, &prng); 278 if (order) 279 break; 280 } while (count >>= 1); 281 if (!count) { 282 kfree(mock_vma_res); 283 return -ENOMEM; 284 } 285 GEM_BUG_ON(!order); 286 287 GEM_BUG_ON(count * BIT_ULL(aligned_size) > vm->total); 288 GEM_BUG_ON(hole_start + count * BIT_ULL(aligned_size) > hole_end); 289 290 /* 291 * Ignore allocation failures (i.e. don't report them as 292 * a test failure) as we are purposefully allocating very 293 * large objects without checking that we have sufficient 294 * memory. We expect to hit -ENOMEM. 295 */ 296 297 obj = fake_dma_object(vm->i915, BIT_ULL(size)); 298 if (IS_ERR(obj)) { 299 kfree(order); 300 break; 301 } 302 303 GEM_BUG_ON(obj->base.size != BIT_ULL(size)); 304 305 if (i915_gem_object_pin_pages_unlocked(obj)) { 306 i915_gem_object_put(obj); 307 kfree(order); 308 break; 309 } 310 311 for (n = 0; n < count; n++) { 312 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 313 intel_wakeref_t wakeref; 314 315 GEM_BUG_ON(addr + BIT_ULL(aligned_size) > vm->total); 316 317 if (igt_timeout(end_time, 318 "%s timed out before %d/%d\n", 319 __func__, n, count)) { 320 hole_end = hole_start; /* quit */ 321 break; 322 } 323 324 if (vm->allocate_va_range) { 325 struct i915_vm_pt_stash stash = {}; 326 struct i915_gem_ww_ctx ww; 327 int err; 328 329 i915_gem_ww_ctx_init(&ww, false); 330 retry: 331 err = i915_vm_lock_objects(vm, &ww); 332 if (err) 333 goto alloc_vm_end; 334 335 err = -ENOMEM; 336 if (i915_vm_alloc_pt_stash(vm, &stash, 337 BIT_ULL(size))) 338 goto alloc_vm_end; 339 340 err = i915_vm_map_pt_stash(vm, &stash); 341 if (!err) 342 vm->allocate_va_range(vm, &stash, 343 addr, BIT_ULL(size)); 344 i915_vm_free_pt_stash(vm, &stash); 345 alloc_vm_end: 346 if (err == -EDEADLK) { 347 err = i915_gem_ww_ctx_backoff(&ww); 348 if (!err) 349 goto retry; 350 } 351 i915_gem_ww_ctx_fini(&ww); 352 353 if (err) 354 break; 355 } 356 357 mock_vma_res->bi.pages = obj->mm.pages; 358 mock_vma_res->node_size = BIT_ULL(aligned_size); 359 mock_vma_res->start = addr; 360 361 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) 362 vm->insert_entries(vm, mock_vma_res, 363 i915_gem_get_pat_index(vm->i915, 364 I915_CACHE_NONE), 365 0); 366 } 367 count = n; 368 369 i915_random_reorder(order, count, &prng); 370 for (n = 0; n < count; n++) { 371 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 372 intel_wakeref_t wakeref; 373 374 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 375 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) 376 vm->clear_range(vm, addr, BIT_ULL(size)); 377 } 378 379 i915_gem_object_unpin_pages(obj); 380 i915_gem_object_put(obj); 381 382 kfree(order); 383 384 cleanup_freed_objects(vm->i915); 385 } 386 387 kfree(mock_vma_res); 388 return 0; 389 } 390 391 static void close_object_list(struct list_head *objects, 392 struct i915_address_space *vm) 393 { 394 struct drm_i915_gem_object *obj, *on; 395 int __maybe_unused ignored; 396 397 list_for_each_entry_safe(obj, on, objects, st_link) { 398 struct i915_vma *vma; 399 400 vma = i915_vma_instance(obj, vm, NULL); 401 if (!IS_ERR(vma)) 402 ignored = i915_vma_unbind_unlocked(vma); 403 404 list_del(&obj->st_link); 405 i915_gem_object_put(obj); 406 } 407 } 408 409 static int fill_hole(struct i915_address_space *vm, 410 u64 hole_start, u64 hole_end, 411 unsigned long end_time) 412 { 413 const u64 hole_size = hole_end - hole_start; 414 struct drm_i915_gem_object *obj; 415 const unsigned int min_alignment = 416 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 417 const unsigned long max_pages = 418 min_t(u64, ULONG_MAX - 1, (hole_size / 2) >> ilog2(min_alignment)); 419 const unsigned long max_step = max(int_sqrt(max_pages), 2UL); 420 unsigned long npages, prime, flags; 421 struct i915_vma *vma; 422 LIST_HEAD(objects); 423 int err; 424 425 /* Try binding many VMA working inwards from either edge */ 426 427 flags = PIN_OFFSET_FIXED | PIN_USER; 428 if (i915_is_ggtt(vm)) 429 flags |= PIN_GLOBAL; 430 431 for_each_prime_number_from(prime, 2, max_step) { 432 for (npages = 1; npages <= max_pages; npages *= prime) { 433 const u64 full_size = npages << PAGE_SHIFT; 434 const struct { 435 const char *name; 436 u64 offset; 437 int step; 438 } phases[] = { 439 { "top-down", hole_end, -1, }, 440 { "bottom-up", hole_start, 1, }, 441 { } 442 }, *p; 443 444 obj = fake_dma_object(vm->i915, full_size); 445 if (IS_ERR(obj)) 446 break; 447 448 list_add(&obj->st_link, &objects); 449 450 /* 451 * Align differing sized objects against the edges, and 452 * check we don't walk off into the void when binding 453 * them into the GTT. 454 */ 455 for (p = phases; p->name; p++) { 456 u64 offset; 457 458 offset = p->offset; 459 list_for_each_entry(obj, &objects, st_link) { 460 u64 aligned_size = round_up(obj->base.size, 461 min_alignment); 462 463 vma = i915_vma_instance(obj, vm, NULL); 464 if (IS_ERR(vma)) 465 continue; 466 467 if (p->step < 0) { 468 if (offset < hole_start + aligned_size) 469 break; 470 offset -= aligned_size; 471 } 472 473 err = i915_vma_pin(vma, 0, 0, offset | flags); 474 if (err) { 475 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 476 __func__, p->name, err, npages, prime, offset); 477 goto err; 478 } 479 480 if (!drm_mm_node_allocated(&vma->node) || 481 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 482 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 483 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 484 offset); 485 err = -EINVAL; 486 goto err; 487 } 488 489 i915_vma_unpin(vma); 490 491 if (p->step > 0) { 492 if (offset + aligned_size > hole_end) 493 break; 494 offset += aligned_size; 495 } 496 } 497 498 offset = p->offset; 499 list_for_each_entry(obj, &objects, st_link) { 500 u64 aligned_size = round_up(obj->base.size, 501 min_alignment); 502 503 vma = i915_vma_instance(obj, vm, NULL); 504 if (IS_ERR(vma)) 505 continue; 506 507 if (p->step < 0) { 508 if (offset < hole_start + aligned_size) 509 break; 510 offset -= aligned_size; 511 } 512 513 if (!drm_mm_node_allocated(&vma->node) || 514 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 515 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n", 516 __func__, p->name, vma->node.start, vma->node.size, 517 offset); 518 err = -EINVAL; 519 goto err; 520 } 521 522 err = i915_vma_unbind_unlocked(vma); 523 if (err) { 524 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n", 525 __func__, p->name, vma->node.start, vma->node.size, 526 err); 527 goto err; 528 } 529 530 if (p->step > 0) { 531 if (offset + aligned_size > hole_end) 532 break; 533 offset += aligned_size; 534 } 535 } 536 537 offset = p->offset; 538 list_for_each_entry_reverse(obj, &objects, st_link) { 539 u64 aligned_size = round_up(obj->base.size, 540 min_alignment); 541 542 vma = i915_vma_instance(obj, vm, NULL); 543 if (IS_ERR(vma)) 544 continue; 545 546 if (p->step < 0) { 547 if (offset < hole_start + aligned_size) 548 break; 549 offset -= aligned_size; 550 } 551 552 err = i915_vma_pin(vma, 0, 0, offset | flags); 553 if (err) { 554 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 555 __func__, p->name, err, npages, prime, offset); 556 goto err; 557 } 558 559 if (!drm_mm_node_allocated(&vma->node) || 560 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 561 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 562 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 563 offset); 564 err = -EINVAL; 565 goto err; 566 } 567 568 i915_vma_unpin(vma); 569 570 if (p->step > 0) { 571 if (offset + aligned_size > hole_end) 572 break; 573 offset += aligned_size; 574 } 575 } 576 577 offset = p->offset; 578 list_for_each_entry_reverse(obj, &objects, st_link) { 579 u64 aligned_size = round_up(obj->base.size, 580 min_alignment); 581 582 vma = i915_vma_instance(obj, vm, NULL); 583 if (IS_ERR(vma)) 584 continue; 585 586 if (p->step < 0) { 587 if (offset < hole_start + aligned_size) 588 break; 589 offset -= aligned_size; 590 } 591 592 if (!drm_mm_node_allocated(&vma->node) || 593 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 594 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 595 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 596 offset); 597 err = -EINVAL; 598 goto err; 599 } 600 601 err = i915_vma_unbind_unlocked(vma); 602 if (err) { 603 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n", 604 __func__, p->name, vma->node.start, vma->node.size, 605 err); 606 goto err; 607 } 608 609 if (p->step > 0) { 610 if (offset + aligned_size > hole_end) 611 break; 612 offset += aligned_size; 613 } 614 } 615 } 616 617 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n", 618 __func__, npages, prime)) { 619 err = -EINTR; 620 goto err; 621 } 622 } 623 624 close_object_list(&objects, vm); 625 cleanup_freed_objects(vm->i915); 626 } 627 628 return 0; 629 630 err: 631 close_object_list(&objects, vm); 632 return err; 633 } 634 635 static int walk_hole(struct i915_address_space *vm, 636 u64 hole_start, u64 hole_end, 637 unsigned long end_time) 638 { 639 const u64 hole_size = hole_end - hole_start; 640 const unsigned long max_pages = 641 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); 642 unsigned long min_alignment; 643 unsigned long flags; 644 u64 size; 645 646 /* Try binding a single VMA in different positions within the hole */ 647 648 flags = PIN_OFFSET_FIXED | PIN_USER; 649 if (i915_is_ggtt(vm)) 650 flags |= PIN_GLOBAL; 651 652 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 653 654 for_each_prime_number_from(size, 1, max_pages) { 655 struct drm_i915_gem_object *obj; 656 struct i915_vma *vma; 657 u64 addr; 658 int err = 0; 659 660 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT); 661 if (IS_ERR(obj)) 662 break; 663 664 vma = i915_vma_instance(obj, vm, NULL); 665 if (IS_ERR(vma)) { 666 err = PTR_ERR(vma); 667 goto err_put; 668 } 669 670 for (addr = hole_start; 671 addr + obj->base.size < hole_end; 672 addr += round_up(obj->base.size, min_alignment)) { 673 err = i915_vma_pin(vma, 0, 0, addr | flags); 674 if (err) { 675 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", 676 __func__, addr, vma->size, 677 hole_start, hole_end, err); 678 goto err_put; 679 } 680 i915_vma_unpin(vma); 681 682 if (!drm_mm_node_allocated(&vma->node) || 683 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 684 pr_err("%s incorrect at %llx + %llx\n", 685 __func__, addr, vma->size); 686 err = -EINVAL; 687 goto err_put; 688 } 689 690 err = i915_vma_unbind_unlocked(vma); 691 if (err) { 692 pr_err("%s unbind failed at %llx + %llx with err=%d\n", 693 __func__, addr, vma->size, err); 694 goto err_put; 695 } 696 697 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 698 699 if (igt_timeout(end_time, 700 "%s timed out at %llx\n", 701 __func__, addr)) { 702 err = -EINTR; 703 goto err_put; 704 } 705 } 706 707 err_put: 708 i915_gem_object_put(obj); 709 if (err) 710 return err; 711 712 cleanup_freed_objects(vm->i915); 713 } 714 715 return 0; 716 } 717 718 static int pot_hole(struct i915_address_space *vm, 719 u64 hole_start, u64 hole_end, 720 unsigned long end_time) 721 { 722 struct drm_i915_gem_object *obj; 723 struct i915_vma *vma; 724 unsigned int min_alignment; 725 unsigned long flags; 726 unsigned int pot; 727 int err = 0; 728 729 flags = PIN_OFFSET_FIXED | PIN_USER; 730 if (i915_is_ggtt(vm)) 731 flags |= PIN_GLOBAL; 732 733 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 734 735 obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE); 736 if (IS_ERR(obj)) 737 return PTR_ERR(obj); 738 739 vma = i915_vma_instance(obj, vm, NULL); 740 if (IS_ERR(vma)) { 741 err = PTR_ERR(vma); 742 goto err_obj; 743 } 744 745 /* Insert a pair of pages across every pot boundary within the hole */ 746 for (pot = fls64(hole_end - 1) - 1; 747 pot > ilog2(2 * min_alignment); 748 pot--) { 749 u64 step = BIT_ULL(pot); 750 u64 addr; 751 752 for (addr = round_up(hole_start + min_alignment, step) - min_alignment; 753 hole_end > addr && hole_end - addr >= 2 * min_alignment; 754 addr += step) { 755 err = i915_vma_pin(vma, 0, 0, addr | flags); 756 if (err) { 757 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n", 758 __func__, 759 addr, 760 hole_start, hole_end, 761 err); 762 goto err_obj; 763 } 764 765 if (!drm_mm_node_allocated(&vma->node) || 766 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 767 pr_err("%s incorrect at %llx + %llx\n", 768 __func__, addr, vma->size); 769 i915_vma_unpin(vma); 770 err = i915_vma_unbind_unlocked(vma); 771 err = -EINVAL; 772 goto err_obj; 773 } 774 775 i915_vma_unpin(vma); 776 err = i915_vma_unbind_unlocked(vma); 777 GEM_BUG_ON(err); 778 } 779 780 if (igt_timeout(end_time, 781 "%s timed out after %d/%d\n", 782 __func__, pot, fls64(hole_end - 1) - 1)) { 783 err = -EINTR; 784 goto err_obj; 785 } 786 } 787 788 err_obj: 789 i915_gem_object_put(obj); 790 return err; 791 } 792 793 static int drunk_hole(struct i915_address_space *vm, 794 u64 hole_start, u64 hole_end, 795 unsigned long end_time) 796 { 797 I915_RND_STATE(prng); 798 unsigned int min_alignment; 799 unsigned int size; 800 unsigned long flags; 801 802 flags = PIN_OFFSET_FIXED | PIN_USER; 803 if (i915_is_ggtt(vm)) 804 flags |= PIN_GLOBAL; 805 806 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 807 808 /* Keep creating larger objects until one cannot fit into the hole */ 809 for (size = 12; (hole_end - hole_start) >> size; size++) { 810 struct drm_i915_gem_object *obj; 811 unsigned int *order, count, n; 812 struct i915_vma *vma; 813 u64 hole_size, aligned_size; 814 int err = -ENODEV; 815 816 aligned_size = max_t(u32, ilog2(min_alignment), size); 817 hole_size = (hole_end - hole_start) >> aligned_size; 818 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 819 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 820 count = hole_size >> 1; 821 if (!count) { 822 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 823 __func__, hole_start, hole_end, size, hole_size); 824 break; 825 } 826 827 do { 828 order = i915_random_order(count, &prng); 829 if (order) 830 break; 831 } while (count >>= 1); 832 if (!count) 833 return -ENOMEM; 834 GEM_BUG_ON(!order); 835 836 /* 837 * Ignore allocation failures (i.e. don't report them as 838 * a test failure) as we are purposefully allocating very 839 * large objects without checking that we have sufficient 840 * memory. We expect to hit -ENOMEM. 841 */ 842 843 obj = fake_dma_object(vm->i915, BIT_ULL(size)); 844 if (IS_ERR(obj)) { 845 kfree(order); 846 break; 847 } 848 849 vma = i915_vma_instance(obj, vm, NULL); 850 if (IS_ERR(vma)) { 851 err = PTR_ERR(vma); 852 goto err_obj; 853 } 854 855 GEM_BUG_ON(vma->size != BIT_ULL(size)); 856 857 for (n = 0; n < count; n++) { 858 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 859 860 err = i915_vma_pin(vma, 0, 0, addr | flags); 861 if (err) { 862 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 863 __func__, 864 addr, BIT_ULL(size), 865 hole_start, hole_end, 866 err); 867 goto err_obj; 868 } 869 870 if (!drm_mm_node_allocated(&vma->node) || 871 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 872 pr_err("%s incorrect at %llx + %llx\n", 873 __func__, addr, BIT_ULL(size)); 874 i915_vma_unpin(vma); 875 err = i915_vma_unbind_unlocked(vma); 876 err = -EINVAL; 877 goto err_obj; 878 } 879 880 i915_vma_unpin(vma); 881 err = i915_vma_unbind_unlocked(vma); 882 GEM_BUG_ON(err); 883 884 if (igt_timeout(end_time, 885 "%s timed out after %d/%d\n", 886 __func__, n, count)) { 887 err = -EINTR; 888 goto err_obj; 889 } 890 } 891 892 err_obj: 893 i915_gem_object_put(obj); 894 kfree(order); 895 if (err) 896 return err; 897 898 cleanup_freed_objects(vm->i915); 899 } 900 901 return 0; 902 } 903 904 static int __shrink_hole(struct i915_address_space *vm, 905 u64 hole_start, u64 hole_end, 906 unsigned long end_time) 907 { 908 struct drm_i915_gem_object *obj; 909 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 910 unsigned int min_alignment; 911 unsigned int order = 12; 912 LIST_HEAD(objects); 913 int err = 0; 914 u64 addr; 915 916 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 917 918 /* Keep creating larger objects until one cannot fit into the hole */ 919 for (addr = hole_start; addr < hole_end; ) { 920 struct i915_vma *vma; 921 u64 size = BIT_ULL(order++); 922 923 size = min(size, hole_end - addr); 924 obj = fake_dma_object(vm->i915, size); 925 if (IS_ERR(obj)) { 926 err = PTR_ERR(obj); 927 break; 928 } 929 930 list_add(&obj->st_link, &objects); 931 932 vma = i915_vma_instance(obj, vm, NULL); 933 if (IS_ERR(vma)) { 934 err = PTR_ERR(vma); 935 break; 936 } 937 938 GEM_BUG_ON(vma->size != size); 939 940 err = i915_vma_pin(vma, 0, 0, addr | flags); 941 if (err) { 942 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 943 __func__, addr, size, hole_start, hole_end, err); 944 break; 945 } 946 947 if (!drm_mm_node_allocated(&vma->node) || 948 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 949 pr_err("%s incorrect at %llx + %llx\n", 950 __func__, addr, size); 951 i915_vma_unpin(vma); 952 err = i915_vma_unbind_unlocked(vma); 953 err = -EINVAL; 954 break; 955 } 956 957 i915_vma_unpin(vma); 958 addr += round_up(size, min_alignment); 959 960 /* 961 * Since we are injecting allocation faults at random intervals, 962 * wait for this allocation to complete before we change the 963 * faultinjection. 964 */ 965 err = i915_vma_sync(vma); 966 if (err) 967 break; 968 969 if (igt_timeout(end_time, 970 "%s timed out at offset %llx [%llx - %llx]\n", 971 __func__, addr, hole_start, hole_end)) { 972 err = -EINTR; 973 break; 974 } 975 } 976 977 close_object_list(&objects, vm); 978 cleanup_freed_objects(vm->i915); 979 return err; 980 } 981 982 static int shrink_hole(struct i915_address_space *vm, 983 u64 hole_start, u64 hole_end, 984 unsigned long end_time) 985 { 986 unsigned long prime; 987 int err; 988 989 vm->fault_attr.probability = 999; 990 atomic_set(&vm->fault_attr.times, -1); 991 992 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) { 993 vm->fault_attr.interval = prime; 994 err = __shrink_hole(vm, hole_start, hole_end, end_time); 995 if (err) 996 break; 997 } 998 999 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 1000 1001 return err; 1002 } 1003 1004 static int shrink_boom(struct i915_address_space *vm, 1005 u64 hole_start, u64 hole_end, 1006 unsigned long end_time) 1007 { 1008 unsigned int sizes[] = { SZ_2M, SZ_1G }; 1009 struct drm_i915_gem_object *purge; 1010 struct drm_i915_gem_object *explode; 1011 int err; 1012 int i; 1013 1014 /* 1015 * Catch the case which shrink_hole seems to miss. The setup here 1016 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while 1017 * ensuring that all vma associated with the respective pd/pdp are 1018 * unpinned at the time. 1019 */ 1020 1021 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1022 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1023 unsigned int size = sizes[i]; 1024 struct i915_vma *vma; 1025 1026 purge = fake_dma_object(vm->i915, size); 1027 if (IS_ERR(purge)) 1028 return PTR_ERR(purge); 1029 1030 vma = i915_vma_instance(purge, vm, NULL); 1031 if (IS_ERR(vma)) { 1032 err = PTR_ERR(vma); 1033 goto err_purge; 1034 } 1035 1036 err = i915_vma_pin(vma, 0, 0, flags); 1037 if (err) 1038 goto err_purge; 1039 1040 /* Should now be ripe for purging */ 1041 i915_vma_unpin(vma); 1042 1043 explode = fake_dma_object(vm->i915, size); 1044 if (IS_ERR(explode)) { 1045 err = PTR_ERR(explode); 1046 goto err_purge; 1047 } 1048 1049 vm->fault_attr.probability = 100; 1050 vm->fault_attr.interval = 1; 1051 atomic_set(&vm->fault_attr.times, -1); 1052 1053 vma = i915_vma_instance(explode, vm, NULL); 1054 if (IS_ERR(vma)) { 1055 err = PTR_ERR(vma); 1056 goto err_explode; 1057 } 1058 1059 err = i915_vma_pin(vma, 0, 0, flags | size); 1060 if (err) 1061 goto err_explode; 1062 1063 i915_vma_unpin(vma); 1064 1065 i915_gem_object_put(purge); 1066 i915_gem_object_put(explode); 1067 1068 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 1069 cleanup_freed_objects(vm->i915); 1070 } 1071 1072 return 0; 1073 1074 err_explode: 1075 i915_gem_object_put(explode); 1076 err_purge: 1077 i915_gem_object_put(purge); 1078 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 1079 return err; 1080 } 1081 1082 static int misaligned_case(struct i915_address_space *vm, struct intel_memory_region *mr, 1083 u64 addr, u64 size, unsigned long flags) 1084 { 1085 struct drm_i915_gem_object *obj; 1086 struct i915_vma *vma; 1087 int err = 0; 1088 u64 expected_vma_size, expected_node_size; 1089 bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM || 1090 mr->type == INTEL_MEMORY_STOLEN_LOCAL; 1091 1092 obj = i915_gem_object_create_region(mr, size, 0, I915_BO_ALLOC_GPU_ONLY); 1093 if (IS_ERR(obj)) { 1094 /* if iGVT-g or DMAR is active, stolen mem will be uninitialized */ 1095 if (PTR_ERR(obj) == -ENODEV && is_stolen) 1096 return 0; 1097 return PTR_ERR(obj); 1098 } 1099 1100 vma = i915_vma_instance(obj, vm, NULL); 1101 if (IS_ERR(vma)) { 1102 err = PTR_ERR(vma); 1103 goto err_put; 1104 } 1105 1106 err = i915_vma_pin(vma, 0, 0, addr | flags); 1107 if (err) 1108 goto err_put; 1109 i915_vma_unpin(vma); 1110 1111 if (!drm_mm_node_allocated(&vma->node)) { 1112 err = -EINVAL; 1113 goto err_put; 1114 } 1115 1116 if (i915_vma_misplaced(vma, 0, 0, addr | flags)) { 1117 err = -EINVAL; 1118 goto err_put; 1119 } 1120 1121 expected_vma_size = round_up(size, 1 << (ffs(vma->resource->page_sizes_gtt) - 1)); 1122 expected_node_size = expected_vma_size; 1123 1124 if (HAS_64K_PAGES(vm->i915) && i915_gem_object_is_lmem(obj)) { 1125 expected_vma_size = round_up(size, I915_GTT_PAGE_SIZE_64K); 1126 expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_64K); 1127 } 1128 1129 if (vma->size != expected_vma_size || vma->node.size != expected_node_size) { 1130 err = i915_vma_unbind_unlocked(vma); 1131 err = -EBADSLT; 1132 goto err_put; 1133 } 1134 1135 err = i915_vma_unbind_unlocked(vma); 1136 if (err) 1137 goto err_put; 1138 1139 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 1140 1141 err_put: 1142 i915_gem_object_put(obj); 1143 cleanup_freed_objects(vm->i915); 1144 return err; 1145 } 1146 1147 static int misaligned_pin(struct i915_address_space *vm, 1148 u64 hole_start, u64 hole_end, 1149 unsigned long end_time) 1150 { 1151 struct intel_memory_region *mr; 1152 enum intel_region_id id; 1153 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 1154 int err = 0; 1155 u64 hole_size = hole_end - hole_start; 1156 1157 if (i915_is_ggtt(vm)) 1158 flags |= PIN_GLOBAL; 1159 1160 for_each_memory_region(mr, vm->i915, id) { 1161 u64 min_alignment = i915_vm_min_alignment(vm, mr->type); 1162 u64 size = min_alignment; 1163 u64 addr = round_down(hole_start + (hole_size / 2), min_alignment); 1164 1165 /* avoid -ENOSPC on very small hole setups */ 1166 if (hole_size < 3 * min_alignment) 1167 continue; 1168 1169 /* we can't test < 4k alignment due to flags being encoded in lower bits */ 1170 if (min_alignment != I915_GTT_PAGE_SIZE_4K) { 1171 err = misaligned_case(vm, mr, addr + (min_alignment / 2), size, flags); 1172 /* misaligned should error with -EINVAL*/ 1173 if (!err) 1174 err = -EBADSLT; 1175 if (err != -EINVAL) 1176 return err; 1177 } 1178 1179 /* test for vma->size expansion to min page size */ 1180 err = misaligned_case(vm, mr, addr, PAGE_SIZE, flags); 1181 if (err) 1182 return err; 1183 1184 /* test for intermediate size not expanding vma->size for large alignments */ 1185 err = misaligned_case(vm, mr, addr, size / 2, flags); 1186 if (err) 1187 return err; 1188 } 1189 1190 return 0; 1191 } 1192 1193 static int exercise_ppgtt(struct drm_i915_private *dev_priv, 1194 int (*func)(struct i915_address_space *vm, 1195 u64 hole_start, u64 hole_end, 1196 unsigned long end_time)) 1197 { 1198 struct i915_ppgtt *ppgtt; 1199 IGT_TIMEOUT(end_time); 1200 struct file *file; 1201 int err; 1202 1203 if (!HAS_FULL_PPGTT(dev_priv)) 1204 return 0; 1205 1206 file = mock_file(dev_priv); 1207 if (IS_ERR(file)) 1208 return PTR_ERR(file); 1209 1210 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); 1211 if (IS_ERR(ppgtt)) { 1212 err = PTR_ERR(ppgtt); 1213 goto out_free; 1214 } 1215 GEM_BUG_ON(offset_in_page(ppgtt->vm.total)); 1216 assert_vm_alive(&ppgtt->vm); 1217 1218 err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time); 1219 1220 i915_vm_put(&ppgtt->vm); 1221 1222 out_free: 1223 fput(file); 1224 return err; 1225 } 1226 1227 static int igt_ppgtt_fill(void *arg) 1228 { 1229 return exercise_ppgtt(arg, fill_hole); 1230 } 1231 1232 static int igt_ppgtt_walk(void *arg) 1233 { 1234 return exercise_ppgtt(arg, walk_hole); 1235 } 1236 1237 static int igt_ppgtt_pot(void *arg) 1238 { 1239 return exercise_ppgtt(arg, pot_hole); 1240 } 1241 1242 static int igt_ppgtt_drunk(void *arg) 1243 { 1244 return exercise_ppgtt(arg, drunk_hole); 1245 } 1246 1247 static int igt_ppgtt_lowlevel(void *arg) 1248 { 1249 return exercise_ppgtt(arg, lowlevel_hole); 1250 } 1251 1252 static int igt_ppgtt_shrink(void *arg) 1253 { 1254 return exercise_ppgtt(arg, shrink_hole); 1255 } 1256 1257 static int igt_ppgtt_shrink_boom(void *arg) 1258 { 1259 return exercise_ppgtt(arg, shrink_boom); 1260 } 1261 1262 static int igt_ppgtt_misaligned_pin(void *arg) 1263 { 1264 return exercise_ppgtt(arg, misaligned_pin); 1265 } 1266 1267 static int sort_holes(void *priv, const struct list_head *A, 1268 const struct list_head *B) 1269 { 1270 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack); 1271 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack); 1272 1273 if (a->start < b->start) 1274 return -1; 1275 else 1276 return 1; 1277 } 1278 1279 static int exercise_ggtt(struct drm_i915_private *i915, 1280 int (*func)(struct i915_address_space *vm, 1281 u64 hole_start, u64 hole_end, 1282 unsigned long end_time)) 1283 { 1284 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1285 u64 hole_start, hole_end, last = 0; 1286 struct drm_mm_node *node; 1287 IGT_TIMEOUT(end_time); 1288 int err = 0; 1289 1290 restart: 1291 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes); 1292 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) { 1293 if (hole_start < last) 1294 continue; 1295 1296 if (ggtt->vm.mm.color_adjust) 1297 ggtt->vm.mm.color_adjust(node, 0, 1298 &hole_start, &hole_end); 1299 if (hole_start >= hole_end) 1300 continue; 1301 1302 err = func(&ggtt->vm, hole_start, hole_end, end_time); 1303 if (err) 1304 break; 1305 1306 /* As we have manipulated the drm_mm, the list may be corrupt */ 1307 last = hole_end; 1308 goto restart; 1309 } 1310 1311 return err; 1312 } 1313 1314 static int igt_ggtt_fill(void *arg) 1315 { 1316 return exercise_ggtt(arg, fill_hole); 1317 } 1318 1319 static int igt_ggtt_walk(void *arg) 1320 { 1321 return exercise_ggtt(arg, walk_hole); 1322 } 1323 1324 static int igt_ggtt_pot(void *arg) 1325 { 1326 return exercise_ggtt(arg, pot_hole); 1327 } 1328 1329 static int igt_ggtt_drunk(void *arg) 1330 { 1331 return exercise_ggtt(arg, drunk_hole); 1332 } 1333 1334 static int igt_ggtt_lowlevel(void *arg) 1335 { 1336 return exercise_ggtt(arg, lowlevel_hole); 1337 } 1338 1339 static int igt_ggtt_misaligned_pin(void *arg) 1340 { 1341 return exercise_ggtt(arg, misaligned_pin); 1342 } 1343 1344 static int igt_ggtt_page(void *arg) 1345 { 1346 const unsigned int count = PAGE_SIZE/sizeof(u32); 1347 I915_RND_STATE(prng); 1348 struct drm_i915_private *i915 = arg; 1349 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1350 struct drm_i915_gem_object *obj; 1351 intel_wakeref_t wakeref; 1352 struct drm_mm_node tmp; 1353 unsigned int *order, n; 1354 int err; 1355 1356 if (!i915_ggtt_has_aperture(ggtt)) 1357 return 0; 1358 1359 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1360 if (IS_ERR(obj)) 1361 return PTR_ERR(obj); 1362 1363 err = i915_gem_object_pin_pages_unlocked(obj); 1364 if (err) 1365 goto out_free; 1366 1367 memset(&tmp, 0, sizeof(tmp)); 1368 mutex_lock(&ggtt->vm.mutex); 1369 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp, 1370 count * PAGE_SIZE, 0, 1371 I915_COLOR_UNEVICTABLE, 1372 0, ggtt->mappable_end, 1373 DRM_MM_INSERT_LOW); 1374 mutex_unlock(&ggtt->vm.mutex); 1375 if (err) 1376 goto out_unpin; 1377 1378 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1379 1380 for (n = 0; n < count; n++) { 1381 u64 offset = tmp.start + n * PAGE_SIZE; 1382 1383 ggtt->vm.insert_page(&ggtt->vm, 1384 i915_gem_object_get_dma_address(obj, 0), 1385 offset, 1386 i915_gem_get_pat_index(i915, 1387 I915_CACHE_NONE), 1388 0); 1389 } 1390 1391 order = i915_random_order(count, &prng); 1392 if (!order) { 1393 err = -ENOMEM; 1394 goto out_remove; 1395 } 1396 1397 for (n = 0; n < count; n++) { 1398 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1399 u32 __iomem *vaddr; 1400 1401 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1402 iowrite32(n, vaddr + n); 1403 io_mapping_unmap_atomic(vaddr); 1404 } 1405 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 1406 1407 i915_random_reorder(order, count, &prng); 1408 for (n = 0; n < count; n++) { 1409 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1410 u32 __iomem *vaddr; 1411 u32 val; 1412 1413 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1414 val = ioread32(vaddr + n); 1415 io_mapping_unmap_atomic(vaddr); 1416 1417 if (val != n) { 1418 pr_err("insert page failed: found %d, expected %d\n", 1419 val, n); 1420 err = -EINVAL; 1421 break; 1422 } 1423 } 1424 1425 kfree(order); 1426 out_remove: 1427 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size); 1428 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1429 mutex_lock(&ggtt->vm.mutex); 1430 drm_mm_remove_node(&tmp); 1431 mutex_unlock(&ggtt->vm.mutex); 1432 out_unpin: 1433 i915_gem_object_unpin_pages(obj); 1434 out_free: 1435 i915_gem_object_put(obj); 1436 return err; 1437 } 1438 1439 static void track_vma_bind(struct i915_vma *vma) 1440 { 1441 struct drm_i915_gem_object *obj = vma->obj; 1442 1443 __i915_gem_object_pin_pages(obj); 1444 1445 GEM_BUG_ON(atomic_read(&vma->pages_count)); 1446 atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE); 1447 __i915_gem_object_pin_pages(obj); 1448 vma->pages = obj->mm.pages; 1449 vma->resource->bi.pages = vma->pages; 1450 1451 mutex_lock(&vma->vm->mutex); 1452 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 1453 mutex_unlock(&vma->vm->mutex); 1454 } 1455 1456 static int exercise_mock(struct drm_i915_private *i915, 1457 int (*func)(struct i915_address_space *vm, 1458 u64 hole_start, u64 hole_end, 1459 unsigned long end_time)) 1460 { 1461 const u64 limit = totalram_pages() << PAGE_SHIFT; 1462 struct i915_address_space *vm; 1463 struct i915_gem_context *ctx; 1464 IGT_TIMEOUT(end_time); 1465 int err; 1466 1467 ctx = mock_context(i915, "mock"); 1468 if (!ctx) 1469 return -ENOMEM; 1470 1471 vm = i915_gem_context_get_eb_vm(ctx); 1472 err = func(vm, 0, min(vm->total, limit), end_time); 1473 i915_vm_put(vm); 1474 1475 mock_context_close(ctx); 1476 return err; 1477 } 1478 1479 static int igt_mock_fill(void *arg) 1480 { 1481 struct i915_ggtt *ggtt = arg; 1482 1483 return exercise_mock(ggtt->vm.i915, fill_hole); 1484 } 1485 1486 static int igt_mock_walk(void *arg) 1487 { 1488 struct i915_ggtt *ggtt = arg; 1489 1490 return exercise_mock(ggtt->vm.i915, walk_hole); 1491 } 1492 1493 static int igt_mock_pot(void *arg) 1494 { 1495 struct i915_ggtt *ggtt = arg; 1496 1497 return exercise_mock(ggtt->vm.i915, pot_hole); 1498 } 1499 1500 static int igt_mock_drunk(void *arg) 1501 { 1502 struct i915_ggtt *ggtt = arg; 1503 1504 return exercise_mock(ggtt->vm.i915, drunk_hole); 1505 } 1506 1507 static int reserve_gtt_with_resource(struct i915_vma *vma, u64 offset) 1508 { 1509 struct i915_address_space *vm = vma->vm; 1510 struct i915_vma_resource *vma_res; 1511 struct drm_i915_gem_object *obj = vma->obj; 1512 int err; 1513 1514 vma_res = i915_vma_resource_alloc(); 1515 if (IS_ERR(vma_res)) 1516 return PTR_ERR(vma_res); 1517 1518 mutex_lock(&vm->mutex); 1519 err = i915_gem_gtt_reserve(vm, NULL, &vma->node, obj->base.size, 1520 offset, 1521 obj->pat_index, 1522 0); 1523 if (!err) { 1524 i915_vma_resource_init_from_vma(vma_res, vma); 1525 vma->resource = vma_res; 1526 } else { 1527 kfree(vma_res); 1528 } 1529 mutex_unlock(&vm->mutex); 1530 1531 return err; 1532 } 1533 1534 static int igt_gtt_reserve(void *arg) 1535 { 1536 struct i915_ggtt *ggtt = arg; 1537 struct drm_i915_gem_object *obj, *on; 1538 I915_RND_STATE(prng); 1539 LIST_HEAD(objects); 1540 u64 total; 1541 int err = -ENODEV; 1542 1543 /* 1544 * i915_gem_gtt_reserve() tries to reserve the precise range 1545 * for the node, and evicts if it has to. So our test checks that 1546 * it can give us the requested space and prevent overlaps. 1547 */ 1548 1549 /* Start by filling the GGTT */ 1550 for (total = 0; 1551 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1552 total += 2 * I915_GTT_PAGE_SIZE) { 1553 struct i915_vma *vma; 1554 1555 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1556 2 * PAGE_SIZE); 1557 if (IS_ERR(obj)) { 1558 err = PTR_ERR(obj); 1559 goto out; 1560 } 1561 1562 err = i915_gem_object_pin_pages_unlocked(obj); 1563 if (err) { 1564 i915_gem_object_put(obj); 1565 goto out; 1566 } 1567 1568 list_add(&obj->st_link, &objects); 1569 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1570 if (IS_ERR(vma)) { 1571 err = PTR_ERR(vma); 1572 goto out; 1573 } 1574 1575 err = reserve_gtt_with_resource(vma, total); 1576 if (err) { 1577 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", 1578 total, ggtt->vm.total, err); 1579 goto out; 1580 } 1581 track_vma_bind(vma); 1582 1583 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1584 if (vma->node.start != total || 1585 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1586 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1587 vma->node.start, vma->node.size, 1588 total, 2*I915_GTT_PAGE_SIZE); 1589 err = -EINVAL; 1590 goto out; 1591 } 1592 } 1593 1594 /* Now we start forcing evictions */ 1595 for (total = I915_GTT_PAGE_SIZE; 1596 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1597 total += 2 * I915_GTT_PAGE_SIZE) { 1598 struct i915_vma *vma; 1599 1600 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1601 2 * PAGE_SIZE); 1602 if (IS_ERR(obj)) { 1603 err = PTR_ERR(obj); 1604 goto out; 1605 } 1606 1607 err = i915_gem_object_pin_pages_unlocked(obj); 1608 if (err) { 1609 i915_gem_object_put(obj); 1610 goto out; 1611 } 1612 1613 list_add(&obj->st_link, &objects); 1614 1615 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1616 if (IS_ERR(vma)) { 1617 err = PTR_ERR(vma); 1618 goto out; 1619 } 1620 1621 err = reserve_gtt_with_resource(vma, total); 1622 if (err) { 1623 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", 1624 total, ggtt->vm.total, err); 1625 goto out; 1626 } 1627 track_vma_bind(vma); 1628 1629 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1630 if (vma->node.start != total || 1631 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1632 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1633 vma->node.start, vma->node.size, 1634 total, 2*I915_GTT_PAGE_SIZE); 1635 err = -EINVAL; 1636 goto out; 1637 } 1638 } 1639 1640 /* And then try at random */ 1641 list_for_each_entry_safe(obj, on, &objects, st_link) { 1642 struct i915_vma *vma; 1643 u64 offset; 1644 1645 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1646 if (IS_ERR(vma)) { 1647 err = PTR_ERR(vma); 1648 goto out; 1649 } 1650 1651 err = i915_vma_unbind_unlocked(vma); 1652 if (err) { 1653 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1654 goto out; 1655 } 1656 1657 offset = igt_random_offset(&prng, 1658 0, ggtt->vm.total, 1659 2 * I915_GTT_PAGE_SIZE, 1660 I915_GTT_MIN_ALIGNMENT); 1661 1662 err = reserve_gtt_with_resource(vma, offset); 1663 if (err) { 1664 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", 1665 total, ggtt->vm.total, err); 1666 goto out; 1667 } 1668 track_vma_bind(vma); 1669 1670 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1671 if (vma->node.start != offset || 1672 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1673 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1674 vma->node.start, vma->node.size, 1675 offset, 2*I915_GTT_PAGE_SIZE); 1676 err = -EINVAL; 1677 goto out; 1678 } 1679 } 1680 1681 out: 1682 list_for_each_entry_safe(obj, on, &objects, st_link) { 1683 i915_gem_object_unpin_pages(obj); 1684 i915_gem_object_put(obj); 1685 } 1686 return err; 1687 } 1688 1689 static int insert_gtt_with_resource(struct i915_vma *vma) 1690 { 1691 struct i915_address_space *vm = vma->vm; 1692 struct i915_vma_resource *vma_res; 1693 struct drm_i915_gem_object *obj = vma->obj; 1694 int err; 1695 1696 vma_res = i915_vma_resource_alloc(); 1697 if (IS_ERR(vma_res)) 1698 return PTR_ERR(vma_res); 1699 1700 mutex_lock(&vm->mutex); 1701 err = i915_gem_gtt_insert(vm, NULL, &vma->node, obj->base.size, 0, 1702 obj->pat_index, 0, vm->total, 0); 1703 if (!err) { 1704 i915_vma_resource_init_from_vma(vma_res, vma); 1705 vma->resource = vma_res; 1706 } else { 1707 kfree(vma_res); 1708 } 1709 mutex_unlock(&vm->mutex); 1710 1711 return err; 1712 } 1713 1714 static int igt_gtt_insert(void *arg) 1715 { 1716 struct i915_ggtt *ggtt = arg; 1717 struct drm_i915_gem_object *obj, *on; 1718 struct drm_mm_node tmp = {}; 1719 const struct invalid_insert { 1720 u64 size; 1721 u64 alignment; 1722 u64 start, end; 1723 } invalid_insert[] = { 1724 { 1725 ggtt->vm.total + I915_GTT_PAGE_SIZE, 0, 1726 0, ggtt->vm.total, 1727 }, 1728 { 1729 2*I915_GTT_PAGE_SIZE, 0, 1730 0, I915_GTT_PAGE_SIZE, 1731 }, 1732 { 1733 -(u64)I915_GTT_PAGE_SIZE, 0, 1734 0, 4*I915_GTT_PAGE_SIZE, 1735 }, 1736 { 1737 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE, 1738 0, 4*I915_GTT_PAGE_SIZE, 1739 }, 1740 { 1741 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1, 1742 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1, 1743 }, 1744 {} 1745 }, *ii; 1746 LIST_HEAD(objects); 1747 u64 total; 1748 int err = -ENODEV; 1749 1750 /* 1751 * i915_gem_gtt_insert() tries to allocate some free space in the GTT 1752 * to the node, evicting if required. 1753 */ 1754 1755 /* Check a couple of obviously invalid requests */ 1756 for (ii = invalid_insert; ii->size; ii++) { 1757 mutex_lock(&ggtt->vm.mutex); 1758 err = i915_gem_gtt_insert(&ggtt->vm, NULL, &tmp, 1759 ii->size, ii->alignment, 1760 I915_COLOR_UNEVICTABLE, 1761 ii->start, ii->end, 1762 0); 1763 mutex_unlock(&ggtt->vm.mutex); 1764 if (err != -ENOSPC) { 1765 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n", 1766 ii->size, ii->alignment, ii->start, ii->end, 1767 err); 1768 return -EINVAL; 1769 } 1770 } 1771 1772 /* Start by filling the GGTT */ 1773 for (total = 0; 1774 total + I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1775 total += I915_GTT_PAGE_SIZE) { 1776 struct i915_vma *vma; 1777 1778 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1779 I915_GTT_PAGE_SIZE); 1780 if (IS_ERR(obj)) { 1781 err = PTR_ERR(obj); 1782 goto out; 1783 } 1784 1785 err = i915_gem_object_pin_pages_unlocked(obj); 1786 if (err) { 1787 i915_gem_object_put(obj); 1788 goto out; 1789 } 1790 1791 list_add(&obj->st_link, &objects); 1792 1793 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1794 if (IS_ERR(vma)) { 1795 err = PTR_ERR(vma); 1796 goto out; 1797 } 1798 1799 err = insert_gtt_with_resource(vma); 1800 if (err == -ENOSPC) { 1801 /* maxed out the GGTT space */ 1802 i915_gem_object_put(obj); 1803 break; 1804 } 1805 if (err) { 1806 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n", 1807 total, ggtt->vm.total, err); 1808 goto out; 1809 } 1810 track_vma_bind(vma); 1811 __i915_vma_pin(vma); 1812 1813 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1814 } 1815 1816 list_for_each_entry(obj, &objects, st_link) { 1817 struct i915_vma *vma; 1818 1819 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1820 if (IS_ERR(vma)) { 1821 err = PTR_ERR(vma); 1822 goto out; 1823 } 1824 1825 if (!drm_mm_node_allocated(&vma->node)) { 1826 pr_err("VMA was unexpectedly evicted!\n"); 1827 err = -EINVAL; 1828 goto out; 1829 } 1830 1831 __i915_vma_unpin(vma); 1832 } 1833 1834 /* If we then reinsert, we should find the same hole */ 1835 list_for_each_entry_safe(obj, on, &objects, st_link) { 1836 struct i915_vma *vma; 1837 u64 offset; 1838 1839 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1840 if (IS_ERR(vma)) { 1841 err = PTR_ERR(vma); 1842 goto out; 1843 } 1844 1845 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1846 offset = vma->node.start; 1847 1848 err = i915_vma_unbind_unlocked(vma); 1849 if (err) { 1850 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1851 goto out; 1852 } 1853 1854 err = insert_gtt_with_resource(vma); 1855 if (err) { 1856 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", 1857 total, ggtt->vm.total, err); 1858 goto out; 1859 } 1860 track_vma_bind(vma); 1861 1862 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1863 if (vma->node.start != offset) { 1864 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n", 1865 offset, vma->node.start); 1866 err = -EINVAL; 1867 goto out; 1868 } 1869 } 1870 1871 /* And then force evictions */ 1872 for (total = 0; 1873 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1874 total += 2 * I915_GTT_PAGE_SIZE) { 1875 struct i915_vma *vma; 1876 1877 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1878 2 * I915_GTT_PAGE_SIZE); 1879 if (IS_ERR(obj)) { 1880 err = PTR_ERR(obj); 1881 goto out; 1882 } 1883 1884 err = i915_gem_object_pin_pages_unlocked(obj); 1885 if (err) { 1886 i915_gem_object_put(obj); 1887 goto out; 1888 } 1889 1890 list_add(&obj->st_link, &objects); 1891 1892 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1893 if (IS_ERR(vma)) { 1894 err = PTR_ERR(vma); 1895 goto out; 1896 } 1897 1898 err = insert_gtt_with_resource(vma); 1899 if (err) { 1900 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", 1901 total, ggtt->vm.total, err); 1902 goto out; 1903 } 1904 track_vma_bind(vma); 1905 1906 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1907 } 1908 1909 out: 1910 list_for_each_entry_safe(obj, on, &objects, st_link) { 1911 i915_gem_object_unpin_pages(obj); 1912 i915_gem_object_put(obj); 1913 } 1914 return err; 1915 } 1916 1917 int i915_gem_gtt_mock_selftests(void) 1918 { 1919 static const struct i915_subtest tests[] = { 1920 SUBTEST(igt_mock_drunk), 1921 SUBTEST(igt_mock_walk), 1922 SUBTEST(igt_mock_pot), 1923 SUBTEST(igt_mock_fill), 1924 SUBTEST(igt_gtt_reserve), 1925 SUBTEST(igt_gtt_insert), 1926 }; 1927 struct drm_i915_private *i915; 1928 struct intel_gt *gt; 1929 int err; 1930 1931 i915 = mock_gem_device(); 1932 if (!i915) 1933 return -ENOMEM; 1934 1935 /* allocate the ggtt */ 1936 err = intel_gt_assign_ggtt(to_gt(i915)); 1937 if (err) 1938 goto out_put; 1939 1940 gt = to_gt(i915); 1941 1942 mock_init_ggtt(gt); 1943 1944 err = i915_subtests(tests, gt->ggtt); 1945 1946 mock_device_flush(i915); 1947 i915_gem_drain_freed_objects(i915); 1948 mock_fini_ggtt(gt->ggtt); 1949 1950 out_put: 1951 mock_destroy_device(i915); 1952 return err; 1953 } 1954 1955 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) 1956 { 1957 static const struct i915_subtest tests[] = { 1958 SUBTEST(igt_ppgtt_alloc), 1959 SUBTEST(igt_ppgtt_lowlevel), 1960 SUBTEST(igt_ppgtt_drunk), 1961 SUBTEST(igt_ppgtt_walk), 1962 SUBTEST(igt_ppgtt_pot), 1963 SUBTEST(igt_ppgtt_fill), 1964 SUBTEST(igt_ppgtt_shrink), 1965 SUBTEST(igt_ppgtt_shrink_boom), 1966 SUBTEST(igt_ppgtt_misaligned_pin), 1967 SUBTEST(igt_ggtt_lowlevel), 1968 SUBTEST(igt_ggtt_drunk), 1969 SUBTEST(igt_ggtt_walk), 1970 SUBTEST(igt_ggtt_pot), 1971 SUBTEST(igt_ggtt_fill), 1972 SUBTEST(igt_ggtt_page), 1973 SUBTEST(igt_ggtt_misaligned_pin), 1974 }; 1975 1976 GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total)); 1977 1978 return i915_live_subtests(tests, i915); 1979 } 1980