1 /* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25 #include <linux/list_sort.h> 26 #include <linux/prime_numbers.h> 27 28 #include "gem/i915_gem_context.h" 29 #include "gem/i915_gem_internal.h" 30 #include "gem/i915_gem_lmem.h" 31 #include "gem/i915_gem_region.h" 32 #include "gem/selftests/mock_context.h" 33 #include "gt/intel_context.h" 34 #include "gt/intel_gpu_commands.h" 35 #include "gt/intel_gtt.h" 36 37 #include "i915_random.h" 38 #include "i915_selftest.h" 39 #include "i915_vma_resource.h" 40 41 #include "mock_drm.h" 42 #include "mock_gem_device.h" 43 #include "mock_gtt.h" 44 #include "igt_flush_test.h" 45 46 static void cleanup_freed_objects(struct drm_i915_private *i915) 47 { 48 i915_gem_drain_freed_objects(i915); 49 } 50 51 static void fake_free_pages(struct drm_i915_gem_object *obj, 52 struct sg_table *pages) 53 { 54 sg_free_table(pages); 55 kfree(pages); 56 } 57 58 static int fake_get_pages(struct drm_i915_gem_object *obj) 59 { 60 #define GFP (GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY) 61 #define PFN_BIAS 0x1000 62 struct sg_table *pages; 63 struct scatterlist *sg; 64 typeof(obj->base.size) rem; 65 66 pages = kmalloc(sizeof(*pages), GFP); 67 if (!pages) 68 return -ENOMEM; 69 70 rem = round_up(obj->base.size, BIT(31)) >> 31; 71 /* restricted by sg_alloc_table */ 72 if (overflows_type(rem, unsigned int)) { 73 kfree(pages); 74 return -E2BIG; 75 } 76 77 if (sg_alloc_table(pages, rem, GFP)) { 78 kfree(pages); 79 return -ENOMEM; 80 } 81 82 rem = obj->base.size; 83 for (sg = pages->sgl; sg; sg = sg_next(sg)) { 84 unsigned long len = min_t(typeof(rem), rem, BIT(31)); 85 86 GEM_BUG_ON(!len); 87 sg_set_page(sg, pfn_to_page(PFN_BIAS), len, 0); 88 sg_dma_address(sg) = page_to_phys(sg_page(sg)); 89 sg_dma_len(sg) = len; 90 91 rem -= len; 92 } 93 GEM_BUG_ON(rem); 94 95 __i915_gem_object_set_pages(obj, pages); 96 97 return 0; 98 #undef GFP 99 } 100 101 static void fake_put_pages(struct drm_i915_gem_object *obj, 102 struct sg_table *pages) 103 { 104 fake_free_pages(obj, pages); 105 obj->mm.dirty = false; 106 } 107 108 static const struct drm_i915_gem_object_ops fake_ops = { 109 .name = "fake-gem", 110 .flags = I915_GEM_OBJECT_IS_SHRINKABLE, 111 .get_pages = fake_get_pages, 112 .put_pages = fake_put_pages, 113 }; 114 115 static struct drm_i915_gem_object * 116 fake_dma_object(struct drm_i915_private *i915, u64 size) 117 { 118 static struct lock_class_key lock_class; 119 struct drm_i915_gem_object *obj; 120 121 GEM_BUG_ON(!size); 122 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); 123 124 if (overflows_type(size, obj->base.size)) 125 return ERR_PTR(-E2BIG); 126 127 obj = i915_gem_object_alloc(); 128 if (!obj) 129 goto err; 130 131 drm_gem_private_object_init(&i915->drm, &obj->base, size); 132 i915_gem_object_init(obj, &fake_ops, &lock_class, 0); 133 134 i915_gem_object_set_volatile(obj); 135 136 obj->write_domain = I915_GEM_DOMAIN_CPU; 137 obj->read_domains = I915_GEM_DOMAIN_CPU; 138 obj->pat_index = i915_gem_get_pat_index(i915, I915_CACHE_NONE); 139 140 /* Preallocate the "backing storage" */ 141 if (i915_gem_object_pin_pages_unlocked(obj)) 142 goto err_obj; 143 144 i915_gem_object_unpin_pages(obj); 145 return obj; 146 147 err_obj: 148 i915_gem_object_put(obj); 149 err: 150 return ERR_PTR(-ENOMEM); 151 } 152 153 static int igt_ppgtt_alloc(void *arg) 154 { 155 struct drm_i915_private *dev_priv = arg; 156 struct i915_ppgtt *ppgtt; 157 struct i915_gem_ww_ctx ww; 158 u64 size, last, limit; 159 int err = 0; 160 161 /* Allocate a ppggt and try to fill the entire range */ 162 163 if (!HAS_PPGTT(dev_priv)) 164 return 0; 165 166 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); 167 if (IS_ERR(ppgtt)) 168 return PTR_ERR(ppgtt); 169 170 if (!ppgtt->vm.allocate_va_range) 171 goto err_ppgtt_cleanup; 172 173 /* 174 * While we only allocate the page tables here and so we could 175 * address a much larger GTT than we could actually fit into 176 * RAM, a practical limit is the amount of physical pages in the system. 177 * This should ensure that we do not run into the oomkiller during 178 * the test and take down the machine wilfully. 179 */ 180 limit = totalram_pages() << PAGE_SHIFT; 181 limit = min(ppgtt->vm.total, limit); 182 183 i915_gem_ww_ctx_init(&ww, false); 184 retry: 185 err = i915_vm_lock_objects(&ppgtt->vm, &ww); 186 if (err) 187 goto err_ppgtt_cleanup; 188 189 /* Check we can allocate the entire range */ 190 for (size = 4096; size <= limit; size <<= 2) { 191 struct i915_vm_pt_stash stash = {}; 192 193 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size); 194 if (err) 195 goto err_ppgtt_cleanup; 196 197 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 198 if (err) { 199 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 200 goto err_ppgtt_cleanup; 201 } 202 203 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size); 204 cond_resched(); 205 206 ppgtt->vm.clear_range(&ppgtt->vm, 0, size); 207 208 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 209 } 210 211 /* Check we can incrementally allocate the entire range */ 212 for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) { 213 struct i915_vm_pt_stash stash = {}; 214 215 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last); 216 if (err) 217 goto err_ppgtt_cleanup; 218 219 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 220 if (err) { 221 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 222 goto err_ppgtt_cleanup; 223 } 224 225 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 226 last, size - last); 227 cond_resched(); 228 229 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 230 } 231 232 err_ppgtt_cleanup: 233 if (err == -EDEADLK) { 234 err = i915_gem_ww_ctx_backoff(&ww); 235 if (!err) 236 goto retry; 237 } 238 i915_gem_ww_ctx_fini(&ww); 239 240 i915_vm_put(&ppgtt->vm); 241 return err; 242 } 243 244 static int lowlevel_hole(struct i915_address_space *vm, 245 u64 hole_start, u64 hole_end, 246 unsigned long end_time) 247 { 248 const unsigned int min_alignment = 249 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 250 I915_RND_STATE(seed_prng); 251 struct i915_vma_resource *mock_vma_res; 252 unsigned int size; 253 254 mock_vma_res = kzalloc(sizeof(*mock_vma_res), GFP_KERNEL); 255 if (!mock_vma_res) 256 return -ENOMEM; 257 258 /* Keep creating larger objects until one cannot fit into the hole */ 259 for (size = 12; (hole_end - hole_start) >> size; size++) { 260 I915_RND_SUBSTATE(prng, seed_prng); 261 struct drm_i915_gem_object *obj; 262 unsigned int *order, count, n; 263 u64 hole_size, aligned_size; 264 265 aligned_size = max_t(u32, ilog2(min_alignment), size); 266 hole_size = (hole_end - hole_start) >> aligned_size; 267 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 268 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 269 count = hole_size >> 1; 270 if (!count) { 271 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 272 __func__, hole_start, hole_end, size, hole_size); 273 break; 274 } 275 276 do { 277 order = i915_random_order(count, &prng); 278 if (order) 279 break; 280 } while (count >>= 1); 281 if (!count) { 282 kfree(mock_vma_res); 283 return -ENOMEM; 284 } 285 GEM_BUG_ON(!order); 286 287 GEM_BUG_ON(count * BIT_ULL(aligned_size) > vm->total); 288 GEM_BUG_ON(hole_start + count * BIT_ULL(aligned_size) > hole_end); 289 290 /* Ignore allocation failures (i.e. don't report them as 291 * a test failure) as we are purposefully allocating very 292 * large objects without checking that we have sufficient 293 * memory. We expect to hit -ENOMEM. 294 */ 295 296 obj = fake_dma_object(vm->i915, BIT_ULL(size)); 297 if (IS_ERR(obj)) { 298 kfree(order); 299 break; 300 } 301 302 GEM_BUG_ON(obj->base.size != BIT_ULL(size)); 303 304 if (i915_gem_object_pin_pages_unlocked(obj)) { 305 i915_gem_object_put(obj); 306 kfree(order); 307 break; 308 } 309 310 for (n = 0; n < count; n++) { 311 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 312 intel_wakeref_t wakeref; 313 314 GEM_BUG_ON(addr + BIT_ULL(aligned_size) > vm->total); 315 316 if (igt_timeout(end_time, 317 "%s timed out before %d/%d\n", 318 __func__, n, count)) { 319 hole_end = hole_start; /* quit */ 320 break; 321 } 322 323 if (vm->allocate_va_range) { 324 struct i915_vm_pt_stash stash = {}; 325 struct i915_gem_ww_ctx ww; 326 int err; 327 328 i915_gem_ww_ctx_init(&ww, false); 329 retry: 330 err = i915_vm_lock_objects(vm, &ww); 331 if (err) 332 goto alloc_vm_end; 333 334 err = -ENOMEM; 335 if (i915_vm_alloc_pt_stash(vm, &stash, 336 BIT_ULL(size))) 337 goto alloc_vm_end; 338 339 err = i915_vm_map_pt_stash(vm, &stash); 340 if (!err) 341 vm->allocate_va_range(vm, &stash, 342 addr, BIT_ULL(size)); 343 i915_vm_free_pt_stash(vm, &stash); 344 alloc_vm_end: 345 if (err == -EDEADLK) { 346 err = i915_gem_ww_ctx_backoff(&ww); 347 if (!err) 348 goto retry; 349 } 350 i915_gem_ww_ctx_fini(&ww); 351 352 if (err) 353 break; 354 } 355 356 mock_vma_res->bi.pages = obj->mm.pages; 357 mock_vma_res->node_size = BIT_ULL(aligned_size); 358 mock_vma_res->start = addr; 359 360 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) 361 vm->insert_entries(vm, mock_vma_res, 362 i915_gem_get_pat_index(vm->i915, 363 I915_CACHE_NONE), 364 0); 365 } 366 count = n; 367 368 i915_random_reorder(order, count, &prng); 369 for (n = 0; n < count; n++) { 370 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 371 intel_wakeref_t wakeref; 372 373 GEM_BUG_ON(addr + BIT_ULL(size) > vm->total); 374 with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref) 375 vm->clear_range(vm, addr, BIT_ULL(size)); 376 } 377 378 i915_gem_object_unpin_pages(obj); 379 i915_gem_object_put(obj); 380 381 kfree(order); 382 383 cleanup_freed_objects(vm->i915); 384 } 385 386 kfree(mock_vma_res); 387 return 0; 388 } 389 390 static void close_object_list(struct list_head *objects, 391 struct i915_address_space *vm) 392 { 393 struct drm_i915_gem_object *obj, *on; 394 int __maybe_unused ignored; 395 396 list_for_each_entry_safe(obj, on, objects, st_link) { 397 struct i915_vma *vma; 398 399 vma = i915_vma_instance(obj, vm, NULL); 400 if (!IS_ERR(vma)) 401 ignored = i915_vma_unbind_unlocked(vma); 402 403 list_del(&obj->st_link); 404 i915_gem_object_put(obj); 405 } 406 } 407 408 static int fill_hole(struct i915_address_space *vm, 409 u64 hole_start, u64 hole_end, 410 unsigned long end_time) 411 { 412 const u64 hole_size = hole_end - hole_start; 413 struct drm_i915_gem_object *obj; 414 const unsigned int min_alignment = 415 i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 416 const unsigned long max_pages = 417 min_t(u64, ULONG_MAX - 1, (hole_size / 2) >> ilog2(min_alignment)); 418 const unsigned long max_step = max(int_sqrt(max_pages), 2UL); 419 unsigned long npages, prime, flags; 420 struct i915_vma *vma; 421 LIST_HEAD(objects); 422 int err; 423 424 /* Try binding many VMA working inwards from either edge */ 425 426 flags = PIN_OFFSET_FIXED | PIN_USER; 427 if (i915_is_ggtt(vm)) 428 flags |= PIN_GLOBAL; 429 430 for_each_prime_number_from(prime, 2, max_step) { 431 for (npages = 1; npages <= max_pages; npages *= prime) { 432 const u64 full_size = npages << PAGE_SHIFT; 433 const struct { 434 const char *name; 435 u64 offset; 436 int step; 437 } phases[] = { 438 { "top-down", hole_end, -1, }, 439 { "bottom-up", hole_start, 1, }, 440 { } 441 }, *p; 442 443 obj = fake_dma_object(vm->i915, full_size); 444 if (IS_ERR(obj)) 445 break; 446 447 list_add(&obj->st_link, &objects); 448 449 /* Align differing sized objects against the edges, and 450 * check we don't walk off into the void when binding 451 * them into the GTT. 452 */ 453 for (p = phases; p->name; p++) { 454 u64 offset; 455 456 offset = p->offset; 457 list_for_each_entry(obj, &objects, st_link) { 458 u64 aligned_size = round_up(obj->base.size, 459 min_alignment); 460 461 vma = i915_vma_instance(obj, vm, NULL); 462 if (IS_ERR(vma)) 463 continue; 464 465 if (p->step < 0) { 466 if (offset < hole_start + aligned_size) 467 break; 468 offset -= aligned_size; 469 } 470 471 err = i915_vma_pin(vma, 0, 0, offset | flags); 472 if (err) { 473 pr_err("%s(%s) pin (forward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 474 __func__, p->name, err, npages, prime, offset); 475 goto err; 476 } 477 478 if (!drm_mm_node_allocated(&vma->node) || 479 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 480 pr_err("%s(%s) (forward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 481 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 482 offset); 483 err = -EINVAL; 484 goto err; 485 } 486 487 i915_vma_unpin(vma); 488 489 if (p->step > 0) { 490 if (offset + aligned_size > hole_end) 491 break; 492 offset += aligned_size; 493 } 494 } 495 496 offset = p->offset; 497 list_for_each_entry(obj, &objects, st_link) { 498 u64 aligned_size = round_up(obj->base.size, 499 min_alignment); 500 501 vma = i915_vma_instance(obj, vm, NULL); 502 if (IS_ERR(vma)) 503 continue; 504 505 if (p->step < 0) { 506 if (offset < hole_start + aligned_size) 507 break; 508 offset -= aligned_size; 509 } 510 511 if (!drm_mm_node_allocated(&vma->node) || 512 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 513 pr_err("%s(%s) (forward) moved vma.node=%llx + %llx, expected offset %llx\n", 514 __func__, p->name, vma->node.start, vma->node.size, 515 offset); 516 err = -EINVAL; 517 goto err; 518 } 519 520 err = i915_vma_unbind_unlocked(vma); 521 if (err) { 522 pr_err("%s(%s) (forward) unbind of vma.node=%llx + %llx failed with err=%d\n", 523 __func__, p->name, vma->node.start, vma->node.size, 524 err); 525 goto err; 526 } 527 528 if (p->step > 0) { 529 if (offset + aligned_size > hole_end) 530 break; 531 offset += aligned_size; 532 } 533 } 534 535 offset = p->offset; 536 list_for_each_entry_reverse(obj, &objects, st_link) { 537 u64 aligned_size = round_up(obj->base.size, 538 min_alignment); 539 540 vma = i915_vma_instance(obj, vm, NULL); 541 if (IS_ERR(vma)) 542 continue; 543 544 if (p->step < 0) { 545 if (offset < hole_start + aligned_size) 546 break; 547 offset -= aligned_size; 548 } 549 550 err = i915_vma_pin(vma, 0, 0, offset | flags); 551 if (err) { 552 pr_err("%s(%s) pin (backward) failed with err=%d on size=%lu pages (prime=%lu), offset=%llx\n", 553 __func__, p->name, err, npages, prime, offset); 554 goto err; 555 } 556 557 if (!drm_mm_node_allocated(&vma->node) || 558 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 559 pr_err("%s(%s) (backward) insert failed: vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 560 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 561 offset); 562 err = -EINVAL; 563 goto err; 564 } 565 566 i915_vma_unpin(vma); 567 568 if (p->step > 0) { 569 if (offset + aligned_size > hole_end) 570 break; 571 offset += aligned_size; 572 } 573 } 574 575 offset = p->offset; 576 list_for_each_entry_reverse(obj, &objects, st_link) { 577 u64 aligned_size = round_up(obj->base.size, 578 min_alignment); 579 580 vma = i915_vma_instance(obj, vm, NULL); 581 if (IS_ERR(vma)) 582 continue; 583 584 if (p->step < 0) { 585 if (offset < hole_start + aligned_size) 586 break; 587 offset -= aligned_size; 588 } 589 590 if (!drm_mm_node_allocated(&vma->node) || 591 i915_vma_misplaced(vma, 0, 0, offset | flags)) { 592 pr_err("%s(%s) (backward) moved vma.node=%llx + %llx [allocated? %d], expected offset %llx\n", 593 __func__, p->name, vma->node.start, vma->node.size, drm_mm_node_allocated(&vma->node), 594 offset); 595 err = -EINVAL; 596 goto err; 597 } 598 599 err = i915_vma_unbind_unlocked(vma); 600 if (err) { 601 pr_err("%s(%s) (backward) unbind of vma.node=%llx + %llx failed with err=%d\n", 602 __func__, p->name, vma->node.start, vma->node.size, 603 err); 604 goto err; 605 } 606 607 if (p->step > 0) { 608 if (offset + aligned_size > hole_end) 609 break; 610 offset += aligned_size; 611 } 612 } 613 } 614 615 if (igt_timeout(end_time, "%s timed out (npages=%lu, prime=%lu)\n", 616 __func__, npages, prime)) { 617 err = -EINTR; 618 goto err; 619 } 620 } 621 622 close_object_list(&objects, vm); 623 cleanup_freed_objects(vm->i915); 624 } 625 626 return 0; 627 628 err: 629 close_object_list(&objects, vm); 630 return err; 631 } 632 633 static int walk_hole(struct i915_address_space *vm, 634 u64 hole_start, u64 hole_end, 635 unsigned long end_time) 636 { 637 const u64 hole_size = hole_end - hole_start; 638 const unsigned long max_pages = 639 min_t(u64, ULONG_MAX - 1, hole_size >> PAGE_SHIFT); 640 unsigned long min_alignment; 641 unsigned long flags; 642 u64 size; 643 644 /* Try binding a single VMA in different positions within the hole */ 645 646 flags = PIN_OFFSET_FIXED | PIN_USER; 647 if (i915_is_ggtt(vm)) 648 flags |= PIN_GLOBAL; 649 650 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 651 652 for_each_prime_number_from(size, 1, max_pages) { 653 struct drm_i915_gem_object *obj; 654 struct i915_vma *vma; 655 u64 addr; 656 int err = 0; 657 658 obj = fake_dma_object(vm->i915, size << PAGE_SHIFT); 659 if (IS_ERR(obj)) 660 break; 661 662 vma = i915_vma_instance(obj, vm, NULL); 663 if (IS_ERR(vma)) { 664 err = PTR_ERR(vma); 665 goto err_put; 666 } 667 668 for (addr = hole_start; 669 addr + obj->base.size < hole_end; 670 addr += round_up(obj->base.size, min_alignment)) { 671 err = i915_vma_pin(vma, 0, 0, addr | flags); 672 if (err) { 673 pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n", 674 __func__, addr, vma->size, 675 hole_start, hole_end, err); 676 goto err_put; 677 } 678 i915_vma_unpin(vma); 679 680 if (!drm_mm_node_allocated(&vma->node) || 681 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 682 pr_err("%s incorrect at %llx + %llx\n", 683 __func__, addr, vma->size); 684 err = -EINVAL; 685 goto err_put; 686 } 687 688 err = i915_vma_unbind_unlocked(vma); 689 if (err) { 690 pr_err("%s unbind failed at %llx + %llx with err=%d\n", 691 __func__, addr, vma->size, err); 692 goto err_put; 693 } 694 695 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 696 697 if (igt_timeout(end_time, 698 "%s timed out at %llx\n", 699 __func__, addr)) { 700 err = -EINTR; 701 goto err_put; 702 } 703 } 704 705 err_put: 706 i915_gem_object_put(obj); 707 if (err) 708 return err; 709 710 cleanup_freed_objects(vm->i915); 711 } 712 713 return 0; 714 } 715 716 static int pot_hole(struct i915_address_space *vm, 717 u64 hole_start, u64 hole_end, 718 unsigned long end_time) 719 { 720 struct drm_i915_gem_object *obj; 721 struct i915_vma *vma; 722 unsigned int min_alignment; 723 unsigned long flags; 724 unsigned int pot; 725 int err = 0; 726 727 flags = PIN_OFFSET_FIXED | PIN_USER; 728 if (i915_is_ggtt(vm)) 729 flags |= PIN_GLOBAL; 730 731 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 732 733 obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE); 734 if (IS_ERR(obj)) 735 return PTR_ERR(obj); 736 737 vma = i915_vma_instance(obj, vm, NULL); 738 if (IS_ERR(vma)) { 739 err = PTR_ERR(vma); 740 goto err_obj; 741 } 742 743 /* Insert a pair of pages across every pot boundary within the hole */ 744 for (pot = fls64(hole_end - 1) - 1; 745 pot > ilog2(2 * min_alignment); 746 pot--) { 747 u64 step = BIT_ULL(pot); 748 u64 addr; 749 750 for (addr = round_up(hole_start + min_alignment, step) - min_alignment; 751 hole_end > addr && hole_end - addr >= 2 * min_alignment; 752 addr += step) { 753 err = i915_vma_pin(vma, 0, 0, addr | flags); 754 if (err) { 755 pr_err("%s failed to pin object at %llx in hole [%llx - %llx], with err=%d\n", 756 __func__, 757 addr, 758 hole_start, hole_end, 759 err); 760 goto err_obj; 761 } 762 763 if (!drm_mm_node_allocated(&vma->node) || 764 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 765 pr_err("%s incorrect at %llx + %llx\n", 766 __func__, addr, vma->size); 767 i915_vma_unpin(vma); 768 err = i915_vma_unbind_unlocked(vma); 769 err = -EINVAL; 770 goto err_obj; 771 } 772 773 i915_vma_unpin(vma); 774 err = i915_vma_unbind_unlocked(vma); 775 GEM_BUG_ON(err); 776 } 777 778 if (igt_timeout(end_time, 779 "%s timed out after %d/%d\n", 780 __func__, pot, fls64(hole_end - 1) - 1)) { 781 err = -EINTR; 782 goto err_obj; 783 } 784 } 785 786 err_obj: 787 i915_gem_object_put(obj); 788 return err; 789 } 790 791 static int drunk_hole(struct i915_address_space *vm, 792 u64 hole_start, u64 hole_end, 793 unsigned long end_time) 794 { 795 I915_RND_STATE(prng); 796 unsigned int min_alignment; 797 unsigned int size; 798 unsigned long flags; 799 800 flags = PIN_OFFSET_FIXED | PIN_USER; 801 if (i915_is_ggtt(vm)) 802 flags |= PIN_GLOBAL; 803 804 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 805 806 /* Keep creating larger objects until one cannot fit into the hole */ 807 for (size = 12; (hole_end - hole_start) >> size; size++) { 808 struct drm_i915_gem_object *obj; 809 unsigned int *order, count, n; 810 struct i915_vma *vma; 811 u64 hole_size, aligned_size; 812 int err = -ENODEV; 813 814 aligned_size = max_t(u32, ilog2(min_alignment), size); 815 hole_size = (hole_end - hole_start) >> aligned_size; 816 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32)) 817 hole_size = KMALLOC_MAX_SIZE / sizeof(u32); 818 count = hole_size >> 1; 819 if (!count) { 820 pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n", 821 __func__, hole_start, hole_end, size, hole_size); 822 break; 823 } 824 825 do { 826 order = i915_random_order(count, &prng); 827 if (order) 828 break; 829 } while (count >>= 1); 830 if (!count) 831 return -ENOMEM; 832 GEM_BUG_ON(!order); 833 834 /* Ignore allocation failures (i.e. don't report them as 835 * a test failure) as we are purposefully allocating very 836 * large objects without checking that we have sufficient 837 * memory. We expect to hit -ENOMEM. 838 */ 839 840 obj = fake_dma_object(vm->i915, BIT_ULL(size)); 841 if (IS_ERR(obj)) { 842 kfree(order); 843 break; 844 } 845 846 vma = i915_vma_instance(obj, vm, NULL); 847 if (IS_ERR(vma)) { 848 err = PTR_ERR(vma); 849 goto err_obj; 850 } 851 852 GEM_BUG_ON(vma->size != BIT_ULL(size)); 853 854 for (n = 0; n < count; n++) { 855 u64 addr = hole_start + order[n] * BIT_ULL(aligned_size); 856 857 err = i915_vma_pin(vma, 0, 0, addr | flags); 858 if (err) { 859 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 860 __func__, 861 addr, BIT_ULL(size), 862 hole_start, hole_end, 863 err); 864 goto err_obj; 865 } 866 867 if (!drm_mm_node_allocated(&vma->node) || 868 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 869 pr_err("%s incorrect at %llx + %llx\n", 870 __func__, addr, BIT_ULL(size)); 871 i915_vma_unpin(vma); 872 err = i915_vma_unbind_unlocked(vma); 873 err = -EINVAL; 874 goto err_obj; 875 } 876 877 i915_vma_unpin(vma); 878 err = i915_vma_unbind_unlocked(vma); 879 GEM_BUG_ON(err); 880 881 if (igt_timeout(end_time, 882 "%s timed out after %d/%d\n", 883 __func__, n, count)) { 884 err = -EINTR; 885 goto err_obj; 886 } 887 } 888 889 err_obj: 890 i915_gem_object_put(obj); 891 kfree(order); 892 if (err) 893 return err; 894 895 cleanup_freed_objects(vm->i915); 896 } 897 898 return 0; 899 } 900 901 static int __shrink_hole(struct i915_address_space *vm, 902 u64 hole_start, u64 hole_end, 903 unsigned long end_time) 904 { 905 struct drm_i915_gem_object *obj; 906 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 907 unsigned int min_alignment; 908 unsigned int order = 12; 909 LIST_HEAD(objects); 910 int err = 0; 911 u64 addr; 912 913 min_alignment = i915_vm_min_alignment(vm, INTEL_MEMORY_SYSTEM); 914 915 /* Keep creating larger objects until one cannot fit into the hole */ 916 for (addr = hole_start; addr < hole_end; ) { 917 struct i915_vma *vma; 918 u64 size = BIT_ULL(order++); 919 920 size = min(size, hole_end - addr); 921 obj = fake_dma_object(vm->i915, size); 922 if (IS_ERR(obj)) { 923 err = PTR_ERR(obj); 924 break; 925 } 926 927 list_add(&obj->st_link, &objects); 928 929 vma = i915_vma_instance(obj, vm, NULL); 930 if (IS_ERR(vma)) { 931 err = PTR_ERR(vma); 932 break; 933 } 934 935 GEM_BUG_ON(vma->size != size); 936 937 err = i915_vma_pin(vma, 0, 0, addr | flags); 938 if (err) { 939 pr_err("%s failed to pin object at %llx + %llx in hole [%llx - %llx], with err=%d\n", 940 __func__, addr, size, hole_start, hole_end, err); 941 break; 942 } 943 944 if (!drm_mm_node_allocated(&vma->node) || 945 i915_vma_misplaced(vma, 0, 0, addr | flags)) { 946 pr_err("%s incorrect at %llx + %llx\n", 947 __func__, addr, size); 948 i915_vma_unpin(vma); 949 err = i915_vma_unbind_unlocked(vma); 950 err = -EINVAL; 951 break; 952 } 953 954 i915_vma_unpin(vma); 955 addr += round_up(size, min_alignment); 956 957 /* 958 * Since we are injecting allocation faults at random intervals, 959 * wait for this allocation to complete before we change the 960 * faultinjection. 961 */ 962 err = i915_vma_sync(vma); 963 if (err) 964 break; 965 966 if (igt_timeout(end_time, 967 "%s timed out at ofset %llx [%llx - %llx]\n", 968 __func__, addr, hole_start, hole_end)) { 969 err = -EINTR; 970 break; 971 } 972 } 973 974 close_object_list(&objects, vm); 975 cleanup_freed_objects(vm->i915); 976 return err; 977 } 978 979 static int shrink_hole(struct i915_address_space *vm, 980 u64 hole_start, u64 hole_end, 981 unsigned long end_time) 982 { 983 unsigned long prime; 984 int err; 985 986 vm->fault_attr.probability = 999; 987 atomic_set(&vm->fault_attr.times, -1); 988 989 for_each_prime_number_from(prime, 0, ULONG_MAX - 1) { 990 vm->fault_attr.interval = prime; 991 err = __shrink_hole(vm, hole_start, hole_end, end_time); 992 if (err) 993 break; 994 } 995 996 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 997 998 return err; 999 } 1000 1001 static int shrink_boom(struct i915_address_space *vm, 1002 u64 hole_start, u64 hole_end, 1003 unsigned long end_time) 1004 { 1005 unsigned int sizes[] = { SZ_2M, SZ_1G }; 1006 struct drm_i915_gem_object *purge; 1007 struct drm_i915_gem_object *explode; 1008 int err; 1009 int i; 1010 1011 /* 1012 * Catch the case which shrink_hole seems to miss. The setup here 1013 * requires invoking the shrinker as we do the alloc_pt/alloc_pd, while 1014 * ensuring that all vma assiocated with the respective pd/pdp are 1015 * unpinned at the time. 1016 */ 1017 1018 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 1019 unsigned int flags = PIN_USER | PIN_OFFSET_FIXED; 1020 unsigned int size = sizes[i]; 1021 struct i915_vma *vma; 1022 1023 purge = fake_dma_object(vm->i915, size); 1024 if (IS_ERR(purge)) 1025 return PTR_ERR(purge); 1026 1027 vma = i915_vma_instance(purge, vm, NULL); 1028 if (IS_ERR(vma)) { 1029 err = PTR_ERR(vma); 1030 goto err_purge; 1031 } 1032 1033 err = i915_vma_pin(vma, 0, 0, flags); 1034 if (err) 1035 goto err_purge; 1036 1037 /* Should now be ripe for purging */ 1038 i915_vma_unpin(vma); 1039 1040 explode = fake_dma_object(vm->i915, size); 1041 if (IS_ERR(explode)) { 1042 err = PTR_ERR(explode); 1043 goto err_purge; 1044 } 1045 1046 vm->fault_attr.probability = 100; 1047 vm->fault_attr.interval = 1; 1048 atomic_set(&vm->fault_attr.times, -1); 1049 1050 vma = i915_vma_instance(explode, vm, NULL); 1051 if (IS_ERR(vma)) { 1052 err = PTR_ERR(vma); 1053 goto err_explode; 1054 } 1055 1056 err = i915_vma_pin(vma, 0, 0, flags | size); 1057 if (err) 1058 goto err_explode; 1059 1060 i915_vma_unpin(vma); 1061 1062 i915_gem_object_put(purge); 1063 i915_gem_object_put(explode); 1064 1065 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 1066 cleanup_freed_objects(vm->i915); 1067 } 1068 1069 return 0; 1070 1071 err_explode: 1072 i915_gem_object_put(explode); 1073 err_purge: 1074 i915_gem_object_put(purge); 1075 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr)); 1076 return err; 1077 } 1078 1079 static int misaligned_case(struct i915_address_space *vm, struct intel_memory_region *mr, 1080 u64 addr, u64 size, unsigned long flags) 1081 { 1082 struct drm_i915_gem_object *obj; 1083 struct i915_vma *vma; 1084 int err = 0; 1085 u64 expected_vma_size, expected_node_size; 1086 bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM || 1087 mr->type == INTEL_MEMORY_STOLEN_LOCAL; 1088 1089 obj = i915_gem_object_create_region(mr, size, 0, I915_BO_ALLOC_GPU_ONLY); 1090 if (IS_ERR(obj)) { 1091 /* if iGVT-g or DMAR is active, stolen mem will be uninitialized */ 1092 if (PTR_ERR(obj) == -ENODEV && is_stolen) 1093 return 0; 1094 return PTR_ERR(obj); 1095 } 1096 1097 vma = i915_vma_instance(obj, vm, NULL); 1098 if (IS_ERR(vma)) { 1099 err = PTR_ERR(vma); 1100 goto err_put; 1101 } 1102 1103 err = i915_vma_pin(vma, 0, 0, addr | flags); 1104 if (err) 1105 goto err_put; 1106 i915_vma_unpin(vma); 1107 1108 if (!drm_mm_node_allocated(&vma->node)) { 1109 err = -EINVAL; 1110 goto err_put; 1111 } 1112 1113 if (i915_vma_misplaced(vma, 0, 0, addr | flags)) { 1114 err = -EINVAL; 1115 goto err_put; 1116 } 1117 1118 expected_vma_size = round_up(size, 1 << (ffs(vma->resource->page_sizes_gtt) - 1)); 1119 expected_node_size = expected_vma_size; 1120 1121 if (HAS_64K_PAGES(vm->i915) && i915_gem_object_is_lmem(obj)) { 1122 expected_vma_size = round_up(size, I915_GTT_PAGE_SIZE_64K); 1123 expected_node_size = round_up(size, I915_GTT_PAGE_SIZE_64K); 1124 } 1125 1126 if (vma->size != expected_vma_size || vma->node.size != expected_node_size) { 1127 err = i915_vma_unbind_unlocked(vma); 1128 err = -EBADSLT; 1129 goto err_put; 1130 } 1131 1132 err = i915_vma_unbind_unlocked(vma); 1133 if (err) 1134 goto err_put; 1135 1136 GEM_BUG_ON(drm_mm_node_allocated(&vma->node)); 1137 1138 err_put: 1139 i915_gem_object_put(obj); 1140 cleanup_freed_objects(vm->i915); 1141 return err; 1142 } 1143 1144 static int misaligned_pin(struct i915_address_space *vm, 1145 u64 hole_start, u64 hole_end, 1146 unsigned long end_time) 1147 { 1148 struct intel_memory_region *mr; 1149 enum intel_region_id id; 1150 unsigned long flags = PIN_OFFSET_FIXED | PIN_USER; 1151 int err = 0; 1152 u64 hole_size = hole_end - hole_start; 1153 1154 if (i915_is_ggtt(vm)) 1155 flags |= PIN_GLOBAL; 1156 1157 for_each_memory_region(mr, vm->i915, id) { 1158 u64 min_alignment = i915_vm_min_alignment(vm, mr->type); 1159 u64 size = min_alignment; 1160 u64 addr = round_down(hole_start + (hole_size / 2), min_alignment); 1161 1162 /* avoid -ENOSPC on very small hole setups */ 1163 if (hole_size < 3 * min_alignment) 1164 continue; 1165 1166 /* we can't test < 4k alignment due to flags being encoded in lower bits */ 1167 if (min_alignment != I915_GTT_PAGE_SIZE_4K) { 1168 err = misaligned_case(vm, mr, addr + (min_alignment / 2), size, flags); 1169 /* misaligned should error with -EINVAL*/ 1170 if (!err) 1171 err = -EBADSLT; 1172 if (err != -EINVAL) 1173 return err; 1174 } 1175 1176 /* test for vma->size expansion to min page size */ 1177 err = misaligned_case(vm, mr, addr, PAGE_SIZE, flags); 1178 if (err) 1179 return err; 1180 1181 /* test for intermediate size not expanding vma->size for large alignments */ 1182 err = misaligned_case(vm, mr, addr, size / 2, flags); 1183 if (err) 1184 return err; 1185 } 1186 1187 return 0; 1188 } 1189 1190 static int exercise_ppgtt(struct drm_i915_private *dev_priv, 1191 int (*func)(struct i915_address_space *vm, 1192 u64 hole_start, u64 hole_end, 1193 unsigned long end_time)) 1194 { 1195 struct i915_ppgtt *ppgtt; 1196 IGT_TIMEOUT(end_time); 1197 struct file *file; 1198 int err; 1199 1200 if (!HAS_FULL_PPGTT(dev_priv)) 1201 return 0; 1202 1203 file = mock_file(dev_priv); 1204 if (IS_ERR(file)) 1205 return PTR_ERR(file); 1206 1207 ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0); 1208 if (IS_ERR(ppgtt)) { 1209 err = PTR_ERR(ppgtt); 1210 goto out_free; 1211 } 1212 GEM_BUG_ON(offset_in_page(ppgtt->vm.total)); 1213 assert_vm_alive(&ppgtt->vm); 1214 1215 err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time); 1216 1217 i915_vm_put(&ppgtt->vm); 1218 1219 out_free: 1220 fput(file); 1221 return err; 1222 } 1223 1224 static int igt_ppgtt_fill(void *arg) 1225 { 1226 return exercise_ppgtt(arg, fill_hole); 1227 } 1228 1229 static int igt_ppgtt_walk(void *arg) 1230 { 1231 return exercise_ppgtt(arg, walk_hole); 1232 } 1233 1234 static int igt_ppgtt_pot(void *arg) 1235 { 1236 return exercise_ppgtt(arg, pot_hole); 1237 } 1238 1239 static int igt_ppgtt_drunk(void *arg) 1240 { 1241 return exercise_ppgtt(arg, drunk_hole); 1242 } 1243 1244 static int igt_ppgtt_lowlevel(void *arg) 1245 { 1246 return exercise_ppgtt(arg, lowlevel_hole); 1247 } 1248 1249 static int igt_ppgtt_shrink(void *arg) 1250 { 1251 return exercise_ppgtt(arg, shrink_hole); 1252 } 1253 1254 static int igt_ppgtt_shrink_boom(void *arg) 1255 { 1256 return exercise_ppgtt(arg, shrink_boom); 1257 } 1258 1259 static int igt_ppgtt_misaligned_pin(void *arg) 1260 { 1261 return exercise_ppgtt(arg, misaligned_pin); 1262 } 1263 1264 static int sort_holes(void *priv, const struct list_head *A, 1265 const struct list_head *B) 1266 { 1267 struct drm_mm_node *a = list_entry(A, typeof(*a), hole_stack); 1268 struct drm_mm_node *b = list_entry(B, typeof(*b), hole_stack); 1269 1270 if (a->start < b->start) 1271 return -1; 1272 else 1273 return 1; 1274 } 1275 1276 static int exercise_ggtt(struct drm_i915_private *i915, 1277 int (*func)(struct i915_address_space *vm, 1278 u64 hole_start, u64 hole_end, 1279 unsigned long end_time)) 1280 { 1281 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1282 u64 hole_start, hole_end, last = 0; 1283 struct drm_mm_node *node; 1284 IGT_TIMEOUT(end_time); 1285 int err = 0; 1286 1287 restart: 1288 list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes); 1289 drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) { 1290 if (hole_start < last) 1291 continue; 1292 1293 if (ggtt->vm.mm.color_adjust) 1294 ggtt->vm.mm.color_adjust(node, 0, 1295 &hole_start, &hole_end); 1296 if (hole_start >= hole_end) 1297 continue; 1298 1299 err = func(&ggtt->vm, hole_start, hole_end, end_time); 1300 if (err) 1301 break; 1302 1303 /* As we have manipulated the drm_mm, the list may be corrupt */ 1304 last = hole_end; 1305 goto restart; 1306 } 1307 1308 return err; 1309 } 1310 1311 static int igt_ggtt_fill(void *arg) 1312 { 1313 return exercise_ggtt(arg, fill_hole); 1314 } 1315 1316 static int igt_ggtt_walk(void *arg) 1317 { 1318 return exercise_ggtt(arg, walk_hole); 1319 } 1320 1321 static int igt_ggtt_pot(void *arg) 1322 { 1323 return exercise_ggtt(arg, pot_hole); 1324 } 1325 1326 static int igt_ggtt_drunk(void *arg) 1327 { 1328 return exercise_ggtt(arg, drunk_hole); 1329 } 1330 1331 static int igt_ggtt_lowlevel(void *arg) 1332 { 1333 return exercise_ggtt(arg, lowlevel_hole); 1334 } 1335 1336 static int igt_ggtt_misaligned_pin(void *arg) 1337 { 1338 return exercise_ggtt(arg, misaligned_pin); 1339 } 1340 1341 static int igt_ggtt_page(void *arg) 1342 { 1343 const unsigned int count = PAGE_SIZE/sizeof(u32); 1344 I915_RND_STATE(prng); 1345 struct drm_i915_private *i915 = arg; 1346 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 1347 struct drm_i915_gem_object *obj; 1348 intel_wakeref_t wakeref; 1349 struct drm_mm_node tmp; 1350 unsigned int *order, n; 1351 int err; 1352 1353 if (!i915_ggtt_has_aperture(ggtt)) 1354 return 0; 1355 1356 obj = i915_gem_object_create_internal(i915, PAGE_SIZE); 1357 if (IS_ERR(obj)) 1358 return PTR_ERR(obj); 1359 1360 err = i915_gem_object_pin_pages_unlocked(obj); 1361 if (err) 1362 goto out_free; 1363 1364 memset(&tmp, 0, sizeof(tmp)); 1365 mutex_lock(&ggtt->vm.mutex); 1366 err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp, 1367 count * PAGE_SIZE, 0, 1368 I915_COLOR_UNEVICTABLE, 1369 0, ggtt->mappable_end, 1370 DRM_MM_INSERT_LOW); 1371 mutex_unlock(&ggtt->vm.mutex); 1372 if (err) 1373 goto out_unpin; 1374 1375 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1376 1377 for (n = 0; n < count; n++) { 1378 u64 offset = tmp.start + n * PAGE_SIZE; 1379 1380 ggtt->vm.insert_page(&ggtt->vm, 1381 i915_gem_object_get_dma_address(obj, 0), 1382 offset, 1383 i915_gem_get_pat_index(i915, 1384 I915_CACHE_NONE), 1385 0); 1386 } 1387 1388 order = i915_random_order(count, &prng); 1389 if (!order) { 1390 err = -ENOMEM; 1391 goto out_remove; 1392 } 1393 1394 for (n = 0; n < count; n++) { 1395 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1396 u32 __iomem *vaddr; 1397 1398 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1399 iowrite32(n, vaddr + n); 1400 io_mapping_unmap_atomic(vaddr); 1401 } 1402 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 1403 1404 i915_random_reorder(order, count, &prng); 1405 for (n = 0; n < count; n++) { 1406 u64 offset = tmp.start + order[n] * PAGE_SIZE; 1407 u32 __iomem *vaddr; 1408 u32 val; 1409 1410 vaddr = io_mapping_map_atomic_wc(&ggtt->iomap, offset); 1411 val = ioread32(vaddr + n); 1412 io_mapping_unmap_atomic(vaddr); 1413 1414 if (val != n) { 1415 pr_err("insert page failed: found %d, expected %d\n", 1416 val, n); 1417 err = -EINVAL; 1418 break; 1419 } 1420 } 1421 1422 kfree(order); 1423 out_remove: 1424 ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size); 1425 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 1426 mutex_lock(&ggtt->vm.mutex); 1427 drm_mm_remove_node(&tmp); 1428 mutex_unlock(&ggtt->vm.mutex); 1429 out_unpin: 1430 i915_gem_object_unpin_pages(obj); 1431 out_free: 1432 i915_gem_object_put(obj); 1433 return err; 1434 } 1435 1436 static void track_vma_bind(struct i915_vma *vma) 1437 { 1438 struct drm_i915_gem_object *obj = vma->obj; 1439 1440 __i915_gem_object_pin_pages(obj); 1441 1442 GEM_BUG_ON(atomic_read(&vma->pages_count)); 1443 atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE); 1444 __i915_gem_object_pin_pages(obj); 1445 vma->pages = obj->mm.pages; 1446 vma->resource->bi.pages = vma->pages; 1447 1448 mutex_lock(&vma->vm->mutex); 1449 list_move_tail(&vma->vm_link, &vma->vm->bound_list); 1450 mutex_unlock(&vma->vm->mutex); 1451 } 1452 1453 static int exercise_mock(struct drm_i915_private *i915, 1454 int (*func)(struct i915_address_space *vm, 1455 u64 hole_start, u64 hole_end, 1456 unsigned long end_time)) 1457 { 1458 const u64 limit = totalram_pages() << PAGE_SHIFT; 1459 struct i915_address_space *vm; 1460 struct i915_gem_context *ctx; 1461 IGT_TIMEOUT(end_time); 1462 int err; 1463 1464 ctx = mock_context(i915, "mock"); 1465 if (!ctx) 1466 return -ENOMEM; 1467 1468 vm = i915_gem_context_get_eb_vm(ctx); 1469 err = func(vm, 0, min(vm->total, limit), end_time); 1470 i915_vm_put(vm); 1471 1472 mock_context_close(ctx); 1473 return err; 1474 } 1475 1476 static int igt_mock_fill(void *arg) 1477 { 1478 struct i915_ggtt *ggtt = arg; 1479 1480 return exercise_mock(ggtt->vm.i915, fill_hole); 1481 } 1482 1483 static int igt_mock_walk(void *arg) 1484 { 1485 struct i915_ggtt *ggtt = arg; 1486 1487 return exercise_mock(ggtt->vm.i915, walk_hole); 1488 } 1489 1490 static int igt_mock_pot(void *arg) 1491 { 1492 struct i915_ggtt *ggtt = arg; 1493 1494 return exercise_mock(ggtt->vm.i915, pot_hole); 1495 } 1496 1497 static int igt_mock_drunk(void *arg) 1498 { 1499 struct i915_ggtt *ggtt = arg; 1500 1501 return exercise_mock(ggtt->vm.i915, drunk_hole); 1502 } 1503 1504 static int reserve_gtt_with_resource(struct i915_vma *vma, u64 offset) 1505 { 1506 struct i915_address_space *vm = vma->vm; 1507 struct i915_vma_resource *vma_res; 1508 struct drm_i915_gem_object *obj = vma->obj; 1509 int err; 1510 1511 vma_res = i915_vma_resource_alloc(); 1512 if (IS_ERR(vma_res)) 1513 return PTR_ERR(vma_res); 1514 1515 mutex_lock(&vm->mutex); 1516 err = i915_gem_gtt_reserve(vm, NULL, &vma->node, obj->base.size, 1517 offset, 1518 obj->pat_index, 1519 0); 1520 if (!err) { 1521 i915_vma_resource_init_from_vma(vma_res, vma); 1522 vma->resource = vma_res; 1523 } else { 1524 kfree(vma_res); 1525 } 1526 mutex_unlock(&vm->mutex); 1527 1528 return err; 1529 } 1530 1531 static int igt_gtt_reserve(void *arg) 1532 { 1533 struct i915_ggtt *ggtt = arg; 1534 struct drm_i915_gem_object *obj, *on; 1535 I915_RND_STATE(prng); 1536 LIST_HEAD(objects); 1537 u64 total; 1538 int err = -ENODEV; 1539 1540 /* i915_gem_gtt_reserve() tries to reserve the precise range 1541 * for the node, and evicts if it has to. So our test checks that 1542 * it can give us the requsted space and prevent overlaps. 1543 */ 1544 1545 /* Start by filling the GGTT */ 1546 for (total = 0; 1547 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1548 total += 2 * I915_GTT_PAGE_SIZE) { 1549 struct i915_vma *vma; 1550 1551 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1552 2 * PAGE_SIZE); 1553 if (IS_ERR(obj)) { 1554 err = PTR_ERR(obj); 1555 goto out; 1556 } 1557 1558 err = i915_gem_object_pin_pages_unlocked(obj); 1559 if (err) { 1560 i915_gem_object_put(obj); 1561 goto out; 1562 } 1563 1564 list_add(&obj->st_link, &objects); 1565 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1566 if (IS_ERR(vma)) { 1567 err = PTR_ERR(vma); 1568 goto out; 1569 } 1570 1571 err = reserve_gtt_with_resource(vma, total); 1572 if (err) { 1573 pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n", 1574 total, ggtt->vm.total, err); 1575 goto out; 1576 } 1577 track_vma_bind(vma); 1578 1579 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1580 if (vma->node.start != total || 1581 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1582 pr_err("i915_gem_gtt_reserve (pass 1) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1583 vma->node.start, vma->node.size, 1584 total, 2*I915_GTT_PAGE_SIZE); 1585 err = -EINVAL; 1586 goto out; 1587 } 1588 } 1589 1590 /* Now we start forcing evictions */ 1591 for (total = I915_GTT_PAGE_SIZE; 1592 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1593 total += 2 * I915_GTT_PAGE_SIZE) { 1594 struct i915_vma *vma; 1595 1596 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1597 2 * PAGE_SIZE); 1598 if (IS_ERR(obj)) { 1599 err = PTR_ERR(obj); 1600 goto out; 1601 } 1602 1603 err = i915_gem_object_pin_pages_unlocked(obj); 1604 if (err) { 1605 i915_gem_object_put(obj); 1606 goto out; 1607 } 1608 1609 list_add(&obj->st_link, &objects); 1610 1611 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1612 if (IS_ERR(vma)) { 1613 err = PTR_ERR(vma); 1614 goto out; 1615 } 1616 1617 err = reserve_gtt_with_resource(vma, total); 1618 if (err) { 1619 pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n", 1620 total, ggtt->vm.total, err); 1621 goto out; 1622 } 1623 track_vma_bind(vma); 1624 1625 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1626 if (vma->node.start != total || 1627 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1628 pr_err("i915_gem_gtt_reserve (pass 2) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1629 vma->node.start, vma->node.size, 1630 total, 2*I915_GTT_PAGE_SIZE); 1631 err = -EINVAL; 1632 goto out; 1633 } 1634 } 1635 1636 /* And then try at random */ 1637 list_for_each_entry_safe(obj, on, &objects, st_link) { 1638 struct i915_vma *vma; 1639 u64 offset; 1640 1641 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1642 if (IS_ERR(vma)) { 1643 err = PTR_ERR(vma); 1644 goto out; 1645 } 1646 1647 err = i915_vma_unbind_unlocked(vma); 1648 if (err) { 1649 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1650 goto out; 1651 } 1652 1653 offset = igt_random_offset(&prng, 1654 0, ggtt->vm.total, 1655 2 * I915_GTT_PAGE_SIZE, 1656 I915_GTT_MIN_ALIGNMENT); 1657 1658 err = reserve_gtt_with_resource(vma, offset); 1659 if (err) { 1660 pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n", 1661 total, ggtt->vm.total, err); 1662 goto out; 1663 } 1664 track_vma_bind(vma); 1665 1666 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1667 if (vma->node.start != offset || 1668 vma->node.size != 2*I915_GTT_PAGE_SIZE) { 1669 pr_err("i915_gem_gtt_reserve (pass 3) placement failed, found (%llx + %llx), expected (%llx + %llx)\n", 1670 vma->node.start, vma->node.size, 1671 offset, 2*I915_GTT_PAGE_SIZE); 1672 err = -EINVAL; 1673 goto out; 1674 } 1675 } 1676 1677 out: 1678 list_for_each_entry_safe(obj, on, &objects, st_link) { 1679 i915_gem_object_unpin_pages(obj); 1680 i915_gem_object_put(obj); 1681 } 1682 return err; 1683 } 1684 1685 static int insert_gtt_with_resource(struct i915_vma *vma) 1686 { 1687 struct i915_address_space *vm = vma->vm; 1688 struct i915_vma_resource *vma_res; 1689 struct drm_i915_gem_object *obj = vma->obj; 1690 int err; 1691 1692 vma_res = i915_vma_resource_alloc(); 1693 if (IS_ERR(vma_res)) 1694 return PTR_ERR(vma_res); 1695 1696 mutex_lock(&vm->mutex); 1697 err = i915_gem_gtt_insert(vm, NULL, &vma->node, obj->base.size, 0, 1698 obj->pat_index, 0, vm->total, 0); 1699 if (!err) { 1700 i915_vma_resource_init_from_vma(vma_res, vma); 1701 vma->resource = vma_res; 1702 } else { 1703 kfree(vma_res); 1704 } 1705 mutex_unlock(&vm->mutex); 1706 1707 return err; 1708 } 1709 1710 static int igt_gtt_insert(void *arg) 1711 { 1712 struct i915_ggtt *ggtt = arg; 1713 struct drm_i915_gem_object *obj, *on; 1714 struct drm_mm_node tmp = {}; 1715 const struct invalid_insert { 1716 u64 size; 1717 u64 alignment; 1718 u64 start, end; 1719 } invalid_insert[] = { 1720 { 1721 ggtt->vm.total + I915_GTT_PAGE_SIZE, 0, 1722 0, ggtt->vm.total, 1723 }, 1724 { 1725 2*I915_GTT_PAGE_SIZE, 0, 1726 0, I915_GTT_PAGE_SIZE, 1727 }, 1728 { 1729 -(u64)I915_GTT_PAGE_SIZE, 0, 1730 0, 4*I915_GTT_PAGE_SIZE, 1731 }, 1732 { 1733 -(u64)2*I915_GTT_PAGE_SIZE, 2*I915_GTT_PAGE_SIZE, 1734 0, 4*I915_GTT_PAGE_SIZE, 1735 }, 1736 { 1737 I915_GTT_PAGE_SIZE, I915_GTT_MIN_ALIGNMENT << 1, 1738 I915_GTT_MIN_ALIGNMENT, I915_GTT_MIN_ALIGNMENT << 1, 1739 }, 1740 {} 1741 }, *ii; 1742 LIST_HEAD(objects); 1743 u64 total; 1744 int err = -ENODEV; 1745 1746 /* i915_gem_gtt_insert() tries to allocate some free space in the GTT 1747 * to the node, evicting if required. 1748 */ 1749 1750 /* Check a couple of obviously invalid requests */ 1751 for (ii = invalid_insert; ii->size; ii++) { 1752 mutex_lock(&ggtt->vm.mutex); 1753 err = i915_gem_gtt_insert(&ggtt->vm, NULL, &tmp, 1754 ii->size, ii->alignment, 1755 I915_COLOR_UNEVICTABLE, 1756 ii->start, ii->end, 1757 0); 1758 mutex_unlock(&ggtt->vm.mutex); 1759 if (err != -ENOSPC) { 1760 pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n", 1761 ii->size, ii->alignment, ii->start, ii->end, 1762 err); 1763 return -EINVAL; 1764 } 1765 } 1766 1767 /* Start by filling the GGTT */ 1768 for (total = 0; 1769 total + I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1770 total += I915_GTT_PAGE_SIZE) { 1771 struct i915_vma *vma; 1772 1773 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1774 I915_GTT_PAGE_SIZE); 1775 if (IS_ERR(obj)) { 1776 err = PTR_ERR(obj); 1777 goto out; 1778 } 1779 1780 err = i915_gem_object_pin_pages_unlocked(obj); 1781 if (err) { 1782 i915_gem_object_put(obj); 1783 goto out; 1784 } 1785 1786 list_add(&obj->st_link, &objects); 1787 1788 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1789 if (IS_ERR(vma)) { 1790 err = PTR_ERR(vma); 1791 goto out; 1792 } 1793 1794 err = insert_gtt_with_resource(vma); 1795 if (err == -ENOSPC) { 1796 /* maxed out the GGTT space */ 1797 i915_gem_object_put(obj); 1798 break; 1799 } 1800 if (err) { 1801 pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n", 1802 total, ggtt->vm.total, err); 1803 goto out; 1804 } 1805 track_vma_bind(vma); 1806 __i915_vma_pin(vma); 1807 1808 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1809 } 1810 1811 list_for_each_entry(obj, &objects, st_link) { 1812 struct i915_vma *vma; 1813 1814 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1815 if (IS_ERR(vma)) { 1816 err = PTR_ERR(vma); 1817 goto out; 1818 } 1819 1820 if (!drm_mm_node_allocated(&vma->node)) { 1821 pr_err("VMA was unexpectedly evicted!\n"); 1822 err = -EINVAL; 1823 goto out; 1824 } 1825 1826 __i915_vma_unpin(vma); 1827 } 1828 1829 /* If we then reinsert, we should find the same hole */ 1830 list_for_each_entry_safe(obj, on, &objects, st_link) { 1831 struct i915_vma *vma; 1832 u64 offset; 1833 1834 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1835 if (IS_ERR(vma)) { 1836 err = PTR_ERR(vma); 1837 goto out; 1838 } 1839 1840 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1841 offset = vma->node.start; 1842 1843 err = i915_vma_unbind_unlocked(vma); 1844 if (err) { 1845 pr_err("i915_vma_unbind failed with err=%d!\n", err); 1846 goto out; 1847 } 1848 1849 err = insert_gtt_with_resource(vma); 1850 if (err) { 1851 pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n", 1852 total, ggtt->vm.total, err); 1853 goto out; 1854 } 1855 track_vma_bind(vma); 1856 1857 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1858 if (vma->node.start != offset) { 1859 pr_err("i915_gem_gtt_insert did not return node to its previous location (the only hole), expected address %llx, found %llx\n", 1860 offset, vma->node.start); 1861 err = -EINVAL; 1862 goto out; 1863 } 1864 } 1865 1866 /* And then force evictions */ 1867 for (total = 0; 1868 total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total; 1869 total += 2 * I915_GTT_PAGE_SIZE) { 1870 struct i915_vma *vma; 1871 1872 obj = i915_gem_object_create_internal(ggtt->vm.i915, 1873 2 * I915_GTT_PAGE_SIZE); 1874 if (IS_ERR(obj)) { 1875 err = PTR_ERR(obj); 1876 goto out; 1877 } 1878 1879 err = i915_gem_object_pin_pages_unlocked(obj); 1880 if (err) { 1881 i915_gem_object_put(obj); 1882 goto out; 1883 } 1884 1885 list_add(&obj->st_link, &objects); 1886 1887 vma = i915_vma_instance(obj, &ggtt->vm, NULL); 1888 if (IS_ERR(vma)) { 1889 err = PTR_ERR(vma); 1890 goto out; 1891 } 1892 1893 err = insert_gtt_with_resource(vma); 1894 if (err) { 1895 pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n", 1896 total, ggtt->vm.total, err); 1897 goto out; 1898 } 1899 track_vma_bind(vma); 1900 1901 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 1902 } 1903 1904 out: 1905 list_for_each_entry_safe(obj, on, &objects, st_link) { 1906 i915_gem_object_unpin_pages(obj); 1907 i915_gem_object_put(obj); 1908 } 1909 return err; 1910 } 1911 1912 int i915_gem_gtt_mock_selftests(void) 1913 { 1914 static const struct i915_subtest tests[] = { 1915 SUBTEST(igt_mock_drunk), 1916 SUBTEST(igt_mock_walk), 1917 SUBTEST(igt_mock_pot), 1918 SUBTEST(igt_mock_fill), 1919 SUBTEST(igt_gtt_reserve), 1920 SUBTEST(igt_gtt_insert), 1921 }; 1922 struct drm_i915_private *i915; 1923 struct intel_gt *gt; 1924 int err; 1925 1926 i915 = mock_gem_device(); 1927 if (!i915) 1928 return -ENOMEM; 1929 1930 /* allocate the ggtt */ 1931 err = intel_gt_assign_ggtt(to_gt(i915)); 1932 if (err) 1933 goto out_put; 1934 1935 gt = to_gt(i915); 1936 1937 mock_init_ggtt(gt); 1938 1939 err = i915_subtests(tests, gt->ggtt); 1940 1941 mock_device_flush(i915); 1942 i915_gem_drain_freed_objects(i915); 1943 mock_fini_ggtt(gt->ggtt); 1944 1945 out_put: 1946 mock_destroy_device(i915); 1947 return err; 1948 } 1949 1950 int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) 1951 { 1952 static const struct i915_subtest tests[] = { 1953 SUBTEST(igt_ppgtt_alloc), 1954 SUBTEST(igt_ppgtt_lowlevel), 1955 SUBTEST(igt_ppgtt_drunk), 1956 SUBTEST(igt_ppgtt_walk), 1957 SUBTEST(igt_ppgtt_pot), 1958 SUBTEST(igt_ppgtt_fill), 1959 SUBTEST(igt_ppgtt_shrink), 1960 SUBTEST(igt_ppgtt_shrink_boom), 1961 SUBTEST(igt_ppgtt_misaligned_pin), 1962 SUBTEST(igt_ggtt_lowlevel), 1963 SUBTEST(igt_ggtt_drunk), 1964 SUBTEST(igt_ggtt_walk), 1965 SUBTEST(igt_ggtt_pot), 1966 SUBTEST(igt_ggtt_fill), 1967 SUBTEST(igt_ggtt_page), 1968 SUBTEST(igt_ggtt_misaligned_pin), 1969 }; 1970 1971 GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total)); 1972 1973 return i915_live_subtests(tests, i915); 1974 } 1975