1 // SPDX-License-Identifier: GPL-2.0 AND MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include <kunit/test.h> 7 #include <kunit/visibility.h> 8 9 #include <linux/iosys-map.h> 10 #include <linux/math64.h> 11 #include <linux/prandom.h> 12 #include <linux/swap.h> 13 14 #include <uapi/linux/sysinfo.h> 15 16 #include "tests/xe_kunit_helpers.h" 17 #include "tests/xe_pci_test.h" 18 #include "tests/xe_test.h" 19 20 #include "xe_bo_evict.h" 21 #include "xe_gt.h" 22 #include "xe_pci.h" 23 #include "xe_pm.h" 24 25 #ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE 26 struct page_size_alloc_saved { 27 enum xe_page_size_alloc_ctrl_mode mode; 28 u32 cur_index; 29 }; 30 31 /* Caller must hold xe->page_size_alloc_ctrl.lock. */ 32 static void page_size_alloc_save(struct xe_device *xe, 33 struct page_size_alloc_saved *s) 34 { 35 s->mode = xe->page_size_alloc_ctrl.mode; 36 s->cur_index = xe->page_size_alloc_ctrl.cur_index; 37 } 38 39 static void page_size_alloc_restore(struct xe_device *xe, 40 const struct page_size_alloc_saved *s) 41 { 42 mutex_lock(&xe->page_size_alloc_ctrl.lock); 43 xe->page_size_alloc_ctrl.mode = s->mode; 44 xe->page_size_alloc_ctrl.cur_index = s->cur_index; 45 mutex_unlock(&xe->page_size_alloc_ctrl.lock); 46 } 47 48 /* Expected properties for a forced page-size allocation mode. */ 49 struct leaf_info { 50 u64 leaf; 51 u64 alloc_size; 52 u32 flag; 53 const char *name; 54 }; 55 56 static const struct leaf_info leaf_2m = { 57 .leaf = SZ_2M, 58 .alloc_size = SZ_2M - PAGE_SIZE, 59 .flag = XE_BO_FLAG_NEEDS_2M, 60 .name = "2M", 61 }; 62 63 static const struct leaf_info leaf_1g = { 64 .leaf = SZ_1G, 65 .alloc_size = SZ_1G - PAGE_SIZE, 66 .flag = XE_BO_FLAG_NEEDS_1G, 67 .name = "1G", 68 }; 69 70 static void run_only_leaf(struct kunit *test, 71 enum xe_page_size_alloc_ctrl_mode mode, 72 const struct leaf_info *li) 73 { 74 struct xe_device *xe = test->priv; 75 struct page_size_alloc_saved saved; 76 struct xe_bo *bo; 77 struct ttm_buffer_object *ttm_bo; 78 u32 other_flags; 79 80 if (!IS_DGFX(xe)) { 81 kunit_skip(test, "requires dGFX VRAM"); 82 return; 83 } 84 85 mutex_lock(&xe->page_size_alloc_ctrl.lock); 86 page_size_alloc_save(xe, &saved); 87 xe->page_size_alloc_ctrl.mode = mode; 88 mutex_unlock(&xe->page_size_alloc_ctrl.lock); 89 90 bo = xe_bo_create_user(xe, NULL, li->alloc_size, 91 DRM_XE_GEM_CPU_CACHING_WC, 92 XE_BO_FLAG_VRAM0, NULL); 93 if (IS_ERR(bo)) { 94 page_size_alloc_restore(xe, &saved); 95 if (PTR_ERR(bo) == -ENOSPC) { 96 kunit_skip(test, 97 "no contiguous %s VRAM available right now", 98 li->name); 99 return; 100 } 101 102 KUNIT_FAIL(test, "%s BO alloc failed: %pe", li->name, bo); 103 return; 104 } 105 106 ttm_bo = &bo->ttm; 107 108 /* 1) The mode added the right NEEDS_* flag. */ 109 KUNIT_EXPECT_TRUE_MSG(test, bo->flags & li->flag, 110 "%s: flag missing, flags=0x%x", 111 li->name, bo->flags); 112 113 /* 2) No other NEEDS_* flags accidentally tagged on. */ 114 other_flags = (XE_BO_FLAG_NEEDS_64K | 115 XE_BO_FLAG_NEEDS_2M | 116 XE_BO_FLAG_NEEDS_1G) & ~li->flag; 117 KUNIT_EXPECT_FALSE_MSG(test, bo->flags & other_flags, 118 "%s: stray flags=0x%x", 119 li->name, bo->flags); 120 /* 3) BO size was rounded up to the expected leaf size. */ 121 KUNIT_EXPECT_EQ_MSG(test, xe_bo_size(bo), li->leaf, 122 "%s: bo size=%llu expected=%llu", 123 li->name, 124 (u64)xe_bo_size(bo), 125 (u64)li->leaf); 126 /* 127 * 4) Allocator honored the requested alignment. 128 * ttm_bo->page_alignment is stored in PAGE_SIZE units, so compare against 129 * the expected leaf size converted with >> PAGE_SHIFT. 130 */ 131 KUNIT_EXPECT_EQ_MSG(test, ttm_bo->page_alignment, 132 li->leaf >> PAGE_SHIFT, 133 "%s: page_alignment=%u pages expected=%llu pages", 134 li->name, ttm_bo->page_alignment, 135 (u64)(li->leaf >> PAGE_SHIFT)); 136 137 xe_bo_put(bo); 138 page_size_alloc_restore(xe, &saved); 139 } 140 141 static void xe_bo_page_size_alloc_only_2m(struct kunit *test) 142 { 143 run_only_leaf(test, XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_2M, &leaf_2m); 144 } 145 146 static void xe_bo_page_size_alloc_only_1g(struct kunit *test) 147 { 148 run_only_leaf(test, XE_PAGE_SIZE_ALLOC_CTRL_MODE_ONLY_1G, &leaf_1g); 149 } 150 151 static void xe_bo_page_size_alloc_mixed_bos(struct kunit *test) 152 { 153 struct xe_device *xe = test->priv; 154 struct page_size_alloc_saved saved; 155 struct xe_bo *bo; 156 struct ttm_buffer_object *ttm_bo; 157 u32 all_flags = XE_BO_FLAG_NEEDS_64K | XE_BO_FLAG_NEEDS_2M | 158 XE_BO_FLAG_NEEDS_1G; 159 u32 flags; 160 u64 expected_align; 161 int i; 162 const int n = 4; 163 164 if (!IS_DGFX(xe)) { 165 kunit_skip(test, "requires dGFX VRAM"); 166 return; 167 } 168 169 mutex_lock(&xe->page_size_alloc_ctrl.lock); 170 page_size_alloc_save(xe, &saved); 171 mutex_unlock(&xe->page_size_alloc_ctrl.lock); 172 173 for (i = 0; i < n; i++) { 174 mutex_lock(&xe->page_size_alloc_ctrl.lock); 175 xe->page_size_alloc_ctrl.mode = XE_PAGE_SIZE_ALLOC_CTRL_MODE_MIXED; 176 xe->page_size_alloc_ctrl.cur_index = i; 177 mutex_unlock(&xe->page_size_alloc_ctrl.lock); 178 /* 179 * Request a size valid for any mixed-mode slot. Since cur_index is 180 * device-global and may be perturbed by concurrent allocations on 181 * a live system, do not assume this iteration will see a specific 182 * slot. 183 */ 184 bo = xe_bo_create_user(xe, NULL, SZ_1G, 185 DRM_XE_GEM_CPU_CACHING_WC, 186 XE_BO_FLAG_VRAM0, NULL); 187 if (IS_ERR(bo)) { 188 int err = PTR_ERR(bo); 189 190 page_size_alloc_restore(xe, &saved); 191 if (err == -ENOSPC) { 192 kunit_skip(test, 193 "mixed mode BO allocation unavailable: %d", 194 err); 195 return; 196 } 197 KUNIT_FAIL(test, "iter=%d alloc failed: %pe", i, bo); 198 return; 199 } 200 201 ttm_bo = &bo->ttm; 202 flags = bo->flags & all_flags; 203 /* 204 * Mixed mode may result in: 205 * 0-> default platform VRAM alignment 206 * XE_BO_FLAG_NEEDS_64K 207 * XE_BO_FLAG_NEEDS_2M 208 * XE_BO_FLAG_NEEDS_1G 209 * Any other combination is invalid. 210 */ 211 if (flags == 0) { 212 expected_align = SZ_4K; 213 if (xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) 214 expected_align = SZ_64K; 215 } else if (flags == XE_BO_FLAG_NEEDS_64K) { 216 expected_align = SZ_64K; 217 } else if (flags == XE_BO_FLAG_NEEDS_2M) { 218 expected_align = SZ_2M; 219 } else if (flags == XE_BO_FLAG_NEEDS_1G) { 220 expected_align = SZ_1G; 221 } else { 222 KUNIT_FAIL(test, 223 "iter=%d invalid mixed-mode flags: 0x%x", 224 i, flags); 225 xe_bo_put(bo); 226 page_size_alloc_restore(xe, &saved); 227 return; 228 } 229 /* 230 * BO size should remain valid for the selected mode. Since the 231 * request is SZ_1G, it should remain unchanged regardless of the 232 * selected page-size policy. 233 */ 234 KUNIT_EXPECT_EQ_MSG(test, xe_bo_size(bo), (u64)SZ_1G, 235 "iter=%d size=%llu expected=%llu", 236 i, 237 (u64)xe_bo_size(bo), 238 (u64)SZ_1G); 239 KUNIT_EXPECT_EQ_MSG(test, ttm_bo->page_alignment, 240 expected_align >> PAGE_SHIFT, 241 "iter=%d flags=0x%x page_alignment=%u pages expected=%llu pages", 242 i, flags, ttm_bo->page_alignment, 243 (u64)(expected_align >> PAGE_SHIFT)); 244 xe_bo_put(bo); 245 } 246 page_size_alloc_restore(xe, &saved); 247 } 248 #endif 249 250 static int ccs_test_migrate(struct xe_tile *tile, struct xe_bo *bo, 251 bool clear, u64 get_val, u64 assign_val, 252 struct kunit *test, struct drm_exec *exec) 253 { 254 struct dma_fence *fence; 255 struct ttm_tt *ttm; 256 struct page *page; 257 pgoff_t ccs_page; 258 long timeout; 259 u64 *cpu_map; 260 int ret; 261 u32 offset; 262 263 /* Move bo to VRAM if not already there. */ 264 ret = xe_bo_validate(bo, NULL, false, exec); 265 if (ret) { 266 KUNIT_FAIL(test, "Failed to validate bo.\n"); 267 return ret; 268 } 269 270 /* Optionally clear bo *and* CCS data in VRAM. */ 271 if (clear) { 272 fence = xe_migrate_clear(tile->migrate, bo, bo->ttm.resource, 273 XE_MIGRATE_CLEAR_FLAG_FULL); 274 if (IS_ERR(fence)) { 275 KUNIT_FAIL(test, "Failed to submit bo clear.\n"); 276 return PTR_ERR(fence); 277 } 278 279 if (dma_fence_wait_timeout(fence, false, 5 * HZ) <= 0) { 280 dma_fence_put(fence); 281 KUNIT_FAIL(test, "Timeout while clearing bo.\n"); 282 return -ETIME; 283 } 284 285 dma_fence_put(fence); 286 } 287 288 /* Evict to system. CCS data should be copied. */ 289 ret = xe_bo_evict(bo, exec); 290 if (ret) { 291 KUNIT_FAIL(test, "Failed to evict bo.\n"); 292 return ret; 293 } 294 295 /* Sync all migration blits */ 296 timeout = dma_resv_wait_timeout(bo->ttm.base.resv, 297 DMA_RESV_USAGE_KERNEL, 298 true, 299 5 * HZ); 300 if (timeout <= 0) { 301 KUNIT_FAIL(test, "Failed to sync bo eviction.\n"); 302 return -ETIME; 303 } 304 305 /* 306 * Bo with CCS data is now in system memory. Verify backing store 307 * and data integrity. Then assign for the next testing round while 308 * we still have a CPU map. 309 */ 310 ttm = bo->ttm.ttm; 311 if (!ttm || !ttm_tt_is_populated(ttm)) { 312 KUNIT_FAIL(test, "Bo was not in expected placement.\n"); 313 return -EINVAL; 314 } 315 316 ccs_page = xe_bo_ccs_pages_start(bo) >> PAGE_SHIFT; 317 if (ccs_page >= ttm->num_pages) { 318 KUNIT_FAIL(test, "No TTM CCS pages present.\n"); 319 return -EINVAL; 320 } 321 322 page = ttm->pages[ccs_page]; 323 cpu_map = kmap_local_page(page); 324 325 /* Check first CCS value */ 326 if (cpu_map[0] != get_val) { 327 KUNIT_FAIL(test, 328 "Expected CCS readout 0x%016llx, got 0x%016llx.\n", 329 (unsigned long long)get_val, 330 (unsigned long long)cpu_map[0]); 331 ret = -EINVAL; 332 } 333 334 /* Check last CCS value, or at least last value in page. */ 335 offset = xe_device_ccs_bytes(tile_to_xe(tile), xe_bo_size(bo)); 336 offset = min_t(u32, offset, PAGE_SIZE) / sizeof(u64) - 1; 337 if (cpu_map[offset] != get_val) { 338 KUNIT_FAIL(test, 339 "Expected CCS readout 0x%016llx, got 0x%016llx.\n", 340 (unsigned long long)get_val, 341 (unsigned long long)cpu_map[offset]); 342 ret = -EINVAL; 343 } 344 345 cpu_map[0] = assign_val; 346 cpu_map[offset] = assign_val; 347 kunmap_local(cpu_map); 348 349 return ret; 350 } 351 352 static void ccs_test_run_tile(struct xe_device *xe, struct xe_tile *tile, 353 struct kunit *test) 354 { 355 struct xe_bo *bo; 356 357 int ret; 358 359 /* TODO: Sanity check */ 360 unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile); 361 struct drm_exec *exec = XE_VALIDATION_OPT_OUT; 362 363 if (IS_DGFX(xe)) 364 kunit_info(test, "Testing vram id %u\n", tile->id); 365 else 366 kunit_info(test, "Testing system memory\n"); 367 368 bo = xe_bo_create_user(xe, NULL, SZ_1M, DRM_XE_GEM_CPU_CACHING_WC, 369 bo_flags, exec); 370 if (IS_ERR(bo)) { 371 KUNIT_FAIL(test, "Failed to create bo.\n"); 372 return; 373 } 374 375 xe_bo_lock(bo, false); 376 377 kunit_info(test, "Verifying that CCS data is cleared on creation.\n"); 378 ret = ccs_test_migrate(tile, bo, false, 0ULL, 0xdeadbeefdeadbeefULL, 379 test, exec); 380 if (ret) 381 goto out_unlock; 382 383 kunit_info(test, "Verifying that CCS data survives migration.\n"); 384 ret = ccs_test_migrate(tile, bo, false, 0xdeadbeefdeadbeefULL, 385 0xdeadbeefdeadbeefULL, test, exec); 386 if (ret) 387 goto out_unlock; 388 389 kunit_info(test, "Verifying that CCS data can be properly cleared.\n"); 390 ret = ccs_test_migrate(tile, bo, true, 0ULL, 0ULL, test, exec); 391 392 out_unlock: 393 xe_bo_unlock(bo); 394 xe_bo_put(bo); 395 } 396 397 static int ccs_test_run_device(struct xe_device *xe) 398 { 399 struct kunit *test = kunit_get_current_test(); 400 struct xe_tile *tile; 401 int id; 402 403 if (!xe_device_has_flat_ccs(xe)) { 404 kunit_skip(test, "non-flat-ccs device\n"); 405 return 0; 406 } 407 408 /* For xe2+ dgfx, we don't handle ccs metadata */ 409 if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)) { 410 kunit_skip(test, "xe2+ dgfx device\n"); 411 return 0; 412 } 413 414 guard(xe_pm_runtime)(xe); 415 for_each_tile(tile, xe, id) { 416 /* For igfx run only for primary tile */ 417 if (!IS_DGFX(xe) && id > 0) 418 continue; 419 ccs_test_run_tile(xe, tile, test); 420 } 421 422 return 0; 423 } 424 425 static void xe_ccs_migrate_kunit(struct kunit *test) 426 { 427 struct xe_device *xe = test->priv; 428 429 ccs_test_run_device(xe); 430 } 431 432 static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struct kunit *test) 433 { 434 struct xe_bo *bo, *external; 435 unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile); 436 struct xe_vm *vm = xe_migrate_get_vm(xe_device_get_root_tile(xe)->migrate); 437 struct drm_exec *exec = XE_VALIDATION_OPT_OUT; 438 struct xe_gt *__gt; 439 int err, i, id; 440 441 kunit_info(test, "Testing device %s vram id %u\n", 442 dev_name(xe->drm.dev), tile->id); 443 444 for (i = 0; i < 2; ++i) { 445 xe_vm_lock(vm, false); 446 bo = xe_bo_create_user(xe, vm, 0x10000, 447 DRM_XE_GEM_CPU_CACHING_WC, 448 bo_flags, exec); 449 xe_vm_unlock(vm); 450 if (IS_ERR(bo)) { 451 KUNIT_FAIL(test, "bo create err=%pe\n", bo); 452 break; 453 } 454 455 external = xe_bo_create_user(xe, NULL, 0x10000, 456 DRM_XE_GEM_CPU_CACHING_WC, 457 bo_flags, NULL); 458 if (IS_ERR(external)) { 459 KUNIT_FAIL(test, "external bo create err=%pe\n", external); 460 goto cleanup_bo; 461 } 462 463 xe_bo_lock(external, false); 464 err = xe_bo_pin_external(external, false, exec); 465 xe_bo_unlock(external); 466 if (err) { 467 KUNIT_FAIL(test, "external bo pin err=%pe\n", 468 ERR_PTR(err)); 469 goto cleanup_external; 470 } 471 472 err = xe_bo_evict_all(xe); 473 if (err) { 474 KUNIT_FAIL(test, "evict err=%pe\n", ERR_PTR(err)); 475 goto cleanup_all; 476 } 477 478 for_each_gt(__gt, xe, id) 479 xe_gt_sanitize(__gt); 480 err = xe_bo_restore_early(xe); 481 /* 482 * Snapshotting the CTB and copying back a potentially old 483 * version seems risky, depending on what might have been 484 * inflight. Also it seems snapshotting the ADS object and 485 * copying back results in serious breakage. Normally when 486 * calling xe_bo_restore_kernel() we always fully restart the 487 * GT, which re-intializes such things. We could potentially 488 * skip saving and restoring such objects in xe_bo_evict_all() 489 * however seems quite fragile not to also restart the GT. Try 490 * to do that here by triggering a GT reset. 491 */ 492 for_each_gt(__gt, xe, id) 493 xe_gt_reset(__gt); 494 495 if (err) { 496 KUNIT_FAIL(test, "restore kernel err=%pe\n", 497 ERR_PTR(err)); 498 goto cleanup_all; 499 } 500 501 err = xe_bo_restore_late(xe); 502 if (err) { 503 KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err)); 504 goto cleanup_all; 505 } 506 507 if (!xe_bo_is_vram(external)) { 508 KUNIT_FAIL(test, "external bo is not vram\n"); 509 err = -EPROTO; 510 goto cleanup_all; 511 } 512 513 if (xe_bo_is_vram(bo)) { 514 KUNIT_FAIL(test, "bo is vram\n"); 515 err = -EPROTO; 516 goto cleanup_all; 517 } 518 519 if (i) { 520 down_read(&vm->lock); 521 xe_vm_lock(vm, false); 522 err = xe_bo_validate(bo, bo->vm, false, exec); 523 xe_vm_unlock(vm); 524 up_read(&vm->lock); 525 if (err) { 526 KUNIT_FAIL(test, "bo valid err=%pe\n", 527 ERR_PTR(err)); 528 goto cleanup_all; 529 } 530 xe_bo_lock(external, false); 531 err = xe_bo_validate(external, NULL, false, exec); 532 xe_bo_unlock(external); 533 if (err) { 534 KUNIT_FAIL(test, "external bo valid err=%pe\n", 535 ERR_PTR(err)); 536 goto cleanup_all; 537 } 538 } 539 540 xe_bo_lock(external, false); 541 xe_bo_unpin_external(external); 542 xe_bo_unlock(external); 543 544 xe_bo_put(external); 545 546 xe_bo_lock(bo, false); 547 __xe_bo_unset_bulk_move(bo); 548 xe_bo_unlock(bo); 549 xe_bo_put(bo); 550 continue; 551 552 cleanup_all: 553 xe_bo_lock(external, false); 554 xe_bo_unpin_external(external); 555 xe_bo_unlock(external); 556 cleanup_external: 557 xe_bo_put(external); 558 cleanup_bo: 559 xe_bo_lock(bo, false); 560 __xe_bo_unset_bulk_move(bo); 561 xe_bo_unlock(bo); 562 xe_bo_put(bo); 563 break; 564 } 565 566 xe_vm_put(vm); 567 568 return 0; 569 } 570 571 static int evict_test_run_device(struct xe_device *xe) 572 { 573 struct kunit *test = kunit_get_current_test(); 574 struct xe_tile *tile; 575 int id; 576 577 if (!IS_DGFX(xe)) { 578 kunit_skip(test, "non-discrete device\n"); 579 return 0; 580 } 581 582 guard(xe_pm_runtime)(xe); 583 for_each_tile(tile, xe, id) 584 evict_test_run_tile(xe, tile, test); 585 586 return 0; 587 } 588 589 static void xe_bo_evict_kunit(struct kunit *test) 590 { 591 struct xe_device *xe = test->priv; 592 593 evict_test_run_device(xe); 594 } 595 596 struct xe_bo_link { 597 struct list_head link; 598 struct xe_bo *bo; 599 u32 val; 600 }; 601 602 #define XE_BO_SHRINK_SIZE ((unsigned long)SZ_64M) 603 604 static int shrink_test_fill_random(struct xe_bo *bo, struct rnd_state *state, 605 struct xe_bo_link *link) 606 { 607 struct iosys_map map; 608 int ret = ttm_bo_vmap(&bo->ttm, &map); 609 size_t __maybe_unused i; 610 611 if (ret) 612 return ret; 613 614 for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) { 615 u32 val = prandom_u32_state(state); 616 617 iosys_map_wr(&map, i, u32, val); 618 if (i == 0) 619 link->val = val; 620 } 621 622 ttm_bo_vunmap(&bo->ttm, &map); 623 return 0; 624 } 625 626 static bool shrink_test_verify(struct kunit *test, struct xe_bo *bo, 627 unsigned int bo_nr, struct rnd_state *state, 628 struct xe_bo_link *link) 629 { 630 struct iosys_map map; 631 int ret = ttm_bo_vmap(&bo->ttm, &map); 632 size_t i; 633 bool failed = false; 634 635 if (ret) { 636 KUNIT_FAIL(test, "Error mapping bo %u for content check.\n", bo_nr); 637 return true; 638 } 639 640 for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) { 641 u32 val = prandom_u32_state(state); 642 643 if (iosys_map_rd(&map, i, u32) != val) { 644 KUNIT_FAIL(test, "Content not preserved, bo %u offset 0x%016llx", 645 bo_nr, (unsigned long long)i); 646 kunit_info(test, "Failed value is 0x%08x, recorded 0x%08x\n", 647 (unsigned int)iosys_map_rd(&map, i, u32), val); 648 if (i == 0 && val != link->val) 649 kunit_info(test, "Looks like PRNG is out of sync.\n"); 650 failed = true; 651 break; 652 } 653 } 654 655 ttm_bo_vunmap(&bo->ttm, &map); 656 657 return failed; 658 } 659 660 /* 661 * Try to create system bos corresponding to twice the amount 662 * of available system memory to test shrinker functionality. 663 * If no swap space is available to accommodate the 664 * memory overcommit, mark bos purgeable. 665 */ 666 static int shrink_test_run_device(struct xe_device *xe) 667 { 668 struct kunit *test = kunit_get_current_test(); 669 LIST_HEAD(bos); 670 struct xe_bo_link *link, *next; 671 struct sysinfo si; 672 u64 ram, ram_and_swap, purgeable = 0, alloced, to_alloc, limit; 673 unsigned int interrupted = 0, successful = 0, count = 0; 674 struct rnd_state prng; 675 u64 rand_seed; 676 bool failed = false; 677 678 rand_seed = get_random_u64(); 679 prandom_seed_state(&prng, rand_seed); 680 kunit_info(test, "Random seed is 0x%016llx.\n", 681 (unsigned long long)rand_seed); 682 683 /* Skip if execution time is expected to be too long. */ 684 685 limit = SZ_32G; 686 /* IGFX with flat CCS needs to copy when swapping / shrinking */ 687 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe)) 688 limit = SZ_16G; 689 690 si_meminfo(&si); 691 ram = (size_t)si.freeram * si.mem_unit; 692 if (ram > limit) { 693 kunit_skip(test, "Too long expected execution time.\n"); 694 return 0; 695 } 696 to_alloc = ram * 2; 697 698 ram_and_swap = ram + get_nr_swap_pages() * PAGE_SIZE; 699 if (to_alloc > ram_and_swap) 700 purgeable = to_alloc - ram_and_swap; 701 purgeable += div64_u64(purgeable, 5); 702 703 kunit_info(test, "Free ram is %lu bytes. Will allocate twice of that.\n", 704 (unsigned long)ram); 705 for (alloced = 0; alloced < to_alloc; alloced += XE_BO_SHRINK_SIZE) { 706 struct xe_bo *bo; 707 unsigned int mem_type; 708 struct xe_ttm_tt *xe_tt; 709 710 link = kzalloc_obj(*link); 711 if (!link) { 712 KUNIT_FAIL(test, "Unexpected link allocation failure\n"); 713 failed = true; 714 break; 715 } 716 717 INIT_LIST_HEAD(&link->link); 718 719 /* We can create bos using WC caching here. But it is slower. */ 720 bo = xe_bo_create_user(xe, NULL, XE_BO_SHRINK_SIZE, 721 DRM_XE_GEM_CPU_CACHING_WB, 722 XE_BO_FLAG_SYSTEM, NULL); 723 if (IS_ERR(bo)) { 724 if (bo != ERR_PTR(-ENOMEM) && bo != ERR_PTR(-ENOSPC) && 725 bo != ERR_PTR(-EINTR) && bo != ERR_PTR(-ERESTARTSYS)) 726 KUNIT_FAIL(test, "Error creating bo: %pe\n", bo); 727 kfree(link); 728 failed = true; 729 break; 730 } 731 xe_bo_lock(bo, false); 732 xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm); 733 734 /* 735 * Allocate purgeable bos first, because if we do it the 736 * other way around, they may not be subject to swapping... 737 */ 738 if (alloced < purgeable) { 739 xe_ttm_tt_account_subtract(xe, &xe_tt->ttm); 740 xe_tt->purgeable = true; 741 xe_ttm_tt_account_add(xe, &xe_tt->ttm); 742 bo->ttm.priority = 0; 743 spin_lock(&bo->ttm.bdev->lru_lock); 744 ttm_bo_move_to_lru_tail(&bo->ttm); 745 spin_unlock(&bo->ttm.bdev->lru_lock); 746 } else { 747 int ret = shrink_test_fill_random(bo, &prng, link); 748 749 if (ret) { 750 xe_bo_unlock(bo); 751 xe_bo_put(bo); 752 KUNIT_FAIL(test, "Error filling bo with random data: %pe\n", 753 ERR_PTR(ret)); 754 kfree(link); 755 failed = true; 756 break; 757 } 758 } 759 760 mem_type = bo->ttm.resource->mem_type; 761 xe_bo_unlock(bo); 762 link->bo = bo; 763 list_add_tail(&link->link, &bos); 764 765 if (mem_type != XE_PL_TT) { 766 KUNIT_FAIL(test, "Bo in incorrect memory type: %u\n", 767 bo->ttm.resource->mem_type); 768 failed = true; 769 } 770 cond_resched(); 771 if (signal_pending(current)) 772 break; 773 } 774 775 /* 776 * Read back and destroy bos. Reset the pseudo-random seed to get an 777 * identical pseudo-random number sequence for readback. 778 */ 779 prandom_seed_state(&prng, rand_seed); 780 list_for_each_entry_safe(link, next, &bos, link) { 781 static struct ttm_operation_ctx ctx = {.interruptible = true}; 782 struct xe_bo *bo = link->bo; 783 struct xe_ttm_tt *xe_tt; 784 int ret; 785 786 count++; 787 if (!signal_pending(current) && !failed) { 788 bool purgeable, intr = false; 789 790 xe_bo_lock(bo, NULL); 791 792 /* xe_tt->purgeable is cleared on validate. */ 793 xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm); 794 purgeable = xe_tt->purgeable; 795 do { 796 ret = ttm_bo_validate(&bo->ttm, &tt_placement, &ctx); 797 if (ret == -EINTR) 798 intr = true; 799 } while (ret == -EINTR && !signal_pending(current)); 800 if (!ret && !purgeable) 801 failed = shrink_test_verify(test, bo, count, &prng, link); 802 803 xe_bo_unlock(bo); 804 if (ret) { 805 KUNIT_FAIL(test, "Validation failed: %pe\n", 806 ERR_PTR(ret)); 807 failed = true; 808 } else if (intr) { 809 interrupted++; 810 } else { 811 successful++; 812 } 813 } 814 xe_bo_put(link->bo); 815 list_del(&link->link); 816 kfree(link); 817 } 818 kunit_info(test, "Readbacks interrupted: %u successful: %u\n", 819 interrupted, successful); 820 821 return 0; 822 } 823 824 static void xe_bo_shrink_kunit(struct kunit *test) 825 { 826 struct xe_device *xe = test->priv; 827 828 shrink_test_run_device(xe); 829 } 830 831 static struct kunit_case xe_bo_tests[] = { 832 KUNIT_CASE_PARAM(xe_ccs_migrate_kunit, xe_pci_live_device_gen_param), 833 KUNIT_CASE_PARAM(xe_bo_evict_kunit, xe_pci_live_device_gen_param), 834 {} 835 }; 836 837 #ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE 838 static struct kunit_case xe_bo_page_size_alloc_cases[] = { 839 KUNIT_CASE_PARAM(xe_bo_page_size_alloc_only_2m, xe_pci_live_device_gen_param), 840 KUNIT_CASE_PARAM(xe_bo_page_size_alloc_only_1g, xe_pci_live_device_gen_param), 841 KUNIT_CASE_PARAM(xe_bo_page_size_alloc_mixed_bos, xe_pci_live_device_gen_param), 842 {} 843 }; 844 845 VISIBLE_IF_KUNIT 846 struct kunit_suite xe_bo_page_size_alloc_suite = { 847 .name = "xe_bo_page_size_alloc", 848 .test_cases = xe_bo_page_size_alloc_cases, 849 .init = xe_kunit_helper_xe_device_live_test_init, 850 }; 851 EXPORT_SYMBOL_IF_KUNIT(xe_bo_page_size_alloc_suite); 852 #endif 853 854 VISIBLE_IF_KUNIT 855 struct kunit_suite xe_bo_test_suite = { 856 .name = "xe_bo", 857 .test_cases = xe_bo_tests, 858 .init = xe_kunit_helper_xe_device_live_test_init, 859 }; 860 EXPORT_SYMBOL_IF_KUNIT(xe_bo_test_suite); 861 862 static struct kunit_case xe_bo_shrink_test[] = { 863 KUNIT_CASE_PARAM_ATTR(xe_bo_shrink_kunit, xe_pci_live_device_gen_param, 864 {.speed = KUNIT_SPEED_SLOW}), 865 {} 866 }; 867 868 VISIBLE_IF_KUNIT 869 struct kunit_suite xe_bo_shrink_test_suite = { 870 .name = "xe_bo_shrink", 871 .test_cases = xe_bo_shrink_test, 872 .init = xe_kunit_helper_xe_device_live_test_init, 873 }; 874 EXPORT_SYMBOL_IF_KUNIT(xe_bo_shrink_test_suite); 875