1 // SPDX-License-Identifier: GPL-2.0 AND MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include <kunit/test.h> 7 #include <kunit/visibility.h> 8 9 #include <linux/iosys-map.h> 10 #include <linux/math64.h> 11 #include <linux/prandom.h> 12 #include <linux/swap.h> 13 14 #include <uapi/linux/sysinfo.h> 15 16 #include "tests/xe_kunit_helpers.h" 17 #include "tests/xe_pci_test.h" 18 #include "tests/xe_test.h" 19 20 #include "xe_bo_evict.h" 21 #include "xe_pci.h" 22 #include "xe_pm.h" 23 24 static int ccs_test_migrate(struct xe_tile *tile, struct xe_bo *bo, 25 bool clear, u64 get_val, u64 assign_val, 26 struct kunit *test) 27 { 28 struct dma_fence *fence; 29 struct ttm_tt *ttm; 30 struct page *page; 31 pgoff_t ccs_page; 32 long timeout; 33 u64 *cpu_map; 34 int ret; 35 u32 offset; 36 37 /* Move bo to VRAM if not already there. */ 38 ret = xe_bo_validate(bo, NULL, false); 39 if (ret) { 40 KUNIT_FAIL(test, "Failed to validate bo.\n"); 41 return ret; 42 } 43 44 /* Optionally clear bo *and* CCS data in VRAM. */ 45 if (clear) { 46 fence = xe_migrate_clear(tile->migrate, bo, bo->ttm.resource, 47 XE_MIGRATE_CLEAR_FLAG_FULL); 48 if (IS_ERR(fence)) { 49 KUNIT_FAIL(test, "Failed to submit bo clear.\n"); 50 return PTR_ERR(fence); 51 } 52 dma_fence_put(fence); 53 } 54 55 /* Evict to system. CCS data should be copied. */ 56 ret = xe_bo_evict(bo, true); 57 if (ret) { 58 KUNIT_FAIL(test, "Failed to evict bo.\n"); 59 return ret; 60 } 61 62 /* Sync all migration blits */ 63 timeout = dma_resv_wait_timeout(bo->ttm.base.resv, 64 DMA_RESV_USAGE_KERNEL, 65 true, 66 5 * HZ); 67 if (timeout <= 0) { 68 KUNIT_FAIL(test, "Failed to sync bo eviction.\n"); 69 return -ETIME; 70 } 71 72 /* 73 * Bo with CCS data is now in system memory. Verify backing store 74 * and data integrity. Then assign for the next testing round while 75 * we still have a CPU map. 76 */ 77 ttm = bo->ttm.ttm; 78 if (!ttm || !ttm_tt_is_populated(ttm)) { 79 KUNIT_FAIL(test, "Bo was not in expected placement.\n"); 80 return -EINVAL; 81 } 82 83 ccs_page = xe_bo_ccs_pages_start(bo) >> PAGE_SHIFT; 84 if (ccs_page >= ttm->num_pages) { 85 KUNIT_FAIL(test, "No TTM CCS pages present.\n"); 86 return -EINVAL; 87 } 88 89 page = ttm->pages[ccs_page]; 90 cpu_map = kmap_local_page(page); 91 92 /* Check first CCS value */ 93 if (cpu_map[0] != get_val) { 94 KUNIT_FAIL(test, 95 "Expected CCS readout 0x%016llx, got 0x%016llx.\n", 96 (unsigned long long)get_val, 97 (unsigned long long)cpu_map[0]); 98 ret = -EINVAL; 99 } 100 101 /* Check last CCS value, or at least last value in page. */ 102 offset = xe_device_ccs_bytes(tile_to_xe(tile), bo->size); 103 offset = min_t(u32, offset, PAGE_SIZE) / sizeof(u64) - 1; 104 if (cpu_map[offset] != get_val) { 105 KUNIT_FAIL(test, 106 "Expected CCS readout 0x%016llx, got 0x%016llx.\n", 107 (unsigned long long)get_val, 108 (unsigned long long)cpu_map[offset]); 109 ret = -EINVAL; 110 } 111 112 cpu_map[0] = assign_val; 113 cpu_map[offset] = assign_val; 114 kunmap_local(cpu_map); 115 116 return ret; 117 } 118 119 static void ccs_test_run_tile(struct xe_device *xe, struct xe_tile *tile, 120 struct kunit *test) 121 { 122 struct xe_bo *bo; 123 124 int ret; 125 126 /* TODO: Sanity check */ 127 unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile); 128 129 if (IS_DGFX(xe)) 130 kunit_info(test, "Testing vram id %u\n", tile->id); 131 else 132 kunit_info(test, "Testing system memory\n"); 133 134 bo = xe_bo_create_user(xe, NULL, NULL, SZ_1M, DRM_XE_GEM_CPU_CACHING_WC, 135 bo_flags); 136 if (IS_ERR(bo)) { 137 KUNIT_FAIL(test, "Failed to create bo.\n"); 138 return; 139 } 140 141 xe_bo_lock(bo, false); 142 143 kunit_info(test, "Verifying that CCS data is cleared on creation.\n"); 144 ret = ccs_test_migrate(tile, bo, false, 0ULL, 0xdeadbeefdeadbeefULL, 145 test); 146 if (ret) 147 goto out_unlock; 148 149 kunit_info(test, "Verifying that CCS data survives migration.\n"); 150 ret = ccs_test_migrate(tile, bo, false, 0xdeadbeefdeadbeefULL, 151 0xdeadbeefdeadbeefULL, test); 152 if (ret) 153 goto out_unlock; 154 155 kunit_info(test, "Verifying that CCS data can be properly cleared.\n"); 156 ret = ccs_test_migrate(tile, bo, true, 0ULL, 0ULL, test); 157 158 out_unlock: 159 xe_bo_unlock(bo); 160 xe_bo_put(bo); 161 } 162 163 static int ccs_test_run_device(struct xe_device *xe) 164 { 165 struct kunit *test = kunit_get_current_test(); 166 struct xe_tile *tile; 167 int id; 168 169 if (!xe_device_has_flat_ccs(xe)) { 170 kunit_skip(test, "non-flat-ccs device\n"); 171 return 0; 172 } 173 174 /* For xe2+ dgfx, we don't handle ccs metadata */ 175 if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)) { 176 kunit_skip(test, "xe2+ dgfx device\n"); 177 return 0; 178 } 179 180 xe_pm_runtime_get(xe); 181 182 for_each_tile(tile, xe, id) { 183 /* For igfx run only for primary tile */ 184 if (!IS_DGFX(xe) && id > 0) 185 continue; 186 ccs_test_run_tile(xe, tile, test); 187 } 188 189 xe_pm_runtime_put(xe); 190 191 return 0; 192 } 193 194 static void xe_ccs_migrate_kunit(struct kunit *test) 195 { 196 struct xe_device *xe = test->priv; 197 198 ccs_test_run_device(xe); 199 } 200 201 static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struct kunit *test) 202 { 203 struct xe_bo *bo, *external; 204 unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile); 205 struct xe_vm *vm = xe_migrate_get_vm(xe_device_get_root_tile(xe)->migrate); 206 struct xe_gt *__gt; 207 int err, i, id; 208 209 kunit_info(test, "Testing device %s vram id %u\n", 210 dev_name(xe->drm.dev), tile->id); 211 212 for (i = 0; i < 2; ++i) { 213 xe_vm_lock(vm, false); 214 bo = xe_bo_create_user(xe, NULL, vm, 0x10000, 215 DRM_XE_GEM_CPU_CACHING_WC, 216 bo_flags); 217 xe_vm_unlock(vm); 218 if (IS_ERR(bo)) { 219 KUNIT_FAIL(test, "bo create err=%pe\n", bo); 220 break; 221 } 222 223 external = xe_bo_create_user(xe, NULL, NULL, 0x10000, 224 DRM_XE_GEM_CPU_CACHING_WC, 225 bo_flags); 226 if (IS_ERR(external)) { 227 KUNIT_FAIL(test, "external bo create err=%pe\n", external); 228 goto cleanup_bo; 229 } 230 231 xe_bo_lock(external, false); 232 err = xe_bo_pin_external(external); 233 xe_bo_unlock(external); 234 if (err) { 235 KUNIT_FAIL(test, "external bo pin err=%pe\n", 236 ERR_PTR(err)); 237 goto cleanup_external; 238 } 239 240 err = xe_bo_evict_all(xe); 241 if (err) { 242 KUNIT_FAIL(test, "evict err=%pe\n", ERR_PTR(err)); 243 goto cleanup_all; 244 } 245 246 for_each_gt(__gt, xe, id) 247 xe_gt_sanitize(__gt); 248 err = xe_bo_restore_kernel(xe); 249 /* 250 * Snapshotting the CTB and copying back a potentially old 251 * version seems risky, depending on what might have been 252 * inflight. Also it seems snapshotting the ADS object and 253 * copying back results in serious breakage. Normally when 254 * calling xe_bo_restore_kernel() we always fully restart the 255 * GT, which re-intializes such things. We could potentially 256 * skip saving and restoring such objects in xe_bo_evict_all() 257 * however seems quite fragile not to also restart the GT. Try 258 * to do that here by triggering a GT reset. 259 */ 260 for_each_gt(__gt, xe, id) 261 xe_gt_reset(__gt); 262 263 if (err) { 264 KUNIT_FAIL(test, "restore kernel err=%pe\n", 265 ERR_PTR(err)); 266 goto cleanup_all; 267 } 268 269 err = xe_bo_restore_user(xe); 270 if (err) { 271 KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err)); 272 goto cleanup_all; 273 } 274 275 if (!xe_bo_is_vram(external)) { 276 KUNIT_FAIL(test, "external bo is not vram\n"); 277 err = -EPROTO; 278 goto cleanup_all; 279 } 280 281 if (xe_bo_is_vram(bo)) { 282 KUNIT_FAIL(test, "bo is vram\n"); 283 err = -EPROTO; 284 goto cleanup_all; 285 } 286 287 if (i) { 288 down_read(&vm->lock); 289 xe_vm_lock(vm, false); 290 err = xe_bo_validate(bo, bo->vm, false); 291 xe_vm_unlock(vm); 292 up_read(&vm->lock); 293 if (err) { 294 KUNIT_FAIL(test, "bo valid err=%pe\n", 295 ERR_PTR(err)); 296 goto cleanup_all; 297 } 298 xe_bo_lock(external, false); 299 err = xe_bo_validate(external, NULL, false); 300 xe_bo_unlock(external); 301 if (err) { 302 KUNIT_FAIL(test, "external bo valid err=%pe\n", 303 ERR_PTR(err)); 304 goto cleanup_all; 305 } 306 } 307 308 xe_bo_lock(external, false); 309 xe_bo_unpin_external(external); 310 xe_bo_unlock(external); 311 312 xe_bo_put(external); 313 314 xe_bo_lock(bo, false); 315 __xe_bo_unset_bulk_move(bo); 316 xe_bo_unlock(bo); 317 xe_bo_put(bo); 318 continue; 319 320 cleanup_all: 321 xe_bo_lock(external, false); 322 xe_bo_unpin_external(external); 323 xe_bo_unlock(external); 324 cleanup_external: 325 xe_bo_put(external); 326 cleanup_bo: 327 xe_bo_lock(bo, false); 328 __xe_bo_unset_bulk_move(bo); 329 xe_bo_unlock(bo); 330 xe_bo_put(bo); 331 break; 332 } 333 334 xe_vm_put(vm); 335 336 return 0; 337 } 338 339 static int evict_test_run_device(struct xe_device *xe) 340 { 341 struct kunit *test = kunit_get_current_test(); 342 struct xe_tile *tile; 343 int id; 344 345 if (!IS_DGFX(xe)) { 346 kunit_skip(test, "non-discrete device\n"); 347 return 0; 348 } 349 350 xe_pm_runtime_get(xe); 351 352 for_each_tile(tile, xe, id) 353 evict_test_run_tile(xe, tile, test); 354 355 xe_pm_runtime_put(xe); 356 357 return 0; 358 } 359 360 static void xe_bo_evict_kunit(struct kunit *test) 361 { 362 struct xe_device *xe = test->priv; 363 364 evict_test_run_device(xe); 365 } 366 367 struct xe_bo_link { 368 struct list_head link; 369 struct xe_bo *bo; 370 u32 val; 371 }; 372 373 #define XE_BO_SHRINK_SIZE ((unsigned long)SZ_64M) 374 375 static int shrink_test_fill_random(struct xe_bo *bo, struct rnd_state *state, 376 struct xe_bo_link *link) 377 { 378 struct iosys_map map; 379 int ret = ttm_bo_vmap(&bo->ttm, &map); 380 size_t __maybe_unused i; 381 382 if (ret) 383 return ret; 384 385 for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) { 386 u32 val = prandom_u32_state(state); 387 388 iosys_map_wr(&map, i, u32, val); 389 if (i == 0) 390 link->val = val; 391 } 392 393 ttm_bo_vunmap(&bo->ttm, &map); 394 return 0; 395 } 396 397 static bool shrink_test_verify(struct kunit *test, struct xe_bo *bo, 398 unsigned int bo_nr, struct rnd_state *state, 399 struct xe_bo_link *link) 400 { 401 struct iosys_map map; 402 int ret = ttm_bo_vmap(&bo->ttm, &map); 403 size_t i; 404 bool failed = false; 405 406 if (ret) { 407 KUNIT_FAIL(test, "Error mapping bo %u for content check.\n", bo_nr); 408 return true; 409 } 410 411 for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) { 412 u32 val = prandom_u32_state(state); 413 414 if (iosys_map_rd(&map, i, u32) != val) { 415 KUNIT_FAIL(test, "Content not preserved, bo %u offset 0x%016llx", 416 bo_nr, (unsigned long long)i); 417 kunit_info(test, "Failed value is 0x%08x, recorded 0x%08x\n", 418 (unsigned int)iosys_map_rd(&map, i, u32), val); 419 if (i == 0 && val != link->val) 420 kunit_info(test, "Looks like PRNG is out of sync.\n"); 421 failed = true; 422 break; 423 } 424 } 425 426 ttm_bo_vunmap(&bo->ttm, &map); 427 428 return failed; 429 } 430 431 /* 432 * Try to create system bos corresponding to twice the amount 433 * of available system memory to test shrinker functionality. 434 * If no swap space is available to accommodate the 435 * memory overcommit, mark bos purgeable. 436 */ 437 static int shrink_test_run_device(struct xe_device *xe) 438 { 439 struct kunit *test = kunit_get_current_test(); 440 LIST_HEAD(bos); 441 struct xe_bo_link *link, *next; 442 struct sysinfo si; 443 u64 ram, ram_and_swap, purgeable = 0, alloced, to_alloc, limit; 444 unsigned int interrupted = 0, successful = 0, count = 0; 445 struct rnd_state prng; 446 u64 rand_seed; 447 bool failed = false; 448 449 rand_seed = get_random_u64(); 450 prandom_seed_state(&prng, rand_seed); 451 kunit_info(test, "Random seed is 0x%016llx.\n", 452 (unsigned long long)rand_seed); 453 454 /* Skip if execution time is expected to be too long. */ 455 456 limit = SZ_32G; 457 /* IGFX with flat CCS needs to copy when swapping / shrinking */ 458 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe)) 459 limit = SZ_16G; 460 461 si_meminfo(&si); 462 ram = (size_t)si.freeram * si.mem_unit; 463 if (ram > limit) { 464 kunit_skip(test, "Too long expected execution time.\n"); 465 return 0; 466 } 467 to_alloc = ram * 2; 468 469 ram_and_swap = ram + get_nr_swap_pages() * PAGE_SIZE; 470 if (to_alloc > ram_and_swap) 471 purgeable = to_alloc - ram_and_swap; 472 purgeable += div64_u64(purgeable, 5); 473 474 kunit_info(test, "Free ram is %lu bytes. Will allocate twice of that.\n", 475 (unsigned long)ram); 476 for (alloced = 0; alloced < to_alloc; alloced += XE_BO_SHRINK_SIZE) { 477 struct xe_bo *bo; 478 unsigned int mem_type; 479 struct xe_ttm_tt *xe_tt; 480 481 link = kzalloc(sizeof(*link), GFP_KERNEL); 482 if (!link) { 483 KUNIT_FAIL(test, "Unexpected link allocation failure\n"); 484 failed = true; 485 break; 486 } 487 488 INIT_LIST_HEAD(&link->link); 489 490 /* We can create bos using WC caching here. But it is slower. */ 491 bo = xe_bo_create_user(xe, NULL, NULL, XE_BO_SHRINK_SIZE, 492 DRM_XE_GEM_CPU_CACHING_WB, 493 XE_BO_FLAG_SYSTEM); 494 if (IS_ERR(bo)) { 495 if (bo != ERR_PTR(-ENOMEM) && bo != ERR_PTR(-ENOSPC) && 496 bo != ERR_PTR(-EINTR) && bo != ERR_PTR(-ERESTARTSYS)) 497 KUNIT_FAIL(test, "Error creating bo: %pe\n", bo); 498 kfree(link); 499 failed = true; 500 break; 501 } 502 xe_bo_lock(bo, false); 503 xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm); 504 505 /* 506 * Allocate purgeable bos first, because if we do it the 507 * other way around, they may not be subject to swapping... 508 */ 509 if (alloced < purgeable) { 510 xe_tt->purgeable = true; 511 bo->ttm.priority = 0; 512 } else { 513 int ret = shrink_test_fill_random(bo, &prng, link); 514 515 if (ret) { 516 xe_bo_unlock(bo); 517 xe_bo_put(bo); 518 KUNIT_FAIL(test, "Error filling bo with random data: %pe\n", 519 ERR_PTR(ret)); 520 kfree(link); 521 failed = true; 522 break; 523 } 524 } 525 526 mem_type = bo->ttm.resource->mem_type; 527 xe_bo_unlock(bo); 528 link->bo = bo; 529 list_add_tail(&link->link, &bos); 530 531 if (mem_type != XE_PL_TT) { 532 KUNIT_FAIL(test, "Bo in incorrect memory type: %u\n", 533 bo->ttm.resource->mem_type); 534 failed = true; 535 } 536 cond_resched(); 537 if (signal_pending(current)) 538 break; 539 } 540 541 /* 542 * Read back and destroy bos. Reset the pseudo-random seed to get an 543 * identical pseudo-random number sequence for readback. 544 */ 545 prandom_seed_state(&prng, rand_seed); 546 list_for_each_entry_safe(link, next, &bos, link) { 547 static struct ttm_operation_ctx ctx = {.interruptible = true}; 548 struct xe_bo *bo = link->bo; 549 struct xe_ttm_tt *xe_tt; 550 int ret; 551 552 count++; 553 if (!signal_pending(current) && !failed) { 554 bool purgeable, intr = false; 555 556 xe_bo_lock(bo, NULL); 557 558 /* xe_tt->purgeable is cleared on validate. */ 559 xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm); 560 purgeable = xe_tt->purgeable; 561 do { 562 ret = ttm_bo_validate(&bo->ttm, &tt_placement, &ctx); 563 if (ret == -EINTR) 564 intr = true; 565 } while (ret == -EINTR && !signal_pending(current)); 566 567 if (!ret && !purgeable) 568 failed = shrink_test_verify(test, bo, count, &prng, link); 569 570 xe_bo_unlock(bo); 571 if (ret) { 572 KUNIT_FAIL(test, "Validation failed: %pe\n", 573 ERR_PTR(ret)); 574 failed = true; 575 } else if (intr) { 576 interrupted++; 577 } else { 578 successful++; 579 } 580 } 581 xe_bo_put(link->bo); 582 list_del(&link->link); 583 kfree(link); 584 } 585 kunit_info(test, "Readbacks interrupted: %u successful: %u\n", 586 interrupted, successful); 587 588 return 0; 589 } 590 591 static void xe_bo_shrink_kunit(struct kunit *test) 592 { 593 struct xe_device *xe = test->priv; 594 595 shrink_test_run_device(xe); 596 } 597 598 static struct kunit_case xe_bo_tests[] = { 599 KUNIT_CASE_PARAM(xe_ccs_migrate_kunit, xe_pci_live_device_gen_param), 600 KUNIT_CASE_PARAM(xe_bo_evict_kunit, xe_pci_live_device_gen_param), 601 KUNIT_CASE_PARAM_ATTR(xe_bo_shrink_kunit, xe_pci_live_device_gen_param, 602 {.speed = KUNIT_SPEED_SLOW}), 603 {} 604 }; 605 606 VISIBLE_IF_KUNIT 607 struct kunit_suite xe_bo_test_suite = { 608 .name = "xe_bo", 609 .test_cases = xe_bo_tests, 610 .init = xe_kunit_helper_xe_device_live_test_init, 611 }; 612 EXPORT_SYMBOL_IF_KUNIT(xe_bo_test_suite); 613