1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020-2022 Intel Corporation 4 */ 5 6 #include <kunit/test.h> 7 #include <kunit/visibility.h> 8 9 #include "tests/xe_migrate_test.h" 10 #include "tests/xe_pci_test.h" 11 12 #include "xe_pci.h" 13 #include "xe_pm.h" 14 15 static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence, 16 const char *str, struct kunit *test) 17 { 18 long ret; 19 20 if (IS_ERR(fence)) { 21 KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str, 22 PTR_ERR(fence)); 23 return true; 24 } 25 if (!fence) 26 return true; 27 28 ret = dma_fence_wait_timeout(fence, false, 5 * HZ); 29 if (ret <= 0) { 30 KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret); 31 return true; 32 } 33 34 return false; 35 } 36 37 static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe, 38 struct xe_bb *bb, u32 second_idx, const char *str, 39 struct kunit *test) 40 { 41 u64 batch_base = xe_migrate_batch_base(m, xe->info.has_usm); 42 struct xe_sched_job *job = xe_bb_create_migration_job(m->q, bb, 43 batch_base, 44 second_idx); 45 struct dma_fence *fence; 46 47 if (IS_ERR(job)) { 48 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", 49 PTR_ERR(job)); 50 return PTR_ERR(job); 51 } 52 53 xe_sched_job_arm(job); 54 fence = dma_fence_get(&job->drm.s_fence->finished); 55 xe_sched_job_push(job); 56 57 if (sanity_fence_failed(xe, fence, str, test)) 58 return -ETIMEDOUT; 59 60 dma_fence_put(fence); 61 kunit_info(test, "%s: Job completed\n", str); 62 return 0; 63 } 64 65 static void 66 sanity_populate_cb(struct xe_migrate_pt_update *pt_update, 67 struct xe_tile *tile, struct iosys_map *map, void *dst, 68 u32 qword_ofs, u32 num_qwords, 69 const struct xe_vm_pgtable_update *update) 70 { 71 struct migrate_test_params *p = 72 to_migrate_test_params(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE)); 73 int i; 74 u64 *ptr = dst; 75 u64 value; 76 77 for (i = 0; i < num_qwords; i++) { 78 value = (qword_ofs + i - update->ofs) * 0x1111111111111111ULL; 79 if (map) 80 xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * 81 sizeof(u64), u64, value); 82 else 83 ptr[i] = value; 84 } 85 86 kunit_info(xe_cur_kunit(), "Used %s.\n", map ? "CPU" : "GPU"); 87 if (p->force_gpu && map) 88 KUNIT_FAIL(xe_cur_kunit(), "GPU pagetable update used CPU.\n"); 89 } 90 91 static const struct xe_migrate_pt_update_ops sanity_ops = { 92 .populate = sanity_populate_cb, 93 }; 94 95 #define check(_retval, _expected, str, _test) \ 96 do { if ((_retval) != (_expected)) { \ 97 KUNIT_FAIL(_test, "Sanity check failed: " str \ 98 " expected %llx, got %llx\n", \ 99 (u64)(_expected), (u64)(_retval)); \ 100 } } while (0) 101 102 static void test_copy(struct xe_migrate *m, struct xe_bo *bo, 103 struct kunit *test, u32 region) 104 { 105 struct xe_device *xe = tile_to_xe(m->tile); 106 u64 retval, expected = 0; 107 bool big = bo->size >= SZ_2M; 108 struct dma_fence *fence; 109 const char *str = big ? "Copying big bo" : "Copying small bo"; 110 int err; 111 112 struct xe_bo *remote = xe_bo_create_locked(xe, m->tile, NULL, 113 bo->size, 114 ttm_bo_type_kernel, 115 region | 116 XE_BO_FLAG_NEEDS_CPU_ACCESS); 117 if (IS_ERR(remote)) { 118 KUNIT_FAIL(test, "Failed to allocate remote bo for %s: %pe\n", 119 str, remote); 120 return; 121 } 122 123 err = xe_bo_validate(remote, NULL, false); 124 if (err) { 125 KUNIT_FAIL(test, "Failed to validate system bo for %s: %i\n", 126 str, err); 127 goto out_unlock; 128 } 129 130 err = xe_bo_vmap(remote); 131 if (err) { 132 KUNIT_FAIL(test, "Failed to vmap system bo for %s: %i\n", 133 str, err); 134 goto out_unlock; 135 } 136 137 xe_map_memset(xe, &remote->vmap, 0, 0xd0, remote->size); 138 fence = xe_migrate_clear(m, remote, remote->ttm.resource); 139 if (!sanity_fence_failed(xe, fence, big ? "Clearing remote big bo" : 140 "Clearing remote small bo", test)) { 141 retval = xe_map_rd(xe, &remote->vmap, 0, u64); 142 check(retval, expected, "remote first offset should be cleared", 143 test); 144 retval = xe_map_rd(xe, &remote->vmap, remote->size - 8, u64); 145 check(retval, expected, "remote last offset should be cleared", 146 test); 147 } 148 dma_fence_put(fence); 149 150 /* Try to copy 0xc0 from remote to vram with 2MB or 64KiB/4KiB pages */ 151 xe_map_memset(xe, &remote->vmap, 0, 0xc0, remote->size); 152 xe_map_memset(xe, &bo->vmap, 0, 0xd0, bo->size); 153 154 expected = 0xc0c0c0c0c0c0c0c0; 155 fence = xe_migrate_copy(m, remote, bo, remote->ttm.resource, 156 bo->ttm.resource, false); 157 if (!sanity_fence_failed(xe, fence, big ? "Copying big bo remote -> vram" : 158 "Copying small bo remote -> vram", test)) { 159 retval = xe_map_rd(xe, &bo->vmap, 0, u64); 160 check(retval, expected, 161 "remote -> vram bo first offset should be copied", test); 162 retval = xe_map_rd(xe, &bo->vmap, bo->size - 8, u64); 163 check(retval, expected, 164 "remote -> vram bo offset should be copied", test); 165 } 166 dma_fence_put(fence); 167 168 /* And other way around.. slightly hacky.. */ 169 xe_map_memset(xe, &remote->vmap, 0, 0xd0, remote->size); 170 xe_map_memset(xe, &bo->vmap, 0, 0xc0, bo->size); 171 172 fence = xe_migrate_copy(m, bo, remote, bo->ttm.resource, 173 remote->ttm.resource, false); 174 if (!sanity_fence_failed(xe, fence, big ? "Copying big bo vram -> remote" : 175 "Copying small bo vram -> remote", test)) { 176 retval = xe_map_rd(xe, &remote->vmap, 0, u64); 177 check(retval, expected, 178 "vram -> remote bo first offset should be copied", test); 179 retval = xe_map_rd(xe, &remote->vmap, bo->size - 8, u64); 180 check(retval, expected, 181 "vram -> remote bo last offset should be copied", test); 182 } 183 dma_fence_put(fence); 184 185 xe_bo_vunmap(remote); 186 out_unlock: 187 xe_bo_unlock(remote); 188 xe_bo_put(remote); 189 } 190 191 static void test_copy_sysmem(struct xe_migrate *m, struct xe_bo *bo, 192 struct kunit *test) 193 { 194 test_copy(m, bo, test, XE_BO_FLAG_SYSTEM); 195 } 196 197 static void test_copy_vram(struct xe_migrate *m, struct xe_bo *bo, 198 struct kunit *test) 199 { 200 u32 region; 201 202 if (bo->ttm.resource->mem_type == XE_PL_SYSTEM) 203 return; 204 205 if (bo->ttm.resource->mem_type == XE_PL_VRAM0) 206 region = XE_BO_FLAG_VRAM1; 207 else 208 region = XE_BO_FLAG_VRAM0; 209 test_copy(m, bo, test, region); 210 } 211 212 static void test_pt_update(struct xe_migrate *m, struct xe_bo *pt, 213 struct kunit *test, bool force_gpu) 214 { 215 struct xe_device *xe = tile_to_xe(m->tile); 216 struct dma_fence *fence; 217 u64 retval, expected; 218 ktime_t then, now; 219 int i; 220 221 struct xe_vm_pgtable_update update = { 222 .ofs = 1, 223 .qwords = 0x10, 224 .pt_bo = pt, 225 }; 226 struct xe_migrate_pt_update pt_update = { 227 .ops = &sanity_ops, 228 }; 229 struct migrate_test_params p = { 230 .base.id = XE_TEST_LIVE_MIGRATE, 231 .force_gpu = force_gpu, 232 }; 233 234 test->priv = &p; 235 /* Test xe_migrate_update_pgtables() updates the pagetable as expected */ 236 expected = 0xf0f0f0f0f0f0f0f0ULL; 237 xe_map_memset(xe, &pt->vmap, 0, (u8)expected, pt->size); 238 239 then = ktime_get(); 240 fence = xe_migrate_update_pgtables(m, m->q->vm, NULL, m->q, &update, 1, 241 NULL, 0, &pt_update); 242 now = ktime_get(); 243 if (sanity_fence_failed(xe, fence, "Migration pagetable update", test)) 244 return; 245 246 kunit_info(test, "Updating without syncing took %llu us,\n", 247 (unsigned long long)ktime_to_us(ktime_sub(now, then))); 248 249 dma_fence_put(fence); 250 retval = xe_map_rd(xe, &pt->vmap, 0, u64); 251 check(retval, expected, "PTE[0] must stay untouched", test); 252 253 for (i = 0; i < update.qwords; i++) { 254 retval = xe_map_rd(xe, &pt->vmap, (update.ofs + i) * 8, u64); 255 check(retval, i * 0x1111111111111111ULL, "PTE update", test); 256 } 257 258 retval = xe_map_rd(xe, &pt->vmap, 8 * (update.ofs + update.qwords), 259 u64); 260 check(retval, expected, "PTE[0x11] must stay untouched", test); 261 } 262 263 static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) 264 { 265 struct xe_tile *tile = m->tile; 266 struct xe_device *xe = tile_to_xe(tile); 267 struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny; 268 struct xe_res_cursor src_it; 269 struct dma_fence *fence; 270 u64 retval, expected; 271 struct xe_bb *bb; 272 int err; 273 u8 id = tile->id; 274 275 err = xe_bo_vmap(bo); 276 if (err) { 277 KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n", 278 PTR_ERR(bo)); 279 return; 280 } 281 282 big = xe_bo_create_pin_map(xe, tile, m->q->vm, SZ_4M, 283 ttm_bo_type_kernel, 284 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 285 XE_BO_FLAG_PINNED); 286 if (IS_ERR(big)) { 287 KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big)); 288 goto vunmap; 289 } 290 291 pt = xe_bo_create_pin_map(xe, tile, m->q->vm, XE_PAGE_SIZE, 292 ttm_bo_type_kernel, 293 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 294 XE_BO_FLAG_PINNED); 295 if (IS_ERR(pt)) { 296 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", 297 PTR_ERR(pt)); 298 goto free_big; 299 } 300 301 tiny = xe_bo_create_pin_map(xe, tile, m->q->vm, 302 2 * SZ_4K, 303 ttm_bo_type_kernel, 304 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 305 XE_BO_FLAG_PINNED); 306 if (IS_ERR(tiny)) { 307 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", 308 PTR_ERR(pt)); 309 goto free_pt; 310 } 311 312 bb = xe_bb_new(tile->primary_gt, 32, xe->info.has_usm); 313 if (IS_ERR(bb)) { 314 KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n", 315 PTR_ERR(bb)); 316 goto free_tiny; 317 } 318 319 kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n", 320 (unsigned long)xe_bo_main_addr(m->q->vm->pt_root[id]->bo, XE_PAGE_SIZE), 321 (unsigned long)xe_bo_main_addr(m->pt_bo, XE_PAGE_SIZE)); 322 323 /* First part of the test, are we updating our pagetable bo with a new entry? */ 324 xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64, 325 0xdeaddeadbeefbeef); 326 expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe->pat.idx[XE_CACHE_WB], 0); 327 if (m->q->vm->flags & XE_VM_FLAG_64K) 328 expected |= XE_PTE_PS64; 329 if (xe_bo_is_vram(pt)) 330 xe_res_first(pt->ttm.resource, 0, pt->size, &src_it); 331 else 332 xe_res_first_sg(xe_bo_sg(pt), 0, pt->size, &src_it); 333 334 emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt), false, 335 &src_it, XE_PAGE_SIZE, pt->ttm.resource); 336 337 run_sanity_job(m, xe, bb, bb->len, "Writing PTE for our fake PT", test); 338 339 retval = xe_map_rd(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), 340 u64); 341 check(retval, expected, "PTE entry write", test); 342 343 /* Now try to write data to our newly mapped 'pagetable', see if it succeeds */ 344 bb->len = 0; 345 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 346 xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead); 347 expected = 0; 348 349 emit_clear(tile->primary_gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4, 350 IS_DGFX(xe)); 351 run_sanity_job(m, xe, bb, 1, "Writing to our newly mapped pagetable", 352 test); 353 354 retval = xe_map_rd(xe, &pt->vmap, 0, u32); 355 check(retval, expected, "Write to PT after adding PTE", test); 356 357 /* Sanity checks passed, try the full ones! */ 358 359 /* Clear a small bo */ 360 kunit_info(test, "Clearing small buffer object\n"); 361 xe_map_memset(xe, &tiny->vmap, 0, 0x22, tiny->size); 362 expected = 0; 363 fence = xe_migrate_clear(m, tiny, tiny->ttm.resource); 364 if (sanity_fence_failed(xe, fence, "Clearing small bo", test)) 365 goto out; 366 367 dma_fence_put(fence); 368 retval = xe_map_rd(xe, &tiny->vmap, 0, u32); 369 check(retval, expected, "Command clear small first value", test); 370 retval = xe_map_rd(xe, &tiny->vmap, tiny->size - 4, u32); 371 check(retval, expected, "Command clear small last value", test); 372 373 kunit_info(test, "Copying small buffer object to system\n"); 374 test_copy_sysmem(m, tiny, test); 375 if (xe->info.tile_count > 1) { 376 kunit_info(test, "Copying small buffer object to other vram\n"); 377 test_copy_vram(m, tiny, test); 378 } 379 380 /* Clear a big bo */ 381 kunit_info(test, "Clearing big buffer object\n"); 382 xe_map_memset(xe, &big->vmap, 0, 0x11, big->size); 383 expected = 0; 384 fence = xe_migrate_clear(m, big, big->ttm.resource); 385 if (sanity_fence_failed(xe, fence, "Clearing big bo", test)) 386 goto out; 387 388 dma_fence_put(fence); 389 retval = xe_map_rd(xe, &big->vmap, 0, u32); 390 check(retval, expected, "Command clear big first value", test); 391 retval = xe_map_rd(xe, &big->vmap, big->size - 4, u32); 392 check(retval, expected, "Command clear big last value", test); 393 394 kunit_info(test, "Copying big buffer object to system\n"); 395 test_copy_sysmem(m, big, test); 396 if (xe->info.tile_count > 1) { 397 kunit_info(test, "Copying big buffer object to other vram\n"); 398 test_copy_vram(m, big, test); 399 } 400 401 kunit_info(test, "Testing page table update using CPU if GPU idle.\n"); 402 test_pt_update(m, pt, test, false); 403 kunit_info(test, "Testing page table update using GPU\n"); 404 test_pt_update(m, pt, test, true); 405 406 out: 407 xe_bb_free(bb, NULL); 408 free_tiny: 409 xe_bo_unpin(tiny); 410 xe_bo_put(tiny); 411 free_pt: 412 xe_bo_unpin(pt); 413 xe_bo_put(pt); 414 free_big: 415 xe_bo_unpin(big); 416 xe_bo_put(big); 417 vunmap: 418 xe_bo_vunmap(m->pt_bo); 419 } 420 421 static int migrate_test_run_device(struct xe_device *xe) 422 { 423 struct kunit *test = xe_cur_kunit(); 424 struct xe_tile *tile; 425 int id; 426 427 xe_pm_runtime_get(xe); 428 429 for_each_tile(tile, xe, id) { 430 struct xe_migrate *m = tile->migrate; 431 432 kunit_info(test, "Testing tile id %d.\n", id); 433 xe_vm_lock(m->q->vm, true); 434 xe_migrate_sanity_test(m, test); 435 xe_vm_unlock(m->q->vm); 436 } 437 438 xe_pm_runtime_put(xe); 439 440 return 0; 441 } 442 443 void xe_migrate_sanity_kunit(struct kunit *test) 444 { 445 xe_call_for_each_device(migrate_test_run_device); 446 } 447 EXPORT_SYMBOL_IF_KUNIT(xe_migrate_sanity_kunit); 448