1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020-2022 Intel Corporation 4 */ 5 6 #include <kunit/test.h> 7 #include <kunit/visibility.h> 8 9 #include "tests/xe_migrate_test.h" 10 #include "tests/xe_pci_test.h" 11 12 #include "xe_pci.h" 13 14 static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence, 15 const char *str, struct kunit *test) 16 { 17 long ret; 18 19 if (IS_ERR(fence)) { 20 KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str, 21 PTR_ERR(fence)); 22 return true; 23 } 24 if (!fence) 25 return true; 26 27 ret = dma_fence_wait_timeout(fence, false, 5 * HZ); 28 if (ret <= 0) { 29 KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret); 30 return true; 31 } 32 33 return false; 34 } 35 36 static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe, 37 struct xe_bb *bb, u32 second_idx, const char *str, 38 struct kunit *test) 39 { 40 u64 batch_base = xe_migrate_batch_base(m, xe->info.supports_usm); 41 struct xe_sched_job *job = xe_bb_create_migration_job(m->eng, bb, 42 batch_base, 43 second_idx); 44 struct dma_fence *fence; 45 46 if (IS_ERR(job)) { 47 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", 48 PTR_ERR(job)); 49 return PTR_ERR(job); 50 } 51 52 xe_sched_job_arm(job); 53 fence = dma_fence_get(&job->drm.s_fence->finished); 54 xe_sched_job_push(job); 55 56 if (sanity_fence_failed(xe, fence, str, test)) 57 return -ETIMEDOUT; 58 59 dma_fence_put(fence); 60 kunit_info(test, "%s: Job completed\n", str); 61 return 0; 62 } 63 64 static void 65 sanity_populate_cb(struct xe_migrate_pt_update *pt_update, 66 struct xe_tile *tile, struct iosys_map *map, void *dst, 67 u32 qword_ofs, u32 num_qwords, 68 const struct xe_vm_pgtable_update *update) 69 { 70 struct migrate_test_params *p = 71 to_migrate_test_params(xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE)); 72 int i; 73 u64 *ptr = dst; 74 u64 value; 75 76 for (i = 0; i < num_qwords; i++) { 77 value = (qword_ofs + i - update->ofs) * 0x1111111111111111ULL; 78 if (map) 79 xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * 80 sizeof(u64), u64, value); 81 else 82 ptr[i] = value; 83 } 84 85 kunit_info(xe_cur_kunit(), "Used %s.\n", map ? "CPU" : "GPU"); 86 if (p->force_gpu && map) 87 KUNIT_FAIL(xe_cur_kunit(), "GPU pagetable update used CPU.\n"); 88 } 89 90 static const struct xe_migrate_pt_update_ops sanity_ops = { 91 .populate = sanity_populate_cb, 92 }; 93 94 #define check(_retval, _expected, str, _test) \ 95 do { if ((_retval) != (_expected)) { \ 96 KUNIT_FAIL(_test, "Sanity check failed: " str \ 97 " expected %llx, got %llx\n", \ 98 (u64)(_expected), (u64)(_retval)); \ 99 } } while (0) 100 101 static void test_copy(struct xe_migrate *m, struct xe_bo *bo, 102 struct kunit *test) 103 { 104 struct xe_device *xe = tile_to_xe(m->tile); 105 u64 retval, expected = 0; 106 bool big = bo->size >= SZ_2M; 107 struct dma_fence *fence; 108 const char *str = big ? "Copying big bo" : "Copying small bo"; 109 int err; 110 111 struct xe_bo *sysmem = xe_bo_create_locked(xe, m->tile, NULL, 112 bo->size, 113 ttm_bo_type_kernel, 114 XE_BO_CREATE_SYSTEM_BIT); 115 if (IS_ERR(sysmem)) { 116 KUNIT_FAIL(test, "Failed to allocate sysmem bo for %s: %li\n", 117 str, PTR_ERR(sysmem)); 118 return; 119 } 120 121 err = xe_bo_validate(sysmem, NULL, false); 122 if (err) { 123 KUNIT_FAIL(test, "Failed to validate system bo for %s: %li\n", 124 str, err); 125 goto out_unlock; 126 } 127 128 err = xe_bo_vmap(sysmem); 129 if (err) { 130 KUNIT_FAIL(test, "Failed to vmap system bo for %s: %li\n", 131 str, err); 132 goto out_unlock; 133 } 134 135 xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size); 136 fence = xe_migrate_clear(m, sysmem, sysmem->ttm.resource); 137 if (!sanity_fence_failed(xe, fence, big ? "Clearing sysmem big bo" : 138 "Clearing sysmem small bo", test)) { 139 retval = xe_map_rd(xe, &sysmem->vmap, 0, u64); 140 check(retval, expected, "sysmem first offset should be cleared", 141 test); 142 retval = xe_map_rd(xe, &sysmem->vmap, sysmem->size - 8, u64); 143 check(retval, expected, "sysmem last offset should be cleared", 144 test); 145 } 146 dma_fence_put(fence); 147 148 /* Try to copy 0xc0 from sysmem to vram with 2MB or 64KiB/4KiB pages */ 149 xe_map_memset(xe, &sysmem->vmap, 0, 0xc0, sysmem->size); 150 xe_map_memset(xe, &bo->vmap, 0, 0xd0, bo->size); 151 152 expected = 0xc0c0c0c0c0c0c0c0; 153 fence = xe_migrate_copy(m, sysmem, bo, sysmem->ttm.resource, 154 bo->ttm.resource); 155 if (!sanity_fence_failed(xe, fence, big ? "Copying big bo sysmem -> vram" : 156 "Copying small bo sysmem -> vram", test)) { 157 retval = xe_map_rd(xe, &bo->vmap, 0, u64); 158 check(retval, expected, 159 "sysmem -> vram bo first offset should be copied", test); 160 retval = xe_map_rd(xe, &bo->vmap, bo->size - 8, u64); 161 check(retval, expected, 162 "sysmem -> vram bo offset should be copied", test); 163 } 164 dma_fence_put(fence); 165 166 /* And other way around.. slightly hacky.. */ 167 xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size); 168 xe_map_memset(xe, &bo->vmap, 0, 0xc0, bo->size); 169 170 fence = xe_migrate_copy(m, bo, sysmem, bo->ttm.resource, 171 sysmem->ttm.resource); 172 if (!sanity_fence_failed(xe, fence, big ? "Copying big bo vram -> sysmem" : 173 "Copying small bo vram -> sysmem", test)) { 174 retval = xe_map_rd(xe, &sysmem->vmap, 0, u64); 175 check(retval, expected, 176 "vram -> sysmem bo first offset should be copied", test); 177 retval = xe_map_rd(xe, &sysmem->vmap, bo->size - 8, u64); 178 check(retval, expected, 179 "vram -> sysmem bo last offset should be copied", test); 180 } 181 dma_fence_put(fence); 182 183 xe_bo_vunmap(sysmem); 184 out_unlock: 185 xe_bo_unlock_no_vm(sysmem); 186 xe_bo_put(sysmem); 187 } 188 189 static void test_pt_update(struct xe_migrate *m, struct xe_bo *pt, 190 struct kunit *test, bool force_gpu) 191 { 192 struct xe_device *xe = tile_to_xe(m->tile); 193 struct dma_fence *fence; 194 u64 retval, expected; 195 ktime_t then, now; 196 int i; 197 198 struct xe_vm_pgtable_update update = { 199 .ofs = 1, 200 .qwords = 0x10, 201 .pt_bo = pt, 202 }; 203 struct xe_migrate_pt_update pt_update = { 204 .ops = &sanity_ops, 205 }; 206 struct migrate_test_params p = { 207 .base.id = XE_TEST_LIVE_MIGRATE, 208 .force_gpu = force_gpu, 209 }; 210 211 test->priv = &p; 212 /* Test xe_migrate_update_pgtables() updates the pagetable as expected */ 213 expected = 0xf0f0f0f0f0f0f0f0ULL; 214 xe_map_memset(xe, &pt->vmap, 0, (u8)expected, pt->size); 215 216 then = ktime_get(); 217 fence = xe_migrate_update_pgtables(m, NULL, NULL, m->eng, &update, 1, 218 NULL, 0, &pt_update); 219 now = ktime_get(); 220 if (sanity_fence_failed(xe, fence, "Migration pagetable update", test)) 221 return; 222 223 kunit_info(test, "Updating without syncing took %llu us,\n", 224 (unsigned long long)ktime_to_us(ktime_sub(now, then))); 225 226 dma_fence_put(fence); 227 retval = xe_map_rd(xe, &pt->vmap, 0, u64); 228 check(retval, expected, "PTE[0] must stay untouched", test); 229 230 for (i = 0; i < update.qwords; i++) { 231 retval = xe_map_rd(xe, &pt->vmap, (update.ofs + i) * 8, u64); 232 check(retval, i * 0x1111111111111111ULL, "PTE update", test); 233 } 234 235 retval = xe_map_rd(xe, &pt->vmap, 8 * (update.ofs + update.qwords), 236 u64); 237 check(retval, expected, "PTE[0x11] must stay untouched", test); 238 } 239 240 static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) 241 { 242 struct xe_tile *tile = m->tile; 243 struct xe_device *xe = tile_to_xe(tile); 244 struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny; 245 struct xe_res_cursor src_it; 246 struct dma_fence *fence; 247 u64 retval, expected; 248 struct xe_bb *bb; 249 int err; 250 u8 id = tile->id; 251 252 err = xe_bo_vmap(bo); 253 if (err) { 254 KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n", 255 PTR_ERR(bo)); 256 return; 257 } 258 259 big = xe_bo_create_pin_map(xe, tile, m->eng->vm, SZ_4M, 260 ttm_bo_type_kernel, 261 XE_BO_CREATE_VRAM_IF_DGFX(tile) | 262 XE_BO_CREATE_PINNED_BIT); 263 if (IS_ERR(big)) { 264 KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big)); 265 goto vunmap; 266 } 267 268 pt = xe_bo_create_pin_map(xe, tile, m->eng->vm, XE_PAGE_SIZE, 269 ttm_bo_type_kernel, 270 XE_BO_CREATE_VRAM_IF_DGFX(tile) | 271 XE_BO_CREATE_PINNED_BIT); 272 if (IS_ERR(pt)) { 273 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", 274 PTR_ERR(pt)); 275 goto free_big; 276 } 277 278 tiny = xe_bo_create_pin_map(xe, tile, m->eng->vm, 279 2 * SZ_4K, 280 ttm_bo_type_kernel, 281 XE_BO_CREATE_VRAM_IF_DGFX(tile) | 282 XE_BO_CREATE_PINNED_BIT); 283 if (IS_ERR(tiny)) { 284 KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", 285 PTR_ERR(pt)); 286 goto free_pt; 287 } 288 289 bb = xe_bb_new(tile->primary_gt, 32, xe->info.supports_usm); 290 if (IS_ERR(bb)) { 291 KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n", 292 PTR_ERR(bb)); 293 goto free_tiny; 294 } 295 296 kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n", 297 (unsigned long)xe_bo_main_addr(m->eng->vm->pt_root[id]->bo, XE_PAGE_SIZE), 298 (unsigned long)xe_bo_main_addr(m->pt_bo, XE_PAGE_SIZE)); 299 300 /* First part of the test, are we updating our pagetable bo with a new entry? */ 301 xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64, 302 0xdeaddeadbeefbeef); 303 expected = gen8_pte_encode(NULL, pt, 0, XE_CACHE_WB, 0, 0); 304 if (m->eng->vm->flags & XE_VM_FLAGS_64K) 305 expected |= XE_PTE_PS64; 306 if (xe_bo_is_vram(pt)) 307 xe_res_first(pt->ttm.resource, 0, pt->size, &src_it); 308 else 309 xe_res_first_sg(xe_bo_get_sg(pt), 0, pt->size, &src_it); 310 311 emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt), 312 &src_it, XE_PAGE_SIZE, pt); 313 314 run_sanity_job(m, xe, bb, bb->len, "Writing PTE for our fake PT", test); 315 316 retval = xe_map_rd(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), 317 u64); 318 check(retval, expected, "PTE entry write", test); 319 320 /* Now try to write data to our newly mapped 'pagetable', see if it succeeds */ 321 bb->len = 0; 322 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 323 xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead); 324 expected = 0; 325 326 emit_clear(tile->primary_gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4, 327 IS_DGFX(xe)); 328 run_sanity_job(m, xe, bb, 1, "Writing to our newly mapped pagetable", 329 test); 330 331 retval = xe_map_rd(xe, &pt->vmap, 0, u32); 332 check(retval, expected, "Write to PT after adding PTE", test); 333 334 /* Sanity checks passed, try the full ones! */ 335 336 /* Clear a small bo */ 337 kunit_info(test, "Clearing small buffer object\n"); 338 xe_map_memset(xe, &tiny->vmap, 0, 0x22, tiny->size); 339 expected = 0; 340 fence = xe_migrate_clear(m, tiny, tiny->ttm.resource); 341 if (sanity_fence_failed(xe, fence, "Clearing small bo", test)) 342 goto out; 343 344 dma_fence_put(fence); 345 retval = xe_map_rd(xe, &tiny->vmap, 0, u32); 346 check(retval, expected, "Command clear small first value", test); 347 retval = xe_map_rd(xe, &tiny->vmap, tiny->size - 4, u32); 348 check(retval, expected, "Command clear small last value", test); 349 350 kunit_info(test, "Copying small buffer object to system\n"); 351 test_copy(m, tiny, test); 352 353 /* Clear a big bo */ 354 kunit_info(test, "Clearing big buffer object\n"); 355 xe_map_memset(xe, &big->vmap, 0, 0x11, big->size); 356 expected = 0; 357 fence = xe_migrate_clear(m, big, big->ttm.resource); 358 if (sanity_fence_failed(xe, fence, "Clearing big bo", test)) 359 goto out; 360 361 dma_fence_put(fence); 362 retval = xe_map_rd(xe, &big->vmap, 0, u32); 363 check(retval, expected, "Command clear big first value", test); 364 retval = xe_map_rd(xe, &big->vmap, big->size - 4, u32); 365 check(retval, expected, "Command clear big last value", test); 366 367 kunit_info(test, "Copying big buffer object to system\n"); 368 test_copy(m, big, test); 369 370 kunit_info(test, "Testing page table update using CPU if GPU idle.\n"); 371 test_pt_update(m, pt, test, false); 372 kunit_info(test, "Testing page table update using GPU\n"); 373 test_pt_update(m, pt, test, true); 374 375 out: 376 xe_bb_free(bb, NULL); 377 free_tiny: 378 xe_bo_unpin(tiny); 379 xe_bo_put(tiny); 380 free_pt: 381 xe_bo_unpin(pt); 382 xe_bo_put(pt); 383 free_big: 384 xe_bo_unpin(big); 385 xe_bo_put(big); 386 vunmap: 387 xe_bo_vunmap(m->pt_bo); 388 } 389 390 static int migrate_test_run_device(struct xe_device *xe) 391 { 392 struct kunit *test = xe_cur_kunit(); 393 struct xe_tile *tile; 394 int id; 395 396 for_each_tile(tile, xe, id) { 397 struct xe_migrate *m = tile->migrate; 398 struct ww_acquire_ctx ww; 399 400 kunit_info(test, "Testing tile id %d.\n", id); 401 xe_vm_lock(m->eng->vm, &ww, 0, true); 402 xe_device_mem_access_get(xe); 403 xe_migrate_sanity_test(m, test); 404 xe_device_mem_access_put(xe); 405 xe_vm_unlock(m->eng->vm, &ww); 406 } 407 408 return 0; 409 } 410 411 void xe_migrate_sanity_kunit(struct kunit *test) 412 { 413 xe_call_for_each_device(migrate_test_run_device); 414 } 415 EXPORT_SYMBOL_IF_KUNIT(xe_migrate_sanity_kunit); 416