1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020-2021 Intel Corporation 4 */ 5 6 #include "gt/intel_migrate.h" 7 #include "gt/intel_gpu_commands.h" 8 #include "gem/i915_gem_ttm_move.h" 9 10 #include "i915_deps.h" 11 12 #include "selftests/igt_reset.h" 13 #include "selftests/igt_spinner.h" 14 15 static int igt_fill_check_buffer(struct drm_i915_gem_object *obj, 16 struct intel_gt *gt, 17 bool fill) 18 { 19 unsigned int i, count = obj->base.size / sizeof(u32); 20 enum i915_map_type map_type = 21 intel_gt_coherent_map_type(gt, obj, false); 22 u32 *cur; 23 int err = 0; 24 25 assert_object_held(obj); 26 cur = i915_gem_object_pin_map(obj, map_type); 27 if (IS_ERR(cur)) 28 return PTR_ERR(cur); 29 30 if (fill) 31 for (i = 0; i < count; ++i) 32 *cur++ = i; 33 else 34 for (i = 0; i < count; ++i) 35 if (*cur++ != i) { 36 pr_err("Object content mismatch at location %d of %d\n", i, count); 37 err = -EINVAL; 38 break; 39 } 40 41 i915_gem_object_unpin_map(obj); 42 43 return err; 44 } 45 46 static int igt_create_migrate(struct intel_gt *gt, enum intel_region_id src, 47 enum intel_region_id dst) 48 { 49 struct drm_i915_private *i915 = gt->i915; 50 struct intel_memory_region *src_mr = i915->mm.regions[src]; 51 struct intel_memory_region *dst_mr = i915->mm.regions[dst]; 52 struct drm_i915_gem_object *obj; 53 struct i915_gem_ww_ctx ww; 54 int err = 0; 55 56 GEM_BUG_ON(!src_mr); 57 GEM_BUG_ON(!dst_mr); 58 59 /* Switch object backing-store on create */ 60 obj = i915_gem_object_create_region(src_mr, dst_mr->min_page_size, 0, 0); 61 if (IS_ERR(obj)) 62 return PTR_ERR(obj); 63 64 for_i915_gem_ww(&ww, err, true) { 65 err = i915_gem_object_lock(obj, &ww); 66 if (err) 67 continue; 68 69 err = igt_fill_check_buffer(obj, gt, true); 70 if (err) 71 continue; 72 73 err = i915_gem_object_migrate(obj, &ww, dst); 74 if (err) 75 continue; 76 77 err = i915_gem_object_pin_pages(obj); 78 if (err) 79 continue; 80 81 if (i915_gem_object_can_migrate(obj, src)) 82 err = -EINVAL; 83 84 i915_gem_object_unpin_pages(obj); 85 err = i915_gem_object_wait_migration(obj, true); 86 if (err) 87 continue; 88 89 err = igt_fill_check_buffer(obj, gt, false); 90 } 91 i915_gem_object_put(obj); 92 93 return err; 94 } 95 96 static int igt_smem_create_migrate(void *arg) 97 { 98 return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_SMEM); 99 } 100 101 static int igt_lmem_create_migrate(void *arg) 102 { 103 return igt_create_migrate(arg, INTEL_REGION_SMEM, INTEL_REGION_LMEM_0); 104 } 105 106 static int igt_same_create_migrate(void *arg) 107 { 108 return igt_create_migrate(arg, INTEL_REGION_LMEM_0, INTEL_REGION_LMEM_0); 109 } 110 111 static int lmem_pages_migrate_one(struct i915_gem_ww_ctx *ww, 112 struct drm_i915_gem_object *obj, 113 struct i915_vma *vma, 114 bool silent_migrate) 115 { 116 int err; 117 118 err = i915_gem_object_lock(obj, ww); 119 if (err) 120 return err; 121 122 if (vma) { 123 err = i915_vma_pin_ww(vma, ww, obj->base.size, 0, 124 0UL | PIN_OFFSET_FIXED | 125 PIN_USER); 126 if (err) { 127 if (err != -EINTR && err != ERESTARTSYS && 128 err != -EDEADLK) 129 pr_err("Failed to pin vma.\n"); 130 return err; 131 } 132 133 i915_vma_unpin(vma); 134 } 135 136 /* 137 * Migration will implicitly unbind (asynchronously) any bound 138 * vmas. 139 */ 140 if (i915_gem_object_is_lmem(obj)) { 141 err = i915_gem_object_migrate(obj, ww, INTEL_REGION_SMEM); 142 if (err) { 143 if (!silent_migrate) 144 pr_err("Object failed migration to smem\n"); 145 if (err) 146 return err; 147 } 148 149 if (i915_gem_object_is_lmem(obj)) { 150 pr_err("object still backed by lmem\n"); 151 err = -EINVAL; 152 } 153 154 if (!i915_gem_object_has_struct_page(obj)) { 155 pr_err("object not backed by struct page\n"); 156 err = -EINVAL; 157 } 158 159 } else { 160 err = i915_gem_object_migrate(obj, ww, INTEL_REGION_LMEM_0); 161 if (err) { 162 if (!silent_migrate) 163 pr_err("Object failed migration to lmem\n"); 164 if (err) 165 return err; 166 } 167 168 if (i915_gem_object_has_struct_page(obj)) { 169 pr_err("object still backed by struct page\n"); 170 err = -EINVAL; 171 } 172 173 if (!i915_gem_object_is_lmem(obj)) { 174 pr_err("object not backed by lmem\n"); 175 err = -EINVAL; 176 } 177 } 178 179 return err; 180 } 181 182 static int __igt_lmem_pages_migrate(struct intel_gt *gt, 183 struct i915_address_space *vm, 184 struct i915_deps *deps, 185 struct igt_spinner *spin, 186 struct dma_fence *spin_fence, 187 bool borked_migrate) 188 { 189 struct drm_i915_private *i915 = gt->i915; 190 struct drm_i915_gem_object *obj; 191 struct i915_vma *vma = NULL; 192 struct i915_gem_ww_ctx ww; 193 struct i915_request *rq; 194 int err; 195 int i; 196 197 /* From LMEM to shmem and back again */ 198 199 obj = i915_gem_object_create_lmem(i915, SZ_2M, 0); 200 if (IS_ERR(obj)) 201 return PTR_ERR(obj); 202 203 if (vm) { 204 vma = i915_vma_instance(obj, vm, NULL); 205 if (IS_ERR(vma)) { 206 err = PTR_ERR(vma); 207 goto out_put; 208 } 209 } 210 211 /* Initial GPU fill, sync, CPU initialization. */ 212 for_i915_gem_ww(&ww, err, true) { 213 err = i915_gem_object_lock(obj, &ww); 214 if (err) 215 continue; 216 217 err = ____i915_gem_object_get_pages(obj); 218 if (err) 219 continue; 220 221 err = intel_migrate_clear(>->migrate, &ww, deps, 222 obj->mm.pages->sgl, obj->pat_index, 223 i915_gem_object_is_lmem(obj), 224 0xdeadbeaf, &rq); 225 if (rq) { 226 err = dma_resv_reserve_fences(obj->base.resv, 1); 227 if (!err) 228 dma_resv_add_fence(obj->base.resv, &rq->fence, 229 DMA_RESV_USAGE_KERNEL); 230 i915_request_put(rq); 231 } 232 if (err) 233 continue; 234 235 if (!vma) { 236 err = igt_fill_check_buffer(obj, gt, true); 237 if (err) 238 continue; 239 } 240 } 241 if (err) 242 goto out_put; 243 244 /* 245 * Migrate to and from smem without explicitly syncing. 246 * Finalize with data in smem for fast readout. 247 */ 248 for (i = 1; i <= 5; ++i) { 249 for_i915_gem_ww(&ww, err, true) 250 err = lmem_pages_migrate_one(&ww, obj, vma, 251 borked_migrate); 252 if (err) 253 goto out_put; 254 } 255 256 err = i915_gem_object_lock_interruptible(obj, NULL); 257 if (err) 258 goto out_put; 259 260 if (spin) { 261 if (dma_fence_is_signaled(spin_fence)) { 262 pr_err("Spinner was terminated by hangcheck.\n"); 263 err = -EBUSY; 264 goto out_unlock; 265 } 266 igt_spinner_end(spin); 267 } 268 269 /* Finally sync migration and check content. */ 270 err = i915_gem_object_wait_migration(obj, true); 271 if (err) 272 goto out_unlock; 273 274 if (vma) { 275 err = i915_vma_wait_for_bind(vma); 276 if (err) 277 goto out_unlock; 278 } else { 279 err = igt_fill_check_buffer(obj, gt, false); 280 } 281 282 out_unlock: 283 i915_gem_object_unlock(obj); 284 out_put: 285 i915_gem_object_put(obj); 286 287 return err; 288 } 289 290 static int igt_lmem_pages_failsafe_migrate(void *arg) 291 { 292 int fail_gpu, fail_alloc, ban_memcpy, ret; 293 struct intel_gt *gt = arg; 294 295 for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) { 296 for (fail_alloc = 0; fail_alloc < 2; ++fail_alloc) { 297 for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) { 298 pr_info("Simulated failure modes: gpu: %d, alloc:%d, ban_memcpy: %d\n", 299 fail_gpu, fail_alloc, ban_memcpy); 300 i915_ttm_migrate_set_ban_memcpy(ban_memcpy); 301 i915_ttm_migrate_set_failure_modes(fail_gpu, 302 fail_alloc); 303 ret = __igt_lmem_pages_migrate(gt, NULL, NULL, 304 NULL, NULL, 305 ban_memcpy && 306 fail_gpu); 307 308 if (ban_memcpy && fail_gpu) { 309 struct intel_gt *__gt; 310 unsigned int id; 311 312 if (ret != -EIO) { 313 pr_err("expected -EIO, got (%d)\n", ret); 314 ret = -EINVAL; 315 } else { 316 ret = 0; 317 } 318 319 for_each_gt(__gt, gt->i915, id) { 320 intel_wakeref_t wakeref; 321 bool wedged; 322 323 mutex_lock(&__gt->reset.mutex); 324 wedged = test_bit(I915_WEDGED, &__gt->reset.flags); 325 mutex_unlock(&__gt->reset.mutex); 326 327 if (fail_gpu && !fail_alloc) { 328 if (!wedged) { 329 pr_err("gt(%u) not wedged\n", id); 330 ret = -EINVAL; 331 continue; 332 } 333 } else if (wedged) { 334 pr_err("gt(%u) incorrectly wedged\n", id); 335 ret = -EINVAL; 336 } else { 337 continue; 338 } 339 340 wakeref = intel_runtime_pm_get(__gt->uncore->rpm); 341 igt_global_reset_lock(__gt); 342 intel_gt_reset(__gt, ALL_ENGINES, NULL); 343 igt_global_reset_unlock(__gt); 344 intel_runtime_pm_put(__gt->uncore->rpm, wakeref); 345 } 346 if (ret) 347 goto out_err; 348 } 349 } 350 } 351 } 352 353 out_err: 354 i915_ttm_migrate_set_failure_modes(false, false); 355 i915_ttm_migrate_set_ban_memcpy(false); 356 return ret; 357 } 358 359 /* 360 * This subtest tests that unbinding at migration is indeed performed 361 * async. We launch a spinner and a number of migrations depending on 362 * that spinner to have terminated. Before each migration we bind a 363 * vma, which should then be async unbound by the migration operation. 364 * If we are able to schedule migrations without blocking while the 365 * spinner is still running, those unbinds are indeed async and non- 366 * blocking. 367 * 368 * Note that each async bind operation is awaiting the previous migration 369 * due to the moving fence resulting from the migration. 370 */ 371 static int igt_async_migrate(struct intel_gt *gt) 372 { 373 struct intel_engine_cs *engine; 374 enum intel_engine_id id; 375 struct i915_ppgtt *ppgtt; 376 struct igt_spinner spin; 377 int err; 378 379 ppgtt = i915_ppgtt_create(gt, 0); 380 if (IS_ERR(ppgtt)) 381 return PTR_ERR(ppgtt); 382 383 if (igt_spinner_init(&spin, gt)) { 384 err = -ENOMEM; 385 goto out_spin; 386 } 387 388 for_each_engine(engine, gt, id) { 389 struct ttm_operation_ctx ctx = { 390 .interruptible = true 391 }; 392 struct dma_fence *spin_fence; 393 struct intel_context *ce; 394 struct i915_request *rq; 395 struct i915_deps deps; 396 397 ce = intel_context_create(engine); 398 if (IS_ERR(ce)) { 399 err = PTR_ERR(ce); 400 goto out_ce; 401 } 402 403 /* 404 * Use MI_NOOP, making the spinner non-preemptible. If there 405 * is a code path where we fail async operation due to the 406 * running spinner, we will block and fail to end the 407 * spinner resulting in a deadlock. But with a non- 408 * preemptible spinner, hangcheck will terminate the spinner 409 * for us, and we will later detect that and fail the test. 410 */ 411 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 412 intel_context_put(ce); 413 if (IS_ERR(rq)) { 414 err = PTR_ERR(rq); 415 goto out_ce; 416 } 417 418 i915_deps_init(&deps, GFP_KERNEL); 419 err = i915_deps_add_dependency(&deps, &rq->fence, &ctx); 420 spin_fence = dma_fence_get(&rq->fence); 421 i915_request_add(rq); 422 if (err) 423 goto out_ce; 424 425 err = __igt_lmem_pages_migrate(gt, &ppgtt->vm, &deps, &spin, 426 spin_fence, false); 427 i915_deps_fini(&deps); 428 dma_fence_put(spin_fence); 429 if (err) 430 goto out_ce; 431 } 432 433 out_ce: 434 igt_spinner_fini(&spin); 435 out_spin: 436 i915_vm_put(&ppgtt->vm); 437 438 return err; 439 } 440 441 /* 442 * Setting ASYNC_FAIL_ALLOC to 2 will simulate memory allocation failure while 443 * arming the migration error check and block async migration. This 444 * will cause us to deadlock and hangcheck will terminate the spinner 445 * causing the test to fail. 446 */ 447 #define ASYNC_FAIL_ALLOC 1 448 static int igt_lmem_async_migrate(void *arg) 449 { 450 int fail_gpu, fail_alloc, ban_memcpy, ret; 451 struct intel_gt *gt = arg; 452 453 for (fail_gpu = 0; fail_gpu < 2; ++fail_gpu) { 454 for (fail_alloc = 0; fail_alloc < ASYNC_FAIL_ALLOC; ++fail_alloc) { 455 for (ban_memcpy = 0; ban_memcpy < 2; ++ban_memcpy) { 456 pr_info("Simulated failure modes: gpu: %d, alloc: %d, ban_memcpy: %d\n", 457 fail_gpu, fail_alloc, ban_memcpy); 458 i915_ttm_migrate_set_ban_memcpy(ban_memcpy); 459 i915_ttm_migrate_set_failure_modes(fail_gpu, 460 fail_alloc); 461 ret = igt_async_migrate(gt); 462 463 if (fail_gpu && ban_memcpy) { 464 struct intel_gt *__gt; 465 unsigned int id; 466 467 if (ret != -EIO) { 468 pr_err("expected -EIO, got (%d)\n", ret); 469 ret = -EINVAL; 470 } else { 471 ret = 0; 472 } 473 474 for_each_gt(__gt, gt->i915, id) { 475 intel_wakeref_t wakeref; 476 bool wedged; 477 478 mutex_lock(&__gt->reset.mutex); 479 wedged = test_bit(I915_WEDGED, &__gt->reset.flags); 480 mutex_unlock(&__gt->reset.mutex); 481 482 if (fail_gpu && !fail_alloc) { 483 if (!wedged) { 484 pr_err("gt(%u) not wedged\n", id); 485 ret = -EINVAL; 486 continue; 487 } 488 } else if (wedged) { 489 pr_err("gt(%u) incorrectly wedged\n", id); 490 ret = -EINVAL; 491 } else { 492 continue; 493 } 494 495 wakeref = intel_runtime_pm_get(__gt->uncore->rpm); 496 igt_global_reset_lock(__gt); 497 intel_gt_reset(__gt, ALL_ENGINES, NULL); 498 igt_global_reset_unlock(__gt); 499 intel_runtime_pm_put(__gt->uncore->rpm, wakeref); 500 } 501 } 502 if (ret) 503 goto out_err; 504 } 505 } 506 } 507 508 out_err: 509 i915_ttm_migrate_set_failure_modes(false, false); 510 i915_ttm_migrate_set_ban_memcpy(false); 511 return ret; 512 } 513 514 int i915_gem_migrate_live_selftests(struct drm_i915_private *i915) 515 { 516 static const struct i915_subtest tests[] = { 517 SUBTEST(igt_smem_create_migrate), 518 SUBTEST(igt_lmem_create_migrate), 519 SUBTEST(igt_same_create_migrate), 520 SUBTEST(igt_lmem_pages_failsafe_migrate), 521 SUBTEST(igt_lmem_async_migrate), 522 }; 523 524 if (!HAS_LMEM(i915)) 525 return 0; 526 527 return intel_gt_live_subtests(tests, to_gt(i915)); 528 } 529