1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2018 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_internal.h" 7 #include "gem/i915_gem_pm.h" 8 #include "gt/intel_engine_user.h" 9 #include "gt/intel_gt.h" 10 #include "i915_selftest.h" 11 #include "intel_reset.h" 12 13 #include "selftests/igt_flush_test.h" 14 #include "selftests/igt_reset.h" 15 #include "selftests/igt_spinner.h" 16 #include "selftests/intel_scheduler_helpers.h" 17 #include "selftests/mock_drm.h" 18 19 #include "gem/selftests/igt_gem_utils.h" 20 #include "gem/selftests/mock_context.h" 21 22 static const struct wo_register { 23 enum intel_platform platform; 24 u32 reg; 25 } wo_registers[] = { 26 { INTEL_GEMINILAKE, 0x731c } 27 }; 28 29 struct wa_lists { 30 struct i915_wa_list gt_wa_list; 31 struct { 32 struct i915_wa_list wa_list; 33 struct i915_wa_list ctx_wa_list; 34 } engine[I915_NUM_ENGINES]; 35 }; 36 37 static int request_add_sync(struct i915_request *rq, int err) 38 { 39 i915_request_get(rq); 40 i915_request_add(rq); 41 if (i915_request_wait(rq, 0, HZ / 5) < 0) 42 err = -EIO; 43 i915_request_put(rq); 44 45 return err; 46 } 47 48 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin) 49 { 50 int err = 0; 51 52 i915_request_get(rq); 53 i915_request_add(rq); 54 if (spin && !igt_wait_for_spinner(spin, rq)) 55 err = -ETIMEDOUT; 56 i915_request_put(rq); 57 58 return err; 59 } 60 61 static void 62 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists) 63 { 64 struct intel_engine_cs *engine; 65 enum intel_engine_id id; 66 67 memset(lists, 0, sizeof(*lists)); 68 69 wa_init_start(&lists->gt_wa_list, gt, "GT_REF", "global"); 70 gt_init_workarounds(gt, &lists->gt_wa_list); 71 wa_init_finish(&lists->gt_wa_list); 72 73 for_each_engine(engine, gt, id) { 74 struct i915_wa_list *wal = &lists->engine[id].wa_list; 75 76 wa_init_start(wal, gt, "REF", engine->name); 77 engine_init_workarounds(engine, wal); 78 wa_init_finish(wal); 79 80 __intel_engine_init_ctx_wa(engine, 81 &lists->engine[id].ctx_wa_list, 82 "CTX_REF"); 83 } 84 } 85 86 static void 87 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists) 88 { 89 struct intel_engine_cs *engine; 90 enum intel_engine_id id; 91 92 for_each_engine(engine, gt, id) 93 intel_wa_list_free(&lists->engine[id].wa_list); 94 95 intel_wa_list_free(&lists->gt_wa_list); 96 } 97 98 static struct drm_i915_gem_object * 99 read_nonprivs(struct intel_context *ce) 100 { 101 struct intel_engine_cs *engine = ce->engine; 102 const u32 base = engine->mmio_base; 103 struct drm_i915_gem_object *result; 104 struct i915_request *rq; 105 struct i915_vma *vma; 106 u32 srm, *cs; 107 int err; 108 int i; 109 110 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); 111 if (IS_ERR(result)) 112 return result; 113 114 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC); 115 116 cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB); 117 if (IS_ERR(cs)) { 118 err = PTR_ERR(cs); 119 goto err_obj; 120 } 121 memset(cs, 0xc5, PAGE_SIZE); 122 i915_gem_object_flush_map(result); 123 i915_gem_object_unpin_map(result); 124 125 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL); 126 if (IS_ERR(vma)) { 127 err = PTR_ERR(vma); 128 goto err_obj; 129 } 130 131 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL); 132 if (err) 133 goto err_obj; 134 135 rq = intel_context_create_request(ce); 136 if (IS_ERR(rq)) { 137 err = PTR_ERR(rq); 138 goto err_pin; 139 } 140 141 i915_vma_lock(vma); 142 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 143 i915_vma_unlock(vma); 144 if (err) 145 goto err_req; 146 147 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 148 if (GRAPHICS_VER(engine->i915) >= 8) 149 srm++; 150 151 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS); 152 if (IS_ERR(cs)) { 153 err = PTR_ERR(cs); 154 goto err_req; 155 } 156 157 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 158 *cs++ = srm; 159 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i)); 160 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i; 161 *cs++ = 0; 162 } 163 intel_ring_advance(rq, cs); 164 165 i915_request_add(rq); 166 i915_vma_unpin(vma); 167 168 return result; 169 170 err_req: 171 i915_request_add(rq); 172 err_pin: 173 i915_vma_unpin(vma); 174 err_obj: 175 i915_gem_object_put(result); 176 return ERR_PTR(err); 177 } 178 179 static u32 180 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i) 181 { 182 i915_reg_t reg = i < engine->whitelist.count ? 183 engine->whitelist.list[i].reg : 184 RING_NOPID(engine->mmio_base); 185 186 return i915_mmio_reg_offset(reg); 187 } 188 189 static void 190 print_results(const struct intel_engine_cs *engine, const u32 *results) 191 { 192 unsigned int i; 193 194 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 195 u32 expected = get_whitelist_reg(engine, i); 196 u32 actual = results[i]; 197 198 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n", 199 i, expected, actual); 200 } 201 } 202 203 static int check_whitelist(struct intel_context *ce) 204 { 205 struct intel_engine_cs *engine = ce->engine; 206 struct drm_i915_gem_object *results; 207 struct intel_wedge_me wedge; 208 u32 *vaddr; 209 int err; 210 int i; 211 212 results = read_nonprivs(ce); 213 if (IS_ERR(results)) 214 return PTR_ERR(results); 215 216 err = 0; 217 i915_gem_object_lock(results, NULL); 218 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */ 219 err = i915_gem_object_set_to_cpu_domain(results, false); 220 221 if (intel_gt_is_wedged(engine->gt)) 222 err = -EIO; 223 if (err) 224 goto out_put; 225 226 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB); 227 if (IS_ERR(vaddr)) { 228 err = PTR_ERR(vaddr); 229 goto out_put; 230 } 231 232 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) { 233 u32 expected = get_whitelist_reg(engine, i); 234 u32 actual = vaddr[i]; 235 236 if (expected != actual) { 237 print_results(engine, vaddr); 238 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n", 239 i, expected, actual); 240 241 err = -EINVAL; 242 break; 243 } 244 } 245 246 i915_gem_object_unpin_map(results); 247 out_put: 248 i915_gem_object_unlock(results); 249 i915_gem_object_put(results); 250 return err; 251 } 252 253 static int do_device_reset(struct intel_engine_cs *engine) 254 { 255 intel_gt_reset(engine->gt, engine->mask, "live_workarounds"); 256 return 0; 257 } 258 259 static int do_engine_reset(struct intel_engine_cs *engine) 260 { 261 return intel_engine_reset(engine, "live_workarounds"); 262 } 263 264 static int do_guc_reset(struct intel_engine_cs *engine) 265 { 266 /* Currently a no-op as the reset is handled by GuC */ 267 return 0; 268 } 269 270 static int 271 switch_to_scratch_context(struct intel_engine_cs *engine, 272 struct igt_spinner *spin, 273 struct i915_request **rq) 274 { 275 struct intel_context *ce; 276 int err = 0; 277 278 ce = intel_context_create(engine); 279 if (IS_ERR(ce)) 280 return PTR_ERR(ce); 281 282 *rq = igt_spinner_create_request(spin, ce, MI_NOOP); 283 intel_context_put(ce); 284 285 if (IS_ERR(*rq)) { 286 spin = NULL; 287 err = PTR_ERR(*rq); 288 goto err; 289 } 290 291 err = request_add_spin(*rq, spin); 292 err: 293 if (err && spin) 294 igt_spinner_end(spin); 295 296 return err; 297 } 298 299 static int check_whitelist_across_reset(struct intel_engine_cs *engine, 300 int (*reset)(struct intel_engine_cs *), 301 const char *name) 302 { 303 struct intel_context *ce, *tmp; 304 struct igt_spinner spin; 305 struct i915_request *rq; 306 intel_wakeref_t wakeref; 307 int err; 308 309 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n", 310 engine->whitelist.count, engine->name, name); 311 312 ce = intel_context_create(engine); 313 if (IS_ERR(ce)) 314 return PTR_ERR(ce); 315 316 err = igt_spinner_init(&spin, engine->gt); 317 if (err) 318 goto out_ctx; 319 320 err = check_whitelist(ce); 321 if (err) { 322 pr_err("Invalid whitelist *before* %s reset!\n", name); 323 goto out_spin; 324 } 325 326 err = switch_to_scratch_context(engine, &spin, &rq); 327 if (err) 328 goto out_spin; 329 330 /* Ensure the spinner hasn't aborted */ 331 if (i915_request_completed(rq)) { 332 pr_err("%s spinner failed to start\n", name); 333 err = -ETIMEDOUT; 334 goto out_spin; 335 } 336 337 with_intel_runtime_pm(engine->uncore->rpm, wakeref) 338 err = reset(engine); 339 340 /* Ensure the reset happens and kills the engine */ 341 if (err == 0) 342 err = intel_selftest_wait_for_rq(rq); 343 344 igt_spinner_end(&spin); 345 346 if (err) { 347 pr_err("%s reset failed\n", name); 348 goto out_spin; 349 } 350 351 err = check_whitelist(ce); 352 if (err) { 353 pr_err("Whitelist not preserved in context across %s reset!\n", 354 name); 355 goto out_spin; 356 } 357 358 tmp = intel_context_create(engine); 359 if (IS_ERR(tmp)) { 360 err = PTR_ERR(tmp); 361 goto out_spin; 362 } 363 intel_context_put(ce); 364 ce = tmp; 365 366 err = check_whitelist(ce); 367 if (err) { 368 pr_err("Invalid whitelist *after* %s reset in fresh context!\n", 369 name); 370 goto out_spin; 371 } 372 373 out_spin: 374 igt_spinner_fini(&spin); 375 out_ctx: 376 intel_context_put(ce); 377 return err; 378 } 379 380 static struct i915_vma *create_batch(struct i915_address_space *vm) 381 { 382 struct drm_i915_gem_object *obj; 383 struct i915_vma *vma; 384 int err; 385 386 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE); 387 if (IS_ERR(obj)) 388 return ERR_CAST(obj); 389 390 vma = i915_vma_instance(obj, vm, NULL); 391 if (IS_ERR(vma)) { 392 err = PTR_ERR(vma); 393 goto err_obj; 394 } 395 396 err = i915_vma_pin(vma, 0, 0, PIN_USER); 397 if (err) 398 goto err_obj; 399 400 return vma; 401 402 err_obj: 403 i915_gem_object_put(obj); 404 return ERR_PTR(err); 405 } 406 407 static u32 reg_write(u32 old, u32 new, u32 rsvd) 408 { 409 if (rsvd == 0x0000ffff) { 410 old &= ~(new >> 16); 411 old |= new & (new >> 16); 412 } else { 413 old &= ~rsvd; 414 old |= new & rsvd; 415 } 416 417 return old; 418 } 419 420 static bool wo_register(struct intel_engine_cs *engine, u32 reg) 421 { 422 enum intel_platform platform = INTEL_INFO(engine->i915)->platform; 423 int i; 424 425 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 426 RING_FORCE_TO_NONPRIV_ACCESS_WR) 427 return true; 428 429 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) { 430 if (wo_registers[i].platform == platform && 431 wo_registers[i].reg == reg) 432 return true; 433 } 434 435 return false; 436 } 437 438 static bool timestamp(const struct intel_engine_cs *engine, u32 reg) 439 { 440 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK; 441 switch (reg) { 442 case 0x358: 443 case 0x35c: 444 case 0x3a8: 445 return true; 446 447 default: 448 return false; 449 } 450 } 451 452 static bool ro_register(u32 reg) 453 { 454 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) == 455 RING_FORCE_TO_NONPRIV_ACCESS_RD) 456 return true; 457 458 return false; 459 } 460 461 static int whitelist_writable_count(struct intel_engine_cs *engine) 462 { 463 int count = engine->whitelist.count; 464 int i; 465 466 for (i = 0; i < engine->whitelist.count; i++) { 467 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 468 469 if (ro_register(reg)) 470 count--; 471 } 472 473 return count; 474 } 475 476 static int check_dirty_whitelist(struct intel_context *ce) 477 { 478 const u32 values[] = { 479 0x00000000, 480 0x01010101, 481 0x10100101, 482 0x03030303, 483 0x30300303, 484 0x05050505, 485 0x50500505, 486 0x0f0f0f0f, 487 0xf00ff00f, 488 0x10101010, 489 0xf0f01010, 490 0x30303030, 491 0xa0a03030, 492 0x50505050, 493 0xc0c05050, 494 0xf0f0f0f0, 495 0x11111111, 496 0x33333333, 497 0x55555555, 498 0x0000ffff, 499 0x00ff00ff, 500 0xff0000ff, 501 0xffff00ff, 502 0xffffffff, 503 }; 504 struct intel_engine_cs *engine = ce->engine; 505 struct i915_vma *scratch; 506 struct i915_vma *batch; 507 int err = 0, i, v, sz; 508 u32 *cs, *results; 509 510 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32); 511 scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz); 512 if (IS_ERR(scratch)) 513 return PTR_ERR(scratch); 514 515 batch = create_batch(ce->vm); 516 if (IS_ERR(batch)) { 517 err = PTR_ERR(batch); 518 goto out_scratch; 519 } 520 521 for (i = 0; i < engine->whitelist.count; i++) { 522 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 523 struct i915_gem_ww_ctx ww; 524 u64 addr = scratch->node.start; 525 struct i915_request *rq; 526 u32 srm, lrm, rsvd; 527 u32 expect; 528 int idx; 529 bool ro_reg; 530 531 if (wo_register(engine, reg)) 532 continue; 533 534 if (timestamp(engine, reg)) 535 continue; /* timestamps are expected to autoincrement */ 536 537 ro_reg = ro_register(reg); 538 539 i915_gem_ww_ctx_init(&ww, false); 540 retry: 541 cs = NULL; 542 err = i915_gem_object_lock(scratch->obj, &ww); 543 if (!err) 544 err = i915_gem_object_lock(batch->obj, &ww); 545 if (!err) 546 err = intel_context_pin_ww(ce, &ww); 547 if (err) 548 goto out; 549 550 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 551 if (IS_ERR(cs)) { 552 err = PTR_ERR(cs); 553 goto out_ctx; 554 } 555 556 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 557 if (IS_ERR(results)) { 558 err = PTR_ERR(results); 559 goto out_unmap_batch; 560 } 561 562 /* Clear non priv flags */ 563 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 564 565 srm = MI_STORE_REGISTER_MEM; 566 lrm = MI_LOAD_REGISTER_MEM; 567 if (GRAPHICS_VER(engine->i915) >= 8) 568 lrm++, srm++; 569 570 pr_debug("%s: Writing garbage to %x\n", 571 engine->name, reg); 572 573 /* SRM original */ 574 *cs++ = srm; 575 *cs++ = reg; 576 *cs++ = lower_32_bits(addr); 577 *cs++ = upper_32_bits(addr); 578 579 idx = 1; 580 for (v = 0; v < ARRAY_SIZE(values); v++) { 581 /* LRI garbage */ 582 *cs++ = MI_LOAD_REGISTER_IMM(1); 583 *cs++ = reg; 584 *cs++ = values[v]; 585 586 /* SRM result */ 587 *cs++ = srm; 588 *cs++ = reg; 589 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 590 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 591 idx++; 592 } 593 for (v = 0; v < ARRAY_SIZE(values); v++) { 594 /* LRI garbage */ 595 *cs++ = MI_LOAD_REGISTER_IMM(1); 596 *cs++ = reg; 597 *cs++ = ~values[v]; 598 599 /* SRM result */ 600 *cs++ = srm; 601 *cs++ = reg; 602 *cs++ = lower_32_bits(addr + sizeof(u32) * idx); 603 *cs++ = upper_32_bits(addr + sizeof(u32) * idx); 604 idx++; 605 } 606 GEM_BUG_ON(idx * sizeof(u32) > scratch->size); 607 608 /* LRM original -- don't leave garbage in the context! */ 609 *cs++ = lrm; 610 *cs++ = reg; 611 *cs++ = lower_32_bits(addr); 612 *cs++ = upper_32_bits(addr); 613 614 *cs++ = MI_BATCH_BUFFER_END; 615 616 i915_gem_object_flush_map(batch->obj); 617 i915_gem_object_unpin_map(batch->obj); 618 intel_gt_chipset_flush(engine->gt); 619 cs = NULL; 620 621 rq = i915_request_create(ce); 622 if (IS_ERR(rq)) { 623 err = PTR_ERR(rq); 624 goto out_unmap_scratch; 625 } 626 627 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 628 err = engine->emit_init_breadcrumb(rq); 629 if (err) 630 goto err_request; 631 } 632 633 err = i915_vma_move_to_active(batch, rq, 0); 634 if (err) 635 goto err_request; 636 637 err = i915_vma_move_to_active(scratch, rq, 638 EXEC_OBJECT_WRITE); 639 if (err) 640 goto err_request; 641 642 err = engine->emit_bb_start(rq, 643 batch->node.start, PAGE_SIZE, 644 0); 645 if (err) 646 goto err_request; 647 648 err_request: 649 err = request_add_sync(rq, err); 650 if (err) { 651 pr_err("%s: Futzing %x timedout; cancelling test\n", 652 engine->name, reg); 653 intel_gt_set_wedged(engine->gt); 654 goto out_unmap_scratch; 655 } 656 657 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff); 658 if (!ro_reg) { 659 /* detect write masking */ 660 rsvd = results[ARRAY_SIZE(values)]; 661 if (!rsvd) { 662 pr_err("%s: Unable to write to whitelisted register %x\n", 663 engine->name, reg); 664 err = -EINVAL; 665 goto out_unmap_scratch; 666 } 667 } else { 668 rsvd = 0; 669 } 670 671 expect = results[0]; 672 idx = 1; 673 for (v = 0; v < ARRAY_SIZE(values); v++) { 674 if (ro_reg) 675 expect = results[0]; 676 else 677 expect = reg_write(expect, values[v], rsvd); 678 679 if (results[idx] != expect) 680 err++; 681 idx++; 682 } 683 for (v = 0; v < ARRAY_SIZE(values); v++) { 684 if (ro_reg) 685 expect = results[0]; 686 else 687 expect = reg_write(expect, ~values[v], rsvd); 688 689 if (results[idx] != expect) 690 err++; 691 idx++; 692 } 693 if (err) { 694 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n", 695 engine->name, err, reg); 696 697 if (ro_reg) 698 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n", 699 engine->name, reg, results[0]); 700 else 701 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n", 702 engine->name, reg, results[0], rsvd); 703 704 expect = results[0]; 705 idx = 1; 706 for (v = 0; v < ARRAY_SIZE(values); v++) { 707 u32 w = values[v]; 708 709 if (ro_reg) 710 expect = results[0]; 711 else 712 expect = reg_write(expect, w, rsvd); 713 pr_info("Wrote %08x, read %08x, expect %08x\n", 714 w, results[idx], expect); 715 idx++; 716 } 717 for (v = 0; v < ARRAY_SIZE(values); v++) { 718 u32 w = ~values[v]; 719 720 if (ro_reg) 721 expect = results[0]; 722 else 723 expect = reg_write(expect, w, rsvd); 724 pr_info("Wrote %08x, read %08x, expect %08x\n", 725 w, results[idx], expect); 726 idx++; 727 } 728 729 err = -EINVAL; 730 } 731 out_unmap_scratch: 732 i915_gem_object_unpin_map(scratch->obj); 733 out_unmap_batch: 734 if (cs) 735 i915_gem_object_unpin_map(batch->obj); 736 out_ctx: 737 intel_context_unpin(ce); 738 out: 739 if (err == -EDEADLK) { 740 err = i915_gem_ww_ctx_backoff(&ww); 741 if (!err) 742 goto retry; 743 } 744 i915_gem_ww_ctx_fini(&ww); 745 if (err) 746 break; 747 } 748 749 if (igt_flush_test(engine->i915)) 750 err = -EIO; 751 752 i915_vma_unpin_and_release(&batch, 0); 753 out_scratch: 754 i915_vma_unpin_and_release(&scratch, 0); 755 return err; 756 } 757 758 static int live_dirty_whitelist(void *arg) 759 { 760 struct intel_gt *gt = arg; 761 struct intel_engine_cs *engine; 762 enum intel_engine_id id; 763 764 /* Can the user write to the whitelisted registers? */ 765 766 if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */ 767 return 0; 768 769 for_each_engine(engine, gt, id) { 770 struct intel_context *ce; 771 int err; 772 773 if (engine->whitelist.count == 0) 774 continue; 775 776 ce = intel_context_create(engine); 777 if (IS_ERR(ce)) 778 return PTR_ERR(ce); 779 780 err = check_dirty_whitelist(ce); 781 intel_context_put(ce); 782 if (err) 783 return err; 784 } 785 786 return 0; 787 } 788 789 static int live_reset_whitelist(void *arg) 790 { 791 struct intel_gt *gt = arg; 792 struct intel_engine_cs *engine; 793 enum intel_engine_id id; 794 int err = 0; 795 796 /* If we reset the gpu, we should not lose the RING_NONPRIV */ 797 igt_global_reset_lock(gt); 798 799 for_each_engine(engine, gt, id) { 800 if (engine->whitelist.count == 0) 801 continue; 802 803 if (intel_has_reset_engine(gt)) { 804 if (intel_engine_uses_guc(engine)) { 805 struct intel_selftest_saved_policy saved; 806 int err2; 807 808 err = intel_selftest_modify_policy(engine, &saved, 809 SELFTEST_SCHEDULER_MODIFY_FAST_RESET); 810 if (err) 811 goto out; 812 813 err = check_whitelist_across_reset(engine, 814 do_guc_reset, 815 "guc"); 816 817 err2 = intel_selftest_restore_policy(engine, &saved); 818 if (err == 0) 819 err = err2; 820 } else { 821 err = check_whitelist_across_reset(engine, 822 do_engine_reset, 823 "engine"); 824 } 825 826 if (err) 827 goto out; 828 } 829 830 if (intel_has_gpu_reset(gt)) { 831 err = check_whitelist_across_reset(engine, 832 do_device_reset, 833 "device"); 834 if (err) 835 goto out; 836 } 837 } 838 839 out: 840 igt_global_reset_unlock(gt); 841 return err; 842 } 843 844 static int read_whitelisted_registers(struct intel_context *ce, 845 struct i915_vma *results) 846 { 847 struct intel_engine_cs *engine = ce->engine; 848 struct i915_request *rq; 849 int i, err = 0; 850 u32 srm, *cs; 851 852 rq = intel_context_create_request(ce); 853 if (IS_ERR(rq)) 854 return PTR_ERR(rq); 855 856 i915_vma_lock(results); 857 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE); 858 i915_vma_unlock(results); 859 if (err) 860 goto err_req; 861 862 srm = MI_STORE_REGISTER_MEM; 863 if (GRAPHICS_VER(engine->i915) >= 8) 864 srm++; 865 866 cs = intel_ring_begin(rq, 4 * engine->whitelist.count); 867 if (IS_ERR(cs)) { 868 err = PTR_ERR(cs); 869 goto err_req; 870 } 871 872 for (i = 0; i < engine->whitelist.count; i++) { 873 u64 offset = results->node.start + sizeof(u32) * i; 874 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 875 876 /* Clear non priv flags */ 877 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 878 879 *cs++ = srm; 880 *cs++ = reg; 881 *cs++ = lower_32_bits(offset); 882 *cs++ = upper_32_bits(offset); 883 } 884 intel_ring_advance(rq, cs); 885 886 err_req: 887 return request_add_sync(rq, err); 888 } 889 890 static int scrub_whitelisted_registers(struct intel_context *ce) 891 { 892 struct intel_engine_cs *engine = ce->engine; 893 struct i915_request *rq; 894 struct i915_vma *batch; 895 int i, err = 0; 896 u32 *cs; 897 898 batch = create_batch(ce->vm); 899 if (IS_ERR(batch)) 900 return PTR_ERR(batch); 901 902 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 903 if (IS_ERR(cs)) { 904 err = PTR_ERR(cs); 905 goto err_batch; 906 } 907 908 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine)); 909 for (i = 0; i < engine->whitelist.count; i++) { 910 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg); 911 912 if (ro_register(reg)) 913 continue; 914 915 /* Clear non priv flags */ 916 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK; 917 918 *cs++ = reg; 919 *cs++ = 0xffffffff; 920 } 921 *cs++ = MI_BATCH_BUFFER_END; 922 923 i915_gem_object_flush_map(batch->obj); 924 intel_gt_chipset_flush(engine->gt); 925 926 rq = intel_context_create_request(ce); 927 if (IS_ERR(rq)) { 928 err = PTR_ERR(rq); 929 goto err_unpin; 930 } 931 932 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */ 933 err = engine->emit_init_breadcrumb(rq); 934 if (err) 935 goto err_request; 936 } 937 938 i915_vma_lock(batch); 939 err = i915_vma_move_to_active(batch, rq, 0); 940 i915_vma_unlock(batch); 941 if (err) 942 goto err_request; 943 944 /* Perform the writes from an unprivileged "user" batch */ 945 err = engine->emit_bb_start(rq, batch->node.start, 0, 0); 946 947 err_request: 948 err = request_add_sync(rq, err); 949 950 err_unpin: 951 i915_gem_object_unpin_map(batch->obj); 952 err_batch: 953 i915_vma_unpin_and_release(&batch, 0); 954 return err; 955 } 956 957 struct regmask { 958 i915_reg_t reg; 959 u8 graphics_ver; 960 }; 961 962 static bool find_reg(struct drm_i915_private *i915, 963 i915_reg_t reg, 964 const struct regmask *tbl, 965 unsigned long count) 966 { 967 u32 offset = i915_mmio_reg_offset(reg); 968 969 while (count--) { 970 if (GRAPHICS_VER(i915) == tbl->graphics_ver && 971 i915_mmio_reg_offset(tbl->reg) == offset) 972 return true; 973 tbl++; 974 } 975 976 return false; 977 } 978 979 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg) 980 { 981 /* Alas, we must pardon some whitelists. Mistakes already made */ 982 static const struct regmask pardon[] = { 983 { GEN9_CTX_PREEMPT_REG, 9 }, 984 { _MMIO(0xb118), 9 }, /* GEN8_L3SQCREG4 */ 985 }; 986 987 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon)); 988 } 989 990 static bool result_eq(struct intel_engine_cs *engine, 991 u32 a, u32 b, i915_reg_t reg) 992 { 993 if (a != b && !pardon_reg(engine->i915, reg)) { 994 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n", 995 i915_mmio_reg_offset(reg), a, b); 996 return false; 997 } 998 999 return true; 1000 } 1001 1002 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg) 1003 { 1004 /* Some registers do not seem to behave and our writes unreadable */ 1005 static const struct regmask wo[] = { 1006 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 }, 1007 }; 1008 1009 return find_reg(i915, reg, wo, ARRAY_SIZE(wo)); 1010 } 1011 1012 static bool result_neq(struct intel_engine_cs *engine, 1013 u32 a, u32 b, i915_reg_t reg) 1014 { 1015 if (a == b && !writeonly_reg(engine->i915, reg)) { 1016 pr_err("Whitelist register 0x%4x:%08x was unwritable\n", 1017 i915_mmio_reg_offset(reg), a); 1018 return false; 1019 } 1020 1021 return true; 1022 } 1023 1024 static int 1025 check_whitelisted_registers(struct intel_engine_cs *engine, 1026 struct i915_vma *A, 1027 struct i915_vma *B, 1028 bool (*fn)(struct intel_engine_cs *engine, 1029 u32 a, u32 b, 1030 i915_reg_t reg)) 1031 { 1032 u32 *a, *b; 1033 int i, err; 1034 1035 a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB); 1036 if (IS_ERR(a)) 1037 return PTR_ERR(a); 1038 1039 b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB); 1040 if (IS_ERR(b)) { 1041 err = PTR_ERR(b); 1042 goto err_a; 1043 } 1044 1045 err = 0; 1046 for (i = 0; i < engine->whitelist.count; i++) { 1047 const struct i915_wa *wa = &engine->whitelist.list[i]; 1048 1049 if (i915_mmio_reg_offset(wa->reg) & 1050 RING_FORCE_TO_NONPRIV_ACCESS_RD) 1051 continue; 1052 1053 if (!fn(engine, a[i], b[i], wa->reg)) 1054 err = -EINVAL; 1055 } 1056 1057 i915_gem_object_unpin_map(B->obj); 1058 err_a: 1059 i915_gem_object_unpin_map(A->obj); 1060 return err; 1061 } 1062 1063 static int live_isolated_whitelist(void *arg) 1064 { 1065 struct intel_gt *gt = arg; 1066 struct { 1067 struct i915_vma *scratch[2]; 1068 } client[2] = {}; 1069 struct intel_engine_cs *engine; 1070 enum intel_engine_id id; 1071 int i, err = 0; 1072 1073 /* 1074 * Check that a write into a whitelist register works, but 1075 * invisible to a second context. 1076 */ 1077 1078 if (!intel_engines_has_context_isolation(gt->i915)) 1079 return 0; 1080 1081 for (i = 0; i < ARRAY_SIZE(client); i++) { 1082 client[i].scratch[0] = 1083 __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1084 if (IS_ERR(client[i].scratch[0])) { 1085 err = PTR_ERR(client[i].scratch[0]); 1086 goto err; 1087 } 1088 1089 client[i].scratch[1] = 1090 __vm_create_scratch_for_read_pinned(gt->vm, 4096); 1091 if (IS_ERR(client[i].scratch[1])) { 1092 err = PTR_ERR(client[i].scratch[1]); 1093 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1094 goto err; 1095 } 1096 } 1097 1098 for_each_engine(engine, gt, id) { 1099 struct intel_context *ce[2]; 1100 1101 if (!engine->kernel_context->vm) 1102 continue; 1103 1104 if (!whitelist_writable_count(engine)) 1105 continue; 1106 1107 ce[0] = intel_context_create(engine); 1108 if (IS_ERR(ce[0])) { 1109 err = PTR_ERR(ce[0]); 1110 break; 1111 } 1112 ce[1] = intel_context_create(engine); 1113 if (IS_ERR(ce[1])) { 1114 err = PTR_ERR(ce[1]); 1115 intel_context_put(ce[0]); 1116 break; 1117 } 1118 1119 /* Read default values */ 1120 err = read_whitelisted_registers(ce[0], client[0].scratch[0]); 1121 if (err) 1122 goto err_ce; 1123 1124 /* Try to overwrite registers (should only affect ctx0) */ 1125 err = scrub_whitelisted_registers(ce[0]); 1126 if (err) 1127 goto err_ce; 1128 1129 /* Read values from ctx1, we expect these to be defaults */ 1130 err = read_whitelisted_registers(ce[1], client[1].scratch[0]); 1131 if (err) 1132 goto err_ce; 1133 1134 /* Verify that both reads return the same default values */ 1135 err = check_whitelisted_registers(engine, 1136 client[0].scratch[0], 1137 client[1].scratch[0], 1138 result_eq); 1139 if (err) 1140 goto err_ce; 1141 1142 /* Read back the updated values in ctx0 */ 1143 err = read_whitelisted_registers(ce[0], client[0].scratch[1]); 1144 if (err) 1145 goto err_ce; 1146 1147 /* User should be granted privilege to overwhite regs */ 1148 err = check_whitelisted_registers(engine, 1149 client[0].scratch[0], 1150 client[0].scratch[1], 1151 result_neq); 1152 err_ce: 1153 intel_context_put(ce[1]); 1154 intel_context_put(ce[0]); 1155 if (err) 1156 break; 1157 } 1158 1159 err: 1160 for (i = 0; i < ARRAY_SIZE(client); i++) { 1161 i915_vma_unpin_and_release(&client[i].scratch[1], 0); 1162 i915_vma_unpin_and_release(&client[i].scratch[0], 0); 1163 } 1164 1165 if (igt_flush_test(gt->i915)) 1166 err = -EIO; 1167 1168 return err; 1169 } 1170 1171 static bool 1172 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists, 1173 const char *str) 1174 { 1175 struct intel_engine_cs *engine; 1176 enum intel_engine_id id; 1177 bool ok = true; 1178 1179 ok &= wa_list_verify(gt, &lists->gt_wa_list, str); 1180 1181 for_each_engine(engine, gt, id) { 1182 struct intel_context *ce; 1183 1184 ce = intel_context_create(engine); 1185 if (IS_ERR(ce)) 1186 return false; 1187 1188 ok &= engine_wa_list_verify(ce, 1189 &lists->engine[id].wa_list, 1190 str) == 0; 1191 1192 ok &= engine_wa_list_verify(ce, 1193 &lists->engine[id].ctx_wa_list, 1194 str) == 0; 1195 1196 intel_context_put(ce); 1197 } 1198 1199 return ok; 1200 } 1201 1202 static int 1203 live_gpu_reset_workarounds(void *arg) 1204 { 1205 struct intel_gt *gt = arg; 1206 intel_wakeref_t wakeref; 1207 struct wa_lists *lists; 1208 bool ok; 1209 1210 if (!intel_has_gpu_reset(gt)) 1211 return 0; 1212 1213 lists = kzalloc(sizeof(*lists), GFP_KERNEL); 1214 if (!lists) 1215 return -ENOMEM; 1216 1217 pr_info("Verifying after GPU reset...\n"); 1218 1219 igt_global_reset_lock(gt); 1220 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1221 1222 reference_lists_init(gt, lists); 1223 1224 ok = verify_wa_lists(gt, lists, "before reset"); 1225 if (!ok) 1226 goto out; 1227 1228 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds"); 1229 1230 ok = verify_wa_lists(gt, lists, "after reset"); 1231 1232 out: 1233 reference_lists_fini(gt, lists); 1234 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1235 igt_global_reset_unlock(gt); 1236 kfree(lists); 1237 1238 return ok ? 0 : -ESRCH; 1239 } 1240 1241 static int 1242 live_engine_reset_workarounds(void *arg) 1243 { 1244 struct intel_gt *gt = arg; 1245 struct intel_engine_cs *engine; 1246 enum intel_engine_id id; 1247 struct intel_context *ce; 1248 struct igt_spinner spin; 1249 struct i915_request *rq; 1250 intel_wakeref_t wakeref; 1251 struct wa_lists *lists; 1252 int ret = 0; 1253 1254 if (!intel_has_reset_engine(gt)) 1255 return 0; 1256 1257 lists = kzalloc(sizeof(*lists), GFP_KERNEL); 1258 if (!lists) 1259 return -ENOMEM; 1260 1261 igt_global_reset_lock(gt); 1262 wakeref = intel_runtime_pm_get(gt->uncore->rpm); 1263 1264 reference_lists_init(gt, lists); 1265 1266 for_each_engine(engine, gt, id) { 1267 struct intel_selftest_saved_policy saved; 1268 bool using_guc = intel_engine_uses_guc(engine); 1269 bool ok; 1270 int ret2; 1271 1272 pr_info("Verifying after %s reset...\n", engine->name); 1273 ret = intel_selftest_modify_policy(engine, &saved, 1274 SELFTEST_SCHEDULER_MODIFY_FAST_RESET); 1275 if (ret) 1276 break; 1277 1278 ce = intel_context_create(engine); 1279 if (IS_ERR(ce)) { 1280 ret = PTR_ERR(ce); 1281 goto restore; 1282 } 1283 1284 if (!using_guc) { 1285 ok = verify_wa_lists(gt, lists, "before reset"); 1286 if (!ok) { 1287 ret = -ESRCH; 1288 goto err; 1289 } 1290 1291 ret = intel_engine_reset(engine, "live_workarounds:idle"); 1292 if (ret) { 1293 pr_err("%s: Reset failed while idle\n", engine->name); 1294 goto err; 1295 } 1296 1297 ok = verify_wa_lists(gt, lists, "after idle reset"); 1298 if (!ok) { 1299 ret = -ESRCH; 1300 goto err; 1301 } 1302 } 1303 1304 ret = igt_spinner_init(&spin, engine->gt); 1305 if (ret) 1306 goto err; 1307 1308 rq = igt_spinner_create_request(&spin, ce, MI_NOOP); 1309 if (IS_ERR(rq)) { 1310 ret = PTR_ERR(rq); 1311 igt_spinner_fini(&spin); 1312 goto err; 1313 } 1314 1315 ret = request_add_spin(rq, &spin); 1316 if (ret) { 1317 pr_err("%s: Spinner failed to start\n", engine->name); 1318 igt_spinner_fini(&spin); 1319 goto err; 1320 } 1321 1322 /* Ensure the spinner hasn't aborted */ 1323 if (i915_request_completed(rq)) { 1324 ret = -ETIMEDOUT; 1325 goto skip; 1326 } 1327 1328 if (!using_guc) { 1329 ret = intel_engine_reset(engine, "live_workarounds:active"); 1330 if (ret) { 1331 pr_err("%s: Reset failed on an active spinner\n", 1332 engine->name); 1333 igt_spinner_fini(&spin); 1334 goto err; 1335 } 1336 } 1337 1338 /* Ensure the reset happens and kills the engine */ 1339 if (ret == 0) 1340 ret = intel_selftest_wait_for_rq(rq); 1341 1342 skip: 1343 igt_spinner_end(&spin); 1344 igt_spinner_fini(&spin); 1345 1346 ok = verify_wa_lists(gt, lists, "after busy reset"); 1347 if (!ok) 1348 ret = -ESRCH; 1349 1350 err: 1351 intel_context_put(ce); 1352 1353 restore: 1354 ret2 = intel_selftest_restore_policy(engine, &saved); 1355 if (ret == 0) 1356 ret = ret2; 1357 if (ret) 1358 break; 1359 } 1360 1361 reference_lists_fini(gt, lists); 1362 intel_runtime_pm_put(gt->uncore->rpm, wakeref); 1363 igt_global_reset_unlock(gt); 1364 kfree(lists); 1365 1366 igt_flush_test(gt->i915); 1367 1368 return ret; 1369 } 1370 1371 int intel_workarounds_live_selftests(struct drm_i915_private *i915) 1372 { 1373 static const struct i915_subtest tests[] = { 1374 SUBTEST(live_dirty_whitelist), 1375 SUBTEST(live_reset_whitelist), 1376 SUBTEST(live_isolated_whitelist), 1377 SUBTEST(live_gpu_reset_workarounds), 1378 SUBTEST(live_engine_reset_workarounds), 1379 }; 1380 1381 if (intel_gt_is_wedged(to_gt(i915))) 1382 return 0; 1383 1384 return intel_gt_live_subtests(tests, to_gt(i915)); 1385 } 1386