1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2018 Intel Corporation 4 */ 5 6 #include <linux/prime_numbers.h> 7 8 #include "gem/i915_gem_internal.h" 9 10 #include "i915_selftest.h" 11 #include "intel_engine_heartbeat.h" 12 #include "intel_engine_pm.h" 13 #include "intel_reset.h" 14 #include "intel_ring.h" 15 #include "selftest_engine_heartbeat.h" 16 #include "selftests/i915_random.h" 17 #include "selftests/igt_flush_test.h" 18 #include "selftests/igt_live_test.h" 19 #include "selftests/igt_spinner.h" 20 #include "selftests/lib_sw_fence.h" 21 #include "shmem_utils.h" 22 23 #include "gem/selftests/igt_gem_utils.h" 24 #include "gem/selftests/mock_context.h" 25 26 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4) 27 #define NUM_GPR 16 28 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ 29 30 static struct i915_vma *create_scratch(struct intel_gt *gt) 31 { 32 return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE); 33 } 34 35 static bool is_active(struct i915_request *rq) 36 { 37 if (i915_request_is_active(rq)) 38 return true; 39 40 if (i915_request_on_hold(rq)) 41 return true; 42 43 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq)) 44 return true; 45 46 return false; 47 } 48 49 static int wait_for_submit(struct intel_engine_cs *engine, 50 struct i915_request *rq, 51 unsigned long timeout) 52 { 53 /* Ignore our own attempts to suppress excess tasklets */ 54 tasklet_hi_schedule(&engine->sched_engine->tasklet); 55 56 timeout += jiffies; 57 do { 58 bool done = time_after(jiffies, timeout); 59 60 if (i915_request_completed(rq)) /* that was quick! */ 61 return 0; 62 63 /* Wait until the HW has acknowleged the submission (or err) */ 64 intel_engine_flush_submission(engine); 65 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) 66 return 0; 67 68 if (done) 69 return -ETIME; 70 71 cond_resched(); 72 } while (1); 73 } 74 75 static int emit_semaphore_signal(struct intel_context *ce, void *slot) 76 { 77 const u32 offset = 78 i915_ggtt_offset(ce->engine->status_page.vma) + 79 offset_in_page(slot); 80 struct i915_request *rq; 81 u32 *cs; 82 83 rq = intel_context_create_request(ce); 84 if (IS_ERR(rq)) 85 return PTR_ERR(rq); 86 87 cs = intel_ring_begin(rq, 4); 88 if (IS_ERR(cs)) { 89 i915_request_add(rq); 90 return PTR_ERR(cs); 91 } 92 93 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 94 *cs++ = offset; 95 *cs++ = 0; 96 *cs++ = 1; 97 98 intel_ring_advance(rq, cs); 99 100 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 101 i915_request_add(rq); 102 return 0; 103 } 104 105 static int context_flush(struct intel_context *ce, long timeout) 106 { 107 struct i915_request *rq; 108 struct dma_fence *fence; 109 int err = 0; 110 111 rq = intel_engine_create_kernel_request(ce->engine); 112 if (IS_ERR(rq)) 113 return PTR_ERR(rq); 114 115 fence = i915_active_fence_get(&ce->timeline->last_request); 116 if (fence) { 117 i915_request_await_dma_fence(rq, fence); 118 dma_fence_put(fence); 119 } 120 121 rq = i915_request_get(rq); 122 i915_request_add(rq); 123 if (i915_request_wait(rq, 0, timeout) < 0) 124 err = -ETIME; 125 i915_request_put(rq); 126 127 rmb(); /* We know the request is written, make sure all state is too! */ 128 return err; 129 } 130 131 static int get_lri_mask(struct intel_engine_cs *engine, u32 lri) 132 { 133 if ((lri & MI_LRI_LRM_CS_MMIO) == 0) 134 return ~0u; 135 136 if (GRAPHICS_VER(engine->i915) < 12) 137 return 0xfff; 138 139 switch (engine->class) { 140 default: 141 case RENDER_CLASS: 142 case COMPUTE_CLASS: 143 return 0x07ff; 144 case COPY_ENGINE_CLASS: 145 return 0x0fff; 146 case VIDEO_DECODE_CLASS: 147 case VIDEO_ENHANCEMENT_CLASS: 148 return 0x3fff; 149 } 150 } 151 152 static int live_lrc_layout(void *arg) 153 { 154 struct intel_gt *gt = arg; 155 struct intel_engine_cs *engine; 156 enum intel_engine_id id; 157 u32 *lrc; 158 int err; 159 160 /* 161 * Check the registers offsets we use to create the initial reg state 162 * match the layout saved by HW. 163 */ 164 165 lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */ 166 if (!lrc) 167 return -ENOMEM; 168 GEM_BUG_ON(offset_in_page(lrc)); 169 170 err = 0; 171 for_each_engine(engine, gt, id) { 172 u32 *hw; 173 int dw; 174 175 if (!engine->default_state) 176 continue; 177 178 hw = shmem_pin_map(engine->default_state); 179 if (IS_ERR(hw)) { 180 err = PTR_ERR(hw); 181 break; 182 } 183 hw += LRC_STATE_OFFSET / sizeof(*hw); 184 185 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE), 186 engine->kernel_context, engine, true); 187 188 dw = 0; 189 do { 190 u32 lri = READ_ONCE(hw[dw]); 191 u32 lri_mask; 192 193 if (lri == 0) { 194 dw++; 195 continue; 196 } 197 198 if (lrc[dw] == 0) { 199 pr_debug("%s: skipped instruction %x at dword %d\n", 200 engine->name, lri, dw); 201 dw++; 202 continue; 203 } 204 205 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 206 pr_err("%s: Expected LRI command at dword %d, found %08x\n", 207 engine->name, dw, lri); 208 err = -EINVAL; 209 break; 210 } 211 212 if (lrc[dw] != lri) { 213 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n", 214 engine->name, dw, lri, lrc[dw]); 215 err = -EINVAL; 216 break; 217 } 218 219 /* 220 * When bit 19 of MI_LOAD_REGISTER_IMM instruction 221 * opcode is set on Gen12+ devices, HW does not 222 * care about certain register address offsets, and 223 * instead check the following for valid address 224 * ranges on specific engines: 225 * RCS && CCS: BITS(0 - 10) 226 * BCS: BITS(0 - 11) 227 * VECS && VCS: BITS(0 - 13) 228 */ 229 lri_mask = get_lri_mask(engine, lri); 230 231 lri &= 0x7f; 232 lri++; 233 dw++; 234 235 while (lri) { 236 u32 offset = READ_ONCE(hw[dw]); 237 238 if ((offset ^ lrc[dw]) & lri_mask) { 239 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n", 240 engine->name, dw, offset, lrc[dw]); 241 err = -EINVAL; 242 break; 243 } 244 245 /* 246 * Skip over the actual register value as we 247 * expect that to differ. 248 */ 249 dw += 2; 250 lri -= 2; 251 } 252 } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 253 254 if (err) { 255 pr_info("%s: HW register image:\n", engine->name); 256 igt_hexdump(hw, PAGE_SIZE); 257 258 pr_info("%s: SW register image:\n", engine->name); 259 igt_hexdump(lrc, PAGE_SIZE); 260 } 261 262 shmem_unpin_map(engine->default_state, hw); 263 if (err) 264 break; 265 } 266 267 free_page((unsigned long)lrc); 268 return err; 269 } 270 271 static int find_offset(const u32 *lri, u32 offset) 272 { 273 int i; 274 275 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) 276 if (lri[i] == offset) 277 return i; 278 279 return -1; 280 } 281 282 static int live_lrc_fixed(void *arg) 283 { 284 struct intel_gt *gt = arg; 285 struct intel_engine_cs *engine; 286 enum intel_engine_id id; 287 int err = 0; 288 289 /* 290 * Check the assumed register offsets match the actual locations in 291 * the context image. 292 */ 293 294 for_each_engine(engine, gt, id) { 295 const struct { 296 u32 reg; 297 u32 offset; 298 const char *name; 299 } tbl[] = { 300 { 301 i915_mmio_reg_offset(RING_START(engine->mmio_base)), 302 CTX_RING_START - 1, 303 "RING_START" 304 }, 305 { 306 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)), 307 CTX_RING_CTL - 1, 308 "RING_CTL" 309 }, 310 { 311 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)), 312 CTX_RING_HEAD - 1, 313 "RING_HEAD" 314 }, 315 { 316 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)), 317 CTX_RING_TAIL - 1, 318 "RING_TAIL" 319 }, 320 { 321 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)), 322 lrc_ring_mi_mode(engine), 323 "RING_MI_MODE" 324 }, 325 { 326 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)), 327 CTX_BB_STATE - 1, 328 "BB_STATE" 329 }, 330 { 331 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)), 332 lrc_ring_wa_bb_per_ctx(engine), 333 "RING_BB_PER_CTX_PTR" 334 }, 335 { 336 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)), 337 lrc_ring_indirect_ptr(engine), 338 "RING_INDIRECT_CTX_PTR" 339 }, 340 { 341 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)), 342 lrc_ring_indirect_offset(engine), 343 "RING_INDIRECT_CTX_OFFSET" 344 }, 345 { 346 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)), 347 CTX_TIMESTAMP - 1, 348 "RING_CTX_TIMESTAMP" 349 }, 350 { 351 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)), 352 lrc_ring_gpr0(engine), 353 "RING_CS_GPR0" 354 }, 355 { 356 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)), 357 lrc_ring_cmd_buf_cctl(engine), 358 "RING_CMD_BUF_CCTL" 359 }, 360 { }, 361 }, *t; 362 u32 *hw; 363 364 if (!engine->default_state) 365 continue; 366 367 hw = shmem_pin_map(engine->default_state); 368 if (IS_ERR(hw)) { 369 err = PTR_ERR(hw); 370 break; 371 } 372 hw += LRC_STATE_OFFSET / sizeof(*hw); 373 374 for (t = tbl; t->name; t++) { 375 int dw = find_offset(hw, t->reg); 376 377 if (dw != t->offset) { 378 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n", 379 engine->name, 380 t->name, 381 t->reg, 382 dw, 383 t->offset); 384 err = -EINVAL; 385 } 386 } 387 388 shmem_unpin_map(engine->default_state, hw); 389 } 390 391 return err; 392 } 393 394 static int __live_lrc_state(struct intel_engine_cs *engine, 395 struct i915_vma *scratch) 396 { 397 struct intel_context *ce; 398 struct i915_request *rq; 399 struct i915_gem_ww_ctx ww; 400 enum { 401 RING_START_IDX = 0, 402 RING_TAIL_IDX, 403 MAX_IDX 404 }; 405 u32 expected[MAX_IDX]; 406 u32 *cs; 407 int err; 408 int n; 409 410 ce = intel_context_create(engine); 411 if (IS_ERR(ce)) 412 return PTR_ERR(ce); 413 414 i915_gem_ww_ctx_init(&ww, false); 415 retry: 416 err = i915_gem_object_lock(scratch->obj, &ww); 417 if (!err) 418 err = intel_context_pin_ww(ce, &ww); 419 if (err) 420 goto err_put; 421 422 rq = i915_request_create(ce); 423 if (IS_ERR(rq)) { 424 err = PTR_ERR(rq); 425 goto err_unpin; 426 } 427 428 cs = intel_ring_begin(rq, 4 * MAX_IDX); 429 if (IS_ERR(cs)) { 430 err = PTR_ERR(cs); 431 i915_request_add(rq); 432 goto err_unpin; 433 } 434 435 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 436 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base)); 437 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32); 438 *cs++ = 0; 439 440 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma); 441 442 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 443 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)); 444 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32); 445 *cs++ = 0; 446 447 err = i915_request_await_object(rq, scratch->obj, true); 448 if (!err) 449 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); 450 451 i915_request_get(rq); 452 i915_request_add(rq); 453 if (err) 454 goto err_rq; 455 456 intel_engine_flush_submission(engine); 457 expected[RING_TAIL_IDX] = ce->ring->tail; 458 459 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 460 err = -ETIME; 461 goto err_rq; 462 } 463 464 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 465 if (IS_ERR(cs)) { 466 err = PTR_ERR(cs); 467 goto err_rq; 468 } 469 470 for (n = 0; n < MAX_IDX; n++) { 471 if (cs[n] != expected[n]) { 472 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n", 473 engine->name, n, cs[n], expected[n]); 474 err = -EINVAL; 475 break; 476 } 477 } 478 479 i915_gem_object_unpin_map(scratch->obj); 480 481 err_rq: 482 i915_request_put(rq); 483 err_unpin: 484 intel_context_unpin(ce); 485 err_put: 486 if (err == -EDEADLK) { 487 err = i915_gem_ww_ctx_backoff(&ww); 488 if (!err) 489 goto retry; 490 } 491 i915_gem_ww_ctx_fini(&ww); 492 intel_context_put(ce); 493 return err; 494 } 495 496 static int live_lrc_state(void *arg) 497 { 498 struct intel_gt *gt = arg; 499 struct intel_engine_cs *engine; 500 struct i915_vma *scratch; 501 enum intel_engine_id id; 502 int err = 0; 503 504 /* 505 * Check the live register state matches what we expect for this 506 * intel_context. 507 */ 508 509 scratch = create_scratch(gt); 510 if (IS_ERR(scratch)) 511 return PTR_ERR(scratch); 512 513 for_each_engine(engine, gt, id) { 514 err = __live_lrc_state(engine, scratch); 515 if (err) 516 break; 517 } 518 519 if (igt_flush_test(gt->i915)) 520 err = -EIO; 521 522 i915_vma_unpin_and_release(&scratch, 0); 523 return err; 524 } 525 526 static int gpr_make_dirty(struct intel_context *ce) 527 { 528 struct i915_request *rq; 529 u32 *cs; 530 int n; 531 532 rq = intel_context_create_request(ce); 533 if (IS_ERR(rq)) 534 return PTR_ERR(rq); 535 536 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2); 537 if (IS_ERR(cs)) { 538 i915_request_add(rq); 539 return PTR_ERR(cs); 540 } 541 542 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW); 543 for (n = 0; n < NUM_GPR_DW; n++) { 544 *cs++ = CS_GPR(ce->engine, n); 545 *cs++ = STACK_MAGIC; 546 } 547 *cs++ = MI_NOOP; 548 549 intel_ring_advance(rq, cs); 550 551 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 552 i915_request_add(rq); 553 554 return 0; 555 } 556 557 static struct i915_request * 558 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot) 559 { 560 const u32 offset = 561 i915_ggtt_offset(ce->engine->status_page.vma) + 562 offset_in_page(slot); 563 struct i915_request *rq; 564 u32 *cs; 565 int err; 566 int n; 567 568 rq = intel_context_create_request(ce); 569 if (IS_ERR(rq)) 570 return rq; 571 572 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW); 573 if (IS_ERR(cs)) { 574 i915_request_add(rq); 575 return ERR_CAST(cs); 576 } 577 578 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 579 *cs++ = MI_NOOP; 580 581 *cs++ = MI_SEMAPHORE_WAIT | 582 MI_SEMAPHORE_GLOBAL_GTT | 583 MI_SEMAPHORE_POLL | 584 MI_SEMAPHORE_SAD_NEQ_SDD; 585 *cs++ = 0; 586 *cs++ = offset; 587 *cs++ = 0; 588 589 for (n = 0; n < NUM_GPR_DW; n++) { 590 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 591 *cs++ = CS_GPR(ce->engine, n); 592 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32); 593 *cs++ = 0; 594 } 595 596 i915_vma_lock(scratch); 597 err = i915_request_await_object(rq, scratch->obj, true); 598 if (!err) 599 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); 600 i915_vma_unlock(scratch); 601 602 i915_request_get(rq); 603 i915_request_add(rq); 604 if (err) { 605 i915_request_put(rq); 606 rq = ERR_PTR(err); 607 } 608 609 return rq; 610 } 611 612 static int __live_lrc_gpr(struct intel_engine_cs *engine, 613 struct i915_vma *scratch, 614 bool preempt) 615 { 616 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4); 617 struct intel_context *ce; 618 struct i915_request *rq; 619 u32 *cs; 620 int err; 621 int n; 622 623 if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS) 624 return 0; /* GPR only on rcs0 for gen8 */ 625 626 err = gpr_make_dirty(engine->kernel_context); 627 if (err) 628 return err; 629 630 ce = intel_context_create(engine); 631 if (IS_ERR(ce)) 632 return PTR_ERR(ce); 633 634 rq = __gpr_read(ce, scratch, slot); 635 if (IS_ERR(rq)) { 636 err = PTR_ERR(rq); 637 goto err_put; 638 } 639 640 err = wait_for_submit(engine, rq, HZ / 2); 641 if (err) 642 goto err_rq; 643 644 if (preempt) { 645 err = gpr_make_dirty(engine->kernel_context); 646 if (err) 647 goto err_rq; 648 649 err = emit_semaphore_signal(engine->kernel_context, slot); 650 if (err) 651 goto err_rq; 652 653 err = wait_for_submit(engine, rq, HZ / 2); 654 if (err) 655 goto err_rq; 656 } else { 657 slot[0] = 1; 658 wmb(); 659 } 660 661 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 662 err = -ETIME; 663 goto err_rq; 664 } 665 666 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB); 667 if (IS_ERR(cs)) { 668 err = PTR_ERR(cs); 669 goto err_rq; 670 } 671 672 for (n = 0; n < NUM_GPR_DW; n++) { 673 if (cs[n]) { 674 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n", 675 engine->name, 676 n / 2, n & 1 ? "udw" : "ldw", 677 cs[n]); 678 err = -EINVAL; 679 break; 680 } 681 } 682 683 i915_gem_object_unpin_map(scratch->obj); 684 685 err_rq: 686 memset32(&slot[0], -1, 4); 687 wmb(); 688 i915_request_put(rq); 689 err_put: 690 intel_context_put(ce); 691 return err; 692 } 693 694 static int live_lrc_gpr(void *arg) 695 { 696 struct intel_gt *gt = arg; 697 struct intel_engine_cs *engine; 698 struct i915_vma *scratch; 699 enum intel_engine_id id; 700 int err = 0; 701 702 /* 703 * Check that GPR registers are cleared in new contexts as we need 704 * to avoid leaking any information from previous contexts. 705 */ 706 707 scratch = create_scratch(gt); 708 if (IS_ERR(scratch)) 709 return PTR_ERR(scratch); 710 711 for_each_engine(engine, gt, id) { 712 st_engine_heartbeat_disable(engine); 713 714 err = __live_lrc_gpr(engine, scratch, false); 715 if (err) 716 goto err; 717 718 err = __live_lrc_gpr(engine, scratch, true); 719 if (err) 720 goto err; 721 722 err: 723 st_engine_heartbeat_enable(engine); 724 if (igt_flush_test(gt->i915)) 725 err = -EIO; 726 if (err) 727 break; 728 } 729 730 i915_vma_unpin_and_release(&scratch, 0); 731 return err; 732 } 733 734 static struct i915_request * 735 create_timestamp(struct intel_context *ce, void *slot, int idx) 736 { 737 const u32 offset = 738 i915_ggtt_offset(ce->engine->status_page.vma) + 739 offset_in_page(slot); 740 struct i915_request *rq; 741 u32 *cs; 742 int err; 743 744 rq = intel_context_create_request(ce); 745 if (IS_ERR(rq)) 746 return rq; 747 748 cs = intel_ring_begin(rq, 10); 749 if (IS_ERR(cs)) { 750 err = PTR_ERR(cs); 751 goto err; 752 } 753 754 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 755 *cs++ = MI_NOOP; 756 757 *cs++ = MI_SEMAPHORE_WAIT | 758 MI_SEMAPHORE_GLOBAL_GTT | 759 MI_SEMAPHORE_POLL | 760 MI_SEMAPHORE_SAD_NEQ_SDD; 761 *cs++ = 0; 762 *cs++ = offset; 763 *cs++ = 0; 764 765 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 766 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base)); 767 *cs++ = offset + idx * sizeof(u32); 768 *cs++ = 0; 769 770 intel_ring_advance(rq, cs); 771 772 err = 0; 773 err: 774 i915_request_get(rq); 775 i915_request_add(rq); 776 if (err) { 777 i915_request_put(rq); 778 return ERR_PTR(err); 779 } 780 781 return rq; 782 } 783 784 struct lrc_timestamp { 785 struct intel_engine_cs *engine; 786 struct intel_context *ce[2]; 787 u32 poison; 788 }; 789 790 static bool timestamp_advanced(u32 start, u32 end) 791 { 792 return (s32)(end - start) > 0; 793 } 794 795 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt) 796 { 797 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4); 798 struct i915_request *rq; 799 u32 timestamp; 800 int err = 0; 801 802 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison; 803 rq = create_timestamp(arg->ce[0], slot, 1); 804 if (IS_ERR(rq)) 805 return PTR_ERR(rq); 806 807 err = wait_for_submit(rq->engine, rq, HZ / 2); 808 if (err) 809 goto err; 810 811 if (preempt) { 812 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef; 813 err = emit_semaphore_signal(arg->ce[1], slot); 814 if (err) 815 goto err; 816 } else { 817 slot[0] = 1; 818 wmb(); 819 } 820 821 /* And wait for switch to kernel (to save our context to memory) */ 822 err = context_flush(arg->ce[0], HZ / 2); 823 if (err) 824 goto err; 825 826 if (!timestamp_advanced(arg->poison, slot[1])) { 827 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n", 828 arg->engine->name, preempt ? "preempt" : "simple", 829 arg->poison, slot[1]); 830 err = -EINVAL; 831 } 832 833 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]); 834 if (!timestamp_advanced(slot[1], timestamp)) { 835 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n", 836 arg->engine->name, preempt ? "preempt" : "simple", 837 slot[1], timestamp); 838 err = -EINVAL; 839 } 840 841 err: 842 memset32(slot, -1, 4); 843 i915_request_put(rq); 844 return err; 845 } 846 847 static int live_lrc_timestamp(void *arg) 848 { 849 struct lrc_timestamp data = {}; 850 struct intel_gt *gt = arg; 851 enum intel_engine_id id; 852 const u32 poison[] = { 853 0, 854 S32_MAX, 855 (u32)S32_MAX + 1, 856 U32_MAX, 857 }; 858 859 /* 860 * We want to verify that the timestamp is saved and restore across 861 * context switches and is monotonic. 862 * 863 * So we do this with a little bit of LRC poisoning to check various 864 * boundary conditions, and see what happens if we preempt the context 865 * with a second request (carrying more poison into the timestamp). 866 */ 867 868 for_each_engine(data.engine, gt, id) { 869 int i, err = 0; 870 871 st_engine_heartbeat_disable(data.engine); 872 873 for (i = 0; i < ARRAY_SIZE(data.ce); i++) { 874 struct intel_context *tmp; 875 876 tmp = intel_context_create(data.engine); 877 if (IS_ERR(tmp)) { 878 err = PTR_ERR(tmp); 879 goto err; 880 } 881 882 err = intel_context_pin(tmp); 883 if (err) { 884 intel_context_put(tmp); 885 goto err; 886 } 887 888 data.ce[i] = tmp; 889 } 890 891 for (i = 0; i < ARRAY_SIZE(poison); i++) { 892 data.poison = poison[i]; 893 894 err = __lrc_timestamp(&data, false); 895 if (err) 896 break; 897 898 err = __lrc_timestamp(&data, true); 899 if (err) 900 break; 901 } 902 903 err: 904 st_engine_heartbeat_enable(data.engine); 905 for (i = 0; i < ARRAY_SIZE(data.ce); i++) { 906 if (!data.ce[i]) 907 break; 908 909 intel_context_unpin(data.ce[i]); 910 intel_context_put(data.ce[i]); 911 } 912 913 if (igt_flush_test(gt->i915)) 914 err = -EIO; 915 if (err) 916 return err; 917 } 918 919 return 0; 920 } 921 922 static struct i915_vma * 923 create_user_vma(struct i915_address_space *vm, unsigned long size) 924 { 925 struct drm_i915_gem_object *obj; 926 struct i915_vma *vma; 927 int err; 928 929 obj = i915_gem_object_create_internal(vm->i915, size); 930 if (IS_ERR(obj)) 931 return ERR_CAST(obj); 932 933 vma = i915_vma_instance(obj, vm, NULL); 934 if (IS_ERR(vma)) { 935 i915_gem_object_put(obj); 936 return vma; 937 } 938 939 err = i915_vma_pin(vma, 0, 0, PIN_USER); 940 if (err) { 941 i915_gem_object_put(obj); 942 return ERR_PTR(err); 943 } 944 945 return vma; 946 } 947 948 static u32 safe_poison(u32 offset, u32 poison) 949 { 950 /* 951 * Do not enable predication as it will nop all subsequent commands, 952 * not only disabling the tests (by preventing all the other SRM) but 953 * also preventing the arbitration events at the end of the request. 954 */ 955 if (offset == i915_mmio_reg_offset(RING_PREDICATE_RESULT(0))) 956 poison &= ~REG_BIT(0); 957 958 return poison; 959 } 960 961 static struct i915_vma * 962 store_context(struct intel_context *ce, struct i915_vma *scratch) 963 { 964 struct i915_vma *batch; 965 u32 dw, x, *cs, *hw; 966 u32 *defaults; 967 968 batch = create_user_vma(ce->vm, SZ_64K); 969 if (IS_ERR(batch)) 970 return batch; 971 972 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 973 if (IS_ERR(cs)) { 974 i915_vma_put(batch); 975 return ERR_CAST(cs); 976 } 977 978 defaults = shmem_pin_map(ce->engine->default_state); 979 if (!defaults) { 980 i915_gem_object_unpin_map(batch->obj); 981 i915_vma_put(batch); 982 return ERR_PTR(-ENOMEM); 983 } 984 985 x = 0; 986 dw = 0; 987 hw = defaults; 988 hw += LRC_STATE_OFFSET / sizeof(*hw); 989 do { 990 u32 len = hw[dw] & 0x7f; 991 992 if (hw[dw] == 0) { 993 dw++; 994 continue; 995 } 996 997 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 998 dw += len + 2; 999 continue; 1000 } 1001 1002 dw++; 1003 len = (len + 1) / 2; 1004 while (len--) { 1005 *cs++ = MI_STORE_REGISTER_MEM_GEN8; 1006 *cs++ = hw[dw]; 1007 *cs++ = lower_32_bits(scratch->node.start + x); 1008 *cs++ = upper_32_bits(scratch->node.start + x); 1009 1010 dw += 2; 1011 x += 4; 1012 } 1013 } while (dw < PAGE_SIZE / sizeof(u32) && 1014 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 1015 1016 *cs++ = MI_BATCH_BUFFER_END; 1017 1018 shmem_unpin_map(ce->engine->default_state, defaults); 1019 1020 i915_gem_object_flush_map(batch->obj); 1021 i915_gem_object_unpin_map(batch->obj); 1022 1023 return batch; 1024 } 1025 1026 static int move_to_active(struct i915_request *rq, 1027 struct i915_vma *vma, 1028 unsigned int flags) 1029 { 1030 int err; 1031 1032 i915_vma_lock(vma); 1033 err = i915_request_await_object(rq, vma->obj, flags); 1034 if (!err) 1035 err = i915_vma_move_to_active(vma, rq, flags); 1036 i915_vma_unlock(vma); 1037 1038 return err; 1039 } 1040 1041 static struct i915_request * 1042 record_registers(struct intel_context *ce, 1043 struct i915_vma *before, 1044 struct i915_vma *after, 1045 u32 *sema) 1046 { 1047 struct i915_vma *b_before, *b_after; 1048 struct i915_request *rq; 1049 u32 *cs; 1050 int err; 1051 1052 b_before = store_context(ce, before); 1053 if (IS_ERR(b_before)) 1054 return ERR_CAST(b_before); 1055 1056 b_after = store_context(ce, after); 1057 if (IS_ERR(b_after)) { 1058 rq = ERR_CAST(b_after); 1059 goto err_before; 1060 } 1061 1062 rq = intel_context_create_request(ce); 1063 if (IS_ERR(rq)) 1064 goto err_after; 1065 1066 err = move_to_active(rq, before, EXEC_OBJECT_WRITE); 1067 if (err) 1068 goto err_rq; 1069 1070 err = move_to_active(rq, b_before, 0); 1071 if (err) 1072 goto err_rq; 1073 1074 err = move_to_active(rq, after, EXEC_OBJECT_WRITE); 1075 if (err) 1076 goto err_rq; 1077 1078 err = move_to_active(rq, b_after, 0); 1079 if (err) 1080 goto err_rq; 1081 1082 cs = intel_ring_begin(rq, 14); 1083 if (IS_ERR(cs)) { 1084 err = PTR_ERR(cs); 1085 goto err_rq; 1086 } 1087 1088 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1089 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1090 *cs++ = lower_32_bits(b_before->node.start); 1091 *cs++ = upper_32_bits(b_before->node.start); 1092 1093 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1094 *cs++ = MI_SEMAPHORE_WAIT | 1095 MI_SEMAPHORE_GLOBAL_GTT | 1096 MI_SEMAPHORE_POLL | 1097 MI_SEMAPHORE_SAD_NEQ_SDD; 1098 *cs++ = 0; 1099 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + 1100 offset_in_page(sema); 1101 *cs++ = 0; 1102 *cs++ = MI_NOOP; 1103 1104 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1105 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1106 *cs++ = lower_32_bits(b_after->node.start); 1107 *cs++ = upper_32_bits(b_after->node.start); 1108 1109 intel_ring_advance(rq, cs); 1110 1111 WRITE_ONCE(*sema, 0); 1112 i915_request_get(rq); 1113 i915_request_add(rq); 1114 err_after: 1115 i915_vma_put(b_after); 1116 err_before: 1117 i915_vma_put(b_before); 1118 return rq; 1119 1120 err_rq: 1121 i915_request_add(rq); 1122 rq = ERR_PTR(err); 1123 goto err_after; 1124 } 1125 1126 static struct i915_vma *load_context(struct intel_context *ce, u32 poison) 1127 { 1128 struct i915_vma *batch; 1129 u32 dw, *cs, *hw; 1130 u32 *defaults; 1131 1132 batch = create_user_vma(ce->vm, SZ_64K); 1133 if (IS_ERR(batch)) 1134 return batch; 1135 1136 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC); 1137 if (IS_ERR(cs)) { 1138 i915_vma_put(batch); 1139 return ERR_CAST(cs); 1140 } 1141 1142 defaults = shmem_pin_map(ce->engine->default_state); 1143 if (!defaults) { 1144 i915_gem_object_unpin_map(batch->obj); 1145 i915_vma_put(batch); 1146 return ERR_PTR(-ENOMEM); 1147 } 1148 1149 dw = 0; 1150 hw = defaults; 1151 hw += LRC_STATE_OFFSET / sizeof(*hw); 1152 do { 1153 u32 len = hw[dw] & 0x7f; 1154 1155 if (hw[dw] == 0) { 1156 dw++; 1157 continue; 1158 } 1159 1160 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 1161 dw += len + 2; 1162 continue; 1163 } 1164 1165 dw++; 1166 len = (len + 1) / 2; 1167 *cs++ = MI_LOAD_REGISTER_IMM(len); 1168 while (len--) { 1169 *cs++ = hw[dw]; 1170 *cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine, 1171 MI_LRI_LRM_CS_MMIO), 1172 poison); 1173 dw += 2; 1174 } 1175 } while (dw < PAGE_SIZE / sizeof(u32) && 1176 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 1177 1178 *cs++ = MI_BATCH_BUFFER_END; 1179 1180 shmem_unpin_map(ce->engine->default_state, defaults); 1181 1182 i915_gem_object_flush_map(batch->obj); 1183 i915_gem_object_unpin_map(batch->obj); 1184 1185 return batch; 1186 } 1187 1188 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema) 1189 { 1190 struct i915_request *rq; 1191 struct i915_vma *batch; 1192 u32 *cs; 1193 int err; 1194 1195 batch = load_context(ce, poison); 1196 if (IS_ERR(batch)) 1197 return PTR_ERR(batch); 1198 1199 rq = intel_context_create_request(ce); 1200 if (IS_ERR(rq)) { 1201 err = PTR_ERR(rq); 1202 goto err_batch; 1203 } 1204 1205 err = move_to_active(rq, batch, 0); 1206 if (err) 1207 goto err_rq; 1208 1209 cs = intel_ring_begin(rq, 8); 1210 if (IS_ERR(cs)) { 1211 err = PTR_ERR(cs); 1212 goto err_rq; 1213 } 1214 1215 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1216 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1217 *cs++ = lower_32_bits(batch->node.start); 1218 *cs++ = upper_32_bits(batch->node.start); 1219 1220 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1221 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + 1222 offset_in_page(sema); 1223 *cs++ = 0; 1224 *cs++ = 1; 1225 1226 intel_ring_advance(rq, cs); 1227 1228 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 1229 err_rq: 1230 i915_request_add(rq); 1231 err_batch: 1232 i915_vma_put(batch); 1233 return err; 1234 } 1235 1236 static bool is_moving(u32 a, u32 b) 1237 { 1238 return a != b; 1239 } 1240 1241 static int compare_isolation(struct intel_engine_cs *engine, 1242 struct i915_vma *ref[2], 1243 struct i915_vma *result[2], 1244 struct intel_context *ce, 1245 u32 poison) 1246 { 1247 u32 x, dw, *hw, *lrc; 1248 u32 *A[2], *B[2]; 1249 u32 *defaults; 1250 int err = 0; 1251 1252 A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC); 1253 if (IS_ERR(A[0])) 1254 return PTR_ERR(A[0]); 1255 1256 A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC); 1257 if (IS_ERR(A[1])) { 1258 err = PTR_ERR(A[1]); 1259 goto err_A0; 1260 } 1261 1262 B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC); 1263 if (IS_ERR(B[0])) { 1264 err = PTR_ERR(B[0]); 1265 goto err_A1; 1266 } 1267 1268 B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC); 1269 if (IS_ERR(B[1])) { 1270 err = PTR_ERR(B[1]); 1271 goto err_B0; 1272 } 1273 1274 lrc = i915_gem_object_pin_map_unlocked(ce->state->obj, 1275 i915_coherent_map_type(engine->i915, 1276 ce->state->obj, 1277 false)); 1278 if (IS_ERR(lrc)) { 1279 err = PTR_ERR(lrc); 1280 goto err_B1; 1281 } 1282 lrc += LRC_STATE_OFFSET / sizeof(*hw); 1283 1284 defaults = shmem_pin_map(ce->engine->default_state); 1285 if (!defaults) { 1286 err = -ENOMEM; 1287 goto err_lrc; 1288 } 1289 1290 x = 0; 1291 dw = 0; 1292 hw = defaults; 1293 hw += LRC_STATE_OFFSET / sizeof(*hw); 1294 do { 1295 u32 len = hw[dw] & 0x7f; 1296 1297 if (hw[dw] == 0) { 1298 dw++; 1299 continue; 1300 } 1301 1302 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 1303 dw += len + 2; 1304 continue; 1305 } 1306 1307 dw++; 1308 len = (len + 1) / 2; 1309 while (len--) { 1310 if (!is_moving(A[0][x], A[1][x]) && 1311 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) { 1312 switch (hw[dw] & 4095) { 1313 case 0x30: /* RING_HEAD */ 1314 case 0x34: /* RING_TAIL */ 1315 break; 1316 1317 default: 1318 pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n", 1319 engine->name, dw, 1320 hw[dw], hw[dw + 1], 1321 A[0][x], B[0][x], B[1][x], 1322 poison, lrc[dw + 1]); 1323 err = -EINVAL; 1324 } 1325 } 1326 dw += 2; 1327 x++; 1328 } 1329 } while (dw < PAGE_SIZE / sizeof(u32) && 1330 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 1331 1332 shmem_unpin_map(ce->engine->default_state, defaults); 1333 err_lrc: 1334 i915_gem_object_unpin_map(ce->state->obj); 1335 err_B1: 1336 i915_gem_object_unpin_map(result[1]->obj); 1337 err_B0: 1338 i915_gem_object_unpin_map(result[0]->obj); 1339 err_A1: 1340 i915_gem_object_unpin_map(ref[1]->obj); 1341 err_A0: 1342 i915_gem_object_unpin_map(ref[0]->obj); 1343 return err; 1344 } 1345 1346 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) 1347 { 1348 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1); 1349 struct i915_vma *ref[2], *result[2]; 1350 struct intel_context *A, *B; 1351 struct i915_request *rq; 1352 int err; 1353 1354 A = intel_context_create(engine); 1355 if (IS_ERR(A)) 1356 return PTR_ERR(A); 1357 1358 B = intel_context_create(engine); 1359 if (IS_ERR(B)) { 1360 err = PTR_ERR(B); 1361 goto err_A; 1362 } 1363 1364 ref[0] = create_user_vma(A->vm, SZ_64K); 1365 if (IS_ERR(ref[0])) { 1366 err = PTR_ERR(ref[0]); 1367 goto err_B; 1368 } 1369 1370 ref[1] = create_user_vma(A->vm, SZ_64K); 1371 if (IS_ERR(ref[1])) { 1372 err = PTR_ERR(ref[1]); 1373 goto err_ref0; 1374 } 1375 1376 rq = record_registers(A, ref[0], ref[1], sema); 1377 if (IS_ERR(rq)) { 1378 err = PTR_ERR(rq); 1379 goto err_ref1; 1380 } 1381 1382 WRITE_ONCE(*sema, 1); 1383 wmb(); 1384 1385 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 1386 i915_request_put(rq); 1387 err = -ETIME; 1388 goto err_ref1; 1389 } 1390 i915_request_put(rq); 1391 1392 result[0] = create_user_vma(A->vm, SZ_64K); 1393 if (IS_ERR(result[0])) { 1394 err = PTR_ERR(result[0]); 1395 goto err_ref1; 1396 } 1397 1398 result[1] = create_user_vma(A->vm, SZ_64K); 1399 if (IS_ERR(result[1])) { 1400 err = PTR_ERR(result[1]); 1401 goto err_result0; 1402 } 1403 1404 rq = record_registers(A, result[0], result[1], sema); 1405 if (IS_ERR(rq)) { 1406 err = PTR_ERR(rq); 1407 goto err_result1; 1408 } 1409 1410 err = poison_registers(B, poison, sema); 1411 if (err) { 1412 WRITE_ONCE(*sema, -1); 1413 i915_request_put(rq); 1414 goto err_result1; 1415 } 1416 1417 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 1418 i915_request_put(rq); 1419 err = -ETIME; 1420 goto err_result1; 1421 } 1422 i915_request_put(rq); 1423 1424 err = compare_isolation(engine, ref, result, A, poison); 1425 1426 err_result1: 1427 i915_vma_put(result[1]); 1428 err_result0: 1429 i915_vma_put(result[0]); 1430 err_ref1: 1431 i915_vma_put(ref[1]); 1432 err_ref0: 1433 i915_vma_put(ref[0]); 1434 err_B: 1435 intel_context_put(B); 1436 err_A: 1437 intel_context_put(A); 1438 return err; 1439 } 1440 1441 static bool skip_isolation(const struct intel_engine_cs *engine) 1442 { 1443 if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9) 1444 return true; 1445 1446 if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11) 1447 return true; 1448 1449 return false; 1450 } 1451 1452 static int live_lrc_isolation(void *arg) 1453 { 1454 struct intel_gt *gt = arg; 1455 struct intel_engine_cs *engine; 1456 enum intel_engine_id id; 1457 const u32 poison[] = { 1458 STACK_MAGIC, 1459 0x3a3a3a3a, 1460 0x5c5c5c5c, 1461 0xffffffff, 1462 0xffff0000, 1463 }; 1464 int err = 0; 1465 1466 /* 1467 * Our goal is try and verify that per-context state cannot be 1468 * tampered with by another non-privileged client. 1469 * 1470 * We take the list of context registers from the LRI in the default 1471 * context image and attempt to modify that list from a remote context. 1472 */ 1473 1474 for_each_engine(engine, gt, id) { 1475 int i; 1476 1477 /* Just don't even ask */ 1478 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) && 1479 skip_isolation(engine)) 1480 continue; 1481 1482 intel_engine_pm_get(engine); 1483 for (i = 0; i < ARRAY_SIZE(poison); i++) { 1484 int result; 1485 1486 result = __lrc_isolation(engine, poison[i]); 1487 if (result && !err) 1488 err = result; 1489 1490 result = __lrc_isolation(engine, ~poison[i]); 1491 if (result && !err) 1492 err = result; 1493 } 1494 intel_engine_pm_put(engine); 1495 if (igt_flush_test(gt->i915)) { 1496 err = -EIO; 1497 break; 1498 } 1499 } 1500 1501 return err; 1502 } 1503 1504 static int indirect_ctx_submit_req(struct intel_context *ce) 1505 { 1506 struct i915_request *rq; 1507 int err = 0; 1508 1509 rq = intel_context_create_request(ce); 1510 if (IS_ERR(rq)) 1511 return PTR_ERR(rq); 1512 1513 i915_request_get(rq); 1514 i915_request_add(rq); 1515 1516 if (i915_request_wait(rq, 0, HZ / 5) < 0) 1517 err = -ETIME; 1518 1519 i915_request_put(rq); 1520 1521 return err; 1522 } 1523 1524 #define CTX_BB_CANARY_OFFSET (3 * 1024) 1525 #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32)) 1526 1527 static u32 * 1528 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs) 1529 { 1530 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | 1531 MI_SRM_LRM_GLOBAL_GTT | 1532 MI_LRI_LRM_CS_MMIO; 1533 *cs++ = i915_mmio_reg_offset(RING_START(0)); 1534 *cs++ = i915_ggtt_offset(ce->state) + 1535 context_wa_bb_offset(ce) + 1536 CTX_BB_CANARY_OFFSET; 1537 *cs++ = 0; 1538 1539 return cs; 1540 } 1541 1542 static void 1543 indirect_ctx_bb_setup(struct intel_context *ce) 1544 { 1545 u32 *cs = context_indirect_bb(ce); 1546 1547 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d; 1548 1549 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary); 1550 } 1551 1552 static bool check_ring_start(struct intel_context *ce) 1553 { 1554 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) - 1555 LRC_STATE_OFFSET + context_wa_bb_offset(ce); 1556 1557 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START]) 1558 return true; 1559 1560 pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n", 1561 ctx_bb[CTX_BB_CANARY_INDEX], 1562 ce->lrc_reg_state[CTX_RING_START]); 1563 1564 return false; 1565 } 1566 1567 static int indirect_ctx_bb_check(struct intel_context *ce) 1568 { 1569 int err; 1570 1571 err = indirect_ctx_submit_req(ce); 1572 if (err) 1573 return err; 1574 1575 if (!check_ring_start(ce)) 1576 return -EINVAL; 1577 1578 return 0; 1579 } 1580 1581 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine) 1582 { 1583 struct intel_context *a, *b; 1584 int err; 1585 1586 a = intel_context_create(engine); 1587 if (IS_ERR(a)) 1588 return PTR_ERR(a); 1589 err = intel_context_pin(a); 1590 if (err) 1591 goto put_a; 1592 1593 b = intel_context_create(engine); 1594 if (IS_ERR(b)) { 1595 err = PTR_ERR(b); 1596 goto unpin_a; 1597 } 1598 err = intel_context_pin(b); 1599 if (err) 1600 goto put_b; 1601 1602 /* We use the already reserved extra page in context state */ 1603 if (!a->wa_bb_page) { 1604 GEM_BUG_ON(b->wa_bb_page); 1605 GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12); 1606 goto unpin_b; 1607 } 1608 1609 /* 1610 * In order to test that our per context bb is truly per context, 1611 * and executes at the intended spot on context restoring process, 1612 * make the batch store the ring start value to memory. 1613 * As ring start is restored apriori of starting the indirect ctx bb and 1614 * as it will be different for each context, it fits to this purpose. 1615 */ 1616 indirect_ctx_bb_setup(a); 1617 indirect_ctx_bb_setup(b); 1618 1619 err = indirect_ctx_bb_check(a); 1620 if (err) 1621 goto unpin_b; 1622 1623 err = indirect_ctx_bb_check(b); 1624 1625 unpin_b: 1626 intel_context_unpin(b); 1627 put_b: 1628 intel_context_put(b); 1629 unpin_a: 1630 intel_context_unpin(a); 1631 put_a: 1632 intel_context_put(a); 1633 1634 return err; 1635 } 1636 1637 static int live_lrc_indirect_ctx_bb(void *arg) 1638 { 1639 struct intel_gt *gt = arg; 1640 struct intel_engine_cs *engine; 1641 enum intel_engine_id id; 1642 int err = 0; 1643 1644 for_each_engine(engine, gt, id) { 1645 intel_engine_pm_get(engine); 1646 err = __live_lrc_indirect_ctx_bb(engine); 1647 intel_engine_pm_put(engine); 1648 1649 if (igt_flush_test(gt->i915)) 1650 err = -EIO; 1651 1652 if (err) 1653 break; 1654 } 1655 1656 return err; 1657 } 1658 1659 static void garbage_reset(struct intel_engine_cs *engine, 1660 struct i915_request *rq) 1661 { 1662 const unsigned int bit = I915_RESET_ENGINE + engine->id; 1663 unsigned long *lock = &engine->gt->reset.flags; 1664 1665 local_bh_disable(); 1666 if (!test_and_set_bit(bit, lock)) { 1667 tasklet_disable(&engine->sched_engine->tasklet); 1668 1669 if (!rq->fence.error) 1670 __intel_engine_reset_bh(engine, NULL); 1671 1672 tasklet_enable(&engine->sched_engine->tasklet); 1673 clear_and_wake_up_bit(bit, lock); 1674 } 1675 local_bh_enable(); 1676 } 1677 1678 static struct i915_request *garbage(struct intel_context *ce, 1679 struct rnd_state *prng) 1680 { 1681 struct i915_request *rq; 1682 int err; 1683 1684 err = intel_context_pin(ce); 1685 if (err) 1686 return ERR_PTR(err); 1687 1688 prandom_bytes_state(prng, 1689 ce->lrc_reg_state, 1690 ce->engine->context_size - 1691 LRC_STATE_OFFSET); 1692 1693 rq = intel_context_create_request(ce); 1694 if (IS_ERR(rq)) { 1695 err = PTR_ERR(rq); 1696 goto err_unpin; 1697 } 1698 1699 i915_request_get(rq); 1700 i915_request_add(rq); 1701 return rq; 1702 1703 err_unpin: 1704 intel_context_unpin(ce); 1705 return ERR_PTR(err); 1706 } 1707 1708 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng) 1709 { 1710 struct intel_context *ce; 1711 struct i915_request *hang; 1712 int err = 0; 1713 1714 ce = intel_context_create(engine); 1715 if (IS_ERR(ce)) 1716 return PTR_ERR(ce); 1717 1718 hang = garbage(ce, prng); 1719 if (IS_ERR(hang)) { 1720 err = PTR_ERR(hang); 1721 goto err_ce; 1722 } 1723 1724 if (wait_for_submit(engine, hang, HZ / 2)) { 1725 i915_request_put(hang); 1726 err = -ETIME; 1727 goto err_ce; 1728 } 1729 1730 intel_context_set_banned(ce); 1731 garbage_reset(engine, hang); 1732 1733 intel_engine_flush_submission(engine); 1734 if (!hang->fence.error) { 1735 i915_request_put(hang); 1736 pr_err("%s: corrupted context was not reset\n", 1737 engine->name); 1738 err = -EINVAL; 1739 goto err_ce; 1740 } 1741 1742 if (i915_request_wait(hang, 0, HZ / 2) < 0) { 1743 pr_err("%s: corrupted context did not recover\n", 1744 engine->name); 1745 i915_request_put(hang); 1746 err = -EIO; 1747 goto err_ce; 1748 } 1749 i915_request_put(hang); 1750 1751 err_ce: 1752 intel_context_put(ce); 1753 return err; 1754 } 1755 1756 static int live_lrc_garbage(void *arg) 1757 { 1758 struct intel_gt *gt = arg; 1759 struct intel_engine_cs *engine; 1760 enum intel_engine_id id; 1761 1762 /* 1763 * Verify that we can recover if one context state is completely 1764 * corrupted. 1765 */ 1766 1767 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN)) 1768 return 0; 1769 1770 for_each_engine(engine, gt, id) { 1771 I915_RND_STATE(prng); 1772 int err = 0, i; 1773 1774 if (!intel_has_reset_engine(engine->gt)) 1775 continue; 1776 1777 intel_engine_pm_get(engine); 1778 for (i = 0; i < 3; i++) { 1779 err = __lrc_garbage(engine, &prng); 1780 if (err) 1781 break; 1782 } 1783 intel_engine_pm_put(engine); 1784 1785 if (igt_flush_test(gt->i915)) 1786 err = -EIO; 1787 if (err) 1788 return err; 1789 } 1790 1791 return 0; 1792 } 1793 1794 static int __live_pphwsp_runtime(struct intel_engine_cs *engine) 1795 { 1796 struct intel_context *ce; 1797 struct i915_request *rq; 1798 IGT_TIMEOUT(end_time); 1799 int err; 1800 1801 ce = intel_context_create(engine); 1802 if (IS_ERR(ce)) 1803 return PTR_ERR(ce); 1804 1805 ce->stats.runtime.num_underflow = 0; 1806 ce->stats.runtime.max_underflow = 0; 1807 1808 do { 1809 unsigned int loop = 1024; 1810 1811 while (loop) { 1812 rq = intel_context_create_request(ce); 1813 if (IS_ERR(rq)) { 1814 err = PTR_ERR(rq); 1815 goto err_rq; 1816 } 1817 1818 if (--loop == 0) 1819 i915_request_get(rq); 1820 1821 i915_request_add(rq); 1822 } 1823 1824 if (__igt_timeout(end_time, NULL)) 1825 break; 1826 1827 i915_request_put(rq); 1828 } while (1); 1829 1830 err = i915_request_wait(rq, 0, HZ / 5); 1831 if (err < 0) { 1832 pr_err("%s: request not completed!\n", engine->name); 1833 goto err_wait; 1834 } 1835 1836 igt_flush_test(engine->i915); 1837 1838 pr_info("%s: pphwsp runtime %lluns, average %lluns\n", 1839 engine->name, 1840 intel_context_get_total_runtime_ns(ce), 1841 intel_context_get_avg_runtime_ns(ce)); 1842 1843 err = 0; 1844 if (ce->stats.runtime.num_underflow) { 1845 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n", 1846 engine->name, 1847 ce->stats.runtime.num_underflow, 1848 ce->stats.runtime.max_underflow); 1849 GEM_TRACE_DUMP(); 1850 err = -EOVERFLOW; 1851 } 1852 1853 err_wait: 1854 i915_request_put(rq); 1855 err_rq: 1856 intel_context_put(ce); 1857 return err; 1858 } 1859 1860 static int live_pphwsp_runtime(void *arg) 1861 { 1862 struct intel_gt *gt = arg; 1863 struct intel_engine_cs *engine; 1864 enum intel_engine_id id; 1865 int err = 0; 1866 1867 /* 1868 * Check that cumulative context runtime as stored in the pphwsp[16] 1869 * is monotonic. 1870 */ 1871 1872 for_each_engine(engine, gt, id) { 1873 err = __live_pphwsp_runtime(engine); 1874 if (err) 1875 break; 1876 } 1877 1878 if (igt_flush_test(gt->i915)) 1879 err = -EIO; 1880 1881 return err; 1882 } 1883 1884 int intel_lrc_live_selftests(struct drm_i915_private *i915) 1885 { 1886 static const struct i915_subtest tests[] = { 1887 SUBTEST(live_lrc_layout), 1888 SUBTEST(live_lrc_fixed), 1889 SUBTEST(live_lrc_state), 1890 SUBTEST(live_lrc_gpr), 1891 SUBTEST(live_lrc_isolation), 1892 SUBTEST(live_lrc_timestamp), 1893 SUBTEST(live_lrc_garbage), 1894 SUBTEST(live_pphwsp_runtime), 1895 SUBTEST(live_lrc_indirect_ctx_bb), 1896 }; 1897 1898 if (!HAS_LOGICAL_RING_CONTEXTS(i915)) 1899 return 0; 1900 1901 return intel_gt_live_subtests(tests, to_gt(i915)); 1902 } 1903