1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2018 Intel Corporation 4 */ 5 6 #include <linux/prime_numbers.h> 7 8 #include "i915_selftest.h" 9 #include "intel_engine_heartbeat.h" 10 #include "intel_engine_pm.h" 11 #include "intel_reset.h" 12 #include "intel_ring.h" 13 #include "selftest_engine_heartbeat.h" 14 #include "selftests/i915_random.h" 15 #include "selftests/igt_flush_test.h" 16 #include "selftests/igt_live_test.h" 17 #include "selftests/igt_spinner.h" 18 #include "selftests/lib_sw_fence.h" 19 #include "shmem_utils.h" 20 21 #include "gem/selftests/igt_gem_utils.h" 22 #include "gem/selftests/mock_context.h" 23 24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4) 25 #define NUM_GPR 16 26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */ 27 28 static struct i915_vma *create_scratch(struct intel_gt *gt) 29 { 30 return __vm_create_scratch_for_read(>->ggtt->vm, PAGE_SIZE); 31 } 32 33 static bool is_active(struct i915_request *rq) 34 { 35 if (i915_request_is_active(rq)) 36 return true; 37 38 if (i915_request_on_hold(rq)) 39 return true; 40 41 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq)) 42 return true; 43 44 return false; 45 } 46 47 static int wait_for_submit(struct intel_engine_cs *engine, 48 struct i915_request *rq, 49 unsigned long timeout) 50 { 51 /* Ignore our own attempts to suppress excess tasklets */ 52 tasklet_hi_schedule(&engine->execlists.tasklet); 53 54 timeout += jiffies; 55 do { 56 bool done = time_after(jiffies, timeout); 57 58 if (i915_request_completed(rq)) /* that was quick! */ 59 return 0; 60 61 /* Wait until the HW has acknowleged the submission (or err) */ 62 intel_engine_flush_submission(engine); 63 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq)) 64 return 0; 65 66 if (done) 67 return -ETIME; 68 69 cond_resched(); 70 } while (1); 71 } 72 73 static int emit_semaphore_signal(struct intel_context *ce, void *slot) 74 { 75 const u32 offset = 76 i915_ggtt_offset(ce->engine->status_page.vma) + 77 offset_in_page(slot); 78 struct i915_request *rq; 79 u32 *cs; 80 81 rq = intel_context_create_request(ce); 82 if (IS_ERR(rq)) 83 return PTR_ERR(rq); 84 85 cs = intel_ring_begin(rq, 4); 86 if (IS_ERR(cs)) { 87 i915_request_add(rq); 88 return PTR_ERR(cs); 89 } 90 91 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 92 *cs++ = offset; 93 *cs++ = 0; 94 *cs++ = 1; 95 96 intel_ring_advance(rq, cs); 97 98 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 99 i915_request_add(rq); 100 return 0; 101 } 102 103 static int context_flush(struct intel_context *ce, long timeout) 104 { 105 struct i915_request *rq; 106 struct dma_fence *fence; 107 int err = 0; 108 109 rq = intel_engine_create_kernel_request(ce->engine); 110 if (IS_ERR(rq)) 111 return PTR_ERR(rq); 112 113 fence = i915_active_fence_get(&ce->timeline->last_request); 114 if (fence) { 115 i915_request_await_dma_fence(rq, fence); 116 dma_fence_put(fence); 117 } 118 119 rq = i915_request_get(rq); 120 i915_request_add(rq); 121 if (i915_request_wait(rq, 0, timeout) < 0) 122 err = -ETIME; 123 i915_request_put(rq); 124 125 rmb(); /* We know the request is written, make sure all state is too! */ 126 return err; 127 } 128 129 static int live_lrc_layout(void *arg) 130 { 131 struct intel_gt *gt = arg; 132 struct intel_engine_cs *engine; 133 enum intel_engine_id id; 134 u32 *lrc; 135 int err; 136 137 /* 138 * Check the registers offsets we use to create the initial reg state 139 * match the layout saved by HW. 140 */ 141 142 lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */ 143 if (!lrc) 144 return -ENOMEM; 145 GEM_BUG_ON(offset_in_page(lrc)); 146 147 err = 0; 148 for_each_engine(engine, gt, id) { 149 u32 *hw; 150 int dw; 151 152 if (!engine->default_state) 153 continue; 154 155 hw = shmem_pin_map(engine->default_state); 156 if (IS_ERR(hw)) { 157 err = PTR_ERR(hw); 158 break; 159 } 160 hw += LRC_STATE_OFFSET / sizeof(*hw); 161 162 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE), 163 engine->kernel_context, engine, true); 164 165 dw = 0; 166 do { 167 u32 lri = READ_ONCE(hw[dw]); 168 169 if (lri == 0) { 170 dw++; 171 continue; 172 } 173 174 if (lrc[dw] == 0) { 175 pr_debug("%s: skipped instruction %x at dword %d\n", 176 engine->name, lri, dw); 177 dw++; 178 continue; 179 } 180 181 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 182 pr_err("%s: Expected LRI command at dword %d, found %08x\n", 183 engine->name, dw, lri); 184 err = -EINVAL; 185 break; 186 } 187 188 if (lrc[dw] != lri) { 189 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n", 190 engine->name, dw, lri, lrc[dw]); 191 err = -EINVAL; 192 break; 193 } 194 195 lri &= 0x7f; 196 lri++; 197 dw++; 198 199 while (lri) { 200 u32 offset = READ_ONCE(hw[dw]); 201 202 if (offset != lrc[dw]) { 203 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n", 204 engine->name, dw, offset, lrc[dw]); 205 err = -EINVAL; 206 break; 207 } 208 209 /* 210 * Skip over the actual register value as we 211 * expect that to differ. 212 */ 213 dw += 2; 214 lri -= 2; 215 } 216 } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 217 218 if (err) { 219 pr_info("%s: HW register image:\n", engine->name); 220 igt_hexdump(hw, PAGE_SIZE); 221 222 pr_info("%s: SW register image:\n", engine->name); 223 igt_hexdump(lrc, PAGE_SIZE); 224 } 225 226 shmem_unpin_map(engine->default_state, hw); 227 if (err) 228 break; 229 } 230 231 free_page((unsigned long)lrc); 232 return err; 233 } 234 235 static int find_offset(const u32 *lri, u32 offset) 236 { 237 int i; 238 239 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++) 240 if (lri[i] == offset) 241 return i; 242 243 return -1; 244 } 245 246 static int live_lrc_fixed(void *arg) 247 { 248 struct intel_gt *gt = arg; 249 struct intel_engine_cs *engine; 250 enum intel_engine_id id; 251 int err = 0; 252 253 /* 254 * Check the assumed register offsets match the actual locations in 255 * the context image. 256 */ 257 258 for_each_engine(engine, gt, id) { 259 const struct { 260 u32 reg; 261 u32 offset; 262 const char *name; 263 } tbl[] = { 264 { 265 i915_mmio_reg_offset(RING_START(engine->mmio_base)), 266 CTX_RING_START - 1, 267 "RING_START" 268 }, 269 { 270 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)), 271 CTX_RING_CTL - 1, 272 "RING_CTL" 273 }, 274 { 275 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)), 276 CTX_RING_HEAD - 1, 277 "RING_HEAD" 278 }, 279 { 280 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)), 281 CTX_RING_TAIL - 1, 282 "RING_TAIL" 283 }, 284 { 285 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)), 286 lrc_ring_mi_mode(engine), 287 "RING_MI_MODE" 288 }, 289 { 290 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)), 291 CTX_BB_STATE - 1, 292 "BB_STATE" 293 }, 294 { 295 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)), 296 lrc_ring_wa_bb_per_ctx(engine), 297 "RING_BB_PER_CTX_PTR" 298 }, 299 { 300 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)), 301 lrc_ring_indirect_ptr(engine), 302 "RING_INDIRECT_CTX_PTR" 303 }, 304 { 305 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)), 306 lrc_ring_indirect_offset(engine), 307 "RING_INDIRECT_CTX_OFFSET" 308 }, 309 { 310 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)), 311 CTX_TIMESTAMP - 1, 312 "RING_CTX_TIMESTAMP" 313 }, 314 { 315 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)), 316 lrc_ring_gpr0(engine), 317 "RING_CS_GPR0" 318 }, 319 { 320 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)), 321 lrc_ring_cmd_buf_cctl(engine), 322 "RING_CMD_BUF_CCTL" 323 }, 324 { }, 325 }, *t; 326 u32 *hw; 327 328 if (!engine->default_state) 329 continue; 330 331 hw = shmem_pin_map(engine->default_state); 332 if (IS_ERR(hw)) { 333 err = PTR_ERR(hw); 334 break; 335 } 336 hw += LRC_STATE_OFFSET / sizeof(*hw); 337 338 for (t = tbl; t->name; t++) { 339 int dw = find_offset(hw, t->reg); 340 341 if (dw != t->offset) { 342 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n", 343 engine->name, 344 t->name, 345 t->reg, 346 dw, 347 t->offset); 348 err = -EINVAL; 349 } 350 } 351 352 shmem_unpin_map(engine->default_state, hw); 353 } 354 355 return err; 356 } 357 358 static int __live_lrc_state(struct intel_engine_cs *engine, 359 struct i915_vma *scratch) 360 { 361 struct intel_context *ce; 362 struct i915_request *rq; 363 struct i915_gem_ww_ctx ww; 364 enum { 365 RING_START_IDX = 0, 366 RING_TAIL_IDX, 367 MAX_IDX 368 }; 369 u32 expected[MAX_IDX]; 370 u32 *cs; 371 int err; 372 int n; 373 374 ce = intel_context_create(engine); 375 if (IS_ERR(ce)) 376 return PTR_ERR(ce); 377 378 i915_gem_ww_ctx_init(&ww, false); 379 retry: 380 err = i915_gem_object_lock(scratch->obj, &ww); 381 if (!err) 382 err = intel_context_pin_ww(ce, &ww); 383 if (err) 384 goto err_put; 385 386 rq = i915_request_create(ce); 387 if (IS_ERR(rq)) { 388 err = PTR_ERR(rq); 389 goto err_unpin; 390 } 391 392 cs = intel_ring_begin(rq, 4 * MAX_IDX); 393 if (IS_ERR(cs)) { 394 err = PTR_ERR(cs); 395 i915_request_add(rq); 396 goto err_unpin; 397 } 398 399 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 400 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base)); 401 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32); 402 *cs++ = 0; 403 404 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma); 405 406 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 407 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)); 408 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32); 409 *cs++ = 0; 410 411 err = i915_request_await_object(rq, scratch->obj, true); 412 if (!err) 413 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); 414 415 i915_request_get(rq); 416 i915_request_add(rq); 417 if (err) 418 goto err_rq; 419 420 intel_engine_flush_submission(engine); 421 expected[RING_TAIL_IDX] = ce->ring->tail; 422 423 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 424 err = -ETIME; 425 goto err_rq; 426 } 427 428 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 429 if (IS_ERR(cs)) { 430 err = PTR_ERR(cs); 431 goto err_rq; 432 } 433 434 for (n = 0; n < MAX_IDX; n++) { 435 if (cs[n] != expected[n]) { 436 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n", 437 engine->name, n, cs[n], expected[n]); 438 err = -EINVAL; 439 break; 440 } 441 } 442 443 i915_gem_object_unpin_map(scratch->obj); 444 445 err_rq: 446 i915_request_put(rq); 447 err_unpin: 448 intel_context_unpin(ce); 449 err_put: 450 if (err == -EDEADLK) { 451 err = i915_gem_ww_ctx_backoff(&ww); 452 if (!err) 453 goto retry; 454 } 455 i915_gem_ww_ctx_fini(&ww); 456 intel_context_put(ce); 457 return err; 458 } 459 460 static int live_lrc_state(void *arg) 461 { 462 struct intel_gt *gt = arg; 463 struct intel_engine_cs *engine; 464 struct i915_vma *scratch; 465 enum intel_engine_id id; 466 int err = 0; 467 468 /* 469 * Check the live register state matches what we expect for this 470 * intel_context. 471 */ 472 473 scratch = create_scratch(gt); 474 if (IS_ERR(scratch)) 475 return PTR_ERR(scratch); 476 477 for_each_engine(engine, gt, id) { 478 err = __live_lrc_state(engine, scratch); 479 if (err) 480 break; 481 } 482 483 if (igt_flush_test(gt->i915)) 484 err = -EIO; 485 486 i915_vma_unpin_and_release(&scratch, 0); 487 return err; 488 } 489 490 static int gpr_make_dirty(struct intel_context *ce) 491 { 492 struct i915_request *rq; 493 u32 *cs; 494 int n; 495 496 rq = intel_context_create_request(ce); 497 if (IS_ERR(rq)) 498 return PTR_ERR(rq); 499 500 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2); 501 if (IS_ERR(cs)) { 502 i915_request_add(rq); 503 return PTR_ERR(cs); 504 } 505 506 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW); 507 for (n = 0; n < NUM_GPR_DW; n++) { 508 *cs++ = CS_GPR(ce->engine, n); 509 *cs++ = STACK_MAGIC; 510 } 511 *cs++ = MI_NOOP; 512 513 intel_ring_advance(rq, cs); 514 515 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 516 i915_request_add(rq); 517 518 return 0; 519 } 520 521 static struct i915_request * 522 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot) 523 { 524 const u32 offset = 525 i915_ggtt_offset(ce->engine->status_page.vma) + 526 offset_in_page(slot); 527 struct i915_request *rq; 528 u32 *cs; 529 int err; 530 int n; 531 532 rq = intel_context_create_request(ce); 533 if (IS_ERR(rq)) 534 return rq; 535 536 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW); 537 if (IS_ERR(cs)) { 538 i915_request_add(rq); 539 return ERR_CAST(cs); 540 } 541 542 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 543 *cs++ = MI_NOOP; 544 545 *cs++ = MI_SEMAPHORE_WAIT | 546 MI_SEMAPHORE_GLOBAL_GTT | 547 MI_SEMAPHORE_POLL | 548 MI_SEMAPHORE_SAD_NEQ_SDD; 549 *cs++ = 0; 550 *cs++ = offset; 551 *cs++ = 0; 552 553 for (n = 0; n < NUM_GPR_DW; n++) { 554 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 555 *cs++ = CS_GPR(ce->engine, n); 556 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32); 557 *cs++ = 0; 558 } 559 560 i915_vma_lock(scratch); 561 err = i915_request_await_object(rq, scratch->obj, true); 562 if (!err) 563 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE); 564 i915_vma_unlock(scratch); 565 566 i915_request_get(rq); 567 i915_request_add(rq); 568 if (err) { 569 i915_request_put(rq); 570 rq = ERR_PTR(err); 571 } 572 573 return rq; 574 } 575 576 static int __live_lrc_gpr(struct intel_engine_cs *engine, 577 struct i915_vma *scratch, 578 bool preempt) 579 { 580 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4); 581 struct intel_context *ce; 582 struct i915_request *rq; 583 u32 *cs; 584 int err; 585 int n; 586 587 if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS) 588 return 0; /* GPR only on rcs0 for gen8 */ 589 590 err = gpr_make_dirty(engine->kernel_context); 591 if (err) 592 return err; 593 594 ce = intel_context_create(engine); 595 if (IS_ERR(ce)) 596 return PTR_ERR(ce); 597 598 rq = __gpr_read(ce, scratch, slot); 599 if (IS_ERR(rq)) { 600 err = PTR_ERR(rq); 601 goto err_put; 602 } 603 604 err = wait_for_submit(engine, rq, HZ / 2); 605 if (err) 606 goto err_rq; 607 608 if (preempt) { 609 err = gpr_make_dirty(engine->kernel_context); 610 if (err) 611 goto err_rq; 612 613 err = emit_semaphore_signal(engine->kernel_context, slot); 614 if (err) 615 goto err_rq; 616 617 err = wait_for_submit(engine, rq, HZ / 2); 618 if (err) 619 goto err_rq; 620 } else { 621 slot[0] = 1; 622 wmb(); 623 } 624 625 if (i915_request_wait(rq, 0, HZ / 5) < 0) { 626 err = -ETIME; 627 goto err_rq; 628 } 629 630 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB); 631 if (IS_ERR(cs)) { 632 err = PTR_ERR(cs); 633 goto err_rq; 634 } 635 636 for (n = 0; n < NUM_GPR_DW; n++) { 637 if (cs[n]) { 638 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n", 639 engine->name, 640 n / 2, n & 1 ? "udw" : "ldw", 641 cs[n]); 642 err = -EINVAL; 643 break; 644 } 645 } 646 647 i915_gem_object_unpin_map(scratch->obj); 648 649 err_rq: 650 memset32(&slot[0], -1, 4); 651 wmb(); 652 i915_request_put(rq); 653 err_put: 654 intel_context_put(ce); 655 return err; 656 } 657 658 static int live_lrc_gpr(void *arg) 659 { 660 struct intel_gt *gt = arg; 661 struct intel_engine_cs *engine; 662 struct i915_vma *scratch; 663 enum intel_engine_id id; 664 int err = 0; 665 666 /* 667 * Check that GPR registers are cleared in new contexts as we need 668 * to avoid leaking any information from previous contexts. 669 */ 670 671 scratch = create_scratch(gt); 672 if (IS_ERR(scratch)) 673 return PTR_ERR(scratch); 674 675 for_each_engine(engine, gt, id) { 676 st_engine_heartbeat_disable(engine); 677 678 err = __live_lrc_gpr(engine, scratch, false); 679 if (err) 680 goto err; 681 682 err = __live_lrc_gpr(engine, scratch, true); 683 if (err) 684 goto err; 685 686 err: 687 st_engine_heartbeat_enable(engine); 688 if (igt_flush_test(gt->i915)) 689 err = -EIO; 690 if (err) 691 break; 692 } 693 694 i915_vma_unpin_and_release(&scratch, 0); 695 return err; 696 } 697 698 static struct i915_request * 699 create_timestamp(struct intel_context *ce, void *slot, int idx) 700 { 701 const u32 offset = 702 i915_ggtt_offset(ce->engine->status_page.vma) + 703 offset_in_page(slot); 704 struct i915_request *rq; 705 u32 *cs; 706 int err; 707 708 rq = intel_context_create_request(ce); 709 if (IS_ERR(rq)) 710 return rq; 711 712 cs = intel_ring_begin(rq, 10); 713 if (IS_ERR(cs)) { 714 err = PTR_ERR(cs); 715 goto err; 716 } 717 718 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 719 *cs++ = MI_NOOP; 720 721 *cs++ = MI_SEMAPHORE_WAIT | 722 MI_SEMAPHORE_GLOBAL_GTT | 723 MI_SEMAPHORE_POLL | 724 MI_SEMAPHORE_SAD_NEQ_SDD; 725 *cs++ = 0; 726 *cs++ = offset; 727 *cs++ = 0; 728 729 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT; 730 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base)); 731 *cs++ = offset + idx * sizeof(u32); 732 *cs++ = 0; 733 734 intel_ring_advance(rq, cs); 735 736 rq->sched.attr.priority = I915_PRIORITY_MASK; 737 err = 0; 738 err: 739 i915_request_get(rq); 740 i915_request_add(rq); 741 if (err) { 742 i915_request_put(rq); 743 return ERR_PTR(err); 744 } 745 746 return rq; 747 } 748 749 struct lrc_timestamp { 750 struct intel_engine_cs *engine; 751 struct intel_context *ce[2]; 752 u32 poison; 753 }; 754 755 static bool timestamp_advanced(u32 start, u32 end) 756 { 757 return (s32)(end - start) > 0; 758 } 759 760 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt) 761 { 762 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4); 763 struct i915_request *rq; 764 u32 timestamp; 765 int err = 0; 766 767 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison; 768 rq = create_timestamp(arg->ce[0], slot, 1); 769 if (IS_ERR(rq)) 770 return PTR_ERR(rq); 771 772 err = wait_for_submit(rq->engine, rq, HZ / 2); 773 if (err) 774 goto err; 775 776 if (preempt) { 777 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef; 778 err = emit_semaphore_signal(arg->ce[1], slot); 779 if (err) 780 goto err; 781 } else { 782 slot[0] = 1; 783 wmb(); 784 } 785 786 /* And wait for switch to kernel (to save our context to memory) */ 787 err = context_flush(arg->ce[0], HZ / 2); 788 if (err) 789 goto err; 790 791 if (!timestamp_advanced(arg->poison, slot[1])) { 792 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n", 793 arg->engine->name, preempt ? "preempt" : "simple", 794 arg->poison, slot[1]); 795 err = -EINVAL; 796 } 797 798 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]); 799 if (!timestamp_advanced(slot[1], timestamp)) { 800 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n", 801 arg->engine->name, preempt ? "preempt" : "simple", 802 slot[1], timestamp); 803 err = -EINVAL; 804 } 805 806 err: 807 memset32(slot, -1, 4); 808 i915_request_put(rq); 809 return err; 810 } 811 812 static int live_lrc_timestamp(void *arg) 813 { 814 struct lrc_timestamp data = {}; 815 struct intel_gt *gt = arg; 816 enum intel_engine_id id; 817 const u32 poison[] = { 818 0, 819 S32_MAX, 820 (u32)S32_MAX + 1, 821 U32_MAX, 822 }; 823 824 /* 825 * We want to verify that the timestamp is saved and restore across 826 * context switches and is monotonic. 827 * 828 * So we do this with a little bit of LRC poisoning to check various 829 * boundary conditions, and see what happens if we preempt the context 830 * with a second request (carrying more poison into the timestamp). 831 */ 832 833 for_each_engine(data.engine, gt, id) { 834 int i, err = 0; 835 836 st_engine_heartbeat_disable(data.engine); 837 838 for (i = 0; i < ARRAY_SIZE(data.ce); i++) { 839 struct intel_context *tmp; 840 841 tmp = intel_context_create(data.engine); 842 if (IS_ERR(tmp)) { 843 err = PTR_ERR(tmp); 844 goto err; 845 } 846 847 err = intel_context_pin(tmp); 848 if (err) { 849 intel_context_put(tmp); 850 goto err; 851 } 852 853 data.ce[i] = tmp; 854 } 855 856 for (i = 0; i < ARRAY_SIZE(poison); i++) { 857 data.poison = poison[i]; 858 859 err = __lrc_timestamp(&data, false); 860 if (err) 861 break; 862 863 err = __lrc_timestamp(&data, true); 864 if (err) 865 break; 866 } 867 868 err: 869 st_engine_heartbeat_enable(data.engine); 870 for (i = 0; i < ARRAY_SIZE(data.ce); i++) { 871 if (!data.ce[i]) 872 break; 873 874 intel_context_unpin(data.ce[i]); 875 intel_context_put(data.ce[i]); 876 } 877 878 if (igt_flush_test(gt->i915)) 879 err = -EIO; 880 if (err) 881 return err; 882 } 883 884 return 0; 885 } 886 887 static struct i915_vma * 888 create_user_vma(struct i915_address_space *vm, unsigned long size) 889 { 890 struct drm_i915_gem_object *obj; 891 struct i915_vma *vma; 892 int err; 893 894 obj = i915_gem_object_create_internal(vm->i915, size); 895 if (IS_ERR(obj)) 896 return ERR_CAST(obj); 897 898 vma = i915_vma_instance(obj, vm, NULL); 899 if (IS_ERR(vma)) { 900 i915_gem_object_put(obj); 901 return vma; 902 } 903 904 err = i915_vma_pin(vma, 0, 0, PIN_USER); 905 if (err) { 906 i915_gem_object_put(obj); 907 return ERR_PTR(err); 908 } 909 910 return vma; 911 } 912 913 static struct i915_vma * 914 store_context(struct intel_context *ce, struct i915_vma *scratch) 915 { 916 struct i915_vma *batch; 917 u32 dw, x, *cs, *hw; 918 u32 *defaults; 919 920 batch = create_user_vma(ce->vm, SZ_64K); 921 if (IS_ERR(batch)) 922 return batch; 923 924 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 925 if (IS_ERR(cs)) { 926 i915_vma_put(batch); 927 return ERR_CAST(cs); 928 } 929 930 defaults = shmem_pin_map(ce->engine->default_state); 931 if (!defaults) { 932 i915_gem_object_unpin_map(batch->obj); 933 i915_vma_put(batch); 934 return ERR_PTR(-ENOMEM); 935 } 936 937 x = 0; 938 dw = 0; 939 hw = defaults; 940 hw += LRC_STATE_OFFSET / sizeof(*hw); 941 do { 942 u32 len = hw[dw] & 0x7f; 943 944 if (hw[dw] == 0) { 945 dw++; 946 continue; 947 } 948 949 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 950 dw += len + 2; 951 continue; 952 } 953 954 dw++; 955 len = (len + 1) / 2; 956 while (len--) { 957 *cs++ = MI_STORE_REGISTER_MEM_GEN8; 958 *cs++ = hw[dw]; 959 *cs++ = lower_32_bits(scratch->node.start + x); 960 *cs++ = upper_32_bits(scratch->node.start + x); 961 962 dw += 2; 963 x += 4; 964 } 965 } while (dw < PAGE_SIZE / sizeof(u32) && 966 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 967 968 *cs++ = MI_BATCH_BUFFER_END; 969 970 shmem_unpin_map(ce->engine->default_state, defaults); 971 972 i915_gem_object_flush_map(batch->obj); 973 i915_gem_object_unpin_map(batch->obj); 974 975 return batch; 976 } 977 978 static int move_to_active(struct i915_request *rq, 979 struct i915_vma *vma, 980 unsigned int flags) 981 { 982 int err; 983 984 i915_vma_lock(vma); 985 err = i915_request_await_object(rq, vma->obj, flags); 986 if (!err) 987 err = i915_vma_move_to_active(vma, rq, flags); 988 i915_vma_unlock(vma); 989 990 return err; 991 } 992 993 static struct i915_request * 994 record_registers(struct intel_context *ce, 995 struct i915_vma *before, 996 struct i915_vma *after, 997 u32 *sema) 998 { 999 struct i915_vma *b_before, *b_after; 1000 struct i915_request *rq; 1001 u32 *cs; 1002 int err; 1003 1004 b_before = store_context(ce, before); 1005 if (IS_ERR(b_before)) 1006 return ERR_CAST(b_before); 1007 1008 b_after = store_context(ce, after); 1009 if (IS_ERR(b_after)) { 1010 rq = ERR_CAST(b_after); 1011 goto err_before; 1012 } 1013 1014 rq = intel_context_create_request(ce); 1015 if (IS_ERR(rq)) 1016 goto err_after; 1017 1018 err = move_to_active(rq, before, EXEC_OBJECT_WRITE); 1019 if (err) 1020 goto err_rq; 1021 1022 err = move_to_active(rq, b_before, 0); 1023 if (err) 1024 goto err_rq; 1025 1026 err = move_to_active(rq, after, EXEC_OBJECT_WRITE); 1027 if (err) 1028 goto err_rq; 1029 1030 err = move_to_active(rq, b_after, 0); 1031 if (err) 1032 goto err_rq; 1033 1034 cs = intel_ring_begin(rq, 14); 1035 if (IS_ERR(cs)) { 1036 err = PTR_ERR(cs); 1037 goto err_rq; 1038 } 1039 1040 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1041 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1042 *cs++ = lower_32_bits(b_before->node.start); 1043 *cs++ = upper_32_bits(b_before->node.start); 1044 1045 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1046 *cs++ = MI_SEMAPHORE_WAIT | 1047 MI_SEMAPHORE_GLOBAL_GTT | 1048 MI_SEMAPHORE_POLL | 1049 MI_SEMAPHORE_SAD_NEQ_SDD; 1050 *cs++ = 0; 1051 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + 1052 offset_in_page(sema); 1053 *cs++ = 0; 1054 *cs++ = MI_NOOP; 1055 1056 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1057 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1058 *cs++ = lower_32_bits(b_after->node.start); 1059 *cs++ = upper_32_bits(b_after->node.start); 1060 1061 intel_ring_advance(rq, cs); 1062 1063 WRITE_ONCE(*sema, 0); 1064 i915_request_get(rq); 1065 i915_request_add(rq); 1066 err_after: 1067 i915_vma_put(b_after); 1068 err_before: 1069 i915_vma_put(b_before); 1070 return rq; 1071 1072 err_rq: 1073 i915_request_add(rq); 1074 rq = ERR_PTR(err); 1075 goto err_after; 1076 } 1077 1078 static struct i915_vma *load_context(struct intel_context *ce, u32 poison) 1079 { 1080 struct i915_vma *batch; 1081 u32 dw, *cs, *hw; 1082 u32 *defaults; 1083 1084 batch = create_user_vma(ce->vm, SZ_64K); 1085 if (IS_ERR(batch)) 1086 return batch; 1087 1088 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC); 1089 if (IS_ERR(cs)) { 1090 i915_vma_put(batch); 1091 return ERR_CAST(cs); 1092 } 1093 1094 defaults = shmem_pin_map(ce->engine->default_state); 1095 if (!defaults) { 1096 i915_gem_object_unpin_map(batch->obj); 1097 i915_vma_put(batch); 1098 return ERR_PTR(-ENOMEM); 1099 } 1100 1101 dw = 0; 1102 hw = defaults; 1103 hw += LRC_STATE_OFFSET / sizeof(*hw); 1104 do { 1105 u32 len = hw[dw] & 0x7f; 1106 1107 if (hw[dw] == 0) { 1108 dw++; 1109 continue; 1110 } 1111 1112 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 1113 dw += len + 2; 1114 continue; 1115 } 1116 1117 dw++; 1118 len = (len + 1) / 2; 1119 *cs++ = MI_LOAD_REGISTER_IMM(len); 1120 while (len--) { 1121 *cs++ = hw[dw]; 1122 *cs++ = poison; 1123 dw += 2; 1124 } 1125 } while (dw < PAGE_SIZE / sizeof(u32) && 1126 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 1127 1128 *cs++ = MI_BATCH_BUFFER_END; 1129 1130 shmem_unpin_map(ce->engine->default_state, defaults); 1131 1132 i915_gem_object_flush_map(batch->obj); 1133 i915_gem_object_unpin_map(batch->obj); 1134 1135 return batch; 1136 } 1137 1138 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema) 1139 { 1140 struct i915_request *rq; 1141 struct i915_vma *batch; 1142 u32 *cs; 1143 int err; 1144 1145 batch = load_context(ce, poison); 1146 if (IS_ERR(batch)) 1147 return PTR_ERR(batch); 1148 1149 rq = intel_context_create_request(ce); 1150 if (IS_ERR(rq)) { 1151 err = PTR_ERR(rq); 1152 goto err_batch; 1153 } 1154 1155 err = move_to_active(rq, batch, 0); 1156 if (err) 1157 goto err_rq; 1158 1159 cs = intel_ring_begin(rq, 8); 1160 if (IS_ERR(cs)) { 1161 err = PTR_ERR(cs); 1162 goto err_rq; 1163 } 1164 1165 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1166 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8); 1167 *cs++ = lower_32_bits(batch->node.start); 1168 *cs++ = upper_32_bits(batch->node.start); 1169 1170 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1171 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) + 1172 offset_in_page(sema); 1173 *cs++ = 0; 1174 *cs++ = 1; 1175 1176 intel_ring_advance(rq, cs); 1177 1178 rq->sched.attr.priority = I915_PRIORITY_BARRIER; 1179 err_rq: 1180 i915_request_add(rq); 1181 err_batch: 1182 i915_vma_put(batch); 1183 return err; 1184 } 1185 1186 static bool is_moving(u32 a, u32 b) 1187 { 1188 return a != b; 1189 } 1190 1191 static int compare_isolation(struct intel_engine_cs *engine, 1192 struct i915_vma *ref[2], 1193 struct i915_vma *result[2], 1194 struct intel_context *ce, 1195 u32 poison) 1196 { 1197 u32 x, dw, *hw, *lrc; 1198 u32 *A[2], *B[2]; 1199 u32 *defaults; 1200 int err = 0; 1201 1202 A[0] = i915_gem_object_pin_map(ref[0]->obj, I915_MAP_WC); 1203 if (IS_ERR(A[0])) 1204 return PTR_ERR(A[0]); 1205 1206 A[1] = i915_gem_object_pin_map(ref[1]->obj, I915_MAP_WC); 1207 if (IS_ERR(A[1])) { 1208 err = PTR_ERR(A[1]); 1209 goto err_A0; 1210 } 1211 1212 B[0] = i915_gem_object_pin_map(result[0]->obj, I915_MAP_WC); 1213 if (IS_ERR(B[0])) { 1214 err = PTR_ERR(B[0]); 1215 goto err_A1; 1216 } 1217 1218 B[1] = i915_gem_object_pin_map(result[1]->obj, I915_MAP_WC); 1219 if (IS_ERR(B[1])) { 1220 err = PTR_ERR(B[1]); 1221 goto err_B0; 1222 } 1223 1224 lrc = i915_gem_object_pin_map(ce->state->obj, 1225 i915_coherent_map_type(engine->i915)); 1226 if (IS_ERR(lrc)) { 1227 err = PTR_ERR(lrc); 1228 goto err_B1; 1229 } 1230 lrc += LRC_STATE_OFFSET / sizeof(*hw); 1231 1232 defaults = shmem_pin_map(ce->engine->default_state); 1233 if (!defaults) { 1234 err = -ENOMEM; 1235 goto err_lrc; 1236 } 1237 1238 x = 0; 1239 dw = 0; 1240 hw = defaults; 1241 hw += LRC_STATE_OFFSET / sizeof(*hw); 1242 do { 1243 u32 len = hw[dw] & 0x7f; 1244 1245 if (hw[dw] == 0) { 1246 dw++; 1247 continue; 1248 } 1249 1250 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) { 1251 dw += len + 2; 1252 continue; 1253 } 1254 1255 dw++; 1256 len = (len + 1) / 2; 1257 while (len--) { 1258 if (!is_moving(A[0][x], A[1][x]) && 1259 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) { 1260 switch (hw[dw] & 4095) { 1261 case 0x30: /* RING_HEAD */ 1262 case 0x34: /* RING_TAIL */ 1263 break; 1264 1265 default: 1266 pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n", 1267 engine->name, dw, 1268 hw[dw], hw[dw + 1], 1269 A[0][x], B[0][x], B[1][x], 1270 poison, lrc[dw + 1]); 1271 err = -EINVAL; 1272 } 1273 } 1274 dw += 2; 1275 x++; 1276 } 1277 } while (dw < PAGE_SIZE / sizeof(u32) && 1278 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END); 1279 1280 shmem_unpin_map(ce->engine->default_state, defaults); 1281 err_lrc: 1282 i915_gem_object_unpin_map(ce->state->obj); 1283 err_B1: 1284 i915_gem_object_unpin_map(result[1]->obj); 1285 err_B0: 1286 i915_gem_object_unpin_map(result[0]->obj); 1287 err_A1: 1288 i915_gem_object_unpin_map(ref[1]->obj); 1289 err_A0: 1290 i915_gem_object_unpin_map(ref[0]->obj); 1291 return err; 1292 } 1293 1294 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison) 1295 { 1296 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1); 1297 struct i915_vma *ref[2], *result[2]; 1298 struct intel_context *A, *B; 1299 struct i915_request *rq; 1300 int err; 1301 1302 A = intel_context_create(engine); 1303 if (IS_ERR(A)) 1304 return PTR_ERR(A); 1305 1306 B = intel_context_create(engine); 1307 if (IS_ERR(B)) { 1308 err = PTR_ERR(B); 1309 goto err_A; 1310 } 1311 1312 ref[0] = create_user_vma(A->vm, SZ_64K); 1313 if (IS_ERR(ref[0])) { 1314 err = PTR_ERR(ref[0]); 1315 goto err_B; 1316 } 1317 1318 ref[1] = create_user_vma(A->vm, SZ_64K); 1319 if (IS_ERR(ref[1])) { 1320 err = PTR_ERR(ref[1]); 1321 goto err_ref0; 1322 } 1323 1324 rq = record_registers(A, ref[0], ref[1], sema); 1325 if (IS_ERR(rq)) { 1326 err = PTR_ERR(rq); 1327 goto err_ref1; 1328 } 1329 1330 WRITE_ONCE(*sema, 1); 1331 wmb(); 1332 1333 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 1334 i915_request_put(rq); 1335 err = -ETIME; 1336 goto err_ref1; 1337 } 1338 i915_request_put(rq); 1339 1340 result[0] = create_user_vma(A->vm, SZ_64K); 1341 if (IS_ERR(result[0])) { 1342 err = PTR_ERR(result[0]); 1343 goto err_ref1; 1344 } 1345 1346 result[1] = create_user_vma(A->vm, SZ_64K); 1347 if (IS_ERR(result[1])) { 1348 err = PTR_ERR(result[1]); 1349 goto err_result0; 1350 } 1351 1352 rq = record_registers(A, result[0], result[1], sema); 1353 if (IS_ERR(rq)) { 1354 err = PTR_ERR(rq); 1355 goto err_result1; 1356 } 1357 1358 err = poison_registers(B, poison, sema); 1359 if (err) { 1360 WRITE_ONCE(*sema, -1); 1361 i915_request_put(rq); 1362 goto err_result1; 1363 } 1364 1365 if (i915_request_wait(rq, 0, HZ / 2) < 0) { 1366 i915_request_put(rq); 1367 err = -ETIME; 1368 goto err_result1; 1369 } 1370 i915_request_put(rq); 1371 1372 err = compare_isolation(engine, ref, result, A, poison); 1373 1374 err_result1: 1375 i915_vma_put(result[1]); 1376 err_result0: 1377 i915_vma_put(result[0]); 1378 err_ref1: 1379 i915_vma_put(ref[1]); 1380 err_ref0: 1381 i915_vma_put(ref[0]); 1382 err_B: 1383 intel_context_put(B); 1384 err_A: 1385 intel_context_put(A); 1386 return err; 1387 } 1388 1389 static bool skip_isolation(const struct intel_engine_cs *engine) 1390 { 1391 if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9) 1392 return true; 1393 1394 if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11) 1395 return true; 1396 1397 return false; 1398 } 1399 1400 static int live_lrc_isolation(void *arg) 1401 { 1402 struct intel_gt *gt = arg; 1403 struct intel_engine_cs *engine; 1404 enum intel_engine_id id; 1405 const u32 poison[] = { 1406 STACK_MAGIC, 1407 0x3a3a3a3a, 1408 0x5c5c5c5c, 1409 0xffffffff, 1410 0xffff0000, 1411 }; 1412 int err = 0; 1413 1414 /* 1415 * Our goal is try and verify that per-context state cannot be 1416 * tampered with by another non-privileged client. 1417 * 1418 * We take the list of context registers from the LRI in the default 1419 * context image and attempt to modify that list from a remote context. 1420 */ 1421 1422 for_each_engine(engine, gt, id) { 1423 int i; 1424 1425 /* Just don't even ask */ 1426 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) && 1427 skip_isolation(engine)) 1428 continue; 1429 1430 intel_engine_pm_get(engine); 1431 for (i = 0; i < ARRAY_SIZE(poison); i++) { 1432 int result; 1433 1434 result = __lrc_isolation(engine, poison[i]); 1435 if (result && !err) 1436 err = result; 1437 1438 result = __lrc_isolation(engine, ~poison[i]); 1439 if (result && !err) 1440 err = result; 1441 } 1442 intel_engine_pm_put(engine); 1443 if (igt_flush_test(gt->i915)) { 1444 err = -EIO; 1445 break; 1446 } 1447 } 1448 1449 return err; 1450 } 1451 1452 static int indirect_ctx_submit_req(struct intel_context *ce) 1453 { 1454 struct i915_request *rq; 1455 int err = 0; 1456 1457 rq = intel_context_create_request(ce); 1458 if (IS_ERR(rq)) 1459 return PTR_ERR(rq); 1460 1461 i915_request_get(rq); 1462 i915_request_add(rq); 1463 1464 if (i915_request_wait(rq, 0, HZ / 5) < 0) 1465 err = -ETIME; 1466 1467 i915_request_put(rq); 1468 1469 return err; 1470 } 1471 1472 #define CTX_BB_CANARY_OFFSET (3 * 1024) 1473 #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32)) 1474 1475 static u32 * 1476 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs) 1477 { 1478 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | 1479 MI_SRM_LRM_GLOBAL_GTT | 1480 MI_LRI_LRM_CS_MMIO; 1481 *cs++ = i915_mmio_reg_offset(RING_START(0)); 1482 *cs++ = i915_ggtt_offset(ce->state) + 1483 context_wa_bb_offset(ce) + 1484 CTX_BB_CANARY_OFFSET; 1485 *cs++ = 0; 1486 1487 return cs; 1488 } 1489 1490 static void 1491 indirect_ctx_bb_setup(struct intel_context *ce) 1492 { 1493 u32 *cs = context_indirect_bb(ce); 1494 1495 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d; 1496 1497 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary); 1498 } 1499 1500 static bool check_ring_start(struct intel_context *ce) 1501 { 1502 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) - 1503 LRC_STATE_OFFSET + context_wa_bb_offset(ce); 1504 1505 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START]) 1506 return true; 1507 1508 pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n", 1509 ctx_bb[CTX_BB_CANARY_INDEX], 1510 ce->lrc_reg_state[CTX_RING_START]); 1511 1512 return false; 1513 } 1514 1515 static int indirect_ctx_bb_check(struct intel_context *ce) 1516 { 1517 int err; 1518 1519 err = indirect_ctx_submit_req(ce); 1520 if (err) 1521 return err; 1522 1523 if (!check_ring_start(ce)) 1524 return -EINVAL; 1525 1526 return 0; 1527 } 1528 1529 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine) 1530 { 1531 struct intel_context *a, *b; 1532 int err; 1533 1534 a = intel_context_create(engine); 1535 if (IS_ERR(a)) 1536 return PTR_ERR(a); 1537 err = intel_context_pin(a); 1538 if (err) 1539 goto put_a; 1540 1541 b = intel_context_create(engine); 1542 if (IS_ERR(b)) { 1543 err = PTR_ERR(b); 1544 goto unpin_a; 1545 } 1546 err = intel_context_pin(b); 1547 if (err) 1548 goto put_b; 1549 1550 /* We use the already reserved extra page in context state */ 1551 if (!a->wa_bb_page) { 1552 GEM_BUG_ON(b->wa_bb_page); 1553 GEM_BUG_ON(INTEL_GEN(engine->i915) == 12); 1554 goto unpin_b; 1555 } 1556 1557 /* 1558 * In order to test that our per context bb is truly per context, 1559 * and executes at the intended spot on context restoring process, 1560 * make the batch store the ring start value to memory. 1561 * As ring start is restored apriori of starting the indirect ctx bb and 1562 * as it will be different for each context, it fits to this purpose. 1563 */ 1564 indirect_ctx_bb_setup(a); 1565 indirect_ctx_bb_setup(b); 1566 1567 err = indirect_ctx_bb_check(a); 1568 if (err) 1569 goto unpin_b; 1570 1571 err = indirect_ctx_bb_check(b); 1572 1573 unpin_b: 1574 intel_context_unpin(b); 1575 put_b: 1576 intel_context_put(b); 1577 unpin_a: 1578 intel_context_unpin(a); 1579 put_a: 1580 intel_context_put(a); 1581 1582 return err; 1583 } 1584 1585 static int live_lrc_indirect_ctx_bb(void *arg) 1586 { 1587 struct intel_gt *gt = arg; 1588 struct intel_engine_cs *engine; 1589 enum intel_engine_id id; 1590 int err = 0; 1591 1592 for_each_engine(engine, gt, id) { 1593 intel_engine_pm_get(engine); 1594 err = __live_lrc_indirect_ctx_bb(engine); 1595 intel_engine_pm_put(engine); 1596 1597 if (igt_flush_test(gt->i915)) 1598 err = -EIO; 1599 1600 if (err) 1601 break; 1602 } 1603 1604 return err; 1605 } 1606 1607 static void garbage_reset(struct intel_engine_cs *engine, 1608 struct i915_request *rq) 1609 { 1610 const unsigned int bit = I915_RESET_ENGINE + engine->id; 1611 unsigned long *lock = &engine->gt->reset.flags; 1612 1613 local_bh_disable(); 1614 if (!test_and_set_bit(bit, lock)) { 1615 tasklet_disable(&engine->execlists.tasklet); 1616 1617 if (!rq->fence.error) 1618 __intel_engine_reset_bh(engine, NULL); 1619 1620 tasklet_enable(&engine->execlists.tasklet); 1621 clear_and_wake_up_bit(bit, lock); 1622 } 1623 local_bh_enable(); 1624 } 1625 1626 static struct i915_request *garbage(struct intel_context *ce, 1627 struct rnd_state *prng) 1628 { 1629 struct i915_request *rq; 1630 int err; 1631 1632 err = intel_context_pin(ce); 1633 if (err) 1634 return ERR_PTR(err); 1635 1636 prandom_bytes_state(prng, 1637 ce->lrc_reg_state, 1638 ce->engine->context_size - 1639 LRC_STATE_OFFSET); 1640 1641 rq = intel_context_create_request(ce); 1642 if (IS_ERR(rq)) { 1643 err = PTR_ERR(rq); 1644 goto err_unpin; 1645 } 1646 1647 i915_request_get(rq); 1648 i915_request_add(rq); 1649 return rq; 1650 1651 err_unpin: 1652 intel_context_unpin(ce); 1653 return ERR_PTR(err); 1654 } 1655 1656 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng) 1657 { 1658 struct intel_context *ce; 1659 struct i915_request *hang; 1660 int err = 0; 1661 1662 ce = intel_context_create(engine); 1663 if (IS_ERR(ce)) 1664 return PTR_ERR(ce); 1665 1666 hang = garbage(ce, prng); 1667 if (IS_ERR(hang)) { 1668 err = PTR_ERR(hang); 1669 goto err_ce; 1670 } 1671 1672 if (wait_for_submit(engine, hang, HZ / 2)) { 1673 i915_request_put(hang); 1674 err = -ETIME; 1675 goto err_ce; 1676 } 1677 1678 intel_context_set_banned(ce); 1679 garbage_reset(engine, hang); 1680 1681 intel_engine_flush_submission(engine); 1682 if (!hang->fence.error) { 1683 i915_request_put(hang); 1684 pr_err("%s: corrupted context was not reset\n", 1685 engine->name); 1686 err = -EINVAL; 1687 goto err_ce; 1688 } 1689 1690 if (i915_request_wait(hang, 0, HZ / 2) < 0) { 1691 pr_err("%s: corrupted context did not recover\n", 1692 engine->name); 1693 i915_request_put(hang); 1694 err = -EIO; 1695 goto err_ce; 1696 } 1697 i915_request_put(hang); 1698 1699 err_ce: 1700 intel_context_put(ce); 1701 return err; 1702 } 1703 1704 static int live_lrc_garbage(void *arg) 1705 { 1706 struct intel_gt *gt = arg; 1707 struct intel_engine_cs *engine; 1708 enum intel_engine_id id; 1709 1710 /* 1711 * Verify that we can recover if one context state is completely 1712 * corrupted. 1713 */ 1714 1715 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN)) 1716 return 0; 1717 1718 for_each_engine(engine, gt, id) { 1719 I915_RND_STATE(prng); 1720 int err = 0, i; 1721 1722 if (!intel_has_reset_engine(engine->gt)) 1723 continue; 1724 1725 intel_engine_pm_get(engine); 1726 for (i = 0; i < 3; i++) { 1727 err = __lrc_garbage(engine, &prng); 1728 if (err) 1729 break; 1730 } 1731 intel_engine_pm_put(engine); 1732 1733 if (igt_flush_test(gt->i915)) 1734 err = -EIO; 1735 if (err) 1736 return err; 1737 } 1738 1739 return 0; 1740 } 1741 1742 static int __live_pphwsp_runtime(struct intel_engine_cs *engine) 1743 { 1744 struct intel_context *ce; 1745 struct i915_request *rq; 1746 IGT_TIMEOUT(end_time); 1747 int err; 1748 1749 ce = intel_context_create(engine); 1750 if (IS_ERR(ce)) 1751 return PTR_ERR(ce); 1752 1753 ce->runtime.num_underflow = 0; 1754 ce->runtime.max_underflow = 0; 1755 1756 do { 1757 unsigned int loop = 1024; 1758 1759 while (loop) { 1760 rq = intel_context_create_request(ce); 1761 if (IS_ERR(rq)) { 1762 err = PTR_ERR(rq); 1763 goto err_rq; 1764 } 1765 1766 if (--loop == 0) 1767 i915_request_get(rq); 1768 1769 i915_request_add(rq); 1770 } 1771 1772 if (__igt_timeout(end_time, NULL)) 1773 break; 1774 1775 i915_request_put(rq); 1776 } while (1); 1777 1778 err = i915_request_wait(rq, 0, HZ / 5); 1779 if (err < 0) { 1780 pr_err("%s: request not completed!\n", engine->name); 1781 goto err_wait; 1782 } 1783 1784 igt_flush_test(engine->i915); 1785 1786 pr_info("%s: pphwsp runtime %lluns, average %lluns\n", 1787 engine->name, 1788 intel_context_get_total_runtime_ns(ce), 1789 intel_context_get_avg_runtime_ns(ce)); 1790 1791 err = 0; 1792 if (ce->runtime.num_underflow) { 1793 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n", 1794 engine->name, 1795 ce->runtime.num_underflow, 1796 ce->runtime.max_underflow); 1797 GEM_TRACE_DUMP(); 1798 err = -EOVERFLOW; 1799 } 1800 1801 err_wait: 1802 i915_request_put(rq); 1803 err_rq: 1804 intel_context_put(ce); 1805 return err; 1806 } 1807 1808 static int live_pphwsp_runtime(void *arg) 1809 { 1810 struct intel_gt *gt = arg; 1811 struct intel_engine_cs *engine; 1812 enum intel_engine_id id; 1813 int err = 0; 1814 1815 /* 1816 * Check that cumulative context runtime as stored in the pphwsp[16] 1817 * is monotonic. 1818 */ 1819 1820 for_each_engine(engine, gt, id) { 1821 err = __live_pphwsp_runtime(engine); 1822 if (err) 1823 break; 1824 } 1825 1826 if (igt_flush_test(gt->i915)) 1827 err = -EIO; 1828 1829 return err; 1830 } 1831 1832 int intel_lrc_live_selftests(struct drm_i915_private *i915) 1833 { 1834 static const struct i915_subtest tests[] = { 1835 SUBTEST(live_lrc_layout), 1836 SUBTEST(live_lrc_fixed), 1837 SUBTEST(live_lrc_state), 1838 SUBTEST(live_lrc_gpr), 1839 SUBTEST(live_lrc_isolation), 1840 SUBTEST(live_lrc_timestamp), 1841 SUBTEST(live_lrc_garbage), 1842 SUBTEST(live_pphwsp_runtime), 1843 SUBTEST(live_lrc_indirect_ctx_bb), 1844 }; 1845 1846 if (!HAS_LOGICAL_RING_CONTEXTS(i915)) 1847 return 0; 1848 1849 return intel_gt_live_subtests(tests, &i915->gt); 1850 } 1851