Lines Matching +full:cs +full:- +full:2
1 // SPDX-License-Identifier: MIT
26 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
28 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
35 return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE);
57 tasklet_hi_schedule(&engine->sched_engine->tasklet);
68 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
72 return -ETIME;
81 i915_ggtt_offset(ce->engine->status_page.vma) +
84 u32 *cs;
90 cs = intel_ring_begin(rq, 4);
91 if (IS_ERR(cs)) {
93 return PTR_ERR(cs);
96 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
97 *cs++ = offset;
98 *cs++ = 0;
99 *cs++ = 1;
101 intel_ring_advance(rq, cs);
103 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
114 rq = intel_engine_create_kernel_request(ce->engine);
118 fence = i915_active_fence_get(&ce->timeline->last_request);
127 err = -ETIME;
139 if (GRAPHICS_VER(engine->i915) < 12)
142 switch (engine->class) {
170 return -ENOMEM;
178 if (!engine->default_state)
181 hw = shmem_pin_map(engine->default_state);
183 err = -ENOMEM;
189 engine->kernel_context, engine, true);
203 engine->name, lri, dw);
210 engine->name, dw, lri);
211 err = -EINVAL;
217 engine->name, dw, lri, lrc[dw]);
218 err = -EINVAL;
228 * RCS && CCS: BITS(0 - 10)
229 * BCS: BITS(0 - 11)
230 * VECS && VCS: BITS(0 - 13)
243 engine->name, dw, offset, lrc[dw]);
244 err = -EINVAL;
252 dw += 2;
253 lri -= 2;
258 pr_info("%s: HW register image:\n", engine->name);
261 pr_info("%s: SW register image:\n", engine->name);
265 shmem_unpin_map(engine->default_state, hw);
282 return -1;
304 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
305 CTX_RING_START - 1,
309 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
310 CTX_RING_CTL - 1,
314 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
315 CTX_RING_HEAD - 1,
319 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
320 CTX_RING_TAIL - 1,
324 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
329 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
330 CTX_BB_STATE - 1,
334 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
339 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
344 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
349 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
350 CTX_TIMESTAMP - 1,
354 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
359 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
364 i915_mmio_reg_offset(RING_BB_OFFSET(engine->mmio_base)),
372 if (!engine->default_state)
375 hw = shmem_pin_map(engine->default_state);
377 err = -ENOMEM;
382 for (t = tbl; t->name; t++) {
383 int dw = find_offset(hw, t->reg);
385 if (dw != t->offset) {
387 engine->name,
388 t->name,
389 t->reg,
391 t->offset);
392 err = -EINVAL;
396 shmem_unpin_map(engine->default_state, hw);
414 u32 *cs;
424 err = i915_gem_object_lock(scratch->obj, &ww);
436 cs = intel_ring_begin(rq, 4 * MAX_IDX);
437 if (IS_ERR(cs)) {
438 err = PTR_ERR(cs);
443 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
444 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
445 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
446 *cs++ = 0;
448 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
450 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
451 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
452 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
453 *cs++ = 0;
463 expected[RING_TAIL_IDX] = ce->ring->tail;
466 err = -ETIME;
470 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
471 if (IS_ERR(cs)) {
472 err = PTR_ERR(cs);
477 if (cs[n] != expected[n]) {
479 engine->name, n, cs[n], expected[n]);
480 err = -EINVAL;
485 i915_gem_object_unpin_map(scratch->obj);
492 if (err == -EDEADLK) {
525 if (igt_flush_test(gt->i915))
526 err = -EIO;
535 u32 *cs;
542 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
543 if (IS_ERR(cs)) {
545 return PTR_ERR(cs);
548 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
550 *cs++ = CS_GPR(ce->engine, n);
551 *cs++ = STACK_MAGIC;
553 *cs++ = MI_NOOP;
555 intel_ring_advance(rq, cs);
557 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
567 i915_ggtt_offset(ce->engine->status_page.vma) +
570 u32 *cs;
578 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
579 if (IS_ERR(cs)) {
581 return ERR_CAST(cs);
584 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
585 *cs++ = MI_NOOP;
587 *cs++ = MI_SEMAPHORE_WAIT |
591 *cs++ = 0;
592 *cs++ = offset;
593 *cs++ = 0;
596 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
597 *cs++ = CS_GPR(ce->engine, n);
598 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
599 *cs++ = 0;
618 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
621 u32 *cs;
625 if (GRAPHICS_VER(engine->i915) < 9 && engine->class != RENDER_CLASS)
628 err = gpr_make_dirty(engine->kernel_context);
642 err = wait_for_submit(engine, rq, HZ / 2);
647 err = gpr_make_dirty(engine->kernel_context);
651 err = emit_semaphore_signal(engine->kernel_context, slot);
655 err = wait_for_submit(engine, rq, HZ / 2);
664 err = -ETIME;
668 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
669 if (IS_ERR(cs)) {
670 err = PTR_ERR(cs);
675 if (cs[n]) {
677 engine->name,
678 n / 2, n & 1 ? "udw" : "ldw",
679 cs[n]);
680 err = -EINVAL;
685 i915_gem_object_unpin_map(scratch->obj);
688 memset32(&slot[0], -1, 4);
726 if (igt_flush_test(gt->i915))
727 err = -EIO;
740 i915_ggtt_offset(ce->engine->status_page.vma) +
743 u32 *cs;
750 cs = intel_ring_begin(rq, 10);
751 if (IS_ERR(cs)) {
752 err = PTR_ERR(cs);
756 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
757 *cs++ = MI_NOOP;
759 *cs++ = MI_SEMAPHORE_WAIT |
763 *cs++ = 0;
764 *cs++ = offset;
765 *cs++ = 0;
767 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
768 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
769 *cs++ = offset + idx * sizeof(u32);
770 *cs++ = 0;
772 intel_ring_advance(rq, cs);
788 struct intel_context *ce[2];
794 return (s32)(end - start) > 0;
799 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
804 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
805 rq = create_timestamp(arg->ce[0], slot, 1);
809 err = wait_for_submit(rq->engine, rq, HZ / 2);
814 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
815 err = emit_semaphore_signal(arg->ce[1], slot);
824 err = context_flush(arg->ce[0], HZ / 2);
828 if (!timestamp_advanced(arg->poison, slot[1])) {
830 arg->engine->name, preempt ? "preempt" : "simple",
831 arg->poison, slot[1]);
832 err = -EINVAL;
835 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
838 arg->engine->name, preempt ? "preempt" : "simple",
840 err = -EINVAL;
844 memset32(slot, -1, 4);
915 if (igt_flush_test(gt->i915))
916 err = -EIO;
931 obj = i915_gem_object_create_internal(vm->i915, size);
967 u32 dw, x, *cs, *hw;
970 batch = create_user_vma(ce->vm, SZ_64K);
974 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
975 if (IS_ERR(cs)) {
977 return ERR_CAST(cs);
980 defaults = shmem_pin_map(ce->engine->default_state);
982 i915_gem_object_unpin_map(batch->obj);
984 return ERR_PTR(-ENOMEM);
1015 dw += len + 2;
1021 ce->engine->name);
1027 len = (len + 1) / 2;
1028 while (len--) {
1029 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
1030 *cs++ = hw[dw];
1031 *cs++ = lower_32_bits(i915_vma_offset(scratch) + x);
1032 *cs++ = upper_32_bits(i915_vma_offset(scratch) + x);
1034 dw += 2;
1040 *cs++ = MI_BATCH_BUFFER_END;
1042 shmem_unpin_map(ce->engine->default_state, defaults);
1044 i915_gem_object_flush_map(batch->obj);
1045 i915_gem_object_unpin_map(batch->obj);
1058 u32 *cs;
1091 cs = intel_ring_begin(rq, 14);
1092 if (IS_ERR(cs)) {
1093 err = PTR_ERR(cs);
1097 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1098 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1099 *cs++ = lower_32_bits(i915_vma_offset(b_before));
1100 *cs++ = upper_32_bits(i915_vma_offset(b_before));
1102 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1103 *cs++ = MI_SEMAPHORE_WAIT |
1107 *cs++ = 0;
1108 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1110 *cs++ = 0;
1111 *cs++ = MI_NOOP;
1113 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1114 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1115 *cs++ = lower_32_bits(i915_vma_offset(b_after));
1116 *cs++ = upper_32_bits(i915_vma_offset(b_after));
1118 intel_ring_advance(rq, cs);
1138 u32 dw, *cs, *hw;
1141 batch = create_user_vma(ce->vm, SZ_64K);
1145 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1146 if (IS_ERR(cs)) {
1148 return ERR_CAST(cs);
1151 defaults = shmem_pin_map(ce->engine->default_state);
1153 i915_gem_object_unpin_map(batch->obj);
1155 return ERR_PTR(-ENOMEM);
1174 dw += len + 2;
1180 ce->engine->name);
1186 len = (len + 1) / 2;
1187 *cs++ = MI_LOAD_REGISTER_IMM(len);
1188 while (len--) {
1189 *cs++ = hw[dw];
1190 *cs++ = safe_poison(hw[dw] & get_lri_mask(ce->engine,
1193 dw += 2;
1198 *cs++ = MI_BATCH_BUFFER_END;
1200 shmem_unpin_map(ce->engine->default_state, defaults);
1202 i915_gem_object_flush_map(batch->obj);
1203 i915_gem_object_unpin_map(batch->obj);
1212 u32 *cs;
1229 cs = intel_ring_begin(rq, 8);
1230 if (IS_ERR(cs)) {
1231 err = PTR_ERR(cs);
1235 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1236 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1237 *cs++ = lower_32_bits(i915_vma_offset(batch));
1238 *cs++ = upper_32_bits(i915_vma_offset(batch));
1240 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1241 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1243 *cs++ = 0;
1244 *cs++ = 1;
1246 intel_ring_advance(rq, cs);
1248 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1262 struct i915_vma *ref[2],
1263 struct i915_vma *result[2],
1268 u32 *A[2], *B[2];
1272 A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1276 A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1282 B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1288 B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1294 lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1295 intel_gt_coherent_map_type(engine->gt,
1296 ce->state->obj,
1304 defaults = shmem_pin_map(ce->engine->default_state);
1306 err = -ENOMEM;
1327 dw += len + 2;
1333 engine->name);
1339 len = (len + 1) / 2;
1340 while (len--) {
1350 engine->name, dw,
1354 err = -EINVAL;
1357 dw += 2;
1363 shmem_unpin_map(ce->engine->default_state, defaults);
1365 i915_gem_object_unpin_map(ce->state->obj);
1367 i915_gem_object_unpin_map(result[1]->obj);
1369 i915_gem_object_unpin_map(result[0]->obj);
1371 i915_gem_object_unpin_map(ref[1]->obj);
1373 i915_gem_object_unpin_map(ref[0]->obj);
1388 ptr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
1394 memset(ptr, POISON_INUSE, vma->size);
1395 i915_gem_object_flush_map(vma->obj);
1396 i915_gem_object_unpin_map(vma->obj);
1403 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1404 struct i915_vma *ref[2], *result[2];
1419 ref[0] = create_result_vma(A->vm, SZ_64K);
1425 ref[1] = create_result_vma(A->vm, SZ_64K);
1440 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1442 err = -ETIME;
1447 result[0] = create_result_vma(A->vm, SZ_64K);
1453 result[1] = create_result_vma(A->vm, SZ_64K);
1466 if (err == 0 && i915_request_wait(rq, 0, HZ / 2) < 0) {
1468 __func__, engine->name);
1469 err = -ETIME;
1473 WRITE_ONCE(*sema, -1);
1497 if (engine->class == COPY_ENGINE_CLASS && GRAPHICS_VER(engine->i915) == 9)
1500 if (engine->class == RENDER_CLASS && GRAPHICS_VER(engine->i915) == 11)
1521 * Our goal is try and verify that per-context state cannot be
1522 * tampered with by another non-privileged client.
1549 if (igt_flush_test(gt->i915)) {
1550 err = -EIO;
1571 err = -ETIME;
1583 u32 *cs, bool per_ctx)
1585 *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1588 *cs++ = i915_mmio_reg_offset(RING_START(0));
1589 *cs++ = i915_ggtt_offset(ce->state) +
1593 *cs++ = 0;
1595 return cs;
1599 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1601 return emit_wabb_ctx_canary(ce, cs, false);
1605 emit_per_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1607 return emit_wabb_ctx_canary(ce, cs, true);
1613 u32 *cs = context_wabb(ce, per_ctx);
1615 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1618 setup_per_ctx_bb(ce, ce->engine, emit_per_ctx_bb_canary);
1620 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1625 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1629 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1634 ce->lrc_reg_state[CTX_RING_START]);
1648 return -EINVAL;
1675 if (!a->wa_bb_page) {
1676 GEM_BUG_ON(b->wa_bb_page);
1677 GEM_BUG_ON(GRAPHICS_VER(engine->i915) == 12);
1721 if (igt_flush_test(gt->i915))
1722 err = -EIO;
1744 const unsigned int bit = I915_RESET_ENGINE + engine->id;
1745 unsigned long *lock = &engine->gt->reset.flags;
1749 tasklet_disable(&engine->sched_engine->tasklet);
1751 if (!rq->fence.error)
1754 tasklet_enable(&engine->sched_engine->tasklet);
1771 ce->lrc_reg_state,
1772 ce->engine->context_size -
1806 if (wait_for_submit(engine, hang, HZ / 2)) {
1808 err = -ETIME;
1816 if (!hang->fence.error) {
1819 engine->name);
1820 err = -EINVAL;
1824 if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1826 engine->name);
1828 err = -EIO;
1856 if (!intel_has_reset_engine(engine->gt))
1867 if (igt_flush_test(gt->i915))
1868 err = -EIO;
1887 ce->stats.runtime.num_underflow = 0;
1888 ce->stats.runtime.max_underflow = 0;
1900 if (--loop == 0)
1914 pr_err("%s: request not completed!\n", engine->name);
1918 igt_flush_test(engine->i915);
1921 engine->name,
1926 if (ce->stats.runtime.num_underflow) {
1928 engine->name,
1929 ce->stats.runtime.num_underflow,
1930 ce->stats.runtime.max_underflow);
1932 err = -EOVERFLOW;
1960 if (igt_flush_test(gt->i915))
1961 err = -EIO;