Lines Matching full:engine
260 * intel_engine_context_size() - return the size of the context for an engine
262 * @class: engine class
264 * Each engine class may require a different amount of space for a context
267 * Return: size (in bytes) of an engine class specific context image
358 static void __sprint_engine_name(struct intel_engine_cs *engine) in __sprint_engine_name() argument
361 * Before we know what the uABI name for this engine will be, in __sprint_engine_name()
362 * we still would like to keep track of this engine in the debug logs. in __sprint_engine_name()
365 GEM_WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s'%u", in __sprint_engine_name()
366 intel_engine_class_repr(engine->class), in __sprint_engine_name()
367 engine->instance) >= sizeof(engine->name)); in __sprint_engine_name()
370 void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask) in intel_engine_set_hwsp_writemask() argument
374 * per-engine HWSTAM until gen6. in intel_engine_set_hwsp_writemask()
376 if (GRAPHICS_VER(engine->i915) < 6 && engine->class != RENDER_CLASS) in intel_engine_set_hwsp_writemask()
379 if (GRAPHICS_VER(engine->i915) >= 3) in intel_engine_set_hwsp_writemask()
380 ENGINE_WRITE(engine, RING_HWSTAM, mask); in intel_engine_set_hwsp_writemask()
382 ENGINE_WRITE16(engine, RING_HWSTAM, mask); in intel_engine_set_hwsp_writemask()
385 static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine) in intel_engine_sanitize_mmio() argument
388 intel_engine_set_hwsp_writemask(engine, ~0u); in intel_engine_sanitize_mmio()
391 static void nop_irq_handler(struct intel_engine_cs *engine, u16 iir) in nop_irq_handler() argument
454 struct intel_engine_cs *engine; in intel_engine_setup() local
462 if (GEM_DEBUG_WARN_ON(id >= ARRAY_SIZE(gt->engine))) in intel_engine_setup()
474 engine = kzalloc(sizeof(*engine), GFP_KERNEL); in intel_engine_setup()
475 if (!engine) in intel_engine_setup()
478 BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES); in intel_engine_setup()
480 INIT_LIST_HEAD(&engine->pinned_contexts_list); in intel_engine_setup()
481 engine->id = id; in intel_engine_setup()
482 engine->legacy_idx = INVALID_ENGINE; in intel_engine_setup()
483 engine->mask = BIT(id); in intel_engine_setup()
484 engine->reset_domain = get_reset_domain(GRAPHICS_VER(gt->i915), in intel_engine_setup()
486 engine->i915 = i915; in intel_engine_setup()
487 engine->gt = gt; in intel_engine_setup()
488 engine->uncore = gt->uncore; in intel_engine_setup()
490 engine->guc_id = MAKE_GUC_ID(guc_class, info->instance); in intel_engine_setup()
491 engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases); in intel_engine_setup()
493 engine->irq_handler = nop_irq_handler; in intel_engine_setup()
495 engine->class = info->class; in intel_engine_setup()
496 engine->instance = info->instance; in intel_engine_setup()
497 engine->logical_mask = BIT(logical_instance); in intel_engine_setup()
498 __sprint_engine_name(engine); in intel_engine_setup()
500 if ((engine->class == COMPUTE_CLASS || engine->class == RENDER_CLASS) && in intel_engine_setup()
501 __ffs(CCS_MASK(engine->gt) | RCS_MASK(engine->gt)) == engine->instance) in intel_engine_setup()
502 engine->flags |= I915_ENGINE_FIRST_RENDER_COMPUTE; in intel_engine_setup()
505 if (engine->class == RENDER_CLASS || engine->class == COMPUTE_CLASS) { in intel_engine_setup()
506 engine->flags |= I915_ENGINE_HAS_RCS_REG_STATE; in intel_engine_setup()
507 engine->flags |= I915_ENGINE_HAS_EU_PRIORITY; in intel_engine_setup()
510 engine->props.heartbeat_interval_ms = in intel_engine_setup()
512 engine->props.max_busywait_duration_ns = in intel_engine_setup()
514 engine->props.preempt_timeout_ms = in intel_engine_setup()
516 engine->props.stop_timeout_ms = in intel_engine_setup()
518 engine->props.timeslice_duration_ms = in intel_engine_setup()
527 if (GRAPHICS_VER(i915) == 12 && (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE)) in intel_engine_setup()
528 engine->props.preempt_timeout_ms = CONFIG_DRM_I915_PREEMPT_TIMEOUT_COMPUTE; in intel_engine_setup()
533 u64 clamp = intel_clamp_##field(engine, engine->props.field); \ in intel_engine_setup()
534 if (clamp != engine->props.field) { \ in intel_engine_setup()
535 drm_notice(&engine->i915->drm, \ in intel_engine_setup()
538 engine->props.field = clamp; \ in intel_engine_setup()
550 engine->defaults = engine->props; /* never to change again */ in intel_engine_setup()
552 engine->context_size = intel_engine_context_size(gt, engine->class); in intel_engine_setup()
553 if (WARN_ON(engine->context_size > BIT(20))) in intel_engine_setup()
554 engine->context_size = 0; in intel_engine_setup()
555 if (engine->context_size) in intel_engine_setup()
558 ewma__engine_latency_init(&engine->latency); in intel_engine_setup()
560 ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier); in intel_engine_setup()
563 intel_engine_sanitize_mmio(engine); in intel_engine_setup()
565 gt->engine_class[info->class][info->instance] = engine; in intel_engine_setup()
566 gt->engine[id] = engine; in intel_engine_setup()
571 u64 intel_clamp_heartbeat_interval_ms(struct intel_engine_cs *engine, u64 value) in intel_clamp_heartbeat_interval_ms() argument
578 u64 intel_clamp_max_busywait_duration_ns(struct intel_engine_cs *engine, u64 value) in intel_clamp_max_busywait_duration_ns() argument
585 u64 intel_clamp_preempt_timeout_ms(struct intel_engine_cs *engine, u64 value) in intel_clamp_preempt_timeout_ms() argument
591 if (intel_guc_submission_is_wanted(gt_to_guc(engine->gt))) in intel_clamp_preempt_timeout_ms()
599 u64 intel_clamp_stop_timeout_ms(struct intel_engine_cs *engine, u64 value) in intel_clamp_stop_timeout_ms() argument
606 u64 intel_clamp_timeslice_duration_ms(struct intel_engine_cs *engine, u64 value) in intel_clamp_timeslice_duration_ms() argument
612 if (intel_guc_submission_is_wanted(gt_to_guc(engine->gt))) in intel_clamp_timeslice_duration_ms()
620 static void __setup_engine_capabilities(struct intel_engine_cs *engine) in __setup_engine_capabilities() argument
622 struct drm_i915_private *i915 = engine->i915; in __setup_engine_capabilities()
624 if (engine->class == VIDEO_DECODE_CLASS) { in __setup_engine_capabilities()
626 * HEVC support is present on first engine instance in __setup_engine_capabilities()
630 (GRAPHICS_VER(i915) >= 9 && engine->instance == 0)) in __setup_engine_capabilities()
631 engine->uabi_capabilities |= in __setup_engine_capabilities()
635 * SFC block is present only on even logical engine in __setup_engine_capabilities()
639 (engine->gt->info.vdbox_sfc_access & in __setup_engine_capabilities()
640 BIT(engine->instance))) || in __setup_engine_capabilities()
641 (GRAPHICS_VER(i915) >= 9 && engine->instance == 0)) in __setup_engine_capabilities()
642 engine->uabi_capabilities |= in __setup_engine_capabilities()
644 } else if (engine->class == VIDEO_ENHANCEMENT_CLASS) { in __setup_engine_capabilities()
646 engine->gt->info.sfc_mask & BIT(engine->instance)) in __setup_engine_capabilities()
647 engine->uabi_capabilities |= in __setup_engine_capabilities()
654 struct intel_engine_cs *engine; in intel_setup_engine_capabilities() local
657 for_each_engine(engine, gt, id) in intel_setup_engine_capabilities()
658 __setup_engine_capabilities(engine); in intel_setup_engine_capabilities()
667 struct intel_engine_cs *engine; in intel_engines_release() local
671 * Before we release the resources held by engine, we must be certain in intel_engines_release()
684 for_each_engine(engine, gt, id) { in intel_engines_release()
685 if (!engine->release) in intel_engines_release()
688 intel_wakeref_wait_for_idle(&engine->wakeref); in intel_engines_release()
689 GEM_BUG_ON(intel_engine_pm_is_awake(engine)); in intel_engines_release()
691 engine->release(engine); in intel_engines_release()
692 engine->release = NULL; in intel_engines_release()
694 memset(&engine->reset, 0, sizeof(engine->reset)); in intel_engines_release()
700 void intel_engine_free_request_pool(struct intel_engine_cs *engine) in intel_engine_free_request_pool() argument
702 if (!engine->request_pool) in intel_engine_free_request_pool()
705 kmem_cache_free(i915_request_slab_cache(), engine->request_pool); in intel_engine_free_request_pool()
710 struct intel_engine_cs *engine; in intel_engines_free() local
716 for_each_engine(engine, gt, id) { in intel_engines_free()
717 intel_engine_free_request_pool(engine); in intel_engines_free()
718 kfree(engine); in intel_engines_free()
719 gt->engine[id] = NULL; in intel_engines_free()
835 * engine is not available for use. in engine_mask_apply_compute_fuses()
846 * the blitter forcewake domain to read the engine fuses, but at the same time
849 * domains based on the full engine mask in the platform capabilities before
881 * All the workload submitted to the first engine will be shared among in init_engine_mask()
892 * changing the CCS engine configuration in init_engine_mask()
896 /* Mask off all the CCS engine */ in init_engine_mask()
898 /* Put back in the first CCS engine */ in init_engine_mask()
949 * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
1012 void intel_engine_init_execlists(struct intel_engine_cs *engine) in intel_engine_init_execlists() argument
1014 struct intel_engine_execlists * const execlists = &engine->execlists; in intel_engine_init_execlists()
1025 static void cleanup_status_page(struct intel_engine_cs *engine) in cleanup_status_page() argument
1030 intel_engine_set_hwsp_writemask(engine, ~0u); in cleanup_status_page()
1032 vma = fetch_and_zero(&engine->status_page.vma); in cleanup_status_page()
1036 if (!HWS_NEEDS_PHYSICAL(engine->i915)) in cleanup_status_page()
1043 static int pin_ggtt_status_page(struct intel_engine_cs *engine, in pin_ggtt_status_page() argument
1049 if (!HAS_LLC(engine->i915) && i915_ggtt_has_aperture(engine->gt->ggtt)) in pin_ggtt_status_page()
1068 static int init_status_page(struct intel_engine_cs *engine) in init_status_page() argument
1076 INIT_LIST_HEAD(&engine->status_page.timelines); in init_status_page()
1085 obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE); in init_status_page()
1087 gt_err(engine->gt, "Failed to allocate status page\n"); in init_status_page()
1093 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); in init_status_page()
1102 if (!ret && !HWS_NEEDS_PHYSICAL(engine->i915)) in init_status_page()
1103 ret = pin_ggtt_status_page(engine, &ww, vma); in init_status_page()
1113 engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE); in init_status_page()
1114 engine->status_page.vma = vma; in init_status_page()
1132 static int intel_engine_init_tlb_invalidation(struct intel_engine_cs *engine) in intel_engine_init_tlb_invalidation() argument
1159 struct drm_i915_private *i915 = engine->i915; in intel_engine_init_tlb_invalidation()
1160 const unsigned int instance = engine->instance; in intel_engine_init_tlb_invalidation()
1161 const unsigned int class = engine->class; in intel_engine_init_tlb_invalidation()
1174 * respective engine registers were moved to masked type. Then after the in intel_engine_init_tlb_invalidation()
1178 if (engine->gt->type == GT_MEDIA) { in intel_engine_init_tlb_invalidation()
1202 if (gt_WARN_ONCE(engine->gt, !num, in intel_engine_init_tlb_invalidation()
1206 if (gt_WARN_ON_ONCE(engine->gt, in intel_engine_init_tlb_invalidation()
1230 engine->tlb_inv.mcr = regs == xehp_regs; in intel_engine_init_tlb_invalidation()
1231 engine->tlb_inv.reg = reg; in intel_engine_init_tlb_invalidation()
1232 engine->tlb_inv.done = val; in intel_engine_init_tlb_invalidation()
1235 (engine->class == VIDEO_DECODE_CLASS || in intel_engine_init_tlb_invalidation()
1236 engine->class == VIDEO_ENHANCEMENT_CLASS || in intel_engine_init_tlb_invalidation()
1237 engine->class == COMPUTE_CLASS || in intel_engine_init_tlb_invalidation()
1238 engine->class == OTHER_CLASS)) in intel_engine_init_tlb_invalidation()
1239 engine->tlb_inv.request = _MASKED_BIT_ENABLE(val); in intel_engine_init_tlb_invalidation()
1241 engine->tlb_inv.request = val; in intel_engine_init_tlb_invalidation()
1246 static int engine_setup_common(struct intel_engine_cs *engine) in engine_setup_common() argument
1250 init_llist_head(&engine->barrier_tasks); in engine_setup_common()
1252 err = intel_engine_init_tlb_invalidation(engine); in engine_setup_common()
1256 err = init_status_page(engine); in engine_setup_common()
1260 engine->breadcrumbs = intel_breadcrumbs_create(engine); in engine_setup_common()
1261 if (!engine->breadcrumbs) { in engine_setup_common()
1266 engine->sched_engine = i915_sched_engine_create(ENGINE_PHYSICAL); in engine_setup_common()
1267 if (!engine->sched_engine) { in engine_setup_common()
1271 engine->sched_engine->private_data = engine; in engine_setup_common()
1273 err = intel_engine_init_cmd_parser(engine); in engine_setup_common()
1277 intel_engine_init_execlists(engine); in engine_setup_common()
1278 intel_engine_init__pm(engine); in engine_setup_common()
1279 intel_engine_init_retire(engine); in engine_setup_common()
1282 engine->sseu = in engine_setup_common()
1283 intel_sseu_from_device_info(&engine->gt->info.sseu); in engine_setup_common()
1285 intel_engine_init_workarounds(engine); in engine_setup_common()
1286 intel_engine_init_whitelist(engine); in engine_setup_common()
1287 intel_engine_init_ctx_wa(engine); in engine_setup_common()
1289 if (GRAPHICS_VER(engine->i915) >= 12) in engine_setup_common()
1290 engine->flags |= I915_ENGINE_HAS_RELATIVE_MMIO; in engine_setup_common()
1295 i915_sched_engine_put(engine->sched_engine); in engine_setup_common()
1297 intel_breadcrumbs_put(engine->breadcrumbs); in engine_setup_common()
1299 cleanup_status_page(engine); in engine_setup_common()
1311 struct intel_engine_cs *engine = ce->engine; in measure_breadcrumb_dw() local
1315 GEM_BUG_ON(!engine->gt->scratch); in measure_breadcrumb_dw()
1321 frame->rq.i915 = engine->i915; in measure_breadcrumb_dw()
1322 frame->rq.engine = engine; in measure_breadcrumb_dw()
1336 spin_lock_irq(&engine->sched_engine->lock); in measure_breadcrumb_dw()
1338 dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs; in measure_breadcrumb_dw()
1340 spin_unlock_irq(&engine->sched_engine->lock); in measure_breadcrumb_dw()
1350 intel_engine_create_pinned_context(struct intel_engine_cs *engine, in intel_engine_create_pinned_context() argument
1360 ce = intel_context_create(engine); in intel_engine_create_pinned_context()
1378 list_add_tail(&ce->pinned_contexts_link, &engine->pinned_contexts_list); in intel_engine_create_pinned_context()
1393 struct intel_engine_cs *engine = ce->engine; in intel_engine_destroy_pinned_context() local
1394 struct i915_vma *hwsp = engine->status_page.vma; in intel_engine_destroy_pinned_context()
1408 create_ggtt_bind_context(struct intel_engine_cs *engine) in create_ggtt_bind_context() argument
1416 return intel_engine_create_pinned_context(engine, engine->gt->vm, SZ_512K, in create_ggtt_bind_context()
1422 create_kernel_context(struct intel_engine_cs *engine) in create_kernel_context() argument
1426 return intel_engine_create_pinned_context(engine, engine->gt->vm, SZ_4K, in create_kernel_context()
1432 * engine_init_common - initialize engine state which might require hw access
1433 * @engine: Engine to initialize.
1435 * Initializes @engine@ structure members shared between legacy and execlists
1438 * Typcally done at later stages of submission mode specific engine setup.
1442 static int engine_init_common(struct intel_engine_cs *engine) in engine_init_common() argument
1447 engine->set_default_submission(engine); in engine_init_common()
1457 ce = create_kernel_context(engine); in engine_init_common()
1461 * Create a separate pinned context for GGTT update with blitter engine in engine_init_common()
1463 * engines as well but BCS should be less busy engine so pick that for in engine_init_common()
1466 if (i915_ggtt_require_binder(engine->i915) && engine->id == BCS0) { in engine_init_common()
1467 bce = create_ggtt_bind_context(engine); in engine_init_common()
1478 engine->emit_fini_breadcrumb_dw = ret; in engine_init_common()
1479 engine->kernel_context = ce; in engine_init_common()
1480 engine->bind_context = bce; in engine_init_common()
1494 int (*setup)(struct intel_engine_cs *engine); in intel_engines_init()
1495 struct intel_engine_cs *engine; in intel_engines_init() local
1510 for_each_engine(engine, gt, id) { in intel_engines_init()
1511 err = engine_setup_common(engine); in intel_engines_init()
1515 err = setup(engine); in intel_engines_init()
1517 intel_engine_cleanup_common(engine); in intel_engines_init()
1522 GEM_BUG_ON(engine->release == NULL); in intel_engines_init()
1524 err = engine_init_common(engine); in intel_engines_init()
1528 intel_engine_add_user(engine); in intel_engines_init()
1535 * intel_engine_cleanup_common - cleans up the engine state created by
1537 * @engine: Engine to cleanup.
1541 void intel_engine_cleanup_common(struct intel_engine_cs *engine) in intel_engine_cleanup_common() argument
1543 GEM_BUG_ON(!list_empty(&engine->sched_engine->requests)); in intel_engine_cleanup_common()
1545 i915_sched_engine_put(engine->sched_engine); in intel_engine_cleanup_common()
1546 intel_breadcrumbs_put(engine->breadcrumbs); in intel_engine_cleanup_common()
1548 intel_engine_fini_retire(engine); in intel_engine_cleanup_common()
1549 intel_engine_cleanup_cmd_parser(engine); in intel_engine_cleanup_common()
1551 if (engine->default_state) in intel_engine_cleanup_common()
1552 fput(engine->default_state); in intel_engine_cleanup_common()
1554 if (engine->kernel_context) in intel_engine_cleanup_common()
1555 intel_engine_destroy_pinned_context(engine->kernel_context); in intel_engine_cleanup_common()
1557 if (engine->bind_context) in intel_engine_cleanup_common()
1558 intel_engine_destroy_pinned_context(engine->bind_context); in intel_engine_cleanup_common()
1561 GEM_BUG_ON(!llist_empty(&engine->barrier_tasks)); in intel_engine_cleanup_common()
1562 cleanup_status_page(engine); in intel_engine_cleanup_common()
1564 intel_wa_list_free(&engine->ctx_wa_list); in intel_engine_cleanup_common()
1565 intel_wa_list_free(&engine->wa_list); in intel_engine_cleanup_common()
1566 intel_wa_list_free(&engine->whitelist); in intel_engine_cleanup_common()
1570 * intel_engine_resume - re-initializes the HW state of the engine
1571 * @engine: Engine to resume.
1575 int intel_engine_resume(struct intel_engine_cs *engine) in intel_engine_resume() argument
1577 intel_engine_apply_workarounds(engine); in intel_engine_resume()
1578 intel_engine_apply_whitelist(engine); in intel_engine_resume()
1580 return engine->resume(engine); in intel_engine_resume()
1583 u64 intel_engine_get_active_head(const struct intel_engine_cs *engine) in intel_engine_get_active_head() argument
1585 struct drm_i915_private *i915 = engine->i915; in intel_engine_get_active_head()
1590 acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW); in intel_engine_get_active_head()
1592 acthd = ENGINE_READ(engine, RING_ACTHD); in intel_engine_get_active_head()
1594 acthd = ENGINE_READ(engine, ACTHD); in intel_engine_get_active_head()
1599 u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine) in intel_engine_get_last_batch_head() argument
1603 if (GRAPHICS_VER(engine->i915) >= 8) in intel_engine_get_last_batch_head()
1604 bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW); in intel_engine_get_last_batch_head()
1606 bbaddr = ENGINE_READ(engine, RING_BBADDR); in intel_engine_get_last_batch_head()
1611 static unsigned long stop_timeout(const struct intel_engine_cs *engine) in stop_timeout() argument
1618 * the engine to quiesce. We've stopped submission to the engine, and in stop_timeout()
1620 * leave the engine idle. So they should not be caught unaware by in stop_timeout()
1623 return READ_ONCE(engine->props.stop_timeout_ms); in stop_timeout()
1626 static int __intel_engine_stop_cs(struct intel_engine_cs *engine, in __intel_engine_stop_cs() argument
1630 struct intel_uncore *uncore = engine->uncore; in __intel_engine_stop_cs()
1631 const i915_reg_t mode = RING_MI_MODE(engine->mmio_base); in __intel_engine_stop_cs()
1640 if (intel_engine_reset_needs_wa_22011802037(engine->gt)) in __intel_engine_stop_cs()
1641 intel_uncore_write_fw(uncore, RING_MODE_GEN7(engine->mmio_base), in __intel_engine_stop_cs()
1644 err = __intel_wait_for_register_fw(engine->uncore, mode, in __intel_engine_stop_cs()
1655 int intel_engine_stop_cs(struct intel_engine_cs *engine) in intel_engine_stop_cs() argument
1659 if (GRAPHICS_VER(engine->i915) < 3) in intel_engine_stop_cs()
1662 ENGINE_TRACE(engine, "\n"); in intel_engine_stop_cs()
1675 if (__intel_engine_stop_cs(engine, 1000, stop_timeout(engine))) { in intel_engine_stop_cs()
1676 ENGINE_TRACE(engine, in intel_engine_stop_cs()
1678 ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR, in intel_engine_stop_cs()
1679 ENGINE_READ_FW(engine, RING_TAIL) & TAIL_ADDR); in intel_engine_stop_cs()
1686 if ((ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) != in intel_engine_stop_cs()
1687 (ENGINE_READ_FW(engine, RING_TAIL) & TAIL_ADDR)) in intel_engine_stop_cs()
1694 void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine) in intel_engine_cancel_stop_cs() argument
1696 ENGINE_TRACE(engine, "\n"); in intel_engine_cancel_stop_cs()
1698 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); in intel_engine_cancel_stop_cs()
1701 static u32 __cs_pending_mi_force_wakes(struct intel_engine_cs *engine) in __cs_pending_mi_force_wakes() argument
1725 if (!_reg[engine->id].reg) in __cs_pending_mi_force_wakes()
1728 val = intel_uncore_read(engine->uncore, _reg[engine->id]); in __cs_pending_mi_force_wakes()
1762 void intel_engine_wait_for_pending_mi_fw(struct intel_engine_cs *engine) in intel_engine_wait_for_pending_mi_fw() argument
1764 u32 fw_pending = __cs_pending_mi_force_wakes(engine); in intel_engine_wait_for_pending_mi_fw()
1767 __gpm_wait_for_fw_complete(engine->gt, fw_pending); in intel_engine_wait_for_pending_mi_fw()
1771 void intel_engine_get_instdone(const struct intel_engine_cs *engine, in intel_engine_get_instdone() argument
1774 struct drm_i915_private *i915 = engine->i915; in intel_engine_get_instdone()
1775 struct intel_uncore *uncore = engine->uncore; in intel_engine_get_instdone()
1776 u32 mmio_base = engine->mmio_base; in intel_engine_get_instdone()
1787 if (engine->id != RCS0) in intel_engine_get_instdone()
1799 for_each_ss_steering(iter, engine->gt, slice, subslice) { in intel_engine_get_instdone()
1801 intel_gt_mcr_read(engine->gt, in intel_engine_get_instdone()
1805 intel_gt_mcr_read(engine->gt, in intel_engine_get_instdone()
1811 for_each_ss_steering(iter, engine->gt, slice, subslice) in intel_engine_get_instdone()
1813 intel_gt_mcr_read(engine->gt, in intel_engine_get_instdone()
1821 if (engine->id != RCS0) in intel_engine_get_instdone()
1833 if (engine->id == RCS0) in intel_engine_get_instdone()
1842 static bool ring_is_idle(struct intel_engine_cs *engine) in ring_is_idle() argument
1846 if (I915_SELFTEST_ONLY(!engine->mmio_base)) in ring_is_idle()
1849 if (!intel_engine_pm_get_if_awake(engine)) in ring_is_idle()
1853 if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) != in ring_is_idle()
1854 (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR)) in ring_is_idle()
1858 if (GRAPHICS_VER(engine->i915) > 2 && in ring_is_idle()
1859 !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE)) in ring_is_idle()
1862 intel_engine_pm_put(engine); in ring_is_idle()
1867 void __intel_engine_flush_submission(struct intel_engine_cs *engine, bool sync) in __intel_engine_flush_submission() argument
1869 struct tasklet_struct *t = &engine->sched_engine->tasklet; in __intel_engine_flush_submission()
1889 * intel_engine_is_idle() - Report if the engine has finished process all work
1890 * @engine: the intel_engine_cs
1893 * to hardware, and that the engine is idle.
1895 bool intel_engine_is_idle(struct intel_engine_cs *engine) in intel_engine_is_idle() argument
1898 if (intel_gt_is_wedged(engine->gt)) in intel_engine_is_idle()
1901 if (!intel_engine_pm_is_awake(engine)) in intel_engine_is_idle()
1905 intel_synchronize_hardirq(engine->i915); in intel_engine_is_idle()
1906 intel_engine_flush_submission(engine); in intel_engine_is_idle()
1909 if (!i915_sched_engine_is_empty(engine->sched_engine)) in intel_engine_is_idle()
1913 return ring_is_idle(engine); in intel_engine_is_idle()
1918 struct intel_engine_cs *engine; in intel_engines_are_idle() local
1932 for_each_engine(engine, gt, id) { in intel_engines_are_idle()
1933 if (!intel_engine_is_idle(engine)) in intel_engines_are_idle()
1940 bool intel_engine_irq_enable(struct intel_engine_cs *engine) in intel_engine_irq_enable() argument
1942 if (!engine->irq_enable) in intel_engine_irq_enable()
1946 spin_lock(engine->gt->irq_lock); in intel_engine_irq_enable()
1947 engine->irq_enable(engine); in intel_engine_irq_enable()
1948 spin_unlock(engine->gt->irq_lock); in intel_engine_irq_enable()
1953 void intel_engine_irq_disable(struct intel_engine_cs *engine) in intel_engine_irq_disable() argument
1955 if (!engine->irq_disable) in intel_engine_irq_disable()
1959 spin_lock(engine->gt->irq_lock); in intel_engine_irq_disable()
1960 engine->irq_disable(engine); in intel_engine_irq_disable()
1961 spin_unlock(engine->gt->irq_lock); in intel_engine_irq_disable()
1966 struct intel_engine_cs *engine; in intel_engines_reset_default_submission() local
1969 for_each_engine(engine, gt, id) { in intel_engines_reset_default_submission()
1970 if (engine->sanitize) in intel_engines_reset_default_submission()
1971 engine->sanitize(engine); in intel_engines_reset_default_submission()
1973 engine->set_default_submission(engine); in intel_engines_reset_default_submission()
1977 bool intel_engine_can_store_dword(struct intel_engine_cs *engine) in intel_engine_can_store_dword() argument
1979 switch (GRAPHICS_VER(engine->i915)) { in intel_engine_can_store_dword()
1984 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915)); in intel_engine_can_store_dword()
1986 return !IS_I965G(engine->i915); /* who knows! */ in intel_engine_can_store_dword()
1988 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */ in intel_engine_can_store_dword()
1999 * Even though we are holding the engine->sched_engine->lock here, there in get_timeline()
2077 static void intel_engine_print_registers(struct intel_engine_cs *engine, in intel_engine_print_registers() argument
2080 struct drm_i915_private *i915 = engine->i915; in intel_engine_print_registers()
2081 struct intel_engine_execlists * const execlists = &engine->execlists; in intel_engine_print_registers()
2084 if (engine->id == RENDER_CLASS && IS_GRAPHICS_VER(i915, 4, 7)) in intel_engine_print_registers()
2085 drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID)); in intel_engine_print_registers()
2088 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI)); in intel_engine_print_registers()
2090 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO)); in intel_engine_print_registers()
2093 ENGINE_READ(engine, RING_START)); in intel_engine_print_registers()
2095 ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR); in intel_engine_print_registers()
2097 ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR); in intel_engine_print_registers()
2099 ENGINE_READ(engine, RING_CTL), in intel_engine_print_registers()
2100 ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : ""); in intel_engine_print_registers()
2101 if (GRAPHICS_VER(engine->i915) > 2) { in intel_engine_print_registers()
2103 ENGINE_READ(engine, RING_MI_MODE), in intel_engine_print_registers()
2104 ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : ""); in intel_engine_print_registers()
2109 ENGINE_READ(engine, RING_IMR)); in intel_engine_print_registers()
2111 ENGINE_READ(engine, RING_ESR)); in intel_engine_print_registers()
2113 ENGINE_READ(engine, RING_EMR)); in intel_engine_print_registers()
2115 ENGINE_READ(engine, RING_EIR)); in intel_engine_print_registers()
2118 addr = intel_engine_get_active_head(engine); in intel_engine_print_registers()
2121 addr = intel_engine_get_last_batch_head(engine); in intel_engine_print_registers()
2125 addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW); in intel_engine_print_registers()
2127 addr = ENGINE_READ(engine, RING_DMA_FADD); in intel_engine_print_registers()
2129 addr = ENGINE_READ(engine, DMA_FADD_I8XX); in intel_engine_print_registers()
2134 ENGINE_READ(engine, RING_IPEIR)); in intel_engine_print_registers()
2136 ENGINE_READ(engine, RING_IPEHR)); in intel_engine_print_registers()
2138 drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR)); in intel_engine_print_registers()
2139 drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR)); in intel_engine_print_registers()
2142 if (HAS_EXECLISTS(i915) && !intel_engine_uses_guc(engine)) { in intel_engine_print_registers()
2145 &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX]; in intel_engine_print_registers()
2151 str_yes_no(test_bit(TASKLET_STATE_SCHED, &engine->sched_engine->tasklet.state)), in intel_engine_print_registers()
2152 str_enabled_disabled(!atomic_read(&engine->sched_engine->tasklet.count)), in intel_engine_print_registers()
2153 repr_timer(&engine->execlists.preempt), in intel_engine_print_registers()
2154 repr_timer(&engine->execlists.timer)); in intel_engine_print_registers()
2160 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO), in intel_engine_print_registers()
2161 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI), in intel_engine_print_registers()
2176 i915_sched_engine_active_lock_bh(engine->sched_engine); in intel_engine_print_registers()
2207 i915_sched_engine_active_unlock_bh(engine->sched_engine); in intel_engine_print_registers()
2210 ENGINE_READ(engine, RING_PP_DIR_BASE)); in intel_engine_print_registers()
2212 ENGINE_READ(engine, RING_PP_DIR_BASE_READ)); in intel_engine_print_registers()
2214 ENGINE_READ(engine, RING_PP_DIR_DCLV)); in intel_engine_print_registers()
2257 static void print_properties(struct intel_engine_cs *engine, in print_properties() argument
2265 .offset = offsetof(typeof(engine->props), x), \ in print_properties()
2283 read_ul(&engine->props, p->offset), in print_properties()
2284 read_ul(&engine->defaults, p->offset)); in print_properties()
2335 msg = "\t\tactive on engine"; in intel_engine_dump_active_requests()
2343 static void engine_dump_active_requests(struct intel_engine_cs *engine, in engine_dump_active_requests() argument
2350 * No need for an engine->irq_seqno_barrier() before the seqno reads. in engine_dump_active_requests()
2356 intel_engine_get_hung_entity(engine, &hung_ce, &hung_rq); in engine_dump_active_requests()
2365 if (intel_uc_uses_guc_submission(&engine->gt->uc)) in engine_dump_active_requests()
2366 intel_guc_dump_active_requests(engine, hung_rq, m); in engine_dump_active_requests()
2368 intel_execlists_dump_active_requests(engine, hung_rq, m); in engine_dump_active_requests()
2374 void intel_engine_dump(struct intel_engine_cs *engine, in intel_engine_dump() argument
2378 struct i915_gpu_error * const error = &engine->i915->gpu_error; in intel_engine_dump()
2391 if (intel_gt_is_wedged(engine->gt)) in intel_engine_dump()
2394 drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count)); in intel_engine_dump()
2396 str_yes_no(!llist_empty(&engine->barrier_tasks))); in intel_engine_dump()
2398 ewma__engine_latency_read(&engine->latency)); in intel_engine_dump()
2399 if (intel_engine_supports_stats(engine)) in intel_engine_dump()
2401 ktime_to_ms(intel_engine_get_busy_time(engine, in intel_engine_dump()
2404 engine->fw_domain, READ_ONCE(engine->fw_active)); in intel_engine_dump()
2407 rq = READ_ONCE(engine->heartbeat.systole); in intel_engine_dump()
2413 i915_reset_engine_count(error, engine), in intel_engine_dump()
2415 print_properties(engine, m); in intel_engine_dump()
2417 engine_dump_active_requests(engine, m); in intel_engine_dump()
2419 drm_printf(m, "\tMMIO base: 0x%08x\n", engine->mmio_base); in intel_engine_dump()
2420 wakeref = intel_runtime_pm_get_if_in_use(engine->uncore->rpm); in intel_engine_dump()
2422 intel_engine_print_registers(engine, m); in intel_engine_dump()
2423 intel_runtime_pm_put(engine->uncore->rpm, wakeref); in intel_engine_dump()
2428 intel_execlists_show_requests(engine, m, i915_request_show, 8); in intel_engine_dump()
2431 hexdump(m, engine->status_page.addr, PAGE_SIZE); in intel_engine_dump()
2433 drm_printf(m, "Idle? %s\n", str_yes_no(intel_engine_is_idle(engine))); in intel_engine_dump()
2435 intel_engine_print_breadcrumbs(engine, m); in intel_engine_dump()
2439 * intel_engine_get_busy_time() - Return current accumulated engine busyness
2440 * @engine: engine to report on
2443 * Returns accumulated time @engine was busy since engine stats were enabled.
2445 ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine, ktime_t *now) in intel_engine_get_busy_time() argument
2447 return engine->busyness(engine, now); in intel_engine_get_busy_time()
2464 static struct i915_request *engine_execlist_find_hung_request(struct intel_engine_cs *engine) in engine_execlist_find_hung_request() argument
2473 GEM_BUG_ON(intel_uc_uses_guc_submission(&engine->gt->uc)); in engine_execlist_find_hung_request()
2476 * We are called by the error capture, reset and to dump engine in engine_execlist_find_hung_request()
2482 * not need an engine->irq_seqno_barrier() before the seqno reads. in engine_execlist_find_hung_request()
2486 lockdep_assert_held(&engine->sched_engine->lock); in engine_execlist_find_hung_request()
2489 request = execlists_active(&engine->execlists); in engine_execlist_find_hung_request()
2504 list_for_each_entry(request, &engine->sched_engine->requests, in engine_execlist_find_hung_request()
2516 void intel_engine_get_hung_entity(struct intel_engine_cs *engine, in intel_engine_get_hung_entity() argument
2521 *ce = intel_engine_get_hung_context(engine); in intel_engine_get_hung_entity()
2523 intel_engine_clear_hung_context(engine); in intel_engine_get_hung_entity()
2533 if (intel_uc_uses_guc_submission(&engine->gt->uc)) in intel_engine_get_hung_entity()
2536 spin_lock_irqsave(&engine->sched_engine->lock, flags); in intel_engine_get_hung_entity()
2537 *rq = engine_execlist_find_hung_request(engine); in intel_engine_get_hung_entity()
2540 spin_unlock_irqrestore(&engine->sched_engine->lock, flags); in intel_engine_get_hung_entity()
2543 void xehp_enable_ccs_engines(struct intel_engine_cs *engine) in xehp_enable_ccs_engines() argument
2548 * so for simplicity we'll take care of this in the RCS engine's in xehp_enable_ccs_engines()
2553 if (!CCS_MASK(engine->gt)) in xehp_enable_ccs_engines()
2556 intel_uncore_write(engine->uncore, GEN12_RCU_MODE, in xehp_enable_ccs_engines()