drm/xe/xe_hw_engine_group.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2024 Intel Corporation
 */

#include <drm/drm_managed.h>

#include "xe_assert.h"
#include "xe_device_types.h"
#include "xe_exec_queue.h"
#include "xe_gt.h"
#include "xe_gt_stats.h"
#include "xe_hw_engine_group.h"
#include "xe_sync.h"
#include "xe_vm.h"

static void
hw_engine_group_resume_lr_jobs_func(struct work_struct *w)
{
	struct xe_exec_queue *q;
	struct xe_hw_engine_group *group = container_of(w, struct xe_hw_engine_group, resume_work);
	int err;
	enum xe_hw_engine_group_execution_mode previous_mode;

	err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode,
					  NULL, 0);
	if (err)
		return;

	if (previous_mode == EXEC_MODE_LR)
		goto put;

	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
		if (!xe_vm_in_fault_mode(q->vm))
			continue;

		/*
		 * Only resume queues that were actually suspended. A queue whose
		 * suspend() failed (e.g. killed/banned/wedged) was never
		 * suspended, so it must not be resumed.
		 */
		if (!READ_ONCE(q->lr.suspended))
			continue;

		WRITE_ONCE(q->lr.suspended, false);
		q->ops->resume(q);
	}

put:
	xe_hw_engine_group_put(group);
}

static struct xe_hw_engine_group *
hw_engine_group_alloc(struct xe_device *xe)
{
	struct xe_hw_engine_group *group;
	int err;

	group = drmm_kzalloc(&xe->drm, sizeof(*group), GFP_KERNEL);
	if (!group)
		return ERR_PTR(-ENOMEM);

	group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", WQ_PERCPU,
					   0);
	if (!group->resume_wq)
		return ERR_PTR(-ENOMEM);

	err = drmm_add_action_or_reset(&xe->drm, __drmm_workqueue_release, group->resume_wq);
	if (err)
		return ERR_PTR(err);

	init_rwsem(&group->mode_sem);
	INIT_WORK(&group->resume_work, hw_engine_group_resume_lr_jobs_func);
	INIT_LIST_HEAD(&group->exec_queue_list);

	return group;
}

/**
 * xe_hw_engine_setup_groups() - Setup the hw engine groups for the gt
 * @gt: The gt for which groups are setup
 *
 * Return: 0 on success, negative error code on error.
 */
int xe_hw_engine_setup_groups(struct xe_gt *gt)
{
	struct xe_hw_engine *hwe;
	enum xe_hw_engine_id id;
	struct xe_hw_engine_group *group_rcs_ccs, *group_bcs, *group_vcs_vecs;
	struct xe_device *xe = gt_to_xe(gt);

	group_rcs_ccs = hw_engine_group_alloc(xe);
	if (IS_ERR(group_rcs_ccs))
		return PTR_ERR(group_rcs_ccs);

	group_bcs = hw_engine_group_alloc(xe);
	if (IS_ERR(group_bcs))
		return PTR_ERR(group_bcs);

	group_vcs_vecs = hw_engine_group_alloc(xe);
	if (IS_ERR(group_vcs_vecs))
		return PTR_ERR(group_vcs_vecs);

	for_each_hw_engine(hwe, gt, id) {
		switch (hwe->class) {
		case XE_ENGINE_CLASS_COPY:
			hwe->hw_engine_group = group_bcs;
			break;
		case XE_ENGINE_CLASS_RENDER:
		case XE_ENGINE_CLASS_COMPUTE:
			hwe->hw_engine_group = group_rcs_ccs;
			break;
		case XE_ENGINE_CLASS_VIDEO_DECODE:
		case XE_ENGINE_CLASS_VIDEO_ENHANCE:
			hwe->hw_engine_group = group_vcs_vecs;
			break;
		case XE_ENGINE_CLASS_OTHER:
			break;
		case XE_ENGINE_CLASS_MAX:
			xe_gt_assert(gt, false);
		}
	}

	return 0;
}

/**
 * xe_hw_engine_group_add_exec_queue() - Add an exec queue to a hw engine group
 * @group: The hw engine group
 * @q: The exec_queue
 *
 * Return: 0 on success,
 *	    -EINTR if the lock could not be acquired
 */
int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q)
{
	int err;
	struct xe_device *xe = gt_to_xe(q->gt);

	xe_assert(xe, group);
	xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM));
	xe_assert(xe, q->vm);

	if (xe_vm_in_preempt_fence_mode(q->vm))
		return 0;

	err = down_write_killable(&group->mode_sem);
	if (err)
		return err;

	if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) {
		/*
		 * suspend() can fail (e.g. killed/banned/wedged), leaving the
		 * queue un-suspended. Propagate the failure so the queue is not
		 * added; on failure nothing was suspended, so there is nothing to
		 * undo. Only record the queue as suspended (and later resume it)
		 * once suspend() has succeeded.
		 */
		err = q->ops->suspend(q);
		if (err)
			goto err_suspend;

		WRITE_ONCE(q->lr.suspended, true);
		err = q->ops->suspend_wait(q);
		if (err)
			goto err_suspend;

		xe_hw_engine_group_resume_faulting_lr_jobs(group);
	}

	list_add(&q->hw_engine_group_link, &group->exec_queue_list);
	up_write(&group->mode_sem);

	return 0;

err_suspend:
	up_write(&group->mode_sem);
	return err;
}
ALLOW_ERROR_INJECTION(xe_hw_engine_group_add_exec_queue, ERRNO);

/**
 * xe_hw_engine_group_del_exec_queue() - Delete an exec queue from a hw engine group
 * @group: The hw engine group
 * @q: The exec_queue
 */
void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q)
{
	struct xe_device *xe = gt_to_xe(q->gt);

	xe_assert(xe, group);
	xe_assert(xe, q->vm);

	down_write(&group->mode_sem);

	if (!list_empty(&q->hw_engine_group_link))
		list_del(&q->hw_engine_group_link);

	up_write(&group->mode_sem);
}

/**
 * xe_hw_engine_group_resume_faulting_lr_jobs() - Asynchronously resume the hw engine group's
 * faulting LR jobs
 * @group: The hw engine group
 */
void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group)
{
	queue_work(group->resume_wq, &group->resume_work);
}

/**
 * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group
 * @group: The hw engine group
 * @has_deps: dma-fence job triggering suspend has dependencies
 *
 * Return: 0 on success, negative error code on error.
 */
static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group,
						       bool has_deps)
{
	int err;
	struct xe_exec_queue *q;
	struct xe_gt *gt = NULL;
	bool need_resume = false;
	ktime_t start = xe_gt_stats_ktime_get();

	lockdep_assert_held_write(&group->mode_sem);

	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {

		if (!xe_vm_in_fault_mode(q->vm))
			continue;

		if (has_deps)
			return -EAGAIN;

		xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1);
		/*
		 * suspend() only fails when the queue is killed/banned/wedged.
		 * Such a queue is being torn down (its removal from HW is handled
		 * by the kill/ban teardown), so it is not a live fault-mode
		 * context the mode switch must preempt. Skip it rather than
		 * failing the switch, otherwise one dying sibling would block a
		 * dma-fence submission on the healthy queues in the group. Only
		 * queues recorded as suspended below are later waited on and
		 * resumed.
		 */
		err = q->ops->suspend(q);
		if (err)
			continue;

		WRITE_ONCE(q->lr.suspended, true);
		need_resume = true;
		gt = q->gt;
	}

	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
		if (!xe_vm_in_fault_mode(q->vm))
			continue;

		/* Only wait on queues that were actually suspended above. */
		if (!READ_ONCE(q->lr.suspended))
			continue;

		err = q->ops->suspend_wait(q);
		if (err)
			goto err_resume;
	}

	if (gt) {
		xe_gt_stats_incr(gt,
				 XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
				 xe_gt_stats_ktime_us_delta(start));
	}

	if (need_resume)
		xe_hw_engine_group_resume_faulting_lr_jobs(group);

	return 0;

err_resume:
	/*
	 * A suspend_wait() failed partway through the mode switch. Resume the
	 * sibling queues that were already suspended in this call so they are
	 * not left suspended forever.
	 *
	 * resume() requires the suspend to have completed (suspend_pending
	 * cleared) or it trips the !suspend_pending assert. So skip the resume
	 * when either:
	 *  - suspend_wait_blocking() fails: on a GuC timeout it bans the queue
	 *    and triggers cleanup, so the queue is being torn down; or
	 *  - reset_status() is true: the queue was reset/killed/banned/wedged.
	 *    suspend_wait() can return success in this case via its killed/
	 *    stopped wait condition while suspend_pending is still set, and the
	 *    queue is being torn down anyway, so its state is resolved by
	 *    teardown rather than by a resume here.
	 * In either case leave the queue marked suspended.
	 *
	 * Use the *blocking* (uninterruptible) wait here: the queues resumed on
	 * this path may belong to a different process than the one that
	 * triggered the mode switch. An interruptible suspend_wait() would
	 * return -ERESTARTSYS if the triggering task is signalled, skip the
	 * resume, and leave the other process's queue suspended forever
	 * (cross-process DoS).
	 */
	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
		if (!xe_vm_in_fault_mode(q->vm))
			continue;

		if (!READ_ONCE(q->lr.suspended))
			continue;

		if (q->ops->suspend_wait_blocking(q) || q->ops->reset_status(q))
			continue;

		WRITE_ONCE(q->lr.suspended, false);
		q->ops->resume(q);
	}

	return err;
}

/**
 * xe_hw_engine_group_wait_for_dma_fence_jobs() - Wait for dma fence jobs to complete
 * @group: The hw engine group
 *
 * This function is not meant to be called directly from a user IOCTL as dma_fence_wait()
 * is not interruptible.
 *
 * Return: 0 on success,
 *	   -ETIME if waiting for one job failed
 */
static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group *group)
{
	long timeout;
	struct xe_exec_queue *q;
	struct xe_gt *gt = NULL;
	struct dma_fence *fence;
	ktime_t start = xe_gt_stats_ktime_get();

	lockdep_assert_held_write(&group->mode_sem);

	list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
		if (xe_vm_in_lr_mode(q->vm))
			continue;

		xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1);
		fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm);
		timeout = dma_fence_wait(fence, false);
		dma_fence_put(fence);
		gt = q->gt;

		if (timeout < 0)
			return -ETIME;
	}

	if (gt) {
		xe_gt_stats_incr(gt,
				 XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
				 xe_gt_stats_ktime_us_delta(start));
	}

	return 0;
}

static int switch_mode(struct xe_hw_engine_group *group, bool has_deps)
{
	int err = 0;
	enum xe_hw_engine_group_execution_mode new_mode;

	lockdep_assert_held_write(&group->mode_sem);

	switch (group->cur_mode) {
	case EXEC_MODE_LR:
		new_mode = EXEC_MODE_DMA_FENCE;
		err = xe_hw_engine_group_suspend_faulting_lr_jobs(group,
								  has_deps);
		break;
	case EXEC_MODE_DMA_FENCE:
		new_mode = EXEC_MODE_LR;
		err = xe_hw_engine_group_wait_for_dma_fence_jobs(group);
		break;
	}

	if (err)
		return err;

	group->cur_mode = new_mode;

	return 0;
}

static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs)
{
	int err, i;

	for (i = 0; i < num_syncs; ++i) {
		err = xe_sync_entry_wait(syncs + i);
		if (err)
			return err;
	}

	return 0;
}

/**
 * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode
 * @group: The hw engine group
 * @new_mode: The new execution mode
 * @previous_mode: Pointer to the previous mode provided for use by caller
 * @syncs: Syncs from exec IOCTL
 * @num_syncs: Number of syncs from exec IOCTL
 *
 * Return: 0 if successful, -EINTR if locking failed.
 */
int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
				enum xe_hw_engine_group_execution_mode new_mode,
				enum xe_hw_engine_group_execution_mode *previous_mode,
				struct xe_sync_entry *syncs, int num_syncs)
__acquires(&group->mode_sem)
{
	bool has_deps = !!num_syncs;
	int err = down_read_interruptible(&group->mode_sem);

	if (err)
		return err;

	*previous_mode = group->cur_mode;

	if (new_mode != group->cur_mode) {
		up_read(&group->mode_sem);
retry:
		err = down_write_killable(&group->mode_sem);
		if (err)
			return err;

		if (new_mode != group->cur_mode) {
			err = switch_mode(group, has_deps);
			if (err) {
				up_write(&group->mode_sem);

				if (err != -EAGAIN)
					return err;

				err = wait_syncs(syncs, num_syncs);
				if (err)
					return err;

				has_deps = false;
				goto retry;
			}
		}
		downgrade_write(&group->mode_sem);
	}

	return err;
}

/**
 * xe_hw_engine_group_put() - Put the group
 * @group: The hw engine group
 */
void xe_hw_engine_group_put(struct xe_hw_engine_group *group)
__releases(&group->mode_sem)
{
	up_read(&group->mode_sem);
}

/**
 * xe_hw_engine_group_find_exec_mode() - Find the execution mode for this exec queue
 * @q: The exec_queue
 */
enum xe_hw_engine_group_execution_mode
xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q)
{
	if (xe_vm_in_fault_mode(q->vm))
		return EXEC_MODE_LR;
	else
		return EXEC_MODE_DMA_FENCE;
}