drm/xe/xe_exec_queue.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2021 Intel Corporation
 */

#include "xe_exec_queue.h"

#include <linux/nospec.h>

#include <drm/drm_device.h>
#include <drm/drm_drv.h>
#include <drm/drm_file.h>
#include <drm/drm_syncobj.h>
#include <uapi/drm/xe_drm.h>

#include "xe_bo.h"
#include "xe_dep_scheduler.h"
#include "xe_device.h"
#include "xe_gt.h"
#include "xe_gt_sriov_pf.h"
#include "xe_gt_sriov_vf.h"
#include "xe_hw_engine_class_sysfs.h"
#include "xe_hw_engine_group.h"
#include "xe_irq.h"
#include "xe_lrc.h"
#include "xe_macros.h"
#include "xe_migrate.h"
#include "xe_pm.h"
#include "xe_trace.h"
#include "xe_vm.h"
#include "xe_pxp.h"

/**
 * DOC: Execution Queue
 *
 * An Execution queue is an interface for the HW context of execution.
 * The user creates an execution queue, submits the GPU jobs through those
 * queues and in the end destroys them.
 *
 * Execution queues can also be created by XeKMD itself for driver internal
 * operations like object migration etc.
 *
 * An execution queue is associated with a specified HW engine or a group of
 * engines (belonging to the same tile and engine class) and any GPU job
 * submitted on the queue will be run on one of these engines.
 *
 * An execution queue is tied to an address space (VM). It holds a reference
 * of the associated VM and the underlying Logical Ring Context/s (LRC/s)
 * until the queue is destroyed.
 *
 * The execution queue sits on top of the submission backend. It opaquely
 * handles the GuC and Execlist backends whichever the platform uses, and
 * the ring operations the different engine classes support.
 */

/**
 * DOC: Multi Queue Group
 *
 * Multi Queue Group is another mode of execution supported by the compute
 * and blitter copy command streamers (CCS and BCS, respectively). It is
 * an enhancement of the existing hardware architecture and leverages the
 * same submission model. It enables support for efficient, parallel
 * execution of multiple queues within a single shared context. The multi
 * queue group functionality is only supported with GuC submission backend.
 * All the queues of a group must use the same address space (VM).
 *
 * The DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE execution queue property
 * supports creating a multi queue group and adding queues to a queue group.
 *
 * The XE_EXEC_QUEUE_CREATE ioctl call with above property with value field
 * set to DRM_XE_MULTI_GROUP_CREATE, will create a new multi queue group with
 * the queue being created as the primary queue (aka q0) of the group. To add
 * secondary queues to the group, they need to be created with the above
 * property with id of the primary queue as the value. The properties of
 * the primary queue (like priority, time slice) applies to the whole group.
 * So, these properties can't be set for secondary queues of a group.
 *
 * The hardware does not support removing a queue from a multi-queue group.
 * However, queues can be dynamically added to the group. A group can have
 * up to 64 queues. To support this, XeKMD holds references to LRCs of the
 * queues even after the queues are destroyed by the user until the whole
 * group is destroyed. The secondary queues hold a reference to the primary
 * queue thus preventing the group from being destroyed when user destroys
 * the primary queue. Once the primary queue is destroyed, secondary queues
 * can't be added to the queue group and new job submissions on existing
 * secondary queues are not allowed.
 *
 * The queues of a multi queue group can set their priority within the group
 * through the DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY property.
 * This multi queue priority can also be set dynamically through the
 * XE_EXEC_QUEUE_SET_PROPERTY ioctl. This is the only other property
 * supported by the secondary queues of a multi queue group, other than
 * DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE.
 *
 * When GuC reports an error on any of the queues of a multi queue group,
 * the queue cleanup mechanism is invoked for all the queues of the group
 * as hardware cannot make progress on the multi queue context.
 *
 * Refer :ref:`multi-queue-group-guc-interface` for multi queue group GuC
 * interface.
 */

enum xe_exec_queue_sched_prop {
	XE_EXEC_QUEUE_JOB_TIMEOUT = 0,
	XE_EXEC_QUEUE_TIMESLICE = 1,
	XE_EXEC_QUEUE_PREEMPT_TIMEOUT = 2,
	XE_EXEC_QUEUE_SCHED_PROP_MAX = 3,
};

static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
				      u64 extensions);

static void xe_exec_queue_group_cleanup(struct xe_exec_queue *q)
{
	struct xe_exec_queue_group *group = q->multi_queue.group;
	struct xe_lrc *lrc;
	unsigned long idx;

	if (xe_exec_queue_is_multi_queue_secondary(q)) {
		/*
		 * Put pairs with get from xe_exec_queue_lookup() call
		 * in xe_exec_queue_group_validate().
		 */
		xe_exec_queue_put(xe_exec_queue_multi_queue_primary(q));
		return;
	}

	if (!group)
		return;

	/* Primary queue cleanup */
	xa_for_each(&group->xa, idx, lrc)
		xe_lrc_put(lrc);

	xa_destroy(&group->xa);
	mutex_destroy(&group->list_lock);
	xe_bo_unpin_map_no_vm(group->cgp_bo);
	kfree(group);
}

static void __xe_exec_queue_free(struct xe_exec_queue *q)
{
	int i;

	for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i)
		if (q->tlb_inval[i].dep_scheduler)
			xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler);

	if (xe_exec_queue_uses_pxp(q))
		xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);

	if (xe_exec_queue_is_multi_queue(q))
		xe_exec_queue_group_cleanup(q);

	if (q->vm)
		xe_vm_put(q->vm);

	if (q->xef)
		xe_file_put(q->xef);

	kvfree(q->replay_state);
	kfree(q);
}

static int alloc_dep_schedulers(struct xe_device *xe, struct xe_exec_queue *q)
{
	struct xe_tile *tile = gt_to_tile(q->gt);
	int i;

	for (i = 0; i < XE_EXEC_QUEUE_TLB_INVAL_COUNT; ++i) {
		struct xe_dep_scheduler *dep_scheduler;
		struct xe_gt *gt;
		struct workqueue_struct *wq;

		if (i == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT)
			gt = tile->primary_gt;
		else
			gt = tile->media_gt;

		if (!gt)
			continue;

		wq = gt->tlb_inval.job_wq;

#define MAX_TLB_INVAL_JOBS	16	/* Picking a reasonable value */
		dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name,
							MAX_TLB_INVAL_JOBS);
		if (IS_ERR(dep_scheduler))
			return PTR_ERR(dep_scheduler);

		q->tlb_inval[i].dep_scheduler = dep_scheduler;
	}
#undef MAX_TLB_INVAL_JOBS

	return 0;
}

static struct xe_exec_queue *__xe_exec_queue_alloc(struct xe_device *xe,
						   struct xe_vm *vm,
						   u32 logical_mask,
						   u16 width, struct xe_hw_engine *hwe,
						   u32 flags, u64 extensions)
{
	struct xe_exec_queue *q;
	struct xe_gt *gt = hwe->gt;
	int err;

	/* only kernel queues can be permanent */
	XE_WARN_ON((flags & EXEC_QUEUE_FLAG_PERMANENT) && !(flags & EXEC_QUEUE_FLAG_KERNEL));

	q = kzalloc(struct_size(q, lrc, width), GFP_KERNEL);
	if (!q)
		return ERR_PTR(-ENOMEM);

	kref_init(&q->refcount);
	q->flags = flags;
	q->hwe = hwe;
	q->gt = gt;
	q->class = hwe->class;
	q->width = width;
	q->msix_vec = XE_IRQ_DEFAULT_MSIX;
	q->logical_mask = logical_mask;
	q->fence_irq = &gt->fence_irq[hwe->class];
	q->ring_ops = gt->ring_ops[hwe->class];
	q->ops = gt->exec_queue_ops;
	INIT_LIST_HEAD(&q->lr.link);
	INIT_LIST_HEAD(&q->multi_gt_link);
	INIT_LIST_HEAD(&q->hw_engine_group_link);
	INIT_LIST_HEAD(&q->pxp.link);
	q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL;

	q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us;
	q->sched_props.preempt_timeout_us =
				hwe->eclass->sched_props.preempt_timeout_us;
	q->sched_props.job_timeout_ms =
				hwe->eclass->sched_props.job_timeout_ms;
	if (q->flags & EXEC_QUEUE_FLAG_KERNEL &&
	    q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY)
		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL;
	else
		q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL;

	if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) {
		err = alloc_dep_schedulers(xe, q);
		if (err) {
			__xe_exec_queue_free(q);
			return ERR_PTR(err);
		}
	}

	if (vm)
		q->vm = xe_vm_get(vm);

	if (extensions) {
		/*
		 * may set q->usm, must come before xe_lrc_create(),
		 * may overwrite q->sched_props, must come before q->ops->init()
		 */
		err = exec_queue_user_extensions(xe, q, extensions);
		if (err) {
			__xe_exec_queue_free(q);
			return ERR_PTR(err);
		}
	}

	return q;
}

static int __xe_exec_queue_init(struct xe_exec_queue *q, u32 exec_queue_flags)
{
	int i, err;
	u32 flags = 0;

	/*
	 * PXP workloads executing on RCS or CCS must run in isolation (i.e. no
	 * other workload can use the EUs at the same time). On MTL this is done
	 * by setting the RUNALONE bit in the LRC, while starting on Xe2 there
	 * is a dedicated bit for it.
	 */
	if (xe_exec_queue_uses_pxp(q) &&
	    (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) {
		if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20)
			flags |= XE_LRC_CREATE_PXP;
		else
			flags |= XE_LRC_CREATE_RUNALONE;
	}

	if (!(exec_queue_flags & EXEC_QUEUE_FLAG_KERNEL))
		flags |= XE_LRC_CREATE_USER_CTX;

	err = q->ops->init(q);
	if (err)
		return err;

	/*
	 * This must occur after q->ops->init to avoid race conditions during VF
	 * post-migration recovery, as the fixups for the LRC GGTT addresses
	 * depend on the queue being present in the backend tracking structure.
	 *
	 * In addition to above, we must wait on inflight GGTT changes to avoid
	 * writing out stale values here. Such wait provides a solid solution
	 * (without a race) only if the function can detect migration instantly
	 * from the moment vCPU resumes execution.
	 */
	for (i = 0; i < q->width; ++i) {
		struct xe_lrc *lrc;

		xe_gt_sriov_vf_wait_valid_ggtt(q->gt);
		lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state,
				    xe_lrc_ring_size(), q->msix_vec, flags);
		if (IS_ERR(lrc)) {
			err = PTR_ERR(lrc);
			goto err_lrc;
		}

		/* Pairs with READ_ONCE to xe_exec_queue_contexts_hwsp_rebase */
		WRITE_ONCE(q->lrc[i], lrc);
	}

	return 0;

err_lrc:
	for (i = i - 1; i >= 0; --i)
		xe_lrc_put(q->lrc[i]);
	return err;
}

static void __xe_exec_queue_fini(struct xe_exec_queue *q)
{
	int i;

	q->ops->fini(q);

	for (i = 0; i < q->width; ++i)
		xe_lrc_put(q->lrc[i]);
}

struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm,
					   u32 logical_mask, u16 width,
					   struct xe_hw_engine *hwe, u32 flags,
					   u64 extensions)
{
	struct xe_exec_queue *q;
	int err;

	/* VMs for GSCCS queues (and only those) must have the XE_VM_FLAG_GSC flag */
	xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0)));

	q = __xe_exec_queue_alloc(xe, vm, logical_mask, width, hwe, flags,
				  extensions);
	if (IS_ERR(q))
		return q;

	err = __xe_exec_queue_init(q, flags);
	if (err)
		goto err_post_alloc;

	/*
	 * We can only add the queue to the PXP list after the init is complete,
	 * because the PXP termination can call exec_queue_kill and that will
	 * go bad if the queue is only half-initialized. This means that we
	 * can't do it when we handle the PXP extension in __xe_exec_queue_alloc
	 * and we need to do it here instead.
	 */
	if (xe_exec_queue_uses_pxp(q)) {
		err = xe_pxp_exec_queue_add(xe->pxp, q);
		if (err)
			goto err_post_init;
	}

	return q;

err_post_init:
	__xe_exec_queue_fini(q);
err_post_alloc:
	__xe_exec_queue_free(q);
	return ERR_PTR(err);
}
ALLOW_ERROR_INJECTION(xe_exec_queue_create, ERRNO);

struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt,
						 struct xe_vm *vm,
						 enum xe_engine_class class,
						 u32 flags, u64 extensions)
{
	struct xe_hw_engine *hwe, *hwe0 = NULL;
	enum xe_hw_engine_id id;
	u32 logical_mask = 0;

	for_each_hw_engine(hwe, gt, id) {
		if (xe_hw_engine_is_reserved(hwe))
			continue;

		if (hwe->class == class) {
			logical_mask |= BIT(hwe->logical_instance);
			if (!hwe0)
				hwe0 = hwe;
		}
	}

	if (!logical_mask)
		return ERR_PTR(-ENODEV);

	return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags, extensions);
}

/**
 * xe_exec_queue_create_bind() - Create bind exec queue.
 * @xe: Xe device.
 * @tile: tile which bind exec queue belongs to.
 * @flags: exec queue creation flags
 * @user_vm: The user VM which this exec queue belongs to
 * @extensions: exec queue creation extensions
 *
 * Normalize bind exec queue creation. Bind exec queue is tied to migration VM
 * for access to physical memory required for page table programming. On a
 * faulting devices the reserved copy engine instance must be used to avoid
 * deadlocking (user binds cannot get stuck behind faults as kernel binds which
 * resolve faults depend on user binds). On non-faulting devices any copy engine
 * can be used.
 *
 * Returns exec queue on success, ERR_PTR on failure
 */
struct xe_exec_queue *xe_exec_queue_create_bind(struct xe_device *xe,
						struct xe_tile *tile,
						struct xe_vm *user_vm,
						u32 flags, u64 extensions)
{
	struct xe_gt *gt = tile->primary_gt;
	struct xe_exec_queue *q;
	struct xe_vm *migrate_vm;

	migrate_vm = xe_migrate_get_vm(tile->migrate);
	if (xe->info.has_usm) {
		struct xe_hw_engine *hwe = xe_gt_hw_engine(gt,
							   XE_ENGINE_CLASS_COPY,
							   gt->usm.reserved_bcs_instance,
							   false);

		if (!hwe) {
			xe_vm_put(migrate_vm);
			return ERR_PTR(-EINVAL);
		}

		q = xe_exec_queue_create(xe, migrate_vm,
					 BIT(hwe->logical_instance), 1, hwe,
					 flags, extensions);
	} else {
		q = xe_exec_queue_create_class(xe, gt, migrate_vm,
					       XE_ENGINE_CLASS_COPY, flags,
					       extensions);
	}
	xe_vm_put(migrate_vm);

	if (!IS_ERR(q)) {
		int err = drm_syncobj_create(&q->ufence_syncobj,
					     DRM_SYNCOBJ_CREATE_SIGNALED,
					     NULL);
		if (err) {
			xe_exec_queue_put(q);
			return ERR_PTR(err);
		}

		if (user_vm)
			q->user_vm = xe_vm_get(user_vm);
	}

	return q;
}
ALLOW_ERROR_INJECTION(xe_exec_queue_create_bind, ERRNO);

void xe_exec_queue_destroy(struct kref *ref)
{
	struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount);
	struct xe_exec_queue *eq, *next;
	int i;

	xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0);

	if (q->ufence_syncobj)
		drm_syncobj_put(q->ufence_syncobj);

	if (xe_exec_queue_uses_pxp(q))
		xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q);

	xe_exec_queue_last_fence_put_unlocked(q);
	for_each_tlb_inval(i)
		xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, i);

	if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) {
		list_for_each_entry_safe(eq, next, &q->multi_gt_list,
					 multi_gt_link)
			xe_exec_queue_put(eq);
	}

	if (q->user_vm) {
		xe_vm_put(q->user_vm);
		q->user_vm = NULL;
	}

	q->ops->destroy(q);
}

void xe_exec_queue_fini(struct xe_exec_queue *q)
{
	/*
	 * Before releasing our ref to lrc and xef, accumulate our run ticks
	 * and wakeup any waiters.
	 */
	xe_exec_queue_update_run_ticks(q);
	if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal))
		wake_up_var(&q->xef->exec_queue.pending_removal);

	__xe_exec_queue_fini(q);
	__xe_exec_queue_free(q);
}

void xe_exec_queue_assign_name(struct xe_exec_queue *q, u32 instance)
{
	switch (q->class) {
	case XE_ENGINE_CLASS_RENDER:
		snprintf(q->name, sizeof(q->name), "rcs%d", instance);
		break;
	case XE_ENGINE_CLASS_VIDEO_DECODE:
		snprintf(q->name, sizeof(q->name), "vcs%d", instance);
		break;
	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
		snprintf(q->name, sizeof(q->name), "vecs%d", instance);
		break;
	case XE_ENGINE_CLASS_COPY:
		snprintf(q->name, sizeof(q->name), "bcs%d", instance);
		break;
	case XE_ENGINE_CLASS_COMPUTE:
		snprintf(q->name, sizeof(q->name), "ccs%d", instance);
		break;
	case XE_ENGINE_CLASS_OTHER:
		snprintf(q->name, sizeof(q->name), "gsccs%d", instance);
		break;
	default:
		XE_WARN_ON(q->class);
	}
}

struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id)
{
	struct xe_exec_queue *q;

	mutex_lock(&xef->exec_queue.lock);
	q = xa_load(&xef->exec_queue.xa, id);
	if (q)
		xe_exec_queue_get(q);
	mutex_unlock(&xef->exec_queue.lock);

	return q;
}

enum xe_exec_queue_priority
xe_exec_queue_device_get_max_priority(struct xe_device *xe)
{
	return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH :
				       XE_EXEC_QUEUE_PRIORITY_NORMAL;
}

static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q,
				   u64 value)
{
	if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH))
		return -EINVAL;

	if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe)))
		return -EPERM;

	q->sched_props.priority = value;
	return 0;
}

static bool xe_exec_queue_enforce_schedule_limit(void)
{
#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)
	return true;
#else
	return !capable(CAP_SYS_NICE);
#endif
}

static void
xe_exec_queue_get_prop_minmax(struct xe_hw_engine_class_intf *eclass,
			      enum xe_exec_queue_sched_prop prop,
			      u32 *min, u32 *max)
{
	switch (prop) {
	case XE_EXEC_QUEUE_JOB_TIMEOUT:
		*min = eclass->sched_props.job_timeout_min;
		*max = eclass->sched_props.job_timeout_max;
		break;
	case XE_EXEC_QUEUE_TIMESLICE:
		*min = eclass->sched_props.timeslice_min;
		*max = eclass->sched_props.timeslice_max;
		break;
	case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:
		*min = eclass->sched_props.preempt_timeout_min;
		*max = eclass->sched_props.preempt_timeout_max;
		break;
	default:
		break;
	}
#if IS_ENABLED(CONFIG_DRM_XE_ENABLE_SCHEDTIMEOUT_LIMIT)
	if (capable(CAP_SYS_NICE)) {
		switch (prop) {
		case XE_EXEC_QUEUE_JOB_TIMEOUT:
			*min = XE_HW_ENGINE_JOB_TIMEOUT_MIN;
			*max = XE_HW_ENGINE_JOB_TIMEOUT_MAX;
			break;
		case XE_EXEC_QUEUE_TIMESLICE:
			*min = XE_HW_ENGINE_TIMESLICE_MIN;
			*max = XE_HW_ENGINE_TIMESLICE_MAX;
			break;
		case XE_EXEC_QUEUE_PREEMPT_TIMEOUT:
			*min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN;
			*max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX;
			break;
		default:
			break;
		}
	}
#endif
}

static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q,
				    u64 value)
{
	u32 min = 0, max = 0;

	xe_exec_queue_get_prop_minmax(q->hwe->eclass,
				      XE_EXEC_QUEUE_TIMESLICE, &min, &max);

	if (xe_exec_queue_enforce_schedule_limit() &&
	    !xe_hw_engine_timeout_in_range(value, min, max))
		return -EINVAL;

	q->sched_props.timeslice_us = value;
	return 0;
}

static int
exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value)
{
	if (value == DRM_XE_PXP_TYPE_NONE)
		return 0;

	/* we only support HWDRM sessions right now */
	if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM))
		return -EINVAL;

	if (!xe_pxp_is_enabled(xe->pxp))
		return -ENODEV;

	return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM);
}

static int exec_queue_set_hang_replay_state(struct xe_device *xe,
					    struct xe_exec_queue *q,
					    u64 value)
{
	size_t size = xe_gt_lrc_hang_replay_size(q->gt, q->class);
	u64 __user *address = u64_to_user_ptr(value);
	void *ptr;

	ptr = vmemdup_user(address, size);
	if (XE_IOCTL_DBG(xe, IS_ERR(ptr)))
		return PTR_ERR(ptr);

	q->replay_state = ptr;

	return 0;
}

static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue *q)
{
	struct xe_tile *tile = gt_to_tile(q->gt);
	struct xe_exec_queue_group *group;
	struct xe_bo *bo;

	group = kzalloc(sizeof(*group), GFP_KERNEL);
	if (!group)
		return -ENOMEM;

	bo = xe_bo_create_pin_map_novm(xe, tile, SZ_4K, ttm_bo_type_kernel,
				       XE_BO_FLAG_VRAM_IF_DGFX(tile) |
				       XE_BO_FLAG_PINNED_LATE_RESTORE |
				       XE_BO_FLAG_FORCE_USER_VRAM |
				       XE_BO_FLAG_GGTT_INVALIDATE |
				       XE_BO_FLAG_GGTT, false);
	if (IS_ERR(bo)) {
		drm_err(&xe->drm, "CGP bo allocation for queue group failed: %ld\n",
			PTR_ERR(bo));
		kfree(group);
		return PTR_ERR(bo);
	}

	xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K);

	group->primary = q;
	group->cgp_bo = bo;
	INIT_LIST_HEAD(&group->list);
	xa_init_flags(&group->xa, XA_FLAGS_ALLOC1);
	mutex_init(&group->list_lock);
	q->multi_queue.group = group;

	/* group->list_lock is used in submission backend */
	if (IS_ENABLED(CONFIG_LOCKDEP)) {
		fs_reclaim_acquire(GFP_KERNEL);
		might_lock(&group->list_lock);
		fs_reclaim_release(GFP_KERNEL);
	}

	return 0;
}

static inline bool xe_exec_queue_supports_multi_queue(struct xe_exec_queue *q)
{
	return q->gt->info.multi_queue_engine_class_mask & BIT(q->class);
}

static int xe_exec_queue_group_validate(struct xe_device *xe, struct xe_exec_queue *q,
					u32 primary_id)
{
	struct xe_exec_queue_group *group;
	struct xe_exec_queue *primary;
	int ret;

	/*
	 * Get from below xe_exec_queue_lookup() pairs with put
	 * in xe_exec_queue_group_cleanup().
	 */
	primary = xe_exec_queue_lookup(q->vm->xef, primary_id);
	if (XE_IOCTL_DBG(xe, !primary))
		return -ENOENT;

	if (XE_IOCTL_DBG(xe, !xe_exec_queue_is_multi_queue_primary(primary)) ||
	    XE_IOCTL_DBG(xe, q->vm != primary->vm) ||
	    XE_IOCTL_DBG(xe, q->logical_mask != primary->logical_mask)) {
		ret = -EINVAL;
		goto put_primary;
	}

	group = primary->multi_queue.group;
	q->multi_queue.valid = true;
	q->multi_queue.group = group;

	return 0;
put_primary:
	xe_exec_queue_put(primary);
	return ret;
}

#define XE_MAX_GROUP_SIZE	64
static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q)
{
	struct xe_exec_queue_group *group = q->multi_queue.group;
	u32 pos;
	int err;

	xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q));

	/* Primary queue holds a reference to LRCs of all secondary queues */
	err = xa_alloc(&group->xa, &pos, xe_lrc_get(q->lrc[0]),
		       XA_LIMIT(1, XE_MAX_GROUP_SIZE - 1), GFP_KERNEL);
	if (XE_IOCTL_DBG(xe, err)) {
		xe_lrc_put(q->lrc[0]);

		/* It is invalid if queue group limit is exceeded */
		if (err == -EBUSY)
			err = -EINVAL;

		return err;
	}

	q->multi_queue.pos = pos;

	return 0;
}

static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queue *q)
{
	struct xe_exec_queue_group *group = q->multi_queue.group;
	struct xe_lrc *lrc;

	xe_assert(xe, xe_exec_queue_is_multi_queue_secondary(q));

	lrc = xa_erase(&group->xa, q->multi_queue.pos);
	xe_assert(xe, lrc);
	xe_lrc_put(lrc);
}

static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q,
				      u64 value)
{
	if (XE_IOCTL_DBG(xe, !xe_exec_queue_supports_multi_queue(q)))
		return -ENODEV;

	if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe)))
		return -EOPNOTSUPP;

	if (XE_IOCTL_DBG(xe, !q->vm->xef))
		return -EINVAL;

	if (XE_IOCTL_DBG(xe, xe_exec_queue_is_parallel(q)))
		return -EINVAL;

	if (XE_IOCTL_DBG(xe, xe_exec_queue_is_multi_queue(q)))
		return -EINVAL;

	if (value & DRM_XE_MULTI_GROUP_CREATE) {
		if (XE_IOCTL_DBG(xe, value & ~DRM_XE_MULTI_GROUP_CREATE))
			return -EINVAL;

		q->multi_queue.valid = true;
		q->multi_queue.is_primary = true;
		q->multi_queue.pos = 0;
		return 0;
	}

	/* While adding secondary queues, the upper 32 bits must be 0 */
	if (XE_IOCTL_DBG(xe, value & (~0ull << 32)))
		return -EINVAL;

	return xe_exec_queue_group_validate(xe, q, value);
}

static int exec_queue_set_multi_queue_priority(struct xe_device *xe, struct xe_exec_queue *q,
					       u64 value)
{
	if (XE_IOCTL_DBG(xe, value > XE_MULTI_QUEUE_PRIORITY_HIGH))
		return -EINVAL;

	/* For queue creation time (!q->xef) setting, just store the priority value */
	if (!q->xef) {
		q->multi_queue.priority = value;
		return 0;
	}

	if (!xe_exec_queue_is_multi_queue(q))
		return -EINVAL;

	return q->ops->set_multi_queue_priority(q, value);
}

typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe,
					     struct xe_exec_queue *q,
					     u64 value);

static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = {
	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority,
	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice,
	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type,
	[DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE] = exec_queue_set_hang_replay_state,
	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP] = exec_queue_set_multi_group,
	[DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY] =
							exec_queue_set_multi_queue_priority,
};

int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data,
				     struct drm_file *file)
{
	struct xe_device *xe = to_xe_device(dev);
	struct xe_file *xef = to_xe_file(file);
	struct drm_xe_exec_queue_set_property *args = data;
	struct xe_exec_queue *q;
	int ret;
	u32 idx;

	if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
		return -EINVAL;

	if (XE_IOCTL_DBG(xe, args->property !=
			 DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY))
		return -EINVAL;

	q = xe_exec_queue_lookup(xef, args->exec_queue_id);
	if (XE_IOCTL_DBG(xe, !q))
		return -ENOENT;

	idx = array_index_nospec(args->property,
				 ARRAY_SIZE(exec_queue_set_property_funcs));
	ret = exec_queue_set_property_funcs[idx](xe, q, args->value);
	if (XE_IOCTL_DBG(xe, ret))
		goto err_post_lookup;

	xe_exec_queue_put(q);
	return 0;

 err_post_lookup:
	xe_exec_queue_put(q);
	return ret;
}

static int exec_queue_user_ext_check(struct xe_exec_queue *q, u64 properties)
{
	u64 secondary_queue_valid_props = BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP) |
				  BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY);

	/*
	 * Only MULTI_QUEUE_PRIORITY property is valid for secondary queues of a
	 * multi-queue group.
	 */
	if (xe_exec_queue_is_multi_queue_secondary(q) &&
	    properties & ~secondary_queue_valid_props)
		return -EINVAL;

	return 0;
}

static int exec_queue_user_ext_check_final(struct xe_exec_queue *q, u64 properties)
{
	/* MULTI_QUEUE_PRIORITY only applies to multi-queue group queues */
	if ((properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY)) &&
	    !(properties & BIT_ULL(DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP)))
		return -EINVAL;

	return 0;
}

static int exec_queue_user_ext_set_property(struct xe_device *xe,
					    struct xe_exec_queue *q,
					    u64 extension, u64 *properties)
{
	u64 __user *address = u64_to_user_ptr(extension);
	struct drm_xe_ext_set_property ext;
	int err;
	u32 idx;

	err = copy_from_user(&ext, address, sizeof(ext));
	if (XE_IOCTL_DBG(xe, err))
		return -EFAULT;

	if (XE_IOCTL_DBG(xe, ext.property >=
			 ARRAY_SIZE(exec_queue_set_property_funcs)) ||
	    XE_IOCTL_DBG(xe, ext.pad) ||
	    XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY &&
			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE &&
			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE &&
			 ext.property != DRM_XE_EXEC_QUEUE_SET_HANG_REPLAY_STATE &&
			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_GROUP &&
			 ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_MULTI_QUEUE_PRIORITY))
		return -EINVAL;

	idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs));
	if (!exec_queue_set_property_funcs[idx])
		return -EINVAL;

	*properties |= BIT_ULL(idx);
	err = exec_queue_user_ext_check(q, *properties);
	if (XE_IOCTL_DBG(xe, err))
		return err;

	return exec_queue_set_property_funcs[idx](xe, q, ext.value);
}

typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe,
					       struct xe_exec_queue *q,
					       u64 extension, u64 *properties);

static const xe_exec_queue_user_extension_fn exec_queue_user_extension_funcs[] = {
	[DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property,
};

#define MAX_USER_EXTENSIONS	16
static int __exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
					u64 extensions, int ext_number, u64 *properties)
{
	u64 __user *address = u64_to_user_ptr(extensions);
	struct drm_xe_user_extension ext;
	int err;
	u32 idx;

	if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS))
		return -E2BIG;

	err = copy_from_user(&ext, address, sizeof(ext));
	if (XE_IOCTL_DBG(xe, err))
		return -EFAULT;

	if (XE_IOCTL_DBG(xe, ext.pad) ||
	    XE_IOCTL_DBG(xe, ext.name >=
			 ARRAY_SIZE(exec_queue_user_extension_funcs)))
		return -EINVAL;

	idx = array_index_nospec(ext.name,
				 ARRAY_SIZE(exec_queue_user_extension_funcs));
	err = exec_queue_user_extension_funcs[idx](xe, q, extensions, properties);
	if (XE_IOCTL_DBG(xe, err))
		return err;

	if (ext.next_extension)
		return __exec_queue_user_extensions(xe, q, ext.next_extension,
						    ++ext_number, properties);

	return 0;
}

static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q,
				      u64 extensions)
{
	u64 properties = 0;
	int err;

	err = __exec_queue_user_extensions(xe, q, extensions, 0, &properties);
	if (XE_IOCTL_DBG(xe, err))
		return err;

	err = exec_queue_user_ext_check_final(q, properties);
	if (XE_IOCTL_DBG(xe, err))
		return err;

	if (xe_exec_queue_is_multi_queue_primary(q)) {
		err = xe_exec_queue_group_init(xe, q);
		if (XE_IOCTL_DBG(xe, err))
			return err;
	}

	return 0;
}

static u32 calc_validate_logical_mask(struct xe_device *xe,
				      struct drm_xe_engine_class_instance *eci,
				      u16 width, u16 num_placements)
{
	int len = width * num_placements;
	int i, j, n;
	u16 class;
	u16 gt_id;
	u32 return_mask = 0, prev_mask;

	if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe) &&
			 len > 1))
		return 0;

	for (i = 0; i < width; ++i) {
		u32 current_mask = 0;

		for (j = 0; j < num_placements; ++j) {
			struct xe_hw_engine *hwe;

			n = j * width + i;

			hwe = xe_hw_engine_lookup(xe, eci[n]);
			if (XE_IOCTL_DBG(xe, !hwe))
				return 0;

			if (XE_IOCTL_DBG(xe, xe_hw_engine_is_reserved(hwe)))
				return 0;

			if (XE_IOCTL_DBG(xe, n && eci[n].gt_id != gt_id) ||
			    XE_IOCTL_DBG(xe, n && eci[n].engine_class != class))
				return 0;

			class = eci[n].engine_class;
			gt_id = eci[n].gt_id;

			if (width == 1 || !i)
				return_mask |= BIT(eci[n].engine_instance);
			current_mask |= BIT(eci[n].engine_instance);
		}

		/* Parallel submissions must be logically contiguous */
		if (i && XE_IOCTL_DBG(xe, current_mask != prev_mask << 1))
			return 0;

		prev_mask = current_mask;
	}

	return return_mask;
}

static bool has_sched_groups(struct xe_gt *gt)
{
	if (IS_SRIOV_PF(gt_to_xe(gt)) && xe_gt_sriov_pf_sched_groups_enabled(gt))
		return true;

	if (IS_SRIOV_VF(gt_to_xe(gt)) && xe_gt_sriov_vf_sched_groups_enabled(gt))
		return true;

	return false;
}

int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data,
			       struct drm_file *file)
{
	struct xe_device *xe = to_xe_device(dev);
	struct xe_file *xef = to_xe_file(file);
	struct drm_xe_exec_queue_create *args = data;
	struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE];
	struct drm_xe_engine_class_instance __user *user_eci =
		u64_to_user_ptr(args->instances);
	struct xe_hw_engine *hwe;
	struct xe_vm *vm;
	struct xe_tile *tile;
	struct xe_exec_queue *q = NULL;
	u32 logical_mask;
	u32 flags = 0;
	u32 id;
	u32 len;
	int err;

	if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) ||
	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
		return -EINVAL;

	len = args->width * args->num_placements;
	if (XE_IOCTL_DBG(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE))
		return -EINVAL;

	err = copy_from_user(eci, user_eci,
			     sizeof(struct drm_xe_engine_class_instance) * len);
	if (XE_IOCTL_DBG(xe, err))
		return -EFAULT;

	if (XE_IOCTL_DBG(xe, !xe_device_get_gt(xe, eci[0].gt_id)))
		return -EINVAL;

	if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT)
		flags |= EXEC_QUEUE_FLAG_LOW_LATENCY;

	if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) {
		if (XE_IOCTL_DBG(xe, args->width != 1) ||
		    XE_IOCTL_DBG(xe, args->num_placements != 1) ||
		    XE_IOCTL_DBG(xe, eci[0].engine_instance != 0))
			return -EINVAL;

		vm = xe_vm_lookup(xef, args->vm_id);
		if (XE_IOCTL_DBG(xe, !vm))
			return -ENOENT;

		err = down_read_interruptible(&vm->lock);
		if (err) {
			xe_vm_put(vm);
			return err;
		}

		if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
			up_read(&vm->lock);
			xe_vm_put(vm);
			return -ENOENT;
		}

		for_each_tile(tile, xe, id) {
			struct xe_exec_queue *new;

			flags |= EXEC_QUEUE_FLAG_VM;
			if (id)
				flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD;

			new = xe_exec_queue_create_bind(xe, tile, vm, flags,
							args->extensions);
			if (IS_ERR(new)) {
				up_read(&vm->lock);
				xe_vm_put(vm);
				err = PTR_ERR(new);
				if (q)
					goto put_exec_queue;
				return err;
			}
			if (id == 0)
				q = new;
			else
				list_add_tail(&new->multi_gt_list,
					      &q->multi_gt_link);
		}
		up_read(&vm->lock);
		xe_vm_put(vm);
	} else {
		logical_mask = calc_validate_logical_mask(xe, eci,
							  args->width,
							  args->num_placements);
		if (XE_IOCTL_DBG(xe, !logical_mask))
			return -EINVAL;

		hwe = xe_hw_engine_lookup(xe, eci[0]);
		if (XE_IOCTL_DBG(xe, !hwe))
			return -EINVAL;

		vm = xe_vm_lookup(xef, args->vm_id);
		if (XE_IOCTL_DBG(xe, !vm))
			return -ENOENT;

		err = down_read_interruptible(&vm->lock);
		if (err) {
			xe_vm_put(vm);
			return err;
		}

		if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) {
			up_read(&vm->lock);
			xe_vm_put(vm);
			return -ENOENT;
		}

		/* SRIOV sched groups are not compatible with multi-lrc */
		if (XE_IOCTL_DBG(xe, args->width > 1 && has_sched_groups(hwe->gt))) {
			up_read(&vm->lock);
			xe_vm_put(vm);
			return -EINVAL;
		}

		q = xe_exec_queue_create(xe, vm, logical_mask,
					 args->width, hwe, flags,
					 args->extensions);
		up_read(&vm->lock);
		xe_vm_put(vm);
		if (IS_ERR(q))
			return PTR_ERR(q);

		if (xe_exec_queue_is_multi_queue_secondary(q)) {
			err = xe_exec_queue_group_add(xe, q);
			if (XE_IOCTL_DBG(xe, err))
				goto put_exec_queue;
		}

		if (xe_vm_in_preempt_fence_mode(vm)) {
			q->lr.context = dma_fence_context_alloc(1);

			err = xe_vm_add_compute_exec_queue(vm, q);
			if (XE_IOCTL_DBG(xe, err))
				goto delete_queue_group;
		}

		if (q->vm && q->hwe->hw_engine_group) {
			err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q);
			if (err)
				goto put_exec_queue;
		}
	}

	q->xef = xe_file_get(xef);

	/* user id alloc must always be last in ioctl to prevent UAF */
	err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL);
	if (err)
		goto kill_exec_queue;

	args->exec_queue_id = id;

	return 0;

kill_exec_queue:
	xe_exec_queue_kill(q);
delete_queue_group:
	if (xe_exec_queue_is_multi_queue_secondary(q))
		xe_exec_queue_group_delete(xe, q);
put_exec_queue:
	xe_exec_queue_put(q);
	return err;
}

int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data,
				     struct drm_file *file)
{
	struct xe_device *xe = to_xe_device(dev);
	struct xe_file *xef = to_xe_file(file);
	struct drm_xe_exec_queue_get_property *args = data;
	struct xe_exec_queue *q;
	int ret;

	if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
		return -EINVAL;

	q = xe_exec_queue_lookup(xef, args->exec_queue_id);
	if (XE_IOCTL_DBG(xe, !q))
		return -ENOENT;

	switch (args->property) {
	case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN:
		args->value = q->ops->reset_status(q);
		ret = 0;
		break;
	default:
		ret = -EINVAL;
	}

	xe_exec_queue_put(q);

	return ret;
}

/**
 * xe_exec_queue_lrc() - Get the LRC from exec queue.
 * @q: The exec_queue.
 *
 * Retrieves the primary LRC for the exec queue. Note that this function
 * returns only the first LRC instance, even when multiple parallel LRCs
 * are configured.
 *
 * Return: Pointer to LRC on success, error on failure
 */
struct xe_lrc *xe_exec_queue_lrc(struct xe_exec_queue *q)
{
	return q->lrc[0];
}

/**
 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running
 * @q: The exec_queue
 *
 * Return: True if the exec_queue is long-running, false otherwise.
 */
bool xe_exec_queue_is_lr(struct xe_exec_queue *q)
{
	return q->vm && xe_vm_in_lr_mode(q->vm) &&
		!(q->flags & EXEC_QUEUE_FLAG_VM);
}

/**
 * xe_exec_queue_is_idle() - Whether an exec_queue is idle.
 * @q: The exec_queue
 *
 * FIXME: Need to determine what to use as the short-lived
 * timeline lock for the exec_queues, so that the return value
 * of this function becomes more than just an advisory
 * snapshot in time. The timeline lock must protect the
 * seqno from racing submissions on the same exec_queue.
 * Typically vm->resv, but user-created timeline locks use the migrate vm
 * and never grabs the migrate vm->resv so we have a race there.
 *
 * Return: True if the exec_queue is idle, false otherwise.
 */
bool xe_exec_queue_is_idle(struct xe_exec_queue *q)
{
	if (xe_exec_queue_is_parallel(q)) {
		int i;

		for (i = 0; i < q->width; ++i) {
			if (xe_lrc_seqno(q->lrc[i]) !=
			    q->lrc[i]->fence_ctx.next_seqno - 1)
				return false;
		}

		return true;
	}

	return xe_lrc_seqno(q->lrc[0]) ==
		q->lrc[0]->fence_ctx.next_seqno - 1;
}

/**
 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue
 * from hw
 * @q: The exec queue
 *
 * Update the timestamp saved by HW for this exec queue and save run ticks
 * calculated by using the delta from last update.
 */
void xe_exec_queue_update_run_ticks(struct xe_exec_queue *q)
{
	struct xe_device *xe = gt_to_xe(q->gt);
	struct xe_lrc *lrc;
	u64 old_ts, new_ts;
	int idx;

	/*
	 * Jobs that are executed by kernel doesn't have a corresponding xe_file
	 * and thus are not accounted.
	 */
	if (!q->xef)
		return;

	/* Synchronize with unbind while holding the xe file open */
	if (!drm_dev_enter(&xe->drm, &idx))
		return;
	/*
	 * Only sample the first LRC. For parallel submission, all of them are
	 * scheduled together and we compensate that below by multiplying by
	 * width - this may introduce errors if that premise is not true and
	 * they don't exit 100% aligned. On the other hand, looping through
	 * the LRCs and reading them in different time could also introduce
	 * errors.
	 */
	lrc = q->lrc[0];
	new_ts = xe_lrc_update_timestamp(lrc, &old_ts);
	q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width;

	drm_dev_exit(idx);
}

/**
 * xe_exec_queue_kill - permanently stop all execution from an exec queue
 * @q: The exec queue
 *
 * This function permanently stops all activity on an exec queue. If the queue
 * is actively executing on the HW, it will be kicked off the engine; any
 * pending jobs are discarded and all future submissions are rejected.
 * This function is safe to call multiple times.
 */
void xe_exec_queue_kill(struct xe_exec_queue *q)
{
	struct xe_exec_queue *eq = q, *next;

	list_for_each_entry_safe(eq, next, &eq->multi_gt_list,
				 multi_gt_link) {
		q->ops->kill(eq);
		xe_vm_remove_compute_exec_queue(q->vm, eq);
	}

	q->ops->kill(q);
	xe_vm_remove_compute_exec_queue(q->vm, q);
}

int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data,
				struct drm_file *file)
{
	struct xe_device *xe = to_xe_device(dev);
	struct xe_file *xef = to_xe_file(file);
	struct drm_xe_exec_queue_destroy *args = data;
	struct xe_exec_queue *q;

	if (XE_IOCTL_DBG(xe, args->pad) ||
	    XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1]))
		return -EINVAL;

	mutex_lock(&xef->exec_queue.lock);
	q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id);
	if (q)
		atomic_inc(&xef->exec_queue.pending_removal);
	mutex_unlock(&xef->exec_queue.lock);

	if (XE_IOCTL_DBG(xe, !q))
		return -ENOENT;

	if (q->vm && q->hwe->hw_engine_group)
		xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);

	xe_exec_queue_kill(q);

	trace_xe_exec_queue_close(q);
	xe_exec_queue_put(q);

	return 0;
}

static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q,
						    struct xe_vm *vm)
{
	if (q->flags & EXEC_QUEUE_FLAG_MIGRATE) {
		xe_migrate_job_lock_assert(q);
	} else if (q->flags & EXEC_QUEUE_FLAG_VM) {
		lockdep_assert_held(&vm->lock);
	} else {
		xe_vm_assert_held(vm);
		lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem);
	}
}

/**
 * xe_exec_queue_last_fence_put() - Drop ref to last fence
 * @q: The exec queue
 * @vm: The VM the engine does a bind or exec for
 */
void xe_exec_queue_last_fence_put(struct xe_exec_queue *q, struct xe_vm *vm)
{
	xe_exec_queue_last_fence_lockdep_assert(q, vm);

	xe_exec_queue_last_fence_put_unlocked(q);
}

/**
 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked
 * @q: The exec queue
 *
 * Only safe to be called from xe_exec_queue_destroy().
 */
void xe_exec_queue_last_fence_put_unlocked(struct xe_exec_queue *q)
{
	if (q->last_fence) {
		dma_fence_put(q->last_fence);
		q->last_fence = NULL;
	}
}

/**
 * xe_exec_queue_last_fence_get() - Get last fence
 * @q: The exec queue
 * @vm: The VM the engine does a bind or exec for
 *
 * Get last fence, takes a ref
 *
 * Returns: last fence if not signaled, dma fence stub if signaled
 */
struct dma_fence *xe_exec_queue_last_fence_get(struct xe_exec_queue *q,
					       struct xe_vm *vm)
{
	struct dma_fence *fence;

	xe_exec_queue_last_fence_lockdep_assert(q, vm);

	if (q->last_fence &&
	    test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags))
		xe_exec_queue_last_fence_put(q, vm);

	fence = q->last_fence ? q->last_fence : dma_fence_get_stub();
	dma_fence_get(fence);
	return fence;
}

/**
 * xe_exec_queue_last_fence_get_for_resume() - Get last fence
 * @q: The exec queue
 * @vm: The VM the engine does a bind or exec for
 *
 * Get last fence, takes a ref. Only safe to be called in the context of
 * resuming the hw engine group's long-running exec queue, when the group
 * semaphore is held.
 *
 * Returns: last fence if not signaled, dma fence stub if signaled
 */
struct dma_fence *xe_exec_queue_last_fence_get_for_resume(struct xe_exec_queue *q,
							  struct xe_vm *vm)
{
	struct dma_fence *fence;

	lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem);

	if (q->last_fence &&
	    test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags))
		xe_exec_queue_last_fence_put_unlocked(q);

	fence = q->last_fence ? q->last_fence : dma_fence_get_stub();
	dma_fence_get(fence);
	return fence;
}

/**
 * xe_exec_queue_last_fence_set() - Set last fence
 * @q: The exec queue
 * @vm: The VM the engine does a bind or exec for
 * @fence: The fence
 *
 * Set the last fence for the engine. Increases reference count for fence, when
 * closing engine xe_exec_queue_last_fence_put should be called.
 */
void xe_exec_queue_last_fence_set(struct xe_exec_queue *q, struct xe_vm *vm,
				  struct dma_fence *fence)
{
	xe_exec_queue_last_fence_lockdep_assert(q, vm);
	xe_assert(vm->xe, !dma_fence_is_container(fence));

	xe_exec_queue_last_fence_put(q, vm);
	q->last_fence = dma_fence_get(fence);
}

/**
 * xe_exec_queue_tlb_inval_last_fence_put() - Drop ref to last TLB invalidation fence
 * @q: The exec queue
 * @vm: The VM the engine does a bind for
 * @type: Either primary or media GT
 */
void xe_exec_queue_tlb_inval_last_fence_put(struct xe_exec_queue *q,
					    struct xe_vm *vm,
					    unsigned int type)
{
	xe_exec_queue_last_fence_lockdep_assert(q, vm);
	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);

	xe_exec_queue_tlb_inval_last_fence_put_unlocked(q, type);
}

/**
 * xe_exec_queue_tlb_inval_last_fence_put_unlocked() - Drop ref to last TLB
 * invalidation fence unlocked
 * @q: The exec queue
 * @type: Either primary or media GT
 *
 * Only safe to be called from xe_exec_queue_destroy().
 */
void xe_exec_queue_tlb_inval_last_fence_put_unlocked(struct xe_exec_queue *q,
						     unsigned int type)
{
	xe_assert(q->vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);

	dma_fence_put(q->tlb_inval[type].last_fence);
	q->tlb_inval[type].last_fence = NULL;
}

/**
 * xe_exec_queue_tlb_inval_last_fence_get() - Get last fence for TLB invalidation
 * @q: The exec queue
 * @vm: The VM the engine does a bind for
 * @type: Either primary or media GT
 *
 * Get last fence, takes a ref
 *
 * Returns: last fence if not signaled, dma fence stub if signaled
 */
struct dma_fence *xe_exec_queue_tlb_inval_last_fence_get(struct xe_exec_queue *q,
							 struct xe_vm *vm,
							 unsigned int type)
{
	struct dma_fence *fence;

	xe_exec_queue_last_fence_lockdep_assert(q, vm);
	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
	xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM |
				      EXEC_QUEUE_FLAG_MIGRATE));

	if (q->tlb_inval[type].last_fence &&
	    test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
		     &q->tlb_inval[type].last_fence->flags))
		xe_exec_queue_tlb_inval_last_fence_put(q, vm, type);

	fence = q->tlb_inval[type].last_fence ?: dma_fence_get_stub();
	dma_fence_get(fence);
	return fence;
}

/**
 * xe_exec_queue_tlb_inval_last_fence_set() - Set last fence for TLB invalidation
 * @q: The exec queue
 * @vm: The VM the engine does a bind for
 * @fence: The fence
 * @type: Either primary or media GT
 *
 * Set the last fence for the tlb invalidation type on the queue. Increases
 * reference count for fence, when closing queue
 * xe_exec_queue_tlb_inval_last_fence_put should be called.
 */
void xe_exec_queue_tlb_inval_last_fence_set(struct xe_exec_queue *q,
					    struct xe_vm *vm,
					    struct dma_fence *fence,
					    unsigned int type)
{
	xe_exec_queue_last_fence_lockdep_assert(q, vm);
	xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT ||
		  type == XE_EXEC_QUEUE_TLB_INVAL_PRIMARY_GT);
	xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM |
				      EXEC_QUEUE_FLAG_MIGRATE));
	xe_assert(vm->xe, !dma_fence_is_container(fence));

	xe_exec_queue_tlb_inval_last_fence_put(q, vm, type);
	q->tlb_inval[type].last_fence = dma_fence_get(fence);
}

/**
 * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references
 * within all LRCs of a queue.
 * @q: the &xe_exec_queue struct instance containing target LRCs
 * @scratch: scratch buffer to be used as temporary storage
 *
 * Returns: zero on success, negative error code on failure
 */
int xe_exec_queue_contexts_hwsp_rebase(struct xe_exec_queue *q, void *scratch)
{
	int i;
	int err = 0;

	for (i = 0; i < q->width; ++i) {
		struct xe_lrc *lrc;

		/* Pairs with WRITE_ONCE in __xe_exec_queue_init  */
		lrc = READ_ONCE(q->lrc[i]);
		if (!lrc)
			continue;

		xe_lrc_update_memirq_regs_with_address(lrc, q->hwe, scratch);
		xe_lrc_update_hwctx_regs_with_address(lrc);
		err = xe_lrc_setup_wa_bb_with_scratch(lrc, q->hwe, scratch);
		if (err)
			break;
	}

	return err;
}