Lines Matching +full:multi +full:- +full:function
1 // SPDX-License-Identifier: MIT
57 * DOC: Multi Queue Group
59 * Multi Queue Group is another mode of execution supported by the compute
63 * execution of multiple queues within a single shared context. The multi
68 * supports creating a multi queue group and adding queues to a queue group.
71 * set to DRM_XE_MULTI_GROUP_CREATE, will create a new multi queue group with
78 * The hardware does not support removing a queue from a multi-queue group.
88 * The queues of a multi queue group can set their priority within the group
90 * This multi queue priority can also be set dynamically through the
92 * supported by the secondary queues of a multi queue group, other than
95 * When GuC reports an error on any of the queues of a multi queue group,
97 * as hardware cannot make progress on the multi queue context.
99 * Refer :ref:`multi-queue-group-guc-interface` for multi queue group GuC
115 struct xe_exec_queue_group *group = q->multi_queue.group; in xe_exec_queue_group_cleanup()
132 xa_for_each(&group->xa, idx, lrc) in xe_exec_queue_group_cleanup()
135 xa_destroy(&group->xa); in xe_exec_queue_group_cleanup()
136 mutex_destroy(&group->list_lock); in xe_exec_queue_group_cleanup()
137 xe_bo_unpin_map_no_vm(group->cgp_bo); in xe_exec_queue_group_cleanup()
146 if (q->tlb_inval[i].dep_scheduler) in __xe_exec_queue_free()
147 xe_dep_scheduler_fini(q->tlb_inval[i].dep_scheduler); in __xe_exec_queue_free()
150 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); in __xe_exec_queue_free()
155 if (q->vm) { in __xe_exec_queue_free()
156 xe_vm_remove_exec_queue(q->vm, q); in __xe_exec_queue_free()
157 xe_vm_put(q->vm); in __xe_exec_queue_free()
160 if (q->xef) in __xe_exec_queue_free()
161 xe_file_put(q->xef); in __xe_exec_queue_free()
163 kvfree(q->replay_state); in __xe_exec_queue_free()
169 struct xe_tile *tile = gt_to_tile(q->gt); in alloc_dep_schedulers()
178 gt = tile->primary_gt; in alloc_dep_schedulers()
180 gt = tile->media_gt; in alloc_dep_schedulers()
185 wq = gt->tlb_inval.job_wq; in alloc_dep_schedulers()
188 dep_scheduler = xe_dep_scheduler_create(xe, wq, q->name, in alloc_dep_schedulers()
193 q->tlb_inval[i].dep_scheduler = dep_scheduler; in alloc_dep_schedulers()
207 struct xe_gt *gt = hwe->gt; in __xe_exec_queue_alloc()
215 return ERR_PTR(-ENOMEM); in __xe_exec_queue_alloc()
217 kref_init(&q->refcount); in __xe_exec_queue_alloc()
218 q->flags = flags; in __xe_exec_queue_alloc()
219 q->hwe = hwe; in __xe_exec_queue_alloc()
220 q->gt = gt; in __xe_exec_queue_alloc()
221 q->class = hwe->class; in __xe_exec_queue_alloc()
222 q->width = width; in __xe_exec_queue_alloc()
223 q->msix_vec = XE_IRQ_DEFAULT_MSIX; in __xe_exec_queue_alloc()
224 q->logical_mask = logical_mask; in __xe_exec_queue_alloc()
225 q->fence_irq = >->fence_irq[hwe->class]; in __xe_exec_queue_alloc()
226 q->ring_ops = gt->ring_ops[hwe->class]; in __xe_exec_queue_alloc()
227 q->ops = gt->exec_queue_ops; in __xe_exec_queue_alloc()
228 INIT_LIST_HEAD(&q->lr.link); in __xe_exec_queue_alloc()
229 INIT_LIST_HEAD(&q->vm_exec_queue_link); in __xe_exec_queue_alloc()
230 INIT_LIST_HEAD(&q->multi_gt_link); in __xe_exec_queue_alloc()
231 INIT_LIST_HEAD(&q->hw_engine_group_link); in __xe_exec_queue_alloc()
232 INIT_LIST_HEAD(&q->pxp.link); in __xe_exec_queue_alloc()
233 spin_lock_init(&q->multi_queue.lock); in __xe_exec_queue_alloc()
234 spin_lock_init(&q->lrc_lookup_lock); in __xe_exec_queue_alloc()
235 q->multi_queue.priority = XE_MULTI_QUEUE_PRIORITY_NORMAL; in __xe_exec_queue_alloc()
237 q->sched_props.timeslice_us = hwe->eclass->sched_props.timeslice_us; in __xe_exec_queue_alloc()
238 q->sched_props.preempt_timeout_us = in __xe_exec_queue_alloc()
239 hwe->eclass->sched_props.preempt_timeout_us; in __xe_exec_queue_alloc()
240 q->sched_props.job_timeout_ms = in __xe_exec_queue_alloc()
241 hwe->eclass->sched_props.job_timeout_ms; in __xe_exec_queue_alloc()
242 if (q->flags & EXEC_QUEUE_FLAG_KERNEL && in __xe_exec_queue_alloc()
243 q->flags & EXEC_QUEUE_FLAG_HIGH_PRIORITY) in __xe_exec_queue_alloc()
244 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; in __xe_exec_queue_alloc()
246 q->sched_props.priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; in __xe_exec_queue_alloc()
248 if (q->flags & (EXEC_QUEUE_FLAG_MIGRATE | EXEC_QUEUE_FLAG_VM)) { in __xe_exec_queue_alloc()
257 q->vm = xe_vm_get(vm); in __xe_exec_queue_alloc()
261 * may set q->usm, must come before xe_lrc_create(), in __xe_exec_queue_alloc()
262 * may overwrite q->sched_props, must come before q->ops->init() in __xe_exec_queue_alloc()
276 xe_assert(gt_to_xe(q->gt), idx < q->width); in xe_exec_queue_set_lrc()
278 scoped_guard(spinlock, &q->lrc_lookup_lock) in xe_exec_queue_set_lrc()
279 q->lrc[idx] = lrc; in xe_exec_queue_set_lrc()
283 * xe_exec_queue_get_lrc() - Get the LRC from exec queue.
285 * @idx: Index within multi-LRC array.
297 xe_assert(gt_to_xe(q->gt), idx < q->width); in xe_exec_queue_get_lrc()
299 scoped_guard(spinlock, &q->lrc_lookup_lock) { in xe_exec_queue_get_lrc()
300 lrc = q->lrc[idx]; in xe_exec_queue_get_lrc()
309 * xe_exec_queue_lrc() - Get the LRC from exec queue.
312 * Retrieves the primary LRC for the exec queue. Note that this function
314 * are configured. This function does not increment reference count,
321 return q->lrc[0]; in xe_exec_queue_lrc()
328 q->ops->fini(q); in __xe_exec_queue_fini()
330 for (i = 0; i < q->width; ++i) in __xe_exec_queue_fini()
331 xe_lrc_put(q->lrc[i]); in __xe_exec_queue_fini()
346 (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { in __xe_exec_queue_init()
347 if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) in __xe_exec_queue_init()
356 if (q->flags & EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX) in __xe_exec_queue_init()
359 err = q->ops->init(q); in __xe_exec_queue_init()
364 * This must occur after q->ops->init to avoid race conditions during VF in __xe_exec_queue_init()
365 * post-migration recovery, as the fixups for the LRC GGTT addresses in __xe_exec_queue_init()
370 * (without a race) only if the function can detect migration instantly in __xe_exec_queue_init()
373 for (i = 0; i < q->width; ++i) { in __xe_exec_queue_init()
380 marker = xe_gt_sriov_vf_wait_valid_ggtt(q->gt); in __xe_exec_queue_init()
382 lrc = xe_lrc_create(q->hwe, q->vm, q->replay_state, in __xe_exec_queue_init()
383 xe_lrc_ring_size(), q->msix_vec, flags); in __xe_exec_queue_init()
395 } while (marker != xe_vf_migration_fixups_complete_count(q->gt)); in __xe_exec_queue_init()
406 * xe_exec_queue_create() - Create an exec queue
428 …xe_assert(xe, !vm || (!!(vm->flags & XE_VM_FLAG_GSC) == !!(hwe->engine_id == XE_HW_ENGINE_GSCCS0))… in xe_exec_queue_create()
442 * go bad if the queue is only half-initialized. This means that we in xe_exec_queue_create()
447 err = xe_pxp_exec_queue_add(xe->pxp, q); in xe_exec_queue_create()
463 * xe_exec_queue_create_class() - Create an exec queue for a specific engine class
488 if (hwe->class == class) { in xe_exec_queue_create_class()
489 logical_mask |= BIT(hwe->logical_instance); in xe_exec_queue_create_class()
496 return ERR_PTR(-ENODEV); in xe_exec_queue_create_class()
502 * xe_exec_queue_create_bind() - Create bind exec queue.
513 * resolve faults depend on user binds). On non-faulting devices any copy engine
523 struct xe_gt *gt = tile->primary_gt; in xe_exec_queue_create_bind()
527 migrate_vm = xe_migrate_get_vm(tile->migrate); in xe_exec_queue_create_bind()
528 if (xe->info.has_usm) { in xe_exec_queue_create_bind()
531 gt->usm.reserved_bcs_instance, in xe_exec_queue_create_bind()
536 return ERR_PTR(-EINVAL); in xe_exec_queue_create_bind()
540 BIT(hwe->logical_instance), 1, hwe, in xe_exec_queue_create_bind()
550 int err = drm_syncobj_create(&q->ufence_syncobj, in xe_exec_queue_create_bind()
559 q->user_vm = xe_vm_get(user_vm); in xe_exec_queue_create_bind()
567 * xe_exec_queue_destroy() - Destroy an exec queue
572 * This function should not be called directly; use xe_exec_queue_put() instead.
580 xe_assert(gt_to_xe(q->gt), atomic_read(&q->job_cnt) == 0); in xe_exec_queue_destroy()
582 if (q->ufence_syncobj) in xe_exec_queue_destroy()
583 drm_syncobj_put(q->ufence_syncobj); in xe_exec_queue_destroy()
586 xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); in xe_exec_queue_destroy()
592 if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { in xe_exec_queue_destroy()
593 list_for_each_entry_safe(eq, next, &q->multi_gt_list, in xe_exec_queue_destroy()
598 if (q->user_vm) { in xe_exec_queue_destroy()
599 xe_vm_put(q->user_vm); in xe_exec_queue_destroy()
600 q->user_vm = NULL; in xe_exec_queue_destroy()
603 q->ops->destroy(q); in xe_exec_queue_destroy()
607 * xe_exec_queue_fini() - Finalize an exec queue
621 if (q->xef && atomic_dec_and_test(&q->xef->exec_queue.pending_removal)) in xe_exec_queue_fini()
622 wake_up_var(&q->xef->exec_queue.pending_removal); in xe_exec_queue_fini()
629 * xe_exec_queue_assign_name() - Assign a name to an exec queue
633 * Assigns a human-readable name to the exec queue based on its engine class
638 switch (q->class) { in xe_exec_queue_assign_name()
640 snprintf(q->name, sizeof(q->name), "rcs%d", instance); in xe_exec_queue_assign_name()
643 snprintf(q->name, sizeof(q->name), "vcs%d", instance); in xe_exec_queue_assign_name()
646 snprintf(q->name, sizeof(q->name), "vecs%d", instance); in xe_exec_queue_assign_name()
649 snprintf(q->name, sizeof(q->name), "bcs%d", instance); in xe_exec_queue_assign_name()
652 snprintf(q->name, sizeof(q->name), "ccs%d", instance); in xe_exec_queue_assign_name()
655 snprintf(q->name, sizeof(q->name), "gsccs%d", instance); in xe_exec_queue_assign_name()
658 XE_WARN_ON(q->class); in xe_exec_queue_assign_name()
663 * xe_exec_queue_lookup() - Look up an exec queue by ID
675 mutex_lock(&xef->exec_queue.lock); in xe_exec_queue_lookup()
676 q = xa_load(&xef->exec_queue.xa, id); in xe_exec_queue_lookup()
679 mutex_unlock(&xef->exec_queue.lock); in xe_exec_queue_lookup()
685 * xe_exec_queue_device_get_max_priority() - Get maximum priority for an exec queues
703 return -EINVAL; in exec_queue_set_priority()
706 return -EPERM; in exec_queue_set_priority()
708 q->sched_props.priority = value; in exec_queue_set_priority()
728 *min = eclass->sched_props.job_timeout_min; in xe_exec_queue_get_prop_minmax()
729 *max = eclass->sched_props.job_timeout_max; in xe_exec_queue_get_prop_minmax()
732 *min = eclass->sched_props.timeslice_min; in xe_exec_queue_get_prop_minmax()
733 *max = eclass->sched_props.timeslice_max; in xe_exec_queue_get_prop_minmax()
736 *min = eclass->sched_props.preempt_timeout_min; in xe_exec_queue_get_prop_minmax()
737 *max = eclass->sched_props.preempt_timeout_max; in xe_exec_queue_get_prop_minmax()
769 xe_exec_queue_get_prop_minmax(q->hwe->eclass, in exec_queue_set_timeslice()
774 return -EINVAL; in exec_queue_set_timeslice()
776 q->sched_props.timeslice_us = value; in exec_queue_set_timeslice()
788 return -EINVAL; in exec_queue_set_pxp_type()
790 if (!xe_pxp_is_enabled(xe->pxp)) in exec_queue_set_pxp_type()
791 return -ENODEV; in exec_queue_set_pxp_type()
793 return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM); in exec_queue_set_pxp_type()
800 size_t size = xe_gt_lrc_hang_replay_size(q->gt, q->class); in exec_queue_set_hang_replay_state()
808 q->replay_state = ptr; in exec_queue_set_hang_replay_state()
815 struct xe_tile *tile = gt_to_tile(q->gt); in xe_exec_queue_group_init()
821 return -ENOMEM; in xe_exec_queue_group_init()
830 drm_err(&xe->drm, "CGP bo allocation for queue group failed: %ld\n", in xe_exec_queue_group_init()
836 xe_map_memset(xe, &bo->vmap, 0, 0, SZ_4K); in xe_exec_queue_group_init()
838 group->primary = q; in xe_exec_queue_group_init()
839 group->cgp_bo = bo; in xe_exec_queue_group_init()
840 INIT_LIST_HEAD(&group->list); in xe_exec_queue_group_init()
841 xa_init_flags(&group->xa, XA_FLAGS_ALLOC1); in xe_exec_queue_group_init()
842 mutex_init(&group->list_lock); in xe_exec_queue_group_init()
843 q->multi_queue.group = group; in xe_exec_queue_group_init()
845 /* group->list_lock is used in submission backend */ in xe_exec_queue_group_init()
848 might_lock(&group->list_lock); in xe_exec_queue_group_init()
857 return q->gt->info.multi_queue_engine_class_mask & BIT(q->class); in xe_exec_queue_supports_multi_queue()
871 primary = xe_exec_queue_lookup(q->vm->xef, primary_id); in xe_exec_queue_group_validate()
873 return -ENOENT; in xe_exec_queue_group_validate()
876 XE_IOCTL_DBG(xe, q->vm != primary->vm) || in xe_exec_queue_group_validate()
877 XE_IOCTL_DBG(xe, q->logical_mask != primary->logical_mask)) { in xe_exec_queue_group_validate()
878 ret = -EINVAL; in xe_exec_queue_group_validate()
882 group = primary->multi_queue.group; in xe_exec_queue_group_validate()
883 q->multi_queue.valid = true; in xe_exec_queue_group_validate()
884 q->multi_queue.group = group; in xe_exec_queue_group_validate()
895 struct xe_exec_queue_group *group = q->multi_queue.group; in xe_exec_queue_group_add()
902 err = xa_alloc(&group->xa, &pos, xe_lrc_get(q->lrc[0]), in xe_exec_queue_group_add()
903 XA_LIMIT(1, XE_MAX_GROUP_SIZE - 1), GFP_KERNEL); in xe_exec_queue_group_add()
905 xe_lrc_put(q->lrc[0]); in xe_exec_queue_group_add()
908 if (err == -EBUSY) in xe_exec_queue_group_add()
909 err = -EINVAL; in xe_exec_queue_group_add()
914 q->multi_queue.pos = pos; in xe_exec_queue_group_add()
921 struct xe_exec_queue_group *group = q->multi_queue.group; in xe_exec_queue_group_delete()
926 lrc = xa_erase(&group->xa, q->multi_queue.pos); in xe_exec_queue_group_delete()
935 return -ENODEV; in exec_queue_set_multi_group()
938 return -EOPNOTSUPP; in exec_queue_set_multi_group()
940 if (XE_IOCTL_DBG(xe, !q->vm->xef)) in exec_queue_set_multi_group()
941 return -EINVAL; in exec_queue_set_multi_group()
944 return -EINVAL; in exec_queue_set_multi_group()
947 return -EINVAL; in exec_queue_set_multi_group()
951 return -EINVAL; in exec_queue_set_multi_group()
953 q->multi_queue.valid = true; in exec_queue_set_multi_group()
954 q->multi_queue.is_primary = true; in exec_queue_set_multi_group()
955 q->multi_queue.pos = 0; in exec_queue_set_multi_group()
961 return -EINVAL; in exec_queue_set_multi_group()
970 return -EINVAL; in exec_queue_set_multi_queue_priority()
972 /* For queue creation time (!q->xef) setting, just store the priority value */ in exec_queue_set_multi_queue_priority()
973 if (!q->xef) { in exec_queue_set_multi_queue_priority()
974 q->multi_queue.priority = value; in exec_queue_set_multi_queue_priority()
979 return -EINVAL; in exec_queue_set_multi_queue_priority()
981 return q->ops->set_multi_queue_priority(q, value); in exec_queue_set_multi_queue_priority()
987 if (XE_IOCTL_DBG(xe, q->class != XE_ENGINE_CLASS_RENDER)) in exec_queue_set_state_cache_perf_fix()
988 return -EOPNOTSUPP; in exec_queue_set_state_cache_perf_fix()
990 q->flags |= value != 0 ? EXEC_QUEUE_FLAG_DISABLE_STATE_CACHE_PERF_FIX : 0; in exec_queue_set_state_cache_perf_fix()
1012 * xe_exec_queue_set_property_ioctl() - Set a property on an exec queue
1018 * supports setting multi-queue priority.
1032 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_exec_queue_set_property_ioctl()
1033 return -EINVAL; in xe_exec_queue_set_property_ioctl()
1035 if (XE_IOCTL_DBG(xe, args->property != in xe_exec_queue_set_property_ioctl()
1037 return -EINVAL; in xe_exec_queue_set_property_ioctl()
1039 q = xe_exec_queue_lookup(xef, args->exec_queue_id); in xe_exec_queue_set_property_ioctl()
1041 return -ENOENT; in xe_exec_queue_set_property_ioctl()
1043 idx = array_index_nospec(args->property, in xe_exec_queue_set_property_ioctl()
1045 ret = exec_queue_set_property_funcs[idx](xe, q, args->value); in xe_exec_queue_set_property_ioctl()
1064 * multi-queue group. in exec_queue_user_ext_check()
1068 return -EINVAL; in exec_queue_user_ext_check()
1075 /* MULTI_QUEUE_PRIORITY only applies to multi-queue group queues */ in exec_queue_user_ext_check_final()
1078 return -EINVAL; in exec_queue_user_ext_check_final()
1094 return -EFAULT; in exec_queue_user_ext_set_property()
1106 return -EINVAL; in exec_queue_user_ext_set_property()
1110 return -EINVAL; in exec_queue_user_ext_set_property()
1138 return -E2BIG; in __exec_queue_user_extensions()
1142 return -EFAULT; in __exec_queue_user_extensions()
1147 return -EINVAL; in __exec_queue_user_extensions()
1248 * xe_exec_queue_create_ioctl() - Create an exec queue via IOCTL
1253 * Creates a new exec queue based on user-provided parameters. Supports
1254 * creating VM bind queues, regular exec queues, multi-lrc exec queues
1255 * and multi-queue groups.
1267 u64_to_user_ptr(args->instances); in xe_exec_queue_create_ioctl()
1278 if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) || in xe_exec_queue_create_ioctl()
1279 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_exec_queue_create_ioctl()
1280 return -EINVAL; in xe_exec_queue_create_ioctl()
1282 len = args->width * args->num_placements; in xe_exec_queue_create_ioctl()
1284 return -EINVAL; in xe_exec_queue_create_ioctl()
1289 return -EFAULT; in xe_exec_queue_create_ioctl()
1292 return -EINVAL; in xe_exec_queue_create_ioctl()
1294 if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) in xe_exec_queue_create_ioctl()
1298 if (XE_IOCTL_DBG(xe, args->width != 1) || in xe_exec_queue_create_ioctl()
1299 XE_IOCTL_DBG(xe, args->num_placements != 1) || in xe_exec_queue_create_ioctl()
1301 return -EINVAL; in xe_exec_queue_create_ioctl()
1303 vm = xe_vm_lookup(xef, args->vm_id); in xe_exec_queue_create_ioctl()
1305 return -ENOENT; in xe_exec_queue_create_ioctl()
1307 err = down_read_interruptible(&vm->lock); in xe_exec_queue_create_ioctl()
1314 up_read(&vm->lock); in xe_exec_queue_create_ioctl()
1316 return -ENOENT; in xe_exec_queue_create_ioctl()
1327 args->extensions); in xe_exec_queue_create_ioctl()
1329 up_read(&vm->lock); in xe_exec_queue_create_ioctl()
1339 list_add_tail(&new->multi_gt_list, in xe_exec_queue_create_ioctl()
1340 &q->multi_gt_link); in xe_exec_queue_create_ioctl()
1342 up_read(&vm->lock); in xe_exec_queue_create_ioctl()
1346 args->width, in xe_exec_queue_create_ioctl()
1347 args->num_placements); in xe_exec_queue_create_ioctl()
1349 return -EINVAL; in xe_exec_queue_create_ioctl()
1353 return -EINVAL; in xe_exec_queue_create_ioctl()
1355 /* multi-lrc is only supported on select engine classes */ in xe_exec_queue_create_ioctl()
1356 if (XE_IOCTL_DBG(xe, args->width > 1 && in xe_exec_queue_create_ioctl()
1357 !(xe->info.multi_lrc_mask & BIT(hwe->class)))) in xe_exec_queue_create_ioctl()
1358 return -EOPNOTSUPP; in xe_exec_queue_create_ioctl()
1360 vm = xe_vm_lookup(xef, args->vm_id); in xe_exec_queue_create_ioctl()
1362 return -ENOENT; in xe_exec_queue_create_ioctl()
1364 err = down_read_interruptible(&vm->lock); in xe_exec_queue_create_ioctl()
1371 up_read(&vm->lock); in xe_exec_queue_create_ioctl()
1373 return -ENOENT; in xe_exec_queue_create_ioctl()
1376 /* SRIOV sched groups are not compatible with multi-lrc */ in xe_exec_queue_create_ioctl()
1377 if (XE_IOCTL_DBG(xe, args->width > 1 && has_sched_groups(hwe->gt))) { in xe_exec_queue_create_ioctl()
1378 up_read(&vm->lock); in xe_exec_queue_create_ioctl()
1380 return -EINVAL; in xe_exec_queue_create_ioctl()
1384 args->width, hwe, flags, in xe_exec_queue_create_ioctl()
1385 args->extensions); in xe_exec_queue_create_ioctl()
1386 up_read(&vm->lock); in xe_exec_queue_create_ioctl()
1398 q->lr.context = dma_fence_context_alloc(1); in xe_exec_queue_create_ioctl()
1405 if (q->vm && q->hwe->hw_engine_group) { in xe_exec_queue_create_ioctl()
1406 err = xe_hw_engine_group_add_exec_queue(q->hwe->hw_engine_group, q); in xe_exec_queue_create_ioctl()
1412 q->xef = xe_file_get(xef); in xe_exec_queue_create_ioctl()
1417 err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); in xe_exec_queue_create_ioctl()
1421 args->exec_queue_id = id; in xe_exec_queue_create_ioctl()
1426 if (q->vm && q->hwe && q->hwe->hw_engine_group) in xe_exec_queue_create_ioctl()
1427 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); in xe_exec_queue_create_ioctl()
1439 * xe_exec_queue_get_property_ioctl() - Get a property from an exec queue
1458 if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_exec_queue_get_property_ioctl()
1459 return -EINVAL; in xe_exec_queue_get_property_ioctl()
1461 q = xe_exec_queue_lookup(xef, args->exec_queue_id); in xe_exec_queue_get_property_ioctl()
1463 return -ENOENT; in xe_exec_queue_get_property_ioctl()
1465 switch (args->property) { in xe_exec_queue_get_property_ioctl()
1467 args->value = q->ops->reset_status(q); in xe_exec_queue_get_property_ioctl()
1471 ret = -EINVAL; in xe_exec_queue_get_property_ioctl()
1480 * xe_exec_queue_is_lr() - Whether an exec_queue is long-running
1483 * Return: True if the exec_queue is long-running, false otherwise.
1487 return q->vm && xe_vm_in_lr_mode(q->vm) && in xe_exec_queue_is_lr()
1488 !(q->flags & EXEC_QUEUE_FLAG_VM); in xe_exec_queue_is_lr()
1492 * xe_exec_queue_is_idle() - Whether an exec_queue is idle.
1495 * FIXME: Need to determine what to use as the short-lived
1497 * of this function becomes more than just an advisory
1500 * Typically vm->resv, but user-created timeline locks use the migrate vm
1501 * and never grabs the migrate vm->resv so we have a race there.
1510 for (i = 0; i < q->width; ++i) { in xe_exec_queue_is_idle()
1511 if (xe_lrc_seqno(q->lrc[i]) != in xe_exec_queue_is_idle()
1512 q->lrc[i]->fence_ctx.next_seqno - 1) in xe_exec_queue_is_idle()
1519 return xe_lrc_seqno(q->lrc[0]) == in xe_exec_queue_is_idle()
1520 q->lrc[0]->fence_ctx.next_seqno - 1; in xe_exec_queue_is_idle()
1524 * xe_exec_queue_update_run_ticks() - Update run time in ticks for this exec queue
1533 struct xe_device *xe = gt_to_xe(q->gt); in xe_exec_queue_update_run_ticks()
1542 if (!q->xef) in xe_exec_queue_update_run_ticks()
1546 if (!drm_dev_enter(&xe->drm, &idx)) in xe_exec_queue_update_run_ticks()
1551 * width - this may introduce errors if that premise is not true and in xe_exec_queue_update_run_ticks()
1556 lrc = q->lrc[0]; in xe_exec_queue_update_run_ticks()
1558 q->xef->run_ticks[q->class] += (new_ts - old_ts) * q->width; in xe_exec_queue_update_run_ticks()
1564 * xe_exec_queue_kill - permanently stop all execution from an exec queue
1567 * This function permanently stops all activity on an exec queue. If the queue
1570 * This function is safe to call multiple times.
1576 list_for_each_entry_safe(eq, next, &eq->multi_gt_list, in xe_exec_queue_kill()
1578 q->ops->kill(eq); in xe_exec_queue_kill()
1579 xe_vm_remove_compute_exec_queue(q->vm, eq); in xe_exec_queue_kill()
1582 q->ops->kill(q); in xe_exec_queue_kill()
1583 xe_vm_remove_compute_exec_queue(q->vm, q); in xe_exec_queue_kill()
1587 * xe_exec_queue_destroy_ioctl() - Destroy an exec queue via IOCTL
1604 if (XE_IOCTL_DBG(xe, args->pad) || in xe_exec_queue_destroy_ioctl()
1605 XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) in xe_exec_queue_destroy_ioctl()
1606 return -EINVAL; in xe_exec_queue_destroy_ioctl()
1608 mutex_lock(&xef->exec_queue.lock); in xe_exec_queue_destroy_ioctl()
1609 q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); in xe_exec_queue_destroy_ioctl()
1611 atomic_inc(&xef->exec_queue.pending_removal); in xe_exec_queue_destroy_ioctl()
1612 mutex_unlock(&xef->exec_queue.lock); in xe_exec_queue_destroy_ioctl()
1615 return -ENOENT; in xe_exec_queue_destroy_ioctl()
1617 if (q->vm && q->hwe->hw_engine_group) in xe_exec_queue_destroy_ioctl()
1618 xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q); in xe_exec_queue_destroy_ioctl()
1631 if (q->flags & EXEC_QUEUE_FLAG_MIGRATE) { in xe_exec_queue_last_fence_lockdep_assert()
1633 } else if (q->flags & EXEC_QUEUE_FLAG_VM) { in xe_exec_queue_last_fence_lockdep_assert()
1634 lockdep_assert_held(&vm->lock); in xe_exec_queue_last_fence_lockdep_assert()
1637 lockdep_assert_held(&q->hwe->hw_engine_group->mode_sem); in xe_exec_queue_last_fence_lockdep_assert()
1642 * xe_exec_queue_last_fence_put() - Drop ref to last fence
1654 * xe_exec_queue_last_fence_put_unlocked() - Drop ref to last fence unlocked
1661 if (q->last_fence) { in xe_exec_queue_last_fence_put_unlocked()
1662 dma_fence_put(q->last_fence); in xe_exec_queue_last_fence_put_unlocked()
1663 q->last_fence = NULL; in xe_exec_queue_last_fence_put_unlocked()
1668 * xe_exec_queue_last_fence_get() - Get last fence
1683 if (q->last_fence && in xe_exec_queue_last_fence_get()
1684 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) in xe_exec_queue_last_fence_get()
1687 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); in xe_exec_queue_last_fence_get()
1693 * xe_exec_queue_last_fence_get_for_resume() - Get last fence
1698 * resuming the hw engine group's long-running exec queue, when the group
1708 lockdep_assert_held_write(&q->hwe->hw_engine_group->mode_sem); in xe_exec_queue_last_fence_get_for_resume()
1710 if (q->last_fence && in xe_exec_queue_last_fence_get_for_resume()
1711 test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &q->last_fence->flags)) in xe_exec_queue_last_fence_get_for_resume()
1714 fence = q->last_fence ? q->last_fence : dma_fence_get_stub(); in xe_exec_queue_last_fence_get_for_resume()
1720 * xe_exec_queue_last_fence_set() - Set last fence
1732 xe_assert(vm->xe, !dma_fence_is_container(fence)); in xe_exec_queue_last_fence_set()
1735 q->last_fence = dma_fence_get(fence); in xe_exec_queue_last_fence_set()
1739 * xe_exec_queue_tlb_inval_last_fence_put() - Drop ref to last TLB invalidation fence
1749 xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || in xe_exec_queue_tlb_inval_last_fence_put()
1756 * xe_exec_queue_tlb_inval_last_fence_put_unlocked() - Drop ref to last TLB
1766 xe_assert(gt_to_xe(q->gt), type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || in xe_exec_queue_tlb_inval_last_fence_put_unlocked()
1769 dma_fence_put(q->tlb_inval[type].last_fence); in xe_exec_queue_tlb_inval_last_fence_put_unlocked()
1770 q->tlb_inval[type].last_fence = NULL; in xe_exec_queue_tlb_inval_last_fence_put_unlocked()
1774 * xe_exec_queue_tlb_inval_last_fence_get() - Get last fence for TLB invalidation
1790 xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || in xe_exec_queue_tlb_inval_last_fence_get()
1792 xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM | in xe_exec_queue_tlb_inval_last_fence_get()
1795 if (q->tlb_inval[type].last_fence && in xe_exec_queue_tlb_inval_last_fence_get()
1797 &q->tlb_inval[type].last_fence->flags)) in xe_exec_queue_tlb_inval_last_fence_get()
1800 fence = q->tlb_inval[type].last_fence ?: dma_fence_get_stub(); in xe_exec_queue_tlb_inval_last_fence_get()
1806 * xe_exec_queue_tlb_inval_last_fence_set() - Set last fence for TLB invalidation
1822 xe_assert(vm->xe, type == XE_EXEC_QUEUE_TLB_INVAL_MEDIA_GT || in xe_exec_queue_tlb_inval_last_fence_set()
1824 xe_assert(vm->xe, q->flags & (EXEC_QUEUE_FLAG_VM | in xe_exec_queue_tlb_inval_last_fence_set()
1826 xe_assert(vm->xe, !dma_fence_is_container(fence)); in xe_exec_queue_tlb_inval_last_fence_set()
1829 q->tlb_inval[type].last_fence = dma_fence_get(fence); in xe_exec_queue_tlb_inval_last_fence_set()
1833 * xe_exec_queue_contexts_hwsp_rebase - Re-compute GGTT references
1845 for (i = 0; i < q->width; ++i) { in xe_exec_queue_contexts_hwsp_rebase()
1852 xe_lrc_update_memirq_regs_with_address(lrc, q->hwe, scratch); in xe_exec_queue_contexts_hwsp_rebase()
1854 err = xe_lrc_setup_wa_bb_with_scratch(lrc, q->hwe, scratch); in xe_exec_queue_contexts_hwsp_rebase()