1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2024 Intel Corporation
4 */
5
6 #include <drm/drm_managed.h>
7
8 #include "xe_assert.h"
9 #include "xe_device_types.h"
10 #include "xe_exec_queue.h"
11 #include "xe_gt.h"
12 #include "xe_gt_stats.h"
13 #include "xe_hw_engine_group.h"
14 #include "xe_sync.h"
15 #include "xe_vm.h"
16
17 static void
hw_engine_group_resume_lr_jobs_func(struct work_struct * w)18 hw_engine_group_resume_lr_jobs_func(struct work_struct *w)
19 {
20 struct xe_exec_queue *q;
21 struct xe_hw_engine_group *group = container_of(w, struct xe_hw_engine_group, resume_work);
22 int err;
23 enum xe_hw_engine_group_execution_mode previous_mode;
24
25 err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode,
26 NULL, 0);
27 if (err)
28 return;
29
30 if (previous_mode == EXEC_MODE_LR)
31 goto put;
32
33 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
34 if (!xe_vm_in_fault_mode(q->vm))
35 continue;
36
37 q->ops->resume(q);
38 }
39
40 put:
41 xe_hw_engine_group_put(group);
42 }
43
44 static struct xe_hw_engine_group *
hw_engine_group_alloc(struct xe_device * xe)45 hw_engine_group_alloc(struct xe_device *xe)
46 {
47 struct xe_hw_engine_group *group;
48 int err;
49
50 group = drmm_kzalloc(&xe->drm, sizeof(*group), GFP_KERNEL);
51 if (!group)
52 return ERR_PTR(-ENOMEM);
53
54 group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", WQ_PERCPU,
55 0);
56 if (!group->resume_wq)
57 return ERR_PTR(-ENOMEM);
58
59 err = drmm_add_action_or_reset(&xe->drm, __drmm_workqueue_release, group->resume_wq);
60 if (err)
61 return ERR_PTR(err);
62
63 init_rwsem(&group->mode_sem);
64 INIT_WORK(&group->resume_work, hw_engine_group_resume_lr_jobs_func);
65 INIT_LIST_HEAD(&group->exec_queue_list);
66
67 return group;
68 }
69
70 /**
71 * xe_hw_engine_setup_groups() - Setup the hw engine groups for the gt
72 * @gt: The gt for which groups are setup
73 *
74 * Return: 0 on success, negative error code on error.
75 */
xe_hw_engine_setup_groups(struct xe_gt * gt)76 int xe_hw_engine_setup_groups(struct xe_gt *gt)
77 {
78 struct xe_hw_engine *hwe;
79 enum xe_hw_engine_id id;
80 struct xe_hw_engine_group *group_rcs_ccs, *group_bcs, *group_vcs_vecs;
81 struct xe_device *xe = gt_to_xe(gt);
82
83 group_rcs_ccs = hw_engine_group_alloc(xe);
84 if (IS_ERR(group_rcs_ccs))
85 return PTR_ERR(group_rcs_ccs);
86
87 group_bcs = hw_engine_group_alloc(xe);
88 if (IS_ERR(group_bcs))
89 return PTR_ERR(group_bcs);
90
91 group_vcs_vecs = hw_engine_group_alloc(xe);
92 if (IS_ERR(group_vcs_vecs))
93 return PTR_ERR(group_vcs_vecs);
94
95 for_each_hw_engine(hwe, gt, id) {
96 switch (hwe->class) {
97 case XE_ENGINE_CLASS_COPY:
98 hwe->hw_engine_group = group_bcs;
99 break;
100 case XE_ENGINE_CLASS_RENDER:
101 case XE_ENGINE_CLASS_COMPUTE:
102 hwe->hw_engine_group = group_rcs_ccs;
103 break;
104 case XE_ENGINE_CLASS_VIDEO_DECODE:
105 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
106 hwe->hw_engine_group = group_vcs_vecs;
107 break;
108 case XE_ENGINE_CLASS_OTHER:
109 break;
110 case XE_ENGINE_CLASS_MAX:
111 xe_gt_assert(gt, false);
112 }
113 }
114
115 return 0;
116 }
117
118 /**
119 * xe_hw_engine_group_add_exec_queue() - Add an exec queue to a hw engine group
120 * @group: The hw engine group
121 * @q: The exec_queue
122 *
123 * Return: 0 on success,
124 * -EINTR if the lock could not be acquired
125 */
xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group * group,struct xe_exec_queue * q)126 int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q)
127 {
128 int err;
129 struct xe_device *xe = gt_to_xe(q->gt);
130
131 xe_assert(xe, group);
132 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM));
133 xe_assert(xe, q->vm);
134
135 if (xe_vm_in_preempt_fence_mode(q->vm))
136 return 0;
137
138 err = down_write_killable(&group->mode_sem);
139 if (err)
140 return err;
141
142 if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) {
143 q->ops->suspend(q);
144 err = q->ops->suspend_wait(q);
145 if (err)
146 goto err_suspend;
147
148 xe_hw_engine_group_resume_faulting_lr_jobs(group);
149 }
150
151 list_add(&q->hw_engine_group_link, &group->exec_queue_list);
152 up_write(&group->mode_sem);
153
154 return 0;
155
156 err_suspend:
157 up_write(&group->mode_sem);
158 return err;
159 }
160 ALLOW_ERROR_INJECTION(xe_hw_engine_group_add_exec_queue, ERRNO);
161
162 /**
163 * xe_hw_engine_group_del_exec_queue() - Delete an exec queue from a hw engine group
164 * @group: The hw engine group
165 * @q: The exec_queue
166 */
xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group * group,struct xe_exec_queue * q)167 void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q)
168 {
169 struct xe_device *xe = gt_to_xe(q->gt);
170
171 xe_assert(xe, group);
172 xe_assert(xe, q->vm);
173
174 down_write(&group->mode_sem);
175
176 if (!list_empty(&q->hw_engine_group_link))
177 list_del(&q->hw_engine_group_link);
178
179 up_write(&group->mode_sem);
180 }
181
182 /**
183 * xe_hw_engine_group_resume_faulting_lr_jobs() - Asynchronously resume the hw engine group's
184 * faulting LR jobs
185 * @group: The hw engine group
186 */
xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group * group)187 void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group)
188 {
189 queue_work(group->resume_wq, &group->resume_work);
190 }
191
192 /**
193 * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group
194 * @group: The hw engine group
195 * @has_deps: dma-fence job triggering suspend has dependencies
196 *
197 * Return: 0 on success, negative error code on error.
198 */
xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group * group,bool has_deps)199 static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group,
200 bool has_deps)
201 {
202 int err;
203 struct xe_exec_queue *q;
204 struct xe_gt *gt = NULL;
205 bool need_resume = false;
206 ktime_t start = xe_gt_stats_ktime_get();
207
208 lockdep_assert_held_write(&group->mode_sem);
209
210 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
211
212 if (!xe_vm_in_fault_mode(q->vm))
213 continue;
214
215 if (has_deps)
216 return -EAGAIN;
217
218 xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1);
219 need_resume = true;
220 q->ops->suspend(q);
221 gt = q->gt;
222 }
223
224 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
225 if (!xe_vm_in_fault_mode(q->vm))
226 continue;
227
228 err = q->ops->suspend_wait(q);
229 if (err)
230 return err;
231 }
232
233 if (gt) {
234 xe_gt_stats_incr(gt,
235 XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US,
236 xe_gt_stats_ktime_us_delta(start));
237 }
238
239 if (need_resume)
240 xe_hw_engine_group_resume_faulting_lr_jobs(group);
241
242 return 0;
243 }
244
245 /**
246 * xe_hw_engine_group_wait_for_dma_fence_jobs() - Wait for dma fence jobs to complete
247 * @group: The hw engine group
248 *
249 * This function is not meant to be called directly from a user IOCTL as dma_fence_wait()
250 * is not interruptible.
251 *
252 * Return: 0 on success,
253 * -ETIME if waiting for one job failed
254 */
xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group * group)255 static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group *group)
256 {
257 long timeout;
258 struct xe_exec_queue *q;
259 struct xe_gt *gt = NULL;
260 struct dma_fence *fence;
261 ktime_t start = xe_gt_stats_ktime_get();
262
263 lockdep_assert_held_write(&group->mode_sem);
264
265 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) {
266 if (xe_vm_in_lr_mode(q->vm))
267 continue;
268
269 xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1);
270 fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm);
271 timeout = dma_fence_wait(fence, false);
272 dma_fence_put(fence);
273 gt = q->gt;
274
275 if (timeout < 0)
276 return -ETIME;
277 }
278
279 if (gt) {
280 xe_gt_stats_incr(gt,
281 XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US,
282 xe_gt_stats_ktime_us_delta(start));
283 }
284
285 return 0;
286 }
287
switch_mode(struct xe_hw_engine_group * group,bool has_deps)288 static int switch_mode(struct xe_hw_engine_group *group, bool has_deps)
289 {
290 int err = 0;
291 enum xe_hw_engine_group_execution_mode new_mode;
292
293 lockdep_assert_held_write(&group->mode_sem);
294
295 switch (group->cur_mode) {
296 case EXEC_MODE_LR:
297 new_mode = EXEC_MODE_DMA_FENCE;
298 err = xe_hw_engine_group_suspend_faulting_lr_jobs(group,
299 has_deps);
300 break;
301 case EXEC_MODE_DMA_FENCE:
302 new_mode = EXEC_MODE_LR;
303 err = xe_hw_engine_group_wait_for_dma_fence_jobs(group);
304 break;
305 }
306
307 if (err)
308 return err;
309
310 group->cur_mode = new_mode;
311
312 return 0;
313 }
314
wait_syncs(struct xe_sync_entry * syncs,int num_syncs)315 static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs)
316 {
317 int err, i;
318
319 for (i = 0; i < num_syncs; ++i) {
320 err = xe_sync_entry_wait(syncs + i);
321 if (err)
322 return err;
323 }
324
325 return 0;
326 }
327
328 /**
329 * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode
330 * @group: The hw engine group
331 * @new_mode: The new execution mode
332 * @previous_mode: Pointer to the previous mode provided for use by caller
333 * @syncs: Syncs from exec IOCTL
334 * @num_syncs: Number of syncs from exec IOCTL
335 *
336 * Return: 0 if successful, -EINTR if locking failed.
337 */
xe_hw_engine_group_get_mode(struct xe_hw_engine_group * group,enum xe_hw_engine_group_execution_mode new_mode,enum xe_hw_engine_group_execution_mode * previous_mode,struct xe_sync_entry * syncs,int num_syncs)338 int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group,
339 enum xe_hw_engine_group_execution_mode new_mode,
340 enum xe_hw_engine_group_execution_mode *previous_mode,
341 struct xe_sync_entry *syncs, int num_syncs)
342 __acquires(&group->mode_sem)
343 {
344 bool has_deps = !!num_syncs;
345 int err = down_read_interruptible(&group->mode_sem);
346
347 if (err)
348 return err;
349
350 *previous_mode = group->cur_mode;
351
352 if (new_mode != group->cur_mode) {
353 up_read(&group->mode_sem);
354 retry:
355 err = down_write_killable(&group->mode_sem);
356 if (err)
357 return err;
358
359 if (new_mode != group->cur_mode) {
360 err = switch_mode(group, has_deps);
361 if (err) {
362 up_write(&group->mode_sem);
363
364 if (err != -EAGAIN)
365 return err;
366
367 err = wait_syncs(syncs, num_syncs);
368 if (err)
369 return err;
370
371 has_deps = false;
372 goto retry;
373 }
374 }
375 downgrade_write(&group->mode_sem);
376 }
377
378 return err;
379 }
380
381 /**
382 * xe_hw_engine_group_put() - Put the group
383 * @group: The hw engine group
384 */
xe_hw_engine_group_put(struct xe_hw_engine_group * group)385 void xe_hw_engine_group_put(struct xe_hw_engine_group *group)
386 __releases(&group->mode_sem)
387 {
388 up_read(&group->mode_sem);
389 }
390
391 /**
392 * xe_hw_engine_group_find_exec_mode() - Find the execution mode for this exec queue
393 * @q: The exec_queue
394 */
395 enum xe_hw_engine_group_execution_mode
xe_hw_engine_group_find_exec_mode(struct xe_exec_queue * q)396 xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q)
397 {
398 if (xe_vm_in_fault_mode(q->vm))
399 return EXEC_MODE_LR;
400 else
401 return EXEC_MODE_DMA_FENCE;
402 }
403