1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #include <drm/drm_managed.h> 7 8 #include "xe_assert.h" 9 #include "xe_device_types.h" 10 #include "xe_exec_queue.h" 11 #include "xe_gt.h" 12 #include "xe_gt_stats.h" 13 #include "xe_hw_engine_group.h" 14 #include "xe_sync.h" 15 #include "xe_vm.h" 16 17 static void 18 hw_engine_group_resume_lr_jobs_func(struct work_struct *w) 19 { 20 struct xe_exec_queue *q; 21 struct xe_hw_engine_group *group = container_of(w, struct xe_hw_engine_group, resume_work); 22 int err; 23 enum xe_hw_engine_group_execution_mode previous_mode; 24 25 err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode, 26 NULL, 0); 27 if (err) 28 return; 29 30 if (previous_mode == EXEC_MODE_LR) 31 goto put; 32 33 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 34 if (!xe_vm_in_fault_mode(q->vm)) 35 continue; 36 37 q->ops->resume(q); 38 } 39 40 put: 41 xe_hw_engine_group_put(group); 42 } 43 44 static struct xe_hw_engine_group * 45 hw_engine_group_alloc(struct xe_device *xe) 46 { 47 struct xe_hw_engine_group *group; 48 int err; 49 50 group = drmm_kzalloc(&xe->drm, sizeof(*group), GFP_KERNEL); 51 if (!group) 52 return ERR_PTR(-ENOMEM); 53 54 group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", 0, 0); 55 if (!group->resume_wq) 56 return ERR_PTR(-ENOMEM); 57 58 err = drmm_add_action_or_reset(&xe->drm, __drmm_workqueue_release, group->resume_wq); 59 if (err) 60 return ERR_PTR(err); 61 62 init_rwsem(&group->mode_sem); 63 INIT_WORK(&group->resume_work, hw_engine_group_resume_lr_jobs_func); 64 INIT_LIST_HEAD(&group->exec_queue_list); 65 66 return group; 67 } 68 69 /** 70 * xe_hw_engine_setup_groups() - Setup the hw engine groups for the gt 71 * @gt: The gt for which groups are setup 72 * 73 * Return: 0 on success, negative error code on error. 74 */ 75 int xe_hw_engine_setup_groups(struct xe_gt *gt) 76 { 77 struct xe_hw_engine *hwe; 78 enum xe_hw_engine_id id; 79 struct xe_hw_engine_group *group_rcs_ccs, *group_bcs, *group_vcs_vecs; 80 struct xe_device *xe = gt_to_xe(gt); 81 82 group_rcs_ccs = hw_engine_group_alloc(xe); 83 if (IS_ERR(group_rcs_ccs)) 84 return PTR_ERR(group_rcs_ccs); 85 86 group_bcs = hw_engine_group_alloc(xe); 87 if (IS_ERR(group_bcs)) 88 return PTR_ERR(group_bcs); 89 90 group_vcs_vecs = hw_engine_group_alloc(xe); 91 if (IS_ERR(group_vcs_vecs)) 92 return PTR_ERR(group_vcs_vecs); 93 94 for_each_hw_engine(hwe, gt, id) { 95 switch (hwe->class) { 96 case XE_ENGINE_CLASS_COPY: 97 hwe->hw_engine_group = group_bcs; 98 break; 99 case XE_ENGINE_CLASS_RENDER: 100 case XE_ENGINE_CLASS_COMPUTE: 101 hwe->hw_engine_group = group_rcs_ccs; 102 break; 103 case XE_ENGINE_CLASS_VIDEO_DECODE: 104 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 105 hwe->hw_engine_group = group_vcs_vecs; 106 break; 107 case XE_ENGINE_CLASS_OTHER: 108 break; 109 case XE_ENGINE_CLASS_MAX: 110 xe_gt_assert(gt, false); 111 } 112 } 113 114 return 0; 115 } 116 117 /** 118 * xe_hw_engine_group_add_exec_queue() - Add an exec queue to a hw engine group 119 * @group: The hw engine group 120 * @q: The exec_queue 121 * 122 * Return: 0 on success, 123 * -EINTR if the lock could not be acquired 124 */ 125 int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q) 126 { 127 int err; 128 struct xe_device *xe = gt_to_xe(q->gt); 129 130 xe_assert(xe, group); 131 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM)); 132 xe_assert(xe, q->vm); 133 134 if (xe_vm_in_preempt_fence_mode(q->vm)) 135 return 0; 136 137 err = down_write_killable(&group->mode_sem); 138 if (err) 139 return err; 140 141 if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) { 142 q->ops->suspend(q); 143 err = q->ops->suspend_wait(q); 144 if (err) 145 goto err_suspend; 146 147 xe_hw_engine_group_resume_faulting_lr_jobs(group); 148 } 149 150 list_add(&q->hw_engine_group_link, &group->exec_queue_list); 151 up_write(&group->mode_sem); 152 153 return 0; 154 155 err_suspend: 156 up_write(&group->mode_sem); 157 return err; 158 } 159 ALLOW_ERROR_INJECTION(xe_hw_engine_group_add_exec_queue, ERRNO); 160 161 /** 162 * xe_hw_engine_group_del_exec_queue() - Delete an exec queue from a hw engine group 163 * @group: The hw engine group 164 * @q: The exec_queue 165 */ 166 void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q) 167 { 168 struct xe_device *xe = gt_to_xe(q->gt); 169 170 xe_assert(xe, group); 171 xe_assert(xe, q->vm); 172 173 down_write(&group->mode_sem); 174 175 if (!list_empty(&q->hw_engine_group_link)) 176 list_del(&q->hw_engine_group_link); 177 178 up_write(&group->mode_sem); 179 } 180 181 /** 182 * xe_hw_engine_group_resume_faulting_lr_jobs() - Asynchronously resume the hw engine group's 183 * faulting LR jobs 184 * @group: The hw engine group 185 */ 186 void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group) 187 { 188 queue_work(group->resume_wq, &group->resume_work); 189 } 190 191 /** 192 * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group 193 * @group: The hw engine group 194 * @has_deps: dma-fence job triggering suspend has dependencies 195 * 196 * Return: 0 on success, negative error code on error. 197 */ 198 static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group, 199 bool has_deps) 200 { 201 int err; 202 struct xe_exec_queue *q; 203 struct xe_gt *gt = NULL; 204 bool need_resume = false; 205 ktime_t start = xe_gt_stats_ktime_get(); 206 207 lockdep_assert_held_write(&group->mode_sem); 208 209 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 210 bool idle_skip_suspend; 211 212 if (!xe_vm_in_fault_mode(q->vm)) 213 continue; 214 215 idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q); 216 if (!idle_skip_suspend && has_deps) 217 return -EAGAIN; 218 219 xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1); 220 if (idle_skip_suspend) 221 xe_gt_stats_incr(q->gt, 222 XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, 1); 223 224 need_resume |= !idle_skip_suspend; 225 q->ops->suspend(q); 226 gt = q->gt; 227 } 228 229 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 230 if (!xe_vm_in_fault_mode(q->vm)) 231 continue; 232 233 err = q->ops->suspend_wait(q); 234 if (err) 235 return err; 236 } 237 238 if (gt) { 239 xe_gt_stats_incr(gt, 240 XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US, 241 xe_gt_stats_ktime_us_delta(start)); 242 } 243 244 if (need_resume) 245 xe_hw_engine_group_resume_faulting_lr_jobs(group); 246 247 return 0; 248 } 249 250 /** 251 * xe_hw_engine_group_wait_for_dma_fence_jobs() - Wait for dma fence jobs to complete 252 * @group: The hw engine group 253 * 254 * This function is not meant to be called directly from a user IOCTL as dma_fence_wait() 255 * is not interruptible. 256 * 257 * Return: 0 on success, 258 * -ETIME if waiting for one job failed 259 */ 260 static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group *group) 261 { 262 long timeout; 263 struct xe_exec_queue *q; 264 struct xe_gt *gt = NULL; 265 struct dma_fence *fence; 266 ktime_t start = xe_gt_stats_ktime_get(); 267 268 lockdep_assert_held_write(&group->mode_sem); 269 270 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 271 if (xe_vm_in_lr_mode(q->vm)) 272 continue; 273 274 xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1); 275 fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm); 276 timeout = dma_fence_wait(fence, false); 277 dma_fence_put(fence); 278 gt = q->gt; 279 280 if (timeout < 0) 281 return -ETIME; 282 } 283 284 if (gt) { 285 xe_gt_stats_incr(gt, 286 XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US, 287 xe_gt_stats_ktime_us_delta(start)); 288 } 289 290 return 0; 291 } 292 293 static int switch_mode(struct xe_hw_engine_group *group, bool has_deps) 294 { 295 int err = 0; 296 enum xe_hw_engine_group_execution_mode new_mode; 297 298 lockdep_assert_held_write(&group->mode_sem); 299 300 switch (group->cur_mode) { 301 case EXEC_MODE_LR: 302 new_mode = EXEC_MODE_DMA_FENCE; 303 err = xe_hw_engine_group_suspend_faulting_lr_jobs(group, 304 has_deps); 305 break; 306 case EXEC_MODE_DMA_FENCE: 307 new_mode = EXEC_MODE_LR; 308 err = xe_hw_engine_group_wait_for_dma_fence_jobs(group); 309 break; 310 } 311 312 if (err) 313 return err; 314 315 group->cur_mode = new_mode; 316 317 return 0; 318 } 319 320 static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs) 321 { 322 int err, i; 323 324 for (i = 0; i < num_syncs; ++i) { 325 err = xe_sync_entry_wait(syncs + i); 326 if (err) 327 return err; 328 } 329 330 return 0; 331 } 332 333 /** 334 * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode 335 * @group: The hw engine group 336 * @new_mode: The new execution mode 337 * @previous_mode: Pointer to the previous mode provided for use by caller 338 * @syncs: Syncs from exec IOCTL 339 * @num_syncs: Number of syncs from exec IOCTL 340 * 341 * Return: 0 if successful, -EINTR if locking failed. 342 */ 343 int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group, 344 enum xe_hw_engine_group_execution_mode new_mode, 345 enum xe_hw_engine_group_execution_mode *previous_mode, 346 struct xe_sync_entry *syncs, int num_syncs) 347 __acquires(&group->mode_sem) 348 { 349 bool has_deps = !!num_syncs; 350 int err = down_read_interruptible(&group->mode_sem); 351 352 if (err) 353 return err; 354 355 *previous_mode = group->cur_mode; 356 357 if (new_mode != group->cur_mode) { 358 up_read(&group->mode_sem); 359 retry: 360 err = down_write_killable(&group->mode_sem); 361 if (err) 362 return err; 363 364 if (new_mode != group->cur_mode) { 365 err = switch_mode(group, has_deps); 366 if (err) { 367 up_write(&group->mode_sem); 368 369 if (err != -EAGAIN) 370 return err; 371 372 err = wait_syncs(syncs, num_syncs); 373 if (err) 374 return err; 375 376 has_deps = false; 377 goto retry; 378 } 379 } 380 downgrade_write(&group->mode_sem); 381 } 382 383 return err; 384 } 385 386 /** 387 * xe_hw_engine_group_put() - Put the group 388 * @group: The hw engine group 389 */ 390 void xe_hw_engine_group_put(struct xe_hw_engine_group *group) 391 __releases(&group->mode_sem) 392 { 393 up_read(&group->mode_sem); 394 } 395 396 /** 397 * xe_hw_engine_group_find_exec_mode() - Find the execution mode for this exec queue 398 * @q: The exec_queue 399 */ 400 enum xe_hw_engine_group_execution_mode 401 xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q) 402 { 403 if (xe_vm_in_fault_mode(q->vm)) 404 return EXEC_MODE_LR; 405 else 406 return EXEC_MODE_DMA_FENCE; 407 } 408