1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #include <linux/error-injection.h> 7 8 #include <drm/drm_managed.h> 9 10 #include "xe_assert.h" 11 #include "xe_device_types.h" 12 #include "xe_exec_queue.h" 13 #include "xe_gt.h" 14 #include "xe_gt_stats.h" 15 #include "xe_hw_engine_group.h" 16 #include "xe_sync.h" 17 #include "xe_vm.h" 18 19 static void 20 hw_engine_group_resume_lr_jobs_func(struct work_struct *w) 21 { 22 struct xe_exec_queue *q; 23 struct xe_hw_engine_group *group = container_of(w, struct xe_hw_engine_group, resume_work); 24 int err; 25 enum xe_hw_engine_group_execution_mode previous_mode; 26 27 err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode, 28 NULL, 0); 29 if (err) 30 return; 31 32 if (previous_mode == EXEC_MODE_LR) 33 goto put; 34 35 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 36 if (!xe_vm_in_fault_mode(q->vm)) 37 continue; 38 39 /* 40 * Only resume queues that were actually suspended. A queue whose 41 * suspend() failed (e.g. killed/banned/wedged) was never 42 * suspended, so it must not be resumed. 43 */ 44 if (!READ_ONCE(q->lr.suspended)) 45 continue; 46 47 WRITE_ONCE(q->lr.suspended, false); 48 q->ops->resume(q); 49 } 50 51 put: 52 xe_hw_engine_group_put(group); 53 } 54 55 static struct xe_hw_engine_group * 56 hw_engine_group_alloc(struct xe_device *xe) 57 { 58 struct xe_hw_engine_group *group; 59 int err; 60 61 group = drmm_kzalloc(&xe->drm, sizeof(*group), GFP_KERNEL); 62 if (!group) 63 return ERR_PTR(-ENOMEM); 64 65 group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", WQ_PERCPU, 66 0); 67 if (!group->resume_wq) 68 return ERR_PTR(-ENOMEM); 69 70 err = drmm_add_action_or_reset(&xe->drm, __drmm_workqueue_release, group->resume_wq); 71 if (err) 72 return ERR_PTR(err); 73 74 init_rwsem(&group->mode_sem); 75 INIT_WORK(&group->resume_work, hw_engine_group_resume_lr_jobs_func); 76 INIT_LIST_HEAD(&group->exec_queue_list); 77 78 return group; 79 } 80 81 /** 82 * xe_hw_engine_setup_groups() - Setup the hw engine groups for the gt 83 * @gt: The gt for which groups are setup 84 * 85 * Return: 0 on success, negative error code on error. 86 */ 87 int xe_hw_engine_setup_groups(struct xe_gt *gt) 88 { 89 struct xe_hw_engine *hwe; 90 enum xe_hw_engine_id id; 91 struct xe_hw_engine_group *group_rcs_ccs, *group_bcs, *group_vcs_vecs; 92 struct xe_device *xe = gt_to_xe(gt); 93 94 group_rcs_ccs = hw_engine_group_alloc(xe); 95 if (IS_ERR(group_rcs_ccs)) 96 return PTR_ERR(group_rcs_ccs); 97 98 group_bcs = hw_engine_group_alloc(xe); 99 if (IS_ERR(group_bcs)) 100 return PTR_ERR(group_bcs); 101 102 group_vcs_vecs = hw_engine_group_alloc(xe); 103 if (IS_ERR(group_vcs_vecs)) 104 return PTR_ERR(group_vcs_vecs); 105 106 for_each_hw_engine(hwe, gt, id) { 107 switch (hwe->class) { 108 case XE_ENGINE_CLASS_COPY: 109 hwe->hw_engine_group = group_bcs; 110 break; 111 case XE_ENGINE_CLASS_RENDER: 112 case XE_ENGINE_CLASS_COMPUTE: 113 hwe->hw_engine_group = group_rcs_ccs; 114 break; 115 case XE_ENGINE_CLASS_VIDEO_DECODE: 116 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 117 hwe->hw_engine_group = group_vcs_vecs; 118 break; 119 case XE_ENGINE_CLASS_OTHER: 120 break; 121 case XE_ENGINE_CLASS_MAX: 122 xe_gt_assert(gt, false); 123 } 124 } 125 126 return 0; 127 } 128 129 /** 130 * xe_hw_engine_group_add_exec_queue() - Add an exec queue to a hw engine group 131 * @group: The hw engine group 132 * @q: The exec_queue 133 * 134 * Return: 0 on success, 135 * -EINTR if the lock could not be acquired 136 */ 137 int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q) 138 { 139 int err; 140 struct xe_device *xe = gt_to_xe(q->gt); 141 142 xe_assert(xe, group); 143 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM)); 144 xe_assert(xe, q->vm); 145 146 if (xe_vm_in_preempt_fence_mode(q->vm)) 147 return 0; 148 149 err = down_write_killable(&group->mode_sem); 150 if (err) 151 return err; 152 153 if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) { 154 /* 155 * suspend() can fail (e.g. killed/banned/wedged), leaving the 156 * queue un-suspended. Propagate the failure so the queue is not 157 * added; on failure nothing was suspended, so there is nothing to 158 * undo. Only record the queue as suspended (and later resume it) 159 * once suspend() has succeeded. 160 */ 161 err = q->ops->suspend(q); 162 if (err) 163 goto err_suspend; 164 165 WRITE_ONCE(q->lr.suspended, true); 166 err = q->ops->suspend_wait(q); 167 if (err) 168 goto err_suspend; 169 170 xe_hw_engine_group_resume_faulting_lr_jobs(group); 171 } 172 173 list_add(&q->hw_engine_group_link, &group->exec_queue_list); 174 up_write(&group->mode_sem); 175 176 return 0; 177 178 err_suspend: 179 up_write(&group->mode_sem); 180 return err; 181 } 182 ALLOW_ERROR_INJECTION(xe_hw_engine_group_add_exec_queue, ERRNO); 183 184 /** 185 * xe_hw_engine_group_del_exec_queue() - Delete an exec queue from a hw engine group 186 * @group: The hw engine group 187 * @q: The exec_queue 188 */ 189 void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q) 190 { 191 struct xe_device *xe = gt_to_xe(q->gt); 192 193 xe_assert(xe, group); 194 xe_assert(xe, q->vm); 195 196 down_write(&group->mode_sem); 197 198 if (!list_empty(&q->hw_engine_group_link)) 199 list_del(&q->hw_engine_group_link); 200 201 up_write(&group->mode_sem); 202 } 203 204 /** 205 * xe_hw_engine_group_resume_faulting_lr_jobs() - Asynchronously resume the hw engine group's 206 * faulting LR jobs 207 * @group: The hw engine group 208 */ 209 void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group) 210 { 211 queue_work(group->resume_wq, &group->resume_work); 212 } 213 214 /** 215 * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group 216 * @group: The hw engine group 217 * @has_deps: dma-fence job triggering suspend has dependencies 218 * 219 * Return: 0 on success, negative error code on error. 220 */ 221 static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group, 222 bool has_deps) 223 { 224 int err; 225 struct xe_exec_queue *q; 226 struct xe_gt *gt = NULL; 227 bool need_resume = false; 228 ktime_t start = xe_gt_stats_ktime_get(); 229 230 lockdep_assert_held_write(&group->mode_sem); 231 232 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 233 234 if (!xe_vm_in_fault_mode(q->vm)) 235 continue; 236 237 if (has_deps) 238 return -EAGAIN; 239 240 xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1); 241 /* 242 * suspend() only fails when the queue is killed/banned/wedged. 243 * Such a queue is being torn down (its removal from HW is handled 244 * by the kill/ban teardown), so it is not a live fault-mode 245 * context the mode switch must preempt. Skip it rather than 246 * failing the switch, otherwise one dying sibling would block a 247 * dma-fence submission on the healthy queues in the group. Only 248 * queues recorded as suspended below are later waited on and 249 * resumed. 250 */ 251 err = q->ops->suspend(q); 252 if (err) 253 continue; 254 255 WRITE_ONCE(q->lr.suspended, true); 256 need_resume = true; 257 gt = q->gt; 258 } 259 260 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 261 if (!xe_vm_in_fault_mode(q->vm)) 262 continue; 263 264 /* Only wait on queues that were actually suspended above. */ 265 if (!READ_ONCE(q->lr.suspended)) 266 continue; 267 268 err = q->ops->suspend_wait(q); 269 if (err) 270 goto err_resume; 271 } 272 273 if (gt) { 274 xe_gt_stats_incr(gt, 275 XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US, 276 xe_gt_stats_ktime_us_delta(start)); 277 } 278 279 if (need_resume) 280 xe_hw_engine_group_resume_faulting_lr_jobs(group); 281 282 return 0; 283 284 err_resume: 285 /* 286 * A suspend_wait() failed partway through the mode switch. Resume the 287 * sibling queues that were already suspended in this call so they are 288 * not left suspended forever. 289 * 290 * resume() requires the suspend to have completed (suspend_pending 291 * cleared) or it trips the !suspend_pending assert. So skip the resume 292 * when either: 293 * - suspend_wait_blocking() fails: on a GuC timeout it bans the queue 294 * and triggers cleanup, so the queue is being torn down; or 295 * - reset_status() is true: the queue was reset/killed/banned/wedged. 296 * suspend_wait() can return success in this case via its killed/ 297 * stopped wait condition while suspend_pending is still set, and the 298 * queue is being torn down anyway, so its state is resolved by 299 * teardown rather than by a resume here. 300 * In either case leave the queue marked suspended. 301 * 302 * Use the *blocking* (uninterruptible) wait here: the queues resumed on 303 * this path may belong to a different process than the one that 304 * triggered the mode switch. An interruptible suspend_wait() would 305 * return -ERESTARTSYS if the triggering task is signalled, skip the 306 * resume, and leave the other process's queue suspended forever 307 * (cross-process DoS). 308 */ 309 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 310 if (!xe_vm_in_fault_mode(q->vm)) 311 continue; 312 313 if (!READ_ONCE(q->lr.suspended)) 314 continue; 315 316 if (q->ops->suspend_wait_blocking(q) || q->ops->reset_status(q)) 317 continue; 318 319 WRITE_ONCE(q->lr.suspended, false); 320 q->ops->resume(q); 321 } 322 323 return err; 324 } 325 326 /** 327 * xe_hw_engine_group_wait_for_dma_fence_jobs() - Wait for dma fence jobs to complete 328 * @group: The hw engine group 329 * 330 * This function is not meant to be called directly from a user IOCTL as dma_fence_wait() 331 * is not interruptible. 332 * 333 * Return: 0 on success, 334 * -ETIME if waiting for one job failed 335 */ 336 static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group *group) 337 { 338 long timeout; 339 struct xe_exec_queue *q; 340 struct xe_gt *gt = NULL; 341 struct dma_fence *fence; 342 ktime_t start = xe_gt_stats_ktime_get(); 343 344 lockdep_assert_held_write(&group->mode_sem); 345 346 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 347 if (xe_vm_in_lr_mode(q->vm)) 348 continue; 349 350 xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1); 351 fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm); 352 timeout = dma_fence_wait(fence, false); 353 dma_fence_put(fence); 354 gt = q->gt; 355 356 if (timeout < 0) 357 return -ETIME; 358 } 359 360 if (gt) { 361 xe_gt_stats_incr(gt, 362 XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US, 363 xe_gt_stats_ktime_us_delta(start)); 364 } 365 366 return 0; 367 } 368 369 static int switch_mode(struct xe_hw_engine_group *group, bool has_deps) 370 { 371 int err = 0; 372 enum xe_hw_engine_group_execution_mode new_mode; 373 374 lockdep_assert_held_write(&group->mode_sem); 375 376 switch (group->cur_mode) { 377 case EXEC_MODE_LR: 378 new_mode = EXEC_MODE_DMA_FENCE; 379 err = xe_hw_engine_group_suspend_faulting_lr_jobs(group, 380 has_deps); 381 break; 382 case EXEC_MODE_DMA_FENCE: 383 new_mode = EXEC_MODE_LR; 384 err = xe_hw_engine_group_wait_for_dma_fence_jobs(group); 385 break; 386 } 387 388 if (err) 389 return err; 390 391 group->cur_mode = new_mode; 392 393 return 0; 394 } 395 396 static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs) 397 { 398 int err, i; 399 400 for (i = 0; i < num_syncs; ++i) { 401 err = xe_sync_entry_wait(syncs + i); 402 if (err) 403 return err; 404 } 405 406 return 0; 407 } 408 409 /** 410 * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode 411 * @group: The hw engine group 412 * @new_mode: The new execution mode 413 * @previous_mode: Pointer to the previous mode provided for use by caller 414 * @syncs: Syncs from exec IOCTL 415 * @num_syncs: Number of syncs from exec IOCTL 416 * 417 * Return: 0 if successful, -EINTR if locking failed. 418 */ 419 int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group, 420 enum xe_hw_engine_group_execution_mode new_mode, 421 enum xe_hw_engine_group_execution_mode *previous_mode, 422 struct xe_sync_entry *syncs, int num_syncs) 423 __acquires(&group->mode_sem) 424 { 425 bool has_deps = !!num_syncs; 426 int err = down_read_interruptible(&group->mode_sem); 427 428 if (err) 429 return err; 430 431 *previous_mode = group->cur_mode; 432 433 if (new_mode != group->cur_mode) { 434 up_read(&group->mode_sem); 435 retry: 436 err = down_write_killable(&group->mode_sem); 437 if (err) 438 return err; 439 440 if (new_mode != group->cur_mode) { 441 err = switch_mode(group, has_deps); 442 if (err) { 443 up_write(&group->mode_sem); 444 445 if (err != -EAGAIN) 446 return err; 447 448 err = wait_syncs(syncs, num_syncs); 449 if (err) 450 return err; 451 452 has_deps = false; 453 goto retry; 454 } 455 } 456 downgrade_write(&group->mode_sem); 457 } 458 459 return err; 460 } 461 462 /** 463 * xe_hw_engine_group_put() - Put the group 464 * @group: The hw engine group 465 */ 466 void xe_hw_engine_group_put(struct xe_hw_engine_group *group) 467 __releases(&group->mode_sem) 468 { 469 up_read(&group->mode_sem); 470 } 471 472 /** 473 * xe_hw_engine_group_find_exec_mode() - Find the execution mode for this exec queue 474 * @q: The exec_queue 475 */ 476 enum xe_hw_engine_group_execution_mode 477 xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q) 478 { 479 if (xe_vm_in_fault_mode(q->vm)) 480 return EXEC_MODE_LR; 481 else 482 return EXEC_MODE_DMA_FENCE; 483 } 484