1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2024 Intel Corporation 4 */ 5 6 #include <drm/drm_managed.h> 7 8 #include "xe_assert.h" 9 #include "xe_device_types.h" 10 #include "xe_exec_queue.h" 11 #include "xe_gt.h" 12 #include "xe_gt_stats.h" 13 #include "xe_hw_engine_group.h" 14 #include "xe_sync.h" 15 #include "xe_vm.h" 16 17 static void 18 hw_engine_group_resume_lr_jobs_func(struct work_struct *w) 19 { 20 struct xe_exec_queue *q; 21 struct xe_hw_engine_group *group = container_of(w, struct xe_hw_engine_group, resume_work); 22 int err; 23 enum xe_hw_engine_group_execution_mode previous_mode; 24 25 err = xe_hw_engine_group_get_mode(group, EXEC_MODE_LR, &previous_mode, 26 NULL, 0); 27 if (err) 28 return; 29 30 if (previous_mode == EXEC_MODE_LR) 31 goto put; 32 33 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 34 if (!xe_vm_in_fault_mode(q->vm)) 35 continue; 36 37 q->ops->resume(q); 38 } 39 40 put: 41 xe_hw_engine_group_put(group); 42 } 43 44 static struct xe_hw_engine_group * 45 hw_engine_group_alloc(struct xe_device *xe) 46 { 47 struct xe_hw_engine_group *group; 48 int err; 49 50 group = drmm_kzalloc(&xe->drm, sizeof(*group), GFP_KERNEL); 51 if (!group) 52 return ERR_PTR(-ENOMEM); 53 54 group->resume_wq = alloc_workqueue("xe-resume-lr-jobs-wq", WQ_PERCPU, 55 0); 56 if (!group->resume_wq) 57 return ERR_PTR(-ENOMEM); 58 59 err = drmm_add_action_or_reset(&xe->drm, __drmm_workqueue_release, group->resume_wq); 60 if (err) 61 return ERR_PTR(err); 62 63 init_rwsem(&group->mode_sem); 64 INIT_WORK(&group->resume_work, hw_engine_group_resume_lr_jobs_func); 65 INIT_LIST_HEAD(&group->exec_queue_list); 66 67 return group; 68 } 69 70 /** 71 * xe_hw_engine_setup_groups() - Setup the hw engine groups for the gt 72 * @gt: The gt for which groups are setup 73 * 74 * Return: 0 on success, negative error code on error. 75 */ 76 int xe_hw_engine_setup_groups(struct xe_gt *gt) 77 { 78 struct xe_hw_engine *hwe; 79 enum xe_hw_engine_id id; 80 struct xe_hw_engine_group *group_rcs_ccs, *group_bcs, *group_vcs_vecs; 81 struct xe_device *xe = gt_to_xe(gt); 82 83 group_rcs_ccs = hw_engine_group_alloc(xe); 84 if (IS_ERR(group_rcs_ccs)) 85 return PTR_ERR(group_rcs_ccs); 86 87 group_bcs = hw_engine_group_alloc(xe); 88 if (IS_ERR(group_bcs)) 89 return PTR_ERR(group_bcs); 90 91 group_vcs_vecs = hw_engine_group_alloc(xe); 92 if (IS_ERR(group_vcs_vecs)) 93 return PTR_ERR(group_vcs_vecs); 94 95 for_each_hw_engine(hwe, gt, id) { 96 switch (hwe->class) { 97 case XE_ENGINE_CLASS_COPY: 98 hwe->hw_engine_group = group_bcs; 99 break; 100 case XE_ENGINE_CLASS_RENDER: 101 case XE_ENGINE_CLASS_COMPUTE: 102 hwe->hw_engine_group = group_rcs_ccs; 103 break; 104 case XE_ENGINE_CLASS_VIDEO_DECODE: 105 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 106 hwe->hw_engine_group = group_vcs_vecs; 107 break; 108 case XE_ENGINE_CLASS_OTHER: 109 break; 110 case XE_ENGINE_CLASS_MAX: 111 xe_gt_assert(gt, false); 112 } 113 } 114 115 return 0; 116 } 117 118 /** 119 * xe_hw_engine_group_add_exec_queue() - Add an exec queue to a hw engine group 120 * @group: The hw engine group 121 * @q: The exec_queue 122 * 123 * Return: 0 on success, 124 * -EINTR if the lock could not be acquired 125 */ 126 int xe_hw_engine_group_add_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q) 127 { 128 int err; 129 struct xe_device *xe = gt_to_xe(q->gt); 130 131 xe_assert(xe, group); 132 xe_assert(xe, !(q->flags & EXEC_QUEUE_FLAG_VM)); 133 xe_assert(xe, q->vm); 134 135 if (xe_vm_in_preempt_fence_mode(q->vm)) 136 return 0; 137 138 err = down_write_killable(&group->mode_sem); 139 if (err) 140 return err; 141 142 if (xe_vm_in_fault_mode(q->vm) && group->cur_mode == EXEC_MODE_DMA_FENCE) { 143 q->ops->suspend(q); 144 err = q->ops->suspend_wait(q); 145 if (err) 146 goto err_suspend; 147 148 xe_hw_engine_group_resume_faulting_lr_jobs(group); 149 } 150 151 list_add(&q->hw_engine_group_link, &group->exec_queue_list); 152 up_write(&group->mode_sem); 153 154 return 0; 155 156 err_suspend: 157 up_write(&group->mode_sem); 158 return err; 159 } 160 ALLOW_ERROR_INJECTION(xe_hw_engine_group_add_exec_queue, ERRNO); 161 162 /** 163 * xe_hw_engine_group_del_exec_queue() - Delete an exec queue from a hw engine group 164 * @group: The hw engine group 165 * @q: The exec_queue 166 */ 167 void xe_hw_engine_group_del_exec_queue(struct xe_hw_engine_group *group, struct xe_exec_queue *q) 168 { 169 struct xe_device *xe = gt_to_xe(q->gt); 170 171 xe_assert(xe, group); 172 xe_assert(xe, q->vm); 173 174 down_write(&group->mode_sem); 175 176 if (!list_empty(&q->hw_engine_group_link)) 177 list_del(&q->hw_engine_group_link); 178 179 up_write(&group->mode_sem); 180 } 181 182 /** 183 * xe_hw_engine_group_resume_faulting_lr_jobs() - Asynchronously resume the hw engine group's 184 * faulting LR jobs 185 * @group: The hw engine group 186 */ 187 void xe_hw_engine_group_resume_faulting_lr_jobs(struct xe_hw_engine_group *group) 188 { 189 queue_work(group->resume_wq, &group->resume_work); 190 } 191 192 /** 193 * xe_hw_engine_group_suspend_faulting_lr_jobs() - Suspend the faulting LR jobs of this group 194 * @group: The hw engine group 195 * @has_deps: dma-fence job triggering suspend has dependencies 196 * 197 * Return: 0 on success, negative error code on error. 198 */ 199 static int xe_hw_engine_group_suspend_faulting_lr_jobs(struct xe_hw_engine_group *group, 200 bool has_deps) 201 { 202 int err; 203 struct xe_exec_queue *q; 204 struct xe_gt *gt = NULL; 205 bool need_resume = false; 206 ktime_t start = xe_gt_stats_ktime_get(); 207 208 lockdep_assert_held_write(&group->mode_sem); 209 210 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 211 bool idle_skip_suspend; 212 213 if (!xe_vm_in_fault_mode(q->vm)) 214 continue; 215 216 idle_skip_suspend = xe_exec_queue_idle_skip_suspend(q); 217 if (!idle_skip_suspend && has_deps) 218 return -EAGAIN; 219 220 xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT, 1); 221 if (idle_skip_suspend) 222 xe_gt_stats_incr(q->gt, 223 XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT, 1); 224 225 need_resume |= !idle_skip_suspend; 226 q->ops->suspend(q); 227 gt = q->gt; 228 } 229 230 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 231 if (!xe_vm_in_fault_mode(q->vm)) 232 continue; 233 234 err = q->ops->suspend_wait(q); 235 if (err) 236 return err; 237 } 238 239 if (gt) { 240 xe_gt_stats_incr(gt, 241 XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US, 242 xe_gt_stats_ktime_us_delta(start)); 243 } 244 245 if (need_resume) 246 xe_hw_engine_group_resume_faulting_lr_jobs(group); 247 248 return 0; 249 } 250 251 /** 252 * xe_hw_engine_group_wait_for_dma_fence_jobs() - Wait for dma fence jobs to complete 253 * @group: The hw engine group 254 * 255 * This function is not meant to be called directly from a user IOCTL as dma_fence_wait() 256 * is not interruptible. 257 * 258 * Return: 0 on success, 259 * -ETIME if waiting for one job failed 260 */ 261 static int xe_hw_engine_group_wait_for_dma_fence_jobs(struct xe_hw_engine_group *group) 262 { 263 long timeout; 264 struct xe_exec_queue *q; 265 struct xe_gt *gt = NULL; 266 struct dma_fence *fence; 267 ktime_t start = xe_gt_stats_ktime_get(); 268 269 lockdep_assert_held_write(&group->mode_sem); 270 271 list_for_each_entry(q, &group->exec_queue_list, hw_engine_group_link) { 272 if (xe_vm_in_lr_mode(q->vm)) 273 continue; 274 275 xe_gt_stats_incr(q->gt, XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT, 1); 276 fence = xe_exec_queue_last_fence_get_for_resume(q, q->vm); 277 timeout = dma_fence_wait(fence, false); 278 dma_fence_put(fence); 279 gt = q->gt; 280 281 if (timeout < 0) 282 return -ETIME; 283 } 284 285 if (gt) { 286 xe_gt_stats_incr(gt, 287 XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US, 288 xe_gt_stats_ktime_us_delta(start)); 289 } 290 291 return 0; 292 } 293 294 static int switch_mode(struct xe_hw_engine_group *group, bool has_deps) 295 { 296 int err = 0; 297 enum xe_hw_engine_group_execution_mode new_mode; 298 299 lockdep_assert_held_write(&group->mode_sem); 300 301 switch (group->cur_mode) { 302 case EXEC_MODE_LR: 303 new_mode = EXEC_MODE_DMA_FENCE; 304 err = xe_hw_engine_group_suspend_faulting_lr_jobs(group, 305 has_deps); 306 break; 307 case EXEC_MODE_DMA_FENCE: 308 new_mode = EXEC_MODE_LR; 309 err = xe_hw_engine_group_wait_for_dma_fence_jobs(group); 310 break; 311 } 312 313 if (err) 314 return err; 315 316 group->cur_mode = new_mode; 317 318 return 0; 319 } 320 321 static int wait_syncs(struct xe_sync_entry *syncs, int num_syncs) 322 { 323 int err, i; 324 325 for (i = 0; i < num_syncs; ++i) { 326 err = xe_sync_entry_wait(syncs + i); 327 if (err) 328 return err; 329 } 330 331 return 0; 332 } 333 334 /** 335 * xe_hw_engine_group_get_mode() - Get the group to execute in the new mode 336 * @group: The hw engine group 337 * @new_mode: The new execution mode 338 * @previous_mode: Pointer to the previous mode provided for use by caller 339 * @syncs: Syncs from exec IOCTL 340 * @num_syncs: Number of syncs from exec IOCTL 341 * 342 * Return: 0 if successful, -EINTR if locking failed. 343 */ 344 int xe_hw_engine_group_get_mode(struct xe_hw_engine_group *group, 345 enum xe_hw_engine_group_execution_mode new_mode, 346 enum xe_hw_engine_group_execution_mode *previous_mode, 347 struct xe_sync_entry *syncs, int num_syncs) 348 __acquires(&group->mode_sem) 349 { 350 bool has_deps = !!num_syncs; 351 int err = down_read_interruptible(&group->mode_sem); 352 353 if (err) 354 return err; 355 356 *previous_mode = group->cur_mode; 357 358 if (new_mode != group->cur_mode) { 359 up_read(&group->mode_sem); 360 retry: 361 err = down_write_killable(&group->mode_sem); 362 if (err) 363 return err; 364 365 if (new_mode != group->cur_mode) { 366 err = switch_mode(group, has_deps); 367 if (err) { 368 up_write(&group->mode_sem); 369 370 if (err != -EAGAIN) 371 return err; 372 373 err = wait_syncs(syncs, num_syncs); 374 if (err) 375 return err; 376 377 has_deps = false; 378 goto retry; 379 } 380 } 381 downgrade_write(&group->mode_sem); 382 } 383 384 return err; 385 } 386 387 /** 388 * xe_hw_engine_group_put() - Put the group 389 * @group: The hw engine group 390 */ 391 void xe_hw_engine_group_put(struct xe_hw_engine_group *group) 392 __releases(&group->mode_sem) 393 { 394 up_read(&group->mode_sem); 395 } 396 397 /** 398 * xe_hw_engine_group_find_exec_mode() - Find the execution mode for this exec queue 399 * @q: The exec_queue 400 */ 401 enum xe_hw_engine_group_execution_mode 402 xe_hw_engine_group_find_exec_mode(struct xe_exec_queue *q) 403 { 404 if (xe_vm_in_fault_mode(q->vm)) 405 return EXEC_MODE_LR; 406 else 407 return EXEC_MODE_DMA_FENCE; 408 } 409