1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_submit.h" 7 8 #include <linux/bitfield.h> 9 #include <uapi/drm/xe_drm.h> 10 #include <linux/bitmap.h> 11 #include <linux/circ_buf.h> 12 #include <linux/dma-fence-array.h> 13 14 #include <drm/drm_drv.h> 15 #include <drm/drm_managed.h> 16 17 #include "abi/guc_actions_abi.h" 18 #include "abi/guc_actions_slpc_abi.h" 19 #include "abi/guc_klvs_abi.h" 20 #include "xe_assert.h" 21 #include "xe_bo.h" 22 #include "xe_cpu_bind.h" 23 #include "xe_devcoredump.h" 24 #include "xe_device.h" 25 #include "xe_exec_queue.h" 26 #include "xe_force_wake.h" 27 #include "xe_gpu_scheduler.h" 28 #include "xe_gt.h" 29 #include "xe_gt_clock.h" 30 #include "xe_gt_printk.h" 31 #include "xe_guc.h" 32 #include "xe_guc_capture.h" 33 #include "xe_guc_ct.h" 34 #include "xe_guc_exec_queue_types.h" 35 #include "xe_guc_id_mgr.h" 36 #include "xe_guc_klv_helpers.h" 37 #include "xe_guc_submit_types.h" 38 #include "xe_hw_engine.h" 39 #include "xe_log.h" 40 #include "xe_lrc.h" 41 #include "xe_macros.h" 42 #include "xe_map.h" 43 #include "xe_mocs.h" 44 #include "xe_module.h" 45 #include "xe_pm.h" 46 #include "xe_pt.h" 47 #include "xe_ring_ops_types.h" 48 #include "xe_sched_job.h" 49 #include "xe_sleep.h" 50 #include "xe_trace.h" 51 #include "xe_uc_fw.h" 52 #include "xe_vm.h" 53 54 #define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN 6 55 56 static int guc_submit_reset_prepare(struct xe_guc *guc); 57 58 static struct xe_guc * 59 exec_queue_to_guc(struct xe_exec_queue *q) 60 { 61 return &q->gt->uc.guc; 62 } 63 64 /* 65 * Helpers for engine state, using an atomic as some of the bits can transition 66 * as the same time (e.g. a suspend can be happning at the same time as schedule 67 * engine done being processed). 68 */ 69 #define EXEC_QUEUE_STATE_REGISTERED (1 << 0) 70 #define EXEC_QUEUE_STATE_ENABLED (1 << 1) 71 #define EXEC_QUEUE_STATE_PENDING_ENABLE (1 << 2) 72 #define EXEC_QUEUE_STATE_PENDING_DISABLE (1 << 3) 73 #define EXEC_QUEUE_STATE_DESTROYED (1 << 4) 74 #define EXEC_QUEUE_STATE_SUSPENDED (1 << 5) 75 #define EXEC_QUEUE_STATE_RESET (1 << 6) 76 #define EXEC_QUEUE_STATE_KILLED (1 << 7) 77 #define EXEC_QUEUE_STATE_WEDGED (1 << 8) 78 #define EXEC_QUEUE_STATE_BANNED (1 << 9) 79 #define EXEC_QUEUE_STATE_PENDING_RESUME (1 << 10) 80 81 static bool exec_queue_registered(struct xe_exec_queue *q) 82 { 83 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED; 84 } 85 86 static void set_exec_queue_registered(struct xe_exec_queue *q) 87 { 88 atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); 89 } 90 91 static void clear_exec_queue_registered(struct xe_exec_queue *q) 92 { 93 atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); 94 } 95 96 static bool exec_queue_enabled(struct xe_exec_queue *q) 97 { 98 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED; 99 } 100 101 static void set_exec_queue_enabled(struct xe_exec_queue *q) 102 { 103 atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state); 104 } 105 106 static void clear_exec_queue_enabled(struct xe_exec_queue *q) 107 { 108 atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state); 109 } 110 111 static bool exec_queue_pending_enable(struct xe_exec_queue *q) 112 { 113 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE; 114 } 115 116 static void set_exec_queue_pending_enable(struct xe_exec_queue *q) 117 { 118 atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); 119 } 120 121 static void clear_exec_queue_pending_enable(struct xe_exec_queue *q) 122 { 123 atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); 124 } 125 126 static bool exec_queue_pending_disable(struct xe_exec_queue *q) 127 { 128 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE; 129 } 130 131 static void set_exec_queue_pending_disable(struct xe_exec_queue *q) 132 { 133 atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); 134 } 135 136 static void clear_exec_queue_pending_disable(struct xe_exec_queue *q) 137 { 138 atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); 139 } 140 141 static bool exec_queue_destroyed(struct xe_exec_queue *q) 142 { 143 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED; 144 } 145 146 static void set_exec_queue_destroyed(struct xe_exec_queue *q) 147 { 148 atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); 149 } 150 151 static void clear_exec_queue_destroyed(struct xe_exec_queue *q) 152 { 153 atomic_and(~EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); 154 } 155 156 static bool exec_queue_banned(struct xe_exec_queue *q) 157 { 158 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED; 159 } 160 161 static void set_exec_queue_banned(struct xe_exec_queue *q) 162 { 163 atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state); 164 } 165 166 static void clear_exec_queue_banned(struct xe_exec_queue *q) 167 { 168 atomic_andnot(EXEC_QUEUE_STATE_BANNED, &q->guc->state); 169 } 170 171 static bool exec_queue_suspended(struct xe_exec_queue *q) 172 { 173 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED; 174 } 175 176 static void set_exec_queue_suspended(struct xe_exec_queue *q) 177 { 178 atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); 179 } 180 181 static void clear_exec_queue_suspended(struct xe_exec_queue *q) 182 { 183 atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); 184 } 185 186 static bool exec_queue_reset(struct xe_exec_queue *q) 187 { 188 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET; 189 } 190 191 static void set_exec_queue_reset(struct xe_exec_queue *q) 192 { 193 atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state); 194 } 195 196 static bool exec_queue_killed(struct xe_exec_queue *q) 197 { 198 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED; 199 } 200 201 static void set_exec_queue_killed(struct xe_exec_queue *q) 202 { 203 atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state); 204 } 205 206 static bool exec_queue_wedged(struct xe_exec_queue *q) 207 { 208 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED; 209 } 210 211 static void set_exec_queue_wedged(struct xe_exec_queue *q) 212 { 213 atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state); 214 } 215 216 static bool exec_queue_pending_resume(struct xe_exec_queue *q) 217 { 218 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_RESUME; 219 } 220 221 static void set_exec_queue_pending_resume(struct xe_exec_queue *q) 222 { 223 atomic_or(EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state); 224 } 225 226 static void clear_exec_queue_pending_resume(struct xe_exec_queue *q) 227 { 228 atomic_and(~EXEC_QUEUE_STATE_PENDING_RESUME, &q->guc->state); 229 } 230 231 static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) 232 { 233 return (atomic_read(&q->guc->state) & 234 (EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED | 235 EXEC_QUEUE_STATE_BANNED)); 236 } 237 238 static void guc_submit_sw_fini(struct drm_device *drm, void *arg) 239 { 240 struct xe_guc *guc = arg; 241 struct xe_gt *gt = guc_to_gt(guc); 242 243 xe_gt_assert(gt, xa_empty(&guc->submission_state.exec_queue_lookup)); 244 245 xa_destroy(&guc->submission_state.exec_queue_lookup); 246 } 247 248 static void guc_submit_fini(void *arg) 249 { 250 struct xe_guc *guc = arg; 251 struct xe_exec_queue *q; 252 unsigned long index; 253 254 /* Drop any wedged queue refs */ 255 mutex_lock(&guc->submission_state.lock); 256 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 257 if (exec_queue_wedged(q)) { 258 mutex_unlock(&guc->submission_state.lock); 259 xe_exec_queue_put(q); 260 mutex_lock(&guc->submission_state.lock); 261 } 262 } 263 mutex_unlock(&guc->submission_state.lock); 264 265 /* Forcefully kill any remaining exec queues */ 266 xe_guc_ct_stop(&guc->ct); 267 guc_submit_reset_prepare(guc); 268 xe_guc_softreset(guc); 269 xe_guc_submit_stop(guc); 270 xe_uc_fw_sanitize(&guc->fw); 271 xe_guc_submit_pause_abort(guc); 272 } 273 274 static const struct xe_exec_queue_ops guc_exec_queue_ops; 275 276 static void primelockdep(struct xe_guc *guc) 277 { 278 if (!IS_ENABLED(CONFIG_LOCKDEP)) 279 return; 280 281 fs_reclaim_acquire(GFP_KERNEL); 282 283 mutex_lock(&guc->submission_state.lock); 284 mutex_unlock(&guc->submission_state.lock); 285 286 fs_reclaim_release(GFP_KERNEL); 287 } 288 289 /** 290 * xe_guc_submit_init() - Initialize GuC submission. 291 * @guc: the &xe_guc to initialize 292 * @num_ids: number of GuC context IDs to use 293 * 294 * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all 295 * GuC context IDs supported by the GuC firmware should be used for submission. 296 * 297 * Only VF drivers will have to provide explicit number of GuC context IDs 298 * that they can use for submission. 299 * 300 * Return: 0 on success or a negative error code on failure. 301 */ 302 int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) 303 { 304 struct xe_device *xe = guc_to_xe(guc); 305 struct xe_gt *gt = guc_to_gt(guc); 306 int err; 307 308 err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock); 309 if (err) 310 return err; 311 312 err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids); 313 if (err) 314 return err; 315 316 gt->exec_queue_ops = &guc_exec_queue_ops; 317 318 xa_init(&guc->submission_state.exec_queue_lookup); 319 320 primelockdep(guc); 321 322 guc->submission_state.initialized = true; 323 324 err = drmm_add_action_or_reset(&xe->drm, guc_submit_sw_fini, guc); 325 if (err) 326 return err; 327 328 return devm_add_action_or_reset(xe->drm.dev, guc_submit_fini, guc); 329 } 330 331 /* 332 * Given that we want to guarantee enough RCS throughput to avoid missing 333 * frames, we set the yield policy to 20% of each 80ms interval. 334 */ 335 #define RC_YIELD_DURATION 80 /* in ms */ 336 #define RC_YIELD_RATIO 20 /* in percent */ 337 static u32 *emit_render_compute_yield_klv(u32 *emit) 338 { 339 *emit++ = PREP_GUC_KLV_TAG(SCHEDULING_POLICIES_RENDER_COMPUTE_YIELD); 340 *emit++ = RC_YIELD_DURATION; 341 *emit++ = RC_YIELD_RATIO; 342 343 return emit; 344 } 345 346 #define SCHEDULING_POLICY_MAX_DWORDS 16 347 static int guc_init_global_schedule_policy(struct xe_guc *guc) 348 { 349 u32 data[SCHEDULING_POLICY_MAX_DWORDS]; 350 u32 *emit = data; 351 u32 count = 0; 352 int ret; 353 354 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0)) 355 return 0; 356 357 *emit++ = XE_GUC_ACTION_UPDATE_SCHEDULING_POLICIES_KLV; 358 359 if (CCS_INSTANCES(guc_to_gt(guc))) 360 emit = emit_render_compute_yield_klv(emit); 361 362 count = emit - data; 363 if (count > 1) { 364 xe_assert(guc_to_xe(guc), count <= SCHEDULING_POLICY_MAX_DWORDS); 365 366 ret = xe_guc_ct_send_block(&guc->ct, data, count); 367 if (ret < 0) { 368 xe_gt_err(guc_to_gt(guc), 369 "failed to enable GuC scheduling policies: %pe\n", 370 ERR_PTR(ret)); 371 return ret; 372 } 373 } 374 375 return 0; 376 } 377 378 int xe_guc_submit_enable(struct xe_guc *guc) 379 { 380 int ret; 381 382 ret = guc_init_global_schedule_policy(guc); 383 if (ret) 384 return ret; 385 386 guc->submission_state.enabled = true; 387 388 return 0; 389 } 390 391 void xe_guc_submit_disable(struct xe_guc *guc) 392 { 393 guc->submission_state.enabled = false; 394 } 395 396 static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, 397 int count) 398 { 399 int i; 400 401 mutex_lock(&guc->submission_state.lock); 402 403 for (i = 0; i < count; ++i) 404 xa_erase(&guc->submission_state.exec_queue_lookup, 405 q->guc->id + i); 406 407 xe_guc_id_mgr_release_locked(&guc->submission_state.idm, 408 q->guc->id, q->width); 409 410 mutex_unlock(&guc->submission_state.lock); 411 } 412 413 static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) 414 { 415 int ret, i; 416 417 mutex_lock(&guc->submission_state.lock); 418 ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm, 419 q->width); 420 mutex_unlock(&guc->submission_state.lock); 421 if (ret < 0) 422 return ret; 423 424 q->guc->id = ret; 425 426 /* Reserve empty slots. */ 427 for (i = 0; i < q->width; ++i) { 428 ret = xa_insert(&guc->submission_state.exec_queue_lookup, 429 q->guc->id + i, NULL, GFP_KERNEL); 430 if (ret) 431 goto err_release; 432 } 433 434 return 0; 435 436 err_release: 437 __release_guc_id(guc, q, i); 438 439 return ret; 440 } 441 442 static void publish_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) 443 { 444 int i; 445 446 lockdep_assert_held(&guc->submission_state.lock); 447 448 for (i = 0; i < q->width; ++i) { 449 void *old; 450 451 old = xa_store(&guc->submission_state.exec_queue_lookup, 452 q->guc->id + i, q, GFP_NOWAIT); 453 XE_WARN_ON(old || xa_is_err(old)); 454 } 455 } 456 457 static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) 458 { 459 __release_guc_id(guc, q, q->width); 460 } 461 462 struct exec_queue_policy { 463 u32 count; 464 struct guc_update_exec_queue_policy h2g; 465 }; 466 467 static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy) 468 { 469 size_t bytes = sizeof(policy->h2g.header) + 470 (sizeof(policy->h2g.klv[0]) * policy->count); 471 472 return bytes / sizeof(u32); 473 } 474 475 static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy, 476 u16 guc_id) 477 { 478 policy->h2g.header.action = 479 XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; 480 policy->h2g.header.guc_id = guc_id; 481 policy->count = 0; 482 } 483 484 #define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \ 485 static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \ 486 u32 data) \ 487 { \ 488 XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ 489 \ 490 policy->h2g.klv[policy->count].kl = \ 491 FIELD_PREP(GUC_KLV_0_KEY, \ 492 GUC_CONTEXT_POLICIES_KLV_ID_##id) | \ 493 FIELD_PREP(GUC_KLV_0_LEN, 1); \ 494 policy->h2g.klv[policy->count].value = data; \ 495 policy->count++; \ 496 } 497 498 MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) 499 MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) 500 MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY) 501 MAKE_EXEC_QUEUE_POLICY_ADD(slpc_exec_queue_freq_req, SLPM_GT_FREQUENCY) 502 #undef MAKE_EXEC_QUEUE_POLICY_ADD 503 504 static const int xe_exec_queue_prio_to_guc[] = { 505 [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, 506 [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, 507 [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, 508 [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, 509 }; 510 511 static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) 512 { 513 struct exec_queue_policy policy; 514 enum xe_exec_queue_priority prio = q->sched_props.priority; 515 u32 timeslice_us = q->sched_props.timeslice_us; 516 u32 slpc_exec_queue_freq_req = 0; 517 u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; 518 519 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q) && 520 !xe_exec_queue_is_multi_queue_secondary(q)); 521 522 if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY) 523 slpc_exec_queue_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE; 524 525 __guc_exec_queue_policy_start_klv(&policy, q->guc->id); 526 __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]); 527 __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us); 528 __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us); 529 __guc_exec_queue_policy_add_slpc_exec_queue_freq_req(&policy, 530 slpc_exec_queue_freq_req); 531 532 xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, 533 __guc_exec_queue_policy_action_size(&policy), 0, 0); 534 } 535 536 static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q) 537 { 538 struct exec_queue_policy policy; 539 540 xe_assert(guc_to_xe(guc), !xe_exec_queue_is_multi_queue_secondary(q)); 541 542 __guc_exec_queue_policy_start_klv(&policy, q->guc->id); 543 __guc_exec_queue_policy_add_preemption_timeout(&policy, 1); 544 545 xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, 546 __guc_exec_queue_policy_action_size(&policy), 0, 0); 547 } 548 549 static bool vf_recovery(struct xe_guc *guc) 550 { 551 return xe_gt_recovery_pending(guc_to_gt(guc)); 552 } 553 554 static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) 555 { 556 struct xe_guc *guc = exec_queue_to_guc(q); 557 struct xe_device *xe = guc_to_xe(guc); 558 559 /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */ 560 wake_up_all(&xe->ufence_wq); 561 562 xe_sched_tdr_queue_imm(&q->guc->sched); 563 } 564 565 static void xe_guc_exec_queue_group_stop(struct xe_exec_queue *q) 566 { 567 struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); 568 struct xe_exec_queue_group *group = q->multi_queue.group; 569 struct xe_exec_queue *eq, *next; 570 LIST_HEAD(tmp); 571 572 xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), 573 xe_exec_queue_is_multi_queue(q)); 574 575 mutex_lock(&group->list_lock); 576 577 /* 578 * Stop all future queues being from executing while group is stopped. 579 */ 580 group->stopped = true; 581 582 list_for_each_entry_safe(eq, next, &group->list, multi_queue.link) 583 /* 584 * Refcount prevents an attempted removal from &group->list, 585 * temporary list allows safe iteration after dropping 586 * &group->list_lock. 587 */ 588 if (xe_exec_queue_get_unless_zero(eq)) 589 list_move_tail(&eq->multi_queue.link, &tmp); 590 591 mutex_unlock(&group->list_lock); 592 593 /* We cannot stop under list lock without getting inversions */ 594 xe_sched_submission_stop(&primary->guc->sched); 595 list_for_each_entry(eq, &tmp, multi_queue.link) 596 xe_sched_submission_stop(&eq->guc->sched); 597 598 mutex_lock(&group->list_lock); 599 list_for_each_entry_safe(eq, next, &tmp, multi_queue.link) { 600 /* 601 * Corner where we got banned while stopping and not on 602 * &group->list 603 */ 604 if (READ_ONCE(group->banned)) 605 xe_guc_exec_queue_trigger_cleanup(eq); 606 607 list_move_tail(&eq->multi_queue.link, &group->list); 608 xe_exec_queue_put(eq); 609 } 610 mutex_unlock(&group->list_lock); 611 } 612 613 static void xe_guc_exec_queue_group_start(struct xe_exec_queue *q) 614 { 615 struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); 616 struct xe_exec_queue_group *group = q->multi_queue.group; 617 struct xe_exec_queue *eq; 618 619 xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), 620 xe_exec_queue_is_multi_queue(q)); 621 622 xe_sched_submission_start(&primary->guc->sched); 623 624 mutex_lock(&group->list_lock); 625 group->stopped = false; 626 list_for_each_entry(eq, &group->list, multi_queue.link) 627 xe_sched_submission_start(&eq->guc->sched); 628 mutex_unlock(&group->list_lock); 629 } 630 631 static void xe_guc_exec_queue_group_trigger_cleanup(struct xe_exec_queue *q) 632 { 633 struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); 634 struct xe_exec_queue_group *group = q->multi_queue.group; 635 struct xe_exec_queue *eq; 636 637 xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), 638 xe_exec_queue_is_multi_queue(q)); 639 640 /* Group banned, skip timeout check in TDR */ 641 WRITE_ONCE(group->banned, true); 642 xe_guc_exec_queue_trigger_cleanup(primary); 643 644 mutex_lock(&group->list_lock); 645 list_for_each_entry(eq, &group->list, multi_queue.link) 646 xe_guc_exec_queue_trigger_cleanup(eq); 647 mutex_unlock(&group->list_lock); 648 } 649 650 static void xe_guc_exec_queue_reset_trigger_cleanup(struct xe_exec_queue *q) 651 { 652 if (xe_exec_queue_is_multi_queue(q)) { 653 struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); 654 struct xe_exec_queue_group *group = q->multi_queue.group; 655 struct xe_exec_queue *eq; 656 657 /* Group banned, skip timeout check in TDR */ 658 WRITE_ONCE(group->banned, true); 659 660 set_exec_queue_reset(primary); 661 if (!exec_queue_banned(primary)) 662 xe_guc_exec_queue_trigger_cleanup(primary); 663 664 mutex_lock(&group->list_lock); 665 list_for_each_entry(eq, &group->list, multi_queue.link) { 666 set_exec_queue_reset(eq); 667 if (!exec_queue_banned(eq)) 668 xe_guc_exec_queue_trigger_cleanup(eq); 669 } 670 mutex_unlock(&group->list_lock); 671 } else { 672 set_exec_queue_reset(q); 673 if (!exec_queue_banned(q)) 674 xe_guc_exec_queue_trigger_cleanup(q); 675 } 676 } 677 678 static void set_exec_queue_group_banned(struct xe_exec_queue *q) 679 { 680 struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); 681 struct xe_exec_queue_group *group = q->multi_queue.group; 682 struct xe_exec_queue *eq; 683 684 /* Ban all queues of the multi-queue group */ 685 xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), 686 xe_exec_queue_is_multi_queue(q)); 687 set_exec_queue_banned(primary); 688 689 mutex_lock(&group->list_lock); 690 list_for_each_entry(eq, &group->list, multi_queue.link) 691 set_exec_queue_banned(eq); 692 mutex_unlock(&group->list_lock); 693 } 694 695 /* Helper for context registration H2G */ 696 struct guc_ctxt_registration_info { 697 u32 flags; 698 u32 context_idx; 699 u32 engine_class; 700 u32 engine_submit_mask; 701 u32 wq_desc_lo; 702 u32 wq_desc_hi; 703 u32 wq_base_lo; 704 u32 wq_base_hi; 705 u32 wq_size; 706 u32 cgp_lo; 707 u32 cgp_hi; 708 u32 hwlrca_lo; 709 u32 hwlrca_hi; 710 }; 711 712 #define parallel_read(xe_, map_, field_) \ 713 xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ 714 field_) 715 #define parallel_write(xe_, map_, field_, val_) \ 716 xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ 717 field_, val_) 718 719 /** 720 * DOC: Multi Queue Group GuC interface 721 * 722 * The multi queue group coordination between KMD and GuC is through a software 723 * construct called Context Group Page (CGP). The CGP is a KMD managed 4KB page 724 * allocated in the global GTT. 725 * 726 * CGP format: 727 * 728 * +-----------+---------------------------+---------------------------------------------+ 729 * | DWORD | Name | Description | 730 * +-----------+---------------------------+---------------------------------------------+ 731 * | 0 | Version | Bits [15:8]=Major ver, [7:0]=Minor ver | 732 * +-----------+---------------------------+---------------------------------------------+ 733 * | 1..15 | RESERVED | MBZ | 734 * +-----------+---------------------------+---------------------------------------------+ 735 * | 16 | KMD_QUEUE_UPDATE_MASK_DW0 | KMD queue mask for queues 31..0 | 736 * +-----------+---------------------------+---------------------------------------------+ 737 * | 17 | KMD_QUEUE_UPDATE_MASK_DW1 | KMD queue mask for queues 63..32 | 738 * +-----------+---------------------------+---------------------------------------------+ 739 * | 18..31 | RESERVED | MBZ | 740 * +-----------+---------------------------+---------------------------------------------+ 741 * | 32 | Q0CD_DW0 | Queue 0 context LRC descriptor lower DWORD | 742 * +-----------+---------------------------+---------------------------------------------+ 743 * | 33 | Q0ContextIndex | Context ID for Queue 0 | 744 * +-----------+---------------------------+---------------------------------------------+ 745 * | 34 | Q1CD_DW0 | Queue 1 context LRC descriptor lower DWORD | 746 * +-----------+---------------------------+---------------------------------------------+ 747 * | 35 | Q1ContextIndex | Context ID for Queue 1 | 748 * +-----------+---------------------------+---------------------------------------------+ 749 * | ... |... | ... | 750 * +-----------+---------------------------+---------------------------------------------+ 751 * | 158 | Q63CD_DW0 | Queue 63 context LRC descriptor lower DWORD | 752 * +-----------+---------------------------+---------------------------------------------+ 753 * | 159 | Q63ContextIndex | Context ID for Queue 63 | 754 * +-----------+---------------------------+---------------------------------------------+ 755 * | 160..1024 | RESERVED | MBZ | 756 * +-----------+---------------------------+---------------------------------------------+ 757 * 758 * While registering Q0 with GuC, CGP is updated with Q0 entry and GuC is notified 759 * through XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE H2G message which specifies 760 * the CGP address. When the secondary queues are added to the group, the CGP is 761 * updated with entry for that queue and GuC is notified through the H2G interface 762 * XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC. GuC responds to these H2G messages 763 * with a XE_GUC_ACTION_NOTIFY_MULTIQ_CONTEXT_CGP_SYNC_DONE G2H message. GuC also 764 * sends a XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR notification for any 765 * error in the CGP. Only one of these CGP update messages can be outstanding 766 * (waiting for GuC response) at any time. The bits in KMD_QUEUE_UPDATE_MASK_DW* 767 * fields indicate which queue entry is being updated in the CGP. 768 * 769 * The primary queue (Q0) represents the multi queue group context in GuC and 770 * submission on any queue of the group must be through Q0 GuC interface only. 771 * 772 * As it is not required to register secondary queues with GuC, the secondary queue 773 * context ids in the CGP are populated with Q0 context id. 774 */ 775 776 #define CGP_VERSION_MAJOR_SHIFT 8 777 778 static void xe_guc_exec_queue_group_cgp_update(struct xe_device *xe, 779 struct xe_exec_queue *q) 780 { 781 struct xe_exec_queue_group *group = q->multi_queue.group; 782 u32 guc_id = group->primary->guc->id; 783 784 /* Currently implementing CGP version 1.0 */ 785 xe_map_wr(xe, &group->cgp_bo->vmap, 0, u32, 786 1 << CGP_VERSION_MAJOR_SHIFT); 787 788 xe_map_wr(xe, &group->cgp_bo->vmap, 789 (32 + q->multi_queue.pos * 2) * sizeof(u32), 790 u32, lower_32_bits(xe_lrc_descriptor(q->lrc[0]))); 791 792 xe_map_wr(xe, &group->cgp_bo->vmap, 793 (33 + q->multi_queue.pos * 2) * sizeof(u32), 794 u32, guc_id); 795 796 if (q->multi_queue.pos / 32) { 797 xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), 798 u32, BIT(q->multi_queue.pos % 32)); 799 xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), u32, 0); 800 } else { 801 xe_map_wr(xe, &group->cgp_bo->vmap, 16 * sizeof(u32), 802 u32, BIT(q->multi_queue.pos)); 803 xe_map_wr(xe, &group->cgp_bo->vmap, 17 * sizeof(u32), u32, 0); 804 } 805 } 806 807 #define CGP_SYNC_REGISTRATION BIT(0) 808 809 static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc, 810 struct xe_exec_queue *q, 811 const u32 *action, u32 len, 812 unsigned int flags) 813 { 814 struct xe_exec_queue_group *group = q->multi_queue.group; 815 struct xe_device *xe = guc_to_xe(guc); 816 enum xe_multi_queue_priority priority; 817 long ret; 818 819 /* 820 * As all queues of a multi queue group use single drm scheduler 821 * submit workqueue, CGP synchronization with GuC are serialized. 822 * Hence, no locking is required here. 823 * Wait for any pending CGP_SYNC_DONE response before updating the 824 * CGP page and sending CGP_SYNC message. 825 */ 826 ret = wait_event_timeout(guc->ct.wq, 827 !READ_ONCE(group->sync_pending) || 828 xe_guc_read_stopped(guc) || vf_recovery(guc), 829 HZ); 830 if ((!ret && !vf_recovery(guc)) || xe_guc_read_stopped(guc)) { 831 /* CGP_SYNC failed. Reset gt, cleanup the group */ 832 xe_gt_warn(guc_to_gt(guc), "Wait for CGP_SYNC_DONE response failed!\n"); 833 set_exec_queue_group_banned(q); 834 xe_gt_reset_async(q->gt); 835 xe_guc_exec_queue_group_trigger_cleanup(q); 836 return; 837 } 838 839 /* 840 * If woken by VF migration recovery, do not touch the CGP or send: the 841 * message would be lost and, for a registration, GuC must (re-)register 842 * the context before its CGP entry may be read. Flag the queue so revert 843 * replays it - a registration by re-registration, a dynamic update by a 844 * replayed CGP_SYNC - and bail. 845 */ 846 if (vf_recovery(guc)) { 847 if (flags & CGP_SYNC_REGISTRATION) 848 q->guc->multi_queue.re_register = true; 849 else 850 q->guc->multi_queue.re_update = true; 851 return; 852 } 853 854 scoped_guard(spinlock, &q->multi_queue.lock) 855 priority = q->multi_queue.priority; 856 857 xe_lrc_set_multi_queue_priority(q->lrc[0], priority); 858 xe_guc_exec_queue_group_cgp_update(xe, q); 859 860 /* 861 * Record the nature of this outstanding sync so revert can replay it if 862 * its CGP_SYNC_DONE is lost across a migration: a registration is 863 * recovered by re-registration, a dynamic update by a replayed CGP_SYNC. 864 */ 865 if (flags & CGP_SYNC_REGISTRATION) { 866 q->guc->multi_queue.registering_cgp = true; 867 q->guc->multi_queue.updating_cgp = false; 868 } else { 869 q->guc->multi_queue.updating_cgp = true; 870 q->guc->multi_queue.registering_cgp = false; 871 } 872 WRITE_ONCE(group->cgp_update_q, q); 873 WRITE_ONCE(group->sync_pending, true); 874 xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1); 875 } 876 877 static void guc_exec_queue_send_cgp_sync(struct xe_exec_queue *q, unsigned int flags) 878 { 879 #define MAX_MULTI_QUEUE_CGP_SYNC_SIZE (2) 880 struct xe_guc *guc = exec_queue_to_guc(q); 881 struct xe_exec_queue_group *group = q->multi_queue.group; 882 u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE]; 883 int len = 0; 884 885 action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC; 886 action[len++] = group->primary->guc->id; 887 888 xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE); 889 #undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE 890 891 xe_guc_exec_queue_group_cgp_sync(guc, q, action, len, flags); 892 } 893 894 static void __register_exec_queue_group(struct xe_exec_queue *q, 895 struct guc_ctxt_registration_info *info) 896 { 897 struct xe_guc *guc = exec_queue_to_guc(q); 898 #define MAX_MULTI_QUEUE_REG_SIZE (8) 899 u32 action[MAX_MULTI_QUEUE_REG_SIZE]; 900 int len = 0; 901 902 action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE; 903 action[len++] = info->flags; 904 action[len++] = info->context_idx; 905 action[len++] = info->engine_class; 906 action[len++] = info->engine_submit_mask; 907 action[len++] = 0; /* Reserved */ 908 action[len++] = info->cgp_lo; 909 action[len++] = info->cgp_hi; 910 911 xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_REG_SIZE); 912 #undef MAX_MULTI_QUEUE_REG_SIZE 913 914 /* 915 * The above XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_QUEUE do expect a 916 * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response 917 * from guc. 918 */ 919 xe_guc_exec_queue_group_cgp_sync(guc, q, action, len, 920 CGP_SYNC_REGISTRATION); 921 } 922 923 static void __register_mlrc_exec_queue(struct xe_guc *guc, 924 struct xe_exec_queue *q, 925 struct guc_ctxt_registration_info *info) 926 { 927 #define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2) 928 u32 action[MAX_MLRC_REG_SIZE]; 929 int len = 0; 930 int i; 931 932 xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_parallel(q)); 933 934 action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; 935 action[len++] = info->flags; 936 action[len++] = info->context_idx; 937 action[len++] = info->engine_class; 938 action[len++] = info->engine_submit_mask; 939 action[len++] = info->wq_desc_lo; 940 action[len++] = info->wq_desc_hi; 941 action[len++] = info->wq_base_lo; 942 action[len++] = info->wq_base_hi; 943 action[len++] = info->wq_size; 944 action[len++] = q->width; 945 action[len++] = info->hwlrca_lo; 946 action[len++] = info->hwlrca_hi; 947 948 for (i = 1; i < q->width; ++i) { 949 struct xe_lrc *lrc = q->lrc[i]; 950 951 action[len++] = lower_32_bits(xe_lrc_descriptor(lrc)); 952 action[len++] = upper_32_bits(xe_lrc_descriptor(lrc)); 953 } 954 955 /* explicitly checks some fields that we might fixup later */ 956 xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo == 957 action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_5_WQ_DESC_ADDR_LOWER]); 958 xe_gt_assert(guc_to_gt(guc), info->wq_base_lo == 959 action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_7_WQ_BUF_BASE_LOWER]); 960 xe_gt_assert(guc_to_gt(guc), q->width == 961 action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_10_NUM_CTXS]); 962 xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo == 963 action[XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_11_HW_LRC_ADDR]); 964 xe_gt_assert(guc_to_gt(guc), len <= MAX_MLRC_REG_SIZE); 965 #undef MAX_MLRC_REG_SIZE 966 967 xe_guc_ct_send(&guc->ct, action, len, 0, 0); 968 } 969 970 static void __register_exec_queue(struct xe_guc *guc, 971 struct guc_ctxt_registration_info *info) 972 { 973 u32 action[] = { 974 XE_GUC_ACTION_REGISTER_CONTEXT, 975 info->flags, 976 info->context_idx, 977 info->engine_class, 978 info->engine_submit_mask, 979 info->wq_desc_lo, 980 info->wq_desc_hi, 981 info->wq_base_lo, 982 info->wq_base_hi, 983 info->wq_size, 984 info->hwlrca_lo, 985 info->hwlrca_hi, 986 }; 987 988 /* explicitly checks some fields that we might fixup later */ 989 xe_gt_assert(guc_to_gt(guc), info->wq_desc_lo == 990 action[XE_GUC_REGISTER_CONTEXT_DATA_5_WQ_DESC_ADDR_LOWER]); 991 xe_gt_assert(guc_to_gt(guc), info->wq_base_lo == 992 action[XE_GUC_REGISTER_CONTEXT_DATA_7_WQ_BUF_BASE_LOWER]); 993 xe_gt_assert(guc_to_gt(guc), info->hwlrca_lo == 994 action[XE_GUC_REGISTER_CONTEXT_DATA_10_HW_LRC_ADDR]); 995 996 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); 997 } 998 999 static u32 xe_hwe_guc_logical_to_submit_mask(struct xe_hw_engine *hwe, u32 logical_mask) 1000 { 1001 struct xe_gt *gt = hwe->gt; 1002 1003 if (xe_gt_is_usm_hwe(gt, hwe)) { 1004 int shift = gt->usm.paging_hwe0->logical_instance; 1005 u32 paging_logical_mask = gt->usm.paging_logical_mask; 1006 1007 xe_gt_assert(gt, (logical_mask & paging_logical_mask) == logical_mask); 1008 1009 /* 1010 * Remap to GUC_PAGING_CLASS logical instance mask, if 1011 * applicable. 1012 */ 1013 if (xe_guc_has_paging_engine(&hwe->gt->uc.guc)) 1014 return logical_mask >> shift; 1015 } 1016 1017 return logical_mask; 1018 } 1019 1020 static void register_exec_queue(struct xe_exec_queue *q, int ctx_type) 1021 { 1022 struct xe_guc *guc = exec_queue_to_guc(q); 1023 struct xe_device *xe = guc_to_xe(guc); 1024 struct xe_lrc *lrc = q->lrc[0]; 1025 struct guc_ctxt_registration_info info; 1026 1027 xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q)); 1028 xe_gt_assert(guc_to_gt(guc), ctx_type < GUC_CONTEXT_COUNT); 1029 1030 memset(&info, 0, sizeof(info)); 1031 info.context_idx = q->guc->id; 1032 info.engine_class = xe_hwe_to_guc_class(q->hwe); 1033 info.engine_submit_mask = 1034 xe_hwe_guc_logical_to_submit_mask(q->hwe, q->logical_mask); 1035 info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc)); 1036 info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); 1037 info.flags = CONTEXT_REGISTRATION_FLAG_KMD | 1038 FIELD_PREP(CONTEXT_REGISTRATION_FLAG_TYPE, ctx_type); 1039 1040 if (xe_exec_queue_is_multi_queue(q)) { 1041 struct xe_exec_queue_group *group = q->multi_queue.group; 1042 1043 info.cgp_lo = xe_bo_ggtt_addr(group->cgp_bo); 1044 info.cgp_hi = 0; 1045 } 1046 1047 if (xe_exec_queue_is_parallel(q)) { 1048 u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); 1049 struct iosys_map map = xe_lrc_parallel_map(lrc); 1050 1051 info.wq_desc_lo = lower_32_bits(ggtt_addr + 1052 offsetof(struct guc_submit_parallel_scratch, wq_desc)); 1053 info.wq_desc_hi = upper_32_bits(ggtt_addr + 1054 offsetof(struct guc_submit_parallel_scratch, wq_desc)); 1055 info.wq_base_lo = lower_32_bits(ggtt_addr + 1056 offsetof(struct guc_submit_parallel_scratch, wq[0])); 1057 info.wq_base_hi = upper_32_bits(ggtt_addr + 1058 offsetof(struct guc_submit_parallel_scratch, wq[0])); 1059 info.wq_size = WQ_SIZE; 1060 1061 q->guc->wqi_head = 0; 1062 q->guc->wqi_tail = 0; 1063 xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE); 1064 parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE); 1065 } 1066 1067 set_exec_queue_registered(q); 1068 trace_xe_exec_queue_register(q); 1069 if (xe_exec_queue_is_multi_queue_primary(q)) 1070 __register_exec_queue_group(q, &info); 1071 else if (xe_exec_queue_is_parallel(q)) 1072 __register_mlrc_exec_queue(guc, q, &info); 1073 else if (!xe_exec_queue_is_multi_queue_secondary(q)) 1074 __register_exec_queue(guc, &info); 1075 1076 if (!xe_exec_queue_is_multi_queue_secondary(q)) 1077 init_policies(guc, q); 1078 1079 if (xe_exec_queue_is_multi_queue_secondary(q)) 1080 guc_exec_queue_send_cgp_sync(q, CGP_SYNC_REGISTRATION); 1081 } 1082 1083 static u32 wq_space_until_wrap(struct xe_exec_queue *q) 1084 { 1085 return (WQ_SIZE - q->guc->wqi_tail); 1086 } 1087 1088 static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) 1089 { 1090 struct xe_guc *guc = exec_queue_to_guc(q); 1091 struct xe_device *xe = guc_to_xe(guc); 1092 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 1093 unsigned int sleep_period_ms = 1, sleep_total_ms = 0; 1094 1095 #define AVAILABLE_SPACE \ 1096 CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE) 1097 if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) { 1098 try_again: 1099 q->guc->wqi_head = parallel_read(xe, map, wq_desc.head); 1100 if (wqi_size > AVAILABLE_SPACE && !vf_recovery(guc)) { 1101 if (sleep_total_ms > 2000) { 1102 xe_gt_reset_async(q->gt); 1103 return -ENODEV; 1104 } 1105 1106 sleep_total_ms += xe_sleep_exponential_ms(&sleep_period_ms, 64); 1107 goto try_again; 1108 } 1109 } 1110 #undef AVAILABLE_SPACE 1111 1112 return 0; 1113 } 1114 1115 static int wq_noop_append(struct xe_exec_queue *q) 1116 { 1117 struct xe_guc *guc = exec_queue_to_guc(q); 1118 struct xe_device *xe = guc_to_xe(guc); 1119 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 1120 u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1; 1121 1122 if (wq_wait_for_space(q, wq_space_until_wrap(q))) 1123 return -ENODEV; 1124 1125 xe_gt_assert(guc_to_gt(guc), FIELD_FIT(WQ_LEN_MASK, len_dw)); 1126 1127 parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)], 1128 FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | 1129 FIELD_PREP(WQ_LEN_MASK, len_dw)); 1130 q->guc->wqi_tail = 0; 1131 1132 return 0; 1133 } 1134 1135 static void wq_item_append(struct xe_exec_queue *q) 1136 { 1137 struct xe_guc *guc = exec_queue_to_guc(q); 1138 struct xe_device *xe = guc_to_xe(guc); 1139 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 1140 #define WQ_HEADER_SIZE 4 /* Includes 1 LRC address too */ 1141 u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)]; 1142 u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32); 1143 u32 len_dw = (wqi_size / sizeof(u32)) - 1; 1144 int i = 0, j; 1145 1146 if (wqi_size > wq_space_until_wrap(q)) { 1147 if (wq_noop_append(q)) 1148 return; 1149 } 1150 if (wq_wait_for_space(q, wqi_size)) 1151 return; 1152 1153 wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | 1154 FIELD_PREP(WQ_LEN_MASK, len_dw); 1155 wqi[i++] = xe_lrc_descriptor(q->lrc[0]); 1156 wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) | 1157 FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64)); 1158 wqi[i++] = 0; 1159 for (j = 1; j < q->width; ++j) { 1160 struct xe_lrc *lrc = q->lrc[j]; 1161 1162 wqi[i++] = lrc->ring.tail / sizeof(u64); 1163 } 1164 1165 xe_gt_assert(guc_to_gt(guc), i == wqi_size / sizeof(u32)); 1166 1167 iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch, 1168 wq[q->guc->wqi_tail / sizeof(u32)])); 1169 xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size); 1170 q->guc->wqi_tail += wqi_size; 1171 xe_gt_assert(guc_to_gt(guc), q->guc->wqi_tail <= WQ_SIZE); 1172 1173 xe_device_wmb(xe); 1174 1175 map = xe_lrc_parallel_map(q->lrc[0]); 1176 parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail); 1177 } 1178 1179 #define RESUME_PENDING ~0x0ull 1180 static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) 1181 { 1182 struct xe_guc *guc = exec_queue_to_guc(q); 1183 struct xe_lrc *lrc = q->lrc[0]; 1184 u32 action[3]; 1185 u32 g2h_len = 0; 1186 u32 num_g2h = 0; 1187 int len = 0; 1188 bool extra_submit = false; 1189 1190 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 1191 1192 if (!job->restore_replay || job->last_replay) { 1193 /* 1194 * A chained ULLS job relies on its predecessor's postamble to 1195 * publish its ring tail. After a reset the context comes back 1196 * disabled with that postamble gone, so publish it here. 1197 */ 1198 if (xe_exec_queue_is_parallel(q)) 1199 wq_item_append(q); 1200 else if (!xe_sched_job_ulls_is_chained(job) || 1201 !exec_queue_enabled(q)) 1202 xe_lrc_set_ring_tail(lrc, lrc->ring.tail); 1203 job->last_replay = false; 1204 } 1205 1206 if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q)) 1207 return; 1208 1209 /* 1210 * All queues in a multi-queue group will use the primary queue 1211 * of the group to interface with GuC. If primay is suspended, 1212 * just return. Jobs will get scheduled once primary is resumed. 1213 */ 1214 q = xe_exec_queue_multi_queue_primary(q); 1215 if (exec_queue_suspended(q)) 1216 return; 1217 1218 if (xe_sched_job_ulls_is_chained(job)) 1219 xe_lrc_set_ulls_semaphore(lrc, xe_sched_job_lrc_seqno(job)); 1220 1221 if (!exec_queue_enabled(q)) { 1222 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; 1223 action[len++] = q->guc->id; 1224 action[len++] = GUC_CONTEXT_ENABLE; 1225 g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; 1226 num_g2h = 1; 1227 if (xe_exec_queue_is_parallel(q)) 1228 extra_submit = true; 1229 1230 q->guc->resume_time = RESUME_PENDING; 1231 set_exec_queue_pending_enable(q); 1232 set_exec_queue_enabled(q); 1233 trace_xe_exec_queue_scheduling_enable(q); 1234 } else if (!xe_sched_job_ulls_is_chained(job)) { 1235 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; 1236 action[len++] = q->guc->id; 1237 trace_xe_exec_queue_submit(q); 1238 } 1239 1240 if (!xe_sched_job_ulls_is_chained(job) || num_g2h) 1241 xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); 1242 1243 if (extra_submit) { 1244 len = 0; 1245 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; 1246 action[len++] = q->guc->id; 1247 trace_xe_exec_queue_submit(q); 1248 1249 xe_guc_ct_send(&guc->ct, action, len, 0, 0); 1250 } 1251 } 1252 1253 static bool is_pt_job(struct xe_sched_job *job) 1254 { 1255 return job->is_pt_job; 1256 } 1257 1258 static void run_pt_job(struct xe_device *xe, struct xe_sched_job *job, 1259 bool force_clear) 1260 { 1261 struct xe_tile *tile; 1262 int id; 1263 1264 for_each_tile(tile, xe, id) { 1265 struct xe_pt_job_ops *pt_job_ops = 1266 job->pt_update[0].pt_job_ops[id]; 1267 1268 if (!pt_job_ops || !pt_job_ops->current_op) 1269 continue; 1270 1271 xe_cpu_bind_update_pgtables_execute(job->pt_update[0].vm, tile, 1272 job->pt_update[0].ops, 1273 pt_job_ops->ops, 1274 pt_job_ops->current_op, 1275 force_clear); 1276 } 1277 } 1278 1279 static void put_pt_job(struct xe_device *xe, struct xe_sched_job *job) 1280 { 1281 struct xe_tile *tile; 1282 int id; 1283 1284 for_each_tile(tile, xe, id) { 1285 struct xe_pt_job_ops *pt_job_ops = 1286 job->pt_update[0].pt_job_ops[id]; 1287 1288 xe_pt_job_ops_put(pt_job_ops); 1289 } 1290 } 1291 1292 static struct dma_fence * 1293 guc_exec_queue_run_job(struct drm_sched_job *drm_job) 1294 { 1295 struct xe_sched_job *job = to_xe_sched_job(drm_job); 1296 struct xe_exec_queue *q = job->q; 1297 struct xe_guc *guc = exec_queue_to_guc(q); 1298 bool killed_or_banned_or_wedged_or_error = 1299 exec_queue_killed_or_banned_or_wedged(q) || 1300 xe_sched_job_is_error(job); 1301 1302 xe_gt_assert(guc_to_gt(guc), !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) || 1303 exec_queue_banned(q) || exec_queue_suspended(q)); 1304 1305 trace_xe_sched_job_run(job); 1306 1307 if (is_pt_job(job)) { 1308 xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q)); 1309 1310 run_pt_job(guc_to_xe(guc), job, 1311 killed_or_banned_or_wedged_or_error); 1312 put_pt_job(guc_to_xe(guc), job); 1313 dma_fence_put(job->fence); /* Drop ref from xe_sched_job_arm */ 1314 1315 return NULL; 1316 } else if (!killed_or_banned_or_wedged_or_error) { 1317 if (xe_exec_queue_is_multi_queue_secondary(q)) { 1318 struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); 1319 1320 if (exec_queue_killed_or_banned_or_wedged(primary)) 1321 goto run_job_out; 1322 1323 if (!exec_queue_registered(primary)) 1324 register_exec_queue(primary, GUC_CONTEXT_NORMAL); 1325 } 1326 1327 if (!exec_queue_registered(q)) 1328 register_exec_queue(q, GUC_CONTEXT_NORMAL); 1329 if (!job->restore_replay) 1330 q->ring_ops->emit_job(job); 1331 submit_exec_queue(q, job); 1332 job->restore_replay = false; 1333 } 1334 1335 run_job_out: 1336 1337 return job->fence; 1338 } 1339 1340 static void guc_exec_queue_free_job(struct drm_sched_job *drm_job) 1341 { 1342 struct xe_sched_job *job = to_xe_sched_job(drm_job); 1343 1344 trace_xe_sched_job_free(job); 1345 xe_sched_job_put(job); 1346 } 1347 1348 int xe_guc_read_stopped(struct xe_guc *guc) 1349 { 1350 return atomic_read(&guc->submission_state.stopped); 1351 } 1352 1353 static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc, 1354 struct xe_exec_queue *q, 1355 u32 runnable_state); 1356 static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q); 1357 1358 #define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \ 1359 u32 action[] = { \ 1360 XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ 1361 q->guc->id, \ 1362 GUC_CONTEXT_##enable_disable, \ 1363 } 1364 1365 static void disable_scheduling_deregister(struct xe_guc *guc, 1366 struct xe_exec_queue *q) 1367 { 1368 MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); 1369 int ret; 1370 1371 if (!xe_exec_queue_is_multi_queue_secondary(q)) 1372 set_min_preemption_timeout(guc, q); 1373 1374 smp_rmb(); 1375 ret = wait_event_timeout(guc->ct.wq, 1376 (!exec_queue_pending_enable(q) && 1377 !exec_queue_pending_disable(q)) || 1378 xe_guc_read_stopped(guc) || 1379 vf_recovery(guc), 1380 HZ * 5); 1381 if (!ret && !vf_recovery(guc)) { 1382 struct xe_gpu_scheduler *sched = &q->guc->sched; 1383 1384 xe_gt_warn(q->gt, "Pending enable/disable failed to respond\n"); 1385 xe_sched_submission_start(sched); 1386 xe_gt_reset_async(q->gt); 1387 xe_sched_tdr_queue_imm(sched); 1388 return; 1389 } 1390 1391 clear_exec_queue_enabled(q); 1392 set_exec_queue_pending_disable(q); 1393 set_exec_queue_destroyed(q); 1394 trace_xe_exec_queue_scheduling_disable(q); 1395 1396 /* 1397 * Reserve space for both G2H here as the 2nd G2H is sent from a G2H 1398 * handler and we are not allowed to reserved G2H space in handlers. 1399 */ 1400 if (xe_exec_queue_is_multi_queue_secondary(q)) 1401 handle_multi_queue_secondary_sched_done(guc, q, 0); 1402 else 1403 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 1404 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + 1405 G2H_LEN_DW_DEREGISTER_CONTEXT, 2); 1406 } 1407 1408 /** 1409 * xe_guc_submit_wedge() - Wedge GuC submission 1410 * @guc: the GuC object 1411 * 1412 * Save exec queue's registered with GuC state by taking a ref to each queue. 1413 * Register a DRMM handler to drop refs upon driver unload. 1414 */ 1415 void xe_guc_submit_wedge(struct xe_guc *guc) 1416 { 1417 struct xe_device *xe = guc_to_xe(guc); 1418 struct xe_exec_queue *q; 1419 unsigned long index; 1420 1421 xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); 1422 1423 /* 1424 * If device is being wedged even before submission_state is 1425 * initialized, there's nothing to do here. 1426 */ 1427 if (!guc->submission_state.initialized) 1428 return; 1429 1430 if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { 1431 mutex_lock(&guc->submission_state.lock); 1432 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 1433 if (xe_exec_queue_get_unless_zero(q)) 1434 set_exec_queue_wedged(q); 1435 mutex_unlock(&guc->submission_state.lock); 1436 } else { 1437 /* Forcefully kill any remaining exec queues, signal fences */ 1438 guc_submit_reset_prepare(guc); 1439 xe_guc_submit_stop(guc); 1440 xe_guc_softreset(guc); 1441 xe_uc_fw_sanitize(&guc->fw); 1442 xe_guc_submit_pause_abort(guc); 1443 } 1444 } 1445 1446 static bool guc_submit_hint_wedged(struct xe_guc *guc) 1447 { 1448 struct xe_device *xe = guc_to_xe(guc); 1449 1450 if (xe->wedged.mode != XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) 1451 return false; 1452 1453 if (xe_device_wedged(xe)) 1454 return true; 1455 1456 xe_device_declare_wedged(xe, DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); 1457 1458 return true; 1459 } 1460 1461 #define ADJUST_FIVE_PERCENT(__t) mul_u64_u32_div(__t, 105, 100) 1462 1463 static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) 1464 { 1465 struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q)); 1466 u32 ctx_timestamp, ctx_job_timestamp; 1467 u32 timeout_ms = q->sched_props.job_timeout_ms; 1468 u32 diff; 1469 u64 running_time_ms; 1470 1471 if (!xe_sched_job_started(job)) { 1472 xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, not started", 1473 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 1474 q->guc->id); 1475 1476 /* GuC never scheduled this job - let the caller trigger a GT reset. */ 1477 return true; 1478 } 1479 1480 ctx_timestamp = lower_32_bits(xe_lrc_timestamp(q->lrc[0])); 1481 if (ctx_timestamp == job->sample_timestamp) { 1482 if (IS_SRIOV_VF(gt_to_xe(gt))) 1483 xe_gt_notice(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck", 1484 xe_sched_job_seqno(job), 1485 xe_sched_job_lrc_seqno(job), q->guc->id); 1486 else 1487 xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, timestamp stuck", 1488 xe_sched_job_seqno(job), 1489 xe_sched_job_lrc_seqno(job), q->guc->id); 1490 1491 return xe_sched_invalidate_job(job, 0); 1492 } 1493 1494 job->sample_timestamp = ctx_timestamp; 1495 ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]); 1496 1497 /* 1498 * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch 1499 * possible overflows with a high timeout. 1500 */ 1501 xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC); 1502 1503 diff = ctx_timestamp - ctx_job_timestamp; 1504 1505 /* 1506 * Ensure timeout is within 5% to account for an GuC scheduling latency 1507 */ 1508 running_time_ms = 1509 ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff)); 1510 1511 xe_gt_dbg(gt, 1512 "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x", 1513 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 1514 q->guc->id, running_time_ms, timeout_ms, diff); 1515 1516 return running_time_ms >= timeout_ms; 1517 } 1518 1519 static void enable_scheduling(struct xe_exec_queue *q) 1520 { 1521 MAKE_SCHED_CONTEXT_ACTION(q, ENABLE); 1522 struct xe_guc *guc = exec_queue_to_guc(q); 1523 int ret; 1524 1525 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); 1526 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 1527 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 1528 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); 1529 1530 set_exec_queue_pending_enable(q); 1531 set_exec_queue_enabled(q); 1532 trace_xe_exec_queue_scheduling_enable(q); 1533 1534 if (xe_exec_queue_is_multi_queue_secondary(q)) 1535 handle_multi_queue_secondary_sched_done(guc, q, 1); 1536 else 1537 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 1538 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); 1539 1540 ret = wait_event_timeout(guc->ct.wq, 1541 !exec_queue_pending_enable(q) || 1542 xe_guc_read_stopped(guc) || 1543 vf_recovery(guc), HZ * 5); 1544 if ((!ret && !vf_recovery(guc)) || xe_guc_read_stopped(guc)) { 1545 xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond"); 1546 set_exec_queue_banned(q); 1547 xe_gt_reset_async(q->gt); 1548 xe_sched_tdr_queue_imm(&q->guc->sched); 1549 } 1550 } 1551 1552 static void disable_scheduling(struct xe_exec_queue *q, bool immediate) 1553 { 1554 MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); 1555 struct xe_guc *guc = exec_queue_to_guc(q); 1556 1557 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); 1558 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 1559 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 1560 1561 if (immediate && !xe_exec_queue_is_multi_queue_secondary(q)) 1562 set_min_preemption_timeout(guc, q); 1563 clear_exec_queue_enabled(q); 1564 set_exec_queue_pending_disable(q); 1565 trace_xe_exec_queue_scheduling_disable(q); 1566 1567 if (xe_exec_queue_is_multi_queue_secondary(q)) 1568 handle_multi_queue_secondary_sched_done(guc, q, 0); 1569 else 1570 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 1571 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); 1572 } 1573 1574 /* 1575 * Recover via GT reset for a kernel queue, or for a GuC scheduling failure (job 1576 * never started) on a queue that was not already killed or banned. An already 1577 * banned queue must stay banned, so its unstarted jobs do not clear the ban or 1578 * trigger a reset. 1579 */ 1580 static bool timeout_needs_gt_reset(struct xe_exec_queue *q, struct xe_sched_job *job, 1581 bool skip_timeout_check) 1582 { 1583 if (q->flags & EXEC_QUEUE_FLAG_KERNEL) 1584 return true; 1585 1586 return !skip_timeout_check && !xe_sched_job_started(job); 1587 } 1588 1589 static enum drm_gpu_sched_stat 1590 guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) 1591 { 1592 struct xe_sched_job *job = to_xe_sched_job(drm_job); 1593 struct drm_sched_job *tmp_job; 1594 struct xe_exec_queue *q = job->q, *primary; 1595 struct xe_gpu_scheduler *sched = &q->guc->sched; 1596 struct xe_guc *guc = exec_queue_to_guc(q); 1597 const char *process_name = "no process"; 1598 struct xe_device *xe = guc_to_xe(guc); 1599 int err = -ETIME; 1600 pid_t pid = -1; 1601 bool wedged = false, wedge_device = false, skip_timeout_check; 1602 int idx; 1603 1604 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); 1605 1606 primary = xe_exec_queue_multi_queue_primary(q); 1607 1608 /* 1609 * TDR has fired before free job worker. Common if exec queue 1610 * immediately closed after last fence signaled. Add back to pending 1611 * list so job can be freed and kick scheduler ensuring free job is not 1612 * lost. 1613 */ 1614 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags) || 1615 vf_recovery(guc)) 1616 return DRM_GPU_SCHED_STAT_NO_HANG; 1617 1618 if (!drm_dev_enter(&xe->drm, &idx)) { 1619 xe_sched_job_set_error(job, -ENODEV); 1620 return DRM_GPU_SCHED_STAT_NO_HANG; 1621 } 1622 1623 /* Kill the run_job entry point */ 1624 if (xe_exec_queue_is_multi_queue(q)) 1625 xe_guc_exec_queue_group_stop(q); 1626 else 1627 xe_sched_submission_stop(sched); 1628 1629 /* Must check all state after stopping scheduler */ 1630 skip_timeout_check = exec_queue_reset(q) || 1631 exec_queue_killed_or_banned_or_wedged(q); 1632 1633 /* Skip timeout check if multi-queue group is banned */ 1634 if (xe_exec_queue_is_multi_queue(q) && 1635 READ_ONCE(q->multi_queue.group->banned)) 1636 skip_timeout_check = true; 1637 1638 /* LR jobs can only get here if queue has been killed or hit an error */ 1639 if (xe_exec_queue_is_lr(q)) 1640 xe_gt_assert(guc_to_gt(guc), skip_timeout_check); 1641 1642 /* 1643 * If devcoredump not captured and GuC capture for the job is not ready 1644 * do manual capture first and decide later if we need to use it 1645 */ 1646 if (!xe_device_is_in_reset(xe) && !exec_queue_killed(q) && !xe->devcoredump.captured && 1647 !xe_guc_capture_get_matching_and_lock(q)) { 1648 /* take force wake before engine register manual capture */ 1649 CLASS(xe_force_wake, fw_ref)(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); 1650 if (!xe_force_wake_ref_has_domain(fw_ref.domains, XE_FORCEWAKE_ALL)) 1651 xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n"); 1652 1653 xe_engine_snapshot_capture_for_queue(q); 1654 } 1655 1656 /* 1657 * Check if job is actually timed out, if so restart job execution and TDR 1658 */ 1659 if (!skip_timeout_check && !check_timeout(q, job)) 1660 goto rearm; 1661 1662 /* 1663 * Killed queues must not newly wedge the device, but preserve an 1664 * already-wedged state to avoid warning on teardown timeouts. 1665 */ 1666 if (!exec_queue_killed(q)) 1667 wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); 1668 else 1669 wedged = xe_device_wedged(xe); 1670 1671 /* 1672 * Only tag as GPU hang if this is the original timeout, not a 1673 * consequence of a prior kill (e.g., page-offline). 1674 */ 1675 if (!exec_queue_killed(q)) 1676 atomic_or(DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG, &q->ban_reason); 1677 set_exec_queue_banned(q); 1678 1679 /* Kick job / queue off hardware */ 1680 if (!xe_device_is_in_reset(xe) && !wedged && 1681 (exec_queue_enabled(primary) || exec_queue_pending_disable(primary))) { 1682 int ret; 1683 1684 if (exec_queue_reset(primary)) 1685 err = -EIO; 1686 1687 if (xe_uc_fw_is_running(&guc->fw)) { 1688 /* 1689 * Wait for any pending G2H to flush out before 1690 * modifying state 1691 */ 1692 ret = wait_event_timeout(guc->ct.wq, 1693 (!exec_queue_pending_enable(primary) && 1694 !exec_queue_pending_disable(primary)) || 1695 xe_guc_read_stopped(guc) || 1696 vf_recovery(guc), HZ * 5); 1697 if (vf_recovery(guc)) 1698 goto handle_vf_resume; 1699 if (!ret || xe_guc_read_stopped(guc)) 1700 goto trigger_reset; 1701 1702 disable_scheduling(primary, skip_timeout_check); 1703 } 1704 1705 /* 1706 * Must wait for scheduling to be disabled before signalling 1707 * any fences, if GT broken the GT reset code should signal us. 1708 * 1709 * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault 1710 * error) messages which can cause the schedule disable to get 1711 * lost. If this occurs, trigger a GT reset to recover. 1712 */ 1713 smp_rmb(); 1714 ret = wait_event_timeout(guc->ct.wq, 1715 !xe_uc_fw_is_running(&guc->fw) || 1716 !exec_queue_pending_disable(primary) || 1717 xe_guc_read_stopped(guc) || 1718 vf_recovery(guc), HZ * 5); 1719 if (vf_recovery(guc)) 1720 goto handle_vf_resume; 1721 if (!ret || xe_guc_read_stopped(guc)) { 1722 trigger_reset: 1723 if (!ret) 1724 xe_gt_warn(guc_to_gt(guc), 1725 "Schedule disable failed to respond, guc_id=%d", 1726 primary->guc->id); 1727 xe_devcoredump(primary, job, 1728 "Schedule disable failed to respond, guc_id=%d, ret=%d, guc_read=%d", 1729 primary->guc->id, ret, xe_guc_read_stopped(guc)); 1730 xe_gt_reset_async(primary->gt); 1731 xe_sched_tdr_queue_imm(sched); 1732 goto rearm; 1733 } 1734 } 1735 1736 if (q->vm && q->vm->xef) { 1737 process_name = q->vm->xef->process_name; 1738 pid = q->vm->xef->pid; 1739 } 1740 1741 if (!exec_queue_killed(q)) 1742 xe_gt_notice(guc_to_gt(guc), 1743 "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]", 1744 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 1745 q->guc->id, q->flags, process_name, pid); 1746 1747 trace_xe_sched_job_timedout(job); 1748 1749 /* Do not access device if in reset */ 1750 if (!xe_device_is_in_reset(xe) && !exec_queue_killed(q)) 1751 xe_devcoredump(q, job, 1752 "Timedout job - seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx", 1753 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 1754 q->guc->id, q->flags); 1755 1756 if (!wedged) { 1757 if (timeout_needs_gt_reset(q, job, skip_timeout_check)) { 1758 if (!xe_sched_invalidate_job(job, 2)) { 1759 clear_exec_queue_banned(q); 1760 /* protect concurrent page offline reasons */ 1761 atomic_andnot(DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG, 1762 &q->ban_reason); 1763 xe_gt_reset_async(q->gt); 1764 goto rearm; 1765 } 1766 if (q->flags & EXEC_QUEUE_FLAG_KERNEL) { 1767 xe_gt_WARN(q->gt, true, "Kernel-submitted job timed out\n"); 1768 wedge_device = true; 1769 } 1770 } else if (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)) { 1771 xe_gt_WARN(q->gt, true, "VM job timed out on non-killed execqueue\n"); 1772 } 1773 } 1774 1775 /* Mark all outstanding jobs as bad, thus completing them */ 1776 xe_sched_job_set_error(job, err); 1777 drm_sched_for_each_pending_job(tmp_job, &sched->base, NULL) 1778 xe_sched_job_set_error(to_xe_sched_job(tmp_job), -ECANCELED); 1779 1780 if (xe_exec_queue_is_multi_queue(q)) { 1781 xe_guc_exec_queue_group_start(q); 1782 xe_guc_exec_queue_group_trigger_cleanup(q); 1783 } else { 1784 xe_sched_submission_start(sched); 1785 xe_guc_exec_queue_trigger_cleanup(q); 1786 } 1787 1788 if (wedge_device) 1789 xe_device_declare_wedged(gt_to_xe(q->gt), 1790 DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); 1791 1792 /* 1793 * We want the job added back to the pending list so it gets freed; this 1794 * is what DRM_GPU_SCHED_STAT_NO_HANG does. 1795 */ 1796 drm_dev_exit(idx); 1797 return DRM_GPU_SCHED_STAT_NO_HANG; 1798 1799 rearm: 1800 /* 1801 * XXX: Ideally want to adjust timeout based on current execution time 1802 * but there is not currently an easy way to do in DRM scheduler. With 1803 * some thought, do this in a follow up. 1804 */ 1805 if (xe_exec_queue_is_multi_queue(q)) 1806 xe_guc_exec_queue_group_start(q); 1807 else 1808 xe_sched_submission_start(sched); 1809 handle_vf_resume: 1810 drm_dev_exit(idx); 1811 return DRM_GPU_SCHED_STAT_NO_HANG; 1812 } 1813 1814 static void guc_exec_queue_multi_queue_drop_suspend(struct xe_exec_queue *q); 1815 static int guc_exec_queue_suspend_wait_blocking(struct xe_exec_queue *q); 1816 1817 static void guc_exec_queue_fini(struct xe_exec_queue *q) 1818 { 1819 struct xe_guc_exec_queue *ge = q->guc; 1820 struct xe_guc *guc = exec_queue_to_guc(q); 1821 struct drm_device *drm = &guc_to_xe(guc)->drm; 1822 1823 /* 1824 * A secondary can leave the group while still preempt suspended (e.g. 1825 * xe_vm_remove_compute_exec_queue() forces its preempt fence to signal, 1826 * which suspends it). It holds one forwarded suspend reference on the 1827 * primary, so drop it and resume the primary if it was the last member 1828 * that had it suspended. Primaries forward to nobody, so they don't need 1829 * this. 1830 * 1831 * First make sure the primary's forwarded suspend has completed. If the 1832 * secondary was killed/reset before its preempt fence worker ran, that 1833 * worker skips suspend_wait() (see preempt_fence_work_func()), leaving 1834 * the primary's suspend possibly in flight. drop_suspend() runs under a 1835 * spinlock and cannot wait, so drain it here with the uninterruptible 1836 * blocking wait; otherwise resuming the primary in drop_suspend() could 1837 * trip the !suspend_pending assert. 1838 */ 1839 if (xe_exec_queue_is_multi_queue_secondary(q)) { 1840 if (READ_ONCE(q->guc->suspend_count)) 1841 guc_exec_queue_suspend_wait_blocking(q); 1842 guc_exec_queue_multi_queue_drop_suspend(q); 1843 } 1844 1845 if (xe_exec_queue_is_multi_queue_secondary(q)) { 1846 struct xe_exec_queue_group *group = q->multi_queue.group; 1847 1848 mutex_lock(&group->list_lock); 1849 list_del(&q->multi_queue.link); 1850 mutex_unlock(&group->list_lock); 1851 } 1852 1853 release_guc_id(guc, q); 1854 xe_sched_entity_fini(&ge->entity); 1855 xe_sched_fini(&ge->sched); 1856 1857 /* 1858 * RCU free due sched being exported via DRM scheduler fences 1859 * (timeline name). 1860 */ 1861 kfree_rcu(ge, rcu); 1862 1863 drm_dev_put(drm); 1864 } 1865 1866 static void guc_exec_queue_do_destroy(struct xe_exec_queue *q) 1867 { 1868 struct xe_guc_exec_queue *ge = q->guc; 1869 struct xe_guc *guc = exec_queue_to_guc(q); 1870 struct xe_device *xe = guc_to_xe(guc); 1871 struct drm_device *drm = &xe->drm; 1872 1873 /* 1874 * guc_exec_queue_fini() drops the queue's drm_device ref. 1875 * Keep the device alive until the PM-runtime guard unwinds. 1876 */ 1877 drm_dev_get(drm); 1878 1879 scoped_guard(xe_pm_runtime, xe) { 1880 trace_xe_exec_queue_destroy(q); 1881 1882 /* Confirm no work left behind accessing device structures */ 1883 cancel_delayed_work_sync(&ge->sched.base.work_tdr); 1884 1885 xe_exec_queue_fini(q); 1886 } 1887 1888 drm_dev_put(drm); 1889 } 1890 1891 static void __guc_exec_queue_destroy_async(struct work_struct *w) 1892 { 1893 struct xe_guc_exec_queue *ge = 1894 container_of(w, struct xe_guc_exec_queue, destroy_async); 1895 1896 guc_exec_queue_do_destroy(ge->q); 1897 } 1898 1899 static void guc_exec_queue_destroy_async(struct xe_exec_queue *q) 1900 { 1901 INIT_WORK(&q->guc->destroy_async, __guc_exec_queue_destroy_async); 1902 xe_destroy_wq_queue(&q->guc->destroy_async); 1903 } 1904 1905 static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg, 1906 bool bound) 1907 { 1908 struct xe_exec_queue *q = msg->private_data; 1909 struct xe_guc *guc = exec_queue_to_guc(q); 1910 1911 trace_xe_exec_queue_cleanup_entity(q); 1912 1913 /* 1914 * Expected state transitions for cleanup: 1915 * - If the exec queue is registered and GuC firmware is running, we must first 1916 * disable scheduling and deregister the queue to ensure proper teardown and 1917 * resource release in the GuC, then destroy the exec queue on driver side. 1918 * - If the GuC is already stopped (e.g., during driver unload or GPU reset), 1919 * we cannot expect a response for the deregister request. In this case, 1920 * it is safe to directly destroy the exec queue on driver side, as the GuC 1921 * will not process further requests and all resources must be cleaned up locally. 1922 */ 1923 /* A wedged GuC won't answer the H2G, so tear down on the driver side. */ 1924 if (bound && !exec_queue_wedged(q) && exec_queue_registered(q) && 1925 xe_uc_fw_is_running(&guc->fw)) 1926 disable_scheduling_deregister(guc, q); 1927 else 1928 guc_exec_queue_destroy_async(q); 1929 } 1930 1931 static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q) 1932 { 1933 return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q); 1934 } 1935 1936 static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg, 1937 bool bound) 1938 { 1939 struct xe_exec_queue *q = msg->private_data; 1940 struct xe_guc *guc = exec_queue_to_guc(q); 1941 1942 if (guc_exec_queue_allowed_to_change_state(q) && bound) 1943 init_policies(guc, q); 1944 kfree(msg); 1945 } 1946 1947 static void suspend_wait_wake(struct xe_exec_queue *q) 1948 { 1949 struct xe_guc *guc = exec_queue_to_guc(q); 1950 struct xe_device *xe = guc_to_xe(guc); 1951 1952 /* 1953 * We use a GuC shared wait queue for VFs because the VF resfix start 1954 * interrupt must be able to wake all instances of suspend_wait. This 1955 * prevents the VF migration worker from being starved during 1956 * scheduling. 1957 */ 1958 if (IS_SRIOV_VF(xe)) 1959 wake_up_all(&guc->ct.wq); 1960 else 1961 wake_up(&q->guc->suspend_wait); 1962 } 1963 1964 static void __suspend_fence_signal(struct xe_exec_queue *q) 1965 { 1966 if (!q->guc->suspend_pending) 1967 return; 1968 1969 WRITE_ONCE(q->guc->suspend_pending, false); 1970 suspend_wait_wake(q); 1971 } 1972 1973 static void suspend_fence_signal(struct xe_exec_queue *q) 1974 { 1975 struct xe_guc *guc = exec_queue_to_guc(q); 1976 1977 xe_gt_assert(guc_to_gt(guc), exec_queue_suspended(q) || exec_queue_killed(q) || 1978 xe_guc_read_stopped(guc)); 1979 xe_gt_assert(guc_to_gt(guc), q->guc->suspend_pending); 1980 1981 __suspend_fence_signal(q); 1982 } 1983 1984 static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg, 1985 bool bound) 1986 { 1987 struct xe_exec_queue *q = msg->private_data; 1988 struct xe_guc *guc = exec_queue_to_guc(q); 1989 1990 if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && 1991 exec_queue_enabled(q) && bound) { 1992 wait_event(guc->ct.wq, vf_recovery(guc) || 1993 ((q->guc->resume_time != RESUME_PENDING || 1994 xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q))); 1995 1996 if (!xe_guc_read_stopped(guc)) { 1997 s64 since_resume_ms = 1998 ktime_ms_delta(ktime_get(), 1999 q->guc->resume_time); 2000 s64 wait_ms = q->vm->preempt.min_run_period_ms - 2001 since_resume_ms; 2002 2003 if (wait_ms > 0 && q->guc->resume_time) 2004 xe_sleep_relaxed_ms(wait_ms); 2005 2006 set_exec_queue_suspended(q); 2007 disable_scheduling(q, false); 2008 } 2009 } else if (q->guc->suspend_pending) { 2010 set_exec_queue_suspended(q); 2011 suspend_fence_signal(q); 2012 } 2013 } 2014 2015 static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg, 2016 bool bound) 2017 { 2018 struct xe_exec_queue *q = msg->private_data; 2019 2020 if (guc_exec_queue_allowed_to_change_state(q) && bound) { 2021 clear_exec_queue_suspended(q); 2022 if (!exec_queue_enabled(q)) { 2023 q->guc->resume_time = RESUME_PENDING; 2024 set_exec_queue_pending_resume(q); 2025 enable_scheduling(q); 2026 } 2027 } else { 2028 clear_exec_queue_suspended(q); 2029 } 2030 } 2031 2032 static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sched_msg *msg, 2033 bool bound) 2034 { 2035 struct xe_exec_queue *q = msg->private_data; 2036 2037 if (guc_exec_queue_allowed_to_change_state(q) && bound) 2038 guc_exec_queue_send_cgp_sync(q, 0); 2039 2040 kfree(msg); 2041 } 2042 2043 static void __guc_exec_queue_process_msg_cgp_sync(struct xe_sched_msg *msg, 2044 bool bound) 2045 { 2046 struct xe_exec_queue *q = msg->private_data; 2047 2048 /* 2049 * Replay a dynamic CGP update lost across VF migration by re-issuing the 2050 * CGP update + CGP_SYNC (re-applies the current priority from 2051 * q->multi_queue.priority). 2052 */ 2053 if (guc_exec_queue_allowed_to_change_state(q) && bound) 2054 guc_exec_queue_send_cgp_sync(q, 0); 2055 } 2056 2057 #define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ 2058 #define SET_SCHED_PROPS 2 2059 #define SUSPEND 3 2060 #define RESUME 4 2061 #define SET_MULTI_QUEUE_PRIORITY 5 2062 #define CGP_SYNC_MSG 6 2063 #define OPCODE_MASK 0xf 2064 #define MSG_LOCKED BIT(8) 2065 #define MSG_HEAD BIT(9) 2066 #define MSG_PM_REF BIT(10) 2067 2068 static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) 2069 { 2070 struct xe_device *xe = guc_to_xe(exec_queue_to_guc(msg->private_data)); 2071 int idx; 2072 bool pm_ref = !!(msg->opcode & MSG_PM_REF); 2073 bool bound = drm_dev_enter(&xe->drm, &idx); 2074 2075 trace_xe_sched_msg_recv(msg); 2076 2077 switch (msg->opcode & OPCODE_MASK) { 2078 case CLEANUP: 2079 __guc_exec_queue_process_msg_cleanup(msg, bound); 2080 break; 2081 case SET_SCHED_PROPS: 2082 __guc_exec_queue_process_msg_set_sched_props(msg, bound); 2083 break; 2084 case SUSPEND: 2085 __guc_exec_queue_process_msg_suspend(msg, bound); 2086 break; 2087 case RESUME: 2088 __guc_exec_queue_process_msg_resume(msg, bound); 2089 break; 2090 case SET_MULTI_QUEUE_PRIORITY: 2091 __guc_exec_queue_process_msg_set_multi_queue_priority(msg, bound); 2092 break; 2093 case CGP_SYNC_MSG: 2094 __guc_exec_queue_process_msg_cgp_sync(msg, bound); 2095 break; 2096 default: 2097 XE_WARN_ON("Unknown message type"); 2098 } 2099 2100 if (pm_ref) 2101 xe_pm_runtime_put(xe); 2102 2103 if (bound) 2104 drm_dev_exit(idx); 2105 } 2106 2107 static const struct drm_sched_backend_ops drm_sched_ops = { 2108 .run_job = guc_exec_queue_run_job, 2109 .free_job = guc_exec_queue_free_job, 2110 .timedout_job = guc_exec_queue_timedout_job, 2111 }; 2112 2113 static const struct xe_sched_backend_ops xe_sched_ops = { 2114 .process_msg = guc_exec_queue_process_msg, 2115 }; 2116 2117 static int guc_exec_queue_init(struct xe_exec_queue *q) 2118 { 2119 struct xe_gpu_scheduler *sched; 2120 struct xe_guc *guc = exec_queue_to_guc(q); 2121 struct drm_device *drm = &guc_to_xe(guc)->drm; 2122 struct workqueue_struct *submit_wq = NULL; 2123 struct xe_guc_exec_queue *ge; 2124 long timeout; 2125 int err, i; 2126 int max_jobs = (xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES); 2127 2128 xe_gt_assert(guc_to_gt(guc), xe_device_uc_enabled(guc_to_xe(guc))); 2129 2130 ge = kzalloc_obj(*ge); 2131 if (!ge) 2132 return -ENOMEM; 2133 2134 drm_dev_get(drm); 2135 2136 q->guc = ge; 2137 ge->q = q; 2138 init_rcu_head(&ge->rcu); 2139 init_waitqueue_head(&ge->suspend_wait); 2140 2141 for (i = 0; i < MAX_STATIC_MSG_TYPE; ++i) 2142 INIT_LIST_HEAD(&ge->static_msgs[i].link); 2143 2144 timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT : 2145 msecs_to_jiffies(q->sched_props.job_timeout_ms); 2146 2147 err = alloc_guc_id(guc, q); 2148 if (err) 2149 goto err_free; 2150 2151 xe_exec_queue_assign_name(q, q->guc->id); 2152 2153 strscpy(ge->name, q->name, sizeof(ge->name)); 2154 2155 /* 2156 * Use primary queue's submit_wq for all secondary queues of a 2157 * multi queue group. This serialization avoids any locking around 2158 * CGP synchronization with GuC. 2159 */ 2160 if (xe_exec_queue_is_multi_queue_secondary(q)) { 2161 struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); 2162 2163 submit_wq = primary->guc->sched.base.submit_wq; 2164 } 2165 2166 if (q->vm && q->vm->flags & XE_VM_FLAG_MIGRATION) 2167 max_jobs = min(max_jobs, LRC_MIGRATION_ULLS_SEMAPHORE_COUNT - 1); 2168 2169 err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, 2170 submit_wq, max_jobs, 64, 2171 timeout, guc_to_gt(guc)->ordered_wq, NULL, 2172 ge->name, gt_to_xe(q->gt)->drm.dev); 2173 if (err) 2174 goto err_release_id; 2175 2176 sched = &ge->sched; 2177 err = xe_sched_entity_init(&ge->entity, sched); 2178 if (err) 2179 goto err_sched; 2180 2181 q->entity = &ge->entity; 2182 2183 mutex_lock(&guc->submission_state.lock); 2184 if (xe_guc_read_stopped(guc) || vf_recovery(guc)) 2185 xe_sched_stop(sched); 2186 publish_guc_id(guc, q); 2187 mutex_unlock(&guc->submission_state.lock); 2188 2189 /* 2190 * Maintain secondary queues of the multi queue group in a list 2191 * for handling dependencies across the queues in the group. 2192 */ 2193 if (xe_exec_queue_is_multi_queue_secondary(q)) { 2194 struct xe_exec_queue_group *group = q->multi_queue.group; 2195 2196 INIT_LIST_HEAD(&q->multi_queue.link); 2197 mutex_lock(&group->list_lock); 2198 if (group->stopped) 2199 WRITE_ONCE(q->guc->sched.base.pause_submit, true); 2200 list_add_tail(&q->multi_queue.link, &group->list); 2201 mutex_unlock(&group->list_lock); 2202 } 2203 2204 if (xe_exec_queue_is_multi_queue(q)) 2205 trace_xe_exec_queue_create_multi_queue(q); 2206 else 2207 trace_xe_exec_queue_create(q); 2208 2209 return 0; 2210 2211 err_sched: 2212 xe_sched_fini(&ge->sched); 2213 err_release_id: 2214 release_guc_id(guc, q); 2215 err_free: 2216 kfree(ge); 2217 drm_dev_put(drm); 2218 2219 return err; 2220 } 2221 2222 static void guc_exec_queue_kill(struct xe_exec_queue *q) 2223 { 2224 trace_xe_exec_queue_kill(q); 2225 set_exec_queue_killed(q); 2226 /* 2227 * Do not signal the suspend fence here, the queue may still be running 2228 * on the hardware. Only wake any suspend waiter so it observes the 2229 * killed state instead of waiting out the full timeout, suspend_pending 2230 * stays set until the queue is torn down. 2231 */ 2232 suspend_wait_wake(q); 2233 xe_guc_exec_queue_trigger_cleanup(q); 2234 } 2235 2236 static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg, 2237 u32 opcode) 2238 { 2239 struct xe_device *xe = guc_to_xe(exec_queue_to_guc(q)); 2240 int idx; 2241 bool bound = drm_dev_enter(&xe->drm, &idx); 2242 2243 INIT_LIST_HEAD(&msg->link); 2244 msg->opcode = opcode & OPCODE_MASK; 2245 if (bound) { 2246 xe_pm_runtime_get_noresume(xe); 2247 msg->opcode |= MSG_PM_REF; 2248 } 2249 msg->private_data = q; 2250 2251 trace_xe_sched_msg_add(msg); 2252 if (opcode & MSG_HEAD) 2253 xe_sched_add_msg_head(&q->guc->sched, msg); 2254 else if (opcode & MSG_LOCKED) 2255 xe_sched_add_msg_locked(&q->guc->sched, msg); 2256 else 2257 xe_sched_add_msg(&q->guc->sched, msg); 2258 2259 if (bound) 2260 drm_dev_exit(idx); 2261 } 2262 2263 static void guc_exec_queue_try_add_msg_head(struct xe_exec_queue *q, 2264 struct xe_sched_msg *msg, 2265 u32 opcode) 2266 { 2267 if (!list_empty(&msg->link)) 2268 return; 2269 2270 guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED | MSG_HEAD); 2271 } 2272 2273 static bool guc_exec_queue_try_add_msg(struct xe_exec_queue *q, 2274 struct xe_sched_msg *msg, 2275 u32 opcode) 2276 { 2277 if (!list_empty(&msg->link)) 2278 return false; 2279 2280 guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED); 2281 2282 return true; 2283 } 2284 2285 #define STATIC_MSG_CLEANUP 0 2286 #define STATIC_MSG_SUSPEND 1 2287 #define STATIC_MSG_RESUME 2 2288 #define STATIC_MSG_CGP_SYNC 3 2289 static void guc_exec_queue_destroy(struct xe_exec_queue *q) 2290 { 2291 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; 2292 2293 guc_exec_queue_add_msg(q, msg, CLEANUP); 2294 } 2295 2296 static int guc_exec_queue_set_priority(struct xe_exec_queue *q, 2297 enum xe_exec_queue_priority priority) 2298 { 2299 struct xe_sched_msg *msg; 2300 2301 if (q->sched_props.priority == priority || 2302 exec_queue_killed_or_banned_or_wedged(q)) 2303 return 0; 2304 2305 msg = kmalloc_obj(*msg); 2306 if (!msg) 2307 return -ENOMEM; 2308 2309 q->sched_props.priority = priority; 2310 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); 2311 2312 return 0; 2313 } 2314 2315 static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) 2316 { 2317 struct xe_sched_msg *msg; 2318 2319 if (q->sched_props.timeslice_us == timeslice_us || 2320 exec_queue_killed_or_banned_or_wedged(q)) 2321 return 0; 2322 2323 msg = kmalloc_obj(*msg); 2324 if (!msg) 2325 return -ENOMEM; 2326 2327 q->sched_props.timeslice_us = timeslice_us; 2328 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); 2329 2330 return 0; 2331 } 2332 2333 static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, 2334 u32 preempt_timeout_us) 2335 { 2336 struct xe_sched_msg *msg; 2337 2338 if (q->sched_props.preempt_timeout_us == preempt_timeout_us || 2339 exec_queue_killed_or_banned_or_wedged(q)) 2340 return 0; 2341 2342 msg = kmalloc_obj(*msg); 2343 if (!msg) 2344 return -ENOMEM; 2345 2346 q->sched_props.preempt_timeout_us = preempt_timeout_us; 2347 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); 2348 2349 return 0; 2350 } 2351 2352 static int guc_exec_queue_set_multi_queue_priority(struct xe_exec_queue *q, 2353 enum xe_multi_queue_priority priority) 2354 { 2355 struct xe_sched_msg *msg; 2356 2357 xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), xe_exec_queue_is_multi_queue(q)); 2358 2359 if (exec_queue_killed_or_banned_or_wedged(q)) 2360 return 0; 2361 2362 msg = kmalloc_obj(*msg); 2363 if (!msg) 2364 return -ENOMEM; 2365 2366 scoped_guard(spinlock, &q->multi_queue.lock) { 2367 if (q->multi_queue.priority == priority) { 2368 kfree(msg); 2369 return 0; 2370 } 2371 2372 q->multi_queue.priority = priority; 2373 } 2374 2375 guc_exec_queue_add_msg(q, msg, SET_MULTI_QUEUE_PRIORITY); 2376 2377 return 0; 2378 } 2379 2380 /* 2381 * Core suspend: take a suspend reference on @q and, on the first reference, 2382 * disable its GuC context so the GPU is actually preempted. Caller must have 2383 * ensured @q is not killed/banned/wedged. Returns true if this was the first 2384 * suspend reference (the 0->1 transition). 2385 */ 2386 static bool __guc_exec_queue_suspend(struct xe_exec_queue *q) 2387 { 2388 struct xe_guc_exec_queue *ge = q->guc; 2389 struct xe_gpu_scheduler *sched = &ge->sched; 2390 struct xe_sched_msg *msg = ge->static_msgs + STATIC_MSG_SUSPEND; 2391 bool first; 2392 2393 xe_sched_msg_lock(sched); 2394 first = (++ge->suspend_count == 1); 2395 if (first) { 2396 bool added = guc_exec_queue_try_add_msg(q, msg, SUSPEND); 2397 2398 /* slot must be free at 0->1 */ 2399 xe_gt_assert(guc_to_gt(exec_queue_to_guc(q)), added); 2400 ge->suspend_pending = true; 2401 } 2402 xe_sched_msg_unlock(sched); 2403 2404 return first; 2405 } 2406 2407 /* 2408 * Core resume: drop a suspend reference on @q and, on the last reference, 2409 * re-enable its GuC context. Returns true if this dropped the last suspend 2410 * reference (the 1->0 transition). 2411 */ 2412 static bool __guc_exec_queue_resume(struct xe_exec_queue *q) 2413 { 2414 struct xe_guc_exec_queue *ge = q->guc; 2415 struct xe_gpu_scheduler *sched = &ge->sched; 2416 struct xe_sched_msg *msg = ge->static_msgs + STATIC_MSG_RESUME; 2417 struct xe_guc *guc = exec_queue_to_guc(q); 2418 bool last; 2419 2420 xe_sched_msg_lock(sched); 2421 /* 2422 * A killed, banned or wedged queue keeps suspend_pending set until it 2423 * is torn down, so only assert the flag is clear otherwise. 2424 */ 2425 xe_gt_assert(guc_to_gt(guc), !ge->suspend_pending || 2426 exec_queue_killed_or_banned_or_wedged(q)); 2427 xe_gt_assert(guc_to_gt(guc), ge->suspend_count > 0); 2428 last = (--ge->suspend_count == 0); 2429 if (last) { 2430 bool added = guc_exec_queue_try_add_msg(q, msg, RESUME); 2431 2432 /* slot must be free at 1->0 */ 2433 xe_gt_assert(guc_to_gt(guc), added); 2434 } 2435 xe_sched_msg_unlock(sched); 2436 2437 return last; 2438 } 2439 2440 static int guc_exec_queue_suspend(struct xe_exec_queue *q) 2441 { 2442 if (exec_queue_killed_or_banned_or_wedged(q)) 2443 return -EINVAL; 2444 2445 /* 2446 * Non-multi-queue queues and multi-queue primaries suspend themselves 2447 * directly: their own msg_lock makes the suspend_count 0->1 transition 2448 * and the suspend_pending update atomic, so no group level serialization 2449 * is needed. 2450 */ 2451 if (!xe_exec_queue_is_multi_queue_secondary(q)) { 2452 __guc_exec_queue_suspend(q); 2453 return 0; 2454 } 2455 2456 /* 2457 * A secondary's suspend is meaningless once the primary - which owns the 2458 * group's GuC context - is gone, so fail it too. This keeps the 2459 * secondary's effective state consistent with guc_exec_queue_reset_status(), 2460 * which already reports the primary's killed/banned/wedged state for 2461 * secondaries. A primary killed *after* this check is still handled at 2462 * message-processing time, where the SUSPEND is a no-op for a killed 2463 * context; this only covers an already-dead primary. 2464 */ 2465 if (exec_queue_killed_or_banned_or_wedged(xe_exec_queue_multi_queue_primary(q))) 2466 return -EINVAL; 2467 2468 /* 2469 * A secondary doesn't interface with GuC: suspend it like any other 2470 * queue (its own suspend_count drives its internally handled scheduler 2471 * state) and, only on its own 0->1 transition, forward the suspend to the 2472 * primary so the GPU is actually preempted. Hold @suspend_lock so that 2473 * observing the secondary's transition and forwarding it to the primary 2474 * happen atomically; this keeps the primary's refcount paired with member 2475 * transitions even if the same secondary is suspended and resumed 2476 * concurrently across rebind cycles. 2477 */ 2478 scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) { 2479 if (__guc_exec_queue_suspend(q)) 2480 __guc_exec_queue_suspend(xe_exec_queue_multi_queue_primary(q)); 2481 } 2482 2483 return 0; 2484 } 2485 2486 static void guc_exec_queue_suspend_timeout_ban(struct xe_exec_queue *q) 2487 { 2488 struct xe_guc *guc = exec_queue_to_guc(q); 2489 2490 xe_gt_warn(guc_to_gt(guc), 2491 "Suspend fence, guc_id=%d, failed to respond, banning queue", 2492 q->guc->id); 2493 /* 2494 * The GuC failed to respond to the suspend within the timeout. This is 2495 * not recoverable for this context, so ban it and tear it down via 2496 * cleanup rather than leave it suspended forever. The suspend fence is 2497 * not signalled here, the queue may still be running on the hardware, 2498 * so suspend_pending stays set until teardown. The waiter that timed 2499 * out is the caller, so no wake up is needed either. 2500 * 2501 * @q is the primary here; it owns the group's GuC context, so a failure 2502 * to suspend it wedges the whole group. Ban and tear down the entire 2503 * group in the multi-queue case. 2504 */ 2505 if (xe_exec_queue_is_multi_queue(q)) { 2506 set_exec_queue_group_banned(q); 2507 xe_guc_exec_queue_group_trigger_cleanup(q); 2508 } else { 2509 set_exec_queue_banned(q); 2510 xe_guc_exec_queue_trigger_cleanup(q); 2511 } 2512 } 2513 2514 /* 2515 * Wait for @q's own suspend to complete: suspend_pending cleared, or the queue 2516 * killed / GuC stopped. With @blocking, wait uninterruptibly and do not handle 2517 * VF recovery (for callers that must complete on behalf of a possibly 2518 * cross-process queue); otherwise wait interruptibly. 2519 * 2520 * Returns 0 on completion or -ETIME on timeout. Interruptible waits may also 2521 * return -EAGAIN (VF recovery in progress, retry) or -ERESTARTSYS (aborted by a 2522 * signal; suspend_pending may still be set, so callers must not resume() 2523 * without re-confirming the suspend). 2524 */ 2525 static int guc_exec_queue_wait_suspend_done(struct xe_exec_queue *q, bool blocking) 2526 { 2527 struct xe_guc *guc = exec_queue_to_guc(q); 2528 struct xe_device *xe = guc_to_xe(guc); 2529 int ret; 2530 2531 /* 2532 * A kill does not clear suspend_pending, the queue may still be running 2533 * on the hardware, so exec_queue_killed() is what ends the wait in that 2534 * case and the queue is torn down asynchronously. 2535 */ 2536 #define WAIT_COND \ 2537 (!READ_ONCE(q->guc->suspend_pending) || exec_queue_killed(q) || \ 2538 xe_guc_read_stopped(guc)) 2539 2540 retry: 2541 if (blocking) { 2542 if (IS_SRIOV_VF(xe)) 2543 ret = wait_event_timeout(guc->ct.wq, WAIT_COND, HZ * 5); 2544 else 2545 ret = wait_event_timeout(q->guc->suspend_wait, WAIT_COND, 2546 HZ * 5); 2547 } else if (IS_SRIOV_VF(xe)) { 2548 ret = wait_event_interruptible_timeout(guc->ct.wq, WAIT_COND || 2549 vf_recovery(guc), HZ * 5); 2550 } else { 2551 ret = wait_event_interruptible_timeout(q->guc->suspend_wait, 2552 WAIT_COND, HZ * 5); 2553 } 2554 2555 if (!blocking && vf_recovery(guc) && !xe_device_wedged(xe)) 2556 return -EAGAIN; 2557 2558 if (!ret) 2559 return -ETIME; 2560 else if (!blocking && IS_SRIOV_VF(xe) && !WAIT_COND) 2561 /* Corner case on RESFIX DONE where vf_recovery() changes */ 2562 goto retry; 2563 2564 #undef WAIT_COND 2565 2566 return ret < 0 ? ret : 0; 2567 } 2568 2569 static int guc_exec_queue_suspend_wait_common(struct xe_exec_queue *q, bool blocking) 2570 { 2571 int ret; 2572 2573 /* 2574 * A secondary's suspend rides the sched-message worker (short-circuited, 2575 * no GuC round-trip) and so is not synchronous with 2576 * guc_exec_queue_suspend(): its own suspend_pending may still be set 2577 * here. Waiting on the primary alone is not sufficient - if the primary 2578 * was already suspended, the forward is a refcount-only transition that 2579 * queues no new primary SUSPEND and leaves the primary's suspend_pending 2580 * clear, so the primary wait would return immediately while the 2581 * secondary's suspend is still in flight, and a later resume() would trip 2582 * the secondary's !suspend_pending assert. So first wait for the 2583 * secondary's own suspend to complete, then wait on the primary. 2584 * 2585 * A timeout on either bans the queue (being multi-queue, that tears down 2586 * the whole group). A secondary suspend has no real GuC round-trip, so 2587 * its timeout is a software scheduler stall rather than a GuC fault, but 2588 * banning is still the safe recovery: otherwise the queue is left with 2589 * suspend_pending set and a subsequent resume() trips the !suspend_pending 2590 * assert. 2591 */ 2592 if (xe_exec_queue_is_multi_queue_secondary(q)) { 2593 ret = guc_exec_queue_wait_suspend_done(q, blocking); 2594 if (ret == -ETIME) 2595 guc_exec_queue_suspend_timeout_ban(q); 2596 if (ret) 2597 return ret; 2598 } 2599 2600 q = xe_exec_queue_multi_queue_primary(q); 2601 ret = guc_exec_queue_wait_suspend_done(q, blocking); 2602 if (ret == -ETIME) 2603 guc_exec_queue_suspend_timeout_ban(q); 2604 2605 return ret; 2606 } 2607 2608 static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q) 2609 { 2610 return guc_exec_queue_suspend_wait_common(q, false); 2611 } 2612 2613 /* 2614 * Uninterruptible variant of guc_exec_queue_suspend_wait() for callers that 2615 * must complete the wait on behalf of a queue possibly owned by a different 2616 * process (e.g. cleanup/undo paths). An interruptible wait could return 2617 * -ERESTARTSYS if the calling task is signalled, leaving that queue suspended 2618 * forever (cross-process DoS). VF recovery is deliberately not handled (no 2619 * -EAGAIN) since a blocking caller cannot retry. 2620 */ 2621 static int guc_exec_queue_suspend_wait_blocking(struct xe_exec_queue *q) 2622 { 2623 return guc_exec_queue_suspend_wait_common(q, true); 2624 } 2625 2626 static void guc_exec_queue_resume(struct xe_exec_queue *q) 2627 { 2628 /* 2629 * Non-multi-queue queues and multi-queue primaries resume themselves 2630 * directly; their own msg_lock is sufficient. 2631 */ 2632 if (!xe_exec_queue_is_multi_queue_secondary(q)) { 2633 __guc_exec_queue_resume(q); 2634 return; 2635 } 2636 2637 /* 2638 * Mirror of guc_exec_queue_suspend(): resume the secondary like any 2639 * other queue and, only on its own 1->0 transition, forward the resume 2640 * to the primary so the primary's GuC context is re-enabled once the 2641 * last member that suspended it resumes. @suspend_lock keeps the 2642 * secondary transition and the primary forward atomic. 2643 */ 2644 scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) { 2645 if (__guc_exec_queue_resume(q)) 2646 __guc_exec_queue_resume(xe_exec_queue_multi_queue_primary(q)); 2647 } 2648 } 2649 2650 /* 2651 * Drop a leaving secondary's forwarded suspend reference on the primary and 2652 * resume the primary if this was the last member that had it suspended. 2653 * See guc_exec_queue_fini(). 2654 */ 2655 static void guc_exec_queue_multi_queue_drop_suspend(struct xe_exec_queue *q) 2656 { 2657 scoped_guard(spinlock, &q->multi_queue.group->suspend_lock) { 2658 struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q); 2659 2660 /* 2661 * A suspended secondary holds exactly one suspend reference on the 2662 * primary (forwarded on its 0->1 transition). If it leaves while 2663 * still suspended, release that reference so the primary is not 2664 * kept disabled forever. 2665 */ 2666 if (!READ_ONCE(q->guc->suspend_count)) 2667 break; 2668 2669 if (exec_queue_killed_or_banned_or_wedged(primary)) 2670 break; 2671 2672 /* 2673 * No suspend_wait() here (and we can't - suspend_lock is a 2674 * spinlock). guc_exec_queue_fini() has already drained the 2675 * primary's forwarded suspend with the blocking wait, so its 2676 * suspend has completed (suspend_pending cleared) by the time we 2677 * resume it here. __guc_exec_queue_resume() asserts this. 2678 */ 2679 __guc_exec_queue_resume(primary); 2680 } 2681 } 2682 2683 static u64 guc_exec_queue_reset_status(struct xe_exec_queue *q) 2684 { 2685 /* TODO: In case of multiqueue, if a secondary queue is banned due to 2686 * page offlining, checking only the primary queue's GuC reset status 2687 * may mask the true reason or race with it. 2688 */ 2689 if (xe_exec_queue_is_multi_queue_secondary(q)) { 2690 u64 status = guc_exec_queue_reset_status(xe_exec_queue_multi_queue_primary(q)); 2691 2692 if (status) 2693 return status; 2694 } 2695 2696 if (exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q)) { 2697 u64 reason = atomic_read_acquire(&q->ban_reason); 2698 2699 /* If no specific reason was recorded, default to GPU hang */ 2700 if (!reason) 2701 reason = DRM_XE_EXEC_QUEUE_BAN_REASON_GPU_HANG; 2702 return reason; 2703 } 2704 2705 return 0; 2706 } 2707 2708 /* 2709 * All of these functions are an abstraction layer which other parts of Xe can 2710 * use to trap into the GuC backend. All of these functions, aside from init, 2711 * really shouldn't do much other than trap into the DRM scheduler which 2712 * synchronizes these operations. 2713 */ 2714 static const struct xe_exec_queue_ops guc_exec_queue_ops = { 2715 .init = guc_exec_queue_init, 2716 .kill = guc_exec_queue_kill, 2717 .fini = guc_exec_queue_fini, 2718 .destroy = guc_exec_queue_destroy, 2719 .set_priority = guc_exec_queue_set_priority, 2720 .set_timeslice = guc_exec_queue_set_timeslice, 2721 .set_preempt_timeout = guc_exec_queue_set_preempt_timeout, 2722 .set_multi_queue_priority = guc_exec_queue_set_multi_queue_priority, 2723 .suspend = guc_exec_queue_suspend, 2724 .suspend_wait = guc_exec_queue_suspend_wait, 2725 .suspend_wait_blocking = guc_exec_queue_suspend_wait_blocking, 2726 .resume = guc_exec_queue_resume, 2727 .reset_status = guc_exec_queue_reset_status, 2728 }; 2729 2730 static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) 2731 { 2732 struct xe_gpu_scheduler *sched = &q->guc->sched; 2733 bool do_destroy = false; 2734 2735 /* Stop scheduling + flush any DRM scheduler operations */ 2736 xe_sched_submission_stop(sched); 2737 2738 /* Clean up lost G2H + reset engine state */ 2739 if (exec_queue_registered(q)) { 2740 if (exec_queue_destroyed(q)) 2741 do_destroy = true; 2742 } 2743 xe_sched_msg_lock(sched); 2744 if (q->guc->suspend_pending) { 2745 set_exec_queue_suspended(q); 2746 suspend_fence_signal(q); 2747 } 2748 xe_sched_msg_unlock(sched); 2749 atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED | 2750 EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED | 2751 EXEC_QUEUE_STATE_SUSPENDED, 2752 &q->guc->state); 2753 q->guc->resume_time = 0; 2754 trace_xe_exec_queue_stop(q); 2755 2756 /* 2757 * Ban any engine (aside from kernel and engines used for VM ops) with a 2758 * started but not complete job or if a job has gone through a GT reset 2759 * more than twice. 2760 */ 2761 if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) { 2762 struct xe_sched_job *job = xe_sched_first_pending_job(sched); 2763 bool ban = false; 2764 2765 if (job) { 2766 if ((xe_sched_job_started(job) && 2767 !xe_sched_job_completed(job)) || 2768 xe_sched_invalidate_job(job, 2)) { 2769 trace_xe_sched_job_ban(job); 2770 ban = true; 2771 } 2772 } 2773 2774 if (ban) { 2775 set_exec_queue_banned(q); 2776 xe_guc_exec_queue_trigger_cleanup(q); 2777 } 2778 } 2779 2780 if (do_destroy) 2781 guc_exec_queue_destroy_async(q); 2782 } 2783 2784 static int guc_submit_reset_prepare(struct xe_guc *guc) 2785 { 2786 int ret; 2787 2788 /* 2789 * Using an atomic here rather than submission_state.lock as this 2790 * function can be called while holding the CT lock (engine reset 2791 * failure). submission_state.lock needs the CT lock to resubmit jobs. 2792 * Atomic is not ideal, but it works to prevent against concurrent reset 2793 * and releasing any TDRs waiting on guc->submission_state.stopped. 2794 */ 2795 ret = atomic_fetch_or(1, &guc->submission_state.stopped); 2796 smp_wmb(); 2797 wake_up_all(&guc->ct.wq); 2798 2799 return ret; 2800 } 2801 2802 int xe_guc_submit_reset_prepare(struct xe_guc *guc) 2803 { 2804 if (xe_gt_WARN_ON(guc_to_gt(guc), vf_recovery(guc))) 2805 return 0; 2806 2807 if (!guc->submission_state.initialized) 2808 return 0; 2809 2810 return guc_submit_reset_prepare(guc); 2811 } 2812 2813 void xe_guc_submit_reset_wait(struct xe_guc *guc) 2814 { 2815 wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) || 2816 !xe_guc_read_stopped(guc)); 2817 } 2818 2819 void xe_guc_submit_stop(struct xe_guc *guc) 2820 { 2821 struct xe_exec_queue *q; 2822 unsigned long index; 2823 2824 xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1); 2825 2826 mutex_lock(&guc->submission_state.lock); 2827 2828 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 2829 /* Prevent redundant attempts to stop parallel queues */ 2830 if (q->guc->id != index) 2831 continue; 2832 2833 guc_exec_queue_stop(guc, q); 2834 } 2835 2836 mutex_unlock(&guc->submission_state.lock); 2837 2838 /* 2839 * No one can enter the backend at this point, aside from new engine 2840 * creation which is protected by guc->submission_state.lock. 2841 */ 2842 2843 } 2844 2845 static void guc_exec_queue_revert_pending_state_change(struct xe_guc *guc, 2846 struct xe_exec_queue *q) 2847 { 2848 bool pending_enable, pending_disable, pending_resume; 2849 2850 pending_enable = exec_queue_pending_enable(q); 2851 pending_resume = exec_queue_pending_resume(q); 2852 2853 if (pending_enable && pending_resume) { 2854 q->guc->needs_resume = true; 2855 xe_gt_dbg(guc_to_gt(guc), "Replay RESUME - guc_id=%d", 2856 q->guc->id); 2857 } 2858 2859 if (pending_enable && !pending_resume) { 2860 clear_exec_queue_registered(q); 2861 xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER - guc_id=%d", 2862 q->guc->id); 2863 } 2864 2865 if (pending_enable) { 2866 clear_exec_queue_enabled(q); 2867 clear_exec_queue_pending_resume(q); 2868 clear_exec_queue_pending_enable(q); 2869 xe_gt_dbg(guc_to_gt(guc), "Replay ENABLE - guc_id=%d", 2870 q->guc->id); 2871 } 2872 2873 if (exec_queue_destroyed(q) && exec_queue_registered(q)) { 2874 clear_exec_queue_destroyed(q); 2875 q->guc->needs_cleanup = true; 2876 xe_gt_dbg(guc_to_gt(guc), "Replay CLEANUP - guc_id=%d", 2877 q->guc->id); 2878 } 2879 2880 pending_disable = exec_queue_pending_disable(q); 2881 2882 if (pending_disable && exec_queue_suspended(q)) { 2883 clear_exec_queue_suspended(q); 2884 q->guc->needs_suspend = true; 2885 xe_gt_dbg(guc_to_gt(guc), "Replay SUSPEND - guc_id=%d", 2886 q->guc->id); 2887 } 2888 2889 if (pending_disable) { 2890 if (!pending_enable) 2891 set_exec_queue_enabled(q); 2892 clear_exec_queue_pending_disable(q); 2893 xe_gt_dbg(guc_to_gt(guc), "Replay DISABLE - guc_id=%d", 2894 q->guc->id); 2895 } 2896 2897 /* 2898 * A registration time CGP update that bailed when woken by VF recovery. 2899 * Re-register the queue. 2900 */ 2901 if (q->guc->multi_queue.re_register) { 2902 clear_exec_queue_registered(q); 2903 q->guc->multi_queue.re_register = false; 2904 xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER (cgp) - guc_id=%d", 2905 q->guc->id); 2906 } 2907 2908 /* 2909 * If a CGP update gets dropped during migration, CGP_SYNC_DONE will not 2910 * be received (sync_pending still set and this queue owns it). Recover 2911 * it the same way and clear the stuck sync_pending. 2912 */ 2913 if (xe_exec_queue_is_multi_queue(q)) { 2914 struct xe_exec_queue_group *group = q->multi_queue.group; 2915 2916 if (q == READ_ONCE(group->cgp_update_q) && 2917 READ_ONCE(group->sync_pending)) { 2918 if (q->guc->multi_queue.registering_cgp) { 2919 clear_exec_queue_registered(q); 2920 xe_gt_dbg(guc_to_gt(guc), "Replay REGISTER (cgp sync) - guc_id=%d", 2921 q->guc->id); 2922 } else if (q->guc->multi_queue.updating_cgp) { 2923 q->guc->multi_queue.needs_cgp_sync = true; 2924 xe_gt_dbg(guc_to_gt(guc), "Replay CGP_SYNC - guc_id=%d", 2925 q->guc->id); 2926 } 2927 q->guc->multi_queue.registering_cgp = false; 2928 q->guc->multi_queue.updating_cgp = false; 2929 WRITE_ONCE(group->cgp_update_q, NULL); 2930 WRITE_ONCE(group->sync_pending, false); 2931 } 2932 } 2933 2934 /* 2935 * A dynamic-time CGP update that bailed when woken by VF recovery. 2936 * Replay the dynamic CGP update unless the queue is registered or being 2937 * re-registered, which re-does the CGP anyway. 2938 */ 2939 if (q->guc->multi_queue.re_update) { 2940 q->guc->multi_queue.re_update = false; 2941 if (exec_queue_registered(q)) { 2942 q->guc->multi_queue.needs_cgp_sync = true; 2943 xe_gt_dbg(guc_to_gt(guc), "Replay CGP_SYNC (re-update) - guc_id=%d", 2944 q->guc->id); 2945 } 2946 } 2947 2948 q->guc->resume_time = 0; 2949 } 2950 2951 static void lrc_parallel_clear(struct xe_lrc *lrc) 2952 { 2953 struct xe_device *xe = gt_to_xe(lrc->gt); 2954 struct iosys_map map = xe_lrc_parallel_map(lrc); 2955 int i; 2956 2957 for (i = 0; i < WQ_SIZE / sizeof(u32); ++i) 2958 parallel_write(xe, map, wq[i], 2959 FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | 2960 FIELD_PREP(WQ_LEN_MASK, 0)); 2961 } 2962 2963 /* 2964 * This function is quite complex but only real way to ensure no state is lost 2965 * during VF resume flows. The function scans the queue state, make adjustments 2966 * as needed, and queues jobs / messages which replayed upon unpause. 2967 */ 2968 static void guc_exec_queue_pause_prepare(struct xe_guc *guc, struct xe_exec_queue *q) 2969 { 2970 struct xe_gpu_scheduler *sched = &q->guc->sched; 2971 2972 lockdep_assert_held(&guc->submission_state.lock); 2973 2974 /* Stop scheduling + flush any DRM scheduler operations */ 2975 xe_sched_submission_stop(sched); 2976 cancel_delayed_work_sync(&sched->base.work_tdr); 2977 } 2978 2979 static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) 2980 { 2981 struct xe_gpu_scheduler *sched = &q->guc->sched; 2982 struct xe_sched_job *job; 2983 int i; 2984 2985 lockdep_assert_held(&guc->submission_state.lock); 2986 2987 guc_exec_queue_revert_pending_state_change(guc, q); 2988 2989 if (xe_exec_queue_is_parallel(q)) { 2990 /* Pairs with WRITE_ONCE in __xe_exec_queue_init */ 2991 struct xe_lrc *lrc = READ_ONCE(q->lrc[0]); 2992 2993 /* 2994 * NOP existing WQ commands that may contain stale GGTT 2995 * addresses. These will be replayed upon unpause. The hardware 2996 * seems to get confused if the WQ head/tail pointers are 2997 * adjusted. 2998 */ 2999 if (lrc) 3000 lrc_parallel_clear(lrc); 3001 } 3002 3003 job = xe_sched_first_pending_job(sched); 3004 if (job) { 3005 job->restore_replay = true; 3006 3007 /* 3008 * Adjust software tail so jobs submitted overwrite previous 3009 * position in ring buffer with new GGTT addresses. 3010 */ 3011 for (i = 0; i < q->width; ++i) 3012 q->lrc[i]->ring.tail = job->ptrs[i].head; 3013 } 3014 } 3015 3016 /** 3017 * xe_guc_submit_pause - Stop further runs of submission tasks on given GuC. 3018 * @guc: the &xe_guc struct instance whose scheduler is to be disabled 3019 */ 3020 void xe_guc_submit_pause(struct xe_guc *guc) 3021 { 3022 struct xe_exec_queue *q; 3023 unsigned long index; 3024 3025 mutex_lock(&guc->submission_state.lock); 3026 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 3027 xe_sched_submission_stop(&q->guc->sched); 3028 mutex_unlock(&guc->submission_state.lock); 3029 } 3030 3031 /** 3032 * xe_guc_submit_pause_vf - Stop further runs of submission tasks for VF. 3033 * @guc: the &xe_guc struct instance whose scheduler is to be disabled 3034 */ 3035 void xe_guc_submit_pause_vf(struct xe_guc *guc) 3036 { 3037 struct xe_exec_queue *q; 3038 unsigned long index; 3039 3040 xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); 3041 xe_gt_assert(guc_to_gt(guc), vf_recovery(guc)); 3042 3043 mutex_lock(&guc->submission_state.lock); 3044 /* 3045 * Stop all schedulers before reverting any queue: in a multi-queue 3046 * group a secondary's run_job() can register the primary, which must 3047 * not race an in-progress revert. 3048 */ 3049 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 3050 /* Prevent redundant attempts to stop parallel queues */ 3051 if (q->guc->id != index) 3052 continue; 3053 3054 guc_exec_queue_pause_prepare(guc, q); 3055 } 3056 3057 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 3058 /* Prevent redundant attempts to stop parallel queues */ 3059 if (q->guc->id != index) 3060 continue; 3061 3062 guc_exec_queue_pause(guc, q); 3063 } 3064 mutex_unlock(&guc->submission_state.lock); 3065 } 3066 3067 static void guc_exec_queue_start(struct xe_exec_queue *q) 3068 { 3069 struct xe_gpu_scheduler *sched = &q->guc->sched; 3070 3071 if (!exec_queue_killed_or_banned_or_wedged(q)) { 3072 struct xe_sched_job *job = xe_sched_first_pending_job(sched); 3073 int i; 3074 3075 trace_xe_exec_queue_resubmit(q); 3076 if (job) { 3077 for (i = 0; i < q->width; ++i) { 3078 /* 3079 * The GuC context is unregistered at this point 3080 * time, adjusting software ring tail ensures 3081 * jobs are rewritten in original placement, 3082 * adjusting LRC tail ensures the newly loaded 3083 * GuC / contexts only view the LRC tail 3084 * increasing as jobs are written out. 3085 */ 3086 q->lrc[i]->ring.tail = job->ptrs[i].head; 3087 /* 3088 * A ULLS job's slot can have been consumed by 3089 * the CS without the job running, leaving the 3090 * saved head past it. Rewind so the re-emitted 3091 * job is ahead of the head, not behind it. 3092 */ 3093 if (xe_sched_job_is_ulls(job)) 3094 xe_lrc_set_ring_head(q->lrc[i], 3095 job->ptrs[i].head); 3096 xe_lrc_set_ring_tail(q->lrc[i], 3097 xe_lrc_ring_head(q->lrc[i])); 3098 } 3099 } 3100 xe_sched_resubmit_jobs(sched); 3101 } 3102 3103 xe_sched_submission_start(sched); 3104 xe_sched_submission_resume_tdr(sched); 3105 } 3106 3107 int xe_guc_submit_start(struct xe_guc *guc) 3108 { 3109 struct xe_exec_queue *q; 3110 unsigned long index; 3111 3112 xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1); 3113 3114 mutex_lock(&guc->submission_state.lock); 3115 atomic_dec(&guc->submission_state.stopped); 3116 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 3117 /* Prevent redundant attempts to start parallel queues */ 3118 if (q->guc->id != index) 3119 continue; 3120 3121 guc_exec_queue_start(q); 3122 } 3123 mutex_unlock(&guc->submission_state.lock); 3124 3125 wake_up_all(&guc->ct.wq); 3126 3127 return 0; 3128 } 3129 3130 static void guc_exec_queue_unpause_prepare(struct xe_guc *guc, 3131 struct xe_exec_queue *q) 3132 { 3133 struct xe_gpu_scheduler *sched = &q->guc->sched; 3134 struct xe_sched_job *job = NULL; 3135 struct drm_sched_job *s_job; 3136 bool restore_replay = false; 3137 3138 drm_sched_for_each_pending_job(s_job, &sched->base, NULL) { 3139 job = to_xe_sched_job(s_job); 3140 restore_replay |= job->restore_replay; 3141 if (restore_replay) { 3142 xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", 3143 q->guc->id, xe_sched_job_seqno(job)); 3144 3145 q->ring_ops->emit_job(job); 3146 job->restore_replay = true; 3147 } 3148 } 3149 3150 if (job) 3151 job->last_replay = true; 3152 } 3153 3154 /** 3155 * xe_guc_submit_unpause_prepare_vf - Prepare unpause submission tasks for VF. 3156 * @guc: the &xe_guc struct instance whose scheduler is to be prepared for unpause 3157 */ 3158 void xe_guc_submit_unpause_prepare_vf(struct xe_guc *guc) 3159 { 3160 struct xe_exec_queue *q; 3161 unsigned long index; 3162 3163 xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); 3164 xe_gt_assert(guc_to_gt(guc), vf_recovery(guc)); 3165 3166 mutex_lock(&guc->submission_state.lock); 3167 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 3168 /* Prevent redundant attempts to stop parallel queues */ 3169 if (q->guc->id != index) 3170 continue; 3171 3172 guc_exec_queue_unpause_prepare(guc, q); 3173 } 3174 mutex_unlock(&guc->submission_state.lock); 3175 } 3176 3177 static void guc_exec_queue_replay_pending_state_change(struct xe_exec_queue *q) 3178 { 3179 struct xe_gpu_scheduler *sched = &q->guc->sched; 3180 struct xe_sched_msg *msg; 3181 3182 if (q->guc->multi_queue.needs_cgp_sync) { 3183 msg = q->guc->static_msgs + STATIC_MSG_CGP_SYNC; 3184 3185 xe_sched_msg_lock(sched); 3186 guc_exec_queue_try_add_msg_head(q, msg, CGP_SYNC_MSG); 3187 xe_sched_msg_unlock(sched); 3188 3189 q->guc->multi_queue.needs_cgp_sync = false; 3190 } 3191 3192 if (q->guc->needs_cleanup) { 3193 msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; 3194 3195 guc_exec_queue_add_msg(q, msg, CLEANUP); 3196 q->guc->needs_cleanup = false; 3197 } 3198 3199 if (q->guc->needs_suspend) { 3200 msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; 3201 3202 xe_sched_msg_lock(sched); 3203 guc_exec_queue_try_add_msg_head(q, msg, SUSPEND); 3204 xe_sched_msg_unlock(sched); 3205 3206 q->guc->needs_suspend = false; 3207 } 3208 3209 /* 3210 * The resume must be in the message queue before the suspend as it is 3211 * not possible for a resume to be issued if a suspend pending is, but 3212 * the inverse is possible. 3213 */ 3214 if (q->guc->needs_resume) { 3215 msg = q->guc->static_msgs + STATIC_MSG_RESUME; 3216 3217 xe_sched_msg_lock(sched); 3218 guc_exec_queue_try_add_msg_head(q, msg, RESUME); 3219 xe_sched_msg_unlock(sched); 3220 3221 q->guc->needs_resume = false; 3222 } 3223 } 3224 3225 static void guc_exec_queue_unpause(struct xe_guc *guc, struct xe_exec_queue *q) 3226 { 3227 struct xe_gpu_scheduler *sched = &q->guc->sched; 3228 bool needs_tdr = exec_queue_killed_or_banned_or_wedged(q); 3229 3230 lockdep_assert_held(&guc->submission_state.lock); 3231 3232 xe_sched_resubmit_jobs(sched); 3233 guc_exec_queue_replay_pending_state_change(q); 3234 xe_sched_submission_start(sched); 3235 if (needs_tdr) 3236 xe_guc_exec_queue_trigger_cleanup(q); 3237 xe_sched_submission_resume_tdr(sched); 3238 } 3239 3240 /** 3241 * xe_guc_submit_unpause - Allow further runs of submission tasks on given GuC. 3242 * @guc: the &xe_guc struct instance whose scheduler is to be enabled 3243 */ 3244 void xe_guc_submit_unpause(struct xe_guc *guc) 3245 { 3246 struct xe_exec_queue *q; 3247 unsigned long index; 3248 3249 mutex_lock(&guc->submission_state.lock); 3250 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 3251 xe_sched_submission_start(&q->guc->sched); 3252 mutex_unlock(&guc->submission_state.lock); 3253 } 3254 3255 /** 3256 * xe_guc_submit_unpause_vf - Allow further runs of submission tasks for VF. 3257 * @guc: the &xe_guc struct instance whose scheduler is to be enabled 3258 */ 3259 void xe_guc_submit_unpause_vf(struct xe_guc *guc) 3260 { 3261 struct xe_exec_queue *q; 3262 unsigned long index; 3263 3264 xe_gt_assert(guc_to_gt(guc), IS_SRIOV_VF(guc_to_xe(guc))); 3265 3266 mutex_lock(&guc->submission_state.lock); 3267 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 3268 /* 3269 * Prevent redundant attempts to stop parallel queues, or queues 3270 * created after resfix done. 3271 */ 3272 if (q->guc->id != index || 3273 !drm_sched_is_stopped(&q->guc->sched.base)) 3274 continue; 3275 3276 guc_exec_queue_unpause(guc, q); 3277 } 3278 mutex_unlock(&guc->submission_state.lock); 3279 } 3280 3281 /** 3282 * xe_guc_submit_pause_abort - Abort all paused submission task on given GuC. 3283 * @guc: the &xe_guc struct instance whose scheduler is to be aborted 3284 */ 3285 void xe_guc_submit_pause_abort(struct xe_guc *guc) 3286 { 3287 struct xe_exec_queue *q; 3288 unsigned long index; 3289 3290 mutex_lock(&guc->submission_state.lock); 3291 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 3292 struct xe_gpu_scheduler *sched = &q->guc->sched; 3293 3294 /* Prevent redundant attempts to stop parallel queues */ 3295 if (q->guc->id != index) 3296 continue; 3297 3298 xe_sched_submission_start(sched); 3299 guc_exec_queue_kill(q); 3300 } 3301 mutex_unlock(&guc->submission_state.lock); 3302 } 3303 3304 static struct xe_exec_queue * 3305 g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) 3306 { 3307 struct xe_gt *gt = guc_to_gt(guc); 3308 struct xe_exec_queue *q; 3309 3310 if (unlikely(guc_id >= GUC_ID_MAX)) { 3311 xe_gt_err(gt, "Invalid guc_id %u\n", guc_id); 3312 return NULL; 3313 } 3314 3315 q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id); 3316 if (unlikely(!q)) { 3317 xe_gt_err(gt, "No exec queue found for guc_id %u\n", guc_id); 3318 return NULL; 3319 } 3320 3321 xe_gt_assert(guc_to_gt(guc), guc_id >= q->guc->id); 3322 xe_gt_assert(guc_to_gt(guc), guc_id < (q->guc->id + q->width)); 3323 3324 return q; 3325 } 3326 3327 static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) 3328 { 3329 u32 action[] = { 3330 XE_GUC_ACTION_DEREGISTER_CONTEXT, 3331 q->guc->id, 3332 }; 3333 3334 xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q)); 3335 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 3336 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 3337 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); 3338 3339 trace_xe_exec_queue_deregister(q); 3340 3341 if (xe_exec_queue_is_multi_queue_secondary(q)) 3342 handle_deregister_done(guc, q); 3343 else 3344 xe_guc_ct_send_g2h_handler(&guc->ct, action, 3345 ARRAY_SIZE(action)); 3346 } 3347 3348 static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, 3349 u32 runnable_state) 3350 { 3351 trace_xe_exec_queue_scheduling_done(q); 3352 3353 if (runnable_state == 1) { 3354 xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q)); 3355 3356 q->guc->resume_time = ktime_get(); 3357 clear_exec_queue_pending_resume(q); 3358 clear_exec_queue_pending_enable(q); 3359 smp_wmb(); 3360 wake_up_all(&guc->ct.wq); 3361 } else { 3362 bool was_pending; 3363 3364 xe_gt_assert(guc_to_gt(guc), runnable_state == 0); 3365 xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q)); 3366 3367 /* 3368 * A TDR stops the scheduler, which can trap a queued SUSPEND 3369 * message, and then issues a disable of its own. suspend_pending 3370 * is still set in that case, so only attribute this SCHED_DONE 3371 * to a suspend if the SUSPEND message actually ran, i.e. it 3372 * marked the queue suspended before disabling scheduling. The 3373 * trapped message completes the suspend once submission 3374 * restarts. 3375 */ 3376 xe_sched_msg_lock(&q->guc->sched); 3377 was_pending = q->guc->suspend_pending && exec_queue_suspended(q); 3378 if (was_pending) { 3379 clear_exec_queue_pending_disable(q); 3380 suspend_fence_signal(q); 3381 } 3382 xe_sched_msg_unlock(&q->guc->sched); 3383 3384 if (!was_pending) { 3385 /* 3386 * Make sure to clear the pending_disable only after 3387 * sampling the destroyed state. We want to ensure we 3388 * don't trigger the unregister too early with something 3389 * intending to only disable scheduling. The caller 3390 * doing the destroy must wait for an ongoing 3391 * pending_disable before marking as destroyed. 3392 */ 3393 bool destroyed = exec_queue_destroyed(q); 3394 3395 clear_exec_queue_pending_disable(q); 3396 if (destroyed) 3397 deregister_exec_queue(guc, q); 3398 } 3399 3400 /* 3401 * Wake after pending_disable is cleared, not before, so waiters 3402 * such as the TDR see the updated state on the first re-check. 3403 */ 3404 smp_wmb(); 3405 wake_up_all(&guc->ct.wq); 3406 } 3407 } 3408 3409 static void handle_multi_queue_secondary_sched_done(struct xe_guc *guc, 3410 struct xe_exec_queue *q, 3411 u32 runnable_state) 3412 { 3413 /* Take CT lock here as handle_sched_done() do send a h2g message */ 3414 mutex_lock(&guc->ct.lock); 3415 handle_sched_done(guc, q, runnable_state); 3416 mutex_unlock(&guc->ct.lock); 3417 } 3418 3419 int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) 3420 { 3421 struct xe_exec_queue *q; 3422 u32 guc_id, runnable_state; 3423 3424 if (unlikely(len < 2)) 3425 return -EPROTO; 3426 3427 guc_id = msg[0]; 3428 runnable_state = msg[1]; 3429 3430 q = g2h_exec_queue_lookup(guc, guc_id); 3431 if (unlikely(!q)) 3432 return -EPROTO; 3433 3434 if (unlikely(!exec_queue_pending_enable(q) && 3435 !exec_queue_pending_disable(q))) { 3436 xe_gt_err(guc_to_gt(guc), 3437 "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u", 3438 atomic_read(&q->guc->state), q->guc->id, 3439 runnable_state); 3440 return -EPROTO; 3441 } 3442 3443 handle_sched_done(guc, q, runnable_state); 3444 3445 return 0; 3446 } 3447 3448 static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q) 3449 { 3450 trace_xe_exec_queue_deregister_done(q); 3451 3452 clear_exec_queue_registered(q); 3453 guc_exec_queue_destroy_async(q); 3454 } 3455 3456 int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) 3457 { 3458 struct xe_exec_queue *q; 3459 u32 guc_id; 3460 3461 if (unlikely(len < 1)) 3462 return -EPROTO; 3463 3464 guc_id = msg[0]; 3465 3466 q = g2h_exec_queue_lookup(guc, guc_id); 3467 if (unlikely(!q)) 3468 return -EPROTO; 3469 3470 if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) || 3471 exec_queue_pending_enable(q) || exec_queue_enabled(q)) { 3472 xe_gt_err(guc_to_gt(guc), 3473 "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d", 3474 atomic_read(&q->guc->state), q->guc->id); 3475 return -EPROTO; 3476 } 3477 3478 handle_deregister_done(guc, q); 3479 3480 return 0; 3481 } 3482 3483 int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) 3484 { 3485 struct xe_gt *gt = guc_to_gt(guc); 3486 struct xe_exec_queue *q; 3487 u32 guc_id; 3488 3489 if (unlikely(len < 1)) 3490 return -EPROTO; 3491 3492 guc_id = msg[0]; 3493 3494 q = g2h_exec_queue_lookup(guc, guc_id); 3495 if (unlikely(!q)) 3496 return -EPROTO; 3497 3498 if (!exec_queue_killed(q)) 3499 xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x", 3500 xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id, 3501 atomic_read(&q->guc->state)); 3502 3503 trace_xe_exec_queue_reset(q); 3504 3505 /* 3506 * A banned engine is a NOP at this point (came from 3507 * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel 3508 * jobs by setting timeout of the job to the minimum value kicking 3509 * guc_exec_queue_timedout_job. 3510 */ 3511 xe_guc_exec_queue_reset_trigger_cleanup(q); 3512 3513 return 0; 3514 } 3515 3516 /* 3517 * xe_guc_error_capture_handler - Handler of GuC captured message 3518 * @guc: The GuC object 3519 * @msg: Point to the message 3520 * @len: The message length 3521 * 3522 * When GuC captured data is ready, GuC will send message 3523 * XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION to host, this function will be 3524 * called 1st to check status before process the data comes with the message. 3525 * 3526 * Returns: error code. 0 if success 3527 */ 3528 int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len) 3529 { 3530 u32 status; 3531 3532 if (unlikely(len != XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION_DATA_LEN)) 3533 return -EPROTO; 3534 3535 status = msg[0] & XE_GUC_STATE_CAPTURE_EVENT_STATUS_MASK; 3536 if (status == XE_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE) 3537 xe_gt_warn(guc_to_gt(guc), "G2H-Error capture no space"); 3538 3539 xe_guc_capture_process(guc); 3540 3541 return 0; 3542 } 3543 3544 int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, 3545 u32 len) 3546 { 3547 struct xe_gt *gt = guc_to_gt(guc); 3548 struct xe_exec_queue *q; 3549 u32 guc_id; 3550 u32 type = XE_GUC_CAT_ERR_TYPE_INVALID; 3551 3552 if (unlikely(!len || len > 2)) 3553 return -EPROTO; 3554 3555 guc_id = msg[0]; 3556 3557 if (len == 2) 3558 type = msg[1]; 3559 3560 if (guc_id == GUC_ID_UNKNOWN) { 3561 /* 3562 * GuC uses GUC_ID_UNKNOWN if it can not map the CAT fault to any PF/VF 3563 * context. In such case only PF will be notified about that fault. 3564 */ 3565 xe_gt_err_ratelimited(gt, "Memory CAT error reported by GuC!\n"); 3566 return 0; 3567 } 3568 3569 q = g2h_exec_queue_lookup(guc, guc_id); 3570 if (unlikely(!q)) 3571 return -EPROTO; 3572 3573 /* 3574 * The type is HW-defined and changes based on platform, so we don't 3575 * decode it in the kernel and only check if it is valid. 3576 * See bspec 54047 and 72187 for details. 3577 */ 3578 if (type != XE_GUC_CAT_ERR_TYPE_INVALID) 3579 xe_gt_info(gt, 3580 "Engine memory CAT error [%u]: class=%s, logical_mask: 0x%x, guc_id=%d", 3581 type, xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); 3582 else 3583 xe_gt_info(gt, 3584 "Engine memory CAT error: class=%s, logical_mask: 0x%x, guc_id=%d", 3585 xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); 3586 3587 trace_xe_exec_queue_memory_cat_error(q); 3588 3589 /* Treat the same as engine reset */ 3590 xe_guc_exec_queue_reset_trigger_cleanup(q); 3591 3592 return 0; 3593 } 3594 3595 int xe_guc_uncorrectable_error_handler(struct xe_guc *guc, u32 *msg, u32 len) 3596 { 3597 struct xe_gt *gt = guc_to_gt(guc); 3598 struct xe_exec_queue *q; 3599 u32 guc_id; 3600 3601 if (unlikely(!len || len > 1)) 3602 return -EPROTO; 3603 3604 guc_id = msg[0]; 3605 3606 if (guc_id == GUC_ID_UNKNOWN) { 3607 xe_gt_err_ratelimited(gt, "GuC: Uncorrectable local error with unknown GuC id\n"); 3608 return 0; 3609 } 3610 3611 q = g2h_exec_queue_lookup(guc, guc_id); 3612 if (unlikely(!q)) 3613 return -EPROTO; 3614 3615 /* 3616 * On CRI, the hardware cannot distinguish GT translation faults 3617 * from true RAS conditions. Demote to debug to avoid false 3618 * positive hardware error logs. 3619 */ 3620 if (gt_to_xe(gt)->info.platform == XE_CRESCENTISLAND) 3621 xe_gt_dbg(gt, 3622 "GuC: Uncorrectable local error! guc_id=%d class=%s, logical_mask=0x%x", 3623 guc_id, xe_hw_engine_class_to_str(q->class), q->logical_mask); 3624 else 3625 xe_gt_err(gt, 3626 "GuC: Uncorrectable local error! guc_id=%d class=%s, logical_mask=0x%x", 3627 guc_id, xe_hw_engine_class_to_str(q->class), q->logical_mask); 3628 3629 trace_xe_guc_uncorrectable_error(q); 3630 3631 /* Treat the same as engine reset */ 3632 xe_guc_exec_queue_reset_trigger_cleanup(q); 3633 3634 return 0; 3635 } 3636 3637 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) 3638 { 3639 struct xe_gt *gt = guc_to_gt(guc); 3640 u8 guc_class, instance; 3641 u32 reason; 3642 3643 if (unlikely(len != 3)) 3644 return -EPROTO; 3645 3646 guc_class = msg[0]; 3647 instance = msg[1]; 3648 reason = msg[2]; 3649 3650 /* Unexpected failure of a hardware feature, log an actual error */ 3651 xe_log_err(gt, GUCSUBMIT, -EIO, 3652 "engine reset failed on %u:%u, reason=%#x\n", 3653 guc_class, instance, reason); 3654 3655 xe_gt_reset_async(gt); 3656 3657 return 0; 3658 } 3659 3660 int xe_guc_exec_queue_cgp_context_error_handler(struct xe_guc *guc, u32 *msg, 3661 u32 len) 3662 { 3663 struct xe_gt *gt = guc_to_gt(guc); 3664 struct xe_device *xe = guc_to_xe(guc); 3665 struct xe_exec_queue *q; 3666 u32 guc_id = msg[2]; 3667 3668 if (unlikely(len != XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN)) { 3669 drm_err(&xe->drm, "Invalid length %u", len); 3670 return -EPROTO; 3671 } 3672 3673 q = g2h_exec_queue_lookup(guc, guc_id); 3674 if (unlikely(!q)) 3675 return -EPROTO; 3676 3677 xe_gt_dbg(gt, 3678 "CGP context error: [%s] err=0x%x, q0_id=0x%x LRCA=0x%x guc_id=0x%x", 3679 msg[0] & 1 ? "uc" : "kmd", msg[1], msg[2], msg[3], msg[4]); 3680 3681 trace_xe_exec_queue_cgp_context_error(q); 3682 3683 /* Treat the same as engine reset */ 3684 xe_guc_exec_queue_reset_trigger_cleanup(q); 3685 3686 return 0; 3687 } 3688 3689 /** 3690 * xe_guc_exec_queue_cgp_sync_done_handler - CGP synchronization done handler 3691 * @guc: guc 3692 * @msg: message indicating CGP sync done 3693 * @len: length of message 3694 * 3695 * Set multi queue group's sync_pending flag to false and wakeup anyone waiting 3696 * for CGP synchronization to complete. 3697 * 3698 * Return: 0 on success, -EPROTO for malformed messages. 3699 */ 3700 int xe_guc_exec_queue_cgp_sync_done_handler(struct xe_guc *guc, u32 *msg, u32 len) 3701 { 3702 struct xe_device *xe = guc_to_xe(guc); 3703 struct xe_exec_queue_group *group; 3704 struct xe_exec_queue *q, *upd_q; 3705 u32 guc_id = msg[0]; 3706 3707 if (unlikely(len < 1)) { 3708 drm_err(&xe->drm, "Invalid CGP_SYNC_DONE length %u", len); 3709 return -EPROTO; 3710 } 3711 3712 q = g2h_exec_queue_lookup(guc, guc_id); 3713 if (unlikely(!q)) 3714 return -EPROTO; 3715 3716 if (!xe_exec_queue_is_multi_queue_primary(q)) { 3717 drm_err(&xe->drm, "Unexpected CGP_SYNC_DONE response"); 3718 return -EPROTO; 3719 } 3720 3721 /* 3722 * The outstanding CGP update is now confirmed; clear the owning queue's 3723 * tracking so a later migration does not needlessly replay it. 3724 */ 3725 group = q->multi_queue.group; 3726 upd_q = READ_ONCE(group->cgp_update_q); 3727 if (upd_q) { 3728 upd_q->guc->multi_queue.registering_cgp = false; 3729 upd_q->guc->multi_queue.updating_cgp = false; 3730 WRITE_ONCE(group->cgp_update_q, NULL); 3731 } 3732 3733 /* Wakeup the serialized cgp update wait */ 3734 WRITE_ONCE(group->sync_pending, false); 3735 xe_guc_ct_wake_waiters(&guc->ct); 3736 3737 return 0; 3738 } 3739 3740 static void 3741 guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q, 3742 struct xe_guc_submit_exec_queue_snapshot *snapshot) 3743 { 3744 struct xe_guc *guc = exec_queue_to_guc(q); 3745 struct xe_device *xe = guc_to_xe(guc); 3746 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 3747 int i; 3748 3749 snapshot->guc.wqi_head = q->guc->wqi_head; 3750 snapshot->guc.wqi_tail = q->guc->wqi_tail; 3751 snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head); 3752 snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail); 3753 snapshot->parallel.wq_desc.status = parallel_read(xe, map, 3754 wq_desc.wq_status); 3755 3756 if (snapshot->parallel.wq_desc.head != 3757 snapshot->parallel.wq_desc.tail) { 3758 for (i = snapshot->parallel.wq_desc.head; 3759 i != snapshot->parallel.wq_desc.tail; 3760 i = (i + sizeof(u32)) % WQ_SIZE) 3761 snapshot->parallel.wq[i / sizeof(u32)] = 3762 parallel_read(xe, map, wq[i / sizeof(u32)]); 3763 } 3764 } 3765 3766 static void 3767 guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, 3768 struct drm_printer *p) 3769 { 3770 int i; 3771 3772 drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n", 3773 snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head); 3774 drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n", 3775 snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail); 3776 drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status); 3777 3778 if (snapshot->parallel.wq_desc.head != 3779 snapshot->parallel.wq_desc.tail) { 3780 for (i = snapshot->parallel.wq_desc.head; 3781 i != snapshot->parallel.wq_desc.tail; 3782 i = (i + sizeof(u32)) % WQ_SIZE) 3783 drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32), 3784 snapshot->parallel.wq[i / sizeof(u32)]); 3785 } 3786 } 3787 3788 /** 3789 * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine. 3790 * @q: faulty exec queue 3791 * 3792 * This can be printed out in a later stage like during dev_coredump 3793 * analysis. 3794 * 3795 * Returns: a GuC Submit Engine snapshot object that must be freed by the 3796 * caller, using `xe_guc_exec_queue_snapshot_free`. 3797 */ 3798 struct xe_guc_submit_exec_queue_snapshot * 3799 xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) 3800 { 3801 struct xe_gpu_scheduler *sched = &q->guc->sched; 3802 struct xe_guc_submit_exec_queue_snapshot *snapshot; 3803 int i; 3804 3805 snapshot = kzalloc_obj(*snapshot, GFP_ATOMIC); 3806 3807 if (!snapshot) 3808 return NULL; 3809 3810 snapshot->guc.id = q->guc->id; 3811 memcpy(&snapshot->name, &q->name, sizeof(snapshot->name)); 3812 snapshot->class = q->class; 3813 snapshot->logical_mask = q->logical_mask; 3814 snapshot->width = q->width; 3815 snapshot->refcount = kref_read(&q->refcount); 3816 snapshot->jobcount = atomic_read(&q->job_cnt); 3817 snapshot->sched_timeout = sched->base.timeout; 3818 snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us; 3819 snapshot->sched_props.preempt_timeout_us = 3820 q->sched_props.preempt_timeout_us; 3821 3822 snapshot->lrc = kmalloc_objs(struct xe_lrc_snapshot *, q->width, 3823 GFP_ATOMIC); 3824 3825 if (snapshot->lrc) { 3826 for (i = 0; i < q->width; ++i) { 3827 struct xe_lrc *lrc = q->lrc[i]; 3828 3829 snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc); 3830 } 3831 } 3832 3833 snapshot->schedule_state = atomic_read(&q->guc->state); 3834 snapshot->exec_queue_flags = q->flags; 3835 3836 snapshot->parallel_execution = xe_exec_queue_is_parallel(q); 3837 if (snapshot->parallel_execution) 3838 guc_exec_queue_wq_snapshot_capture(q, snapshot); 3839 3840 if (xe_exec_queue_is_multi_queue(q)) { 3841 snapshot->multi_queue.valid = true; 3842 snapshot->multi_queue.primary = xe_exec_queue_multi_queue_primary(q)->guc->id; 3843 snapshot->multi_queue.pos = q->multi_queue.pos; 3844 } 3845 3846 return snapshot; 3847 } 3848 3849 /** 3850 * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine. 3851 * @snapshot: Previously captured snapshot of job. 3852 * 3853 * This captures some data that requires taking some locks, so it cannot be done in signaling path. 3854 */ 3855 void 3856 xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot) 3857 { 3858 int i; 3859 3860 if (!snapshot || !snapshot->lrc) 3861 return; 3862 3863 for (i = 0; i < snapshot->width; ++i) 3864 xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]); 3865 } 3866 3867 /** 3868 * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot. 3869 * @snapshot: GuC Submit Engine snapshot object. 3870 * @p: drm_printer where it will be printed out. 3871 * 3872 * This function prints out a given GuC Submit Engine snapshot object. 3873 */ 3874 void 3875 xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, 3876 struct drm_printer *p) 3877 { 3878 int i; 3879 3880 if (!snapshot) 3881 return; 3882 3883 drm_printf(p, "GuC ID: %d\n", snapshot->guc.id); 3884 drm_printf(p, "\tName: %s\n", snapshot->name); 3885 drm_printf(p, "\tClass: %d\n", snapshot->class); 3886 drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask); 3887 drm_printf(p, "\tWidth: %d\n", snapshot->width); 3888 drm_printf(p, "\tRef: %d\n", snapshot->refcount); 3889 drm_printf(p, "\tJob count: %u\n", snapshot->jobcount); 3890 drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout); 3891 drm_printf(p, "\tTimeslice: %u (us)\n", 3892 snapshot->sched_props.timeslice_us); 3893 drm_printf(p, "\tPreempt timeout: %u (us)\n", 3894 snapshot->sched_props.preempt_timeout_us); 3895 3896 for (i = 0; snapshot->lrc && i < snapshot->width; ++i) 3897 xe_lrc_snapshot_print(snapshot->lrc[i], p); 3898 3899 drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state); 3900 drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags); 3901 3902 if (snapshot->parallel_execution) 3903 guc_exec_queue_wq_snapshot_print(snapshot, p); 3904 3905 if (snapshot->multi_queue.valid) { 3906 drm_printf(p, "\tMulti queue primary GuC ID: %d\n", snapshot->multi_queue.primary); 3907 drm_printf(p, "\tMulti queue position: %d\n", snapshot->multi_queue.pos); 3908 } 3909 } 3910 3911 /** 3912 * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given 3913 * snapshot. 3914 * @snapshot: GuC Submit Engine snapshot object. 3915 * 3916 * This function free all the memory that needed to be allocated at capture 3917 * time. 3918 */ 3919 void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot) 3920 { 3921 int i; 3922 3923 if (!snapshot) 3924 return; 3925 3926 if (snapshot->lrc) { 3927 for (i = 0; i < snapshot->width; i++) 3928 xe_lrc_snapshot_free(snapshot->lrc[i]); 3929 kfree(snapshot->lrc); 3930 } 3931 kfree(snapshot); 3932 } 3933 3934 static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p) 3935 { 3936 struct xe_guc_submit_exec_queue_snapshot *snapshot; 3937 3938 snapshot = xe_guc_exec_queue_snapshot_capture(q); 3939 xe_guc_exec_queue_snapshot_print(snapshot, p); 3940 xe_guc_exec_queue_snapshot_free(snapshot); 3941 } 3942 3943 /** 3944 * xe_guc_register_vf_exec_queue - Register exec queue for a given context type. 3945 * @q: Execution queue 3946 * @ctx_type: Type of the context 3947 * 3948 * This function registers the execution queue with the guc. Special context 3949 * types like GUC_CONTEXT_COMPRESSION_SAVE and GUC_CONTEXT_COMPRESSION_RESTORE 3950 * are only applicable for IGPU and in the VF. 3951 * Submits the execution queue to GUC after registering it. 3952 * 3953 * Returns - None. 3954 */ 3955 void xe_guc_register_vf_exec_queue(struct xe_exec_queue *q, int ctx_type) 3956 { 3957 struct xe_guc *guc = exec_queue_to_guc(q); 3958 struct xe_device *xe = guc_to_xe(guc); 3959 struct xe_gt *gt = guc_to_gt(guc); 3960 3961 xe_gt_assert(gt, IS_SRIOV_VF(xe)); 3962 xe_gt_assert(gt, !IS_DGFX(xe)); 3963 xe_gt_assert(gt, ctx_type == GUC_CONTEXT_COMPRESSION_SAVE || 3964 ctx_type == GUC_CONTEXT_COMPRESSION_RESTORE); 3965 xe_gt_assert(gt, GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 23, 0)); 3966 3967 register_exec_queue(q, ctx_type); 3968 enable_scheduling(q); 3969 } 3970 3971 /** 3972 * xe_guc_submit_print - GuC Submit Print. 3973 * @guc: GuC. 3974 * @p: drm_printer where it will be printed out. 3975 * 3976 * This function capture and prints snapshots of **all** GuC Engines. 3977 */ 3978 void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) 3979 { 3980 struct xe_exec_queue *q; 3981 unsigned long index; 3982 3983 if (!xe_device_uc_enabled(guc_to_xe(guc))) 3984 return; 3985 3986 mutex_lock(&guc->submission_state.lock); 3987 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 3988 guc_exec_queue_print(q, p); 3989 mutex_unlock(&guc->submission_state.lock); 3990 } 3991 3992 /** 3993 * xe_guc_has_registered_mlrc_queues - check whether there are any MLRC queues 3994 * registered with the GuC 3995 * @guc: GuC. 3996 * 3997 * Return: true if any MLRC queue is registered with the GuC, false otherwise. 3998 */ 3999 bool xe_guc_has_registered_mlrc_queues(struct xe_guc *guc) 4000 { 4001 struct xe_exec_queue *q; 4002 unsigned long index; 4003 4004 guard(mutex)(&guc->submission_state.lock); 4005 4006 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 4007 if (q->width > 1) 4008 return true; 4009 4010 return false; 4011 } 4012 4013 /** 4014 * xe_guc_submit_active_multi_queue_lrca() - Resolve the LRCA of the active 4015 * queue in the multi-queue group currently running on an engine. 4016 * @guc: the &xe_guc managing the exec queues 4017 * @hwe: the &xe_hw_engine whose active queue is being resolved 4018 * @cur_lrca: value read from RING_CURRENT_LRCA, identifies the running group 4019 * @active_id: current Active Queue ID read from CSMQDEBUG (position in group) 4020 * 4021 * The running group is identified by matching @cur_lrca against the group's 4022 * primary LRCA; @active_id then selects the active queue within that group. 4023 * 4024 * Return: the LRCA of the active queue, or 0 if no matching queue is found. 4025 */ 4026 u32 xe_guc_submit_active_multi_queue_lrca(struct xe_guc *guc, 4027 struct xe_hw_engine *hwe, 4028 u32 cur_lrca, u32 active_id) 4029 { 4030 struct xe_exec_queue *q; 4031 unsigned long index; 4032 u32 lrca = 0; 4033 4034 /* 4035 * submission_state.lock also protects exec_queue teardown: an exec 4036 * queue is removed from exec_queue_lookup before its group/primary 4037 * are freed, so any q found in the xarray below has a live group 4038 * and primary for as long as we hold the lock. 4039 */ 4040 guard(mutex)(&guc->submission_state.lock); 4041 4042 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 4043 struct xe_exec_queue_group *group = q->multi_queue.group; 4044 struct xe_lrc *active_lrc; 4045 struct xe_lrc *primary_lrc; 4046 4047 if (!q->multi_queue.valid || !group || !group->primary) 4048 continue; 4049 /* 4050 * Multi-queue exec queues are bound to a hw engine class; 4051 * GuC dynamically schedules them onto one of the class's 4052 * physical instances, so there is no fixed queue-to-instance 4053 * mapping to filter on here. 4054 */ 4055 if (q->class != hwe->class) 4056 continue; 4057 if (q->multi_queue.pos != active_id) 4058 continue; 4059 /* 4060 * LRCAs are page-aligned (4K) addresses in GGTT; the low 4061 * bits reported by RING_CURRENT_LRCA are not meaningful, so 4062 * only compare bits [31:12]. 4063 */ 4064 primary_lrc = xe_exec_queue_get_lrc(group->primary, 0); 4065 if (!primary_lrc) 4066 continue; 4067 4068 if ((xe_lrc_ggtt_addr(primary_lrc) ^ cur_lrca) & GENMASK(31, 12)) { 4069 xe_lrc_put(primary_lrc); 4070 continue; 4071 } 4072 4073 active_lrc = xe_exec_queue_get_lrc(q, 0); 4074 xe_lrc_put(primary_lrc); 4075 if (!active_lrc) 4076 continue; 4077 4078 lrca = xe_lrc_ggtt_addr(active_lrc); 4079 xe_lrc_put(active_lrc); 4080 break; 4081 } 4082 4083 return lrca; 4084 } 4085 4086 /** 4087 * xe_guc_contexts_hwsp_rebase - Re-compute GGTT references within all 4088 * exec queues registered to given GuC. 4089 * @guc: the &xe_guc struct instance 4090 * @scratch: scratch buffer to be used as temporary storage 4091 * 4092 * Returns: zero on success, negative error code on failure. 4093 */ 4094 int xe_guc_contexts_hwsp_rebase(struct xe_guc *guc, void *scratch) 4095 { 4096 struct xe_exec_queue *q; 4097 unsigned long index; 4098 int err = 0; 4099 4100 mutex_lock(&guc->submission_state.lock); 4101 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 4102 /* Prevent redundant attempts to stop parallel queues */ 4103 if (q->guc->id != index) 4104 continue; 4105 4106 err = xe_exec_queue_contexts_hwsp_rebase(q, scratch); 4107 if (err) 4108 break; 4109 } 4110 mutex_unlock(&guc->submission_state.lock); 4111 4112 return err; 4113 } 4114