1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_submit.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/bitmap.h> 10 #include <linux/circ_buf.h> 11 #include <linux/delay.h> 12 #include <linux/dma-fence-array.h> 13 #include <linux/math64.h> 14 15 #include <drm/drm_managed.h> 16 17 #include "abi/guc_actions_abi.h" 18 #include "abi/guc_actions_slpc_abi.h" 19 #include "abi/guc_klvs_abi.h" 20 #include "regs/xe_lrc_layout.h" 21 #include "xe_assert.h" 22 #include "xe_devcoredump.h" 23 #include "xe_device.h" 24 #include "xe_exec_queue.h" 25 #include "xe_force_wake.h" 26 #include "xe_gpu_scheduler.h" 27 #include "xe_gt.h" 28 #include "xe_gt_clock.h" 29 #include "xe_gt_printk.h" 30 #include "xe_guc.h" 31 #include "xe_guc_capture.h" 32 #include "xe_guc_ct.h" 33 #include "xe_guc_exec_queue_types.h" 34 #include "xe_guc_id_mgr.h" 35 #include "xe_guc_submit_types.h" 36 #include "xe_hw_engine.h" 37 #include "xe_hw_fence.h" 38 #include "xe_lrc.h" 39 #include "xe_macros.h" 40 #include "xe_map.h" 41 #include "xe_mocs.h" 42 #include "xe_pm.h" 43 #include "xe_ring_ops_types.h" 44 #include "xe_sched_job.h" 45 #include "xe_trace.h" 46 #include "xe_vm.h" 47 48 static struct xe_guc * 49 exec_queue_to_guc(struct xe_exec_queue *q) 50 { 51 return &q->gt->uc.guc; 52 } 53 54 /* 55 * Helpers for engine state, using an atomic as some of the bits can transition 56 * as the same time (e.g. a suspend can be happning at the same time as schedule 57 * engine done being processed). 58 */ 59 #define EXEC_QUEUE_STATE_REGISTERED (1 << 0) 60 #define EXEC_QUEUE_STATE_ENABLED (1 << 1) 61 #define EXEC_QUEUE_STATE_PENDING_ENABLE (1 << 2) 62 #define EXEC_QUEUE_STATE_PENDING_DISABLE (1 << 3) 63 #define EXEC_QUEUE_STATE_DESTROYED (1 << 4) 64 #define EXEC_QUEUE_STATE_SUSPENDED (1 << 5) 65 #define EXEC_QUEUE_STATE_RESET (1 << 6) 66 #define EXEC_QUEUE_STATE_KILLED (1 << 7) 67 #define EXEC_QUEUE_STATE_WEDGED (1 << 8) 68 #define EXEC_QUEUE_STATE_BANNED (1 << 9) 69 #define EXEC_QUEUE_STATE_CHECK_TIMEOUT (1 << 10) 70 #define EXEC_QUEUE_STATE_EXTRA_REF (1 << 11) 71 72 static bool exec_queue_registered(struct xe_exec_queue *q) 73 { 74 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED; 75 } 76 77 static void set_exec_queue_registered(struct xe_exec_queue *q) 78 { 79 atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); 80 } 81 82 static void clear_exec_queue_registered(struct xe_exec_queue *q) 83 { 84 atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); 85 } 86 87 static bool exec_queue_enabled(struct xe_exec_queue *q) 88 { 89 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_ENABLED; 90 } 91 92 static void set_exec_queue_enabled(struct xe_exec_queue *q) 93 { 94 atomic_or(EXEC_QUEUE_STATE_ENABLED, &q->guc->state); 95 } 96 97 static void clear_exec_queue_enabled(struct xe_exec_queue *q) 98 { 99 atomic_and(~EXEC_QUEUE_STATE_ENABLED, &q->guc->state); 100 } 101 102 static bool exec_queue_pending_enable(struct xe_exec_queue *q) 103 { 104 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE; 105 } 106 107 static void set_exec_queue_pending_enable(struct xe_exec_queue *q) 108 { 109 atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); 110 } 111 112 static void clear_exec_queue_pending_enable(struct xe_exec_queue *q) 113 { 114 atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); 115 } 116 117 static bool exec_queue_pending_disable(struct xe_exec_queue *q) 118 { 119 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE; 120 } 121 122 static void set_exec_queue_pending_disable(struct xe_exec_queue *q) 123 { 124 atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); 125 } 126 127 static void clear_exec_queue_pending_disable(struct xe_exec_queue *q) 128 { 129 atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); 130 } 131 132 static bool exec_queue_destroyed(struct xe_exec_queue *q) 133 { 134 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED; 135 } 136 137 static void set_exec_queue_destroyed(struct xe_exec_queue *q) 138 { 139 atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); 140 } 141 142 static bool exec_queue_banned(struct xe_exec_queue *q) 143 { 144 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_BANNED; 145 } 146 147 static void set_exec_queue_banned(struct xe_exec_queue *q) 148 { 149 atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state); 150 } 151 152 static bool exec_queue_suspended(struct xe_exec_queue *q) 153 { 154 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED; 155 } 156 157 static void set_exec_queue_suspended(struct xe_exec_queue *q) 158 { 159 atomic_or(EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); 160 } 161 162 static void clear_exec_queue_suspended(struct xe_exec_queue *q) 163 { 164 atomic_and(~EXEC_QUEUE_STATE_SUSPENDED, &q->guc->state); 165 } 166 167 static bool exec_queue_reset(struct xe_exec_queue *q) 168 { 169 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET; 170 } 171 172 static void set_exec_queue_reset(struct xe_exec_queue *q) 173 { 174 atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state); 175 } 176 177 static bool exec_queue_killed(struct xe_exec_queue *q) 178 { 179 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_KILLED; 180 } 181 182 static void set_exec_queue_killed(struct xe_exec_queue *q) 183 { 184 atomic_or(EXEC_QUEUE_STATE_KILLED, &q->guc->state); 185 } 186 187 static bool exec_queue_wedged(struct xe_exec_queue *q) 188 { 189 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_WEDGED; 190 } 191 192 static void set_exec_queue_wedged(struct xe_exec_queue *q) 193 { 194 atomic_or(EXEC_QUEUE_STATE_WEDGED, &q->guc->state); 195 } 196 197 static bool exec_queue_check_timeout(struct xe_exec_queue *q) 198 { 199 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_CHECK_TIMEOUT; 200 } 201 202 static void set_exec_queue_check_timeout(struct xe_exec_queue *q) 203 { 204 atomic_or(EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state); 205 } 206 207 static void clear_exec_queue_check_timeout(struct xe_exec_queue *q) 208 { 209 atomic_and(~EXEC_QUEUE_STATE_CHECK_TIMEOUT, &q->guc->state); 210 } 211 212 static bool exec_queue_extra_ref(struct xe_exec_queue *q) 213 { 214 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_EXTRA_REF; 215 } 216 217 static void set_exec_queue_extra_ref(struct xe_exec_queue *q) 218 { 219 atomic_or(EXEC_QUEUE_STATE_EXTRA_REF, &q->guc->state); 220 } 221 222 static bool exec_queue_killed_or_banned_or_wedged(struct xe_exec_queue *q) 223 { 224 return (atomic_read(&q->guc->state) & 225 (EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_KILLED | 226 EXEC_QUEUE_STATE_BANNED)); 227 } 228 229 static void guc_submit_fini(struct drm_device *drm, void *arg) 230 { 231 struct xe_guc *guc = arg; 232 233 xa_destroy(&guc->submission_state.exec_queue_lookup); 234 } 235 236 static void guc_submit_wedged_fini(void *arg) 237 { 238 struct xe_guc *guc = arg; 239 struct xe_exec_queue *q; 240 unsigned long index; 241 242 mutex_lock(&guc->submission_state.lock); 243 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 244 if (exec_queue_wedged(q)) { 245 mutex_unlock(&guc->submission_state.lock); 246 xe_exec_queue_put(q); 247 mutex_lock(&guc->submission_state.lock); 248 } 249 } 250 mutex_unlock(&guc->submission_state.lock); 251 } 252 253 static const struct xe_exec_queue_ops guc_exec_queue_ops; 254 255 static void primelockdep(struct xe_guc *guc) 256 { 257 if (!IS_ENABLED(CONFIG_LOCKDEP)) 258 return; 259 260 fs_reclaim_acquire(GFP_KERNEL); 261 262 mutex_lock(&guc->submission_state.lock); 263 mutex_unlock(&guc->submission_state.lock); 264 265 fs_reclaim_release(GFP_KERNEL); 266 } 267 268 /** 269 * xe_guc_submit_init() - Initialize GuC submission. 270 * @guc: the &xe_guc to initialize 271 * @num_ids: number of GuC context IDs to use 272 * 273 * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all 274 * GuC context IDs supported by the GuC firmware should be used for submission. 275 * 276 * Only VF drivers will have to provide explicit number of GuC context IDs 277 * that they can use for submission. 278 * 279 * Return: 0 on success or a negative error code on failure. 280 */ 281 int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) 282 { 283 struct xe_device *xe = guc_to_xe(guc); 284 struct xe_gt *gt = guc_to_gt(guc); 285 int err; 286 287 err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock); 288 if (err) 289 return err; 290 291 err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids); 292 if (err) 293 return err; 294 295 gt->exec_queue_ops = &guc_exec_queue_ops; 296 297 xa_init(&guc->submission_state.exec_queue_lookup); 298 299 init_waitqueue_head(&guc->submission_state.fini_wq); 300 301 primelockdep(guc); 302 303 return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc); 304 } 305 306 static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count) 307 { 308 int i; 309 310 lockdep_assert_held(&guc->submission_state.lock); 311 312 for (i = 0; i < xa_count; ++i) 313 xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i); 314 315 xe_guc_id_mgr_release_locked(&guc->submission_state.idm, 316 q->guc->id, q->width); 317 318 if (xa_empty(&guc->submission_state.exec_queue_lookup)) 319 wake_up(&guc->submission_state.fini_wq); 320 } 321 322 static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) 323 { 324 int ret; 325 int i; 326 327 /* 328 * Must use GFP_NOWAIT as this lock is in the dma fence signalling path, 329 * worse case user gets -ENOMEM on engine create and has to try again. 330 * 331 * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent 332 * failure. 333 */ 334 lockdep_assert_held(&guc->submission_state.lock); 335 336 ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm, 337 q->width); 338 if (ret < 0) 339 return ret; 340 341 q->guc->id = ret; 342 343 for (i = 0; i < q->width; ++i) { 344 ret = xa_err(xa_store(&guc->submission_state.exec_queue_lookup, 345 q->guc->id + i, q, GFP_NOWAIT)); 346 if (ret) 347 goto err_release; 348 } 349 350 return 0; 351 352 err_release: 353 __release_guc_id(guc, q, i); 354 355 return ret; 356 } 357 358 static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) 359 { 360 mutex_lock(&guc->submission_state.lock); 361 __release_guc_id(guc, q, q->width); 362 mutex_unlock(&guc->submission_state.lock); 363 } 364 365 struct exec_queue_policy { 366 u32 count; 367 struct guc_update_exec_queue_policy h2g; 368 }; 369 370 static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy) 371 { 372 size_t bytes = sizeof(policy->h2g.header) + 373 (sizeof(policy->h2g.klv[0]) * policy->count); 374 375 return bytes / sizeof(u32); 376 } 377 378 static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy, 379 u16 guc_id) 380 { 381 policy->h2g.header.action = 382 XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; 383 policy->h2g.header.guc_id = guc_id; 384 policy->count = 0; 385 } 386 387 #define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \ 388 static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \ 389 u32 data) \ 390 { \ 391 XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ 392 \ 393 policy->h2g.klv[policy->count].kl = \ 394 FIELD_PREP(GUC_KLV_0_KEY, \ 395 GUC_CONTEXT_POLICIES_KLV_ID_##id) | \ 396 FIELD_PREP(GUC_KLV_0_LEN, 1); \ 397 policy->h2g.klv[policy->count].value = data; \ 398 policy->count++; \ 399 } 400 401 MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) 402 MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) 403 MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY) 404 MAKE_EXEC_QUEUE_POLICY_ADD(slpc_exec_queue_freq_req, SLPM_GT_FREQUENCY) 405 #undef MAKE_EXEC_QUEUE_POLICY_ADD 406 407 static const int xe_exec_queue_prio_to_guc[] = { 408 [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, 409 [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, 410 [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, 411 [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, 412 }; 413 414 static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) 415 { 416 struct exec_queue_policy policy; 417 enum xe_exec_queue_priority prio = q->sched_props.priority; 418 u32 timeslice_us = q->sched_props.timeslice_us; 419 u32 slpc_exec_queue_freq_req = 0; 420 u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; 421 422 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 423 424 if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY) 425 slpc_exec_queue_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE; 426 427 __guc_exec_queue_policy_start_klv(&policy, q->guc->id); 428 __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]); 429 __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us); 430 __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us); 431 __guc_exec_queue_policy_add_slpc_exec_queue_freq_req(&policy, 432 slpc_exec_queue_freq_req); 433 434 xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, 435 __guc_exec_queue_policy_action_size(&policy), 0, 0); 436 } 437 438 static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q) 439 { 440 struct exec_queue_policy policy; 441 442 __guc_exec_queue_policy_start_klv(&policy, q->guc->id); 443 __guc_exec_queue_policy_add_preemption_timeout(&policy, 1); 444 445 xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, 446 __guc_exec_queue_policy_action_size(&policy), 0, 0); 447 } 448 449 #define parallel_read(xe_, map_, field_) \ 450 xe_map_rd_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ 451 field_) 452 #define parallel_write(xe_, map_, field_, val_) \ 453 xe_map_wr_field(xe_, &map_, 0, struct guc_submit_parallel_scratch, \ 454 field_, val_) 455 456 static void __register_mlrc_exec_queue(struct xe_guc *guc, 457 struct xe_exec_queue *q, 458 struct guc_ctxt_registration_info *info) 459 { 460 #define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2) 461 u32 action[MAX_MLRC_REG_SIZE]; 462 int len = 0; 463 int i; 464 465 xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_parallel(q)); 466 467 action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; 468 action[len++] = info->flags; 469 action[len++] = info->context_idx; 470 action[len++] = info->engine_class; 471 action[len++] = info->engine_submit_mask; 472 action[len++] = info->wq_desc_lo; 473 action[len++] = info->wq_desc_hi; 474 action[len++] = info->wq_base_lo; 475 action[len++] = info->wq_base_hi; 476 action[len++] = info->wq_size; 477 action[len++] = q->width; 478 action[len++] = info->hwlrca_lo; 479 action[len++] = info->hwlrca_hi; 480 481 for (i = 1; i < q->width; ++i) { 482 struct xe_lrc *lrc = q->lrc[i]; 483 484 action[len++] = lower_32_bits(xe_lrc_descriptor(lrc)); 485 action[len++] = upper_32_bits(xe_lrc_descriptor(lrc)); 486 } 487 488 xe_gt_assert(guc_to_gt(guc), len <= MAX_MLRC_REG_SIZE); 489 #undef MAX_MLRC_REG_SIZE 490 491 xe_guc_ct_send(&guc->ct, action, len, 0, 0); 492 } 493 494 static void __register_exec_queue(struct xe_guc *guc, 495 struct guc_ctxt_registration_info *info) 496 { 497 u32 action[] = { 498 XE_GUC_ACTION_REGISTER_CONTEXT, 499 info->flags, 500 info->context_idx, 501 info->engine_class, 502 info->engine_submit_mask, 503 info->wq_desc_lo, 504 info->wq_desc_hi, 505 info->wq_base_lo, 506 info->wq_base_hi, 507 info->wq_size, 508 info->hwlrca_lo, 509 info->hwlrca_hi, 510 }; 511 512 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); 513 } 514 515 static void register_exec_queue(struct xe_exec_queue *q) 516 { 517 struct xe_guc *guc = exec_queue_to_guc(q); 518 struct xe_device *xe = guc_to_xe(guc); 519 struct xe_lrc *lrc = q->lrc[0]; 520 struct guc_ctxt_registration_info info; 521 522 xe_gt_assert(guc_to_gt(guc), !exec_queue_registered(q)); 523 524 memset(&info, 0, sizeof(info)); 525 info.context_idx = q->guc->id; 526 info.engine_class = xe_engine_class_to_guc_class(q->class); 527 info.engine_submit_mask = q->logical_mask; 528 info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc)); 529 info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); 530 info.flags = CONTEXT_REGISTRATION_FLAG_KMD; 531 532 if (xe_exec_queue_is_parallel(q)) { 533 u64 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); 534 struct iosys_map map = xe_lrc_parallel_map(lrc); 535 536 info.wq_desc_lo = lower_32_bits(ggtt_addr + 537 offsetof(struct guc_submit_parallel_scratch, wq_desc)); 538 info.wq_desc_hi = upper_32_bits(ggtt_addr + 539 offsetof(struct guc_submit_parallel_scratch, wq_desc)); 540 info.wq_base_lo = lower_32_bits(ggtt_addr + 541 offsetof(struct guc_submit_parallel_scratch, wq[0])); 542 info.wq_base_hi = upper_32_bits(ggtt_addr + 543 offsetof(struct guc_submit_parallel_scratch, wq[0])); 544 info.wq_size = WQ_SIZE; 545 546 q->guc->wqi_head = 0; 547 q->guc->wqi_tail = 0; 548 xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE); 549 parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE); 550 } 551 552 /* 553 * We must keep a reference for LR engines if engine is registered with 554 * the GuC as jobs signal immediately and can't destroy an engine if the 555 * GuC has a reference to it. 556 */ 557 if (xe_exec_queue_is_lr(q)) 558 xe_exec_queue_get(q); 559 560 set_exec_queue_registered(q); 561 trace_xe_exec_queue_register(q); 562 if (xe_exec_queue_is_parallel(q)) 563 __register_mlrc_exec_queue(guc, q, &info); 564 else 565 __register_exec_queue(guc, &info); 566 init_policies(guc, q); 567 } 568 569 static u32 wq_space_until_wrap(struct xe_exec_queue *q) 570 { 571 return (WQ_SIZE - q->guc->wqi_tail); 572 } 573 574 static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) 575 { 576 struct xe_guc *guc = exec_queue_to_guc(q); 577 struct xe_device *xe = guc_to_xe(guc); 578 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 579 unsigned int sleep_period_ms = 1; 580 581 #define AVAILABLE_SPACE \ 582 CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE) 583 if (wqi_size > AVAILABLE_SPACE) { 584 try_again: 585 q->guc->wqi_head = parallel_read(xe, map, wq_desc.head); 586 if (wqi_size > AVAILABLE_SPACE) { 587 if (sleep_period_ms == 1024) { 588 xe_gt_reset_async(q->gt); 589 return -ENODEV; 590 } 591 592 msleep(sleep_period_ms); 593 sleep_period_ms <<= 1; 594 goto try_again; 595 } 596 } 597 #undef AVAILABLE_SPACE 598 599 return 0; 600 } 601 602 static int wq_noop_append(struct xe_exec_queue *q) 603 { 604 struct xe_guc *guc = exec_queue_to_guc(q); 605 struct xe_device *xe = guc_to_xe(guc); 606 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 607 u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1; 608 609 if (wq_wait_for_space(q, wq_space_until_wrap(q))) 610 return -ENODEV; 611 612 xe_gt_assert(guc_to_gt(guc), FIELD_FIT(WQ_LEN_MASK, len_dw)); 613 614 parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)], 615 FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | 616 FIELD_PREP(WQ_LEN_MASK, len_dw)); 617 q->guc->wqi_tail = 0; 618 619 return 0; 620 } 621 622 static void wq_item_append(struct xe_exec_queue *q) 623 { 624 struct xe_guc *guc = exec_queue_to_guc(q); 625 struct xe_device *xe = guc_to_xe(guc); 626 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 627 #define WQ_HEADER_SIZE 4 /* Includes 1 LRC address too */ 628 u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)]; 629 u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32); 630 u32 len_dw = (wqi_size / sizeof(u32)) - 1; 631 int i = 0, j; 632 633 if (wqi_size > wq_space_until_wrap(q)) { 634 if (wq_noop_append(q)) 635 return; 636 } 637 if (wq_wait_for_space(q, wqi_size)) 638 return; 639 640 wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | 641 FIELD_PREP(WQ_LEN_MASK, len_dw); 642 wqi[i++] = xe_lrc_descriptor(q->lrc[0]); 643 wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) | 644 FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc[0]->ring.tail / sizeof(u64)); 645 wqi[i++] = 0; 646 for (j = 1; j < q->width; ++j) { 647 struct xe_lrc *lrc = q->lrc[j]; 648 649 wqi[i++] = lrc->ring.tail / sizeof(u64); 650 } 651 652 xe_gt_assert(guc_to_gt(guc), i == wqi_size / sizeof(u32)); 653 654 iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch, 655 wq[q->guc->wqi_tail / sizeof(u32)])); 656 xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size); 657 q->guc->wqi_tail += wqi_size; 658 xe_gt_assert(guc_to_gt(guc), q->guc->wqi_tail <= WQ_SIZE); 659 660 xe_device_wmb(xe); 661 662 map = xe_lrc_parallel_map(q->lrc[0]); 663 parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail); 664 } 665 666 #define RESUME_PENDING ~0x0ull 667 static void submit_exec_queue(struct xe_exec_queue *q) 668 { 669 struct xe_guc *guc = exec_queue_to_guc(q); 670 struct xe_lrc *lrc = q->lrc[0]; 671 u32 action[3]; 672 u32 g2h_len = 0; 673 u32 num_g2h = 0; 674 int len = 0; 675 bool extra_submit = false; 676 677 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 678 679 if (xe_exec_queue_is_parallel(q)) 680 wq_item_append(q); 681 else 682 xe_lrc_set_ring_tail(lrc, lrc->ring.tail); 683 684 if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q)) 685 return; 686 687 if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) { 688 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; 689 action[len++] = q->guc->id; 690 action[len++] = GUC_CONTEXT_ENABLE; 691 g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; 692 num_g2h = 1; 693 if (xe_exec_queue_is_parallel(q)) 694 extra_submit = true; 695 696 q->guc->resume_time = RESUME_PENDING; 697 set_exec_queue_pending_enable(q); 698 set_exec_queue_enabled(q); 699 trace_xe_exec_queue_scheduling_enable(q); 700 } else { 701 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; 702 action[len++] = q->guc->id; 703 trace_xe_exec_queue_submit(q); 704 } 705 706 xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); 707 708 if (extra_submit) { 709 len = 0; 710 action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; 711 action[len++] = q->guc->id; 712 trace_xe_exec_queue_submit(q); 713 714 xe_guc_ct_send(&guc->ct, action, len, 0, 0); 715 } 716 } 717 718 static struct dma_fence * 719 guc_exec_queue_run_job(struct drm_sched_job *drm_job) 720 { 721 struct xe_sched_job *job = to_xe_sched_job(drm_job); 722 struct xe_exec_queue *q = job->q; 723 struct xe_guc *guc = exec_queue_to_guc(q); 724 struct dma_fence *fence = NULL; 725 bool lr = xe_exec_queue_is_lr(q); 726 727 xe_gt_assert(guc_to_gt(guc), !(exec_queue_destroyed(q) || exec_queue_pending_disable(q)) || 728 exec_queue_banned(q) || exec_queue_suspended(q)); 729 730 trace_xe_sched_job_run(job); 731 732 if (!exec_queue_killed_or_banned_or_wedged(q) && !xe_sched_job_is_error(job)) { 733 if (!exec_queue_registered(q)) 734 register_exec_queue(q); 735 if (!lr) /* LR jobs are emitted in the exec IOCTL */ 736 q->ring_ops->emit_job(job); 737 submit_exec_queue(q); 738 } 739 740 if (lr) { 741 xe_sched_job_set_error(job, -EOPNOTSUPP); 742 dma_fence_put(job->fence); /* Drop ref from xe_sched_job_arm */ 743 } else { 744 fence = job->fence; 745 } 746 747 return fence; 748 } 749 750 static void guc_exec_queue_free_job(struct drm_sched_job *drm_job) 751 { 752 struct xe_sched_job *job = to_xe_sched_job(drm_job); 753 754 trace_xe_sched_job_free(job); 755 xe_sched_job_put(job); 756 } 757 758 int xe_guc_read_stopped(struct xe_guc *guc) 759 { 760 return atomic_read(&guc->submission_state.stopped); 761 } 762 763 #define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \ 764 u32 action[] = { \ 765 XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ 766 q->guc->id, \ 767 GUC_CONTEXT_##enable_disable, \ 768 } 769 770 static void disable_scheduling_deregister(struct xe_guc *guc, 771 struct xe_exec_queue *q) 772 { 773 MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); 774 int ret; 775 776 set_min_preemption_timeout(guc, q); 777 smp_rmb(); 778 ret = wait_event_timeout(guc->ct.wq, 779 (!exec_queue_pending_enable(q) && 780 !exec_queue_pending_disable(q)) || 781 xe_guc_read_stopped(guc), 782 HZ * 5); 783 if (!ret) { 784 struct xe_gpu_scheduler *sched = &q->guc->sched; 785 786 xe_gt_warn(q->gt, "Pending enable/disable failed to respond\n"); 787 xe_sched_submission_start(sched); 788 xe_gt_reset_async(q->gt); 789 xe_sched_tdr_queue_imm(sched); 790 return; 791 } 792 793 clear_exec_queue_enabled(q); 794 set_exec_queue_pending_disable(q); 795 set_exec_queue_destroyed(q); 796 trace_xe_exec_queue_scheduling_disable(q); 797 798 /* 799 * Reserve space for both G2H here as the 2nd G2H is sent from a G2H 800 * handler and we are not allowed to reserved G2H space in handlers. 801 */ 802 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 803 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + 804 G2H_LEN_DW_DEREGISTER_CONTEXT, 2); 805 } 806 807 static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) 808 { 809 struct xe_guc *guc = exec_queue_to_guc(q); 810 struct xe_device *xe = guc_to_xe(guc); 811 812 /** to wakeup xe_wait_user_fence ioctl if exec queue is reset */ 813 wake_up_all(&xe->ufence_wq); 814 815 if (xe_exec_queue_is_lr(q)) 816 queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr); 817 else 818 xe_sched_tdr_queue_imm(&q->guc->sched); 819 } 820 821 /** 822 * xe_guc_submit_wedge() - Wedge GuC submission 823 * @guc: the GuC object 824 * 825 * Save exec queue's registered with GuC state by taking a ref to each queue. 826 * Register a DRMM handler to drop refs upon driver unload. 827 */ 828 void xe_guc_submit_wedge(struct xe_guc *guc) 829 { 830 struct xe_gt *gt = guc_to_gt(guc); 831 struct xe_exec_queue *q; 832 unsigned long index; 833 int err; 834 835 xe_gt_assert(guc_to_gt(guc), guc_to_xe(guc)->wedged.mode); 836 837 err = devm_add_action_or_reset(guc_to_xe(guc)->drm.dev, 838 guc_submit_wedged_fini, guc); 839 if (err) { 840 xe_gt_err(gt, "Failed to register clean-up on wedged.mode=2; " 841 "Although device is wedged.\n"); 842 return; 843 } 844 845 mutex_lock(&guc->submission_state.lock); 846 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 847 if (xe_exec_queue_get_unless_zero(q)) 848 set_exec_queue_wedged(q); 849 mutex_unlock(&guc->submission_state.lock); 850 } 851 852 static bool guc_submit_hint_wedged(struct xe_guc *guc) 853 { 854 struct xe_device *xe = guc_to_xe(guc); 855 856 if (xe->wedged.mode != 2) 857 return false; 858 859 if (xe_device_wedged(xe)) 860 return true; 861 862 xe_device_declare_wedged(xe); 863 864 return true; 865 } 866 867 static void xe_guc_exec_queue_lr_cleanup(struct work_struct *w) 868 { 869 struct xe_guc_exec_queue *ge = 870 container_of(w, struct xe_guc_exec_queue, lr_tdr); 871 struct xe_exec_queue *q = ge->q; 872 struct xe_guc *guc = exec_queue_to_guc(q); 873 struct xe_gpu_scheduler *sched = &ge->sched; 874 bool wedged; 875 876 xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_lr(q)); 877 trace_xe_exec_queue_lr_cleanup(q); 878 879 wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); 880 881 /* Kill the run_job / process_msg entry points */ 882 xe_sched_submission_stop(sched); 883 884 /* 885 * Engine state now mostly stable, disable scheduling / deregister if 886 * needed. This cleanup routine might be called multiple times, where 887 * the actual async engine deregister drops the final engine ref. 888 * Calling disable_scheduling_deregister will mark the engine as 889 * destroyed and fire off the CT requests to disable scheduling / 890 * deregister, which we only want to do once. We also don't want to mark 891 * the engine as pending_disable again as this may race with the 892 * xe_guc_deregister_done_handler() which treats it as an unexpected 893 * state. 894 */ 895 if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) { 896 struct xe_guc *guc = exec_queue_to_guc(q); 897 int ret; 898 899 set_exec_queue_banned(q); 900 disable_scheduling_deregister(guc, q); 901 902 /* 903 * Must wait for scheduling to be disabled before signalling 904 * any fences, if GT broken the GT reset code should signal us. 905 */ 906 ret = wait_event_timeout(guc->ct.wq, 907 !exec_queue_pending_disable(q) || 908 xe_guc_read_stopped(guc), HZ * 5); 909 if (!ret) { 910 xe_gt_warn(q->gt, "Schedule disable failed to respond, guc_id=%d\n", 911 q->guc->id); 912 xe_devcoredump(q, NULL, "Schedule disable failed to respond, guc_id=%d\n", 913 q->guc->id); 914 xe_sched_submission_start(sched); 915 xe_gt_reset_async(q->gt); 916 return; 917 } 918 } 919 920 if (!exec_queue_killed(q) && !xe_lrc_ring_is_idle(q->lrc[0])) 921 xe_devcoredump(q, NULL, "LR job cleanup, guc_id=%d", q->guc->id); 922 923 xe_sched_submission_start(sched); 924 } 925 926 #define ADJUST_FIVE_PERCENT(__t) mul_u64_u32_div(__t, 105, 100) 927 928 static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) 929 { 930 struct xe_gt *gt = guc_to_gt(exec_queue_to_guc(q)); 931 u32 ctx_timestamp, ctx_job_timestamp; 932 u32 timeout_ms = q->sched_props.job_timeout_ms; 933 u32 diff; 934 u64 running_time_ms; 935 936 if (!xe_sched_job_started(job)) { 937 xe_gt_warn(gt, "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, not started", 938 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 939 q->guc->id); 940 941 return xe_sched_invalidate_job(job, 2); 942 } 943 944 ctx_timestamp = xe_lrc_ctx_timestamp(q->lrc[0]); 945 ctx_job_timestamp = xe_lrc_ctx_job_timestamp(q->lrc[0]); 946 947 /* 948 * Counter wraps at ~223s at the usual 19.2MHz, be paranoid catch 949 * possible overflows with a high timeout. 950 */ 951 xe_gt_assert(gt, timeout_ms < 100 * MSEC_PER_SEC); 952 953 if (ctx_timestamp < ctx_job_timestamp) 954 diff = ctx_timestamp + U32_MAX - ctx_job_timestamp; 955 else 956 diff = ctx_timestamp - ctx_job_timestamp; 957 958 /* 959 * Ensure timeout is within 5% to account for an GuC scheduling latency 960 */ 961 running_time_ms = 962 ADJUST_FIVE_PERCENT(xe_gt_clock_interval_to_ms(gt, diff)); 963 964 xe_gt_dbg(gt, 965 "Check job timeout: seqno=%u, lrc_seqno=%u, guc_id=%d, running_time_ms=%llu, timeout_ms=%u, diff=0x%08x", 966 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 967 q->guc->id, running_time_ms, timeout_ms, diff); 968 969 return running_time_ms >= timeout_ms; 970 } 971 972 static void enable_scheduling(struct xe_exec_queue *q) 973 { 974 MAKE_SCHED_CONTEXT_ACTION(q, ENABLE); 975 struct xe_guc *guc = exec_queue_to_guc(q); 976 int ret; 977 978 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); 979 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 980 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 981 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); 982 983 set_exec_queue_pending_enable(q); 984 set_exec_queue_enabled(q); 985 trace_xe_exec_queue_scheduling_enable(q); 986 987 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 988 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); 989 990 ret = wait_event_timeout(guc->ct.wq, 991 !exec_queue_pending_enable(q) || 992 xe_guc_read_stopped(guc), HZ * 5); 993 if (!ret || xe_guc_read_stopped(guc)) { 994 xe_gt_warn(guc_to_gt(guc), "Schedule enable failed to respond"); 995 set_exec_queue_banned(q); 996 xe_gt_reset_async(q->gt); 997 xe_sched_tdr_queue_imm(&q->guc->sched); 998 } 999 } 1000 1001 static void disable_scheduling(struct xe_exec_queue *q, bool immediate) 1002 { 1003 MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); 1004 struct xe_guc *guc = exec_queue_to_guc(q); 1005 1006 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); 1007 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 1008 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 1009 1010 if (immediate) 1011 set_min_preemption_timeout(guc, q); 1012 clear_exec_queue_enabled(q); 1013 set_exec_queue_pending_disable(q); 1014 trace_xe_exec_queue_scheduling_disable(q); 1015 1016 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 1017 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); 1018 } 1019 1020 static void __deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) 1021 { 1022 u32 action[] = { 1023 XE_GUC_ACTION_DEREGISTER_CONTEXT, 1024 q->guc->id, 1025 }; 1026 1027 xe_gt_assert(guc_to_gt(guc), !exec_queue_destroyed(q)); 1028 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 1029 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); 1030 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 1031 1032 set_exec_queue_destroyed(q); 1033 trace_xe_exec_queue_deregister(q); 1034 1035 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 1036 G2H_LEN_DW_DEREGISTER_CONTEXT, 1); 1037 } 1038 1039 static enum drm_gpu_sched_stat 1040 guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) 1041 { 1042 struct xe_sched_job *job = to_xe_sched_job(drm_job); 1043 struct xe_sched_job *tmp_job; 1044 struct xe_exec_queue *q = job->q; 1045 struct xe_gpu_scheduler *sched = &q->guc->sched; 1046 struct xe_guc *guc = exec_queue_to_guc(q); 1047 const char *process_name = "no process"; 1048 struct xe_device *xe = guc_to_xe(guc); 1049 unsigned int fw_ref; 1050 int err = -ETIME; 1051 pid_t pid = -1; 1052 int i = 0; 1053 bool wedged, skip_timeout_check; 1054 1055 /* 1056 * TDR has fired before free job worker. Common if exec queue 1057 * immediately closed after last fence signaled. Add back to pending 1058 * list so job can be freed and kick scheduler ensuring free job is not 1059 * lost. 1060 */ 1061 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) { 1062 xe_sched_add_pending_job(sched, job); 1063 xe_sched_submission_start(sched); 1064 1065 return DRM_GPU_SCHED_STAT_NOMINAL; 1066 } 1067 1068 /* Kill the run_job entry point */ 1069 xe_sched_submission_stop(sched); 1070 1071 /* Must check all state after stopping scheduler */ 1072 skip_timeout_check = exec_queue_reset(q) || 1073 exec_queue_killed_or_banned_or_wedged(q) || 1074 exec_queue_destroyed(q); 1075 1076 /* 1077 * If devcoredump not captured and GuC capture for the job is not ready 1078 * do manual capture first and decide later if we need to use it 1079 */ 1080 if (!exec_queue_killed(q) && !xe->devcoredump.captured && 1081 !xe_guc_capture_get_matching_and_lock(q)) { 1082 /* take force wake before engine register manual capture */ 1083 fw_ref = xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); 1084 if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) 1085 xe_gt_info(q->gt, "failed to get forcewake for coredump capture\n"); 1086 1087 xe_engine_snapshot_capture_for_queue(q); 1088 1089 xe_force_wake_put(gt_to_fw(q->gt), fw_ref); 1090 } 1091 1092 /* 1093 * XXX: Sampling timeout doesn't work in wedged mode as we have to 1094 * modify scheduling state to read timestamp. We could read the 1095 * timestamp from a register to accumulate current running time but this 1096 * doesn't work for SRIOV. For now assuming timeouts in wedged mode are 1097 * genuine timeouts. 1098 */ 1099 wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); 1100 1101 /* Engine state now stable, disable scheduling to check timestamp */ 1102 if (!wedged && exec_queue_registered(q)) { 1103 int ret; 1104 1105 if (exec_queue_reset(q)) 1106 err = -EIO; 1107 1108 if (!exec_queue_destroyed(q)) { 1109 /* 1110 * Wait for any pending G2H to flush out before 1111 * modifying state 1112 */ 1113 ret = wait_event_timeout(guc->ct.wq, 1114 (!exec_queue_pending_enable(q) && 1115 !exec_queue_pending_disable(q)) || 1116 xe_guc_read_stopped(guc), HZ * 5); 1117 if (!ret || xe_guc_read_stopped(guc)) 1118 goto trigger_reset; 1119 1120 /* 1121 * Flag communicates to G2H handler that schedule 1122 * disable originated from a timeout check. The G2H then 1123 * avoid triggering cleanup or deregistering the exec 1124 * queue. 1125 */ 1126 set_exec_queue_check_timeout(q); 1127 disable_scheduling(q, skip_timeout_check); 1128 } 1129 1130 /* 1131 * Must wait for scheduling to be disabled before signalling 1132 * any fences, if GT broken the GT reset code should signal us. 1133 * 1134 * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault 1135 * error) messages which can cause the schedule disable to get 1136 * lost. If this occurs, trigger a GT reset to recover. 1137 */ 1138 smp_rmb(); 1139 ret = wait_event_timeout(guc->ct.wq, 1140 !exec_queue_pending_disable(q) || 1141 xe_guc_read_stopped(guc), HZ * 5); 1142 if (!ret || xe_guc_read_stopped(guc)) { 1143 trigger_reset: 1144 if (!ret) 1145 xe_gt_warn(guc_to_gt(guc), 1146 "Schedule disable failed to respond, guc_id=%d", 1147 q->guc->id); 1148 xe_devcoredump(q, job, 1149 "Schedule disable failed to respond, guc_id=%d, ret=%d, guc_read=%d", 1150 q->guc->id, ret, xe_guc_read_stopped(guc)); 1151 set_exec_queue_extra_ref(q); 1152 xe_exec_queue_get(q); /* GT reset owns this */ 1153 set_exec_queue_banned(q); 1154 xe_gt_reset_async(q->gt); 1155 xe_sched_tdr_queue_imm(sched); 1156 goto rearm; 1157 } 1158 } 1159 1160 /* 1161 * Check if job is actually timed out, if so restart job execution and TDR 1162 */ 1163 if (!wedged && !skip_timeout_check && !check_timeout(q, job) && 1164 !exec_queue_reset(q) && exec_queue_registered(q)) { 1165 clear_exec_queue_check_timeout(q); 1166 goto sched_enable; 1167 } 1168 1169 if (q->vm && q->vm->xef) { 1170 process_name = q->vm->xef->process_name; 1171 pid = q->vm->xef->pid; 1172 } 1173 xe_gt_notice(guc_to_gt(guc), "Timedout job: seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx in %s [%d]", 1174 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 1175 q->guc->id, q->flags, process_name, pid); 1176 1177 trace_xe_sched_job_timedout(job); 1178 1179 if (!exec_queue_killed(q)) 1180 xe_devcoredump(q, job, 1181 "Timedout job - seqno=%u, lrc_seqno=%u, guc_id=%d, flags=0x%lx", 1182 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job), 1183 q->guc->id, q->flags); 1184 1185 /* 1186 * Kernel jobs should never fail, nor should VM jobs if they do 1187 * somethings has gone wrong and the GT needs a reset 1188 */ 1189 xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL, 1190 "Kernel-submitted job timed out\n"); 1191 xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q), 1192 "VM job timed out on non-killed execqueue\n"); 1193 if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL || 1194 (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) { 1195 if (!xe_sched_invalidate_job(job, 2)) { 1196 clear_exec_queue_check_timeout(q); 1197 xe_gt_reset_async(q->gt); 1198 goto rearm; 1199 } 1200 } 1201 1202 /* Finish cleaning up exec queue via deregister */ 1203 set_exec_queue_banned(q); 1204 if (!wedged && exec_queue_registered(q) && !exec_queue_destroyed(q)) { 1205 set_exec_queue_extra_ref(q); 1206 xe_exec_queue_get(q); 1207 __deregister_exec_queue(guc, q); 1208 } 1209 1210 /* Stop fence signaling */ 1211 xe_hw_fence_irq_stop(q->fence_irq); 1212 1213 /* 1214 * Fence state now stable, stop / start scheduler which cleans up any 1215 * fences that are complete 1216 */ 1217 xe_sched_add_pending_job(sched, job); 1218 xe_sched_submission_start(sched); 1219 1220 xe_guc_exec_queue_trigger_cleanup(q); 1221 1222 /* Mark all outstanding jobs as bad, thus completing them */ 1223 spin_lock(&sched->base.job_list_lock); 1224 list_for_each_entry(tmp_job, &sched->base.pending_list, drm.list) 1225 xe_sched_job_set_error(tmp_job, !i++ ? err : -ECANCELED); 1226 spin_unlock(&sched->base.job_list_lock); 1227 1228 /* Start fence signaling */ 1229 xe_hw_fence_irq_start(q->fence_irq); 1230 1231 return DRM_GPU_SCHED_STAT_NOMINAL; 1232 1233 sched_enable: 1234 enable_scheduling(q); 1235 rearm: 1236 /* 1237 * XXX: Ideally want to adjust timeout based on current execution time 1238 * but there is not currently an easy way to do in DRM scheduler. With 1239 * some thought, do this in a follow up. 1240 */ 1241 xe_sched_add_pending_job(sched, job); 1242 xe_sched_submission_start(sched); 1243 1244 return DRM_GPU_SCHED_STAT_NOMINAL; 1245 } 1246 1247 static void __guc_exec_queue_fini_async(struct work_struct *w) 1248 { 1249 struct xe_guc_exec_queue *ge = 1250 container_of(w, struct xe_guc_exec_queue, fini_async); 1251 struct xe_exec_queue *q = ge->q; 1252 struct xe_guc *guc = exec_queue_to_guc(q); 1253 1254 xe_pm_runtime_get(guc_to_xe(guc)); 1255 trace_xe_exec_queue_destroy(q); 1256 1257 release_guc_id(guc, q); 1258 if (xe_exec_queue_is_lr(q)) 1259 cancel_work_sync(&ge->lr_tdr); 1260 /* Confirm no work left behind accessing device structures */ 1261 cancel_delayed_work_sync(&ge->sched.base.work_tdr); 1262 xe_sched_entity_fini(&ge->entity); 1263 xe_sched_fini(&ge->sched); 1264 1265 kfree(ge); 1266 xe_exec_queue_fini(q); 1267 xe_pm_runtime_put(guc_to_xe(guc)); 1268 } 1269 1270 static void guc_exec_queue_fini_async(struct xe_exec_queue *q) 1271 { 1272 struct xe_guc *guc = exec_queue_to_guc(q); 1273 struct xe_device *xe = guc_to_xe(guc); 1274 1275 INIT_WORK(&q->guc->fini_async, __guc_exec_queue_fini_async); 1276 1277 /* We must block on kernel engines so slabs are empty on driver unload */ 1278 if (q->flags & EXEC_QUEUE_FLAG_PERMANENT || exec_queue_wedged(q)) 1279 __guc_exec_queue_fini_async(&q->guc->fini_async); 1280 else 1281 queue_work(xe->destroy_wq, &q->guc->fini_async); 1282 } 1283 1284 static void __guc_exec_queue_fini(struct xe_guc *guc, struct xe_exec_queue *q) 1285 { 1286 /* 1287 * Might be done from within the GPU scheduler, need to do async as we 1288 * fini the scheduler when the engine is fini'd, the scheduler can't 1289 * complete fini within itself (circular dependency). Async resolves 1290 * this we and don't really care when everything is fini'd, just that it 1291 * is. 1292 */ 1293 guc_exec_queue_fini_async(q); 1294 } 1295 1296 static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg) 1297 { 1298 struct xe_exec_queue *q = msg->private_data; 1299 struct xe_guc *guc = exec_queue_to_guc(q); 1300 1301 xe_gt_assert(guc_to_gt(guc), !(q->flags & EXEC_QUEUE_FLAG_PERMANENT)); 1302 trace_xe_exec_queue_cleanup_entity(q); 1303 1304 if (exec_queue_registered(q)) 1305 disable_scheduling_deregister(guc, q); 1306 else 1307 __guc_exec_queue_fini(guc, q); 1308 } 1309 1310 static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q) 1311 { 1312 return !exec_queue_killed_or_banned_or_wedged(q) && exec_queue_registered(q); 1313 } 1314 1315 static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg) 1316 { 1317 struct xe_exec_queue *q = msg->private_data; 1318 struct xe_guc *guc = exec_queue_to_guc(q); 1319 1320 if (guc_exec_queue_allowed_to_change_state(q)) 1321 init_policies(guc, q); 1322 kfree(msg); 1323 } 1324 1325 static void __suspend_fence_signal(struct xe_exec_queue *q) 1326 { 1327 if (!q->guc->suspend_pending) 1328 return; 1329 1330 WRITE_ONCE(q->guc->suspend_pending, false); 1331 wake_up(&q->guc->suspend_wait); 1332 } 1333 1334 static void suspend_fence_signal(struct xe_exec_queue *q) 1335 { 1336 struct xe_guc *guc = exec_queue_to_guc(q); 1337 1338 xe_gt_assert(guc_to_gt(guc), exec_queue_suspended(q) || exec_queue_killed(q) || 1339 xe_guc_read_stopped(guc)); 1340 xe_gt_assert(guc_to_gt(guc), q->guc->suspend_pending); 1341 1342 __suspend_fence_signal(q); 1343 } 1344 1345 static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) 1346 { 1347 struct xe_exec_queue *q = msg->private_data; 1348 struct xe_guc *guc = exec_queue_to_guc(q); 1349 1350 if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && 1351 exec_queue_enabled(q)) { 1352 wait_event(guc->ct.wq, (q->guc->resume_time != RESUME_PENDING || 1353 xe_guc_read_stopped(guc)) && !exec_queue_pending_disable(q)); 1354 1355 if (!xe_guc_read_stopped(guc)) { 1356 s64 since_resume_ms = 1357 ktime_ms_delta(ktime_get(), 1358 q->guc->resume_time); 1359 s64 wait_ms = q->vm->preempt.min_run_period_ms - 1360 since_resume_ms; 1361 1362 if (wait_ms > 0 && q->guc->resume_time) 1363 msleep(wait_ms); 1364 1365 set_exec_queue_suspended(q); 1366 disable_scheduling(q, false); 1367 } 1368 } else if (q->guc->suspend_pending) { 1369 set_exec_queue_suspended(q); 1370 suspend_fence_signal(q); 1371 } 1372 } 1373 1374 static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) 1375 { 1376 struct xe_exec_queue *q = msg->private_data; 1377 1378 if (guc_exec_queue_allowed_to_change_state(q)) { 1379 clear_exec_queue_suspended(q); 1380 if (!exec_queue_enabled(q)) { 1381 q->guc->resume_time = RESUME_PENDING; 1382 enable_scheduling(q); 1383 } 1384 } else { 1385 clear_exec_queue_suspended(q); 1386 } 1387 } 1388 1389 #define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ 1390 #define SET_SCHED_PROPS 2 1391 #define SUSPEND 3 1392 #define RESUME 4 1393 #define OPCODE_MASK 0xf 1394 #define MSG_LOCKED BIT(8) 1395 1396 static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) 1397 { 1398 struct xe_device *xe = guc_to_xe(exec_queue_to_guc(msg->private_data)); 1399 1400 trace_xe_sched_msg_recv(msg); 1401 1402 switch (msg->opcode) { 1403 case CLEANUP: 1404 __guc_exec_queue_process_msg_cleanup(msg); 1405 break; 1406 case SET_SCHED_PROPS: 1407 __guc_exec_queue_process_msg_set_sched_props(msg); 1408 break; 1409 case SUSPEND: 1410 __guc_exec_queue_process_msg_suspend(msg); 1411 break; 1412 case RESUME: 1413 __guc_exec_queue_process_msg_resume(msg); 1414 break; 1415 default: 1416 XE_WARN_ON("Unknown message type"); 1417 } 1418 1419 xe_pm_runtime_put(xe); 1420 } 1421 1422 static const struct drm_sched_backend_ops drm_sched_ops = { 1423 .run_job = guc_exec_queue_run_job, 1424 .free_job = guc_exec_queue_free_job, 1425 .timedout_job = guc_exec_queue_timedout_job, 1426 }; 1427 1428 static const struct xe_sched_backend_ops xe_sched_ops = { 1429 .process_msg = guc_exec_queue_process_msg, 1430 }; 1431 1432 static int guc_exec_queue_init(struct xe_exec_queue *q) 1433 { 1434 struct xe_gpu_scheduler *sched; 1435 struct xe_guc *guc = exec_queue_to_guc(q); 1436 struct xe_guc_exec_queue *ge; 1437 long timeout; 1438 int err, i; 1439 1440 xe_gt_assert(guc_to_gt(guc), xe_device_uc_enabled(guc_to_xe(guc))); 1441 1442 ge = kzalloc(sizeof(*ge), GFP_KERNEL); 1443 if (!ge) 1444 return -ENOMEM; 1445 1446 q->guc = ge; 1447 ge->q = q; 1448 init_waitqueue_head(&ge->suspend_wait); 1449 1450 for (i = 0; i < MAX_STATIC_MSG_TYPE; ++i) 1451 INIT_LIST_HEAD(&ge->static_msgs[i].link); 1452 1453 timeout = (q->vm && xe_vm_in_lr_mode(q->vm)) ? MAX_SCHEDULE_TIMEOUT : 1454 msecs_to_jiffies(q->sched_props.job_timeout_ms); 1455 err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, 1456 NULL, q->lrc[0]->ring.size / MAX_JOB_SIZE_BYTES, 64, 1457 timeout, guc_to_gt(guc)->ordered_wq, NULL, 1458 q->name, gt_to_xe(q->gt)->drm.dev); 1459 if (err) 1460 goto err_free; 1461 1462 sched = &ge->sched; 1463 err = xe_sched_entity_init(&ge->entity, sched); 1464 if (err) 1465 goto err_sched; 1466 1467 if (xe_exec_queue_is_lr(q)) 1468 INIT_WORK(&q->guc->lr_tdr, xe_guc_exec_queue_lr_cleanup); 1469 1470 mutex_lock(&guc->submission_state.lock); 1471 1472 err = alloc_guc_id(guc, q); 1473 if (err) 1474 goto err_entity; 1475 1476 q->entity = &ge->entity; 1477 1478 if (xe_guc_read_stopped(guc)) 1479 xe_sched_stop(sched); 1480 1481 mutex_unlock(&guc->submission_state.lock); 1482 1483 xe_exec_queue_assign_name(q, q->guc->id); 1484 1485 trace_xe_exec_queue_create(q); 1486 1487 return 0; 1488 1489 err_entity: 1490 mutex_unlock(&guc->submission_state.lock); 1491 xe_sched_entity_fini(&ge->entity); 1492 err_sched: 1493 xe_sched_fini(&ge->sched); 1494 err_free: 1495 kfree(ge); 1496 1497 return err; 1498 } 1499 1500 static void guc_exec_queue_kill(struct xe_exec_queue *q) 1501 { 1502 trace_xe_exec_queue_kill(q); 1503 set_exec_queue_killed(q); 1504 __suspend_fence_signal(q); 1505 xe_guc_exec_queue_trigger_cleanup(q); 1506 } 1507 1508 static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg, 1509 u32 opcode) 1510 { 1511 xe_pm_runtime_get_noresume(guc_to_xe(exec_queue_to_guc(q))); 1512 1513 INIT_LIST_HEAD(&msg->link); 1514 msg->opcode = opcode & OPCODE_MASK; 1515 msg->private_data = q; 1516 1517 trace_xe_sched_msg_add(msg); 1518 if (opcode & MSG_LOCKED) 1519 xe_sched_add_msg_locked(&q->guc->sched, msg); 1520 else 1521 xe_sched_add_msg(&q->guc->sched, msg); 1522 } 1523 1524 static bool guc_exec_queue_try_add_msg(struct xe_exec_queue *q, 1525 struct xe_sched_msg *msg, 1526 u32 opcode) 1527 { 1528 if (!list_empty(&msg->link)) 1529 return false; 1530 1531 guc_exec_queue_add_msg(q, msg, opcode | MSG_LOCKED); 1532 1533 return true; 1534 } 1535 1536 #define STATIC_MSG_CLEANUP 0 1537 #define STATIC_MSG_SUSPEND 1 1538 #define STATIC_MSG_RESUME 2 1539 static void guc_exec_queue_fini(struct xe_exec_queue *q) 1540 { 1541 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; 1542 1543 if (!(q->flags & EXEC_QUEUE_FLAG_PERMANENT) && !exec_queue_wedged(q)) 1544 guc_exec_queue_add_msg(q, msg, CLEANUP); 1545 else 1546 __guc_exec_queue_fini(exec_queue_to_guc(q), q); 1547 } 1548 1549 static int guc_exec_queue_set_priority(struct xe_exec_queue *q, 1550 enum xe_exec_queue_priority priority) 1551 { 1552 struct xe_sched_msg *msg; 1553 1554 if (q->sched_props.priority == priority || 1555 exec_queue_killed_or_banned_or_wedged(q)) 1556 return 0; 1557 1558 msg = kmalloc(sizeof(*msg), GFP_KERNEL); 1559 if (!msg) 1560 return -ENOMEM; 1561 1562 q->sched_props.priority = priority; 1563 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); 1564 1565 return 0; 1566 } 1567 1568 static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) 1569 { 1570 struct xe_sched_msg *msg; 1571 1572 if (q->sched_props.timeslice_us == timeslice_us || 1573 exec_queue_killed_or_banned_or_wedged(q)) 1574 return 0; 1575 1576 msg = kmalloc(sizeof(*msg), GFP_KERNEL); 1577 if (!msg) 1578 return -ENOMEM; 1579 1580 q->sched_props.timeslice_us = timeslice_us; 1581 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); 1582 1583 return 0; 1584 } 1585 1586 static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, 1587 u32 preempt_timeout_us) 1588 { 1589 struct xe_sched_msg *msg; 1590 1591 if (q->sched_props.preempt_timeout_us == preempt_timeout_us || 1592 exec_queue_killed_or_banned_or_wedged(q)) 1593 return 0; 1594 1595 msg = kmalloc(sizeof(*msg), GFP_KERNEL); 1596 if (!msg) 1597 return -ENOMEM; 1598 1599 q->sched_props.preempt_timeout_us = preempt_timeout_us; 1600 guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); 1601 1602 return 0; 1603 } 1604 1605 static int guc_exec_queue_suspend(struct xe_exec_queue *q) 1606 { 1607 struct xe_gpu_scheduler *sched = &q->guc->sched; 1608 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; 1609 1610 if (exec_queue_killed_or_banned_or_wedged(q)) 1611 return -EINVAL; 1612 1613 xe_sched_msg_lock(sched); 1614 if (guc_exec_queue_try_add_msg(q, msg, SUSPEND)) 1615 q->guc->suspend_pending = true; 1616 xe_sched_msg_unlock(sched); 1617 1618 return 0; 1619 } 1620 1621 static int guc_exec_queue_suspend_wait(struct xe_exec_queue *q) 1622 { 1623 struct xe_guc *guc = exec_queue_to_guc(q); 1624 int ret; 1625 1626 /* 1627 * Likely don't need to check exec_queue_killed() as we clear 1628 * suspend_pending upon kill but to be paranoid but races in which 1629 * suspend_pending is set after kill also check kill here. 1630 */ 1631 ret = wait_event_interruptible_timeout(q->guc->suspend_wait, 1632 !READ_ONCE(q->guc->suspend_pending) || 1633 exec_queue_killed(q) || 1634 xe_guc_read_stopped(guc), 1635 HZ * 5); 1636 1637 if (!ret) { 1638 xe_gt_warn(guc_to_gt(guc), 1639 "Suspend fence, guc_id=%d, failed to respond", 1640 q->guc->id); 1641 /* XXX: Trigger GT reset? */ 1642 return -ETIME; 1643 } 1644 1645 return ret < 0 ? ret : 0; 1646 } 1647 1648 static void guc_exec_queue_resume(struct xe_exec_queue *q) 1649 { 1650 struct xe_gpu_scheduler *sched = &q->guc->sched; 1651 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME; 1652 struct xe_guc *guc = exec_queue_to_guc(q); 1653 1654 xe_gt_assert(guc_to_gt(guc), !q->guc->suspend_pending); 1655 1656 xe_sched_msg_lock(sched); 1657 guc_exec_queue_try_add_msg(q, msg, RESUME); 1658 xe_sched_msg_unlock(sched); 1659 } 1660 1661 static bool guc_exec_queue_reset_status(struct xe_exec_queue *q) 1662 { 1663 return exec_queue_reset(q) || exec_queue_killed_or_banned_or_wedged(q); 1664 } 1665 1666 /* 1667 * All of these functions are an abstraction layer which other parts of XE can 1668 * use to trap into the GuC backend. All of these functions, aside from init, 1669 * really shouldn't do much other than trap into the DRM scheduler which 1670 * synchronizes these operations. 1671 */ 1672 static const struct xe_exec_queue_ops guc_exec_queue_ops = { 1673 .init = guc_exec_queue_init, 1674 .kill = guc_exec_queue_kill, 1675 .fini = guc_exec_queue_fini, 1676 .set_priority = guc_exec_queue_set_priority, 1677 .set_timeslice = guc_exec_queue_set_timeslice, 1678 .set_preempt_timeout = guc_exec_queue_set_preempt_timeout, 1679 .suspend = guc_exec_queue_suspend, 1680 .suspend_wait = guc_exec_queue_suspend_wait, 1681 .resume = guc_exec_queue_resume, 1682 .reset_status = guc_exec_queue_reset_status, 1683 }; 1684 1685 static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) 1686 { 1687 struct xe_gpu_scheduler *sched = &q->guc->sched; 1688 1689 /* Stop scheduling + flush any DRM scheduler operations */ 1690 xe_sched_submission_stop(sched); 1691 1692 /* Clean up lost G2H + reset engine state */ 1693 if (exec_queue_registered(q)) { 1694 if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q)) 1695 xe_exec_queue_put(q); 1696 else if (exec_queue_destroyed(q)) 1697 __guc_exec_queue_fini(guc, q); 1698 } 1699 if (q->guc->suspend_pending) { 1700 set_exec_queue_suspended(q); 1701 suspend_fence_signal(q); 1702 } 1703 atomic_and(EXEC_QUEUE_STATE_WEDGED | EXEC_QUEUE_STATE_BANNED | 1704 EXEC_QUEUE_STATE_KILLED | EXEC_QUEUE_STATE_DESTROYED | 1705 EXEC_QUEUE_STATE_SUSPENDED, 1706 &q->guc->state); 1707 q->guc->resume_time = 0; 1708 trace_xe_exec_queue_stop(q); 1709 1710 /* 1711 * Ban any engine (aside from kernel and engines used for VM ops) with a 1712 * started but not complete job or if a job has gone through a GT reset 1713 * more than twice. 1714 */ 1715 if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) { 1716 struct xe_sched_job *job = xe_sched_first_pending_job(sched); 1717 bool ban = false; 1718 1719 if (job) { 1720 if ((xe_sched_job_started(job) && 1721 !xe_sched_job_completed(job)) || 1722 xe_sched_invalidate_job(job, 2)) { 1723 trace_xe_sched_job_ban(job); 1724 ban = true; 1725 } 1726 } else if (xe_exec_queue_is_lr(q) && 1727 !xe_lrc_ring_is_idle(q->lrc[0])) { 1728 ban = true; 1729 } 1730 1731 if (ban) { 1732 set_exec_queue_banned(q); 1733 xe_guc_exec_queue_trigger_cleanup(q); 1734 } 1735 } 1736 } 1737 1738 int xe_guc_submit_reset_prepare(struct xe_guc *guc) 1739 { 1740 int ret; 1741 1742 /* 1743 * Using an atomic here rather than submission_state.lock as this 1744 * function can be called while holding the CT lock (engine reset 1745 * failure). submission_state.lock needs the CT lock to resubmit jobs. 1746 * Atomic is not ideal, but it works to prevent against concurrent reset 1747 * and releasing any TDRs waiting on guc->submission_state.stopped. 1748 */ 1749 ret = atomic_fetch_or(1, &guc->submission_state.stopped); 1750 smp_wmb(); 1751 wake_up_all(&guc->ct.wq); 1752 1753 return ret; 1754 } 1755 1756 void xe_guc_submit_reset_wait(struct xe_guc *guc) 1757 { 1758 wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) || 1759 !xe_guc_read_stopped(guc)); 1760 } 1761 1762 void xe_guc_submit_stop(struct xe_guc *guc) 1763 { 1764 struct xe_exec_queue *q; 1765 unsigned long index; 1766 1767 xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1); 1768 1769 mutex_lock(&guc->submission_state.lock); 1770 1771 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 1772 /* Prevent redundant attempts to stop parallel queues */ 1773 if (q->guc->id != index) 1774 continue; 1775 1776 guc_exec_queue_stop(guc, q); 1777 } 1778 1779 mutex_unlock(&guc->submission_state.lock); 1780 1781 /* 1782 * No one can enter the backend at this point, aside from new engine 1783 * creation which is protected by guc->submission_state.lock. 1784 */ 1785 1786 } 1787 1788 static void guc_exec_queue_start(struct xe_exec_queue *q) 1789 { 1790 struct xe_gpu_scheduler *sched = &q->guc->sched; 1791 1792 if (!exec_queue_killed_or_banned_or_wedged(q)) { 1793 int i; 1794 1795 trace_xe_exec_queue_resubmit(q); 1796 for (i = 0; i < q->width; ++i) 1797 xe_lrc_set_ring_head(q->lrc[i], q->lrc[i]->ring.tail); 1798 xe_sched_resubmit_jobs(sched); 1799 } 1800 1801 xe_sched_submission_start(sched); 1802 xe_sched_submission_resume_tdr(sched); 1803 } 1804 1805 int xe_guc_submit_start(struct xe_guc *guc) 1806 { 1807 struct xe_exec_queue *q; 1808 unsigned long index; 1809 1810 xe_gt_assert(guc_to_gt(guc), xe_guc_read_stopped(guc) == 1); 1811 1812 mutex_lock(&guc->submission_state.lock); 1813 atomic_dec(&guc->submission_state.stopped); 1814 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) { 1815 /* Prevent redundant attempts to start parallel queues */ 1816 if (q->guc->id != index) 1817 continue; 1818 1819 guc_exec_queue_start(q); 1820 } 1821 mutex_unlock(&guc->submission_state.lock); 1822 1823 wake_up_all(&guc->ct.wq); 1824 1825 return 0; 1826 } 1827 1828 static struct xe_exec_queue * 1829 g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) 1830 { 1831 struct xe_gt *gt = guc_to_gt(guc); 1832 struct xe_exec_queue *q; 1833 1834 if (unlikely(guc_id >= GUC_ID_MAX)) { 1835 xe_gt_err(gt, "Invalid guc_id %u\n", guc_id); 1836 return NULL; 1837 } 1838 1839 q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id); 1840 if (unlikely(!q)) { 1841 xe_gt_err(gt, "Not engine present for guc_id %u\n", guc_id); 1842 return NULL; 1843 } 1844 1845 xe_gt_assert(guc_to_gt(guc), guc_id >= q->guc->id); 1846 xe_gt_assert(guc_to_gt(guc), guc_id < (q->guc->id + q->width)); 1847 1848 return q; 1849 } 1850 1851 static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) 1852 { 1853 u32 action[] = { 1854 XE_GUC_ACTION_DEREGISTER_CONTEXT, 1855 q->guc->id, 1856 }; 1857 1858 xe_gt_assert(guc_to_gt(guc), exec_queue_destroyed(q)); 1859 xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); 1860 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_disable(q)); 1861 xe_gt_assert(guc_to_gt(guc), !exec_queue_pending_enable(q)); 1862 1863 trace_xe_exec_queue_deregister(q); 1864 1865 xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); 1866 } 1867 1868 static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q, 1869 u32 runnable_state) 1870 { 1871 trace_xe_exec_queue_scheduling_done(q); 1872 1873 if (runnable_state == 1) { 1874 xe_gt_assert(guc_to_gt(guc), exec_queue_pending_enable(q)); 1875 1876 q->guc->resume_time = ktime_get(); 1877 clear_exec_queue_pending_enable(q); 1878 smp_wmb(); 1879 wake_up_all(&guc->ct.wq); 1880 } else { 1881 bool check_timeout = exec_queue_check_timeout(q); 1882 1883 xe_gt_assert(guc_to_gt(guc), runnable_state == 0); 1884 xe_gt_assert(guc_to_gt(guc), exec_queue_pending_disable(q)); 1885 1886 if (q->guc->suspend_pending) { 1887 suspend_fence_signal(q); 1888 clear_exec_queue_pending_disable(q); 1889 } else { 1890 if (exec_queue_banned(q) || check_timeout) { 1891 smp_wmb(); 1892 wake_up_all(&guc->ct.wq); 1893 } 1894 if (!check_timeout && exec_queue_destroyed(q)) { 1895 /* 1896 * Make sure to clear the pending_disable only 1897 * after sampling the destroyed state. We want 1898 * to ensure we don't trigger the unregister too 1899 * early with something intending to only 1900 * disable scheduling. The caller doing the 1901 * destroy must wait for an ongoing 1902 * pending_disable before marking as destroyed. 1903 */ 1904 clear_exec_queue_pending_disable(q); 1905 deregister_exec_queue(guc, q); 1906 } else { 1907 clear_exec_queue_pending_disable(q); 1908 } 1909 } 1910 } 1911 } 1912 1913 int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) 1914 { 1915 struct xe_exec_queue *q; 1916 u32 guc_id, runnable_state; 1917 1918 if (unlikely(len < 2)) 1919 return -EPROTO; 1920 1921 guc_id = msg[0]; 1922 runnable_state = msg[1]; 1923 1924 q = g2h_exec_queue_lookup(guc, guc_id); 1925 if (unlikely(!q)) 1926 return -EPROTO; 1927 1928 if (unlikely(!exec_queue_pending_enable(q) && 1929 !exec_queue_pending_disable(q))) { 1930 xe_gt_err(guc_to_gt(guc), 1931 "SCHED_DONE: Unexpected engine state 0x%04x, guc_id=%d, runnable_state=%u", 1932 atomic_read(&q->guc->state), q->guc->id, 1933 runnable_state); 1934 return -EPROTO; 1935 } 1936 1937 handle_sched_done(guc, q, runnable_state); 1938 1939 return 0; 1940 } 1941 1942 static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q) 1943 { 1944 trace_xe_exec_queue_deregister_done(q); 1945 1946 clear_exec_queue_registered(q); 1947 1948 if (exec_queue_extra_ref(q) || xe_exec_queue_is_lr(q)) 1949 xe_exec_queue_put(q); 1950 else 1951 __guc_exec_queue_fini(guc, q); 1952 } 1953 1954 int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) 1955 { 1956 struct xe_exec_queue *q; 1957 u32 guc_id; 1958 1959 if (unlikely(len < 1)) 1960 return -EPROTO; 1961 1962 guc_id = msg[0]; 1963 1964 q = g2h_exec_queue_lookup(guc, guc_id); 1965 if (unlikely(!q)) 1966 return -EPROTO; 1967 1968 if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) || 1969 exec_queue_pending_enable(q) || exec_queue_enabled(q)) { 1970 xe_gt_err(guc_to_gt(guc), 1971 "DEREGISTER_DONE: Unexpected engine state 0x%04x, guc_id=%d", 1972 atomic_read(&q->guc->state), q->guc->id); 1973 return -EPROTO; 1974 } 1975 1976 handle_deregister_done(guc, q); 1977 1978 return 0; 1979 } 1980 1981 int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) 1982 { 1983 struct xe_gt *gt = guc_to_gt(guc); 1984 struct xe_exec_queue *q; 1985 u32 guc_id; 1986 1987 if (unlikely(len < 1)) 1988 return -EPROTO; 1989 1990 guc_id = msg[0]; 1991 1992 q = g2h_exec_queue_lookup(guc, guc_id); 1993 if (unlikely(!q)) 1994 return -EPROTO; 1995 1996 xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d", 1997 xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); 1998 1999 trace_xe_exec_queue_reset(q); 2000 2001 /* 2002 * A banned engine is a NOP at this point (came from 2003 * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel 2004 * jobs by setting timeout of the job to the minimum value kicking 2005 * guc_exec_queue_timedout_job. 2006 */ 2007 set_exec_queue_reset(q); 2008 if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) 2009 xe_guc_exec_queue_trigger_cleanup(q); 2010 2011 return 0; 2012 } 2013 2014 /* 2015 * xe_guc_error_capture_handler - Handler of GuC captured message 2016 * @guc: The GuC object 2017 * @msg: Point to the message 2018 * @len: The message length 2019 * 2020 * When GuC captured data is ready, GuC will send message 2021 * XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION to host, this function will be 2022 * called 1st to check status before process the data comes with the message. 2023 * 2024 * Returns: error code. 0 if success 2025 */ 2026 int xe_guc_error_capture_handler(struct xe_guc *guc, u32 *msg, u32 len) 2027 { 2028 u32 status; 2029 2030 if (unlikely(len != XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION_DATA_LEN)) 2031 return -EPROTO; 2032 2033 status = msg[0] & XE_GUC_STATE_CAPTURE_EVENT_STATUS_MASK; 2034 if (status == XE_GUC_STATE_CAPTURE_EVENT_STATUS_NOSPACE) 2035 xe_gt_warn(guc_to_gt(guc), "G2H-Error capture no space"); 2036 2037 xe_guc_capture_process(guc); 2038 2039 return 0; 2040 } 2041 2042 int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, 2043 u32 len) 2044 { 2045 struct xe_gt *gt = guc_to_gt(guc); 2046 struct xe_exec_queue *q; 2047 u32 guc_id; 2048 2049 if (unlikely(len < 1)) 2050 return -EPROTO; 2051 2052 guc_id = msg[0]; 2053 2054 if (guc_id == GUC_ID_UNKNOWN) { 2055 /* 2056 * GuC uses GUC_ID_UNKNOWN if it can not map the CAT fault to any PF/VF 2057 * context. In such case only PF will be notified about that fault. 2058 */ 2059 xe_gt_err_ratelimited(gt, "Memory CAT error reported by GuC!\n"); 2060 return 0; 2061 } 2062 2063 q = g2h_exec_queue_lookup(guc, guc_id); 2064 if (unlikely(!q)) 2065 return -EPROTO; 2066 2067 xe_gt_dbg(gt, "Engine memory cat error: engine_class=%s, logical_mask: 0x%x, guc_id=%d", 2068 xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id); 2069 2070 trace_xe_exec_queue_memory_cat_error(q); 2071 2072 /* Treat the same as engine reset */ 2073 set_exec_queue_reset(q); 2074 if (!exec_queue_banned(q) && !exec_queue_check_timeout(q)) 2075 xe_guc_exec_queue_trigger_cleanup(q); 2076 2077 return 0; 2078 } 2079 2080 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) 2081 { 2082 struct xe_gt *gt = guc_to_gt(guc); 2083 u8 guc_class, instance; 2084 u32 reason; 2085 2086 if (unlikely(len != 3)) 2087 return -EPROTO; 2088 2089 guc_class = msg[0]; 2090 instance = msg[1]; 2091 reason = msg[2]; 2092 2093 /* Unexpected failure of a hardware feature, log an actual error */ 2094 xe_gt_err(gt, "GuC engine reset request failed on %d:%d because 0x%08X", 2095 guc_class, instance, reason); 2096 2097 xe_gt_reset_async(gt); 2098 2099 return 0; 2100 } 2101 2102 static void 2103 guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q, 2104 struct xe_guc_submit_exec_queue_snapshot *snapshot) 2105 { 2106 struct xe_guc *guc = exec_queue_to_guc(q); 2107 struct xe_device *xe = guc_to_xe(guc); 2108 struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); 2109 int i; 2110 2111 snapshot->guc.wqi_head = q->guc->wqi_head; 2112 snapshot->guc.wqi_tail = q->guc->wqi_tail; 2113 snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head); 2114 snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail); 2115 snapshot->parallel.wq_desc.status = parallel_read(xe, map, 2116 wq_desc.wq_status); 2117 2118 if (snapshot->parallel.wq_desc.head != 2119 snapshot->parallel.wq_desc.tail) { 2120 for (i = snapshot->parallel.wq_desc.head; 2121 i != snapshot->parallel.wq_desc.tail; 2122 i = (i + sizeof(u32)) % WQ_SIZE) 2123 snapshot->parallel.wq[i / sizeof(u32)] = 2124 parallel_read(xe, map, wq[i / sizeof(u32)]); 2125 } 2126 } 2127 2128 static void 2129 guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, 2130 struct drm_printer *p) 2131 { 2132 int i; 2133 2134 drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n", 2135 snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head); 2136 drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n", 2137 snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail); 2138 drm_printf(p, "\tWQ status: %u\n", snapshot->parallel.wq_desc.status); 2139 2140 if (snapshot->parallel.wq_desc.head != 2141 snapshot->parallel.wq_desc.tail) { 2142 for (i = snapshot->parallel.wq_desc.head; 2143 i != snapshot->parallel.wq_desc.tail; 2144 i = (i + sizeof(u32)) % WQ_SIZE) 2145 drm_printf(p, "\tWQ[%zu]: 0x%08x\n", i / sizeof(u32), 2146 snapshot->parallel.wq[i / sizeof(u32)]); 2147 } 2148 } 2149 2150 /** 2151 * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine. 2152 * @q: faulty exec queue 2153 * 2154 * This can be printed out in a later stage like during dev_coredump 2155 * analysis. 2156 * 2157 * Returns: a GuC Submit Engine snapshot object that must be freed by the 2158 * caller, using `xe_guc_exec_queue_snapshot_free`. 2159 */ 2160 struct xe_guc_submit_exec_queue_snapshot * 2161 xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) 2162 { 2163 struct xe_gpu_scheduler *sched = &q->guc->sched; 2164 struct xe_guc_submit_exec_queue_snapshot *snapshot; 2165 int i; 2166 2167 snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC); 2168 2169 if (!snapshot) 2170 return NULL; 2171 2172 snapshot->guc.id = q->guc->id; 2173 memcpy(&snapshot->name, &q->name, sizeof(snapshot->name)); 2174 snapshot->class = q->class; 2175 snapshot->logical_mask = q->logical_mask; 2176 snapshot->width = q->width; 2177 snapshot->refcount = kref_read(&q->refcount); 2178 snapshot->sched_timeout = sched->base.timeout; 2179 snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us; 2180 snapshot->sched_props.preempt_timeout_us = 2181 q->sched_props.preempt_timeout_us; 2182 2183 snapshot->lrc = kmalloc_array(q->width, sizeof(struct xe_lrc_snapshot *), 2184 GFP_ATOMIC); 2185 2186 if (snapshot->lrc) { 2187 for (i = 0; i < q->width; ++i) { 2188 struct xe_lrc *lrc = q->lrc[i]; 2189 2190 snapshot->lrc[i] = xe_lrc_snapshot_capture(lrc); 2191 } 2192 } 2193 2194 snapshot->schedule_state = atomic_read(&q->guc->state); 2195 snapshot->exec_queue_flags = q->flags; 2196 2197 snapshot->parallel_execution = xe_exec_queue_is_parallel(q); 2198 if (snapshot->parallel_execution) 2199 guc_exec_queue_wq_snapshot_capture(q, snapshot); 2200 2201 spin_lock(&sched->base.job_list_lock); 2202 snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list); 2203 snapshot->pending_list = kmalloc_array(snapshot->pending_list_size, 2204 sizeof(struct pending_list_snapshot), 2205 GFP_ATOMIC); 2206 2207 if (snapshot->pending_list) { 2208 struct xe_sched_job *job_iter; 2209 2210 i = 0; 2211 list_for_each_entry(job_iter, &sched->base.pending_list, drm.list) { 2212 snapshot->pending_list[i].seqno = 2213 xe_sched_job_seqno(job_iter); 2214 snapshot->pending_list[i].fence = 2215 dma_fence_is_signaled(job_iter->fence) ? 1 : 0; 2216 snapshot->pending_list[i].finished = 2217 dma_fence_is_signaled(&job_iter->drm.s_fence->finished) 2218 ? 1 : 0; 2219 i++; 2220 } 2221 } 2222 2223 spin_unlock(&sched->base.job_list_lock); 2224 2225 return snapshot; 2226 } 2227 2228 /** 2229 * xe_guc_exec_queue_snapshot_capture_delayed - Take delayed part of snapshot of the GuC Engine. 2230 * @snapshot: Previously captured snapshot of job. 2231 * 2232 * This captures some data that requires taking some locks, so it cannot be done in signaling path. 2233 */ 2234 void 2235 xe_guc_exec_queue_snapshot_capture_delayed(struct xe_guc_submit_exec_queue_snapshot *snapshot) 2236 { 2237 int i; 2238 2239 if (!snapshot || !snapshot->lrc) 2240 return; 2241 2242 for (i = 0; i < snapshot->width; ++i) 2243 xe_lrc_snapshot_capture_delayed(snapshot->lrc[i]); 2244 } 2245 2246 /** 2247 * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot. 2248 * @snapshot: GuC Submit Engine snapshot object. 2249 * @p: drm_printer where it will be printed out. 2250 * 2251 * This function prints out a given GuC Submit Engine snapshot object. 2252 */ 2253 void 2254 xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, 2255 struct drm_printer *p) 2256 { 2257 int i; 2258 2259 if (!snapshot) 2260 return; 2261 2262 drm_printf(p, "GuC ID: %d\n", snapshot->guc.id); 2263 drm_printf(p, "\tName: %s\n", snapshot->name); 2264 drm_printf(p, "\tClass: %d\n", snapshot->class); 2265 drm_printf(p, "\tLogical mask: 0x%x\n", snapshot->logical_mask); 2266 drm_printf(p, "\tWidth: %d\n", snapshot->width); 2267 drm_printf(p, "\tRef: %d\n", snapshot->refcount); 2268 drm_printf(p, "\tTimeout: %ld (ms)\n", snapshot->sched_timeout); 2269 drm_printf(p, "\tTimeslice: %u (us)\n", 2270 snapshot->sched_props.timeslice_us); 2271 drm_printf(p, "\tPreempt timeout: %u (us)\n", 2272 snapshot->sched_props.preempt_timeout_us); 2273 2274 for (i = 0; snapshot->lrc && i < snapshot->width; ++i) 2275 xe_lrc_snapshot_print(snapshot->lrc[i], p); 2276 2277 drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state); 2278 drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags); 2279 2280 if (snapshot->parallel_execution) 2281 guc_exec_queue_wq_snapshot_print(snapshot, p); 2282 2283 for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size; 2284 i++) 2285 drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n", 2286 snapshot->pending_list[i].seqno, 2287 snapshot->pending_list[i].fence, 2288 snapshot->pending_list[i].finished); 2289 } 2290 2291 /** 2292 * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given 2293 * snapshot. 2294 * @snapshot: GuC Submit Engine snapshot object. 2295 * 2296 * This function free all the memory that needed to be allocated at capture 2297 * time. 2298 */ 2299 void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot) 2300 { 2301 int i; 2302 2303 if (!snapshot) 2304 return; 2305 2306 if (snapshot->lrc) { 2307 for (i = 0; i < snapshot->width; i++) 2308 xe_lrc_snapshot_free(snapshot->lrc[i]); 2309 kfree(snapshot->lrc); 2310 } 2311 kfree(snapshot->pending_list); 2312 kfree(snapshot); 2313 } 2314 2315 static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p) 2316 { 2317 struct xe_guc_submit_exec_queue_snapshot *snapshot; 2318 2319 snapshot = xe_guc_exec_queue_snapshot_capture(q); 2320 xe_guc_exec_queue_snapshot_print(snapshot, p); 2321 xe_guc_exec_queue_snapshot_free(snapshot); 2322 } 2323 2324 /** 2325 * xe_guc_submit_print - GuC Submit Print. 2326 * @guc: GuC. 2327 * @p: drm_printer where it will be printed out. 2328 * 2329 * This function capture and prints snapshots of **all** GuC Engines. 2330 */ 2331 void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) 2332 { 2333 struct xe_exec_queue *q; 2334 unsigned long index; 2335 2336 if (!xe_device_uc_enabled(guc_to_xe(guc))) 2337 return; 2338 2339 mutex_lock(&guc->submission_state.lock); 2340 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) 2341 guc_exec_queue_print(q, p); 2342 mutex_unlock(&guc->submission_state.lock); 2343 } 2344