1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ct.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 #include <linux/delay.h> 11 #include <linux/fault-inject.h> 12 13 #include <kunit/static_stub.h> 14 15 #include <drm/drm_managed.h> 16 17 #include "abi/guc_actions_abi.h" 18 #include "abi/guc_actions_sriov_abi.h" 19 #include "abi/guc_klvs_abi.h" 20 #include "xe_bo.h" 21 #include "xe_devcoredump.h" 22 #include "xe_device.h" 23 #include "xe_gt.h" 24 #include "xe_gt_printk.h" 25 #include "xe_gt_sriov_pf_control.h" 26 #include "xe_gt_sriov_pf_monitor.h" 27 #include "xe_guc.h" 28 #include "xe_guc_log.h" 29 #include "xe_guc_pagefault.h" 30 #include "xe_guc_relay.h" 31 #include "xe_guc_submit.h" 32 #include "xe_guc_tlb_inval.h" 33 #include "xe_map.h" 34 #include "xe_page_reclaim.h" 35 #include "xe_pm.h" 36 #include "xe_sleep.h" 37 #include "xe_sriov_vf.h" 38 #include "xe_trace_guc.h" 39 40 static void receive_g2h(struct xe_guc_ct *ct); 41 static void g2h_worker_func(struct work_struct *w); 42 static void safe_mode_worker_func(struct work_struct *w); 43 static void ct_exit_safe_mode(struct xe_guc_ct *ct); 44 static void guc_ct_change_state(struct xe_guc_ct *ct, 45 enum xe_guc_ct_state state); 46 47 static struct xe_guc *ct_to_guc(struct xe_guc_ct *ct) 48 { 49 return container_of(ct, struct xe_guc, ct); 50 } 51 52 static struct xe_gt *ct_to_gt(struct xe_guc_ct *ct) 53 { 54 return container_of(ct, struct xe_gt, uc.guc.ct); 55 } 56 57 static struct xe_device *ct_to_xe(struct xe_guc_ct *ct) 58 { 59 return gt_to_xe(ct_to_gt(ct)); 60 } 61 62 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 63 enum { 64 /* Internal states, not error conditions */ 65 CT_DEAD_STATE_REARM, /* 0x0001 */ 66 CT_DEAD_STATE_CAPTURE, /* 0x0002 */ 67 68 /* Error conditions */ 69 CT_DEAD_SETUP, /* 0x0004 */ 70 CT_DEAD_H2G_WRITE, /* 0x0008 */ 71 CT_DEAD_H2G_HAS_ROOM, /* 0x0010 */ 72 CT_DEAD_G2H_READ, /* 0x0020 */ 73 CT_DEAD_G2H_RECV, /* 0x0040 */ 74 CT_DEAD_G2H_RELEASE, /* 0x0080 */ 75 CT_DEAD_DEADLOCK, /* 0x0100 */ 76 CT_DEAD_PROCESS_FAILED, /* 0x0200 */ 77 CT_DEAD_FAST_G2H, /* 0x0400 */ 78 CT_DEAD_PARSE_G2H_RESPONSE, /* 0x0800 */ 79 CT_DEAD_PARSE_G2H_UNKNOWN, /* 0x1000 */ 80 CT_DEAD_PARSE_G2H_ORIGIN, /* 0x2000 */ 81 CT_DEAD_PARSE_G2H_TYPE, /* 0x4000 */ 82 CT_DEAD_CRASH, /* 0x8000 */ 83 }; 84 85 static void ct_dead_worker_func(struct work_struct *w); 86 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code); 87 88 static void ct_dead_fini(struct xe_guc_ct *ct) 89 { 90 cancel_work_sync(&ct->dead.worker); 91 } 92 93 static void ct_dead_init(struct xe_guc_ct *ct) 94 { 95 spin_lock_init(&ct->dead.lock); 96 INIT_WORK(&ct->dead.worker, ct_dead_worker_func); 97 98 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 99 stack_depot_init(); 100 #endif 101 } 102 103 static void fast_req_stack_save(struct xe_guc_ct *ct, unsigned int slot) 104 { 105 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 106 unsigned long entries[SZ_32]; 107 unsigned int n; 108 109 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 110 /* May be called under spinlock, so avoid sleeping */ 111 ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT); 112 #endif 113 } 114 115 static void fast_req_dump(struct xe_guc_ct *ct, u16 fence, unsigned int slot) 116 { 117 struct xe_gt *gt = ct_to_gt(ct); 118 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 119 char *buf __cleanup(kfree) = kmalloc(SZ_4K, GFP_NOWAIT); 120 121 if (buf && stack_depot_snprint(ct->fast_req[slot].stack, buf, SZ_4K, 0)) 122 xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s\n", 123 fence, ct->fast_req[slot].action, buf); 124 else 125 xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n", 126 fence, ct->fast_req[slot].action); 127 #else 128 xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n", 129 fence, ct->fast_req[slot].action); 130 #endif 131 } 132 133 static void fast_req_report(struct xe_guc_ct *ct, u16 fence) 134 { 135 u16 fence_min = U16_MAX, fence_max = 0; 136 struct xe_gt *gt = ct_to_gt(ct); 137 unsigned int n; 138 139 lockdep_assert_held(&ct->lock); 140 141 for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) { 142 if (ct->fast_req[n].fence < fence_min) 143 fence_min = ct->fast_req[n].fence; 144 if (ct->fast_req[n].fence > fence_max) 145 fence_max = ct->fast_req[n].fence; 146 147 if (ct->fast_req[n].fence != fence) 148 continue; 149 150 return fast_req_dump(ct, fence, n); 151 } 152 153 xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n", 154 fence, fence_min, fence_max, ct->fence_seqno); 155 } 156 157 static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) 158 { 159 unsigned int slot = fence % ARRAY_SIZE(ct->fast_req); 160 161 fast_req_stack_save(ct, slot); 162 ct->fast_req[slot].fence = fence; 163 ct->fast_req[slot].action = action; 164 } 165 166 #define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code) 167 168 #else 169 170 static void ct_dead_fini(struct xe_guc_ct *ct) { } 171 static void ct_dead_init(struct xe_guc_ct *ct) { } 172 173 static void fast_req_report(struct xe_guc_ct *ct, u16 fence) { } 174 static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { } 175 176 #define CT_DEAD(ct, ctb, reason) \ 177 do { \ 178 struct guc_ctb *_ctb = (ctb); \ 179 if (_ctb) \ 180 _ctb->info.broken = true; \ 181 } while (0) 182 183 #endif 184 185 /* Used when a CT send wants to block and / or receive data */ 186 struct g2h_fence { 187 u32 *response_buffer; 188 u32 seqno; 189 /* fields below this point are setup based on the response */ 190 u32 response_data; 191 u16 response_len; 192 u16 error; 193 u16 hint; 194 u16 reason; 195 u32 counter; 196 bool cancel; 197 bool retry; 198 bool wait; 199 bool fail; 200 bool done; 201 }; 202 203 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) 204 { 205 memset(g2h_fence, 0, sizeof(*g2h_fence)); 206 g2h_fence->response_buffer = response_buffer; 207 g2h_fence->seqno = ~0x0; 208 } 209 210 static void g2h_fence_reinit(struct g2h_fence *g2h_fence) 211 { 212 memset_after(g2h_fence, 0, seqno); 213 } 214 215 static void g2h_fence_cancel(struct g2h_fence *g2h_fence) 216 { 217 g2h_fence->cancel = true; 218 g2h_fence->fail = true; 219 220 /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ 221 WRITE_ONCE(g2h_fence->done, true); 222 } 223 224 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) 225 { 226 return g2h_fence->seqno == ~0x0; 227 } 228 229 /** 230 * DOC: GuC CTB Blob 231 * 232 * We allocate single blob to hold both CTB descriptors and buffers: 233 * 234 * +--------+-----------------------------------------------+------+ 235 * | offset | contents | size | 236 * +========+===============================================+======+ 237 * | 0x0000 | H2G CTB Descriptor (send) | | 238 * +--------+-----------------------------------------------+ 4K | 239 * | 0x0800 | G2H CTB Descriptor (g2h) | | 240 * +--------+-----------------------------------------------+------+ 241 * | 0x1000 | H2G CT Buffer (send) | n*4K | 242 * | | | | 243 * +--------+-----------------------------------------------+------+ 244 * | 0x1000 | G2H CT Buffer (g2h) | m*4K | 245 * | + n*4K | | | 246 * +--------+-----------------------------------------------+------+ 247 * 248 * Size of each ``CT Buffer`` must be multiple of 4K. 249 * We don't expect too many messages in flight at any time, unless we are 250 * using the GuC submission. In that case each request requires a minimum 251 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 252 * enough space to avoid backpressure on the driver. We increase the size 253 * of the receive buffer (relative to the send) to ensure a G2H response 254 * CTB has a landing spot. 255 * 256 * In addition to submissions, the G2H buffer needs to be able to hold 257 * enough space for recoverable page fault notifications. The number of 258 * page faults is interrupt driven and can be as much as the number of 259 * compute resources available. However, most of the actual work for these 260 * is in a separate page fault worker thread. Therefore we only need to 261 * make sure the queue has enough space to handle all of the submissions 262 * and responses and an extra buffer for incoming page faults. 263 */ 264 265 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 266 #define CTB_H2G_BUFFER_OFFSET (CTB_DESC_SIZE * 2) 267 #define CTB_G2H_BUFFER_OFFSET (CTB_DESC_SIZE * 2) 268 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 269 #define CTB_H2G_BUFFER_DWORDS (CTB_H2G_BUFFER_SIZE / sizeof(u32)) 270 #define CTB_G2H_BUFFER_SIZE (SZ_128K) 271 #define CTB_G2H_BUFFER_DWORDS (CTB_G2H_BUFFER_SIZE / sizeof(u32)) 272 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2) 273 #define G2H_ROOM_BUFFER_DWORDS (CTB_G2H_BUFFER_DWORDS / 2) 274 275 /** 276 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full 277 * CT command queue 278 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future. 279 * 280 * Observation is that a 4KiB buffer full of commands takes a little over a 281 * second to process. Use that to calculate maximum time to process a full CT 282 * command queue. 283 * 284 * Return: Maximum time to process a full CT queue in jiffies. 285 */ 286 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct) 287 { 288 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4K)); 289 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ; 290 } 291 292 static size_t guc_h2g_size(void) 293 { 294 return CTB_H2G_BUFFER_OFFSET + CTB_H2G_BUFFER_SIZE; 295 } 296 297 static size_t guc_g2h_size(void) 298 { 299 return CTB_G2H_BUFFER_OFFSET + CTB_G2H_BUFFER_SIZE; 300 } 301 302 static void guc_ct_fini(struct drm_device *drm, void *arg) 303 { 304 struct xe_guc_ct *ct = arg; 305 306 ct_dead_fini(ct); 307 ct_exit_safe_mode(ct); 308 destroy_workqueue(ct->g2h_wq); 309 xa_destroy(&ct->fence_lookup); 310 } 311 312 static void primelockdep(struct xe_guc_ct *ct) 313 { 314 if (!IS_ENABLED(CONFIG_LOCKDEP)) 315 return; 316 317 fs_reclaim_acquire(GFP_KERNEL); 318 might_lock(&ct->lock); 319 fs_reclaim_release(GFP_KERNEL); 320 } 321 322 int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) 323 { 324 struct xe_device *xe = ct_to_xe(ct); 325 struct xe_gt *gt = ct_to_gt(ct); 326 int err; 327 328 xe_gt_assert(gt, !(guc_h2g_size() % PAGE_SIZE)); 329 xe_gt_assert(gt, !(guc_g2h_size() % PAGE_SIZE)); 330 331 err = drmm_mutex_init(&xe->drm, &ct->lock); 332 if (err) 333 return err; 334 335 primelockdep(ct); 336 337 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM); 338 if (!ct->g2h_wq) 339 return -ENOMEM; 340 341 spin_lock_init(&ct->fast_lock); 342 xa_init(&ct->fence_lookup); 343 INIT_WORK(&ct->g2h_worker, g2h_worker_func); 344 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func); 345 346 ct_dead_init(ct); 347 init_waitqueue_head(&ct->wq); 348 init_waitqueue_head(&ct->g2h_fence_wq); 349 350 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); 351 if (err) 352 return err; 353 354 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED); 355 ct->state = XE_GUC_CT_STATE_DISABLED; 356 return 0; 357 } 358 ALLOW_ERROR_INJECTION(xe_guc_ct_init_noalloc, ERRNO); /* See xe_pci_probe() */ 359 360 static void guc_action_disable_ct(void *arg) 361 { 362 struct xe_guc_ct *ct = arg; 363 364 xe_guc_ct_stop(ct); 365 guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); 366 } 367 368 int xe_guc_ct_init(struct xe_guc_ct *ct) 369 { 370 struct xe_device *xe = ct_to_xe(ct); 371 struct xe_gt *gt = ct_to_gt(ct); 372 struct xe_tile *tile = gt_to_tile(gt); 373 struct xe_bo *bo; 374 375 bo = xe_managed_bo_create_pin_map(xe, tile, guc_h2g_size(), 376 XE_BO_FLAG_SYSTEM | 377 XE_BO_FLAG_GGTT | 378 XE_BO_FLAG_GGTT_INVALIDATE | 379 XE_BO_FLAG_PINNED_NORESTORE); 380 if (IS_ERR(bo)) 381 return PTR_ERR(bo); 382 383 ct->ctbs.h2g.bo = bo; 384 385 bo = xe_managed_bo_create_pin_map(xe, tile, guc_g2h_size(), 386 XE_BO_FLAG_SYSTEM | 387 XE_BO_FLAG_GGTT | 388 XE_BO_FLAG_GGTT_INVALIDATE | 389 XE_BO_FLAG_PINNED_NORESTORE); 390 if (IS_ERR(bo)) 391 return PTR_ERR(bo); 392 393 ct->ctbs.g2h.bo = bo; 394 395 return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct); 396 } 397 ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */ 398 399 /** 400 * xe_guc_ct_init_post_hwconfig - Reinitialize the GuC CTB in VRAM 401 * @ct: the &xe_guc_ct 402 * 403 * Allocate a new BO in VRAM and free the previous BO that was allocated 404 * in system memory (SMEM). Applicable only for DGFX products. 405 * 406 * Return: 0 on success, or a negative errno on failure. 407 */ 408 int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct) 409 { 410 struct xe_device *xe = ct_to_xe(ct); 411 struct xe_gt *gt = ct_to_gt(ct); 412 struct xe_tile *tile = gt_to_tile(gt); 413 int ret; 414 415 xe_assert(xe, !xe_guc_ct_enabled(ct)); 416 417 if (IS_DGFX(xe)) { 418 ret = xe_managed_bo_reinit_in_vram(xe, tile, &ct->ctbs.h2g.bo); 419 if (ret) 420 return ret; 421 } 422 423 devm_remove_action(xe->drm.dev, guc_action_disable_ct, ct); 424 return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct); 425 } 426 427 #define desc_read(xe_, guc_ctb__, field_) \ 428 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \ 429 struct guc_ct_buffer_desc, field_) 430 431 #define desc_write(xe_, guc_ctb__, field_, val_) \ 432 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \ 433 struct guc_ct_buffer_desc, field_, val_) 434 435 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, 436 struct iosys_map *map) 437 { 438 h2g->info.size = CTB_H2G_BUFFER_DWORDS; 439 h2g->info.resv_space = 0; 440 h2g->info.tail = 0; 441 h2g->info.head = 0; 442 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 443 h2g->info.size) - 444 h2g->info.resv_space; 445 h2g->info.broken = false; 446 447 h2g->desc = *map; 448 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 449 450 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_H2G_BUFFER_OFFSET); 451 } 452 453 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, 454 struct iosys_map *map) 455 { 456 g2h->info.size = CTB_G2H_BUFFER_DWORDS; 457 g2h->info.resv_space = G2H_ROOM_BUFFER_DWORDS; 458 g2h->info.head = 0; 459 g2h->info.tail = 0; 460 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head, 461 g2h->info.size) - 462 g2h->info.resv_space; 463 g2h->info.broken = false; 464 465 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE); 466 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 467 468 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_G2H_BUFFER_OFFSET); 469 } 470 471 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct) 472 { 473 struct xe_guc *guc = ct_to_guc(ct); 474 u32 desc_addr, ctb_addr, size; 475 int err; 476 477 desc_addr = xe_bo_ggtt_addr(ct->ctbs.h2g.bo); 478 ctb_addr = xe_bo_ggtt_addr(ct->ctbs.h2g.bo) + CTB_H2G_BUFFER_OFFSET; 479 size = ct->ctbs.h2g.info.size * sizeof(u32); 480 481 err = xe_guc_self_cfg64(guc, 482 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY, 483 desc_addr); 484 if (err) 485 return err; 486 487 err = xe_guc_self_cfg64(guc, 488 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY, 489 ctb_addr); 490 if (err) 491 return err; 492 493 return xe_guc_self_cfg32(guc, 494 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY, 495 size); 496 } 497 498 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct) 499 { 500 struct xe_guc *guc = ct_to_guc(ct); 501 u32 desc_addr, ctb_addr, size; 502 int err; 503 504 desc_addr = xe_bo_ggtt_addr(ct->ctbs.g2h.bo) + CTB_DESC_SIZE; 505 ctb_addr = xe_bo_ggtt_addr(ct->ctbs.g2h.bo) + CTB_G2H_BUFFER_OFFSET; 506 size = ct->ctbs.g2h.info.size * sizeof(u32); 507 508 err = xe_guc_self_cfg64(guc, 509 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 510 desc_addr); 511 if (err) 512 return err; 513 514 err = xe_guc_self_cfg64(guc, 515 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 516 ctb_addr); 517 if (err) 518 return err; 519 520 return xe_guc_self_cfg32(guc, 521 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 522 size); 523 } 524 525 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) 526 { 527 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 528 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 529 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 530 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, 531 GUC_ACTION_HOST2GUC_CONTROL_CTB), 532 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, 533 enable ? GUC_CTB_CONTROL_ENABLE : 534 GUC_CTB_CONTROL_DISABLE), 535 }; 536 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request)); 537 538 return ret > 0 ? -EPROTO : ret; 539 } 540 541 static void guc_ct_change_state(struct xe_guc_ct *ct, 542 enum xe_guc_ct_state state) 543 { 544 struct xe_gt *gt = ct_to_gt(ct); 545 struct g2h_fence *g2h_fence; 546 unsigned long idx; 547 548 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ 549 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ 550 551 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 || 552 state == XE_GUC_CT_STATE_STOPPED); 553 554 if (ct->g2h_outstanding) 555 xe_pm_runtime_put(ct_to_xe(ct)); 556 ct->g2h_outstanding = 0; 557 558 /* 559 * WRITE_ONCE pairs with READ_ONCEs in xe_guc_ct_initialized and 560 * xe_guc_ct_enabled. 561 */ 562 WRITE_ONCE(ct->state, state); 563 564 xe_gt_dbg(gt, "GuC CT communication channel %s\n", 565 state == XE_GUC_CT_STATE_STOPPED ? "stopped" : 566 str_enabled_disabled(state == XE_GUC_CT_STATE_ENABLED)); 567 568 spin_unlock_irq(&ct->fast_lock); 569 570 /* cancel all in-flight send-recv requests */ 571 xa_for_each(&ct->fence_lookup, idx, g2h_fence) 572 g2h_fence_cancel(g2h_fence); 573 574 /* make sure guc_ct_send_recv() will see g2h_fence changes */ 575 smp_mb(); 576 wake_up_all(&ct->g2h_fence_wq); 577 578 /* 579 * Lockdep doesn't like this under the fast lock and he destroy only 580 * needs to be serialized with the send path which ct lock provides. 581 */ 582 xa_destroy(&ct->fence_lookup); 583 584 mutex_unlock(&ct->lock); 585 } 586 587 static bool ct_needs_safe_mode(struct xe_guc_ct *ct) 588 { 589 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev)); 590 } 591 592 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct) 593 { 594 if (!ct_needs_safe_mode(ct)) 595 return false; 596 597 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10); 598 return true; 599 } 600 601 static void safe_mode_worker_func(struct work_struct *w) 602 { 603 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work); 604 605 receive_g2h(ct); 606 607 if (!ct_restart_safe_mode_worker(ct)) 608 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n"); 609 } 610 611 static void ct_enter_safe_mode(struct xe_guc_ct *ct) 612 { 613 if (ct_restart_safe_mode_worker(ct)) 614 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n"); 615 } 616 617 static void ct_exit_safe_mode(struct xe_guc_ct *ct) 618 { 619 if (cancel_delayed_work_sync(&ct->safe_mode_worker)) 620 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n"); 621 } 622 623 static int __xe_guc_ct_start(struct xe_guc_ct *ct, bool needs_register) 624 { 625 struct xe_device *xe = ct_to_xe(ct); 626 struct xe_gt *gt = ct_to_gt(ct); 627 int err; 628 629 xe_gt_assert(gt, !xe_guc_ct_enabled(ct)); 630 631 if (needs_register) { 632 xe_map_memset(xe, &ct->ctbs.h2g.bo->vmap, 0, 0, 633 xe_bo_size(ct->ctbs.h2g.bo)); 634 xe_map_memset(xe, &ct->ctbs.g2h.bo->vmap, 0, 0, 635 xe_bo_size(ct->ctbs.g2h.bo)); 636 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->ctbs.h2g.bo->vmap); 637 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->ctbs.g2h.bo->vmap); 638 639 err = guc_ct_ctb_h2g_register(ct); 640 if (err) 641 goto err_out; 642 643 err = guc_ct_ctb_g2h_register(ct); 644 if (err) 645 goto err_out; 646 647 err = guc_ct_control_toggle(ct, true); 648 if (err) 649 goto err_out; 650 } else { 651 ct->ctbs.h2g.info.broken = false; 652 ct->ctbs.g2h.info.broken = false; 653 /* Skip everything in H2G buffer */ 654 xe_map_memset(xe, &ct->ctbs.h2g.bo->vmap, CTB_H2G_BUFFER_OFFSET, 0, 655 CTB_H2G_BUFFER_SIZE); 656 } 657 658 guc_ct_change_state(ct, XE_GUC_CT_STATE_ENABLED); 659 660 smp_mb(); 661 wake_up_all(&ct->wq); 662 663 if (ct_needs_safe_mode(ct)) 664 ct_enter_safe_mode(ct); 665 666 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 667 /* 668 * The CT has now been reset so the dumper can be re-armed 669 * after any existing dead state has been dumped. 670 */ 671 spin_lock_irq(&ct->dead.lock); 672 if (ct->dead.reason) { 673 ct->dead.reason |= (1 << CT_DEAD_STATE_REARM); 674 queue_work(system_dfl_wq, &ct->dead.worker); 675 } 676 spin_unlock_irq(&ct->dead.lock); 677 #endif 678 679 return 0; 680 681 err_out: 682 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err)); 683 CT_DEAD(ct, NULL, SETUP); 684 685 return err; 686 } 687 688 /** 689 * xe_guc_ct_restart() - Restart GuC CT 690 * @ct: the &xe_guc_ct 691 * 692 * Restart GuC CT to an empty state without issuing a CT register MMIO command. 693 * 694 * Return: 0 on success, or a negative errno on failure. 695 */ 696 int xe_guc_ct_restart(struct xe_guc_ct *ct) 697 { 698 return __xe_guc_ct_start(ct, false); 699 } 700 701 /** 702 * xe_guc_ct_enable() - Enable GuC CT 703 * @ct: the &xe_guc_ct 704 * 705 * Enable GuC CT to an empty state and issue a CT register MMIO command. 706 * 707 * Return: 0 on success, or a negative errno on failure. 708 */ 709 int xe_guc_ct_enable(struct xe_guc_ct *ct) 710 { 711 return __xe_guc_ct_start(ct, true); 712 } 713 714 static void stop_g2h_handler(struct xe_guc_ct *ct) 715 { 716 cancel_work_sync(&ct->g2h_worker); 717 } 718 719 /** 720 * xe_guc_ct_disable - Set GuC to disabled state 721 * @ct: the &xe_guc_ct 722 * 723 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected 724 * in this transition. 725 */ 726 void xe_guc_ct_disable(struct xe_guc_ct *ct) 727 { 728 guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); 729 ct_exit_safe_mode(ct); 730 stop_g2h_handler(ct); 731 } 732 733 /** 734 * xe_guc_ct_flush_and_stop - Flush and stop all processing of G2H / H2G 735 * @ct: the &xe_guc_ct 736 */ 737 void xe_guc_ct_flush_and_stop(struct xe_guc_ct *ct) 738 { 739 receive_g2h(ct); 740 xe_guc_ct_stop(ct); 741 } 742 743 /** 744 * xe_guc_ct_stop - Set GuC to stopped state 745 * @ct: the &xe_guc_ct 746 * 747 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h 748 */ 749 void xe_guc_ct_stop(struct xe_guc_ct *ct) 750 { 751 if (!xe_guc_ct_initialized(ct)) 752 return; 753 754 guc_ct_change_state(ct, XE_GUC_CT_STATE_STOPPED); 755 stop_g2h_handler(ct); 756 } 757 758 /** 759 * xe_guc_ct_runtime_suspend() - GuC CT runtime suspend 760 * @ct: the &xe_guc_ct 761 * 762 * Set GuC CT to disabled state. 763 */ 764 void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct) 765 { 766 struct guc_ctb *g2h = &ct->ctbs.g2h; 767 u32 credits = CIRC_SPACE(0, 0, CTB_G2H_BUFFER_DWORDS) - G2H_ROOM_BUFFER_DWORDS; 768 769 /* We should be back to guc_ct_ctb_g2h_init() values */ 770 xe_gt_assert(ct_to_gt(ct), g2h->info.space == credits); 771 772 /* 773 * Since we're already in runtime suspend path, we shouldn't have pending 774 * messages. But if there happen to be any, we'd probably want them to be 775 * thrown as errors for further investigation. 776 */ 777 xe_guc_ct_disable(ct); 778 } 779 780 /** 781 * xe_guc_ct_runtime_resume() - GuC CT runtime resume 782 * @ct: the &xe_guc_ct 783 * 784 * Restart GuC CT and set it to enabled state. 785 */ 786 void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct) 787 { 788 xe_guc_ct_restart(ct); 789 } 790 791 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) 792 { 793 struct guc_ctb *h2g = &ct->ctbs.h2g; 794 795 lockdep_assert_held(&ct->lock); 796 797 if (cmd_len > h2g->info.space) { 798 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); 799 800 if (h2g->info.head > h2g->info.size) { 801 struct xe_device *xe = ct_to_xe(ct); 802 u32 desc_status = desc_read(xe, h2g, status); 803 804 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 805 806 xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n", 807 h2g->info.head, h2g->info.size); 808 CT_DEAD(ct, h2g, H2G_HAS_ROOM); 809 return false; 810 } 811 812 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 813 h2g->info.size) - 814 h2g->info.resv_space; 815 if (cmd_len > h2g->info.space) 816 return false; 817 } 818 819 return true; 820 } 821 822 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len) 823 { 824 if (!g2h_len) 825 return true; 826 827 lockdep_assert_held(&ct->fast_lock); 828 829 return ct->ctbs.g2h.info.space > g2h_len; 830 } 831 832 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len) 833 { 834 lockdep_assert_held(&ct->lock); 835 836 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len)) 837 return -EBUSY; 838 839 return 0; 840 } 841 842 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len) 843 { 844 lockdep_assert_held(&ct->lock); 845 ct->ctbs.h2g.info.space -= cmd_len; 846 } 847 848 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h) 849 { 850 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space); 851 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) || 852 (g2h_len && num_g2h)); 853 854 if (g2h_len) { 855 lockdep_assert_held(&ct->fast_lock); 856 857 if (!ct->g2h_outstanding) 858 xe_pm_runtime_get_noresume(ct_to_xe(ct)); 859 860 ct->ctbs.g2h.info.space -= g2h_len; 861 ct->g2h_outstanding += num_g2h; 862 } 863 } 864 865 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 866 { 867 bool bad = false; 868 869 lockdep_assert_held(&ct->fast_lock); 870 871 bad = ct->ctbs.g2h.info.space + g2h_len > 872 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space; 873 bad |= !ct->g2h_outstanding; 874 875 if (bad) { 876 xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n", 877 ct->ctbs.g2h.info.space, g2h_len, 878 ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space, 879 ct->ctbs.g2h.info.space + g2h_len, 880 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space, 881 ct->g2h_outstanding); 882 CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE); 883 return; 884 } 885 886 ct->ctbs.g2h.info.space += g2h_len; 887 if (!--ct->g2h_outstanding) 888 xe_pm_runtime_put(ct_to_xe(ct)); 889 } 890 891 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 892 { 893 spin_lock_irq(&ct->fast_lock); 894 __g2h_release_space(ct, g2h_len); 895 spin_unlock_irq(&ct->fast_lock); 896 } 897 898 /* 899 * The CT protocol accepts a 16 bits fence. This field is fully owned by the 900 * driver, the GuC will just copy it to the reply message. Since we need to 901 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages, 902 * we use one bit of the seqno as an indicator for that and a rolling counter 903 * for the remaining 15 bits. 904 */ 905 #define CT_SEQNO_MASK GENMASK(14, 0) 906 #define CT_SEQNO_UNTRACKED BIT(15) 907 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence) 908 { 909 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK; 910 911 if (!is_g2h_fence) 912 seqno |= CT_SEQNO_UNTRACKED; 913 914 return seqno; 915 } 916 917 #define MAKE_ACTION(type, __action) \ 918 ({ \ 919 FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | \ 920 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | \ 921 GUC_HXG_EVENT_MSG_0_DATA0, __action); \ 922 }) 923 924 static bool vf_action_can_safely_fail(struct xe_device *xe, u32 action) 925 { 926 /* 927 * When resuming a VF, we can't reliably track whether context 928 * registration has completed in the GuC state machine. It is harmless 929 * to resend the request, as it will fail silently if GUC_HXG_TYPE_EVENT 930 * is used. Additionally, if there is an H2G protocol issue on a VF, 931 * subsequent H2G messages sent as GUC_HXG_TYPE_FAST_REQUEST will likely 932 * fail. 933 */ 934 return IS_SRIOV_VF(xe) && xe_sriov_vf_migration_supported(xe) && 935 (action == XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC || 936 action == XE_GUC_ACTION_REGISTER_CONTEXT); 937 } 938 939 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */ 940 941 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, 942 u32 ct_fence_value, bool want_response) 943 { 944 struct xe_device *xe = ct_to_xe(ct); 945 struct xe_gt *gt = ct_to_gt(ct); 946 struct guc_ctb *h2g = &ct->ctbs.h2g; 947 u32 cmd[H2G_CT_HEADERS]; 948 u32 tail = h2g->info.tail; 949 u32 full_len; 950 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds, 951 tail * sizeof(u32)); 952 953 full_len = len + GUC_CTB_HDR_LEN; 954 955 lockdep_assert_held(&ct->lock); 956 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN); 957 958 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 959 u32 desc_tail = desc_read(xe, h2g, tail); 960 u32 desc_head = desc_read(xe, h2g, head); 961 u32 desc_status; 962 963 desc_status = desc_read(xe, h2g, status); 964 if (desc_status) { 965 xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status); 966 goto corrupted; 967 } 968 969 if (tail != desc_tail) { 970 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH); 971 xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail); 972 goto corrupted; 973 } 974 975 if (tail > h2g->info.size) { 976 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 977 xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n", 978 tail, h2g->info.size); 979 goto corrupted; 980 } 981 982 if (desc_head >= h2g->info.size) { 983 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 984 xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n", 985 desc_head, h2g->info.size); 986 goto corrupted; 987 } 988 } 989 990 /* Command will wrap, zero fill (NOPs), return and check credits again */ 991 if (tail + full_len > h2g->info.size) { 992 xe_map_memset(xe, &map, 0, 0, 993 (h2g->info.size - tail) * sizeof(u32)); 994 h2g_reserve_space(ct, (h2g->info.size - tail)); 995 h2g->info.tail = 0; 996 desc_write(xe, h2g, tail, h2g->info.tail); 997 998 return -EAGAIN; 999 } 1000 1001 /* 1002 * dw0: CT header (including fence) 1003 * dw1: HXG header (including action code) 1004 * dw2+: action data 1005 */ 1006 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 1007 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 1008 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value); 1009 if (want_response) { 1010 cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_REQUEST, action[0]); 1011 } else if (vf_action_can_safely_fail(xe, action[0])) { 1012 cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_EVENT, action[0]); 1013 } else { 1014 fast_req_track(ct, ct_fence_value, 1015 FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0])); 1016 1017 cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_FAST_REQUEST, action[0]); 1018 } 1019 1020 /* H2G header in cmd[1] replaces action[0] so: */ 1021 --len; 1022 ++action; 1023 1024 /* Write H2G ensuring visible before descriptor update */ 1025 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32)); 1026 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32)); 1027 xe_device_wmb(xe); 1028 1029 /* Update local copies */ 1030 h2g->info.tail = (tail + full_len) % h2g->info.size; 1031 h2g_reserve_space(ct, full_len); 1032 1033 /* Update descriptor */ 1034 desc_write(xe, h2g, tail, h2g->info.tail); 1035 1036 /* 1037 * desc_read() performs an VRAM read which serializes the CPU and drains 1038 * posted writes on dGPU platforms. Tracepoints evaluate arguments even 1039 * when disabled, so guard the event to avoid adding µs-scale latency to 1040 * the fast H2G submission path when tracing is not active. 1041 */ 1042 if (trace_xe_guc_ctb_h2g_enabled()) 1043 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len, 1044 desc_read(xe, h2g, head), h2g->info.tail); 1045 1046 return 0; 1047 1048 corrupted: 1049 CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE); 1050 return -EPIPE; 1051 } 1052 1053 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, 1054 u32 len, u32 g2h_len, u32 num_g2h, 1055 struct g2h_fence *g2h_fence) 1056 { 1057 struct xe_gt *gt = ct_to_gt(ct); 1058 u16 seqno; 1059 int ret; 1060 1061 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 1062 xe_gt_assert(gt, !g2h_len || !g2h_fence); 1063 xe_gt_assert(gt, !num_g2h || !g2h_fence); 1064 xe_gt_assert(gt, !g2h_len || num_g2h); 1065 xe_gt_assert(gt, g2h_len || !num_g2h); 1066 lockdep_assert_held(&ct->lock); 1067 1068 if (xe_device_wedged(ct_to_xe(ct))) { 1069 ret = -ENOTRECOVERABLE; 1070 goto out; 1071 } 1072 1073 if (unlikely(ct->ctbs.h2g.info.broken)) { 1074 ret = -EPIPE; 1075 goto out; 1076 } 1077 1078 if (ct->state == XE_GUC_CT_STATE_DISABLED) { 1079 ret = -ENODEV; 1080 goto out; 1081 } 1082 1083 if (ct->state == XE_GUC_CT_STATE_STOPPED || xe_gt_recovery_pending(gt)) { 1084 ret = -ECANCELED; 1085 goto out; 1086 } 1087 1088 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1089 1090 if (g2h_fence) { 1091 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; 1092 num_g2h = 1; 1093 1094 if (g2h_fence_needs_alloc(g2h_fence)) { 1095 g2h_fence->seqno = next_ct_seqno(ct, true); 1096 ret = xa_err(xa_store(&ct->fence_lookup, 1097 g2h_fence->seqno, g2h_fence, 1098 GFP_ATOMIC)); 1099 if (ret) 1100 goto out; 1101 } 1102 1103 seqno = g2h_fence->seqno; 1104 } else { 1105 seqno = next_ct_seqno(ct, false); 1106 } 1107 1108 if (g2h_len) 1109 spin_lock_irq(&ct->fast_lock); 1110 retry: 1111 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len); 1112 if (unlikely(ret)) 1113 goto out_unlock; 1114 1115 ret = h2g_write(ct, action, len, seqno, !!g2h_fence); 1116 if (unlikely(ret)) { 1117 if (ret == -EAGAIN) 1118 goto retry; 1119 goto out_unlock; 1120 } 1121 1122 __g2h_reserve_space(ct, g2h_len, num_g2h); 1123 xe_guc_notify(ct_to_guc(ct)); 1124 out_unlock: 1125 if (g2h_len) 1126 spin_unlock_irq(&ct->fast_lock); 1127 out: 1128 return ret; 1129 } 1130 1131 static void kick_reset(struct xe_guc_ct *ct) 1132 { 1133 xe_gt_reset_async(ct_to_gt(ct)); 1134 } 1135 1136 static int dequeue_one_g2h(struct xe_guc_ct *ct); 1137 1138 /* 1139 * wait before retry of sending h2g message 1140 * Return: true if ready for retry, false if the wait timeouted 1141 */ 1142 static bool guc_ct_send_wait_for_retry(struct xe_guc_ct *ct, u32 len, 1143 u32 g2h_len, struct g2h_fence *g2h_fence, 1144 unsigned int *sleep_period_ms, 1145 unsigned int *sleep_total_ms) 1146 { 1147 struct xe_device *xe = ct_to_xe(ct); 1148 1149 /* 1150 * We wait to try to restore credits for about 1 second before bailing. 1151 * In the case of H2G credits we have no choice but just to wait for the 1152 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In 1153 * the case of G2H we process any G2H in the channel, hopefully freeing 1154 * credits as we consume the G2H messages. 1155 */ 1156 if (!h2g_has_room(ct, len + GUC_CTB_HDR_LEN)) { 1157 struct guc_ctb *h2g = &ct->ctbs.h2g; 1158 1159 if (*sleep_total_ms > 1000) 1160 return false; 1161 1162 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail, 1163 h2g->info.size, 1164 h2g->info.space, 1165 len + GUC_CTB_HDR_LEN); 1166 *sleep_total_ms += xe_sleep_exponential_ms(sleep_period_ms, 64); 1167 } else { 1168 struct guc_ctb *g2h = &ct->ctbs.g2h; 1169 int ret; 1170 1171 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head, 1172 desc_read(xe, g2h, tail), 1173 g2h->info.size, 1174 g2h->info.space, 1175 g2h_fence ? 1176 GUC_CTB_HXG_MSG_MAX_LEN : 1177 g2h_len); 1178 1179 #define g2h_avail(ct) \ 1180 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head) 1181 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding || 1182 g2h_avail(ct), HZ)) 1183 return false; 1184 #undef g2h_avail 1185 1186 ret = dequeue_one_g2h(ct); 1187 if (ret < 0) { 1188 if (ret != -ECANCELED) 1189 xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)\n", 1190 ERR_PTR(ret)); 1191 return false; 1192 } 1193 } 1194 return true; 1195 } 1196 1197 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 1198 u32 g2h_len, u32 num_g2h, 1199 struct g2h_fence *g2h_fence) 1200 { 1201 struct xe_gt *gt = ct_to_gt(ct); 1202 unsigned int sleep_period_ms = 1; 1203 unsigned int sleep_total_ms = 0; 1204 int ret; 1205 1206 xe_gt_assert(gt, !g2h_len || !g2h_fence); 1207 lockdep_assert_held(&ct->lock); 1208 xe_device_assert_mem_access(ct_to_xe(ct)); 1209 1210 try_again: 1211 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, 1212 g2h_fence); 1213 1214 if (unlikely(ret == -EBUSY)) { 1215 if (!guc_ct_send_wait_for_retry(ct, len, g2h_len, g2h_fence, 1216 &sleep_period_ms, &sleep_total_ms)) 1217 goto broken; 1218 goto try_again; 1219 } 1220 1221 return ret; 1222 1223 broken: 1224 xe_gt_err(gt, "No forward process on H2G, reset required\n"); 1225 CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK); 1226 1227 return -EDEADLK; 1228 } 1229 1230 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 1231 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) 1232 { 1233 int ret; 1234 1235 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence); 1236 1237 mutex_lock(&ct->lock); 1238 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence); 1239 mutex_unlock(&ct->lock); 1240 1241 return ret; 1242 } 1243 1244 /** 1245 * xe_guc_ct_send - Send an HXG message to the GuC over CT 1246 * @ct: the &xe_guc_ct 1247 * @action: dword array with the HXG message (can't be NULL) 1248 * @len: length of the HXG message in dwords (can't be 0) 1249 * @g2h_len: G2H response space to reserve in dwords, or 0 1250 * @num_g2h: number of G2H messages expected, or 0 1251 * 1252 * Return codes from the non-blocking send helpers are: 1253 * 1254 * * -ENOTRECOVERABLE: the xe device is wedged. Stop submitting new GuC work; the 1255 * request cannot make progress until the device is recovered. 1256 * * -EPIPE: the H2G CTB is marked broken. The channel stays unusable until the 1257 * CT is restarted, which clears the broken flag. 1258 * * -ENODEV: the CT channel is disabled, messages not expected in this state. 1259 * Don't retry until it is enabled again. 1260 * * -ECANCELED: the CT channel is stopped or a GT recovery is pending; the 1261 * message was dropped. Often benign. Cancel-tolerant callers (e.g. TLB 1262 * invalidations, GuC submission) rely on the stop/start flow to recover; 1263 * others should retry once the CT is re-enabled or the reset/recovery 1264 * completes. 1265 * * -EDEADLK: no CTB room and the wait for space timed out. The send helpers 1266 * have already requested an async GT reset before returning this error. 1267 * 1268 * -ENOMEM may also be returned if an internal allocation fails; the blocking 1269 * xe_guc_ct_send_recv() path retries that allocation. -EBUSY and 1270 * -EAGAIN are internal flow-control results handled by the send helpers. 1271 * 1272 * Return: 0 on success, or a negative error code on failure. 1273 */ 1274 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 1275 u32 g2h_len, u32 num_g2h) 1276 { 1277 int ret; 1278 1279 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL); 1280 if (ret == -EDEADLK) 1281 kick_reset(ct); 1282 1283 return ret; 1284 } 1285 1286 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 1287 u32 g2h_len, u32 num_g2h) 1288 { 1289 int ret; 1290 1291 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL); 1292 if (ret == -EDEADLK) 1293 kick_reset(ct); 1294 1295 return ret; 1296 } 1297 1298 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len) 1299 { 1300 int ret; 1301 1302 lockdep_assert_held(&ct->lock); 1303 1304 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL); 1305 if (ret == -EDEADLK) 1306 kick_reset(ct); 1307 1308 return ret; 1309 } 1310 1311 /* 1312 * Check if a GT reset is in progress or will occur and if GT reset brought the 1313 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset. 1314 */ 1315 static bool retry_failure(struct xe_guc_ct *ct, int ret) 1316 { 1317 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV)) 1318 return false; 1319 1320 #define ct_alive(ct) \ 1321 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \ 1322 !ct->ctbs.g2h.info.broken) 1323 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) 1324 return false; 1325 #undef ct_alive 1326 1327 return true; 1328 } 1329 1330 #define GUC_SEND_RETRY_LIMIT 50 1331 #define GUC_SEND_RETRY_MSLEEP 5 1332 1333 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1334 u32 *response_buffer, bool no_fail) 1335 { 1336 struct xe_gt *gt = ct_to_gt(ct); 1337 struct g2h_fence g2h_fence; 1338 unsigned int retries = 0; 1339 int ret = 0; 1340 1341 /* 1342 * We use a fence to implement blocking sends / receiving response data. 1343 * The seqno of the fence is sent in the H2G, returned in the G2H, and 1344 * an xarray is used as storage media with the seqno being to key. 1345 * Fields in the fence hold success, failure, retry status and the 1346 * response data. Safe to allocate on the stack as the xarray is the 1347 * only reference and it cannot be present after this function exits. 1348 */ 1349 retry: 1350 g2h_fence_init(&g2h_fence, response_buffer); 1351 retry_same_fence: 1352 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence); 1353 if (unlikely(ret == -ENOMEM)) { 1354 /* Retry allocation /w GFP_KERNEL */ 1355 ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno, 1356 &g2h_fence, GFP_KERNEL)); 1357 if (ret) 1358 return ret; 1359 1360 goto retry_same_fence; 1361 } else if (unlikely(ret)) { 1362 if (ret == -EDEADLK) 1363 kick_reset(ct); 1364 1365 if (no_fail && retry_failure(ct, ret)) 1366 goto retry_same_fence; 1367 1368 if (!g2h_fence_needs_alloc(&g2h_fence)) 1369 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1370 1371 return ret; 1372 } 1373 1374 /* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response 1375 * and g2h_fence_cancel. 1376 */ 1377 wait_again: 1378 ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ); 1379 if (!ret) { 1380 LNL_FLUSH_WORK(&ct->g2h_worker); 1381 if (READ_ONCE(g2h_fence.done)) { 1382 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", 1383 g2h_fence.seqno, action[0]); 1384 ret = 1; 1385 } 1386 } 1387 1388 /* 1389 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on 1390 * the stack, since we have no clue if it will fire after the timeout before we can erase 1391 * from the xa. Also we have some dependent loads and stores below for which we need the 1392 * correct ordering, and we lack the needed barriers. 1393 */ 1394 mutex_lock(&ct->lock); 1395 if (!ret) { 1396 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s\n", 1397 g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done)); 1398 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1399 mutex_unlock(&ct->lock); 1400 return -ETIME; 1401 } 1402 1403 if (g2h_fence.wait) { 1404 xe_gt_dbg(gt, "H2G action %#x busy: counter %u\n", 1405 action[0], g2h_fence.counter); 1406 /* we can't leave any response data if we want to wait again */ 1407 g2h_fence_reinit(&g2h_fence); 1408 mutex_unlock(&ct->lock); 1409 goto wait_again; 1410 } 1411 if (g2h_fence.retry) { 1412 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", 1413 action[0], g2h_fence.reason); 1414 mutex_unlock(&ct->lock); 1415 if (++retries > GUC_SEND_RETRY_LIMIT) { 1416 xe_gt_err(gt, "H2G action %#x reached retry limit=%u, aborting\n", 1417 action[0], GUC_SEND_RETRY_LIMIT); 1418 return -ELOOP; 1419 } 1420 msleep(GUC_SEND_RETRY_MSLEEP * retries); 1421 goto retry; 1422 } 1423 if (g2h_fence.fail) { 1424 if (g2h_fence.cancel) { 1425 xe_gt_dbg(gt, "H2G request %#x canceled!\n", action[0]); 1426 ret = xe_device_wedged(ct_to_xe(ct)) ? -ENOTRECOVERABLE : -ECANCELED; 1427 goto unlock; 1428 } 1429 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n", 1430 action[0], g2h_fence.error, g2h_fence.hint); 1431 ret = -EIO; 1432 } 1433 1434 if (ret > 0) 1435 ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data; 1436 1437 unlock: 1438 mutex_unlock(&ct->lock); 1439 1440 return ret; 1441 } 1442 1443 /** 1444 * xe_guc_ct_send_recv - Send and receive HXG to the GuC 1445 * @ct: the &xe_guc_ct 1446 * @action: the dword array with `HXG Request`_ message (can't be NULL) 1447 * @len: length of the `HXG Request`_ message (in dwords, can't be 0) 1448 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL) 1449 * 1450 * Send a `HXG Request`_ message to the GuC over CT communication channel and 1451 * blocks until GuC replies with a `HXG Response`_ message. 1452 * 1453 * For non-blocking communication with GuC use xe_guc_ct_send(). 1454 * 1455 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_. 1456 * 1457 * Return: response length (in dwords) if &response_buffer was not NULL, or 1458 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or 1459 * a negative error code on failure. 1460 */ 1461 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1462 u32 *response_buffer) 1463 { 1464 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer); 1465 return guc_ct_send_recv(ct, action, len, response_buffer, false); 1466 } 1467 ALLOW_ERROR_INJECTION(xe_guc_ct_send_recv, ERRNO); 1468 1469 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, 1470 u32 len, u32 *response_buffer) 1471 { 1472 return guc_ct_send_recv(ct, action, len, response_buffer, true); 1473 } 1474 1475 static u32 *msg_to_hxg(u32 *msg) 1476 { 1477 return msg + GUC_CTB_MSG_MIN_LEN; 1478 } 1479 1480 static u32 msg_len_to_hxg_len(u32 len) 1481 { 1482 return len - GUC_CTB_MSG_MIN_LEN; 1483 } 1484 1485 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) 1486 { 1487 u32 *hxg = msg_to_hxg(msg); 1488 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1489 1490 lockdep_assert_held(&ct->lock); 1491 1492 switch (action) { 1493 case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: 1494 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1495 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1496 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1497 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1498 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1499 g2h_release_space(ct, len); 1500 } 1501 1502 return 0; 1503 } 1504 1505 static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) 1506 { 1507 struct xe_gt *gt = ct_to_gt(ct); 1508 1509 if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED) 1510 xe_gt_err(gt, "GuC Crash dump notification\n"); 1511 else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION) 1512 xe_gt_err(gt, "GuC Exception notification\n"); 1513 else 1514 xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action); 1515 1516 CT_DEAD(ct, NULL, CRASH); 1517 1518 kick_reset(ct); 1519 1520 return 0; 1521 } 1522 1523 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) 1524 { 1525 struct xe_gt *gt = ct_to_gt(ct); 1526 u32 *hxg = msg_to_hxg(msg); 1527 u32 hxg_len = msg_len_to_hxg_len(len); 1528 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); 1529 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1530 struct g2h_fence *g2h_fence; 1531 1532 lockdep_assert_held(&ct->lock); 1533 1534 /* 1535 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup. 1536 * Those messages should never fail, so if we do get an error back it 1537 * means we're likely doing an illegal operation and the GuC is 1538 * rejecting it. We have no way to inform the code that submitted the 1539 * H2G that the message was rejected, so we need to escalate the 1540 * failure to trigger a reset. 1541 */ 1542 if (fence & CT_SEQNO_UNTRACKED) { 1543 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) 1544 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n", 1545 fence, 1546 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]), 1547 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0])); 1548 else 1549 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n", 1550 type, fence); 1551 1552 fast_req_report(ct, fence); 1553 1554 /* FIXME: W/A race in the GuC, will get in firmware soon */ 1555 if (xe_gt_recovery_pending(gt)) 1556 return 0; 1557 1558 CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE); 1559 1560 return -EPROTO; 1561 } 1562 1563 /* don't erase as we still expect a final response with the same fence */ 1564 if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) 1565 g2h_fence = xa_load(&ct->fence_lookup, fence); 1566 else 1567 g2h_fence = xa_erase(&ct->fence_lookup, fence); 1568 1569 if (unlikely(!g2h_fence)) { 1570 /* Don't tear down channel, as send could've timed out */ 1571 /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ 1572 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence); 1573 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1574 return 0; 1575 } 1576 1577 xe_gt_assert(gt, fence == g2h_fence->seqno); 1578 1579 /* 1580 * reinit as we might have already process this g2h_fence before 1581 * if we received a NO_RESPONSE_BUSY reply 1582 */ 1583 g2h_fence_reinit(g2h_fence); 1584 1585 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { 1586 g2h_fence->fail = true; 1587 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); 1588 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]); 1589 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 1590 g2h_fence->retry = true; 1591 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); 1592 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) { 1593 g2h_fence->wait = true; 1594 g2h_fence->counter = FIELD_GET(GUC_HXG_BUSY_MSG_0_COUNTER, hxg[0]); 1595 } else if (g2h_fence->response_buffer) { 1596 g2h_fence->response_len = hxg_len; 1597 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); 1598 } else { 1599 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); 1600 } 1601 1602 /* don't release any space if it was an intermediate message */ 1603 if (!g2h_fence->wait) 1604 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1605 1606 /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ 1607 WRITE_ONCE(g2h_fence->done, true); 1608 smp_mb(); 1609 1610 wake_up_all(&ct->g2h_fence_wq); 1611 1612 return 0; 1613 } 1614 1615 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1616 { 1617 struct xe_gt *gt = ct_to_gt(ct); 1618 u32 *hxg = msg_to_hxg(msg); 1619 u32 origin, type; 1620 int ret; 1621 1622 lockdep_assert_held(&ct->lock); 1623 1624 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1625 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1626 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n", 1627 origin); 1628 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN); 1629 1630 return -EPROTO; 1631 } 1632 1633 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1634 switch (type) { 1635 case GUC_HXG_TYPE_EVENT: 1636 ret = parse_g2h_event(ct, msg, len); 1637 break; 1638 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1639 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1640 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1641 case GUC_HXG_TYPE_NO_RESPONSE_BUSY: 1642 ret = parse_g2h_response(ct, msg, len); 1643 break; 1644 default: 1645 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n", 1646 type); 1647 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE); 1648 1649 ret = -EOPNOTSUPP; 1650 } 1651 1652 return ret; 1653 } 1654 1655 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1656 { 1657 struct xe_guc *guc = ct_to_guc(ct); 1658 struct xe_gt *gt = ct_to_gt(ct); 1659 u32 hxg_len = msg_len_to_hxg_len(len); 1660 u32 *hxg = msg_to_hxg(msg); 1661 u32 action, adj_len; 1662 u32 *payload; 1663 int ret = 0; 1664 1665 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1666 return 0; 1667 1668 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1669 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN; 1670 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN; 1671 1672 switch (action) { 1673 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1674 ret = xe_guc_sched_done_handler(guc, payload, adj_len); 1675 break; 1676 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1677 ret = xe_guc_deregister_done_handler(guc, payload, adj_len); 1678 break; 1679 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1680 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len); 1681 break; 1682 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1683 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload, 1684 adj_len); 1685 break; 1686 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1687 /* Selftest only at the moment */ 1688 break; 1689 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1690 ret = xe_guc_error_capture_handler(guc, payload, adj_len); 1691 break; 1692 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1693 /* FIXME: Handle this */ 1694 break; 1695 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: 1696 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload, 1697 adj_len); 1698 break; 1699 case XE_GUC_ACTION_NOTIFY_UNCORRECTABLE_LOCAL_ERROR: 1700 ret = xe_guc_uncorrectable_error_handler(guc, payload, adj_len); 1701 break; 1702 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1703 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1704 break; 1705 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1706 ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); 1707 break; 1708 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1709 ret = xe_guc_page_reclaim_done_handler(guc, payload, adj_len); 1710 break; 1711 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: 1712 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len); 1713 break; 1714 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF: 1715 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len); 1716 break; 1717 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY: 1718 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len); 1719 break; 1720 case GUC_ACTION_GUC2PF_ADVERSE_EVENT: 1721 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len); 1722 break; 1723 case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1724 case XE_GUC_ACTION_NOTIFY_EXCEPTION: 1725 ret = guc_crash_process_msg(ct, action); 1726 break; 1727 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1728 case XE_GUC_ACTION_TEST_G2G_RECV: 1729 ret = xe_guc_g2g_test_notification(guc, payload, adj_len); 1730 break; 1731 #endif 1732 case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: 1733 ret = xe_guc_exec_queue_cgp_sync_done_handler(guc, payload, adj_len); 1734 break; 1735 case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR: 1736 ret = xe_guc_exec_queue_cgp_context_error_handler(guc, payload, 1737 adj_len); 1738 break; 1739 default: 1740 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); 1741 } 1742 1743 if (ret) { 1744 xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n", 1745 action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg); 1746 CT_DEAD(ct, NULL, PROCESS_FAILED); 1747 } 1748 1749 return 0; 1750 } 1751 1752 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) 1753 { 1754 struct xe_device *xe = ct_to_xe(ct); 1755 struct xe_gt *gt = ct_to_gt(ct); 1756 struct guc_ctb *g2h = &ct->ctbs.g2h; 1757 u32 tail, head, len, desc_status; 1758 s32 avail; 1759 u32 action; 1760 u32 *hxg; 1761 1762 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 1763 lockdep_assert_held(&ct->fast_lock); 1764 1765 if (xe_device_wedged(xe)) 1766 return -ENOTRECOVERABLE; 1767 1768 if (ct->state == XE_GUC_CT_STATE_DISABLED) 1769 return -ENODEV; 1770 1771 if (ct->state == XE_GUC_CT_STATE_STOPPED) 1772 return -ECANCELED; 1773 1774 if (g2h->info.broken) 1775 return -EPIPE; 1776 1777 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1778 1779 desc_status = desc_read(xe, g2h, status); 1780 if (desc_status) { 1781 if (desc_status & GUC_CTB_STATUS_DISABLED) { 1782 /* 1783 * Potentially valid if a CLIENT_RESET request resulted in 1784 * contexts/engines being reset. But should never happen as 1785 * no contexts should be active when CLIENT_RESET is sent. 1786 */ 1787 xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n"); 1788 desc_status &= ~GUC_CTB_STATUS_DISABLED; 1789 } 1790 1791 if (desc_status) { 1792 xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status); 1793 goto corrupted; 1794 } 1795 } 1796 1797 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 1798 u32 desc_tail = desc_read(xe, g2h, tail); 1799 /* 1800 u32 desc_head = desc_read(xe, g2h, head); 1801 1802 * info.head and desc_head are updated back-to-back at the end of 1803 * this function and nowhere else. Hence, they cannot be different 1804 * unless two g2h_read calls are running concurrently. Which is not 1805 * possible because it is guarded by ct->fast_lock. And yet, some 1806 * discrete platforms are regularly hitting this error :(. 1807 * 1808 * desc_head rolling backwards shouldn't cause any noticeable 1809 * problems - just a delay in GuC being allowed to proceed past that 1810 * point in the queue. So for now, just disable the error until it 1811 * can be root caused. 1812 * 1813 if (g2h->info.head != desc_head) { 1814 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH); 1815 xe_gt_err(gt, "CT read: head was modified %u != %u\n", 1816 desc_head, g2h->info.head); 1817 goto corrupted; 1818 } 1819 */ 1820 1821 if (g2h->info.head > g2h->info.size) { 1822 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1823 xe_gt_err(gt, "CT read: head out of range: %u vs %u\n", 1824 g2h->info.head, g2h->info.size); 1825 goto corrupted; 1826 } 1827 1828 if (desc_tail >= g2h->info.size) { 1829 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1830 xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n", 1831 desc_tail, g2h->info.size); 1832 goto corrupted; 1833 } 1834 } 1835 1836 /* Calculate DW available to read */ 1837 tail = desc_read(xe, g2h, tail); 1838 avail = tail - g2h->info.head; 1839 if (unlikely(avail == 0)) 1840 return 0; 1841 1842 if (avail < 0) 1843 avail += g2h->info.size; 1844 1845 /* Read header */ 1846 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head, 1847 sizeof(u32)); 1848 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; 1849 if (len > avail) { 1850 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n", 1851 avail, len); 1852 goto corrupted; 1853 } 1854 1855 head = (g2h->info.head + 1) % g2h->info.size; 1856 avail = len - 1; 1857 1858 /* Read G2H message */ 1859 if (avail + head > g2h->info.size) { 1860 u32 avail_til_wrap = g2h->info.size - head; 1861 1862 xe_map_memcpy_from(xe, msg + 1, 1863 &g2h->cmds, sizeof(u32) * head, 1864 avail_til_wrap * sizeof(u32)); 1865 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap, 1866 &g2h->cmds, 0, 1867 (avail - avail_til_wrap) * sizeof(u32)); 1868 } else { 1869 xe_map_memcpy_from(xe, msg + 1, 1870 &g2h->cmds, sizeof(u32) * head, 1871 avail * sizeof(u32)); 1872 } 1873 1874 hxg = msg_to_hxg(msg); 1875 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1876 1877 if (fast_path) { 1878 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1879 return 0; 1880 1881 switch (action) { 1882 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1883 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1884 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1885 break; /* Process these in fast-path */ 1886 default: 1887 return 0; 1888 } 1889 } 1890 1891 /* Update local / descriptor header */ 1892 g2h->info.head = (head + avail) % g2h->info.size; 1893 desc_write(xe, g2h, head, g2h->info.head); 1894 1895 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id, 1896 action, len, g2h->info.head, tail); 1897 1898 return len; 1899 1900 corrupted: 1901 CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ); 1902 return -EPROTO; 1903 } 1904 1905 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) 1906 { 1907 struct xe_gt *gt = ct_to_gt(ct); 1908 struct xe_guc *guc = ct_to_guc(ct); 1909 u32 hxg_len = msg_len_to_hxg_len(len); 1910 u32 *hxg = msg_to_hxg(msg); 1911 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1912 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN; 1913 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1914 int ret = 0; 1915 1916 switch (action) { 1917 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1918 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1919 break; 1920 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1921 __g2h_release_space(ct, len); 1922 ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); 1923 break; 1924 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1925 __g2h_release_space(ct, len); 1926 ret = xe_guc_page_reclaim_done_handler(guc, payload, adj_len); 1927 break; 1928 default: 1929 xe_gt_warn(gt, "NOT_POSSIBLE\n"); 1930 } 1931 1932 if (ret) { 1933 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n", 1934 action, ERR_PTR(ret)); 1935 CT_DEAD(ct, NULL, FAST_G2H); 1936 } 1937 } 1938 1939 /** 1940 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler 1941 * @ct: GuC CT object 1942 * 1943 * Anything related to page faults is critical for performance, process these 1944 * critical G2H in the IRQ. This is safe as these handlers either just wake up 1945 * waiters or queue another worker. 1946 */ 1947 void xe_guc_ct_fast_path(struct xe_guc_ct *ct) 1948 { 1949 struct xe_device *xe = ct_to_xe(ct); 1950 bool ongoing; 1951 int len; 1952 1953 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1954 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1955 return; 1956 1957 spin_lock(&ct->fast_lock); 1958 do { 1959 len = g2h_read(ct, ct->fast_msg, true); 1960 if (len > 0) 1961 g2h_fast_path(ct, ct->fast_msg, len); 1962 } while (len > 0); 1963 spin_unlock(&ct->fast_lock); 1964 1965 if (ongoing) 1966 xe_pm_runtime_put(xe); 1967 } 1968 1969 /* Returns less than zero on error, 0 on done, 1 on more available */ 1970 static int dequeue_one_g2h(struct xe_guc_ct *ct) 1971 { 1972 int len; 1973 int ret; 1974 1975 lockdep_assert_held(&ct->lock); 1976 1977 spin_lock_irq(&ct->fast_lock); 1978 len = g2h_read(ct, ct->msg, false); 1979 spin_unlock_irq(&ct->fast_lock); 1980 if (len <= 0) 1981 return len; 1982 1983 ret = parse_g2h_msg(ct, ct->msg, len); 1984 if (unlikely(ret < 0)) 1985 return ret; 1986 1987 ret = process_g2h_msg(ct, ct->msg, len); 1988 if (unlikely(ret < 0)) 1989 return ret; 1990 1991 return 1; 1992 } 1993 1994 static void receive_g2h(struct xe_guc_ct *ct) 1995 { 1996 bool ongoing; 1997 int ret; 1998 1999 /* 2000 * Normal users must always hold mem_access.ref around CT calls. However 2001 * during the runtime pm callbacks we rely on CT to talk to the GuC, but 2002 * at this stage we can't rely on mem_access.ref and even the 2003 * callback_task will be different than current. For such cases we just 2004 * need to ensure we always process the responses from any blocking 2005 * ct_send requests or where we otherwise expect some response when 2006 * initiated from those callbacks (which will need to wait for the below 2007 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if 2008 * the device has suspended to the point that the CT communication has 2009 * been disabled. 2010 * 2011 * If we are inside the runtime pm callback, we can be the only task 2012 * still issuing CT requests (since that requires having the 2013 * mem_access.ref). It seems like it might in theory be possible to 2014 * receive unsolicited events from the GuC just as we are 2015 * suspending-resuming, but those will currently anyway be lost when 2016 * eventually exiting from suspend, hence no need to wake up the device 2017 * here. If we ever need something stronger than get_if_ongoing() then 2018 * we need to be careful with blocking the pm callbacks from getting CT 2019 * responses, if the worker here is blocked on those callbacks 2020 * completing, creating a deadlock. 2021 */ 2022 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 2023 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 2024 return; 2025 2026 do { 2027 mutex_lock(&ct->lock); 2028 ret = dequeue_one_g2h(ct); 2029 mutex_unlock(&ct->lock); 2030 2031 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) { 2032 xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d\n", ret); 2033 CT_DEAD(ct, NULL, G2H_RECV); 2034 kick_reset(ct); 2035 } 2036 } while (ret == 1); 2037 2038 if (ongoing) 2039 xe_pm_runtime_put(ct_to_xe(ct)); 2040 } 2041 2042 static void g2h_worker_func(struct work_struct *w) 2043 { 2044 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker); 2045 2046 receive_g2h(ct); 2047 } 2048 2049 static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic, 2050 bool want_ctb) 2051 { 2052 struct xe_guc_ct_snapshot *snapshot; 2053 2054 snapshot = kzalloc_obj(*snapshot, atomic ? GFP_ATOMIC : GFP_KERNEL); 2055 if (!snapshot) 2056 return NULL; 2057 2058 if (ct->ctbs.h2g.bo && ct->ctbs.g2h.bo && want_ctb) { 2059 snapshot->ctb_size = xe_bo_size(ct->ctbs.h2g.bo) + 2060 xe_bo_size(ct->ctbs.g2h.bo); 2061 snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL); 2062 } 2063 2064 return snapshot; 2065 } 2066 2067 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb, 2068 struct guc_ctb_snapshot *snapshot) 2069 { 2070 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0, 2071 sizeof(struct guc_ct_buffer_desc)); 2072 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info)); 2073 } 2074 2075 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot, 2076 struct drm_printer *p) 2077 { 2078 drm_printf(p, "\tsize: %d\n", snapshot->info.size); 2079 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space); 2080 drm_printf(p, "\thead: %d\n", snapshot->info.head); 2081 drm_printf(p, "\ttail: %d\n", snapshot->info.tail); 2082 drm_printf(p, "\tspace: %d\n", snapshot->info.space); 2083 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken); 2084 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head); 2085 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail); 2086 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status); 2087 } 2088 2089 static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic, 2090 bool want_ctb) 2091 { 2092 struct xe_device *xe = ct_to_xe(ct); 2093 struct xe_guc_ct_snapshot *snapshot; 2094 2095 snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb); 2096 if (!snapshot) { 2097 xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n"); 2098 return NULL; 2099 } 2100 2101 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) { 2102 snapshot->ct_enabled = true; 2103 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding); 2104 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g); 2105 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h); 2106 } 2107 2108 if (ct->ctbs.h2g.bo && ct->ctbs.g2h.bo && snapshot->ctb) { 2109 xe_map_memcpy_from(xe, snapshot->ctb, &ct->ctbs.h2g.bo->vmap, 0, 2110 xe_bo_size(ct->ctbs.h2g.bo)); 2111 xe_map_memcpy_from(xe, snapshot->ctb + xe_bo_size(ct->ctbs.h2g.bo), 2112 &ct->ctbs.g2h.bo->vmap, 0, 2113 xe_bo_size(ct->ctbs.g2h.bo)); 2114 } 2115 2116 return snapshot; 2117 } 2118 2119 /** 2120 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state. 2121 * @ct: GuC CT object. 2122 * 2123 * This can be printed out in a later stage like during dev_coredump 2124 * analysis. This is safe to be called during atomic context. 2125 * 2126 * Returns: a GuC CT snapshot object that must be freed by the caller 2127 * by using `xe_guc_ct_snapshot_free`. 2128 */ 2129 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct) 2130 { 2131 return guc_ct_snapshot_capture(ct, true, true); 2132 } 2133 2134 /** 2135 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot. 2136 * @snapshot: GuC CT snapshot object. 2137 * @p: drm_printer where it will be printed out. 2138 * 2139 * This function prints out a given GuC CT snapshot object. 2140 */ 2141 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, 2142 struct drm_printer *p) 2143 { 2144 if (!snapshot) 2145 return; 2146 2147 if (snapshot->ct_enabled) { 2148 drm_puts(p, "H2G CTB (all sizes in DW):\n"); 2149 guc_ctb_snapshot_print(&snapshot->h2g, p); 2150 2151 drm_puts(p, "G2H CTB (all sizes in DW):\n"); 2152 guc_ctb_snapshot_print(&snapshot->g2h, p); 2153 drm_printf(p, "\tg2h outstanding: %d\n", 2154 snapshot->g2h_outstanding); 2155 2156 if (snapshot->ctb) { 2157 drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size); 2158 xe_print_blob_ascii85(p, "[CTB].data", '\n', 2159 snapshot->ctb, 0, snapshot->ctb_size); 2160 } 2161 } else { 2162 drm_puts(p, "CT disabled\n"); 2163 } 2164 } 2165 2166 /** 2167 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot. 2168 * @snapshot: GuC CT snapshot object. 2169 * 2170 * This function free all the memory that needed to be allocated at capture 2171 * time. 2172 */ 2173 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot) 2174 { 2175 if (!snapshot) 2176 return; 2177 2178 kfree(snapshot->ctb); 2179 kfree(snapshot); 2180 } 2181 2182 /** 2183 * xe_guc_ct_print - GuC CT Print. 2184 * @ct: GuC CT. 2185 * @p: drm_printer where it will be printed out. 2186 * @want_ctb: Should the full CTB content be dumped (vs just the headers) 2187 * 2188 * This function will quickly capture a snapshot of the CT state 2189 * and immediately print it out. 2190 */ 2191 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb) 2192 { 2193 struct xe_guc_ct_snapshot *snapshot; 2194 2195 snapshot = guc_ct_snapshot_capture(ct, false, want_ctb); 2196 xe_guc_ct_snapshot_print(snapshot, p); 2197 xe_guc_ct_snapshot_free(snapshot); 2198 } 2199 2200 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 2201 2202 #ifdef CONFIG_FUNCTION_ERROR_INJECTION 2203 /* 2204 * This is a helper function which assists the driver in identifying if a fault 2205 * injection test is currently active, allowing it to reduce unnecessary debug 2206 * output. Typically, the function returns zero, but the fault injection 2207 * framework can alter this to return an error. Since faults are injected 2208 * through this function, it's important to ensure the compiler doesn't optimize 2209 * it into an inline function. To avoid such optimization, the 'noinline' 2210 * attribute is applied. Compiler optimizes the static function defined in the 2211 * header file as an inline function. 2212 */ 2213 noinline int xe_is_injection_active(void) { return 0; } 2214 ALLOW_ERROR_INJECTION(xe_is_injection_active, ERRNO); 2215 #else 2216 int xe_is_injection_active(void) { return 0; } 2217 #endif 2218 2219 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code) 2220 { 2221 struct xe_guc_log_snapshot *snapshot_log; 2222 struct xe_guc_ct_snapshot *snapshot_ct; 2223 struct xe_guc *guc = ct_to_guc(ct); 2224 unsigned long flags; 2225 bool have_capture; 2226 2227 if (ctb) 2228 ctb->info.broken = true; 2229 /* 2230 * Huge dump is getting generated when injecting error for guc CT/MMIO 2231 * functions. So, let us suppress the dump when fault is injected. 2232 */ 2233 if (xe_is_injection_active()) 2234 return; 2235 2236 /* Ignore further errors after the first dump until a reset */ 2237 if (ct->dead.reported) 2238 return; 2239 2240 spin_lock_irqsave(&ct->dead.lock, flags); 2241 2242 /* And only capture one dump at a time */ 2243 have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE); 2244 ct->dead.reason |= (1 << reason_code) | 2245 (1 << CT_DEAD_STATE_CAPTURE); 2246 2247 spin_unlock_irqrestore(&ct->dead.lock, flags); 2248 2249 if (have_capture) 2250 return; 2251 2252 snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true); 2253 snapshot_ct = xe_guc_ct_snapshot_capture((ct)); 2254 2255 spin_lock_irqsave(&ct->dead.lock, flags); 2256 2257 if (ct->dead.snapshot_log || ct->dead.snapshot_ct) { 2258 xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n"); 2259 xe_guc_log_snapshot_free(snapshot_log); 2260 xe_guc_ct_snapshot_free(snapshot_ct); 2261 } else { 2262 ct->dead.snapshot_log = snapshot_log; 2263 ct->dead.snapshot_ct = snapshot_ct; 2264 } 2265 2266 spin_unlock_irqrestore(&ct->dead.lock, flags); 2267 2268 queue_work(system_dfl_wq, &(ct)->dead.worker); 2269 } 2270 2271 static void ct_dead_print(struct xe_dead_ct *dead) 2272 { 2273 struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead); 2274 struct xe_device *xe = ct_to_xe(ct); 2275 struct xe_gt *gt = ct_to_gt(ct); 2276 static int g_count; 2277 struct drm_printer ip = xe_gt_info_printer(gt); 2278 struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count); 2279 2280 if (!dead->reason) { 2281 xe_gt_err(gt, "CTB is dead for no reason!?\n"); 2282 return; 2283 } 2284 2285 /* Can't generate a genuine core dump at this point, so just do the good bits */ 2286 drm_puts(&lp, "**** Xe Device Coredump ****\n"); 2287 drm_printf(&lp, "Reason: CTB is dead - 0x%X\n", dead->reason); 2288 xe_device_snapshot_print(xe, &lp); 2289 2290 drm_printf(&lp, "**** GT #%d ****\n", gt->info.id); 2291 drm_printf(&lp, "\tTile: %d\n", gt->tile->id); 2292 2293 drm_puts(&lp, "**** GuC Log ****\n"); 2294 xe_guc_log_snapshot_print(dead->snapshot_log, &lp); 2295 2296 drm_puts(&lp, "**** GuC CT ****\n"); 2297 xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp); 2298 2299 drm_puts(&lp, "Done.\n"); 2300 } 2301 2302 static void ct_dead_worker_func(struct work_struct *w) 2303 { 2304 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker); 2305 2306 if (!ct->dead.reported) { 2307 ct->dead.reported = true; 2308 ct_dead_print(&ct->dead); 2309 } 2310 2311 spin_lock_irq(&ct->dead.lock); 2312 2313 xe_guc_log_snapshot_free(ct->dead.snapshot_log); 2314 ct->dead.snapshot_log = NULL; 2315 xe_guc_ct_snapshot_free(ct->dead.snapshot_ct); 2316 ct->dead.snapshot_ct = NULL; 2317 2318 if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) { 2319 /* A reset has occurred so re-arm the error reporting */ 2320 ct->dead.reason = 0; 2321 ct->dead.reported = false; 2322 } 2323 2324 spin_unlock_irq(&ct->dead.lock); 2325 } 2326 #endif 2327