1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ct.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 #include <linux/delay.h> 11 #include <linux/fault-inject.h> 12 13 #include <kunit/static_stub.h> 14 15 #include <drm/drm_managed.h> 16 17 #include "abi/guc_actions_abi.h" 18 #include "abi/guc_actions_sriov_abi.h" 19 #include "abi/guc_klvs_abi.h" 20 #include "xe_bo.h" 21 #include "xe_devcoredump.h" 22 #include "xe_device.h" 23 #include "xe_gt.h" 24 #include "xe_gt_pagefault.h" 25 #include "xe_gt_printk.h" 26 #include "xe_gt_sriov_pf_control.h" 27 #include "xe_gt_sriov_pf_monitor.h" 28 #include "xe_gt_tlb_invalidation.h" 29 #include "xe_guc.h" 30 #include "xe_guc_log.h" 31 #include "xe_guc_relay.h" 32 #include "xe_guc_submit.h" 33 #include "xe_map.h" 34 #include "xe_pm.h" 35 #include "xe_trace_guc.h" 36 37 static void receive_g2h(struct xe_guc_ct *ct); 38 static void g2h_worker_func(struct work_struct *w); 39 static void safe_mode_worker_func(struct work_struct *w); 40 static void ct_exit_safe_mode(struct xe_guc_ct *ct); 41 42 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 43 enum { 44 /* Internal states, not error conditions */ 45 CT_DEAD_STATE_REARM, /* 0x0001 */ 46 CT_DEAD_STATE_CAPTURE, /* 0x0002 */ 47 48 /* Error conditions */ 49 CT_DEAD_SETUP, /* 0x0004 */ 50 CT_DEAD_H2G_WRITE, /* 0x0008 */ 51 CT_DEAD_H2G_HAS_ROOM, /* 0x0010 */ 52 CT_DEAD_G2H_READ, /* 0x0020 */ 53 CT_DEAD_G2H_RECV, /* 0x0040 */ 54 CT_DEAD_G2H_RELEASE, /* 0x0080 */ 55 CT_DEAD_DEADLOCK, /* 0x0100 */ 56 CT_DEAD_PROCESS_FAILED, /* 0x0200 */ 57 CT_DEAD_FAST_G2H, /* 0x0400 */ 58 CT_DEAD_PARSE_G2H_RESPONSE, /* 0x0800 */ 59 CT_DEAD_PARSE_G2H_UNKNOWN, /* 0x1000 */ 60 CT_DEAD_PARSE_G2H_ORIGIN, /* 0x2000 */ 61 CT_DEAD_PARSE_G2H_TYPE, /* 0x4000 */ 62 CT_DEAD_CRASH, /* 0x8000 */ 63 }; 64 65 static void ct_dead_worker_func(struct work_struct *w); 66 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code); 67 68 #define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code) 69 #else 70 #define CT_DEAD(ct, ctb, reason) \ 71 do { \ 72 struct guc_ctb *_ctb = (ctb); \ 73 if (_ctb) \ 74 _ctb->info.broken = true; \ 75 } while (0) 76 #endif 77 78 /* Used when a CT send wants to block and / or receive data */ 79 struct g2h_fence { 80 u32 *response_buffer; 81 u32 seqno; 82 u32 response_data; 83 u16 response_len; 84 u16 error; 85 u16 hint; 86 u16 reason; 87 bool retry; 88 bool fail; 89 bool done; 90 }; 91 92 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) 93 { 94 g2h_fence->response_buffer = response_buffer; 95 g2h_fence->response_data = 0; 96 g2h_fence->response_len = 0; 97 g2h_fence->fail = false; 98 g2h_fence->retry = false; 99 g2h_fence->done = false; 100 g2h_fence->seqno = ~0x0; 101 } 102 103 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) 104 { 105 return g2h_fence->seqno == ~0x0; 106 } 107 108 static struct xe_guc * 109 ct_to_guc(struct xe_guc_ct *ct) 110 { 111 return container_of(ct, struct xe_guc, ct); 112 } 113 114 static struct xe_gt * 115 ct_to_gt(struct xe_guc_ct *ct) 116 { 117 return container_of(ct, struct xe_gt, uc.guc.ct); 118 } 119 120 static struct xe_device * 121 ct_to_xe(struct xe_guc_ct *ct) 122 { 123 return gt_to_xe(ct_to_gt(ct)); 124 } 125 126 /** 127 * DOC: GuC CTB Blob 128 * 129 * We allocate single blob to hold both CTB descriptors and buffers: 130 * 131 * +--------+-----------------------------------------------+------+ 132 * | offset | contents | size | 133 * +========+===============================================+======+ 134 * | 0x0000 | H2G CTB Descriptor (send) | | 135 * +--------+-----------------------------------------------+ 4K | 136 * | 0x0800 | G2H CTB Descriptor (g2h) | | 137 * +--------+-----------------------------------------------+------+ 138 * | 0x1000 | H2G CT Buffer (send) | n*4K | 139 * | | | | 140 * +--------+-----------------------------------------------+------+ 141 * | 0x1000 | G2H CT Buffer (g2h) | m*4K | 142 * | + n*4K | | | 143 * +--------+-----------------------------------------------+------+ 144 * 145 * Size of each ``CT Buffer`` must be multiple of 4K. 146 * We don't expect too many messages in flight at any time, unless we are 147 * using the GuC submission. In that case each request requires a minimum 148 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 149 * enough space to avoid backpressure on the driver. We increase the size 150 * of the receive buffer (relative to the send) to ensure a G2H response 151 * CTB has a landing spot. 152 * 153 * In addition to submissions, the G2H buffer needs to be able to hold 154 * enough space for recoverable page fault notifications. The number of 155 * page faults is interrupt driven and can be as much as the number of 156 * compute resources available. However, most of the actual work for these 157 * is in a separate page fault worker thread. Therefore we only need to 158 * make sure the queue has enough space to handle all of the submissions 159 * and responses and an extra buffer for incoming page faults. 160 */ 161 162 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 163 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 164 #define CTB_G2H_BUFFER_SIZE (SZ_128K) 165 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2) 166 167 /** 168 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full 169 * CT command queue 170 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future. 171 * 172 * Observation is that a 4KiB buffer full of commands takes a little over a 173 * second to process. Use that to calculate maximum time to process a full CT 174 * command queue. 175 * 176 * Return: Maximum time to process a full CT queue in jiffies. 177 */ 178 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct) 179 { 180 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4)); 181 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ; 182 } 183 184 static size_t guc_ct_size(void) 185 { 186 return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + 187 CTB_G2H_BUFFER_SIZE; 188 } 189 190 static void guc_ct_fini(struct drm_device *drm, void *arg) 191 { 192 struct xe_guc_ct *ct = arg; 193 194 ct_exit_safe_mode(ct); 195 destroy_workqueue(ct->g2h_wq); 196 xa_destroy(&ct->fence_lookup); 197 } 198 199 static void primelockdep(struct xe_guc_ct *ct) 200 { 201 if (!IS_ENABLED(CONFIG_LOCKDEP)) 202 return; 203 204 fs_reclaim_acquire(GFP_KERNEL); 205 might_lock(&ct->lock); 206 fs_reclaim_release(GFP_KERNEL); 207 } 208 209 int xe_guc_ct_init(struct xe_guc_ct *ct) 210 { 211 struct xe_device *xe = ct_to_xe(ct); 212 struct xe_gt *gt = ct_to_gt(ct); 213 struct xe_tile *tile = gt_to_tile(gt); 214 struct xe_bo *bo; 215 int err; 216 217 xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE)); 218 219 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM); 220 if (!ct->g2h_wq) 221 return -ENOMEM; 222 223 spin_lock_init(&ct->fast_lock); 224 xa_init(&ct->fence_lookup); 225 INIT_WORK(&ct->g2h_worker, g2h_worker_func); 226 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func); 227 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 228 spin_lock_init(&ct->dead.lock); 229 INIT_WORK(&ct->dead.worker, ct_dead_worker_func); 230 #endif 231 init_waitqueue_head(&ct->wq); 232 init_waitqueue_head(&ct->g2h_fence_wq); 233 234 err = drmm_mutex_init(&xe->drm, &ct->lock); 235 if (err) 236 return err; 237 238 primelockdep(ct); 239 240 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(), 241 XE_BO_FLAG_SYSTEM | 242 XE_BO_FLAG_GGTT | 243 XE_BO_FLAG_GGTT_INVALIDATE | 244 XE_BO_FLAG_PINNED_NORESTORE); 245 if (IS_ERR(bo)) 246 return PTR_ERR(bo); 247 248 ct->bo = bo; 249 250 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); 251 if (err) 252 return err; 253 254 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED); 255 ct->state = XE_GUC_CT_STATE_DISABLED; 256 return 0; 257 } 258 ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */ 259 260 #define desc_read(xe_, guc_ctb__, field_) \ 261 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \ 262 struct guc_ct_buffer_desc, field_) 263 264 #define desc_write(xe_, guc_ctb__, field_, val_) \ 265 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \ 266 struct guc_ct_buffer_desc, field_, val_) 267 268 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, 269 struct iosys_map *map) 270 { 271 h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32); 272 h2g->info.resv_space = 0; 273 h2g->info.tail = 0; 274 h2g->info.head = 0; 275 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 276 h2g->info.size) - 277 h2g->info.resv_space; 278 h2g->info.broken = false; 279 280 h2g->desc = *map; 281 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 282 283 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2); 284 } 285 286 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, 287 struct iosys_map *map) 288 { 289 g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32); 290 g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32); 291 g2h->info.head = 0; 292 g2h->info.tail = 0; 293 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head, 294 g2h->info.size) - 295 g2h->info.resv_space; 296 g2h->info.broken = false; 297 298 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE); 299 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 300 301 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 + 302 CTB_H2G_BUFFER_SIZE); 303 } 304 305 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct) 306 { 307 struct xe_guc *guc = ct_to_guc(ct); 308 u32 desc_addr, ctb_addr, size; 309 int err; 310 311 desc_addr = xe_bo_ggtt_addr(ct->bo); 312 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2; 313 size = ct->ctbs.h2g.info.size * sizeof(u32); 314 315 err = xe_guc_self_cfg64(guc, 316 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY, 317 desc_addr); 318 if (err) 319 return err; 320 321 err = xe_guc_self_cfg64(guc, 322 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY, 323 ctb_addr); 324 if (err) 325 return err; 326 327 return xe_guc_self_cfg32(guc, 328 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY, 329 size); 330 } 331 332 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct) 333 { 334 struct xe_guc *guc = ct_to_guc(ct); 335 u32 desc_addr, ctb_addr, size; 336 int err; 337 338 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE; 339 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 + 340 CTB_H2G_BUFFER_SIZE; 341 size = ct->ctbs.g2h.info.size * sizeof(u32); 342 343 err = xe_guc_self_cfg64(guc, 344 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 345 desc_addr); 346 if (err) 347 return err; 348 349 err = xe_guc_self_cfg64(guc, 350 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 351 ctb_addr); 352 if (err) 353 return err; 354 355 return xe_guc_self_cfg32(guc, 356 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 357 size); 358 } 359 360 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) 361 { 362 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 363 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 364 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 365 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, 366 GUC_ACTION_HOST2GUC_CONTROL_CTB), 367 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, 368 enable ? GUC_CTB_CONTROL_ENABLE : 369 GUC_CTB_CONTROL_DISABLE), 370 }; 371 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request)); 372 373 return ret > 0 ? -EPROTO : ret; 374 } 375 376 static void xe_guc_ct_set_state(struct xe_guc_ct *ct, 377 enum xe_guc_ct_state state) 378 { 379 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ 380 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ 381 382 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 || 383 state == XE_GUC_CT_STATE_STOPPED); 384 385 if (ct->g2h_outstanding) 386 xe_pm_runtime_put(ct_to_xe(ct)); 387 ct->g2h_outstanding = 0; 388 ct->state = state; 389 390 spin_unlock_irq(&ct->fast_lock); 391 392 /* 393 * Lockdep doesn't like this under the fast lock and he destroy only 394 * needs to be serialized with the send path which ct lock provides. 395 */ 396 xa_destroy(&ct->fence_lookup); 397 398 mutex_unlock(&ct->lock); 399 } 400 401 static bool ct_needs_safe_mode(struct xe_guc_ct *ct) 402 { 403 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev)); 404 } 405 406 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct) 407 { 408 if (!ct_needs_safe_mode(ct)) 409 return false; 410 411 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10); 412 return true; 413 } 414 415 static void safe_mode_worker_func(struct work_struct *w) 416 { 417 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work); 418 419 receive_g2h(ct); 420 421 if (!ct_restart_safe_mode_worker(ct)) 422 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n"); 423 } 424 425 static void ct_enter_safe_mode(struct xe_guc_ct *ct) 426 { 427 if (ct_restart_safe_mode_worker(ct)) 428 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n"); 429 } 430 431 static void ct_exit_safe_mode(struct xe_guc_ct *ct) 432 { 433 if (cancel_delayed_work_sync(&ct->safe_mode_worker)) 434 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n"); 435 } 436 437 int xe_guc_ct_enable(struct xe_guc_ct *ct) 438 { 439 struct xe_device *xe = ct_to_xe(ct); 440 struct xe_gt *gt = ct_to_gt(ct); 441 int err; 442 443 xe_gt_assert(gt, !xe_guc_ct_enabled(ct)); 444 445 xe_map_memset(xe, &ct->bo->vmap, 0, 0, ct->bo->size); 446 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap); 447 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap); 448 449 err = guc_ct_ctb_h2g_register(ct); 450 if (err) 451 goto err_out; 452 453 err = guc_ct_ctb_g2h_register(ct); 454 if (err) 455 goto err_out; 456 457 err = guc_ct_control_toggle(ct, true); 458 if (err) 459 goto err_out; 460 461 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED); 462 463 smp_mb(); 464 wake_up_all(&ct->wq); 465 xe_gt_dbg(gt, "GuC CT communication channel enabled\n"); 466 467 if (ct_needs_safe_mode(ct)) 468 ct_enter_safe_mode(ct); 469 470 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 471 /* 472 * The CT has now been reset so the dumper can be re-armed 473 * after any existing dead state has been dumped. 474 */ 475 spin_lock_irq(&ct->dead.lock); 476 if (ct->dead.reason) { 477 ct->dead.reason |= (1 << CT_DEAD_STATE_REARM); 478 queue_work(system_unbound_wq, &ct->dead.worker); 479 } 480 spin_unlock_irq(&ct->dead.lock); 481 #endif 482 483 return 0; 484 485 err_out: 486 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err)); 487 CT_DEAD(ct, NULL, SETUP); 488 489 return err; 490 } 491 492 static void stop_g2h_handler(struct xe_guc_ct *ct) 493 { 494 cancel_work_sync(&ct->g2h_worker); 495 } 496 497 /** 498 * xe_guc_ct_disable - Set GuC to disabled state 499 * @ct: the &xe_guc_ct 500 * 501 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected 502 * in this transition. 503 */ 504 void xe_guc_ct_disable(struct xe_guc_ct *ct) 505 { 506 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED); 507 ct_exit_safe_mode(ct); 508 stop_g2h_handler(ct); 509 } 510 511 /** 512 * xe_guc_ct_stop - Set GuC to stopped state 513 * @ct: the &xe_guc_ct 514 * 515 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h 516 */ 517 void xe_guc_ct_stop(struct xe_guc_ct *ct) 518 { 519 if (!xe_guc_ct_initialized(ct)) 520 return; 521 522 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED); 523 stop_g2h_handler(ct); 524 } 525 526 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) 527 { 528 struct guc_ctb *h2g = &ct->ctbs.h2g; 529 530 lockdep_assert_held(&ct->lock); 531 532 if (cmd_len > h2g->info.space) { 533 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); 534 535 if (h2g->info.head > h2g->info.size) { 536 struct xe_device *xe = ct_to_xe(ct); 537 u32 desc_status = desc_read(xe, h2g, status); 538 539 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 540 541 xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n", 542 h2g->info.head, h2g->info.size); 543 CT_DEAD(ct, h2g, H2G_HAS_ROOM); 544 return false; 545 } 546 547 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 548 h2g->info.size) - 549 h2g->info.resv_space; 550 if (cmd_len > h2g->info.space) 551 return false; 552 } 553 554 return true; 555 } 556 557 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len) 558 { 559 if (!g2h_len) 560 return true; 561 562 lockdep_assert_held(&ct->fast_lock); 563 564 return ct->ctbs.g2h.info.space > g2h_len; 565 } 566 567 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len) 568 { 569 lockdep_assert_held(&ct->lock); 570 571 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len)) 572 return -EBUSY; 573 574 return 0; 575 } 576 577 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len) 578 { 579 lockdep_assert_held(&ct->lock); 580 ct->ctbs.h2g.info.space -= cmd_len; 581 } 582 583 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h) 584 { 585 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space); 586 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) || 587 (g2h_len && num_g2h)); 588 589 if (g2h_len) { 590 lockdep_assert_held(&ct->fast_lock); 591 592 if (!ct->g2h_outstanding) 593 xe_pm_runtime_get_noresume(ct_to_xe(ct)); 594 595 ct->ctbs.g2h.info.space -= g2h_len; 596 ct->g2h_outstanding += num_g2h; 597 } 598 } 599 600 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 601 { 602 bool bad = false; 603 604 lockdep_assert_held(&ct->fast_lock); 605 606 bad = ct->ctbs.g2h.info.space + g2h_len > 607 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space; 608 bad |= !ct->g2h_outstanding; 609 610 if (bad) { 611 xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n", 612 ct->ctbs.g2h.info.space, g2h_len, 613 ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space, 614 ct->ctbs.g2h.info.space + g2h_len, 615 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space, 616 ct->g2h_outstanding); 617 CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE); 618 return; 619 } 620 621 ct->ctbs.g2h.info.space += g2h_len; 622 if (!--ct->g2h_outstanding) 623 xe_pm_runtime_put(ct_to_xe(ct)); 624 } 625 626 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 627 { 628 spin_lock_irq(&ct->fast_lock); 629 __g2h_release_space(ct, g2h_len); 630 spin_unlock_irq(&ct->fast_lock); 631 } 632 633 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */ 634 635 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, 636 u32 ct_fence_value, bool want_response) 637 { 638 struct xe_device *xe = ct_to_xe(ct); 639 struct xe_gt *gt = ct_to_gt(ct); 640 struct guc_ctb *h2g = &ct->ctbs.h2g; 641 u32 cmd[H2G_CT_HEADERS]; 642 u32 tail = h2g->info.tail; 643 u32 full_len; 644 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds, 645 tail * sizeof(u32)); 646 u32 desc_status; 647 648 full_len = len + GUC_CTB_HDR_LEN; 649 650 lockdep_assert_held(&ct->lock); 651 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN); 652 653 desc_status = desc_read(xe, h2g, status); 654 if (desc_status) { 655 xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status); 656 goto corrupted; 657 } 658 659 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 660 u32 desc_tail = desc_read(xe, h2g, tail); 661 u32 desc_head = desc_read(xe, h2g, head); 662 663 if (tail != desc_tail) { 664 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH); 665 xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail); 666 goto corrupted; 667 } 668 669 if (tail > h2g->info.size) { 670 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 671 xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n", 672 tail, h2g->info.size); 673 goto corrupted; 674 } 675 676 if (desc_head >= h2g->info.size) { 677 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 678 xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n", 679 desc_head, h2g->info.size); 680 goto corrupted; 681 } 682 } 683 684 /* Command will wrap, zero fill (NOPs), return and check credits again */ 685 if (tail + full_len > h2g->info.size) { 686 xe_map_memset(xe, &map, 0, 0, 687 (h2g->info.size - tail) * sizeof(u32)); 688 h2g_reserve_space(ct, (h2g->info.size - tail)); 689 h2g->info.tail = 0; 690 desc_write(xe, h2g, tail, h2g->info.tail); 691 692 return -EAGAIN; 693 } 694 695 /* 696 * dw0: CT header (including fence) 697 * dw1: HXG header (including action code) 698 * dw2+: action data 699 */ 700 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 701 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 702 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value); 703 if (want_response) { 704 cmd[1] = 705 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 706 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 707 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 708 } else { 709 cmd[1] = 710 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) | 711 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 712 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 713 } 714 715 /* H2G header in cmd[1] replaces action[0] so: */ 716 --len; 717 ++action; 718 719 /* Write H2G ensuring visible before descriptor update */ 720 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32)); 721 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32)); 722 xe_device_wmb(xe); 723 724 /* Update local copies */ 725 h2g->info.tail = (tail + full_len) % h2g->info.size; 726 h2g_reserve_space(ct, full_len); 727 728 /* Update descriptor */ 729 desc_write(xe, h2g, tail, h2g->info.tail); 730 731 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len, 732 desc_read(xe, h2g, head), h2g->info.tail); 733 734 return 0; 735 736 corrupted: 737 CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE); 738 return -EPIPE; 739 } 740 741 /* 742 * The CT protocol accepts a 16 bits fence. This field is fully owned by the 743 * driver, the GuC will just copy it to the reply message. Since we need to 744 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages, 745 * we use one bit of the seqno as an indicator for that and a rolling counter 746 * for the remaining 15 bits. 747 */ 748 #define CT_SEQNO_MASK GENMASK(14, 0) 749 #define CT_SEQNO_UNTRACKED BIT(15) 750 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence) 751 { 752 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK; 753 754 if (!is_g2h_fence) 755 seqno |= CT_SEQNO_UNTRACKED; 756 757 return seqno; 758 } 759 760 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, 761 u32 len, u32 g2h_len, u32 num_g2h, 762 struct g2h_fence *g2h_fence) 763 { 764 struct xe_gt *gt __maybe_unused = ct_to_gt(ct); 765 u16 seqno; 766 int ret; 767 768 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 769 xe_gt_assert(gt, !g2h_len || !g2h_fence); 770 xe_gt_assert(gt, !num_g2h || !g2h_fence); 771 xe_gt_assert(gt, !g2h_len || num_g2h); 772 xe_gt_assert(gt, g2h_len || !num_g2h); 773 lockdep_assert_held(&ct->lock); 774 775 if (unlikely(ct->ctbs.h2g.info.broken)) { 776 ret = -EPIPE; 777 goto out; 778 } 779 780 if (ct->state == XE_GUC_CT_STATE_DISABLED) { 781 ret = -ENODEV; 782 goto out; 783 } 784 785 if (ct->state == XE_GUC_CT_STATE_STOPPED) { 786 ret = -ECANCELED; 787 goto out; 788 } 789 790 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 791 792 if (g2h_fence) { 793 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; 794 num_g2h = 1; 795 796 if (g2h_fence_needs_alloc(g2h_fence)) { 797 g2h_fence->seqno = next_ct_seqno(ct, true); 798 ret = xa_err(xa_store(&ct->fence_lookup, 799 g2h_fence->seqno, g2h_fence, 800 GFP_ATOMIC)); 801 if (ret) 802 goto out; 803 } 804 805 seqno = g2h_fence->seqno; 806 } else { 807 seqno = next_ct_seqno(ct, false); 808 } 809 810 if (g2h_len) 811 spin_lock_irq(&ct->fast_lock); 812 retry: 813 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len); 814 if (unlikely(ret)) 815 goto out_unlock; 816 817 ret = h2g_write(ct, action, len, seqno, !!g2h_fence); 818 if (unlikely(ret)) { 819 if (ret == -EAGAIN) 820 goto retry; 821 goto out_unlock; 822 } 823 824 __g2h_reserve_space(ct, g2h_len, num_g2h); 825 xe_guc_notify(ct_to_guc(ct)); 826 out_unlock: 827 if (g2h_len) 828 spin_unlock_irq(&ct->fast_lock); 829 out: 830 return ret; 831 } 832 833 static void kick_reset(struct xe_guc_ct *ct) 834 { 835 xe_gt_reset_async(ct_to_gt(ct)); 836 } 837 838 static int dequeue_one_g2h(struct xe_guc_ct *ct); 839 840 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 841 u32 g2h_len, u32 num_g2h, 842 struct g2h_fence *g2h_fence) 843 { 844 struct xe_device *xe = ct_to_xe(ct); 845 struct xe_gt *gt = ct_to_gt(ct); 846 unsigned int sleep_period_ms = 1; 847 int ret; 848 849 xe_gt_assert(gt, !g2h_len || !g2h_fence); 850 lockdep_assert_held(&ct->lock); 851 xe_device_assert_mem_access(ct_to_xe(ct)); 852 853 try_again: 854 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, 855 g2h_fence); 856 857 /* 858 * We wait to try to restore credits for about 1 second before bailing. 859 * In the case of H2G credits we have no choice but just to wait for the 860 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In 861 * the case of G2H we process any G2H in the channel, hopefully freeing 862 * credits as we consume the G2H messages. 863 */ 864 if (unlikely(ret == -EBUSY && 865 !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) { 866 struct guc_ctb *h2g = &ct->ctbs.h2g; 867 868 if (sleep_period_ms == 1024) 869 goto broken; 870 871 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail, 872 h2g->info.size, 873 h2g->info.space, 874 len + GUC_CTB_HDR_LEN); 875 msleep(sleep_period_ms); 876 sleep_period_ms <<= 1; 877 878 goto try_again; 879 } else if (unlikely(ret == -EBUSY)) { 880 struct xe_device *xe = ct_to_xe(ct); 881 struct guc_ctb *g2h = &ct->ctbs.g2h; 882 883 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head, 884 desc_read(xe, g2h, tail), 885 g2h->info.size, 886 g2h->info.space, 887 g2h_fence ? 888 GUC_CTB_HXG_MSG_MAX_LEN : 889 g2h_len); 890 891 #define g2h_avail(ct) \ 892 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head) 893 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding || 894 g2h_avail(ct), HZ)) 895 goto broken; 896 #undef g2h_avail 897 898 ret = dequeue_one_g2h(ct); 899 if (ret < 0) { 900 if (ret != -ECANCELED) 901 xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)", 902 ERR_PTR(ret)); 903 goto broken; 904 } 905 906 goto try_again; 907 } 908 909 return ret; 910 911 broken: 912 xe_gt_err(gt, "No forward process on H2G, reset required\n"); 913 CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK); 914 915 return -EDEADLK; 916 } 917 918 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 919 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) 920 { 921 int ret; 922 923 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence); 924 925 mutex_lock(&ct->lock); 926 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence); 927 mutex_unlock(&ct->lock); 928 929 return ret; 930 } 931 932 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 933 u32 g2h_len, u32 num_g2h) 934 { 935 int ret; 936 937 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL); 938 if (ret == -EDEADLK) 939 kick_reset(ct); 940 941 return ret; 942 } 943 944 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 945 u32 g2h_len, u32 num_g2h) 946 { 947 int ret; 948 949 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL); 950 if (ret == -EDEADLK) 951 kick_reset(ct); 952 953 return ret; 954 } 955 956 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len) 957 { 958 int ret; 959 960 lockdep_assert_held(&ct->lock); 961 962 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL); 963 if (ret == -EDEADLK) 964 kick_reset(ct); 965 966 return ret; 967 } 968 969 /* 970 * Check if a GT reset is in progress or will occur and if GT reset brought the 971 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset. 972 */ 973 static bool retry_failure(struct xe_guc_ct *ct, int ret) 974 { 975 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV)) 976 return false; 977 978 #define ct_alive(ct) \ 979 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \ 980 !ct->ctbs.g2h.info.broken) 981 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) 982 return false; 983 #undef ct_alive 984 985 return true; 986 } 987 988 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 989 u32 *response_buffer, bool no_fail) 990 { 991 struct xe_gt *gt = ct_to_gt(ct); 992 struct g2h_fence g2h_fence; 993 int ret = 0; 994 995 /* 996 * We use a fence to implement blocking sends / receiving response data. 997 * The seqno of the fence is sent in the H2G, returned in the G2H, and 998 * an xarray is used as storage media with the seqno being to key. 999 * Fields in the fence hold success, failure, retry status and the 1000 * response data. Safe to allocate on the stack as the xarray is the 1001 * only reference and it cannot be present after this function exits. 1002 */ 1003 retry: 1004 g2h_fence_init(&g2h_fence, response_buffer); 1005 retry_same_fence: 1006 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence); 1007 if (unlikely(ret == -ENOMEM)) { 1008 /* Retry allocation /w GFP_KERNEL */ 1009 ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno, 1010 &g2h_fence, GFP_KERNEL)); 1011 if (ret) 1012 return ret; 1013 1014 goto retry_same_fence; 1015 } else if (unlikely(ret)) { 1016 if (ret == -EDEADLK) 1017 kick_reset(ct); 1018 1019 if (no_fail && retry_failure(ct, ret)) 1020 goto retry_same_fence; 1021 1022 if (!g2h_fence_needs_alloc(&g2h_fence)) 1023 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1024 1025 return ret; 1026 } 1027 1028 ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ); 1029 if (!ret) { 1030 LNL_FLUSH_WORK(&ct->g2h_worker); 1031 if (g2h_fence.done) { 1032 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", 1033 g2h_fence.seqno, action[0]); 1034 ret = 1; 1035 } 1036 } 1037 1038 /* 1039 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on 1040 * the stack, since we have no clue if it will fire after the timeout before we can erase 1041 * from the xa. Also we have some dependent loads and stores below for which we need the 1042 * correct ordering, and we lack the needed barriers. 1043 */ 1044 mutex_lock(&ct->lock); 1045 if (!ret) { 1046 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s", 1047 g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done)); 1048 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1049 mutex_unlock(&ct->lock); 1050 return -ETIME; 1051 } 1052 1053 if (g2h_fence.retry) { 1054 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", 1055 action[0], g2h_fence.reason); 1056 mutex_unlock(&ct->lock); 1057 goto retry; 1058 } 1059 if (g2h_fence.fail) { 1060 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n", 1061 action[0], g2h_fence.error, g2h_fence.hint); 1062 ret = -EIO; 1063 } 1064 1065 if (ret > 0) 1066 ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data; 1067 1068 mutex_unlock(&ct->lock); 1069 1070 return ret; 1071 } 1072 1073 /** 1074 * xe_guc_ct_send_recv - Send and receive HXG to the GuC 1075 * @ct: the &xe_guc_ct 1076 * @action: the dword array with `HXG Request`_ message (can't be NULL) 1077 * @len: length of the `HXG Request`_ message (in dwords, can't be 0) 1078 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL) 1079 * 1080 * Send a `HXG Request`_ message to the GuC over CT communication channel and 1081 * blocks until GuC replies with a `HXG Response`_ message. 1082 * 1083 * For non-blocking communication with GuC use xe_guc_ct_send(). 1084 * 1085 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_. 1086 * 1087 * Return: response length (in dwords) if &response_buffer was not NULL, or 1088 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or 1089 * a negative error code on failure. 1090 */ 1091 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1092 u32 *response_buffer) 1093 { 1094 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer); 1095 return guc_ct_send_recv(ct, action, len, response_buffer, false); 1096 } 1097 ALLOW_ERROR_INJECTION(xe_guc_ct_send_recv, ERRNO); 1098 1099 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, 1100 u32 len, u32 *response_buffer) 1101 { 1102 return guc_ct_send_recv(ct, action, len, response_buffer, true); 1103 } 1104 1105 static u32 *msg_to_hxg(u32 *msg) 1106 { 1107 return msg + GUC_CTB_MSG_MIN_LEN; 1108 } 1109 1110 static u32 msg_len_to_hxg_len(u32 len) 1111 { 1112 return len - GUC_CTB_MSG_MIN_LEN; 1113 } 1114 1115 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) 1116 { 1117 u32 *hxg = msg_to_hxg(msg); 1118 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1119 1120 lockdep_assert_held(&ct->lock); 1121 1122 switch (action) { 1123 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1124 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1125 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1126 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1127 g2h_release_space(ct, len); 1128 } 1129 1130 return 0; 1131 } 1132 1133 static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) 1134 { 1135 struct xe_gt *gt = ct_to_gt(ct); 1136 1137 if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED) 1138 xe_gt_err(gt, "GuC Crash dump notification\n"); 1139 else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION) 1140 xe_gt_err(gt, "GuC Exception notification\n"); 1141 else 1142 xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action); 1143 1144 CT_DEAD(ct, NULL, CRASH); 1145 1146 kick_reset(ct); 1147 1148 return 0; 1149 } 1150 1151 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) 1152 { 1153 struct xe_gt *gt = ct_to_gt(ct); 1154 u32 *hxg = msg_to_hxg(msg); 1155 u32 hxg_len = msg_len_to_hxg_len(len); 1156 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); 1157 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1158 struct g2h_fence *g2h_fence; 1159 1160 lockdep_assert_held(&ct->lock); 1161 1162 /* 1163 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup. 1164 * Those messages should never fail, so if we do get an error back it 1165 * means we're likely doing an illegal operation and the GuC is 1166 * rejecting it. We have no way to inform the code that submitted the 1167 * H2G that the message was rejected, so we need to escalate the 1168 * failure to trigger a reset. 1169 */ 1170 if (fence & CT_SEQNO_UNTRACKED) { 1171 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) 1172 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n", 1173 fence, 1174 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]), 1175 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0])); 1176 else 1177 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n", 1178 type, fence); 1179 CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE); 1180 1181 return -EPROTO; 1182 } 1183 1184 g2h_fence = xa_erase(&ct->fence_lookup, fence); 1185 if (unlikely(!g2h_fence)) { 1186 /* Don't tear down channel, as send could've timed out */ 1187 /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ 1188 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence); 1189 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1190 return 0; 1191 } 1192 1193 xe_gt_assert(gt, fence == g2h_fence->seqno); 1194 1195 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { 1196 g2h_fence->fail = true; 1197 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); 1198 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]); 1199 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 1200 g2h_fence->retry = true; 1201 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); 1202 } else if (g2h_fence->response_buffer) { 1203 g2h_fence->response_len = hxg_len; 1204 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); 1205 } else { 1206 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); 1207 } 1208 1209 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1210 1211 g2h_fence->done = true; 1212 smp_mb(); 1213 1214 wake_up_all(&ct->g2h_fence_wq); 1215 1216 return 0; 1217 } 1218 1219 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1220 { 1221 struct xe_gt *gt = ct_to_gt(ct); 1222 u32 *hxg = msg_to_hxg(msg); 1223 u32 origin, type; 1224 int ret; 1225 1226 lockdep_assert_held(&ct->lock); 1227 1228 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1229 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1230 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n", 1231 origin); 1232 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN); 1233 1234 return -EPROTO; 1235 } 1236 1237 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1238 switch (type) { 1239 case GUC_HXG_TYPE_EVENT: 1240 ret = parse_g2h_event(ct, msg, len); 1241 break; 1242 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1243 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1244 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1245 ret = parse_g2h_response(ct, msg, len); 1246 break; 1247 default: 1248 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n", 1249 type); 1250 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE); 1251 1252 ret = -EOPNOTSUPP; 1253 } 1254 1255 return ret; 1256 } 1257 1258 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1259 { 1260 struct xe_guc *guc = ct_to_guc(ct); 1261 struct xe_gt *gt = ct_to_gt(ct); 1262 u32 hxg_len = msg_len_to_hxg_len(len); 1263 u32 *hxg = msg_to_hxg(msg); 1264 u32 action, adj_len; 1265 u32 *payload; 1266 int ret = 0; 1267 1268 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1269 return 0; 1270 1271 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1272 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN; 1273 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN; 1274 1275 switch (action) { 1276 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1277 ret = xe_guc_sched_done_handler(guc, payload, adj_len); 1278 break; 1279 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1280 ret = xe_guc_deregister_done_handler(guc, payload, adj_len); 1281 break; 1282 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1283 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len); 1284 break; 1285 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1286 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload, 1287 adj_len); 1288 break; 1289 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1290 /* Selftest only at the moment */ 1291 break; 1292 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1293 ret = xe_guc_error_capture_handler(guc, payload, adj_len); 1294 break; 1295 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1296 /* FIXME: Handle this */ 1297 break; 1298 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: 1299 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload, 1300 adj_len); 1301 break; 1302 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1303 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1304 break; 1305 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1306 ret = xe_guc_tlb_invalidation_done_handler(guc, payload, 1307 adj_len); 1308 break; 1309 case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY: 1310 ret = xe_guc_access_counter_notify_handler(guc, payload, 1311 adj_len); 1312 break; 1313 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: 1314 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len); 1315 break; 1316 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF: 1317 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len); 1318 break; 1319 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY: 1320 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len); 1321 break; 1322 case GUC_ACTION_GUC2PF_ADVERSE_EVENT: 1323 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len); 1324 break; 1325 case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1326 case XE_GUC_ACTION_NOTIFY_EXCEPTION: 1327 ret = guc_crash_process_msg(ct, action); 1328 break; 1329 default: 1330 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); 1331 } 1332 1333 if (ret) { 1334 xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n", 1335 action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg); 1336 CT_DEAD(ct, NULL, PROCESS_FAILED); 1337 } 1338 1339 return 0; 1340 } 1341 1342 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) 1343 { 1344 struct xe_device *xe = ct_to_xe(ct); 1345 struct xe_gt *gt = ct_to_gt(ct); 1346 struct guc_ctb *g2h = &ct->ctbs.g2h; 1347 u32 tail, head, len, desc_status; 1348 s32 avail; 1349 u32 action; 1350 u32 *hxg; 1351 1352 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 1353 lockdep_assert_held(&ct->fast_lock); 1354 1355 if (ct->state == XE_GUC_CT_STATE_DISABLED) 1356 return -ENODEV; 1357 1358 if (ct->state == XE_GUC_CT_STATE_STOPPED) 1359 return -ECANCELED; 1360 1361 if (g2h->info.broken) 1362 return -EPIPE; 1363 1364 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1365 1366 desc_status = desc_read(xe, g2h, status); 1367 if (desc_status) { 1368 if (desc_status & GUC_CTB_STATUS_DISABLED) { 1369 /* 1370 * Potentially valid if a CLIENT_RESET request resulted in 1371 * contexts/engines being reset. But should never happen as 1372 * no contexts should be active when CLIENT_RESET is sent. 1373 */ 1374 xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n"); 1375 desc_status &= ~GUC_CTB_STATUS_DISABLED; 1376 } 1377 1378 if (desc_status) { 1379 xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status); 1380 goto corrupted; 1381 } 1382 } 1383 1384 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 1385 u32 desc_tail = desc_read(xe, g2h, tail); 1386 /* 1387 u32 desc_head = desc_read(xe, g2h, head); 1388 1389 * info.head and desc_head are updated back-to-back at the end of 1390 * this function and nowhere else. Hence, they cannot be different 1391 * unless two g2h_read calls are running concurrently. Which is not 1392 * possible because it is guarded by ct->fast_lock. And yet, some 1393 * discrete platforms are regularly hitting this error :(. 1394 * 1395 * desc_head rolling backwards shouldn't cause any noticeable 1396 * problems - just a delay in GuC being allowed to proceed past that 1397 * point in the queue. So for now, just disable the error until it 1398 * can be root caused. 1399 * 1400 if (g2h->info.head != desc_head) { 1401 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH); 1402 xe_gt_err(gt, "CT read: head was modified %u != %u\n", 1403 desc_head, g2h->info.head); 1404 goto corrupted; 1405 } 1406 */ 1407 1408 if (g2h->info.head > g2h->info.size) { 1409 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1410 xe_gt_err(gt, "CT read: head out of range: %u vs %u\n", 1411 g2h->info.head, g2h->info.size); 1412 goto corrupted; 1413 } 1414 1415 if (desc_tail >= g2h->info.size) { 1416 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1417 xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n", 1418 desc_tail, g2h->info.size); 1419 goto corrupted; 1420 } 1421 } 1422 1423 /* Calculate DW available to read */ 1424 tail = desc_read(xe, g2h, tail); 1425 avail = tail - g2h->info.head; 1426 if (unlikely(avail == 0)) 1427 return 0; 1428 1429 if (avail < 0) 1430 avail += g2h->info.size; 1431 1432 /* Read header */ 1433 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head, 1434 sizeof(u32)); 1435 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; 1436 if (len > avail) { 1437 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n", 1438 avail, len); 1439 goto corrupted; 1440 } 1441 1442 head = (g2h->info.head + 1) % g2h->info.size; 1443 avail = len - 1; 1444 1445 /* Read G2H message */ 1446 if (avail + head > g2h->info.size) { 1447 u32 avail_til_wrap = g2h->info.size - head; 1448 1449 xe_map_memcpy_from(xe, msg + 1, 1450 &g2h->cmds, sizeof(u32) * head, 1451 avail_til_wrap * sizeof(u32)); 1452 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap, 1453 &g2h->cmds, 0, 1454 (avail - avail_til_wrap) * sizeof(u32)); 1455 } else { 1456 xe_map_memcpy_from(xe, msg + 1, 1457 &g2h->cmds, sizeof(u32) * head, 1458 avail * sizeof(u32)); 1459 } 1460 1461 hxg = msg_to_hxg(msg); 1462 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1463 1464 if (fast_path) { 1465 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1466 return 0; 1467 1468 switch (action) { 1469 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1470 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1471 break; /* Process these in fast-path */ 1472 default: 1473 return 0; 1474 } 1475 } 1476 1477 /* Update local / descriptor header */ 1478 g2h->info.head = (head + avail) % g2h->info.size; 1479 desc_write(xe, g2h, head, g2h->info.head); 1480 1481 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id, 1482 action, len, g2h->info.head, tail); 1483 1484 return len; 1485 1486 corrupted: 1487 CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ); 1488 return -EPROTO; 1489 } 1490 1491 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) 1492 { 1493 struct xe_gt *gt = ct_to_gt(ct); 1494 struct xe_guc *guc = ct_to_guc(ct); 1495 u32 hxg_len = msg_len_to_hxg_len(len); 1496 u32 *hxg = msg_to_hxg(msg); 1497 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1498 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN; 1499 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1500 int ret = 0; 1501 1502 switch (action) { 1503 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1504 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1505 break; 1506 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1507 __g2h_release_space(ct, len); 1508 ret = xe_guc_tlb_invalidation_done_handler(guc, payload, 1509 adj_len); 1510 break; 1511 default: 1512 xe_gt_warn(gt, "NOT_POSSIBLE"); 1513 } 1514 1515 if (ret) { 1516 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n", 1517 action, ERR_PTR(ret)); 1518 CT_DEAD(ct, NULL, FAST_G2H); 1519 } 1520 } 1521 1522 /** 1523 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler 1524 * @ct: GuC CT object 1525 * 1526 * Anything related to page faults is critical for performance, process these 1527 * critical G2H in the IRQ. This is safe as these handlers either just wake up 1528 * waiters or queue another worker. 1529 */ 1530 void xe_guc_ct_fast_path(struct xe_guc_ct *ct) 1531 { 1532 struct xe_device *xe = ct_to_xe(ct); 1533 bool ongoing; 1534 int len; 1535 1536 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1537 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1538 return; 1539 1540 spin_lock(&ct->fast_lock); 1541 do { 1542 len = g2h_read(ct, ct->fast_msg, true); 1543 if (len > 0) 1544 g2h_fast_path(ct, ct->fast_msg, len); 1545 } while (len > 0); 1546 spin_unlock(&ct->fast_lock); 1547 1548 if (ongoing) 1549 xe_pm_runtime_put(xe); 1550 } 1551 1552 /* Returns less than zero on error, 0 on done, 1 on more available */ 1553 static int dequeue_one_g2h(struct xe_guc_ct *ct) 1554 { 1555 int len; 1556 int ret; 1557 1558 lockdep_assert_held(&ct->lock); 1559 1560 spin_lock_irq(&ct->fast_lock); 1561 len = g2h_read(ct, ct->msg, false); 1562 spin_unlock_irq(&ct->fast_lock); 1563 if (len <= 0) 1564 return len; 1565 1566 ret = parse_g2h_msg(ct, ct->msg, len); 1567 if (unlikely(ret < 0)) 1568 return ret; 1569 1570 ret = process_g2h_msg(ct, ct->msg, len); 1571 if (unlikely(ret < 0)) 1572 return ret; 1573 1574 return 1; 1575 } 1576 1577 static void receive_g2h(struct xe_guc_ct *ct) 1578 { 1579 bool ongoing; 1580 int ret; 1581 1582 /* 1583 * Normal users must always hold mem_access.ref around CT calls. However 1584 * during the runtime pm callbacks we rely on CT to talk to the GuC, but 1585 * at this stage we can't rely on mem_access.ref and even the 1586 * callback_task will be different than current. For such cases we just 1587 * need to ensure we always process the responses from any blocking 1588 * ct_send requests or where we otherwise expect some response when 1589 * initiated from those callbacks (which will need to wait for the below 1590 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if 1591 * the device has suspended to the point that the CT communication has 1592 * been disabled. 1593 * 1594 * If we are inside the runtime pm callback, we can be the only task 1595 * still issuing CT requests (since that requires having the 1596 * mem_access.ref). It seems like it might in theory be possible to 1597 * receive unsolicited events from the GuC just as we are 1598 * suspending-resuming, but those will currently anyway be lost when 1599 * eventually exiting from suspend, hence no need to wake up the device 1600 * here. If we ever need something stronger than get_if_ongoing() then 1601 * we need to be careful with blocking the pm callbacks from getting CT 1602 * responses, if the worker here is blocked on those callbacks 1603 * completing, creating a deadlock. 1604 */ 1605 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1606 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1607 return; 1608 1609 do { 1610 mutex_lock(&ct->lock); 1611 ret = dequeue_one_g2h(ct); 1612 mutex_unlock(&ct->lock); 1613 1614 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) { 1615 xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d", ret); 1616 CT_DEAD(ct, NULL, G2H_RECV); 1617 kick_reset(ct); 1618 } 1619 } while (ret == 1); 1620 1621 if (ongoing) 1622 xe_pm_runtime_put(ct_to_xe(ct)); 1623 } 1624 1625 static void g2h_worker_func(struct work_struct *w) 1626 { 1627 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker); 1628 1629 receive_g2h(ct); 1630 } 1631 1632 static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic, 1633 bool want_ctb) 1634 { 1635 struct xe_guc_ct_snapshot *snapshot; 1636 1637 snapshot = kzalloc(sizeof(*snapshot), atomic ? GFP_ATOMIC : GFP_KERNEL); 1638 if (!snapshot) 1639 return NULL; 1640 1641 if (ct->bo && want_ctb) { 1642 snapshot->ctb_size = ct->bo->size; 1643 snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL); 1644 } 1645 1646 return snapshot; 1647 } 1648 1649 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb, 1650 struct guc_ctb_snapshot *snapshot) 1651 { 1652 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0, 1653 sizeof(struct guc_ct_buffer_desc)); 1654 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info)); 1655 } 1656 1657 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot, 1658 struct drm_printer *p) 1659 { 1660 drm_printf(p, "\tsize: %d\n", snapshot->info.size); 1661 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space); 1662 drm_printf(p, "\thead: %d\n", snapshot->info.head); 1663 drm_printf(p, "\ttail: %d\n", snapshot->info.tail); 1664 drm_printf(p, "\tspace: %d\n", snapshot->info.space); 1665 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken); 1666 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head); 1667 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail); 1668 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status); 1669 } 1670 1671 static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic, 1672 bool want_ctb) 1673 { 1674 struct xe_device *xe = ct_to_xe(ct); 1675 struct xe_guc_ct_snapshot *snapshot; 1676 1677 snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb); 1678 if (!snapshot) { 1679 xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n"); 1680 return NULL; 1681 } 1682 1683 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) { 1684 snapshot->ct_enabled = true; 1685 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding); 1686 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g); 1687 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h); 1688 } 1689 1690 if (ct->bo && snapshot->ctb) 1691 xe_map_memcpy_from(xe, snapshot->ctb, &ct->bo->vmap, 0, snapshot->ctb_size); 1692 1693 return snapshot; 1694 } 1695 1696 /** 1697 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state. 1698 * @ct: GuC CT object. 1699 * 1700 * This can be printed out in a later stage like during dev_coredump 1701 * analysis. This is safe to be called during atomic context. 1702 * 1703 * Returns: a GuC CT snapshot object that must be freed by the caller 1704 * by using `xe_guc_ct_snapshot_free`. 1705 */ 1706 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct) 1707 { 1708 return guc_ct_snapshot_capture(ct, true, true); 1709 } 1710 1711 /** 1712 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot. 1713 * @snapshot: GuC CT snapshot object. 1714 * @p: drm_printer where it will be printed out. 1715 * 1716 * This function prints out a given GuC CT snapshot object. 1717 */ 1718 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, 1719 struct drm_printer *p) 1720 { 1721 if (!snapshot) 1722 return; 1723 1724 if (snapshot->ct_enabled) { 1725 drm_puts(p, "H2G CTB (all sizes in DW):\n"); 1726 guc_ctb_snapshot_print(&snapshot->h2g, p); 1727 1728 drm_puts(p, "G2H CTB (all sizes in DW):\n"); 1729 guc_ctb_snapshot_print(&snapshot->g2h, p); 1730 drm_printf(p, "\tg2h outstanding: %d\n", 1731 snapshot->g2h_outstanding); 1732 1733 if (snapshot->ctb) { 1734 drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size); 1735 xe_print_blob_ascii85(p, "[CTB].data", '\n', 1736 snapshot->ctb, 0, snapshot->ctb_size); 1737 } 1738 } else { 1739 drm_puts(p, "CT disabled\n"); 1740 } 1741 } 1742 1743 /** 1744 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot. 1745 * @snapshot: GuC CT snapshot object. 1746 * 1747 * This function free all the memory that needed to be allocated at capture 1748 * time. 1749 */ 1750 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot) 1751 { 1752 if (!snapshot) 1753 return; 1754 1755 kfree(snapshot->ctb); 1756 kfree(snapshot); 1757 } 1758 1759 /** 1760 * xe_guc_ct_print - GuC CT Print. 1761 * @ct: GuC CT. 1762 * @p: drm_printer where it will be printed out. 1763 * @want_ctb: Should the full CTB content be dumped (vs just the headers) 1764 * 1765 * This function will quickly capture a snapshot of the CT state 1766 * and immediately print it out. 1767 */ 1768 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb) 1769 { 1770 struct xe_guc_ct_snapshot *snapshot; 1771 1772 snapshot = guc_ct_snapshot_capture(ct, false, want_ctb); 1773 xe_guc_ct_snapshot_print(snapshot, p); 1774 xe_guc_ct_snapshot_free(snapshot); 1775 } 1776 1777 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 1778 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code) 1779 { 1780 struct xe_guc_log_snapshot *snapshot_log; 1781 struct xe_guc_ct_snapshot *snapshot_ct; 1782 struct xe_guc *guc = ct_to_guc(ct); 1783 unsigned long flags; 1784 bool have_capture; 1785 1786 if (ctb) 1787 ctb->info.broken = true; 1788 1789 /* Ignore further errors after the first dump until a reset */ 1790 if (ct->dead.reported) 1791 return; 1792 1793 spin_lock_irqsave(&ct->dead.lock, flags); 1794 1795 /* And only capture one dump at a time */ 1796 have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE); 1797 ct->dead.reason |= (1 << reason_code) | 1798 (1 << CT_DEAD_STATE_CAPTURE); 1799 1800 spin_unlock_irqrestore(&ct->dead.lock, flags); 1801 1802 if (have_capture) 1803 return; 1804 1805 snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true); 1806 snapshot_ct = xe_guc_ct_snapshot_capture((ct)); 1807 1808 spin_lock_irqsave(&ct->dead.lock, flags); 1809 1810 if (ct->dead.snapshot_log || ct->dead.snapshot_ct) { 1811 xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n"); 1812 xe_guc_log_snapshot_free(snapshot_log); 1813 xe_guc_ct_snapshot_free(snapshot_ct); 1814 } else { 1815 ct->dead.snapshot_log = snapshot_log; 1816 ct->dead.snapshot_ct = snapshot_ct; 1817 } 1818 1819 spin_unlock_irqrestore(&ct->dead.lock, flags); 1820 1821 queue_work(system_unbound_wq, &(ct)->dead.worker); 1822 } 1823 1824 static void ct_dead_print(struct xe_dead_ct *dead) 1825 { 1826 struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead); 1827 struct xe_device *xe = ct_to_xe(ct); 1828 struct xe_gt *gt = ct_to_gt(ct); 1829 static int g_count; 1830 struct drm_printer ip = xe_gt_info_printer(gt); 1831 struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count); 1832 1833 if (!dead->reason) { 1834 xe_gt_err(gt, "CTB is dead for no reason!?\n"); 1835 return; 1836 } 1837 1838 1839 /* Can't generate a genuine core dump at this point, so just do the good bits */ 1840 drm_puts(&lp, "**** Xe Device Coredump ****\n"); 1841 drm_printf(&lp, "Reason: CTB is dead - 0x%X\n", dead->reason); 1842 xe_device_snapshot_print(xe, &lp); 1843 1844 drm_printf(&lp, "**** GT #%d ****\n", gt->info.id); 1845 drm_printf(&lp, "\tTile: %d\n", gt->tile->id); 1846 1847 drm_puts(&lp, "**** GuC Log ****\n"); 1848 xe_guc_log_snapshot_print(dead->snapshot_log, &lp); 1849 1850 drm_puts(&lp, "**** GuC CT ****\n"); 1851 xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp); 1852 1853 drm_puts(&lp, "Done.\n"); 1854 } 1855 1856 static void ct_dead_worker_func(struct work_struct *w) 1857 { 1858 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker); 1859 1860 if (!ct->dead.reported) { 1861 ct->dead.reported = true; 1862 ct_dead_print(&ct->dead); 1863 } 1864 1865 spin_lock_irq(&ct->dead.lock); 1866 1867 xe_guc_log_snapshot_free(ct->dead.snapshot_log); 1868 ct->dead.snapshot_log = NULL; 1869 xe_guc_ct_snapshot_free(ct->dead.snapshot_ct); 1870 ct->dead.snapshot_ct = NULL; 1871 1872 if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) { 1873 /* A reset has occurred so re-arm the error reporting */ 1874 ct->dead.reason = 0; 1875 ct->dead.reported = false; 1876 } 1877 1878 spin_unlock_irq(&ct->dead.lock); 1879 } 1880 #endif 1881