1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ct.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 #include <linux/delay.h> 11 #include <linux/fault-inject.h> 12 13 #include <kunit/static_stub.h> 14 15 #include <drm/drm_managed.h> 16 17 #include "abi/guc_actions_abi.h" 18 #include "abi/guc_actions_sriov_abi.h" 19 #include "abi/guc_klvs_abi.h" 20 #include "xe_bo.h" 21 #include "xe_devcoredump.h" 22 #include "xe_device.h" 23 #include "xe_gt.h" 24 #include "xe_gt_pagefault.h" 25 #include "xe_gt_printk.h" 26 #include "xe_gt_sriov_pf_control.h" 27 #include "xe_gt_sriov_pf_monitor.h" 28 #include "xe_gt_tlb_invalidation.h" 29 #include "xe_guc.h" 30 #include "xe_guc_log.h" 31 #include "xe_guc_relay.h" 32 #include "xe_guc_submit.h" 33 #include "xe_map.h" 34 #include "xe_pm.h" 35 #include "xe_trace_guc.h" 36 37 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 38 enum { 39 /* Internal states, not error conditions */ 40 CT_DEAD_STATE_REARM, /* 0x0001 */ 41 CT_DEAD_STATE_CAPTURE, /* 0x0002 */ 42 43 /* Error conditions */ 44 CT_DEAD_SETUP, /* 0x0004 */ 45 CT_DEAD_H2G_WRITE, /* 0x0008 */ 46 CT_DEAD_H2G_HAS_ROOM, /* 0x0010 */ 47 CT_DEAD_G2H_READ, /* 0x0020 */ 48 CT_DEAD_G2H_RECV, /* 0x0040 */ 49 CT_DEAD_G2H_RELEASE, /* 0x0080 */ 50 CT_DEAD_DEADLOCK, /* 0x0100 */ 51 CT_DEAD_PROCESS_FAILED, /* 0x0200 */ 52 CT_DEAD_FAST_G2H, /* 0x0400 */ 53 CT_DEAD_PARSE_G2H_RESPONSE, /* 0x0800 */ 54 CT_DEAD_PARSE_G2H_UNKNOWN, /* 0x1000 */ 55 CT_DEAD_PARSE_G2H_ORIGIN, /* 0x2000 */ 56 CT_DEAD_PARSE_G2H_TYPE, /* 0x4000 */ 57 CT_DEAD_CRASH, /* 0x8000 */ 58 }; 59 60 static void ct_dead_worker_func(struct work_struct *w); 61 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code); 62 63 #define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code) 64 #else 65 #define CT_DEAD(ct, ctb, reason) \ 66 do { \ 67 struct guc_ctb *_ctb = (ctb); \ 68 if (_ctb) \ 69 _ctb->info.broken = true; \ 70 } while (0) 71 #endif 72 73 /* Used when a CT send wants to block and / or receive data */ 74 struct g2h_fence { 75 u32 *response_buffer; 76 u32 seqno; 77 u32 response_data; 78 u16 response_len; 79 u16 error; 80 u16 hint; 81 u16 reason; 82 bool retry; 83 bool fail; 84 bool done; 85 }; 86 87 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) 88 { 89 g2h_fence->response_buffer = response_buffer; 90 g2h_fence->response_data = 0; 91 g2h_fence->response_len = 0; 92 g2h_fence->fail = false; 93 g2h_fence->retry = false; 94 g2h_fence->done = false; 95 g2h_fence->seqno = ~0x0; 96 } 97 98 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) 99 { 100 return g2h_fence->seqno == ~0x0; 101 } 102 103 static struct xe_guc * 104 ct_to_guc(struct xe_guc_ct *ct) 105 { 106 return container_of(ct, struct xe_guc, ct); 107 } 108 109 static struct xe_gt * 110 ct_to_gt(struct xe_guc_ct *ct) 111 { 112 return container_of(ct, struct xe_gt, uc.guc.ct); 113 } 114 115 static struct xe_device * 116 ct_to_xe(struct xe_guc_ct *ct) 117 { 118 return gt_to_xe(ct_to_gt(ct)); 119 } 120 121 /** 122 * DOC: GuC CTB Blob 123 * 124 * We allocate single blob to hold both CTB descriptors and buffers: 125 * 126 * +--------+-----------------------------------------------+------+ 127 * | offset | contents | size | 128 * +========+===============================================+======+ 129 * | 0x0000 | H2G CTB Descriptor (send) | | 130 * +--------+-----------------------------------------------+ 4K | 131 * | 0x0800 | G2H CTB Descriptor (g2h) | | 132 * +--------+-----------------------------------------------+------+ 133 * | 0x1000 | H2G CT Buffer (send) | n*4K | 134 * | | | | 135 * +--------+-----------------------------------------------+------+ 136 * | 0x1000 | G2H CT Buffer (g2h) | m*4K | 137 * | + n*4K | | | 138 * +--------+-----------------------------------------------+------+ 139 * 140 * Size of each ``CT Buffer`` must be multiple of 4K. 141 * We don't expect too many messages in flight at any time, unless we are 142 * using the GuC submission. In that case each request requires a minimum 143 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 144 * enough space to avoid backpressure on the driver. We increase the size 145 * of the receive buffer (relative to the send) to ensure a G2H response 146 * CTB has a landing spot. 147 * 148 * In addition to submissions, the G2H buffer needs to be able to hold 149 * enough space for recoverable page fault notifications. The number of 150 * page faults is interrupt driven and can be as much as the number of 151 * compute resources available. However, most of the actual work for these 152 * is in a separate page fault worker thread. Therefore we only need to 153 * make sure the queue has enough space to handle all of the submissions 154 * and responses and an extra buffer for incoming page faults. 155 */ 156 157 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 158 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 159 #define CTB_G2H_BUFFER_SIZE (SZ_128K) 160 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2) 161 162 /** 163 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full 164 * CT command queue 165 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future. 166 * 167 * Observation is that a 4KiB buffer full of commands takes a little over a 168 * second to process. Use that to calculate maximum time to process a full CT 169 * command queue. 170 * 171 * Return: Maximum time to process a full CT queue in jiffies. 172 */ 173 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct) 174 { 175 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4)); 176 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ; 177 } 178 179 static size_t guc_ct_size(void) 180 { 181 return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + 182 CTB_G2H_BUFFER_SIZE; 183 } 184 185 static void guc_ct_fini(struct drm_device *drm, void *arg) 186 { 187 struct xe_guc_ct *ct = arg; 188 189 destroy_workqueue(ct->g2h_wq); 190 xa_destroy(&ct->fence_lookup); 191 } 192 193 static void receive_g2h(struct xe_guc_ct *ct); 194 static void g2h_worker_func(struct work_struct *w); 195 static void safe_mode_worker_func(struct work_struct *w); 196 197 static void primelockdep(struct xe_guc_ct *ct) 198 { 199 if (!IS_ENABLED(CONFIG_LOCKDEP)) 200 return; 201 202 fs_reclaim_acquire(GFP_KERNEL); 203 might_lock(&ct->lock); 204 fs_reclaim_release(GFP_KERNEL); 205 } 206 207 int xe_guc_ct_init(struct xe_guc_ct *ct) 208 { 209 struct xe_device *xe = ct_to_xe(ct); 210 struct xe_gt *gt = ct_to_gt(ct); 211 struct xe_tile *tile = gt_to_tile(gt); 212 struct xe_bo *bo; 213 int err; 214 215 xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE)); 216 217 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM); 218 if (!ct->g2h_wq) 219 return -ENOMEM; 220 221 spin_lock_init(&ct->fast_lock); 222 xa_init(&ct->fence_lookup); 223 INIT_WORK(&ct->g2h_worker, g2h_worker_func); 224 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func); 225 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 226 spin_lock_init(&ct->dead.lock); 227 INIT_WORK(&ct->dead.worker, ct_dead_worker_func); 228 #endif 229 init_waitqueue_head(&ct->wq); 230 init_waitqueue_head(&ct->g2h_fence_wq); 231 232 err = drmm_mutex_init(&xe->drm, &ct->lock); 233 if (err) 234 return err; 235 236 primelockdep(ct); 237 238 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(), 239 XE_BO_FLAG_SYSTEM | 240 XE_BO_FLAG_GGTT | 241 XE_BO_FLAG_GGTT_INVALIDATE | 242 XE_BO_FLAG_PINNED_NORESTORE); 243 if (IS_ERR(bo)) 244 return PTR_ERR(bo); 245 246 ct->bo = bo; 247 248 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); 249 if (err) 250 return err; 251 252 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED); 253 ct->state = XE_GUC_CT_STATE_DISABLED; 254 return 0; 255 } 256 ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */ 257 258 #define desc_read(xe_, guc_ctb__, field_) \ 259 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \ 260 struct guc_ct_buffer_desc, field_) 261 262 #define desc_write(xe_, guc_ctb__, field_, val_) \ 263 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \ 264 struct guc_ct_buffer_desc, field_, val_) 265 266 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, 267 struct iosys_map *map) 268 { 269 h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32); 270 h2g->info.resv_space = 0; 271 h2g->info.tail = 0; 272 h2g->info.head = 0; 273 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 274 h2g->info.size) - 275 h2g->info.resv_space; 276 h2g->info.broken = false; 277 278 h2g->desc = *map; 279 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 280 281 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2); 282 } 283 284 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, 285 struct iosys_map *map) 286 { 287 g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32); 288 g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32); 289 g2h->info.head = 0; 290 g2h->info.tail = 0; 291 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head, 292 g2h->info.size) - 293 g2h->info.resv_space; 294 g2h->info.broken = false; 295 296 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE); 297 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 298 299 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 + 300 CTB_H2G_BUFFER_SIZE); 301 } 302 303 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct) 304 { 305 struct xe_guc *guc = ct_to_guc(ct); 306 u32 desc_addr, ctb_addr, size; 307 int err; 308 309 desc_addr = xe_bo_ggtt_addr(ct->bo); 310 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2; 311 size = ct->ctbs.h2g.info.size * sizeof(u32); 312 313 err = xe_guc_self_cfg64(guc, 314 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY, 315 desc_addr); 316 if (err) 317 return err; 318 319 err = xe_guc_self_cfg64(guc, 320 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY, 321 ctb_addr); 322 if (err) 323 return err; 324 325 return xe_guc_self_cfg32(guc, 326 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY, 327 size); 328 } 329 330 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct) 331 { 332 struct xe_guc *guc = ct_to_guc(ct); 333 u32 desc_addr, ctb_addr, size; 334 int err; 335 336 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE; 337 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 + 338 CTB_H2G_BUFFER_SIZE; 339 size = ct->ctbs.g2h.info.size * sizeof(u32); 340 341 err = xe_guc_self_cfg64(guc, 342 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 343 desc_addr); 344 if (err) 345 return err; 346 347 err = xe_guc_self_cfg64(guc, 348 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 349 ctb_addr); 350 if (err) 351 return err; 352 353 return xe_guc_self_cfg32(guc, 354 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 355 size); 356 } 357 358 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) 359 { 360 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 361 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 362 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 363 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, 364 GUC_ACTION_HOST2GUC_CONTROL_CTB), 365 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, 366 enable ? GUC_CTB_CONTROL_ENABLE : 367 GUC_CTB_CONTROL_DISABLE), 368 }; 369 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request)); 370 371 return ret > 0 ? -EPROTO : ret; 372 } 373 374 static void xe_guc_ct_set_state(struct xe_guc_ct *ct, 375 enum xe_guc_ct_state state) 376 { 377 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ 378 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ 379 380 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 || 381 state == XE_GUC_CT_STATE_STOPPED); 382 383 if (ct->g2h_outstanding) 384 xe_pm_runtime_put(ct_to_xe(ct)); 385 ct->g2h_outstanding = 0; 386 ct->state = state; 387 388 spin_unlock_irq(&ct->fast_lock); 389 390 /* 391 * Lockdep doesn't like this under the fast lock and he destroy only 392 * needs to be serialized with the send path which ct lock provides. 393 */ 394 xa_destroy(&ct->fence_lookup); 395 396 mutex_unlock(&ct->lock); 397 } 398 399 static bool ct_needs_safe_mode(struct xe_guc_ct *ct) 400 { 401 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev)); 402 } 403 404 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct) 405 { 406 if (!ct_needs_safe_mode(ct)) 407 return false; 408 409 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10); 410 return true; 411 } 412 413 static void safe_mode_worker_func(struct work_struct *w) 414 { 415 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work); 416 417 receive_g2h(ct); 418 419 if (!ct_restart_safe_mode_worker(ct)) 420 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n"); 421 } 422 423 static void ct_enter_safe_mode(struct xe_guc_ct *ct) 424 { 425 if (ct_restart_safe_mode_worker(ct)) 426 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n"); 427 } 428 429 static void ct_exit_safe_mode(struct xe_guc_ct *ct) 430 { 431 if (cancel_delayed_work_sync(&ct->safe_mode_worker)) 432 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n"); 433 } 434 435 int xe_guc_ct_enable(struct xe_guc_ct *ct) 436 { 437 struct xe_device *xe = ct_to_xe(ct); 438 struct xe_gt *gt = ct_to_gt(ct); 439 int err; 440 441 xe_gt_assert(gt, !xe_guc_ct_enabled(ct)); 442 443 xe_map_memset(xe, &ct->bo->vmap, 0, 0, ct->bo->size); 444 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap); 445 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap); 446 447 err = guc_ct_ctb_h2g_register(ct); 448 if (err) 449 goto err_out; 450 451 err = guc_ct_ctb_g2h_register(ct); 452 if (err) 453 goto err_out; 454 455 err = guc_ct_control_toggle(ct, true); 456 if (err) 457 goto err_out; 458 459 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED); 460 461 smp_mb(); 462 wake_up_all(&ct->wq); 463 xe_gt_dbg(gt, "GuC CT communication channel enabled\n"); 464 465 if (ct_needs_safe_mode(ct)) 466 ct_enter_safe_mode(ct); 467 468 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 469 /* 470 * The CT has now been reset so the dumper can be re-armed 471 * after any existing dead state has been dumped. 472 */ 473 spin_lock_irq(&ct->dead.lock); 474 if (ct->dead.reason) { 475 ct->dead.reason |= (1 << CT_DEAD_STATE_REARM); 476 queue_work(system_unbound_wq, &ct->dead.worker); 477 } 478 spin_unlock_irq(&ct->dead.lock); 479 #endif 480 481 return 0; 482 483 err_out: 484 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err)); 485 CT_DEAD(ct, NULL, SETUP); 486 487 return err; 488 } 489 490 static void stop_g2h_handler(struct xe_guc_ct *ct) 491 { 492 cancel_work_sync(&ct->g2h_worker); 493 } 494 495 /** 496 * xe_guc_ct_disable - Set GuC to disabled state 497 * @ct: the &xe_guc_ct 498 * 499 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected 500 * in this transition. 501 */ 502 void xe_guc_ct_disable(struct xe_guc_ct *ct) 503 { 504 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED); 505 ct_exit_safe_mode(ct); 506 stop_g2h_handler(ct); 507 } 508 509 /** 510 * xe_guc_ct_stop - Set GuC to stopped state 511 * @ct: the &xe_guc_ct 512 * 513 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h 514 */ 515 void xe_guc_ct_stop(struct xe_guc_ct *ct) 516 { 517 if (!xe_guc_ct_initialized(ct)) 518 return; 519 520 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED); 521 stop_g2h_handler(ct); 522 } 523 524 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) 525 { 526 struct guc_ctb *h2g = &ct->ctbs.h2g; 527 528 lockdep_assert_held(&ct->lock); 529 530 if (cmd_len > h2g->info.space) { 531 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); 532 533 if (h2g->info.head > h2g->info.size) { 534 struct xe_device *xe = ct_to_xe(ct); 535 u32 desc_status = desc_read(xe, h2g, status); 536 537 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 538 539 xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n", 540 h2g->info.head, h2g->info.size); 541 CT_DEAD(ct, h2g, H2G_HAS_ROOM); 542 return false; 543 } 544 545 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 546 h2g->info.size) - 547 h2g->info.resv_space; 548 if (cmd_len > h2g->info.space) 549 return false; 550 } 551 552 return true; 553 } 554 555 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len) 556 { 557 if (!g2h_len) 558 return true; 559 560 lockdep_assert_held(&ct->fast_lock); 561 562 return ct->ctbs.g2h.info.space > g2h_len; 563 } 564 565 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len) 566 { 567 lockdep_assert_held(&ct->lock); 568 569 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len)) 570 return -EBUSY; 571 572 return 0; 573 } 574 575 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len) 576 { 577 lockdep_assert_held(&ct->lock); 578 ct->ctbs.h2g.info.space -= cmd_len; 579 } 580 581 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h) 582 { 583 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space); 584 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) || 585 (g2h_len && num_g2h)); 586 587 if (g2h_len) { 588 lockdep_assert_held(&ct->fast_lock); 589 590 if (!ct->g2h_outstanding) 591 xe_pm_runtime_get_noresume(ct_to_xe(ct)); 592 593 ct->ctbs.g2h.info.space -= g2h_len; 594 ct->g2h_outstanding += num_g2h; 595 } 596 } 597 598 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 599 { 600 bool bad = false; 601 602 lockdep_assert_held(&ct->fast_lock); 603 604 bad = ct->ctbs.g2h.info.space + g2h_len > 605 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space; 606 bad |= !ct->g2h_outstanding; 607 608 if (bad) { 609 xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n", 610 ct->ctbs.g2h.info.space, g2h_len, 611 ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space, 612 ct->ctbs.g2h.info.space + g2h_len, 613 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space, 614 ct->g2h_outstanding); 615 CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE); 616 return; 617 } 618 619 ct->ctbs.g2h.info.space += g2h_len; 620 if (!--ct->g2h_outstanding) 621 xe_pm_runtime_put(ct_to_xe(ct)); 622 } 623 624 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 625 { 626 spin_lock_irq(&ct->fast_lock); 627 __g2h_release_space(ct, g2h_len); 628 spin_unlock_irq(&ct->fast_lock); 629 } 630 631 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */ 632 633 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, 634 u32 ct_fence_value, bool want_response) 635 { 636 struct xe_device *xe = ct_to_xe(ct); 637 struct xe_gt *gt = ct_to_gt(ct); 638 struct guc_ctb *h2g = &ct->ctbs.h2g; 639 u32 cmd[H2G_CT_HEADERS]; 640 u32 tail = h2g->info.tail; 641 u32 full_len; 642 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds, 643 tail * sizeof(u32)); 644 u32 desc_status; 645 646 full_len = len + GUC_CTB_HDR_LEN; 647 648 lockdep_assert_held(&ct->lock); 649 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN); 650 651 desc_status = desc_read(xe, h2g, status); 652 if (desc_status) { 653 xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status); 654 goto corrupted; 655 } 656 657 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 658 u32 desc_tail = desc_read(xe, h2g, tail); 659 u32 desc_head = desc_read(xe, h2g, head); 660 661 if (tail != desc_tail) { 662 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH); 663 xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail); 664 goto corrupted; 665 } 666 667 if (tail > h2g->info.size) { 668 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 669 xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n", 670 tail, h2g->info.size); 671 goto corrupted; 672 } 673 674 if (desc_head >= h2g->info.size) { 675 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 676 xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n", 677 desc_head, h2g->info.size); 678 goto corrupted; 679 } 680 } 681 682 /* Command will wrap, zero fill (NOPs), return and check credits again */ 683 if (tail + full_len > h2g->info.size) { 684 xe_map_memset(xe, &map, 0, 0, 685 (h2g->info.size - tail) * sizeof(u32)); 686 h2g_reserve_space(ct, (h2g->info.size - tail)); 687 h2g->info.tail = 0; 688 desc_write(xe, h2g, tail, h2g->info.tail); 689 690 return -EAGAIN; 691 } 692 693 /* 694 * dw0: CT header (including fence) 695 * dw1: HXG header (including action code) 696 * dw2+: action data 697 */ 698 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 699 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 700 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value); 701 if (want_response) { 702 cmd[1] = 703 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 704 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 705 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 706 } else { 707 cmd[1] = 708 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) | 709 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 710 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 711 } 712 713 /* H2G header in cmd[1] replaces action[0] so: */ 714 --len; 715 ++action; 716 717 /* Write H2G ensuring visible before descriptor update */ 718 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32)); 719 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32)); 720 xe_device_wmb(xe); 721 722 /* Update local copies */ 723 h2g->info.tail = (tail + full_len) % h2g->info.size; 724 h2g_reserve_space(ct, full_len); 725 726 /* Update descriptor */ 727 desc_write(xe, h2g, tail, h2g->info.tail); 728 729 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len, 730 desc_read(xe, h2g, head), h2g->info.tail); 731 732 return 0; 733 734 corrupted: 735 CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE); 736 return -EPIPE; 737 } 738 739 /* 740 * The CT protocol accepts a 16 bits fence. This field is fully owned by the 741 * driver, the GuC will just copy it to the reply message. Since we need to 742 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages, 743 * we use one bit of the seqno as an indicator for that and a rolling counter 744 * for the remaining 15 bits. 745 */ 746 #define CT_SEQNO_MASK GENMASK(14, 0) 747 #define CT_SEQNO_UNTRACKED BIT(15) 748 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence) 749 { 750 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK; 751 752 if (!is_g2h_fence) 753 seqno |= CT_SEQNO_UNTRACKED; 754 755 return seqno; 756 } 757 758 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, 759 u32 len, u32 g2h_len, u32 num_g2h, 760 struct g2h_fence *g2h_fence) 761 { 762 struct xe_gt *gt __maybe_unused = ct_to_gt(ct); 763 u16 seqno; 764 int ret; 765 766 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 767 xe_gt_assert(gt, !g2h_len || !g2h_fence); 768 xe_gt_assert(gt, !num_g2h || !g2h_fence); 769 xe_gt_assert(gt, !g2h_len || num_g2h); 770 xe_gt_assert(gt, g2h_len || !num_g2h); 771 lockdep_assert_held(&ct->lock); 772 773 if (unlikely(ct->ctbs.h2g.info.broken)) { 774 ret = -EPIPE; 775 goto out; 776 } 777 778 if (ct->state == XE_GUC_CT_STATE_DISABLED) { 779 ret = -ENODEV; 780 goto out; 781 } 782 783 if (ct->state == XE_GUC_CT_STATE_STOPPED) { 784 ret = -ECANCELED; 785 goto out; 786 } 787 788 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 789 790 if (g2h_fence) { 791 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; 792 num_g2h = 1; 793 794 if (g2h_fence_needs_alloc(g2h_fence)) { 795 g2h_fence->seqno = next_ct_seqno(ct, true); 796 ret = xa_err(xa_store(&ct->fence_lookup, 797 g2h_fence->seqno, g2h_fence, 798 GFP_ATOMIC)); 799 if (ret) 800 goto out; 801 } 802 803 seqno = g2h_fence->seqno; 804 } else { 805 seqno = next_ct_seqno(ct, false); 806 } 807 808 if (g2h_len) 809 spin_lock_irq(&ct->fast_lock); 810 retry: 811 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len); 812 if (unlikely(ret)) 813 goto out_unlock; 814 815 ret = h2g_write(ct, action, len, seqno, !!g2h_fence); 816 if (unlikely(ret)) { 817 if (ret == -EAGAIN) 818 goto retry; 819 goto out_unlock; 820 } 821 822 __g2h_reserve_space(ct, g2h_len, num_g2h); 823 xe_guc_notify(ct_to_guc(ct)); 824 out_unlock: 825 if (g2h_len) 826 spin_unlock_irq(&ct->fast_lock); 827 out: 828 return ret; 829 } 830 831 static void kick_reset(struct xe_guc_ct *ct) 832 { 833 xe_gt_reset_async(ct_to_gt(ct)); 834 } 835 836 static int dequeue_one_g2h(struct xe_guc_ct *ct); 837 838 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 839 u32 g2h_len, u32 num_g2h, 840 struct g2h_fence *g2h_fence) 841 { 842 struct xe_device *xe = ct_to_xe(ct); 843 struct xe_gt *gt = ct_to_gt(ct); 844 unsigned int sleep_period_ms = 1; 845 int ret; 846 847 xe_gt_assert(gt, !g2h_len || !g2h_fence); 848 lockdep_assert_held(&ct->lock); 849 xe_device_assert_mem_access(ct_to_xe(ct)); 850 851 try_again: 852 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, 853 g2h_fence); 854 855 /* 856 * We wait to try to restore credits for about 1 second before bailing. 857 * In the case of H2G credits we have no choice but just to wait for the 858 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In 859 * the case of G2H we process any G2H in the channel, hopefully freeing 860 * credits as we consume the G2H messages. 861 */ 862 if (unlikely(ret == -EBUSY && 863 !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) { 864 struct guc_ctb *h2g = &ct->ctbs.h2g; 865 866 if (sleep_period_ms == 1024) 867 goto broken; 868 869 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail, 870 h2g->info.size, 871 h2g->info.space, 872 len + GUC_CTB_HDR_LEN); 873 msleep(sleep_period_ms); 874 sleep_period_ms <<= 1; 875 876 goto try_again; 877 } else if (unlikely(ret == -EBUSY)) { 878 struct xe_device *xe = ct_to_xe(ct); 879 struct guc_ctb *g2h = &ct->ctbs.g2h; 880 881 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head, 882 desc_read(xe, g2h, tail), 883 g2h->info.size, 884 g2h->info.space, 885 g2h_fence ? 886 GUC_CTB_HXG_MSG_MAX_LEN : 887 g2h_len); 888 889 #define g2h_avail(ct) \ 890 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head) 891 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding || 892 g2h_avail(ct), HZ)) 893 goto broken; 894 #undef g2h_avail 895 896 ret = dequeue_one_g2h(ct); 897 if (ret < 0) { 898 if (ret != -ECANCELED) 899 xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)", 900 ERR_PTR(ret)); 901 goto broken; 902 } 903 904 goto try_again; 905 } 906 907 return ret; 908 909 broken: 910 xe_gt_err(gt, "No forward process on H2G, reset required\n"); 911 CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK); 912 913 return -EDEADLK; 914 } 915 916 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 917 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) 918 { 919 int ret; 920 921 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence); 922 923 mutex_lock(&ct->lock); 924 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence); 925 mutex_unlock(&ct->lock); 926 927 return ret; 928 } 929 930 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 931 u32 g2h_len, u32 num_g2h) 932 { 933 int ret; 934 935 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL); 936 if (ret == -EDEADLK) 937 kick_reset(ct); 938 939 return ret; 940 } 941 942 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 943 u32 g2h_len, u32 num_g2h) 944 { 945 int ret; 946 947 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL); 948 if (ret == -EDEADLK) 949 kick_reset(ct); 950 951 return ret; 952 } 953 954 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len) 955 { 956 int ret; 957 958 lockdep_assert_held(&ct->lock); 959 960 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL); 961 if (ret == -EDEADLK) 962 kick_reset(ct); 963 964 return ret; 965 } 966 967 /* 968 * Check if a GT reset is in progress or will occur and if GT reset brought the 969 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset. 970 */ 971 static bool retry_failure(struct xe_guc_ct *ct, int ret) 972 { 973 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV)) 974 return false; 975 976 #define ct_alive(ct) \ 977 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \ 978 !ct->ctbs.g2h.info.broken) 979 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) 980 return false; 981 #undef ct_alive 982 983 return true; 984 } 985 986 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 987 u32 *response_buffer, bool no_fail) 988 { 989 struct xe_gt *gt = ct_to_gt(ct); 990 struct g2h_fence g2h_fence; 991 int ret = 0; 992 993 /* 994 * We use a fence to implement blocking sends / receiving response data. 995 * The seqno of the fence is sent in the H2G, returned in the G2H, and 996 * an xarray is used as storage media with the seqno being to key. 997 * Fields in the fence hold success, failure, retry status and the 998 * response data. Safe to allocate on the stack as the xarray is the 999 * only reference and it cannot be present after this function exits. 1000 */ 1001 retry: 1002 g2h_fence_init(&g2h_fence, response_buffer); 1003 retry_same_fence: 1004 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence); 1005 if (unlikely(ret == -ENOMEM)) { 1006 /* Retry allocation /w GFP_KERNEL */ 1007 ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno, 1008 &g2h_fence, GFP_KERNEL)); 1009 if (ret) 1010 return ret; 1011 1012 goto retry_same_fence; 1013 } else if (unlikely(ret)) { 1014 if (ret == -EDEADLK) 1015 kick_reset(ct); 1016 1017 if (no_fail && retry_failure(ct, ret)) 1018 goto retry_same_fence; 1019 1020 if (!g2h_fence_needs_alloc(&g2h_fence)) 1021 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1022 1023 return ret; 1024 } 1025 1026 ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ); 1027 if (!ret) { 1028 LNL_FLUSH_WORK(&ct->g2h_worker); 1029 if (g2h_fence.done) { 1030 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", 1031 g2h_fence.seqno, action[0]); 1032 ret = 1; 1033 } 1034 } 1035 1036 /* 1037 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on 1038 * the stack, since we have no clue if it will fire after the timeout before we can erase 1039 * from the xa. Also we have some dependent loads and stores below for which we need the 1040 * correct ordering, and we lack the needed barriers. 1041 */ 1042 mutex_lock(&ct->lock); 1043 if (!ret) { 1044 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s", 1045 g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done)); 1046 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1047 mutex_unlock(&ct->lock); 1048 return -ETIME; 1049 } 1050 1051 if (g2h_fence.retry) { 1052 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", 1053 action[0], g2h_fence.reason); 1054 mutex_unlock(&ct->lock); 1055 goto retry; 1056 } 1057 if (g2h_fence.fail) { 1058 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n", 1059 action[0], g2h_fence.error, g2h_fence.hint); 1060 ret = -EIO; 1061 } 1062 1063 if (ret > 0) 1064 ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data; 1065 1066 mutex_unlock(&ct->lock); 1067 1068 return ret; 1069 } 1070 1071 /** 1072 * xe_guc_ct_send_recv - Send and receive HXG to the GuC 1073 * @ct: the &xe_guc_ct 1074 * @action: the dword array with `HXG Request`_ message (can't be NULL) 1075 * @len: length of the `HXG Request`_ message (in dwords, can't be 0) 1076 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL) 1077 * 1078 * Send a `HXG Request`_ message to the GuC over CT communication channel and 1079 * blocks until GuC replies with a `HXG Response`_ message. 1080 * 1081 * For non-blocking communication with GuC use xe_guc_ct_send(). 1082 * 1083 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_. 1084 * 1085 * Return: response length (in dwords) if &response_buffer was not NULL, or 1086 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or 1087 * a negative error code on failure. 1088 */ 1089 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1090 u32 *response_buffer) 1091 { 1092 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer); 1093 return guc_ct_send_recv(ct, action, len, response_buffer, false); 1094 } 1095 ALLOW_ERROR_INJECTION(xe_guc_ct_send_recv, ERRNO); 1096 1097 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, 1098 u32 len, u32 *response_buffer) 1099 { 1100 return guc_ct_send_recv(ct, action, len, response_buffer, true); 1101 } 1102 1103 static u32 *msg_to_hxg(u32 *msg) 1104 { 1105 return msg + GUC_CTB_MSG_MIN_LEN; 1106 } 1107 1108 static u32 msg_len_to_hxg_len(u32 len) 1109 { 1110 return len - GUC_CTB_MSG_MIN_LEN; 1111 } 1112 1113 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) 1114 { 1115 u32 *hxg = msg_to_hxg(msg); 1116 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1117 1118 lockdep_assert_held(&ct->lock); 1119 1120 switch (action) { 1121 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1122 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1123 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1124 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1125 g2h_release_space(ct, len); 1126 } 1127 1128 return 0; 1129 } 1130 1131 static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) 1132 { 1133 struct xe_gt *gt = ct_to_gt(ct); 1134 1135 if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED) 1136 xe_gt_err(gt, "GuC Crash dump notification\n"); 1137 else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION) 1138 xe_gt_err(gt, "GuC Exception notification\n"); 1139 else 1140 xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action); 1141 1142 CT_DEAD(ct, NULL, CRASH); 1143 1144 kick_reset(ct); 1145 1146 return 0; 1147 } 1148 1149 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) 1150 { 1151 struct xe_gt *gt = ct_to_gt(ct); 1152 u32 *hxg = msg_to_hxg(msg); 1153 u32 hxg_len = msg_len_to_hxg_len(len); 1154 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); 1155 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1156 struct g2h_fence *g2h_fence; 1157 1158 lockdep_assert_held(&ct->lock); 1159 1160 /* 1161 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup. 1162 * Those messages should never fail, so if we do get an error back it 1163 * means we're likely doing an illegal operation and the GuC is 1164 * rejecting it. We have no way to inform the code that submitted the 1165 * H2G that the message was rejected, so we need to escalate the 1166 * failure to trigger a reset. 1167 */ 1168 if (fence & CT_SEQNO_UNTRACKED) { 1169 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) 1170 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n", 1171 fence, 1172 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]), 1173 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0])); 1174 else 1175 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n", 1176 type, fence); 1177 CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE); 1178 1179 return -EPROTO; 1180 } 1181 1182 g2h_fence = xa_erase(&ct->fence_lookup, fence); 1183 if (unlikely(!g2h_fence)) { 1184 /* Don't tear down channel, as send could've timed out */ 1185 /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ 1186 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence); 1187 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1188 return 0; 1189 } 1190 1191 xe_gt_assert(gt, fence == g2h_fence->seqno); 1192 1193 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { 1194 g2h_fence->fail = true; 1195 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); 1196 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]); 1197 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 1198 g2h_fence->retry = true; 1199 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); 1200 } else if (g2h_fence->response_buffer) { 1201 g2h_fence->response_len = hxg_len; 1202 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); 1203 } else { 1204 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); 1205 } 1206 1207 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1208 1209 g2h_fence->done = true; 1210 smp_mb(); 1211 1212 wake_up_all(&ct->g2h_fence_wq); 1213 1214 return 0; 1215 } 1216 1217 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1218 { 1219 struct xe_gt *gt = ct_to_gt(ct); 1220 u32 *hxg = msg_to_hxg(msg); 1221 u32 origin, type; 1222 int ret; 1223 1224 lockdep_assert_held(&ct->lock); 1225 1226 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1227 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1228 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n", 1229 origin); 1230 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN); 1231 1232 return -EPROTO; 1233 } 1234 1235 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1236 switch (type) { 1237 case GUC_HXG_TYPE_EVENT: 1238 ret = parse_g2h_event(ct, msg, len); 1239 break; 1240 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1241 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1242 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1243 ret = parse_g2h_response(ct, msg, len); 1244 break; 1245 default: 1246 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n", 1247 type); 1248 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE); 1249 1250 ret = -EOPNOTSUPP; 1251 } 1252 1253 return ret; 1254 } 1255 1256 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1257 { 1258 struct xe_guc *guc = ct_to_guc(ct); 1259 struct xe_gt *gt = ct_to_gt(ct); 1260 u32 hxg_len = msg_len_to_hxg_len(len); 1261 u32 *hxg = msg_to_hxg(msg); 1262 u32 action, adj_len; 1263 u32 *payload; 1264 int ret = 0; 1265 1266 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1267 return 0; 1268 1269 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1270 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN; 1271 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN; 1272 1273 switch (action) { 1274 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1275 ret = xe_guc_sched_done_handler(guc, payload, adj_len); 1276 break; 1277 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1278 ret = xe_guc_deregister_done_handler(guc, payload, adj_len); 1279 break; 1280 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1281 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len); 1282 break; 1283 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1284 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload, 1285 adj_len); 1286 break; 1287 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1288 /* Selftest only at the moment */ 1289 break; 1290 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1291 ret = xe_guc_error_capture_handler(guc, payload, adj_len); 1292 break; 1293 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1294 /* FIXME: Handle this */ 1295 break; 1296 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: 1297 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload, 1298 adj_len); 1299 break; 1300 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1301 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1302 break; 1303 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1304 ret = xe_guc_tlb_invalidation_done_handler(guc, payload, 1305 adj_len); 1306 break; 1307 case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY: 1308 ret = xe_guc_access_counter_notify_handler(guc, payload, 1309 adj_len); 1310 break; 1311 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: 1312 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len); 1313 break; 1314 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF: 1315 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len); 1316 break; 1317 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY: 1318 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len); 1319 break; 1320 case GUC_ACTION_GUC2PF_ADVERSE_EVENT: 1321 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len); 1322 break; 1323 case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1324 case XE_GUC_ACTION_NOTIFY_EXCEPTION: 1325 ret = guc_crash_process_msg(ct, action); 1326 break; 1327 default: 1328 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); 1329 } 1330 1331 if (ret) { 1332 xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n", 1333 action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg); 1334 CT_DEAD(ct, NULL, PROCESS_FAILED); 1335 } 1336 1337 return 0; 1338 } 1339 1340 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) 1341 { 1342 struct xe_device *xe = ct_to_xe(ct); 1343 struct xe_gt *gt = ct_to_gt(ct); 1344 struct guc_ctb *g2h = &ct->ctbs.g2h; 1345 u32 tail, head, len, desc_status; 1346 s32 avail; 1347 u32 action; 1348 u32 *hxg; 1349 1350 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 1351 lockdep_assert_held(&ct->fast_lock); 1352 1353 if (ct->state == XE_GUC_CT_STATE_DISABLED) 1354 return -ENODEV; 1355 1356 if (ct->state == XE_GUC_CT_STATE_STOPPED) 1357 return -ECANCELED; 1358 1359 if (g2h->info.broken) 1360 return -EPIPE; 1361 1362 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1363 1364 desc_status = desc_read(xe, g2h, status); 1365 if (desc_status) { 1366 if (desc_status & GUC_CTB_STATUS_DISABLED) { 1367 /* 1368 * Potentially valid if a CLIENT_RESET request resulted in 1369 * contexts/engines being reset. But should never happen as 1370 * no contexts should be active when CLIENT_RESET is sent. 1371 */ 1372 xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n"); 1373 desc_status &= ~GUC_CTB_STATUS_DISABLED; 1374 } 1375 1376 if (desc_status) { 1377 xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status); 1378 goto corrupted; 1379 } 1380 } 1381 1382 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 1383 u32 desc_tail = desc_read(xe, g2h, tail); 1384 /* 1385 u32 desc_head = desc_read(xe, g2h, head); 1386 1387 * info.head and desc_head are updated back-to-back at the end of 1388 * this function and nowhere else. Hence, they cannot be different 1389 * unless two g2h_read calls are running concurrently. Which is not 1390 * possible because it is guarded by ct->fast_lock. And yet, some 1391 * discrete platforms are regularly hitting this error :(. 1392 * 1393 * desc_head rolling backwards shouldn't cause any noticeable 1394 * problems - just a delay in GuC being allowed to proceed past that 1395 * point in the queue. So for now, just disable the error until it 1396 * can be root caused. 1397 * 1398 if (g2h->info.head != desc_head) { 1399 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH); 1400 xe_gt_err(gt, "CT read: head was modified %u != %u\n", 1401 desc_head, g2h->info.head); 1402 goto corrupted; 1403 } 1404 */ 1405 1406 if (g2h->info.head > g2h->info.size) { 1407 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1408 xe_gt_err(gt, "CT read: head out of range: %u vs %u\n", 1409 g2h->info.head, g2h->info.size); 1410 goto corrupted; 1411 } 1412 1413 if (desc_tail >= g2h->info.size) { 1414 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1415 xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n", 1416 desc_tail, g2h->info.size); 1417 goto corrupted; 1418 } 1419 } 1420 1421 /* Calculate DW available to read */ 1422 tail = desc_read(xe, g2h, tail); 1423 avail = tail - g2h->info.head; 1424 if (unlikely(avail == 0)) 1425 return 0; 1426 1427 if (avail < 0) 1428 avail += g2h->info.size; 1429 1430 /* Read header */ 1431 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head, 1432 sizeof(u32)); 1433 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; 1434 if (len > avail) { 1435 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n", 1436 avail, len); 1437 goto corrupted; 1438 } 1439 1440 head = (g2h->info.head + 1) % g2h->info.size; 1441 avail = len - 1; 1442 1443 /* Read G2H message */ 1444 if (avail + head > g2h->info.size) { 1445 u32 avail_til_wrap = g2h->info.size - head; 1446 1447 xe_map_memcpy_from(xe, msg + 1, 1448 &g2h->cmds, sizeof(u32) * head, 1449 avail_til_wrap * sizeof(u32)); 1450 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap, 1451 &g2h->cmds, 0, 1452 (avail - avail_til_wrap) * sizeof(u32)); 1453 } else { 1454 xe_map_memcpy_from(xe, msg + 1, 1455 &g2h->cmds, sizeof(u32) * head, 1456 avail * sizeof(u32)); 1457 } 1458 1459 hxg = msg_to_hxg(msg); 1460 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1461 1462 if (fast_path) { 1463 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1464 return 0; 1465 1466 switch (action) { 1467 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1468 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1469 break; /* Process these in fast-path */ 1470 default: 1471 return 0; 1472 } 1473 } 1474 1475 /* Update local / descriptor header */ 1476 g2h->info.head = (head + avail) % g2h->info.size; 1477 desc_write(xe, g2h, head, g2h->info.head); 1478 1479 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id, 1480 action, len, g2h->info.head, tail); 1481 1482 return len; 1483 1484 corrupted: 1485 CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ); 1486 return -EPROTO; 1487 } 1488 1489 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) 1490 { 1491 struct xe_gt *gt = ct_to_gt(ct); 1492 struct xe_guc *guc = ct_to_guc(ct); 1493 u32 hxg_len = msg_len_to_hxg_len(len); 1494 u32 *hxg = msg_to_hxg(msg); 1495 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1496 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN; 1497 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1498 int ret = 0; 1499 1500 switch (action) { 1501 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1502 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1503 break; 1504 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1505 __g2h_release_space(ct, len); 1506 ret = xe_guc_tlb_invalidation_done_handler(guc, payload, 1507 adj_len); 1508 break; 1509 default: 1510 xe_gt_warn(gt, "NOT_POSSIBLE"); 1511 } 1512 1513 if (ret) { 1514 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n", 1515 action, ERR_PTR(ret)); 1516 CT_DEAD(ct, NULL, FAST_G2H); 1517 } 1518 } 1519 1520 /** 1521 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler 1522 * @ct: GuC CT object 1523 * 1524 * Anything related to page faults is critical for performance, process these 1525 * critical G2H in the IRQ. This is safe as these handlers either just wake up 1526 * waiters or queue another worker. 1527 */ 1528 void xe_guc_ct_fast_path(struct xe_guc_ct *ct) 1529 { 1530 struct xe_device *xe = ct_to_xe(ct); 1531 bool ongoing; 1532 int len; 1533 1534 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1535 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1536 return; 1537 1538 spin_lock(&ct->fast_lock); 1539 do { 1540 len = g2h_read(ct, ct->fast_msg, true); 1541 if (len > 0) 1542 g2h_fast_path(ct, ct->fast_msg, len); 1543 } while (len > 0); 1544 spin_unlock(&ct->fast_lock); 1545 1546 if (ongoing) 1547 xe_pm_runtime_put(xe); 1548 } 1549 1550 /* Returns less than zero on error, 0 on done, 1 on more available */ 1551 static int dequeue_one_g2h(struct xe_guc_ct *ct) 1552 { 1553 int len; 1554 int ret; 1555 1556 lockdep_assert_held(&ct->lock); 1557 1558 spin_lock_irq(&ct->fast_lock); 1559 len = g2h_read(ct, ct->msg, false); 1560 spin_unlock_irq(&ct->fast_lock); 1561 if (len <= 0) 1562 return len; 1563 1564 ret = parse_g2h_msg(ct, ct->msg, len); 1565 if (unlikely(ret < 0)) 1566 return ret; 1567 1568 ret = process_g2h_msg(ct, ct->msg, len); 1569 if (unlikely(ret < 0)) 1570 return ret; 1571 1572 return 1; 1573 } 1574 1575 static void receive_g2h(struct xe_guc_ct *ct) 1576 { 1577 bool ongoing; 1578 int ret; 1579 1580 /* 1581 * Normal users must always hold mem_access.ref around CT calls. However 1582 * during the runtime pm callbacks we rely on CT to talk to the GuC, but 1583 * at this stage we can't rely on mem_access.ref and even the 1584 * callback_task will be different than current. For such cases we just 1585 * need to ensure we always process the responses from any blocking 1586 * ct_send requests or where we otherwise expect some response when 1587 * initiated from those callbacks (which will need to wait for the below 1588 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if 1589 * the device has suspended to the point that the CT communication has 1590 * been disabled. 1591 * 1592 * If we are inside the runtime pm callback, we can be the only task 1593 * still issuing CT requests (since that requires having the 1594 * mem_access.ref). It seems like it might in theory be possible to 1595 * receive unsolicited events from the GuC just as we are 1596 * suspending-resuming, but those will currently anyway be lost when 1597 * eventually exiting from suspend, hence no need to wake up the device 1598 * here. If we ever need something stronger than get_if_ongoing() then 1599 * we need to be careful with blocking the pm callbacks from getting CT 1600 * responses, if the worker here is blocked on those callbacks 1601 * completing, creating a deadlock. 1602 */ 1603 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1604 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1605 return; 1606 1607 do { 1608 mutex_lock(&ct->lock); 1609 ret = dequeue_one_g2h(ct); 1610 mutex_unlock(&ct->lock); 1611 1612 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) { 1613 xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d", ret); 1614 CT_DEAD(ct, NULL, G2H_RECV); 1615 kick_reset(ct); 1616 } 1617 } while (ret == 1); 1618 1619 if (ongoing) 1620 xe_pm_runtime_put(ct_to_xe(ct)); 1621 } 1622 1623 static void g2h_worker_func(struct work_struct *w) 1624 { 1625 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker); 1626 1627 receive_g2h(ct); 1628 } 1629 1630 static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic, 1631 bool want_ctb) 1632 { 1633 struct xe_guc_ct_snapshot *snapshot; 1634 1635 snapshot = kzalloc(sizeof(*snapshot), atomic ? GFP_ATOMIC : GFP_KERNEL); 1636 if (!snapshot) 1637 return NULL; 1638 1639 if (ct->bo && want_ctb) { 1640 snapshot->ctb_size = ct->bo->size; 1641 snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL); 1642 } 1643 1644 return snapshot; 1645 } 1646 1647 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb, 1648 struct guc_ctb_snapshot *snapshot) 1649 { 1650 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0, 1651 sizeof(struct guc_ct_buffer_desc)); 1652 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info)); 1653 } 1654 1655 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot, 1656 struct drm_printer *p) 1657 { 1658 drm_printf(p, "\tsize: %d\n", snapshot->info.size); 1659 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space); 1660 drm_printf(p, "\thead: %d\n", snapshot->info.head); 1661 drm_printf(p, "\ttail: %d\n", snapshot->info.tail); 1662 drm_printf(p, "\tspace: %d\n", snapshot->info.space); 1663 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken); 1664 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head); 1665 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail); 1666 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status); 1667 } 1668 1669 static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic, 1670 bool want_ctb) 1671 { 1672 struct xe_device *xe = ct_to_xe(ct); 1673 struct xe_guc_ct_snapshot *snapshot; 1674 1675 snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb); 1676 if (!snapshot) { 1677 xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n"); 1678 return NULL; 1679 } 1680 1681 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) { 1682 snapshot->ct_enabled = true; 1683 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding); 1684 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g); 1685 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h); 1686 } 1687 1688 if (ct->bo && snapshot->ctb) 1689 xe_map_memcpy_from(xe, snapshot->ctb, &ct->bo->vmap, 0, snapshot->ctb_size); 1690 1691 return snapshot; 1692 } 1693 1694 /** 1695 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state. 1696 * @ct: GuC CT object. 1697 * 1698 * This can be printed out in a later stage like during dev_coredump 1699 * analysis. This is safe to be called during atomic context. 1700 * 1701 * Returns: a GuC CT snapshot object that must be freed by the caller 1702 * by using `xe_guc_ct_snapshot_free`. 1703 */ 1704 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct) 1705 { 1706 return guc_ct_snapshot_capture(ct, true, true); 1707 } 1708 1709 /** 1710 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot. 1711 * @snapshot: GuC CT snapshot object. 1712 * @p: drm_printer where it will be printed out. 1713 * 1714 * This function prints out a given GuC CT snapshot object. 1715 */ 1716 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, 1717 struct drm_printer *p) 1718 { 1719 if (!snapshot) 1720 return; 1721 1722 if (snapshot->ct_enabled) { 1723 drm_puts(p, "H2G CTB (all sizes in DW):\n"); 1724 guc_ctb_snapshot_print(&snapshot->h2g, p); 1725 1726 drm_puts(p, "G2H CTB (all sizes in DW):\n"); 1727 guc_ctb_snapshot_print(&snapshot->g2h, p); 1728 drm_printf(p, "\tg2h outstanding: %d\n", 1729 snapshot->g2h_outstanding); 1730 1731 if (snapshot->ctb) { 1732 drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size); 1733 xe_print_blob_ascii85(p, "[CTB].data", '\n', 1734 snapshot->ctb, 0, snapshot->ctb_size); 1735 } 1736 } else { 1737 drm_puts(p, "CT disabled\n"); 1738 } 1739 } 1740 1741 /** 1742 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot. 1743 * @snapshot: GuC CT snapshot object. 1744 * 1745 * This function free all the memory that needed to be allocated at capture 1746 * time. 1747 */ 1748 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot) 1749 { 1750 if (!snapshot) 1751 return; 1752 1753 kfree(snapshot->ctb); 1754 kfree(snapshot); 1755 } 1756 1757 /** 1758 * xe_guc_ct_print - GuC CT Print. 1759 * @ct: GuC CT. 1760 * @p: drm_printer where it will be printed out. 1761 * @want_ctb: Should the full CTB content be dumped (vs just the headers) 1762 * 1763 * This function will quickly capture a snapshot of the CT state 1764 * and immediately print it out. 1765 */ 1766 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb) 1767 { 1768 struct xe_guc_ct_snapshot *snapshot; 1769 1770 snapshot = guc_ct_snapshot_capture(ct, false, want_ctb); 1771 xe_guc_ct_snapshot_print(snapshot, p); 1772 xe_guc_ct_snapshot_free(snapshot); 1773 } 1774 1775 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 1776 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code) 1777 { 1778 struct xe_guc_log_snapshot *snapshot_log; 1779 struct xe_guc_ct_snapshot *snapshot_ct; 1780 struct xe_guc *guc = ct_to_guc(ct); 1781 unsigned long flags; 1782 bool have_capture; 1783 1784 if (ctb) 1785 ctb->info.broken = true; 1786 1787 /* Ignore further errors after the first dump until a reset */ 1788 if (ct->dead.reported) 1789 return; 1790 1791 spin_lock_irqsave(&ct->dead.lock, flags); 1792 1793 /* And only capture one dump at a time */ 1794 have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE); 1795 ct->dead.reason |= (1 << reason_code) | 1796 (1 << CT_DEAD_STATE_CAPTURE); 1797 1798 spin_unlock_irqrestore(&ct->dead.lock, flags); 1799 1800 if (have_capture) 1801 return; 1802 1803 snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true); 1804 snapshot_ct = xe_guc_ct_snapshot_capture((ct)); 1805 1806 spin_lock_irqsave(&ct->dead.lock, flags); 1807 1808 if (ct->dead.snapshot_log || ct->dead.snapshot_ct) { 1809 xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n"); 1810 xe_guc_log_snapshot_free(snapshot_log); 1811 xe_guc_ct_snapshot_free(snapshot_ct); 1812 } else { 1813 ct->dead.snapshot_log = snapshot_log; 1814 ct->dead.snapshot_ct = snapshot_ct; 1815 } 1816 1817 spin_unlock_irqrestore(&ct->dead.lock, flags); 1818 1819 queue_work(system_unbound_wq, &(ct)->dead.worker); 1820 } 1821 1822 static void ct_dead_print(struct xe_dead_ct *dead) 1823 { 1824 struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead); 1825 struct xe_device *xe = ct_to_xe(ct); 1826 struct xe_gt *gt = ct_to_gt(ct); 1827 static int g_count; 1828 struct drm_printer ip = xe_gt_info_printer(gt); 1829 struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count); 1830 1831 if (!dead->reason) { 1832 xe_gt_err(gt, "CTB is dead for no reason!?\n"); 1833 return; 1834 } 1835 1836 1837 /* Can't generate a genuine core dump at this point, so just do the good bits */ 1838 drm_puts(&lp, "**** Xe Device Coredump ****\n"); 1839 drm_printf(&lp, "Reason: CTB is dead - 0x%X\n", dead->reason); 1840 xe_device_snapshot_print(xe, &lp); 1841 1842 drm_printf(&lp, "**** GT #%d ****\n", gt->info.id); 1843 drm_printf(&lp, "\tTile: %d\n", gt->tile->id); 1844 1845 drm_puts(&lp, "**** GuC Log ****\n"); 1846 xe_guc_log_snapshot_print(dead->snapshot_log, &lp); 1847 1848 drm_puts(&lp, "**** GuC CT ****\n"); 1849 xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp); 1850 1851 drm_puts(&lp, "Done.\n"); 1852 } 1853 1854 static void ct_dead_worker_func(struct work_struct *w) 1855 { 1856 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker); 1857 1858 if (!ct->dead.reported) { 1859 ct->dead.reported = true; 1860 ct_dead_print(&ct->dead); 1861 } 1862 1863 spin_lock_irq(&ct->dead.lock); 1864 1865 xe_guc_log_snapshot_free(ct->dead.snapshot_log); 1866 ct->dead.snapshot_log = NULL; 1867 xe_guc_ct_snapshot_free(ct->dead.snapshot_ct); 1868 ct->dead.snapshot_ct = NULL; 1869 1870 if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) { 1871 /* A reset has occurred so re-arm the error reporting */ 1872 ct->dead.reason = 0; 1873 ct->dead.reported = false; 1874 } 1875 1876 spin_unlock_irq(&ct->dead.lock); 1877 } 1878 #endif 1879