1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ct.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 #include <linux/delay.h> 11 #include <linux/fault-inject.h> 12 13 #include <kunit/static_stub.h> 14 15 #include <drm/drm_managed.h> 16 17 #include "abi/guc_actions_abi.h" 18 #include "abi/guc_actions_sriov_abi.h" 19 #include "abi/guc_klvs_abi.h" 20 #include "xe_bo.h" 21 #include "xe_devcoredump.h" 22 #include "xe_device.h" 23 #include "xe_gt.h" 24 #include "xe_gt_pagefault.h" 25 #include "xe_gt_printk.h" 26 #include "xe_gt_sriov_pf_control.h" 27 #include "xe_gt_sriov_pf_monitor.h" 28 #include "xe_gt_sriov_printk.h" 29 #include "xe_gt_tlb_invalidation.h" 30 #include "xe_guc.h" 31 #include "xe_guc_log.h" 32 #include "xe_guc_relay.h" 33 #include "xe_guc_submit.h" 34 #include "xe_map.h" 35 #include "xe_pm.h" 36 #include "xe_trace_guc.h" 37 38 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 39 enum { 40 /* Internal states, not error conditions */ 41 CT_DEAD_STATE_REARM, /* 0x0001 */ 42 CT_DEAD_STATE_CAPTURE, /* 0x0002 */ 43 44 /* Error conditions */ 45 CT_DEAD_SETUP, /* 0x0004 */ 46 CT_DEAD_H2G_WRITE, /* 0x0008 */ 47 CT_DEAD_H2G_HAS_ROOM, /* 0x0010 */ 48 CT_DEAD_G2H_READ, /* 0x0020 */ 49 CT_DEAD_G2H_RECV, /* 0x0040 */ 50 CT_DEAD_G2H_RELEASE, /* 0x0080 */ 51 CT_DEAD_DEADLOCK, /* 0x0100 */ 52 CT_DEAD_PROCESS_FAILED, /* 0x0200 */ 53 CT_DEAD_FAST_G2H, /* 0x0400 */ 54 CT_DEAD_PARSE_G2H_RESPONSE, /* 0x0800 */ 55 CT_DEAD_PARSE_G2H_UNKNOWN, /* 0x1000 */ 56 CT_DEAD_PARSE_G2H_ORIGIN, /* 0x2000 */ 57 CT_DEAD_PARSE_G2H_TYPE, /* 0x4000 */ 58 CT_DEAD_CRASH, /* 0x8000 */ 59 }; 60 61 static void ct_dead_worker_func(struct work_struct *w); 62 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code); 63 64 #define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code) 65 #else 66 #define CT_DEAD(ct, ctb, reason) \ 67 do { \ 68 struct guc_ctb *_ctb = (ctb); \ 69 if (_ctb) \ 70 _ctb->info.broken = true; \ 71 } while (0) 72 #endif 73 74 /* Used when a CT send wants to block and / or receive data */ 75 struct g2h_fence { 76 u32 *response_buffer; 77 u32 seqno; 78 u32 response_data; 79 u16 response_len; 80 u16 error; 81 u16 hint; 82 u16 reason; 83 bool retry; 84 bool fail; 85 bool done; 86 }; 87 88 #define make_u64(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo))) 89 90 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) 91 { 92 g2h_fence->response_buffer = response_buffer; 93 g2h_fence->response_data = 0; 94 g2h_fence->response_len = 0; 95 g2h_fence->fail = false; 96 g2h_fence->retry = false; 97 g2h_fence->done = false; 98 g2h_fence->seqno = ~0x0; 99 } 100 101 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) 102 { 103 return g2h_fence->seqno == ~0x0; 104 } 105 106 static struct xe_guc * 107 ct_to_guc(struct xe_guc_ct *ct) 108 { 109 return container_of(ct, struct xe_guc, ct); 110 } 111 112 static struct xe_gt * 113 ct_to_gt(struct xe_guc_ct *ct) 114 { 115 return container_of(ct, struct xe_gt, uc.guc.ct); 116 } 117 118 static struct xe_device * 119 ct_to_xe(struct xe_guc_ct *ct) 120 { 121 return gt_to_xe(ct_to_gt(ct)); 122 } 123 124 /** 125 * DOC: GuC CTB Blob 126 * 127 * We allocate single blob to hold both CTB descriptors and buffers: 128 * 129 * +--------+-----------------------------------------------+------+ 130 * | offset | contents | size | 131 * +========+===============================================+======+ 132 * | 0x0000 | H2G CTB Descriptor (send) | | 133 * +--------+-----------------------------------------------+ 4K | 134 * | 0x0800 | G2H CTB Descriptor (g2h) | | 135 * +--------+-----------------------------------------------+------+ 136 * | 0x1000 | H2G CT Buffer (send) | n*4K | 137 * | | | | 138 * +--------+-----------------------------------------------+------+ 139 * | 0x1000 | G2H CT Buffer (g2h) | m*4K | 140 * | + n*4K | | | 141 * +--------+-----------------------------------------------+------+ 142 * 143 * Size of each ``CT Buffer`` must be multiple of 4K. 144 * We don't expect too many messages in flight at any time, unless we are 145 * using the GuC submission. In that case each request requires a minimum 146 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 147 * enough space to avoid backpressure on the driver. We increase the size 148 * of the receive buffer (relative to the send) to ensure a G2H response 149 * CTB has a landing spot. 150 * 151 * In addition to submissions, the G2H buffer needs to be able to hold 152 * enough space for recoverable page fault notifications. The number of 153 * page faults is interrupt driven and can be as much as the number of 154 * compute resources available. However, most of the actual work for these 155 * is in a separate page fault worker thread. Therefore we only need to 156 * make sure the queue has enough space to handle all of the submissions 157 * and responses and an extra buffer for incoming page faults. 158 */ 159 160 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 161 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 162 #define CTB_G2H_BUFFER_SIZE (SZ_128K) 163 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2) 164 165 /** 166 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full 167 * CT command queue 168 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future. 169 * 170 * Observation is that a 4KiB buffer full of commands takes a little over a 171 * second to process. Use that to calculate maximum time to process a full CT 172 * command queue. 173 * 174 * Return: Maximum time to process a full CT queue in jiffies. 175 */ 176 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct) 177 { 178 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4)); 179 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ; 180 } 181 182 static size_t guc_ct_size(void) 183 { 184 return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + 185 CTB_G2H_BUFFER_SIZE; 186 } 187 188 static void guc_ct_fini(struct drm_device *drm, void *arg) 189 { 190 struct xe_guc_ct *ct = arg; 191 192 destroy_workqueue(ct->g2h_wq); 193 xa_destroy(&ct->fence_lookup); 194 } 195 196 static void receive_g2h(struct xe_guc_ct *ct); 197 static void g2h_worker_func(struct work_struct *w); 198 static void safe_mode_worker_func(struct work_struct *w); 199 200 static void primelockdep(struct xe_guc_ct *ct) 201 { 202 if (!IS_ENABLED(CONFIG_LOCKDEP)) 203 return; 204 205 fs_reclaim_acquire(GFP_KERNEL); 206 might_lock(&ct->lock); 207 fs_reclaim_release(GFP_KERNEL); 208 } 209 210 int xe_guc_ct_init(struct xe_guc_ct *ct) 211 { 212 struct xe_device *xe = ct_to_xe(ct); 213 struct xe_gt *gt = ct_to_gt(ct); 214 struct xe_tile *tile = gt_to_tile(gt); 215 struct xe_bo *bo; 216 int err; 217 218 xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE)); 219 220 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM); 221 if (!ct->g2h_wq) 222 return -ENOMEM; 223 224 spin_lock_init(&ct->fast_lock); 225 xa_init(&ct->fence_lookup); 226 INIT_WORK(&ct->g2h_worker, g2h_worker_func); 227 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func); 228 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 229 spin_lock_init(&ct->dead.lock); 230 INIT_WORK(&ct->dead.worker, ct_dead_worker_func); 231 #endif 232 init_waitqueue_head(&ct->wq); 233 init_waitqueue_head(&ct->g2h_fence_wq); 234 235 err = drmm_mutex_init(&xe->drm, &ct->lock); 236 if (err) 237 return err; 238 239 primelockdep(ct); 240 241 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(), 242 XE_BO_FLAG_SYSTEM | 243 XE_BO_FLAG_GGTT | 244 XE_BO_FLAG_GGTT_INVALIDATE | 245 XE_BO_FLAG_PINNED_NORESTORE); 246 if (IS_ERR(bo)) 247 return PTR_ERR(bo); 248 249 ct->bo = bo; 250 251 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); 252 if (err) 253 return err; 254 255 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED); 256 ct->state = XE_GUC_CT_STATE_DISABLED; 257 return 0; 258 } 259 ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */ 260 261 #define desc_read(xe_, guc_ctb__, field_) \ 262 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \ 263 struct guc_ct_buffer_desc, field_) 264 265 #define desc_write(xe_, guc_ctb__, field_, val_) \ 266 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \ 267 struct guc_ct_buffer_desc, field_, val_) 268 269 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, 270 struct iosys_map *map) 271 { 272 h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32); 273 h2g->info.resv_space = 0; 274 h2g->info.tail = 0; 275 h2g->info.head = 0; 276 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 277 h2g->info.size) - 278 h2g->info.resv_space; 279 h2g->info.broken = false; 280 281 h2g->desc = *map; 282 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 283 284 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2); 285 } 286 287 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, 288 struct iosys_map *map) 289 { 290 g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32); 291 g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32); 292 g2h->info.head = 0; 293 g2h->info.tail = 0; 294 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head, 295 g2h->info.size) - 296 g2h->info.resv_space; 297 g2h->info.broken = false; 298 299 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE); 300 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 301 302 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 + 303 CTB_H2G_BUFFER_SIZE); 304 } 305 306 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct) 307 { 308 struct xe_guc *guc = ct_to_guc(ct); 309 u32 desc_addr, ctb_addr, size; 310 int err; 311 312 desc_addr = xe_bo_ggtt_addr(ct->bo); 313 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2; 314 size = ct->ctbs.h2g.info.size * sizeof(u32); 315 316 err = xe_guc_self_cfg64(guc, 317 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY, 318 desc_addr); 319 if (err) 320 return err; 321 322 err = xe_guc_self_cfg64(guc, 323 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY, 324 ctb_addr); 325 if (err) 326 return err; 327 328 return xe_guc_self_cfg32(guc, 329 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY, 330 size); 331 } 332 333 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct) 334 { 335 struct xe_guc *guc = ct_to_guc(ct); 336 u32 desc_addr, ctb_addr, size; 337 int err; 338 339 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE; 340 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 + 341 CTB_H2G_BUFFER_SIZE; 342 size = ct->ctbs.g2h.info.size * sizeof(u32); 343 344 err = xe_guc_self_cfg64(guc, 345 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 346 desc_addr); 347 if (err) 348 return err; 349 350 err = xe_guc_self_cfg64(guc, 351 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 352 ctb_addr); 353 if (err) 354 return err; 355 356 return xe_guc_self_cfg32(guc, 357 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 358 size); 359 } 360 361 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) 362 { 363 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 364 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 365 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 366 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, 367 GUC_ACTION_HOST2GUC_CONTROL_CTB), 368 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, 369 enable ? GUC_CTB_CONTROL_ENABLE : 370 GUC_CTB_CONTROL_DISABLE), 371 }; 372 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request)); 373 374 return ret > 0 ? -EPROTO : ret; 375 } 376 377 static void xe_guc_ct_set_state(struct xe_guc_ct *ct, 378 enum xe_guc_ct_state state) 379 { 380 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ 381 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ 382 383 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 || 384 state == XE_GUC_CT_STATE_STOPPED); 385 386 if (ct->g2h_outstanding) 387 xe_pm_runtime_put(ct_to_xe(ct)); 388 ct->g2h_outstanding = 0; 389 ct->state = state; 390 391 spin_unlock_irq(&ct->fast_lock); 392 393 /* 394 * Lockdep doesn't like this under the fast lock and he destroy only 395 * needs to be serialized with the send path which ct lock provides. 396 */ 397 xa_destroy(&ct->fence_lookup); 398 399 mutex_unlock(&ct->lock); 400 } 401 402 static bool ct_needs_safe_mode(struct xe_guc_ct *ct) 403 { 404 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev)); 405 } 406 407 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct) 408 { 409 if (!ct_needs_safe_mode(ct)) 410 return false; 411 412 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10); 413 return true; 414 } 415 416 static void safe_mode_worker_func(struct work_struct *w) 417 { 418 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work); 419 420 receive_g2h(ct); 421 422 if (!ct_restart_safe_mode_worker(ct)) 423 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n"); 424 } 425 426 static void ct_enter_safe_mode(struct xe_guc_ct *ct) 427 { 428 if (ct_restart_safe_mode_worker(ct)) 429 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n"); 430 } 431 432 static void ct_exit_safe_mode(struct xe_guc_ct *ct) 433 { 434 if (cancel_delayed_work_sync(&ct->safe_mode_worker)) 435 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n"); 436 } 437 438 int xe_guc_ct_enable(struct xe_guc_ct *ct) 439 { 440 struct xe_device *xe = ct_to_xe(ct); 441 struct xe_gt *gt = ct_to_gt(ct); 442 int err; 443 444 xe_gt_assert(gt, !xe_guc_ct_enabled(ct)); 445 446 xe_map_memset(xe, &ct->bo->vmap, 0, 0, ct->bo->size); 447 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap); 448 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap); 449 450 err = guc_ct_ctb_h2g_register(ct); 451 if (err) 452 goto err_out; 453 454 err = guc_ct_ctb_g2h_register(ct); 455 if (err) 456 goto err_out; 457 458 err = guc_ct_control_toggle(ct, true); 459 if (err) 460 goto err_out; 461 462 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED); 463 464 smp_mb(); 465 wake_up_all(&ct->wq); 466 xe_gt_dbg(gt, "GuC CT communication channel enabled\n"); 467 468 if (ct_needs_safe_mode(ct)) 469 ct_enter_safe_mode(ct); 470 471 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 472 /* 473 * The CT has now been reset so the dumper can be re-armed 474 * after any existing dead state has been dumped. 475 */ 476 spin_lock_irq(&ct->dead.lock); 477 if (ct->dead.reason) { 478 ct->dead.reason |= (1 << CT_DEAD_STATE_REARM); 479 queue_work(system_unbound_wq, &ct->dead.worker); 480 } 481 spin_unlock_irq(&ct->dead.lock); 482 #endif 483 484 return 0; 485 486 err_out: 487 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err)); 488 CT_DEAD(ct, NULL, SETUP); 489 490 return err; 491 } 492 493 static void stop_g2h_handler(struct xe_guc_ct *ct) 494 { 495 cancel_work_sync(&ct->g2h_worker); 496 } 497 498 /** 499 * xe_guc_ct_disable - Set GuC to disabled state 500 * @ct: the &xe_guc_ct 501 * 502 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected 503 * in this transition. 504 */ 505 void xe_guc_ct_disable(struct xe_guc_ct *ct) 506 { 507 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED); 508 ct_exit_safe_mode(ct); 509 stop_g2h_handler(ct); 510 } 511 512 /** 513 * xe_guc_ct_stop - Set GuC to stopped state 514 * @ct: the &xe_guc_ct 515 * 516 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h 517 */ 518 void xe_guc_ct_stop(struct xe_guc_ct *ct) 519 { 520 if (!xe_guc_ct_initialized(ct)) 521 return; 522 523 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED); 524 stop_g2h_handler(ct); 525 } 526 527 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) 528 { 529 struct guc_ctb *h2g = &ct->ctbs.h2g; 530 531 lockdep_assert_held(&ct->lock); 532 533 if (cmd_len > h2g->info.space) { 534 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); 535 536 if (h2g->info.head > h2g->info.size) { 537 struct xe_device *xe = ct_to_xe(ct); 538 u32 desc_status = desc_read(xe, h2g, status); 539 540 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 541 542 xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n", 543 h2g->info.head, h2g->info.size); 544 CT_DEAD(ct, h2g, H2G_HAS_ROOM); 545 return false; 546 } 547 548 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 549 h2g->info.size) - 550 h2g->info.resv_space; 551 if (cmd_len > h2g->info.space) 552 return false; 553 } 554 555 return true; 556 } 557 558 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len) 559 { 560 if (!g2h_len) 561 return true; 562 563 lockdep_assert_held(&ct->fast_lock); 564 565 return ct->ctbs.g2h.info.space > g2h_len; 566 } 567 568 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len) 569 { 570 lockdep_assert_held(&ct->lock); 571 572 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len)) 573 return -EBUSY; 574 575 return 0; 576 } 577 578 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len) 579 { 580 lockdep_assert_held(&ct->lock); 581 ct->ctbs.h2g.info.space -= cmd_len; 582 } 583 584 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h) 585 { 586 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space); 587 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) || 588 (g2h_len && num_g2h)); 589 590 if (g2h_len) { 591 lockdep_assert_held(&ct->fast_lock); 592 593 if (!ct->g2h_outstanding) 594 xe_pm_runtime_get_noresume(ct_to_xe(ct)); 595 596 ct->ctbs.g2h.info.space -= g2h_len; 597 ct->g2h_outstanding += num_g2h; 598 } 599 } 600 601 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 602 { 603 bool bad = false; 604 605 lockdep_assert_held(&ct->fast_lock); 606 607 bad = ct->ctbs.g2h.info.space + g2h_len > 608 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space; 609 bad |= !ct->g2h_outstanding; 610 611 if (bad) { 612 xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n", 613 ct->ctbs.g2h.info.space, g2h_len, 614 ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space, 615 ct->ctbs.g2h.info.space + g2h_len, 616 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space, 617 ct->g2h_outstanding); 618 CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE); 619 return; 620 } 621 622 ct->ctbs.g2h.info.space += g2h_len; 623 if (!--ct->g2h_outstanding) 624 xe_pm_runtime_put(ct_to_xe(ct)); 625 } 626 627 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 628 { 629 spin_lock_irq(&ct->fast_lock); 630 __g2h_release_space(ct, g2h_len); 631 spin_unlock_irq(&ct->fast_lock); 632 } 633 634 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 635 static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) 636 { 637 unsigned int slot = fence % ARRAY_SIZE(ct->fast_req); 638 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 639 unsigned long entries[SZ_32]; 640 unsigned int n; 641 642 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 643 644 /* May be called under spinlock, so avoid sleeping */ 645 ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT); 646 #endif 647 ct->fast_req[slot].fence = fence; 648 ct->fast_req[slot].action = action; 649 } 650 #else 651 static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) 652 { 653 } 654 #endif 655 656 /* 657 * The CT protocol accepts a 16 bits fence. This field is fully owned by the 658 * driver, the GuC will just copy it to the reply message. Since we need to 659 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages, 660 * we use one bit of the seqno as an indicator for that and a rolling counter 661 * for the remaining 15 bits. 662 */ 663 #define CT_SEQNO_MASK GENMASK(14, 0) 664 #define CT_SEQNO_UNTRACKED BIT(15) 665 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence) 666 { 667 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK; 668 669 if (!is_g2h_fence) 670 seqno |= CT_SEQNO_UNTRACKED; 671 672 return seqno; 673 } 674 675 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */ 676 677 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, 678 u32 ct_fence_value, bool want_response) 679 { 680 struct xe_device *xe = ct_to_xe(ct); 681 struct xe_gt *gt = ct_to_gt(ct); 682 struct guc_ctb *h2g = &ct->ctbs.h2g; 683 u32 cmd[H2G_CT_HEADERS]; 684 u32 tail = h2g->info.tail; 685 u32 full_len; 686 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds, 687 tail * sizeof(u32)); 688 u32 desc_status; 689 690 full_len = len + GUC_CTB_HDR_LEN; 691 692 lockdep_assert_held(&ct->lock); 693 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN); 694 695 desc_status = desc_read(xe, h2g, status); 696 if (desc_status) { 697 xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status); 698 goto corrupted; 699 } 700 701 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 702 u32 desc_tail = desc_read(xe, h2g, tail); 703 u32 desc_head = desc_read(xe, h2g, head); 704 705 if (tail != desc_tail) { 706 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH); 707 xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail); 708 goto corrupted; 709 } 710 711 if (tail > h2g->info.size) { 712 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 713 xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n", 714 tail, h2g->info.size); 715 goto corrupted; 716 } 717 718 if (desc_head >= h2g->info.size) { 719 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 720 xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n", 721 desc_head, h2g->info.size); 722 goto corrupted; 723 } 724 } 725 726 /* Command will wrap, zero fill (NOPs), return and check credits again */ 727 if (tail + full_len > h2g->info.size) { 728 xe_map_memset(xe, &map, 0, 0, 729 (h2g->info.size - tail) * sizeof(u32)); 730 h2g_reserve_space(ct, (h2g->info.size - tail)); 731 h2g->info.tail = 0; 732 desc_write(xe, h2g, tail, h2g->info.tail); 733 734 return -EAGAIN; 735 } 736 737 /* 738 * dw0: CT header (including fence) 739 * dw1: HXG header (including action code) 740 * dw2+: action data 741 */ 742 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 743 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 744 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value); 745 if (want_response) { 746 cmd[1] = 747 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 748 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 749 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 750 } else { 751 fast_req_track(ct, ct_fence_value, 752 FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0])); 753 754 cmd[1] = 755 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) | 756 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 757 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 758 } 759 760 /* H2G header in cmd[1] replaces action[0] so: */ 761 --len; 762 ++action; 763 764 /* Write H2G ensuring visible before descriptor update */ 765 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32)); 766 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32)); 767 xe_device_wmb(xe); 768 769 /* Update local copies */ 770 h2g->info.tail = (tail + full_len) % h2g->info.size; 771 h2g_reserve_space(ct, full_len); 772 773 /* Update descriptor */ 774 desc_write(xe, h2g, tail, h2g->info.tail); 775 776 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len, 777 desc_read(xe, h2g, head), h2g->info.tail); 778 779 return 0; 780 781 corrupted: 782 CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE); 783 return -EPIPE; 784 } 785 786 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, 787 u32 len, u32 g2h_len, u32 num_g2h, 788 struct g2h_fence *g2h_fence) 789 { 790 struct xe_gt *gt __maybe_unused = ct_to_gt(ct); 791 u16 seqno; 792 int ret; 793 794 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 795 xe_gt_assert(gt, !g2h_len || !g2h_fence); 796 xe_gt_assert(gt, !num_g2h || !g2h_fence); 797 xe_gt_assert(gt, !g2h_len || num_g2h); 798 xe_gt_assert(gt, g2h_len || !num_g2h); 799 lockdep_assert_held(&ct->lock); 800 801 if (unlikely(ct->ctbs.h2g.info.broken)) { 802 ret = -EPIPE; 803 goto out; 804 } 805 806 if (ct->state == XE_GUC_CT_STATE_DISABLED) { 807 ret = -ENODEV; 808 goto out; 809 } 810 811 if (ct->state == XE_GUC_CT_STATE_STOPPED) { 812 ret = -ECANCELED; 813 goto out; 814 } 815 816 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 817 818 if (g2h_fence) { 819 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; 820 num_g2h = 1; 821 822 if (g2h_fence_needs_alloc(g2h_fence)) { 823 g2h_fence->seqno = next_ct_seqno(ct, true); 824 ret = xa_err(xa_store(&ct->fence_lookup, 825 g2h_fence->seqno, g2h_fence, 826 GFP_ATOMIC)); 827 if (ret) 828 goto out; 829 } 830 831 seqno = g2h_fence->seqno; 832 } else { 833 seqno = next_ct_seqno(ct, false); 834 } 835 836 if (g2h_len) 837 spin_lock_irq(&ct->fast_lock); 838 retry: 839 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len); 840 if (unlikely(ret)) 841 goto out_unlock; 842 843 ret = h2g_write(ct, action, len, seqno, !!g2h_fence); 844 if (unlikely(ret)) { 845 if (ret == -EAGAIN) 846 goto retry; 847 goto out_unlock; 848 } 849 850 __g2h_reserve_space(ct, g2h_len, num_g2h); 851 xe_guc_notify(ct_to_guc(ct)); 852 out_unlock: 853 if (g2h_len) 854 spin_unlock_irq(&ct->fast_lock); 855 out: 856 return ret; 857 } 858 859 static void kick_reset(struct xe_guc_ct *ct) 860 { 861 xe_gt_reset_async(ct_to_gt(ct)); 862 } 863 864 static int dequeue_one_g2h(struct xe_guc_ct *ct); 865 866 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 867 u32 g2h_len, u32 num_g2h, 868 struct g2h_fence *g2h_fence) 869 { 870 struct xe_device *xe = ct_to_xe(ct); 871 struct xe_gt *gt = ct_to_gt(ct); 872 unsigned int sleep_period_ms = 1; 873 int ret; 874 875 xe_gt_assert(gt, !g2h_len || !g2h_fence); 876 lockdep_assert_held(&ct->lock); 877 xe_device_assert_mem_access(ct_to_xe(ct)); 878 879 try_again: 880 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, 881 g2h_fence); 882 883 /* 884 * We wait to try to restore credits for about 1 second before bailing. 885 * In the case of H2G credits we have no choice but just to wait for the 886 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In 887 * the case of G2H we process any G2H in the channel, hopefully freeing 888 * credits as we consume the G2H messages. 889 */ 890 if (unlikely(ret == -EBUSY && 891 !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) { 892 struct guc_ctb *h2g = &ct->ctbs.h2g; 893 894 if (sleep_period_ms == 1024) 895 goto broken; 896 897 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail, 898 h2g->info.size, 899 h2g->info.space, 900 len + GUC_CTB_HDR_LEN); 901 msleep(sleep_period_ms); 902 sleep_period_ms <<= 1; 903 904 goto try_again; 905 } else if (unlikely(ret == -EBUSY)) { 906 struct xe_device *xe = ct_to_xe(ct); 907 struct guc_ctb *g2h = &ct->ctbs.g2h; 908 909 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head, 910 desc_read(xe, g2h, tail), 911 g2h->info.size, 912 g2h->info.space, 913 g2h_fence ? 914 GUC_CTB_HXG_MSG_MAX_LEN : 915 g2h_len); 916 917 #define g2h_avail(ct) \ 918 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head) 919 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding || 920 g2h_avail(ct), HZ)) 921 goto broken; 922 #undef g2h_avail 923 924 ret = dequeue_one_g2h(ct); 925 if (ret < 0) { 926 if (ret != -ECANCELED) 927 xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)", 928 ERR_PTR(ret)); 929 goto broken; 930 } 931 932 goto try_again; 933 } 934 935 return ret; 936 937 broken: 938 xe_gt_err(gt, "No forward process on H2G, reset required\n"); 939 CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK); 940 941 return -EDEADLK; 942 } 943 944 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 945 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) 946 { 947 int ret; 948 949 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence); 950 951 mutex_lock(&ct->lock); 952 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence); 953 mutex_unlock(&ct->lock); 954 955 return ret; 956 } 957 958 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 959 u32 g2h_len, u32 num_g2h) 960 { 961 int ret; 962 963 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL); 964 if (ret == -EDEADLK) 965 kick_reset(ct); 966 967 return ret; 968 } 969 970 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 971 u32 g2h_len, u32 num_g2h) 972 { 973 int ret; 974 975 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL); 976 if (ret == -EDEADLK) 977 kick_reset(ct); 978 979 return ret; 980 } 981 982 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len) 983 { 984 int ret; 985 986 lockdep_assert_held(&ct->lock); 987 988 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL); 989 if (ret == -EDEADLK) 990 kick_reset(ct); 991 992 return ret; 993 } 994 995 /* 996 * Check if a GT reset is in progress or will occur and if GT reset brought the 997 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset. 998 */ 999 static bool retry_failure(struct xe_guc_ct *ct, int ret) 1000 { 1001 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV)) 1002 return false; 1003 1004 #define ct_alive(ct) \ 1005 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \ 1006 !ct->ctbs.g2h.info.broken) 1007 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) 1008 return false; 1009 #undef ct_alive 1010 1011 return true; 1012 } 1013 1014 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1015 u32 *response_buffer, bool no_fail) 1016 { 1017 struct xe_gt *gt = ct_to_gt(ct); 1018 struct g2h_fence g2h_fence; 1019 int ret = 0; 1020 1021 /* 1022 * We use a fence to implement blocking sends / receiving response data. 1023 * The seqno of the fence is sent in the H2G, returned in the G2H, and 1024 * an xarray is used as storage media with the seqno being to key. 1025 * Fields in the fence hold success, failure, retry status and the 1026 * response data. Safe to allocate on the stack as the xarray is the 1027 * only reference and it cannot be present after this function exits. 1028 */ 1029 retry: 1030 g2h_fence_init(&g2h_fence, response_buffer); 1031 retry_same_fence: 1032 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence); 1033 if (unlikely(ret == -ENOMEM)) { 1034 /* Retry allocation /w GFP_KERNEL */ 1035 ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno, 1036 &g2h_fence, GFP_KERNEL)); 1037 if (ret) 1038 return ret; 1039 1040 goto retry_same_fence; 1041 } else if (unlikely(ret)) { 1042 if (ret == -EDEADLK) 1043 kick_reset(ct); 1044 1045 if (no_fail && retry_failure(ct, ret)) 1046 goto retry_same_fence; 1047 1048 if (!g2h_fence_needs_alloc(&g2h_fence)) 1049 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1050 1051 return ret; 1052 } 1053 1054 ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ); 1055 if (!ret) { 1056 LNL_FLUSH_WORK(&ct->g2h_worker); 1057 if (g2h_fence.done) { 1058 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", 1059 g2h_fence.seqno, action[0]); 1060 ret = 1; 1061 } 1062 } 1063 1064 /* 1065 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on 1066 * the stack, since we have no clue if it will fire after the timeout before we can erase 1067 * from the xa. Also we have some dependent loads and stores below for which we need the 1068 * correct ordering, and we lack the needed barriers. 1069 */ 1070 mutex_lock(&ct->lock); 1071 if (!ret) { 1072 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s", 1073 g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done)); 1074 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1075 mutex_unlock(&ct->lock); 1076 return -ETIME; 1077 } 1078 1079 if (g2h_fence.retry) { 1080 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", 1081 action[0], g2h_fence.reason); 1082 mutex_unlock(&ct->lock); 1083 goto retry; 1084 } 1085 if (g2h_fence.fail) { 1086 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n", 1087 action[0], g2h_fence.error, g2h_fence.hint); 1088 ret = -EIO; 1089 } 1090 1091 if (ret > 0) 1092 ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data; 1093 1094 mutex_unlock(&ct->lock); 1095 1096 return ret; 1097 } 1098 1099 /** 1100 * xe_guc_ct_send_recv - Send and receive HXG to the GuC 1101 * @ct: the &xe_guc_ct 1102 * @action: the dword array with `HXG Request`_ message (can't be NULL) 1103 * @len: length of the `HXG Request`_ message (in dwords, can't be 0) 1104 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL) 1105 * 1106 * Send a `HXG Request`_ message to the GuC over CT communication channel and 1107 * blocks until GuC replies with a `HXG Response`_ message. 1108 * 1109 * For non-blocking communication with GuC use xe_guc_ct_send(). 1110 * 1111 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_. 1112 * 1113 * Return: response length (in dwords) if &response_buffer was not NULL, or 1114 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or 1115 * a negative error code on failure. 1116 */ 1117 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1118 u32 *response_buffer) 1119 { 1120 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer); 1121 return guc_ct_send_recv(ct, action, len, response_buffer, false); 1122 } 1123 ALLOW_ERROR_INJECTION(xe_guc_ct_send_recv, ERRNO); 1124 1125 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, 1126 u32 len, u32 *response_buffer) 1127 { 1128 return guc_ct_send_recv(ct, action, len, response_buffer, true); 1129 } 1130 1131 static u32 *msg_to_hxg(u32 *msg) 1132 { 1133 return msg + GUC_CTB_MSG_MIN_LEN; 1134 } 1135 1136 static u32 msg_len_to_hxg_len(u32 len) 1137 { 1138 return len - GUC_CTB_MSG_MIN_LEN; 1139 } 1140 1141 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) 1142 { 1143 u32 *hxg = msg_to_hxg(msg); 1144 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1145 1146 lockdep_assert_held(&ct->lock); 1147 1148 switch (action) { 1149 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1150 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1151 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1152 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1153 g2h_release_space(ct, len); 1154 } 1155 1156 return 0; 1157 } 1158 1159 static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) 1160 { 1161 struct xe_gt *gt = ct_to_gt(ct); 1162 1163 if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED) 1164 xe_gt_err(gt, "GuC Crash dump notification\n"); 1165 else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION) 1166 xe_gt_err(gt, "GuC Exception notification\n"); 1167 else 1168 xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action); 1169 1170 CT_DEAD(ct, NULL, CRASH); 1171 1172 kick_reset(ct); 1173 1174 return 0; 1175 } 1176 1177 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 1178 static void fast_req_report(struct xe_guc_ct *ct, u16 fence) 1179 { 1180 u16 fence_min = U16_MAX, fence_max = 0; 1181 struct xe_gt *gt = ct_to_gt(ct); 1182 bool found = false; 1183 unsigned int n; 1184 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 1185 char *buf; 1186 #endif 1187 1188 lockdep_assert_held(&ct->lock); 1189 1190 for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) { 1191 if (ct->fast_req[n].fence < fence_min) 1192 fence_min = ct->fast_req[n].fence; 1193 if (ct->fast_req[n].fence > fence_max) 1194 fence_max = ct->fast_req[n].fence; 1195 1196 if (ct->fast_req[n].fence != fence) 1197 continue; 1198 found = true; 1199 1200 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 1201 buf = kmalloc(SZ_4K, GFP_NOWAIT); 1202 if (buf && stack_depot_snprint(ct->fast_req[n].stack, buf, SZ_4K, 0)) 1203 xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s", 1204 fence, ct->fast_req[n].action, buf); 1205 else 1206 xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n", 1207 fence, ct->fast_req[n].action); 1208 kfree(buf); 1209 #else 1210 xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n", 1211 fence, ct->fast_req[n].action); 1212 #endif 1213 break; 1214 } 1215 1216 if (!found) 1217 xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n", 1218 fence, fence_min, fence_max, ct->fence_seqno); 1219 } 1220 #else 1221 static void fast_req_report(struct xe_guc_ct *ct, u16 fence) 1222 { 1223 } 1224 #endif 1225 1226 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) 1227 { 1228 struct xe_gt *gt = ct_to_gt(ct); 1229 u32 *hxg = msg_to_hxg(msg); 1230 u32 hxg_len = msg_len_to_hxg_len(len); 1231 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); 1232 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1233 struct g2h_fence *g2h_fence; 1234 1235 lockdep_assert_held(&ct->lock); 1236 1237 /* 1238 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup. 1239 * Those messages should never fail, so if we do get an error back it 1240 * means we're likely doing an illegal operation and the GuC is 1241 * rejecting it. We have no way to inform the code that submitted the 1242 * H2G that the message was rejected, so we need to escalate the 1243 * failure to trigger a reset. 1244 */ 1245 if (fence & CT_SEQNO_UNTRACKED) { 1246 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) 1247 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n", 1248 fence, 1249 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]), 1250 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0])); 1251 else 1252 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n", 1253 type, fence); 1254 1255 fast_req_report(ct, fence); 1256 1257 CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE); 1258 1259 return -EPROTO; 1260 } 1261 1262 g2h_fence = xa_erase(&ct->fence_lookup, fence); 1263 if (unlikely(!g2h_fence)) { 1264 /* Don't tear down channel, as send could've timed out */ 1265 /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ 1266 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence); 1267 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1268 return 0; 1269 } 1270 1271 xe_gt_assert(gt, fence == g2h_fence->seqno); 1272 1273 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { 1274 g2h_fence->fail = true; 1275 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); 1276 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]); 1277 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 1278 g2h_fence->retry = true; 1279 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); 1280 } else if (g2h_fence->response_buffer) { 1281 g2h_fence->response_len = hxg_len; 1282 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); 1283 } else { 1284 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); 1285 } 1286 1287 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1288 1289 g2h_fence->done = true; 1290 smp_mb(); 1291 1292 wake_up_all(&ct->g2h_fence_wq); 1293 1294 return 0; 1295 } 1296 1297 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1298 { 1299 struct xe_gt *gt = ct_to_gt(ct); 1300 u32 *hxg = msg_to_hxg(msg); 1301 u32 origin, type; 1302 int ret; 1303 1304 lockdep_assert_held(&ct->lock); 1305 1306 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1307 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1308 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n", 1309 origin); 1310 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN); 1311 1312 return -EPROTO; 1313 } 1314 1315 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1316 switch (type) { 1317 case GUC_HXG_TYPE_EVENT: 1318 ret = parse_g2h_event(ct, msg, len); 1319 break; 1320 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1321 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1322 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1323 ret = parse_g2h_response(ct, msg, len); 1324 break; 1325 default: 1326 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n", 1327 type); 1328 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE); 1329 1330 ret = -EOPNOTSUPP; 1331 } 1332 1333 return ret; 1334 } 1335 1336 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1337 { 1338 struct xe_guc *guc = ct_to_guc(ct); 1339 struct xe_gt *gt = ct_to_gt(ct); 1340 u32 hxg_len = msg_len_to_hxg_len(len); 1341 u32 *hxg = msg_to_hxg(msg); 1342 u32 action, adj_len; 1343 u32 *payload; 1344 int ret = 0; 1345 1346 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1347 return 0; 1348 1349 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1350 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN; 1351 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN; 1352 1353 switch (action) { 1354 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1355 ret = xe_guc_sched_done_handler(guc, payload, adj_len); 1356 break; 1357 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1358 ret = xe_guc_deregister_done_handler(guc, payload, adj_len); 1359 break; 1360 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1361 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len); 1362 break; 1363 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1364 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload, 1365 adj_len); 1366 break; 1367 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1368 /* Selftest only at the moment */ 1369 break; 1370 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1371 ret = xe_guc_error_capture_handler(guc, payload, adj_len); 1372 break; 1373 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1374 /* FIXME: Handle this */ 1375 break; 1376 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: 1377 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload, 1378 adj_len); 1379 break; 1380 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1381 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1382 break; 1383 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1384 ret = xe_guc_tlb_invalidation_done_handler(guc, payload, 1385 adj_len); 1386 break; 1387 case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY: 1388 ret = xe_guc_access_counter_notify_handler(guc, payload, 1389 adj_len); 1390 break; 1391 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: 1392 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len); 1393 break; 1394 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF: 1395 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len); 1396 break; 1397 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY: 1398 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len); 1399 break; 1400 case GUC_ACTION_GUC2PF_ADVERSE_EVENT: 1401 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len); 1402 break; 1403 case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1404 case XE_GUC_ACTION_NOTIFY_EXCEPTION: 1405 ret = guc_crash_process_msg(ct, action); 1406 break; 1407 default: 1408 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); 1409 } 1410 1411 if (ret) { 1412 xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n", 1413 action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg); 1414 CT_DEAD(ct, NULL, PROCESS_FAILED); 1415 } 1416 1417 return 0; 1418 } 1419 1420 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) 1421 { 1422 struct xe_device *xe = ct_to_xe(ct); 1423 struct xe_gt *gt = ct_to_gt(ct); 1424 struct guc_ctb *g2h = &ct->ctbs.g2h; 1425 u32 tail, head, len, desc_status; 1426 s32 avail; 1427 u32 action; 1428 u32 *hxg; 1429 1430 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 1431 lockdep_assert_held(&ct->fast_lock); 1432 1433 if (ct->state == XE_GUC_CT_STATE_DISABLED) 1434 return -ENODEV; 1435 1436 if (ct->state == XE_GUC_CT_STATE_STOPPED) 1437 return -ECANCELED; 1438 1439 if (g2h->info.broken) 1440 return -EPIPE; 1441 1442 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1443 1444 desc_status = desc_read(xe, g2h, status); 1445 if (desc_status) { 1446 if (desc_status & GUC_CTB_STATUS_DISABLED) { 1447 /* 1448 * Potentially valid if a CLIENT_RESET request resulted in 1449 * contexts/engines being reset. But should never happen as 1450 * no contexts should be active when CLIENT_RESET is sent. 1451 */ 1452 xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n"); 1453 desc_status &= ~GUC_CTB_STATUS_DISABLED; 1454 } 1455 1456 if (desc_status) { 1457 xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status); 1458 goto corrupted; 1459 } 1460 } 1461 1462 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 1463 u32 desc_tail = desc_read(xe, g2h, tail); 1464 /* 1465 u32 desc_head = desc_read(xe, g2h, head); 1466 1467 * info.head and desc_head are updated back-to-back at the end of 1468 * this function and nowhere else. Hence, they cannot be different 1469 * unless two g2h_read calls are running concurrently. Which is not 1470 * possible because it is guarded by ct->fast_lock. And yet, some 1471 * discrete platforms are regularly hitting this error :(. 1472 * 1473 * desc_head rolling backwards shouldn't cause any noticeable 1474 * problems - just a delay in GuC being allowed to proceed past that 1475 * point in the queue. So for now, just disable the error until it 1476 * can be root caused. 1477 * 1478 if (g2h->info.head != desc_head) { 1479 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH); 1480 xe_gt_err(gt, "CT read: head was modified %u != %u\n", 1481 desc_head, g2h->info.head); 1482 goto corrupted; 1483 } 1484 */ 1485 1486 if (g2h->info.head > g2h->info.size) { 1487 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1488 xe_gt_err(gt, "CT read: head out of range: %u vs %u\n", 1489 g2h->info.head, g2h->info.size); 1490 goto corrupted; 1491 } 1492 1493 if (desc_tail >= g2h->info.size) { 1494 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1495 xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n", 1496 desc_tail, g2h->info.size); 1497 goto corrupted; 1498 } 1499 } 1500 1501 /* Calculate DW available to read */ 1502 tail = desc_read(xe, g2h, tail); 1503 avail = tail - g2h->info.head; 1504 if (unlikely(avail == 0)) 1505 return 0; 1506 1507 if (avail < 0) 1508 avail += g2h->info.size; 1509 1510 /* Read header */ 1511 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head, 1512 sizeof(u32)); 1513 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; 1514 if (len > avail) { 1515 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n", 1516 avail, len); 1517 goto corrupted; 1518 } 1519 1520 head = (g2h->info.head + 1) % g2h->info.size; 1521 avail = len - 1; 1522 1523 /* Read G2H message */ 1524 if (avail + head > g2h->info.size) { 1525 u32 avail_til_wrap = g2h->info.size - head; 1526 1527 xe_map_memcpy_from(xe, msg + 1, 1528 &g2h->cmds, sizeof(u32) * head, 1529 avail_til_wrap * sizeof(u32)); 1530 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap, 1531 &g2h->cmds, 0, 1532 (avail - avail_til_wrap) * sizeof(u32)); 1533 } else { 1534 xe_map_memcpy_from(xe, msg + 1, 1535 &g2h->cmds, sizeof(u32) * head, 1536 avail * sizeof(u32)); 1537 } 1538 1539 hxg = msg_to_hxg(msg); 1540 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1541 1542 if (fast_path) { 1543 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1544 return 0; 1545 1546 switch (action) { 1547 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1548 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1549 break; /* Process these in fast-path */ 1550 default: 1551 return 0; 1552 } 1553 } 1554 1555 /* Update local / descriptor header */ 1556 g2h->info.head = (head + avail) % g2h->info.size; 1557 desc_write(xe, g2h, head, g2h->info.head); 1558 1559 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id, 1560 action, len, g2h->info.head, tail); 1561 1562 return len; 1563 1564 corrupted: 1565 CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ); 1566 return -EPROTO; 1567 } 1568 1569 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) 1570 { 1571 struct xe_gt *gt = ct_to_gt(ct); 1572 struct xe_guc *guc = ct_to_guc(ct); 1573 u32 hxg_len = msg_len_to_hxg_len(len); 1574 u32 *hxg = msg_to_hxg(msg); 1575 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1576 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN; 1577 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1578 int ret = 0; 1579 1580 switch (action) { 1581 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1582 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1583 break; 1584 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1585 __g2h_release_space(ct, len); 1586 ret = xe_guc_tlb_invalidation_done_handler(guc, payload, 1587 adj_len); 1588 break; 1589 default: 1590 xe_gt_warn(gt, "NOT_POSSIBLE"); 1591 } 1592 1593 if (ret) { 1594 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n", 1595 action, ERR_PTR(ret)); 1596 CT_DEAD(ct, NULL, FAST_G2H); 1597 } 1598 } 1599 1600 /** 1601 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler 1602 * @ct: GuC CT object 1603 * 1604 * Anything related to page faults is critical for performance, process these 1605 * critical G2H in the IRQ. This is safe as these handlers either just wake up 1606 * waiters or queue another worker. 1607 */ 1608 void xe_guc_ct_fast_path(struct xe_guc_ct *ct) 1609 { 1610 struct xe_device *xe = ct_to_xe(ct); 1611 bool ongoing; 1612 int len; 1613 1614 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1615 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1616 return; 1617 1618 spin_lock(&ct->fast_lock); 1619 do { 1620 len = g2h_read(ct, ct->fast_msg, true); 1621 if (len > 0) 1622 g2h_fast_path(ct, ct->fast_msg, len); 1623 } while (len > 0); 1624 spin_unlock(&ct->fast_lock); 1625 1626 if (ongoing) 1627 xe_pm_runtime_put(xe); 1628 } 1629 1630 /* Returns less than zero on error, 0 on done, 1 on more available */ 1631 static int dequeue_one_g2h(struct xe_guc_ct *ct) 1632 { 1633 int len; 1634 int ret; 1635 1636 lockdep_assert_held(&ct->lock); 1637 1638 spin_lock_irq(&ct->fast_lock); 1639 len = g2h_read(ct, ct->msg, false); 1640 spin_unlock_irq(&ct->fast_lock); 1641 if (len <= 0) 1642 return len; 1643 1644 ret = parse_g2h_msg(ct, ct->msg, len); 1645 if (unlikely(ret < 0)) 1646 return ret; 1647 1648 ret = process_g2h_msg(ct, ct->msg, len); 1649 if (unlikely(ret < 0)) 1650 return ret; 1651 1652 return 1; 1653 } 1654 1655 static void receive_g2h(struct xe_guc_ct *ct) 1656 { 1657 bool ongoing; 1658 int ret; 1659 1660 /* 1661 * Normal users must always hold mem_access.ref around CT calls. However 1662 * during the runtime pm callbacks we rely on CT to talk to the GuC, but 1663 * at this stage we can't rely on mem_access.ref and even the 1664 * callback_task will be different than current. For such cases we just 1665 * need to ensure we always process the responses from any blocking 1666 * ct_send requests or where we otherwise expect some response when 1667 * initiated from those callbacks (which will need to wait for the below 1668 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if 1669 * the device has suspended to the point that the CT communication has 1670 * been disabled. 1671 * 1672 * If we are inside the runtime pm callback, we can be the only task 1673 * still issuing CT requests (since that requires having the 1674 * mem_access.ref). It seems like it might in theory be possible to 1675 * receive unsolicited events from the GuC just as we are 1676 * suspending-resuming, but those will currently anyway be lost when 1677 * eventually exiting from suspend, hence no need to wake up the device 1678 * here. If we ever need something stronger than get_if_ongoing() then 1679 * we need to be careful with blocking the pm callbacks from getting CT 1680 * responses, if the worker here is blocked on those callbacks 1681 * completing, creating a deadlock. 1682 */ 1683 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1684 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1685 return; 1686 1687 do { 1688 mutex_lock(&ct->lock); 1689 ret = dequeue_one_g2h(ct); 1690 mutex_unlock(&ct->lock); 1691 1692 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) { 1693 xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d", ret); 1694 CT_DEAD(ct, NULL, G2H_RECV); 1695 kick_reset(ct); 1696 } 1697 } while (ret == 1); 1698 1699 if (ongoing) 1700 xe_pm_runtime_put(ct_to_xe(ct)); 1701 } 1702 1703 static void g2h_worker_func(struct work_struct *w) 1704 { 1705 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker); 1706 1707 receive_g2h(ct); 1708 } 1709 1710 static void xe_fixup_u64_in_cmds(struct xe_device *xe, struct iosys_map *cmds, 1711 u32 size, u32 idx, s64 shift) 1712 { 1713 u32 hi, lo; 1714 u64 offset; 1715 1716 lo = xe_map_rd_ring_u32(xe, cmds, idx, size); 1717 hi = xe_map_rd_ring_u32(xe, cmds, idx + 1, size); 1718 offset = make_u64(hi, lo); 1719 offset += shift; 1720 lo = lower_32_bits(offset); 1721 hi = upper_32_bits(offset); 1722 xe_map_wr_ring_u32(xe, cmds, idx, size, lo); 1723 xe_map_wr_ring_u32(xe, cmds, idx + 1, size, hi); 1724 } 1725 1726 /* 1727 * Shift any GGTT addresses within a single message left within CTB from 1728 * before post-migration recovery. 1729 * @ct: pointer to CT struct of the target GuC 1730 * @cmds: iomap buffer containing CT messages 1731 * @head: start of the target message within the buffer 1732 * @len: length of the target message 1733 * @size: size of the commands buffer 1734 * @shift: the address shift to be added to each GGTT reference 1735 * Return: true if the message was fixed or needed no fixups, false on failure 1736 */ 1737 static bool ct_fixup_ggtt_in_message(struct xe_guc_ct *ct, 1738 struct iosys_map *cmds, u32 head, 1739 u32 len, u32 size, s64 shift) 1740 { 1741 struct xe_gt *gt = ct_to_gt(ct); 1742 struct xe_device *xe = ct_to_xe(ct); 1743 u32 msg[GUC_HXG_MSG_MIN_LEN]; 1744 u32 action, i, n; 1745 1746 xe_gt_assert(gt, len >= GUC_HXG_MSG_MIN_LEN); 1747 1748 msg[0] = xe_map_rd_ring_u32(xe, cmds, head, size); 1749 action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]); 1750 1751 xe_gt_sriov_dbg_verbose(gt, "fixing H2G %#x\n", action); 1752 1753 switch (action) { 1754 case XE_GUC_ACTION_REGISTER_CONTEXT: 1755 if (len != XE_GUC_REGISTER_CONTEXT_MSG_LEN) 1756 goto err_len; 1757 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1758 XE_GUC_REGISTER_CONTEXT_DATA_5_WQ_DESC_ADDR_LOWER, 1759 shift); 1760 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1761 XE_GUC_REGISTER_CONTEXT_DATA_7_WQ_BUF_BASE_LOWER, 1762 shift); 1763 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1764 XE_GUC_REGISTER_CONTEXT_DATA_10_HW_LRC_ADDR, shift); 1765 break; 1766 case XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC: 1767 if (len < XE_GUC_REGISTER_CONTEXT_MULTI_LRC_MSG_MIN_LEN) 1768 goto err_len; 1769 n = xe_map_rd_ring_u32(xe, cmds, head + 1770 XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_10_NUM_CTXS, size); 1771 if (len != XE_GUC_REGISTER_CONTEXT_MULTI_LRC_MSG_MIN_LEN + 2 * n) 1772 goto err_len; 1773 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1774 XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_5_WQ_DESC_ADDR_LOWER, 1775 shift); 1776 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1777 XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_7_WQ_BUF_BASE_LOWER, 1778 shift); 1779 for (i = 0; i < n; i++) 1780 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1781 XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_11_HW_LRC_ADDR 1782 + 2 * i, shift); 1783 break; 1784 default: 1785 break; 1786 } 1787 return true; 1788 1789 err_len: 1790 xe_gt_err(gt, "Skipped G2G %#x message fixups, unexpected length (%u)\n", action, len); 1791 return false; 1792 } 1793 1794 /* 1795 * Apply fixups to the next outgoing CT message within given CTB 1796 * @ct: the &xe_guc_ct struct instance representing the target GuC 1797 * @h2g: the &guc_ctb struct instance of the target buffer 1798 * @shift: shift to be added to all GGTT addresses within the CTB 1799 * @mhead: pointer to an integer storing message start position; the 1800 * position is changed to next message before this function return 1801 * @avail: size of the area available for parsing, that is length 1802 * of all remaining messages stored within the CTB 1803 * Return: size of the area available for parsing after one message 1804 * has been parsed, that is length remaining from the updated mhead 1805 */ 1806 static int ct_fixup_ggtt_in_buffer(struct xe_guc_ct *ct, struct guc_ctb *h2g, 1807 s64 shift, u32 *mhead, s32 avail) 1808 { 1809 struct xe_gt *gt = ct_to_gt(ct); 1810 struct xe_device *xe = ct_to_xe(ct); 1811 u32 msg[GUC_HXG_MSG_MIN_LEN]; 1812 u32 size = h2g->info.size; 1813 u32 head = *mhead; 1814 u32 len; 1815 1816 xe_gt_assert(gt, avail >= (s32)GUC_CTB_MSG_MIN_LEN); 1817 1818 /* Read header */ 1819 msg[0] = xe_map_rd_ring_u32(xe, &h2g->cmds, head, size); 1820 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; 1821 1822 if (unlikely(len > (u32)avail)) { 1823 xe_gt_err(gt, "H2G channel broken on read, avail=%d, len=%d, fixups skipped\n", 1824 avail, len); 1825 return 0; 1826 } 1827 1828 head = (head + GUC_CTB_MSG_MIN_LEN) % size; 1829 if (!ct_fixup_ggtt_in_message(ct, &h2g->cmds, head, msg_len_to_hxg_len(len), size, shift)) 1830 return 0; 1831 *mhead = (head + msg_len_to_hxg_len(len)) % size; 1832 1833 return avail - len; 1834 } 1835 1836 /** 1837 * xe_guc_ct_fixup_messages_with_ggtt - Fixup any pending H2G CTB messages 1838 * @ct: pointer to CT struct of the target GuC 1839 * @ggtt_shift: shift to be added to all GGTT addresses within the CTB 1840 * 1841 * Messages in GuC to Host CTB are owned by GuC and any fixups in them 1842 * are made by GuC. But content of the Host to GuC CTB is owned by the 1843 * KMD, so fixups to GGTT references in any pending messages need to be 1844 * applied here. 1845 * This function updates GGTT offsets in payloads of pending H2G CTB 1846 * messages (messages which were not consumed by GuC before the VF got 1847 * paused). 1848 */ 1849 void xe_guc_ct_fixup_messages_with_ggtt(struct xe_guc_ct *ct, s64 ggtt_shift) 1850 { 1851 struct guc_ctb *h2g = &ct->ctbs.h2g; 1852 struct xe_guc *guc = ct_to_guc(ct); 1853 struct xe_gt *gt = guc_to_gt(guc); 1854 u32 head, tail, size; 1855 s32 avail; 1856 1857 if (unlikely(h2g->info.broken)) 1858 return; 1859 1860 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); 1861 head = h2g->info.head; 1862 tail = READ_ONCE(h2g->info.tail); 1863 size = h2g->info.size; 1864 1865 if (unlikely(head > size)) 1866 goto corrupted; 1867 1868 if (unlikely(tail >= size)) 1869 goto corrupted; 1870 1871 avail = tail - head; 1872 1873 /* beware of buffer wrap case */ 1874 if (unlikely(avail < 0)) 1875 avail += size; 1876 xe_gt_dbg(gt, "available %d (%u:%u:%u)\n", avail, head, tail, size); 1877 xe_gt_assert(gt, avail >= 0); 1878 1879 while (avail > 0) 1880 avail = ct_fixup_ggtt_in_buffer(ct, h2g, ggtt_shift, &head, avail); 1881 1882 return; 1883 1884 corrupted: 1885 xe_gt_err(gt, "Corrupted H2G descriptor head=%u tail=%u size=%u, fixups not applied\n", 1886 head, tail, size); 1887 h2g->info.broken = true; 1888 } 1889 1890 static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic, 1891 bool want_ctb) 1892 { 1893 struct xe_guc_ct_snapshot *snapshot; 1894 1895 snapshot = kzalloc(sizeof(*snapshot), atomic ? GFP_ATOMIC : GFP_KERNEL); 1896 if (!snapshot) 1897 return NULL; 1898 1899 if (ct->bo && want_ctb) { 1900 snapshot->ctb_size = ct->bo->size; 1901 snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL); 1902 } 1903 1904 return snapshot; 1905 } 1906 1907 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb, 1908 struct guc_ctb_snapshot *snapshot) 1909 { 1910 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0, 1911 sizeof(struct guc_ct_buffer_desc)); 1912 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info)); 1913 } 1914 1915 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot, 1916 struct drm_printer *p) 1917 { 1918 drm_printf(p, "\tsize: %d\n", snapshot->info.size); 1919 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space); 1920 drm_printf(p, "\thead: %d\n", snapshot->info.head); 1921 drm_printf(p, "\ttail: %d\n", snapshot->info.tail); 1922 drm_printf(p, "\tspace: %d\n", snapshot->info.space); 1923 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken); 1924 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head); 1925 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail); 1926 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status); 1927 } 1928 1929 static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic, 1930 bool want_ctb) 1931 { 1932 struct xe_device *xe = ct_to_xe(ct); 1933 struct xe_guc_ct_snapshot *snapshot; 1934 1935 snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb); 1936 if (!snapshot) { 1937 xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n"); 1938 return NULL; 1939 } 1940 1941 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) { 1942 snapshot->ct_enabled = true; 1943 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding); 1944 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g); 1945 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h); 1946 } 1947 1948 if (ct->bo && snapshot->ctb) 1949 xe_map_memcpy_from(xe, snapshot->ctb, &ct->bo->vmap, 0, snapshot->ctb_size); 1950 1951 return snapshot; 1952 } 1953 1954 /** 1955 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state. 1956 * @ct: GuC CT object. 1957 * 1958 * This can be printed out in a later stage like during dev_coredump 1959 * analysis. This is safe to be called during atomic context. 1960 * 1961 * Returns: a GuC CT snapshot object that must be freed by the caller 1962 * by using `xe_guc_ct_snapshot_free`. 1963 */ 1964 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct) 1965 { 1966 return guc_ct_snapshot_capture(ct, true, true); 1967 } 1968 1969 /** 1970 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot. 1971 * @snapshot: GuC CT snapshot object. 1972 * @p: drm_printer where it will be printed out. 1973 * 1974 * This function prints out a given GuC CT snapshot object. 1975 */ 1976 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, 1977 struct drm_printer *p) 1978 { 1979 if (!snapshot) 1980 return; 1981 1982 if (snapshot->ct_enabled) { 1983 drm_puts(p, "H2G CTB (all sizes in DW):\n"); 1984 guc_ctb_snapshot_print(&snapshot->h2g, p); 1985 1986 drm_puts(p, "G2H CTB (all sizes in DW):\n"); 1987 guc_ctb_snapshot_print(&snapshot->g2h, p); 1988 drm_printf(p, "\tg2h outstanding: %d\n", 1989 snapshot->g2h_outstanding); 1990 1991 if (snapshot->ctb) { 1992 drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size); 1993 xe_print_blob_ascii85(p, "[CTB].data", '\n', 1994 snapshot->ctb, 0, snapshot->ctb_size); 1995 } 1996 } else { 1997 drm_puts(p, "CT disabled\n"); 1998 } 1999 } 2000 2001 /** 2002 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot. 2003 * @snapshot: GuC CT snapshot object. 2004 * 2005 * This function free all the memory that needed to be allocated at capture 2006 * time. 2007 */ 2008 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot) 2009 { 2010 if (!snapshot) 2011 return; 2012 2013 kfree(snapshot->ctb); 2014 kfree(snapshot); 2015 } 2016 2017 /** 2018 * xe_guc_ct_print - GuC CT Print. 2019 * @ct: GuC CT. 2020 * @p: drm_printer where it will be printed out. 2021 * @want_ctb: Should the full CTB content be dumped (vs just the headers) 2022 * 2023 * This function will quickly capture a snapshot of the CT state 2024 * and immediately print it out. 2025 */ 2026 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb) 2027 { 2028 struct xe_guc_ct_snapshot *snapshot; 2029 2030 snapshot = guc_ct_snapshot_capture(ct, false, want_ctb); 2031 xe_guc_ct_snapshot_print(snapshot, p); 2032 xe_guc_ct_snapshot_free(snapshot); 2033 } 2034 2035 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 2036 2037 #ifdef CONFIG_FUNCTION_ERROR_INJECTION 2038 /* 2039 * This is a helper function which assists the driver in identifying if a fault 2040 * injection test is currently active, allowing it to reduce unnecessary debug 2041 * output. Typically, the function returns zero, but the fault injection 2042 * framework can alter this to return an error. Since faults are injected 2043 * through this function, it's important to ensure the compiler doesn't optimize 2044 * it into an inline function. To avoid such optimization, the 'noinline' 2045 * attribute is applied. Compiler optimizes the static function defined in the 2046 * header file as an inline function. 2047 */ 2048 noinline int xe_is_injection_active(void) { return 0; } 2049 ALLOW_ERROR_INJECTION(xe_is_injection_active, ERRNO); 2050 #else 2051 int xe_is_injection_active(void) { return 0; } 2052 #endif 2053 2054 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code) 2055 { 2056 struct xe_guc_log_snapshot *snapshot_log; 2057 struct xe_guc_ct_snapshot *snapshot_ct; 2058 struct xe_guc *guc = ct_to_guc(ct); 2059 unsigned long flags; 2060 bool have_capture; 2061 2062 if (ctb) 2063 ctb->info.broken = true; 2064 /* 2065 * Huge dump is getting generated when injecting error for guc CT/MMIO 2066 * functions. So, let us suppress the dump when fault is injected. 2067 */ 2068 if (xe_is_injection_active()) 2069 return; 2070 2071 /* Ignore further errors after the first dump until a reset */ 2072 if (ct->dead.reported) 2073 return; 2074 2075 spin_lock_irqsave(&ct->dead.lock, flags); 2076 2077 /* And only capture one dump at a time */ 2078 have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE); 2079 ct->dead.reason |= (1 << reason_code) | 2080 (1 << CT_DEAD_STATE_CAPTURE); 2081 2082 spin_unlock_irqrestore(&ct->dead.lock, flags); 2083 2084 if (have_capture) 2085 return; 2086 2087 snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true); 2088 snapshot_ct = xe_guc_ct_snapshot_capture((ct)); 2089 2090 spin_lock_irqsave(&ct->dead.lock, flags); 2091 2092 if (ct->dead.snapshot_log || ct->dead.snapshot_ct) { 2093 xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n"); 2094 xe_guc_log_snapshot_free(snapshot_log); 2095 xe_guc_ct_snapshot_free(snapshot_ct); 2096 } else { 2097 ct->dead.snapshot_log = snapshot_log; 2098 ct->dead.snapshot_ct = snapshot_ct; 2099 } 2100 2101 spin_unlock_irqrestore(&ct->dead.lock, flags); 2102 2103 queue_work(system_unbound_wq, &(ct)->dead.worker); 2104 } 2105 2106 static void ct_dead_print(struct xe_dead_ct *dead) 2107 { 2108 struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead); 2109 struct xe_device *xe = ct_to_xe(ct); 2110 struct xe_gt *gt = ct_to_gt(ct); 2111 static int g_count; 2112 struct drm_printer ip = xe_gt_info_printer(gt); 2113 struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count); 2114 2115 if (!dead->reason) { 2116 xe_gt_err(gt, "CTB is dead for no reason!?\n"); 2117 return; 2118 } 2119 2120 /* Can't generate a genuine core dump at this point, so just do the good bits */ 2121 drm_puts(&lp, "**** Xe Device Coredump ****\n"); 2122 drm_printf(&lp, "Reason: CTB is dead - 0x%X\n", dead->reason); 2123 xe_device_snapshot_print(xe, &lp); 2124 2125 drm_printf(&lp, "**** GT #%d ****\n", gt->info.id); 2126 drm_printf(&lp, "\tTile: %d\n", gt->tile->id); 2127 2128 drm_puts(&lp, "**** GuC Log ****\n"); 2129 xe_guc_log_snapshot_print(dead->snapshot_log, &lp); 2130 2131 drm_puts(&lp, "**** GuC CT ****\n"); 2132 xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp); 2133 2134 drm_puts(&lp, "Done.\n"); 2135 } 2136 2137 static void ct_dead_worker_func(struct work_struct *w) 2138 { 2139 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker); 2140 2141 if (!ct->dead.reported) { 2142 ct->dead.reported = true; 2143 ct_dead_print(&ct->dead); 2144 } 2145 2146 spin_lock_irq(&ct->dead.lock); 2147 2148 xe_guc_log_snapshot_free(ct->dead.snapshot_log); 2149 ct->dead.snapshot_log = NULL; 2150 xe_guc_ct_snapshot_free(ct->dead.snapshot_ct); 2151 ct->dead.snapshot_ct = NULL; 2152 2153 if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) { 2154 /* A reset has occurred so re-arm the error reporting */ 2155 ct->dead.reason = 0; 2156 ct->dead.reported = false; 2157 } 2158 2159 spin_unlock_irq(&ct->dead.lock); 2160 } 2161 #endif 2162