1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ct.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 #include <linux/delay.h> 11 #include <linux/fault-inject.h> 12 13 #include <kunit/static_stub.h> 14 15 #include <drm/drm_managed.h> 16 17 #include "abi/guc_actions_abi.h" 18 #include "abi/guc_actions_sriov_abi.h" 19 #include "abi/guc_klvs_abi.h" 20 #include "xe_bo.h" 21 #include "xe_devcoredump.h" 22 #include "xe_device.h" 23 #include "xe_gt.h" 24 #include "xe_gt_pagefault.h" 25 #include "xe_gt_printk.h" 26 #include "xe_gt_sriov_pf_control.h" 27 #include "xe_gt_sriov_pf_monitor.h" 28 #include "xe_gt_sriov_printk.h" 29 #include "xe_gt_tlb_invalidation.h" 30 #include "xe_guc.h" 31 #include "xe_guc_log.h" 32 #include "xe_guc_relay.h" 33 #include "xe_guc_submit.h" 34 #include "xe_map.h" 35 #include "xe_pm.h" 36 #include "xe_trace_guc.h" 37 38 static void receive_g2h(struct xe_guc_ct *ct); 39 static void g2h_worker_func(struct work_struct *w); 40 static void safe_mode_worker_func(struct work_struct *w); 41 static void ct_exit_safe_mode(struct xe_guc_ct *ct); 42 43 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 44 enum { 45 /* Internal states, not error conditions */ 46 CT_DEAD_STATE_REARM, /* 0x0001 */ 47 CT_DEAD_STATE_CAPTURE, /* 0x0002 */ 48 49 /* Error conditions */ 50 CT_DEAD_SETUP, /* 0x0004 */ 51 CT_DEAD_H2G_WRITE, /* 0x0008 */ 52 CT_DEAD_H2G_HAS_ROOM, /* 0x0010 */ 53 CT_DEAD_G2H_READ, /* 0x0020 */ 54 CT_DEAD_G2H_RECV, /* 0x0040 */ 55 CT_DEAD_G2H_RELEASE, /* 0x0080 */ 56 CT_DEAD_DEADLOCK, /* 0x0100 */ 57 CT_DEAD_PROCESS_FAILED, /* 0x0200 */ 58 CT_DEAD_FAST_G2H, /* 0x0400 */ 59 CT_DEAD_PARSE_G2H_RESPONSE, /* 0x0800 */ 60 CT_DEAD_PARSE_G2H_UNKNOWN, /* 0x1000 */ 61 CT_DEAD_PARSE_G2H_ORIGIN, /* 0x2000 */ 62 CT_DEAD_PARSE_G2H_TYPE, /* 0x4000 */ 63 CT_DEAD_CRASH, /* 0x8000 */ 64 }; 65 66 static void ct_dead_worker_func(struct work_struct *w); 67 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code); 68 69 #define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code) 70 #else 71 #define CT_DEAD(ct, ctb, reason) \ 72 do { \ 73 struct guc_ctb *_ctb = (ctb); \ 74 if (_ctb) \ 75 _ctb->info.broken = true; \ 76 } while (0) 77 #endif 78 79 /* Used when a CT send wants to block and / or receive data */ 80 struct g2h_fence { 81 u32 *response_buffer; 82 u32 seqno; 83 u32 response_data; 84 u16 response_len; 85 u16 error; 86 u16 hint; 87 u16 reason; 88 bool retry; 89 bool fail; 90 bool done; 91 }; 92 93 #define make_u64(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo))) 94 95 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) 96 { 97 g2h_fence->response_buffer = response_buffer; 98 g2h_fence->response_data = 0; 99 g2h_fence->response_len = 0; 100 g2h_fence->fail = false; 101 g2h_fence->retry = false; 102 g2h_fence->done = false; 103 g2h_fence->seqno = ~0x0; 104 } 105 106 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) 107 { 108 return g2h_fence->seqno == ~0x0; 109 } 110 111 static struct xe_guc * 112 ct_to_guc(struct xe_guc_ct *ct) 113 { 114 return container_of(ct, struct xe_guc, ct); 115 } 116 117 static struct xe_gt * 118 ct_to_gt(struct xe_guc_ct *ct) 119 { 120 return container_of(ct, struct xe_gt, uc.guc.ct); 121 } 122 123 static struct xe_device * 124 ct_to_xe(struct xe_guc_ct *ct) 125 { 126 return gt_to_xe(ct_to_gt(ct)); 127 } 128 129 /** 130 * DOC: GuC CTB Blob 131 * 132 * We allocate single blob to hold both CTB descriptors and buffers: 133 * 134 * +--------+-----------------------------------------------+------+ 135 * | offset | contents | size | 136 * +========+===============================================+======+ 137 * | 0x0000 | H2G CTB Descriptor (send) | | 138 * +--------+-----------------------------------------------+ 4K | 139 * | 0x0800 | G2H CTB Descriptor (g2h) | | 140 * +--------+-----------------------------------------------+------+ 141 * | 0x1000 | H2G CT Buffer (send) | n*4K | 142 * | | | | 143 * +--------+-----------------------------------------------+------+ 144 * | 0x1000 | G2H CT Buffer (g2h) | m*4K | 145 * | + n*4K | | | 146 * +--------+-----------------------------------------------+------+ 147 * 148 * Size of each ``CT Buffer`` must be multiple of 4K. 149 * We don't expect too many messages in flight at any time, unless we are 150 * using the GuC submission. In that case each request requires a minimum 151 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 152 * enough space to avoid backpressure on the driver. We increase the size 153 * of the receive buffer (relative to the send) to ensure a G2H response 154 * CTB has a landing spot. 155 * 156 * In addition to submissions, the G2H buffer needs to be able to hold 157 * enough space for recoverable page fault notifications. The number of 158 * page faults is interrupt driven and can be as much as the number of 159 * compute resources available. However, most of the actual work for these 160 * is in a separate page fault worker thread. Therefore we only need to 161 * make sure the queue has enough space to handle all of the submissions 162 * and responses and an extra buffer for incoming page faults. 163 */ 164 165 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 166 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 167 #define CTB_G2H_BUFFER_SIZE (SZ_128K) 168 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2) 169 170 /** 171 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full 172 * CT command queue 173 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future. 174 * 175 * Observation is that a 4KiB buffer full of commands takes a little over a 176 * second to process. Use that to calculate maximum time to process a full CT 177 * command queue. 178 * 179 * Return: Maximum time to process a full CT queue in jiffies. 180 */ 181 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct) 182 { 183 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4)); 184 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ; 185 } 186 187 static size_t guc_ct_size(void) 188 { 189 return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + 190 CTB_G2H_BUFFER_SIZE; 191 } 192 193 static void guc_ct_fini(struct drm_device *drm, void *arg) 194 { 195 struct xe_guc_ct *ct = arg; 196 197 ct_exit_safe_mode(ct); 198 destroy_workqueue(ct->g2h_wq); 199 xa_destroy(&ct->fence_lookup); 200 } 201 202 static void primelockdep(struct xe_guc_ct *ct) 203 { 204 if (!IS_ENABLED(CONFIG_LOCKDEP)) 205 return; 206 207 fs_reclaim_acquire(GFP_KERNEL); 208 might_lock(&ct->lock); 209 fs_reclaim_release(GFP_KERNEL); 210 } 211 212 int xe_guc_ct_init(struct xe_guc_ct *ct) 213 { 214 struct xe_device *xe = ct_to_xe(ct); 215 struct xe_gt *gt = ct_to_gt(ct); 216 struct xe_tile *tile = gt_to_tile(gt); 217 struct xe_bo *bo; 218 int err; 219 220 xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE)); 221 222 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM); 223 if (!ct->g2h_wq) 224 return -ENOMEM; 225 226 spin_lock_init(&ct->fast_lock); 227 xa_init(&ct->fence_lookup); 228 INIT_WORK(&ct->g2h_worker, g2h_worker_func); 229 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func); 230 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 231 spin_lock_init(&ct->dead.lock); 232 INIT_WORK(&ct->dead.worker, ct_dead_worker_func); 233 #endif 234 init_waitqueue_head(&ct->wq); 235 init_waitqueue_head(&ct->g2h_fence_wq); 236 237 err = drmm_mutex_init(&xe->drm, &ct->lock); 238 if (err) 239 return err; 240 241 primelockdep(ct); 242 243 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(), 244 XE_BO_FLAG_SYSTEM | 245 XE_BO_FLAG_GGTT | 246 XE_BO_FLAG_GGTT_INVALIDATE | 247 XE_BO_FLAG_PINNED_NORESTORE); 248 if (IS_ERR(bo)) 249 return PTR_ERR(bo); 250 251 ct->bo = bo; 252 253 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); 254 if (err) 255 return err; 256 257 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED); 258 ct->state = XE_GUC_CT_STATE_DISABLED; 259 return 0; 260 } 261 ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */ 262 263 #define desc_read(xe_, guc_ctb__, field_) \ 264 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \ 265 struct guc_ct_buffer_desc, field_) 266 267 #define desc_write(xe_, guc_ctb__, field_, val_) \ 268 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \ 269 struct guc_ct_buffer_desc, field_, val_) 270 271 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, 272 struct iosys_map *map) 273 { 274 h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32); 275 h2g->info.resv_space = 0; 276 h2g->info.tail = 0; 277 h2g->info.head = 0; 278 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 279 h2g->info.size) - 280 h2g->info.resv_space; 281 h2g->info.broken = false; 282 283 h2g->desc = *map; 284 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 285 286 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2); 287 } 288 289 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, 290 struct iosys_map *map) 291 { 292 g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32); 293 g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32); 294 g2h->info.head = 0; 295 g2h->info.tail = 0; 296 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head, 297 g2h->info.size) - 298 g2h->info.resv_space; 299 g2h->info.broken = false; 300 301 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE); 302 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 303 304 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 + 305 CTB_H2G_BUFFER_SIZE); 306 } 307 308 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct) 309 { 310 struct xe_guc *guc = ct_to_guc(ct); 311 u32 desc_addr, ctb_addr, size; 312 int err; 313 314 desc_addr = xe_bo_ggtt_addr(ct->bo); 315 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2; 316 size = ct->ctbs.h2g.info.size * sizeof(u32); 317 318 err = xe_guc_self_cfg64(guc, 319 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY, 320 desc_addr); 321 if (err) 322 return err; 323 324 err = xe_guc_self_cfg64(guc, 325 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY, 326 ctb_addr); 327 if (err) 328 return err; 329 330 return xe_guc_self_cfg32(guc, 331 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY, 332 size); 333 } 334 335 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct) 336 { 337 struct xe_guc *guc = ct_to_guc(ct); 338 u32 desc_addr, ctb_addr, size; 339 int err; 340 341 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE; 342 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 + 343 CTB_H2G_BUFFER_SIZE; 344 size = ct->ctbs.g2h.info.size * sizeof(u32); 345 346 err = xe_guc_self_cfg64(guc, 347 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 348 desc_addr); 349 if (err) 350 return err; 351 352 err = xe_guc_self_cfg64(guc, 353 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 354 ctb_addr); 355 if (err) 356 return err; 357 358 return xe_guc_self_cfg32(guc, 359 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 360 size); 361 } 362 363 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) 364 { 365 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 366 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 367 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 368 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, 369 GUC_ACTION_HOST2GUC_CONTROL_CTB), 370 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, 371 enable ? GUC_CTB_CONTROL_ENABLE : 372 GUC_CTB_CONTROL_DISABLE), 373 }; 374 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request)); 375 376 return ret > 0 ? -EPROTO : ret; 377 } 378 379 static void xe_guc_ct_set_state(struct xe_guc_ct *ct, 380 enum xe_guc_ct_state state) 381 { 382 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ 383 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ 384 385 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 || 386 state == XE_GUC_CT_STATE_STOPPED); 387 388 if (ct->g2h_outstanding) 389 xe_pm_runtime_put(ct_to_xe(ct)); 390 ct->g2h_outstanding = 0; 391 ct->state = state; 392 393 spin_unlock_irq(&ct->fast_lock); 394 395 /* 396 * Lockdep doesn't like this under the fast lock and he destroy only 397 * needs to be serialized with the send path which ct lock provides. 398 */ 399 xa_destroy(&ct->fence_lookup); 400 401 mutex_unlock(&ct->lock); 402 } 403 404 static bool ct_needs_safe_mode(struct xe_guc_ct *ct) 405 { 406 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev)); 407 } 408 409 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct) 410 { 411 if (!ct_needs_safe_mode(ct)) 412 return false; 413 414 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10); 415 return true; 416 } 417 418 static void safe_mode_worker_func(struct work_struct *w) 419 { 420 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work); 421 422 receive_g2h(ct); 423 424 if (!ct_restart_safe_mode_worker(ct)) 425 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n"); 426 } 427 428 static void ct_enter_safe_mode(struct xe_guc_ct *ct) 429 { 430 if (ct_restart_safe_mode_worker(ct)) 431 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n"); 432 } 433 434 static void ct_exit_safe_mode(struct xe_guc_ct *ct) 435 { 436 if (cancel_delayed_work_sync(&ct->safe_mode_worker)) 437 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n"); 438 } 439 440 int xe_guc_ct_enable(struct xe_guc_ct *ct) 441 { 442 struct xe_device *xe = ct_to_xe(ct); 443 struct xe_gt *gt = ct_to_gt(ct); 444 int err; 445 446 xe_gt_assert(gt, !xe_guc_ct_enabled(ct)); 447 448 xe_map_memset(xe, &ct->bo->vmap, 0, 0, ct->bo->size); 449 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap); 450 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap); 451 452 err = guc_ct_ctb_h2g_register(ct); 453 if (err) 454 goto err_out; 455 456 err = guc_ct_ctb_g2h_register(ct); 457 if (err) 458 goto err_out; 459 460 err = guc_ct_control_toggle(ct, true); 461 if (err) 462 goto err_out; 463 464 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED); 465 466 smp_mb(); 467 wake_up_all(&ct->wq); 468 xe_gt_dbg(gt, "GuC CT communication channel enabled\n"); 469 470 if (ct_needs_safe_mode(ct)) 471 ct_enter_safe_mode(ct); 472 473 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 474 /* 475 * The CT has now been reset so the dumper can be re-armed 476 * after any existing dead state has been dumped. 477 */ 478 spin_lock_irq(&ct->dead.lock); 479 if (ct->dead.reason) { 480 ct->dead.reason |= (1 << CT_DEAD_STATE_REARM); 481 queue_work(system_unbound_wq, &ct->dead.worker); 482 } 483 spin_unlock_irq(&ct->dead.lock); 484 #endif 485 486 return 0; 487 488 err_out: 489 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err)); 490 CT_DEAD(ct, NULL, SETUP); 491 492 return err; 493 } 494 495 static void stop_g2h_handler(struct xe_guc_ct *ct) 496 { 497 cancel_work_sync(&ct->g2h_worker); 498 } 499 500 /** 501 * xe_guc_ct_disable - Set GuC to disabled state 502 * @ct: the &xe_guc_ct 503 * 504 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected 505 * in this transition. 506 */ 507 void xe_guc_ct_disable(struct xe_guc_ct *ct) 508 { 509 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED); 510 ct_exit_safe_mode(ct); 511 stop_g2h_handler(ct); 512 } 513 514 /** 515 * xe_guc_ct_stop - Set GuC to stopped state 516 * @ct: the &xe_guc_ct 517 * 518 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h 519 */ 520 void xe_guc_ct_stop(struct xe_guc_ct *ct) 521 { 522 if (!xe_guc_ct_initialized(ct)) 523 return; 524 525 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED); 526 stop_g2h_handler(ct); 527 } 528 529 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) 530 { 531 struct guc_ctb *h2g = &ct->ctbs.h2g; 532 533 lockdep_assert_held(&ct->lock); 534 535 if (cmd_len > h2g->info.space) { 536 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); 537 538 if (h2g->info.head > h2g->info.size) { 539 struct xe_device *xe = ct_to_xe(ct); 540 u32 desc_status = desc_read(xe, h2g, status); 541 542 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 543 544 xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n", 545 h2g->info.head, h2g->info.size); 546 CT_DEAD(ct, h2g, H2G_HAS_ROOM); 547 return false; 548 } 549 550 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 551 h2g->info.size) - 552 h2g->info.resv_space; 553 if (cmd_len > h2g->info.space) 554 return false; 555 } 556 557 return true; 558 } 559 560 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len) 561 { 562 if (!g2h_len) 563 return true; 564 565 lockdep_assert_held(&ct->fast_lock); 566 567 return ct->ctbs.g2h.info.space > g2h_len; 568 } 569 570 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len) 571 { 572 lockdep_assert_held(&ct->lock); 573 574 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len)) 575 return -EBUSY; 576 577 return 0; 578 } 579 580 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len) 581 { 582 lockdep_assert_held(&ct->lock); 583 ct->ctbs.h2g.info.space -= cmd_len; 584 } 585 586 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h) 587 { 588 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space); 589 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) || 590 (g2h_len && num_g2h)); 591 592 if (g2h_len) { 593 lockdep_assert_held(&ct->fast_lock); 594 595 if (!ct->g2h_outstanding) 596 xe_pm_runtime_get_noresume(ct_to_xe(ct)); 597 598 ct->ctbs.g2h.info.space -= g2h_len; 599 ct->g2h_outstanding += num_g2h; 600 } 601 } 602 603 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 604 { 605 bool bad = false; 606 607 lockdep_assert_held(&ct->fast_lock); 608 609 bad = ct->ctbs.g2h.info.space + g2h_len > 610 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space; 611 bad |= !ct->g2h_outstanding; 612 613 if (bad) { 614 xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n", 615 ct->ctbs.g2h.info.space, g2h_len, 616 ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space, 617 ct->ctbs.g2h.info.space + g2h_len, 618 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space, 619 ct->g2h_outstanding); 620 CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE); 621 return; 622 } 623 624 ct->ctbs.g2h.info.space += g2h_len; 625 if (!--ct->g2h_outstanding) 626 xe_pm_runtime_put(ct_to_xe(ct)); 627 } 628 629 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 630 { 631 spin_lock_irq(&ct->fast_lock); 632 __g2h_release_space(ct, g2h_len); 633 spin_unlock_irq(&ct->fast_lock); 634 } 635 636 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 637 static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) 638 { 639 unsigned int slot = fence % ARRAY_SIZE(ct->fast_req); 640 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 641 unsigned long entries[SZ_32]; 642 unsigned int n; 643 644 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 645 646 /* May be called under spinlock, so avoid sleeping */ 647 ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT); 648 #endif 649 ct->fast_req[slot].fence = fence; 650 ct->fast_req[slot].action = action; 651 } 652 #else 653 static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) 654 { 655 } 656 #endif 657 658 /* 659 * The CT protocol accepts a 16 bits fence. This field is fully owned by the 660 * driver, the GuC will just copy it to the reply message. Since we need to 661 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages, 662 * we use one bit of the seqno as an indicator for that and a rolling counter 663 * for the remaining 15 bits. 664 */ 665 #define CT_SEQNO_MASK GENMASK(14, 0) 666 #define CT_SEQNO_UNTRACKED BIT(15) 667 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence) 668 { 669 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK; 670 671 if (!is_g2h_fence) 672 seqno |= CT_SEQNO_UNTRACKED; 673 674 return seqno; 675 } 676 677 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */ 678 679 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, 680 u32 ct_fence_value, bool want_response) 681 { 682 struct xe_device *xe = ct_to_xe(ct); 683 struct xe_gt *gt = ct_to_gt(ct); 684 struct guc_ctb *h2g = &ct->ctbs.h2g; 685 u32 cmd[H2G_CT_HEADERS]; 686 u32 tail = h2g->info.tail; 687 u32 full_len; 688 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds, 689 tail * sizeof(u32)); 690 u32 desc_status; 691 692 full_len = len + GUC_CTB_HDR_LEN; 693 694 lockdep_assert_held(&ct->lock); 695 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN); 696 697 desc_status = desc_read(xe, h2g, status); 698 if (desc_status) { 699 xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status); 700 goto corrupted; 701 } 702 703 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 704 u32 desc_tail = desc_read(xe, h2g, tail); 705 u32 desc_head = desc_read(xe, h2g, head); 706 707 if (tail != desc_tail) { 708 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH); 709 xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail); 710 goto corrupted; 711 } 712 713 if (tail > h2g->info.size) { 714 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 715 xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n", 716 tail, h2g->info.size); 717 goto corrupted; 718 } 719 720 if (desc_head >= h2g->info.size) { 721 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 722 xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n", 723 desc_head, h2g->info.size); 724 goto corrupted; 725 } 726 } 727 728 /* Command will wrap, zero fill (NOPs), return and check credits again */ 729 if (tail + full_len > h2g->info.size) { 730 xe_map_memset(xe, &map, 0, 0, 731 (h2g->info.size - tail) * sizeof(u32)); 732 h2g_reserve_space(ct, (h2g->info.size - tail)); 733 h2g->info.tail = 0; 734 desc_write(xe, h2g, tail, h2g->info.tail); 735 736 return -EAGAIN; 737 } 738 739 /* 740 * dw0: CT header (including fence) 741 * dw1: HXG header (including action code) 742 * dw2+: action data 743 */ 744 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 745 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 746 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value); 747 if (want_response) { 748 cmd[1] = 749 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 750 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 751 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 752 } else { 753 fast_req_track(ct, ct_fence_value, 754 FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0])); 755 756 cmd[1] = 757 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) | 758 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 759 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 760 } 761 762 /* H2G header in cmd[1] replaces action[0] so: */ 763 --len; 764 ++action; 765 766 /* Write H2G ensuring visible before descriptor update */ 767 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32)); 768 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32)); 769 xe_device_wmb(xe); 770 771 /* Update local copies */ 772 h2g->info.tail = (tail + full_len) % h2g->info.size; 773 h2g_reserve_space(ct, full_len); 774 775 /* Update descriptor */ 776 desc_write(xe, h2g, tail, h2g->info.tail); 777 778 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len, 779 desc_read(xe, h2g, head), h2g->info.tail); 780 781 return 0; 782 783 corrupted: 784 CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE); 785 return -EPIPE; 786 } 787 788 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, 789 u32 len, u32 g2h_len, u32 num_g2h, 790 struct g2h_fence *g2h_fence) 791 { 792 struct xe_gt *gt __maybe_unused = ct_to_gt(ct); 793 u16 seqno; 794 int ret; 795 796 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 797 xe_gt_assert(gt, !g2h_len || !g2h_fence); 798 xe_gt_assert(gt, !num_g2h || !g2h_fence); 799 xe_gt_assert(gt, !g2h_len || num_g2h); 800 xe_gt_assert(gt, g2h_len || !num_g2h); 801 lockdep_assert_held(&ct->lock); 802 803 if (unlikely(ct->ctbs.h2g.info.broken)) { 804 ret = -EPIPE; 805 goto out; 806 } 807 808 if (ct->state == XE_GUC_CT_STATE_DISABLED) { 809 ret = -ENODEV; 810 goto out; 811 } 812 813 if (ct->state == XE_GUC_CT_STATE_STOPPED) { 814 ret = -ECANCELED; 815 goto out; 816 } 817 818 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 819 820 if (g2h_fence) { 821 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; 822 num_g2h = 1; 823 824 if (g2h_fence_needs_alloc(g2h_fence)) { 825 g2h_fence->seqno = next_ct_seqno(ct, true); 826 ret = xa_err(xa_store(&ct->fence_lookup, 827 g2h_fence->seqno, g2h_fence, 828 GFP_ATOMIC)); 829 if (ret) 830 goto out; 831 } 832 833 seqno = g2h_fence->seqno; 834 } else { 835 seqno = next_ct_seqno(ct, false); 836 } 837 838 if (g2h_len) 839 spin_lock_irq(&ct->fast_lock); 840 retry: 841 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len); 842 if (unlikely(ret)) 843 goto out_unlock; 844 845 ret = h2g_write(ct, action, len, seqno, !!g2h_fence); 846 if (unlikely(ret)) { 847 if (ret == -EAGAIN) 848 goto retry; 849 goto out_unlock; 850 } 851 852 __g2h_reserve_space(ct, g2h_len, num_g2h); 853 xe_guc_notify(ct_to_guc(ct)); 854 out_unlock: 855 if (g2h_len) 856 spin_unlock_irq(&ct->fast_lock); 857 out: 858 return ret; 859 } 860 861 static void kick_reset(struct xe_guc_ct *ct) 862 { 863 xe_gt_reset_async(ct_to_gt(ct)); 864 } 865 866 static int dequeue_one_g2h(struct xe_guc_ct *ct); 867 868 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 869 u32 g2h_len, u32 num_g2h, 870 struct g2h_fence *g2h_fence) 871 { 872 struct xe_device *xe = ct_to_xe(ct); 873 struct xe_gt *gt = ct_to_gt(ct); 874 unsigned int sleep_period_ms = 1; 875 int ret; 876 877 xe_gt_assert(gt, !g2h_len || !g2h_fence); 878 lockdep_assert_held(&ct->lock); 879 xe_device_assert_mem_access(ct_to_xe(ct)); 880 881 try_again: 882 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, 883 g2h_fence); 884 885 /* 886 * We wait to try to restore credits for about 1 second before bailing. 887 * In the case of H2G credits we have no choice but just to wait for the 888 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In 889 * the case of G2H we process any G2H in the channel, hopefully freeing 890 * credits as we consume the G2H messages. 891 */ 892 if (unlikely(ret == -EBUSY && 893 !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) { 894 struct guc_ctb *h2g = &ct->ctbs.h2g; 895 896 if (sleep_period_ms == 1024) 897 goto broken; 898 899 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail, 900 h2g->info.size, 901 h2g->info.space, 902 len + GUC_CTB_HDR_LEN); 903 msleep(sleep_period_ms); 904 sleep_period_ms <<= 1; 905 906 goto try_again; 907 } else if (unlikely(ret == -EBUSY)) { 908 struct xe_device *xe = ct_to_xe(ct); 909 struct guc_ctb *g2h = &ct->ctbs.g2h; 910 911 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head, 912 desc_read(xe, g2h, tail), 913 g2h->info.size, 914 g2h->info.space, 915 g2h_fence ? 916 GUC_CTB_HXG_MSG_MAX_LEN : 917 g2h_len); 918 919 #define g2h_avail(ct) \ 920 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head) 921 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding || 922 g2h_avail(ct), HZ)) 923 goto broken; 924 #undef g2h_avail 925 926 ret = dequeue_one_g2h(ct); 927 if (ret < 0) { 928 if (ret != -ECANCELED) 929 xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)", 930 ERR_PTR(ret)); 931 goto broken; 932 } 933 934 goto try_again; 935 } 936 937 return ret; 938 939 broken: 940 xe_gt_err(gt, "No forward process on H2G, reset required\n"); 941 CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK); 942 943 return -EDEADLK; 944 } 945 946 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 947 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) 948 { 949 int ret; 950 951 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence); 952 953 mutex_lock(&ct->lock); 954 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence); 955 mutex_unlock(&ct->lock); 956 957 return ret; 958 } 959 960 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 961 u32 g2h_len, u32 num_g2h) 962 { 963 int ret; 964 965 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL); 966 if (ret == -EDEADLK) 967 kick_reset(ct); 968 969 return ret; 970 } 971 972 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 973 u32 g2h_len, u32 num_g2h) 974 { 975 int ret; 976 977 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL); 978 if (ret == -EDEADLK) 979 kick_reset(ct); 980 981 return ret; 982 } 983 984 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len) 985 { 986 int ret; 987 988 lockdep_assert_held(&ct->lock); 989 990 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL); 991 if (ret == -EDEADLK) 992 kick_reset(ct); 993 994 return ret; 995 } 996 997 /* 998 * Check if a GT reset is in progress or will occur and if GT reset brought the 999 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset. 1000 */ 1001 static bool retry_failure(struct xe_guc_ct *ct, int ret) 1002 { 1003 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV)) 1004 return false; 1005 1006 #define ct_alive(ct) \ 1007 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \ 1008 !ct->ctbs.g2h.info.broken) 1009 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) 1010 return false; 1011 #undef ct_alive 1012 1013 return true; 1014 } 1015 1016 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1017 u32 *response_buffer, bool no_fail) 1018 { 1019 struct xe_gt *gt = ct_to_gt(ct); 1020 struct g2h_fence g2h_fence; 1021 int ret = 0; 1022 1023 /* 1024 * We use a fence to implement blocking sends / receiving response data. 1025 * The seqno of the fence is sent in the H2G, returned in the G2H, and 1026 * an xarray is used as storage media with the seqno being to key. 1027 * Fields in the fence hold success, failure, retry status and the 1028 * response data. Safe to allocate on the stack as the xarray is the 1029 * only reference and it cannot be present after this function exits. 1030 */ 1031 retry: 1032 g2h_fence_init(&g2h_fence, response_buffer); 1033 retry_same_fence: 1034 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence); 1035 if (unlikely(ret == -ENOMEM)) { 1036 /* Retry allocation /w GFP_KERNEL */ 1037 ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno, 1038 &g2h_fence, GFP_KERNEL)); 1039 if (ret) 1040 return ret; 1041 1042 goto retry_same_fence; 1043 } else if (unlikely(ret)) { 1044 if (ret == -EDEADLK) 1045 kick_reset(ct); 1046 1047 if (no_fail && retry_failure(ct, ret)) 1048 goto retry_same_fence; 1049 1050 if (!g2h_fence_needs_alloc(&g2h_fence)) 1051 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1052 1053 return ret; 1054 } 1055 1056 ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ); 1057 if (!ret) { 1058 LNL_FLUSH_WORK(&ct->g2h_worker); 1059 if (g2h_fence.done) { 1060 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", 1061 g2h_fence.seqno, action[0]); 1062 ret = 1; 1063 } 1064 } 1065 1066 /* 1067 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on 1068 * the stack, since we have no clue if it will fire after the timeout before we can erase 1069 * from the xa. Also we have some dependent loads and stores below for which we need the 1070 * correct ordering, and we lack the needed barriers. 1071 */ 1072 mutex_lock(&ct->lock); 1073 if (!ret) { 1074 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s", 1075 g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done)); 1076 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1077 mutex_unlock(&ct->lock); 1078 return -ETIME; 1079 } 1080 1081 if (g2h_fence.retry) { 1082 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", 1083 action[0], g2h_fence.reason); 1084 mutex_unlock(&ct->lock); 1085 goto retry; 1086 } 1087 if (g2h_fence.fail) { 1088 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n", 1089 action[0], g2h_fence.error, g2h_fence.hint); 1090 ret = -EIO; 1091 } 1092 1093 if (ret > 0) 1094 ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data; 1095 1096 mutex_unlock(&ct->lock); 1097 1098 return ret; 1099 } 1100 1101 /** 1102 * xe_guc_ct_send_recv - Send and receive HXG to the GuC 1103 * @ct: the &xe_guc_ct 1104 * @action: the dword array with `HXG Request`_ message (can't be NULL) 1105 * @len: length of the `HXG Request`_ message (in dwords, can't be 0) 1106 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL) 1107 * 1108 * Send a `HXG Request`_ message to the GuC over CT communication channel and 1109 * blocks until GuC replies with a `HXG Response`_ message. 1110 * 1111 * For non-blocking communication with GuC use xe_guc_ct_send(). 1112 * 1113 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_. 1114 * 1115 * Return: response length (in dwords) if &response_buffer was not NULL, or 1116 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or 1117 * a negative error code on failure. 1118 */ 1119 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1120 u32 *response_buffer) 1121 { 1122 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer); 1123 return guc_ct_send_recv(ct, action, len, response_buffer, false); 1124 } 1125 ALLOW_ERROR_INJECTION(xe_guc_ct_send_recv, ERRNO); 1126 1127 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, 1128 u32 len, u32 *response_buffer) 1129 { 1130 return guc_ct_send_recv(ct, action, len, response_buffer, true); 1131 } 1132 1133 static u32 *msg_to_hxg(u32 *msg) 1134 { 1135 return msg + GUC_CTB_MSG_MIN_LEN; 1136 } 1137 1138 static u32 msg_len_to_hxg_len(u32 len) 1139 { 1140 return len - GUC_CTB_MSG_MIN_LEN; 1141 } 1142 1143 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) 1144 { 1145 u32 *hxg = msg_to_hxg(msg); 1146 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1147 1148 lockdep_assert_held(&ct->lock); 1149 1150 switch (action) { 1151 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1152 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1153 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1154 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1155 g2h_release_space(ct, len); 1156 } 1157 1158 return 0; 1159 } 1160 1161 static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) 1162 { 1163 struct xe_gt *gt = ct_to_gt(ct); 1164 1165 if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED) 1166 xe_gt_err(gt, "GuC Crash dump notification\n"); 1167 else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION) 1168 xe_gt_err(gt, "GuC Exception notification\n"); 1169 else 1170 xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action); 1171 1172 CT_DEAD(ct, NULL, CRASH); 1173 1174 kick_reset(ct); 1175 1176 return 0; 1177 } 1178 1179 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 1180 static void fast_req_report(struct xe_guc_ct *ct, u16 fence) 1181 { 1182 u16 fence_min = U16_MAX, fence_max = 0; 1183 struct xe_gt *gt = ct_to_gt(ct); 1184 bool found = false; 1185 unsigned int n; 1186 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 1187 char *buf; 1188 #endif 1189 1190 lockdep_assert_held(&ct->lock); 1191 1192 for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) { 1193 if (ct->fast_req[n].fence < fence_min) 1194 fence_min = ct->fast_req[n].fence; 1195 if (ct->fast_req[n].fence > fence_max) 1196 fence_max = ct->fast_req[n].fence; 1197 1198 if (ct->fast_req[n].fence != fence) 1199 continue; 1200 found = true; 1201 1202 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 1203 buf = kmalloc(SZ_4K, GFP_NOWAIT); 1204 if (buf && stack_depot_snprint(ct->fast_req[n].stack, buf, SZ_4K, 0)) 1205 xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s", 1206 fence, ct->fast_req[n].action, buf); 1207 else 1208 xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n", 1209 fence, ct->fast_req[n].action); 1210 kfree(buf); 1211 #else 1212 xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n", 1213 fence, ct->fast_req[n].action); 1214 #endif 1215 break; 1216 } 1217 1218 if (!found) 1219 xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n", 1220 fence, fence_min, fence_max, ct->fence_seqno); 1221 } 1222 #else 1223 static void fast_req_report(struct xe_guc_ct *ct, u16 fence) 1224 { 1225 } 1226 #endif 1227 1228 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) 1229 { 1230 struct xe_gt *gt = ct_to_gt(ct); 1231 u32 *hxg = msg_to_hxg(msg); 1232 u32 hxg_len = msg_len_to_hxg_len(len); 1233 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); 1234 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1235 struct g2h_fence *g2h_fence; 1236 1237 lockdep_assert_held(&ct->lock); 1238 1239 /* 1240 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup. 1241 * Those messages should never fail, so if we do get an error back it 1242 * means we're likely doing an illegal operation and the GuC is 1243 * rejecting it. We have no way to inform the code that submitted the 1244 * H2G that the message was rejected, so we need to escalate the 1245 * failure to trigger a reset. 1246 */ 1247 if (fence & CT_SEQNO_UNTRACKED) { 1248 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) 1249 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n", 1250 fence, 1251 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]), 1252 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0])); 1253 else 1254 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n", 1255 type, fence); 1256 1257 fast_req_report(ct, fence); 1258 1259 CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE); 1260 1261 return -EPROTO; 1262 } 1263 1264 g2h_fence = xa_erase(&ct->fence_lookup, fence); 1265 if (unlikely(!g2h_fence)) { 1266 /* Don't tear down channel, as send could've timed out */ 1267 /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ 1268 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence); 1269 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1270 return 0; 1271 } 1272 1273 xe_gt_assert(gt, fence == g2h_fence->seqno); 1274 1275 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { 1276 g2h_fence->fail = true; 1277 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); 1278 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]); 1279 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 1280 g2h_fence->retry = true; 1281 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); 1282 } else if (g2h_fence->response_buffer) { 1283 g2h_fence->response_len = hxg_len; 1284 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); 1285 } else { 1286 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); 1287 } 1288 1289 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1290 1291 g2h_fence->done = true; 1292 smp_mb(); 1293 1294 wake_up_all(&ct->g2h_fence_wq); 1295 1296 return 0; 1297 } 1298 1299 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1300 { 1301 struct xe_gt *gt = ct_to_gt(ct); 1302 u32 *hxg = msg_to_hxg(msg); 1303 u32 origin, type; 1304 int ret; 1305 1306 lockdep_assert_held(&ct->lock); 1307 1308 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1309 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1310 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n", 1311 origin); 1312 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN); 1313 1314 return -EPROTO; 1315 } 1316 1317 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1318 switch (type) { 1319 case GUC_HXG_TYPE_EVENT: 1320 ret = parse_g2h_event(ct, msg, len); 1321 break; 1322 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1323 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1324 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1325 ret = parse_g2h_response(ct, msg, len); 1326 break; 1327 default: 1328 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n", 1329 type); 1330 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE); 1331 1332 ret = -EOPNOTSUPP; 1333 } 1334 1335 return ret; 1336 } 1337 1338 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1339 { 1340 struct xe_guc *guc = ct_to_guc(ct); 1341 struct xe_gt *gt = ct_to_gt(ct); 1342 u32 hxg_len = msg_len_to_hxg_len(len); 1343 u32 *hxg = msg_to_hxg(msg); 1344 u32 action, adj_len; 1345 u32 *payload; 1346 int ret = 0; 1347 1348 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1349 return 0; 1350 1351 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1352 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN; 1353 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN; 1354 1355 switch (action) { 1356 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1357 ret = xe_guc_sched_done_handler(guc, payload, adj_len); 1358 break; 1359 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1360 ret = xe_guc_deregister_done_handler(guc, payload, adj_len); 1361 break; 1362 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1363 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len); 1364 break; 1365 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1366 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload, 1367 adj_len); 1368 break; 1369 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1370 /* Selftest only at the moment */ 1371 break; 1372 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1373 ret = xe_guc_error_capture_handler(guc, payload, adj_len); 1374 break; 1375 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1376 /* FIXME: Handle this */ 1377 break; 1378 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: 1379 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload, 1380 adj_len); 1381 break; 1382 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1383 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1384 break; 1385 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1386 ret = xe_guc_tlb_invalidation_done_handler(guc, payload, 1387 adj_len); 1388 break; 1389 case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY: 1390 ret = xe_guc_access_counter_notify_handler(guc, payload, 1391 adj_len); 1392 break; 1393 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: 1394 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len); 1395 break; 1396 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF: 1397 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len); 1398 break; 1399 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY: 1400 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len); 1401 break; 1402 case GUC_ACTION_GUC2PF_ADVERSE_EVENT: 1403 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len); 1404 break; 1405 case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1406 case XE_GUC_ACTION_NOTIFY_EXCEPTION: 1407 ret = guc_crash_process_msg(ct, action); 1408 break; 1409 default: 1410 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); 1411 } 1412 1413 if (ret) { 1414 xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n", 1415 action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg); 1416 CT_DEAD(ct, NULL, PROCESS_FAILED); 1417 } 1418 1419 return 0; 1420 } 1421 1422 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) 1423 { 1424 struct xe_device *xe = ct_to_xe(ct); 1425 struct xe_gt *gt = ct_to_gt(ct); 1426 struct guc_ctb *g2h = &ct->ctbs.g2h; 1427 u32 tail, head, len, desc_status; 1428 s32 avail; 1429 u32 action; 1430 u32 *hxg; 1431 1432 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 1433 lockdep_assert_held(&ct->fast_lock); 1434 1435 if (ct->state == XE_GUC_CT_STATE_DISABLED) 1436 return -ENODEV; 1437 1438 if (ct->state == XE_GUC_CT_STATE_STOPPED) 1439 return -ECANCELED; 1440 1441 if (g2h->info.broken) 1442 return -EPIPE; 1443 1444 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1445 1446 desc_status = desc_read(xe, g2h, status); 1447 if (desc_status) { 1448 if (desc_status & GUC_CTB_STATUS_DISABLED) { 1449 /* 1450 * Potentially valid if a CLIENT_RESET request resulted in 1451 * contexts/engines being reset. But should never happen as 1452 * no contexts should be active when CLIENT_RESET is sent. 1453 */ 1454 xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n"); 1455 desc_status &= ~GUC_CTB_STATUS_DISABLED; 1456 } 1457 1458 if (desc_status) { 1459 xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status); 1460 goto corrupted; 1461 } 1462 } 1463 1464 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 1465 u32 desc_tail = desc_read(xe, g2h, tail); 1466 /* 1467 u32 desc_head = desc_read(xe, g2h, head); 1468 1469 * info.head and desc_head are updated back-to-back at the end of 1470 * this function and nowhere else. Hence, they cannot be different 1471 * unless two g2h_read calls are running concurrently. Which is not 1472 * possible because it is guarded by ct->fast_lock. And yet, some 1473 * discrete platforms are regularly hitting this error :(. 1474 * 1475 * desc_head rolling backwards shouldn't cause any noticeable 1476 * problems - just a delay in GuC being allowed to proceed past that 1477 * point in the queue. So for now, just disable the error until it 1478 * can be root caused. 1479 * 1480 if (g2h->info.head != desc_head) { 1481 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH); 1482 xe_gt_err(gt, "CT read: head was modified %u != %u\n", 1483 desc_head, g2h->info.head); 1484 goto corrupted; 1485 } 1486 */ 1487 1488 if (g2h->info.head > g2h->info.size) { 1489 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1490 xe_gt_err(gt, "CT read: head out of range: %u vs %u\n", 1491 g2h->info.head, g2h->info.size); 1492 goto corrupted; 1493 } 1494 1495 if (desc_tail >= g2h->info.size) { 1496 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1497 xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n", 1498 desc_tail, g2h->info.size); 1499 goto corrupted; 1500 } 1501 } 1502 1503 /* Calculate DW available to read */ 1504 tail = desc_read(xe, g2h, tail); 1505 avail = tail - g2h->info.head; 1506 if (unlikely(avail == 0)) 1507 return 0; 1508 1509 if (avail < 0) 1510 avail += g2h->info.size; 1511 1512 /* Read header */ 1513 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head, 1514 sizeof(u32)); 1515 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; 1516 if (len > avail) { 1517 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n", 1518 avail, len); 1519 goto corrupted; 1520 } 1521 1522 head = (g2h->info.head + 1) % g2h->info.size; 1523 avail = len - 1; 1524 1525 /* Read G2H message */ 1526 if (avail + head > g2h->info.size) { 1527 u32 avail_til_wrap = g2h->info.size - head; 1528 1529 xe_map_memcpy_from(xe, msg + 1, 1530 &g2h->cmds, sizeof(u32) * head, 1531 avail_til_wrap * sizeof(u32)); 1532 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap, 1533 &g2h->cmds, 0, 1534 (avail - avail_til_wrap) * sizeof(u32)); 1535 } else { 1536 xe_map_memcpy_from(xe, msg + 1, 1537 &g2h->cmds, sizeof(u32) * head, 1538 avail * sizeof(u32)); 1539 } 1540 1541 hxg = msg_to_hxg(msg); 1542 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1543 1544 if (fast_path) { 1545 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1546 return 0; 1547 1548 switch (action) { 1549 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1550 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1551 break; /* Process these in fast-path */ 1552 default: 1553 return 0; 1554 } 1555 } 1556 1557 /* Update local / descriptor header */ 1558 g2h->info.head = (head + avail) % g2h->info.size; 1559 desc_write(xe, g2h, head, g2h->info.head); 1560 1561 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id, 1562 action, len, g2h->info.head, tail); 1563 1564 return len; 1565 1566 corrupted: 1567 CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ); 1568 return -EPROTO; 1569 } 1570 1571 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) 1572 { 1573 struct xe_gt *gt = ct_to_gt(ct); 1574 struct xe_guc *guc = ct_to_guc(ct); 1575 u32 hxg_len = msg_len_to_hxg_len(len); 1576 u32 *hxg = msg_to_hxg(msg); 1577 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1578 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN; 1579 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1580 int ret = 0; 1581 1582 switch (action) { 1583 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1584 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1585 break; 1586 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1587 __g2h_release_space(ct, len); 1588 ret = xe_guc_tlb_invalidation_done_handler(guc, payload, 1589 adj_len); 1590 break; 1591 default: 1592 xe_gt_warn(gt, "NOT_POSSIBLE"); 1593 } 1594 1595 if (ret) { 1596 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n", 1597 action, ERR_PTR(ret)); 1598 CT_DEAD(ct, NULL, FAST_G2H); 1599 } 1600 } 1601 1602 /** 1603 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler 1604 * @ct: GuC CT object 1605 * 1606 * Anything related to page faults is critical for performance, process these 1607 * critical G2H in the IRQ. This is safe as these handlers either just wake up 1608 * waiters or queue another worker. 1609 */ 1610 void xe_guc_ct_fast_path(struct xe_guc_ct *ct) 1611 { 1612 struct xe_device *xe = ct_to_xe(ct); 1613 bool ongoing; 1614 int len; 1615 1616 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1617 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1618 return; 1619 1620 spin_lock(&ct->fast_lock); 1621 do { 1622 len = g2h_read(ct, ct->fast_msg, true); 1623 if (len > 0) 1624 g2h_fast_path(ct, ct->fast_msg, len); 1625 } while (len > 0); 1626 spin_unlock(&ct->fast_lock); 1627 1628 if (ongoing) 1629 xe_pm_runtime_put(xe); 1630 } 1631 1632 /* Returns less than zero on error, 0 on done, 1 on more available */ 1633 static int dequeue_one_g2h(struct xe_guc_ct *ct) 1634 { 1635 int len; 1636 int ret; 1637 1638 lockdep_assert_held(&ct->lock); 1639 1640 spin_lock_irq(&ct->fast_lock); 1641 len = g2h_read(ct, ct->msg, false); 1642 spin_unlock_irq(&ct->fast_lock); 1643 if (len <= 0) 1644 return len; 1645 1646 ret = parse_g2h_msg(ct, ct->msg, len); 1647 if (unlikely(ret < 0)) 1648 return ret; 1649 1650 ret = process_g2h_msg(ct, ct->msg, len); 1651 if (unlikely(ret < 0)) 1652 return ret; 1653 1654 return 1; 1655 } 1656 1657 static void receive_g2h(struct xe_guc_ct *ct) 1658 { 1659 bool ongoing; 1660 int ret; 1661 1662 /* 1663 * Normal users must always hold mem_access.ref around CT calls. However 1664 * during the runtime pm callbacks we rely on CT to talk to the GuC, but 1665 * at this stage we can't rely on mem_access.ref and even the 1666 * callback_task will be different than current. For such cases we just 1667 * need to ensure we always process the responses from any blocking 1668 * ct_send requests or where we otherwise expect some response when 1669 * initiated from those callbacks (which will need to wait for the below 1670 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if 1671 * the device has suspended to the point that the CT communication has 1672 * been disabled. 1673 * 1674 * If we are inside the runtime pm callback, we can be the only task 1675 * still issuing CT requests (since that requires having the 1676 * mem_access.ref). It seems like it might in theory be possible to 1677 * receive unsolicited events from the GuC just as we are 1678 * suspending-resuming, but those will currently anyway be lost when 1679 * eventually exiting from suspend, hence no need to wake up the device 1680 * here. If we ever need something stronger than get_if_ongoing() then 1681 * we need to be careful with blocking the pm callbacks from getting CT 1682 * responses, if the worker here is blocked on those callbacks 1683 * completing, creating a deadlock. 1684 */ 1685 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1686 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1687 return; 1688 1689 do { 1690 mutex_lock(&ct->lock); 1691 ret = dequeue_one_g2h(ct); 1692 mutex_unlock(&ct->lock); 1693 1694 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) { 1695 xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d", ret); 1696 CT_DEAD(ct, NULL, G2H_RECV); 1697 kick_reset(ct); 1698 } 1699 } while (ret == 1); 1700 1701 if (ongoing) 1702 xe_pm_runtime_put(ct_to_xe(ct)); 1703 } 1704 1705 static void g2h_worker_func(struct work_struct *w) 1706 { 1707 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker); 1708 1709 receive_g2h(ct); 1710 } 1711 1712 static void xe_fixup_u64_in_cmds(struct xe_device *xe, struct iosys_map *cmds, 1713 u32 size, u32 idx, s64 shift) 1714 { 1715 u32 hi, lo; 1716 u64 offset; 1717 1718 lo = xe_map_rd_ring_u32(xe, cmds, idx, size); 1719 hi = xe_map_rd_ring_u32(xe, cmds, idx + 1, size); 1720 offset = make_u64(hi, lo); 1721 offset += shift; 1722 lo = lower_32_bits(offset); 1723 hi = upper_32_bits(offset); 1724 xe_map_wr_ring_u32(xe, cmds, idx, size, lo); 1725 xe_map_wr_ring_u32(xe, cmds, idx + 1, size, hi); 1726 } 1727 1728 /* 1729 * Shift any GGTT addresses within a single message left within CTB from 1730 * before post-migration recovery. 1731 * @ct: pointer to CT struct of the target GuC 1732 * @cmds: iomap buffer containing CT messages 1733 * @head: start of the target message within the buffer 1734 * @len: length of the target message 1735 * @size: size of the commands buffer 1736 * @shift: the address shift to be added to each GGTT reference 1737 * Return: true if the message was fixed or needed no fixups, false on failure 1738 */ 1739 static bool ct_fixup_ggtt_in_message(struct xe_guc_ct *ct, 1740 struct iosys_map *cmds, u32 head, 1741 u32 len, u32 size, s64 shift) 1742 { 1743 struct xe_gt *gt = ct_to_gt(ct); 1744 struct xe_device *xe = ct_to_xe(ct); 1745 u32 msg[GUC_HXG_MSG_MIN_LEN]; 1746 u32 action, i, n; 1747 1748 xe_gt_assert(gt, len >= GUC_HXG_MSG_MIN_LEN); 1749 1750 msg[0] = xe_map_rd_ring_u32(xe, cmds, head, size); 1751 action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]); 1752 1753 xe_gt_sriov_dbg_verbose(gt, "fixing H2G %#x\n", action); 1754 1755 switch (action) { 1756 case XE_GUC_ACTION_REGISTER_CONTEXT: 1757 if (len != XE_GUC_REGISTER_CONTEXT_MSG_LEN) 1758 goto err_len; 1759 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1760 XE_GUC_REGISTER_CONTEXT_DATA_5_WQ_DESC_ADDR_LOWER, 1761 shift); 1762 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1763 XE_GUC_REGISTER_CONTEXT_DATA_7_WQ_BUF_BASE_LOWER, 1764 shift); 1765 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1766 XE_GUC_REGISTER_CONTEXT_DATA_10_HW_LRC_ADDR, shift); 1767 break; 1768 case XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC: 1769 if (len < XE_GUC_REGISTER_CONTEXT_MULTI_LRC_MSG_MIN_LEN) 1770 goto err_len; 1771 n = xe_map_rd_ring_u32(xe, cmds, head + 1772 XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_10_NUM_CTXS, size); 1773 if (len != XE_GUC_REGISTER_CONTEXT_MULTI_LRC_MSG_MIN_LEN + 2 * n) 1774 goto err_len; 1775 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1776 XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_5_WQ_DESC_ADDR_LOWER, 1777 shift); 1778 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1779 XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_7_WQ_BUF_BASE_LOWER, 1780 shift); 1781 for (i = 0; i < n; i++) 1782 xe_fixup_u64_in_cmds(xe, cmds, size, head + 1783 XE_GUC_REGISTER_CONTEXT_MULTI_LRC_DATA_11_HW_LRC_ADDR 1784 + 2 * i, shift); 1785 break; 1786 default: 1787 break; 1788 } 1789 return true; 1790 1791 err_len: 1792 xe_gt_err(gt, "Skipped G2G %#x message fixups, unexpected length (%u)\n", action, len); 1793 return false; 1794 } 1795 1796 /* 1797 * Apply fixups to the next outgoing CT message within given CTB 1798 * @ct: the &xe_guc_ct struct instance representing the target GuC 1799 * @h2g: the &guc_ctb struct instance of the target buffer 1800 * @shift: shift to be added to all GGTT addresses within the CTB 1801 * @mhead: pointer to an integer storing message start position; the 1802 * position is changed to next message before this function return 1803 * @avail: size of the area available for parsing, that is length 1804 * of all remaining messages stored within the CTB 1805 * Return: size of the area available for parsing after one message 1806 * has been parsed, that is length remaining from the updated mhead 1807 */ 1808 static int ct_fixup_ggtt_in_buffer(struct xe_guc_ct *ct, struct guc_ctb *h2g, 1809 s64 shift, u32 *mhead, s32 avail) 1810 { 1811 struct xe_gt *gt = ct_to_gt(ct); 1812 struct xe_device *xe = ct_to_xe(ct); 1813 u32 msg[GUC_HXG_MSG_MIN_LEN]; 1814 u32 size = h2g->info.size; 1815 u32 head = *mhead; 1816 u32 len; 1817 1818 xe_gt_assert(gt, avail >= (s32)GUC_CTB_MSG_MIN_LEN); 1819 1820 /* Read header */ 1821 msg[0] = xe_map_rd_ring_u32(xe, &h2g->cmds, head, size); 1822 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; 1823 1824 if (unlikely(len > (u32)avail)) { 1825 xe_gt_err(gt, "H2G channel broken on read, avail=%d, len=%d, fixups skipped\n", 1826 avail, len); 1827 return 0; 1828 } 1829 1830 head = (head + GUC_CTB_MSG_MIN_LEN) % size; 1831 if (!ct_fixup_ggtt_in_message(ct, &h2g->cmds, head, msg_len_to_hxg_len(len), size, shift)) 1832 return 0; 1833 *mhead = (head + msg_len_to_hxg_len(len)) % size; 1834 1835 return avail - len; 1836 } 1837 1838 /** 1839 * xe_guc_ct_fixup_messages_with_ggtt - Fixup any pending H2G CTB messages 1840 * @ct: pointer to CT struct of the target GuC 1841 * @ggtt_shift: shift to be added to all GGTT addresses within the CTB 1842 * 1843 * Messages in GuC to Host CTB are owned by GuC and any fixups in them 1844 * are made by GuC. But content of the Host to GuC CTB is owned by the 1845 * KMD, so fixups to GGTT references in any pending messages need to be 1846 * applied here. 1847 * This function updates GGTT offsets in payloads of pending H2G CTB 1848 * messages (messages which were not consumed by GuC before the VF got 1849 * paused). 1850 */ 1851 void xe_guc_ct_fixup_messages_with_ggtt(struct xe_guc_ct *ct, s64 ggtt_shift) 1852 { 1853 struct guc_ctb *h2g = &ct->ctbs.h2g; 1854 struct xe_guc *guc = ct_to_guc(ct); 1855 struct xe_gt *gt = guc_to_gt(guc); 1856 u32 head, tail, size; 1857 s32 avail; 1858 1859 if (unlikely(h2g->info.broken)) 1860 return; 1861 1862 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); 1863 head = h2g->info.head; 1864 tail = READ_ONCE(h2g->info.tail); 1865 size = h2g->info.size; 1866 1867 if (unlikely(head > size)) 1868 goto corrupted; 1869 1870 if (unlikely(tail >= size)) 1871 goto corrupted; 1872 1873 avail = tail - head; 1874 1875 /* beware of buffer wrap case */ 1876 if (unlikely(avail < 0)) 1877 avail += size; 1878 xe_gt_dbg(gt, "available %d (%u:%u:%u)\n", avail, head, tail, size); 1879 xe_gt_assert(gt, avail >= 0); 1880 1881 while (avail > 0) 1882 avail = ct_fixup_ggtt_in_buffer(ct, h2g, ggtt_shift, &head, avail); 1883 1884 return; 1885 1886 corrupted: 1887 xe_gt_err(gt, "Corrupted H2G descriptor head=%u tail=%u size=%u, fixups not applied\n", 1888 head, tail, size); 1889 h2g->info.broken = true; 1890 } 1891 1892 static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic, 1893 bool want_ctb) 1894 { 1895 struct xe_guc_ct_snapshot *snapshot; 1896 1897 snapshot = kzalloc(sizeof(*snapshot), atomic ? GFP_ATOMIC : GFP_KERNEL); 1898 if (!snapshot) 1899 return NULL; 1900 1901 if (ct->bo && want_ctb) { 1902 snapshot->ctb_size = ct->bo->size; 1903 snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL); 1904 } 1905 1906 return snapshot; 1907 } 1908 1909 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb, 1910 struct guc_ctb_snapshot *snapshot) 1911 { 1912 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0, 1913 sizeof(struct guc_ct_buffer_desc)); 1914 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info)); 1915 } 1916 1917 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot, 1918 struct drm_printer *p) 1919 { 1920 drm_printf(p, "\tsize: %d\n", snapshot->info.size); 1921 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space); 1922 drm_printf(p, "\thead: %d\n", snapshot->info.head); 1923 drm_printf(p, "\ttail: %d\n", snapshot->info.tail); 1924 drm_printf(p, "\tspace: %d\n", snapshot->info.space); 1925 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken); 1926 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head); 1927 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail); 1928 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status); 1929 } 1930 1931 static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic, 1932 bool want_ctb) 1933 { 1934 struct xe_device *xe = ct_to_xe(ct); 1935 struct xe_guc_ct_snapshot *snapshot; 1936 1937 snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb); 1938 if (!snapshot) { 1939 xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n"); 1940 return NULL; 1941 } 1942 1943 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) { 1944 snapshot->ct_enabled = true; 1945 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding); 1946 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g); 1947 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h); 1948 } 1949 1950 if (ct->bo && snapshot->ctb) 1951 xe_map_memcpy_from(xe, snapshot->ctb, &ct->bo->vmap, 0, snapshot->ctb_size); 1952 1953 return snapshot; 1954 } 1955 1956 /** 1957 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state. 1958 * @ct: GuC CT object. 1959 * 1960 * This can be printed out in a later stage like during dev_coredump 1961 * analysis. This is safe to be called during atomic context. 1962 * 1963 * Returns: a GuC CT snapshot object that must be freed by the caller 1964 * by using `xe_guc_ct_snapshot_free`. 1965 */ 1966 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct) 1967 { 1968 return guc_ct_snapshot_capture(ct, true, true); 1969 } 1970 1971 /** 1972 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot. 1973 * @snapshot: GuC CT snapshot object. 1974 * @p: drm_printer where it will be printed out. 1975 * 1976 * This function prints out a given GuC CT snapshot object. 1977 */ 1978 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, 1979 struct drm_printer *p) 1980 { 1981 if (!snapshot) 1982 return; 1983 1984 if (snapshot->ct_enabled) { 1985 drm_puts(p, "H2G CTB (all sizes in DW):\n"); 1986 guc_ctb_snapshot_print(&snapshot->h2g, p); 1987 1988 drm_puts(p, "G2H CTB (all sizes in DW):\n"); 1989 guc_ctb_snapshot_print(&snapshot->g2h, p); 1990 drm_printf(p, "\tg2h outstanding: %d\n", 1991 snapshot->g2h_outstanding); 1992 1993 if (snapshot->ctb) { 1994 drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size); 1995 xe_print_blob_ascii85(p, "[CTB].data", '\n', 1996 snapshot->ctb, 0, snapshot->ctb_size); 1997 } 1998 } else { 1999 drm_puts(p, "CT disabled\n"); 2000 } 2001 } 2002 2003 /** 2004 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot. 2005 * @snapshot: GuC CT snapshot object. 2006 * 2007 * This function free all the memory that needed to be allocated at capture 2008 * time. 2009 */ 2010 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot) 2011 { 2012 if (!snapshot) 2013 return; 2014 2015 kfree(snapshot->ctb); 2016 kfree(snapshot); 2017 } 2018 2019 /** 2020 * xe_guc_ct_print - GuC CT Print. 2021 * @ct: GuC CT. 2022 * @p: drm_printer where it will be printed out. 2023 * @want_ctb: Should the full CTB content be dumped (vs just the headers) 2024 * 2025 * This function will quickly capture a snapshot of the CT state 2026 * and immediately print it out. 2027 */ 2028 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb) 2029 { 2030 struct xe_guc_ct_snapshot *snapshot; 2031 2032 snapshot = guc_ct_snapshot_capture(ct, false, want_ctb); 2033 xe_guc_ct_snapshot_print(snapshot, p); 2034 xe_guc_ct_snapshot_free(snapshot); 2035 } 2036 2037 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 2038 2039 #ifdef CONFIG_FUNCTION_ERROR_INJECTION 2040 /* 2041 * This is a helper function which assists the driver in identifying if a fault 2042 * injection test is currently active, allowing it to reduce unnecessary debug 2043 * output. Typically, the function returns zero, but the fault injection 2044 * framework can alter this to return an error. Since faults are injected 2045 * through this function, it's important to ensure the compiler doesn't optimize 2046 * it into an inline function. To avoid such optimization, the 'noinline' 2047 * attribute is applied. Compiler optimizes the static function defined in the 2048 * header file as an inline function. 2049 */ 2050 noinline int xe_is_injection_active(void) { return 0; } 2051 ALLOW_ERROR_INJECTION(xe_is_injection_active, ERRNO); 2052 #else 2053 int xe_is_injection_active(void) { return 0; } 2054 #endif 2055 2056 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code) 2057 { 2058 struct xe_guc_log_snapshot *snapshot_log; 2059 struct xe_guc_ct_snapshot *snapshot_ct; 2060 struct xe_guc *guc = ct_to_guc(ct); 2061 unsigned long flags; 2062 bool have_capture; 2063 2064 if (ctb) 2065 ctb->info.broken = true; 2066 /* 2067 * Huge dump is getting generated when injecting error for guc CT/MMIO 2068 * functions. So, let us suppress the dump when fault is injected. 2069 */ 2070 if (xe_is_injection_active()) 2071 return; 2072 2073 /* Ignore further errors after the first dump until a reset */ 2074 if (ct->dead.reported) 2075 return; 2076 2077 spin_lock_irqsave(&ct->dead.lock, flags); 2078 2079 /* And only capture one dump at a time */ 2080 have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE); 2081 ct->dead.reason |= (1 << reason_code) | 2082 (1 << CT_DEAD_STATE_CAPTURE); 2083 2084 spin_unlock_irqrestore(&ct->dead.lock, flags); 2085 2086 if (have_capture) 2087 return; 2088 2089 snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true); 2090 snapshot_ct = xe_guc_ct_snapshot_capture((ct)); 2091 2092 spin_lock_irqsave(&ct->dead.lock, flags); 2093 2094 if (ct->dead.snapshot_log || ct->dead.snapshot_ct) { 2095 xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n"); 2096 xe_guc_log_snapshot_free(snapshot_log); 2097 xe_guc_ct_snapshot_free(snapshot_ct); 2098 } else { 2099 ct->dead.snapshot_log = snapshot_log; 2100 ct->dead.snapshot_ct = snapshot_ct; 2101 } 2102 2103 spin_unlock_irqrestore(&ct->dead.lock, flags); 2104 2105 queue_work(system_unbound_wq, &(ct)->dead.worker); 2106 } 2107 2108 static void ct_dead_print(struct xe_dead_ct *dead) 2109 { 2110 struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead); 2111 struct xe_device *xe = ct_to_xe(ct); 2112 struct xe_gt *gt = ct_to_gt(ct); 2113 static int g_count; 2114 struct drm_printer ip = xe_gt_info_printer(gt); 2115 struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count); 2116 2117 if (!dead->reason) { 2118 xe_gt_err(gt, "CTB is dead for no reason!?\n"); 2119 return; 2120 } 2121 2122 /* Can't generate a genuine core dump at this point, so just do the good bits */ 2123 drm_puts(&lp, "**** Xe Device Coredump ****\n"); 2124 drm_printf(&lp, "Reason: CTB is dead - 0x%X\n", dead->reason); 2125 xe_device_snapshot_print(xe, &lp); 2126 2127 drm_printf(&lp, "**** GT #%d ****\n", gt->info.id); 2128 drm_printf(&lp, "\tTile: %d\n", gt->tile->id); 2129 2130 drm_puts(&lp, "**** GuC Log ****\n"); 2131 xe_guc_log_snapshot_print(dead->snapshot_log, &lp); 2132 2133 drm_puts(&lp, "**** GuC CT ****\n"); 2134 xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp); 2135 2136 drm_puts(&lp, "Done.\n"); 2137 } 2138 2139 static void ct_dead_worker_func(struct work_struct *w) 2140 { 2141 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker); 2142 2143 if (!ct->dead.reported) { 2144 ct->dead.reported = true; 2145 ct_dead_print(&ct->dead); 2146 } 2147 2148 spin_lock_irq(&ct->dead.lock); 2149 2150 xe_guc_log_snapshot_free(ct->dead.snapshot_log); 2151 ct->dead.snapshot_log = NULL; 2152 xe_guc_ct_snapshot_free(ct->dead.snapshot_ct); 2153 ct->dead.snapshot_ct = NULL; 2154 2155 if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) { 2156 /* A reset has occurred so re-arm the error reporting */ 2157 ct->dead.reason = 0; 2158 ct->dead.reported = false; 2159 } 2160 2161 spin_unlock_irq(&ct->dead.lock); 2162 } 2163 #endif 2164