1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ct.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 #include <linux/delay.h> 11 #include <linux/fault-inject.h> 12 13 #include <kunit/static_stub.h> 14 15 #include <drm/drm_managed.h> 16 17 #include "abi/guc_actions_abi.h" 18 #include "abi/guc_actions_sriov_abi.h" 19 #include "abi/guc_klvs_abi.h" 20 #include "xe_bo.h" 21 #include "xe_devcoredump.h" 22 #include "xe_device.h" 23 #include "xe_gt.h" 24 #include "xe_gt_printk.h" 25 #include "xe_gt_sriov_pf_control.h" 26 #include "xe_gt_sriov_pf_monitor.h" 27 #include "xe_guc.h" 28 #include "xe_guc_log.h" 29 #include "xe_guc_pagefault.h" 30 #include "xe_guc_relay.h" 31 #include "xe_guc_submit.h" 32 #include "xe_guc_tlb_inval.h" 33 #include "xe_map.h" 34 #include "xe_pm.h" 35 #include "xe_sriov_vf.h" 36 #include "xe_trace_guc.h" 37 38 static void receive_g2h(struct xe_guc_ct *ct); 39 static void g2h_worker_func(struct work_struct *w); 40 static void safe_mode_worker_func(struct work_struct *w); 41 static void ct_exit_safe_mode(struct xe_guc_ct *ct); 42 static void guc_ct_change_state(struct xe_guc_ct *ct, 43 enum xe_guc_ct_state state); 44 45 static struct xe_guc *ct_to_guc(struct xe_guc_ct *ct) 46 { 47 return container_of(ct, struct xe_guc, ct); 48 } 49 50 static struct xe_gt *ct_to_gt(struct xe_guc_ct *ct) 51 { 52 return container_of(ct, struct xe_gt, uc.guc.ct); 53 } 54 55 static struct xe_device *ct_to_xe(struct xe_guc_ct *ct) 56 { 57 return gt_to_xe(ct_to_gt(ct)); 58 } 59 60 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 61 enum { 62 /* Internal states, not error conditions */ 63 CT_DEAD_STATE_REARM, /* 0x0001 */ 64 CT_DEAD_STATE_CAPTURE, /* 0x0002 */ 65 66 /* Error conditions */ 67 CT_DEAD_SETUP, /* 0x0004 */ 68 CT_DEAD_H2G_WRITE, /* 0x0008 */ 69 CT_DEAD_H2G_HAS_ROOM, /* 0x0010 */ 70 CT_DEAD_G2H_READ, /* 0x0020 */ 71 CT_DEAD_G2H_RECV, /* 0x0040 */ 72 CT_DEAD_G2H_RELEASE, /* 0x0080 */ 73 CT_DEAD_DEADLOCK, /* 0x0100 */ 74 CT_DEAD_PROCESS_FAILED, /* 0x0200 */ 75 CT_DEAD_FAST_G2H, /* 0x0400 */ 76 CT_DEAD_PARSE_G2H_RESPONSE, /* 0x0800 */ 77 CT_DEAD_PARSE_G2H_UNKNOWN, /* 0x1000 */ 78 CT_DEAD_PARSE_G2H_ORIGIN, /* 0x2000 */ 79 CT_DEAD_PARSE_G2H_TYPE, /* 0x4000 */ 80 CT_DEAD_CRASH, /* 0x8000 */ 81 }; 82 83 static void ct_dead_worker_func(struct work_struct *w); 84 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code); 85 86 static void ct_dead_fini(struct xe_guc_ct *ct) 87 { 88 cancel_work_sync(&ct->dead.worker); 89 } 90 91 static void ct_dead_init(struct xe_guc_ct *ct) 92 { 93 spin_lock_init(&ct->dead.lock); 94 INIT_WORK(&ct->dead.worker, ct_dead_worker_func); 95 96 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 97 stack_depot_init(); 98 #endif 99 } 100 101 static void fast_req_stack_save(struct xe_guc_ct *ct, unsigned int slot) 102 { 103 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 104 unsigned long entries[SZ_32]; 105 unsigned int n; 106 107 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 108 /* May be called under spinlock, so avoid sleeping */ 109 ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT); 110 #endif 111 } 112 113 static void fast_req_dump(struct xe_guc_ct *ct, u16 fence, unsigned int slot) 114 { 115 struct xe_gt *gt = ct_to_gt(ct); 116 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 117 char *buf __cleanup(kfree) = kmalloc(SZ_4K, GFP_NOWAIT); 118 119 if (buf && stack_depot_snprint(ct->fast_req[slot].stack, buf, SZ_4K, 0)) 120 xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s\n", 121 fence, ct->fast_req[slot].action, buf); 122 else 123 xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n", 124 fence, ct->fast_req[slot].action); 125 #else 126 xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n", 127 fence, ct->fast_req[slot].action); 128 #endif 129 } 130 131 static void fast_req_report(struct xe_guc_ct *ct, u16 fence) 132 { 133 u16 fence_min = U16_MAX, fence_max = 0; 134 struct xe_gt *gt = ct_to_gt(ct); 135 unsigned int n; 136 137 lockdep_assert_held(&ct->lock); 138 139 for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) { 140 if (ct->fast_req[n].fence < fence_min) 141 fence_min = ct->fast_req[n].fence; 142 if (ct->fast_req[n].fence > fence_max) 143 fence_max = ct->fast_req[n].fence; 144 145 if (ct->fast_req[n].fence != fence) 146 continue; 147 148 return fast_req_dump(ct, fence, n); 149 } 150 151 xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n", 152 fence, fence_min, fence_max, ct->fence_seqno); 153 } 154 155 static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) 156 { 157 unsigned int slot = fence % ARRAY_SIZE(ct->fast_req); 158 159 fast_req_stack_save(ct, slot); 160 ct->fast_req[slot].fence = fence; 161 ct->fast_req[slot].action = action; 162 } 163 164 #define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code) 165 166 #else 167 168 static void ct_dead_fini(struct xe_guc_ct *ct) { } 169 static void ct_dead_init(struct xe_guc_ct *ct) { } 170 171 static void fast_req_report(struct xe_guc_ct *ct, u16 fence) { } 172 static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { } 173 174 #define CT_DEAD(ct, ctb, reason) \ 175 do { \ 176 struct guc_ctb *_ctb = (ctb); \ 177 if (_ctb) \ 178 _ctb->info.broken = true; \ 179 } while (0) 180 181 #endif 182 183 /* Used when a CT send wants to block and / or receive data */ 184 struct g2h_fence { 185 u32 *response_buffer; 186 u32 seqno; 187 u32 response_data; 188 u16 response_len; 189 u16 error; 190 u16 hint; 191 u16 reason; 192 bool cancel; 193 bool retry; 194 bool fail; 195 bool done; 196 }; 197 198 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) 199 { 200 memset(g2h_fence, 0, sizeof(*g2h_fence)); 201 g2h_fence->response_buffer = response_buffer; 202 g2h_fence->seqno = ~0x0; 203 } 204 205 static void g2h_fence_cancel(struct g2h_fence *g2h_fence) 206 { 207 g2h_fence->cancel = true; 208 g2h_fence->fail = true; 209 210 /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ 211 WRITE_ONCE(g2h_fence->done, true); 212 } 213 214 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) 215 { 216 return g2h_fence->seqno == ~0x0; 217 } 218 219 /** 220 * DOC: GuC CTB Blob 221 * 222 * We allocate single blob to hold both CTB descriptors and buffers: 223 * 224 * +--------+-----------------------------------------------+------+ 225 * | offset | contents | size | 226 * +========+===============================================+======+ 227 * | 0x0000 | H2G CTB Descriptor (send) | | 228 * +--------+-----------------------------------------------+ 4K | 229 * | 0x0800 | G2H CTB Descriptor (g2h) | | 230 * +--------+-----------------------------------------------+------+ 231 * | 0x1000 | H2G CT Buffer (send) | n*4K | 232 * | | | | 233 * +--------+-----------------------------------------------+------+ 234 * | 0x1000 | G2H CT Buffer (g2h) | m*4K | 235 * | + n*4K | | | 236 * +--------+-----------------------------------------------+------+ 237 * 238 * Size of each ``CT Buffer`` must be multiple of 4K. 239 * We don't expect too many messages in flight at any time, unless we are 240 * using the GuC submission. In that case each request requires a minimum 241 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 242 * enough space to avoid backpressure on the driver. We increase the size 243 * of the receive buffer (relative to the send) to ensure a G2H response 244 * CTB has a landing spot. 245 * 246 * In addition to submissions, the G2H buffer needs to be able to hold 247 * enough space for recoverable page fault notifications. The number of 248 * page faults is interrupt driven and can be as much as the number of 249 * compute resources available. However, most of the actual work for these 250 * is in a separate page fault worker thread. Therefore we only need to 251 * make sure the queue has enough space to handle all of the submissions 252 * and responses and an extra buffer for incoming page faults. 253 */ 254 255 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 256 #define CTB_H2G_BUFFER_OFFSET (CTB_DESC_SIZE * 2) 257 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 258 #define CTB_H2G_BUFFER_DWORDS (CTB_H2G_BUFFER_SIZE / sizeof(u32)) 259 #define CTB_G2H_BUFFER_SIZE (SZ_128K) 260 #define CTB_G2H_BUFFER_DWORDS (CTB_G2H_BUFFER_SIZE / sizeof(u32)) 261 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2) 262 #define G2H_ROOM_BUFFER_DWORDS (CTB_G2H_BUFFER_DWORDS / 2) 263 264 /** 265 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full 266 * CT command queue 267 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future. 268 * 269 * Observation is that a 4KiB buffer full of commands takes a little over a 270 * second to process. Use that to calculate maximum time to process a full CT 271 * command queue. 272 * 273 * Return: Maximum time to process a full CT queue in jiffies. 274 */ 275 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct) 276 { 277 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4)); 278 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ; 279 } 280 281 static size_t guc_ct_size(void) 282 { 283 return CTB_H2G_BUFFER_OFFSET + CTB_H2G_BUFFER_SIZE + 284 CTB_G2H_BUFFER_SIZE; 285 } 286 287 static void guc_ct_fini(struct drm_device *drm, void *arg) 288 { 289 struct xe_guc_ct *ct = arg; 290 291 ct_dead_fini(ct); 292 ct_exit_safe_mode(ct); 293 destroy_workqueue(ct->g2h_wq); 294 xa_destroy(&ct->fence_lookup); 295 } 296 297 static void primelockdep(struct xe_guc_ct *ct) 298 { 299 if (!IS_ENABLED(CONFIG_LOCKDEP)) 300 return; 301 302 fs_reclaim_acquire(GFP_KERNEL); 303 might_lock(&ct->lock); 304 fs_reclaim_release(GFP_KERNEL); 305 } 306 307 int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) 308 { 309 struct xe_device *xe = ct_to_xe(ct); 310 struct xe_gt *gt = ct_to_gt(ct); 311 int err; 312 313 xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE)); 314 315 err = drmm_mutex_init(&xe->drm, &ct->lock); 316 if (err) 317 return err; 318 319 primelockdep(ct); 320 321 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM); 322 if (!ct->g2h_wq) 323 return -ENOMEM; 324 325 spin_lock_init(&ct->fast_lock); 326 xa_init(&ct->fence_lookup); 327 INIT_WORK(&ct->g2h_worker, g2h_worker_func); 328 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func); 329 330 ct_dead_init(ct); 331 init_waitqueue_head(&ct->wq); 332 init_waitqueue_head(&ct->g2h_fence_wq); 333 334 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); 335 if (err) 336 return err; 337 338 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED); 339 ct->state = XE_GUC_CT_STATE_DISABLED; 340 return 0; 341 } 342 ALLOW_ERROR_INJECTION(xe_guc_ct_init_noalloc, ERRNO); /* See xe_pci_probe() */ 343 344 static void guc_action_disable_ct(void *arg) 345 { 346 struct xe_guc_ct *ct = arg; 347 348 xe_guc_ct_stop(ct); 349 guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); 350 } 351 352 int xe_guc_ct_init(struct xe_guc_ct *ct) 353 { 354 struct xe_device *xe = ct_to_xe(ct); 355 struct xe_gt *gt = ct_to_gt(ct); 356 struct xe_tile *tile = gt_to_tile(gt); 357 struct xe_bo *bo; 358 359 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(), 360 XE_BO_FLAG_SYSTEM | 361 XE_BO_FLAG_GGTT | 362 XE_BO_FLAG_GGTT_INVALIDATE | 363 XE_BO_FLAG_PINNED_NORESTORE); 364 if (IS_ERR(bo)) 365 return PTR_ERR(bo); 366 367 ct->bo = bo; 368 369 return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct); 370 } 371 ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */ 372 373 /** 374 * xe_guc_ct_init_post_hwconfig - Reinitialize the GuC CTB in VRAM 375 * @ct: the &xe_guc_ct 376 * 377 * Allocate a new BO in VRAM and free the previous BO that was allocated 378 * in system memory (SMEM). Applicable only for DGFX products. 379 * 380 * Return: 0 on success, or a negative errno on failure. 381 */ 382 int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct) 383 { 384 struct xe_device *xe = ct_to_xe(ct); 385 struct xe_gt *gt = ct_to_gt(ct); 386 struct xe_tile *tile = gt_to_tile(gt); 387 int ret; 388 389 xe_assert(xe, !xe_guc_ct_enabled(ct)); 390 391 if (IS_DGFX(xe)) { 392 ret = xe_managed_bo_reinit_in_vram(xe, tile, &ct->bo); 393 if (ret) 394 return ret; 395 } 396 397 devm_remove_action(xe->drm.dev, guc_action_disable_ct, ct); 398 return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct); 399 } 400 401 #define desc_read(xe_, guc_ctb__, field_) \ 402 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \ 403 struct guc_ct_buffer_desc, field_) 404 405 #define desc_write(xe_, guc_ctb__, field_, val_) \ 406 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \ 407 struct guc_ct_buffer_desc, field_, val_) 408 409 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, 410 struct iosys_map *map) 411 { 412 h2g->info.size = CTB_H2G_BUFFER_DWORDS; 413 h2g->info.resv_space = 0; 414 h2g->info.tail = 0; 415 h2g->info.head = 0; 416 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 417 h2g->info.size) - 418 h2g->info.resv_space; 419 h2g->info.broken = false; 420 421 h2g->desc = *map; 422 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 423 424 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_H2G_BUFFER_OFFSET); 425 } 426 427 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, 428 struct iosys_map *map) 429 { 430 g2h->info.size = CTB_G2H_BUFFER_DWORDS; 431 g2h->info.resv_space = G2H_ROOM_BUFFER_DWORDS; 432 g2h->info.head = 0; 433 g2h->info.tail = 0; 434 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head, 435 g2h->info.size) - 436 g2h->info.resv_space; 437 g2h->info.broken = false; 438 439 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE); 440 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 441 442 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_H2G_BUFFER_OFFSET + 443 CTB_H2G_BUFFER_SIZE); 444 } 445 446 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct) 447 { 448 struct xe_guc *guc = ct_to_guc(ct); 449 u32 desc_addr, ctb_addr, size; 450 int err; 451 452 desc_addr = xe_bo_ggtt_addr(ct->bo); 453 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_H2G_BUFFER_OFFSET; 454 size = ct->ctbs.h2g.info.size * sizeof(u32); 455 456 err = xe_guc_self_cfg64(guc, 457 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY, 458 desc_addr); 459 if (err) 460 return err; 461 462 err = xe_guc_self_cfg64(guc, 463 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY, 464 ctb_addr); 465 if (err) 466 return err; 467 468 return xe_guc_self_cfg32(guc, 469 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY, 470 size); 471 } 472 473 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct) 474 { 475 struct xe_guc *guc = ct_to_guc(ct); 476 u32 desc_addr, ctb_addr, size; 477 int err; 478 479 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE; 480 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_H2G_BUFFER_OFFSET + 481 CTB_H2G_BUFFER_SIZE; 482 size = ct->ctbs.g2h.info.size * sizeof(u32); 483 484 err = xe_guc_self_cfg64(guc, 485 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 486 desc_addr); 487 if (err) 488 return err; 489 490 err = xe_guc_self_cfg64(guc, 491 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 492 ctb_addr); 493 if (err) 494 return err; 495 496 return xe_guc_self_cfg32(guc, 497 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 498 size); 499 } 500 501 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) 502 { 503 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 504 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 505 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 506 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, 507 GUC_ACTION_HOST2GUC_CONTROL_CTB), 508 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, 509 enable ? GUC_CTB_CONTROL_ENABLE : 510 GUC_CTB_CONTROL_DISABLE), 511 }; 512 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request)); 513 514 return ret > 0 ? -EPROTO : ret; 515 } 516 517 static void guc_ct_change_state(struct xe_guc_ct *ct, 518 enum xe_guc_ct_state state) 519 { 520 struct xe_gt *gt = ct_to_gt(ct); 521 struct g2h_fence *g2h_fence; 522 unsigned long idx; 523 524 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ 525 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ 526 527 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 || 528 state == XE_GUC_CT_STATE_STOPPED); 529 530 if (ct->g2h_outstanding) 531 xe_pm_runtime_put(ct_to_xe(ct)); 532 ct->g2h_outstanding = 0; 533 534 /* 535 * WRITE_ONCE pairs with READ_ONCEs in xe_guc_ct_initialized and 536 * xe_guc_ct_enabled. 537 */ 538 WRITE_ONCE(ct->state, state); 539 540 xe_gt_dbg(gt, "GuC CT communication channel %s\n", 541 state == XE_GUC_CT_STATE_STOPPED ? "stopped" : 542 str_enabled_disabled(state == XE_GUC_CT_STATE_ENABLED)); 543 544 spin_unlock_irq(&ct->fast_lock); 545 546 /* cancel all in-flight send-recv requests */ 547 xa_for_each(&ct->fence_lookup, idx, g2h_fence) 548 g2h_fence_cancel(g2h_fence); 549 550 /* make sure guc_ct_send_recv() will see g2h_fence changes */ 551 smp_mb(); 552 wake_up_all(&ct->g2h_fence_wq); 553 554 /* 555 * Lockdep doesn't like this under the fast lock and he destroy only 556 * needs to be serialized with the send path which ct lock provides. 557 */ 558 xa_destroy(&ct->fence_lookup); 559 560 mutex_unlock(&ct->lock); 561 } 562 563 static bool ct_needs_safe_mode(struct xe_guc_ct *ct) 564 { 565 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev)); 566 } 567 568 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct) 569 { 570 if (!ct_needs_safe_mode(ct)) 571 return false; 572 573 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10); 574 return true; 575 } 576 577 static void safe_mode_worker_func(struct work_struct *w) 578 { 579 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work); 580 581 receive_g2h(ct); 582 583 if (!ct_restart_safe_mode_worker(ct)) 584 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n"); 585 } 586 587 static void ct_enter_safe_mode(struct xe_guc_ct *ct) 588 { 589 if (ct_restart_safe_mode_worker(ct)) 590 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n"); 591 } 592 593 static void ct_exit_safe_mode(struct xe_guc_ct *ct) 594 { 595 if (cancel_delayed_work_sync(&ct->safe_mode_worker)) 596 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n"); 597 } 598 599 static int __xe_guc_ct_start(struct xe_guc_ct *ct, bool needs_register) 600 { 601 struct xe_device *xe = ct_to_xe(ct); 602 struct xe_gt *gt = ct_to_gt(ct); 603 int err; 604 605 xe_gt_assert(gt, !xe_guc_ct_enabled(ct)); 606 607 if (needs_register) { 608 xe_map_memset(xe, &ct->bo->vmap, 0, 0, xe_bo_size(ct->bo)); 609 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap); 610 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap); 611 612 err = guc_ct_ctb_h2g_register(ct); 613 if (err) 614 goto err_out; 615 616 err = guc_ct_ctb_g2h_register(ct); 617 if (err) 618 goto err_out; 619 620 err = guc_ct_control_toggle(ct, true); 621 if (err) 622 goto err_out; 623 } else { 624 ct->ctbs.h2g.info.broken = false; 625 ct->ctbs.g2h.info.broken = false; 626 /* Skip everything in H2G buffer */ 627 xe_map_memset(xe, &ct->bo->vmap, CTB_H2G_BUFFER_OFFSET, 0, 628 CTB_H2G_BUFFER_SIZE); 629 } 630 631 guc_ct_change_state(ct, XE_GUC_CT_STATE_ENABLED); 632 633 smp_mb(); 634 wake_up_all(&ct->wq); 635 636 if (ct_needs_safe_mode(ct)) 637 ct_enter_safe_mode(ct); 638 639 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 640 /* 641 * The CT has now been reset so the dumper can be re-armed 642 * after any existing dead state has been dumped. 643 */ 644 spin_lock_irq(&ct->dead.lock); 645 if (ct->dead.reason) { 646 ct->dead.reason |= (1 << CT_DEAD_STATE_REARM); 647 queue_work(system_unbound_wq, &ct->dead.worker); 648 } 649 spin_unlock_irq(&ct->dead.lock); 650 #endif 651 652 return 0; 653 654 err_out: 655 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err)); 656 CT_DEAD(ct, NULL, SETUP); 657 658 return err; 659 } 660 661 /** 662 * xe_guc_ct_restart() - Restart GuC CT 663 * @ct: the &xe_guc_ct 664 * 665 * Restart GuC CT to an empty state without issuing a CT register MMIO command. 666 * 667 * Return: 0 on success, or a negative errno on failure. 668 */ 669 int xe_guc_ct_restart(struct xe_guc_ct *ct) 670 { 671 return __xe_guc_ct_start(ct, false); 672 } 673 674 /** 675 * xe_guc_ct_enable() - Enable GuC CT 676 * @ct: the &xe_guc_ct 677 * 678 * Enable GuC CT to an empty state and issue a CT register MMIO command. 679 * 680 * Return: 0 on success, or a negative errno on failure. 681 */ 682 int xe_guc_ct_enable(struct xe_guc_ct *ct) 683 { 684 return __xe_guc_ct_start(ct, true); 685 } 686 687 static void stop_g2h_handler(struct xe_guc_ct *ct) 688 { 689 cancel_work_sync(&ct->g2h_worker); 690 } 691 692 /** 693 * xe_guc_ct_disable - Set GuC to disabled state 694 * @ct: the &xe_guc_ct 695 * 696 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected 697 * in this transition. 698 */ 699 void xe_guc_ct_disable(struct xe_guc_ct *ct) 700 { 701 guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); 702 ct_exit_safe_mode(ct); 703 stop_g2h_handler(ct); 704 } 705 706 /** 707 * xe_guc_ct_flush_and_stop - Flush and stop all processing of G2H / H2G 708 * @ct: the &xe_guc_ct 709 */ 710 void xe_guc_ct_flush_and_stop(struct xe_guc_ct *ct) 711 { 712 receive_g2h(ct); 713 xe_guc_ct_stop(ct); 714 } 715 716 /** 717 * xe_guc_ct_stop - Set GuC to stopped state 718 * @ct: the &xe_guc_ct 719 * 720 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h 721 */ 722 void xe_guc_ct_stop(struct xe_guc_ct *ct) 723 { 724 if (!xe_guc_ct_initialized(ct)) 725 return; 726 727 guc_ct_change_state(ct, XE_GUC_CT_STATE_STOPPED); 728 stop_g2h_handler(ct); 729 } 730 731 /** 732 * xe_guc_ct_runtime_suspend() - GuC CT runtime suspend 733 * @ct: the &xe_guc_ct 734 * 735 * Set GuC CT to disabled state. 736 */ 737 void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct) 738 { 739 struct guc_ctb *g2h = &ct->ctbs.g2h; 740 u32 credits = CIRC_SPACE(0, 0, CTB_G2H_BUFFER_DWORDS) - G2H_ROOM_BUFFER_DWORDS; 741 742 /* We should be back to guc_ct_ctb_g2h_init() values */ 743 xe_gt_assert(ct_to_gt(ct), g2h->info.space == credits); 744 745 /* 746 * Since we're already in runtime suspend path, we shouldn't have pending 747 * messages. But if there happen to be any, we'd probably want them to be 748 * thrown as errors for further investigation. 749 */ 750 xe_guc_ct_disable(ct); 751 } 752 753 /** 754 * xe_guc_ct_runtime_resume() - GuC CT runtime resume 755 * @ct: the &xe_guc_ct 756 * 757 * Restart GuC CT and set it to enabled state. 758 */ 759 void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct) 760 { 761 xe_guc_ct_restart(ct); 762 } 763 764 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) 765 { 766 struct guc_ctb *h2g = &ct->ctbs.h2g; 767 768 lockdep_assert_held(&ct->lock); 769 770 if (cmd_len > h2g->info.space) { 771 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); 772 773 if (h2g->info.head > h2g->info.size) { 774 struct xe_device *xe = ct_to_xe(ct); 775 u32 desc_status = desc_read(xe, h2g, status); 776 777 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 778 779 xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n", 780 h2g->info.head, h2g->info.size); 781 CT_DEAD(ct, h2g, H2G_HAS_ROOM); 782 return false; 783 } 784 785 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 786 h2g->info.size) - 787 h2g->info.resv_space; 788 if (cmd_len > h2g->info.space) 789 return false; 790 } 791 792 return true; 793 } 794 795 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len) 796 { 797 if (!g2h_len) 798 return true; 799 800 lockdep_assert_held(&ct->fast_lock); 801 802 return ct->ctbs.g2h.info.space > g2h_len; 803 } 804 805 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len) 806 { 807 lockdep_assert_held(&ct->lock); 808 809 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len)) 810 return -EBUSY; 811 812 return 0; 813 } 814 815 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len) 816 { 817 lockdep_assert_held(&ct->lock); 818 ct->ctbs.h2g.info.space -= cmd_len; 819 } 820 821 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h) 822 { 823 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space); 824 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) || 825 (g2h_len && num_g2h)); 826 827 if (g2h_len) { 828 lockdep_assert_held(&ct->fast_lock); 829 830 if (!ct->g2h_outstanding) 831 xe_pm_runtime_get_noresume(ct_to_xe(ct)); 832 833 ct->ctbs.g2h.info.space -= g2h_len; 834 ct->g2h_outstanding += num_g2h; 835 } 836 } 837 838 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 839 { 840 bool bad = false; 841 842 lockdep_assert_held(&ct->fast_lock); 843 844 bad = ct->ctbs.g2h.info.space + g2h_len > 845 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space; 846 bad |= !ct->g2h_outstanding; 847 848 if (bad) { 849 xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n", 850 ct->ctbs.g2h.info.space, g2h_len, 851 ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space, 852 ct->ctbs.g2h.info.space + g2h_len, 853 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space, 854 ct->g2h_outstanding); 855 CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE); 856 return; 857 } 858 859 ct->ctbs.g2h.info.space += g2h_len; 860 if (!--ct->g2h_outstanding) 861 xe_pm_runtime_put(ct_to_xe(ct)); 862 } 863 864 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 865 { 866 spin_lock_irq(&ct->fast_lock); 867 __g2h_release_space(ct, g2h_len); 868 spin_unlock_irq(&ct->fast_lock); 869 } 870 871 /* 872 * The CT protocol accepts a 16 bits fence. This field is fully owned by the 873 * driver, the GuC will just copy it to the reply message. Since we need to 874 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages, 875 * we use one bit of the seqno as an indicator for that and a rolling counter 876 * for the remaining 15 bits. 877 */ 878 #define CT_SEQNO_MASK GENMASK(14, 0) 879 #define CT_SEQNO_UNTRACKED BIT(15) 880 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence) 881 { 882 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK; 883 884 if (!is_g2h_fence) 885 seqno |= CT_SEQNO_UNTRACKED; 886 887 return seqno; 888 } 889 890 #define MAKE_ACTION(type, __action) \ 891 ({ \ 892 FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | \ 893 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | \ 894 GUC_HXG_EVENT_MSG_0_DATA0, __action); \ 895 }) 896 897 static bool vf_action_can_safely_fail(struct xe_device *xe, u32 action) 898 { 899 /* 900 * When resuming a VF, we can't reliably track whether context 901 * registration has completed in the GuC state machine. It is harmless 902 * to resend the request, as it will fail silently if GUC_HXG_TYPE_EVENT 903 * is used. Additionally, if there is an H2G protocol issue on a VF, 904 * subsequent H2G messages sent as GUC_HXG_TYPE_FAST_REQUEST will likely 905 * fail. 906 */ 907 return IS_SRIOV_VF(xe) && xe_sriov_vf_migration_supported(xe) && 908 (action == XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC || 909 action == XE_GUC_ACTION_REGISTER_CONTEXT); 910 } 911 912 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */ 913 914 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, 915 u32 ct_fence_value, bool want_response) 916 { 917 struct xe_device *xe = ct_to_xe(ct); 918 struct xe_gt *gt = ct_to_gt(ct); 919 struct guc_ctb *h2g = &ct->ctbs.h2g; 920 u32 cmd[H2G_CT_HEADERS]; 921 u32 tail = h2g->info.tail; 922 u32 full_len; 923 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds, 924 tail * sizeof(u32)); 925 u32 desc_status; 926 927 full_len = len + GUC_CTB_HDR_LEN; 928 929 lockdep_assert_held(&ct->lock); 930 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN); 931 932 desc_status = desc_read(xe, h2g, status); 933 if (desc_status) { 934 xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status); 935 goto corrupted; 936 } 937 938 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 939 u32 desc_tail = desc_read(xe, h2g, tail); 940 u32 desc_head = desc_read(xe, h2g, head); 941 942 if (tail != desc_tail) { 943 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH); 944 xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail); 945 goto corrupted; 946 } 947 948 if (tail > h2g->info.size) { 949 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 950 xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n", 951 tail, h2g->info.size); 952 goto corrupted; 953 } 954 955 if (desc_head >= h2g->info.size) { 956 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 957 xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n", 958 desc_head, h2g->info.size); 959 goto corrupted; 960 } 961 } 962 963 /* Command will wrap, zero fill (NOPs), return and check credits again */ 964 if (tail + full_len > h2g->info.size) { 965 xe_map_memset(xe, &map, 0, 0, 966 (h2g->info.size - tail) * sizeof(u32)); 967 h2g_reserve_space(ct, (h2g->info.size - tail)); 968 h2g->info.tail = 0; 969 desc_write(xe, h2g, tail, h2g->info.tail); 970 971 return -EAGAIN; 972 } 973 974 /* 975 * dw0: CT header (including fence) 976 * dw1: HXG header (including action code) 977 * dw2+: action data 978 */ 979 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 980 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 981 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value); 982 if (want_response) { 983 cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_REQUEST, action[0]); 984 } else if (vf_action_can_safely_fail(xe, action[0])) { 985 cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_EVENT, action[0]); 986 } else { 987 fast_req_track(ct, ct_fence_value, 988 FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0])); 989 990 cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_FAST_REQUEST, action[0]); 991 } 992 993 /* H2G header in cmd[1] replaces action[0] so: */ 994 --len; 995 ++action; 996 997 /* Write H2G ensuring visible before descriptor update */ 998 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32)); 999 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32)); 1000 xe_device_wmb(xe); 1001 1002 /* Update local copies */ 1003 h2g->info.tail = (tail + full_len) % h2g->info.size; 1004 h2g_reserve_space(ct, full_len); 1005 1006 /* Update descriptor */ 1007 desc_write(xe, h2g, tail, h2g->info.tail); 1008 1009 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len, 1010 desc_read(xe, h2g, head), h2g->info.tail); 1011 1012 return 0; 1013 1014 corrupted: 1015 CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE); 1016 return -EPIPE; 1017 } 1018 1019 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, 1020 u32 len, u32 g2h_len, u32 num_g2h, 1021 struct g2h_fence *g2h_fence) 1022 { 1023 struct xe_gt *gt = ct_to_gt(ct); 1024 u16 seqno; 1025 int ret; 1026 1027 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 1028 xe_gt_assert(gt, !g2h_len || !g2h_fence); 1029 xe_gt_assert(gt, !num_g2h || !g2h_fence); 1030 xe_gt_assert(gt, !g2h_len || num_g2h); 1031 xe_gt_assert(gt, g2h_len || !num_g2h); 1032 lockdep_assert_held(&ct->lock); 1033 1034 if (unlikely(ct->ctbs.h2g.info.broken)) { 1035 ret = -EPIPE; 1036 goto out; 1037 } 1038 1039 if (ct->state == XE_GUC_CT_STATE_DISABLED) { 1040 ret = -ENODEV; 1041 goto out; 1042 } 1043 1044 if (ct->state == XE_GUC_CT_STATE_STOPPED || xe_gt_recovery_pending(gt)) { 1045 ret = -ECANCELED; 1046 goto out; 1047 } 1048 1049 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1050 1051 if (g2h_fence) { 1052 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; 1053 num_g2h = 1; 1054 1055 if (g2h_fence_needs_alloc(g2h_fence)) { 1056 g2h_fence->seqno = next_ct_seqno(ct, true); 1057 ret = xa_err(xa_store(&ct->fence_lookup, 1058 g2h_fence->seqno, g2h_fence, 1059 GFP_ATOMIC)); 1060 if (ret) 1061 goto out; 1062 } 1063 1064 seqno = g2h_fence->seqno; 1065 } else { 1066 seqno = next_ct_seqno(ct, false); 1067 } 1068 1069 if (g2h_len) 1070 spin_lock_irq(&ct->fast_lock); 1071 retry: 1072 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len); 1073 if (unlikely(ret)) 1074 goto out_unlock; 1075 1076 ret = h2g_write(ct, action, len, seqno, !!g2h_fence); 1077 if (unlikely(ret)) { 1078 if (ret == -EAGAIN) 1079 goto retry; 1080 goto out_unlock; 1081 } 1082 1083 __g2h_reserve_space(ct, g2h_len, num_g2h); 1084 xe_guc_notify(ct_to_guc(ct)); 1085 out_unlock: 1086 if (g2h_len) 1087 spin_unlock_irq(&ct->fast_lock); 1088 out: 1089 return ret; 1090 } 1091 1092 static void kick_reset(struct xe_guc_ct *ct) 1093 { 1094 xe_gt_reset_async(ct_to_gt(ct)); 1095 } 1096 1097 static int dequeue_one_g2h(struct xe_guc_ct *ct); 1098 1099 /* 1100 * wait before retry of sending h2g message 1101 * Return: true if ready for retry, false if the wait timeouted 1102 */ 1103 static bool guc_ct_send_wait_for_retry(struct xe_guc_ct *ct, u32 len, 1104 u32 g2h_len, struct g2h_fence *g2h_fence, 1105 unsigned int *sleep_period_ms) 1106 { 1107 struct xe_device *xe = ct_to_xe(ct); 1108 1109 /* 1110 * We wait to try to restore credits for about 1 second before bailing. 1111 * In the case of H2G credits we have no choice but just to wait for the 1112 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In 1113 * the case of G2H we process any G2H in the channel, hopefully freeing 1114 * credits as we consume the G2H messages. 1115 */ 1116 if (!h2g_has_room(ct, len + GUC_CTB_HDR_LEN)) { 1117 struct guc_ctb *h2g = &ct->ctbs.h2g; 1118 1119 if (*sleep_period_ms == 1024) 1120 return false; 1121 1122 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail, 1123 h2g->info.size, 1124 h2g->info.space, 1125 len + GUC_CTB_HDR_LEN); 1126 msleep(*sleep_period_ms); 1127 *sleep_period_ms <<= 1; 1128 } else { 1129 struct xe_device *xe = ct_to_xe(ct); 1130 struct guc_ctb *g2h = &ct->ctbs.g2h; 1131 int ret; 1132 1133 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head, 1134 desc_read(xe, g2h, tail), 1135 g2h->info.size, 1136 g2h->info.space, 1137 g2h_fence ? 1138 GUC_CTB_HXG_MSG_MAX_LEN : 1139 g2h_len); 1140 1141 #define g2h_avail(ct) \ 1142 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head) 1143 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding || 1144 g2h_avail(ct), HZ)) 1145 return false; 1146 #undef g2h_avail 1147 1148 ret = dequeue_one_g2h(ct); 1149 if (ret < 0) { 1150 if (ret != -ECANCELED) 1151 xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)", 1152 ERR_PTR(ret)); 1153 return false; 1154 } 1155 } 1156 return true; 1157 } 1158 1159 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 1160 u32 g2h_len, u32 num_g2h, 1161 struct g2h_fence *g2h_fence) 1162 { 1163 struct xe_gt *gt = ct_to_gt(ct); 1164 unsigned int sleep_period_ms = 1; 1165 int ret; 1166 1167 xe_gt_assert(gt, !g2h_len || !g2h_fence); 1168 lockdep_assert_held(&ct->lock); 1169 xe_device_assert_mem_access(ct_to_xe(ct)); 1170 1171 try_again: 1172 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, 1173 g2h_fence); 1174 1175 if (unlikely(ret == -EBUSY)) { 1176 if (!guc_ct_send_wait_for_retry(ct, len, g2h_len, g2h_fence, 1177 &sleep_period_ms)) 1178 goto broken; 1179 goto try_again; 1180 } 1181 1182 return ret; 1183 1184 broken: 1185 xe_gt_err(gt, "No forward process on H2G, reset required\n"); 1186 CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK); 1187 1188 return -EDEADLK; 1189 } 1190 1191 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 1192 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) 1193 { 1194 int ret; 1195 1196 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence); 1197 1198 mutex_lock(&ct->lock); 1199 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence); 1200 mutex_unlock(&ct->lock); 1201 1202 return ret; 1203 } 1204 1205 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 1206 u32 g2h_len, u32 num_g2h) 1207 { 1208 int ret; 1209 1210 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL); 1211 if (ret == -EDEADLK) 1212 kick_reset(ct); 1213 1214 return ret; 1215 } 1216 1217 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 1218 u32 g2h_len, u32 num_g2h) 1219 { 1220 int ret; 1221 1222 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL); 1223 if (ret == -EDEADLK) 1224 kick_reset(ct); 1225 1226 return ret; 1227 } 1228 1229 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len) 1230 { 1231 int ret; 1232 1233 lockdep_assert_held(&ct->lock); 1234 1235 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL); 1236 if (ret == -EDEADLK) 1237 kick_reset(ct); 1238 1239 return ret; 1240 } 1241 1242 /* 1243 * Check if a GT reset is in progress or will occur and if GT reset brought the 1244 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset. 1245 */ 1246 static bool retry_failure(struct xe_guc_ct *ct, int ret) 1247 { 1248 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV)) 1249 return false; 1250 1251 #define ct_alive(ct) \ 1252 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \ 1253 !ct->ctbs.g2h.info.broken) 1254 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) 1255 return false; 1256 #undef ct_alive 1257 1258 return true; 1259 } 1260 1261 #define GUC_SEND_RETRY_LIMIT 50 1262 #define GUC_SEND_RETRY_MSLEEP 5 1263 1264 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1265 u32 *response_buffer, bool no_fail) 1266 { 1267 struct xe_gt *gt = ct_to_gt(ct); 1268 struct g2h_fence g2h_fence; 1269 unsigned int retries = 0; 1270 int ret = 0; 1271 1272 /* 1273 * We use a fence to implement blocking sends / receiving response data. 1274 * The seqno of the fence is sent in the H2G, returned in the G2H, and 1275 * an xarray is used as storage media with the seqno being to key. 1276 * Fields in the fence hold success, failure, retry status and the 1277 * response data. Safe to allocate on the stack as the xarray is the 1278 * only reference and it cannot be present after this function exits. 1279 */ 1280 retry: 1281 g2h_fence_init(&g2h_fence, response_buffer); 1282 retry_same_fence: 1283 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence); 1284 if (unlikely(ret == -ENOMEM)) { 1285 /* Retry allocation /w GFP_KERNEL */ 1286 ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno, 1287 &g2h_fence, GFP_KERNEL)); 1288 if (ret) 1289 return ret; 1290 1291 goto retry_same_fence; 1292 } else if (unlikely(ret)) { 1293 if (ret == -EDEADLK) 1294 kick_reset(ct); 1295 1296 if (no_fail && retry_failure(ct, ret)) 1297 goto retry_same_fence; 1298 1299 if (!g2h_fence_needs_alloc(&g2h_fence)) 1300 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1301 1302 return ret; 1303 } 1304 1305 /* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response 1306 * and g2h_fence_cancel. 1307 */ 1308 ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ); 1309 if (!ret) { 1310 LNL_FLUSH_WORK(&ct->g2h_worker); 1311 if (READ_ONCE(g2h_fence.done)) { 1312 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", 1313 g2h_fence.seqno, action[0]); 1314 ret = 1; 1315 } 1316 } 1317 1318 /* 1319 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on 1320 * the stack, since we have no clue if it will fire after the timeout before we can erase 1321 * from the xa. Also we have some dependent loads and stores below for which we need the 1322 * correct ordering, and we lack the needed barriers. 1323 */ 1324 mutex_lock(&ct->lock); 1325 if (!ret) { 1326 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s", 1327 g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done)); 1328 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1329 mutex_unlock(&ct->lock); 1330 return -ETIME; 1331 } 1332 1333 if (g2h_fence.retry) { 1334 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", 1335 action[0], g2h_fence.reason); 1336 mutex_unlock(&ct->lock); 1337 if (++retries > GUC_SEND_RETRY_LIMIT) { 1338 xe_gt_err(gt, "H2G action %#x reached retry limit=%u, aborting\n", 1339 action[0], GUC_SEND_RETRY_LIMIT); 1340 return -ELOOP; 1341 } 1342 msleep(GUC_SEND_RETRY_MSLEEP * retries); 1343 goto retry; 1344 } 1345 if (g2h_fence.fail) { 1346 if (g2h_fence.cancel) { 1347 xe_gt_dbg(gt, "H2G request %#x canceled!\n", action[0]); 1348 ret = -ECANCELED; 1349 goto unlock; 1350 } 1351 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n", 1352 action[0], g2h_fence.error, g2h_fence.hint); 1353 ret = -EIO; 1354 } 1355 1356 if (ret > 0) 1357 ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data; 1358 1359 unlock: 1360 mutex_unlock(&ct->lock); 1361 1362 return ret; 1363 } 1364 1365 /** 1366 * xe_guc_ct_send_recv - Send and receive HXG to the GuC 1367 * @ct: the &xe_guc_ct 1368 * @action: the dword array with `HXG Request`_ message (can't be NULL) 1369 * @len: length of the `HXG Request`_ message (in dwords, can't be 0) 1370 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL) 1371 * 1372 * Send a `HXG Request`_ message to the GuC over CT communication channel and 1373 * blocks until GuC replies with a `HXG Response`_ message. 1374 * 1375 * For non-blocking communication with GuC use xe_guc_ct_send(). 1376 * 1377 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_. 1378 * 1379 * Return: response length (in dwords) if &response_buffer was not NULL, or 1380 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or 1381 * a negative error code on failure. 1382 */ 1383 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1384 u32 *response_buffer) 1385 { 1386 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer); 1387 return guc_ct_send_recv(ct, action, len, response_buffer, false); 1388 } 1389 ALLOW_ERROR_INJECTION(xe_guc_ct_send_recv, ERRNO); 1390 1391 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, 1392 u32 len, u32 *response_buffer) 1393 { 1394 return guc_ct_send_recv(ct, action, len, response_buffer, true); 1395 } 1396 1397 static u32 *msg_to_hxg(u32 *msg) 1398 { 1399 return msg + GUC_CTB_MSG_MIN_LEN; 1400 } 1401 1402 static u32 msg_len_to_hxg_len(u32 len) 1403 { 1404 return len - GUC_CTB_MSG_MIN_LEN; 1405 } 1406 1407 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) 1408 { 1409 u32 *hxg = msg_to_hxg(msg); 1410 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1411 1412 lockdep_assert_held(&ct->lock); 1413 1414 switch (action) { 1415 case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: 1416 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1417 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1418 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1419 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1420 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1421 g2h_release_space(ct, len); 1422 } 1423 1424 return 0; 1425 } 1426 1427 static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) 1428 { 1429 struct xe_gt *gt = ct_to_gt(ct); 1430 1431 if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED) 1432 xe_gt_err(gt, "GuC Crash dump notification\n"); 1433 else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION) 1434 xe_gt_err(gt, "GuC Exception notification\n"); 1435 else 1436 xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action); 1437 1438 CT_DEAD(ct, NULL, CRASH); 1439 1440 kick_reset(ct); 1441 1442 return 0; 1443 } 1444 1445 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) 1446 { 1447 struct xe_gt *gt = ct_to_gt(ct); 1448 u32 *hxg = msg_to_hxg(msg); 1449 u32 hxg_len = msg_len_to_hxg_len(len); 1450 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); 1451 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1452 struct g2h_fence *g2h_fence; 1453 1454 lockdep_assert_held(&ct->lock); 1455 1456 /* 1457 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup. 1458 * Those messages should never fail, so if we do get an error back it 1459 * means we're likely doing an illegal operation and the GuC is 1460 * rejecting it. We have no way to inform the code that submitted the 1461 * H2G that the message was rejected, so we need to escalate the 1462 * failure to trigger a reset. 1463 */ 1464 if (fence & CT_SEQNO_UNTRACKED) { 1465 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) 1466 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n", 1467 fence, 1468 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]), 1469 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0])); 1470 else 1471 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n", 1472 type, fence); 1473 1474 fast_req_report(ct, fence); 1475 1476 /* FIXME: W/A race in the GuC, will get in firmware soon */ 1477 if (xe_gt_recovery_pending(gt)) 1478 return 0; 1479 1480 CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE); 1481 1482 return -EPROTO; 1483 } 1484 1485 g2h_fence = xa_erase(&ct->fence_lookup, fence); 1486 if (unlikely(!g2h_fence)) { 1487 /* Don't tear down channel, as send could've timed out */ 1488 /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ 1489 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence); 1490 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1491 return 0; 1492 } 1493 1494 xe_gt_assert(gt, fence == g2h_fence->seqno); 1495 1496 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { 1497 g2h_fence->fail = true; 1498 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); 1499 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]); 1500 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 1501 g2h_fence->retry = true; 1502 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); 1503 } else if (g2h_fence->response_buffer) { 1504 g2h_fence->response_len = hxg_len; 1505 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); 1506 } else { 1507 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); 1508 } 1509 1510 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1511 1512 /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ 1513 WRITE_ONCE(g2h_fence->done, true); 1514 smp_mb(); 1515 1516 wake_up_all(&ct->g2h_fence_wq); 1517 1518 return 0; 1519 } 1520 1521 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1522 { 1523 struct xe_gt *gt = ct_to_gt(ct); 1524 u32 *hxg = msg_to_hxg(msg); 1525 u32 origin, type; 1526 int ret; 1527 1528 lockdep_assert_held(&ct->lock); 1529 1530 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1531 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1532 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n", 1533 origin); 1534 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN); 1535 1536 return -EPROTO; 1537 } 1538 1539 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1540 switch (type) { 1541 case GUC_HXG_TYPE_EVENT: 1542 ret = parse_g2h_event(ct, msg, len); 1543 break; 1544 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1545 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1546 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1547 ret = parse_g2h_response(ct, msg, len); 1548 break; 1549 default: 1550 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n", 1551 type); 1552 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE); 1553 1554 ret = -EOPNOTSUPP; 1555 } 1556 1557 return ret; 1558 } 1559 1560 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1561 { 1562 struct xe_guc *guc = ct_to_guc(ct); 1563 struct xe_gt *gt = ct_to_gt(ct); 1564 u32 hxg_len = msg_len_to_hxg_len(len); 1565 u32 *hxg = msg_to_hxg(msg); 1566 u32 action, adj_len; 1567 u32 *payload; 1568 int ret = 0; 1569 1570 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1571 return 0; 1572 1573 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1574 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN; 1575 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN; 1576 1577 switch (action) { 1578 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1579 ret = xe_guc_sched_done_handler(guc, payload, adj_len); 1580 break; 1581 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1582 ret = xe_guc_deregister_done_handler(guc, payload, adj_len); 1583 break; 1584 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1585 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len); 1586 break; 1587 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1588 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload, 1589 adj_len); 1590 break; 1591 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1592 /* Selftest only at the moment */ 1593 break; 1594 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1595 ret = xe_guc_error_capture_handler(guc, payload, adj_len); 1596 break; 1597 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1598 /* FIXME: Handle this */ 1599 break; 1600 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: 1601 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload, 1602 adj_len); 1603 break; 1604 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1605 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1606 break; 1607 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1608 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1609 /* 1610 * Page reclamation is an extension of TLB invalidation. Both 1611 * operations share the same seqno and fence. When either 1612 * action completes, we need to signal the corresponding 1613 * fence. Since the handling logic (lookup fence by seqno, 1614 * fence signalling) is identical, we use the same handler 1615 * for both G2H events. 1616 */ 1617 ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); 1618 break; 1619 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: 1620 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len); 1621 break; 1622 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF: 1623 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len); 1624 break; 1625 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY: 1626 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len); 1627 break; 1628 case GUC_ACTION_GUC2PF_ADVERSE_EVENT: 1629 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len); 1630 break; 1631 case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1632 case XE_GUC_ACTION_NOTIFY_EXCEPTION: 1633 ret = guc_crash_process_msg(ct, action); 1634 break; 1635 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1636 case XE_GUC_ACTION_TEST_G2G_RECV: 1637 ret = xe_guc_g2g_test_notification(guc, payload, adj_len); 1638 break; 1639 #endif 1640 case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: 1641 ret = xe_guc_exec_queue_cgp_sync_done_handler(guc, payload, adj_len); 1642 break; 1643 case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR: 1644 ret = xe_guc_exec_queue_cgp_context_error_handler(guc, payload, 1645 adj_len); 1646 break; 1647 default: 1648 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); 1649 } 1650 1651 if (ret) { 1652 xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n", 1653 action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg); 1654 CT_DEAD(ct, NULL, PROCESS_FAILED); 1655 } 1656 1657 return 0; 1658 } 1659 1660 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) 1661 { 1662 struct xe_device *xe = ct_to_xe(ct); 1663 struct xe_gt *gt = ct_to_gt(ct); 1664 struct guc_ctb *g2h = &ct->ctbs.g2h; 1665 u32 tail, head, len, desc_status; 1666 s32 avail; 1667 u32 action; 1668 u32 *hxg; 1669 1670 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 1671 lockdep_assert_held(&ct->fast_lock); 1672 1673 if (ct->state == XE_GUC_CT_STATE_DISABLED) 1674 return -ENODEV; 1675 1676 if (ct->state == XE_GUC_CT_STATE_STOPPED) 1677 return -ECANCELED; 1678 1679 if (g2h->info.broken) 1680 return -EPIPE; 1681 1682 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1683 1684 desc_status = desc_read(xe, g2h, status); 1685 if (desc_status) { 1686 if (desc_status & GUC_CTB_STATUS_DISABLED) { 1687 /* 1688 * Potentially valid if a CLIENT_RESET request resulted in 1689 * contexts/engines being reset. But should never happen as 1690 * no contexts should be active when CLIENT_RESET is sent. 1691 */ 1692 xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n"); 1693 desc_status &= ~GUC_CTB_STATUS_DISABLED; 1694 } 1695 1696 if (desc_status) { 1697 xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status); 1698 goto corrupted; 1699 } 1700 } 1701 1702 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 1703 u32 desc_tail = desc_read(xe, g2h, tail); 1704 /* 1705 u32 desc_head = desc_read(xe, g2h, head); 1706 1707 * info.head and desc_head are updated back-to-back at the end of 1708 * this function and nowhere else. Hence, they cannot be different 1709 * unless two g2h_read calls are running concurrently. Which is not 1710 * possible because it is guarded by ct->fast_lock. And yet, some 1711 * discrete platforms are regularly hitting this error :(. 1712 * 1713 * desc_head rolling backwards shouldn't cause any noticeable 1714 * problems - just a delay in GuC being allowed to proceed past that 1715 * point in the queue. So for now, just disable the error until it 1716 * can be root caused. 1717 * 1718 if (g2h->info.head != desc_head) { 1719 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH); 1720 xe_gt_err(gt, "CT read: head was modified %u != %u\n", 1721 desc_head, g2h->info.head); 1722 goto corrupted; 1723 } 1724 */ 1725 1726 if (g2h->info.head > g2h->info.size) { 1727 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1728 xe_gt_err(gt, "CT read: head out of range: %u vs %u\n", 1729 g2h->info.head, g2h->info.size); 1730 goto corrupted; 1731 } 1732 1733 if (desc_tail >= g2h->info.size) { 1734 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1735 xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n", 1736 desc_tail, g2h->info.size); 1737 goto corrupted; 1738 } 1739 } 1740 1741 /* Calculate DW available to read */ 1742 tail = desc_read(xe, g2h, tail); 1743 avail = tail - g2h->info.head; 1744 if (unlikely(avail == 0)) 1745 return 0; 1746 1747 if (avail < 0) 1748 avail += g2h->info.size; 1749 1750 /* Read header */ 1751 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head, 1752 sizeof(u32)); 1753 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; 1754 if (len > avail) { 1755 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n", 1756 avail, len); 1757 goto corrupted; 1758 } 1759 1760 head = (g2h->info.head + 1) % g2h->info.size; 1761 avail = len - 1; 1762 1763 /* Read G2H message */ 1764 if (avail + head > g2h->info.size) { 1765 u32 avail_til_wrap = g2h->info.size - head; 1766 1767 xe_map_memcpy_from(xe, msg + 1, 1768 &g2h->cmds, sizeof(u32) * head, 1769 avail_til_wrap * sizeof(u32)); 1770 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap, 1771 &g2h->cmds, 0, 1772 (avail - avail_til_wrap) * sizeof(u32)); 1773 } else { 1774 xe_map_memcpy_from(xe, msg + 1, 1775 &g2h->cmds, sizeof(u32) * head, 1776 avail * sizeof(u32)); 1777 } 1778 1779 hxg = msg_to_hxg(msg); 1780 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1781 1782 if (fast_path) { 1783 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1784 return 0; 1785 1786 switch (action) { 1787 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1788 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1789 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1790 break; /* Process these in fast-path */ 1791 default: 1792 return 0; 1793 } 1794 } 1795 1796 /* Update local / descriptor header */ 1797 g2h->info.head = (head + avail) % g2h->info.size; 1798 desc_write(xe, g2h, head, g2h->info.head); 1799 1800 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id, 1801 action, len, g2h->info.head, tail); 1802 1803 return len; 1804 1805 corrupted: 1806 CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ); 1807 return -EPROTO; 1808 } 1809 1810 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) 1811 { 1812 struct xe_gt *gt = ct_to_gt(ct); 1813 struct xe_guc *guc = ct_to_guc(ct); 1814 u32 hxg_len = msg_len_to_hxg_len(len); 1815 u32 *hxg = msg_to_hxg(msg); 1816 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1817 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN; 1818 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1819 int ret = 0; 1820 1821 switch (action) { 1822 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1823 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1824 break; 1825 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1826 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1827 /* 1828 * Seqno and fence handling of page reclamation and TLB 1829 * invalidation is identical, so we can use the same handler 1830 * for both actions. 1831 */ 1832 __g2h_release_space(ct, len); 1833 ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); 1834 break; 1835 default: 1836 xe_gt_warn(gt, "NOT_POSSIBLE"); 1837 } 1838 1839 if (ret) { 1840 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n", 1841 action, ERR_PTR(ret)); 1842 CT_DEAD(ct, NULL, FAST_G2H); 1843 } 1844 } 1845 1846 /** 1847 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler 1848 * @ct: GuC CT object 1849 * 1850 * Anything related to page faults is critical for performance, process these 1851 * critical G2H in the IRQ. This is safe as these handlers either just wake up 1852 * waiters or queue another worker. 1853 */ 1854 void xe_guc_ct_fast_path(struct xe_guc_ct *ct) 1855 { 1856 struct xe_device *xe = ct_to_xe(ct); 1857 bool ongoing; 1858 int len; 1859 1860 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1861 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1862 return; 1863 1864 spin_lock(&ct->fast_lock); 1865 do { 1866 len = g2h_read(ct, ct->fast_msg, true); 1867 if (len > 0) 1868 g2h_fast_path(ct, ct->fast_msg, len); 1869 } while (len > 0); 1870 spin_unlock(&ct->fast_lock); 1871 1872 if (ongoing) 1873 xe_pm_runtime_put(xe); 1874 } 1875 1876 /* Returns less than zero on error, 0 on done, 1 on more available */ 1877 static int dequeue_one_g2h(struct xe_guc_ct *ct) 1878 { 1879 int len; 1880 int ret; 1881 1882 lockdep_assert_held(&ct->lock); 1883 1884 spin_lock_irq(&ct->fast_lock); 1885 len = g2h_read(ct, ct->msg, false); 1886 spin_unlock_irq(&ct->fast_lock); 1887 if (len <= 0) 1888 return len; 1889 1890 ret = parse_g2h_msg(ct, ct->msg, len); 1891 if (unlikely(ret < 0)) 1892 return ret; 1893 1894 ret = process_g2h_msg(ct, ct->msg, len); 1895 if (unlikely(ret < 0)) 1896 return ret; 1897 1898 return 1; 1899 } 1900 1901 static void receive_g2h(struct xe_guc_ct *ct) 1902 { 1903 bool ongoing; 1904 int ret; 1905 1906 /* 1907 * Normal users must always hold mem_access.ref around CT calls. However 1908 * during the runtime pm callbacks we rely on CT to talk to the GuC, but 1909 * at this stage we can't rely on mem_access.ref and even the 1910 * callback_task will be different than current. For such cases we just 1911 * need to ensure we always process the responses from any blocking 1912 * ct_send requests or where we otherwise expect some response when 1913 * initiated from those callbacks (which will need to wait for the below 1914 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if 1915 * the device has suspended to the point that the CT communication has 1916 * been disabled. 1917 * 1918 * If we are inside the runtime pm callback, we can be the only task 1919 * still issuing CT requests (since that requires having the 1920 * mem_access.ref). It seems like it might in theory be possible to 1921 * receive unsolicited events from the GuC just as we are 1922 * suspending-resuming, but those will currently anyway be lost when 1923 * eventually exiting from suspend, hence no need to wake up the device 1924 * here. If we ever need something stronger than get_if_ongoing() then 1925 * we need to be careful with blocking the pm callbacks from getting CT 1926 * responses, if the worker here is blocked on those callbacks 1927 * completing, creating a deadlock. 1928 */ 1929 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1930 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1931 return; 1932 1933 do { 1934 mutex_lock(&ct->lock); 1935 ret = dequeue_one_g2h(ct); 1936 mutex_unlock(&ct->lock); 1937 1938 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) { 1939 xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d", ret); 1940 CT_DEAD(ct, NULL, G2H_RECV); 1941 kick_reset(ct); 1942 } 1943 } while (ret == 1); 1944 1945 if (ongoing) 1946 xe_pm_runtime_put(ct_to_xe(ct)); 1947 } 1948 1949 static void g2h_worker_func(struct work_struct *w) 1950 { 1951 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker); 1952 1953 receive_g2h(ct); 1954 } 1955 1956 static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic, 1957 bool want_ctb) 1958 { 1959 struct xe_guc_ct_snapshot *snapshot; 1960 1961 snapshot = kzalloc_obj(*snapshot, atomic ? GFP_ATOMIC : GFP_KERNEL); 1962 if (!snapshot) 1963 return NULL; 1964 1965 if (ct->bo && want_ctb) { 1966 snapshot->ctb_size = xe_bo_size(ct->bo); 1967 snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL); 1968 } 1969 1970 return snapshot; 1971 } 1972 1973 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb, 1974 struct guc_ctb_snapshot *snapshot) 1975 { 1976 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0, 1977 sizeof(struct guc_ct_buffer_desc)); 1978 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info)); 1979 } 1980 1981 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot, 1982 struct drm_printer *p) 1983 { 1984 drm_printf(p, "\tsize: %d\n", snapshot->info.size); 1985 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space); 1986 drm_printf(p, "\thead: %d\n", snapshot->info.head); 1987 drm_printf(p, "\ttail: %d\n", snapshot->info.tail); 1988 drm_printf(p, "\tspace: %d\n", snapshot->info.space); 1989 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken); 1990 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head); 1991 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail); 1992 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status); 1993 } 1994 1995 static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic, 1996 bool want_ctb) 1997 { 1998 struct xe_device *xe = ct_to_xe(ct); 1999 struct xe_guc_ct_snapshot *snapshot; 2000 2001 snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb); 2002 if (!snapshot) { 2003 xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n"); 2004 return NULL; 2005 } 2006 2007 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) { 2008 snapshot->ct_enabled = true; 2009 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding); 2010 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g); 2011 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h); 2012 } 2013 2014 if (ct->bo && snapshot->ctb) 2015 xe_map_memcpy_from(xe, snapshot->ctb, &ct->bo->vmap, 0, snapshot->ctb_size); 2016 2017 return snapshot; 2018 } 2019 2020 /** 2021 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state. 2022 * @ct: GuC CT object. 2023 * 2024 * This can be printed out in a later stage like during dev_coredump 2025 * analysis. This is safe to be called during atomic context. 2026 * 2027 * Returns: a GuC CT snapshot object that must be freed by the caller 2028 * by using `xe_guc_ct_snapshot_free`. 2029 */ 2030 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct) 2031 { 2032 return guc_ct_snapshot_capture(ct, true, true); 2033 } 2034 2035 /** 2036 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot. 2037 * @snapshot: GuC CT snapshot object. 2038 * @p: drm_printer where it will be printed out. 2039 * 2040 * This function prints out a given GuC CT snapshot object. 2041 */ 2042 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, 2043 struct drm_printer *p) 2044 { 2045 if (!snapshot) 2046 return; 2047 2048 if (snapshot->ct_enabled) { 2049 drm_puts(p, "H2G CTB (all sizes in DW):\n"); 2050 guc_ctb_snapshot_print(&snapshot->h2g, p); 2051 2052 drm_puts(p, "G2H CTB (all sizes in DW):\n"); 2053 guc_ctb_snapshot_print(&snapshot->g2h, p); 2054 drm_printf(p, "\tg2h outstanding: %d\n", 2055 snapshot->g2h_outstanding); 2056 2057 if (snapshot->ctb) { 2058 drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size); 2059 xe_print_blob_ascii85(p, "[CTB].data", '\n', 2060 snapshot->ctb, 0, snapshot->ctb_size); 2061 } 2062 } else { 2063 drm_puts(p, "CT disabled\n"); 2064 } 2065 } 2066 2067 /** 2068 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot. 2069 * @snapshot: GuC CT snapshot object. 2070 * 2071 * This function free all the memory that needed to be allocated at capture 2072 * time. 2073 */ 2074 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot) 2075 { 2076 if (!snapshot) 2077 return; 2078 2079 kfree(snapshot->ctb); 2080 kfree(snapshot); 2081 } 2082 2083 /** 2084 * xe_guc_ct_print - GuC CT Print. 2085 * @ct: GuC CT. 2086 * @p: drm_printer where it will be printed out. 2087 * @want_ctb: Should the full CTB content be dumped (vs just the headers) 2088 * 2089 * This function will quickly capture a snapshot of the CT state 2090 * and immediately print it out. 2091 */ 2092 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb) 2093 { 2094 struct xe_guc_ct_snapshot *snapshot; 2095 2096 snapshot = guc_ct_snapshot_capture(ct, false, want_ctb); 2097 xe_guc_ct_snapshot_print(snapshot, p); 2098 xe_guc_ct_snapshot_free(snapshot); 2099 } 2100 2101 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 2102 2103 #ifdef CONFIG_FUNCTION_ERROR_INJECTION 2104 /* 2105 * This is a helper function which assists the driver in identifying if a fault 2106 * injection test is currently active, allowing it to reduce unnecessary debug 2107 * output. Typically, the function returns zero, but the fault injection 2108 * framework can alter this to return an error. Since faults are injected 2109 * through this function, it's important to ensure the compiler doesn't optimize 2110 * it into an inline function. To avoid such optimization, the 'noinline' 2111 * attribute is applied. Compiler optimizes the static function defined in the 2112 * header file as an inline function. 2113 */ 2114 noinline int xe_is_injection_active(void) { return 0; } 2115 ALLOW_ERROR_INJECTION(xe_is_injection_active, ERRNO); 2116 #else 2117 int xe_is_injection_active(void) { return 0; } 2118 #endif 2119 2120 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code) 2121 { 2122 struct xe_guc_log_snapshot *snapshot_log; 2123 struct xe_guc_ct_snapshot *snapshot_ct; 2124 struct xe_guc *guc = ct_to_guc(ct); 2125 unsigned long flags; 2126 bool have_capture; 2127 2128 if (ctb) 2129 ctb->info.broken = true; 2130 /* 2131 * Huge dump is getting generated when injecting error for guc CT/MMIO 2132 * functions. So, let us suppress the dump when fault is injected. 2133 */ 2134 if (xe_is_injection_active()) 2135 return; 2136 2137 /* Ignore further errors after the first dump until a reset */ 2138 if (ct->dead.reported) 2139 return; 2140 2141 spin_lock_irqsave(&ct->dead.lock, flags); 2142 2143 /* And only capture one dump at a time */ 2144 have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE); 2145 ct->dead.reason |= (1 << reason_code) | 2146 (1 << CT_DEAD_STATE_CAPTURE); 2147 2148 spin_unlock_irqrestore(&ct->dead.lock, flags); 2149 2150 if (have_capture) 2151 return; 2152 2153 snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true); 2154 snapshot_ct = xe_guc_ct_snapshot_capture((ct)); 2155 2156 spin_lock_irqsave(&ct->dead.lock, flags); 2157 2158 if (ct->dead.snapshot_log || ct->dead.snapshot_ct) { 2159 xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n"); 2160 xe_guc_log_snapshot_free(snapshot_log); 2161 xe_guc_ct_snapshot_free(snapshot_ct); 2162 } else { 2163 ct->dead.snapshot_log = snapshot_log; 2164 ct->dead.snapshot_ct = snapshot_ct; 2165 } 2166 2167 spin_unlock_irqrestore(&ct->dead.lock, flags); 2168 2169 queue_work(system_unbound_wq, &(ct)->dead.worker); 2170 } 2171 2172 static void ct_dead_print(struct xe_dead_ct *dead) 2173 { 2174 struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead); 2175 struct xe_device *xe = ct_to_xe(ct); 2176 struct xe_gt *gt = ct_to_gt(ct); 2177 static int g_count; 2178 struct drm_printer ip = xe_gt_info_printer(gt); 2179 struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count); 2180 2181 if (!dead->reason) { 2182 xe_gt_err(gt, "CTB is dead for no reason!?\n"); 2183 return; 2184 } 2185 2186 /* Can't generate a genuine core dump at this point, so just do the good bits */ 2187 drm_puts(&lp, "**** Xe Device Coredump ****\n"); 2188 drm_printf(&lp, "Reason: CTB is dead - 0x%X\n", dead->reason); 2189 xe_device_snapshot_print(xe, &lp); 2190 2191 drm_printf(&lp, "**** GT #%d ****\n", gt->info.id); 2192 drm_printf(&lp, "\tTile: %d\n", gt->tile->id); 2193 2194 drm_puts(&lp, "**** GuC Log ****\n"); 2195 xe_guc_log_snapshot_print(dead->snapshot_log, &lp); 2196 2197 drm_puts(&lp, "**** GuC CT ****\n"); 2198 xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp); 2199 2200 drm_puts(&lp, "Done.\n"); 2201 } 2202 2203 static void ct_dead_worker_func(struct work_struct *w) 2204 { 2205 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker); 2206 2207 if (!ct->dead.reported) { 2208 ct->dead.reported = true; 2209 ct_dead_print(&ct->dead); 2210 } 2211 2212 spin_lock_irq(&ct->dead.lock); 2213 2214 xe_guc_log_snapshot_free(ct->dead.snapshot_log); 2215 ct->dead.snapshot_log = NULL; 2216 xe_guc_ct_snapshot_free(ct->dead.snapshot_ct); 2217 ct->dead.snapshot_ct = NULL; 2218 2219 if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) { 2220 /* A reset has occurred so re-arm the error reporting */ 2221 ct->dead.reason = 0; 2222 ct->dead.reported = false; 2223 } 2224 2225 spin_unlock_irq(&ct->dead.lock); 2226 } 2227 #endif 2228