1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ct.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 #include <linux/delay.h> 11 #include <linux/fault-inject.h> 12 13 #include <kunit/static_stub.h> 14 15 #include <drm/drm_managed.h> 16 17 #include "abi/guc_actions_abi.h" 18 #include "abi/guc_actions_sriov_abi.h" 19 #include "abi/guc_klvs_abi.h" 20 #include "xe_bo.h" 21 #include "xe_devcoredump.h" 22 #include "xe_device.h" 23 #include "xe_gt.h" 24 #include "xe_gt_printk.h" 25 #include "xe_gt_sriov_pf_control.h" 26 #include "xe_gt_sriov_pf_monitor.h" 27 #include "xe_guc.h" 28 #include "xe_guc_log.h" 29 #include "xe_guc_pagefault.h" 30 #include "xe_guc_relay.h" 31 #include "xe_guc_submit.h" 32 #include "xe_guc_tlb_inval.h" 33 #include "xe_map.h" 34 #include "xe_pm.h" 35 #include "xe_sleep.h" 36 #include "xe_sriov_vf.h" 37 #include "xe_trace_guc.h" 38 39 static void receive_g2h(struct xe_guc_ct *ct); 40 static void g2h_worker_func(struct work_struct *w); 41 static void safe_mode_worker_func(struct work_struct *w); 42 static void ct_exit_safe_mode(struct xe_guc_ct *ct); 43 static void guc_ct_change_state(struct xe_guc_ct *ct, 44 enum xe_guc_ct_state state); 45 46 static struct xe_guc *ct_to_guc(struct xe_guc_ct *ct) 47 { 48 return container_of(ct, struct xe_guc, ct); 49 } 50 51 static struct xe_gt *ct_to_gt(struct xe_guc_ct *ct) 52 { 53 return container_of(ct, struct xe_gt, uc.guc.ct); 54 } 55 56 static struct xe_device *ct_to_xe(struct xe_guc_ct *ct) 57 { 58 return gt_to_xe(ct_to_gt(ct)); 59 } 60 61 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 62 enum { 63 /* Internal states, not error conditions */ 64 CT_DEAD_STATE_REARM, /* 0x0001 */ 65 CT_DEAD_STATE_CAPTURE, /* 0x0002 */ 66 67 /* Error conditions */ 68 CT_DEAD_SETUP, /* 0x0004 */ 69 CT_DEAD_H2G_WRITE, /* 0x0008 */ 70 CT_DEAD_H2G_HAS_ROOM, /* 0x0010 */ 71 CT_DEAD_G2H_READ, /* 0x0020 */ 72 CT_DEAD_G2H_RECV, /* 0x0040 */ 73 CT_DEAD_G2H_RELEASE, /* 0x0080 */ 74 CT_DEAD_DEADLOCK, /* 0x0100 */ 75 CT_DEAD_PROCESS_FAILED, /* 0x0200 */ 76 CT_DEAD_FAST_G2H, /* 0x0400 */ 77 CT_DEAD_PARSE_G2H_RESPONSE, /* 0x0800 */ 78 CT_DEAD_PARSE_G2H_UNKNOWN, /* 0x1000 */ 79 CT_DEAD_PARSE_G2H_ORIGIN, /* 0x2000 */ 80 CT_DEAD_PARSE_G2H_TYPE, /* 0x4000 */ 81 CT_DEAD_CRASH, /* 0x8000 */ 82 }; 83 84 static void ct_dead_worker_func(struct work_struct *w); 85 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code); 86 87 static void ct_dead_fini(struct xe_guc_ct *ct) 88 { 89 cancel_work_sync(&ct->dead.worker); 90 } 91 92 static void ct_dead_init(struct xe_guc_ct *ct) 93 { 94 spin_lock_init(&ct->dead.lock); 95 INIT_WORK(&ct->dead.worker, ct_dead_worker_func); 96 97 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 98 stack_depot_init(); 99 #endif 100 } 101 102 static void fast_req_stack_save(struct xe_guc_ct *ct, unsigned int slot) 103 { 104 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 105 unsigned long entries[SZ_32]; 106 unsigned int n; 107 108 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1); 109 /* May be called under spinlock, so avoid sleeping */ 110 ct->fast_req[slot].stack = stack_depot_save(entries, n, GFP_NOWAIT); 111 #endif 112 } 113 114 static void fast_req_dump(struct xe_guc_ct *ct, u16 fence, unsigned int slot) 115 { 116 struct xe_gt *gt = ct_to_gt(ct); 117 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) 118 char *buf __cleanup(kfree) = kmalloc(SZ_4K, GFP_NOWAIT); 119 120 if (buf && stack_depot_snprint(ct->fast_req[slot].stack, buf, SZ_4K, 0)) 121 xe_gt_err(gt, "Fence 0x%x was used by action %#04x sent at:\n%s\n", 122 fence, ct->fast_req[slot].action, buf); 123 else 124 xe_gt_err(gt, "Fence 0x%x was used by action %#04x [failed to retrieve stack]\n", 125 fence, ct->fast_req[slot].action); 126 #else 127 xe_gt_err(gt, "Fence 0x%x was used by action %#04x\n", 128 fence, ct->fast_req[slot].action); 129 #endif 130 } 131 132 static void fast_req_report(struct xe_guc_ct *ct, u16 fence) 133 { 134 u16 fence_min = U16_MAX, fence_max = 0; 135 struct xe_gt *gt = ct_to_gt(ct); 136 unsigned int n; 137 138 lockdep_assert_held(&ct->lock); 139 140 for (n = 0; n < ARRAY_SIZE(ct->fast_req); n++) { 141 if (ct->fast_req[n].fence < fence_min) 142 fence_min = ct->fast_req[n].fence; 143 if (ct->fast_req[n].fence > fence_max) 144 fence_max = ct->fast_req[n].fence; 145 146 if (ct->fast_req[n].fence != fence) 147 continue; 148 149 return fast_req_dump(ct, fence, n); 150 } 151 152 xe_gt_warn(gt, "Fence 0x%x not found - tracking buffer wrapped? [range = 0x%x -> 0x%x, next = 0x%X]\n", 153 fence, fence_min, fence_max, ct->fence_seqno); 154 } 155 156 static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) 157 { 158 unsigned int slot = fence % ARRAY_SIZE(ct->fast_req); 159 160 fast_req_stack_save(ct, slot); 161 ct->fast_req[slot].fence = fence; 162 ct->fast_req[slot].action = action; 163 } 164 165 #define CT_DEAD(ct, ctb, reason_code) ct_dead_capture((ct), (ctb), CT_DEAD_##reason_code) 166 167 #else 168 169 static void ct_dead_fini(struct xe_guc_ct *ct) { } 170 static void ct_dead_init(struct xe_guc_ct *ct) { } 171 172 static void fast_req_report(struct xe_guc_ct *ct, u16 fence) { } 173 static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { } 174 175 #define CT_DEAD(ct, ctb, reason) \ 176 do { \ 177 struct guc_ctb *_ctb = (ctb); \ 178 if (_ctb) \ 179 _ctb->info.broken = true; \ 180 } while (0) 181 182 #endif 183 184 /* Used when a CT send wants to block and / or receive data */ 185 struct g2h_fence { 186 u32 *response_buffer; 187 u32 seqno; 188 u32 response_data; 189 u16 response_len; 190 u16 error; 191 u16 hint; 192 u16 reason; 193 bool cancel; 194 bool retry; 195 bool fail; 196 bool done; 197 }; 198 199 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) 200 { 201 memset(g2h_fence, 0, sizeof(*g2h_fence)); 202 g2h_fence->response_buffer = response_buffer; 203 g2h_fence->seqno = ~0x0; 204 } 205 206 static void g2h_fence_cancel(struct g2h_fence *g2h_fence) 207 { 208 g2h_fence->cancel = true; 209 g2h_fence->fail = true; 210 211 /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ 212 WRITE_ONCE(g2h_fence->done, true); 213 } 214 215 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) 216 { 217 return g2h_fence->seqno == ~0x0; 218 } 219 220 /** 221 * DOC: GuC CTB Blob 222 * 223 * We allocate single blob to hold both CTB descriptors and buffers: 224 * 225 * +--------+-----------------------------------------------+------+ 226 * | offset | contents | size | 227 * +========+===============================================+======+ 228 * | 0x0000 | H2G CTB Descriptor (send) | | 229 * +--------+-----------------------------------------------+ 4K | 230 * | 0x0800 | G2H CTB Descriptor (g2h) | | 231 * +--------+-----------------------------------------------+------+ 232 * | 0x1000 | H2G CT Buffer (send) | n*4K | 233 * | | | | 234 * +--------+-----------------------------------------------+------+ 235 * | 0x1000 | G2H CT Buffer (g2h) | m*4K | 236 * | + n*4K | | | 237 * +--------+-----------------------------------------------+------+ 238 * 239 * Size of each ``CT Buffer`` must be multiple of 4K. 240 * We don't expect too many messages in flight at any time, unless we are 241 * using the GuC submission. In that case each request requires a minimum 242 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 243 * enough space to avoid backpressure on the driver. We increase the size 244 * of the receive buffer (relative to the send) to ensure a G2H response 245 * CTB has a landing spot. 246 * 247 * In addition to submissions, the G2H buffer needs to be able to hold 248 * enough space for recoverable page fault notifications. The number of 249 * page faults is interrupt driven and can be as much as the number of 250 * compute resources available. However, most of the actual work for these 251 * is in a separate page fault worker thread. Therefore we only need to 252 * make sure the queue has enough space to handle all of the submissions 253 * and responses and an extra buffer for incoming page faults. 254 */ 255 256 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 257 #define CTB_H2G_BUFFER_OFFSET (CTB_DESC_SIZE * 2) 258 #define CTB_G2H_BUFFER_OFFSET (CTB_DESC_SIZE * 2) 259 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 260 #define CTB_H2G_BUFFER_DWORDS (CTB_H2G_BUFFER_SIZE / sizeof(u32)) 261 #define CTB_G2H_BUFFER_SIZE (SZ_128K) 262 #define CTB_G2H_BUFFER_DWORDS (CTB_G2H_BUFFER_SIZE / sizeof(u32)) 263 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 2) 264 #define G2H_ROOM_BUFFER_DWORDS (CTB_G2H_BUFFER_DWORDS / 2) 265 266 /** 267 * xe_guc_ct_queue_proc_time_jiffies - Return maximum time to process a full 268 * CT command queue 269 * @ct: the &xe_guc_ct. Unused at this moment but will be used in the future. 270 * 271 * Observation is that a 4KiB buffer full of commands takes a little over a 272 * second to process. Use that to calculate maximum time to process a full CT 273 * command queue. 274 * 275 * Return: Maximum time to process a full CT queue in jiffies. 276 */ 277 long xe_guc_ct_queue_proc_time_jiffies(struct xe_guc_ct *ct) 278 { 279 BUILD_BUG_ON(!IS_ALIGNED(CTB_H2G_BUFFER_SIZE, SZ_4K)); 280 return (CTB_H2G_BUFFER_SIZE / SZ_4K) * HZ; 281 } 282 283 static size_t guc_h2g_size(void) 284 { 285 return CTB_H2G_BUFFER_OFFSET + CTB_H2G_BUFFER_SIZE; 286 } 287 288 static size_t guc_g2h_size(void) 289 { 290 return CTB_G2H_BUFFER_OFFSET + CTB_G2H_BUFFER_SIZE; 291 } 292 293 static void guc_ct_fini(struct drm_device *drm, void *arg) 294 { 295 struct xe_guc_ct *ct = arg; 296 297 ct_dead_fini(ct); 298 ct_exit_safe_mode(ct); 299 destroy_workqueue(ct->g2h_wq); 300 xa_destroy(&ct->fence_lookup); 301 } 302 303 static void primelockdep(struct xe_guc_ct *ct) 304 { 305 if (!IS_ENABLED(CONFIG_LOCKDEP)) 306 return; 307 308 fs_reclaim_acquire(GFP_KERNEL); 309 might_lock(&ct->lock); 310 fs_reclaim_release(GFP_KERNEL); 311 } 312 313 int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) 314 { 315 struct xe_device *xe = ct_to_xe(ct); 316 struct xe_gt *gt = ct_to_gt(ct); 317 int err; 318 319 xe_gt_assert(gt, !(guc_h2g_size() % PAGE_SIZE)); 320 xe_gt_assert(gt, !(guc_g2h_size() % PAGE_SIZE)); 321 322 err = drmm_mutex_init(&xe->drm, &ct->lock); 323 if (err) 324 return err; 325 326 primelockdep(ct); 327 328 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM); 329 if (!ct->g2h_wq) 330 return -ENOMEM; 331 332 spin_lock_init(&ct->fast_lock); 333 xa_init(&ct->fence_lookup); 334 INIT_WORK(&ct->g2h_worker, g2h_worker_func); 335 INIT_DELAYED_WORK(&ct->safe_mode_worker, safe_mode_worker_func); 336 337 ct_dead_init(ct); 338 init_waitqueue_head(&ct->wq); 339 init_waitqueue_head(&ct->g2h_fence_wq); 340 341 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); 342 if (err) 343 return err; 344 345 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED); 346 ct->state = XE_GUC_CT_STATE_DISABLED; 347 return 0; 348 } 349 ALLOW_ERROR_INJECTION(xe_guc_ct_init_noalloc, ERRNO); /* See xe_pci_probe() */ 350 351 static void guc_action_disable_ct(void *arg) 352 { 353 struct xe_guc_ct *ct = arg; 354 355 guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); 356 } 357 358 int xe_guc_ct_init(struct xe_guc_ct *ct) 359 { 360 struct xe_device *xe = ct_to_xe(ct); 361 struct xe_gt *gt = ct_to_gt(ct); 362 struct xe_tile *tile = gt_to_tile(gt); 363 struct xe_bo *bo; 364 365 bo = xe_managed_bo_create_pin_map(xe, tile, guc_h2g_size(), 366 XE_BO_FLAG_SYSTEM | 367 XE_BO_FLAG_GGTT | 368 XE_BO_FLAG_GGTT_INVALIDATE | 369 XE_BO_FLAG_PINNED_NORESTORE); 370 if (IS_ERR(bo)) 371 return PTR_ERR(bo); 372 373 ct->ctbs.h2g.bo = bo; 374 375 bo = xe_managed_bo_create_pin_map(xe, tile, guc_g2h_size(), 376 XE_BO_FLAG_SYSTEM | 377 XE_BO_FLAG_GGTT | 378 XE_BO_FLAG_GGTT_INVALIDATE | 379 XE_BO_FLAG_PINNED_NORESTORE); 380 if (IS_ERR(bo)) 381 return PTR_ERR(bo); 382 383 ct->ctbs.g2h.bo = bo; 384 385 return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct); 386 } 387 ALLOW_ERROR_INJECTION(xe_guc_ct_init, ERRNO); /* See xe_pci_probe() */ 388 389 /** 390 * xe_guc_ct_init_post_hwconfig - Reinitialize the GuC CTB in VRAM 391 * @ct: the &xe_guc_ct 392 * 393 * Allocate a new BO in VRAM and free the previous BO that was allocated 394 * in system memory (SMEM). Applicable only for DGFX products. 395 * 396 * Return: 0 on success, or a negative errno on failure. 397 */ 398 int xe_guc_ct_init_post_hwconfig(struct xe_guc_ct *ct) 399 { 400 struct xe_device *xe = ct_to_xe(ct); 401 struct xe_gt *gt = ct_to_gt(ct); 402 struct xe_tile *tile = gt_to_tile(gt); 403 int ret; 404 405 xe_assert(xe, !xe_guc_ct_enabled(ct)); 406 407 if (IS_DGFX(xe)) { 408 ret = xe_managed_bo_reinit_in_vram(xe, tile, &ct->ctbs.h2g.bo); 409 if (ret) 410 return ret; 411 } 412 413 devm_remove_action(xe->drm.dev, guc_action_disable_ct, ct); 414 return devm_add_action_or_reset(xe->drm.dev, guc_action_disable_ct, ct); 415 } 416 417 #define desc_read(xe_, guc_ctb__, field_) \ 418 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \ 419 struct guc_ct_buffer_desc, field_) 420 421 #define desc_write(xe_, guc_ctb__, field_, val_) \ 422 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \ 423 struct guc_ct_buffer_desc, field_, val_) 424 425 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, 426 struct iosys_map *map) 427 { 428 h2g->info.size = CTB_H2G_BUFFER_DWORDS; 429 h2g->info.resv_space = 0; 430 h2g->info.tail = 0; 431 h2g->info.head = 0; 432 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 433 h2g->info.size) - 434 h2g->info.resv_space; 435 h2g->info.broken = false; 436 437 h2g->desc = *map; 438 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 439 440 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_H2G_BUFFER_OFFSET); 441 } 442 443 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, 444 struct iosys_map *map) 445 { 446 g2h->info.size = CTB_G2H_BUFFER_DWORDS; 447 g2h->info.resv_space = G2H_ROOM_BUFFER_DWORDS; 448 g2h->info.head = 0; 449 g2h->info.tail = 0; 450 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head, 451 g2h->info.size) - 452 g2h->info.resv_space; 453 g2h->info.broken = false; 454 455 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE); 456 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 457 458 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_G2H_BUFFER_OFFSET); 459 } 460 461 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct) 462 { 463 struct xe_guc *guc = ct_to_guc(ct); 464 u32 desc_addr, ctb_addr, size; 465 int err; 466 467 desc_addr = xe_bo_ggtt_addr(ct->ctbs.h2g.bo); 468 ctb_addr = xe_bo_ggtt_addr(ct->ctbs.h2g.bo) + CTB_H2G_BUFFER_OFFSET; 469 size = ct->ctbs.h2g.info.size * sizeof(u32); 470 471 err = xe_guc_self_cfg64(guc, 472 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY, 473 desc_addr); 474 if (err) 475 return err; 476 477 err = xe_guc_self_cfg64(guc, 478 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY, 479 ctb_addr); 480 if (err) 481 return err; 482 483 return xe_guc_self_cfg32(guc, 484 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY, 485 size); 486 } 487 488 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct) 489 { 490 struct xe_guc *guc = ct_to_guc(ct); 491 u32 desc_addr, ctb_addr, size; 492 int err; 493 494 desc_addr = xe_bo_ggtt_addr(ct->ctbs.g2h.bo) + CTB_DESC_SIZE; 495 ctb_addr = xe_bo_ggtt_addr(ct->ctbs.g2h.bo) + CTB_G2H_BUFFER_OFFSET; 496 size = ct->ctbs.g2h.info.size * sizeof(u32); 497 498 err = xe_guc_self_cfg64(guc, 499 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 500 desc_addr); 501 if (err) 502 return err; 503 504 err = xe_guc_self_cfg64(guc, 505 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 506 ctb_addr); 507 if (err) 508 return err; 509 510 return xe_guc_self_cfg32(guc, 511 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 512 size); 513 } 514 515 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) 516 { 517 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 518 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 519 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 520 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, 521 GUC_ACTION_HOST2GUC_CONTROL_CTB), 522 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, 523 enable ? GUC_CTB_CONTROL_ENABLE : 524 GUC_CTB_CONTROL_DISABLE), 525 }; 526 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request)); 527 528 return ret > 0 ? -EPROTO : ret; 529 } 530 531 static void guc_ct_change_state(struct xe_guc_ct *ct, 532 enum xe_guc_ct_state state) 533 { 534 struct xe_gt *gt = ct_to_gt(ct); 535 struct g2h_fence *g2h_fence; 536 unsigned long idx; 537 538 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ 539 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ 540 541 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 || 542 state == XE_GUC_CT_STATE_STOPPED); 543 544 if (ct->g2h_outstanding) 545 xe_pm_runtime_put(ct_to_xe(ct)); 546 ct->g2h_outstanding = 0; 547 548 /* 549 * WRITE_ONCE pairs with READ_ONCEs in xe_guc_ct_initialized and 550 * xe_guc_ct_enabled. 551 */ 552 WRITE_ONCE(ct->state, state); 553 554 xe_gt_dbg(gt, "GuC CT communication channel %s\n", 555 state == XE_GUC_CT_STATE_STOPPED ? "stopped" : 556 str_enabled_disabled(state == XE_GUC_CT_STATE_ENABLED)); 557 558 spin_unlock_irq(&ct->fast_lock); 559 560 /* cancel all in-flight send-recv requests */ 561 xa_for_each(&ct->fence_lookup, idx, g2h_fence) 562 g2h_fence_cancel(g2h_fence); 563 564 /* make sure guc_ct_send_recv() will see g2h_fence changes */ 565 smp_mb(); 566 wake_up_all(&ct->g2h_fence_wq); 567 568 /* 569 * Lockdep doesn't like this under the fast lock and he destroy only 570 * needs to be serialized with the send path which ct lock provides. 571 */ 572 xa_destroy(&ct->fence_lookup); 573 574 mutex_unlock(&ct->lock); 575 } 576 577 static bool ct_needs_safe_mode(struct xe_guc_ct *ct) 578 { 579 return !pci_dev_msi_enabled(to_pci_dev(ct_to_xe(ct)->drm.dev)); 580 } 581 582 static bool ct_restart_safe_mode_worker(struct xe_guc_ct *ct) 583 { 584 if (!ct_needs_safe_mode(ct)) 585 return false; 586 587 queue_delayed_work(ct->g2h_wq, &ct->safe_mode_worker, HZ / 10); 588 return true; 589 } 590 591 static void safe_mode_worker_func(struct work_struct *w) 592 { 593 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, safe_mode_worker.work); 594 595 receive_g2h(ct); 596 597 if (!ct_restart_safe_mode_worker(ct)) 598 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode canceled\n"); 599 } 600 601 static void ct_enter_safe_mode(struct xe_guc_ct *ct) 602 { 603 if (ct_restart_safe_mode_worker(ct)) 604 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode enabled\n"); 605 } 606 607 static void ct_exit_safe_mode(struct xe_guc_ct *ct) 608 { 609 if (cancel_delayed_work_sync(&ct->safe_mode_worker)) 610 xe_gt_dbg(ct_to_gt(ct), "GuC CT safe-mode disabled\n"); 611 } 612 613 static int __xe_guc_ct_start(struct xe_guc_ct *ct, bool needs_register) 614 { 615 struct xe_device *xe = ct_to_xe(ct); 616 struct xe_gt *gt = ct_to_gt(ct); 617 int err; 618 619 xe_gt_assert(gt, !xe_guc_ct_enabled(ct)); 620 621 if (needs_register) { 622 xe_map_memset(xe, &ct->ctbs.h2g.bo->vmap, 0, 0, 623 xe_bo_size(ct->ctbs.h2g.bo)); 624 xe_map_memset(xe, &ct->ctbs.g2h.bo->vmap, 0, 0, 625 xe_bo_size(ct->ctbs.g2h.bo)); 626 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->ctbs.h2g.bo->vmap); 627 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->ctbs.g2h.bo->vmap); 628 629 err = guc_ct_ctb_h2g_register(ct); 630 if (err) 631 goto err_out; 632 633 err = guc_ct_ctb_g2h_register(ct); 634 if (err) 635 goto err_out; 636 637 err = guc_ct_control_toggle(ct, true); 638 if (err) 639 goto err_out; 640 } else { 641 ct->ctbs.h2g.info.broken = false; 642 ct->ctbs.g2h.info.broken = false; 643 /* Skip everything in H2G buffer */ 644 xe_map_memset(xe, &ct->ctbs.h2g.bo->vmap, CTB_H2G_BUFFER_OFFSET, 0, 645 CTB_H2G_BUFFER_SIZE); 646 } 647 648 guc_ct_change_state(ct, XE_GUC_CT_STATE_ENABLED); 649 650 smp_mb(); 651 wake_up_all(&ct->wq); 652 653 if (ct_needs_safe_mode(ct)) 654 ct_enter_safe_mode(ct); 655 656 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 657 /* 658 * The CT has now been reset so the dumper can be re-armed 659 * after any existing dead state has been dumped. 660 */ 661 spin_lock_irq(&ct->dead.lock); 662 if (ct->dead.reason) { 663 ct->dead.reason |= (1 << CT_DEAD_STATE_REARM); 664 queue_work(system_dfl_wq, &ct->dead.worker); 665 } 666 spin_unlock_irq(&ct->dead.lock); 667 #endif 668 669 return 0; 670 671 err_out: 672 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err)); 673 CT_DEAD(ct, NULL, SETUP); 674 675 return err; 676 } 677 678 /** 679 * xe_guc_ct_restart() - Restart GuC CT 680 * @ct: the &xe_guc_ct 681 * 682 * Restart GuC CT to an empty state without issuing a CT register MMIO command. 683 * 684 * Return: 0 on success, or a negative errno on failure. 685 */ 686 int xe_guc_ct_restart(struct xe_guc_ct *ct) 687 { 688 return __xe_guc_ct_start(ct, false); 689 } 690 691 /** 692 * xe_guc_ct_enable() - Enable GuC CT 693 * @ct: the &xe_guc_ct 694 * 695 * Enable GuC CT to an empty state and issue a CT register MMIO command. 696 * 697 * Return: 0 on success, or a negative errno on failure. 698 */ 699 int xe_guc_ct_enable(struct xe_guc_ct *ct) 700 { 701 return __xe_guc_ct_start(ct, true); 702 } 703 704 static void stop_g2h_handler(struct xe_guc_ct *ct) 705 { 706 cancel_work_sync(&ct->g2h_worker); 707 } 708 709 /** 710 * xe_guc_ct_disable - Set GuC to disabled state 711 * @ct: the &xe_guc_ct 712 * 713 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected 714 * in this transition. 715 */ 716 void xe_guc_ct_disable(struct xe_guc_ct *ct) 717 { 718 guc_ct_change_state(ct, XE_GUC_CT_STATE_DISABLED); 719 ct_exit_safe_mode(ct); 720 stop_g2h_handler(ct); 721 } 722 723 /** 724 * xe_guc_ct_flush_and_stop - Flush and stop all processing of G2H / H2G 725 * @ct: the &xe_guc_ct 726 */ 727 void xe_guc_ct_flush_and_stop(struct xe_guc_ct *ct) 728 { 729 receive_g2h(ct); 730 xe_guc_ct_stop(ct); 731 } 732 733 /** 734 * xe_guc_ct_stop - Set GuC to stopped state 735 * @ct: the &xe_guc_ct 736 * 737 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h 738 */ 739 void xe_guc_ct_stop(struct xe_guc_ct *ct) 740 { 741 if (!xe_guc_ct_initialized(ct)) 742 return; 743 744 guc_ct_change_state(ct, XE_GUC_CT_STATE_STOPPED); 745 stop_g2h_handler(ct); 746 } 747 748 /** 749 * xe_guc_ct_runtime_suspend() - GuC CT runtime suspend 750 * @ct: the &xe_guc_ct 751 * 752 * Set GuC CT to disabled state. 753 */ 754 void xe_guc_ct_runtime_suspend(struct xe_guc_ct *ct) 755 { 756 struct guc_ctb *g2h = &ct->ctbs.g2h; 757 u32 credits = CIRC_SPACE(0, 0, CTB_G2H_BUFFER_DWORDS) - G2H_ROOM_BUFFER_DWORDS; 758 759 /* We should be back to guc_ct_ctb_g2h_init() values */ 760 xe_gt_assert(ct_to_gt(ct), g2h->info.space == credits); 761 762 /* 763 * Since we're already in runtime suspend path, we shouldn't have pending 764 * messages. But if there happen to be any, we'd probably want them to be 765 * thrown as errors for further investigation. 766 */ 767 xe_guc_ct_disable(ct); 768 } 769 770 /** 771 * xe_guc_ct_runtime_resume() - GuC CT runtime resume 772 * @ct: the &xe_guc_ct 773 * 774 * Restart GuC CT and set it to enabled state. 775 */ 776 void xe_guc_ct_runtime_resume(struct xe_guc_ct *ct) 777 { 778 xe_guc_ct_restart(ct); 779 } 780 781 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) 782 { 783 struct guc_ctb *h2g = &ct->ctbs.h2g; 784 785 lockdep_assert_held(&ct->lock); 786 787 if (cmd_len > h2g->info.space) { 788 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); 789 790 if (h2g->info.head > h2g->info.size) { 791 struct xe_device *xe = ct_to_xe(ct); 792 u32 desc_status = desc_read(xe, h2g, status); 793 794 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 795 796 xe_gt_err(ct_to_gt(ct), "CT: invalid head offset %u >= %u)\n", 797 h2g->info.head, h2g->info.size); 798 CT_DEAD(ct, h2g, H2G_HAS_ROOM); 799 return false; 800 } 801 802 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 803 h2g->info.size) - 804 h2g->info.resv_space; 805 if (cmd_len > h2g->info.space) 806 return false; 807 } 808 809 return true; 810 } 811 812 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len) 813 { 814 if (!g2h_len) 815 return true; 816 817 lockdep_assert_held(&ct->fast_lock); 818 819 return ct->ctbs.g2h.info.space > g2h_len; 820 } 821 822 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len) 823 { 824 lockdep_assert_held(&ct->lock); 825 826 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len)) 827 return -EBUSY; 828 829 return 0; 830 } 831 832 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len) 833 { 834 lockdep_assert_held(&ct->lock); 835 ct->ctbs.h2g.info.space -= cmd_len; 836 } 837 838 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h) 839 { 840 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space); 841 xe_gt_assert(ct_to_gt(ct), (!g2h_len && !num_g2h) || 842 (g2h_len && num_g2h)); 843 844 if (g2h_len) { 845 lockdep_assert_held(&ct->fast_lock); 846 847 if (!ct->g2h_outstanding) 848 xe_pm_runtime_get_noresume(ct_to_xe(ct)); 849 850 ct->ctbs.g2h.info.space -= g2h_len; 851 ct->g2h_outstanding += num_g2h; 852 } 853 } 854 855 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 856 { 857 bool bad = false; 858 859 lockdep_assert_held(&ct->fast_lock); 860 861 bad = ct->ctbs.g2h.info.space + g2h_len > 862 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space; 863 bad |= !ct->g2h_outstanding; 864 865 if (bad) { 866 xe_gt_err(ct_to_gt(ct), "Invalid G2H release: %d + %d vs %d - %d -> %d vs %d, outstanding = %d!\n", 867 ct->ctbs.g2h.info.space, g2h_len, 868 ct->ctbs.g2h.info.size, ct->ctbs.g2h.info.resv_space, 869 ct->ctbs.g2h.info.space + g2h_len, 870 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space, 871 ct->g2h_outstanding); 872 CT_DEAD(ct, &ct->ctbs.g2h, G2H_RELEASE); 873 return; 874 } 875 876 ct->ctbs.g2h.info.space += g2h_len; 877 if (!--ct->g2h_outstanding) 878 xe_pm_runtime_put(ct_to_xe(ct)); 879 } 880 881 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 882 { 883 spin_lock_irq(&ct->fast_lock); 884 __g2h_release_space(ct, g2h_len); 885 spin_unlock_irq(&ct->fast_lock); 886 } 887 888 /* 889 * The CT protocol accepts a 16 bits fence. This field is fully owned by the 890 * driver, the GuC will just copy it to the reply message. Since we need to 891 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages, 892 * we use one bit of the seqno as an indicator for that and a rolling counter 893 * for the remaining 15 bits. 894 */ 895 #define CT_SEQNO_MASK GENMASK(14, 0) 896 #define CT_SEQNO_UNTRACKED BIT(15) 897 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence) 898 { 899 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK; 900 901 if (!is_g2h_fence) 902 seqno |= CT_SEQNO_UNTRACKED; 903 904 return seqno; 905 } 906 907 #define MAKE_ACTION(type, __action) \ 908 ({ \ 909 FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | \ 910 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | \ 911 GUC_HXG_EVENT_MSG_0_DATA0, __action); \ 912 }) 913 914 static bool vf_action_can_safely_fail(struct xe_device *xe, u32 action) 915 { 916 /* 917 * When resuming a VF, we can't reliably track whether context 918 * registration has completed in the GuC state machine. It is harmless 919 * to resend the request, as it will fail silently if GUC_HXG_TYPE_EVENT 920 * is used. Additionally, if there is an H2G protocol issue on a VF, 921 * subsequent H2G messages sent as GUC_HXG_TYPE_FAST_REQUEST will likely 922 * fail. 923 */ 924 return IS_SRIOV_VF(xe) && xe_sriov_vf_migration_supported(xe) && 925 (action == XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC || 926 action == XE_GUC_ACTION_REGISTER_CONTEXT); 927 } 928 929 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */ 930 931 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, 932 u32 ct_fence_value, bool want_response) 933 { 934 struct xe_device *xe = ct_to_xe(ct); 935 struct xe_gt *gt = ct_to_gt(ct); 936 struct guc_ctb *h2g = &ct->ctbs.h2g; 937 u32 cmd[H2G_CT_HEADERS]; 938 u32 tail = h2g->info.tail; 939 u32 full_len; 940 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds, 941 tail * sizeof(u32)); 942 943 full_len = len + GUC_CTB_HDR_LEN; 944 945 lockdep_assert_held(&ct->lock); 946 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN); 947 948 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 949 u32 desc_tail = desc_read(xe, h2g, tail); 950 u32 desc_head = desc_read(xe, h2g, head); 951 u32 desc_status; 952 953 desc_status = desc_read(xe, h2g, status); 954 if (desc_status) { 955 xe_gt_err(gt, "CT write: non-zero status: %u\n", desc_status); 956 goto corrupted; 957 } 958 959 if (tail != desc_tail) { 960 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_MISMATCH); 961 xe_gt_err(gt, "CT write: tail was modified %u != %u\n", desc_tail, tail); 962 goto corrupted; 963 } 964 965 if (tail > h2g->info.size) { 966 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 967 xe_gt_err(gt, "CT write: tail out of range: %u vs %u\n", 968 tail, h2g->info.size); 969 goto corrupted; 970 } 971 972 if (desc_head >= h2g->info.size) { 973 desc_write(xe, h2g, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 974 xe_gt_err(gt, "CT write: invalid head offset %u >= %u)\n", 975 desc_head, h2g->info.size); 976 goto corrupted; 977 } 978 } 979 980 /* Command will wrap, zero fill (NOPs), return and check credits again */ 981 if (tail + full_len > h2g->info.size) { 982 xe_map_memset(xe, &map, 0, 0, 983 (h2g->info.size - tail) * sizeof(u32)); 984 h2g_reserve_space(ct, (h2g->info.size - tail)); 985 h2g->info.tail = 0; 986 desc_write(xe, h2g, tail, h2g->info.tail); 987 988 return -EAGAIN; 989 } 990 991 /* 992 * dw0: CT header (including fence) 993 * dw1: HXG header (including action code) 994 * dw2+: action data 995 */ 996 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 997 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 998 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value); 999 if (want_response) { 1000 cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_REQUEST, action[0]); 1001 } else if (vf_action_can_safely_fail(xe, action[0])) { 1002 cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_EVENT, action[0]); 1003 } else { 1004 fast_req_track(ct, ct_fence_value, 1005 FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0])); 1006 1007 cmd[1] = MAKE_ACTION(GUC_HXG_TYPE_FAST_REQUEST, action[0]); 1008 } 1009 1010 /* H2G header in cmd[1] replaces action[0] so: */ 1011 --len; 1012 ++action; 1013 1014 /* Write H2G ensuring visible before descriptor update */ 1015 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32)); 1016 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32)); 1017 xe_device_wmb(xe); 1018 1019 /* Update local copies */ 1020 h2g->info.tail = (tail + full_len) % h2g->info.size; 1021 h2g_reserve_space(ct, full_len); 1022 1023 /* Update descriptor */ 1024 desc_write(xe, h2g, tail, h2g->info.tail); 1025 1026 /* 1027 * desc_read() performs an VRAM read which serializes the CPU and drains 1028 * posted writes on dGPU platforms. Tracepoints evaluate arguments even 1029 * when disabled, so guard the event to avoid adding µs-scale latency to 1030 * the fast H2G submission path when tracing is not active. 1031 */ 1032 if (trace_xe_guc_ctb_h2g_enabled()) 1033 trace_xe_guc_ctb_h2g(xe, gt->info.id, *(action - 1), full_len, 1034 desc_read(xe, h2g, head), h2g->info.tail); 1035 1036 return 0; 1037 1038 corrupted: 1039 CT_DEAD(ct, &ct->ctbs.h2g, H2G_WRITE); 1040 return -EPIPE; 1041 } 1042 1043 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, 1044 u32 len, u32 g2h_len, u32 num_g2h, 1045 struct g2h_fence *g2h_fence) 1046 { 1047 struct xe_gt *gt = ct_to_gt(ct); 1048 u16 seqno; 1049 int ret; 1050 1051 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 1052 xe_gt_assert(gt, !g2h_len || !g2h_fence); 1053 xe_gt_assert(gt, !num_g2h || !g2h_fence); 1054 xe_gt_assert(gt, !g2h_len || num_g2h); 1055 xe_gt_assert(gt, g2h_len || !num_g2h); 1056 lockdep_assert_held(&ct->lock); 1057 1058 if (unlikely(ct->ctbs.h2g.info.broken)) { 1059 ret = -EPIPE; 1060 goto out; 1061 } 1062 1063 if (ct->state == XE_GUC_CT_STATE_DISABLED) { 1064 ret = -ENODEV; 1065 goto out; 1066 } 1067 1068 if (ct->state == XE_GUC_CT_STATE_STOPPED || xe_gt_recovery_pending(gt)) { 1069 ret = -ECANCELED; 1070 goto out; 1071 } 1072 1073 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1074 1075 if (g2h_fence) { 1076 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; 1077 num_g2h = 1; 1078 1079 if (g2h_fence_needs_alloc(g2h_fence)) { 1080 g2h_fence->seqno = next_ct_seqno(ct, true); 1081 ret = xa_err(xa_store(&ct->fence_lookup, 1082 g2h_fence->seqno, g2h_fence, 1083 GFP_ATOMIC)); 1084 if (ret) 1085 goto out; 1086 } 1087 1088 seqno = g2h_fence->seqno; 1089 } else { 1090 seqno = next_ct_seqno(ct, false); 1091 } 1092 1093 if (g2h_len) 1094 spin_lock_irq(&ct->fast_lock); 1095 retry: 1096 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len); 1097 if (unlikely(ret)) 1098 goto out_unlock; 1099 1100 ret = h2g_write(ct, action, len, seqno, !!g2h_fence); 1101 if (unlikely(ret)) { 1102 if (ret == -EAGAIN) 1103 goto retry; 1104 goto out_unlock; 1105 } 1106 1107 __g2h_reserve_space(ct, g2h_len, num_g2h); 1108 xe_guc_notify(ct_to_guc(ct)); 1109 out_unlock: 1110 if (g2h_len) 1111 spin_unlock_irq(&ct->fast_lock); 1112 out: 1113 return ret; 1114 } 1115 1116 static void kick_reset(struct xe_guc_ct *ct) 1117 { 1118 xe_gt_reset_async(ct_to_gt(ct)); 1119 } 1120 1121 static int dequeue_one_g2h(struct xe_guc_ct *ct); 1122 1123 /* 1124 * wait before retry of sending h2g message 1125 * Return: true if ready for retry, false if the wait timeouted 1126 */ 1127 static bool guc_ct_send_wait_for_retry(struct xe_guc_ct *ct, u32 len, 1128 u32 g2h_len, struct g2h_fence *g2h_fence, 1129 unsigned int *sleep_period_ms, 1130 unsigned int *sleep_total_ms) 1131 { 1132 struct xe_device *xe = ct_to_xe(ct); 1133 1134 /* 1135 * We wait to try to restore credits for about 1 second before bailing. 1136 * In the case of H2G credits we have no choice but just to wait for the 1137 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In 1138 * the case of G2H we process any G2H in the channel, hopefully freeing 1139 * credits as we consume the G2H messages. 1140 */ 1141 if (!h2g_has_room(ct, len + GUC_CTB_HDR_LEN)) { 1142 struct guc_ctb *h2g = &ct->ctbs.h2g; 1143 1144 if (*sleep_total_ms > 1000) 1145 return false; 1146 1147 trace_xe_guc_ct_h2g_flow_control(xe, h2g->info.head, h2g->info.tail, 1148 h2g->info.size, 1149 h2g->info.space, 1150 len + GUC_CTB_HDR_LEN); 1151 *sleep_total_ms += xe_sleep_exponential_ms(sleep_period_ms, 64); 1152 } else { 1153 struct guc_ctb *g2h = &ct->ctbs.g2h; 1154 int ret; 1155 1156 trace_xe_guc_ct_g2h_flow_control(xe, g2h->info.head, 1157 desc_read(xe, g2h, tail), 1158 g2h->info.size, 1159 g2h->info.space, 1160 g2h_fence ? 1161 GUC_CTB_HXG_MSG_MAX_LEN : 1162 g2h_len); 1163 1164 #define g2h_avail(ct) \ 1165 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head) 1166 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding || 1167 g2h_avail(ct), HZ)) 1168 return false; 1169 #undef g2h_avail 1170 1171 ret = dequeue_one_g2h(ct); 1172 if (ret < 0) { 1173 if (ret != -ECANCELED) 1174 xe_gt_err(ct_to_gt(ct), "CTB receive failed (%pe)\n", 1175 ERR_PTR(ret)); 1176 return false; 1177 } 1178 } 1179 return true; 1180 } 1181 1182 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 1183 u32 g2h_len, u32 num_g2h, 1184 struct g2h_fence *g2h_fence) 1185 { 1186 struct xe_gt *gt = ct_to_gt(ct); 1187 unsigned int sleep_period_ms = 1; 1188 unsigned int sleep_total_ms = 0; 1189 int ret; 1190 1191 xe_gt_assert(gt, !g2h_len || !g2h_fence); 1192 lockdep_assert_held(&ct->lock); 1193 xe_device_assert_mem_access(ct_to_xe(ct)); 1194 1195 try_again: 1196 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, 1197 g2h_fence); 1198 1199 if (unlikely(ret == -EBUSY)) { 1200 if (!guc_ct_send_wait_for_retry(ct, len, g2h_len, g2h_fence, 1201 &sleep_period_ms, &sleep_total_ms)) 1202 goto broken; 1203 goto try_again; 1204 } 1205 1206 return ret; 1207 1208 broken: 1209 xe_gt_err(gt, "No forward process on H2G, reset required\n"); 1210 CT_DEAD(ct, &ct->ctbs.h2g, DEADLOCK); 1211 1212 return -EDEADLK; 1213 } 1214 1215 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 1216 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) 1217 { 1218 int ret; 1219 1220 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence); 1221 1222 mutex_lock(&ct->lock); 1223 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence); 1224 mutex_unlock(&ct->lock); 1225 1226 return ret; 1227 } 1228 1229 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 1230 u32 g2h_len, u32 num_g2h) 1231 { 1232 int ret; 1233 1234 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL); 1235 if (ret == -EDEADLK) 1236 kick_reset(ct); 1237 1238 return ret; 1239 } 1240 1241 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 1242 u32 g2h_len, u32 num_g2h) 1243 { 1244 int ret; 1245 1246 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL); 1247 if (ret == -EDEADLK) 1248 kick_reset(ct); 1249 1250 return ret; 1251 } 1252 1253 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len) 1254 { 1255 int ret; 1256 1257 lockdep_assert_held(&ct->lock); 1258 1259 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL); 1260 if (ret == -EDEADLK) 1261 kick_reset(ct); 1262 1263 return ret; 1264 } 1265 1266 /* 1267 * Check if a GT reset is in progress or will occur and if GT reset brought the 1268 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset. 1269 */ 1270 static bool retry_failure(struct xe_guc_ct *ct, int ret) 1271 { 1272 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV)) 1273 return false; 1274 1275 #define ct_alive(ct) \ 1276 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \ 1277 !ct->ctbs.g2h.info.broken) 1278 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) 1279 return false; 1280 #undef ct_alive 1281 1282 return true; 1283 } 1284 1285 #define GUC_SEND_RETRY_LIMIT 50 1286 #define GUC_SEND_RETRY_MSLEEP 5 1287 1288 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1289 u32 *response_buffer, bool no_fail) 1290 { 1291 struct xe_gt *gt = ct_to_gt(ct); 1292 struct g2h_fence g2h_fence; 1293 unsigned int retries = 0; 1294 int ret = 0; 1295 1296 /* 1297 * We use a fence to implement blocking sends / receiving response data. 1298 * The seqno of the fence is sent in the H2G, returned in the G2H, and 1299 * an xarray is used as storage media with the seqno being to key. 1300 * Fields in the fence hold success, failure, retry status and the 1301 * response data. Safe to allocate on the stack as the xarray is the 1302 * only reference and it cannot be present after this function exits. 1303 */ 1304 retry: 1305 g2h_fence_init(&g2h_fence, response_buffer); 1306 retry_same_fence: 1307 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence); 1308 if (unlikely(ret == -ENOMEM)) { 1309 /* Retry allocation /w GFP_KERNEL */ 1310 ret = xa_err(xa_store(&ct->fence_lookup, g2h_fence.seqno, 1311 &g2h_fence, GFP_KERNEL)); 1312 if (ret) 1313 return ret; 1314 1315 goto retry_same_fence; 1316 } else if (unlikely(ret)) { 1317 if (ret == -EDEADLK) 1318 kick_reset(ct); 1319 1320 if (no_fail && retry_failure(ct, ret)) 1321 goto retry_same_fence; 1322 1323 if (!g2h_fence_needs_alloc(&g2h_fence)) 1324 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1325 1326 return ret; 1327 } 1328 1329 /* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response 1330 * and g2h_fence_cancel. 1331 */ 1332 ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ); 1333 if (!ret) { 1334 LNL_FLUSH_WORK(&ct->g2h_worker); 1335 if (READ_ONCE(g2h_fence.done)) { 1336 xe_gt_warn(gt, "G2H fence %u, action %04x, done\n", 1337 g2h_fence.seqno, action[0]); 1338 ret = 1; 1339 } 1340 } 1341 1342 /* 1343 * Ensure we serialize with completion side to prevent UAF with fence going out of scope on 1344 * the stack, since we have no clue if it will fire after the timeout before we can erase 1345 * from the xa. Also we have some dependent loads and stores below for which we need the 1346 * correct ordering, and we lack the needed barriers. 1347 */ 1348 mutex_lock(&ct->lock); 1349 if (!ret) { 1350 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x, done %s\n", 1351 g2h_fence.seqno, action[0], str_yes_no(g2h_fence.done)); 1352 xa_erase(&ct->fence_lookup, g2h_fence.seqno); 1353 mutex_unlock(&ct->lock); 1354 return -ETIME; 1355 } 1356 1357 if (g2h_fence.retry) { 1358 xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n", 1359 action[0], g2h_fence.reason); 1360 mutex_unlock(&ct->lock); 1361 if (++retries > GUC_SEND_RETRY_LIMIT) { 1362 xe_gt_err(gt, "H2G action %#x reached retry limit=%u, aborting\n", 1363 action[0], GUC_SEND_RETRY_LIMIT); 1364 return -ELOOP; 1365 } 1366 msleep(GUC_SEND_RETRY_MSLEEP * retries); 1367 goto retry; 1368 } 1369 if (g2h_fence.fail) { 1370 if (g2h_fence.cancel) { 1371 xe_gt_dbg(gt, "H2G request %#x canceled!\n", action[0]); 1372 ret = -ECANCELED; 1373 goto unlock; 1374 } 1375 xe_gt_err(gt, "H2G request %#x failed: error %#x hint %#x\n", 1376 action[0], g2h_fence.error, g2h_fence.hint); 1377 ret = -EIO; 1378 } 1379 1380 if (ret > 0) 1381 ret = response_buffer ? g2h_fence.response_len : g2h_fence.response_data; 1382 1383 unlock: 1384 mutex_unlock(&ct->lock); 1385 1386 return ret; 1387 } 1388 1389 /** 1390 * xe_guc_ct_send_recv - Send and receive HXG to the GuC 1391 * @ct: the &xe_guc_ct 1392 * @action: the dword array with `HXG Request`_ message (can't be NULL) 1393 * @len: length of the `HXG Request`_ message (in dwords, can't be 0) 1394 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL) 1395 * 1396 * Send a `HXG Request`_ message to the GuC over CT communication channel and 1397 * blocks until GuC replies with a `HXG Response`_ message. 1398 * 1399 * For non-blocking communication with GuC use xe_guc_ct_send(). 1400 * 1401 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_. 1402 * 1403 * Return: response length (in dwords) if &response_buffer was not NULL, or 1404 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or 1405 * a negative error code on failure. 1406 */ 1407 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 1408 u32 *response_buffer) 1409 { 1410 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer); 1411 return guc_ct_send_recv(ct, action, len, response_buffer, false); 1412 } 1413 ALLOW_ERROR_INJECTION(xe_guc_ct_send_recv, ERRNO); 1414 1415 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, 1416 u32 len, u32 *response_buffer) 1417 { 1418 return guc_ct_send_recv(ct, action, len, response_buffer, true); 1419 } 1420 1421 static u32 *msg_to_hxg(u32 *msg) 1422 { 1423 return msg + GUC_CTB_MSG_MIN_LEN; 1424 } 1425 1426 static u32 msg_len_to_hxg_len(u32 len) 1427 { 1428 return len - GUC_CTB_MSG_MIN_LEN; 1429 } 1430 1431 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) 1432 { 1433 u32 *hxg = msg_to_hxg(msg); 1434 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1435 1436 lockdep_assert_held(&ct->lock); 1437 1438 switch (action) { 1439 case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: 1440 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1441 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1442 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1443 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1444 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1445 g2h_release_space(ct, len); 1446 } 1447 1448 return 0; 1449 } 1450 1451 static int guc_crash_process_msg(struct xe_guc_ct *ct, u32 action) 1452 { 1453 struct xe_gt *gt = ct_to_gt(ct); 1454 1455 if (action == XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED) 1456 xe_gt_err(gt, "GuC Crash dump notification\n"); 1457 else if (action == XE_GUC_ACTION_NOTIFY_EXCEPTION) 1458 xe_gt_err(gt, "GuC Exception notification\n"); 1459 else 1460 xe_gt_err(gt, "Unknown GuC crash notification: 0x%04X\n", action); 1461 1462 CT_DEAD(ct, NULL, CRASH); 1463 1464 kick_reset(ct); 1465 1466 return 0; 1467 } 1468 1469 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) 1470 { 1471 struct xe_gt *gt = ct_to_gt(ct); 1472 u32 *hxg = msg_to_hxg(msg); 1473 u32 hxg_len = msg_len_to_hxg_len(len); 1474 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); 1475 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1476 struct g2h_fence *g2h_fence; 1477 1478 lockdep_assert_held(&ct->lock); 1479 1480 /* 1481 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup. 1482 * Those messages should never fail, so if we do get an error back it 1483 * means we're likely doing an illegal operation and the GuC is 1484 * rejecting it. We have no way to inform the code that submitted the 1485 * H2G that the message was rejected, so we need to escalate the 1486 * failure to trigger a reset. 1487 */ 1488 if (fence & CT_SEQNO_UNTRACKED) { 1489 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) 1490 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n", 1491 fence, 1492 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]), 1493 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0])); 1494 else 1495 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n", 1496 type, fence); 1497 1498 fast_req_report(ct, fence); 1499 1500 /* FIXME: W/A race in the GuC, will get in firmware soon */ 1501 if (xe_gt_recovery_pending(gt)) 1502 return 0; 1503 1504 CT_DEAD(ct, NULL, PARSE_G2H_RESPONSE); 1505 1506 return -EPROTO; 1507 } 1508 1509 g2h_fence = xa_erase(&ct->fence_lookup, fence); 1510 if (unlikely(!g2h_fence)) { 1511 /* Don't tear down channel, as send could've timed out */ 1512 /* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */ 1513 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence); 1514 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1515 return 0; 1516 } 1517 1518 xe_gt_assert(gt, fence == g2h_fence->seqno); 1519 1520 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { 1521 g2h_fence->fail = true; 1522 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); 1523 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]); 1524 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 1525 g2h_fence->retry = true; 1526 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); 1527 } else if (g2h_fence->response_buffer) { 1528 g2h_fence->response_len = hxg_len; 1529 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); 1530 } else { 1531 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); 1532 } 1533 1534 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 1535 1536 /* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */ 1537 WRITE_ONCE(g2h_fence->done, true); 1538 smp_mb(); 1539 1540 wake_up_all(&ct->g2h_fence_wq); 1541 1542 return 0; 1543 } 1544 1545 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1546 { 1547 struct xe_gt *gt = ct_to_gt(ct); 1548 u32 *hxg = msg_to_hxg(msg); 1549 u32 origin, type; 1550 int ret; 1551 1552 lockdep_assert_held(&ct->lock); 1553 1554 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 1555 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 1556 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n", 1557 origin); 1558 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_ORIGIN); 1559 1560 return -EPROTO; 1561 } 1562 1563 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 1564 switch (type) { 1565 case GUC_HXG_TYPE_EVENT: 1566 ret = parse_g2h_event(ct, msg, len); 1567 break; 1568 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 1569 case GUC_HXG_TYPE_RESPONSE_FAILURE: 1570 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1571 ret = parse_g2h_response(ct, msg, len); 1572 break; 1573 default: 1574 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n", 1575 type); 1576 CT_DEAD(ct, &ct->ctbs.g2h, PARSE_G2H_TYPE); 1577 1578 ret = -EOPNOTSUPP; 1579 } 1580 1581 return ret; 1582 } 1583 1584 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1585 { 1586 struct xe_guc *guc = ct_to_guc(ct); 1587 struct xe_gt *gt = ct_to_gt(ct); 1588 u32 hxg_len = msg_len_to_hxg_len(len); 1589 u32 *hxg = msg_to_hxg(msg); 1590 u32 action, adj_len; 1591 u32 *payload; 1592 int ret = 0; 1593 1594 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1595 return 0; 1596 1597 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1598 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN; 1599 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN; 1600 1601 switch (action) { 1602 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1603 ret = xe_guc_sched_done_handler(guc, payload, adj_len); 1604 break; 1605 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1606 ret = xe_guc_deregister_done_handler(guc, payload, adj_len); 1607 break; 1608 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1609 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len); 1610 break; 1611 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1612 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload, 1613 adj_len); 1614 break; 1615 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1616 /* Selftest only at the moment */ 1617 break; 1618 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1619 ret = xe_guc_error_capture_handler(guc, payload, adj_len); 1620 break; 1621 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1622 /* FIXME: Handle this */ 1623 break; 1624 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: 1625 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload, 1626 adj_len); 1627 break; 1628 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1629 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1630 break; 1631 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1632 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1633 /* 1634 * Page reclamation is an extension of TLB invalidation. Both 1635 * operations share the same seqno and fence. When either 1636 * action completes, we need to signal the corresponding 1637 * fence. Since the handling logic (lookup fence by seqno, 1638 * fence signalling) is identical, we use the same handler 1639 * for both G2H events. 1640 */ 1641 ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); 1642 break; 1643 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: 1644 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len); 1645 break; 1646 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF: 1647 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len); 1648 break; 1649 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY: 1650 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len); 1651 break; 1652 case GUC_ACTION_GUC2PF_ADVERSE_EVENT: 1653 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len); 1654 break; 1655 case XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED: 1656 case XE_GUC_ACTION_NOTIFY_EXCEPTION: 1657 ret = guc_crash_process_msg(ct, action); 1658 break; 1659 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1660 case XE_GUC_ACTION_TEST_G2G_RECV: 1661 ret = xe_guc_g2g_test_notification(guc, payload, adj_len); 1662 break; 1663 #endif 1664 case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE: 1665 ret = xe_guc_exec_queue_cgp_sync_done_handler(guc, payload, adj_len); 1666 break; 1667 case XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CGP_CONTEXT_ERROR: 1668 ret = xe_guc_exec_queue_cgp_context_error_handler(guc, payload, 1669 adj_len); 1670 break; 1671 default: 1672 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); 1673 } 1674 1675 if (ret) { 1676 xe_gt_err(gt, "G2H action %#04x failed (%pe) len %u msg %*ph\n", 1677 action, ERR_PTR(ret), hxg_len, (int)sizeof(u32) * hxg_len, hxg); 1678 CT_DEAD(ct, NULL, PROCESS_FAILED); 1679 } 1680 1681 return 0; 1682 } 1683 1684 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) 1685 { 1686 struct xe_device *xe = ct_to_xe(ct); 1687 struct xe_gt *gt = ct_to_gt(ct); 1688 struct guc_ctb *g2h = &ct->ctbs.g2h; 1689 u32 tail, head, len, desc_status; 1690 s32 avail; 1691 u32 action; 1692 u32 *hxg; 1693 1694 xe_gt_assert(gt, xe_guc_ct_initialized(ct)); 1695 lockdep_assert_held(&ct->fast_lock); 1696 1697 if (ct->state == XE_GUC_CT_STATE_DISABLED) 1698 return -ENODEV; 1699 1700 if (ct->state == XE_GUC_CT_STATE_STOPPED) 1701 return -ECANCELED; 1702 1703 if (g2h->info.broken) 1704 return -EPIPE; 1705 1706 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1707 1708 desc_status = desc_read(xe, g2h, status); 1709 if (desc_status) { 1710 if (desc_status & GUC_CTB_STATUS_DISABLED) { 1711 /* 1712 * Potentially valid if a CLIENT_RESET request resulted in 1713 * contexts/engines being reset. But should never happen as 1714 * no contexts should be active when CLIENT_RESET is sent. 1715 */ 1716 xe_gt_err(gt, "CT read: unexpected G2H after GuC has stopped!\n"); 1717 desc_status &= ~GUC_CTB_STATUS_DISABLED; 1718 } 1719 1720 if (desc_status) { 1721 xe_gt_err(gt, "CT read: non-zero status: %u\n", desc_status); 1722 goto corrupted; 1723 } 1724 } 1725 1726 if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) { 1727 u32 desc_tail = desc_read(xe, g2h, tail); 1728 /* 1729 u32 desc_head = desc_read(xe, g2h, head); 1730 1731 * info.head and desc_head are updated back-to-back at the end of 1732 * this function and nowhere else. Hence, they cannot be different 1733 * unless two g2h_read calls are running concurrently. Which is not 1734 * possible because it is guarded by ct->fast_lock. And yet, some 1735 * discrete platforms are regularly hitting this error :(. 1736 * 1737 * desc_head rolling backwards shouldn't cause any noticeable 1738 * problems - just a delay in GuC being allowed to proceed past that 1739 * point in the queue. So for now, just disable the error until it 1740 * can be root caused. 1741 * 1742 if (g2h->info.head != desc_head) { 1743 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_MISMATCH); 1744 xe_gt_err(gt, "CT read: head was modified %u != %u\n", 1745 desc_head, g2h->info.head); 1746 goto corrupted; 1747 } 1748 */ 1749 1750 if (g2h->info.head > g2h->info.size) { 1751 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1752 xe_gt_err(gt, "CT read: head out of range: %u vs %u\n", 1753 g2h->info.head, g2h->info.size); 1754 goto corrupted; 1755 } 1756 1757 if (desc_tail >= g2h->info.size) { 1758 desc_write(xe, g2h, status, desc_status | GUC_CTB_STATUS_OVERFLOW); 1759 xe_gt_err(gt, "CT read: invalid tail offset %u >= %u)\n", 1760 desc_tail, g2h->info.size); 1761 goto corrupted; 1762 } 1763 } 1764 1765 /* Calculate DW available to read */ 1766 tail = desc_read(xe, g2h, tail); 1767 avail = tail - g2h->info.head; 1768 if (unlikely(avail == 0)) 1769 return 0; 1770 1771 if (avail < 0) 1772 avail += g2h->info.size; 1773 1774 /* Read header */ 1775 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head, 1776 sizeof(u32)); 1777 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; 1778 if (len > avail) { 1779 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n", 1780 avail, len); 1781 goto corrupted; 1782 } 1783 1784 head = (g2h->info.head + 1) % g2h->info.size; 1785 avail = len - 1; 1786 1787 /* Read G2H message */ 1788 if (avail + head > g2h->info.size) { 1789 u32 avail_til_wrap = g2h->info.size - head; 1790 1791 xe_map_memcpy_from(xe, msg + 1, 1792 &g2h->cmds, sizeof(u32) * head, 1793 avail_til_wrap * sizeof(u32)); 1794 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap, 1795 &g2h->cmds, 0, 1796 (avail - avail_til_wrap) * sizeof(u32)); 1797 } else { 1798 xe_map_memcpy_from(xe, msg + 1, 1799 &g2h->cmds, sizeof(u32) * head, 1800 avail * sizeof(u32)); 1801 } 1802 1803 hxg = msg_to_hxg(msg); 1804 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1805 1806 if (fast_path) { 1807 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1808 return 0; 1809 1810 switch (action) { 1811 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1812 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1813 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1814 break; /* Process these in fast-path */ 1815 default: 1816 return 0; 1817 } 1818 } 1819 1820 /* Update local / descriptor header */ 1821 g2h->info.head = (head + avail) % g2h->info.size; 1822 desc_write(xe, g2h, head, g2h->info.head); 1823 1824 trace_xe_guc_ctb_g2h(xe, ct_to_gt(ct)->info.id, 1825 action, len, g2h->info.head, tail); 1826 1827 return len; 1828 1829 corrupted: 1830 CT_DEAD(ct, &ct->ctbs.g2h, G2H_READ); 1831 return -EPROTO; 1832 } 1833 1834 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) 1835 { 1836 struct xe_gt *gt = ct_to_gt(ct); 1837 struct xe_guc *guc = ct_to_guc(ct); 1838 u32 hxg_len = msg_len_to_hxg_len(len); 1839 u32 *hxg = msg_to_hxg(msg); 1840 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1841 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN; 1842 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1843 int ret = 0; 1844 1845 switch (action) { 1846 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1847 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1848 break; 1849 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1850 case XE_GUC_ACTION_PAGE_RECLAMATION_DONE: 1851 /* 1852 * Seqno and fence handling of page reclamation and TLB 1853 * invalidation is identical, so we can use the same handler 1854 * for both actions. 1855 */ 1856 __g2h_release_space(ct, len); 1857 ret = xe_guc_tlb_inval_done_handler(guc, payload, adj_len); 1858 break; 1859 default: 1860 xe_gt_warn(gt, "NOT_POSSIBLE\n"); 1861 } 1862 1863 if (ret) { 1864 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n", 1865 action, ERR_PTR(ret)); 1866 CT_DEAD(ct, NULL, FAST_G2H); 1867 } 1868 } 1869 1870 /** 1871 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler 1872 * @ct: GuC CT object 1873 * 1874 * Anything related to page faults is critical for performance, process these 1875 * critical G2H in the IRQ. This is safe as these handlers either just wake up 1876 * waiters or queue another worker. 1877 */ 1878 void xe_guc_ct_fast_path(struct xe_guc_ct *ct) 1879 { 1880 struct xe_device *xe = ct_to_xe(ct); 1881 bool ongoing; 1882 int len; 1883 1884 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1885 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1886 return; 1887 1888 spin_lock(&ct->fast_lock); 1889 do { 1890 len = g2h_read(ct, ct->fast_msg, true); 1891 if (len > 0) 1892 g2h_fast_path(ct, ct->fast_msg, len); 1893 } while (len > 0); 1894 spin_unlock(&ct->fast_lock); 1895 1896 if (ongoing) 1897 xe_pm_runtime_put(xe); 1898 } 1899 1900 /* Returns less than zero on error, 0 on done, 1 on more available */ 1901 static int dequeue_one_g2h(struct xe_guc_ct *ct) 1902 { 1903 int len; 1904 int ret; 1905 1906 lockdep_assert_held(&ct->lock); 1907 1908 spin_lock_irq(&ct->fast_lock); 1909 len = g2h_read(ct, ct->msg, false); 1910 spin_unlock_irq(&ct->fast_lock); 1911 if (len <= 0) 1912 return len; 1913 1914 ret = parse_g2h_msg(ct, ct->msg, len); 1915 if (unlikely(ret < 0)) 1916 return ret; 1917 1918 ret = process_g2h_msg(ct, ct->msg, len); 1919 if (unlikely(ret < 0)) 1920 return ret; 1921 1922 return 1; 1923 } 1924 1925 static void receive_g2h(struct xe_guc_ct *ct) 1926 { 1927 bool ongoing; 1928 int ret; 1929 1930 /* 1931 * Normal users must always hold mem_access.ref around CT calls. However 1932 * during the runtime pm callbacks we rely on CT to talk to the GuC, but 1933 * at this stage we can't rely on mem_access.ref and even the 1934 * callback_task will be different than current. For such cases we just 1935 * need to ensure we always process the responses from any blocking 1936 * ct_send requests or where we otherwise expect some response when 1937 * initiated from those callbacks (which will need to wait for the below 1938 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if 1939 * the device has suspended to the point that the CT communication has 1940 * been disabled. 1941 * 1942 * If we are inside the runtime pm callback, we can be the only task 1943 * still issuing CT requests (since that requires having the 1944 * mem_access.ref). It seems like it might in theory be possible to 1945 * receive unsolicited events from the GuC just as we are 1946 * suspending-resuming, but those will currently anyway be lost when 1947 * eventually exiting from suspend, hence no need to wake up the device 1948 * here. If we ever need something stronger than get_if_ongoing() then 1949 * we need to be careful with blocking the pm callbacks from getting CT 1950 * responses, if the worker here is blocked on those callbacks 1951 * completing, creating a deadlock. 1952 */ 1953 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1954 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1955 return; 1956 1957 do { 1958 mutex_lock(&ct->lock); 1959 ret = dequeue_one_g2h(ct); 1960 mutex_unlock(&ct->lock); 1961 1962 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) { 1963 xe_gt_err(ct_to_gt(ct), "CT dequeue failed: %d\n", ret); 1964 CT_DEAD(ct, NULL, G2H_RECV); 1965 kick_reset(ct); 1966 } 1967 } while (ret == 1); 1968 1969 if (ongoing) 1970 xe_pm_runtime_put(ct_to_xe(ct)); 1971 } 1972 1973 static void g2h_worker_func(struct work_struct *w) 1974 { 1975 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker); 1976 1977 receive_g2h(ct); 1978 } 1979 1980 static struct xe_guc_ct_snapshot *guc_ct_snapshot_alloc(struct xe_guc_ct *ct, bool atomic, 1981 bool want_ctb) 1982 { 1983 struct xe_guc_ct_snapshot *snapshot; 1984 1985 snapshot = kzalloc_obj(*snapshot, atomic ? GFP_ATOMIC : GFP_KERNEL); 1986 if (!snapshot) 1987 return NULL; 1988 1989 if (ct->ctbs.h2g.bo && ct->ctbs.g2h.bo && want_ctb) { 1990 snapshot->ctb_size = xe_bo_size(ct->ctbs.h2g.bo) + 1991 xe_bo_size(ct->ctbs.g2h.bo); 1992 snapshot->ctb = kmalloc(snapshot->ctb_size, atomic ? GFP_ATOMIC : GFP_KERNEL); 1993 } 1994 1995 return snapshot; 1996 } 1997 1998 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb, 1999 struct guc_ctb_snapshot *snapshot) 2000 { 2001 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0, 2002 sizeof(struct guc_ct_buffer_desc)); 2003 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info)); 2004 } 2005 2006 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot, 2007 struct drm_printer *p) 2008 { 2009 drm_printf(p, "\tsize: %d\n", snapshot->info.size); 2010 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space); 2011 drm_printf(p, "\thead: %d\n", snapshot->info.head); 2012 drm_printf(p, "\ttail: %d\n", snapshot->info.tail); 2013 drm_printf(p, "\tspace: %d\n", snapshot->info.space); 2014 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken); 2015 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head); 2016 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail); 2017 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status); 2018 } 2019 2020 static struct xe_guc_ct_snapshot *guc_ct_snapshot_capture(struct xe_guc_ct *ct, bool atomic, 2021 bool want_ctb) 2022 { 2023 struct xe_device *xe = ct_to_xe(ct); 2024 struct xe_guc_ct_snapshot *snapshot; 2025 2026 snapshot = guc_ct_snapshot_alloc(ct, atomic, want_ctb); 2027 if (!snapshot) { 2028 xe_gt_err(ct_to_gt(ct), "Skipping CTB snapshot entirely.\n"); 2029 return NULL; 2030 } 2031 2032 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) { 2033 snapshot->ct_enabled = true; 2034 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding); 2035 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, &snapshot->h2g); 2036 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, &snapshot->g2h); 2037 } 2038 2039 if (ct->ctbs.h2g.bo && ct->ctbs.g2h.bo && snapshot->ctb) { 2040 xe_map_memcpy_from(xe, snapshot->ctb, &ct->ctbs.h2g.bo->vmap, 0, 2041 xe_bo_size(ct->ctbs.h2g.bo)); 2042 xe_map_memcpy_from(xe, snapshot->ctb + xe_bo_size(ct->ctbs.h2g.bo), 2043 &ct->ctbs.g2h.bo->vmap, 0, 2044 xe_bo_size(ct->ctbs.g2h.bo)); 2045 } 2046 2047 return snapshot; 2048 } 2049 2050 /** 2051 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state. 2052 * @ct: GuC CT object. 2053 * 2054 * This can be printed out in a later stage like during dev_coredump 2055 * analysis. This is safe to be called during atomic context. 2056 * 2057 * Returns: a GuC CT snapshot object that must be freed by the caller 2058 * by using `xe_guc_ct_snapshot_free`. 2059 */ 2060 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct) 2061 { 2062 return guc_ct_snapshot_capture(ct, true, true); 2063 } 2064 2065 /** 2066 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot. 2067 * @snapshot: GuC CT snapshot object. 2068 * @p: drm_printer where it will be printed out. 2069 * 2070 * This function prints out a given GuC CT snapshot object. 2071 */ 2072 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, 2073 struct drm_printer *p) 2074 { 2075 if (!snapshot) 2076 return; 2077 2078 if (snapshot->ct_enabled) { 2079 drm_puts(p, "H2G CTB (all sizes in DW):\n"); 2080 guc_ctb_snapshot_print(&snapshot->h2g, p); 2081 2082 drm_puts(p, "G2H CTB (all sizes in DW):\n"); 2083 guc_ctb_snapshot_print(&snapshot->g2h, p); 2084 drm_printf(p, "\tg2h outstanding: %d\n", 2085 snapshot->g2h_outstanding); 2086 2087 if (snapshot->ctb) { 2088 drm_printf(p, "[CTB].length: 0x%zx\n", snapshot->ctb_size); 2089 xe_print_blob_ascii85(p, "[CTB].data", '\n', 2090 snapshot->ctb, 0, snapshot->ctb_size); 2091 } 2092 } else { 2093 drm_puts(p, "CT disabled\n"); 2094 } 2095 } 2096 2097 /** 2098 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot. 2099 * @snapshot: GuC CT snapshot object. 2100 * 2101 * This function free all the memory that needed to be allocated at capture 2102 * time. 2103 */ 2104 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot) 2105 { 2106 if (!snapshot) 2107 return; 2108 2109 kfree(snapshot->ctb); 2110 kfree(snapshot); 2111 } 2112 2113 /** 2114 * xe_guc_ct_print - GuC CT Print. 2115 * @ct: GuC CT. 2116 * @p: drm_printer where it will be printed out. 2117 * @want_ctb: Should the full CTB content be dumped (vs just the headers) 2118 * 2119 * This function will quickly capture a snapshot of the CT state 2120 * and immediately print it out. 2121 */ 2122 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool want_ctb) 2123 { 2124 struct xe_guc_ct_snapshot *snapshot; 2125 2126 snapshot = guc_ct_snapshot_capture(ct, false, want_ctb); 2127 xe_guc_ct_snapshot_print(snapshot, p); 2128 xe_guc_ct_snapshot_free(snapshot); 2129 } 2130 2131 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 2132 2133 #ifdef CONFIG_FUNCTION_ERROR_INJECTION 2134 /* 2135 * This is a helper function which assists the driver in identifying if a fault 2136 * injection test is currently active, allowing it to reduce unnecessary debug 2137 * output. Typically, the function returns zero, but the fault injection 2138 * framework can alter this to return an error. Since faults are injected 2139 * through this function, it's important to ensure the compiler doesn't optimize 2140 * it into an inline function. To avoid such optimization, the 'noinline' 2141 * attribute is applied. Compiler optimizes the static function defined in the 2142 * header file as an inline function. 2143 */ 2144 noinline int xe_is_injection_active(void) { return 0; } 2145 ALLOW_ERROR_INJECTION(xe_is_injection_active, ERRNO); 2146 #else 2147 int xe_is_injection_active(void) { return 0; } 2148 #endif 2149 2150 static void ct_dead_capture(struct xe_guc_ct *ct, struct guc_ctb *ctb, u32 reason_code) 2151 { 2152 struct xe_guc_log_snapshot *snapshot_log; 2153 struct xe_guc_ct_snapshot *snapshot_ct; 2154 struct xe_guc *guc = ct_to_guc(ct); 2155 unsigned long flags; 2156 bool have_capture; 2157 2158 if (ctb) 2159 ctb->info.broken = true; 2160 /* 2161 * Huge dump is getting generated when injecting error for guc CT/MMIO 2162 * functions. So, let us suppress the dump when fault is injected. 2163 */ 2164 if (xe_is_injection_active()) 2165 return; 2166 2167 /* Ignore further errors after the first dump until a reset */ 2168 if (ct->dead.reported) 2169 return; 2170 2171 spin_lock_irqsave(&ct->dead.lock, flags); 2172 2173 /* And only capture one dump at a time */ 2174 have_capture = ct->dead.reason & (1 << CT_DEAD_STATE_CAPTURE); 2175 ct->dead.reason |= (1 << reason_code) | 2176 (1 << CT_DEAD_STATE_CAPTURE); 2177 2178 spin_unlock_irqrestore(&ct->dead.lock, flags); 2179 2180 if (have_capture) 2181 return; 2182 2183 snapshot_log = xe_guc_log_snapshot_capture(&guc->log, true); 2184 snapshot_ct = xe_guc_ct_snapshot_capture((ct)); 2185 2186 spin_lock_irqsave(&ct->dead.lock, flags); 2187 2188 if (ct->dead.snapshot_log || ct->dead.snapshot_ct) { 2189 xe_gt_err(ct_to_gt(ct), "Got unexpected dead CT capture!\n"); 2190 xe_guc_log_snapshot_free(snapshot_log); 2191 xe_guc_ct_snapshot_free(snapshot_ct); 2192 } else { 2193 ct->dead.snapshot_log = snapshot_log; 2194 ct->dead.snapshot_ct = snapshot_ct; 2195 } 2196 2197 spin_unlock_irqrestore(&ct->dead.lock, flags); 2198 2199 queue_work(system_dfl_wq, &(ct)->dead.worker); 2200 } 2201 2202 static void ct_dead_print(struct xe_dead_ct *dead) 2203 { 2204 struct xe_guc_ct *ct = container_of(dead, struct xe_guc_ct, dead); 2205 struct xe_device *xe = ct_to_xe(ct); 2206 struct xe_gt *gt = ct_to_gt(ct); 2207 static int g_count; 2208 struct drm_printer ip = xe_gt_info_printer(gt); 2209 struct drm_printer lp = drm_line_printer(&ip, "Capture", ++g_count); 2210 2211 if (!dead->reason) { 2212 xe_gt_err(gt, "CTB is dead for no reason!?\n"); 2213 return; 2214 } 2215 2216 /* Can't generate a genuine core dump at this point, so just do the good bits */ 2217 drm_puts(&lp, "**** Xe Device Coredump ****\n"); 2218 drm_printf(&lp, "Reason: CTB is dead - 0x%X\n", dead->reason); 2219 xe_device_snapshot_print(xe, &lp); 2220 2221 drm_printf(&lp, "**** GT #%d ****\n", gt->info.id); 2222 drm_printf(&lp, "\tTile: %d\n", gt->tile->id); 2223 2224 drm_puts(&lp, "**** GuC Log ****\n"); 2225 xe_guc_log_snapshot_print(dead->snapshot_log, &lp); 2226 2227 drm_puts(&lp, "**** GuC CT ****\n"); 2228 xe_guc_ct_snapshot_print(dead->snapshot_ct, &lp); 2229 2230 drm_puts(&lp, "Done.\n"); 2231 } 2232 2233 static void ct_dead_worker_func(struct work_struct *w) 2234 { 2235 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, dead.worker); 2236 2237 if (!ct->dead.reported) { 2238 ct->dead.reported = true; 2239 ct_dead_print(&ct->dead); 2240 } 2241 2242 spin_lock_irq(&ct->dead.lock); 2243 2244 xe_guc_log_snapshot_free(ct->dead.snapshot_log); 2245 ct->dead.snapshot_log = NULL; 2246 xe_guc_ct_snapshot_free(ct->dead.snapshot_ct); 2247 ct->dead.snapshot_ct = NULL; 2248 2249 if (ct->dead.reason & (1 << CT_DEAD_STATE_REARM)) { 2250 /* A reset has occurred so re-arm the error reporting */ 2251 ct->dead.reason = 0; 2252 ct->dead.reported = false; 2253 } 2254 2255 spin_unlock_irq(&ct->dead.lock); 2256 } 2257 #endif 2258