1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_guc_ct.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/circ_buf.h> 10 #include <linux/delay.h> 11 12 #include <kunit/static_stub.h> 13 14 #include <drm/drm_managed.h> 15 16 #include "abi/guc_actions_abi.h" 17 #include "abi/guc_actions_sriov_abi.h" 18 #include "abi/guc_klvs_abi.h" 19 #include "xe_bo.h" 20 #include "xe_device.h" 21 #include "xe_gt.h" 22 #include "xe_gt_pagefault.h" 23 #include "xe_gt_printk.h" 24 #include "xe_gt_sriov_pf_control.h" 25 #include "xe_gt_sriov_pf_monitor.h" 26 #include "xe_gt_tlb_invalidation.h" 27 #include "xe_guc.h" 28 #include "xe_guc_relay.h" 29 #include "xe_guc_submit.h" 30 #include "xe_map.h" 31 #include "xe_pm.h" 32 #include "xe_trace.h" 33 34 /* Used when a CT send wants to block and / or receive data */ 35 struct g2h_fence { 36 u32 *response_buffer; 37 u32 seqno; 38 u32 response_data; 39 u16 response_len; 40 u16 error; 41 u16 hint; 42 u16 reason; 43 bool retry; 44 bool fail; 45 bool done; 46 }; 47 48 static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) 49 { 50 g2h_fence->response_buffer = response_buffer; 51 g2h_fence->response_data = 0; 52 g2h_fence->response_len = 0; 53 g2h_fence->fail = false; 54 g2h_fence->retry = false; 55 g2h_fence->done = false; 56 g2h_fence->seqno = ~0x0; 57 } 58 59 static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) 60 { 61 return g2h_fence->seqno == ~0x0; 62 } 63 64 static struct xe_guc * 65 ct_to_guc(struct xe_guc_ct *ct) 66 { 67 return container_of(ct, struct xe_guc, ct); 68 } 69 70 static struct xe_gt * 71 ct_to_gt(struct xe_guc_ct *ct) 72 { 73 return container_of(ct, struct xe_gt, uc.guc.ct); 74 } 75 76 static struct xe_device * 77 ct_to_xe(struct xe_guc_ct *ct) 78 { 79 return gt_to_xe(ct_to_gt(ct)); 80 } 81 82 /** 83 * DOC: GuC CTB Blob 84 * 85 * We allocate single blob to hold both CTB descriptors and buffers: 86 * 87 * +--------+-----------------------------------------------+------+ 88 * | offset | contents | size | 89 * +========+===============================================+======+ 90 * | 0x0000 | H2G CTB Descriptor (send) | | 91 * +--------+-----------------------------------------------+ 4K | 92 * | 0x0800 | G2H CTB Descriptor (g2h) | | 93 * +--------+-----------------------------------------------+------+ 94 * | 0x1000 | H2G CT Buffer (send) | n*4K | 95 * | | | | 96 * +--------+-----------------------------------------------+------+ 97 * | 0x1000 | G2H CT Buffer (g2h) | m*4K | 98 * | + n*4K | | | 99 * +--------+-----------------------------------------------+------+ 100 * 101 * Size of each ``CT Buffer`` must be multiple of 4K. 102 * We don't expect too many messages in flight at any time, unless we are 103 * using the GuC submission. In that case each request requires a minimum 104 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this 105 * enough space to avoid backpressure on the driver. We increase the size 106 * of the receive buffer (relative to the send) to ensure a G2H response 107 * CTB has a landing spot. 108 */ 109 110 #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) 111 #define CTB_H2G_BUFFER_SIZE (SZ_4K) 112 #define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE) 113 #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4) 114 115 static size_t guc_ct_size(void) 116 { 117 return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + 118 CTB_G2H_BUFFER_SIZE; 119 } 120 121 static void guc_ct_fini(struct drm_device *drm, void *arg) 122 { 123 struct xe_guc_ct *ct = arg; 124 125 destroy_workqueue(ct->g2h_wq); 126 xa_destroy(&ct->fence_lookup); 127 } 128 129 static void g2h_worker_func(struct work_struct *w); 130 131 static void primelockdep(struct xe_guc_ct *ct) 132 { 133 if (!IS_ENABLED(CONFIG_LOCKDEP)) 134 return; 135 136 fs_reclaim_acquire(GFP_KERNEL); 137 might_lock(&ct->lock); 138 fs_reclaim_release(GFP_KERNEL); 139 } 140 141 int xe_guc_ct_init(struct xe_guc_ct *ct) 142 { 143 struct xe_device *xe = ct_to_xe(ct); 144 struct xe_gt *gt = ct_to_gt(ct); 145 struct xe_tile *tile = gt_to_tile(gt); 146 struct xe_bo *bo; 147 int err; 148 149 xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE)); 150 151 ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", 0); 152 if (!ct->g2h_wq) 153 return -ENOMEM; 154 155 spin_lock_init(&ct->fast_lock); 156 xa_init(&ct->fence_lookup); 157 INIT_WORK(&ct->g2h_worker, g2h_worker_func); 158 init_waitqueue_head(&ct->wq); 159 init_waitqueue_head(&ct->g2h_fence_wq); 160 161 err = drmm_mutex_init(&xe->drm, &ct->lock); 162 if (err) 163 return err; 164 165 primelockdep(ct); 166 167 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(), 168 XE_BO_FLAG_SYSTEM | 169 XE_BO_FLAG_GGTT | 170 XE_BO_FLAG_GGTT_INVALIDATE); 171 if (IS_ERR(bo)) 172 return PTR_ERR(bo); 173 174 ct->bo = bo; 175 176 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); 177 if (err) 178 return err; 179 180 xe_gt_assert(gt, ct->state == XE_GUC_CT_STATE_NOT_INITIALIZED); 181 ct->state = XE_GUC_CT_STATE_DISABLED; 182 return 0; 183 } 184 185 #define desc_read(xe_, guc_ctb__, field_) \ 186 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \ 187 struct guc_ct_buffer_desc, field_) 188 189 #define desc_write(xe_, guc_ctb__, field_, val_) \ 190 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \ 191 struct guc_ct_buffer_desc, field_, val_) 192 193 static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, 194 struct iosys_map *map) 195 { 196 h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32); 197 h2g->info.resv_space = 0; 198 h2g->info.tail = 0; 199 h2g->info.head = 0; 200 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 201 h2g->info.size) - 202 h2g->info.resv_space; 203 h2g->info.broken = false; 204 205 h2g->desc = *map; 206 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 207 208 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2); 209 } 210 211 static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, 212 struct iosys_map *map) 213 { 214 g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32); 215 g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32); 216 g2h->info.head = 0; 217 g2h->info.tail = 0; 218 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head, 219 g2h->info.size) - 220 g2h->info.resv_space; 221 g2h->info.broken = false; 222 223 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE); 224 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); 225 226 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 + 227 CTB_H2G_BUFFER_SIZE); 228 } 229 230 static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct) 231 { 232 struct xe_guc *guc = ct_to_guc(ct); 233 u32 desc_addr, ctb_addr, size; 234 int err; 235 236 desc_addr = xe_bo_ggtt_addr(ct->bo); 237 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2; 238 size = ct->ctbs.h2g.info.size * sizeof(u32); 239 240 err = xe_guc_self_cfg64(guc, 241 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY, 242 desc_addr); 243 if (err) 244 return err; 245 246 err = xe_guc_self_cfg64(guc, 247 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY, 248 ctb_addr); 249 if (err) 250 return err; 251 252 return xe_guc_self_cfg32(guc, 253 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY, 254 size); 255 } 256 257 static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct) 258 { 259 struct xe_guc *guc = ct_to_guc(ct); 260 u32 desc_addr, ctb_addr, size; 261 int err; 262 263 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE; 264 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 + 265 CTB_H2G_BUFFER_SIZE; 266 size = ct->ctbs.g2h.info.size * sizeof(u32); 267 268 err = xe_guc_self_cfg64(guc, 269 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, 270 desc_addr); 271 if (err) 272 return err; 273 274 err = xe_guc_self_cfg64(guc, 275 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, 276 ctb_addr); 277 if (err) 278 return err; 279 280 return xe_guc_self_cfg32(guc, 281 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, 282 size); 283 } 284 285 static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) 286 { 287 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { 288 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | 289 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 290 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, 291 GUC_ACTION_HOST2GUC_CONTROL_CTB), 292 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, 293 enable ? GUC_CTB_CONTROL_ENABLE : 294 GUC_CTB_CONTROL_DISABLE), 295 }; 296 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request)); 297 298 return ret > 0 ? -EPROTO : ret; 299 } 300 301 static void xe_guc_ct_set_state(struct xe_guc_ct *ct, 302 enum xe_guc_ct_state state) 303 { 304 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */ 305 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */ 306 307 xe_gt_assert(ct_to_gt(ct), ct->g2h_outstanding == 0 || 308 state == XE_GUC_CT_STATE_STOPPED); 309 310 ct->g2h_outstanding = 0; 311 ct->state = state; 312 313 spin_unlock_irq(&ct->fast_lock); 314 315 /* 316 * Lockdep doesn't like this under the fast lock and he destroy only 317 * needs to be serialized with the send path which ct lock provides. 318 */ 319 xa_destroy(&ct->fence_lookup); 320 321 mutex_unlock(&ct->lock); 322 } 323 324 int xe_guc_ct_enable(struct xe_guc_ct *ct) 325 { 326 struct xe_device *xe = ct_to_xe(ct); 327 struct xe_gt *gt = ct_to_gt(ct); 328 int err; 329 330 xe_gt_assert(gt, !xe_guc_ct_enabled(ct)); 331 332 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap); 333 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap); 334 335 err = guc_ct_ctb_h2g_register(ct); 336 if (err) 337 goto err_out; 338 339 err = guc_ct_ctb_g2h_register(ct); 340 if (err) 341 goto err_out; 342 343 err = guc_ct_control_toggle(ct, true); 344 if (err) 345 goto err_out; 346 347 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_ENABLED); 348 349 smp_mb(); 350 wake_up_all(&ct->wq); 351 xe_gt_dbg(gt, "GuC CT communication channel enabled\n"); 352 353 return 0; 354 355 err_out: 356 xe_gt_err(gt, "Failed to enable GuC CT (%pe)\n", ERR_PTR(err)); 357 358 return err; 359 } 360 361 static void stop_g2h_handler(struct xe_guc_ct *ct) 362 { 363 cancel_work_sync(&ct->g2h_worker); 364 } 365 366 /** 367 * xe_guc_ct_disable - Set GuC to disabled state 368 * @ct: the &xe_guc_ct 369 * 370 * Set GuC CT to disabled state and stop g2h handler. No outstanding g2h expected 371 * in this transition. 372 */ 373 void xe_guc_ct_disable(struct xe_guc_ct *ct) 374 { 375 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_DISABLED); 376 stop_g2h_handler(ct); 377 } 378 379 /** 380 * xe_guc_ct_stop - Set GuC to stopped state 381 * @ct: the &xe_guc_ct 382 * 383 * Set GuC CT to stopped state, stop g2h handler, and clear any outstanding g2h 384 */ 385 void xe_guc_ct_stop(struct xe_guc_ct *ct) 386 { 387 xe_guc_ct_set_state(ct, XE_GUC_CT_STATE_STOPPED); 388 stop_g2h_handler(ct); 389 } 390 391 static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) 392 { 393 struct guc_ctb *h2g = &ct->ctbs.h2g; 394 395 lockdep_assert_held(&ct->lock); 396 397 if (cmd_len > h2g->info.space) { 398 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head); 399 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head, 400 h2g->info.size) - 401 h2g->info.resv_space; 402 if (cmd_len > h2g->info.space) 403 return false; 404 } 405 406 return true; 407 } 408 409 static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len) 410 { 411 if (!g2h_len) 412 return true; 413 414 lockdep_assert_held(&ct->fast_lock); 415 416 return ct->ctbs.g2h.info.space > g2h_len; 417 } 418 419 static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len) 420 { 421 lockdep_assert_held(&ct->lock); 422 423 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len)) 424 return -EBUSY; 425 426 return 0; 427 } 428 429 static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len) 430 { 431 lockdep_assert_held(&ct->lock); 432 ct->ctbs.h2g.info.space -= cmd_len; 433 } 434 435 static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h) 436 { 437 xe_gt_assert(ct_to_gt(ct), g2h_len <= ct->ctbs.g2h.info.space); 438 439 if (g2h_len) { 440 lockdep_assert_held(&ct->fast_lock); 441 442 ct->ctbs.g2h.info.space -= g2h_len; 443 ct->g2h_outstanding += num_g2h; 444 } 445 } 446 447 static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 448 { 449 lockdep_assert_held(&ct->fast_lock); 450 xe_gt_assert(ct_to_gt(ct), ct->ctbs.g2h.info.space + g2h_len <= 451 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space); 452 453 ct->ctbs.g2h.info.space += g2h_len; 454 --ct->g2h_outstanding; 455 } 456 457 static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) 458 { 459 spin_lock_irq(&ct->fast_lock); 460 __g2h_release_space(ct, g2h_len); 461 spin_unlock_irq(&ct->fast_lock); 462 } 463 464 #define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */ 465 466 static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, 467 u32 ct_fence_value, bool want_response) 468 { 469 struct xe_device *xe = ct_to_xe(ct); 470 struct xe_gt *gt = ct_to_gt(ct); 471 struct guc_ctb *h2g = &ct->ctbs.h2g; 472 u32 cmd[H2G_CT_HEADERS]; 473 u32 tail = h2g->info.tail; 474 u32 full_len; 475 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds, 476 tail * sizeof(u32)); 477 478 full_len = len + GUC_CTB_HDR_LEN; 479 480 lockdep_assert_held(&ct->lock); 481 xe_gt_assert(gt, full_len <= GUC_CTB_MSG_MAX_LEN); 482 xe_gt_assert(gt, tail <= h2g->info.size); 483 484 /* Command will wrap, zero fill (NOPs), return and check credits again */ 485 if (tail + full_len > h2g->info.size) { 486 xe_map_memset(xe, &map, 0, 0, 487 (h2g->info.size - tail) * sizeof(u32)); 488 h2g_reserve_space(ct, (h2g->info.size - tail)); 489 h2g->info.tail = 0; 490 desc_write(xe, h2g, tail, h2g->info.tail); 491 492 return -EAGAIN; 493 } 494 495 /* 496 * dw0: CT header (including fence) 497 * dw1: HXG header (including action code) 498 * dw2+: action data 499 */ 500 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | 501 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | 502 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value); 503 if (want_response) { 504 cmd[1] = 505 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | 506 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 507 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 508 } else { 509 cmd[1] = 510 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_FAST_REQUEST) | 511 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | 512 GUC_HXG_EVENT_MSG_0_DATA0, action[0]); 513 } 514 515 /* H2G header in cmd[1] replaces action[0] so: */ 516 --len; 517 ++action; 518 519 /* Write H2G ensuring visable before descriptor update */ 520 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32)); 521 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32)); 522 xe_device_wmb(xe); 523 524 /* Update local copies */ 525 h2g->info.tail = (tail + full_len) % h2g->info.size; 526 h2g_reserve_space(ct, full_len); 527 528 /* Update descriptor */ 529 desc_write(xe, h2g, tail, h2g->info.tail); 530 531 trace_xe_guc_ctb_h2g(gt->info.id, *(action - 1), full_len, 532 desc_read(xe, h2g, head), h2g->info.tail); 533 534 return 0; 535 } 536 537 /* 538 * The CT protocol accepts a 16 bits fence. This field is fully owned by the 539 * driver, the GuC will just copy it to the reply message. Since we need to 540 * be able to distinguish between replies to REQUEST and FAST_REQUEST messages, 541 * we use one bit of the seqno as an indicator for that and a rolling counter 542 * for the remaining 15 bits. 543 */ 544 #define CT_SEQNO_MASK GENMASK(14, 0) 545 #define CT_SEQNO_UNTRACKED BIT(15) 546 static u16 next_ct_seqno(struct xe_guc_ct *ct, bool is_g2h_fence) 547 { 548 u32 seqno = ct->fence_seqno++ & CT_SEQNO_MASK; 549 550 if (!is_g2h_fence) 551 seqno |= CT_SEQNO_UNTRACKED; 552 553 return seqno; 554 } 555 556 static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, 557 u32 len, u32 g2h_len, u32 num_g2h, 558 struct g2h_fence *g2h_fence) 559 { 560 struct xe_gt *gt __maybe_unused = ct_to_gt(ct); 561 u16 seqno; 562 int ret; 563 564 xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED); 565 xe_gt_assert(gt, !g2h_len || !g2h_fence); 566 xe_gt_assert(gt, !num_g2h || !g2h_fence); 567 xe_gt_assert(gt, !g2h_len || num_g2h); 568 xe_gt_assert(gt, g2h_len || !num_g2h); 569 lockdep_assert_held(&ct->lock); 570 571 if (unlikely(ct->ctbs.h2g.info.broken)) { 572 ret = -EPIPE; 573 goto out; 574 } 575 576 if (ct->state == XE_GUC_CT_STATE_DISABLED) { 577 ret = -ENODEV; 578 goto out; 579 } 580 581 if (ct->state == XE_GUC_CT_STATE_STOPPED) { 582 ret = -ECANCELED; 583 goto out; 584 } 585 586 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 587 588 if (g2h_fence) { 589 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; 590 num_g2h = 1; 591 592 if (g2h_fence_needs_alloc(g2h_fence)) { 593 void *ptr; 594 595 g2h_fence->seqno = next_ct_seqno(ct, true); 596 ptr = xa_store(&ct->fence_lookup, 597 g2h_fence->seqno, 598 g2h_fence, GFP_ATOMIC); 599 if (IS_ERR(ptr)) { 600 ret = PTR_ERR(ptr); 601 goto out; 602 } 603 } 604 605 seqno = g2h_fence->seqno; 606 } else { 607 seqno = next_ct_seqno(ct, false); 608 } 609 610 if (g2h_len) 611 spin_lock_irq(&ct->fast_lock); 612 retry: 613 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len); 614 if (unlikely(ret)) 615 goto out_unlock; 616 617 ret = h2g_write(ct, action, len, seqno, !!g2h_fence); 618 if (unlikely(ret)) { 619 if (ret == -EAGAIN) 620 goto retry; 621 goto out_unlock; 622 } 623 624 __g2h_reserve_space(ct, g2h_len, num_g2h); 625 xe_guc_notify(ct_to_guc(ct)); 626 out_unlock: 627 if (g2h_len) 628 spin_unlock_irq(&ct->fast_lock); 629 out: 630 return ret; 631 } 632 633 static void kick_reset(struct xe_guc_ct *ct) 634 { 635 xe_gt_reset_async(ct_to_gt(ct)); 636 } 637 638 static int dequeue_one_g2h(struct xe_guc_ct *ct); 639 640 static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 641 u32 g2h_len, u32 num_g2h, 642 struct g2h_fence *g2h_fence) 643 { 644 struct xe_gt *gt = ct_to_gt(ct); 645 struct drm_printer p = xe_gt_info_printer(gt); 646 unsigned int sleep_period_ms = 1; 647 int ret; 648 649 xe_gt_assert(gt, !g2h_len || !g2h_fence); 650 lockdep_assert_held(&ct->lock); 651 xe_device_assert_mem_access(ct_to_xe(ct)); 652 653 try_again: 654 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, 655 g2h_fence); 656 657 /* 658 * We wait to try to restore credits for about 1 second before bailing. 659 * In the case of H2G credits we have no choice but just to wait for the 660 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In 661 * the case of G2H we process any G2H in the channel, hopefully freeing 662 * credits as we consume the G2H messages. 663 */ 664 if (unlikely(ret == -EBUSY && 665 !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) { 666 struct guc_ctb *h2g = &ct->ctbs.h2g; 667 668 if (sleep_period_ms == 1024) 669 goto broken; 670 671 trace_xe_guc_ct_h2g_flow_control(h2g->info.head, h2g->info.tail, 672 h2g->info.size, 673 h2g->info.space, 674 len + GUC_CTB_HDR_LEN); 675 msleep(sleep_period_ms); 676 sleep_period_ms <<= 1; 677 678 goto try_again; 679 } else if (unlikely(ret == -EBUSY)) { 680 struct xe_device *xe = ct_to_xe(ct); 681 struct guc_ctb *g2h = &ct->ctbs.g2h; 682 683 trace_xe_guc_ct_g2h_flow_control(g2h->info.head, 684 desc_read(xe, g2h, tail), 685 g2h->info.size, 686 g2h->info.space, 687 g2h_fence ? 688 GUC_CTB_HXG_MSG_MAX_LEN : 689 g2h_len); 690 691 #define g2h_avail(ct) \ 692 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head) 693 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding || 694 g2h_avail(ct), HZ)) 695 goto broken; 696 #undef g2h_avail 697 698 if (dequeue_one_g2h(ct) < 0) 699 goto broken; 700 701 goto try_again; 702 } 703 704 return ret; 705 706 broken: 707 xe_gt_err(gt, "No forward process on H2G, reset required\n"); 708 xe_guc_ct_print(ct, &p, true); 709 ct->ctbs.h2g.info.broken = true; 710 711 return -EDEADLK; 712 } 713 714 static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 715 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) 716 { 717 int ret; 718 719 xe_gt_assert(ct_to_gt(ct), !g2h_len || !g2h_fence); 720 721 mutex_lock(&ct->lock); 722 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence); 723 mutex_unlock(&ct->lock); 724 725 return ret; 726 } 727 728 int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, 729 u32 g2h_len, u32 num_g2h) 730 { 731 int ret; 732 733 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL); 734 if (ret == -EDEADLK) 735 kick_reset(ct); 736 737 return ret; 738 } 739 740 int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, 741 u32 g2h_len, u32 num_g2h) 742 { 743 int ret; 744 745 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL); 746 if (ret == -EDEADLK) 747 kick_reset(ct); 748 749 return ret; 750 } 751 752 int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len) 753 { 754 int ret; 755 756 lockdep_assert_held(&ct->lock); 757 758 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL); 759 if (ret == -EDEADLK) 760 kick_reset(ct); 761 762 return ret; 763 } 764 765 /* 766 * Check if a GT reset is in progress or will occur and if GT reset brought the 767 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset. 768 */ 769 static bool retry_failure(struct xe_guc_ct *ct, int ret) 770 { 771 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV)) 772 return false; 773 774 #define ct_alive(ct) \ 775 (xe_guc_ct_enabled(ct) && !ct->ctbs.h2g.info.broken && \ 776 !ct->ctbs.g2h.info.broken) 777 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) 778 return false; 779 #undef ct_alive 780 781 return true; 782 } 783 784 static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 785 u32 *response_buffer, bool no_fail) 786 { 787 struct xe_gt *gt = ct_to_gt(ct); 788 struct g2h_fence g2h_fence; 789 int ret = 0; 790 791 /* 792 * We use a fence to implement blocking sends / receiving response data. 793 * The seqno of the fence is sent in the H2G, returned in the G2H, and 794 * an xarray is used as storage media with the seqno being to key. 795 * Fields in the fence hold success, failure, retry status and the 796 * response data. Safe to allocate on the stack as the xarray is the 797 * only reference and it cannot be present after this function exits. 798 */ 799 retry: 800 g2h_fence_init(&g2h_fence, response_buffer); 801 retry_same_fence: 802 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence); 803 if (unlikely(ret == -ENOMEM)) { 804 void *ptr; 805 806 /* Retry allocation /w GFP_KERNEL */ 807 ptr = xa_store(&ct->fence_lookup, 808 g2h_fence.seqno, 809 &g2h_fence, GFP_KERNEL); 810 if (IS_ERR(ptr)) 811 return PTR_ERR(ptr); 812 813 goto retry_same_fence; 814 } else if (unlikely(ret)) { 815 if (ret == -EDEADLK) 816 kick_reset(ct); 817 818 if (no_fail && retry_failure(ct, ret)) 819 goto retry_same_fence; 820 821 if (!g2h_fence_needs_alloc(&g2h_fence)) 822 xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno); 823 824 return ret; 825 } 826 827 ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ); 828 if (!ret) { 829 xe_gt_err(gt, "Timed out wait for G2H, fence %u, action %04x", 830 g2h_fence.seqno, action[0]); 831 xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno); 832 return -ETIME; 833 } 834 835 if (g2h_fence.retry) { 836 xe_gt_warn(gt, "H2G retry, action 0x%04x, reason %u", 837 action[0], g2h_fence.reason); 838 goto retry; 839 } 840 if (g2h_fence.fail) { 841 xe_gt_err(gt, "H2G send failed, action 0x%04x, error %d, hint %u", 842 action[0], g2h_fence.error, g2h_fence.hint); 843 ret = -EIO; 844 } 845 846 return ret > 0 ? response_buffer ? g2h_fence.response_len : g2h_fence.response_data : ret; 847 } 848 849 /** 850 * xe_guc_ct_send_recv - Send and receive HXG to the GuC 851 * @ct: the &xe_guc_ct 852 * @action: the dword array with `HXG Request`_ message (can't be NULL) 853 * @len: length of the `HXG Request`_ message (in dwords, can't be 0) 854 * @response_buffer: placeholder for the `HXG Response`_ message (can be NULL) 855 * 856 * Send a `HXG Request`_ message to the GuC over CT communication channel and 857 * blocks until GuC replies with a `HXG Response`_ message. 858 * 859 * For non-blocking communication with GuC use xe_guc_ct_send(). 860 * 861 * Note: The size of &response_buffer must be at least GUC_CTB_MAX_DWORDS_. 862 * 863 * Return: response length (in dwords) if &response_buffer was not NULL, or 864 * DATA0 from `HXG Response`_ if &response_buffer was NULL, or 865 * a negative error code on failure. 866 */ 867 int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, 868 u32 *response_buffer) 869 { 870 KUNIT_STATIC_STUB_REDIRECT(xe_guc_ct_send_recv, ct, action, len, response_buffer); 871 return guc_ct_send_recv(ct, action, len, response_buffer, false); 872 } 873 874 int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, 875 u32 len, u32 *response_buffer) 876 { 877 return guc_ct_send_recv(ct, action, len, response_buffer, true); 878 } 879 880 static u32 *msg_to_hxg(u32 *msg) 881 { 882 return msg + GUC_CTB_MSG_MIN_LEN; 883 } 884 885 static u32 msg_len_to_hxg_len(u32 len) 886 { 887 return len - GUC_CTB_MSG_MIN_LEN; 888 } 889 890 static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) 891 { 892 u32 *hxg = msg_to_hxg(msg); 893 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 894 895 lockdep_assert_held(&ct->lock); 896 897 switch (action) { 898 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 899 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 900 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 901 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 902 g2h_release_space(ct, len); 903 } 904 905 return 0; 906 } 907 908 static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) 909 { 910 struct xe_gt *gt = ct_to_gt(ct); 911 u32 *hxg = msg_to_hxg(msg); 912 u32 hxg_len = msg_len_to_hxg_len(len); 913 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); 914 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 915 struct g2h_fence *g2h_fence; 916 917 lockdep_assert_held(&ct->lock); 918 919 /* 920 * Fences for FAST_REQUEST messages are not tracked in ct->fence_lookup. 921 * Those messages should never fail, so if we do get an error back it 922 * means we're likely doing an illegal operation and the GuC is 923 * rejecting it. We have no way to inform the code that submitted the 924 * H2G that the message was rejected, so we need to escalate the 925 * failure to trigger a reset. 926 */ 927 if (fence & CT_SEQNO_UNTRACKED) { 928 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) 929 xe_gt_err(gt, "FAST_REQ H2G fence 0x%x failed! e=0x%x, h=%u\n", 930 fence, 931 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]), 932 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0])); 933 else 934 xe_gt_err(gt, "unexpected response %u for FAST_REQ H2G fence 0x%x!\n", 935 type, fence); 936 937 return -EPROTO; 938 } 939 940 g2h_fence = xa_erase(&ct->fence_lookup, fence); 941 if (unlikely(!g2h_fence)) { 942 /* Don't tear down channel, as send could've timed out */ 943 xe_gt_warn(gt, "G2H fence (%u) not found!\n", fence); 944 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 945 return 0; 946 } 947 948 xe_gt_assert(gt, fence == g2h_fence->seqno); 949 950 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { 951 g2h_fence->fail = true; 952 g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]); 953 g2h_fence->hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, hxg[0]); 954 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { 955 g2h_fence->retry = true; 956 g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]); 957 } else if (g2h_fence->response_buffer) { 958 g2h_fence->response_len = hxg_len; 959 memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32)); 960 } else { 961 g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]); 962 } 963 964 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); 965 966 g2h_fence->done = true; 967 smp_mb(); 968 969 wake_up_all(&ct->g2h_fence_wq); 970 971 return 0; 972 } 973 974 static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 975 { 976 struct xe_gt *gt = ct_to_gt(ct); 977 u32 *hxg = msg_to_hxg(msg); 978 u32 origin, type; 979 int ret; 980 981 lockdep_assert_held(&ct->lock); 982 983 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); 984 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { 985 xe_gt_err(gt, "G2H channel broken on read, origin=%u, reset required\n", 986 origin); 987 ct->ctbs.g2h.info.broken = true; 988 989 return -EPROTO; 990 } 991 992 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); 993 switch (type) { 994 case GUC_HXG_TYPE_EVENT: 995 ret = parse_g2h_event(ct, msg, len); 996 break; 997 case GUC_HXG_TYPE_RESPONSE_SUCCESS: 998 case GUC_HXG_TYPE_RESPONSE_FAILURE: 999 case GUC_HXG_TYPE_NO_RESPONSE_RETRY: 1000 ret = parse_g2h_response(ct, msg, len); 1001 break; 1002 default: 1003 xe_gt_err(gt, "G2H channel broken on read, type=%u, reset required\n", 1004 type); 1005 ct->ctbs.g2h.info.broken = true; 1006 1007 ret = -EOPNOTSUPP; 1008 } 1009 1010 return ret; 1011 } 1012 1013 static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) 1014 { 1015 struct xe_guc *guc = ct_to_guc(ct); 1016 struct xe_gt *gt = ct_to_gt(ct); 1017 u32 hxg_len = msg_len_to_hxg_len(len); 1018 u32 *hxg = msg_to_hxg(msg); 1019 u32 action, adj_len; 1020 u32 *payload; 1021 int ret = 0; 1022 1023 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1024 return 0; 1025 1026 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1027 payload = hxg + GUC_HXG_EVENT_MSG_MIN_LEN; 1028 adj_len = hxg_len - GUC_HXG_EVENT_MSG_MIN_LEN; 1029 1030 switch (action) { 1031 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: 1032 ret = xe_guc_sched_done_handler(guc, payload, adj_len); 1033 break; 1034 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: 1035 ret = xe_guc_deregister_done_handler(guc, payload, adj_len); 1036 break; 1037 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: 1038 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len); 1039 break; 1040 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: 1041 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload, 1042 adj_len); 1043 break; 1044 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: 1045 /* Selftest only at the moment */ 1046 break; 1047 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: 1048 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: 1049 /* FIXME: Handle this */ 1050 break; 1051 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: 1052 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload, 1053 adj_len); 1054 break; 1055 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1056 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1057 break; 1058 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1059 ret = xe_guc_tlb_invalidation_done_handler(guc, payload, 1060 adj_len); 1061 break; 1062 case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY: 1063 ret = xe_guc_access_counter_notify_handler(guc, payload, 1064 adj_len); 1065 break; 1066 case XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF: 1067 ret = xe_guc_relay_process_guc2pf(&guc->relay, hxg, hxg_len); 1068 break; 1069 case XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF: 1070 ret = xe_guc_relay_process_guc2vf(&guc->relay, hxg, hxg_len); 1071 break; 1072 case GUC_ACTION_GUC2PF_VF_STATE_NOTIFY: 1073 ret = xe_gt_sriov_pf_control_process_guc2pf(gt, hxg, hxg_len); 1074 break; 1075 case GUC_ACTION_GUC2PF_ADVERSE_EVENT: 1076 ret = xe_gt_sriov_pf_monitor_process_guc2pf(gt, hxg, hxg_len); 1077 break; 1078 default: 1079 xe_gt_err(gt, "unexpected G2H action 0x%04x\n", action); 1080 } 1081 1082 if (ret) 1083 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n", 1084 action, ERR_PTR(ret)); 1085 1086 return 0; 1087 } 1088 1089 static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) 1090 { 1091 struct xe_device *xe = ct_to_xe(ct); 1092 struct xe_gt *gt = ct_to_gt(ct); 1093 struct guc_ctb *g2h = &ct->ctbs.g2h; 1094 u32 tail, head, len; 1095 s32 avail; 1096 u32 action; 1097 u32 *hxg; 1098 1099 xe_gt_assert(gt, ct->state != XE_GUC_CT_STATE_NOT_INITIALIZED); 1100 lockdep_assert_held(&ct->fast_lock); 1101 1102 if (ct->state == XE_GUC_CT_STATE_DISABLED) 1103 return -ENODEV; 1104 1105 if (ct->state == XE_GUC_CT_STATE_STOPPED) 1106 return -ECANCELED; 1107 1108 if (g2h->info.broken) 1109 return -EPIPE; 1110 1111 xe_gt_assert(gt, xe_guc_ct_enabled(ct)); 1112 1113 /* Calculate DW available to read */ 1114 tail = desc_read(xe, g2h, tail); 1115 avail = tail - g2h->info.head; 1116 if (unlikely(avail == 0)) 1117 return 0; 1118 1119 if (avail < 0) 1120 avail += g2h->info.size; 1121 1122 /* Read header */ 1123 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head, 1124 sizeof(u32)); 1125 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; 1126 if (len > avail) { 1127 xe_gt_err(gt, "G2H channel broken on read, avail=%d, len=%d, reset required\n", 1128 avail, len); 1129 g2h->info.broken = true; 1130 1131 return -EPROTO; 1132 } 1133 1134 head = (g2h->info.head + 1) % g2h->info.size; 1135 avail = len - 1; 1136 1137 /* Read G2H message */ 1138 if (avail + head > g2h->info.size) { 1139 u32 avail_til_wrap = g2h->info.size - head; 1140 1141 xe_map_memcpy_from(xe, msg + 1, 1142 &g2h->cmds, sizeof(u32) * head, 1143 avail_til_wrap * sizeof(u32)); 1144 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap, 1145 &g2h->cmds, 0, 1146 (avail - avail_til_wrap) * sizeof(u32)); 1147 } else { 1148 xe_map_memcpy_from(xe, msg + 1, 1149 &g2h->cmds, sizeof(u32) * head, 1150 avail * sizeof(u32)); 1151 } 1152 1153 hxg = msg_to_hxg(msg); 1154 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1155 1156 if (fast_path) { 1157 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT) 1158 return 0; 1159 1160 switch (action) { 1161 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1162 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1163 break; /* Process these in fast-path */ 1164 default: 1165 return 0; 1166 } 1167 } 1168 1169 /* Update local / descriptor header */ 1170 g2h->info.head = (head + avail) % g2h->info.size; 1171 desc_write(xe, g2h, head, g2h->info.head); 1172 1173 trace_xe_guc_ctb_g2h(ct_to_gt(ct)->info.id, action, len, 1174 g2h->info.head, tail); 1175 1176 return len; 1177 } 1178 1179 static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) 1180 { 1181 struct xe_gt *gt = ct_to_gt(ct); 1182 struct xe_guc *guc = ct_to_guc(ct); 1183 u32 hxg_len = msg_len_to_hxg_len(len); 1184 u32 *hxg = msg_to_hxg(msg); 1185 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); 1186 u32 *payload = hxg + GUC_HXG_MSG_MIN_LEN; 1187 u32 adj_len = hxg_len - GUC_HXG_MSG_MIN_LEN; 1188 int ret = 0; 1189 1190 switch (action) { 1191 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: 1192 ret = xe_guc_pagefault_handler(guc, payload, adj_len); 1193 break; 1194 case XE_GUC_ACTION_TLB_INVALIDATION_DONE: 1195 __g2h_release_space(ct, len); 1196 ret = xe_guc_tlb_invalidation_done_handler(guc, payload, 1197 adj_len); 1198 break; 1199 default: 1200 xe_gt_warn(gt, "NOT_POSSIBLE"); 1201 } 1202 1203 if (ret) 1204 xe_gt_err(gt, "G2H action 0x%04x failed (%pe)\n", 1205 action, ERR_PTR(ret)); 1206 } 1207 1208 /** 1209 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler 1210 * @ct: GuC CT object 1211 * 1212 * Anything related to page faults is critical for performance, process these 1213 * critical G2H in the IRQ. This is safe as these handlers either just wake up 1214 * waiters or queue another worker. 1215 */ 1216 void xe_guc_ct_fast_path(struct xe_guc_ct *ct) 1217 { 1218 struct xe_device *xe = ct_to_xe(ct); 1219 bool ongoing; 1220 int len; 1221 1222 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1223 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1224 return; 1225 1226 spin_lock(&ct->fast_lock); 1227 do { 1228 len = g2h_read(ct, ct->fast_msg, true); 1229 if (len > 0) 1230 g2h_fast_path(ct, ct->fast_msg, len); 1231 } while (len > 0); 1232 spin_unlock(&ct->fast_lock); 1233 1234 if (ongoing) 1235 xe_pm_runtime_put(xe); 1236 } 1237 1238 /* Returns less than zero on error, 0 on done, 1 on more available */ 1239 static int dequeue_one_g2h(struct xe_guc_ct *ct) 1240 { 1241 int len; 1242 int ret; 1243 1244 lockdep_assert_held(&ct->lock); 1245 1246 spin_lock_irq(&ct->fast_lock); 1247 len = g2h_read(ct, ct->msg, false); 1248 spin_unlock_irq(&ct->fast_lock); 1249 if (len <= 0) 1250 return len; 1251 1252 ret = parse_g2h_msg(ct, ct->msg, len); 1253 if (unlikely(ret < 0)) 1254 return ret; 1255 1256 ret = process_g2h_msg(ct, ct->msg, len); 1257 if (unlikely(ret < 0)) 1258 return ret; 1259 1260 return 1; 1261 } 1262 1263 static void g2h_worker_func(struct work_struct *w) 1264 { 1265 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker); 1266 struct xe_gt *gt = ct_to_gt(ct); 1267 bool ongoing; 1268 int ret; 1269 1270 /* 1271 * Normal users must always hold mem_access.ref around CT calls. However 1272 * during the runtime pm callbacks we rely on CT to talk to the GuC, but 1273 * at this stage we can't rely on mem_access.ref and even the 1274 * callback_task will be different than current. For such cases we just 1275 * need to ensure we always process the responses from any blocking 1276 * ct_send requests or where we otherwise expect some response when 1277 * initiated from those callbacks (which will need to wait for the below 1278 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if 1279 * the device has suspended to the point that the CT communication has 1280 * been disabled. 1281 * 1282 * If we are inside the runtime pm callback, we can be the only task 1283 * still issuing CT requests (since that requires having the 1284 * mem_access.ref). It seems like it might in theory be possible to 1285 * receive unsolicited events from the GuC just as we are 1286 * suspending-resuming, but those will currently anyway be lost when 1287 * eventually exiting from suspend, hence no need to wake up the device 1288 * here. If we ever need something stronger than get_if_ongoing() then 1289 * we need to be careful with blocking the pm callbacks from getting CT 1290 * responses, if the worker here is blocked on those callbacks 1291 * completing, creating a deadlock. 1292 */ 1293 ongoing = xe_pm_runtime_get_if_active(ct_to_xe(ct)); 1294 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL) 1295 return; 1296 1297 do { 1298 mutex_lock(&ct->lock); 1299 ret = dequeue_one_g2h(ct); 1300 mutex_unlock(&ct->lock); 1301 1302 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) { 1303 struct drm_printer p = xe_gt_info_printer(gt); 1304 1305 xe_guc_ct_print(ct, &p, false); 1306 kick_reset(ct); 1307 } 1308 } while (ret == 1); 1309 1310 if (ongoing) 1311 xe_pm_runtime_put(ct_to_xe(ct)); 1312 } 1313 1314 static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb, 1315 struct guc_ctb_snapshot *snapshot, 1316 bool atomic) 1317 { 1318 u32 head, tail; 1319 1320 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0, 1321 sizeof(struct guc_ct_buffer_desc)); 1322 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info)); 1323 1324 snapshot->cmds = kmalloc_array(ctb->info.size, sizeof(u32), 1325 atomic ? GFP_ATOMIC : GFP_KERNEL); 1326 1327 if (!snapshot->cmds) { 1328 drm_err(&xe->drm, "Skipping CTB commands snapshot. Only CTB info will be available.\n"); 1329 return; 1330 } 1331 1332 head = snapshot->desc.head; 1333 tail = snapshot->desc.tail; 1334 1335 if (head != tail) { 1336 struct iosys_map map = 1337 IOSYS_MAP_INIT_OFFSET(&ctb->cmds, head * sizeof(u32)); 1338 1339 while (head != tail) { 1340 snapshot->cmds[head] = xe_map_rd(xe, &map, 0, u32); 1341 ++head; 1342 if (head == ctb->info.size) { 1343 head = 0; 1344 map = ctb->cmds; 1345 } else { 1346 iosys_map_incr(&map, sizeof(u32)); 1347 } 1348 } 1349 } 1350 } 1351 1352 static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot, 1353 struct drm_printer *p) 1354 { 1355 u32 head, tail; 1356 1357 drm_printf(p, "\tsize: %d\n", snapshot->info.size); 1358 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space); 1359 drm_printf(p, "\thead: %d\n", snapshot->info.head); 1360 drm_printf(p, "\ttail: %d\n", snapshot->info.tail); 1361 drm_printf(p, "\tspace: %d\n", snapshot->info.space); 1362 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken); 1363 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head); 1364 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail); 1365 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status); 1366 1367 if (!snapshot->cmds) 1368 return; 1369 1370 head = snapshot->desc.head; 1371 tail = snapshot->desc.tail; 1372 1373 while (head != tail) { 1374 drm_printf(p, "\tcmd[%d]: 0x%08x\n", head, 1375 snapshot->cmds[head]); 1376 ++head; 1377 if (head == snapshot->info.size) 1378 head = 0; 1379 } 1380 } 1381 1382 static void guc_ctb_snapshot_free(struct guc_ctb_snapshot *snapshot) 1383 { 1384 kfree(snapshot->cmds); 1385 } 1386 1387 /** 1388 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state. 1389 * @ct: GuC CT object. 1390 * @atomic: Boolean to indicate if this is called from atomic context like 1391 * reset or CTB handler or from some regular path like debugfs. 1392 * 1393 * This can be printed out in a later stage like during dev_coredump 1394 * analysis. 1395 * 1396 * Returns: a GuC CT snapshot object that must be freed by the caller 1397 * by using `xe_guc_ct_snapshot_free`. 1398 */ 1399 struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct, 1400 bool atomic) 1401 { 1402 struct xe_device *xe = ct_to_xe(ct); 1403 struct xe_guc_ct_snapshot *snapshot; 1404 1405 snapshot = kzalloc(sizeof(*snapshot), 1406 atomic ? GFP_ATOMIC : GFP_KERNEL); 1407 1408 if (!snapshot) { 1409 drm_err(&xe->drm, "Skipping CTB snapshot entirely.\n"); 1410 return NULL; 1411 } 1412 1413 if (xe_guc_ct_enabled(ct) || ct->state == XE_GUC_CT_STATE_STOPPED) { 1414 snapshot->ct_enabled = true; 1415 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding); 1416 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g, 1417 &snapshot->h2g, atomic); 1418 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h, 1419 &snapshot->g2h, atomic); 1420 } 1421 1422 return snapshot; 1423 } 1424 1425 /** 1426 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot. 1427 * @snapshot: GuC CT snapshot object. 1428 * @p: drm_printer where it will be printed out. 1429 * 1430 * This function prints out a given GuC CT snapshot object. 1431 */ 1432 void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot, 1433 struct drm_printer *p) 1434 { 1435 if (!snapshot) 1436 return; 1437 1438 if (snapshot->ct_enabled) { 1439 drm_puts(p, "H2G CTB (all sizes in DW):\n"); 1440 guc_ctb_snapshot_print(&snapshot->h2g, p); 1441 1442 drm_puts(p, "\nG2H CTB (all sizes in DW):\n"); 1443 guc_ctb_snapshot_print(&snapshot->g2h, p); 1444 1445 drm_printf(p, "\tg2h outstanding: %d\n", 1446 snapshot->g2h_outstanding); 1447 } else { 1448 drm_puts(p, "CT disabled\n"); 1449 } 1450 } 1451 1452 /** 1453 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot. 1454 * @snapshot: GuC CT snapshot object. 1455 * 1456 * This function free all the memory that needed to be allocated at capture 1457 * time. 1458 */ 1459 void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot) 1460 { 1461 if (!snapshot) 1462 return; 1463 1464 guc_ctb_snapshot_free(&snapshot->h2g); 1465 guc_ctb_snapshot_free(&snapshot->g2h); 1466 kfree(snapshot); 1467 } 1468 1469 /** 1470 * xe_guc_ct_print - GuC CT Print. 1471 * @ct: GuC CT. 1472 * @p: drm_printer where it will be printed out. 1473 * @atomic: Boolean to indicate if this is called from atomic context like 1474 * reset or CTB handler or from some regular path like debugfs. 1475 * 1476 * This function quickly capture a snapshot and immediately print it out. 1477 */ 1478 void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool atomic) 1479 { 1480 struct xe_guc_ct_snapshot *snapshot; 1481 1482 snapshot = xe_guc_ct_snapshot_capture(ct, atomic); 1483 xe_guc_ct_snapshot_print(snapshot, p); 1484 xe_guc_ct_snapshot_free(snapshot); 1485 } 1486