1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_lrc.h" 7 8 #include <generated/xe_wa_oob.h> 9 10 #include <linux/ascii85.h> 11 #include <linux/panic.h> 12 13 #include "instructions/xe_mi_commands.h" 14 #include "instructions/xe_gfxpipe_commands.h" 15 #include "instructions/xe_gfx_state_commands.h" 16 #include "regs/xe_engine_regs.h" 17 #include "regs/xe_gt_regs.h" 18 #include "regs/xe_lrc_layout.h" 19 #include "xe_bb.h" 20 #include "xe_bo.h" 21 #include "xe_configfs.h" 22 #include "xe_device.h" 23 #include "xe_drm_client.h" 24 #include "xe_exec_queue.h" 25 #include "xe_exec_queue_types.h" 26 #include "xe_gt.h" 27 #include "xe_gt_clock.h" 28 #include "xe_gt_printk.h" 29 #include "xe_hw_fence.h" 30 #include "xe_map.h" 31 #include "xe_memirq.h" 32 #include "xe_mmio.h" 33 #include "xe_ring_ops.h" 34 #include "xe_sriov.h" 35 #include "xe_trace_lrc.h" 36 #include "xe_vm.h" 37 #include "xe_wa.h" 38 39 #define LRC_VALID BIT_ULL(0) 40 #define LRC_PRIVILEGE BIT_ULL(8) 41 #define LRC_ADDRESSING_MODE GENMASK_ULL(4, 3) 42 #define LRC_LEGACY_64B_CONTEXT 3 43 44 #define LRC_ENGINE_CLASS GENMASK_ULL(63, 61) 45 #define LRC_ENGINE_INSTANCE GENMASK_ULL(53, 48) 46 47 #define LRC_PPHWSP_SIZE SZ_4K 48 #define LRC_INDIRECT_CTX_BO_SIZE SZ_4K 49 #define LRC_INDIRECT_RING_STATE_SIZE SZ_4K 50 51 #define LRC_PRIORITY GENMASK_ULL(10, 9) 52 #define LRC_PRIORITY_LOW 0 53 #define LRC_PRIORITY_NORMAL 1 54 #define LRC_PRIORITY_HIGH 2 55 56 /* 57 * Layout of the LRC and associated data allocated as 58 * lrc->bo: 59 * 60 * Region Size 61 * +============================+=================================+ <- __xe_lrc_ring_offset() 62 * | Ring | ring_size, see | 63 * | | xe_lrc_init() | 64 * +============================+=================================+ <- __xe_lrc_pphwsp_offset() 65 * | PPHWSP (includes SW state) | 4K | 66 * +----------------------------+---------------------------------+ <- __xe_lrc_regs_offset() 67 * | Engine Context Image | n * 4K, see | 68 * | | xe_gt_lrc_size() | 69 * +----------------------------+---------------------------------+ <- __xe_lrc_indirect_ring_offset() 70 * | Indirect Ring State Page | 0 or 4k, see | 71 * | | XE_LRC_FLAG_INDIRECT_RING_STATE | 72 * +============================+=================================+ <- __xe_lrc_indirect_ctx_offset() 73 * | Indirect Context Page | 0 or 4k, see | 74 * | | XE_LRC_FLAG_INDIRECT_CTX | 75 * +============================+=================================+ <- __xe_lrc_wa_bb_offset() 76 * | WA BB Per Ctx | 4k | 77 * +============================+=================================+ <- xe_bo_size(lrc->bo) 78 */ 79 80 static struct xe_device * 81 lrc_to_xe(struct xe_lrc *lrc) 82 { 83 return gt_to_xe(lrc->fence_ctx.gt); 84 } 85 86 static bool 87 gt_engine_needs_indirect_ctx(struct xe_gt *gt, enum xe_engine_class class) 88 { 89 struct xe_device *xe = gt_to_xe(gt); 90 91 if (XE_GT_WA(gt, 16010904313) && 92 (class == XE_ENGINE_CLASS_RENDER || 93 class == XE_ENGINE_CLASS_COMPUTE)) 94 return true; 95 96 if (xe_configfs_get_ctx_restore_mid_bb(to_pci_dev(xe->drm.dev), 97 class, NULL)) 98 return true; 99 100 if (gt->ring_ops[class]->emit_aux_table_inv) 101 return true; 102 103 return false; 104 } 105 106 /** 107 * xe_gt_lrc_hang_replay_size() - Hang replay size 108 * @gt: The GT 109 * @class: Hardware engine class 110 * 111 * Determine size of GPU hang replay state for a GT and hardware engine class. 112 * 113 * Return: Size of GPU hang replay size 114 */ 115 size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class) 116 { 117 struct xe_device *xe = gt_to_xe(gt); 118 size_t size = 0; 119 120 /* Engine context image */ 121 switch (class) { 122 case XE_ENGINE_CLASS_RENDER: 123 if (GRAPHICS_VERx100(xe) >= 3510) 124 size += 7 * SZ_4K; 125 else if (GRAPHICS_VER(xe) >= 20) 126 size += 3 * SZ_4K; 127 else 128 size += 13 * SZ_4K; 129 break; 130 case XE_ENGINE_CLASS_COMPUTE: 131 if (GRAPHICS_VERx100(xe) >= 3510) 132 size += 5 * SZ_4K; 133 else if (GRAPHICS_VER(xe) >= 20) 134 size += 2 * SZ_4K; 135 else 136 size += 13 * SZ_4K; 137 break; 138 default: 139 WARN(1, "Unknown engine class: %d", class); 140 fallthrough; 141 case XE_ENGINE_CLASS_COPY: 142 case XE_ENGINE_CLASS_VIDEO_DECODE: 143 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 144 case XE_ENGINE_CLASS_OTHER: 145 size += 1 * SZ_4K; 146 } 147 148 return size; 149 } 150 151 size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class) 152 { 153 size_t size = xe_gt_lrc_hang_replay_size(gt, class); 154 155 /* Add indirect ring state page */ 156 if (xe_gt_has_indirect_ring_state(gt)) 157 size += LRC_INDIRECT_RING_STATE_SIZE; 158 159 return size + LRC_PPHWSP_SIZE; 160 } 161 162 /* 163 * The per-platform tables are u8-encoded in @data. Decode @data and set the 164 * addresses' offset and commands in @regs. The following encoding is used 165 * for each byte. There are 2 steps: decoding commands and decoding addresses. 166 * 167 * Commands: 168 * [7]: create NOPs - number of NOPs are set in lower bits 169 * [6]: When creating MI_LOAD_REGISTER_IMM command, allow to set 170 * MI_LRI_FORCE_POSTED 171 * [5:0]: Number of NOPs or registers to set values to in case of 172 * MI_LOAD_REGISTER_IMM 173 * 174 * Addresses: these are decoded after a MI_LOAD_REGISTER_IMM command by "count" 175 * number of registers. They are set by using the REG/REG16 macros: the former 176 * is used for offsets smaller than 0x200 while the latter is for values bigger 177 * than that. Those macros already set all the bits documented below correctly: 178 * 179 * [7]: When a register offset needs more than 6 bits, use additional bytes, to 180 * follow, for the lower bits 181 * [6:0]: Register offset, without considering the engine base. 182 * 183 * This function only tweaks the commands and register offsets. Values are not 184 * filled out. 185 */ 186 static void set_offsets(u32 *regs, 187 const u8 *data, 188 const struct xe_hw_engine *hwe) 189 #define NOP(x) (BIT(7) | (x)) 190 #define LRI(count, flags) ((flags) << 6 | (count) | \ 191 BUILD_BUG_ON_ZERO(count >= BIT(6))) 192 #define POSTED BIT(0) 193 #define REG(x) (((x) >> 2) | BUILD_BUG_ON_ZERO(x >= 0x200)) 194 #define REG16(x) \ 195 (((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \ 196 (((x) >> 2) & 0x7f) 197 { 198 const u32 base = hwe->mmio_base; 199 200 while (*data) { 201 u8 count, flags; 202 203 if (*data & BIT(7)) { /* skip */ 204 count = *data++ & ~BIT(7); 205 regs += count; 206 continue; 207 } 208 209 count = *data & 0x3f; 210 flags = *data >> 6; 211 data++; 212 213 *regs = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count); 214 if (flags & POSTED) 215 *regs |= MI_LRI_FORCE_POSTED; 216 *regs |= MI_LRI_LRM_CS_MMIO; 217 regs++; 218 219 xe_gt_assert(hwe->gt, count); 220 do { 221 u32 offset = 0; 222 u8 v; 223 224 do { 225 v = *data++; 226 offset <<= 7; 227 offset |= v & ~BIT(7); 228 } while (v & BIT(7)); 229 230 regs[0] = base + (offset << 2); 231 regs += 2; 232 } while (--count); 233 } 234 235 *regs = MI_BATCH_BUFFER_END | BIT(0); 236 } 237 238 static const u8 gen12_xcs_offsets[] = { 239 NOP(1), 240 LRI(13, POSTED), 241 REG16(0x244), 242 REG(0x034), 243 REG(0x030), 244 REG(0x038), 245 REG(0x03c), 246 REG(0x168), 247 REG(0x140), 248 REG(0x110), 249 REG(0x1c0), 250 REG(0x1c4), 251 REG(0x1c8), 252 REG(0x180), 253 REG16(0x2b4), 254 255 NOP(5), 256 LRI(9, POSTED), 257 REG16(0x3a8), 258 REG16(0x28c), 259 REG16(0x288), 260 REG16(0x284), 261 REG16(0x280), 262 REG16(0x27c), 263 REG16(0x278), 264 REG16(0x274), 265 REG16(0x270), 266 267 0 268 }; 269 270 static const u8 dg2_xcs_offsets[] = { 271 NOP(1), 272 LRI(15, POSTED), 273 REG16(0x244), 274 REG(0x034), 275 REG(0x030), 276 REG(0x038), 277 REG(0x03c), 278 REG(0x168), 279 REG(0x140), 280 REG(0x110), 281 REG(0x1c0), 282 REG(0x1c4), 283 REG(0x1c8), 284 REG(0x180), 285 REG16(0x2b4), 286 REG(0x120), 287 REG(0x124), 288 289 NOP(1), 290 LRI(9, POSTED), 291 REG16(0x3a8), 292 REG16(0x28c), 293 REG16(0x288), 294 REG16(0x284), 295 REG16(0x280), 296 REG16(0x27c), 297 REG16(0x278), 298 REG16(0x274), 299 REG16(0x270), 300 301 0 302 }; 303 304 static const u8 gen12_rcs_offsets[] = { 305 NOP(1), 306 LRI(13, POSTED), 307 REG16(0x244), 308 REG(0x034), 309 REG(0x030), 310 REG(0x038), 311 REG(0x03c), 312 REG(0x168), 313 REG(0x140), 314 REG(0x110), 315 REG(0x1c0), 316 REG(0x1c4), 317 REG(0x1c8), 318 REG(0x180), 319 REG16(0x2b4), 320 321 NOP(5), 322 LRI(9, POSTED), 323 REG16(0x3a8), 324 REG16(0x28c), 325 REG16(0x288), 326 REG16(0x284), 327 REG16(0x280), 328 REG16(0x27c), 329 REG16(0x278), 330 REG16(0x274), 331 REG16(0x270), 332 333 LRI(3, POSTED), 334 REG(0x1b0), 335 REG16(0x5a8), 336 REG16(0x5ac), 337 338 NOP(6), 339 LRI(1, 0), 340 REG(0x0c8), 341 NOP(3 + 9 + 1), 342 343 LRI(51, POSTED), 344 REG16(0x588), 345 REG16(0x588), 346 REG16(0x588), 347 REG16(0x588), 348 REG16(0x588), 349 REG16(0x588), 350 REG(0x028), 351 REG(0x09c), 352 REG(0x0c0), 353 REG(0x178), 354 REG(0x17c), 355 REG16(0x358), 356 REG(0x170), 357 REG(0x150), 358 REG(0x154), 359 REG(0x158), 360 REG16(0x41c), 361 REG16(0x600), 362 REG16(0x604), 363 REG16(0x608), 364 REG16(0x60c), 365 REG16(0x610), 366 REG16(0x614), 367 REG16(0x618), 368 REG16(0x61c), 369 REG16(0x620), 370 REG16(0x624), 371 REG16(0x628), 372 REG16(0x62c), 373 REG16(0x630), 374 REG16(0x634), 375 REG16(0x638), 376 REG16(0x63c), 377 REG16(0x640), 378 REG16(0x644), 379 REG16(0x648), 380 REG16(0x64c), 381 REG16(0x650), 382 REG16(0x654), 383 REG16(0x658), 384 REG16(0x65c), 385 REG16(0x660), 386 REG16(0x664), 387 REG16(0x668), 388 REG16(0x66c), 389 REG16(0x670), 390 REG16(0x674), 391 REG16(0x678), 392 REG16(0x67c), 393 REG(0x068), 394 REG(0x084), 395 NOP(1), 396 397 0 398 }; 399 400 static const u8 xehp_rcs_offsets[] = { 401 NOP(1), 402 LRI(13, POSTED), 403 REG16(0x244), 404 REG(0x034), 405 REG(0x030), 406 REG(0x038), 407 REG(0x03c), 408 REG(0x168), 409 REG(0x140), 410 REG(0x110), 411 REG(0x1c0), 412 REG(0x1c4), 413 REG(0x1c8), 414 REG(0x180), 415 REG16(0x2b4), 416 417 NOP(5), 418 LRI(9, POSTED), 419 REG16(0x3a8), 420 REG16(0x28c), 421 REG16(0x288), 422 REG16(0x284), 423 REG16(0x280), 424 REG16(0x27c), 425 REG16(0x278), 426 REG16(0x274), 427 REG16(0x270), 428 429 LRI(3, POSTED), 430 REG(0x1b0), 431 REG16(0x5a8), 432 REG16(0x5ac), 433 434 NOP(6), 435 LRI(1, 0), 436 REG(0x0c8), 437 438 0 439 }; 440 441 static const u8 dg2_rcs_offsets[] = { 442 NOP(1), 443 LRI(15, POSTED), 444 REG16(0x244), 445 REG(0x034), 446 REG(0x030), 447 REG(0x038), 448 REG(0x03c), 449 REG(0x168), 450 REG(0x140), 451 REG(0x110), 452 REG(0x1c0), 453 REG(0x1c4), 454 REG(0x1c8), 455 REG(0x180), 456 REG16(0x2b4), 457 REG(0x120), 458 REG(0x124), 459 460 NOP(1), 461 LRI(9, POSTED), 462 REG16(0x3a8), 463 REG16(0x28c), 464 REG16(0x288), 465 REG16(0x284), 466 REG16(0x280), 467 REG16(0x27c), 468 REG16(0x278), 469 REG16(0x274), 470 REG16(0x270), 471 472 LRI(3, POSTED), 473 REG(0x1b0), 474 REG16(0x5a8), 475 REG16(0x5ac), 476 477 NOP(6), 478 LRI(1, 0), 479 REG(0x0c8), 480 481 0 482 }; 483 484 static const u8 mtl_rcs_offsets[] = { 485 NOP(1), 486 LRI(15, POSTED), 487 REG16(0x244), 488 REG(0x034), 489 REG(0x030), 490 REG(0x038), 491 REG(0x03c), 492 REG(0x168), 493 REG(0x140), 494 REG(0x110), 495 REG(0x1c0), 496 REG(0x1c4), 497 REG(0x1c8), 498 REG(0x180), 499 REG16(0x2b4), 500 REG(0x120), 501 REG(0x124), 502 503 NOP(1), 504 LRI(9, POSTED), 505 REG16(0x3a8), 506 REG16(0x28c), 507 REG16(0x288), 508 REG16(0x284), 509 REG16(0x280), 510 REG16(0x27c), 511 REG16(0x278), 512 REG16(0x274), 513 REG16(0x270), 514 515 NOP(2), 516 LRI(2, POSTED), 517 REG16(0x5a8), 518 REG16(0x5ac), 519 520 NOP(6), 521 LRI(1, 0), 522 REG(0x0c8), 523 524 0 525 }; 526 527 #define XE2_CTX_COMMON \ 528 NOP(1), /* [0x00] */ \ 529 LRI(15, POSTED), /* [0x01] */ \ 530 REG16(0x244), /* [0x02] CTXT_SR_CTL */ \ 531 REG(0x034), /* [0x04] RING_BUFFER_HEAD */ \ 532 REG(0x030), /* [0x06] RING_BUFFER_TAIL */ \ 533 REG(0x038), /* [0x08] RING_BUFFER_START */ \ 534 REG(0x03c), /* [0x0a] RING_BUFFER_CONTROL */ \ 535 REG(0x168), /* [0x0c] BB_ADDR_UDW */ \ 536 REG(0x140), /* [0x0e] BB_ADDR */ \ 537 REG(0x110), /* [0x10] BB_STATE */ \ 538 REG(0x1c0), /* [0x12] BB_PER_CTX_PTR */ \ 539 REG(0x1c4), /* [0x14] RCS_INDIRECT_CTX */ \ 540 REG(0x1c8), /* [0x16] RCS_INDIRECT_CTX_OFFSET */ \ 541 REG(0x180), /* [0x18] CCID */ \ 542 REG16(0x2b4), /* [0x1a] SEMAPHORE_TOKEN */ \ 543 REG(0x120), /* [0x1c] PRT_BB_STATE */ \ 544 REG(0x124), /* [0x1e] PRT_BB_STATE_UDW */ \ 545 \ 546 NOP(1), /* [0x20] */ \ 547 LRI(9, POSTED), /* [0x21] */ \ 548 REG16(0x3a8), /* [0x22] CTX_TIMESTAMP */ \ 549 REG16(0x3ac), /* [0x24] CTX_TIMESTAMP_UDW */ \ 550 REG(0x108), /* [0x26] INDIRECT_RING_STATE */ \ 551 REG16(0x284), /* [0x28] dummy reg */ \ 552 REG16(0x280), /* [0x2a] CS_ACC_CTR_THOLD */ \ 553 REG16(0x27c), /* [0x2c] CS_CTX_SYS_PASID */ \ 554 REG16(0x278), /* [0x2e] CS_CTX_ASID */ \ 555 REG16(0x274), /* [0x30] PTBP_UDW */ \ 556 REG16(0x270) /* [0x32] PTBP_LDW */ 557 558 static const u8 xe2_rcs_offsets[] = { 559 XE2_CTX_COMMON, 560 561 NOP(2), /* [0x34] */ 562 LRI(2, POSTED), /* [0x36] */ 563 REG16(0x5a8), /* [0x37] CONTEXT_SCHEDULING_ATTRIBUTES */ 564 REG16(0x5ac), /* [0x39] PREEMPTION_STATUS */ 565 566 NOP(6), /* [0x41] */ 567 LRI(1, 0), /* [0x47] */ 568 REG(0x0c8), /* [0x48] R_PWR_CLK_STATE */ 569 570 0 571 }; 572 573 static const u8 xe2_bcs_offsets[] = { 574 XE2_CTX_COMMON, 575 576 NOP(4 + 8 + 1), /* [0x34] */ 577 LRI(2, POSTED), /* [0x41] */ 578 REG16(0x200), /* [0x42] BCS_SWCTRL */ 579 REG16(0x204), /* [0x44] BLIT_CCTL */ 580 581 0 582 }; 583 584 static const u8 xe2_xcs_offsets[] = { 585 XE2_CTX_COMMON, 586 587 0 588 }; 589 590 static const u8 xe2_indirect_ring_state_offsets[] = { 591 NOP(1), /* [0x00] */ 592 LRI(5, POSTED), /* [0x01] */ 593 REG(0x034), /* [0x02] RING_BUFFER_HEAD */ 594 REG(0x030), /* [0x04] RING_BUFFER_TAIL */ 595 REG(0x038), /* [0x06] RING_BUFFER_START */ 596 REG(0x048), /* [0x08] RING_BUFFER_START_UDW */ 597 REG(0x03c), /* [0x0a] RING_BUFFER_CONTROL */ 598 599 NOP(5), /* [0x0c] */ 600 LRI(9, POSTED), /* [0x11] */ 601 REG(0x168), /* [0x12] BB_ADDR_UDW */ 602 REG(0x140), /* [0x14] BB_ADDR */ 603 REG(0x110), /* [0x16] BB_STATE */ 604 REG16(0x588), /* [0x18] BB_STACK_WRITE_PORT */ 605 REG16(0x588), /* [0x20] BB_STACK_WRITE_PORT */ 606 REG16(0x588), /* [0x22] BB_STACK_WRITE_PORT */ 607 REG16(0x588), /* [0x24] BB_STACK_WRITE_PORT */ 608 REG16(0x588), /* [0x26] BB_STACK_WRITE_PORT */ 609 REG16(0x588), /* [0x28] BB_STACK_WRITE_PORT */ 610 611 NOP(12), /* [0x00] */ 612 613 0 614 }; 615 616 #undef REG16 617 #undef REG 618 #undef LRI 619 #undef NOP 620 621 static const u8 *reg_offsets(struct xe_device *xe, enum xe_engine_class class) 622 { 623 if (class == XE_ENGINE_CLASS_RENDER) { 624 if (GRAPHICS_VER(xe) >= 20) 625 return xe2_rcs_offsets; 626 else if (GRAPHICS_VERx100(xe) >= 1270) 627 return mtl_rcs_offsets; 628 else if (GRAPHICS_VERx100(xe) >= 1255) 629 return dg2_rcs_offsets; 630 else if (GRAPHICS_VERx100(xe) >= 1250) 631 return xehp_rcs_offsets; 632 else 633 return gen12_rcs_offsets; 634 } else if (class == XE_ENGINE_CLASS_COPY) { 635 if (GRAPHICS_VER(xe) >= 20) 636 return xe2_bcs_offsets; 637 else 638 return gen12_xcs_offsets; 639 } else { 640 if (GRAPHICS_VER(xe) >= 20) 641 return xe2_xcs_offsets; 642 else if (GRAPHICS_VERx100(xe) >= 1255) 643 return dg2_xcs_offsets; 644 else 645 return gen12_xcs_offsets; 646 } 647 } 648 649 static void set_context_control(u32 *regs, struct xe_hw_engine *hwe) 650 { 651 regs[CTX_CONTEXT_CONTROL] = REG_MASKED_FIELD_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | 652 CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); 653 654 if (xe_gt_has_indirect_ring_state(hwe->gt)) 655 regs[CTX_CONTEXT_CONTROL] |= 656 REG_MASKED_FIELD_ENABLE(CTX_CTRL_INDIRECT_RING_STATE_ENABLE); 657 } 658 659 static void set_memory_based_intr(u32 *regs, struct xe_hw_engine *hwe) 660 { 661 struct xe_memirq *memirq = >_to_tile(hwe->gt)->memirq; 662 struct xe_device *xe = gt_to_xe(hwe->gt); 663 u8 num_regs; 664 665 if (!xe_device_uses_memirq(xe)) 666 return; 667 668 regs[CTX_LRM_INT_MASK_ENABLE] = MI_LOAD_REGISTER_MEM | 669 MI_LRI_LRM_CS_MMIO | MI_LRM_USE_GGTT; 670 regs[CTX_INT_MASK_ENABLE_REG] = RING_IMR(0).addr; 671 regs[CTX_INT_MASK_ENABLE_PTR] = xe_memirq_enable_ptr(memirq); 672 673 num_regs = xe_device_has_msix(xe) ? 3 : 2; 674 regs[CTX_LRI_INT_REPORT_PTR] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(num_regs) | 675 MI_LRI_LRM_CS_MMIO | MI_LRI_FORCE_POSTED; 676 regs[CTX_INT_STATUS_REPORT_REG] = RING_INT_STATUS_RPT_PTR(0).addr; 677 regs[CTX_INT_STATUS_REPORT_PTR] = xe_memirq_status_ptr(memirq, hwe); 678 regs[CTX_INT_SRC_REPORT_REG] = RING_INT_SRC_RPT_PTR(0).addr; 679 regs[CTX_INT_SRC_REPORT_PTR] = xe_memirq_source_ptr(memirq, hwe); 680 681 if (xe_device_has_msix(xe)) { 682 regs[CTX_CS_INT_VEC_REG] = CS_INT_VEC(0).addr; 683 /* CTX_CS_INT_VEC_DATA will be set in xe_lrc_init */ 684 } 685 } 686 687 static inline bool xe_lrc_has_indirect_ring_state(struct xe_lrc *lrc) 688 { 689 return lrc->flags & XE_LRC_FLAG_INDIRECT_RING_STATE; 690 } 691 692 static inline u32 __xe_lrc_ring_offset(struct xe_lrc *lrc) 693 { 694 return 0; 695 } 696 697 u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc) 698 { 699 return lrc->ring.size; 700 } 701 702 /* Make the magic macros work */ 703 #define __xe_lrc_pphwsp_offset xe_lrc_pphwsp_offset 704 #define __xe_lrc_regs_offset xe_lrc_regs_offset 705 706 #define LRC_CTX_JOB_TIMESTAMP_OFFSET 512 707 #define LRC_ENGINE_ID_PPHWSP_OFFSET 1024 708 #define LRC_PARALLEL_PPHWSP_OFFSET 2048 709 #define LRC_ULLS_PPHWSP_OFFSET 2048 /* Mutually exclusive with parallel */ 710 711 #define LRC_SEQNO_OFFSET 0 712 #define LRC_START_SEQNO_OFFSET (LRC_SEQNO_OFFSET + 8) 713 714 u32 xe_lrc_regs_offset(struct xe_lrc *lrc) 715 { 716 return xe_lrc_pphwsp_offset(lrc) + LRC_PPHWSP_SIZE; 717 } 718 719 /** 720 * xe_lrc_reg_size() - Get size of the LRC registers area within queues 721 * @xe: the &xe_device struct instance 722 * 723 * Returns: Size of the LRC registers area for current platform 724 */ 725 size_t xe_lrc_reg_size(struct xe_device *xe) 726 { 727 if (GRAPHICS_VERx100(xe) >= 1250) 728 return 96 * sizeof(u32); 729 else 730 return 80 * sizeof(u32); 731 } 732 733 /** 734 * xe_lrc_engine_state_size() - Get size of the engine state within LRC 735 * @gt: the &xe_gt struct instance 736 * @class: Hardware engine class 737 * 738 * Returns: Size of the engine state 739 */ 740 size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class) 741 { 742 return xe_gt_lrc_hang_replay_size(gt, class) - xe_lrc_reg_size(gt_to_xe(gt)); 743 } 744 745 static inline u32 __xe_lrc_seqno_offset(struct xe_lrc *lrc) 746 { 747 return LRC_SEQNO_OFFSET; 748 } 749 750 static inline u32 __xe_lrc_start_seqno_offset(struct xe_lrc *lrc) 751 { 752 return LRC_START_SEQNO_OFFSET; 753 } 754 755 static u32 __xe_lrc_ctx_job_timestamp_offset(struct xe_lrc *lrc) 756 { 757 /* This is stored in the driver-defined portion of PPHWSP */ 758 return xe_lrc_pphwsp_offset(lrc) + LRC_CTX_JOB_TIMESTAMP_OFFSET; 759 } 760 761 static inline u32 __xe_lrc_parallel_offset(struct xe_lrc *lrc) 762 { 763 /* The parallel is stored in the driver-defined portion of PPHWSP */ 764 return xe_lrc_pphwsp_offset(lrc) + LRC_PARALLEL_PPHWSP_OFFSET; 765 } 766 767 static inline u32 __xe_lrc_engine_id_offset(struct xe_lrc *lrc) 768 { 769 return xe_lrc_pphwsp_offset(lrc) + LRC_ENGINE_ID_PPHWSP_OFFSET; 770 } 771 772 static u32 __xe_lrc_ulls_offset(struct xe_lrc *lrc) 773 { 774 /* The ulls is stored in the driver-defined portion of PPHWSP */ 775 return xe_lrc_pphwsp_offset(lrc) + LRC_ULLS_PPHWSP_OFFSET; 776 } 777 778 static u32 __xe_lrc_ctx_timestamp_offset(struct xe_lrc *lrc) 779 { 780 return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP * sizeof(u32); 781 } 782 783 static u32 __xe_lrc_ctx_timestamp_udw_offset(struct xe_lrc *lrc) 784 { 785 return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP_UDW * sizeof(u32); 786 } 787 788 static u32 __xe_lrc_queue_timestamp_offset(struct xe_lrc *lrc) 789 { 790 return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP * sizeof(u32); 791 } 792 793 static u32 __xe_lrc_queue_timestamp_udw_offset(struct xe_lrc *lrc) 794 { 795 return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP_UDW * sizeof(u32); 796 } 797 798 static inline u32 __xe_lrc_indirect_ring_offset(struct xe_lrc *lrc) 799 { 800 u32 offset = xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE - 801 LRC_INDIRECT_RING_STATE_SIZE; 802 803 if (lrc->flags & XE_LRC_FLAG_INDIRECT_CTX) 804 offset -= LRC_INDIRECT_CTX_BO_SIZE; 805 806 return offset; 807 } 808 809 static inline u32 __xe_lrc_indirect_ctx_offset(struct xe_lrc *lrc) 810 { 811 return xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE - LRC_INDIRECT_CTX_BO_SIZE; 812 } 813 814 static inline u32 __xe_lrc_wa_bb_offset(struct xe_lrc *lrc) 815 { 816 return xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE; 817 } 818 819 #define DECL_MAP_ADDR_HELPERS(elem, bo_expr) \ 820 static inline struct iosys_map __xe_lrc_##elem##_map(struct xe_lrc *lrc) \ 821 { \ 822 struct xe_bo *bo = (bo_expr); \ 823 struct iosys_map map = bo->vmap; \ 824 \ 825 xe_assert(lrc_to_xe(lrc), !iosys_map_is_null(&map)); \ 826 iosys_map_incr(&map, __xe_lrc_##elem##_offset(lrc)); \ 827 return map; \ 828 } \ 829 static inline u32 __maybe_unused __xe_lrc_##elem##_ggtt_addr(struct xe_lrc *lrc) \ 830 { \ 831 struct xe_bo *bo = (bo_expr); \ 832 \ 833 return xe_bo_ggtt_addr(bo) + __xe_lrc_##elem##_offset(lrc); \ 834 } \ 835 836 DECL_MAP_ADDR_HELPERS(ring, lrc->bo) 837 DECL_MAP_ADDR_HELPERS(pphwsp, lrc->bo) 838 DECL_MAP_ADDR_HELPERS(seqno, lrc->seqno_bo) 839 DECL_MAP_ADDR_HELPERS(regs, lrc->bo) 840 DECL_MAP_ADDR_HELPERS(start_seqno, lrc->seqno_bo) 841 DECL_MAP_ADDR_HELPERS(ctx_job_timestamp, lrc->bo) 842 DECL_MAP_ADDR_HELPERS(ctx_timestamp, lrc->bo) 843 DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo) 844 DECL_MAP_ADDR_HELPERS(parallel, lrc->bo) 845 DECL_MAP_ADDR_HELPERS(ulls, lrc->bo) 846 DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo) 847 DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo) 848 DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo) 849 DECL_MAP_ADDR_HELPERS(queue_timestamp_udw, lrc->bo) 850 851 #undef DECL_MAP_ADDR_HELPERS 852 853 /** 854 * xe_lrc_ctx_timestamp_ggtt_addr() - Get ctx timestamp GGTT address 855 * @lrc: Pointer to the lrc. 856 * 857 * Returns: ctx timestamp GGTT address 858 */ 859 u32 xe_lrc_ctx_timestamp_ggtt_addr(struct xe_lrc *lrc) 860 { 861 return __xe_lrc_ctx_timestamp_ggtt_addr(lrc); 862 } 863 864 /** 865 * xe_lrc_ctx_timestamp_udw_ggtt_addr() - Get ctx timestamp udw GGTT address 866 * @lrc: Pointer to the lrc. 867 * 868 * Returns: ctx timestamp udw GGTT address 869 */ 870 u32 xe_lrc_ctx_timestamp_udw_ggtt_addr(struct xe_lrc *lrc) 871 { 872 return __xe_lrc_ctx_timestamp_udw_ggtt_addr(lrc); 873 } 874 875 /** 876 * xe_lrc_ctx_timestamp() - Read ctx timestamp value 877 * @lrc: Pointer to the lrc. 878 * 879 * Returns: ctx timestamp value 880 */ 881 static u64 xe_lrc_ctx_timestamp(struct xe_lrc *lrc) 882 { 883 struct xe_device *xe = lrc_to_xe(lrc); 884 struct iosys_map map; 885 u32 ldw, udw = 0; 886 887 map = __xe_lrc_ctx_timestamp_map(lrc); 888 ldw = xe_map_read32(xe, &map); 889 890 if (xe->info.has_64bit_timestamp) { 891 map = __xe_lrc_ctx_timestamp_udw_map(lrc); 892 udw = xe_map_read32(xe, &map); 893 } 894 895 return (u64)udw << 32 | ldw; 896 } 897 898 /** 899 * xe_lrc_queue_timestamp() - Read queue timestamp value 900 * @lrc: Pointer to the lrc. 901 * 902 * Returns: queue timestamp value 903 */ 904 static u64 xe_lrc_queue_timestamp(struct xe_lrc *lrc) 905 { 906 struct xe_device *xe = lrc_to_xe(lrc); 907 struct iosys_map map; 908 u32 ldw, udw = 0; 909 910 xe_assert(xe, xe_lrc_is_multi_queue(lrc)); 911 912 map = __xe_lrc_queue_timestamp_map(lrc); 913 ldw = xe_map_read32(xe, &map); 914 915 map = __xe_lrc_queue_timestamp_udw_map(lrc); 916 udw = xe_map_read32(xe, &map); 917 918 return (u64)udw << 32 | ldw; 919 } 920 921 /** 922 * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address 923 * @lrc: Pointer to the lrc. 924 * 925 * Returns: ctx timestamp job GGTT address 926 */ 927 u32 xe_lrc_ctx_job_timestamp_ggtt_addr(struct xe_lrc *lrc) 928 { 929 return __xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); 930 } 931 932 /** 933 * xe_lrc_ctx_job_timestamp() - Read ctx job timestamp value 934 * @lrc: Pointer to the lrc. 935 * 936 * Returns: ctx timestamp job value 937 */ 938 u32 xe_lrc_ctx_job_timestamp(struct xe_lrc *lrc) 939 { 940 struct xe_device *xe = lrc_to_xe(lrc); 941 struct iosys_map map; 942 943 map = __xe_lrc_ctx_job_timestamp_map(lrc); 944 return xe_map_read32(xe, &map); 945 } 946 947 u32 xe_lrc_ggtt_addr(struct xe_lrc *lrc) 948 { 949 return __xe_lrc_pphwsp_ggtt_addr(lrc); 950 } 951 952 u32 xe_lrc_indirect_ring_ggtt_addr(struct xe_lrc *lrc) 953 { 954 if (!xe_lrc_has_indirect_ring_state(lrc)) 955 return 0; 956 957 return __xe_lrc_indirect_ring_ggtt_addr(lrc); 958 } 959 960 static u32 xe_lrc_read_indirect_ctx_reg(struct xe_lrc *lrc, int reg_nr) 961 { 962 struct xe_device *xe = lrc_to_xe(lrc); 963 struct iosys_map map; 964 965 map = __xe_lrc_indirect_ring_map(lrc); 966 iosys_map_incr(&map, reg_nr * sizeof(u32)); 967 return xe_map_read32(xe, &map); 968 } 969 970 static void xe_lrc_write_indirect_ctx_reg(struct xe_lrc *lrc, 971 int reg_nr, u32 val) 972 { 973 struct xe_device *xe = lrc_to_xe(lrc); 974 struct iosys_map map; 975 976 map = __xe_lrc_indirect_ring_map(lrc); 977 iosys_map_incr(&map, reg_nr * sizeof(u32)); 978 xe_map_write32(xe, &map, val); 979 } 980 981 u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr) 982 { 983 struct xe_device *xe = lrc_to_xe(lrc); 984 struct iosys_map map; 985 986 map = __xe_lrc_regs_map(lrc); 987 iosys_map_incr(&map, reg_nr * sizeof(u32)); 988 return xe_map_read32(xe, &map); 989 } 990 991 void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val) 992 { 993 struct xe_device *xe = lrc_to_xe(lrc); 994 struct iosys_map map; 995 996 map = __xe_lrc_regs_map(lrc); 997 iosys_map_incr(&map, reg_nr * sizeof(u32)); 998 xe_map_write32(xe, &map, val); 999 } 1000 1001 static void *empty_lrc_data(struct xe_hw_engine *hwe) 1002 { 1003 struct xe_gt *gt = hwe->gt; 1004 void *data; 1005 u32 *regs; 1006 1007 data = kzalloc(xe_gt_lrc_size(gt, hwe->class), GFP_KERNEL); 1008 if (!data) 1009 return NULL; 1010 1011 /* 1st page: Per-Process of HW status Page */ 1012 regs = data + LRC_PPHWSP_SIZE; 1013 set_offsets(regs, reg_offsets(gt_to_xe(gt), hwe->class), hwe); 1014 set_context_control(regs, hwe); 1015 set_memory_based_intr(regs, hwe); 1016 if (xe_gt_has_indirect_ring_state(gt)) { 1017 regs = data + xe_gt_lrc_size(gt, hwe->class) - 1018 LRC_INDIRECT_RING_STATE_SIZE; 1019 set_offsets(regs, xe2_indirect_ring_state_offsets, hwe); 1020 } 1021 1022 return data; 1023 } 1024 1025 /** 1026 * xe_default_lrc_update_memirq_regs_with_address - Re-compute GGTT references in default LRC 1027 * of given engine. 1028 * @hwe: the &xe_hw_engine struct instance 1029 */ 1030 void xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine *hwe) 1031 { 1032 struct xe_gt *gt = hwe->gt; 1033 u32 *regs; 1034 1035 if (!gt->default_lrc[hwe->class]) 1036 return; 1037 1038 regs = gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE; 1039 set_memory_based_intr(regs, hwe); 1040 } 1041 1042 /** 1043 * xe_lrc_update_memirq_regs_with_address - Re-compute GGTT references in mem interrupt data 1044 * for given LRC. 1045 * @lrc: the &xe_lrc struct instance 1046 * @hwe: the &xe_hw_engine struct instance 1047 * @regs: scratch buffer to be used as temporary storage 1048 */ 1049 void xe_lrc_update_memirq_regs_with_address(struct xe_lrc *lrc, struct xe_hw_engine *hwe, 1050 u32 *regs) 1051 { 1052 struct xe_gt *gt = hwe->gt; 1053 struct iosys_map map; 1054 size_t regs_len; 1055 1056 if (!xe_device_uses_memirq(gt_to_xe(gt))) 1057 return; 1058 1059 map = __xe_lrc_regs_map(lrc); 1060 regs_len = xe_lrc_reg_size(gt_to_xe(gt)); 1061 xe_map_memcpy_from(gt_to_xe(gt), regs, &map, 0, regs_len); 1062 set_memory_based_intr(regs, hwe); 1063 xe_map_memcpy_to(gt_to_xe(gt), &map, 0, regs, regs_len); 1064 } 1065 1066 static void xe_lrc_set_ppgtt(struct xe_lrc *lrc, struct xe_vm *vm) 1067 { 1068 u64 desc = xe_vm_pdp4_descriptor(vm, gt_to_tile(lrc->gt)); 1069 1070 xe_lrc_write_ctx_reg(lrc, CTX_PDP0_UDW, upper_32_bits(desc)); 1071 xe_lrc_write_ctx_reg(lrc, CTX_PDP0_LDW, lower_32_bits(desc)); 1072 } 1073 1074 static void xe_lrc_finish(struct xe_lrc *lrc) 1075 { 1076 xe_hw_fence_ctx_finish(&lrc->fence_ctx); 1077 xe_bo_unpin_map_no_vm(lrc->bo); 1078 xe_bo_unpin_map_no_vm(lrc->seqno_bo); 1079 } 1080 1081 /* 1082 * wa_bb_setup_utilization() - Write commands to wa bb to assist 1083 * in calculating active context run ticks. 1084 * 1085 * Context Timestamp (CTX_TIMESTAMP) in the LRC accumulates the run ticks of the 1086 * context, but only gets updated when the context switches out. In order to 1087 * check how long a context has been active before it switches out, two things 1088 * are required: 1089 * 1090 * (1) Determine if the context is running: 1091 * To do so, we program the WA BB to set an initial value for CTX_TIMESTAMP in 1092 * the LRC. The value chosen is 1 since 0 is the initial value when the LRC is 1093 * initialized. During a query, we just check for this value to determine if the 1094 * context is active. If the context switched out, it would overwrite this 1095 * location with the actual CTX_TIMESTAMP MMIO value. Note that WA BB runs as 1096 * the last part of context restore, so reusing this LRC location will not 1097 * clobber anything. 1098 * 1099 * (2) Calculate the time that the context has been active for: 1100 * The CTX_TIMESTAMP ticks only when the context is active. If a context is 1101 * active, we just use the CTX_TIMESTAMP MMIO as the new value of utilization. 1102 * While doing so, we need to read the CTX_TIMESTAMP MMIO for the specific 1103 * engine instance. Since we do not know which instance the context is running 1104 * on until it is scheduled, we also read the ENGINE_ID MMIO in the WA BB and 1105 * store it in the PPHSWP. 1106 */ 1107 #define CONTEXT_ACTIVE XE_LRC_CTX_TIMESTAMP_ACTIVE 1108 static ssize_t setup_utilization_wa(struct xe_lrc *lrc, 1109 struct xe_hw_engine *hwe, 1110 u32 *batch, size_t max_len, bool indirect) 1111 { 1112 u32 *cmd = batch; 1113 1114 if (IS_SRIOV_VF(gt_to_xe(lrc->gt))) 1115 return 0; 1116 1117 if (xe_gt_WARN_ON(lrc->gt, max_len < 12)) 1118 return -ENOSPC; 1119 1120 *cmd++ = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; 1121 *cmd++ = ENGINE_ID(0).addr; 1122 *cmd++ = __xe_lrc_engine_id_ggtt_addr(lrc); 1123 *cmd++ = 0; 1124 1125 *cmd++ = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); 1126 *cmd++ = __xe_lrc_ctx_timestamp_ggtt_addr(lrc); 1127 *cmd++ = 0; 1128 *cmd++ = lower_32_bits(CONTEXT_ACTIVE); 1129 1130 if (lrc_to_xe(lrc)->info.has_64bit_timestamp) { 1131 *cmd++ = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); 1132 *cmd++ = __xe_lrc_ctx_timestamp_udw_ggtt_addr(lrc); 1133 *cmd++ = 0; 1134 *cmd++ = upper_32_bits(CONTEXT_ACTIVE); 1135 } 1136 1137 return cmd - batch; 1138 } 1139 1140 static ssize_t setup_timestamp_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, 1141 u32 *batch, size_t max_len, bool indirect) 1142 { 1143 const u32 ts_addr = __xe_lrc_ctx_timestamp_ggtt_addr(lrc); 1144 u32 *cmd = batch; 1145 1146 if (!XE_GT_WA(lrc->gt, 16010904313)) 1147 return 0; 1148 1149 if (!indirect && 1150 !(hwe->class == XE_ENGINE_CLASS_COPY || 1151 hwe->class == XE_ENGINE_CLASS_VIDEO_DECODE || 1152 hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE)) 1153 return 0; 1154 1155 if (indirect && 1156 !(hwe->class == XE_ENGINE_CLASS_RENDER || 1157 hwe->class == XE_ENGINE_CLASS_COMPUTE)) 1158 return 0; 1159 1160 if (xe_gt_WARN_ON(lrc->gt, max_len < 12)) 1161 return -ENOSPC; 1162 1163 *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO | 1164 MI_LRM_ASYNC; 1165 *cmd++ = RING_CTX_TIMESTAMP(0).addr; 1166 *cmd++ = ts_addr; 1167 *cmd++ = 0; 1168 1169 *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO | 1170 MI_LRM_ASYNC; 1171 *cmd++ = RING_CTX_TIMESTAMP(0).addr; 1172 *cmd++ = ts_addr; 1173 *cmd++ = 0; 1174 1175 *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO; 1176 *cmd++ = RING_CTX_TIMESTAMP(0).addr; 1177 *cmd++ = ts_addr; 1178 *cmd++ = 0; 1179 1180 return cmd - batch; 1181 } 1182 1183 static ssize_t setup_configfs_post_ctx_restore_bb(struct xe_lrc *lrc, 1184 struct xe_hw_engine *hwe, 1185 u32 *batch, size_t max_len, 1186 bool indirect) 1187 { 1188 struct xe_device *xe = gt_to_xe(lrc->gt); 1189 const u32 *user_batch; 1190 u32 *cmd = batch; 1191 u32 count; 1192 1193 count = xe_configfs_get_ctx_restore_post_bb(to_pci_dev(xe->drm.dev), 1194 hwe->class, &user_batch); 1195 if (!count) 1196 return 0; 1197 1198 if (count > max_len) 1199 return -ENOSPC; 1200 1201 /* 1202 * This should be used only for tests and validation. Taint the kernel 1203 * as anything could be submitted directly in context switches 1204 */ 1205 add_taint(TAINT_TEST, LOCKDEP_STILL_OK); 1206 1207 memcpy(cmd, user_batch, count * sizeof(u32)); 1208 cmd += count; 1209 1210 return cmd - batch; 1211 } 1212 1213 static ssize_t setup_configfs_mid_ctx_restore_bb(struct xe_lrc *lrc, 1214 struct xe_hw_engine *hwe, 1215 u32 *batch, size_t max_len, 1216 bool indirect) 1217 { 1218 struct xe_device *xe = gt_to_xe(lrc->gt); 1219 const u32 *user_batch; 1220 u32 *cmd = batch; 1221 u32 count; 1222 1223 count = xe_configfs_get_ctx_restore_mid_bb(to_pci_dev(xe->drm.dev), 1224 hwe->class, &user_batch); 1225 if (!count) 1226 return 0; 1227 1228 if (count > max_len) 1229 return -ENOSPC; 1230 1231 /* 1232 * This should be used only for tests and validation. Taint the kernel 1233 * as anything could be submitted directly in context switches 1234 */ 1235 add_taint(TAINT_TEST, LOCKDEP_STILL_OK); 1236 1237 memcpy(cmd, user_batch, count * sizeof(u32)); 1238 cmd += count; 1239 1240 return cmd - batch; 1241 } 1242 1243 static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc, 1244 struct xe_hw_engine *hwe, 1245 u32 *batch, size_t max_len, 1246 bool indirect) 1247 { 1248 u32 *cmd = batch; 1249 1250 if (!XE_GT_WA(lrc->gt, 18022495364) || 1251 hwe->class != XE_ENGINE_CLASS_RENDER) 1252 return 0; 1253 1254 if (xe_gt_WARN_ON(lrc->gt, max_len < 3)) 1255 return -ENOSPC; 1256 1257 *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_LRM_CS_MMIO | MI_LRI_NUM_REGS(1); 1258 *cmd++ = CS_DEBUG_MODE2(0).addr; 1259 *cmd++ = REG_MASKED_FIELD_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); 1260 1261 return cmd - batch; 1262 } 1263 1264 static ssize_t setup_invalidate_auxccs_wa(struct xe_lrc *lrc, 1265 struct xe_hw_engine *hwe, 1266 u32 *batch, size_t max_len, 1267 bool indirect) 1268 { 1269 struct xe_gt *gt = lrc->gt; 1270 u32 *(*emit)(struct xe_gt *gt, u32 *cmd) = 1271 gt->ring_ops[hwe->class]->emit_aux_table_inv; 1272 1273 if (!emit) 1274 return 0; 1275 1276 if (xe_gt_WARN_ON(gt, max_len < 8)) 1277 return -ENOSPC; 1278 1279 return emit(gt, batch) - batch; 1280 } 1281 1282 struct bo_setup { 1283 ssize_t (*setup)(struct xe_lrc *lrc, struct xe_hw_engine *hwe, 1284 u32 *batch, size_t max_size, bool indirect); 1285 }; 1286 1287 struct bo_setup_state { 1288 /* Input: */ 1289 struct xe_lrc *lrc; 1290 struct xe_hw_engine *hwe; 1291 size_t max_size; 1292 size_t reserve_dw; 1293 bool indirect; 1294 unsigned int offset; 1295 const struct bo_setup *funcs; 1296 unsigned int num_funcs; 1297 1298 /* State: */ 1299 u32 *buffer; 1300 u32 *ptr; 1301 unsigned int written; 1302 }; 1303 1304 static int setup_bo(struct bo_setup_state *state) 1305 { 1306 ssize_t remain; 1307 1308 if (state->lrc->bo->vmap.is_iomem) { 1309 xe_gt_assert(state->hwe->gt, state->buffer); 1310 state->ptr = state->buffer; 1311 } else { 1312 state->ptr = state->lrc->bo->vmap.vaddr + state->offset; 1313 } 1314 1315 remain = state->max_size / sizeof(u32); 1316 1317 for (size_t i = 0; i < state->num_funcs; i++) { 1318 ssize_t len = state->funcs[i].setup(state->lrc, state->hwe, 1319 state->ptr, remain, 1320 state->indirect); 1321 1322 remain -= len; 1323 1324 /* 1325 * Caller has asked for at least reserve_dw to remain unused. 1326 */ 1327 if (len < 0 || 1328 xe_gt_WARN_ON(state->lrc->gt, remain < state->reserve_dw)) 1329 goto fail; 1330 1331 state->ptr += len; 1332 state->written += len; 1333 } 1334 1335 return 0; 1336 1337 fail: 1338 return -ENOSPC; 1339 } 1340 1341 static void finish_bo(struct bo_setup_state *state) 1342 { 1343 if (!state->lrc->bo->vmap.is_iomem) 1344 return; 1345 1346 xe_map_memcpy_to(gt_to_xe(state->lrc->gt), &state->lrc->bo->vmap, 1347 state->offset, state->buffer, 1348 state->written * sizeof(u32)); 1349 } 1350 1351 /** 1352 * xe_lrc_setup_wa_bb_with_scratch - Execute all wa bb setup callbacks. 1353 * @lrc: the &xe_lrc struct instance 1354 * @hwe: the &xe_hw_engine struct instance 1355 * @scratch: preallocated scratch buffer for temporary storage 1356 * Return: 0 on success, negative error code on failure 1357 */ 1358 int xe_lrc_setup_wa_bb_with_scratch(struct xe_lrc *lrc, struct xe_hw_engine *hwe, u32 *scratch) 1359 { 1360 static const struct bo_setup funcs[] = { 1361 { .setup = setup_timestamp_wa }, 1362 { .setup = setup_invalidate_state_cache_wa }, 1363 { .setup = setup_utilization_wa }, 1364 { .setup = setup_configfs_post_ctx_restore_bb }, 1365 }; 1366 struct bo_setup_state state = { 1367 .lrc = lrc, 1368 .hwe = hwe, 1369 .max_size = LRC_WA_BB_SIZE, 1370 .buffer = scratch, 1371 .reserve_dw = 1, 1372 .offset = __xe_lrc_wa_bb_offset(lrc), 1373 .funcs = funcs, 1374 .num_funcs = ARRAY_SIZE(funcs), 1375 }; 1376 int ret; 1377 1378 ret = setup_bo(&state); 1379 if (ret) 1380 return ret; 1381 1382 *state.ptr++ = MI_BATCH_BUFFER_END; 1383 state.written++; 1384 1385 finish_bo(&state); 1386 1387 xe_lrc_write_ctx_reg(lrc, CTX_BB_PER_CTX_PTR, 1388 xe_bo_ggtt_addr(lrc->bo) + state.offset + 1); 1389 1390 return 0; 1391 } 1392 1393 static int setup_wa_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe) 1394 { 1395 u32 *buf = NULL; 1396 int ret; 1397 1398 if (lrc->bo->vmap.is_iomem) { 1399 buf = kmalloc(LRC_WA_BB_SIZE, GFP_KERNEL); 1400 if (!buf) 1401 return -ENOMEM; 1402 } 1403 1404 ret = xe_lrc_setup_wa_bb_with_scratch(lrc, hwe, buf); 1405 1406 kfree(buf); 1407 1408 return ret; 1409 } 1410 1411 static int 1412 setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) 1413 { 1414 static const struct bo_setup rcs_funcs[] = { 1415 { .setup = setup_timestamp_wa }, 1416 { .setup = setup_invalidate_auxccs_wa }, 1417 { .setup = setup_configfs_mid_ctx_restore_bb }, 1418 }; 1419 static const struct bo_setup xcs_funcs[] = { 1420 { .setup = setup_invalidate_auxccs_wa }, 1421 { .setup = setup_configfs_mid_ctx_restore_bb }, 1422 }; 1423 struct bo_setup_state state = { 1424 .lrc = lrc, 1425 .hwe = hwe, 1426 .indirect = true, 1427 .max_size = (63 * 64) /* max 63 cachelines */, 1428 .buffer = NULL, 1429 .offset = __xe_lrc_indirect_ctx_offset(lrc), 1430 }; 1431 int ret; 1432 1433 if (!(lrc->flags & XE_LRC_FLAG_INDIRECT_CTX)) 1434 return 0; 1435 1436 if (hwe->class == XE_ENGINE_CLASS_RENDER || 1437 hwe->class == XE_ENGINE_CLASS_COMPUTE) { 1438 state.funcs = rcs_funcs; 1439 state.num_funcs = ARRAY_SIZE(rcs_funcs); 1440 } else { 1441 state.funcs = xcs_funcs; 1442 state.num_funcs = ARRAY_SIZE(xcs_funcs); 1443 } 1444 1445 if (xe_gt_WARN_ON(lrc->gt, !state.funcs)) 1446 return 0; 1447 1448 if (lrc->bo->vmap.is_iomem) { 1449 state.buffer = kmalloc(state.max_size, GFP_KERNEL); 1450 if (!state.buffer) 1451 return -ENOMEM; 1452 } 1453 1454 ret = setup_bo(&state); 1455 if (ret) { 1456 kfree(state.buffer); 1457 return ret; 1458 } 1459 1460 /* 1461 * Align to 64B cacheline so there's no garbage at the end for CS to 1462 * execute: size for indirect ctx must be a multiple of 64. 1463 */ 1464 while (state.written & 0xf) { 1465 *state.ptr++ = MI_NOOP; 1466 state.written++; 1467 } 1468 1469 finish_bo(&state); 1470 kfree(state.buffer); 1471 1472 /* 1473 * Enable INDIRECT_CTX leaving INDIRECT_CTX_OFFSET at its default: it 1474 * varies per engine class, but the default is good enough 1475 */ 1476 xe_lrc_write_ctx_reg(lrc, 1477 CTX_CS_INDIRECT_CTX, 1478 (xe_bo_ggtt_addr(lrc->bo) + state.offset) | 1479 /* Size in CLs. */ 1480 (state.written * sizeof(u32) / 64)); 1481 1482 return 0; 1483 } 1484 1485 static u8 xe_multi_queue_prio_to_lrc(struct xe_lrc *lrc, enum xe_multi_queue_priority priority) 1486 { 1487 struct xe_device *xe = gt_to_xe(lrc->gt); 1488 1489 xe_assert(xe, (priority >= XE_MULTI_QUEUE_PRIORITY_LOW && 1490 priority <= XE_MULTI_QUEUE_PRIORITY_HIGH)); 1491 1492 /* xe_multi_queue_priority is directly mapped to LRC priority values */ 1493 return priority; 1494 } 1495 1496 /** 1497 * xe_lrc_set_multi_queue_priority() - Set multi queue priority in LRC 1498 * @lrc: Logical Ring Context 1499 * @priority: Multi queue priority of the exec queue 1500 * 1501 * Convert @priority to LRC multi queue priority and update the @lrc descriptor 1502 */ 1503 void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority) 1504 { 1505 lrc->desc &= ~LRC_PRIORITY; 1506 lrc->desc |= FIELD_PREP(LRC_PRIORITY, xe_multi_queue_prio_to_lrc(lrc, priority)); 1507 } 1508 1509 static void xe_lrc_set_gpgpu_preemption_level(struct xe_lrc *lrc, struct xe_gt *gt) 1510 { 1511 enum xe_gpgpu_preempt_level level = gt->gpgpu_preemption_level; 1512 u32 level_bits; 1513 u32 val; 1514 1515 if (level == XE_GPGPU_PREEMPT_DEFAULT) 1516 return; 1517 1518 switch (level) { 1519 case XE_GPGPU_PREEMPT_MID_THREAD: 1520 level_bits = PREEMPT_GPGPU_MID_THREAD_LEVEL; 1521 break; 1522 case XE_GPGPU_PREEMPT_THREAD_GROUP: 1523 level_bits = PREEMPT_GPGPU_THREAD_GROUP_LEVEL; 1524 break; 1525 case XE_GPGPU_PREEMPT_COMMAND: 1526 level_bits = PREEMPT_GPGPU_COMMAND_LEVEL; 1527 break; 1528 default: 1529 xe_gt_WARN(gt, true, "Invalid GPGPU preemption level: %d\n", level); 1530 return; 1531 } 1532 1533 val = xe_lrc_read_ctx_reg(lrc, CTX_CS_CHICKEN1); 1534 val &= ~PREEMPT_GPGPU_LEVEL_MASK; 1535 val |= REG_MASKED_FIELD(PREEMPT_GPGPU_LEVEL_MASK, level_bits); 1536 xe_lrc_write_ctx_reg(lrc, CTX_CS_CHICKEN1, val); 1537 } 1538 1539 static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm, 1540 void *replay_state, u16 msix_vec, u32 init_flags) 1541 { 1542 struct xe_gt *gt = hwe->gt; 1543 struct xe_tile *tile = gt_to_tile(gt); 1544 struct xe_device *xe = gt_to_xe(gt); 1545 struct iosys_map map; 1546 u32 arb_enable; 1547 u32 state_cache_perf_fix[3]; 1548 int err; 1549 1550 /* 1551 * Init Per-Process of HW status Page, LRC / context state to known 1552 * values. If there's already a primed default_lrc, just copy it, otherwise 1553 * it's the early submission to record the lrc: build a new empty one from 1554 * scratch. 1555 */ 1556 map = __xe_lrc_pphwsp_map(lrc); 1557 if (gt->default_lrc[hwe->class] || replay_state) { 1558 xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE); /* PPHWSP */ 1559 xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, 1560 gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE, 1561 lrc->size - LRC_PPHWSP_SIZE); 1562 if (replay_state) 1563 xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, 1564 replay_state, lrc->replay_size); 1565 } else { 1566 void *init_data = empty_lrc_data(hwe); 1567 1568 if (!init_data) { 1569 return -ENOMEM; 1570 } 1571 1572 xe_map_memcpy_to(xe, &map, 0, init_data, lrc->size); 1573 kfree(init_data); 1574 } 1575 1576 if (vm) 1577 xe_lrc_set_ppgtt(lrc, vm); 1578 1579 if (xe_device_has_msix(xe)) { 1580 xe_lrc_write_ctx_reg(lrc, CTX_INT_STATUS_REPORT_PTR, 1581 xe_memirq_status_ptr(&tile->memirq, hwe)); 1582 xe_lrc_write_ctx_reg(lrc, CTX_INT_SRC_REPORT_PTR, 1583 xe_memirq_source_ptr(&tile->memirq, hwe)); 1584 xe_lrc_write_ctx_reg(lrc, CTX_CS_INT_VEC_DATA, msix_vec << 16 | msix_vec); 1585 } 1586 1587 if (xe_gt_has_indirect_ring_state(gt)) { 1588 xe_lrc_write_ctx_reg(lrc, CTX_INDIRECT_RING_STATE, 1589 __xe_lrc_indirect_ring_ggtt_addr(lrc)); 1590 1591 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START, 1592 __xe_lrc_ring_ggtt_addr(lrc)); 1593 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START_UDW, 0); 1594 1595 /* Match head and tail pointers */ 1596 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, lrc->ring.tail); 1597 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, lrc->ring.tail); 1598 1599 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_CTL, 1600 RING_CTL_SIZE(lrc->ring.size) | RING_VALID); 1601 } else { 1602 xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc)); 1603 1604 /* Match head and tail pointers */ 1605 xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, lrc->ring.tail); 1606 xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); 1607 1608 xe_lrc_write_ctx_reg(lrc, CTX_RING_CTL, 1609 RING_CTL_SIZE(lrc->ring.size) | RING_VALID); 1610 } 1611 1612 if (init_flags & XE_LRC_CREATE_RUNALONE) 1613 xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL, 1614 xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) | 1615 REG_MASKED_FIELD_ENABLE(CTX_CTRL_RUN_ALONE)); 1616 1617 if (init_flags & XE_LRC_CREATE_PXP) 1618 xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL, 1619 xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) | 1620 REG_MASKED_FIELD_ENABLE(CTX_CTRL_PXP_ENABLE)); 1621 1622 lrc->ctx_timestamp = 0; 1623 xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP, 0); 1624 if (lrc_to_xe(lrc)->info.has_64bit_timestamp) 1625 xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP_UDW, 0); 1626 1627 /* 1628 * Note: It's possible that this LRC may belong to an exec_queue that is 1629 * not part of a multi-queue group. That said, it doesn't hurt to set 1630 * this field anyways since any class that supports multi-queue will 1631 * have these LRC fields defined. 1632 */ 1633 if (xe_gt_supports_multi_queue(gt, hwe->class)) { 1634 lrc->queue_timestamp = 0; 1635 xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP, 0); 1636 xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP_UDW, 0); 1637 } 1638 1639 if (xe->info.has_asid && vm) 1640 xe_lrc_write_ctx_reg(lrc, CTX_ASID, vm->usm.asid); 1641 1642 if (GRAPHICS_VER(xe) >= 20 && hwe->class == XE_ENGINE_CLASS_RENDER) 1643 xe_lrc_set_gpgpu_preemption_level(lrc, gt); 1644 1645 lrc->desc = LRC_VALID; 1646 lrc->desc |= FIELD_PREP(LRC_ADDRESSING_MODE, LRC_LEGACY_64B_CONTEXT); 1647 /* TODO: Priority */ 1648 1649 /* While this appears to have something about privileged batches or 1650 * some such, it really just means PPGTT mode. 1651 */ 1652 if (vm) 1653 lrc->desc |= LRC_PRIVILEGE; 1654 1655 if (GRAPHICS_VERx100(xe) < 1250) { 1656 lrc->desc |= FIELD_PREP(LRC_ENGINE_INSTANCE, hwe->instance); 1657 lrc->desc |= FIELD_PREP(LRC_ENGINE_CLASS, hwe->class); 1658 } 1659 1660 arb_enable = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1661 xe_lrc_write_ring(lrc, &arb_enable, sizeof(arb_enable)); 1662 1663 if (init_flags & XE_LRC_DISABLE_STATE_CACHE_PERF_FIX) { 1664 state_cache_perf_fix[0] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); 1665 state_cache_perf_fix[1] = COMMON_SLICE_CHICKEN3.addr; 1666 state_cache_perf_fix[2] = REG_MASKED_FIELD_ENABLE(DISABLE_STATE_CACHE_PERF_FIX); 1667 xe_lrc_write_ring(lrc, state_cache_perf_fix, sizeof(state_cache_perf_fix)); 1668 } 1669 1670 map = __xe_lrc_seqno_map(lrc); 1671 xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); 1672 1673 map = __xe_lrc_start_seqno_map(lrc); 1674 xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); 1675 1676 err = setup_wa_bb(lrc, hwe); 1677 if (err) 1678 return err; 1679 1680 err = setup_indirect_ctx(lrc, hwe); 1681 1682 return err; 1683 } 1684 1685 static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm, 1686 void *replay_state, u32 ring_size, u16 msix_vec, u32 init_flags) 1687 { 1688 struct xe_gt *gt = hwe->gt; 1689 const u32 lrc_size = xe_gt_lrc_size(gt, hwe->class); 1690 u32 bo_size = ring_size + lrc_size + LRC_WA_BB_SIZE; 1691 struct xe_tile *tile = gt_to_tile(gt); 1692 struct xe_device *xe = gt_to_xe(gt); 1693 struct xe_bo *bo; 1694 u32 bo_flags; 1695 int err; 1696 1697 kref_init(&lrc->refcount); 1698 lrc->gt = gt; 1699 lrc->replay_size = xe_gt_lrc_hang_replay_size(gt, hwe->class); 1700 lrc->size = lrc_size; 1701 lrc->flags = 0; 1702 lrc->ring.size = ring_size; 1703 lrc->ring.tail = 0; 1704 1705 if (gt_engine_needs_indirect_ctx(gt, hwe->class)) { 1706 lrc->flags |= XE_LRC_FLAG_INDIRECT_CTX; 1707 bo_size += LRC_INDIRECT_CTX_BO_SIZE; 1708 } 1709 1710 if (xe_gt_has_indirect_ring_state(gt)) 1711 lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE; 1712 1713 bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT | 1714 XE_BO_FLAG_GGTT_INVALIDATE; 1715 1716 if ((vm && vm->xef) || init_flags & XE_LRC_CREATE_USER_CTX) /* userspace */ 1717 bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM; 1718 1719 bo = xe_bo_create_pin_map_novm(xe, tile, bo_size, 1720 ttm_bo_type_kernel, 1721 bo_flags, false); 1722 if (IS_ERR(bo)) 1723 return PTR_ERR(bo); 1724 1725 lrc->bo = bo; 1726 1727 bo = xe_bo_create_pin_map_novm(xe, tile, PAGE_SIZE, 1728 ttm_bo_type_kernel, 1729 XE_BO_FLAG_GGTT | 1730 XE_BO_FLAG_GGTT_INVALIDATE | 1731 XE_BO_FLAG_SYSTEM, false); 1732 if (IS_ERR(bo)) { 1733 err = PTR_ERR(bo); 1734 goto err_lrc_finish; 1735 } 1736 lrc->seqno_bo = bo; 1737 1738 xe_hw_fence_ctx_init(&lrc->fence_ctx, hwe->gt, 1739 hwe->fence_irq, hwe->name); 1740 1741 err = xe_lrc_ctx_init(lrc, hwe, vm, replay_state, msix_vec, init_flags); 1742 if (err) 1743 goto err_lrc_finish; 1744 1745 if (vm && vm->xef) 1746 xe_drm_client_add_bo(vm->xef->client, lrc->bo); 1747 1748 return 0; 1749 1750 err_lrc_finish: 1751 xe_lrc_finish(lrc); 1752 return err; 1753 } 1754 1755 /** 1756 * xe_lrc_create - Create a LRC 1757 * @hwe: Hardware Engine 1758 * @vm: The VM (address space) 1759 * @replay_state: GPU hang replay state 1760 * @ring_size: LRC ring size 1761 * @msix_vec: MSI-X interrupt vector (for platforms that support it) 1762 * @flags: LRC initialization flags 1763 * 1764 * Allocate and initialize the Logical Ring Context (LRC). 1765 * 1766 * Return pointer to created LRC upon success and an error pointer 1767 * upon failure. 1768 */ 1769 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, 1770 void *replay_state, u32 ring_size, u16 msix_vec, u32 flags) 1771 { 1772 struct xe_lrc *lrc; 1773 int err; 1774 1775 lrc = kzalloc_obj(*lrc); 1776 if (!lrc) 1777 return ERR_PTR(-ENOMEM); 1778 1779 err = xe_lrc_init(lrc, hwe, vm, replay_state, ring_size, msix_vec, flags); 1780 if (err) { 1781 kfree(lrc); 1782 return ERR_PTR(err); 1783 } 1784 1785 return lrc; 1786 } 1787 1788 /** 1789 * xe_lrc_destroy - Destroy the LRC 1790 * @ref: reference to LRC 1791 * 1792 * Called when ref == 0, release resources held by the Logical Ring Context 1793 * (LRC) and free the LRC memory. 1794 */ 1795 void xe_lrc_destroy(struct kref *ref) 1796 { 1797 struct xe_lrc *lrc = container_of(ref, struct xe_lrc, refcount); 1798 1799 xe_lrc_finish(lrc); 1800 kfree(lrc); 1801 } 1802 1803 /** 1804 * xe_lrc_update_hwctx_regs_with_address - Re-compute GGTT references within given LRC. 1805 * @lrc: the &xe_lrc struct instance 1806 */ 1807 void xe_lrc_update_hwctx_regs_with_address(struct xe_lrc *lrc) 1808 { 1809 if (xe_lrc_has_indirect_ring_state(lrc)) { 1810 xe_lrc_write_ctx_reg(lrc, CTX_INDIRECT_RING_STATE, 1811 __xe_lrc_indirect_ring_ggtt_addr(lrc)); 1812 1813 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START, 1814 __xe_lrc_ring_ggtt_addr(lrc)); 1815 } else { 1816 xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc)); 1817 } 1818 } 1819 1820 void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail) 1821 { 1822 if (xe_lrc_has_indirect_ring_state(lrc)) 1823 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, tail); 1824 else 1825 xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, tail); 1826 } 1827 1828 /** 1829 * xe_lrc_ring_tail_ggtt_addr() - Saved ring tail GGTT address 1830 * @lrc: Pointer to the lrc. 1831 * 1832 * GGTT address of the ring tail as saved for this context - in the indirect 1833 * ring state on platforms which have it, otherwise in the context image. 1834 * This is what a context restore loads the tail register from, so a ring 1835 * which advances the tail register itself must keep this in sync. 1836 * 1837 * Returns: saved ring tail GGTT address 1838 */ 1839 u32 xe_lrc_ring_tail_ggtt_addr(struct xe_lrc *lrc) 1840 { 1841 if (xe_lrc_has_indirect_ring_state(lrc)) 1842 return __xe_lrc_indirect_ring_ggtt_addr(lrc) + 1843 INDIRECT_CTX_RING_TAIL * sizeof(u32); 1844 1845 return __xe_lrc_regs_ggtt_addr(lrc) + CTX_RING_TAIL * sizeof(u32); 1846 } 1847 1848 u32 xe_lrc_ring_tail(struct xe_lrc *lrc) 1849 { 1850 if (xe_lrc_has_indirect_ring_state(lrc)) 1851 return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL) & TAIL_ADDR; 1852 else 1853 return xe_lrc_read_ctx_reg(lrc, CTX_RING_TAIL) & TAIL_ADDR; 1854 } 1855 1856 static u32 xe_lrc_ring_start(struct xe_lrc *lrc) 1857 { 1858 if (xe_lrc_has_indirect_ring_state(lrc)) 1859 return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START); 1860 else 1861 return xe_lrc_read_ctx_reg(lrc, CTX_RING_START); 1862 } 1863 1864 void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head) 1865 { 1866 if (xe_lrc_has_indirect_ring_state(lrc)) 1867 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, head); 1868 else 1869 xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, head); 1870 } 1871 1872 u32 xe_lrc_ring_head(struct xe_lrc *lrc) 1873 { 1874 if (xe_lrc_has_indirect_ring_state(lrc)) 1875 return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD) & HEAD_ADDR; 1876 else 1877 return xe_lrc_read_ctx_reg(lrc, CTX_RING_HEAD) & HEAD_ADDR; 1878 } 1879 1880 u32 xe_lrc_ring_space(struct xe_lrc *lrc) 1881 { 1882 const u32 head = xe_lrc_ring_head(lrc); 1883 const u32 tail = lrc->ring.tail; 1884 const u32 size = lrc->ring.size; 1885 1886 return ((head - tail - 1) & (size - 1)) + 1; 1887 } 1888 1889 static void __xe_lrc_write_ring(struct xe_lrc *lrc, struct iosys_map ring, 1890 const void *data, size_t size) 1891 { 1892 struct xe_device *xe = lrc_to_xe(lrc); 1893 1894 iosys_map_incr(&ring, lrc->ring.tail); 1895 xe_map_memcpy_to(xe, &ring, 0, data, size); 1896 lrc->ring.tail = (lrc->ring.tail + size) & (lrc->ring.size - 1); 1897 } 1898 1899 void xe_lrc_write_ring(struct xe_lrc *lrc, const void *data, size_t size) 1900 { 1901 struct xe_device *xe = lrc_to_xe(lrc); 1902 struct iosys_map ring; 1903 u32 rhs; 1904 size_t aligned_size; 1905 1906 xe_assert(xe, IS_ALIGNED(size, 4)); 1907 aligned_size = ALIGN(size, 8); 1908 1909 ring = __xe_lrc_ring_map(lrc); 1910 1911 xe_assert(xe, lrc->ring.tail < lrc->ring.size); 1912 rhs = lrc->ring.size - lrc->ring.tail; 1913 if (size > rhs) { 1914 __xe_lrc_write_ring(lrc, ring, data, rhs); 1915 __xe_lrc_write_ring(lrc, ring, data + rhs, size - rhs); 1916 } else { 1917 __xe_lrc_write_ring(lrc, ring, data, size); 1918 } 1919 1920 if (aligned_size > size) { 1921 u32 noop = MI_NOOP; 1922 1923 __xe_lrc_write_ring(lrc, ring, &noop, sizeof(noop)); 1924 } 1925 1926 /* 1927 * The ring and the LRC context image are both WC, so the ring tail 1928 * update which publishes these writes can become visible to the device 1929 * first. Ensure the ring contents are visible before returning. 1930 */ 1931 xe_device_wmb(xe); 1932 } 1933 1934 u64 xe_lrc_descriptor(struct xe_lrc *lrc) 1935 { 1936 return lrc->desc | xe_lrc_ggtt_addr(lrc); 1937 } 1938 1939 u32 xe_lrc_seqno_ggtt_addr(struct xe_lrc *lrc) 1940 { 1941 return __xe_lrc_seqno_ggtt_addr(lrc); 1942 } 1943 1944 /** 1945 * xe_lrc_alloc_seqno_fence() - Allocate an lrc seqno fence. 1946 * 1947 * Allocate but don't initialize an lrc seqno fence. 1948 * 1949 * Return: Pointer to the allocated fence or 1950 * negative error pointer on error. 1951 */ 1952 struct dma_fence *xe_lrc_alloc_seqno_fence(void) 1953 { 1954 return xe_hw_fence_alloc(); 1955 } 1956 1957 /** 1958 * xe_lrc_free_seqno_fence() - Free an lrc seqno fence. 1959 * @fence: Pointer to the fence to free. 1960 * 1961 * Frees an lrc seqno fence that hasn't yet been 1962 * initialized. 1963 */ 1964 void xe_lrc_free_seqno_fence(struct dma_fence *fence) 1965 { 1966 xe_hw_fence_free(fence); 1967 } 1968 1969 /** 1970 * xe_lrc_init_seqno_fence() - Initialize an lrc seqno fence. 1971 * @lrc: Pointer to the lrc. 1972 * @fence: Pointer to the fence to initialize. 1973 * 1974 * Initializes a pre-allocated lrc seqno fence. 1975 * After initialization, the fence is subject to normal 1976 * dma-fence refcounting. 1977 */ 1978 void xe_lrc_init_seqno_fence(struct xe_lrc *lrc, struct dma_fence *fence) 1979 { 1980 xe_hw_fence_init(fence, &lrc->fence_ctx, __xe_lrc_seqno_map(lrc)); 1981 } 1982 1983 s32 xe_lrc_seqno(struct xe_lrc *lrc) 1984 { 1985 struct iosys_map map = __xe_lrc_seqno_map(lrc); 1986 1987 return xe_map_read32(lrc_to_xe(lrc), &map); 1988 } 1989 1990 s32 xe_lrc_start_seqno(struct xe_lrc *lrc) 1991 { 1992 struct iosys_map map = __xe_lrc_start_seqno_map(lrc); 1993 1994 return xe_map_read32(lrc_to_xe(lrc), &map); 1995 } 1996 1997 u32 xe_lrc_start_seqno_ggtt_addr(struct xe_lrc *lrc) 1998 { 1999 return __xe_lrc_start_seqno_ggtt_addr(lrc); 2000 } 2001 2002 u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc) 2003 { 2004 return __xe_lrc_parallel_ggtt_addr(lrc); 2005 } 2006 2007 struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc) 2008 { 2009 return __xe_lrc_parallel_map(lrc); 2010 } 2011 2012 /** 2013 * xe_lrc_engine_id() - Read engine id value 2014 * @lrc: Pointer to the lrc. 2015 * 2016 * Returns: context id value 2017 */ 2018 static u32 xe_lrc_engine_id(struct xe_lrc *lrc) 2019 { 2020 struct xe_device *xe = lrc_to_xe(lrc); 2021 struct iosys_map map; 2022 2023 map = __xe_lrc_engine_id_map(lrc); 2024 return xe_map_read32(xe, &map); 2025 } 2026 2027 #define semaphore_offset(seqno) \ 2028 (sizeof(u32) * ((seqno) % LRC_MIGRATION_ULLS_SEMAPHORE_COUNT)) 2029 2030 /** 2031 * xe_lrc_ulls_semaphore_ggtt_addr() - ULLS semaphore GGTT address 2032 * @lrc: Pointer to the lrc. 2033 * @seqno: seqno of current job. 2034 * 2035 * Calculate ULLS semaphore GGTT address based on input seqno 2036 * 2037 * Returns: ULLS semaphore GGTT address 2038 */ 2039 u32 xe_lrc_ulls_semaphore_ggtt_addr(struct xe_lrc *lrc, u32 seqno) 2040 { 2041 xe_assert(lrc_to_xe(lrc), semaphore_offset(seqno) < 2042 LRC_PPHWSP_SIZE - LRC_ULLS_PPHWSP_OFFSET); 2043 2044 return __xe_lrc_ulls_ggtt_addr(lrc) + semaphore_offset(seqno); 2045 } 2046 2047 /** 2048 * xe_lrc_set_ulls_semaphore() - Set ULLS semaphore 2049 * @lrc: Pointer to the lrc. 2050 * @seqno: seqno of current job. 2051 * 2052 * Set ULLS semaphore based on input seqno 2053 */ 2054 void xe_lrc_set_ulls_semaphore(struct xe_lrc *lrc, u32 seqno) 2055 { 2056 struct xe_device *xe = lrc_to_xe(lrc); 2057 struct iosys_map map = __xe_lrc_ulls_map(lrc); 2058 2059 xe_assert(xe, semaphore_offset(seqno) < 2060 LRC_PPHWSP_SIZE - LRC_ULLS_PPHWSP_OFFSET); 2061 2062 /* 2063 * The ring contents this semaphore releases are ordered by the 2064 * xe_device_wmb() at the end of xe_lrc_write_ring(). 2065 */ 2066 iosys_map_incr(&map, semaphore_offset(seqno)); 2067 xe_map_write32(xe, &map, LRC_MIGRATION_ULLS_SEMAPHORE_SIGNAL); 2068 2069 xe_device_wmb(xe); /* Flush write to hardware */ 2070 } 2071 2072 static int instr_dw(u32 cmd_header) 2073 { 2074 /* GFXPIPE "SINGLE_DW" opcodes are a single dword */ 2075 if ((cmd_header & (XE_INSTR_CMD_TYPE | GFXPIPE_PIPELINE)) == 2076 GFXPIPE_SINGLE_DW_CMD(0, 0)) 2077 return 1; 2078 2079 /* 3DSTATE_SO_DECL_LIST has a 9-bit dword length rather than 8 */ 2080 if ((cmd_header & GFXPIPE_MATCH_MASK) == CMD_3DSTATE_SO_DECL_LIST) 2081 return REG_FIELD_GET(CMD_3DSTATE_SO_DECL_LIST_DW_LEN, cmd_header) + 2; 2082 2083 /* Most instructions have the # of dwords (minus 2) in 7:0 */ 2084 return REG_FIELD_GET(XE_INSTR_LEN_MASK, cmd_header) + 2; 2085 } 2086 2087 static int dump_mi_command(struct drm_printer *p, 2088 struct xe_gt *gt, 2089 u32 *start, 2090 u32 *dw, 2091 int remaining_dw) 2092 { 2093 u32 inst_header = *dw; 2094 u32 numdw = instr_dw(inst_header); 2095 u32 opcode = REG_FIELD_GET(MI_OPCODE, inst_header); 2096 int num_noop; 2097 2098 /* First check for commands that don't have/use a '# DW' field */ 2099 switch (inst_header & MI_OPCODE) { 2100 case MI_NOOP: 2101 num_noop = 1; 2102 while (num_noop < remaining_dw && 2103 (*(++dw) & REG_GENMASK(31, 23)) == MI_NOOP) 2104 num_noop++; 2105 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_NOOP (%d dwords)\n", 2106 dw - num_noop - start, inst_header, num_noop); 2107 return num_noop; 2108 2109 case MI_TOPOLOGY_FILTER: 2110 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_TOPOLOGY_FILTER\n", 2111 dw - start, inst_header); 2112 return 1; 2113 2114 case MI_BATCH_BUFFER_END: 2115 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_BATCH_BUFFER_END\n", 2116 dw - start, inst_header); 2117 /* Return 'remaining_dw' to consume the rest of the LRC */ 2118 return remaining_dw; 2119 } 2120 2121 /* 2122 * Any remaining commands include a # of dwords. We should make sure 2123 * it doesn't exceed the remaining size of the LRC. 2124 */ 2125 if (xe_gt_WARN_ON(gt, numdw > remaining_dw)) 2126 numdw = remaining_dw; 2127 2128 switch (inst_header & MI_OPCODE) { 2129 case MI_LOAD_REGISTER_IMM: 2130 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_LOAD_REGISTER_IMM: %d regs\n", 2131 dw - start, inst_header, (numdw - 1) / 2); 2132 for (int i = 1; i < numdw; i += 2) 2133 drm_printf(p, "LRC[%#5tx] = - %#6x = %#010x\n", 2134 &dw[i] - start, dw[i], dw[i + 1]); 2135 return numdw; 2136 2137 case MI_LOAD_REGISTER_MEM & MI_OPCODE: 2138 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_LOAD_REGISTER_MEM: %s%s\n", 2139 dw - start, inst_header, 2140 dw[0] & MI_LRI_LRM_CS_MMIO ? "CS_MMIO " : "", 2141 dw[0] & MI_LRM_USE_GGTT ? "USE_GGTT " : ""); 2142 if (numdw == 4) 2143 drm_printf(p, "LRC[%#5tx] = - %#6x = %#010llx\n", 2144 dw - start, 2145 dw[1], ((u64)(dw[3]) << 32 | (u64)(dw[2]))); 2146 else 2147 drm_printf(p, "LRC[%#5tx] = - %*ph (%s)\n", 2148 dw - start, (int)sizeof(u32) * (numdw - 1), 2149 dw + 1, numdw < 4 ? "truncated" : "malformed"); 2150 return numdw; 2151 2152 case MI_FORCE_WAKEUP: 2153 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_FORCE_WAKEUP\n", 2154 dw - start, inst_header); 2155 return numdw; 2156 2157 default: 2158 drm_printf(p, "LRC[%#5tx] = [%#010x] unknown MI opcode %#x, likely %d dwords\n", 2159 dw - start, inst_header, opcode, numdw); 2160 return numdw; 2161 } 2162 } 2163 2164 static int dump_gfxpipe_command(struct drm_printer *p, 2165 struct xe_gt *gt, 2166 u32 *start, 2167 u32 *dw, 2168 int remaining_dw) 2169 { 2170 u32 numdw = instr_dw(*dw); 2171 u32 pipeline = REG_FIELD_GET(GFXPIPE_PIPELINE, *dw); 2172 u32 opcode = REG_FIELD_GET(GFXPIPE_OPCODE, *dw); 2173 u32 subopcode = REG_FIELD_GET(GFXPIPE_SUBOPCODE, *dw); 2174 2175 /* 2176 * Make sure we haven't mis-parsed a number of dwords that exceeds the 2177 * remaining size of the LRC. 2178 */ 2179 if (xe_gt_WARN_ON(gt, numdw > remaining_dw)) 2180 numdw = remaining_dw; 2181 2182 switch (*dw & GFXPIPE_MATCH_MASK) { 2183 #define MATCH(cmd) \ 2184 case cmd: \ 2185 drm_printf(p, "LRC[%#5tx] = [%#010x] " #cmd " (%d dwords)\n", \ 2186 dw - start, *dw, numdw); \ 2187 return numdw 2188 #define MATCH3D(cmd) \ 2189 case CMD_##cmd: \ 2190 drm_printf(p, "LRC[%#5tx] = [%#010x] " #cmd " (%d dwords)\n", \ 2191 dw - start, *dw, numdw); \ 2192 return numdw 2193 2194 MATCH(STATE_BASE_ADDRESS); 2195 MATCH(STATE_SIP); 2196 MATCH(GPGPU_CSR_BASE_ADDRESS); 2197 MATCH(STATE_COMPUTE_MODE); 2198 MATCH3D(3DSTATE_BTD); 2199 MATCH(STATE_SYSTEM_MEM_FENCE_ADDRESS); 2200 MATCH(STATE_CONTEXT_DATA_BASE_ADDRESS); 2201 2202 MATCH3D(3DSTATE_VF_STATISTICS); 2203 2204 MATCH(PIPELINE_SELECT); 2205 2206 MATCH3D(3DSTATE_DRAWING_RECTANGLE_FAST); 2207 MATCH3D(3DSTATE_CUSTOM_SAMPLE_PATTERN); 2208 MATCH3D(3DSTATE_CLEAR_PARAMS); 2209 MATCH3D(3DSTATE_DEPTH_BUFFER); 2210 MATCH3D(3DSTATE_STENCIL_BUFFER); 2211 MATCH3D(3DSTATE_HIER_DEPTH_BUFFER); 2212 MATCH3D(3DSTATE_VERTEX_BUFFERS); 2213 MATCH3D(3DSTATE_VERTEX_ELEMENTS); 2214 MATCH3D(3DSTATE_INDEX_BUFFER); 2215 MATCH3D(3DSTATE_VF); 2216 MATCH3D(3DSTATE_MULTISAMPLE); 2217 MATCH3D(3DSTATE_CC_STATE_POINTERS); 2218 MATCH3D(3DSTATE_SCISSOR_STATE_POINTERS); 2219 MATCH3D(3DSTATE_VS); 2220 MATCH3D(3DSTATE_GS); 2221 MATCH3D(3DSTATE_CLIP); 2222 MATCH3D(3DSTATE_SF); 2223 MATCH3D(3DSTATE_WM); 2224 MATCH3D(3DSTATE_CONSTANT_VS); 2225 MATCH3D(3DSTATE_CONSTANT_GS); 2226 MATCH3D(3DSTATE_CONSTANT_PS); 2227 MATCH3D(3DSTATE_SAMPLE_MASK); 2228 MATCH3D(3DSTATE_CONSTANT_HS); 2229 MATCH3D(3DSTATE_CONSTANT_DS); 2230 MATCH3D(3DSTATE_HS); 2231 MATCH3D(3DSTATE_TE); 2232 MATCH3D(3DSTATE_DS); 2233 MATCH3D(3DSTATE_STREAMOUT); 2234 MATCH3D(3DSTATE_SBE); 2235 MATCH3D(3DSTATE_PS); 2236 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP); 2237 MATCH3D(3DSTATE_CPS_POINTERS); 2238 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_CC); 2239 MATCH3D(3DSTATE_BLEND_STATE_POINTERS); 2240 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_VS); 2241 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_HS); 2242 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_DS); 2243 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_GS); 2244 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_PS); 2245 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_VS); 2246 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_HS); 2247 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_DS); 2248 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_GS); 2249 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_PS); 2250 MATCH3D(3DSTATE_VF_INSTANCING); 2251 MATCH3D(3DSTATE_VF_SGVS); 2252 MATCH3D(3DSTATE_VF_TOPOLOGY); 2253 MATCH3D(3DSTATE_WM_CHROMAKEY); 2254 MATCH3D(3DSTATE_PS_BLEND); 2255 MATCH3D(3DSTATE_WM_DEPTH_STENCIL); 2256 MATCH3D(3DSTATE_PS_EXTRA); 2257 MATCH3D(3DSTATE_RASTER); 2258 MATCH3D(3DSTATE_SBE_SWIZ); 2259 MATCH3D(3DSTATE_WM_HZ_OP); 2260 MATCH3D(3DSTATE_VF_COMPONENT_PACKING); 2261 MATCH3D(3DSTATE_VF_SGVS_2); 2262 MATCH3D(3DSTATE_VFG); 2263 MATCH3D(3DSTATE_URB_ALLOC_VS); 2264 MATCH3D(3DSTATE_URB_ALLOC_HS); 2265 MATCH3D(3DSTATE_URB_ALLOC_DS); 2266 MATCH3D(3DSTATE_URB_ALLOC_GS); 2267 MATCH3D(3DSTATE_SO_BUFFER_INDEX_0); 2268 MATCH3D(3DSTATE_SO_BUFFER_INDEX_1); 2269 MATCH3D(3DSTATE_SO_BUFFER_INDEX_2); 2270 MATCH3D(3DSTATE_SO_BUFFER_INDEX_3); 2271 MATCH3D(3DSTATE_PRIMITIVE_REPLICATION); 2272 MATCH3D(3DSTATE_TBIMR_TILE_PASS_INFO); 2273 MATCH3D(3DSTATE_AMFS); 2274 MATCH3D(3DSTATE_DEPTH_BOUNDS); 2275 MATCH3D(3DSTATE_AMFS_TEXTURE_POINTERS); 2276 MATCH3D(3DSTATE_CONSTANT_TS_POINTER); 2277 MATCH3D(3DSTATE_MESH_CONTROL); 2278 MATCH3D(3DSTATE_MESH_DISTRIB); 2279 MATCH3D(3DSTATE_TASK_REDISTRIB); 2280 MATCH3D(3DSTATE_MESH_SHADER); 2281 MATCH3D(3DSTATE_MESH_SHADER_DATA); 2282 MATCH3D(3DSTATE_TASK_CONTROL); 2283 MATCH3D(3DSTATE_TASK_SHADER); 2284 MATCH3D(3DSTATE_TASK_SHADER_DATA); 2285 MATCH3D(3DSTATE_URB_ALLOC_MESH); 2286 MATCH3D(3DSTATE_URB_ALLOC_TASK); 2287 MATCH3D(3DSTATE_CLIP_MESH); 2288 MATCH3D(3DSTATE_SBE_MESH); 2289 MATCH3D(3DSTATE_CPSIZE_CONTROL_BUFFER); 2290 MATCH3D(3DSTATE_COARSE_PIXEL); 2291 MATCH3D(3DSTATE_MESH_SHADER_DATA_EXT); 2292 MATCH3D(3DSTATE_TASK_SHADER_DATA_EXT); 2293 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_CC_2); 2294 MATCH3D(3DSTATE_CC_STATE_POINTERS_2); 2295 MATCH3D(3DSTATE_SCISSOR_STATE_POINTERS_2); 2296 MATCH3D(3DSTATE_BLEND_STATE_POINTERS_2); 2297 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP_2); 2298 2299 MATCH3D(3DSTATE_DRAWING_RECTANGLE); 2300 MATCH3D(3DSTATE_URB_MEMORY); 2301 MATCH3D(3DSTATE_CHROMA_KEY); 2302 MATCH3D(3DSTATE_POLY_STIPPLE_OFFSET); 2303 MATCH3D(3DSTATE_POLY_STIPPLE_PATTERN); 2304 MATCH3D(3DSTATE_LINE_STIPPLE); 2305 MATCH3D(3DSTATE_AA_LINE_PARAMETERS); 2306 MATCH3D(3DSTATE_MONOFILTER_SIZE); 2307 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_VS); 2308 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_HS); 2309 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_DS); 2310 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_GS); 2311 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_PS); 2312 MATCH3D(3DSTATE_SO_DECL_LIST); 2313 MATCH3D(3DSTATE_SO_BUFFER); 2314 MATCH3D(3DSTATE_BINDING_TABLE_POOL_ALLOC); 2315 MATCH3D(3DSTATE_SAMPLE_PATTERN); 2316 MATCH3D(3DSTATE_3D_MODE); 2317 MATCH3D(3DSTATE_SUBSLICE_HASH_TABLE); 2318 MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTERS); 2319 MATCH3D(3DSTATE_PTBR_TILE_PASS_INFO); 2320 MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTER_2); 2321 2322 default: 2323 drm_printf(p, "LRC[%#5tx] = [%#010x] unknown GFXPIPE command (pipeline=%#x, opcode=%#x, subopcode=%#x), likely %d dwords\n", 2324 dw - start, *dw, pipeline, opcode, subopcode, numdw); 2325 return numdw; 2326 } 2327 } 2328 2329 static int dump_gfx_state_command(struct drm_printer *p, 2330 struct xe_gt *gt, 2331 u32 *start, 2332 u32 *dw, 2333 int remaining_dw) 2334 { 2335 u32 numdw = instr_dw(*dw); 2336 u32 opcode = REG_FIELD_GET(GFX_STATE_OPCODE, *dw); 2337 2338 /* 2339 * Make sure we haven't mis-parsed a number of dwords that exceeds the 2340 * remaining size of the LRC. 2341 */ 2342 if (xe_gt_WARN_ON(gt, numdw > remaining_dw)) 2343 numdw = remaining_dw; 2344 2345 switch (*dw & (XE_INSTR_GFX_STATE | GFX_STATE_OPCODE)) { 2346 MATCH(STATE_WRITE_INLINE); 2347 2348 default: 2349 drm_printf(p, "LRC[%#5tx] = [%#010x] unknown GFX_STATE command (opcode=%#x), likely %d dwords\n", 2350 dw - start, *dw, opcode, numdw); 2351 return numdw; 2352 } 2353 } 2354 2355 void xe_lrc_dump_default(struct drm_printer *p, 2356 struct xe_gt *gt, 2357 enum xe_engine_class hwe_class) 2358 { 2359 u32 *dw, *start; 2360 int remaining_dw, num_dw; 2361 2362 if (!gt->default_lrc[hwe_class]) { 2363 drm_printf(p, "No default LRC for class %d\n", hwe_class); 2364 return; 2365 } 2366 2367 /* 2368 * Skip the beginning of the LRC since it contains the per-process 2369 * hardware status page. 2370 */ 2371 dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE; 2372 start = dw; 2373 remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4; 2374 2375 while (remaining_dw > 0) { 2376 if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_MI) { 2377 num_dw = dump_mi_command(p, gt, start, dw, remaining_dw); 2378 } else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE) { 2379 num_dw = dump_gfxpipe_command(p, gt, start, dw, remaining_dw); 2380 } else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFX_STATE) { 2381 num_dw = dump_gfx_state_command(p, gt, start, dw, remaining_dw); 2382 } else { 2383 num_dw = min(instr_dw(*dw), remaining_dw); 2384 drm_printf(p, "LRC[%#5tx] = [%#10x] Unknown instruction of type %#x, likely %d dwords\n", 2385 dw - start, 2386 *dw, REG_FIELD_GET(XE_INSTR_CMD_TYPE, *dw), 2387 num_dw); 2388 } 2389 2390 dw += num_dw; 2391 remaining_dw -= num_dw; 2392 } 2393 } 2394 2395 /* 2396 * Lookup the value of a register within the offset/value pairs of an 2397 * MI_LOAD_REGISTER_IMM instruction. 2398 * 2399 * Return -ENOENT if the register is not present in the MI_LRI instruction. 2400 */ 2401 static int lookup_reg_in_mi_lri(u32 offset, u32 *value, 2402 const u32 *dword_pair, int num_regs) 2403 { 2404 for (int i = 0; i < num_regs; i++) { 2405 if (dword_pair[2 * i] == offset) { 2406 *value = dword_pair[2 * i + 1]; 2407 return 0; 2408 } 2409 } 2410 2411 return -ENOENT; 2412 } 2413 2414 /* 2415 * Lookup the value of a register in a specific engine type's default LRC. 2416 * 2417 * Return -EINVAL if the default LRC doesn't exist, or ENOENT if the register 2418 * cannot be found in the default LRC. 2419 */ 2420 int xe_lrc_lookup_default_reg_value(struct xe_gt *gt, 2421 enum xe_engine_class hwe_class, 2422 u32 offset, 2423 u32 *value) 2424 { 2425 u32 *dw; 2426 int remaining_dw, ret; 2427 2428 if (!gt->default_lrc[hwe_class]) 2429 return -EINVAL; 2430 2431 /* 2432 * Skip the beginning of the LRC since it contains the per-process 2433 * hardware status page. 2434 */ 2435 dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE; 2436 remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4; 2437 2438 while (remaining_dw > 0) { 2439 u32 num_dw = instr_dw(*dw); 2440 2441 if (num_dw > remaining_dw) 2442 num_dw = remaining_dw; 2443 2444 switch (*dw & XE_INSTR_CMD_TYPE) { 2445 case XE_INSTR_MI: 2446 switch (*dw & MI_OPCODE) { 2447 case MI_BATCH_BUFFER_END: 2448 /* End of LRC; register not found */ 2449 return -ENOENT; 2450 2451 case MI_NOOP: 2452 case MI_TOPOLOGY_FILTER: 2453 /* 2454 * MI_NOOP and MI_TOPOLOGY_FILTER don't have 2455 * a length field and are always 1-dword 2456 * instructions. 2457 */ 2458 remaining_dw--; 2459 dw++; 2460 break; 2461 2462 case MI_LOAD_REGISTER_IMM: 2463 ret = lookup_reg_in_mi_lri(offset, value, 2464 dw + 1, (num_dw - 1) / 2); 2465 if (ret == 0) 2466 return 0; 2467 2468 fallthrough; 2469 2470 default: 2471 /* 2472 * Jump to next instruction based on length 2473 * field. 2474 */ 2475 remaining_dw -= num_dw; 2476 dw += num_dw; 2477 break; 2478 } 2479 break; 2480 2481 default: 2482 /* Jump to next instruction based on length field. */ 2483 remaining_dw -= num_dw; 2484 dw += num_dw; 2485 } 2486 } 2487 2488 return -ENOENT; 2489 } 2490 2491 struct instr_state { 2492 u32 instr; 2493 u16 num_dw; 2494 }; 2495 2496 static const struct instr_state xe_hpg_svg_state[] = { 2497 { .instr = CMD_3DSTATE_CONSTANT_VS, .num_dw = 11 }, 2498 { .instr = CMD_3DSTATE_CONSTANT_HS, .num_dw = 11 }, 2499 { .instr = CMD_3DSTATE_CONSTANT_DS, .num_dw = 11 }, 2500 { .instr = CMD_3DSTATE_CONSTANT_GS, .num_dw = 11 }, 2501 { .instr = CMD_3DSTATE_VERTEX_ELEMENTS, .num_dw = 69 }, 2502 { .instr = CMD_3DSTATE_VF_COMPONENT_PACKING, .num_dw = 5 }, 2503 { .instr = CMD_3DSTATE_VF_SGVS, .num_dw = 2 }, 2504 { .instr = CMD_3DSTATE_VF_SGVS_2, .num_dw = 3 }, 2505 { .instr = CMD_3DSTATE_VS, .num_dw = 9 }, 2506 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_VS, .num_dw = 2 }, 2507 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_VS, .num_dw = 2 }, 2508 { .instr = CMD_3DSTATE_URB_ALLOC_VS, .num_dw = 3 }, 2509 { .instr = CMD_3DSTATE_STREAMOUT, .num_dw = 5 }, 2510 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_0, .num_dw = 8 }, 2511 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_1, .num_dw = 8 }, 2512 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_2, .num_dw = 8 }, 2513 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_3, .num_dw = 8 }, 2514 { .instr = CMD_3DSTATE_CLIP, .num_dw = 4 }, 2515 { .instr = CMD_3DSTATE_PRIMITIVE_REPLICATION, .num_dw = 6 }, 2516 { .instr = CMD_3DSTATE_CLIP_MESH, .num_dw = 2 }, 2517 { .instr = CMD_3DSTATE_SF, .num_dw = 4 }, 2518 { .instr = CMD_3DSTATE_SCISSOR_STATE_POINTERS, .num_dw = 2 }, 2519 { .instr = CMD_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP, .num_dw = 2 }, 2520 { .instr = CMD_3DSTATE_RASTER, .num_dw = 5 }, 2521 { .instr = CMD_3DSTATE_TBIMR_TILE_PASS_INFO, .num_dw = 4 }, 2522 { .instr = CMD_3DSTATE_WM_HZ_OP, .num_dw = 6 }, 2523 { .instr = CMD_3DSTATE_MULTISAMPLE, .num_dw = 2 }, 2524 { .instr = CMD_3DSTATE_HS, .num_dw = 9 }, 2525 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_HS, .num_dw = 2 }, 2526 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_HS, .num_dw = 2 }, 2527 { .instr = CMD_3DSTATE_URB_ALLOC_HS, .num_dw = 3 }, 2528 { .instr = CMD_3DSTATE_TASK_CONTROL, .num_dw = 3 }, 2529 { .instr = CMD_3DSTATE_TASK_SHADER, .num_dw = 7 }, 2530 { .instr = CMD_3DSTATE_TASK_SHADER_DATA, .num_dw = 10 }, 2531 { .instr = CMD_3DSTATE_URB_ALLOC_TASK, .num_dw = 3 }, 2532 { .instr = CMD_3DSTATE_TE, .num_dw = 5 }, 2533 { .instr = CMD_3DSTATE_TASK_REDISTRIB, .num_dw = 2 }, 2534 { .instr = CMD_3DSTATE_DS, .num_dw = 11 }, 2535 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_DS, .num_dw = 2 }, 2536 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_DS, .num_dw = 2 }, 2537 { .instr = CMD_3DSTATE_URB_ALLOC_DS, .num_dw = 3 }, 2538 { .instr = CMD_3DSTATE_GS, .num_dw = 10 }, 2539 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_GS, .num_dw = 2 }, 2540 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_GS, .num_dw = 2 }, 2541 { .instr = CMD_3DSTATE_URB_ALLOC_GS, .num_dw = 3 }, 2542 { .instr = CMD_3DSTATE_MESH_CONTROL, .num_dw = 3 }, 2543 { .instr = CMD_3DSTATE_MESH_SHADER_DATA, .num_dw = 10 }, 2544 { .instr = CMD_3DSTATE_URB_ALLOC_MESH, .num_dw = 3 }, 2545 { .instr = CMD_3DSTATE_MESH_SHADER, .num_dw = 8 }, 2546 { .instr = CMD_3DSTATE_DRAWING_RECTANGLE, .num_dw = 4 }, 2547 }; 2548 2549 u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs) 2550 { 2551 struct xe_gt *gt = q->hwe->gt; 2552 struct xe_device *xe = gt_to_xe(gt); 2553 const struct instr_state *state_table = NULL; 2554 int state_table_size = 0; 2555 2556 /* 2557 * Wa_14019789679 2558 * 2559 * If the driver doesn't explicitly emit the SVG instructions while 2560 * setting up the default LRC, the context switch will write 0's 2561 * (noops) into the LRC memory rather than the expected instruction 2562 * headers. Application contexts start out as a copy of the default 2563 * LRC, and if they also do not emit specific settings for some SVG 2564 * state, then on context restore they'll unintentionally inherit 2565 * whatever state setting the previous context had programmed into the 2566 * hardware (i.e., the lack of a 3DSTATE_* instruction in the LRC will 2567 * prevent the hardware from resetting that state back to any specific 2568 * value). 2569 * 2570 * The official workaround only requires emitting 3DSTATE_MESH_CONTROL 2571 * since that's a specific state setting that can easily cause GPU 2572 * hangs if unintentionally inherited. However to be safe we'll 2573 * continue to emit all of the SVG state since it's best not to leak 2574 * any of the state between contexts, even if that leakage is harmless. 2575 */ 2576 if (XE_GT_WA(gt, 14019789679) && q->hwe->class == XE_ENGINE_CLASS_RENDER) { 2577 state_table = xe_hpg_svg_state; 2578 state_table_size = ARRAY_SIZE(xe_hpg_svg_state); 2579 } 2580 2581 if (!state_table) { 2582 xe_gt_dbg(gt, "No non-register state to emit on graphics ver %d.%02d\n", 2583 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); 2584 return cs; 2585 } 2586 2587 for (int i = 0; i < state_table_size; i++) { 2588 u32 instr = state_table[i].instr; 2589 u16 num_dw = state_table[i].num_dw; 2590 bool is_single_dw = ((instr & GFXPIPE_PIPELINE) == PIPELINE_SINGLE_DW); 2591 2592 xe_gt_assert(gt, (instr & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE); 2593 xe_gt_assert(gt, num_dw != 0); 2594 xe_gt_assert(gt, is_single_dw ^ (num_dw > 1)); 2595 2596 /* 2597 * Xe2's SVG context is the same as the one on DG2 / MTL 2598 * except that 3DSTATE_DRAWING_RECTANGLE (non-pipelined) has 2599 * been replaced by 3DSTATE_DRAWING_RECTANGLE_FAST (pipelined). 2600 * Just make the replacement here rather than defining a 2601 * whole separate table for the single trivial change. 2602 */ 2603 if (GRAPHICS_VER(xe) >= 20 && 2604 instr == CMD_3DSTATE_DRAWING_RECTANGLE) 2605 instr = CMD_3DSTATE_DRAWING_RECTANGLE_FAST; 2606 2607 *cs = instr; 2608 if (!is_single_dw) 2609 *cs |= (num_dw - 2); 2610 2611 cs += num_dw; 2612 } 2613 2614 return cs; 2615 } 2616 2617 struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc) 2618 { 2619 struct xe_lrc_snapshot *snapshot = kmalloc_obj(*snapshot, GFP_NOWAIT); 2620 2621 if (!snapshot) 2622 return NULL; 2623 2624 snapshot->context_desc = xe_lrc_ggtt_addr(lrc); 2625 snapshot->ring_addr = __xe_lrc_ring_ggtt_addr(lrc); 2626 snapshot->indirect_context_desc = xe_lrc_indirect_ring_ggtt_addr(lrc); 2627 snapshot->head = xe_lrc_ring_head(lrc); 2628 snapshot->tail.internal = lrc->ring.tail; 2629 snapshot->tail.memory = xe_lrc_ring_tail(lrc); 2630 snapshot->start = xe_lrc_ring_start(lrc); 2631 snapshot->start_seqno = xe_lrc_start_seqno(lrc); 2632 snapshot->seqno = xe_lrc_seqno(lrc); 2633 snapshot->lrc_bo = xe_bo_get(lrc->bo); 2634 snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc); 2635 snapshot->lrc_size = lrc->size; 2636 snapshot->replay_offset = 0; 2637 snapshot->replay_size = lrc->replay_size; 2638 snapshot->lrc_snapshot = NULL; 2639 snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc); 2640 snapshot->ctx_timestamp_ms = 2641 xe_gt_clock_interval_to_ms(lrc->gt, xe_lrc_ctx_timestamp(lrc)); 2642 if (xe_lrc_is_multi_queue(lrc)) { 2643 snapshot->queue_timestamp = xe_lrc_queue_timestamp(lrc); 2644 snapshot->queue_timestamp_ms = 2645 xe_gt_clock_interval_to_ms(lrc->gt, snapshot->queue_timestamp); 2646 } else { 2647 snapshot->queue_timestamp = 0; 2648 snapshot->queue_timestamp_ms = 0; 2649 } 2650 snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); 2651 return snapshot; 2652 } 2653 2654 void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot) 2655 { 2656 struct xe_bo *bo; 2657 struct iosys_map src; 2658 2659 if (!snapshot) 2660 return; 2661 2662 bo = snapshot->lrc_bo; 2663 snapshot->lrc_bo = NULL; 2664 2665 snapshot->lrc_snapshot = kvmalloc(snapshot->lrc_size, GFP_KERNEL); 2666 if (!snapshot->lrc_snapshot) 2667 goto put_bo; 2668 2669 xe_bo_lock(bo, false); 2670 if (!ttm_bo_vmap(&bo->ttm, &src)) { 2671 xe_map_memcpy_from(xe_bo_device(bo), 2672 snapshot->lrc_snapshot, &src, snapshot->lrc_offset, 2673 snapshot->lrc_size); 2674 ttm_bo_vunmap(&bo->ttm, &src); 2675 } else { 2676 kvfree(snapshot->lrc_snapshot); 2677 snapshot->lrc_snapshot = NULL; 2678 } 2679 xe_bo_unlock(bo); 2680 put_bo: 2681 xe_bo_put(bo); 2682 } 2683 2684 void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p) 2685 { 2686 unsigned long i; 2687 2688 if (!snapshot) 2689 return; 2690 2691 drm_printf(p, "\tHW Context Desc: 0x%08x\n", snapshot->context_desc); 2692 drm_printf(p, "\tHW Ring address: 0x%08x\n", 2693 snapshot->ring_addr); 2694 drm_printf(p, "\tHW Indirect Ring State: 0x%08x\n", 2695 snapshot->indirect_context_desc); 2696 drm_printf(p, "\tLRC Head: (memory) %u\n", snapshot->head); 2697 drm_printf(p, "\tLRC Tail: (internal) %u, (memory) %u\n", 2698 snapshot->tail.internal, snapshot->tail.memory); 2699 drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start); 2700 drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno); 2701 drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); 2702 drm_printf(p, "\tTimestamp: 0x%016llx\n", snapshot->ctx_timestamp); 2703 drm_printf(p, "\tTimestamp ms: %llu\n", snapshot->ctx_timestamp_ms); 2704 drm_printf(p, "\tQueue Timestamp: 0x%016llx\n", snapshot->queue_timestamp); 2705 drm_printf(p, "\tQueue Timestamp ms: %llu\n", snapshot->queue_timestamp_ms); 2706 drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); 2707 2708 if (!snapshot->lrc_snapshot) 2709 return; 2710 2711 drm_printf(p, "\t[HWSP].length: 0x%x\n", LRC_PPHWSP_SIZE); 2712 drm_puts(p, "\t[HWSP].data: "); 2713 for (i = 0; i < LRC_PPHWSP_SIZE; i += sizeof(u32)) { 2714 u32 *val = snapshot->lrc_snapshot + i; 2715 char dumped[ASCII85_BUFSZ]; 2716 2717 drm_puts(p, ascii85_encode(*val, dumped)); 2718 } 2719 2720 drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE); 2721 drm_printf(p, "\n\t[HWCTX].replay_offset: 0x%lx\n", snapshot->replay_offset); 2722 drm_printf(p, "\n\t[HWCTX].replay_length: 0x%lx\n", snapshot->replay_size); 2723 2724 drm_puts(p, "\t[HWCTX].data: "); 2725 for (; i < snapshot->lrc_size; i += sizeof(u32)) { 2726 u32 *val = snapshot->lrc_snapshot + i; 2727 char dumped[ASCII85_BUFSZ]; 2728 2729 drm_puts(p, ascii85_encode(*val, dumped)); 2730 } 2731 drm_puts(p, "\n"); 2732 } 2733 2734 void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot) 2735 { 2736 if (!snapshot) 2737 return; 2738 2739 kvfree(snapshot->lrc_snapshot); 2740 if (snapshot->lrc_bo) 2741 xe_bo_put(snapshot->lrc_bo); 2742 2743 kfree(snapshot); 2744 } 2745 2746 static bool engine_valid_for_utilization(struct xe_gt *gt, struct xe_hw_engine *hwe) 2747 { 2748 /* The USM-reserved copy engine runs kernel migrate contexts queried here */ 2749 return hwe && (!xe_hw_engine_is_reserved(hwe) || xe_gt_is_usm_hwe(gt, hwe)); 2750 } 2751 2752 static struct xe_hw_engine *engine_id_to_hwe(struct xe_gt *gt, u32 engine_id) 2753 { 2754 u16 class = REG_FIELD_GET(ENGINE_CLASS_ID, engine_id); 2755 u16 instance = REG_FIELD_GET(ENGINE_INSTANCE_ID, engine_id); 2756 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, class, instance, false); 2757 2758 if (xe_gt_WARN_ONCE(gt, !engine_valid_for_utilization(gt, hwe), 2759 "Unexpected engine class:instance %d:%d for utilization\n", 2760 class, instance)) 2761 return NULL; 2762 2763 return hwe; 2764 } 2765 2766 static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) 2767 { 2768 struct xe_hw_engine *hwe; 2769 u64 val; 2770 2771 hwe = engine_id_to_hwe(lrc->gt, engine_id); 2772 if (!hwe) 2773 return -1; 2774 2775 if (lrc_to_xe(lrc)->info.has_64bit_timestamp) 2776 val = xe_mmio_read64_2x32(&hwe->gt->mmio, 2777 RING_CTX_TIMESTAMP(hwe->mmio_base)); 2778 else 2779 val = xe_mmio_read32(&hwe->gt->mmio, 2780 RING_CTX_TIMESTAMP(hwe->mmio_base)); 2781 2782 *reg_ctx_ts = val; 2783 2784 return 0; 2785 } 2786 2787 static u64 get_queue_timestamp(struct xe_hw_engine *hwe) 2788 { 2789 return xe_mmio_read64_2x32(&hwe->gt->mmio, 2790 RING_QUEUE_TIMESTAMP(hwe->mmio_base)); 2791 } 2792 2793 u32 xe_lrc_get_multi_queue_active_queue_id(struct xe_hw_engine *hwe) 2794 { 2795 u32 val = xe_mmio_read32(&hwe->gt->mmio, 2796 RING_CSMQDEBUG(hwe->mmio_base)); 2797 2798 return REG_FIELD_GET(CURRENT_ACTIVE_QUEUE_ID_MASK, val); 2799 } 2800 2801 static bool context_active(struct xe_lrc *lrc) 2802 { 2803 return xe_lrc_ctx_timestamp(lrc) == CONTEXT_ACTIVE; 2804 } 2805 2806 static u64 xe_lrc_multi_queue_timestamp(struct xe_lrc *lrc) 2807 { 2808 struct xe_device *xe = lrc_to_xe(lrc); 2809 struct xe_lrc *primary_lrc = lrc->multi_queue.primary_lrc; 2810 struct xe_hw_engine *hwe; 2811 u64 reg_queue_ts = lrc->queue_timestamp; 2812 2813 if (IS_SRIOV_VF(xe)) 2814 return xe_lrc_queue_timestamp(lrc); 2815 2816 xe_assert(xe, primary_lrc); 2817 2818 /* WA BB populates CONTEXT_ACTIVE cookie for primary context only */ 2819 if (!context_active(primary_lrc)) 2820 return xe_lrc_queue_timestamp(lrc); 2821 2822 /* WA BB populates engine id in PPHWSP of primary context only */ 2823 hwe = engine_id_to_hwe(primary_lrc->gt, xe_lrc_engine_id(primary_lrc)); 2824 if (!hwe) 2825 return xe_lrc_queue_timestamp(lrc); 2826 2827 if (xe_lrc_get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos) 2828 return xe_lrc_queue_timestamp(lrc); 2829 2830 /* queue is active, so store the queue timestamp register */ 2831 reg_queue_ts = get_queue_timestamp(hwe); 2832 2833 /* double check queue and primary queue are both still active */ 2834 if (xe_lrc_get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos || 2835 !context_active(primary_lrc)) 2836 return xe_lrc_queue_timestamp(lrc); 2837 2838 return reg_queue_ts; 2839 } 2840 2841 static u64 xe_lrc_update_multi_queue_timestamp(struct xe_lrc *lrc, u64 *old_ts) 2842 { 2843 *old_ts = lrc->queue_timestamp; 2844 lrc->queue_timestamp = xe_lrc_multi_queue_timestamp(lrc); 2845 2846 trace_xe_lrc_update_queue_timestamp(lrc, *old_ts); 2847 2848 return lrc->queue_timestamp; 2849 } 2850 2851 static u64 xe_lrc_context_timestamp(struct xe_lrc *lrc) 2852 { 2853 u64 reg_ts, new_ts = lrc->ctx_timestamp; 2854 u64 stored; 2855 2856 /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ 2857 if (IS_SRIOV_VF(lrc_to_xe(lrc))) 2858 return xe_lrc_ctx_timestamp(lrc); 2859 2860 /* Safely read CTX_TIMESTAMP: Avoid TOCTOU on LRC-stored CONTEXT_ACTIVE sentinel */ 2861 stored = xe_lrc_ctx_timestamp(lrc); 2862 if (stored != CONTEXT_ACTIVE) 2863 return stored; 2864 2865 /* Context is active: read the live timestamp from the engine's MMIO register */ 2866 if (!get_ctx_timestamp(lrc, xe_lrc_engine_id(lrc), ®_ts)) 2867 new_ts = reg_ts; 2868 2869 /* If the context switched out prefer using the value 2870 * from context-save over the stale MMIO read. 2871 */ 2872 stored = xe_lrc_ctx_timestamp(lrc); 2873 if (stored != CONTEXT_ACTIVE) 2874 return stored; 2875 2876 return new_ts; 2877 } 2878 2879 static u64 xe_lrc_update_context_timestamp(struct xe_lrc *lrc, u64 *old_ts) 2880 { 2881 *old_ts = lrc->ctx_timestamp; 2882 lrc->ctx_timestamp = xe_lrc_context_timestamp(lrc); 2883 2884 trace_xe_lrc_update_timestamp(lrc, *old_ts); 2885 2886 return lrc->ctx_timestamp; 2887 } 2888 2889 /** 2890 * xe_lrc_timestamp() - Current lrc timestamp 2891 * @lrc: Pointer to the lrc. 2892 * 2893 * Return latest lrc timestamp. With support for active contexts/queues, the 2894 * calculation may be slightly racy, so follow a read-again logic to ensure that 2895 * the context/queue is still active before returning the right timestamp. 2896 * 2897 * Returns: New lrc timestamp value 2898 */ 2899 u64 xe_lrc_timestamp(struct xe_lrc *lrc) 2900 { 2901 if (xe_lrc_is_multi_queue(lrc)) 2902 return xe_lrc_multi_queue_timestamp(lrc); 2903 else 2904 return xe_lrc_context_timestamp(lrc); 2905 } 2906 2907 /** 2908 * xe_lrc_update_timestamp() - Update lrc timestamp 2909 * @lrc: Pointer to the lrc. 2910 * @old_ts: Old timestamp value 2911 * 2912 * Populate @old_ts with current saved lrc timestamp, read new lrc timestamp and 2913 * update saved value. 2914 * 2915 * Returns: New lrc timestamp value 2916 */ 2917 u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts) 2918 { 2919 if (xe_lrc_is_multi_queue(lrc)) 2920 return xe_lrc_update_multi_queue_timestamp(lrc, old_ts); 2921 else 2922 return xe_lrc_update_context_timestamp(lrc, old_ts); 2923 } 2924 2925 /** 2926 * xe_lrc_ring_is_idle() - LRC is idle 2927 * @lrc: Pointer to the lrc. 2928 * 2929 * Compare LRC ring head and tail to determine if idle. 2930 * 2931 * Return: True is ring is idle, False otherwise 2932 */ 2933 bool xe_lrc_ring_is_idle(struct xe_lrc *lrc) 2934 { 2935 return xe_lrc_ring_head(lrc) == xe_lrc_ring_tail(lrc); 2936 } 2937