1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_lrc.h" 7 8 #include <generated/xe_wa_oob.h> 9 10 #include <linux/ascii85.h> 11 #include <linux/panic.h> 12 13 #include "instructions/xe_mi_commands.h" 14 #include "instructions/xe_gfxpipe_commands.h" 15 #include "instructions/xe_gfx_state_commands.h" 16 #include "regs/xe_engine_regs.h" 17 #include "regs/xe_gt_regs.h" 18 #include "regs/xe_lrc_layout.h" 19 #include "xe_bb.h" 20 #include "xe_bo.h" 21 #include "xe_configfs.h" 22 #include "xe_device.h" 23 #include "xe_drm_client.h" 24 #include "xe_exec_queue.h" 25 #include "xe_exec_queue_types.h" 26 #include "xe_gt.h" 27 #include "xe_gt_clock.h" 28 #include "xe_gt_printk.h" 29 #include "xe_hw_fence.h" 30 #include "xe_map.h" 31 #include "xe_memirq.h" 32 #include "xe_mmio.h" 33 #include "xe_ring_ops.h" 34 #include "xe_sriov.h" 35 #include "xe_trace_lrc.h" 36 #include "xe_vm.h" 37 #include "xe_wa.h" 38 39 #define LRC_VALID BIT_ULL(0) 40 #define LRC_PRIVILEGE BIT_ULL(8) 41 #define LRC_ADDRESSING_MODE GENMASK_ULL(4, 3) 42 #define LRC_LEGACY_64B_CONTEXT 3 43 44 #define LRC_ENGINE_CLASS GENMASK_ULL(63, 61) 45 #define LRC_ENGINE_INSTANCE GENMASK_ULL(53, 48) 46 47 #define LRC_PPHWSP_SIZE SZ_4K 48 #define LRC_INDIRECT_CTX_BO_SIZE SZ_4K 49 #define LRC_INDIRECT_RING_STATE_SIZE SZ_4K 50 51 #define LRC_PRIORITY GENMASK_ULL(10, 9) 52 #define LRC_PRIORITY_LOW 0 53 #define LRC_PRIORITY_NORMAL 1 54 #define LRC_PRIORITY_HIGH 2 55 56 /* 57 * Layout of the LRC and associated data allocated as 58 * lrc->bo: 59 * 60 * Region Size 61 * +============================+=================================+ <- __xe_lrc_ring_offset() 62 * | Ring | ring_size, see | 63 * | | xe_lrc_init() | 64 * +============================+=================================+ <- __xe_lrc_pphwsp_offset() 65 * | PPHWSP (includes SW state) | 4K | 66 * +----------------------------+---------------------------------+ <- __xe_lrc_regs_offset() 67 * | Engine Context Image | n * 4K, see | 68 * | | xe_gt_lrc_size() | 69 * +----------------------------+---------------------------------+ <- __xe_lrc_indirect_ring_offset() 70 * | Indirect Ring State Page | 0 or 4k, see | 71 * | | XE_LRC_FLAG_INDIRECT_RING_STATE | 72 * +============================+=================================+ <- __xe_lrc_indirect_ctx_offset() 73 * | Indirect Context Page | 0 or 4k, see | 74 * | | XE_LRC_FLAG_INDIRECT_CTX | 75 * +============================+=================================+ <- __xe_lrc_wa_bb_offset() 76 * | WA BB Per Ctx | 4k | 77 * +============================+=================================+ <- xe_bo_size(lrc->bo) 78 */ 79 80 static struct xe_device * 81 lrc_to_xe(struct xe_lrc *lrc) 82 { 83 return gt_to_xe(lrc->fence_ctx.gt); 84 } 85 86 static bool 87 gt_engine_needs_indirect_ctx(struct xe_gt *gt, enum xe_engine_class class) 88 { 89 struct xe_device *xe = gt_to_xe(gt); 90 91 if (XE_GT_WA(gt, 16010904313) && 92 (class == XE_ENGINE_CLASS_RENDER || 93 class == XE_ENGINE_CLASS_COMPUTE)) 94 return true; 95 96 if (xe_configfs_get_ctx_restore_mid_bb(to_pci_dev(xe->drm.dev), 97 class, NULL)) 98 return true; 99 100 if (gt->ring_ops[class]->emit_aux_table_inv) 101 return true; 102 103 return false; 104 } 105 106 /** 107 * xe_gt_lrc_hang_replay_size() - Hang replay size 108 * @gt: The GT 109 * @class: Hardware engine class 110 * 111 * Determine size of GPU hang replay state for a GT and hardware engine class. 112 * 113 * Return: Size of GPU hang replay size 114 */ 115 size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class) 116 { 117 struct xe_device *xe = gt_to_xe(gt); 118 size_t size = 0; 119 120 /* Engine context image */ 121 switch (class) { 122 case XE_ENGINE_CLASS_RENDER: 123 if (GRAPHICS_VERx100(xe) >= 3510) 124 size += 7 * SZ_4K; 125 else if (GRAPHICS_VER(xe) >= 20) 126 size += 3 * SZ_4K; 127 else 128 size += 13 * SZ_4K; 129 break; 130 case XE_ENGINE_CLASS_COMPUTE: 131 if (GRAPHICS_VERx100(xe) >= 3510) 132 size += 5 * SZ_4K; 133 else if (GRAPHICS_VER(xe) >= 20) 134 size += 2 * SZ_4K; 135 else 136 size += 13 * SZ_4K; 137 break; 138 default: 139 WARN(1, "Unknown engine class: %d", class); 140 fallthrough; 141 case XE_ENGINE_CLASS_COPY: 142 case XE_ENGINE_CLASS_VIDEO_DECODE: 143 case XE_ENGINE_CLASS_VIDEO_ENHANCE: 144 case XE_ENGINE_CLASS_OTHER: 145 size += 1 * SZ_4K; 146 } 147 148 return size; 149 } 150 151 size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class) 152 { 153 size_t size = xe_gt_lrc_hang_replay_size(gt, class); 154 155 /* Add indirect ring state page */ 156 if (xe_gt_has_indirect_ring_state(gt)) 157 size += LRC_INDIRECT_RING_STATE_SIZE; 158 159 return size + LRC_PPHWSP_SIZE; 160 } 161 162 /* 163 * The per-platform tables are u8-encoded in @data. Decode @data and set the 164 * addresses' offset and commands in @regs. The following encoding is used 165 * for each byte. There are 2 steps: decoding commands and decoding addresses. 166 * 167 * Commands: 168 * [7]: create NOPs - number of NOPs are set in lower bits 169 * [6]: When creating MI_LOAD_REGISTER_IMM command, allow to set 170 * MI_LRI_FORCE_POSTED 171 * [5:0]: Number of NOPs or registers to set values to in case of 172 * MI_LOAD_REGISTER_IMM 173 * 174 * Addresses: these are decoded after a MI_LOAD_REGISTER_IMM command by "count" 175 * number of registers. They are set by using the REG/REG16 macros: the former 176 * is used for offsets smaller than 0x200 while the latter is for values bigger 177 * than that. Those macros already set all the bits documented below correctly: 178 * 179 * [7]: When a register offset needs more than 6 bits, use additional bytes, to 180 * follow, for the lower bits 181 * [6:0]: Register offset, without considering the engine base. 182 * 183 * This function only tweaks the commands and register offsets. Values are not 184 * filled out. 185 */ 186 static void set_offsets(u32 *regs, 187 const u8 *data, 188 const struct xe_hw_engine *hwe) 189 #define NOP(x) (BIT(7) | (x)) 190 #define LRI(count, flags) ((flags) << 6 | (count) | \ 191 BUILD_BUG_ON_ZERO(count >= BIT(6))) 192 #define POSTED BIT(0) 193 #define REG(x) (((x) >> 2) | BUILD_BUG_ON_ZERO(x >= 0x200)) 194 #define REG16(x) \ 195 (((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \ 196 (((x) >> 2) & 0x7f) 197 { 198 const u32 base = hwe->mmio_base; 199 200 while (*data) { 201 u8 count, flags; 202 203 if (*data & BIT(7)) { /* skip */ 204 count = *data++ & ~BIT(7); 205 regs += count; 206 continue; 207 } 208 209 count = *data & 0x3f; 210 flags = *data >> 6; 211 data++; 212 213 *regs = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count); 214 if (flags & POSTED) 215 *regs |= MI_LRI_FORCE_POSTED; 216 *regs |= MI_LRI_LRM_CS_MMIO; 217 regs++; 218 219 xe_gt_assert(hwe->gt, count); 220 do { 221 u32 offset = 0; 222 u8 v; 223 224 do { 225 v = *data++; 226 offset <<= 7; 227 offset |= v & ~BIT(7); 228 } while (v & BIT(7)); 229 230 regs[0] = base + (offset << 2); 231 regs += 2; 232 } while (--count); 233 } 234 235 *regs = MI_BATCH_BUFFER_END | BIT(0); 236 } 237 238 static const u8 gen12_xcs_offsets[] = { 239 NOP(1), 240 LRI(13, POSTED), 241 REG16(0x244), 242 REG(0x034), 243 REG(0x030), 244 REG(0x038), 245 REG(0x03c), 246 REG(0x168), 247 REG(0x140), 248 REG(0x110), 249 REG(0x1c0), 250 REG(0x1c4), 251 REG(0x1c8), 252 REG(0x180), 253 REG16(0x2b4), 254 255 NOP(5), 256 LRI(9, POSTED), 257 REG16(0x3a8), 258 REG16(0x28c), 259 REG16(0x288), 260 REG16(0x284), 261 REG16(0x280), 262 REG16(0x27c), 263 REG16(0x278), 264 REG16(0x274), 265 REG16(0x270), 266 267 0 268 }; 269 270 static const u8 dg2_xcs_offsets[] = { 271 NOP(1), 272 LRI(15, POSTED), 273 REG16(0x244), 274 REG(0x034), 275 REG(0x030), 276 REG(0x038), 277 REG(0x03c), 278 REG(0x168), 279 REG(0x140), 280 REG(0x110), 281 REG(0x1c0), 282 REG(0x1c4), 283 REG(0x1c8), 284 REG(0x180), 285 REG16(0x2b4), 286 REG(0x120), 287 REG(0x124), 288 289 NOP(1), 290 LRI(9, POSTED), 291 REG16(0x3a8), 292 REG16(0x28c), 293 REG16(0x288), 294 REG16(0x284), 295 REG16(0x280), 296 REG16(0x27c), 297 REG16(0x278), 298 REG16(0x274), 299 REG16(0x270), 300 301 0 302 }; 303 304 static const u8 gen12_rcs_offsets[] = { 305 NOP(1), 306 LRI(13, POSTED), 307 REG16(0x244), 308 REG(0x034), 309 REG(0x030), 310 REG(0x038), 311 REG(0x03c), 312 REG(0x168), 313 REG(0x140), 314 REG(0x110), 315 REG(0x1c0), 316 REG(0x1c4), 317 REG(0x1c8), 318 REG(0x180), 319 REG16(0x2b4), 320 321 NOP(5), 322 LRI(9, POSTED), 323 REG16(0x3a8), 324 REG16(0x28c), 325 REG16(0x288), 326 REG16(0x284), 327 REG16(0x280), 328 REG16(0x27c), 329 REG16(0x278), 330 REG16(0x274), 331 REG16(0x270), 332 333 LRI(3, POSTED), 334 REG(0x1b0), 335 REG16(0x5a8), 336 REG16(0x5ac), 337 338 NOP(6), 339 LRI(1, 0), 340 REG(0x0c8), 341 NOP(3 + 9 + 1), 342 343 LRI(51, POSTED), 344 REG16(0x588), 345 REG16(0x588), 346 REG16(0x588), 347 REG16(0x588), 348 REG16(0x588), 349 REG16(0x588), 350 REG(0x028), 351 REG(0x09c), 352 REG(0x0c0), 353 REG(0x178), 354 REG(0x17c), 355 REG16(0x358), 356 REG(0x170), 357 REG(0x150), 358 REG(0x154), 359 REG(0x158), 360 REG16(0x41c), 361 REG16(0x600), 362 REG16(0x604), 363 REG16(0x608), 364 REG16(0x60c), 365 REG16(0x610), 366 REG16(0x614), 367 REG16(0x618), 368 REG16(0x61c), 369 REG16(0x620), 370 REG16(0x624), 371 REG16(0x628), 372 REG16(0x62c), 373 REG16(0x630), 374 REG16(0x634), 375 REG16(0x638), 376 REG16(0x63c), 377 REG16(0x640), 378 REG16(0x644), 379 REG16(0x648), 380 REG16(0x64c), 381 REG16(0x650), 382 REG16(0x654), 383 REG16(0x658), 384 REG16(0x65c), 385 REG16(0x660), 386 REG16(0x664), 387 REG16(0x668), 388 REG16(0x66c), 389 REG16(0x670), 390 REG16(0x674), 391 REG16(0x678), 392 REG16(0x67c), 393 REG(0x068), 394 REG(0x084), 395 NOP(1), 396 397 0 398 }; 399 400 static const u8 xehp_rcs_offsets[] = { 401 NOP(1), 402 LRI(13, POSTED), 403 REG16(0x244), 404 REG(0x034), 405 REG(0x030), 406 REG(0x038), 407 REG(0x03c), 408 REG(0x168), 409 REG(0x140), 410 REG(0x110), 411 REG(0x1c0), 412 REG(0x1c4), 413 REG(0x1c8), 414 REG(0x180), 415 REG16(0x2b4), 416 417 NOP(5), 418 LRI(9, POSTED), 419 REG16(0x3a8), 420 REG16(0x28c), 421 REG16(0x288), 422 REG16(0x284), 423 REG16(0x280), 424 REG16(0x27c), 425 REG16(0x278), 426 REG16(0x274), 427 REG16(0x270), 428 429 LRI(3, POSTED), 430 REG(0x1b0), 431 REG16(0x5a8), 432 REG16(0x5ac), 433 434 NOP(6), 435 LRI(1, 0), 436 REG(0x0c8), 437 438 0 439 }; 440 441 static const u8 dg2_rcs_offsets[] = { 442 NOP(1), 443 LRI(15, POSTED), 444 REG16(0x244), 445 REG(0x034), 446 REG(0x030), 447 REG(0x038), 448 REG(0x03c), 449 REG(0x168), 450 REG(0x140), 451 REG(0x110), 452 REG(0x1c0), 453 REG(0x1c4), 454 REG(0x1c8), 455 REG(0x180), 456 REG16(0x2b4), 457 REG(0x120), 458 REG(0x124), 459 460 NOP(1), 461 LRI(9, POSTED), 462 REG16(0x3a8), 463 REG16(0x28c), 464 REG16(0x288), 465 REG16(0x284), 466 REG16(0x280), 467 REG16(0x27c), 468 REG16(0x278), 469 REG16(0x274), 470 REG16(0x270), 471 472 LRI(3, POSTED), 473 REG(0x1b0), 474 REG16(0x5a8), 475 REG16(0x5ac), 476 477 NOP(6), 478 LRI(1, 0), 479 REG(0x0c8), 480 481 0 482 }; 483 484 static const u8 mtl_rcs_offsets[] = { 485 NOP(1), 486 LRI(15, POSTED), 487 REG16(0x244), 488 REG(0x034), 489 REG(0x030), 490 REG(0x038), 491 REG(0x03c), 492 REG(0x168), 493 REG(0x140), 494 REG(0x110), 495 REG(0x1c0), 496 REG(0x1c4), 497 REG(0x1c8), 498 REG(0x180), 499 REG16(0x2b4), 500 REG(0x120), 501 REG(0x124), 502 503 NOP(1), 504 LRI(9, POSTED), 505 REG16(0x3a8), 506 REG16(0x28c), 507 REG16(0x288), 508 REG16(0x284), 509 REG16(0x280), 510 REG16(0x27c), 511 REG16(0x278), 512 REG16(0x274), 513 REG16(0x270), 514 515 NOP(2), 516 LRI(2, POSTED), 517 REG16(0x5a8), 518 REG16(0x5ac), 519 520 NOP(6), 521 LRI(1, 0), 522 REG(0x0c8), 523 524 0 525 }; 526 527 #define XE2_CTX_COMMON \ 528 NOP(1), /* [0x00] */ \ 529 LRI(15, POSTED), /* [0x01] */ \ 530 REG16(0x244), /* [0x02] CTXT_SR_CTL */ \ 531 REG(0x034), /* [0x04] RING_BUFFER_HEAD */ \ 532 REG(0x030), /* [0x06] RING_BUFFER_TAIL */ \ 533 REG(0x038), /* [0x08] RING_BUFFER_START */ \ 534 REG(0x03c), /* [0x0a] RING_BUFFER_CONTROL */ \ 535 REG(0x168), /* [0x0c] BB_ADDR_UDW */ \ 536 REG(0x140), /* [0x0e] BB_ADDR */ \ 537 REG(0x110), /* [0x10] BB_STATE */ \ 538 REG(0x1c0), /* [0x12] BB_PER_CTX_PTR */ \ 539 REG(0x1c4), /* [0x14] RCS_INDIRECT_CTX */ \ 540 REG(0x1c8), /* [0x16] RCS_INDIRECT_CTX_OFFSET */ \ 541 REG(0x180), /* [0x18] CCID */ \ 542 REG16(0x2b4), /* [0x1a] SEMAPHORE_TOKEN */ \ 543 REG(0x120), /* [0x1c] PRT_BB_STATE */ \ 544 REG(0x124), /* [0x1e] PRT_BB_STATE_UDW */ \ 545 \ 546 NOP(1), /* [0x20] */ \ 547 LRI(9, POSTED), /* [0x21] */ \ 548 REG16(0x3a8), /* [0x22] CTX_TIMESTAMP */ \ 549 REG16(0x3ac), /* [0x24] CTX_TIMESTAMP_UDW */ \ 550 REG(0x108), /* [0x26] INDIRECT_RING_STATE */ \ 551 REG16(0x284), /* [0x28] dummy reg */ \ 552 REG16(0x280), /* [0x2a] CS_ACC_CTR_THOLD */ \ 553 REG16(0x27c), /* [0x2c] CS_CTX_SYS_PASID */ \ 554 REG16(0x278), /* [0x2e] CS_CTX_ASID */ \ 555 REG16(0x274), /* [0x30] PTBP_UDW */ \ 556 REG16(0x270) /* [0x32] PTBP_LDW */ 557 558 static const u8 xe2_rcs_offsets[] = { 559 XE2_CTX_COMMON, 560 561 NOP(2), /* [0x34] */ 562 LRI(2, POSTED), /* [0x36] */ 563 REG16(0x5a8), /* [0x37] CONTEXT_SCHEDULING_ATTRIBUTES */ 564 REG16(0x5ac), /* [0x39] PREEMPTION_STATUS */ 565 566 NOP(6), /* [0x41] */ 567 LRI(1, 0), /* [0x47] */ 568 REG(0x0c8), /* [0x48] R_PWR_CLK_STATE */ 569 570 0 571 }; 572 573 static const u8 xe2_bcs_offsets[] = { 574 XE2_CTX_COMMON, 575 576 NOP(4 + 8 + 1), /* [0x34] */ 577 LRI(2, POSTED), /* [0x41] */ 578 REG16(0x200), /* [0x42] BCS_SWCTRL */ 579 REG16(0x204), /* [0x44] BLIT_CCTL */ 580 581 0 582 }; 583 584 static const u8 xe2_xcs_offsets[] = { 585 XE2_CTX_COMMON, 586 587 0 588 }; 589 590 static const u8 xe2_indirect_ring_state_offsets[] = { 591 NOP(1), /* [0x00] */ 592 LRI(5, POSTED), /* [0x01] */ 593 REG(0x034), /* [0x02] RING_BUFFER_HEAD */ 594 REG(0x030), /* [0x04] RING_BUFFER_TAIL */ 595 REG(0x038), /* [0x06] RING_BUFFER_START */ 596 REG(0x048), /* [0x08] RING_BUFFER_START_UDW */ 597 REG(0x03c), /* [0x0a] RING_BUFFER_CONTROL */ 598 599 NOP(5), /* [0x0c] */ 600 LRI(9, POSTED), /* [0x11] */ 601 REG(0x168), /* [0x12] BB_ADDR_UDW */ 602 REG(0x140), /* [0x14] BB_ADDR */ 603 REG(0x110), /* [0x16] BB_STATE */ 604 REG16(0x588), /* [0x18] BB_STACK_WRITE_PORT */ 605 REG16(0x588), /* [0x20] BB_STACK_WRITE_PORT */ 606 REG16(0x588), /* [0x22] BB_STACK_WRITE_PORT */ 607 REG16(0x588), /* [0x24] BB_STACK_WRITE_PORT */ 608 REG16(0x588), /* [0x26] BB_STACK_WRITE_PORT */ 609 REG16(0x588), /* [0x28] BB_STACK_WRITE_PORT */ 610 611 NOP(12), /* [0x00] */ 612 613 0 614 }; 615 616 #undef REG16 617 #undef REG 618 #undef LRI 619 #undef NOP 620 621 static const u8 *reg_offsets(struct xe_device *xe, enum xe_engine_class class) 622 { 623 if (class == XE_ENGINE_CLASS_RENDER) { 624 if (GRAPHICS_VER(xe) >= 20) 625 return xe2_rcs_offsets; 626 else if (GRAPHICS_VERx100(xe) >= 1270) 627 return mtl_rcs_offsets; 628 else if (GRAPHICS_VERx100(xe) >= 1255) 629 return dg2_rcs_offsets; 630 else if (GRAPHICS_VERx100(xe) >= 1250) 631 return xehp_rcs_offsets; 632 else 633 return gen12_rcs_offsets; 634 } else if (class == XE_ENGINE_CLASS_COPY) { 635 if (GRAPHICS_VER(xe) >= 20) 636 return xe2_bcs_offsets; 637 else 638 return gen12_xcs_offsets; 639 } else { 640 if (GRAPHICS_VER(xe) >= 20) 641 return xe2_xcs_offsets; 642 else if (GRAPHICS_VERx100(xe) >= 1255) 643 return dg2_xcs_offsets; 644 else 645 return gen12_xcs_offsets; 646 } 647 } 648 649 static void set_context_control(u32 *regs, struct xe_hw_engine *hwe) 650 { 651 regs[CTX_CONTEXT_CONTROL] = REG_MASKED_FIELD_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH | 652 CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT); 653 654 if (xe_gt_has_indirect_ring_state(hwe->gt)) 655 regs[CTX_CONTEXT_CONTROL] |= 656 REG_MASKED_FIELD_ENABLE(CTX_CTRL_INDIRECT_RING_STATE_ENABLE); 657 } 658 659 static void set_memory_based_intr(u32 *regs, struct xe_hw_engine *hwe) 660 { 661 struct xe_memirq *memirq = >_to_tile(hwe->gt)->memirq; 662 struct xe_device *xe = gt_to_xe(hwe->gt); 663 u8 num_regs; 664 665 if (!xe_device_uses_memirq(xe)) 666 return; 667 668 regs[CTX_LRM_INT_MASK_ENABLE] = MI_LOAD_REGISTER_MEM | 669 MI_LRI_LRM_CS_MMIO | MI_LRM_USE_GGTT; 670 regs[CTX_INT_MASK_ENABLE_REG] = RING_IMR(0).addr; 671 regs[CTX_INT_MASK_ENABLE_PTR] = xe_memirq_enable_ptr(memirq); 672 673 num_regs = xe_device_has_msix(xe) ? 3 : 2; 674 regs[CTX_LRI_INT_REPORT_PTR] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(num_regs) | 675 MI_LRI_LRM_CS_MMIO | MI_LRI_FORCE_POSTED; 676 regs[CTX_INT_STATUS_REPORT_REG] = RING_INT_STATUS_RPT_PTR(0).addr; 677 regs[CTX_INT_STATUS_REPORT_PTR] = xe_memirq_status_ptr(memirq, hwe); 678 regs[CTX_INT_SRC_REPORT_REG] = RING_INT_SRC_RPT_PTR(0).addr; 679 regs[CTX_INT_SRC_REPORT_PTR] = xe_memirq_source_ptr(memirq, hwe); 680 681 if (xe_device_has_msix(xe)) { 682 regs[CTX_CS_INT_VEC_REG] = CS_INT_VEC(0).addr; 683 /* CTX_CS_INT_VEC_DATA will be set in xe_lrc_init */ 684 } 685 } 686 687 static inline bool xe_lrc_has_indirect_ring_state(struct xe_lrc *lrc) 688 { 689 return lrc->flags & XE_LRC_FLAG_INDIRECT_RING_STATE; 690 } 691 692 static inline u32 __xe_lrc_ring_offset(struct xe_lrc *lrc) 693 { 694 return 0; 695 } 696 697 u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc) 698 { 699 return lrc->ring.size; 700 } 701 702 /* Make the magic macros work */ 703 #define __xe_lrc_pphwsp_offset xe_lrc_pphwsp_offset 704 #define __xe_lrc_regs_offset xe_lrc_regs_offset 705 706 #define LRC_CTX_JOB_TIMESTAMP_OFFSET 512 707 #define LRC_ENGINE_ID_PPHWSP_OFFSET 1024 708 #define LRC_PARALLEL_PPHWSP_OFFSET 2048 709 710 #define LRC_SEQNO_OFFSET 0 711 #define LRC_START_SEQNO_OFFSET (LRC_SEQNO_OFFSET + 8) 712 713 u32 xe_lrc_regs_offset(struct xe_lrc *lrc) 714 { 715 return xe_lrc_pphwsp_offset(lrc) + LRC_PPHWSP_SIZE; 716 } 717 718 /** 719 * xe_lrc_reg_size() - Get size of the LRC registers area within queues 720 * @xe: the &xe_device struct instance 721 * 722 * Returns: Size of the LRC registers area for current platform 723 */ 724 size_t xe_lrc_reg_size(struct xe_device *xe) 725 { 726 if (GRAPHICS_VERx100(xe) >= 1250) 727 return 96 * sizeof(u32); 728 else 729 return 80 * sizeof(u32); 730 } 731 732 /** 733 * xe_lrc_engine_state_size() - Get size of the engine state within LRC 734 * @gt: the &xe_gt struct instance 735 * @class: Hardware engine class 736 * 737 * Returns: Size of the engine state 738 */ 739 size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class) 740 { 741 return xe_gt_lrc_hang_replay_size(gt, class) - xe_lrc_reg_size(gt_to_xe(gt)); 742 } 743 744 static inline u32 __xe_lrc_seqno_offset(struct xe_lrc *lrc) 745 { 746 return LRC_SEQNO_OFFSET; 747 } 748 749 static inline u32 __xe_lrc_start_seqno_offset(struct xe_lrc *lrc) 750 { 751 return LRC_START_SEQNO_OFFSET; 752 } 753 754 static u32 __xe_lrc_ctx_job_timestamp_offset(struct xe_lrc *lrc) 755 { 756 /* This is stored in the driver-defined portion of PPHWSP */ 757 return xe_lrc_pphwsp_offset(lrc) + LRC_CTX_JOB_TIMESTAMP_OFFSET; 758 } 759 760 static inline u32 __xe_lrc_parallel_offset(struct xe_lrc *lrc) 761 { 762 /* The parallel is stored in the driver-defined portion of PPHWSP */ 763 return xe_lrc_pphwsp_offset(lrc) + LRC_PARALLEL_PPHWSP_OFFSET; 764 } 765 766 static inline u32 __xe_lrc_engine_id_offset(struct xe_lrc *lrc) 767 { 768 return xe_lrc_pphwsp_offset(lrc) + LRC_ENGINE_ID_PPHWSP_OFFSET; 769 } 770 771 static u32 __xe_lrc_ctx_timestamp_offset(struct xe_lrc *lrc) 772 { 773 return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP * sizeof(u32); 774 } 775 776 static u32 __xe_lrc_ctx_timestamp_udw_offset(struct xe_lrc *lrc) 777 { 778 return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP_UDW * sizeof(u32); 779 } 780 781 static u32 __xe_lrc_queue_timestamp_offset(struct xe_lrc *lrc) 782 { 783 return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP * sizeof(u32); 784 } 785 786 static u32 __xe_lrc_queue_timestamp_udw_offset(struct xe_lrc *lrc) 787 { 788 return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP_UDW * sizeof(u32); 789 } 790 791 static inline u32 __xe_lrc_indirect_ring_offset(struct xe_lrc *lrc) 792 { 793 u32 offset = xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE - 794 LRC_INDIRECT_RING_STATE_SIZE; 795 796 if (lrc->flags & XE_LRC_FLAG_INDIRECT_CTX) 797 offset -= LRC_INDIRECT_CTX_BO_SIZE; 798 799 return offset; 800 } 801 802 static inline u32 __xe_lrc_indirect_ctx_offset(struct xe_lrc *lrc) 803 { 804 return xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE - LRC_INDIRECT_CTX_BO_SIZE; 805 } 806 807 static inline u32 __xe_lrc_wa_bb_offset(struct xe_lrc *lrc) 808 { 809 return xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE; 810 } 811 812 #define DECL_MAP_ADDR_HELPERS(elem, bo_expr) \ 813 static inline struct iosys_map __xe_lrc_##elem##_map(struct xe_lrc *lrc) \ 814 { \ 815 struct xe_bo *bo = (bo_expr); \ 816 struct iosys_map map = bo->vmap; \ 817 \ 818 xe_assert(lrc_to_xe(lrc), !iosys_map_is_null(&map)); \ 819 iosys_map_incr(&map, __xe_lrc_##elem##_offset(lrc)); \ 820 return map; \ 821 } \ 822 static inline u32 __maybe_unused __xe_lrc_##elem##_ggtt_addr(struct xe_lrc *lrc) \ 823 { \ 824 struct xe_bo *bo = (bo_expr); \ 825 \ 826 return xe_bo_ggtt_addr(bo) + __xe_lrc_##elem##_offset(lrc); \ 827 } \ 828 829 DECL_MAP_ADDR_HELPERS(ring, lrc->bo) 830 DECL_MAP_ADDR_HELPERS(pphwsp, lrc->bo) 831 DECL_MAP_ADDR_HELPERS(seqno, lrc->seqno_bo) 832 DECL_MAP_ADDR_HELPERS(regs, lrc->bo) 833 DECL_MAP_ADDR_HELPERS(start_seqno, lrc->seqno_bo) 834 DECL_MAP_ADDR_HELPERS(ctx_job_timestamp, lrc->bo) 835 DECL_MAP_ADDR_HELPERS(ctx_timestamp, lrc->bo) 836 DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo) 837 DECL_MAP_ADDR_HELPERS(parallel, lrc->bo) 838 DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo) 839 DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo) 840 DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo) 841 DECL_MAP_ADDR_HELPERS(queue_timestamp_udw, lrc->bo) 842 843 #undef DECL_MAP_ADDR_HELPERS 844 845 /** 846 * xe_lrc_ctx_timestamp_ggtt_addr() - Get ctx timestamp GGTT address 847 * @lrc: Pointer to the lrc. 848 * 849 * Returns: ctx timestamp GGTT address 850 */ 851 u32 xe_lrc_ctx_timestamp_ggtt_addr(struct xe_lrc *lrc) 852 { 853 return __xe_lrc_ctx_timestamp_ggtt_addr(lrc); 854 } 855 856 /** 857 * xe_lrc_ctx_timestamp_udw_ggtt_addr() - Get ctx timestamp udw GGTT address 858 * @lrc: Pointer to the lrc. 859 * 860 * Returns: ctx timestamp udw GGTT address 861 */ 862 u32 xe_lrc_ctx_timestamp_udw_ggtt_addr(struct xe_lrc *lrc) 863 { 864 return __xe_lrc_ctx_timestamp_udw_ggtt_addr(lrc); 865 } 866 867 /** 868 * xe_lrc_ctx_timestamp() - Read ctx timestamp value 869 * @lrc: Pointer to the lrc. 870 * 871 * Returns: ctx timestamp value 872 */ 873 static u64 xe_lrc_ctx_timestamp(struct xe_lrc *lrc) 874 { 875 struct xe_device *xe = lrc_to_xe(lrc); 876 struct iosys_map map; 877 u32 ldw, udw = 0; 878 879 map = __xe_lrc_ctx_timestamp_map(lrc); 880 ldw = xe_map_read32(xe, &map); 881 882 if (xe->info.has_64bit_timestamp) { 883 map = __xe_lrc_ctx_timestamp_udw_map(lrc); 884 udw = xe_map_read32(xe, &map); 885 } 886 887 return (u64)udw << 32 | ldw; 888 } 889 890 /** 891 * xe_lrc_queue_timestamp() - Read queue timestamp value 892 * @lrc: Pointer to the lrc. 893 * 894 * Returns: queue timestamp value 895 */ 896 static u64 xe_lrc_queue_timestamp(struct xe_lrc *lrc) 897 { 898 struct xe_device *xe = lrc_to_xe(lrc); 899 struct iosys_map map; 900 u32 ldw, udw = 0; 901 902 xe_assert(xe, xe_lrc_is_multi_queue(lrc)); 903 904 map = __xe_lrc_queue_timestamp_map(lrc); 905 ldw = xe_map_read32(xe, &map); 906 907 map = __xe_lrc_queue_timestamp_udw_map(lrc); 908 udw = xe_map_read32(xe, &map); 909 910 return (u64)udw << 32 | ldw; 911 } 912 913 /** 914 * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address 915 * @lrc: Pointer to the lrc. 916 * 917 * Returns: ctx timestamp job GGTT address 918 */ 919 u32 xe_lrc_ctx_job_timestamp_ggtt_addr(struct xe_lrc *lrc) 920 { 921 return __xe_lrc_ctx_job_timestamp_ggtt_addr(lrc); 922 } 923 924 /** 925 * xe_lrc_ctx_job_timestamp() - Read ctx job timestamp value 926 * @lrc: Pointer to the lrc. 927 * 928 * Returns: ctx timestamp job value 929 */ 930 u32 xe_lrc_ctx_job_timestamp(struct xe_lrc *lrc) 931 { 932 struct xe_device *xe = lrc_to_xe(lrc); 933 struct iosys_map map; 934 935 map = __xe_lrc_ctx_job_timestamp_map(lrc); 936 return xe_map_read32(xe, &map); 937 } 938 939 u32 xe_lrc_ggtt_addr(struct xe_lrc *lrc) 940 { 941 return __xe_lrc_pphwsp_ggtt_addr(lrc); 942 } 943 944 u32 xe_lrc_indirect_ring_ggtt_addr(struct xe_lrc *lrc) 945 { 946 if (!xe_lrc_has_indirect_ring_state(lrc)) 947 return 0; 948 949 return __xe_lrc_indirect_ring_ggtt_addr(lrc); 950 } 951 952 static u32 xe_lrc_read_indirect_ctx_reg(struct xe_lrc *lrc, int reg_nr) 953 { 954 struct xe_device *xe = lrc_to_xe(lrc); 955 struct iosys_map map; 956 957 map = __xe_lrc_indirect_ring_map(lrc); 958 iosys_map_incr(&map, reg_nr * sizeof(u32)); 959 return xe_map_read32(xe, &map); 960 } 961 962 static void xe_lrc_write_indirect_ctx_reg(struct xe_lrc *lrc, 963 int reg_nr, u32 val) 964 { 965 struct xe_device *xe = lrc_to_xe(lrc); 966 struct iosys_map map; 967 968 map = __xe_lrc_indirect_ring_map(lrc); 969 iosys_map_incr(&map, reg_nr * sizeof(u32)); 970 xe_map_write32(xe, &map, val); 971 } 972 973 u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr) 974 { 975 struct xe_device *xe = lrc_to_xe(lrc); 976 struct iosys_map map; 977 978 map = __xe_lrc_regs_map(lrc); 979 iosys_map_incr(&map, reg_nr * sizeof(u32)); 980 return xe_map_read32(xe, &map); 981 } 982 983 void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val) 984 { 985 struct xe_device *xe = lrc_to_xe(lrc); 986 struct iosys_map map; 987 988 map = __xe_lrc_regs_map(lrc); 989 iosys_map_incr(&map, reg_nr * sizeof(u32)); 990 xe_map_write32(xe, &map, val); 991 } 992 993 static void *empty_lrc_data(struct xe_hw_engine *hwe) 994 { 995 struct xe_gt *gt = hwe->gt; 996 void *data; 997 u32 *regs; 998 999 data = kzalloc(xe_gt_lrc_size(gt, hwe->class), GFP_KERNEL); 1000 if (!data) 1001 return NULL; 1002 1003 /* 1st page: Per-Process of HW status Page */ 1004 regs = data + LRC_PPHWSP_SIZE; 1005 set_offsets(regs, reg_offsets(gt_to_xe(gt), hwe->class), hwe); 1006 set_context_control(regs, hwe); 1007 set_memory_based_intr(regs, hwe); 1008 if (xe_gt_has_indirect_ring_state(gt)) { 1009 regs = data + xe_gt_lrc_size(gt, hwe->class) - 1010 LRC_INDIRECT_RING_STATE_SIZE; 1011 set_offsets(regs, xe2_indirect_ring_state_offsets, hwe); 1012 } 1013 1014 return data; 1015 } 1016 1017 /** 1018 * xe_default_lrc_update_memirq_regs_with_address - Re-compute GGTT references in default LRC 1019 * of given engine. 1020 * @hwe: the &xe_hw_engine struct instance 1021 */ 1022 void xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine *hwe) 1023 { 1024 struct xe_gt *gt = hwe->gt; 1025 u32 *regs; 1026 1027 if (!gt->default_lrc[hwe->class]) 1028 return; 1029 1030 regs = gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE; 1031 set_memory_based_intr(regs, hwe); 1032 } 1033 1034 /** 1035 * xe_lrc_update_memirq_regs_with_address - Re-compute GGTT references in mem interrupt data 1036 * for given LRC. 1037 * @lrc: the &xe_lrc struct instance 1038 * @hwe: the &xe_hw_engine struct instance 1039 * @regs: scratch buffer to be used as temporary storage 1040 */ 1041 void xe_lrc_update_memirq_regs_with_address(struct xe_lrc *lrc, struct xe_hw_engine *hwe, 1042 u32 *regs) 1043 { 1044 struct xe_gt *gt = hwe->gt; 1045 struct iosys_map map; 1046 size_t regs_len; 1047 1048 if (!xe_device_uses_memirq(gt_to_xe(gt))) 1049 return; 1050 1051 map = __xe_lrc_regs_map(lrc); 1052 regs_len = xe_lrc_reg_size(gt_to_xe(gt)); 1053 xe_map_memcpy_from(gt_to_xe(gt), regs, &map, 0, regs_len); 1054 set_memory_based_intr(regs, hwe); 1055 xe_map_memcpy_to(gt_to_xe(gt), &map, 0, regs, regs_len); 1056 } 1057 1058 static void xe_lrc_set_ppgtt(struct xe_lrc *lrc, struct xe_vm *vm) 1059 { 1060 u64 desc = xe_vm_pdp4_descriptor(vm, gt_to_tile(lrc->gt)); 1061 1062 xe_lrc_write_ctx_reg(lrc, CTX_PDP0_UDW, upper_32_bits(desc)); 1063 xe_lrc_write_ctx_reg(lrc, CTX_PDP0_LDW, lower_32_bits(desc)); 1064 } 1065 1066 static void xe_lrc_finish(struct xe_lrc *lrc) 1067 { 1068 xe_hw_fence_ctx_finish(&lrc->fence_ctx); 1069 xe_bo_unpin_map_no_vm(lrc->bo); 1070 xe_bo_unpin_map_no_vm(lrc->seqno_bo); 1071 } 1072 1073 /* 1074 * wa_bb_setup_utilization() - Write commands to wa bb to assist 1075 * in calculating active context run ticks. 1076 * 1077 * Context Timestamp (CTX_TIMESTAMP) in the LRC accumulates the run ticks of the 1078 * context, but only gets updated when the context switches out. In order to 1079 * check how long a context has been active before it switches out, two things 1080 * are required: 1081 * 1082 * (1) Determine if the context is running: 1083 * To do so, we program the WA BB to set an initial value for CTX_TIMESTAMP in 1084 * the LRC. The value chosen is 1 since 0 is the initial value when the LRC is 1085 * initialized. During a query, we just check for this value to determine if the 1086 * context is active. If the context switched out, it would overwrite this 1087 * location with the actual CTX_TIMESTAMP MMIO value. Note that WA BB runs as 1088 * the last part of context restore, so reusing this LRC location will not 1089 * clobber anything. 1090 * 1091 * (2) Calculate the time that the context has been active for: 1092 * The CTX_TIMESTAMP ticks only when the context is active. If a context is 1093 * active, we just use the CTX_TIMESTAMP MMIO as the new value of utilization. 1094 * While doing so, we need to read the CTX_TIMESTAMP MMIO for the specific 1095 * engine instance. Since we do not know which instance the context is running 1096 * on until it is scheduled, we also read the ENGINE_ID MMIO in the WA BB and 1097 * store it in the PPHSWP. 1098 */ 1099 #define CONTEXT_ACTIVE XE_LRC_CTX_TIMESTAMP_ACTIVE 1100 static ssize_t setup_utilization_wa(struct xe_lrc *lrc, 1101 struct xe_hw_engine *hwe, 1102 u32 *batch, size_t max_len, bool indirect) 1103 { 1104 u32 *cmd = batch; 1105 1106 if (IS_SRIOV_VF(gt_to_xe(lrc->gt))) 1107 return 0; 1108 1109 if (xe_gt_WARN_ON(lrc->gt, max_len < 12)) 1110 return -ENOSPC; 1111 1112 *cmd++ = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET; 1113 *cmd++ = ENGINE_ID(0).addr; 1114 *cmd++ = __xe_lrc_engine_id_ggtt_addr(lrc); 1115 *cmd++ = 0; 1116 1117 *cmd++ = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); 1118 *cmd++ = __xe_lrc_ctx_timestamp_ggtt_addr(lrc); 1119 *cmd++ = 0; 1120 *cmd++ = lower_32_bits(CONTEXT_ACTIVE); 1121 1122 if (lrc_to_xe(lrc)->info.has_64bit_timestamp) { 1123 *cmd++ = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1); 1124 *cmd++ = __xe_lrc_ctx_timestamp_udw_ggtt_addr(lrc); 1125 *cmd++ = 0; 1126 *cmd++ = upper_32_bits(CONTEXT_ACTIVE); 1127 } 1128 1129 return cmd - batch; 1130 } 1131 1132 static ssize_t setup_timestamp_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe, 1133 u32 *batch, size_t max_len, bool indirect) 1134 { 1135 const u32 ts_addr = __xe_lrc_ctx_timestamp_ggtt_addr(lrc); 1136 u32 *cmd = batch; 1137 1138 if (!XE_GT_WA(lrc->gt, 16010904313)) 1139 return 0; 1140 1141 if (!indirect && 1142 !(hwe->class == XE_ENGINE_CLASS_COPY || 1143 hwe->class == XE_ENGINE_CLASS_VIDEO_DECODE || 1144 hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE)) 1145 return 0; 1146 1147 if (indirect && 1148 !(hwe->class == XE_ENGINE_CLASS_RENDER || 1149 hwe->class == XE_ENGINE_CLASS_COMPUTE)) 1150 return 0; 1151 1152 if (xe_gt_WARN_ON(lrc->gt, max_len < 12)) 1153 return -ENOSPC; 1154 1155 *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO | 1156 MI_LRM_ASYNC; 1157 *cmd++ = RING_CTX_TIMESTAMP(0).addr; 1158 *cmd++ = ts_addr; 1159 *cmd++ = 0; 1160 1161 *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO | 1162 MI_LRM_ASYNC; 1163 *cmd++ = RING_CTX_TIMESTAMP(0).addr; 1164 *cmd++ = ts_addr; 1165 *cmd++ = 0; 1166 1167 *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO; 1168 *cmd++ = RING_CTX_TIMESTAMP(0).addr; 1169 *cmd++ = ts_addr; 1170 *cmd++ = 0; 1171 1172 return cmd - batch; 1173 } 1174 1175 static ssize_t setup_configfs_post_ctx_restore_bb(struct xe_lrc *lrc, 1176 struct xe_hw_engine *hwe, 1177 u32 *batch, size_t max_len, 1178 bool indirect) 1179 { 1180 struct xe_device *xe = gt_to_xe(lrc->gt); 1181 const u32 *user_batch; 1182 u32 *cmd = batch; 1183 u32 count; 1184 1185 count = xe_configfs_get_ctx_restore_post_bb(to_pci_dev(xe->drm.dev), 1186 hwe->class, &user_batch); 1187 if (!count) 1188 return 0; 1189 1190 if (count > max_len) 1191 return -ENOSPC; 1192 1193 /* 1194 * This should be used only for tests and validation. Taint the kernel 1195 * as anything could be submitted directly in context switches 1196 */ 1197 add_taint(TAINT_TEST, LOCKDEP_STILL_OK); 1198 1199 memcpy(cmd, user_batch, count * sizeof(u32)); 1200 cmd += count; 1201 1202 return cmd - batch; 1203 } 1204 1205 static ssize_t setup_configfs_mid_ctx_restore_bb(struct xe_lrc *lrc, 1206 struct xe_hw_engine *hwe, 1207 u32 *batch, size_t max_len, 1208 bool indirect) 1209 { 1210 struct xe_device *xe = gt_to_xe(lrc->gt); 1211 const u32 *user_batch; 1212 u32 *cmd = batch; 1213 u32 count; 1214 1215 count = xe_configfs_get_ctx_restore_mid_bb(to_pci_dev(xe->drm.dev), 1216 hwe->class, &user_batch); 1217 if (!count) 1218 return 0; 1219 1220 if (count > max_len) 1221 return -ENOSPC; 1222 1223 /* 1224 * This should be used only for tests and validation. Taint the kernel 1225 * as anything could be submitted directly in context switches 1226 */ 1227 add_taint(TAINT_TEST, LOCKDEP_STILL_OK); 1228 1229 memcpy(cmd, user_batch, count * sizeof(u32)); 1230 cmd += count; 1231 1232 return cmd - batch; 1233 } 1234 1235 static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc, 1236 struct xe_hw_engine *hwe, 1237 u32 *batch, size_t max_len, 1238 bool indirect) 1239 { 1240 u32 *cmd = batch; 1241 1242 if (!XE_GT_WA(lrc->gt, 18022495364) || 1243 hwe->class != XE_ENGINE_CLASS_RENDER) 1244 return 0; 1245 1246 if (xe_gt_WARN_ON(lrc->gt, max_len < 3)) 1247 return -ENOSPC; 1248 1249 *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_LRM_CS_MMIO | MI_LRI_NUM_REGS(1); 1250 *cmd++ = CS_DEBUG_MODE2(0).addr; 1251 *cmd++ = REG_MASKED_FIELD_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE); 1252 1253 return cmd - batch; 1254 } 1255 1256 static ssize_t setup_invalidate_auxccs_wa(struct xe_lrc *lrc, 1257 struct xe_hw_engine *hwe, 1258 u32 *batch, size_t max_len, 1259 bool indirect) 1260 { 1261 struct xe_gt *gt = lrc->gt; 1262 u32 *(*emit)(struct xe_gt *gt, u32 *cmd) = 1263 gt->ring_ops[hwe->class]->emit_aux_table_inv; 1264 1265 if (!emit) 1266 return 0; 1267 1268 if (xe_gt_WARN_ON(gt, max_len < 8)) 1269 return -ENOSPC; 1270 1271 return emit(gt, batch) - batch; 1272 } 1273 1274 struct bo_setup { 1275 ssize_t (*setup)(struct xe_lrc *lrc, struct xe_hw_engine *hwe, 1276 u32 *batch, size_t max_size, bool indirect); 1277 }; 1278 1279 struct bo_setup_state { 1280 /* Input: */ 1281 struct xe_lrc *lrc; 1282 struct xe_hw_engine *hwe; 1283 size_t max_size; 1284 size_t reserve_dw; 1285 bool indirect; 1286 unsigned int offset; 1287 const struct bo_setup *funcs; 1288 unsigned int num_funcs; 1289 1290 /* State: */ 1291 u32 *buffer; 1292 u32 *ptr; 1293 unsigned int written; 1294 }; 1295 1296 static int setup_bo(struct bo_setup_state *state) 1297 { 1298 ssize_t remain; 1299 1300 if (state->lrc->bo->vmap.is_iomem) { 1301 xe_gt_assert(state->hwe->gt, state->buffer); 1302 state->ptr = state->buffer; 1303 } else { 1304 state->ptr = state->lrc->bo->vmap.vaddr + state->offset; 1305 } 1306 1307 remain = state->max_size / sizeof(u32); 1308 1309 for (size_t i = 0; i < state->num_funcs; i++) { 1310 ssize_t len = state->funcs[i].setup(state->lrc, state->hwe, 1311 state->ptr, remain, 1312 state->indirect); 1313 1314 remain -= len; 1315 1316 /* 1317 * Caller has asked for at least reserve_dw to remain unused. 1318 */ 1319 if (len < 0 || 1320 xe_gt_WARN_ON(state->lrc->gt, remain < state->reserve_dw)) 1321 goto fail; 1322 1323 state->ptr += len; 1324 state->written += len; 1325 } 1326 1327 return 0; 1328 1329 fail: 1330 return -ENOSPC; 1331 } 1332 1333 static void finish_bo(struct bo_setup_state *state) 1334 { 1335 if (!state->lrc->bo->vmap.is_iomem) 1336 return; 1337 1338 xe_map_memcpy_to(gt_to_xe(state->lrc->gt), &state->lrc->bo->vmap, 1339 state->offset, state->buffer, 1340 state->written * sizeof(u32)); 1341 } 1342 1343 /** 1344 * xe_lrc_setup_wa_bb_with_scratch - Execute all wa bb setup callbacks. 1345 * @lrc: the &xe_lrc struct instance 1346 * @hwe: the &xe_hw_engine struct instance 1347 * @scratch: preallocated scratch buffer for temporary storage 1348 * Return: 0 on success, negative error code on failure 1349 */ 1350 int xe_lrc_setup_wa_bb_with_scratch(struct xe_lrc *lrc, struct xe_hw_engine *hwe, u32 *scratch) 1351 { 1352 static const struct bo_setup funcs[] = { 1353 { .setup = setup_timestamp_wa }, 1354 { .setup = setup_invalidate_state_cache_wa }, 1355 { .setup = setup_utilization_wa }, 1356 { .setup = setup_configfs_post_ctx_restore_bb }, 1357 }; 1358 struct bo_setup_state state = { 1359 .lrc = lrc, 1360 .hwe = hwe, 1361 .max_size = LRC_WA_BB_SIZE, 1362 .buffer = scratch, 1363 .reserve_dw = 1, 1364 .offset = __xe_lrc_wa_bb_offset(lrc), 1365 .funcs = funcs, 1366 .num_funcs = ARRAY_SIZE(funcs), 1367 }; 1368 int ret; 1369 1370 ret = setup_bo(&state); 1371 if (ret) 1372 return ret; 1373 1374 *state.ptr++ = MI_BATCH_BUFFER_END; 1375 state.written++; 1376 1377 finish_bo(&state); 1378 1379 xe_lrc_write_ctx_reg(lrc, CTX_BB_PER_CTX_PTR, 1380 xe_bo_ggtt_addr(lrc->bo) + state.offset + 1); 1381 1382 return 0; 1383 } 1384 1385 static int setup_wa_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe) 1386 { 1387 u32 *buf = NULL; 1388 int ret; 1389 1390 if (lrc->bo->vmap.is_iomem) { 1391 buf = kmalloc(LRC_WA_BB_SIZE, GFP_KERNEL); 1392 if (!buf) 1393 return -ENOMEM; 1394 } 1395 1396 ret = xe_lrc_setup_wa_bb_with_scratch(lrc, hwe, buf); 1397 1398 kfree(buf); 1399 1400 return ret; 1401 } 1402 1403 static int 1404 setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe) 1405 { 1406 static const struct bo_setup rcs_funcs[] = { 1407 { .setup = setup_timestamp_wa }, 1408 { .setup = setup_invalidate_auxccs_wa }, 1409 { .setup = setup_configfs_mid_ctx_restore_bb }, 1410 }; 1411 static const struct bo_setup xcs_funcs[] = { 1412 { .setup = setup_invalidate_auxccs_wa }, 1413 { .setup = setup_configfs_mid_ctx_restore_bb }, 1414 }; 1415 struct bo_setup_state state = { 1416 .lrc = lrc, 1417 .hwe = hwe, 1418 .indirect = true, 1419 .max_size = (63 * 64) /* max 63 cachelines */, 1420 .buffer = NULL, 1421 .offset = __xe_lrc_indirect_ctx_offset(lrc), 1422 }; 1423 int ret; 1424 1425 if (!(lrc->flags & XE_LRC_FLAG_INDIRECT_CTX)) 1426 return 0; 1427 1428 if (hwe->class == XE_ENGINE_CLASS_RENDER || 1429 hwe->class == XE_ENGINE_CLASS_COMPUTE) { 1430 state.funcs = rcs_funcs; 1431 state.num_funcs = ARRAY_SIZE(rcs_funcs); 1432 } else { 1433 state.funcs = xcs_funcs; 1434 state.num_funcs = ARRAY_SIZE(xcs_funcs); 1435 } 1436 1437 if (xe_gt_WARN_ON(lrc->gt, !state.funcs)) 1438 return 0; 1439 1440 if (lrc->bo->vmap.is_iomem) { 1441 state.buffer = kmalloc(state.max_size, GFP_KERNEL); 1442 if (!state.buffer) 1443 return -ENOMEM; 1444 } 1445 1446 ret = setup_bo(&state); 1447 if (ret) { 1448 kfree(state.buffer); 1449 return ret; 1450 } 1451 1452 /* 1453 * Align to 64B cacheline so there's no garbage at the end for CS to 1454 * execute: size for indirect ctx must be a multiple of 64. 1455 */ 1456 while (state.written & 0xf) { 1457 *state.ptr++ = MI_NOOP; 1458 state.written++; 1459 } 1460 1461 finish_bo(&state); 1462 kfree(state.buffer); 1463 1464 /* 1465 * Enable INDIRECT_CTX leaving INDIRECT_CTX_OFFSET at its default: it 1466 * varies per engine class, but the default is good enough 1467 */ 1468 xe_lrc_write_ctx_reg(lrc, 1469 CTX_CS_INDIRECT_CTX, 1470 (xe_bo_ggtt_addr(lrc->bo) + state.offset) | 1471 /* Size in CLs. */ 1472 (state.written * sizeof(u32) / 64)); 1473 1474 return 0; 1475 } 1476 1477 static u8 xe_multi_queue_prio_to_lrc(struct xe_lrc *lrc, enum xe_multi_queue_priority priority) 1478 { 1479 struct xe_device *xe = gt_to_xe(lrc->gt); 1480 1481 xe_assert(xe, (priority >= XE_MULTI_QUEUE_PRIORITY_LOW && 1482 priority <= XE_MULTI_QUEUE_PRIORITY_HIGH)); 1483 1484 /* xe_multi_queue_priority is directly mapped to LRC priority values */ 1485 return priority; 1486 } 1487 1488 /** 1489 * xe_lrc_set_multi_queue_priority() - Set multi queue priority in LRC 1490 * @lrc: Logical Ring Context 1491 * @priority: Multi queue priority of the exec queue 1492 * 1493 * Convert @priority to LRC multi queue priority and update the @lrc descriptor 1494 */ 1495 void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority) 1496 { 1497 lrc->desc &= ~LRC_PRIORITY; 1498 lrc->desc |= FIELD_PREP(LRC_PRIORITY, xe_multi_queue_prio_to_lrc(lrc, priority)); 1499 } 1500 1501 static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm, 1502 void *replay_state, u16 msix_vec, u32 init_flags) 1503 { 1504 struct xe_gt *gt = hwe->gt; 1505 struct xe_tile *tile = gt_to_tile(gt); 1506 struct xe_device *xe = gt_to_xe(gt); 1507 struct iosys_map map; 1508 u32 arb_enable; 1509 u32 state_cache_perf_fix[3]; 1510 int err; 1511 1512 /* 1513 * Init Per-Process of HW status Page, LRC / context state to known 1514 * values. If there's already a primed default_lrc, just copy it, otherwise 1515 * it's the early submission to record the lrc: build a new empty one from 1516 * scratch. 1517 */ 1518 map = __xe_lrc_pphwsp_map(lrc); 1519 if (gt->default_lrc[hwe->class] || replay_state) { 1520 xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE); /* PPHWSP */ 1521 xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, 1522 gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE, 1523 lrc->size - LRC_PPHWSP_SIZE); 1524 if (replay_state) 1525 xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, 1526 replay_state, lrc->replay_size); 1527 } else { 1528 void *init_data = empty_lrc_data(hwe); 1529 1530 if (!init_data) { 1531 return -ENOMEM; 1532 } 1533 1534 xe_map_memcpy_to(xe, &map, 0, init_data, lrc->size); 1535 kfree(init_data); 1536 } 1537 1538 if (vm) 1539 xe_lrc_set_ppgtt(lrc, vm); 1540 1541 if (xe_device_has_msix(xe)) { 1542 xe_lrc_write_ctx_reg(lrc, CTX_INT_STATUS_REPORT_PTR, 1543 xe_memirq_status_ptr(&tile->memirq, hwe)); 1544 xe_lrc_write_ctx_reg(lrc, CTX_INT_SRC_REPORT_PTR, 1545 xe_memirq_source_ptr(&tile->memirq, hwe)); 1546 xe_lrc_write_ctx_reg(lrc, CTX_CS_INT_VEC_DATA, msix_vec << 16 | msix_vec); 1547 } 1548 1549 if (xe_gt_has_indirect_ring_state(gt)) { 1550 xe_lrc_write_ctx_reg(lrc, CTX_INDIRECT_RING_STATE, 1551 __xe_lrc_indirect_ring_ggtt_addr(lrc)); 1552 1553 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START, 1554 __xe_lrc_ring_ggtt_addr(lrc)); 1555 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START_UDW, 0); 1556 1557 /* Match head and tail pointers */ 1558 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, lrc->ring.tail); 1559 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, lrc->ring.tail); 1560 1561 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_CTL, 1562 RING_CTL_SIZE(lrc->ring.size) | RING_VALID); 1563 } else { 1564 xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc)); 1565 1566 /* Match head and tail pointers */ 1567 xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, lrc->ring.tail); 1568 xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); 1569 1570 xe_lrc_write_ctx_reg(lrc, CTX_RING_CTL, 1571 RING_CTL_SIZE(lrc->ring.size) | RING_VALID); 1572 } 1573 1574 if (init_flags & XE_LRC_CREATE_RUNALONE) 1575 xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL, 1576 xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) | 1577 REG_MASKED_FIELD_ENABLE(CTX_CTRL_RUN_ALONE)); 1578 1579 if (init_flags & XE_LRC_CREATE_PXP) 1580 xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL, 1581 xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) | 1582 REG_MASKED_FIELD_ENABLE(CTX_CTRL_PXP_ENABLE)); 1583 1584 lrc->ctx_timestamp = 0; 1585 xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP, 0); 1586 if (lrc_to_xe(lrc)->info.has_64bit_timestamp) 1587 xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP_UDW, 0); 1588 1589 /* 1590 * Note: It's possible that this LRC may belong to an exec_queue that is 1591 * not part of a multi-queue group. That said, it doesn't hurt to set 1592 * this field anyways since any class that supports multi-queue will 1593 * have these LRC fields defined. 1594 */ 1595 if (xe_gt_supports_multi_queue(gt, hwe->class)) { 1596 lrc->queue_timestamp = 0; 1597 xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP, 0); 1598 xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP_UDW, 0); 1599 } 1600 1601 if (xe->info.has_asid && vm) 1602 xe_lrc_write_ctx_reg(lrc, CTX_ASID, vm->usm.asid); 1603 1604 lrc->desc = LRC_VALID; 1605 lrc->desc |= FIELD_PREP(LRC_ADDRESSING_MODE, LRC_LEGACY_64B_CONTEXT); 1606 /* TODO: Priority */ 1607 1608 /* While this appears to have something about privileged batches or 1609 * some such, it really just means PPGTT mode. 1610 */ 1611 if (vm) 1612 lrc->desc |= LRC_PRIVILEGE; 1613 1614 if (GRAPHICS_VERx100(xe) < 1250) { 1615 lrc->desc |= FIELD_PREP(LRC_ENGINE_INSTANCE, hwe->instance); 1616 lrc->desc |= FIELD_PREP(LRC_ENGINE_CLASS, hwe->class); 1617 } 1618 1619 arb_enable = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1620 xe_lrc_write_ring(lrc, &arb_enable, sizeof(arb_enable)); 1621 1622 if (init_flags & XE_LRC_DISABLE_STATE_CACHE_PERF_FIX) { 1623 state_cache_perf_fix[0] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1); 1624 state_cache_perf_fix[1] = COMMON_SLICE_CHICKEN3.addr; 1625 state_cache_perf_fix[2] = REG_MASKED_FIELD_ENABLE(DISABLE_STATE_CACHE_PERF_FIX); 1626 xe_lrc_write_ring(lrc, state_cache_perf_fix, sizeof(state_cache_perf_fix)); 1627 } 1628 1629 map = __xe_lrc_seqno_map(lrc); 1630 xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); 1631 1632 map = __xe_lrc_start_seqno_map(lrc); 1633 xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1); 1634 1635 err = setup_wa_bb(lrc, hwe); 1636 if (err) 1637 return err; 1638 1639 err = setup_indirect_ctx(lrc, hwe); 1640 1641 return err; 1642 } 1643 1644 static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm, 1645 void *replay_state, u32 ring_size, u16 msix_vec, u32 init_flags) 1646 { 1647 struct xe_gt *gt = hwe->gt; 1648 const u32 lrc_size = xe_gt_lrc_size(gt, hwe->class); 1649 u32 bo_size = ring_size + lrc_size + LRC_WA_BB_SIZE; 1650 struct xe_tile *tile = gt_to_tile(gt); 1651 struct xe_device *xe = gt_to_xe(gt); 1652 struct xe_bo *bo; 1653 u32 bo_flags; 1654 int err; 1655 1656 kref_init(&lrc->refcount); 1657 lrc->gt = gt; 1658 lrc->replay_size = xe_gt_lrc_hang_replay_size(gt, hwe->class); 1659 lrc->size = lrc_size; 1660 lrc->flags = 0; 1661 lrc->ring.size = ring_size; 1662 lrc->ring.tail = 0; 1663 1664 if (gt_engine_needs_indirect_ctx(gt, hwe->class)) { 1665 lrc->flags |= XE_LRC_FLAG_INDIRECT_CTX; 1666 bo_size += LRC_INDIRECT_CTX_BO_SIZE; 1667 } 1668 1669 if (xe_gt_has_indirect_ring_state(gt)) 1670 lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE; 1671 1672 bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT | 1673 XE_BO_FLAG_GGTT_INVALIDATE; 1674 1675 if ((vm && vm->xef) || init_flags & XE_LRC_CREATE_USER_CTX) /* userspace */ 1676 bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM; 1677 1678 bo = xe_bo_create_pin_map_novm(xe, tile, bo_size, 1679 ttm_bo_type_kernel, 1680 bo_flags, false); 1681 if (IS_ERR(bo)) 1682 return PTR_ERR(bo); 1683 1684 lrc->bo = bo; 1685 1686 bo = xe_bo_create_pin_map_novm(xe, tile, PAGE_SIZE, 1687 ttm_bo_type_kernel, 1688 XE_BO_FLAG_GGTT | 1689 XE_BO_FLAG_GGTT_INVALIDATE | 1690 XE_BO_FLAG_SYSTEM, false); 1691 if (IS_ERR(bo)) { 1692 err = PTR_ERR(bo); 1693 goto err_lrc_finish; 1694 } 1695 lrc->seqno_bo = bo; 1696 1697 xe_hw_fence_ctx_init(&lrc->fence_ctx, hwe->gt, 1698 hwe->fence_irq, hwe->name); 1699 1700 err = xe_lrc_ctx_init(lrc, hwe, vm, replay_state, msix_vec, init_flags); 1701 if (err) 1702 goto err_lrc_finish; 1703 1704 if (vm && vm->xef) 1705 xe_drm_client_add_bo(vm->xef->client, lrc->bo); 1706 1707 return 0; 1708 1709 err_lrc_finish: 1710 xe_lrc_finish(lrc); 1711 return err; 1712 } 1713 1714 /** 1715 * xe_lrc_create - Create a LRC 1716 * @hwe: Hardware Engine 1717 * @vm: The VM (address space) 1718 * @replay_state: GPU hang replay state 1719 * @ring_size: LRC ring size 1720 * @msix_vec: MSI-X interrupt vector (for platforms that support it) 1721 * @flags: LRC initialization flags 1722 * 1723 * Allocate and initialize the Logical Ring Context (LRC). 1724 * 1725 * Return pointer to created LRC upon success and an error pointer 1726 * upon failure. 1727 */ 1728 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, 1729 void *replay_state, u32 ring_size, u16 msix_vec, u32 flags) 1730 { 1731 struct xe_lrc *lrc; 1732 int err; 1733 1734 lrc = kzalloc_obj(*lrc); 1735 if (!lrc) 1736 return ERR_PTR(-ENOMEM); 1737 1738 err = xe_lrc_init(lrc, hwe, vm, replay_state, ring_size, msix_vec, flags); 1739 if (err) { 1740 kfree(lrc); 1741 return ERR_PTR(err); 1742 } 1743 1744 return lrc; 1745 } 1746 1747 /** 1748 * xe_lrc_destroy - Destroy the LRC 1749 * @ref: reference to LRC 1750 * 1751 * Called when ref == 0, release resources held by the Logical Ring Context 1752 * (LRC) and free the LRC memory. 1753 */ 1754 void xe_lrc_destroy(struct kref *ref) 1755 { 1756 struct xe_lrc *lrc = container_of(ref, struct xe_lrc, refcount); 1757 1758 xe_lrc_finish(lrc); 1759 kfree(lrc); 1760 } 1761 1762 /** 1763 * xe_lrc_update_hwctx_regs_with_address - Re-compute GGTT references within given LRC. 1764 * @lrc: the &xe_lrc struct instance 1765 */ 1766 void xe_lrc_update_hwctx_regs_with_address(struct xe_lrc *lrc) 1767 { 1768 if (xe_lrc_has_indirect_ring_state(lrc)) { 1769 xe_lrc_write_ctx_reg(lrc, CTX_INDIRECT_RING_STATE, 1770 __xe_lrc_indirect_ring_ggtt_addr(lrc)); 1771 1772 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START, 1773 __xe_lrc_ring_ggtt_addr(lrc)); 1774 } else { 1775 xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc)); 1776 } 1777 } 1778 1779 void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail) 1780 { 1781 if (xe_lrc_has_indirect_ring_state(lrc)) 1782 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, tail); 1783 else 1784 xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, tail); 1785 } 1786 1787 u32 xe_lrc_ring_tail(struct xe_lrc *lrc) 1788 { 1789 if (xe_lrc_has_indirect_ring_state(lrc)) 1790 return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL) & TAIL_ADDR; 1791 else 1792 return xe_lrc_read_ctx_reg(lrc, CTX_RING_TAIL) & TAIL_ADDR; 1793 } 1794 1795 static u32 xe_lrc_ring_start(struct xe_lrc *lrc) 1796 { 1797 if (xe_lrc_has_indirect_ring_state(lrc)) 1798 return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START); 1799 else 1800 return xe_lrc_read_ctx_reg(lrc, CTX_RING_START); 1801 } 1802 1803 void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head) 1804 { 1805 if (xe_lrc_has_indirect_ring_state(lrc)) 1806 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, head); 1807 else 1808 xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, head); 1809 } 1810 1811 u32 xe_lrc_ring_head(struct xe_lrc *lrc) 1812 { 1813 if (xe_lrc_has_indirect_ring_state(lrc)) 1814 return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD) & HEAD_ADDR; 1815 else 1816 return xe_lrc_read_ctx_reg(lrc, CTX_RING_HEAD) & HEAD_ADDR; 1817 } 1818 1819 u32 xe_lrc_ring_space(struct xe_lrc *lrc) 1820 { 1821 const u32 head = xe_lrc_ring_head(lrc); 1822 const u32 tail = lrc->ring.tail; 1823 const u32 size = lrc->ring.size; 1824 1825 return ((head - tail - 1) & (size - 1)) + 1; 1826 } 1827 1828 static void __xe_lrc_write_ring(struct xe_lrc *lrc, struct iosys_map ring, 1829 const void *data, size_t size) 1830 { 1831 struct xe_device *xe = lrc_to_xe(lrc); 1832 1833 iosys_map_incr(&ring, lrc->ring.tail); 1834 xe_map_memcpy_to(xe, &ring, 0, data, size); 1835 lrc->ring.tail = (lrc->ring.tail + size) & (lrc->ring.size - 1); 1836 } 1837 1838 void xe_lrc_write_ring(struct xe_lrc *lrc, const void *data, size_t size) 1839 { 1840 struct xe_device *xe = lrc_to_xe(lrc); 1841 struct iosys_map ring; 1842 u32 rhs; 1843 size_t aligned_size; 1844 1845 xe_assert(xe, IS_ALIGNED(size, 4)); 1846 aligned_size = ALIGN(size, 8); 1847 1848 ring = __xe_lrc_ring_map(lrc); 1849 1850 xe_assert(xe, lrc->ring.tail < lrc->ring.size); 1851 rhs = lrc->ring.size - lrc->ring.tail; 1852 if (size > rhs) { 1853 __xe_lrc_write_ring(lrc, ring, data, rhs); 1854 __xe_lrc_write_ring(lrc, ring, data + rhs, size - rhs); 1855 } else { 1856 __xe_lrc_write_ring(lrc, ring, data, size); 1857 } 1858 1859 if (aligned_size > size) { 1860 u32 noop = MI_NOOP; 1861 1862 __xe_lrc_write_ring(lrc, ring, &noop, sizeof(noop)); 1863 } 1864 1865 /* 1866 * The ring and the LRC context image are both WC, so the ring tail 1867 * update which publishes these writes can become visible to the device 1868 * first. Ensure the ring contents are visible before returning. 1869 */ 1870 xe_device_wmb(xe); 1871 } 1872 1873 u64 xe_lrc_descriptor(struct xe_lrc *lrc) 1874 { 1875 return lrc->desc | xe_lrc_ggtt_addr(lrc); 1876 } 1877 1878 u32 xe_lrc_seqno_ggtt_addr(struct xe_lrc *lrc) 1879 { 1880 return __xe_lrc_seqno_ggtt_addr(lrc); 1881 } 1882 1883 /** 1884 * xe_lrc_alloc_seqno_fence() - Allocate an lrc seqno fence. 1885 * 1886 * Allocate but don't initialize an lrc seqno fence. 1887 * 1888 * Return: Pointer to the allocated fence or 1889 * negative error pointer on error. 1890 */ 1891 struct dma_fence *xe_lrc_alloc_seqno_fence(void) 1892 { 1893 return xe_hw_fence_alloc(); 1894 } 1895 1896 /** 1897 * xe_lrc_free_seqno_fence() - Free an lrc seqno fence. 1898 * @fence: Pointer to the fence to free. 1899 * 1900 * Frees an lrc seqno fence that hasn't yet been 1901 * initialized. 1902 */ 1903 void xe_lrc_free_seqno_fence(struct dma_fence *fence) 1904 { 1905 xe_hw_fence_free(fence); 1906 } 1907 1908 /** 1909 * xe_lrc_init_seqno_fence() - Initialize an lrc seqno fence. 1910 * @lrc: Pointer to the lrc. 1911 * @fence: Pointer to the fence to initialize. 1912 * 1913 * Initializes a pre-allocated lrc seqno fence. 1914 * After initialization, the fence is subject to normal 1915 * dma-fence refcounting. 1916 */ 1917 void xe_lrc_init_seqno_fence(struct xe_lrc *lrc, struct dma_fence *fence) 1918 { 1919 xe_hw_fence_init(fence, &lrc->fence_ctx, __xe_lrc_seqno_map(lrc)); 1920 } 1921 1922 s32 xe_lrc_seqno(struct xe_lrc *lrc) 1923 { 1924 struct iosys_map map = __xe_lrc_seqno_map(lrc); 1925 1926 return xe_map_read32(lrc_to_xe(lrc), &map); 1927 } 1928 1929 s32 xe_lrc_start_seqno(struct xe_lrc *lrc) 1930 { 1931 struct iosys_map map = __xe_lrc_start_seqno_map(lrc); 1932 1933 return xe_map_read32(lrc_to_xe(lrc), &map); 1934 } 1935 1936 u32 xe_lrc_start_seqno_ggtt_addr(struct xe_lrc *lrc) 1937 { 1938 return __xe_lrc_start_seqno_ggtt_addr(lrc); 1939 } 1940 1941 u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc) 1942 { 1943 return __xe_lrc_parallel_ggtt_addr(lrc); 1944 } 1945 1946 struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc) 1947 { 1948 return __xe_lrc_parallel_map(lrc); 1949 } 1950 1951 /** 1952 * xe_lrc_engine_id() - Read engine id value 1953 * @lrc: Pointer to the lrc. 1954 * 1955 * Returns: context id value 1956 */ 1957 static u32 xe_lrc_engine_id(struct xe_lrc *lrc) 1958 { 1959 struct xe_device *xe = lrc_to_xe(lrc); 1960 struct iosys_map map; 1961 1962 map = __xe_lrc_engine_id_map(lrc); 1963 return xe_map_read32(xe, &map); 1964 } 1965 1966 static int instr_dw(u32 cmd_header) 1967 { 1968 /* GFXPIPE "SINGLE_DW" opcodes are a single dword */ 1969 if ((cmd_header & (XE_INSTR_CMD_TYPE | GFXPIPE_PIPELINE)) == 1970 GFXPIPE_SINGLE_DW_CMD(0, 0)) 1971 return 1; 1972 1973 /* 3DSTATE_SO_DECL_LIST has a 9-bit dword length rather than 8 */ 1974 if ((cmd_header & GFXPIPE_MATCH_MASK) == CMD_3DSTATE_SO_DECL_LIST) 1975 return REG_FIELD_GET(CMD_3DSTATE_SO_DECL_LIST_DW_LEN, cmd_header) + 2; 1976 1977 /* Most instructions have the # of dwords (minus 2) in 7:0 */ 1978 return REG_FIELD_GET(XE_INSTR_LEN_MASK, cmd_header) + 2; 1979 } 1980 1981 static int dump_mi_command(struct drm_printer *p, 1982 struct xe_gt *gt, 1983 u32 *start, 1984 u32 *dw, 1985 int remaining_dw) 1986 { 1987 u32 inst_header = *dw; 1988 u32 numdw = instr_dw(inst_header); 1989 u32 opcode = REG_FIELD_GET(MI_OPCODE, inst_header); 1990 int num_noop; 1991 1992 /* First check for commands that don't have/use a '# DW' field */ 1993 switch (inst_header & MI_OPCODE) { 1994 case MI_NOOP: 1995 num_noop = 1; 1996 while (num_noop < remaining_dw && 1997 (*(++dw) & REG_GENMASK(31, 23)) == MI_NOOP) 1998 num_noop++; 1999 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_NOOP (%d dwords)\n", 2000 dw - num_noop - start, inst_header, num_noop); 2001 return num_noop; 2002 2003 case MI_TOPOLOGY_FILTER: 2004 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_TOPOLOGY_FILTER\n", 2005 dw - start, inst_header); 2006 return 1; 2007 2008 case MI_BATCH_BUFFER_END: 2009 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_BATCH_BUFFER_END\n", 2010 dw - start, inst_header); 2011 /* Return 'remaining_dw' to consume the rest of the LRC */ 2012 return remaining_dw; 2013 } 2014 2015 /* 2016 * Any remaining commands include a # of dwords. We should make sure 2017 * it doesn't exceed the remaining size of the LRC. 2018 */ 2019 if (xe_gt_WARN_ON(gt, numdw > remaining_dw)) 2020 numdw = remaining_dw; 2021 2022 switch (inst_header & MI_OPCODE) { 2023 case MI_LOAD_REGISTER_IMM: 2024 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_LOAD_REGISTER_IMM: %d regs\n", 2025 dw - start, inst_header, (numdw - 1) / 2); 2026 for (int i = 1; i < numdw; i += 2) 2027 drm_printf(p, "LRC[%#5tx] = - %#6x = %#010x\n", 2028 &dw[i] - start, dw[i], dw[i + 1]); 2029 return numdw; 2030 2031 case MI_LOAD_REGISTER_MEM & MI_OPCODE: 2032 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_LOAD_REGISTER_MEM: %s%s\n", 2033 dw - start, inst_header, 2034 dw[0] & MI_LRI_LRM_CS_MMIO ? "CS_MMIO " : "", 2035 dw[0] & MI_LRM_USE_GGTT ? "USE_GGTT " : ""); 2036 if (numdw == 4) 2037 drm_printf(p, "LRC[%#5tx] = - %#6x = %#010llx\n", 2038 dw - start, 2039 dw[1], ((u64)(dw[3]) << 32 | (u64)(dw[2]))); 2040 else 2041 drm_printf(p, "LRC[%#5tx] = - %*ph (%s)\n", 2042 dw - start, (int)sizeof(u32) * (numdw - 1), 2043 dw + 1, numdw < 4 ? "truncated" : "malformed"); 2044 return numdw; 2045 2046 case MI_FORCE_WAKEUP: 2047 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_FORCE_WAKEUP\n", 2048 dw - start, inst_header); 2049 return numdw; 2050 2051 default: 2052 drm_printf(p, "LRC[%#5tx] = [%#010x] unknown MI opcode %#x, likely %d dwords\n", 2053 dw - start, inst_header, opcode, numdw); 2054 return numdw; 2055 } 2056 } 2057 2058 static int dump_gfxpipe_command(struct drm_printer *p, 2059 struct xe_gt *gt, 2060 u32 *start, 2061 u32 *dw, 2062 int remaining_dw) 2063 { 2064 u32 numdw = instr_dw(*dw); 2065 u32 pipeline = REG_FIELD_GET(GFXPIPE_PIPELINE, *dw); 2066 u32 opcode = REG_FIELD_GET(GFXPIPE_OPCODE, *dw); 2067 u32 subopcode = REG_FIELD_GET(GFXPIPE_SUBOPCODE, *dw); 2068 2069 /* 2070 * Make sure we haven't mis-parsed a number of dwords that exceeds the 2071 * remaining size of the LRC. 2072 */ 2073 if (xe_gt_WARN_ON(gt, numdw > remaining_dw)) 2074 numdw = remaining_dw; 2075 2076 switch (*dw & GFXPIPE_MATCH_MASK) { 2077 #define MATCH(cmd) \ 2078 case cmd: \ 2079 drm_printf(p, "LRC[%#5tx] = [%#010x] " #cmd " (%d dwords)\n", \ 2080 dw - start, *dw, numdw); \ 2081 return numdw 2082 #define MATCH3D(cmd) \ 2083 case CMD_##cmd: \ 2084 drm_printf(p, "LRC[%#5tx] = [%#010x] " #cmd " (%d dwords)\n", \ 2085 dw - start, *dw, numdw); \ 2086 return numdw 2087 2088 MATCH(STATE_BASE_ADDRESS); 2089 MATCH(STATE_SIP); 2090 MATCH(GPGPU_CSR_BASE_ADDRESS); 2091 MATCH(STATE_COMPUTE_MODE); 2092 MATCH3D(3DSTATE_BTD); 2093 MATCH(STATE_SYSTEM_MEM_FENCE_ADDRESS); 2094 MATCH(STATE_CONTEXT_DATA_BASE_ADDRESS); 2095 2096 MATCH3D(3DSTATE_VF_STATISTICS); 2097 2098 MATCH(PIPELINE_SELECT); 2099 2100 MATCH3D(3DSTATE_DRAWING_RECTANGLE_FAST); 2101 MATCH3D(3DSTATE_CUSTOM_SAMPLE_PATTERN); 2102 MATCH3D(3DSTATE_CLEAR_PARAMS); 2103 MATCH3D(3DSTATE_DEPTH_BUFFER); 2104 MATCH3D(3DSTATE_STENCIL_BUFFER); 2105 MATCH3D(3DSTATE_HIER_DEPTH_BUFFER); 2106 MATCH3D(3DSTATE_VERTEX_BUFFERS); 2107 MATCH3D(3DSTATE_VERTEX_ELEMENTS); 2108 MATCH3D(3DSTATE_INDEX_BUFFER); 2109 MATCH3D(3DSTATE_VF); 2110 MATCH3D(3DSTATE_MULTISAMPLE); 2111 MATCH3D(3DSTATE_CC_STATE_POINTERS); 2112 MATCH3D(3DSTATE_SCISSOR_STATE_POINTERS); 2113 MATCH3D(3DSTATE_VS); 2114 MATCH3D(3DSTATE_GS); 2115 MATCH3D(3DSTATE_CLIP); 2116 MATCH3D(3DSTATE_SF); 2117 MATCH3D(3DSTATE_WM); 2118 MATCH3D(3DSTATE_CONSTANT_VS); 2119 MATCH3D(3DSTATE_CONSTANT_GS); 2120 MATCH3D(3DSTATE_CONSTANT_PS); 2121 MATCH3D(3DSTATE_SAMPLE_MASK); 2122 MATCH3D(3DSTATE_CONSTANT_HS); 2123 MATCH3D(3DSTATE_CONSTANT_DS); 2124 MATCH3D(3DSTATE_HS); 2125 MATCH3D(3DSTATE_TE); 2126 MATCH3D(3DSTATE_DS); 2127 MATCH3D(3DSTATE_STREAMOUT); 2128 MATCH3D(3DSTATE_SBE); 2129 MATCH3D(3DSTATE_PS); 2130 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP); 2131 MATCH3D(3DSTATE_CPS_POINTERS); 2132 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_CC); 2133 MATCH3D(3DSTATE_BLEND_STATE_POINTERS); 2134 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_VS); 2135 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_HS); 2136 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_DS); 2137 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_GS); 2138 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_PS); 2139 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_VS); 2140 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_HS); 2141 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_DS); 2142 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_GS); 2143 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_PS); 2144 MATCH3D(3DSTATE_VF_INSTANCING); 2145 MATCH3D(3DSTATE_VF_SGVS); 2146 MATCH3D(3DSTATE_VF_TOPOLOGY); 2147 MATCH3D(3DSTATE_WM_CHROMAKEY); 2148 MATCH3D(3DSTATE_PS_BLEND); 2149 MATCH3D(3DSTATE_WM_DEPTH_STENCIL); 2150 MATCH3D(3DSTATE_PS_EXTRA); 2151 MATCH3D(3DSTATE_RASTER); 2152 MATCH3D(3DSTATE_SBE_SWIZ); 2153 MATCH3D(3DSTATE_WM_HZ_OP); 2154 MATCH3D(3DSTATE_VF_COMPONENT_PACKING); 2155 MATCH3D(3DSTATE_VF_SGVS_2); 2156 MATCH3D(3DSTATE_VFG); 2157 MATCH3D(3DSTATE_URB_ALLOC_VS); 2158 MATCH3D(3DSTATE_URB_ALLOC_HS); 2159 MATCH3D(3DSTATE_URB_ALLOC_DS); 2160 MATCH3D(3DSTATE_URB_ALLOC_GS); 2161 MATCH3D(3DSTATE_SO_BUFFER_INDEX_0); 2162 MATCH3D(3DSTATE_SO_BUFFER_INDEX_1); 2163 MATCH3D(3DSTATE_SO_BUFFER_INDEX_2); 2164 MATCH3D(3DSTATE_SO_BUFFER_INDEX_3); 2165 MATCH3D(3DSTATE_PRIMITIVE_REPLICATION); 2166 MATCH3D(3DSTATE_TBIMR_TILE_PASS_INFO); 2167 MATCH3D(3DSTATE_AMFS); 2168 MATCH3D(3DSTATE_DEPTH_BOUNDS); 2169 MATCH3D(3DSTATE_AMFS_TEXTURE_POINTERS); 2170 MATCH3D(3DSTATE_CONSTANT_TS_POINTER); 2171 MATCH3D(3DSTATE_MESH_CONTROL); 2172 MATCH3D(3DSTATE_MESH_DISTRIB); 2173 MATCH3D(3DSTATE_TASK_REDISTRIB); 2174 MATCH3D(3DSTATE_MESH_SHADER); 2175 MATCH3D(3DSTATE_MESH_SHADER_DATA); 2176 MATCH3D(3DSTATE_TASK_CONTROL); 2177 MATCH3D(3DSTATE_TASK_SHADER); 2178 MATCH3D(3DSTATE_TASK_SHADER_DATA); 2179 MATCH3D(3DSTATE_URB_ALLOC_MESH); 2180 MATCH3D(3DSTATE_URB_ALLOC_TASK); 2181 MATCH3D(3DSTATE_CLIP_MESH); 2182 MATCH3D(3DSTATE_SBE_MESH); 2183 MATCH3D(3DSTATE_CPSIZE_CONTROL_BUFFER); 2184 MATCH3D(3DSTATE_COARSE_PIXEL); 2185 MATCH3D(3DSTATE_MESH_SHADER_DATA_EXT); 2186 MATCH3D(3DSTATE_TASK_SHADER_DATA_EXT); 2187 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_CC_2); 2188 MATCH3D(3DSTATE_CC_STATE_POINTERS_2); 2189 MATCH3D(3DSTATE_SCISSOR_STATE_POINTERS_2); 2190 MATCH3D(3DSTATE_BLEND_STATE_POINTERS_2); 2191 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP_2); 2192 2193 MATCH3D(3DSTATE_DRAWING_RECTANGLE); 2194 MATCH3D(3DSTATE_URB_MEMORY); 2195 MATCH3D(3DSTATE_CHROMA_KEY); 2196 MATCH3D(3DSTATE_POLY_STIPPLE_OFFSET); 2197 MATCH3D(3DSTATE_POLY_STIPPLE_PATTERN); 2198 MATCH3D(3DSTATE_LINE_STIPPLE); 2199 MATCH3D(3DSTATE_AA_LINE_PARAMETERS); 2200 MATCH3D(3DSTATE_MONOFILTER_SIZE); 2201 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_VS); 2202 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_HS); 2203 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_DS); 2204 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_GS); 2205 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_PS); 2206 MATCH3D(3DSTATE_SO_DECL_LIST); 2207 MATCH3D(3DSTATE_SO_BUFFER); 2208 MATCH3D(3DSTATE_BINDING_TABLE_POOL_ALLOC); 2209 MATCH3D(3DSTATE_SAMPLE_PATTERN); 2210 MATCH3D(3DSTATE_3D_MODE); 2211 MATCH3D(3DSTATE_SUBSLICE_HASH_TABLE); 2212 MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTERS); 2213 MATCH3D(3DSTATE_PTBR_TILE_PASS_INFO); 2214 MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTER_2); 2215 2216 default: 2217 drm_printf(p, "LRC[%#5tx] = [%#010x] unknown GFXPIPE command (pipeline=%#x, opcode=%#x, subopcode=%#x), likely %d dwords\n", 2218 dw - start, *dw, pipeline, opcode, subopcode, numdw); 2219 return numdw; 2220 } 2221 } 2222 2223 static int dump_gfx_state_command(struct drm_printer *p, 2224 struct xe_gt *gt, 2225 u32 *start, 2226 u32 *dw, 2227 int remaining_dw) 2228 { 2229 u32 numdw = instr_dw(*dw); 2230 u32 opcode = REG_FIELD_GET(GFX_STATE_OPCODE, *dw); 2231 2232 /* 2233 * Make sure we haven't mis-parsed a number of dwords that exceeds the 2234 * remaining size of the LRC. 2235 */ 2236 if (xe_gt_WARN_ON(gt, numdw > remaining_dw)) 2237 numdw = remaining_dw; 2238 2239 switch (*dw & (XE_INSTR_GFX_STATE | GFX_STATE_OPCODE)) { 2240 MATCH(STATE_WRITE_INLINE); 2241 2242 default: 2243 drm_printf(p, "LRC[%#5tx] = [%#010x] unknown GFX_STATE command (opcode=%#x), likely %d dwords\n", 2244 dw - start, *dw, opcode, numdw); 2245 return numdw; 2246 } 2247 } 2248 2249 void xe_lrc_dump_default(struct drm_printer *p, 2250 struct xe_gt *gt, 2251 enum xe_engine_class hwe_class) 2252 { 2253 u32 *dw, *start; 2254 int remaining_dw, num_dw; 2255 2256 if (!gt->default_lrc[hwe_class]) { 2257 drm_printf(p, "No default LRC for class %d\n", hwe_class); 2258 return; 2259 } 2260 2261 /* 2262 * Skip the beginning of the LRC since it contains the per-process 2263 * hardware status page. 2264 */ 2265 dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE; 2266 start = dw; 2267 remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4; 2268 2269 while (remaining_dw > 0) { 2270 if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_MI) { 2271 num_dw = dump_mi_command(p, gt, start, dw, remaining_dw); 2272 } else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE) { 2273 num_dw = dump_gfxpipe_command(p, gt, start, dw, remaining_dw); 2274 } else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFX_STATE) { 2275 num_dw = dump_gfx_state_command(p, gt, start, dw, remaining_dw); 2276 } else { 2277 num_dw = min(instr_dw(*dw), remaining_dw); 2278 drm_printf(p, "LRC[%#5tx] = [%#10x] Unknown instruction of type %#x, likely %d dwords\n", 2279 dw - start, 2280 *dw, REG_FIELD_GET(XE_INSTR_CMD_TYPE, *dw), 2281 num_dw); 2282 } 2283 2284 dw += num_dw; 2285 remaining_dw -= num_dw; 2286 } 2287 } 2288 2289 /* 2290 * Lookup the value of a register within the offset/value pairs of an 2291 * MI_LOAD_REGISTER_IMM instruction. 2292 * 2293 * Return -ENOENT if the register is not present in the MI_LRI instruction. 2294 */ 2295 static int lookup_reg_in_mi_lri(u32 offset, u32 *value, 2296 const u32 *dword_pair, int num_regs) 2297 { 2298 for (int i = 0; i < num_regs; i++) { 2299 if (dword_pair[2 * i] == offset) { 2300 *value = dword_pair[2 * i + 1]; 2301 return 0; 2302 } 2303 } 2304 2305 return -ENOENT; 2306 } 2307 2308 /* 2309 * Lookup the value of a register in a specific engine type's default LRC. 2310 * 2311 * Return -EINVAL if the default LRC doesn't exist, or ENOENT if the register 2312 * cannot be found in the default LRC. 2313 */ 2314 int xe_lrc_lookup_default_reg_value(struct xe_gt *gt, 2315 enum xe_engine_class hwe_class, 2316 u32 offset, 2317 u32 *value) 2318 { 2319 u32 *dw; 2320 int remaining_dw, ret; 2321 2322 if (!gt->default_lrc[hwe_class]) 2323 return -EINVAL; 2324 2325 /* 2326 * Skip the beginning of the LRC since it contains the per-process 2327 * hardware status page. 2328 */ 2329 dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE; 2330 remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4; 2331 2332 while (remaining_dw > 0) { 2333 u32 num_dw = instr_dw(*dw); 2334 2335 if (num_dw > remaining_dw) 2336 num_dw = remaining_dw; 2337 2338 switch (*dw & XE_INSTR_CMD_TYPE) { 2339 case XE_INSTR_MI: 2340 switch (*dw & MI_OPCODE) { 2341 case MI_BATCH_BUFFER_END: 2342 /* End of LRC; register not found */ 2343 return -ENOENT; 2344 2345 case MI_NOOP: 2346 case MI_TOPOLOGY_FILTER: 2347 /* 2348 * MI_NOOP and MI_TOPOLOGY_FILTER don't have 2349 * a length field and are always 1-dword 2350 * instructions. 2351 */ 2352 remaining_dw--; 2353 dw++; 2354 break; 2355 2356 case MI_LOAD_REGISTER_IMM: 2357 ret = lookup_reg_in_mi_lri(offset, value, 2358 dw + 1, (num_dw - 1) / 2); 2359 if (ret == 0) 2360 return 0; 2361 2362 fallthrough; 2363 2364 default: 2365 /* 2366 * Jump to next instruction based on length 2367 * field. 2368 */ 2369 remaining_dw -= num_dw; 2370 dw += num_dw; 2371 break; 2372 } 2373 break; 2374 2375 default: 2376 /* Jump to next instruction based on length field. */ 2377 remaining_dw -= num_dw; 2378 dw += num_dw; 2379 } 2380 } 2381 2382 return -ENOENT; 2383 } 2384 2385 struct instr_state { 2386 u32 instr; 2387 u16 num_dw; 2388 }; 2389 2390 static const struct instr_state xe_hpg_svg_state[] = { 2391 { .instr = CMD_3DSTATE_CONSTANT_VS, .num_dw = 11 }, 2392 { .instr = CMD_3DSTATE_CONSTANT_HS, .num_dw = 11 }, 2393 { .instr = CMD_3DSTATE_CONSTANT_DS, .num_dw = 11 }, 2394 { .instr = CMD_3DSTATE_CONSTANT_GS, .num_dw = 11 }, 2395 { .instr = CMD_3DSTATE_VERTEX_ELEMENTS, .num_dw = 69 }, 2396 { .instr = CMD_3DSTATE_VF_COMPONENT_PACKING, .num_dw = 5 }, 2397 { .instr = CMD_3DSTATE_VF_SGVS, .num_dw = 2 }, 2398 { .instr = CMD_3DSTATE_VF_SGVS_2, .num_dw = 3 }, 2399 { .instr = CMD_3DSTATE_VS, .num_dw = 9 }, 2400 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_VS, .num_dw = 2 }, 2401 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_VS, .num_dw = 2 }, 2402 { .instr = CMD_3DSTATE_URB_ALLOC_VS, .num_dw = 3 }, 2403 { .instr = CMD_3DSTATE_STREAMOUT, .num_dw = 5 }, 2404 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_0, .num_dw = 8 }, 2405 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_1, .num_dw = 8 }, 2406 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_2, .num_dw = 8 }, 2407 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_3, .num_dw = 8 }, 2408 { .instr = CMD_3DSTATE_CLIP, .num_dw = 4 }, 2409 { .instr = CMD_3DSTATE_PRIMITIVE_REPLICATION, .num_dw = 6 }, 2410 { .instr = CMD_3DSTATE_CLIP_MESH, .num_dw = 2 }, 2411 { .instr = CMD_3DSTATE_SF, .num_dw = 4 }, 2412 { .instr = CMD_3DSTATE_SCISSOR_STATE_POINTERS, .num_dw = 2 }, 2413 { .instr = CMD_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP, .num_dw = 2 }, 2414 { .instr = CMD_3DSTATE_RASTER, .num_dw = 5 }, 2415 { .instr = CMD_3DSTATE_TBIMR_TILE_PASS_INFO, .num_dw = 4 }, 2416 { .instr = CMD_3DSTATE_WM_HZ_OP, .num_dw = 6 }, 2417 { .instr = CMD_3DSTATE_MULTISAMPLE, .num_dw = 2 }, 2418 { .instr = CMD_3DSTATE_HS, .num_dw = 9 }, 2419 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_HS, .num_dw = 2 }, 2420 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_HS, .num_dw = 2 }, 2421 { .instr = CMD_3DSTATE_URB_ALLOC_HS, .num_dw = 3 }, 2422 { .instr = CMD_3DSTATE_TASK_CONTROL, .num_dw = 3 }, 2423 { .instr = CMD_3DSTATE_TASK_SHADER, .num_dw = 7 }, 2424 { .instr = CMD_3DSTATE_TASK_SHADER_DATA, .num_dw = 10 }, 2425 { .instr = CMD_3DSTATE_URB_ALLOC_TASK, .num_dw = 3 }, 2426 { .instr = CMD_3DSTATE_TE, .num_dw = 5 }, 2427 { .instr = CMD_3DSTATE_TASK_REDISTRIB, .num_dw = 2 }, 2428 { .instr = CMD_3DSTATE_DS, .num_dw = 11 }, 2429 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_DS, .num_dw = 2 }, 2430 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_DS, .num_dw = 2 }, 2431 { .instr = CMD_3DSTATE_URB_ALLOC_DS, .num_dw = 3 }, 2432 { .instr = CMD_3DSTATE_GS, .num_dw = 10 }, 2433 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_GS, .num_dw = 2 }, 2434 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_GS, .num_dw = 2 }, 2435 { .instr = CMD_3DSTATE_URB_ALLOC_GS, .num_dw = 3 }, 2436 { .instr = CMD_3DSTATE_MESH_CONTROL, .num_dw = 3 }, 2437 { .instr = CMD_3DSTATE_MESH_SHADER_DATA, .num_dw = 10 }, 2438 { .instr = CMD_3DSTATE_URB_ALLOC_MESH, .num_dw = 3 }, 2439 { .instr = CMD_3DSTATE_MESH_SHADER, .num_dw = 8 }, 2440 { .instr = CMD_3DSTATE_DRAWING_RECTANGLE, .num_dw = 4 }, 2441 }; 2442 2443 u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs) 2444 { 2445 struct xe_gt *gt = q->hwe->gt; 2446 struct xe_device *xe = gt_to_xe(gt); 2447 const struct instr_state *state_table = NULL; 2448 int state_table_size = 0; 2449 2450 /* 2451 * Wa_14019789679 2452 * 2453 * If the driver doesn't explicitly emit the SVG instructions while 2454 * setting up the default LRC, the context switch will write 0's 2455 * (noops) into the LRC memory rather than the expected instruction 2456 * headers. Application contexts start out as a copy of the default 2457 * LRC, and if they also do not emit specific settings for some SVG 2458 * state, then on context restore they'll unintentionally inherit 2459 * whatever state setting the previous context had programmed into the 2460 * hardware (i.e., the lack of a 3DSTATE_* instruction in the LRC will 2461 * prevent the hardware from resetting that state back to any specific 2462 * value). 2463 * 2464 * The official workaround only requires emitting 3DSTATE_MESH_CONTROL 2465 * since that's a specific state setting that can easily cause GPU 2466 * hangs if unintentionally inherited. However to be safe we'll 2467 * continue to emit all of the SVG state since it's best not to leak 2468 * any of the state between contexts, even if that leakage is harmless. 2469 */ 2470 if (XE_GT_WA(gt, 14019789679) && q->hwe->class == XE_ENGINE_CLASS_RENDER) { 2471 state_table = xe_hpg_svg_state; 2472 state_table_size = ARRAY_SIZE(xe_hpg_svg_state); 2473 } 2474 2475 if (!state_table) { 2476 xe_gt_dbg(gt, "No non-register state to emit on graphics ver %d.%02d\n", 2477 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); 2478 return cs; 2479 } 2480 2481 for (int i = 0; i < state_table_size; i++) { 2482 u32 instr = state_table[i].instr; 2483 u16 num_dw = state_table[i].num_dw; 2484 bool is_single_dw = ((instr & GFXPIPE_PIPELINE) == PIPELINE_SINGLE_DW); 2485 2486 xe_gt_assert(gt, (instr & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE); 2487 xe_gt_assert(gt, num_dw != 0); 2488 xe_gt_assert(gt, is_single_dw ^ (num_dw > 1)); 2489 2490 /* 2491 * Xe2's SVG context is the same as the one on DG2 / MTL 2492 * except that 3DSTATE_DRAWING_RECTANGLE (non-pipelined) has 2493 * been replaced by 3DSTATE_DRAWING_RECTANGLE_FAST (pipelined). 2494 * Just make the replacement here rather than defining a 2495 * whole separate table for the single trivial change. 2496 */ 2497 if (GRAPHICS_VER(xe) >= 20 && 2498 instr == CMD_3DSTATE_DRAWING_RECTANGLE) 2499 instr = CMD_3DSTATE_DRAWING_RECTANGLE_FAST; 2500 2501 *cs = instr; 2502 if (!is_single_dw) 2503 *cs |= (num_dw - 2); 2504 2505 cs += num_dw; 2506 } 2507 2508 return cs; 2509 } 2510 2511 struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc) 2512 { 2513 struct xe_lrc_snapshot *snapshot = kmalloc_obj(*snapshot, GFP_NOWAIT); 2514 2515 if (!snapshot) 2516 return NULL; 2517 2518 snapshot->context_desc = xe_lrc_ggtt_addr(lrc); 2519 snapshot->ring_addr = __xe_lrc_ring_ggtt_addr(lrc); 2520 snapshot->indirect_context_desc = xe_lrc_indirect_ring_ggtt_addr(lrc); 2521 snapshot->head = xe_lrc_ring_head(lrc); 2522 snapshot->tail.internal = lrc->ring.tail; 2523 snapshot->tail.memory = xe_lrc_ring_tail(lrc); 2524 snapshot->start = xe_lrc_ring_start(lrc); 2525 snapshot->start_seqno = xe_lrc_start_seqno(lrc); 2526 snapshot->seqno = xe_lrc_seqno(lrc); 2527 snapshot->lrc_bo = xe_bo_get(lrc->bo); 2528 snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc); 2529 snapshot->lrc_size = lrc->size; 2530 snapshot->replay_offset = 0; 2531 snapshot->replay_size = lrc->replay_size; 2532 snapshot->lrc_snapshot = NULL; 2533 snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc); 2534 snapshot->ctx_timestamp_ms = 2535 xe_gt_clock_interval_to_ms(lrc->gt, xe_lrc_ctx_timestamp(lrc)); 2536 if (xe_lrc_is_multi_queue(lrc)) { 2537 snapshot->queue_timestamp = xe_lrc_queue_timestamp(lrc); 2538 snapshot->queue_timestamp_ms = 2539 xe_gt_clock_interval_to_ms(lrc->gt, snapshot->queue_timestamp); 2540 } else { 2541 snapshot->queue_timestamp = 0; 2542 snapshot->queue_timestamp_ms = 0; 2543 } 2544 snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc); 2545 return snapshot; 2546 } 2547 2548 void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot) 2549 { 2550 struct xe_bo *bo; 2551 struct iosys_map src; 2552 2553 if (!snapshot) 2554 return; 2555 2556 bo = snapshot->lrc_bo; 2557 snapshot->lrc_bo = NULL; 2558 2559 snapshot->lrc_snapshot = kvmalloc(snapshot->lrc_size, GFP_KERNEL); 2560 if (!snapshot->lrc_snapshot) 2561 goto put_bo; 2562 2563 xe_bo_lock(bo, false); 2564 if (!ttm_bo_vmap(&bo->ttm, &src)) { 2565 xe_map_memcpy_from(xe_bo_device(bo), 2566 snapshot->lrc_snapshot, &src, snapshot->lrc_offset, 2567 snapshot->lrc_size); 2568 ttm_bo_vunmap(&bo->ttm, &src); 2569 } else { 2570 kvfree(snapshot->lrc_snapshot); 2571 snapshot->lrc_snapshot = NULL; 2572 } 2573 xe_bo_unlock(bo); 2574 put_bo: 2575 xe_bo_put(bo); 2576 } 2577 2578 void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p) 2579 { 2580 unsigned long i; 2581 2582 if (!snapshot) 2583 return; 2584 2585 drm_printf(p, "\tHW Context Desc: 0x%08x\n", snapshot->context_desc); 2586 drm_printf(p, "\tHW Ring address: 0x%08x\n", 2587 snapshot->ring_addr); 2588 drm_printf(p, "\tHW Indirect Ring State: 0x%08x\n", 2589 snapshot->indirect_context_desc); 2590 drm_printf(p, "\tLRC Head: (memory) %u\n", snapshot->head); 2591 drm_printf(p, "\tLRC Tail: (internal) %u, (memory) %u\n", 2592 snapshot->tail.internal, snapshot->tail.memory); 2593 drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start); 2594 drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno); 2595 drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno); 2596 drm_printf(p, "\tTimestamp: 0x%016llx\n", snapshot->ctx_timestamp); 2597 drm_printf(p, "\tTimestamp ms: %llu\n", snapshot->ctx_timestamp_ms); 2598 drm_printf(p, "\tQueue Timestamp: 0x%016llx\n", snapshot->queue_timestamp); 2599 drm_printf(p, "\tQueue Timestamp ms: %llu\n", snapshot->queue_timestamp_ms); 2600 drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp); 2601 2602 if (!snapshot->lrc_snapshot) 2603 return; 2604 2605 drm_printf(p, "\t[HWSP].length: 0x%x\n", LRC_PPHWSP_SIZE); 2606 drm_puts(p, "\t[HWSP].data: "); 2607 for (i = 0; i < LRC_PPHWSP_SIZE; i += sizeof(u32)) { 2608 u32 *val = snapshot->lrc_snapshot + i; 2609 char dumped[ASCII85_BUFSZ]; 2610 2611 drm_puts(p, ascii85_encode(*val, dumped)); 2612 } 2613 2614 drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE); 2615 drm_printf(p, "\n\t[HWCTX].replay_offset: 0x%lx\n", snapshot->replay_offset); 2616 drm_printf(p, "\n\t[HWCTX].replay_length: 0x%lx\n", snapshot->replay_size); 2617 2618 drm_puts(p, "\t[HWCTX].data: "); 2619 for (; i < snapshot->lrc_size; i += sizeof(u32)) { 2620 u32 *val = snapshot->lrc_snapshot + i; 2621 char dumped[ASCII85_BUFSZ]; 2622 2623 drm_puts(p, ascii85_encode(*val, dumped)); 2624 } 2625 drm_puts(p, "\n"); 2626 } 2627 2628 void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot) 2629 { 2630 if (!snapshot) 2631 return; 2632 2633 kvfree(snapshot->lrc_snapshot); 2634 if (snapshot->lrc_bo) 2635 xe_bo_put(snapshot->lrc_bo); 2636 2637 kfree(snapshot); 2638 } 2639 2640 static bool engine_valid_for_utilization(struct xe_gt *gt, struct xe_hw_engine *hwe) 2641 { 2642 /* The USM-reserved copy engine runs kernel migrate contexts queried here */ 2643 return hwe && (!xe_hw_engine_is_reserved(hwe) || xe_gt_is_usm_hwe(gt, hwe)); 2644 } 2645 2646 static struct xe_hw_engine *engine_id_to_hwe(struct xe_gt *gt, u32 engine_id) 2647 { 2648 u16 class = REG_FIELD_GET(ENGINE_CLASS_ID, engine_id); 2649 u16 instance = REG_FIELD_GET(ENGINE_INSTANCE_ID, engine_id); 2650 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, class, instance, false); 2651 2652 if (xe_gt_WARN_ONCE(gt, !engine_valid_for_utilization(gt, hwe), 2653 "Unexpected engine class:instance %d:%d for utilization\n", 2654 class, instance)) 2655 return NULL; 2656 2657 return hwe; 2658 } 2659 2660 static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts) 2661 { 2662 struct xe_hw_engine *hwe; 2663 u64 val; 2664 2665 hwe = engine_id_to_hwe(lrc->gt, engine_id); 2666 if (!hwe) 2667 return -1; 2668 2669 if (lrc_to_xe(lrc)->info.has_64bit_timestamp) 2670 val = xe_mmio_read64_2x32(&hwe->gt->mmio, 2671 RING_CTX_TIMESTAMP(hwe->mmio_base)); 2672 else 2673 val = xe_mmio_read32(&hwe->gt->mmio, 2674 RING_CTX_TIMESTAMP(hwe->mmio_base)); 2675 2676 *reg_ctx_ts = val; 2677 2678 return 0; 2679 } 2680 2681 static u64 get_queue_timestamp(struct xe_hw_engine *hwe) 2682 { 2683 return xe_mmio_read64_2x32(&hwe->gt->mmio, 2684 RING_QUEUE_TIMESTAMP(hwe->mmio_base)); 2685 } 2686 2687 static u32 get_multi_queue_active_queue_id(struct xe_hw_engine *hwe) 2688 { 2689 u32 val = xe_mmio_read32(&hwe->gt->mmio, 2690 RING_CSMQDEBUG(hwe->mmio_base)); 2691 2692 return REG_FIELD_GET(CURRENT_ACTIVE_QUEUE_ID_MASK, val); 2693 } 2694 2695 static bool context_active(struct xe_lrc *lrc) 2696 { 2697 return xe_lrc_ctx_timestamp(lrc) == CONTEXT_ACTIVE; 2698 } 2699 2700 static u64 xe_lrc_multi_queue_timestamp(struct xe_lrc *lrc) 2701 { 2702 struct xe_device *xe = lrc_to_xe(lrc); 2703 struct xe_lrc *primary_lrc = lrc->multi_queue.primary_lrc; 2704 struct xe_hw_engine *hwe; 2705 u64 reg_queue_ts = lrc->queue_timestamp; 2706 2707 if (IS_SRIOV_VF(xe)) 2708 return xe_lrc_queue_timestamp(lrc); 2709 2710 xe_assert(xe, primary_lrc); 2711 2712 /* WA BB populates CONTEXT_ACTIVE cookie for primary context only */ 2713 if (!context_active(primary_lrc)) 2714 return xe_lrc_queue_timestamp(lrc); 2715 2716 /* WA BB populates engine id in PPHWSP of primary context only */ 2717 hwe = engine_id_to_hwe(primary_lrc->gt, xe_lrc_engine_id(primary_lrc)); 2718 if (!hwe) 2719 return xe_lrc_queue_timestamp(lrc); 2720 2721 if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos) 2722 return xe_lrc_queue_timestamp(lrc); 2723 2724 /* queue is active, so store the queue timestamp register */ 2725 reg_queue_ts = get_queue_timestamp(hwe); 2726 2727 /* double check queue and primary queue are both still active */ 2728 if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos || 2729 !context_active(primary_lrc)) 2730 return xe_lrc_queue_timestamp(lrc); 2731 2732 return reg_queue_ts; 2733 } 2734 2735 static u64 xe_lrc_update_multi_queue_timestamp(struct xe_lrc *lrc, u64 *old_ts) 2736 { 2737 *old_ts = lrc->queue_timestamp; 2738 lrc->queue_timestamp = xe_lrc_multi_queue_timestamp(lrc); 2739 2740 trace_xe_lrc_update_queue_timestamp(lrc, *old_ts); 2741 2742 return lrc->queue_timestamp; 2743 } 2744 2745 static u64 xe_lrc_context_timestamp(struct xe_lrc *lrc) 2746 { 2747 u64 reg_ts, new_ts = lrc->ctx_timestamp; 2748 u64 stored; 2749 2750 /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */ 2751 if (IS_SRIOV_VF(lrc_to_xe(lrc))) 2752 return xe_lrc_ctx_timestamp(lrc); 2753 2754 /* Safely read CTX_TIMESTAMP: Avoid TOCTOU on LRC-stored CONTEXT_ACTIVE sentinel */ 2755 stored = xe_lrc_ctx_timestamp(lrc); 2756 if (stored != CONTEXT_ACTIVE) 2757 return stored; 2758 2759 /* Context is active: read the live timestamp from the engine's MMIO register */ 2760 if (!get_ctx_timestamp(lrc, xe_lrc_engine_id(lrc), ®_ts)) 2761 new_ts = reg_ts; 2762 2763 /* If the context switched out prefer using the value 2764 * from context-save over the stale MMIO read. 2765 */ 2766 stored = xe_lrc_ctx_timestamp(lrc); 2767 if (stored != CONTEXT_ACTIVE) 2768 return stored; 2769 2770 return new_ts; 2771 } 2772 2773 static u64 xe_lrc_update_context_timestamp(struct xe_lrc *lrc, u64 *old_ts) 2774 { 2775 *old_ts = lrc->ctx_timestamp; 2776 lrc->ctx_timestamp = xe_lrc_context_timestamp(lrc); 2777 2778 trace_xe_lrc_update_timestamp(lrc, *old_ts); 2779 2780 return lrc->ctx_timestamp; 2781 } 2782 2783 /** 2784 * xe_lrc_timestamp() - Current lrc timestamp 2785 * @lrc: Pointer to the lrc. 2786 * 2787 * Return latest lrc timestamp. With support for active contexts/queues, the 2788 * calculation may be slightly racy, so follow a read-again logic to ensure that 2789 * the context/queue is still active before returning the right timestamp. 2790 * 2791 * Returns: New lrc timestamp value 2792 */ 2793 u64 xe_lrc_timestamp(struct xe_lrc *lrc) 2794 { 2795 if (xe_lrc_is_multi_queue(lrc)) 2796 return xe_lrc_multi_queue_timestamp(lrc); 2797 else 2798 return xe_lrc_context_timestamp(lrc); 2799 } 2800 2801 /** 2802 * xe_lrc_update_timestamp() - Update lrc timestamp 2803 * @lrc: Pointer to the lrc. 2804 * @old_ts: Old timestamp value 2805 * 2806 * Populate @old_ts with current saved lrc timestamp, read new lrc timestamp and 2807 * update saved value. 2808 * 2809 * Returns: New lrc timestamp value 2810 */ 2811 u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts) 2812 { 2813 if (xe_lrc_is_multi_queue(lrc)) 2814 return xe_lrc_update_multi_queue_timestamp(lrc, old_ts); 2815 else 2816 return xe_lrc_update_context_timestamp(lrc, old_ts); 2817 } 2818 2819 /** 2820 * xe_lrc_ring_is_idle() - LRC is idle 2821 * @lrc: Pointer to the lrc. 2822 * 2823 * Compare LRC ring head and tail to determine if idle. 2824 * 2825 * Return: True is ring is idle, False otherwise 2826 */ 2827 bool xe_lrc_ring_is_idle(struct xe_lrc *lrc) 2828 { 2829 return xe_lrc_ring_head(lrc) == xe_lrc_ring_tail(lrc); 2830 } 2831