1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2021 Intel Corporation
4 */
5
6 #include "xe_lrc.h"
7
8 #include <generated/xe_wa_oob.h>
9
10 #include <linux/ascii85.h>
11 #include <linux/panic.h>
12
13 #include "instructions/xe_mi_commands.h"
14 #include "instructions/xe_gfxpipe_commands.h"
15 #include "instructions/xe_gfx_state_commands.h"
16 #include "regs/xe_engine_regs.h"
17 #include "regs/xe_gt_regs.h"
18 #include "regs/xe_lrc_layout.h"
19 #include "xe_bb.h"
20 #include "xe_bo.h"
21 #include "xe_configfs.h"
22 #include "xe_device.h"
23 #include "xe_drm_client.h"
24 #include "xe_exec_queue.h"
25 #include "xe_exec_queue_types.h"
26 #include "xe_gt.h"
27 #include "xe_gt_clock.h"
28 #include "xe_gt_printk.h"
29 #include "xe_hw_fence.h"
30 #include "xe_map.h"
31 #include "xe_memirq.h"
32 #include "xe_mmio.h"
33 #include "xe_ring_ops.h"
34 #include "xe_sriov.h"
35 #include "xe_trace_lrc.h"
36 #include "xe_vm.h"
37 #include "xe_wa.h"
38
39 #define LRC_VALID BIT_ULL(0)
40 #define LRC_PRIVILEGE BIT_ULL(8)
41 #define LRC_ADDRESSING_MODE GENMASK_ULL(4, 3)
42 #define LRC_LEGACY_64B_CONTEXT 3
43
44 #define LRC_ENGINE_CLASS GENMASK_ULL(63, 61)
45 #define LRC_ENGINE_INSTANCE GENMASK_ULL(53, 48)
46
47 #define LRC_PPHWSP_SIZE SZ_4K
48 #define LRC_INDIRECT_CTX_BO_SIZE SZ_4K
49 #define LRC_INDIRECT_RING_STATE_SIZE SZ_4K
50
51 #define LRC_PRIORITY GENMASK_ULL(10, 9)
52 #define LRC_PRIORITY_LOW 0
53 #define LRC_PRIORITY_NORMAL 1
54 #define LRC_PRIORITY_HIGH 2
55
56 /*
57 * Layout of the LRC and associated data allocated as
58 * lrc->bo:
59 *
60 * Region Size
61 * +============================+=================================+ <- __xe_lrc_ring_offset()
62 * | Ring | ring_size, see |
63 * | | xe_lrc_init() |
64 * +============================+=================================+ <- __xe_lrc_pphwsp_offset()
65 * | PPHWSP (includes SW state) | 4K |
66 * +----------------------------+---------------------------------+ <- __xe_lrc_regs_offset()
67 * | Engine Context Image | n * 4K, see |
68 * | | xe_gt_lrc_size() |
69 * +----------------------------+---------------------------------+ <- __xe_lrc_indirect_ring_offset()
70 * | Indirect Ring State Page | 0 or 4k, see |
71 * | | XE_LRC_FLAG_INDIRECT_RING_STATE |
72 * +============================+=================================+ <- __xe_lrc_indirect_ctx_offset()
73 * | Indirect Context Page | 0 or 4k, see |
74 * | | XE_LRC_FLAG_INDIRECT_CTX |
75 * +============================+=================================+ <- __xe_lrc_wa_bb_offset()
76 * | WA BB Per Ctx | 4k |
77 * +============================+=================================+ <- xe_bo_size(lrc->bo)
78 */
79
80 static struct xe_device *
lrc_to_xe(struct xe_lrc * lrc)81 lrc_to_xe(struct xe_lrc *lrc)
82 {
83 return gt_to_xe(lrc->fence_ctx.gt);
84 }
85
86 static bool
gt_engine_needs_indirect_ctx(struct xe_gt * gt,enum xe_engine_class class)87 gt_engine_needs_indirect_ctx(struct xe_gt *gt, enum xe_engine_class class)
88 {
89 struct xe_device *xe = gt_to_xe(gt);
90
91 if (XE_GT_WA(gt, 16010904313) &&
92 (class == XE_ENGINE_CLASS_RENDER ||
93 class == XE_ENGINE_CLASS_COMPUTE))
94 return true;
95
96 if (xe_configfs_get_ctx_restore_mid_bb(to_pci_dev(xe->drm.dev),
97 class, NULL))
98 return true;
99
100 if (gt->ring_ops[class]->emit_aux_table_inv)
101 return true;
102
103 return false;
104 }
105
106 /**
107 * xe_gt_lrc_hang_replay_size() - Hang replay size
108 * @gt: The GT
109 * @class: Hardware engine class
110 *
111 * Determine size of GPU hang replay state for a GT and hardware engine class.
112 *
113 * Return: Size of GPU hang replay size
114 */
xe_gt_lrc_hang_replay_size(struct xe_gt * gt,enum xe_engine_class class)115 size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class)
116 {
117 struct xe_device *xe = gt_to_xe(gt);
118 size_t size = 0;
119
120 /* Engine context image */
121 switch (class) {
122 case XE_ENGINE_CLASS_RENDER:
123 if (GRAPHICS_VERx100(xe) >= 3510)
124 size += 7 * SZ_4K;
125 else if (GRAPHICS_VER(xe) >= 20)
126 size += 3 * SZ_4K;
127 else
128 size += 13 * SZ_4K;
129 break;
130 case XE_ENGINE_CLASS_COMPUTE:
131 if (GRAPHICS_VERx100(xe) >= 3510)
132 size += 5 * SZ_4K;
133 else if (GRAPHICS_VER(xe) >= 20)
134 size += 2 * SZ_4K;
135 else
136 size += 13 * SZ_4K;
137 break;
138 default:
139 WARN(1, "Unknown engine class: %d", class);
140 fallthrough;
141 case XE_ENGINE_CLASS_COPY:
142 case XE_ENGINE_CLASS_VIDEO_DECODE:
143 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
144 case XE_ENGINE_CLASS_OTHER:
145 size += 1 * SZ_4K;
146 }
147
148 return size;
149 }
150
xe_gt_lrc_size(struct xe_gt * gt,enum xe_engine_class class)151 size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class)
152 {
153 size_t size = xe_gt_lrc_hang_replay_size(gt, class);
154
155 /* Add indirect ring state page */
156 if (xe_gt_has_indirect_ring_state(gt))
157 size += LRC_INDIRECT_RING_STATE_SIZE;
158
159 return size + LRC_PPHWSP_SIZE;
160 }
161
162 /*
163 * The per-platform tables are u8-encoded in @data. Decode @data and set the
164 * addresses' offset and commands in @regs. The following encoding is used
165 * for each byte. There are 2 steps: decoding commands and decoding addresses.
166 *
167 * Commands:
168 * [7]: create NOPs - number of NOPs are set in lower bits
169 * [6]: When creating MI_LOAD_REGISTER_IMM command, allow to set
170 * MI_LRI_FORCE_POSTED
171 * [5:0]: Number of NOPs or registers to set values to in case of
172 * MI_LOAD_REGISTER_IMM
173 *
174 * Addresses: these are decoded after a MI_LOAD_REGISTER_IMM command by "count"
175 * number of registers. They are set by using the REG/REG16 macros: the former
176 * is used for offsets smaller than 0x200 while the latter is for values bigger
177 * than that. Those macros already set all the bits documented below correctly:
178 *
179 * [7]: When a register offset needs more than 6 bits, use additional bytes, to
180 * follow, for the lower bits
181 * [6:0]: Register offset, without considering the engine base.
182 *
183 * This function only tweaks the commands and register offsets. Values are not
184 * filled out.
185 */
set_offsets(u32 * regs,const u8 * data,const struct xe_hw_engine * hwe)186 static void set_offsets(u32 *regs,
187 const u8 *data,
188 const struct xe_hw_engine *hwe)
189 #define NOP(x) (BIT(7) | (x))
190 #define LRI(count, flags) ((flags) << 6 | (count) | \
191 BUILD_BUG_ON_ZERO(count >= BIT(6)))
192 #define POSTED BIT(0)
193 #define REG(x) (((x) >> 2) | BUILD_BUG_ON_ZERO(x >= 0x200))
194 #define REG16(x) \
195 (((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \
196 (((x) >> 2) & 0x7f)
197 {
198 const u32 base = hwe->mmio_base;
199
200 while (*data) {
201 u8 count, flags;
202
203 if (*data & BIT(7)) { /* skip */
204 count = *data++ & ~BIT(7);
205 regs += count;
206 continue;
207 }
208
209 count = *data & 0x3f;
210 flags = *data >> 6;
211 data++;
212
213 *regs = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(count);
214 if (flags & POSTED)
215 *regs |= MI_LRI_FORCE_POSTED;
216 *regs |= MI_LRI_LRM_CS_MMIO;
217 regs++;
218
219 xe_gt_assert(hwe->gt, count);
220 do {
221 u32 offset = 0;
222 u8 v;
223
224 do {
225 v = *data++;
226 offset <<= 7;
227 offset |= v & ~BIT(7);
228 } while (v & BIT(7));
229
230 regs[0] = base + (offset << 2);
231 regs += 2;
232 } while (--count);
233 }
234
235 *regs = MI_BATCH_BUFFER_END | BIT(0);
236 }
237
238 static const u8 gen12_xcs_offsets[] = {
239 NOP(1),
240 LRI(13, POSTED),
241 REG16(0x244),
242 REG(0x034),
243 REG(0x030),
244 REG(0x038),
245 REG(0x03c),
246 REG(0x168),
247 REG(0x140),
248 REG(0x110),
249 REG(0x1c0),
250 REG(0x1c4),
251 REG(0x1c8),
252 REG(0x180),
253 REG16(0x2b4),
254
255 NOP(5),
256 LRI(9, POSTED),
257 REG16(0x3a8),
258 REG16(0x28c),
259 REG16(0x288),
260 REG16(0x284),
261 REG16(0x280),
262 REG16(0x27c),
263 REG16(0x278),
264 REG16(0x274),
265 REG16(0x270),
266
267 0
268 };
269
270 static const u8 dg2_xcs_offsets[] = {
271 NOP(1),
272 LRI(15, POSTED),
273 REG16(0x244),
274 REG(0x034),
275 REG(0x030),
276 REG(0x038),
277 REG(0x03c),
278 REG(0x168),
279 REG(0x140),
280 REG(0x110),
281 REG(0x1c0),
282 REG(0x1c4),
283 REG(0x1c8),
284 REG(0x180),
285 REG16(0x2b4),
286 REG(0x120),
287 REG(0x124),
288
289 NOP(1),
290 LRI(9, POSTED),
291 REG16(0x3a8),
292 REG16(0x28c),
293 REG16(0x288),
294 REG16(0x284),
295 REG16(0x280),
296 REG16(0x27c),
297 REG16(0x278),
298 REG16(0x274),
299 REG16(0x270),
300
301 0
302 };
303
304 static const u8 gen12_rcs_offsets[] = {
305 NOP(1),
306 LRI(13, POSTED),
307 REG16(0x244),
308 REG(0x034),
309 REG(0x030),
310 REG(0x038),
311 REG(0x03c),
312 REG(0x168),
313 REG(0x140),
314 REG(0x110),
315 REG(0x1c0),
316 REG(0x1c4),
317 REG(0x1c8),
318 REG(0x180),
319 REG16(0x2b4),
320
321 NOP(5),
322 LRI(9, POSTED),
323 REG16(0x3a8),
324 REG16(0x28c),
325 REG16(0x288),
326 REG16(0x284),
327 REG16(0x280),
328 REG16(0x27c),
329 REG16(0x278),
330 REG16(0x274),
331 REG16(0x270),
332
333 LRI(3, POSTED),
334 REG(0x1b0),
335 REG16(0x5a8),
336 REG16(0x5ac),
337
338 NOP(6),
339 LRI(1, 0),
340 REG(0x0c8),
341 NOP(3 + 9 + 1),
342
343 LRI(51, POSTED),
344 REG16(0x588),
345 REG16(0x588),
346 REG16(0x588),
347 REG16(0x588),
348 REG16(0x588),
349 REG16(0x588),
350 REG(0x028),
351 REG(0x09c),
352 REG(0x0c0),
353 REG(0x178),
354 REG(0x17c),
355 REG16(0x358),
356 REG(0x170),
357 REG(0x150),
358 REG(0x154),
359 REG(0x158),
360 REG16(0x41c),
361 REG16(0x600),
362 REG16(0x604),
363 REG16(0x608),
364 REG16(0x60c),
365 REG16(0x610),
366 REG16(0x614),
367 REG16(0x618),
368 REG16(0x61c),
369 REG16(0x620),
370 REG16(0x624),
371 REG16(0x628),
372 REG16(0x62c),
373 REG16(0x630),
374 REG16(0x634),
375 REG16(0x638),
376 REG16(0x63c),
377 REG16(0x640),
378 REG16(0x644),
379 REG16(0x648),
380 REG16(0x64c),
381 REG16(0x650),
382 REG16(0x654),
383 REG16(0x658),
384 REG16(0x65c),
385 REG16(0x660),
386 REG16(0x664),
387 REG16(0x668),
388 REG16(0x66c),
389 REG16(0x670),
390 REG16(0x674),
391 REG16(0x678),
392 REG16(0x67c),
393 REG(0x068),
394 REG(0x084),
395 NOP(1),
396
397 0
398 };
399
400 static const u8 xehp_rcs_offsets[] = {
401 NOP(1),
402 LRI(13, POSTED),
403 REG16(0x244),
404 REG(0x034),
405 REG(0x030),
406 REG(0x038),
407 REG(0x03c),
408 REG(0x168),
409 REG(0x140),
410 REG(0x110),
411 REG(0x1c0),
412 REG(0x1c4),
413 REG(0x1c8),
414 REG(0x180),
415 REG16(0x2b4),
416
417 NOP(5),
418 LRI(9, POSTED),
419 REG16(0x3a8),
420 REG16(0x28c),
421 REG16(0x288),
422 REG16(0x284),
423 REG16(0x280),
424 REG16(0x27c),
425 REG16(0x278),
426 REG16(0x274),
427 REG16(0x270),
428
429 LRI(3, POSTED),
430 REG(0x1b0),
431 REG16(0x5a8),
432 REG16(0x5ac),
433
434 NOP(6),
435 LRI(1, 0),
436 REG(0x0c8),
437
438 0
439 };
440
441 static const u8 dg2_rcs_offsets[] = {
442 NOP(1),
443 LRI(15, POSTED),
444 REG16(0x244),
445 REG(0x034),
446 REG(0x030),
447 REG(0x038),
448 REG(0x03c),
449 REG(0x168),
450 REG(0x140),
451 REG(0x110),
452 REG(0x1c0),
453 REG(0x1c4),
454 REG(0x1c8),
455 REG(0x180),
456 REG16(0x2b4),
457 REG(0x120),
458 REG(0x124),
459
460 NOP(1),
461 LRI(9, POSTED),
462 REG16(0x3a8),
463 REG16(0x28c),
464 REG16(0x288),
465 REG16(0x284),
466 REG16(0x280),
467 REG16(0x27c),
468 REG16(0x278),
469 REG16(0x274),
470 REG16(0x270),
471
472 LRI(3, POSTED),
473 REG(0x1b0),
474 REG16(0x5a8),
475 REG16(0x5ac),
476
477 NOP(6),
478 LRI(1, 0),
479 REG(0x0c8),
480
481 0
482 };
483
484 static const u8 mtl_rcs_offsets[] = {
485 NOP(1),
486 LRI(15, POSTED),
487 REG16(0x244),
488 REG(0x034),
489 REG(0x030),
490 REG(0x038),
491 REG(0x03c),
492 REG(0x168),
493 REG(0x140),
494 REG(0x110),
495 REG(0x1c0),
496 REG(0x1c4),
497 REG(0x1c8),
498 REG(0x180),
499 REG16(0x2b4),
500 REG(0x120),
501 REG(0x124),
502
503 NOP(1),
504 LRI(9, POSTED),
505 REG16(0x3a8),
506 REG16(0x28c),
507 REG16(0x288),
508 REG16(0x284),
509 REG16(0x280),
510 REG16(0x27c),
511 REG16(0x278),
512 REG16(0x274),
513 REG16(0x270),
514
515 NOP(2),
516 LRI(2, POSTED),
517 REG16(0x5a8),
518 REG16(0x5ac),
519
520 NOP(6),
521 LRI(1, 0),
522 REG(0x0c8),
523
524 0
525 };
526
527 #define XE2_CTX_COMMON \
528 NOP(1), /* [0x00] */ \
529 LRI(15, POSTED), /* [0x01] */ \
530 REG16(0x244), /* [0x02] CTXT_SR_CTL */ \
531 REG(0x034), /* [0x04] RING_BUFFER_HEAD */ \
532 REG(0x030), /* [0x06] RING_BUFFER_TAIL */ \
533 REG(0x038), /* [0x08] RING_BUFFER_START */ \
534 REG(0x03c), /* [0x0a] RING_BUFFER_CONTROL */ \
535 REG(0x168), /* [0x0c] BB_ADDR_UDW */ \
536 REG(0x140), /* [0x0e] BB_ADDR */ \
537 REG(0x110), /* [0x10] BB_STATE */ \
538 REG(0x1c0), /* [0x12] BB_PER_CTX_PTR */ \
539 REG(0x1c4), /* [0x14] RCS_INDIRECT_CTX */ \
540 REG(0x1c8), /* [0x16] RCS_INDIRECT_CTX_OFFSET */ \
541 REG(0x180), /* [0x18] CCID */ \
542 REG16(0x2b4), /* [0x1a] SEMAPHORE_TOKEN */ \
543 REG(0x120), /* [0x1c] PRT_BB_STATE */ \
544 REG(0x124), /* [0x1e] PRT_BB_STATE_UDW */ \
545 \
546 NOP(1), /* [0x20] */ \
547 LRI(9, POSTED), /* [0x21] */ \
548 REG16(0x3a8), /* [0x22] CTX_TIMESTAMP */ \
549 REG16(0x3ac), /* [0x24] CTX_TIMESTAMP_UDW */ \
550 REG(0x108), /* [0x26] INDIRECT_RING_STATE */ \
551 REG16(0x284), /* [0x28] dummy reg */ \
552 REG16(0x280), /* [0x2a] CS_ACC_CTR_THOLD */ \
553 REG16(0x27c), /* [0x2c] CS_CTX_SYS_PASID */ \
554 REG16(0x278), /* [0x2e] CS_CTX_ASID */ \
555 REG16(0x274), /* [0x30] PTBP_UDW */ \
556 REG16(0x270) /* [0x32] PTBP_LDW */
557
558 static const u8 xe2_rcs_offsets[] = {
559 XE2_CTX_COMMON,
560
561 NOP(2), /* [0x34] */
562 LRI(2, POSTED), /* [0x36] */
563 REG16(0x5a8), /* [0x37] CONTEXT_SCHEDULING_ATTRIBUTES */
564 REG16(0x5ac), /* [0x39] PREEMPTION_STATUS */
565
566 NOP(6), /* [0x41] */
567 LRI(1, 0), /* [0x47] */
568 REG(0x0c8), /* [0x48] R_PWR_CLK_STATE */
569
570 0
571 };
572
573 static const u8 xe2_bcs_offsets[] = {
574 XE2_CTX_COMMON,
575
576 NOP(4 + 8 + 1), /* [0x34] */
577 LRI(2, POSTED), /* [0x41] */
578 REG16(0x200), /* [0x42] BCS_SWCTRL */
579 REG16(0x204), /* [0x44] BLIT_CCTL */
580
581 0
582 };
583
584 static const u8 xe2_xcs_offsets[] = {
585 XE2_CTX_COMMON,
586
587 0
588 };
589
590 static const u8 xe2_indirect_ring_state_offsets[] = {
591 NOP(1), /* [0x00] */
592 LRI(5, POSTED), /* [0x01] */
593 REG(0x034), /* [0x02] RING_BUFFER_HEAD */
594 REG(0x030), /* [0x04] RING_BUFFER_TAIL */
595 REG(0x038), /* [0x06] RING_BUFFER_START */
596 REG(0x048), /* [0x08] RING_BUFFER_START_UDW */
597 REG(0x03c), /* [0x0a] RING_BUFFER_CONTROL */
598
599 NOP(5), /* [0x0c] */
600 LRI(9, POSTED), /* [0x11] */
601 REG(0x168), /* [0x12] BB_ADDR_UDW */
602 REG(0x140), /* [0x14] BB_ADDR */
603 REG(0x110), /* [0x16] BB_STATE */
604 REG16(0x588), /* [0x18] BB_STACK_WRITE_PORT */
605 REG16(0x588), /* [0x20] BB_STACK_WRITE_PORT */
606 REG16(0x588), /* [0x22] BB_STACK_WRITE_PORT */
607 REG16(0x588), /* [0x24] BB_STACK_WRITE_PORT */
608 REG16(0x588), /* [0x26] BB_STACK_WRITE_PORT */
609 REG16(0x588), /* [0x28] BB_STACK_WRITE_PORT */
610
611 NOP(12), /* [0x00] */
612
613 0
614 };
615
616 #undef REG16
617 #undef REG
618 #undef LRI
619 #undef NOP
620
reg_offsets(struct xe_device * xe,enum xe_engine_class class)621 static const u8 *reg_offsets(struct xe_device *xe, enum xe_engine_class class)
622 {
623 if (class == XE_ENGINE_CLASS_RENDER) {
624 if (GRAPHICS_VER(xe) >= 20)
625 return xe2_rcs_offsets;
626 else if (GRAPHICS_VERx100(xe) >= 1270)
627 return mtl_rcs_offsets;
628 else if (GRAPHICS_VERx100(xe) >= 1255)
629 return dg2_rcs_offsets;
630 else if (GRAPHICS_VERx100(xe) >= 1250)
631 return xehp_rcs_offsets;
632 else
633 return gen12_rcs_offsets;
634 } else if (class == XE_ENGINE_CLASS_COPY) {
635 if (GRAPHICS_VER(xe) >= 20)
636 return xe2_bcs_offsets;
637 else
638 return gen12_xcs_offsets;
639 } else {
640 if (GRAPHICS_VER(xe) >= 20)
641 return xe2_xcs_offsets;
642 else if (GRAPHICS_VERx100(xe) >= 1255)
643 return dg2_xcs_offsets;
644 else
645 return gen12_xcs_offsets;
646 }
647 }
648
set_context_control(u32 * regs,struct xe_hw_engine * hwe)649 static void set_context_control(u32 *regs, struct xe_hw_engine *hwe)
650 {
651 regs[CTX_CONTEXT_CONTROL] = REG_MASKED_FIELD_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
652 CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
653
654 if (xe_gt_has_indirect_ring_state(hwe->gt))
655 regs[CTX_CONTEXT_CONTROL] |=
656 REG_MASKED_FIELD_ENABLE(CTX_CTRL_INDIRECT_RING_STATE_ENABLE);
657 }
658
set_memory_based_intr(u32 * regs,struct xe_hw_engine * hwe)659 static void set_memory_based_intr(u32 *regs, struct xe_hw_engine *hwe)
660 {
661 struct xe_memirq *memirq = >_to_tile(hwe->gt)->memirq;
662 struct xe_device *xe = gt_to_xe(hwe->gt);
663 u8 num_regs;
664
665 if (!xe_device_uses_memirq(xe))
666 return;
667
668 regs[CTX_LRM_INT_MASK_ENABLE] = MI_LOAD_REGISTER_MEM |
669 MI_LRI_LRM_CS_MMIO | MI_LRM_USE_GGTT;
670 regs[CTX_INT_MASK_ENABLE_REG] = RING_IMR(0).addr;
671 regs[CTX_INT_MASK_ENABLE_PTR] = xe_memirq_enable_ptr(memirq);
672
673 num_regs = xe_device_has_msix(xe) ? 3 : 2;
674 regs[CTX_LRI_INT_REPORT_PTR] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(num_regs) |
675 MI_LRI_LRM_CS_MMIO | MI_LRI_FORCE_POSTED;
676 regs[CTX_INT_STATUS_REPORT_REG] = RING_INT_STATUS_RPT_PTR(0).addr;
677 regs[CTX_INT_STATUS_REPORT_PTR] = xe_memirq_status_ptr(memirq, hwe);
678 regs[CTX_INT_SRC_REPORT_REG] = RING_INT_SRC_RPT_PTR(0).addr;
679 regs[CTX_INT_SRC_REPORT_PTR] = xe_memirq_source_ptr(memirq, hwe);
680
681 if (xe_device_has_msix(xe)) {
682 regs[CTX_CS_INT_VEC_REG] = CS_INT_VEC(0).addr;
683 /* CTX_CS_INT_VEC_DATA will be set in xe_lrc_init */
684 }
685 }
686
xe_lrc_has_indirect_ring_state(struct xe_lrc * lrc)687 static inline bool xe_lrc_has_indirect_ring_state(struct xe_lrc *lrc)
688 {
689 return lrc->flags & XE_LRC_FLAG_INDIRECT_RING_STATE;
690 }
691
__xe_lrc_ring_offset(struct xe_lrc * lrc)692 static inline u32 __xe_lrc_ring_offset(struct xe_lrc *lrc)
693 {
694 return 0;
695 }
696
xe_lrc_pphwsp_offset(struct xe_lrc * lrc)697 u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc)
698 {
699 return lrc->ring.size;
700 }
701
702 /* Make the magic macros work */
703 #define __xe_lrc_pphwsp_offset xe_lrc_pphwsp_offset
704 #define __xe_lrc_regs_offset xe_lrc_regs_offset
705
706 #define LRC_CTX_JOB_TIMESTAMP_OFFSET 512
707 #define LRC_ENGINE_ID_PPHWSP_OFFSET 1024
708 #define LRC_PARALLEL_PPHWSP_OFFSET 2048
709
710 #define LRC_SEQNO_OFFSET 0
711 #define LRC_START_SEQNO_OFFSET (LRC_SEQNO_OFFSET + 8)
712
xe_lrc_regs_offset(struct xe_lrc * lrc)713 u32 xe_lrc_regs_offset(struct xe_lrc *lrc)
714 {
715 return xe_lrc_pphwsp_offset(lrc) + LRC_PPHWSP_SIZE;
716 }
717
718 /**
719 * xe_lrc_reg_size() - Get size of the LRC registers area within queues
720 * @xe: the &xe_device struct instance
721 *
722 * Returns: Size of the LRC registers area for current platform
723 */
xe_lrc_reg_size(struct xe_device * xe)724 size_t xe_lrc_reg_size(struct xe_device *xe)
725 {
726 if (GRAPHICS_VERx100(xe) >= 1250)
727 return 96 * sizeof(u32);
728 else
729 return 80 * sizeof(u32);
730 }
731
732 /**
733 * xe_lrc_engine_state_size() - Get size of the engine state within LRC
734 * @gt: the &xe_gt struct instance
735 * @class: Hardware engine class
736 *
737 * Returns: Size of the engine state
738 */
xe_lrc_engine_state_size(struct xe_gt * gt,enum xe_engine_class class)739 size_t xe_lrc_engine_state_size(struct xe_gt *gt, enum xe_engine_class class)
740 {
741 return xe_gt_lrc_hang_replay_size(gt, class) - xe_lrc_reg_size(gt_to_xe(gt));
742 }
743
__xe_lrc_seqno_offset(struct xe_lrc * lrc)744 static inline u32 __xe_lrc_seqno_offset(struct xe_lrc *lrc)
745 {
746 return LRC_SEQNO_OFFSET;
747 }
748
__xe_lrc_start_seqno_offset(struct xe_lrc * lrc)749 static inline u32 __xe_lrc_start_seqno_offset(struct xe_lrc *lrc)
750 {
751 return LRC_START_SEQNO_OFFSET;
752 }
753
__xe_lrc_ctx_job_timestamp_offset(struct xe_lrc * lrc)754 static u32 __xe_lrc_ctx_job_timestamp_offset(struct xe_lrc *lrc)
755 {
756 /* This is stored in the driver-defined portion of PPHWSP */
757 return xe_lrc_pphwsp_offset(lrc) + LRC_CTX_JOB_TIMESTAMP_OFFSET;
758 }
759
__xe_lrc_parallel_offset(struct xe_lrc * lrc)760 static inline u32 __xe_lrc_parallel_offset(struct xe_lrc *lrc)
761 {
762 /* The parallel is stored in the driver-defined portion of PPHWSP */
763 return xe_lrc_pphwsp_offset(lrc) + LRC_PARALLEL_PPHWSP_OFFSET;
764 }
765
__xe_lrc_engine_id_offset(struct xe_lrc * lrc)766 static inline u32 __xe_lrc_engine_id_offset(struct xe_lrc *lrc)
767 {
768 return xe_lrc_pphwsp_offset(lrc) + LRC_ENGINE_ID_PPHWSP_OFFSET;
769 }
770
__xe_lrc_ctx_timestamp_offset(struct xe_lrc * lrc)771 static u32 __xe_lrc_ctx_timestamp_offset(struct xe_lrc *lrc)
772 {
773 return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP * sizeof(u32);
774 }
775
__xe_lrc_ctx_timestamp_udw_offset(struct xe_lrc * lrc)776 static u32 __xe_lrc_ctx_timestamp_udw_offset(struct xe_lrc *lrc)
777 {
778 return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP_UDW * sizeof(u32);
779 }
780
__xe_lrc_queue_timestamp_offset(struct xe_lrc * lrc)781 static u32 __xe_lrc_queue_timestamp_offset(struct xe_lrc *lrc)
782 {
783 return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP * sizeof(u32);
784 }
785
__xe_lrc_queue_timestamp_udw_offset(struct xe_lrc * lrc)786 static u32 __xe_lrc_queue_timestamp_udw_offset(struct xe_lrc *lrc)
787 {
788 return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP_UDW * sizeof(u32);
789 }
790
__xe_lrc_indirect_ring_offset(struct xe_lrc * lrc)791 static inline u32 __xe_lrc_indirect_ring_offset(struct xe_lrc *lrc)
792 {
793 u32 offset = xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE -
794 LRC_INDIRECT_RING_STATE_SIZE;
795
796 if (lrc->flags & XE_LRC_FLAG_INDIRECT_CTX)
797 offset -= LRC_INDIRECT_CTX_BO_SIZE;
798
799 return offset;
800 }
801
__xe_lrc_indirect_ctx_offset(struct xe_lrc * lrc)802 static inline u32 __xe_lrc_indirect_ctx_offset(struct xe_lrc *lrc)
803 {
804 return xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE - LRC_INDIRECT_CTX_BO_SIZE;
805 }
806
__xe_lrc_wa_bb_offset(struct xe_lrc * lrc)807 static inline u32 __xe_lrc_wa_bb_offset(struct xe_lrc *lrc)
808 {
809 return xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE;
810 }
811
812 #define DECL_MAP_ADDR_HELPERS(elem, bo_expr) \
813 static inline struct iosys_map __xe_lrc_##elem##_map(struct xe_lrc *lrc) \
814 { \
815 struct xe_bo *bo = (bo_expr); \
816 struct iosys_map map = bo->vmap; \
817 \
818 xe_assert(lrc_to_xe(lrc), !iosys_map_is_null(&map)); \
819 iosys_map_incr(&map, __xe_lrc_##elem##_offset(lrc)); \
820 return map; \
821 } \
822 static inline u32 __maybe_unused __xe_lrc_##elem##_ggtt_addr(struct xe_lrc *lrc) \
823 { \
824 struct xe_bo *bo = (bo_expr); \
825 \
826 return xe_bo_ggtt_addr(bo) + __xe_lrc_##elem##_offset(lrc); \
827 } \
828
829 DECL_MAP_ADDR_HELPERS(ring, lrc->bo)
830 DECL_MAP_ADDR_HELPERS(pphwsp, lrc->bo)
831 DECL_MAP_ADDR_HELPERS(seqno, lrc->seqno_bo)
832 DECL_MAP_ADDR_HELPERS(regs, lrc->bo)
833 DECL_MAP_ADDR_HELPERS(start_seqno, lrc->seqno_bo)
834 DECL_MAP_ADDR_HELPERS(ctx_job_timestamp, lrc->bo)
835 DECL_MAP_ADDR_HELPERS(ctx_timestamp, lrc->bo)
836 DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo)
837 DECL_MAP_ADDR_HELPERS(parallel, lrc->bo)
838 DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo)
839 DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo)
840 DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo)
841 DECL_MAP_ADDR_HELPERS(queue_timestamp_udw, lrc->bo)
842
843 #undef DECL_MAP_ADDR_HELPERS
844
845 /**
846 * xe_lrc_ctx_timestamp_ggtt_addr() - Get ctx timestamp GGTT address
847 * @lrc: Pointer to the lrc.
848 *
849 * Returns: ctx timestamp GGTT address
850 */
xe_lrc_ctx_timestamp_ggtt_addr(struct xe_lrc * lrc)851 u32 xe_lrc_ctx_timestamp_ggtt_addr(struct xe_lrc *lrc)
852 {
853 return __xe_lrc_ctx_timestamp_ggtt_addr(lrc);
854 }
855
856 /**
857 * xe_lrc_ctx_timestamp_udw_ggtt_addr() - Get ctx timestamp udw GGTT address
858 * @lrc: Pointer to the lrc.
859 *
860 * Returns: ctx timestamp udw GGTT address
861 */
xe_lrc_ctx_timestamp_udw_ggtt_addr(struct xe_lrc * lrc)862 u32 xe_lrc_ctx_timestamp_udw_ggtt_addr(struct xe_lrc *lrc)
863 {
864 return __xe_lrc_ctx_timestamp_udw_ggtt_addr(lrc);
865 }
866
867 /**
868 * xe_lrc_ctx_timestamp() - Read ctx timestamp value
869 * @lrc: Pointer to the lrc.
870 *
871 * Returns: ctx timestamp value
872 */
xe_lrc_ctx_timestamp(struct xe_lrc * lrc)873 static u64 xe_lrc_ctx_timestamp(struct xe_lrc *lrc)
874 {
875 struct xe_device *xe = lrc_to_xe(lrc);
876 struct iosys_map map;
877 u32 ldw, udw = 0;
878
879 map = __xe_lrc_ctx_timestamp_map(lrc);
880 ldw = xe_map_read32(xe, &map);
881
882 if (xe->info.has_64bit_timestamp) {
883 map = __xe_lrc_ctx_timestamp_udw_map(lrc);
884 udw = xe_map_read32(xe, &map);
885 }
886
887 return (u64)udw << 32 | ldw;
888 }
889
890 /**
891 * xe_lrc_queue_timestamp() - Read queue timestamp value
892 * @lrc: Pointer to the lrc.
893 *
894 * Returns: queue timestamp value
895 */
xe_lrc_queue_timestamp(struct xe_lrc * lrc)896 static u64 xe_lrc_queue_timestamp(struct xe_lrc *lrc)
897 {
898 struct xe_device *xe = lrc_to_xe(lrc);
899 struct iosys_map map;
900 u32 ldw, udw = 0;
901
902 xe_assert(xe, xe_lrc_is_multi_queue(lrc));
903
904 map = __xe_lrc_queue_timestamp_map(lrc);
905 ldw = xe_map_read32(xe, &map);
906
907 map = __xe_lrc_queue_timestamp_udw_map(lrc);
908 udw = xe_map_read32(xe, &map);
909
910 return (u64)udw << 32 | ldw;
911 }
912
913 /**
914 * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address
915 * @lrc: Pointer to the lrc.
916 *
917 * Returns: ctx timestamp job GGTT address
918 */
xe_lrc_ctx_job_timestamp_ggtt_addr(struct xe_lrc * lrc)919 u32 xe_lrc_ctx_job_timestamp_ggtt_addr(struct xe_lrc *lrc)
920 {
921 return __xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);
922 }
923
924 /**
925 * xe_lrc_ctx_job_timestamp() - Read ctx job timestamp value
926 * @lrc: Pointer to the lrc.
927 *
928 * Returns: ctx timestamp job value
929 */
xe_lrc_ctx_job_timestamp(struct xe_lrc * lrc)930 u32 xe_lrc_ctx_job_timestamp(struct xe_lrc *lrc)
931 {
932 struct xe_device *xe = lrc_to_xe(lrc);
933 struct iosys_map map;
934
935 map = __xe_lrc_ctx_job_timestamp_map(lrc);
936 return xe_map_read32(xe, &map);
937 }
938
xe_lrc_ggtt_addr(struct xe_lrc * lrc)939 u32 xe_lrc_ggtt_addr(struct xe_lrc *lrc)
940 {
941 return __xe_lrc_pphwsp_ggtt_addr(lrc);
942 }
943
xe_lrc_indirect_ring_ggtt_addr(struct xe_lrc * lrc)944 u32 xe_lrc_indirect_ring_ggtt_addr(struct xe_lrc *lrc)
945 {
946 if (!xe_lrc_has_indirect_ring_state(lrc))
947 return 0;
948
949 return __xe_lrc_indirect_ring_ggtt_addr(lrc);
950 }
951
xe_lrc_read_indirect_ctx_reg(struct xe_lrc * lrc,int reg_nr)952 static u32 xe_lrc_read_indirect_ctx_reg(struct xe_lrc *lrc, int reg_nr)
953 {
954 struct xe_device *xe = lrc_to_xe(lrc);
955 struct iosys_map map;
956
957 map = __xe_lrc_indirect_ring_map(lrc);
958 iosys_map_incr(&map, reg_nr * sizeof(u32));
959 return xe_map_read32(xe, &map);
960 }
961
xe_lrc_write_indirect_ctx_reg(struct xe_lrc * lrc,int reg_nr,u32 val)962 static void xe_lrc_write_indirect_ctx_reg(struct xe_lrc *lrc,
963 int reg_nr, u32 val)
964 {
965 struct xe_device *xe = lrc_to_xe(lrc);
966 struct iosys_map map;
967
968 map = __xe_lrc_indirect_ring_map(lrc);
969 iosys_map_incr(&map, reg_nr * sizeof(u32));
970 xe_map_write32(xe, &map, val);
971 }
972
xe_lrc_read_ctx_reg(struct xe_lrc * lrc,int reg_nr)973 u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr)
974 {
975 struct xe_device *xe = lrc_to_xe(lrc);
976 struct iosys_map map;
977
978 map = __xe_lrc_regs_map(lrc);
979 iosys_map_incr(&map, reg_nr * sizeof(u32));
980 return xe_map_read32(xe, &map);
981 }
982
xe_lrc_write_ctx_reg(struct xe_lrc * lrc,int reg_nr,u32 val)983 void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val)
984 {
985 struct xe_device *xe = lrc_to_xe(lrc);
986 struct iosys_map map;
987
988 map = __xe_lrc_regs_map(lrc);
989 iosys_map_incr(&map, reg_nr * sizeof(u32));
990 xe_map_write32(xe, &map, val);
991 }
992
empty_lrc_data(struct xe_hw_engine * hwe)993 static void *empty_lrc_data(struct xe_hw_engine *hwe)
994 {
995 struct xe_gt *gt = hwe->gt;
996 void *data;
997 u32 *regs;
998
999 data = kzalloc(xe_gt_lrc_size(gt, hwe->class), GFP_KERNEL);
1000 if (!data)
1001 return NULL;
1002
1003 /* 1st page: Per-Process of HW status Page */
1004 regs = data + LRC_PPHWSP_SIZE;
1005 set_offsets(regs, reg_offsets(gt_to_xe(gt), hwe->class), hwe);
1006 set_context_control(regs, hwe);
1007 set_memory_based_intr(regs, hwe);
1008 if (xe_gt_has_indirect_ring_state(gt)) {
1009 regs = data + xe_gt_lrc_size(gt, hwe->class) -
1010 LRC_INDIRECT_RING_STATE_SIZE;
1011 set_offsets(regs, xe2_indirect_ring_state_offsets, hwe);
1012 }
1013
1014 return data;
1015 }
1016
1017 /**
1018 * xe_default_lrc_update_memirq_regs_with_address - Re-compute GGTT references in default LRC
1019 * of given engine.
1020 * @hwe: the &xe_hw_engine struct instance
1021 */
xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine * hwe)1022 void xe_default_lrc_update_memirq_regs_with_address(struct xe_hw_engine *hwe)
1023 {
1024 struct xe_gt *gt = hwe->gt;
1025 u32 *regs;
1026
1027 if (!gt->default_lrc[hwe->class])
1028 return;
1029
1030 regs = gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE;
1031 set_memory_based_intr(regs, hwe);
1032 }
1033
1034 /**
1035 * xe_lrc_update_memirq_regs_with_address - Re-compute GGTT references in mem interrupt data
1036 * for given LRC.
1037 * @lrc: the &xe_lrc struct instance
1038 * @hwe: the &xe_hw_engine struct instance
1039 * @regs: scratch buffer to be used as temporary storage
1040 */
xe_lrc_update_memirq_regs_with_address(struct xe_lrc * lrc,struct xe_hw_engine * hwe,u32 * regs)1041 void xe_lrc_update_memirq_regs_with_address(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
1042 u32 *regs)
1043 {
1044 struct xe_gt *gt = hwe->gt;
1045 struct iosys_map map;
1046 size_t regs_len;
1047
1048 if (!xe_device_uses_memirq(gt_to_xe(gt)))
1049 return;
1050
1051 map = __xe_lrc_regs_map(lrc);
1052 regs_len = xe_lrc_reg_size(gt_to_xe(gt));
1053 xe_map_memcpy_from(gt_to_xe(gt), regs, &map, 0, regs_len);
1054 set_memory_based_intr(regs, hwe);
1055 xe_map_memcpy_to(gt_to_xe(gt), &map, 0, regs, regs_len);
1056 }
1057
xe_lrc_set_ppgtt(struct xe_lrc * lrc,struct xe_vm * vm)1058 static void xe_lrc_set_ppgtt(struct xe_lrc *lrc, struct xe_vm *vm)
1059 {
1060 u64 desc = xe_vm_pdp4_descriptor(vm, gt_to_tile(lrc->gt));
1061
1062 xe_lrc_write_ctx_reg(lrc, CTX_PDP0_UDW, upper_32_bits(desc));
1063 xe_lrc_write_ctx_reg(lrc, CTX_PDP0_LDW, lower_32_bits(desc));
1064 }
1065
xe_lrc_finish(struct xe_lrc * lrc)1066 static void xe_lrc_finish(struct xe_lrc *lrc)
1067 {
1068 xe_hw_fence_ctx_finish(&lrc->fence_ctx);
1069 xe_bo_unpin_map_no_vm(lrc->bo);
1070 xe_bo_unpin_map_no_vm(lrc->seqno_bo);
1071 }
1072
1073 /*
1074 * wa_bb_setup_utilization() - Write commands to wa bb to assist
1075 * in calculating active context run ticks.
1076 *
1077 * Context Timestamp (CTX_TIMESTAMP) in the LRC accumulates the run ticks of the
1078 * context, but only gets updated when the context switches out. In order to
1079 * check how long a context has been active before it switches out, two things
1080 * are required:
1081 *
1082 * (1) Determine if the context is running:
1083 * To do so, we program the WA BB to set an initial value for CTX_TIMESTAMP in
1084 * the LRC. The value chosen is 1 since 0 is the initial value when the LRC is
1085 * initialized. During a query, we just check for this value to determine if the
1086 * context is active. If the context switched out, it would overwrite this
1087 * location with the actual CTX_TIMESTAMP MMIO value. Note that WA BB runs as
1088 * the last part of context restore, so reusing this LRC location will not
1089 * clobber anything.
1090 *
1091 * (2) Calculate the time that the context has been active for:
1092 * The CTX_TIMESTAMP ticks only when the context is active. If a context is
1093 * active, we just use the CTX_TIMESTAMP MMIO as the new value of utilization.
1094 * While doing so, we need to read the CTX_TIMESTAMP MMIO for the specific
1095 * engine instance. Since we do not know which instance the context is running
1096 * on until it is scheduled, we also read the ENGINE_ID MMIO in the WA BB and
1097 * store it in the PPHSWP.
1098 */
1099 #define CONTEXT_ACTIVE XE_LRC_CTX_TIMESTAMP_ACTIVE
setup_utilization_wa(struct xe_lrc * lrc,struct xe_hw_engine * hwe,u32 * batch,size_t max_len,bool indirect)1100 static ssize_t setup_utilization_wa(struct xe_lrc *lrc,
1101 struct xe_hw_engine *hwe,
1102 u32 *batch, size_t max_len, bool indirect)
1103 {
1104 u32 *cmd = batch;
1105
1106 if (IS_SRIOV_VF(gt_to_xe(lrc->gt)))
1107 return 0;
1108
1109 if (xe_gt_WARN_ON(lrc->gt, max_len < 12))
1110 return -ENOSPC;
1111
1112 *cmd++ = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET;
1113 *cmd++ = ENGINE_ID(0).addr;
1114 *cmd++ = __xe_lrc_engine_id_ggtt_addr(lrc);
1115 *cmd++ = 0;
1116
1117 *cmd++ = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
1118 *cmd++ = __xe_lrc_ctx_timestamp_ggtt_addr(lrc);
1119 *cmd++ = 0;
1120 *cmd++ = lower_32_bits(CONTEXT_ACTIVE);
1121
1122 if (lrc_to_xe(lrc)->info.has_64bit_timestamp) {
1123 *cmd++ = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
1124 *cmd++ = __xe_lrc_ctx_timestamp_udw_ggtt_addr(lrc);
1125 *cmd++ = 0;
1126 *cmd++ = upper_32_bits(CONTEXT_ACTIVE);
1127 }
1128
1129 return cmd - batch;
1130 }
1131
setup_timestamp_wa(struct xe_lrc * lrc,struct xe_hw_engine * hwe,u32 * batch,size_t max_len,bool indirect)1132 static ssize_t setup_timestamp_wa(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
1133 u32 *batch, size_t max_len, bool indirect)
1134 {
1135 const u32 ts_addr = __xe_lrc_ctx_timestamp_ggtt_addr(lrc);
1136 u32 *cmd = batch;
1137
1138 if (!XE_GT_WA(lrc->gt, 16010904313))
1139 return 0;
1140
1141 if (!indirect &&
1142 !(hwe->class == XE_ENGINE_CLASS_COPY ||
1143 hwe->class == XE_ENGINE_CLASS_VIDEO_DECODE ||
1144 hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE))
1145 return 0;
1146
1147 if (indirect &&
1148 !(hwe->class == XE_ENGINE_CLASS_RENDER ||
1149 hwe->class == XE_ENGINE_CLASS_COMPUTE))
1150 return 0;
1151
1152 if (xe_gt_WARN_ON(lrc->gt, max_len < 12))
1153 return -ENOSPC;
1154
1155 *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO |
1156 MI_LRM_ASYNC;
1157 *cmd++ = RING_CTX_TIMESTAMP(0).addr;
1158 *cmd++ = ts_addr;
1159 *cmd++ = 0;
1160
1161 *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO |
1162 MI_LRM_ASYNC;
1163 *cmd++ = RING_CTX_TIMESTAMP(0).addr;
1164 *cmd++ = ts_addr;
1165 *cmd++ = 0;
1166
1167 *cmd++ = MI_LOAD_REGISTER_MEM | MI_LRM_USE_GGTT | MI_LRI_LRM_CS_MMIO;
1168 *cmd++ = RING_CTX_TIMESTAMP(0).addr;
1169 *cmd++ = ts_addr;
1170 *cmd++ = 0;
1171
1172 return cmd - batch;
1173 }
1174
setup_configfs_post_ctx_restore_bb(struct xe_lrc * lrc,struct xe_hw_engine * hwe,u32 * batch,size_t max_len,bool indirect)1175 static ssize_t setup_configfs_post_ctx_restore_bb(struct xe_lrc *lrc,
1176 struct xe_hw_engine *hwe,
1177 u32 *batch, size_t max_len,
1178 bool indirect)
1179 {
1180 struct xe_device *xe = gt_to_xe(lrc->gt);
1181 const u32 *user_batch;
1182 u32 *cmd = batch;
1183 u32 count;
1184
1185 count = xe_configfs_get_ctx_restore_post_bb(to_pci_dev(xe->drm.dev),
1186 hwe->class, &user_batch);
1187 if (!count)
1188 return 0;
1189
1190 if (count > max_len)
1191 return -ENOSPC;
1192
1193 /*
1194 * This should be used only for tests and validation. Taint the kernel
1195 * as anything could be submitted directly in context switches
1196 */
1197 add_taint(TAINT_TEST, LOCKDEP_STILL_OK);
1198
1199 memcpy(cmd, user_batch, count * sizeof(u32));
1200 cmd += count;
1201
1202 return cmd - batch;
1203 }
1204
setup_configfs_mid_ctx_restore_bb(struct xe_lrc * lrc,struct xe_hw_engine * hwe,u32 * batch,size_t max_len,bool indirect)1205 static ssize_t setup_configfs_mid_ctx_restore_bb(struct xe_lrc *lrc,
1206 struct xe_hw_engine *hwe,
1207 u32 *batch, size_t max_len,
1208 bool indirect)
1209 {
1210 struct xe_device *xe = gt_to_xe(lrc->gt);
1211 const u32 *user_batch;
1212 u32 *cmd = batch;
1213 u32 count;
1214
1215 count = xe_configfs_get_ctx_restore_mid_bb(to_pci_dev(xe->drm.dev),
1216 hwe->class, &user_batch);
1217 if (!count)
1218 return 0;
1219
1220 if (count > max_len)
1221 return -ENOSPC;
1222
1223 /*
1224 * This should be used only for tests and validation. Taint the kernel
1225 * as anything could be submitted directly in context switches
1226 */
1227 add_taint(TAINT_TEST, LOCKDEP_STILL_OK);
1228
1229 memcpy(cmd, user_batch, count * sizeof(u32));
1230 cmd += count;
1231
1232 return cmd - batch;
1233 }
1234
setup_invalidate_state_cache_wa(struct xe_lrc * lrc,struct xe_hw_engine * hwe,u32 * batch,size_t max_len,bool indirect)1235 static ssize_t setup_invalidate_state_cache_wa(struct xe_lrc *lrc,
1236 struct xe_hw_engine *hwe,
1237 u32 *batch, size_t max_len,
1238 bool indirect)
1239 {
1240 u32 *cmd = batch;
1241
1242 if (!XE_GT_WA(lrc->gt, 18022495364) ||
1243 hwe->class != XE_ENGINE_CLASS_RENDER)
1244 return 0;
1245
1246 if (xe_gt_WARN_ON(lrc->gt, max_len < 3))
1247 return -ENOSPC;
1248
1249 *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_LRM_CS_MMIO | MI_LRI_NUM_REGS(1);
1250 *cmd++ = CS_DEBUG_MODE2(0).addr;
1251 *cmd++ = REG_MASKED_FIELD_ENABLE(INSTRUCTION_STATE_CACHE_INVALIDATE);
1252
1253 return cmd - batch;
1254 }
1255
setup_invalidate_auxccs_wa(struct xe_lrc * lrc,struct xe_hw_engine * hwe,u32 * batch,size_t max_len,bool indirect)1256 static ssize_t setup_invalidate_auxccs_wa(struct xe_lrc *lrc,
1257 struct xe_hw_engine *hwe,
1258 u32 *batch, size_t max_len,
1259 bool indirect)
1260 {
1261 struct xe_gt *gt = lrc->gt;
1262 u32 *(*emit)(struct xe_gt *gt, u32 *cmd) =
1263 gt->ring_ops[hwe->class]->emit_aux_table_inv;
1264
1265 if (!emit)
1266 return 0;
1267
1268 if (xe_gt_WARN_ON(gt, max_len < 8))
1269 return -ENOSPC;
1270
1271 return emit(gt, batch) - batch;
1272 }
1273
1274 struct bo_setup {
1275 ssize_t (*setup)(struct xe_lrc *lrc, struct xe_hw_engine *hwe,
1276 u32 *batch, size_t max_size, bool indirect);
1277 };
1278
1279 struct bo_setup_state {
1280 /* Input: */
1281 struct xe_lrc *lrc;
1282 struct xe_hw_engine *hwe;
1283 size_t max_size;
1284 size_t reserve_dw;
1285 bool indirect;
1286 unsigned int offset;
1287 const struct bo_setup *funcs;
1288 unsigned int num_funcs;
1289
1290 /* State: */
1291 u32 *buffer;
1292 u32 *ptr;
1293 unsigned int written;
1294 };
1295
setup_bo(struct bo_setup_state * state)1296 static int setup_bo(struct bo_setup_state *state)
1297 {
1298 ssize_t remain;
1299
1300 if (state->lrc->bo->vmap.is_iomem) {
1301 xe_gt_assert(state->hwe->gt, state->buffer);
1302 state->ptr = state->buffer;
1303 } else {
1304 state->ptr = state->lrc->bo->vmap.vaddr + state->offset;
1305 }
1306
1307 remain = state->max_size / sizeof(u32);
1308
1309 for (size_t i = 0; i < state->num_funcs; i++) {
1310 ssize_t len = state->funcs[i].setup(state->lrc, state->hwe,
1311 state->ptr, remain,
1312 state->indirect);
1313
1314 remain -= len;
1315
1316 /*
1317 * Caller has asked for at least reserve_dw to remain unused.
1318 */
1319 if (len < 0 ||
1320 xe_gt_WARN_ON(state->lrc->gt, remain < state->reserve_dw))
1321 goto fail;
1322
1323 state->ptr += len;
1324 state->written += len;
1325 }
1326
1327 return 0;
1328
1329 fail:
1330 return -ENOSPC;
1331 }
1332
finish_bo(struct bo_setup_state * state)1333 static void finish_bo(struct bo_setup_state *state)
1334 {
1335 if (!state->lrc->bo->vmap.is_iomem)
1336 return;
1337
1338 xe_map_memcpy_to(gt_to_xe(state->lrc->gt), &state->lrc->bo->vmap,
1339 state->offset, state->buffer,
1340 state->written * sizeof(u32));
1341 }
1342
1343 /**
1344 * xe_lrc_setup_wa_bb_with_scratch - Execute all wa bb setup callbacks.
1345 * @lrc: the &xe_lrc struct instance
1346 * @hwe: the &xe_hw_engine struct instance
1347 * @scratch: preallocated scratch buffer for temporary storage
1348 * Return: 0 on success, negative error code on failure
1349 */
xe_lrc_setup_wa_bb_with_scratch(struct xe_lrc * lrc,struct xe_hw_engine * hwe,u32 * scratch)1350 int xe_lrc_setup_wa_bb_with_scratch(struct xe_lrc *lrc, struct xe_hw_engine *hwe, u32 *scratch)
1351 {
1352 static const struct bo_setup funcs[] = {
1353 { .setup = setup_timestamp_wa },
1354 { .setup = setup_invalidate_state_cache_wa },
1355 { .setup = setup_utilization_wa },
1356 { .setup = setup_configfs_post_ctx_restore_bb },
1357 };
1358 struct bo_setup_state state = {
1359 .lrc = lrc,
1360 .hwe = hwe,
1361 .max_size = LRC_WA_BB_SIZE,
1362 .buffer = scratch,
1363 .reserve_dw = 1,
1364 .offset = __xe_lrc_wa_bb_offset(lrc),
1365 .funcs = funcs,
1366 .num_funcs = ARRAY_SIZE(funcs),
1367 };
1368 int ret;
1369
1370 ret = setup_bo(&state);
1371 if (ret)
1372 return ret;
1373
1374 *state.ptr++ = MI_BATCH_BUFFER_END;
1375 state.written++;
1376
1377 finish_bo(&state);
1378
1379 xe_lrc_write_ctx_reg(lrc, CTX_BB_PER_CTX_PTR,
1380 xe_bo_ggtt_addr(lrc->bo) + state.offset + 1);
1381
1382 return 0;
1383 }
1384
setup_wa_bb(struct xe_lrc * lrc,struct xe_hw_engine * hwe)1385 static int setup_wa_bb(struct xe_lrc *lrc, struct xe_hw_engine *hwe)
1386 {
1387 u32 *buf = NULL;
1388 int ret;
1389
1390 if (lrc->bo->vmap.is_iomem) {
1391 buf = kmalloc(LRC_WA_BB_SIZE, GFP_KERNEL);
1392 if (!buf)
1393 return -ENOMEM;
1394 }
1395
1396 ret = xe_lrc_setup_wa_bb_with_scratch(lrc, hwe, buf);
1397
1398 kfree(buf);
1399
1400 return ret;
1401 }
1402
1403 static int
setup_indirect_ctx(struct xe_lrc * lrc,struct xe_hw_engine * hwe)1404 setup_indirect_ctx(struct xe_lrc *lrc, struct xe_hw_engine *hwe)
1405 {
1406 static const struct bo_setup rcs_funcs[] = {
1407 { .setup = setup_timestamp_wa },
1408 { .setup = setup_invalidate_auxccs_wa },
1409 { .setup = setup_configfs_mid_ctx_restore_bb },
1410 };
1411 static const struct bo_setup xcs_funcs[] = {
1412 { .setup = setup_invalidate_auxccs_wa },
1413 { .setup = setup_configfs_mid_ctx_restore_bb },
1414 };
1415 struct bo_setup_state state = {
1416 .lrc = lrc,
1417 .hwe = hwe,
1418 .indirect = true,
1419 .max_size = (63 * 64) /* max 63 cachelines */,
1420 .buffer = NULL,
1421 .offset = __xe_lrc_indirect_ctx_offset(lrc),
1422 };
1423 int ret;
1424
1425 if (!(lrc->flags & XE_LRC_FLAG_INDIRECT_CTX))
1426 return 0;
1427
1428 if (hwe->class == XE_ENGINE_CLASS_RENDER ||
1429 hwe->class == XE_ENGINE_CLASS_COMPUTE) {
1430 state.funcs = rcs_funcs;
1431 state.num_funcs = ARRAY_SIZE(rcs_funcs);
1432 } else {
1433 state.funcs = xcs_funcs;
1434 state.num_funcs = ARRAY_SIZE(xcs_funcs);
1435 }
1436
1437 if (xe_gt_WARN_ON(lrc->gt, !state.funcs))
1438 return 0;
1439
1440 if (lrc->bo->vmap.is_iomem) {
1441 state.buffer = kmalloc(state.max_size, GFP_KERNEL);
1442 if (!state.buffer)
1443 return -ENOMEM;
1444 }
1445
1446 ret = setup_bo(&state);
1447 if (ret) {
1448 kfree(state.buffer);
1449 return ret;
1450 }
1451
1452 /*
1453 * Align to 64B cacheline so there's no garbage at the end for CS to
1454 * execute: size for indirect ctx must be a multiple of 64.
1455 */
1456 while (state.written & 0xf) {
1457 *state.ptr++ = MI_NOOP;
1458 state.written++;
1459 }
1460
1461 finish_bo(&state);
1462 kfree(state.buffer);
1463
1464 /*
1465 * Enable INDIRECT_CTX leaving INDIRECT_CTX_OFFSET at its default: it
1466 * varies per engine class, but the default is good enough
1467 */
1468 xe_lrc_write_ctx_reg(lrc,
1469 CTX_CS_INDIRECT_CTX,
1470 (xe_bo_ggtt_addr(lrc->bo) + state.offset) |
1471 /* Size in CLs. */
1472 (state.written * sizeof(u32) / 64));
1473
1474 return 0;
1475 }
1476
xe_multi_queue_prio_to_lrc(struct xe_lrc * lrc,enum xe_multi_queue_priority priority)1477 static u8 xe_multi_queue_prio_to_lrc(struct xe_lrc *lrc, enum xe_multi_queue_priority priority)
1478 {
1479 struct xe_device *xe = gt_to_xe(lrc->gt);
1480
1481 xe_assert(xe, (priority >= XE_MULTI_QUEUE_PRIORITY_LOW &&
1482 priority <= XE_MULTI_QUEUE_PRIORITY_HIGH));
1483
1484 /* xe_multi_queue_priority is directly mapped to LRC priority values */
1485 return priority;
1486 }
1487
1488 /**
1489 * xe_lrc_set_multi_queue_priority() - Set multi queue priority in LRC
1490 * @lrc: Logical Ring Context
1491 * @priority: Multi queue priority of the exec queue
1492 *
1493 * Convert @priority to LRC multi queue priority and update the @lrc descriptor
1494 */
xe_lrc_set_multi_queue_priority(struct xe_lrc * lrc,enum xe_multi_queue_priority priority)1495 void xe_lrc_set_multi_queue_priority(struct xe_lrc *lrc, enum xe_multi_queue_priority priority)
1496 {
1497 lrc->desc &= ~LRC_PRIORITY;
1498 lrc->desc |= FIELD_PREP(LRC_PRIORITY, xe_multi_queue_prio_to_lrc(lrc, priority));
1499 }
1500
xe_lrc_ctx_init(struct xe_lrc * lrc,struct xe_hw_engine * hwe,struct xe_vm * vm,void * replay_state,u16 msix_vec,u32 init_flags)1501 static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm,
1502 void *replay_state, u16 msix_vec, u32 init_flags)
1503 {
1504 struct xe_gt *gt = hwe->gt;
1505 struct xe_tile *tile = gt_to_tile(gt);
1506 struct xe_device *xe = gt_to_xe(gt);
1507 struct iosys_map map;
1508 u32 arb_enable;
1509 u32 state_cache_perf_fix[3];
1510 int err;
1511
1512 /*
1513 * Init Per-Process of HW status Page, LRC / context state to known
1514 * values. If there's already a primed default_lrc, just copy it, otherwise
1515 * it's the early submission to record the lrc: build a new empty one from
1516 * scratch.
1517 */
1518 map = __xe_lrc_pphwsp_map(lrc);
1519 if (gt->default_lrc[hwe->class] || replay_state) {
1520 xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE); /* PPHWSP */
1521 xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE,
1522 gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE,
1523 lrc->size - LRC_PPHWSP_SIZE);
1524 if (replay_state)
1525 xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE,
1526 replay_state, lrc->replay_size);
1527 } else {
1528 void *init_data = empty_lrc_data(hwe);
1529
1530 if (!init_data) {
1531 return -ENOMEM;
1532 }
1533
1534 xe_map_memcpy_to(xe, &map, 0, init_data, lrc->size);
1535 kfree(init_data);
1536 }
1537
1538 if (vm)
1539 xe_lrc_set_ppgtt(lrc, vm);
1540
1541 if (xe_device_has_msix(xe)) {
1542 xe_lrc_write_ctx_reg(lrc, CTX_INT_STATUS_REPORT_PTR,
1543 xe_memirq_status_ptr(&tile->memirq, hwe));
1544 xe_lrc_write_ctx_reg(lrc, CTX_INT_SRC_REPORT_PTR,
1545 xe_memirq_source_ptr(&tile->memirq, hwe));
1546 xe_lrc_write_ctx_reg(lrc, CTX_CS_INT_VEC_DATA, msix_vec << 16 | msix_vec);
1547 }
1548
1549 if (xe_gt_has_indirect_ring_state(gt)) {
1550 xe_lrc_write_ctx_reg(lrc, CTX_INDIRECT_RING_STATE,
1551 __xe_lrc_indirect_ring_ggtt_addr(lrc));
1552
1553 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START,
1554 __xe_lrc_ring_ggtt_addr(lrc));
1555 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START_UDW, 0);
1556
1557 /* Match head and tail pointers */
1558 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, lrc->ring.tail);
1559 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, lrc->ring.tail);
1560
1561 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_CTL,
1562 RING_CTL_SIZE(lrc->ring.size) | RING_VALID);
1563 } else {
1564 xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc));
1565
1566 /* Match head and tail pointers */
1567 xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, lrc->ring.tail);
1568 xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail);
1569
1570 xe_lrc_write_ctx_reg(lrc, CTX_RING_CTL,
1571 RING_CTL_SIZE(lrc->ring.size) | RING_VALID);
1572 }
1573
1574 if (init_flags & XE_LRC_CREATE_RUNALONE)
1575 xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL,
1576 xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) |
1577 REG_MASKED_FIELD_ENABLE(CTX_CTRL_RUN_ALONE));
1578
1579 if (init_flags & XE_LRC_CREATE_PXP)
1580 xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL,
1581 xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) |
1582 REG_MASKED_FIELD_ENABLE(CTX_CTRL_PXP_ENABLE));
1583
1584 lrc->ctx_timestamp = 0;
1585 xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP, 0);
1586 if (lrc_to_xe(lrc)->info.has_64bit_timestamp)
1587 xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP_UDW, 0);
1588
1589 /*
1590 * Note: It's possible that this LRC may belong to an exec_queue that is
1591 * not part of a multi-queue group. That said, it doesn't hurt to set
1592 * this field anyways since any class that supports multi-queue will
1593 * have these LRC fields defined.
1594 */
1595 if (xe_gt_supports_multi_queue(gt, hwe->class)) {
1596 lrc->queue_timestamp = 0;
1597 xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP, 0);
1598 xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP_UDW, 0);
1599 }
1600
1601 if (xe->info.has_asid && vm)
1602 xe_lrc_write_ctx_reg(lrc, CTX_ASID, vm->usm.asid);
1603
1604 lrc->desc = LRC_VALID;
1605 lrc->desc |= FIELD_PREP(LRC_ADDRESSING_MODE, LRC_LEGACY_64B_CONTEXT);
1606 /* TODO: Priority */
1607
1608 /* While this appears to have something about privileged batches or
1609 * some such, it really just means PPGTT mode.
1610 */
1611 if (vm)
1612 lrc->desc |= LRC_PRIVILEGE;
1613
1614 if (GRAPHICS_VERx100(xe) < 1250) {
1615 lrc->desc |= FIELD_PREP(LRC_ENGINE_INSTANCE, hwe->instance);
1616 lrc->desc |= FIELD_PREP(LRC_ENGINE_CLASS, hwe->class);
1617 }
1618
1619 arb_enable = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1620 xe_lrc_write_ring(lrc, &arb_enable, sizeof(arb_enable));
1621
1622 if (init_flags & XE_LRC_DISABLE_STATE_CACHE_PERF_FIX) {
1623 state_cache_perf_fix[0] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1);
1624 state_cache_perf_fix[1] = COMMON_SLICE_CHICKEN3.addr;
1625 state_cache_perf_fix[2] = REG_MASKED_FIELD_ENABLE(DISABLE_STATE_CACHE_PERF_FIX);
1626 xe_lrc_write_ring(lrc, state_cache_perf_fix, sizeof(state_cache_perf_fix));
1627 }
1628
1629 map = __xe_lrc_seqno_map(lrc);
1630 xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1);
1631
1632 map = __xe_lrc_start_seqno_map(lrc);
1633 xe_map_write32(lrc_to_xe(lrc), &map, lrc->fence_ctx.next_seqno - 1);
1634
1635 err = setup_wa_bb(lrc, hwe);
1636 if (err)
1637 return err;
1638
1639 err = setup_indirect_ctx(lrc, hwe);
1640
1641 return err;
1642 }
1643
xe_lrc_init(struct xe_lrc * lrc,struct xe_hw_engine * hwe,struct xe_vm * vm,void * replay_state,u32 ring_size,u16 msix_vec,u32 init_flags)1644 static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_vm *vm,
1645 void *replay_state, u32 ring_size, u16 msix_vec, u32 init_flags)
1646 {
1647 struct xe_gt *gt = hwe->gt;
1648 const u32 lrc_size = xe_gt_lrc_size(gt, hwe->class);
1649 u32 bo_size = ring_size + lrc_size + LRC_WA_BB_SIZE;
1650 struct xe_tile *tile = gt_to_tile(gt);
1651 struct xe_device *xe = gt_to_xe(gt);
1652 struct xe_bo *bo;
1653 u32 bo_flags;
1654 int err;
1655
1656 kref_init(&lrc->refcount);
1657 lrc->gt = gt;
1658 lrc->replay_size = xe_gt_lrc_hang_replay_size(gt, hwe->class);
1659 lrc->size = lrc_size;
1660 lrc->flags = 0;
1661 lrc->ring.size = ring_size;
1662 lrc->ring.tail = 0;
1663
1664 if (gt_engine_needs_indirect_ctx(gt, hwe->class)) {
1665 lrc->flags |= XE_LRC_FLAG_INDIRECT_CTX;
1666 bo_size += LRC_INDIRECT_CTX_BO_SIZE;
1667 }
1668
1669 if (xe_gt_has_indirect_ring_state(gt))
1670 lrc->flags |= XE_LRC_FLAG_INDIRECT_RING_STATE;
1671
1672 bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile) | XE_BO_FLAG_GGTT |
1673 XE_BO_FLAG_GGTT_INVALIDATE;
1674
1675 if ((vm && vm->xef) || init_flags & XE_LRC_CREATE_USER_CTX) /* userspace */
1676 bo_flags |= XE_BO_FLAG_PINNED_LATE_RESTORE | XE_BO_FLAG_FORCE_USER_VRAM;
1677
1678 bo = xe_bo_create_pin_map_novm(xe, tile, bo_size,
1679 ttm_bo_type_kernel,
1680 bo_flags, false);
1681 if (IS_ERR(bo))
1682 return PTR_ERR(bo);
1683
1684 lrc->bo = bo;
1685
1686 bo = xe_bo_create_pin_map_novm(xe, tile, PAGE_SIZE,
1687 ttm_bo_type_kernel,
1688 XE_BO_FLAG_GGTT |
1689 XE_BO_FLAG_GGTT_INVALIDATE |
1690 XE_BO_FLAG_SYSTEM, false);
1691 if (IS_ERR(bo)) {
1692 err = PTR_ERR(bo);
1693 goto err_lrc_finish;
1694 }
1695 lrc->seqno_bo = bo;
1696
1697 xe_hw_fence_ctx_init(&lrc->fence_ctx, hwe->gt,
1698 hwe->fence_irq, hwe->name);
1699
1700 err = xe_lrc_ctx_init(lrc, hwe, vm, replay_state, msix_vec, init_flags);
1701 if (err)
1702 goto err_lrc_finish;
1703
1704 if (vm && vm->xef)
1705 xe_drm_client_add_bo(vm->xef->client, lrc->bo);
1706
1707 return 0;
1708
1709 err_lrc_finish:
1710 xe_lrc_finish(lrc);
1711 return err;
1712 }
1713
1714 /**
1715 * xe_lrc_create - Create a LRC
1716 * @hwe: Hardware Engine
1717 * @vm: The VM (address space)
1718 * @replay_state: GPU hang replay state
1719 * @ring_size: LRC ring size
1720 * @msix_vec: MSI-X interrupt vector (for platforms that support it)
1721 * @flags: LRC initialization flags
1722 *
1723 * Allocate and initialize the Logical Ring Context (LRC).
1724 *
1725 * Return pointer to created LRC upon success and an error pointer
1726 * upon failure.
1727 */
xe_lrc_create(struct xe_hw_engine * hwe,struct xe_vm * vm,void * replay_state,u32 ring_size,u16 msix_vec,u32 flags)1728 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm,
1729 void *replay_state, u32 ring_size, u16 msix_vec, u32 flags)
1730 {
1731 struct xe_lrc *lrc;
1732 int err;
1733
1734 lrc = kzalloc_obj(*lrc);
1735 if (!lrc)
1736 return ERR_PTR(-ENOMEM);
1737
1738 err = xe_lrc_init(lrc, hwe, vm, replay_state, ring_size, msix_vec, flags);
1739 if (err) {
1740 kfree(lrc);
1741 return ERR_PTR(err);
1742 }
1743
1744 return lrc;
1745 }
1746
1747 /**
1748 * xe_lrc_destroy - Destroy the LRC
1749 * @ref: reference to LRC
1750 *
1751 * Called when ref == 0, release resources held by the Logical Ring Context
1752 * (LRC) and free the LRC memory.
1753 */
xe_lrc_destroy(struct kref * ref)1754 void xe_lrc_destroy(struct kref *ref)
1755 {
1756 struct xe_lrc *lrc = container_of(ref, struct xe_lrc, refcount);
1757
1758 xe_lrc_finish(lrc);
1759 kfree(lrc);
1760 }
1761
1762 /**
1763 * xe_lrc_update_hwctx_regs_with_address - Re-compute GGTT references within given LRC.
1764 * @lrc: the &xe_lrc struct instance
1765 */
xe_lrc_update_hwctx_regs_with_address(struct xe_lrc * lrc)1766 void xe_lrc_update_hwctx_regs_with_address(struct xe_lrc *lrc)
1767 {
1768 if (xe_lrc_has_indirect_ring_state(lrc)) {
1769 xe_lrc_write_ctx_reg(lrc, CTX_INDIRECT_RING_STATE,
1770 __xe_lrc_indirect_ring_ggtt_addr(lrc));
1771
1772 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START,
1773 __xe_lrc_ring_ggtt_addr(lrc));
1774 } else {
1775 xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc));
1776 }
1777 }
1778
xe_lrc_set_ring_tail(struct xe_lrc * lrc,u32 tail)1779 void xe_lrc_set_ring_tail(struct xe_lrc *lrc, u32 tail)
1780 {
1781 if (xe_lrc_has_indirect_ring_state(lrc))
1782 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL, tail);
1783 else
1784 xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, tail);
1785 }
1786
xe_lrc_ring_tail(struct xe_lrc * lrc)1787 u32 xe_lrc_ring_tail(struct xe_lrc *lrc)
1788 {
1789 if (xe_lrc_has_indirect_ring_state(lrc))
1790 return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_TAIL) & TAIL_ADDR;
1791 else
1792 return xe_lrc_read_ctx_reg(lrc, CTX_RING_TAIL) & TAIL_ADDR;
1793 }
1794
xe_lrc_ring_start(struct xe_lrc * lrc)1795 static u32 xe_lrc_ring_start(struct xe_lrc *lrc)
1796 {
1797 if (xe_lrc_has_indirect_ring_state(lrc))
1798 return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_START);
1799 else
1800 return xe_lrc_read_ctx_reg(lrc, CTX_RING_START);
1801 }
1802
xe_lrc_set_ring_head(struct xe_lrc * lrc,u32 head)1803 void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head)
1804 {
1805 if (xe_lrc_has_indirect_ring_state(lrc))
1806 xe_lrc_write_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD, head);
1807 else
1808 xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, head);
1809 }
1810
xe_lrc_ring_head(struct xe_lrc * lrc)1811 u32 xe_lrc_ring_head(struct xe_lrc *lrc)
1812 {
1813 if (xe_lrc_has_indirect_ring_state(lrc))
1814 return xe_lrc_read_indirect_ctx_reg(lrc, INDIRECT_CTX_RING_HEAD) & HEAD_ADDR;
1815 else
1816 return xe_lrc_read_ctx_reg(lrc, CTX_RING_HEAD) & HEAD_ADDR;
1817 }
1818
xe_lrc_ring_space(struct xe_lrc * lrc)1819 u32 xe_lrc_ring_space(struct xe_lrc *lrc)
1820 {
1821 const u32 head = xe_lrc_ring_head(lrc);
1822 const u32 tail = lrc->ring.tail;
1823 const u32 size = lrc->ring.size;
1824
1825 return ((head - tail - 1) & (size - 1)) + 1;
1826 }
1827
__xe_lrc_write_ring(struct xe_lrc * lrc,struct iosys_map ring,const void * data,size_t size)1828 static void __xe_lrc_write_ring(struct xe_lrc *lrc, struct iosys_map ring,
1829 const void *data, size_t size)
1830 {
1831 struct xe_device *xe = lrc_to_xe(lrc);
1832
1833 iosys_map_incr(&ring, lrc->ring.tail);
1834 xe_map_memcpy_to(xe, &ring, 0, data, size);
1835 lrc->ring.tail = (lrc->ring.tail + size) & (lrc->ring.size - 1);
1836 }
1837
xe_lrc_write_ring(struct xe_lrc * lrc,const void * data,size_t size)1838 void xe_lrc_write_ring(struct xe_lrc *lrc, const void *data, size_t size)
1839 {
1840 struct xe_device *xe = lrc_to_xe(lrc);
1841 struct iosys_map ring;
1842 u32 rhs;
1843 size_t aligned_size;
1844
1845 xe_assert(xe, IS_ALIGNED(size, 4));
1846 aligned_size = ALIGN(size, 8);
1847
1848 ring = __xe_lrc_ring_map(lrc);
1849
1850 xe_assert(xe, lrc->ring.tail < lrc->ring.size);
1851 rhs = lrc->ring.size - lrc->ring.tail;
1852 if (size > rhs) {
1853 __xe_lrc_write_ring(lrc, ring, data, rhs);
1854 __xe_lrc_write_ring(lrc, ring, data + rhs, size - rhs);
1855 } else {
1856 __xe_lrc_write_ring(lrc, ring, data, size);
1857 }
1858
1859 if (aligned_size > size) {
1860 u32 noop = MI_NOOP;
1861
1862 __xe_lrc_write_ring(lrc, ring, &noop, sizeof(noop));
1863 }
1864
1865 /*
1866 * The ring and the LRC context image are both WC, so the ring tail
1867 * update which publishes these writes can become visible to the device
1868 * first. Ensure the ring contents are visible before returning.
1869 */
1870 xe_device_wmb(xe);
1871 }
1872
xe_lrc_descriptor(struct xe_lrc * lrc)1873 u64 xe_lrc_descriptor(struct xe_lrc *lrc)
1874 {
1875 return lrc->desc | xe_lrc_ggtt_addr(lrc);
1876 }
1877
xe_lrc_seqno_ggtt_addr(struct xe_lrc * lrc)1878 u32 xe_lrc_seqno_ggtt_addr(struct xe_lrc *lrc)
1879 {
1880 return __xe_lrc_seqno_ggtt_addr(lrc);
1881 }
1882
1883 /**
1884 * xe_lrc_alloc_seqno_fence() - Allocate an lrc seqno fence.
1885 *
1886 * Allocate but don't initialize an lrc seqno fence.
1887 *
1888 * Return: Pointer to the allocated fence or
1889 * negative error pointer on error.
1890 */
xe_lrc_alloc_seqno_fence(void)1891 struct dma_fence *xe_lrc_alloc_seqno_fence(void)
1892 {
1893 return xe_hw_fence_alloc();
1894 }
1895
1896 /**
1897 * xe_lrc_free_seqno_fence() - Free an lrc seqno fence.
1898 * @fence: Pointer to the fence to free.
1899 *
1900 * Frees an lrc seqno fence that hasn't yet been
1901 * initialized.
1902 */
xe_lrc_free_seqno_fence(struct dma_fence * fence)1903 void xe_lrc_free_seqno_fence(struct dma_fence *fence)
1904 {
1905 xe_hw_fence_free(fence);
1906 }
1907
1908 /**
1909 * xe_lrc_init_seqno_fence() - Initialize an lrc seqno fence.
1910 * @lrc: Pointer to the lrc.
1911 * @fence: Pointer to the fence to initialize.
1912 *
1913 * Initializes a pre-allocated lrc seqno fence.
1914 * After initialization, the fence is subject to normal
1915 * dma-fence refcounting.
1916 */
xe_lrc_init_seqno_fence(struct xe_lrc * lrc,struct dma_fence * fence)1917 void xe_lrc_init_seqno_fence(struct xe_lrc *lrc, struct dma_fence *fence)
1918 {
1919 xe_hw_fence_init(fence, &lrc->fence_ctx, __xe_lrc_seqno_map(lrc));
1920 }
1921
xe_lrc_seqno(struct xe_lrc * lrc)1922 s32 xe_lrc_seqno(struct xe_lrc *lrc)
1923 {
1924 struct iosys_map map = __xe_lrc_seqno_map(lrc);
1925
1926 return xe_map_read32(lrc_to_xe(lrc), &map);
1927 }
1928
xe_lrc_start_seqno(struct xe_lrc * lrc)1929 s32 xe_lrc_start_seqno(struct xe_lrc *lrc)
1930 {
1931 struct iosys_map map = __xe_lrc_start_seqno_map(lrc);
1932
1933 return xe_map_read32(lrc_to_xe(lrc), &map);
1934 }
1935
xe_lrc_start_seqno_ggtt_addr(struct xe_lrc * lrc)1936 u32 xe_lrc_start_seqno_ggtt_addr(struct xe_lrc *lrc)
1937 {
1938 return __xe_lrc_start_seqno_ggtt_addr(lrc);
1939 }
1940
xe_lrc_parallel_ggtt_addr(struct xe_lrc * lrc)1941 u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc)
1942 {
1943 return __xe_lrc_parallel_ggtt_addr(lrc);
1944 }
1945
xe_lrc_parallel_map(struct xe_lrc * lrc)1946 struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc)
1947 {
1948 return __xe_lrc_parallel_map(lrc);
1949 }
1950
1951 /**
1952 * xe_lrc_engine_id() - Read engine id value
1953 * @lrc: Pointer to the lrc.
1954 *
1955 * Returns: context id value
1956 */
xe_lrc_engine_id(struct xe_lrc * lrc)1957 static u32 xe_lrc_engine_id(struct xe_lrc *lrc)
1958 {
1959 struct xe_device *xe = lrc_to_xe(lrc);
1960 struct iosys_map map;
1961
1962 map = __xe_lrc_engine_id_map(lrc);
1963 return xe_map_read32(xe, &map);
1964 }
1965
instr_dw(u32 cmd_header)1966 static int instr_dw(u32 cmd_header)
1967 {
1968 /* GFXPIPE "SINGLE_DW" opcodes are a single dword */
1969 if ((cmd_header & (XE_INSTR_CMD_TYPE | GFXPIPE_PIPELINE)) ==
1970 GFXPIPE_SINGLE_DW_CMD(0, 0))
1971 return 1;
1972
1973 /* 3DSTATE_SO_DECL_LIST has a 9-bit dword length rather than 8 */
1974 if ((cmd_header & GFXPIPE_MATCH_MASK) == CMD_3DSTATE_SO_DECL_LIST)
1975 return REG_FIELD_GET(CMD_3DSTATE_SO_DECL_LIST_DW_LEN, cmd_header) + 2;
1976
1977 /* Most instructions have the # of dwords (minus 2) in 7:0 */
1978 return REG_FIELD_GET(XE_INSTR_LEN_MASK, cmd_header) + 2;
1979 }
1980
dump_mi_command(struct drm_printer * p,struct xe_gt * gt,u32 * start,u32 * dw,int remaining_dw)1981 static int dump_mi_command(struct drm_printer *p,
1982 struct xe_gt *gt,
1983 u32 *start,
1984 u32 *dw,
1985 int remaining_dw)
1986 {
1987 u32 inst_header = *dw;
1988 u32 numdw = instr_dw(inst_header);
1989 u32 opcode = REG_FIELD_GET(MI_OPCODE, inst_header);
1990 int num_noop;
1991
1992 /* First check for commands that don't have/use a '# DW' field */
1993 switch (inst_header & MI_OPCODE) {
1994 case MI_NOOP:
1995 num_noop = 1;
1996 while (num_noop < remaining_dw &&
1997 (*(++dw) & REG_GENMASK(31, 23)) == MI_NOOP)
1998 num_noop++;
1999 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_NOOP (%d dwords)\n",
2000 dw - num_noop - start, inst_header, num_noop);
2001 return num_noop;
2002
2003 case MI_TOPOLOGY_FILTER:
2004 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_TOPOLOGY_FILTER\n",
2005 dw - start, inst_header);
2006 return 1;
2007
2008 case MI_BATCH_BUFFER_END:
2009 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_BATCH_BUFFER_END\n",
2010 dw - start, inst_header);
2011 /* Return 'remaining_dw' to consume the rest of the LRC */
2012 return remaining_dw;
2013 }
2014
2015 /*
2016 * Any remaining commands include a # of dwords. We should make sure
2017 * it doesn't exceed the remaining size of the LRC.
2018 */
2019 if (xe_gt_WARN_ON(gt, numdw > remaining_dw))
2020 numdw = remaining_dw;
2021
2022 switch (inst_header & MI_OPCODE) {
2023 case MI_LOAD_REGISTER_IMM:
2024 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_LOAD_REGISTER_IMM: %d regs\n",
2025 dw - start, inst_header, (numdw - 1) / 2);
2026 for (int i = 1; i < numdw; i += 2)
2027 drm_printf(p, "LRC[%#5tx] = - %#6x = %#010x\n",
2028 &dw[i] - start, dw[i], dw[i + 1]);
2029 return numdw;
2030
2031 case MI_LOAD_REGISTER_MEM & MI_OPCODE:
2032 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_LOAD_REGISTER_MEM: %s%s\n",
2033 dw - start, inst_header,
2034 dw[0] & MI_LRI_LRM_CS_MMIO ? "CS_MMIO " : "",
2035 dw[0] & MI_LRM_USE_GGTT ? "USE_GGTT " : "");
2036 if (numdw == 4)
2037 drm_printf(p, "LRC[%#5tx] = - %#6x = %#010llx\n",
2038 dw - start,
2039 dw[1], ((u64)(dw[3]) << 32 | (u64)(dw[2])));
2040 else
2041 drm_printf(p, "LRC[%#5tx] = - %*ph (%s)\n",
2042 dw - start, (int)sizeof(u32) * (numdw - 1),
2043 dw + 1, numdw < 4 ? "truncated" : "malformed");
2044 return numdw;
2045
2046 case MI_FORCE_WAKEUP:
2047 drm_printf(p, "LRC[%#5tx] = [%#010x] MI_FORCE_WAKEUP\n",
2048 dw - start, inst_header);
2049 return numdw;
2050
2051 default:
2052 drm_printf(p, "LRC[%#5tx] = [%#010x] unknown MI opcode %#x, likely %d dwords\n",
2053 dw - start, inst_header, opcode, numdw);
2054 return numdw;
2055 }
2056 }
2057
dump_gfxpipe_command(struct drm_printer * p,struct xe_gt * gt,u32 * start,u32 * dw,int remaining_dw)2058 static int dump_gfxpipe_command(struct drm_printer *p,
2059 struct xe_gt *gt,
2060 u32 *start,
2061 u32 *dw,
2062 int remaining_dw)
2063 {
2064 u32 numdw = instr_dw(*dw);
2065 u32 pipeline = REG_FIELD_GET(GFXPIPE_PIPELINE, *dw);
2066 u32 opcode = REG_FIELD_GET(GFXPIPE_OPCODE, *dw);
2067 u32 subopcode = REG_FIELD_GET(GFXPIPE_SUBOPCODE, *dw);
2068
2069 /*
2070 * Make sure we haven't mis-parsed a number of dwords that exceeds the
2071 * remaining size of the LRC.
2072 */
2073 if (xe_gt_WARN_ON(gt, numdw > remaining_dw))
2074 numdw = remaining_dw;
2075
2076 switch (*dw & GFXPIPE_MATCH_MASK) {
2077 #define MATCH(cmd) \
2078 case cmd: \
2079 drm_printf(p, "LRC[%#5tx] = [%#010x] " #cmd " (%d dwords)\n", \
2080 dw - start, *dw, numdw); \
2081 return numdw
2082 #define MATCH3D(cmd) \
2083 case CMD_##cmd: \
2084 drm_printf(p, "LRC[%#5tx] = [%#010x] " #cmd " (%d dwords)\n", \
2085 dw - start, *dw, numdw); \
2086 return numdw
2087
2088 MATCH(STATE_BASE_ADDRESS);
2089 MATCH(STATE_SIP);
2090 MATCH(GPGPU_CSR_BASE_ADDRESS);
2091 MATCH(STATE_COMPUTE_MODE);
2092 MATCH3D(3DSTATE_BTD);
2093 MATCH(STATE_SYSTEM_MEM_FENCE_ADDRESS);
2094 MATCH(STATE_CONTEXT_DATA_BASE_ADDRESS);
2095
2096 MATCH3D(3DSTATE_VF_STATISTICS);
2097
2098 MATCH(PIPELINE_SELECT);
2099
2100 MATCH3D(3DSTATE_DRAWING_RECTANGLE_FAST);
2101 MATCH3D(3DSTATE_CUSTOM_SAMPLE_PATTERN);
2102 MATCH3D(3DSTATE_CLEAR_PARAMS);
2103 MATCH3D(3DSTATE_DEPTH_BUFFER);
2104 MATCH3D(3DSTATE_STENCIL_BUFFER);
2105 MATCH3D(3DSTATE_HIER_DEPTH_BUFFER);
2106 MATCH3D(3DSTATE_VERTEX_BUFFERS);
2107 MATCH3D(3DSTATE_VERTEX_ELEMENTS);
2108 MATCH3D(3DSTATE_INDEX_BUFFER);
2109 MATCH3D(3DSTATE_VF);
2110 MATCH3D(3DSTATE_MULTISAMPLE);
2111 MATCH3D(3DSTATE_CC_STATE_POINTERS);
2112 MATCH3D(3DSTATE_SCISSOR_STATE_POINTERS);
2113 MATCH3D(3DSTATE_VS);
2114 MATCH3D(3DSTATE_GS);
2115 MATCH3D(3DSTATE_CLIP);
2116 MATCH3D(3DSTATE_SF);
2117 MATCH3D(3DSTATE_WM);
2118 MATCH3D(3DSTATE_CONSTANT_VS);
2119 MATCH3D(3DSTATE_CONSTANT_GS);
2120 MATCH3D(3DSTATE_CONSTANT_PS);
2121 MATCH3D(3DSTATE_SAMPLE_MASK);
2122 MATCH3D(3DSTATE_CONSTANT_HS);
2123 MATCH3D(3DSTATE_CONSTANT_DS);
2124 MATCH3D(3DSTATE_HS);
2125 MATCH3D(3DSTATE_TE);
2126 MATCH3D(3DSTATE_DS);
2127 MATCH3D(3DSTATE_STREAMOUT);
2128 MATCH3D(3DSTATE_SBE);
2129 MATCH3D(3DSTATE_PS);
2130 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP);
2131 MATCH3D(3DSTATE_CPS_POINTERS);
2132 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_CC);
2133 MATCH3D(3DSTATE_BLEND_STATE_POINTERS);
2134 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_VS);
2135 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_HS);
2136 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_DS);
2137 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_GS);
2138 MATCH3D(3DSTATE_BINDING_TABLE_POINTERS_PS);
2139 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_VS);
2140 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_HS);
2141 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_DS);
2142 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_GS);
2143 MATCH3D(3DSTATE_SAMPLER_STATE_POINTERS_PS);
2144 MATCH3D(3DSTATE_VF_INSTANCING);
2145 MATCH3D(3DSTATE_VF_SGVS);
2146 MATCH3D(3DSTATE_VF_TOPOLOGY);
2147 MATCH3D(3DSTATE_WM_CHROMAKEY);
2148 MATCH3D(3DSTATE_PS_BLEND);
2149 MATCH3D(3DSTATE_WM_DEPTH_STENCIL);
2150 MATCH3D(3DSTATE_PS_EXTRA);
2151 MATCH3D(3DSTATE_RASTER);
2152 MATCH3D(3DSTATE_SBE_SWIZ);
2153 MATCH3D(3DSTATE_WM_HZ_OP);
2154 MATCH3D(3DSTATE_VF_COMPONENT_PACKING);
2155 MATCH3D(3DSTATE_VF_SGVS_2);
2156 MATCH3D(3DSTATE_VFG);
2157 MATCH3D(3DSTATE_URB_ALLOC_VS);
2158 MATCH3D(3DSTATE_URB_ALLOC_HS);
2159 MATCH3D(3DSTATE_URB_ALLOC_DS);
2160 MATCH3D(3DSTATE_URB_ALLOC_GS);
2161 MATCH3D(3DSTATE_SO_BUFFER_INDEX_0);
2162 MATCH3D(3DSTATE_SO_BUFFER_INDEX_1);
2163 MATCH3D(3DSTATE_SO_BUFFER_INDEX_2);
2164 MATCH3D(3DSTATE_SO_BUFFER_INDEX_3);
2165 MATCH3D(3DSTATE_PRIMITIVE_REPLICATION);
2166 MATCH3D(3DSTATE_TBIMR_TILE_PASS_INFO);
2167 MATCH3D(3DSTATE_AMFS);
2168 MATCH3D(3DSTATE_DEPTH_BOUNDS);
2169 MATCH3D(3DSTATE_AMFS_TEXTURE_POINTERS);
2170 MATCH3D(3DSTATE_CONSTANT_TS_POINTER);
2171 MATCH3D(3DSTATE_MESH_CONTROL);
2172 MATCH3D(3DSTATE_MESH_DISTRIB);
2173 MATCH3D(3DSTATE_TASK_REDISTRIB);
2174 MATCH3D(3DSTATE_MESH_SHADER);
2175 MATCH3D(3DSTATE_MESH_SHADER_DATA);
2176 MATCH3D(3DSTATE_TASK_CONTROL);
2177 MATCH3D(3DSTATE_TASK_SHADER);
2178 MATCH3D(3DSTATE_TASK_SHADER_DATA);
2179 MATCH3D(3DSTATE_URB_ALLOC_MESH);
2180 MATCH3D(3DSTATE_URB_ALLOC_TASK);
2181 MATCH3D(3DSTATE_CLIP_MESH);
2182 MATCH3D(3DSTATE_SBE_MESH);
2183 MATCH3D(3DSTATE_CPSIZE_CONTROL_BUFFER);
2184 MATCH3D(3DSTATE_COARSE_PIXEL);
2185 MATCH3D(3DSTATE_MESH_SHADER_DATA_EXT);
2186 MATCH3D(3DSTATE_TASK_SHADER_DATA_EXT);
2187 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_CC_2);
2188 MATCH3D(3DSTATE_CC_STATE_POINTERS_2);
2189 MATCH3D(3DSTATE_SCISSOR_STATE_POINTERS_2);
2190 MATCH3D(3DSTATE_BLEND_STATE_POINTERS_2);
2191 MATCH3D(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP_2);
2192
2193 MATCH3D(3DSTATE_DRAWING_RECTANGLE);
2194 MATCH3D(3DSTATE_URB_MEMORY);
2195 MATCH3D(3DSTATE_CHROMA_KEY);
2196 MATCH3D(3DSTATE_POLY_STIPPLE_OFFSET);
2197 MATCH3D(3DSTATE_POLY_STIPPLE_PATTERN);
2198 MATCH3D(3DSTATE_LINE_STIPPLE);
2199 MATCH3D(3DSTATE_AA_LINE_PARAMETERS);
2200 MATCH3D(3DSTATE_MONOFILTER_SIZE);
2201 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_VS);
2202 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_HS);
2203 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_DS);
2204 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_GS);
2205 MATCH3D(3DSTATE_PUSH_CONSTANT_ALLOC_PS);
2206 MATCH3D(3DSTATE_SO_DECL_LIST);
2207 MATCH3D(3DSTATE_SO_BUFFER);
2208 MATCH3D(3DSTATE_BINDING_TABLE_POOL_ALLOC);
2209 MATCH3D(3DSTATE_SAMPLE_PATTERN);
2210 MATCH3D(3DSTATE_3D_MODE);
2211 MATCH3D(3DSTATE_SUBSLICE_HASH_TABLE);
2212 MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTERS);
2213 MATCH3D(3DSTATE_PTBR_TILE_PASS_INFO);
2214 MATCH3D(3DSTATE_SLICE_TABLE_STATE_POINTER_2);
2215
2216 default:
2217 drm_printf(p, "LRC[%#5tx] = [%#010x] unknown GFXPIPE command (pipeline=%#x, opcode=%#x, subopcode=%#x), likely %d dwords\n",
2218 dw - start, *dw, pipeline, opcode, subopcode, numdw);
2219 return numdw;
2220 }
2221 }
2222
dump_gfx_state_command(struct drm_printer * p,struct xe_gt * gt,u32 * start,u32 * dw,int remaining_dw)2223 static int dump_gfx_state_command(struct drm_printer *p,
2224 struct xe_gt *gt,
2225 u32 *start,
2226 u32 *dw,
2227 int remaining_dw)
2228 {
2229 u32 numdw = instr_dw(*dw);
2230 u32 opcode = REG_FIELD_GET(GFX_STATE_OPCODE, *dw);
2231
2232 /*
2233 * Make sure we haven't mis-parsed a number of dwords that exceeds the
2234 * remaining size of the LRC.
2235 */
2236 if (xe_gt_WARN_ON(gt, numdw > remaining_dw))
2237 numdw = remaining_dw;
2238
2239 switch (*dw & (XE_INSTR_GFX_STATE | GFX_STATE_OPCODE)) {
2240 MATCH(STATE_WRITE_INLINE);
2241
2242 default:
2243 drm_printf(p, "LRC[%#5tx] = [%#010x] unknown GFX_STATE command (opcode=%#x), likely %d dwords\n",
2244 dw - start, *dw, opcode, numdw);
2245 return numdw;
2246 }
2247 }
2248
xe_lrc_dump_default(struct drm_printer * p,struct xe_gt * gt,enum xe_engine_class hwe_class)2249 void xe_lrc_dump_default(struct drm_printer *p,
2250 struct xe_gt *gt,
2251 enum xe_engine_class hwe_class)
2252 {
2253 u32 *dw, *start;
2254 int remaining_dw, num_dw;
2255
2256 if (!gt->default_lrc[hwe_class]) {
2257 drm_printf(p, "No default LRC for class %d\n", hwe_class);
2258 return;
2259 }
2260
2261 /*
2262 * Skip the beginning of the LRC since it contains the per-process
2263 * hardware status page.
2264 */
2265 dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE;
2266 start = dw;
2267 remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4;
2268
2269 while (remaining_dw > 0) {
2270 if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_MI) {
2271 num_dw = dump_mi_command(p, gt, start, dw, remaining_dw);
2272 } else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE) {
2273 num_dw = dump_gfxpipe_command(p, gt, start, dw, remaining_dw);
2274 } else if ((*dw & XE_INSTR_CMD_TYPE) == XE_INSTR_GFX_STATE) {
2275 num_dw = dump_gfx_state_command(p, gt, start, dw, remaining_dw);
2276 } else {
2277 num_dw = min(instr_dw(*dw), remaining_dw);
2278 drm_printf(p, "LRC[%#5tx] = [%#10x] Unknown instruction of type %#x, likely %d dwords\n",
2279 dw - start,
2280 *dw, REG_FIELD_GET(XE_INSTR_CMD_TYPE, *dw),
2281 num_dw);
2282 }
2283
2284 dw += num_dw;
2285 remaining_dw -= num_dw;
2286 }
2287 }
2288
2289 /*
2290 * Lookup the value of a register within the offset/value pairs of an
2291 * MI_LOAD_REGISTER_IMM instruction.
2292 *
2293 * Return -ENOENT if the register is not present in the MI_LRI instruction.
2294 */
lookup_reg_in_mi_lri(u32 offset,u32 * value,const u32 * dword_pair,int num_regs)2295 static int lookup_reg_in_mi_lri(u32 offset, u32 *value,
2296 const u32 *dword_pair, int num_regs)
2297 {
2298 for (int i = 0; i < num_regs; i++) {
2299 if (dword_pair[2 * i] == offset) {
2300 *value = dword_pair[2 * i + 1];
2301 return 0;
2302 }
2303 }
2304
2305 return -ENOENT;
2306 }
2307
2308 /*
2309 * Lookup the value of a register in a specific engine type's default LRC.
2310 *
2311 * Return -EINVAL if the default LRC doesn't exist, or ENOENT if the register
2312 * cannot be found in the default LRC.
2313 */
xe_lrc_lookup_default_reg_value(struct xe_gt * gt,enum xe_engine_class hwe_class,u32 offset,u32 * value)2314 int xe_lrc_lookup_default_reg_value(struct xe_gt *gt,
2315 enum xe_engine_class hwe_class,
2316 u32 offset,
2317 u32 *value)
2318 {
2319 u32 *dw;
2320 int remaining_dw, ret;
2321
2322 if (!gt->default_lrc[hwe_class])
2323 return -EINVAL;
2324
2325 /*
2326 * Skip the beginning of the LRC since it contains the per-process
2327 * hardware status page.
2328 */
2329 dw = gt->default_lrc[hwe_class] + LRC_PPHWSP_SIZE;
2330 remaining_dw = (xe_gt_lrc_size(gt, hwe_class) - LRC_PPHWSP_SIZE) / 4;
2331
2332 while (remaining_dw > 0) {
2333 u32 num_dw = instr_dw(*dw);
2334
2335 if (num_dw > remaining_dw)
2336 num_dw = remaining_dw;
2337
2338 switch (*dw & XE_INSTR_CMD_TYPE) {
2339 case XE_INSTR_MI:
2340 switch (*dw & MI_OPCODE) {
2341 case MI_BATCH_BUFFER_END:
2342 /* End of LRC; register not found */
2343 return -ENOENT;
2344
2345 case MI_NOOP:
2346 case MI_TOPOLOGY_FILTER:
2347 /*
2348 * MI_NOOP and MI_TOPOLOGY_FILTER don't have
2349 * a length field and are always 1-dword
2350 * instructions.
2351 */
2352 remaining_dw--;
2353 dw++;
2354 break;
2355
2356 case MI_LOAD_REGISTER_IMM:
2357 ret = lookup_reg_in_mi_lri(offset, value,
2358 dw + 1, (num_dw - 1) / 2);
2359 if (ret == 0)
2360 return 0;
2361
2362 fallthrough;
2363
2364 default:
2365 /*
2366 * Jump to next instruction based on length
2367 * field.
2368 */
2369 remaining_dw -= num_dw;
2370 dw += num_dw;
2371 break;
2372 }
2373 break;
2374
2375 default:
2376 /* Jump to next instruction based on length field. */
2377 remaining_dw -= num_dw;
2378 dw += num_dw;
2379 }
2380 }
2381
2382 return -ENOENT;
2383 }
2384
2385 struct instr_state {
2386 u32 instr;
2387 u16 num_dw;
2388 };
2389
2390 static const struct instr_state xe_hpg_svg_state[] = {
2391 { .instr = CMD_3DSTATE_CONSTANT_VS, .num_dw = 11 },
2392 { .instr = CMD_3DSTATE_CONSTANT_HS, .num_dw = 11 },
2393 { .instr = CMD_3DSTATE_CONSTANT_DS, .num_dw = 11 },
2394 { .instr = CMD_3DSTATE_CONSTANT_GS, .num_dw = 11 },
2395 { .instr = CMD_3DSTATE_VERTEX_ELEMENTS, .num_dw = 69 },
2396 { .instr = CMD_3DSTATE_VF_COMPONENT_PACKING, .num_dw = 5 },
2397 { .instr = CMD_3DSTATE_VF_SGVS, .num_dw = 2 },
2398 { .instr = CMD_3DSTATE_VF_SGVS_2, .num_dw = 3 },
2399 { .instr = CMD_3DSTATE_VS, .num_dw = 9 },
2400 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_VS, .num_dw = 2 },
2401 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_VS, .num_dw = 2 },
2402 { .instr = CMD_3DSTATE_URB_ALLOC_VS, .num_dw = 3 },
2403 { .instr = CMD_3DSTATE_STREAMOUT, .num_dw = 5 },
2404 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_0, .num_dw = 8 },
2405 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_1, .num_dw = 8 },
2406 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_2, .num_dw = 8 },
2407 { .instr = CMD_3DSTATE_SO_BUFFER_INDEX_3, .num_dw = 8 },
2408 { .instr = CMD_3DSTATE_CLIP, .num_dw = 4 },
2409 { .instr = CMD_3DSTATE_PRIMITIVE_REPLICATION, .num_dw = 6 },
2410 { .instr = CMD_3DSTATE_CLIP_MESH, .num_dw = 2 },
2411 { .instr = CMD_3DSTATE_SF, .num_dw = 4 },
2412 { .instr = CMD_3DSTATE_SCISSOR_STATE_POINTERS, .num_dw = 2 },
2413 { .instr = CMD_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP, .num_dw = 2 },
2414 { .instr = CMD_3DSTATE_RASTER, .num_dw = 5 },
2415 { .instr = CMD_3DSTATE_TBIMR_TILE_PASS_INFO, .num_dw = 4 },
2416 { .instr = CMD_3DSTATE_WM_HZ_OP, .num_dw = 6 },
2417 { .instr = CMD_3DSTATE_MULTISAMPLE, .num_dw = 2 },
2418 { .instr = CMD_3DSTATE_HS, .num_dw = 9 },
2419 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_HS, .num_dw = 2 },
2420 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_HS, .num_dw = 2 },
2421 { .instr = CMD_3DSTATE_URB_ALLOC_HS, .num_dw = 3 },
2422 { .instr = CMD_3DSTATE_TASK_CONTROL, .num_dw = 3 },
2423 { .instr = CMD_3DSTATE_TASK_SHADER, .num_dw = 7 },
2424 { .instr = CMD_3DSTATE_TASK_SHADER_DATA, .num_dw = 10 },
2425 { .instr = CMD_3DSTATE_URB_ALLOC_TASK, .num_dw = 3 },
2426 { .instr = CMD_3DSTATE_TE, .num_dw = 5 },
2427 { .instr = CMD_3DSTATE_TASK_REDISTRIB, .num_dw = 2 },
2428 { .instr = CMD_3DSTATE_DS, .num_dw = 11 },
2429 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_DS, .num_dw = 2 },
2430 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_DS, .num_dw = 2 },
2431 { .instr = CMD_3DSTATE_URB_ALLOC_DS, .num_dw = 3 },
2432 { .instr = CMD_3DSTATE_GS, .num_dw = 10 },
2433 { .instr = CMD_3DSTATE_BINDING_TABLE_POINTERS_GS, .num_dw = 2 },
2434 { .instr = CMD_3DSTATE_SAMPLER_STATE_POINTERS_GS, .num_dw = 2 },
2435 { .instr = CMD_3DSTATE_URB_ALLOC_GS, .num_dw = 3 },
2436 { .instr = CMD_3DSTATE_MESH_CONTROL, .num_dw = 3 },
2437 { .instr = CMD_3DSTATE_MESH_SHADER_DATA, .num_dw = 10 },
2438 { .instr = CMD_3DSTATE_URB_ALLOC_MESH, .num_dw = 3 },
2439 { .instr = CMD_3DSTATE_MESH_SHADER, .num_dw = 8 },
2440 { .instr = CMD_3DSTATE_DRAWING_RECTANGLE, .num_dw = 4 },
2441 };
2442
xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue * q,u32 * cs)2443 u32 *xe_lrc_emit_hwe_state_instructions(struct xe_exec_queue *q, u32 *cs)
2444 {
2445 struct xe_gt *gt = q->hwe->gt;
2446 struct xe_device *xe = gt_to_xe(gt);
2447 const struct instr_state *state_table = NULL;
2448 int state_table_size = 0;
2449
2450 /*
2451 * Wa_14019789679
2452 *
2453 * If the driver doesn't explicitly emit the SVG instructions while
2454 * setting up the default LRC, the context switch will write 0's
2455 * (noops) into the LRC memory rather than the expected instruction
2456 * headers. Application contexts start out as a copy of the default
2457 * LRC, and if they also do not emit specific settings for some SVG
2458 * state, then on context restore they'll unintentionally inherit
2459 * whatever state setting the previous context had programmed into the
2460 * hardware (i.e., the lack of a 3DSTATE_* instruction in the LRC will
2461 * prevent the hardware from resetting that state back to any specific
2462 * value).
2463 *
2464 * The official workaround only requires emitting 3DSTATE_MESH_CONTROL
2465 * since that's a specific state setting that can easily cause GPU
2466 * hangs if unintentionally inherited. However to be safe we'll
2467 * continue to emit all of the SVG state since it's best not to leak
2468 * any of the state between contexts, even if that leakage is harmless.
2469 */
2470 if (XE_GT_WA(gt, 14019789679) && q->hwe->class == XE_ENGINE_CLASS_RENDER) {
2471 state_table = xe_hpg_svg_state;
2472 state_table_size = ARRAY_SIZE(xe_hpg_svg_state);
2473 }
2474
2475 if (!state_table) {
2476 xe_gt_dbg(gt, "No non-register state to emit on graphics ver %d.%02d\n",
2477 GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100);
2478 return cs;
2479 }
2480
2481 for (int i = 0; i < state_table_size; i++) {
2482 u32 instr = state_table[i].instr;
2483 u16 num_dw = state_table[i].num_dw;
2484 bool is_single_dw = ((instr & GFXPIPE_PIPELINE) == PIPELINE_SINGLE_DW);
2485
2486 xe_gt_assert(gt, (instr & XE_INSTR_CMD_TYPE) == XE_INSTR_GFXPIPE);
2487 xe_gt_assert(gt, num_dw != 0);
2488 xe_gt_assert(gt, is_single_dw ^ (num_dw > 1));
2489
2490 /*
2491 * Xe2's SVG context is the same as the one on DG2 / MTL
2492 * except that 3DSTATE_DRAWING_RECTANGLE (non-pipelined) has
2493 * been replaced by 3DSTATE_DRAWING_RECTANGLE_FAST (pipelined).
2494 * Just make the replacement here rather than defining a
2495 * whole separate table for the single trivial change.
2496 */
2497 if (GRAPHICS_VER(xe) >= 20 &&
2498 instr == CMD_3DSTATE_DRAWING_RECTANGLE)
2499 instr = CMD_3DSTATE_DRAWING_RECTANGLE_FAST;
2500
2501 *cs = instr;
2502 if (!is_single_dw)
2503 *cs |= (num_dw - 2);
2504
2505 cs += num_dw;
2506 }
2507
2508 return cs;
2509 }
2510
xe_lrc_snapshot_capture(struct xe_lrc * lrc)2511 struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc)
2512 {
2513 struct xe_lrc_snapshot *snapshot = kmalloc_obj(*snapshot, GFP_NOWAIT);
2514
2515 if (!snapshot)
2516 return NULL;
2517
2518 snapshot->context_desc = xe_lrc_ggtt_addr(lrc);
2519 snapshot->ring_addr = __xe_lrc_ring_ggtt_addr(lrc);
2520 snapshot->indirect_context_desc = xe_lrc_indirect_ring_ggtt_addr(lrc);
2521 snapshot->head = xe_lrc_ring_head(lrc);
2522 snapshot->tail.internal = lrc->ring.tail;
2523 snapshot->tail.memory = xe_lrc_ring_tail(lrc);
2524 snapshot->start = xe_lrc_ring_start(lrc);
2525 snapshot->start_seqno = xe_lrc_start_seqno(lrc);
2526 snapshot->seqno = xe_lrc_seqno(lrc);
2527 snapshot->lrc_bo = xe_bo_get(lrc->bo);
2528 snapshot->lrc_offset = xe_lrc_pphwsp_offset(lrc);
2529 snapshot->lrc_size = lrc->size;
2530 snapshot->replay_offset = 0;
2531 snapshot->replay_size = lrc->replay_size;
2532 snapshot->lrc_snapshot = NULL;
2533 snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc);
2534 snapshot->ctx_timestamp_ms =
2535 xe_gt_clock_interval_to_ms(lrc->gt, xe_lrc_ctx_timestamp(lrc));
2536 if (xe_lrc_is_multi_queue(lrc)) {
2537 snapshot->queue_timestamp = xe_lrc_queue_timestamp(lrc);
2538 snapshot->queue_timestamp_ms =
2539 xe_gt_clock_interval_to_ms(lrc->gt, snapshot->queue_timestamp);
2540 } else {
2541 snapshot->queue_timestamp = 0;
2542 snapshot->queue_timestamp_ms = 0;
2543 }
2544 snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc);
2545 return snapshot;
2546 }
2547
xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot * snapshot)2548 void xe_lrc_snapshot_capture_delayed(struct xe_lrc_snapshot *snapshot)
2549 {
2550 struct xe_bo *bo;
2551 struct iosys_map src;
2552
2553 if (!snapshot)
2554 return;
2555
2556 bo = snapshot->lrc_bo;
2557 snapshot->lrc_bo = NULL;
2558
2559 snapshot->lrc_snapshot = kvmalloc(snapshot->lrc_size, GFP_KERNEL);
2560 if (!snapshot->lrc_snapshot)
2561 goto put_bo;
2562
2563 xe_bo_lock(bo, false);
2564 if (!ttm_bo_vmap(&bo->ttm, &src)) {
2565 xe_map_memcpy_from(xe_bo_device(bo),
2566 snapshot->lrc_snapshot, &src, snapshot->lrc_offset,
2567 snapshot->lrc_size);
2568 ttm_bo_vunmap(&bo->ttm, &src);
2569 } else {
2570 kvfree(snapshot->lrc_snapshot);
2571 snapshot->lrc_snapshot = NULL;
2572 }
2573 xe_bo_unlock(bo);
2574 put_bo:
2575 xe_bo_put(bo);
2576 }
2577
xe_lrc_snapshot_print(struct xe_lrc_snapshot * snapshot,struct drm_printer * p)2578 void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer *p)
2579 {
2580 unsigned long i;
2581
2582 if (!snapshot)
2583 return;
2584
2585 drm_printf(p, "\tHW Context Desc: 0x%08x\n", snapshot->context_desc);
2586 drm_printf(p, "\tHW Ring address: 0x%08x\n",
2587 snapshot->ring_addr);
2588 drm_printf(p, "\tHW Indirect Ring State: 0x%08x\n",
2589 snapshot->indirect_context_desc);
2590 drm_printf(p, "\tLRC Head: (memory) %u\n", snapshot->head);
2591 drm_printf(p, "\tLRC Tail: (internal) %u, (memory) %u\n",
2592 snapshot->tail.internal, snapshot->tail.memory);
2593 drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start);
2594 drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno);
2595 drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno);
2596 drm_printf(p, "\tTimestamp: 0x%016llx\n", snapshot->ctx_timestamp);
2597 drm_printf(p, "\tTimestamp ms: %llu\n", snapshot->ctx_timestamp_ms);
2598 drm_printf(p, "\tQueue Timestamp: 0x%016llx\n", snapshot->queue_timestamp);
2599 drm_printf(p, "\tQueue Timestamp ms: %llu\n", snapshot->queue_timestamp_ms);
2600 drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp);
2601
2602 if (!snapshot->lrc_snapshot)
2603 return;
2604
2605 drm_printf(p, "\t[HWSP].length: 0x%x\n", LRC_PPHWSP_SIZE);
2606 drm_puts(p, "\t[HWSP].data: ");
2607 for (i = 0; i < LRC_PPHWSP_SIZE; i += sizeof(u32)) {
2608 u32 *val = snapshot->lrc_snapshot + i;
2609 char dumped[ASCII85_BUFSZ];
2610
2611 drm_puts(p, ascii85_encode(*val, dumped));
2612 }
2613
2614 drm_printf(p, "\n\t[HWCTX].length: 0x%lx\n", snapshot->lrc_size - LRC_PPHWSP_SIZE);
2615 drm_printf(p, "\n\t[HWCTX].replay_offset: 0x%lx\n", snapshot->replay_offset);
2616 drm_printf(p, "\n\t[HWCTX].replay_length: 0x%lx\n", snapshot->replay_size);
2617
2618 drm_puts(p, "\t[HWCTX].data: ");
2619 for (; i < snapshot->lrc_size; i += sizeof(u32)) {
2620 u32 *val = snapshot->lrc_snapshot + i;
2621 char dumped[ASCII85_BUFSZ];
2622
2623 drm_puts(p, ascii85_encode(*val, dumped));
2624 }
2625 drm_puts(p, "\n");
2626 }
2627
xe_lrc_snapshot_free(struct xe_lrc_snapshot * snapshot)2628 void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot)
2629 {
2630 if (!snapshot)
2631 return;
2632
2633 kvfree(snapshot->lrc_snapshot);
2634 if (snapshot->lrc_bo)
2635 xe_bo_put(snapshot->lrc_bo);
2636
2637 kfree(snapshot);
2638 }
2639
engine_valid_for_utilization(struct xe_gt * gt,struct xe_hw_engine * hwe)2640 static bool engine_valid_for_utilization(struct xe_gt *gt, struct xe_hw_engine *hwe)
2641 {
2642 /* The USM-reserved copy engine runs kernel migrate contexts queried here */
2643 return hwe && (!xe_hw_engine_is_reserved(hwe) || xe_gt_is_usm_hwe(gt, hwe));
2644 }
2645
engine_id_to_hwe(struct xe_gt * gt,u32 engine_id)2646 static struct xe_hw_engine *engine_id_to_hwe(struct xe_gt *gt, u32 engine_id)
2647 {
2648 u16 class = REG_FIELD_GET(ENGINE_CLASS_ID, engine_id);
2649 u16 instance = REG_FIELD_GET(ENGINE_INSTANCE_ID, engine_id);
2650 struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, class, instance, false);
2651
2652 if (xe_gt_WARN_ONCE(gt, !engine_valid_for_utilization(gt, hwe),
2653 "Unexpected engine class:instance %d:%d for utilization\n",
2654 class, instance))
2655 return NULL;
2656
2657 return hwe;
2658 }
2659
get_ctx_timestamp(struct xe_lrc * lrc,u32 engine_id,u64 * reg_ctx_ts)2660 static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts)
2661 {
2662 struct xe_hw_engine *hwe;
2663 u64 val;
2664
2665 hwe = engine_id_to_hwe(lrc->gt, engine_id);
2666 if (!hwe)
2667 return -1;
2668
2669 if (lrc_to_xe(lrc)->info.has_64bit_timestamp)
2670 val = xe_mmio_read64_2x32(&hwe->gt->mmio,
2671 RING_CTX_TIMESTAMP(hwe->mmio_base));
2672 else
2673 val = xe_mmio_read32(&hwe->gt->mmio,
2674 RING_CTX_TIMESTAMP(hwe->mmio_base));
2675
2676 *reg_ctx_ts = val;
2677
2678 return 0;
2679 }
2680
get_queue_timestamp(struct xe_hw_engine * hwe)2681 static u64 get_queue_timestamp(struct xe_hw_engine *hwe)
2682 {
2683 return xe_mmio_read64_2x32(&hwe->gt->mmio,
2684 RING_QUEUE_TIMESTAMP(hwe->mmio_base));
2685 }
2686
get_multi_queue_active_queue_id(struct xe_hw_engine * hwe)2687 static u32 get_multi_queue_active_queue_id(struct xe_hw_engine *hwe)
2688 {
2689 u32 val = xe_mmio_read32(&hwe->gt->mmio,
2690 RING_CSMQDEBUG(hwe->mmio_base));
2691
2692 return REG_FIELD_GET(CURRENT_ACTIVE_QUEUE_ID_MASK, val);
2693 }
2694
context_active(struct xe_lrc * lrc)2695 static bool context_active(struct xe_lrc *lrc)
2696 {
2697 return xe_lrc_ctx_timestamp(lrc) == CONTEXT_ACTIVE;
2698 }
2699
xe_lrc_multi_queue_timestamp(struct xe_lrc * lrc)2700 static u64 xe_lrc_multi_queue_timestamp(struct xe_lrc *lrc)
2701 {
2702 struct xe_device *xe = lrc_to_xe(lrc);
2703 struct xe_lrc *primary_lrc = lrc->multi_queue.primary_lrc;
2704 struct xe_hw_engine *hwe;
2705 u64 reg_queue_ts = lrc->queue_timestamp;
2706
2707 if (IS_SRIOV_VF(xe))
2708 return xe_lrc_queue_timestamp(lrc);
2709
2710 xe_assert(xe, primary_lrc);
2711
2712 /* WA BB populates CONTEXT_ACTIVE cookie for primary context only */
2713 if (!context_active(primary_lrc))
2714 return xe_lrc_queue_timestamp(lrc);
2715
2716 /* WA BB populates engine id in PPHWSP of primary context only */
2717 hwe = engine_id_to_hwe(primary_lrc->gt, xe_lrc_engine_id(primary_lrc));
2718 if (!hwe)
2719 return xe_lrc_queue_timestamp(lrc);
2720
2721 if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos)
2722 return xe_lrc_queue_timestamp(lrc);
2723
2724 /* queue is active, so store the queue timestamp register */
2725 reg_queue_ts = get_queue_timestamp(hwe);
2726
2727 /* double check queue and primary queue are both still active */
2728 if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos ||
2729 !context_active(primary_lrc))
2730 return xe_lrc_queue_timestamp(lrc);
2731
2732 return reg_queue_ts;
2733 }
2734
xe_lrc_update_multi_queue_timestamp(struct xe_lrc * lrc,u64 * old_ts)2735 static u64 xe_lrc_update_multi_queue_timestamp(struct xe_lrc *lrc, u64 *old_ts)
2736 {
2737 *old_ts = lrc->queue_timestamp;
2738 lrc->queue_timestamp = xe_lrc_multi_queue_timestamp(lrc);
2739
2740 trace_xe_lrc_update_queue_timestamp(lrc, *old_ts);
2741
2742 return lrc->queue_timestamp;
2743 }
2744
xe_lrc_context_timestamp(struct xe_lrc * lrc)2745 static u64 xe_lrc_context_timestamp(struct xe_lrc *lrc)
2746 {
2747 u64 reg_ts, new_ts = lrc->ctx_timestamp;
2748 u64 stored;
2749
2750 /* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */
2751 if (IS_SRIOV_VF(lrc_to_xe(lrc)))
2752 return xe_lrc_ctx_timestamp(lrc);
2753
2754 /* Safely read CTX_TIMESTAMP: Avoid TOCTOU on LRC-stored CONTEXT_ACTIVE sentinel */
2755 stored = xe_lrc_ctx_timestamp(lrc);
2756 if (stored != CONTEXT_ACTIVE)
2757 return stored;
2758
2759 /* Context is active: read the live timestamp from the engine's MMIO register */
2760 if (!get_ctx_timestamp(lrc, xe_lrc_engine_id(lrc), ®_ts))
2761 new_ts = reg_ts;
2762
2763 /* If the context switched out prefer using the value
2764 * from context-save over the stale MMIO read.
2765 */
2766 stored = xe_lrc_ctx_timestamp(lrc);
2767 if (stored != CONTEXT_ACTIVE)
2768 return stored;
2769
2770 return new_ts;
2771 }
2772
xe_lrc_update_context_timestamp(struct xe_lrc * lrc,u64 * old_ts)2773 static u64 xe_lrc_update_context_timestamp(struct xe_lrc *lrc, u64 *old_ts)
2774 {
2775 *old_ts = lrc->ctx_timestamp;
2776 lrc->ctx_timestamp = xe_lrc_context_timestamp(lrc);
2777
2778 trace_xe_lrc_update_timestamp(lrc, *old_ts);
2779
2780 return lrc->ctx_timestamp;
2781 }
2782
2783 /**
2784 * xe_lrc_timestamp() - Current lrc timestamp
2785 * @lrc: Pointer to the lrc.
2786 *
2787 * Return latest lrc timestamp. With support for active contexts/queues, the
2788 * calculation may be slightly racy, so follow a read-again logic to ensure that
2789 * the context/queue is still active before returning the right timestamp.
2790 *
2791 * Returns: New lrc timestamp value
2792 */
xe_lrc_timestamp(struct xe_lrc * lrc)2793 u64 xe_lrc_timestamp(struct xe_lrc *lrc)
2794 {
2795 if (xe_lrc_is_multi_queue(lrc))
2796 return xe_lrc_multi_queue_timestamp(lrc);
2797 else
2798 return xe_lrc_context_timestamp(lrc);
2799 }
2800
2801 /**
2802 * xe_lrc_update_timestamp() - Update lrc timestamp
2803 * @lrc: Pointer to the lrc.
2804 * @old_ts: Old timestamp value
2805 *
2806 * Populate @old_ts with current saved lrc timestamp, read new lrc timestamp and
2807 * update saved value.
2808 *
2809 * Returns: New lrc timestamp value
2810 */
xe_lrc_update_timestamp(struct xe_lrc * lrc,u64 * old_ts)2811 u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts)
2812 {
2813 if (xe_lrc_is_multi_queue(lrc))
2814 return xe_lrc_update_multi_queue_timestamp(lrc, old_ts);
2815 else
2816 return xe_lrc_update_context_timestamp(lrc, old_ts);
2817 }
2818
2819 /**
2820 * xe_lrc_ring_is_idle() - LRC is idle
2821 * @lrc: Pointer to the lrc.
2822 *
2823 * Compare LRC ring head and tail to determine if idle.
2824 *
2825 * Return: True is ring is idle, False otherwise
2826 */
xe_lrc_ring_is_idle(struct xe_lrc * lrc)2827 bool xe_lrc_ring_is_idle(struct xe_lrc *lrc)
2828 {
2829 return xe_lrc_ring_head(lrc) == xe_lrc_ring_tail(lrc);
2830 }
2831