1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2021 Intel Corporation
4 */
5
6 #include "xe_hw_engine.h"
7
8 #include <linux/nospec.h>
9
10 #include <drm/drm_managed.h>
11 #include <drm/drm_print.h>
12 #include <uapi/drm/xe_drm.h>
13 #include <generated/xe_wa_oob.h>
14
15 #include "regs/xe_engine_regs.h"
16 #include "regs/xe_gt_regs.h"
17 #include "regs/xe_irq_regs.h"
18 #include "xe_assert.h"
19 #include "xe_bo.h"
20 #include "xe_configfs.h"
21 #include "xe_device.h"
22 #include "xe_execlist.h"
23 #include "xe_force_wake.h"
24 #include "xe_gsc.h"
25 #include "xe_gt.h"
26 #include "xe_gt_ccs_mode.h"
27 #include "xe_gt_clock.h"
28 #include "xe_gt_printk.h"
29 #include "xe_gt_mcr.h"
30 #include "xe_gt_topology.h"
31 #include "xe_guc_capture.h"
32 #include "xe_hw_engine_group.h"
33 #include "xe_hw_fence.h"
34 #include "xe_irq.h"
35 #include "xe_lrc.h"
36 #include "xe_mmio.h"
37 #include "xe_reg_sr.h"
38 #include "xe_reg_whitelist.h"
39 #include "xe_rtp.h"
40 #include "xe_sched_job.h"
41 #include "xe_sriov.h"
42 #include "xe_tuning.h"
43 #include "xe_uc_fw.h"
44 #include "xe_wa.h"
45
46 #define MAX_MMIO_BASES 3
47 struct engine_info {
48 const char *name;
49 unsigned int class : 8;
50 unsigned int instance : 8;
51 unsigned int irq_offset : 8;
52 enum xe_force_wake_domains domain;
53 u32 mmio_base;
54 };
55
56 static const struct engine_info engine_infos[] = {
57 [XE_HW_ENGINE_RCS0] = {
58 .name = "rcs0",
59 .class = XE_ENGINE_CLASS_RENDER,
60 .instance = 0,
61 .irq_offset = ilog2(INTR_RCS0),
62 .domain = XE_FW_RENDER,
63 .mmio_base = RENDER_RING_BASE,
64 },
65 [XE_HW_ENGINE_BCS0] = {
66 .name = "bcs0",
67 .class = XE_ENGINE_CLASS_COPY,
68 .instance = 0,
69 .irq_offset = ilog2(INTR_BCS(0)),
70 .domain = XE_FW_GT,
71 .mmio_base = BLT_RING_BASE,
72 },
73 [XE_HW_ENGINE_BCS1] = {
74 .name = "bcs1",
75 .class = XE_ENGINE_CLASS_COPY,
76 .instance = 1,
77 .irq_offset = ilog2(INTR_BCS(1)),
78 .domain = XE_FW_GT,
79 .mmio_base = XEHPC_BCS1_RING_BASE,
80 },
81 [XE_HW_ENGINE_BCS2] = {
82 .name = "bcs2",
83 .class = XE_ENGINE_CLASS_COPY,
84 .instance = 2,
85 .irq_offset = ilog2(INTR_BCS(2)),
86 .domain = XE_FW_GT,
87 .mmio_base = XEHPC_BCS2_RING_BASE,
88 },
89 [XE_HW_ENGINE_BCS3] = {
90 .name = "bcs3",
91 .class = XE_ENGINE_CLASS_COPY,
92 .instance = 3,
93 .irq_offset = ilog2(INTR_BCS(3)),
94 .domain = XE_FW_GT,
95 .mmio_base = XEHPC_BCS3_RING_BASE,
96 },
97 [XE_HW_ENGINE_BCS4] = {
98 .name = "bcs4",
99 .class = XE_ENGINE_CLASS_COPY,
100 .instance = 4,
101 .irq_offset = ilog2(INTR_BCS(4)),
102 .domain = XE_FW_GT,
103 .mmio_base = XEHPC_BCS4_RING_BASE,
104 },
105 [XE_HW_ENGINE_BCS5] = {
106 .name = "bcs5",
107 .class = XE_ENGINE_CLASS_COPY,
108 .instance = 5,
109 .irq_offset = ilog2(INTR_BCS(5)),
110 .domain = XE_FW_GT,
111 .mmio_base = XEHPC_BCS5_RING_BASE,
112 },
113 [XE_HW_ENGINE_BCS6] = {
114 .name = "bcs6",
115 .class = XE_ENGINE_CLASS_COPY,
116 .instance = 6,
117 .irq_offset = ilog2(INTR_BCS(6)),
118 .domain = XE_FW_GT,
119 .mmio_base = XEHPC_BCS6_RING_BASE,
120 },
121 [XE_HW_ENGINE_BCS7] = {
122 .name = "bcs7",
123 .class = XE_ENGINE_CLASS_COPY,
124 .irq_offset = ilog2(INTR_BCS(7)),
125 .instance = 7,
126 .domain = XE_FW_GT,
127 .mmio_base = XEHPC_BCS7_RING_BASE,
128 },
129 [XE_HW_ENGINE_BCS8] = {
130 .name = "bcs8",
131 .class = XE_ENGINE_CLASS_COPY,
132 .instance = 8,
133 .irq_offset = ilog2(INTR_BCS8),
134 .domain = XE_FW_GT,
135 .mmio_base = XEHPC_BCS8_RING_BASE,
136 },
137
138 [XE_HW_ENGINE_VCS0] = {
139 .name = "vcs0",
140 .class = XE_ENGINE_CLASS_VIDEO_DECODE,
141 .instance = 0,
142 .irq_offset = 32 + ilog2(INTR_VCS(0)),
143 .domain = XE_FW_MEDIA_VDBOX0,
144 .mmio_base = BSD_RING_BASE,
145 },
146 [XE_HW_ENGINE_VCS1] = {
147 .name = "vcs1",
148 .class = XE_ENGINE_CLASS_VIDEO_DECODE,
149 .instance = 1,
150 .irq_offset = 32 + ilog2(INTR_VCS(1)),
151 .domain = XE_FW_MEDIA_VDBOX1,
152 .mmio_base = BSD2_RING_BASE,
153 },
154 [XE_HW_ENGINE_VCS2] = {
155 .name = "vcs2",
156 .class = XE_ENGINE_CLASS_VIDEO_DECODE,
157 .instance = 2,
158 .irq_offset = 32 + ilog2(INTR_VCS(2)),
159 .domain = XE_FW_MEDIA_VDBOX2,
160 .mmio_base = BSD3_RING_BASE,
161 },
162 [XE_HW_ENGINE_VCS3] = {
163 .name = "vcs3",
164 .class = XE_ENGINE_CLASS_VIDEO_DECODE,
165 .instance = 3,
166 .irq_offset = 32 + ilog2(INTR_VCS(3)),
167 .domain = XE_FW_MEDIA_VDBOX3,
168 .mmio_base = BSD4_RING_BASE,
169 },
170 [XE_HW_ENGINE_VCS4] = {
171 .name = "vcs4",
172 .class = XE_ENGINE_CLASS_VIDEO_DECODE,
173 .instance = 4,
174 .irq_offset = 32 + ilog2(INTR_VCS(4)),
175 .domain = XE_FW_MEDIA_VDBOX4,
176 .mmio_base = XEHP_BSD5_RING_BASE,
177 },
178 [XE_HW_ENGINE_VCS5] = {
179 .name = "vcs5",
180 .class = XE_ENGINE_CLASS_VIDEO_DECODE,
181 .instance = 5,
182 .irq_offset = 32 + ilog2(INTR_VCS(5)),
183 .domain = XE_FW_MEDIA_VDBOX5,
184 .mmio_base = XEHP_BSD6_RING_BASE,
185 },
186 [XE_HW_ENGINE_VCS6] = {
187 .name = "vcs6",
188 .class = XE_ENGINE_CLASS_VIDEO_DECODE,
189 .instance = 6,
190 .irq_offset = 32 + ilog2(INTR_VCS(6)),
191 .domain = XE_FW_MEDIA_VDBOX6,
192 .mmio_base = XEHP_BSD7_RING_BASE,
193 },
194 [XE_HW_ENGINE_VCS7] = {
195 .name = "vcs7",
196 .class = XE_ENGINE_CLASS_VIDEO_DECODE,
197 .instance = 7,
198 .irq_offset = 32 + ilog2(INTR_VCS(7)),
199 .domain = XE_FW_MEDIA_VDBOX7,
200 .mmio_base = XEHP_BSD8_RING_BASE,
201 },
202 [XE_HW_ENGINE_VECS0] = {
203 .name = "vecs0",
204 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE,
205 .instance = 0,
206 .irq_offset = 32 + ilog2(INTR_VECS(0)),
207 .domain = XE_FW_MEDIA_VEBOX0,
208 .mmio_base = VEBOX_RING_BASE,
209 },
210 [XE_HW_ENGINE_VECS1] = {
211 .name = "vecs1",
212 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE,
213 .instance = 1,
214 .irq_offset = 32 + ilog2(INTR_VECS(1)),
215 .domain = XE_FW_MEDIA_VEBOX1,
216 .mmio_base = VEBOX2_RING_BASE,
217 },
218 [XE_HW_ENGINE_VECS2] = {
219 .name = "vecs2",
220 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE,
221 .instance = 2,
222 .irq_offset = 32 + ilog2(INTR_VECS(2)),
223 .domain = XE_FW_MEDIA_VEBOX2,
224 .mmio_base = XEHP_VEBOX3_RING_BASE,
225 },
226 [XE_HW_ENGINE_VECS3] = {
227 .name = "vecs3",
228 .class = XE_ENGINE_CLASS_VIDEO_ENHANCE,
229 .instance = 3,
230 .irq_offset = 32 + ilog2(INTR_VECS(3)),
231 .domain = XE_FW_MEDIA_VEBOX3,
232 .mmio_base = XEHP_VEBOX4_RING_BASE,
233 },
234 [XE_HW_ENGINE_CCS0] = {
235 .name = "ccs0",
236 .class = XE_ENGINE_CLASS_COMPUTE,
237 .instance = 0,
238 .irq_offset = ilog2(INTR_CCS(0)),
239 .domain = XE_FW_RENDER,
240 .mmio_base = COMPUTE0_RING_BASE,
241 },
242 [XE_HW_ENGINE_CCS1] = {
243 .name = "ccs1",
244 .class = XE_ENGINE_CLASS_COMPUTE,
245 .instance = 1,
246 .irq_offset = ilog2(INTR_CCS(1)),
247 .domain = XE_FW_RENDER,
248 .mmio_base = COMPUTE1_RING_BASE,
249 },
250 [XE_HW_ENGINE_CCS2] = {
251 .name = "ccs2",
252 .class = XE_ENGINE_CLASS_COMPUTE,
253 .instance = 2,
254 .irq_offset = ilog2(INTR_CCS(2)),
255 .domain = XE_FW_RENDER,
256 .mmio_base = COMPUTE2_RING_BASE,
257 },
258 [XE_HW_ENGINE_CCS3] = {
259 .name = "ccs3",
260 .class = XE_ENGINE_CLASS_COMPUTE,
261 .instance = 3,
262 .irq_offset = ilog2(INTR_CCS(3)),
263 .domain = XE_FW_RENDER,
264 .mmio_base = COMPUTE3_RING_BASE,
265 },
266 [XE_HW_ENGINE_GSCCS0] = {
267 .name = "gsccs0",
268 .class = XE_ENGINE_CLASS_OTHER,
269 .instance = OTHER_GSC_INSTANCE,
270 .domain = XE_FW_GSC,
271 .mmio_base = GSCCS_RING_BASE,
272 },
273 };
274
hw_engine_fini(void * arg)275 static void hw_engine_fini(void *arg)
276 {
277 struct xe_hw_engine *hwe = arg;
278
279 if (hwe->exl_port)
280 xe_execlist_port_destroy(hwe->exl_port);
281
282 hwe->gt = NULL;
283 }
284
285 /**
286 * xe_hw_engine_mmio_read32() - Read engine register
287 * @hwe: engine
288 * @reg: register to read from
289 *
290 * This function will read from an engine specific register.
291 * Forcewake must be held by the caller.
292 *
293 * Return: value of the 32-bit register.
294 */
xe_hw_engine_mmio_read32(struct xe_hw_engine * hwe,struct xe_reg reg)295 u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg)
296 {
297 xe_gt_assert(hwe->gt, !(reg.addr & hwe->mmio_base));
298 xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain);
299
300 reg.addr += hwe->mmio_base;
301
302 return xe_mmio_read32(&hwe->gt->mmio, reg);
303 }
304
xe_hw_engine_enable_ring(struct xe_hw_engine * hwe)305 void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe)
306 {
307 xe_mmio_write32(&hwe->gt->mmio, RING_HWS_PGA(hwe->mmio_base),
308 xe_bo_ggtt_addr(hwe->hwsp));
309 }
310
xe_hw_engine_match_fixed_cslice_mode(const struct xe_device * xe,const struct xe_gt * gt,const struct xe_hw_engine * hwe)311 static bool xe_hw_engine_match_fixed_cslice_mode(const struct xe_device *xe,
312 const struct xe_gt *gt,
313 const struct xe_hw_engine *hwe)
314 {
315 /*
316 * Xe3p no longer supports load balance mode, so "fixed cslice" mode
317 * is automatic and no RCU_MODE programming is required.
318 */
319 if (GRAPHICS_VER(gt_to_xe(gt)) >= 35)
320 return false;
321
322 return xe_gt_ccs_mode_enabled(gt) &&
323 xe_rtp_match_first_render_or_compute(xe, gt, hwe);
324 }
325
xe_rtp_cfeg_wmtp_disabled(const struct xe_device * xe,const struct xe_gt * gt,const struct xe_hw_engine * hwe)326 static bool xe_rtp_cfeg_wmtp_disabled(const struct xe_device *xe,
327 const struct xe_gt *gt,
328 const struct xe_hw_engine *hwe)
329 {
330 if (GRAPHICS_VER(xe) < 20)
331 return false;
332
333 if (hwe->class != XE_ENGINE_CLASS_COMPUTE &&
334 hwe->class != XE_ENGINE_CLASS_RENDER)
335 return false;
336
337 return xe_mmio_read32(&hwe->gt->mmio, XEHP_FUSE4) & CFEG_WMTP_DISABLE;
338 }
339
blit_cctl_val(struct xe_gt * gt,struct xe_hw_engine * hwe)340 static u32 blit_cctl_val(struct xe_gt *gt, struct xe_hw_engine *hwe)
341 {
342 return REG_FIELD_PREP(BLIT_CCTL_DST_MOCS_MASK, gt->mocs.uc_index) |
343 REG_FIELD_PREP(BLIT_CCTL_SRC_MOCS_MASK, gt->mocs.uc_index);
344 }
345
346 static const struct xe_rtp_table_sr lrc_setup = XE_RTP_TABLE_SR(
347 /*
348 * Some blitter commands do not have a field for MOCS, those
349 * commands will use MOCS index pointed by BLIT_CCTL.
350 * BLIT_CCTL registers are needed to be programmed to un-cached.
351 */
352 { XE_RTP_NAME("BLIT_CCTL_default_MOCS"),
353 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1274),
354 ENGINE_CLASS(COPY)),
355 XE_RTP_ACTIONS(FIELD_SET_FUNC(BLIT_CCTL(0),
356 BLIT_CCTL_DST_MOCS_MASK |
357 BLIT_CCTL_SRC_MOCS_MASK,
358 blit_cctl_val,
359 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
360 },
361 /* Disable WMTP if HW doesn't support it */
362 { XE_RTP_NAME("DISABLE_WMTP_ON_UNSUPPORTED_HW"),
363 XE_RTP_RULES(FUNC(xe_rtp_cfeg_wmtp_disabled)),
364 XE_RTP_ACTIONS(FIELD_SET(CS_CHICKEN1(0),
365 PREEMPT_GPGPU_LEVEL_MASK,
366 PREEMPT_GPGPU_THREAD_GROUP_LEVEL)),
367 XE_RTP_ENTRY_FLAG(FOREACH_ENGINE)
368 },
369 );
370
371 static void
hw_engine_setup_default_lrc_state(struct xe_hw_engine * hwe)372 hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe)
373 {
374 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);
375
376 xe_rtp_process_to_sr(&ctx, &lrc_setup, &hwe->reg_lrc, true);
377 }
378
xe_hw_engine_setup_reg_lrc(struct xe_hw_engine * hwe)379 void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe)
380 {
381 struct xe_gt *gt = hwe->gt;
382 struct xe_device *xe = gt_to_xe(gt);
383
384 xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe);
385 xe_wa_process_lrc(hwe);
386 hw_engine_setup_default_lrc_state(hwe);
387 xe_tuning_process_lrc(hwe);
388 }
389
390 /*
391 * RING_CMD_CCTL specifies the default MOCS entry that will be
392 * used by the command streamer when executing commands that
393 * don't have a way to explicitly specify a MOCS setting.
394 * The default should usually reference whichever MOCS entry
395 * corresponds to uncached behavior, although use of a WB cached
396 * entry is recommended by the spec in certain circumstances on
397 * specific platforms.
398 * Bspec: 72161
399 */
ring_cmd_cctl_val(struct xe_gt * gt,struct xe_hw_engine * hwe)400 static u32 ring_cmd_cctl_val(struct xe_gt *gt, struct xe_hw_engine *hwe)
401 {
402 struct xe_device *xe = gt_to_xe(gt);
403 u8 mocs_read_idx = gt->mocs.uc_index;
404
405 if (hwe->class == XE_ENGINE_CLASS_COMPUTE && IS_DGFX(xe) &&
406 (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC))
407 mocs_read_idx = gt->mocs.wb_index;
408
409 return REG_FIELD_PREP(CMD_CCTL_WRITE_OVERRIDE_MASK, gt->mocs.uc_index) |
410 REG_FIELD_PREP(CMD_CCTL_READ_OVERRIDE_MASK, mocs_read_idx);
411 }
412
413 static const struct xe_rtp_table_sr engine_sr = XE_RTP_TABLE_SR(
414 { XE_RTP_NAME("RING_CMD_CCTL_default_MOCS"),
415 XE_RTP_RULES(FUNC(xe_rtp_match_always)),
416 XE_RTP_ACTIONS(FIELD_SET_FUNC(RING_CMD_CCTL(0),
417 CMD_CCTL_WRITE_OVERRIDE_MASK |
418 CMD_CCTL_READ_OVERRIDE_MASK,
419 ring_cmd_cctl_val,
420 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
421 },
422 { XE_RTP_NAME("Disable HW status page updates for interrupts"),
423 XE_RTP_RULES(FUNC(xe_rtp_match_always)),
424 XE_RTP_ACTIONS(SET(RING_HWSTAM(0), ~0x0,
425 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
426 },
427 { XE_RTP_NAME("Disable engine 'legacy' mode"),
428 XE_RTP_RULES(FUNC(xe_rtp_match_always)),
429 XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_DISABLE_LEGACY_MODE,
430 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
431 },
432 /*
433 * To allow the GSC engine to go idle on MTL we need to enable
434 * idle messaging and set the hysteresis value (we use 0xA=5us
435 * as recommended in spec). On platforms after MTL this is
436 * enabled by default.
437 */
438 { XE_RTP_NAME("MTL GSCCS IDLE MSG enable"),
439 XE_RTP_RULES(MEDIA_VERSION(1300), ENGINE_CLASS(OTHER)),
440 XE_RTP_ACTIONS(CLR(RING_PSMI_CTL(0),
441 IDLE_MSG_DISABLE,
442 XE_RTP_ACTION_FLAG(ENGINE_BASE)),
443 FIELD_SET(RING_PWRCTX_MAXCNT(0),
444 IDLE_WAIT_TIME,
445 0xA,
446 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
447 },
448 /* Enable Priority Mem Read */
449 { XE_RTP_NAME("Priority_Mem_Read"),
450 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED)),
451 XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0), CS_PRIORITY_MEM_READ,
452 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
453 },
454 { XE_RTP_NAME("Enable CCS Engine(s)"),
455 XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, XE_RTP_END_VERSION_UNDEFINED),
456 FUNC(xe_rtp_match_first_render_or_compute)),
457 XE_RTP_ACTIONS(SET(RCU_MODE, RCU_MODE_CCS_ENABLE))
458 },
459 /* Use Fixed slice CCS mode */
460 { XE_RTP_NAME("RCU_MODE_FIXED_SLICE_CCS_MODE"),
461 XE_RTP_RULES(FUNC(xe_hw_engine_match_fixed_cslice_mode)),
462 XE_RTP_ACTIONS(FIELD_SET(RCU_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE,
463 RCU_MODE_FIXED_SLICE_CCS_MODE))
464 },
465 { XE_RTP_NAME("Enable MSI-X interrupt support"),
466 XE_RTP_RULES(FUNC(xe_rtp_match_has_msix)),
467 XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_MSIX_INTERRUPT_ENABLE,
468 XE_RTP_ACTION_FLAG(ENGINE_BASE)))
469 },
470 );
471
472 static void
hw_engine_setup_default_state(struct xe_hw_engine * hwe)473 hw_engine_setup_default_state(struct xe_hw_engine *hwe)
474 {
475 struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);
476
477 xe_rtp_process_to_sr(&ctx, &engine_sr, &hwe->reg_sr, false);
478 }
479
find_engine_info(enum xe_engine_class class,int instance)480 static const struct engine_info *find_engine_info(enum xe_engine_class class, int instance)
481 {
482 const struct engine_info *info;
483 enum xe_hw_engine_id id;
484
485 for (id = 0; id < XE_NUM_HW_ENGINES; ++id) {
486 info = &engine_infos[id];
487 if (info->class == class && info->instance == instance)
488 return info;
489 }
490
491 return NULL;
492 }
493
get_msix_irq_offset(struct xe_gt * gt,enum xe_engine_class class)494 static u16 get_msix_irq_offset(struct xe_gt *gt, enum xe_engine_class class)
495 {
496 /* For MSI-X, hw engines report to offset of engine instance zero */
497 const struct engine_info *info = find_engine_info(class, 0);
498
499 xe_gt_assert(gt, info);
500
501 return info ? info->irq_offset : 0;
502 }
503
hw_engine_init_early(struct xe_gt * gt,struct xe_hw_engine * hwe,enum xe_hw_engine_id id)504 static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe,
505 enum xe_hw_engine_id id)
506 {
507 const struct engine_info *info;
508
509 if (WARN_ON(id >= ARRAY_SIZE(engine_infos) || !engine_infos[id].name))
510 return;
511
512 if (!(gt->info.engine_mask & BIT(id)))
513 return;
514
515 info = &engine_infos[id];
516
517 xe_gt_assert(gt, !hwe->gt);
518
519 hwe->gt = gt;
520 hwe->class = info->class;
521 hwe->instance = info->instance;
522 hwe->mmio_base = info->mmio_base;
523 if (xe_device_has_msix(gt_to_xe(gt))) {
524 hwe->irq_offset = get_msix_irq_offset(gt, info->class);
525 hwe->irq_page = info->instance;
526
527 } else {
528 hwe->irq_offset = info->irq_offset;
529 hwe->irq_page = 0;
530 }
531 hwe->domain = info->domain;
532 hwe->name = info->name;
533 hwe->fence_irq = >->fence_irq[info->class];
534 hwe->engine_id = id;
535
536 hwe->eclass = >->eclass[hwe->class];
537 if (!hwe->eclass->sched_props.job_timeout_ms) {
538 hwe->eclass->sched_props.job_timeout_ms = 5 * 1000;
539 hwe->eclass->sched_props.job_timeout_min = XE_HW_ENGINE_JOB_TIMEOUT_MIN;
540 hwe->eclass->sched_props.job_timeout_max = XE_HW_ENGINE_JOB_TIMEOUT_MAX;
541 hwe->eclass->sched_props.timeslice_us = 1 * 1000;
542 hwe->eclass->sched_props.timeslice_min = XE_HW_ENGINE_TIMESLICE_MIN;
543 hwe->eclass->sched_props.timeslice_max = XE_HW_ENGINE_TIMESLICE_MAX;
544 hwe->eclass->sched_props.preempt_timeout_us = XE_HW_ENGINE_PREEMPT_TIMEOUT;
545 hwe->eclass->sched_props.preempt_timeout_min = XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN;
546 hwe->eclass->sched_props.preempt_timeout_max = XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX;
547
548 /*
549 * The GSC engine can accept submissions while the GSC shim is
550 * being reset, during which time the submission is stalled. In
551 * the worst case, the shim reset can take up to the maximum GSC
552 * command execution time (250ms), so the request start can be
553 * delayed by that much; the request itself can take that long
554 * without being preemptible, which means worst case it can
555 * theoretically take up to 500ms for a preemption to go through
556 * on the GSC engine. Adding to that an extra 100ms as a safety
557 * margin, we get a minimum recommended timeout of 600ms.
558 * The preempt_timeout value can't be tuned for OTHER_CLASS
559 * because the class is reserved for kernel usage, so we just
560 * need to make sure that the starting value is above that
561 * threshold; since our default value (640ms) is greater than
562 * 600ms, the only way we can go below is via a kconfig setting.
563 * If that happens, log it in dmesg and update the value.
564 */
565 if (hwe->class == XE_ENGINE_CLASS_OTHER) {
566 const u32 min_preempt_timeout = 600 * 1000;
567 if (hwe->eclass->sched_props.preempt_timeout_us < min_preempt_timeout) {
568 hwe->eclass->sched_props.preempt_timeout_us = min_preempt_timeout;
569 xe_gt_notice(gt, "Increasing preempt_timeout for GSC to 600ms\n");
570 }
571 }
572
573 /* Record default props */
574 hwe->eclass->defaults = hwe->eclass->sched_props;
575 }
576
577 xe_reg_sr_init(&hwe->reg_sr, hwe->name, gt_to_xe(gt));
578 xe_tuning_process_engine(hwe);
579 xe_wa_process_engine(hwe);
580 hw_engine_setup_default_state(hwe);
581
582 xe_reg_sr_init(&hwe->reg_whitelist, hwe->name, gt_to_xe(gt));
583 xe_reg_sr_init(&hwe->oa_whitelist, hwe->name, gt_to_xe(gt));
584 xe_reg_sr_init(&hwe->oa_sr, hwe->name, gt_to_xe(gt));
585 xe_reg_whitelist_process_engine(hwe);
586 }
587
idledly_floor_ticks(u32 idledly_ns,u32 idledly_units_ps)588 static u32 idledly_floor_ticks(u32 idledly_ns, u32 idledly_units_ps)
589 {
590 return DIV_ROUND_DOWN_ULL((u64)idledly_ns * 1000, idledly_units_ps);
591 }
592
adjust_idledly(struct xe_hw_engine * hwe)593 static void adjust_idledly(struct xe_hw_engine *hwe)
594 {
595 struct xe_gt *gt = hwe->gt;
596 u32 idledly, idledly_hw, idledly_reg_val, maxcnt;
597 u32 idledly_units_ps = 8 * gt->info.timestamp_base;
598 u32 maxcnt_units_ns = 640;
599 bool inhibit_switch = false;
600 bool wa_applied = false;
601 bool clamped_below_maxcnt = false;
602
603 if ((!IS_SRIOV_VF(gt_to_xe(gt)) && XE_GT_WA(gt, 16023105232)) ||
604 XE_GT_WA(gt, 14025941587)) {
605 u32 mincnt_idledly_ns = 5000;
606
607 /* xe_gt_clock_init() warns and zeroes timestamp_base on unknown crystal clock. */
608 if (!idledly_units_ps)
609 return;
610
611 idledly_reg_val = xe_mmio_read32(>->mmio, RING_IDLEDLY(hwe->mmio_base));
612 maxcnt = xe_mmio_read32(>->mmio, RING_PWRCTX_MAXCNT(hwe->mmio_base));
613
614 inhibit_switch = idledly_reg_val & INHIBIT_SWITCH_UNTIL_PREEMPTED;
615 idledly = REG_FIELD_GET(IDLE_DELAY, idledly_reg_val);
616 idledly = DIV_ROUND_CLOSEST_ULL((u64)idledly * idledly_units_ps, 1000);
617 idledly_hw = idledly;
618 maxcnt = REG_FIELD_GET(IDLE_WAIT_TIME, maxcnt);
619 maxcnt *= maxcnt_units_ns;
620
621 /*
622 * Wa_14025941587 is applied before Wa_16023105232, which takes
623 * priority if the two ever conflict (not expected in practice).
624 */
625 if (XE_GT_WA(gt, 14025941587) &&
626 idledly < mincnt_idledly_ns) {
627 idledly = mincnt_idledly_ns;
628 wa_applied = true;
629 }
630
631 if (XE_GT_WA(gt, 16023105232)) {
632 /* Clear the inhibit switch without disturbing a valid delay. */
633 if (inhibit_switch) {
634 idledly_reg_val &= ~INHIBIT_SWITCH_UNTIL_PREEMPTED;
635 wa_applied = true;
636 }
637
638 /* Warn only on the value read from hardware. */
639 xe_gt_WARN_ON(gt, idledly_hw >= maxcnt);
640
641 if (idledly >= maxcnt) {
642 /* maxcnt may be 0 if IDLE_WAIT_TIME is unprogrammed. */
643 idledly = maxcnt ? maxcnt - 1 : 0;
644 clamped_below_maxcnt = true;
645 wa_applied = true;
646 }
647 }
648
649 if (wa_applied) {
650 u32 idledly_ticks;
651
652 /*
653 * Wa_16023105232 requires idledly < maxcnt, so floor
654 * that clamp; otherwise round up to guarantee the
655 * Wa_14025941587 minimum survives tick quantization.
656 */
657 if (clamped_below_maxcnt)
658 idledly_ticks = idledly_floor_ticks(idledly, idledly_units_ps);
659 else
660 idledly_ticks = DIV_ROUND_UP_ULL((u64)idledly * 1000,
661 idledly_units_ps);
662
663 /*
664 * Tick quantization can still push the rounded-up value
665 * to/above maxcnt; re-floor here so Wa_16023105232 keeps
666 * priority even in that case.
667 */
668 if (!clamped_below_maxcnt && XE_GT_WA(gt, 16023105232) &&
669 (u64)idledly_ticks * idledly_units_ps >= (u64)maxcnt * 1000) {
670 xe_gt_dbg(gt, "idledly %s: %u ticks would exceed maxcnt=%u, so flooring\n",
671 hwe->name, idledly_ticks, maxcnt);
672 idledly = maxcnt ? maxcnt - 1 : 0;
673 idledly_ticks = idledly_floor_ticks(idledly, idledly_units_ps);
674 }
675
676 idledly_reg_val &= ~IDLE_DELAY;
677 idledly_reg_val |= REG_FIELD_PREP(IDLE_DELAY, idledly_ticks);
678 xe_gt_dbg(gt, "idledly %s: set %u max=%u inh=%u ts=%u\n",
679 hwe->name, idledly, maxcnt,
680 !!inhibit_switch, gt->info.timestamp_base);
681 xe_mmio_write32(>->mmio,
682 RING_IDLEDLY(hwe->mmio_base),
683 idledly_reg_val);
684 }
685 }
686 }
687
hw_engine_init(struct xe_gt * gt,struct xe_hw_engine * hwe,enum xe_hw_engine_id id)688 static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe,
689 enum xe_hw_engine_id id)
690 {
691 struct xe_device *xe = gt_to_xe(gt);
692 struct xe_tile *tile = gt_to_tile(gt);
693 int err;
694
695 xe_gt_assert(gt, id < ARRAY_SIZE(engine_infos) && engine_infos[id].name);
696 xe_gt_assert(gt, gt->info.engine_mask & BIT(id));
697
698 xe_reg_sr_apply_mmio(&hwe->reg_sr, gt);
699
700 hwe->hwsp = xe_managed_bo_create_pin_map(xe, tile, SZ_4K,
701 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
702 XE_BO_FLAG_GGTT |
703 XE_BO_FLAG_GGTT_INVALIDATE);
704 if (IS_ERR(hwe->hwsp)) {
705 err = PTR_ERR(hwe->hwsp);
706 goto err_name;
707 }
708
709 if (!xe_device_uc_enabled(xe)) {
710 hwe->exl_port = xe_execlist_port_create(xe, hwe);
711 if (IS_ERR(hwe->exl_port)) {
712 err = PTR_ERR(hwe->exl_port);
713 goto err_name;
714 }
715 } else {
716 /* GSCCS has a special interrupt for reset */
717 if (hwe->class == XE_ENGINE_CLASS_OTHER)
718 hwe->irq_handler = xe_gsc_hwe_irq_handler;
719
720 if (!IS_SRIOV_VF(xe))
721 xe_hw_engine_enable_ring(hwe);
722 }
723
724 /* Ensure IDLEDLY is lower than MAXCNT */
725 adjust_idledly(hwe);
726
727 return devm_add_action_or_reset(xe->drm.dev, hw_engine_fini, hwe);
728
729 err_name:
730 hwe->name = NULL;
731
732 return err;
733 }
734
hw_engine_setup_logical_and_paging_mapping(struct xe_gt * gt)735 static int hw_engine_setup_logical_and_paging_mapping(struct xe_gt *gt)
736 {
737 struct xe_device *xe = gt_to_xe(gt);
738 unsigned int num_copy_engines = 0, num_paging_engines = 0;
739 unsigned int reserved_logical_bcs_start;
740 struct xe_hw_engine *hwe;
741 enum xe_hw_engine_id id;
742 int class;
743
744 for_each_hw_engine(hwe, gt, id)
745 if (hwe->class == XE_ENGINE_CLASS_COPY)
746 num_copy_engines++;
747
748 if (num_copy_engines && xe->info.has_usm)
749 num_paging_engines = 1;
750
751 if (IS_SRIOV_VF(xe)) {
752 u32 vf_num_paging_engines;
753
754 /*
755 * PF could in theory reserve multiple paging engines, which
756 * internally the submission/scheduling backend can load balance
757 * from. Not something we currently expect, but we are at the
758 * mercy of the PF, so we just need try our best to mirror the
759 * paging configuration.
760 */
761 vf_num_paging_engines = xe_gt_sriov_vf_paging_engines(gt);
762 if (vf_num_paging_engines) {
763 /* This should only be non-zero on NVL-S+ */
764 if (xe_gt_WARN_ON(gt, xe->info.platform < XE_NOVALAKE_S))
765 return -EINVAL;
766
767 num_paging_engines = vf_num_paging_engines;
768 }
769 }
770
771 if (xe_gt_WARN_ON(gt, num_paging_engines > num_copy_engines))
772 return -EINVAL;
773
774 /*
775 * On PF, we just reserve the highest BCS instance for USM.
776 *
777 * Note: This is now a requirement going forward. The PF must ALWAYS
778 * reserve BCS instances in top-down order, that way the VF has a chance
779 * of discovering the physical BCS instance mappings for paging engines,
780 * in conjunction with vf_num_paging_engines. In some places we might
781 * only have the physical instance, and from hw pov there is no such
782 * thing as a paging engine. For example, the page fault descriptor,
783 * which comes directly from the hw, will use the physical engine
784 * instance.
785 */
786 reserved_logical_bcs_start = num_copy_engines - num_paging_engines;
787
788 /* FIXME: Doing a simple logical mapping that works for most hardware */
789 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
790 int logical_instance = 0;
791
792 for_each_hw_engine(hwe, gt, id) {
793 if (hwe->class == class) {
794 hwe->logical_instance = logical_instance++;
795
796 if (class == XE_ENGINE_CLASS_COPY &&
797 hwe->logical_instance >=
798 reserved_logical_bcs_start) {
799 if (!gt->usm.paging_hwe0)
800 gt->usm.paging_hwe0 = hwe;
801 gt->usm.paging_logical_mask |=
802 BIT(hwe->logical_instance);
803 }
804 }
805 }
806 }
807
808 return 0;
809 }
810
read_media_fuses(struct xe_gt * gt)811 static void read_media_fuses(struct xe_gt *gt)
812 {
813 struct xe_device *xe = gt_to_xe(gt);
814 u32 media_fuse;
815 u16 vdbox_mask;
816 u16 vebox_mask;
817 int i, j;
818
819 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
820
821 media_fuse = xe_mmio_read32(>->mmio, GT_VEBOX_VDBOX_DISABLE);
822
823 /*
824 * Pre-Xe_HP platforms had register bits representing absent engines,
825 * whereas Xe_HP and beyond have bits representing present engines.
826 * Invert the polarity on old platforms so that we can use common
827 * handling below.
828 */
829 if (GRAPHICS_VERx100(xe) < 1250)
830 media_fuse = ~media_fuse;
831
832 vdbox_mask = REG_FIELD_GET(GT_VDBOX_DISABLE_MASK, media_fuse);
833 vebox_mask = REG_FIELD_GET(GT_VEBOX_DISABLE_MASK, media_fuse);
834
835 for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) {
836 if (!(gt->info.engine_mask & BIT(i)))
837 continue;
838
839 if (!(BIT(j) & vdbox_mask)) {
840 gt->info.engine_mask &= ~BIT(i);
841 xe_gt_info(gt, "vcs%u fused off\n", j);
842 }
843 }
844
845 for (i = XE_HW_ENGINE_VECS0, j = 0; i <= XE_HW_ENGINE_VECS3; ++i, ++j) {
846 if (!(gt->info.engine_mask & BIT(i)))
847 continue;
848
849 if (!(BIT(j) & vebox_mask)) {
850 gt->info.engine_mask &= ~BIT(i);
851 xe_gt_info(gt, "vecs%u fused off\n", j);
852 }
853 }
854 }
855
infer_svccopy_from_meml3(struct xe_gt * gt)856 static u32 infer_svccopy_from_meml3(struct xe_gt *gt)
857 {
858 u32 meml3 = REG_FIELD_GET(MEML3_EN_MASK,
859 xe_mmio_read32(>->mmio, MIRROR_FUSE3));
860 u32 svccopy_mask = 0;
861
862 /*
863 * Each of the four meml3 bits determines the fusing of two service
864 * copy engines.
865 */
866 for (int i = 0; i < 4; i++)
867 svccopy_mask |= (meml3 & BIT(i)) ? 0b11 << 2 * i : 0;
868
869 return svccopy_mask;
870 }
871
read_svccopy_fuses(struct xe_gt * gt)872 static u32 read_svccopy_fuses(struct xe_gt *gt)
873 {
874 return REG_FIELD_GET(FUSE_SERVICE_COPY_ENABLE_MASK,
875 xe_mmio_read32(>->mmio, SERVICE_COPY_ENABLE));
876 }
877
read_copy_fuses(struct xe_gt * gt)878 static void read_copy_fuses(struct xe_gt *gt)
879 {
880 struct xe_device *xe = gt_to_xe(gt);
881 u32 bcs_mask;
882
883 xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT);
884
885 if (GRAPHICS_VER(xe) >= 35)
886 bcs_mask = read_svccopy_fuses(gt);
887 else if (GRAPHICS_VERx100(xe) == 1260)
888 bcs_mask = infer_svccopy_from_meml3(gt);
889 else
890 return;
891
892 /* Only BCS1-BCS8 may be fused off */
893 bcs_mask <<= XE_HW_ENGINE_BCS1;
894 for (int i = XE_HW_ENGINE_BCS1; i <= XE_HW_ENGINE_BCS8; ++i) {
895 if (!(gt->info.engine_mask & BIT(i)))
896 continue;
897
898 if (!(bcs_mask & BIT(i))) {
899 gt->info.engine_mask &= ~BIT(i);
900 xe_gt_info(gt, "bcs%u fused off\n",
901 i - XE_HW_ENGINE_BCS0);
902 }
903 }
904 }
905
read_compute_fuses_from_dss(struct xe_gt * gt)906 static void read_compute_fuses_from_dss(struct xe_gt *gt)
907 {
908 /*
909 * CCS fusing based on DSS masks only applies to platforms that can
910 * have more than one CCS.
911 */
912 if (hweight64(gt->info.engine_mask &
913 GENMASK_ULL(XE_HW_ENGINE_CCS3, XE_HW_ENGINE_CCS0)) <= 1)
914 return;
915
916 /*
917 * CCS availability on Xe_HP is inferred from the presence of DSS in
918 * each quadrant.
919 */
920 for (int i = XE_HW_ENGINE_CCS0, j = 0; i <= XE_HW_ENGINE_CCS3; ++i, ++j) {
921 if (!(gt->info.engine_mask & BIT(i)))
922 continue;
923
924 if (!xe_gt_topology_has_dss_in_quadrant(gt, j)) {
925 gt->info.engine_mask &= ~BIT(i);
926 xe_gt_info(gt, "ccs%u fused off\n", j);
927 }
928 }
929 }
930
read_compute_fuses_from_reg(struct xe_gt * gt)931 static void read_compute_fuses_from_reg(struct xe_gt *gt)
932 {
933 u32 ccs_mask;
934
935 ccs_mask = xe_mmio_read32(>->mmio, XEHP_FUSE4);
936 ccs_mask = REG_FIELD_GET(CCS_EN_MASK, ccs_mask);
937
938 for (int i = XE_HW_ENGINE_CCS0, j = 0; i <= XE_HW_ENGINE_CCS3; ++i, ++j) {
939 if (!(gt->info.engine_mask & BIT(i)))
940 continue;
941
942 if ((ccs_mask & BIT(j)) == 0) {
943 gt->info.engine_mask &= ~BIT(i);
944 xe_gt_info(gt, "ccs%u fused off\n", j);
945 }
946 }
947 }
948
read_compute_fuses(struct xe_gt * gt)949 static void read_compute_fuses(struct xe_gt *gt)
950 {
951 if (GRAPHICS_VER(gt_to_xe(gt)) >= 20)
952 read_compute_fuses_from_reg(gt);
953 else
954 read_compute_fuses_from_dss(gt);
955 }
956
check_gsc_availability(struct xe_gt * gt)957 static void check_gsc_availability(struct xe_gt *gt)
958 {
959 if (!(gt->info.engine_mask & BIT(XE_HW_ENGINE_GSCCS0)))
960 return;
961
962 /*
963 * The GSCCS is only used to communicate with the GSC FW, so if we don't
964 * have the FW there is nothing we need the engine for and can therefore
965 * skip its initialization.
966 */
967 if (!xe_uc_fw_is_available(>->uc.gsc.fw)) {
968 gt->info.engine_mask &= ~BIT(XE_HW_ENGINE_GSCCS0);
969
970 /* interrupts where previously enabled, so turn them off */
971 xe_mmio_write32(>->mmio, GUNIT_GSC_INTR_ENABLE, 0);
972 xe_mmio_write32(>->mmio, GUNIT_GSC_INTR_MASK, ~0);
973
974 xe_gt_dbg(gt, "GSC FW not used, disabling gsccs\n");
975 }
976 }
977
check_sw_disable(struct xe_gt * gt)978 static void check_sw_disable(struct xe_gt *gt)
979 {
980 struct xe_device *xe = gt_to_xe(gt);
981 u64 sw_allowed = xe_configfs_get_engines_allowed(to_pci_dev(xe->drm.dev));
982 enum xe_hw_engine_id id;
983
984 for (id = 0; id < XE_NUM_HW_ENGINES; ++id) {
985 if (!(gt->info.engine_mask & BIT(id)))
986 continue;
987
988 if (!(sw_allowed & BIT(id))) {
989 gt->info.engine_mask &= ~BIT(id);
990 xe_gt_info(gt, "%s disabled via configfs\n",
991 engine_infos[id].name);
992 }
993 }
994 }
995
xe_hw_engines_init_early(struct xe_gt * gt)996 int xe_hw_engines_init_early(struct xe_gt *gt)
997 {
998 int i;
999
1000 read_media_fuses(gt);
1001 read_copy_fuses(gt);
1002 read_compute_fuses(gt);
1003 check_gsc_availability(gt);
1004 check_sw_disable(gt);
1005
1006 BUILD_BUG_ON(XE_HW_ENGINE_PREEMPT_TIMEOUT < XE_HW_ENGINE_PREEMPT_TIMEOUT_MIN);
1007 BUILD_BUG_ON(XE_HW_ENGINE_PREEMPT_TIMEOUT > XE_HW_ENGINE_PREEMPT_TIMEOUT_MAX);
1008
1009 for (i = 0; i < ARRAY_SIZE(gt->hw_engines); i++)
1010 hw_engine_init_early(gt, >->hw_engines[i], i);
1011
1012 return 0;
1013 }
1014
xe_hw_engines_init(struct xe_gt * gt)1015 int xe_hw_engines_init(struct xe_gt *gt)
1016 {
1017 int err;
1018 struct xe_hw_engine *hwe;
1019 enum xe_hw_engine_id id;
1020
1021 for_each_hw_engine(hwe, gt, id) {
1022 err = hw_engine_init(gt, hwe, id);
1023 if (err)
1024 return err;
1025 }
1026
1027 err = hw_engine_setup_logical_and_paging_mapping(gt);
1028 if (err)
1029 return err;
1030
1031 err = xe_hw_engine_setup_groups(gt);
1032 if (err)
1033 return err;
1034
1035 return 0;
1036 }
1037
xe_hw_engine_handle_irq(struct xe_hw_engine * hwe,u16 intr_vec)1038 void xe_hw_engine_handle_irq(struct xe_hw_engine *hwe, u16 intr_vec)
1039 {
1040 wake_up_all(>_to_xe(hwe->gt)->ufence_wq);
1041
1042 if (hwe->irq_handler)
1043 hwe->irq_handler(hwe, intr_vec);
1044
1045 if (intr_vec & GT_MI_USER_INTERRUPT)
1046 xe_hw_fence_irq_run(hwe->fence_irq);
1047 }
1048
1049 /**
1050 * xe_hw_engine_snapshot_capture - Take a quick snapshot of the HW Engine.
1051 * @hwe: Xe HW Engine.
1052 * @q: The exec queue object.
1053 *
1054 * This can be printed out in a later stage like during dev_coredump
1055 * analysis.
1056 *
1057 * Returns: a Xe HW Engine snapshot object that must be freed by the
1058 * caller, using `xe_hw_engine_snapshot_free`.
1059 */
1060 struct xe_hw_engine_snapshot *
xe_hw_engine_snapshot_capture(struct xe_hw_engine * hwe,struct xe_exec_queue * q)1061 xe_hw_engine_snapshot_capture(struct xe_hw_engine *hwe, struct xe_exec_queue *q)
1062 {
1063 struct xe_hw_engine_snapshot *snapshot;
1064 struct __guc_capture_parsed_output *node;
1065
1066 if (!xe_hw_engine_is_valid(hwe))
1067 return NULL;
1068
1069 snapshot = kzalloc_obj(*snapshot, GFP_ATOMIC);
1070
1071 if (!snapshot)
1072 return NULL;
1073
1074 snapshot->name = kstrdup(hwe->name, GFP_ATOMIC);
1075 snapshot->hwe = hwe;
1076 snapshot->logical_instance = hwe->logical_instance;
1077 snapshot->forcewake.domain = hwe->domain;
1078 snapshot->forcewake.ref = xe_force_wake_ref(gt_to_fw(hwe->gt),
1079 hwe->domain);
1080 snapshot->mmio_base = hwe->mmio_base;
1081 snapshot->kernel_reserved = xe_hw_engine_is_reserved(hwe);
1082
1083 /* no more VF accessible data below this point */
1084 if (IS_SRIOV_VF(gt_to_xe(hwe->gt)))
1085 return snapshot;
1086
1087 if (q) {
1088 /* If got guc capture, set source to GuC */
1089 node = xe_guc_capture_get_matching_and_lock(q);
1090 if (node) {
1091 struct xe_device *xe = gt_to_xe(hwe->gt);
1092 struct xe_devcoredump *coredump = &xe->devcoredump;
1093
1094 coredump->snapshot.matched_node = node;
1095 xe_gt_dbg(hwe->gt, "Found and locked GuC-err-capture node");
1096 return snapshot;
1097 }
1098 }
1099
1100 /* otherwise, do manual capture */
1101 xe_engine_manual_capture(hwe, snapshot);
1102 xe_gt_dbg(hwe->gt, "Proceeding with manual engine snapshot");
1103
1104 return snapshot;
1105 }
1106
1107 /**
1108 * xe_hw_engine_snapshot_free - Free all allocated objects for a given snapshot.
1109 * @snapshot: Xe HW Engine snapshot object.
1110 *
1111 * This function free all the memory that needed to be allocated at capture
1112 * time.
1113 */
xe_hw_engine_snapshot_free(struct xe_hw_engine_snapshot * snapshot)1114 void xe_hw_engine_snapshot_free(struct xe_hw_engine_snapshot *snapshot)
1115 {
1116 struct xe_gt *gt;
1117 if (!snapshot)
1118 return;
1119
1120 gt = snapshot->hwe->gt;
1121 /*
1122 * xe_guc_capture_put_matched_nodes is called here and from
1123 * xe_devcoredump_snapshot_free, to cover the 2 calling paths
1124 * of hw_engines - debugfs and devcoredump free.
1125 */
1126 xe_guc_capture_put_matched_nodes(>->uc.guc);
1127
1128 kfree(snapshot->name);
1129 kfree(snapshot);
1130 }
1131
1132 /**
1133 * xe_hw_engine_print - Xe HW Engine Print.
1134 * @hwe: Hardware Engine.
1135 * @p: drm_printer.
1136 *
1137 * This function quickly capture a snapshot and immediately print it out.
1138 */
xe_hw_engine_print(struct xe_hw_engine * hwe,struct drm_printer * p)1139 void xe_hw_engine_print(struct xe_hw_engine *hwe, struct drm_printer *p)
1140 {
1141 struct xe_hw_engine_snapshot *snapshot;
1142
1143 snapshot = xe_hw_engine_snapshot_capture(hwe, NULL);
1144 xe_engine_snapshot_print(snapshot, p);
1145 xe_hw_engine_snapshot_free(snapshot);
1146 }
1147
xe_hw_engine_mask_per_class(struct xe_gt * gt,enum xe_engine_class engine_class)1148 u32 xe_hw_engine_mask_per_class(struct xe_gt *gt,
1149 enum xe_engine_class engine_class)
1150 {
1151 u32 mask = 0;
1152 enum xe_hw_engine_id id;
1153
1154 for (id = 0; id < XE_NUM_HW_ENGINES; ++id) {
1155 if (engine_infos[id].class == engine_class &&
1156 gt->info.engine_mask & BIT(id))
1157 mask |= BIT(engine_infos[id].instance);
1158 }
1159 return mask;
1160 }
1161
xe_hw_engine_is_reserved(struct xe_hw_engine * hwe)1162 bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe)
1163 {
1164 struct xe_gt *gt = hwe->gt;
1165 struct xe_device *xe = gt_to_xe(gt);
1166
1167 if (xe_device_is_admin_only(xe))
1168 return true;
1169
1170 if (hwe->class == XE_ENGINE_CLASS_OTHER)
1171 return true;
1172
1173 /* Check for engines disabled by ccs_mode setting */
1174 if (xe_gt_ccs_mode_enabled(gt) &&
1175 hwe->class == XE_ENGINE_CLASS_COMPUTE &&
1176 hwe->logical_instance >= gt->ccs_mode)
1177 return true;
1178
1179 return xe_gt_is_usm_hwe(gt, hwe);
1180 }
1181
xe_hw_engine_class_to_str(enum xe_engine_class class)1182 const char *xe_hw_engine_class_to_str(enum xe_engine_class class)
1183 {
1184 switch (class) {
1185 case XE_ENGINE_CLASS_RENDER:
1186 return "rcs";
1187 case XE_ENGINE_CLASS_VIDEO_DECODE:
1188 return "vcs";
1189 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
1190 return "vecs";
1191 case XE_ENGINE_CLASS_COPY:
1192 return "bcs";
1193 case XE_ENGINE_CLASS_OTHER:
1194 return "other";
1195 case XE_ENGINE_CLASS_COMPUTE:
1196 return "ccs";
1197 case XE_ENGINE_CLASS_MAX:
1198 break;
1199 }
1200
1201 return NULL;
1202 }
1203
xe_hw_engine_read_timestamp(struct xe_hw_engine * hwe)1204 u64 xe_hw_engine_read_timestamp(struct xe_hw_engine *hwe)
1205 {
1206 return xe_mmio_read64_2x32(&hwe->gt->mmio, RING_TIMESTAMP(hwe->mmio_base));
1207 }
1208
xe_hw_engine_to_fw_domain(struct xe_hw_engine * hwe)1209 enum xe_force_wake_domains xe_hw_engine_to_fw_domain(struct xe_hw_engine *hwe)
1210 {
1211 return engine_infos[hwe->engine_id].domain;
1212 }
1213
1214 static const enum xe_engine_class user_to_xe_engine_class[] = {
1215 [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER,
1216 [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY,
1217 [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE,
1218 [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE,
1219 [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE,
1220 };
1221
1222 /**
1223 * xe_hw_engine_lookup() - Lookup hardware engine for class:instance
1224 * @xe: xe device
1225 * @eci: engine class and instance
1226 *
1227 * This function will find a hardware engine for given engine
1228 * class and instance.
1229 *
1230 * Return: If found xe_hw_engine pointer, NULL otherwise.
1231 */
1232 struct xe_hw_engine *
xe_hw_engine_lookup(struct xe_device * xe,struct drm_xe_engine_class_instance eci)1233 xe_hw_engine_lookup(struct xe_device *xe,
1234 struct drm_xe_engine_class_instance eci)
1235 {
1236 struct xe_gt *gt = xe_device_get_gt(xe, eci.gt_id);
1237 unsigned int idx;
1238
1239 if (eci.engine_class >= ARRAY_SIZE(user_to_xe_engine_class))
1240 return NULL;
1241
1242 if (!gt)
1243 return NULL;
1244
1245 idx = array_index_nospec(eci.engine_class,
1246 ARRAY_SIZE(user_to_xe_engine_class));
1247
1248 return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id),
1249 user_to_xe_engine_class[idx],
1250 eci.engine_instance, true);
1251 }
1252