1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_ring_ops.h"
7
8 #include <generated/xe_wa_oob.h>
9
10 #include "instructions/xe_gpu_commands.h"
11 #include "instructions/xe_mi_commands.h"
12 #include "regs/xe_engine_regs.h"
13 #include "regs/xe_gt_regs.h"
14 #include "xe_exec_queue.h"
15 #include "xe_gt_types.h"
16 #include "xe_lrc.h"
17 #include "xe_sched_job.h"
18 #include "xe_sriov.h"
19 #include "xe_vm_types.h"
20 #include "xe_vm.h"
21 #include "xe_wa.h"
22
23 /*
24 * 3D-related flags that can't be set on _engines_ that lack access to the 3D
25 * pipeline (i.e., CCS engines).
26 */
27 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\
28 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \
29 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
30 PIPE_CONTROL_TILE_CACHE_FLUSH | \
31 PIPE_CONTROL_DEPTH_STALL | \
32 PIPE_CONTROL_STALL_AT_SCOREBOARD | \
33 PIPE_CONTROL_PSD_SYNC | \
34 PIPE_CONTROL_AMFS_FLUSH | \
35 PIPE_CONTROL_VF_CACHE_INVALIDATE | \
36 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET)
37
38 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */
39 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \
40 PIPE_CONTROL_3D_ENGINE_FLAGS | \
41 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \
42 PIPE_CONTROL_FLUSH_ENABLE | \
43 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
44 PIPE_CONTROL_DC_FLUSH_ENABLE)
45
preparser_disable(bool state)46 static u32 preparser_disable(bool state)
47 {
48 return MI_ARB_CHECK | BIT(8) | state;
49 }
50
51 static u32 *
__emit_aux_table_inv(u32 * cmd,const struct xe_reg reg,u32 adj_offset)52 __emit_aux_table_inv(u32 *cmd, const struct xe_reg reg, u32 adj_offset)
53 {
54 *cmd++ = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) |
55 MI_LRI_MMIO_REMAP_EN;
56 *cmd++ = reg.addr + adj_offset;
57 *cmd++ = AUX_INV;
58 *cmd++ = MI_SEMAPHORE_WAIT_TOKEN | MI_SEMAPHORE_REGISTER_POLL |
59 MI_SEMAPHORE_POLL | MI_SEMAPHORE_SAD_EQ_SDD;
60 *cmd++ = 0;
61 *cmd++ = reg.addr + adj_offset;
62 *cmd++ = 0;
63 *cmd++ = 0;
64
65 return cmd;
66 }
67
emit_aux_table_inv_render_compute(struct xe_gt * gt,u32 * cmd)68 static u32 *emit_aux_table_inv_render_compute(struct xe_gt *gt, u32 *cmd)
69 {
70 return __emit_aux_table_inv(cmd, CCS_AUX_INV, gt->mmio.adj_offset);
71 }
72
emit_aux_table_inv_video_decode(struct xe_gt * gt,u32 * cmd)73 static u32 *emit_aux_table_inv_video_decode(struct xe_gt *gt, u32 *cmd)
74 {
75 return __emit_aux_table_inv(cmd, VD0_AUX_INV, gt->mmio.adj_offset);
76 }
77
emit_aux_table_inv_video_enhance(struct xe_gt * gt,u32 * cmd)78 static u32 *emit_aux_table_inv_video_enhance(struct xe_gt *gt, u32 *cmd)
79 {
80 return __emit_aux_table_inv(cmd, VE0_AUX_INV, gt->mmio.adj_offset);
81 }
82
emit_aux_table_inv(struct xe_hw_engine * hwe,u32 * dw,int i)83 static int emit_aux_table_inv(struct xe_hw_engine *hwe, u32 *dw, int i)
84 {
85 struct xe_gt *gt = hwe->gt;
86 u32 *(*emit)(struct xe_gt *gt, u32 *cmd) =
87 gt->ring_ops[hwe->class]->emit_aux_table_inv;
88
89 if (emit)
90 return emit(gt, dw + i) - dw;
91 else
92 return i;
93 }
94
emit_user_interrupt(u32 * dw,int i)95 static int emit_user_interrupt(u32 *dw, int i)
96 {
97 dw[i++] = MI_USER_INTERRUPT;
98 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
99 dw[i++] = MI_ARB_CHECK;
100
101 return i;
102 }
103
emit_store_imm_ggtt(u32 addr,u32 value,u32 * dw,int i)104 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)
105 {
106 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
107 dw[i++] = addr;
108 dw[i++] = 0;
109 dw[i++] = value;
110
111 return i;
112 }
113
emit_flush_dw(u32 * dw,int i)114 static int emit_flush_dw(u32 *dw, int i)
115 {
116 dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;
117 dw[i++] = 0;
118 dw[i++] = 0;
119 dw[i++] = 0;
120
121 return i;
122 }
123
emit_flush_imm_ggtt(u32 addr,u32 value,u32 flags,u32 * dw,int i)124 static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i)
125 {
126 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
127 flags;
128 dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
129 dw[i++] = 0;
130 dw[i++] = value;
131
132 return i;
133 }
134
emit_bb_start(u64 batch_addr,u32 ppgtt_flag,u32 * dw,int i)135 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
136 {
137 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);
138 dw[i++] = lower_32_bits(batch_addr);
139 dw[i++] = upper_32_bits(batch_addr);
140
141 return i;
142 }
143
emit_flush_invalidate(u32 addr,u32 val,u32 flush_flags,u32 * dw,int i)144 static int emit_flush_invalidate(u32 addr, u32 val, u32 flush_flags, u32 *dw, int i)
145 {
146 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW |
147 MI_FLUSH_IMM_DW | (flush_flags & MI_INVALIDATE_TLB) ?: 0;
148
149 dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
150 dw[i++] = 0;
151 dw[i++] = val;
152
153 return i;
154 }
155
156 static int
emit_pipe_control(u32 * dw,int i,u32 bit_group_0,u32 bit_group_1,u32 offset,u32 value)157 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)
158 {
159 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;
160 dw[i++] = bit_group_1;
161 dw[i++] = offset;
162 dw[i++] = 0;
163 dw[i++] = value;
164 dw[i++] = 0;
165
166 return i;
167 }
168
emit_pipe_invalidate(struct xe_exec_queue * q,u32 mask_flags,bool invalidate_tlb,u32 * dw,int i)169 static int emit_pipe_invalidate(struct xe_exec_queue *q, u32 mask_flags,
170 bool invalidate_tlb, u32 *dw, int i)
171 {
172 u32 flags0 = 0;
173 u32 flags1 = PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
174 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
175 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
176 PIPE_CONTROL_VF_CACHE_INVALIDATE |
177 PIPE_CONTROL_CONST_CACHE_INVALIDATE |
178 PIPE_CONTROL_STATE_CACHE_INVALIDATE |
179 PIPE_CONTROL_QW_WRITE |
180 PIPE_CONTROL_STORE_DATA_INDEX;
181
182 if (invalidate_tlb)
183 flags1 |= PIPE_CONTROL_TLB_INVALIDATE;
184
185 if (xe_exec_queue_is_multi_queue(q))
186 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
187 else
188 flags1 |= PIPE_CONTROL_CS_STALL;
189
190 flags1 &= ~mask_flags;
191
192 if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE)
193 flags0 |= PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE;
194
195 return emit_pipe_control(dw, i, flags0, flags1,
196 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
197 }
198
emit_store_imm_ppgtt_posted(u64 addr,u64 value,u32 * dw,int i)199 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
200 u32 *dw, int i)
201 {
202 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);
203 dw[i++] = lower_32_bits(addr);
204 dw[i++] = upper_32_bits(addr);
205 dw[i++] = lower_32_bits(value);
206 dw[i++] = upper_32_bits(value);
207
208 return i;
209 }
210
emit_render_cache_flush(struct xe_sched_job * job,u32 * dw,int i)211 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
212 {
213 struct xe_exec_queue *q = job->q;
214 struct xe_gt *gt = q->gt;
215 struct xe_device *xe = gt_to_xe(gt);
216 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
217 u32 flags0, flags1;
218
219 if (XE_GT_WA(gt, 14016712196))
220 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,
221 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
222
223 flags0 = PIPE_CONTROL0_HDC_PIPELINE_FLUSH;
224 /*
225 * Prior to MTL, HDC Pipeline Flush reliably also flushes the LSC
226 * untyped L1 dataport cache, provided HDC_CHICKEN0 is programmed
227 * correctly. Starting with MTL that coupling no longer holds
228 * regardless of how HDC_CHICKEN0 is programmed, but explicitly
229 * requesting the flush via PIPE_CONTROL is itself only reliable
230 * from Xe2 onward, so only gate it in on Xe2+.
231 */
232 if (GRAPHICS_VERx100(xe) >= 2000)
233 flags0 |= PIPE_CONTROL0_UNTYPED_DATAPORT_CACHE_FLUSH;
234 flags1 = (PIPE_CONTROL_TILE_CACHE_FLUSH |
235 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
236 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
237 PIPE_CONTROL_DC_FLUSH_ENABLE |
238 PIPE_CONTROL_FLUSH_ENABLE);
239
240 if (XE_GT_WA(gt, 1409600907))
241 flags1 |= PIPE_CONTROL_DEPTH_STALL;
242
243 if (lacks_render)
244 flags1 &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
245 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
246 flags1 &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
247
248 if (xe_exec_queue_is_multi_queue(q))
249 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
250 else
251 flags1 |= PIPE_CONTROL_CS_STALL;
252
253 return emit_pipe_control(dw, i, flags0, flags1, 0, 0);
254 }
255
emit_pipe_imm_ggtt(struct xe_exec_queue * q,u32 addr,u32 value,bool stall_only,u32 * dw,int i)256 static int emit_pipe_imm_ggtt(struct xe_exec_queue *q, u32 addr, u32 value,
257 bool stall_only, u32 *dw, int i)
258 {
259 u32 flags0 = 0, flags1 = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE;
260
261 if (!stall_only)
262 flags1 |= PIPE_CONTROL_FLUSH_ENABLE;
263
264 if (xe_exec_queue_is_multi_queue(q))
265 flags0 |= PIPE_CONTROL0_QUEUE_DRAIN_MODE;
266 else
267 flags1 |= PIPE_CONTROL_CS_STALL;
268
269 return emit_pipe_control(dw, i, flags0, flags1, addr, value);
270 }
271
get_ppgtt_flag(struct xe_sched_job * job)272 static u32 get_ppgtt_flag(struct xe_sched_job *job)
273 {
274 if (job->q->vm && !job->ggtt)
275 return BIT(8);
276
277 return 0;
278 }
279
emit_copy_timestamp(struct xe_device * xe,struct xe_lrc * lrc,u32 * dw,int i)280 static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc,
281 u32 *dw, int i)
282 {
283 const struct xe_reg reg = xe_lrc_is_multi_queue(lrc) ?
284 RING_QUEUE_TIMESTAMP(0) :
285 RING_CTX_TIMESTAMP(0);
286
287 dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET;
288 dw[i++] = reg.addr;
289 dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);
290 dw[i++] = 0;
291
292 /*
293 * Ensure CTX timestamp >= Job timestamp during VF sampling to avoid
294 * arithmetic wraparound in TDR.
295 */
296 if (IS_SRIOV_VF(xe)) {
297 dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT |
298 MI_SRM_ADD_CS_OFFSET;
299 dw[i++] = reg.addr;
300 dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc);
301 dw[i++] = 0;
302 }
303
304 return i;
305 }
306
emit_fake_watchdog(struct xe_lrc * lrc,u32 * dw,int i)307 static int emit_fake_watchdog(struct xe_lrc *lrc, u32 *dw, int i)
308 {
309 /*
310 * Setup a watchdog with impossible condition to always trigger an
311 * hardware interrupt that would force the GuC to reset the engine.
312 */
313
314 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(2) | MI_LRI_LRM_CS_MMIO;
315 dw[i++] = PR_CTR_THRSH(0).addr;
316 dw[i++] = 2; /* small threshold */
317 dw[i++] = PR_CTR_CTRL(0).addr;
318 dw[i++] = CTR_LOGIC_OP(START);
319
320 dw[i++] = MI_SEMAPHORE_WAIT | MI_SEMW_GGTT | MI_SEMW_POLL | MI_SEMW_COMPARE(SAD_EQ_SDD);
321 dw[i++] = 0xdead; /* this should never be seen */
322 dw[i++] = lower_32_bits(xe_lrc_ggtt_addr(lrc));
323 dw[i++] = upper_32_bits(xe_lrc_ggtt_addr(lrc));
324 dw[i++] = 0; /* unused token */
325
326 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_LRM_CS_MMIO;
327 dw[i++] = PR_CTR_CTRL(0).addr;
328 dw[i++] = CTR_LOGIC_OP(STOP);
329
330 return i;
331 }
332
333 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */
__emit_job_gen12_simple(struct xe_sched_job * job,struct xe_lrc * lrc,u64 batch_addr,u32 * head,u32 seqno)334 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc,
335 u64 batch_addr, u32 *head, u32 seqno)
336 {
337 u32 dw[MAX_JOB_SIZE_DW], i = 0;
338 u32 ppgtt_flag = get_ppgtt_flag(job);
339 struct xe_gt *gt = job->q->gt;
340
341 *head = lrc->ring.tail;
342
343 if (job->ring_ops_force_reset)
344 i = emit_fake_watchdog(lrc, dw, i);
345
346 i = emit_copy_timestamp(gt_to_xe(gt), lrc, dw, i);
347
348 if (job->ring_ops_flush_tlb) {
349 dw[i++] = preparser_disable(true);
350 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
351 seqno, MI_INVALIDATE_TLB, dw, i);
352 dw[i++] = preparser_disable(false);
353 } else {
354 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
355 seqno, dw, i);
356 }
357
358 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
359
360 /* Don't preempt fence signaling */
361 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE;
362
363 if (job->user_fence.used) {
364 i = emit_flush_dw(dw, i);
365 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
366 job->user_fence.value,
367 dw, i);
368 }
369
370 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);
371
372 i = emit_user_interrupt(dw, i);
373
374 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
375
376 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
377 }
378
has_aux_ccs(struct xe_device * xe)379 static bool has_aux_ccs(struct xe_device *xe)
380 {
381 /*
382 * PVC is a special case that has no compression of either type
383 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2
384 * onward, so any future platforms with no FlatCCS will not have
385 * AuxCCS, and we explicitly do not want to support it on MTL.
386 */
387 if (GRAPHICS_VERx100(xe) >= 1270 || xe->info.platform == XE_PVC)
388 return false;
389
390 return !xe->info.has_flat_ccs;
391 }
392
__emit_job_gen12_video(struct xe_sched_job * job,struct xe_lrc * lrc,u64 batch_addr,u32 * head,u32 seqno)393 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc,
394 u64 batch_addr, u32 *head, u32 seqno)
395 {
396 u32 dw[MAX_JOB_SIZE_DW], i = 0;
397 u32 ppgtt_flag = get_ppgtt_flag(job);
398 struct xe_gt *gt = job->q->gt;
399 struct xe_device *xe = gt_to_xe(gt);
400
401 *head = lrc->ring.tail;
402
403 if (job->ring_ops_force_reset)
404 i = emit_fake_watchdog(lrc, dw, i);
405
406 i = emit_copy_timestamp(xe, lrc, dw, i);
407
408 dw[i++] = preparser_disable(true);
409
410 /* hsdes: 1809175790 */
411 i = emit_aux_table_inv(job->q->hwe, dw, i);
412
413 if (job->ring_ops_flush_tlb)
414 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
415 seqno, MI_INVALIDATE_TLB, dw, i);
416
417 dw[i++] = preparser_disable(false);
418
419 if (!job->ring_ops_flush_tlb)
420 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
421 seqno, dw, i);
422
423 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
424
425 /* Don't preempt fence signaling */
426 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE;
427
428 if (job->user_fence.used) {
429 i = emit_flush_dw(dw, i);
430 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
431 job->user_fence.value,
432 dw, i);
433 }
434
435 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);
436
437 i = emit_user_interrupt(dw, i);
438
439 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
440
441 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
442 }
443
__emit_job_gen12_render_compute(struct xe_sched_job * job,struct xe_lrc * lrc,u64 batch_addr,u32 * head,u32 seqno)444 static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
445 struct xe_lrc *lrc,
446 u64 batch_addr, u32 *head,
447 u32 seqno)
448 {
449 u32 dw[MAX_JOB_SIZE_DW], i = 0;
450 u32 ppgtt_flag = get_ppgtt_flag(job);
451 struct xe_gt *gt = job->q->gt;
452 struct xe_device *xe = gt_to_xe(gt);
453 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
454 u32 mask_flags = 0;
455
456 *head = lrc->ring.tail;
457
458 if (job->ring_ops_force_reset)
459 i = emit_fake_watchdog(lrc, dw, i);
460
461 i = emit_copy_timestamp(xe, lrc, dw, i);
462
463 /*
464 * On AuxCCS platforms the invalidation of the Aux table requires
465 * quiescing the memory traffic beforehand.
466 */
467 if (has_aux_ccs(xe))
468 i = emit_render_cache_flush(job, dw, i);
469
470 dw[i++] = preparser_disable(true);
471 if (lacks_render)
472 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
473 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
474 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
475
476 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
477 i = emit_pipe_invalidate(job->q, mask_flags, job->ring_ops_flush_tlb, dw, i);
478
479 /* hsdes: 1809175790 */
480 i = emit_aux_table_inv(job->q->hwe, dw, i);
481
482 dw[i++] = preparser_disable(false);
483
484 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
485 seqno, dw, i);
486
487 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
488
489 /* Don't preempt fence signaling */
490 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE;
491
492 i = emit_render_cache_flush(job, dw, i);
493
494 if (job->user_fence.used)
495 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
496 job->user_fence.value,
497 dw, i);
498
499 i = emit_pipe_imm_ggtt(job->q, xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
500
501 i = emit_user_interrupt(dw, i);
502
503 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
504
505 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
506 }
507
emit_migration_job_gen12(struct xe_sched_job * job,struct xe_lrc * lrc,u32 * head,u32 seqno)508 static void emit_migration_job_gen12(struct xe_sched_job *job,
509 struct xe_lrc *lrc, u32 *head,
510 u32 seqno)
511 {
512 struct xe_gt *gt = job->q->gt;
513 struct xe_device *xe = gt_to_xe(gt);
514 u32 saddr = xe_lrc_start_seqno_ggtt_addr(lrc);
515 u32 dw[MAX_JOB_SIZE_DW], i = 0;
516
517 *head = lrc->ring.tail;
518
519 xe_gt_assert(gt, !job->ring_ops_force_reset);
520
521 i = emit_copy_timestamp(xe, lrc, dw, i);
522
523 i = emit_store_imm_ggtt(saddr, seqno, dw, i);
524
525 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
526
527 i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);
528
529 dw[i++] = preparser_disable(true);
530 i = emit_flush_invalidate(saddr, seqno, job->migrate_flush_flags, dw, i);
531 dw[i++] = preparser_disable(false);
532
533 i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);
534
535 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno,
536 job->migrate_flush_flags,
537 dw, i);
538
539 i = emit_user_interrupt(dw, i);
540
541 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
542
543 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
544 }
545
emit_job_gen12_gsc(struct xe_sched_job * job)546 static void emit_job_gen12_gsc(struct xe_sched_job *job)
547 {
548 struct xe_gt *gt = job->q->gt;
549
550 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */
551
552 __emit_job_gen12_simple(job, job->q->lrc[0],
553 job->ptrs[0].batch_addr,
554 &job->ptrs[0].head,
555 xe_sched_job_lrc_seqno(job));
556 }
557
emit_job_gen12_copy(struct xe_sched_job * job)558 static void emit_job_gen12_copy(struct xe_sched_job *job)
559 {
560 int i;
561
562 if (xe_sched_job_is_migration(job->q)) {
563 emit_migration_job_gen12(job, job->q->lrc[0],
564 &job->ptrs[0].head,
565 xe_sched_job_lrc_seqno(job));
566 return;
567 }
568
569 for (i = 0; i < job->q->width; ++i)
570 __emit_job_gen12_simple(job, job->q->lrc[i],
571 job->ptrs[i].batch_addr,
572 &job->ptrs[i].head,
573 xe_sched_job_lrc_seqno(job));
574 }
575
emit_job_gen12_video(struct xe_sched_job * job)576 static void emit_job_gen12_video(struct xe_sched_job *job)
577 {
578 int i;
579
580 /* FIXME: Not doing parallel handshake for now */
581 for (i = 0; i < job->q->width; ++i)
582 __emit_job_gen12_video(job, job->q->lrc[i],
583 job->ptrs[i].batch_addr,
584 &job->ptrs[i].head,
585 xe_sched_job_lrc_seqno(job));
586 }
587
emit_job_gen12_render_compute(struct xe_sched_job * job)588 static void emit_job_gen12_render_compute(struct xe_sched_job *job)
589 {
590 int i;
591
592 for (i = 0; i < job->q->width; ++i)
593 __emit_job_gen12_render_compute(job, job->q->lrc[i],
594 job->ptrs[i].batch_addr,
595 &job->ptrs[i].head,
596 xe_sched_job_lrc_seqno(job));
597 }
598
599 static const struct xe_ring_ops ring_ops_gen12_gsc = {
600 .emit_job = emit_job_gen12_gsc,
601 };
602
603 static const struct xe_ring_ops ring_ops_gen12_copy = {
604 .emit_job = emit_job_gen12_copy,
605 };
606
607 static const struct xe_ring_ops ring_ops_gen12_video_decode = {
608 .emit_job = emit_job_gen12_video,
609 };
610
611 static const struct xe_ring_ops ring_ops_gen12_video_enhance = {
612 .emit_job = emit_job_gen12_video,
613 };
614
615 static const struct xe_ring_ops ring_ops_gen12_render_compute = {
616 .emit_job = emit_job_gen12_render_compute,
617 };
618
619 static const struct xe_ring_ops auxccs_ring_ops_gen12_video_decode = {
620 .emit_job = emit_job_gen12_video,
621 .emit_aux_table_inv = emit_aux_table_inv_video_decode,
622 };
623
624 static const struct xe_ring_ops auxccs_ring_ops_gen12_video_enhance = {
625 .emit_job = emit_job_gen12_video,
626 .emit_aux_table_inv = emit_aux_table_inv_video_enhance,
627 };
628
629 static const struct xe_ring_ops auxccs_ring_ops_gen12_render_compute = {
630 .emit_job = emit_job_gen12_render_compute,
631 .emit_aux_table_inv = emit_aux_table_inv_render_compute,
632 };
633
634 const struct xe_ring_ops *
xe_ring_ops_get(struct xe_gt * gt,enum xe_engine_class class)635 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)
636 {
637 struct xe_device *xe = gt_to_xe(gt);
638
639 switch (class) {
640 case XE_ENGINE_CLASS_OTHER:
641 return &ring_ops_gen12_gsc;
642 case XE_ENGINE_CLASS_COPY:
643 return &ring_ops_gen12_copy;
644 case XE_ENGINE_CLASS_VIDEO_DECODE:
645 if (has_aux_ccs(xe))
646 return &auxccs_ring_ops_gen12_video_decode;
647 else
648 return &ring_ops_gen12_video_decode;
649 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
650 if (has_aux_ccs(xe))
651 return &auxccs_ring_ops_gen12_video_enhance;
652 else
653 return &ring_ops_gen12_video_enhance;
654 case XE_ENGINE_CLASS_RENDER:
655 case XE_ENGINE_CLASS_COMPUTE:
656 if (has_aux_ccs(xe))
657 return &auxccs_ring_ops_gen12_render_compute;
658 else
659 return &ring_ops_gen12_render_compute;
660 default:
661 return NULL;
662 }
663 }
664