1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_ring_ops.h"
7
8 #include <generated/xe_wa_oob.h>
9
10 #include "instructions/xe_gpu_commands.h"
11 #include "instructions/xe_mi_commands.h"
12 #include "regs/xe_engine_regs.h"
13 #include "regs/xe_gt_regs.h"
14 #include "regs/xe_lrc_layout.h"
15 #include "xe_exec_queue_types.h"
16 #include "xe_gt.h"
17 #include "xe_lrc.h"
18 #include "xe_macros.h"
19 #include "xe_sched_job.h"
20 #include "xe_sriov.h"
21 #include "xe_vm_types.h"
22 #include "xe_vm.h"
23 #include "xe_wa.h"
24
25 /*
26 * 3D-related flags that can't be set on _engines_ that lack access to the 3D
27 * pipeline (i.e., CCS engines).
28 */
29 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\
30 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \
31 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
32 PIPE_CONTROL_TILE_CACHE_FLUSH | \
33 PIPE_CONTROL_DEPTH_STALL | \
34 PIPE_CONTROL_STALL_AT_SCOREBOARD | \
35 PIPE_CONTROL_PSD_SYNC | \
36 PIPE_CONTROL_AMFS_FLUSH | \
37 PIPE_CONTROL_VF_CACHE_INVALIDATE | \
38 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET)
39
40 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */
41 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \
42 PIPE_CONTROL_3D_ENGINE_FLAGS | \
43 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \
44 PIPE_CONTROL_FLUSH_ENABLE | \
45 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
46 PIPE_CONTROL_DC_FLUSH_ENABLE)
47
preparser_disable(bool state)48 static u32 preparser_disable(bool state)
49 {
50 return MI_ARB_CHECK | BIT(8) | state;
51 }
52
emit_aux_table_inv(struct xe_gt * gt,struct xe_reg reg,u32 * dw,int i)53 static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg,
54 u32 *dw, int i)
55 {
56 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;
57 dw[i++] = reg.addr + gt->mmio.adj_offset;
58 dw[i++] = AUX_INV;
59 dw[i++] = MI_NOOP;
60
61 return i;
62 }
63
emit_user_interrupt(u32 * dw,int i)64 static int emit_user_interrupt(u32 *dw, int i)
65 {
66 dw[i++] = MI_USER_INTERRUPT;
67 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
68 dw[i++] = MI_ARB_CHECK;
69
70 return i;
71 }
72
emit_store_imm_ggtt(u32 addr,u32 value,u32 * dw,int i)73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)
74 {
75 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
76 dw[i++] = addr;
77 dw[i++] = 0;
78 dw[i++] = value;
79
80 return i;
81 }
82
emit_flush_dw(u32 * dw,int i)83 static int emit_flush_dw(u32 *dw, int i)
84 {
85 dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;
86 dw[i++] = 0;
87 dw[i++] = 0;
88 dw[i++] = 0;
89
90 return i;
91 }
92
emit_flush_imm_ggtt(u32 addr,u32 value,bool invalidate_tlb,u32 * dw,int i)93 static int emit_flush_imm_ggtt(u32 addr, u32 value, bool invalidate_tlb,
94 u32 *dw, int i)
95 {
96 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
97 (invalidate_tlb ? MI_INVALIDATE_TLB : 0);
98 dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
99 dw[i++] = 0;
100 dw[i++] = value;
101
102 return i;
103 }
104
emit_bb_start(u64 batch_addr,u32 ppgtt_flag,u32 * dw,int i)105 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
106 {
107 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);
108 dw[i++] = lower_32_bits(batch_addr);
109 dw[i++] = upper_32_bits(batch_addr);
110
111 return i;
112 }
113
emit_flush_invalidate(u32 flag,u32 * dw,int i)114 static int emit_flush_invalidate(u32 flag, u32 *dw, int i)
115 {
116 dw[i] = MI_FLUSH_DW;
117 dw[i] |= flag;
118 dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
119 MI_FLUSH_DW_STORE_INDEX;
120
121 dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
122 dw[i++] = 0;
123 dw[i++] = ~0U;
124
125 return i;
126 }
127
128 static int
emit_pipe_control(u32 * dw,int i,u32 bit_group_0,u32 bit_group_1,u32 offset,u32 value)129 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)
130 {
131 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;
132 dw[i++] = bit_group_1;
133 dw[i++] = offset;
134 dw[i++] = 0;
135 dw[i++] = value;
136 dw[i++] = 0;
137
138 return i;
139 }
140
emit_pipe_invalidate(u32 mask_flags,bool invalidate_tlb,u32 * dw,int i)141 static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
142 int i)
143 {
144 u32 flags = PIPE_CONTROL_CS_STALL |
145 PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
146 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
147 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
148 PIPE_CONTROL_VF_CACHE_INVALIDATE |
149 PIPE_CONTROL_CONST_CACHE_INVALIDATE |
150 PIPE_CONTROL_STATE_CACHE_INVALIDATE |
151 PIPE_CONTROL_QW_WRITE |
152 PIPE_CONTROL_STORE_DATA_INDEX;
153
154 if (invalidate_tlb)
155 flags |= PIPE_CONTROL_TLB_INVALIDATE;
156
157 flags &= ~mask_flags;
158
159 return emit_pipe_control(dw, i, 0, flags, LRC_PPHWSP_SCRATCH_ADDR, 0);
160 }
161
emit_store_imm_ppgtt_posted(u64 addr,u64 value,u32 * dw,int i)162 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
163 u32 *dw, int i)
164 {
165 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);
166 dw[i++] = lower_32_bits(addr);
167 dw[i++] = upper_32_bits(addr);
168 dw[i++] = lower_32_bits(value);
169 dw[i++] = upper_32_bits(value);
170
171 return i;
172 }
173
emit_render_cache_flush(struct xe_sched_job * job,u32 * dw,int i)174 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
175 {
176 struct xe_gt *gt = job->q->gt;
177 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
178 u32 flags;
179
180 flags = (PIPE_CONTROL_CS_STALL |
181 PIPE_CONTROL_TILE_CACHE_FLUSH |
182 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
183 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
184 PIPE_CONTROL_DC_FLUSH_ENABLE |
185 PIPE_CONTROL_FLUSH_ENABLE);
186
187 if (XE_WA(gt, 1409600907))
188 flags |= PIPE_CONTROL_DEPTH_STALL;
189
190 if (lacks_render)
191 flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
192 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
193 flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
194
195 return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);
196 }
197
emit_pipe_control_to_ring_end(struct xe_hw_engine * hwe,u32 * dw,int i)198 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)
199 {
200 if (hwe->class != XE_ENGINE_CLASS_RENDER)
201 return i;
202
203 if (XE_WA(hwe->gt, 16020292621))
204 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
205 RING_NOPID(hwe->mmio_base).addr, 0);
206
207 return i;
208 }
209
emit_pipe_imm_ggtt(u32 addr,u32 value,bool stall_only,u32 * dw,int i)210 static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,
211 int i)
212 {
213 u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB |
214 PIPE_CONTROL_QW_WRITE;
215
216 if (!stall_only)
217 flags |= PIPE_CONTROL_FLUSH_ENABLE;
218
219 return emit_pipe_control(dw, i, 0, flags, addr, value);
220 }
221
get_ppgtt_flag(struct xe_sched_job * job)222 static u32 get_ppgtt_flag(struct xe_sched_job *job)
223 {
224 return job->q->vm ? BIT(8) : 0;
225 }
226
emit_copy_timestamp(struct xe_lrc * lrc,u32 * dw,int i)227 static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)
228 {
229 dw[i++] = MI_COPY_MEM_MEM | MI_COPY_MEM_MEM_SRC_GGTT |
230 MI_COPY_MEM_MEM_DST_GGTT;
231 dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);
232 dw[i++] = 0;
233 dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc);
234 dw[i++] = 0;
235 dw[i++] = MI_NOOP;
236
237 return i;
238 }
239
240 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */
__emit_job_gen12_simple(struct xe_sched_job * job,struct xe_lrc * lrc,u64 batch_addr,u32 seqno)241 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc,
242 u64 batch_addr, u32 seqno)
243 {
244 u32 dw[MAX_JOB_SIZE_DW], i = 0;
245 u32 ppgtt_flag = get_ppgtt_flag(job);
246 struct xe_gt *gt = job->q->gt;
247
248 i = emit_copy_timestamp(lrc, dw, i);
249
250 if (job->ring_ops_flush_tlb) {
251 dw[i++] = preparser_disable(true);
252 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
253 seqno, true, dw, i);
254 dw[i++] = preparser_disable(false);
255 } else {
256 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
257 seqno, dw, i);
258 }
259
260 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
261
262 if (job->user_fence.used) {
263 i = emit_flush_dw(dw, i);
264 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
265 job->user_fence.value,
266 dw, i);
267 }
268
269 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);
270
271 i = emit_user_interrupt(dw, i);
272
273 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
274
275 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
276 }
277
has_aux_ccs(struct xe_device * xe)278 static bool has_aux_ccs(struct xe_device *xe)
279 {
280 /*
281 * PVC is a special case that has no compression of either type
282 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2
283 * onward, so any future platforms with no FlatCCS will not have
284 * AuxCCS either.
285 */
286 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
287 return false;
288
289 return !xe->info.has_flat_ccs;
290 }
291
__emit_job_gen12_video(struct xe_sched_job * job,struct xe_lrc * lrc,u64 batch_addr,u32 seqno)292 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc,
293 u64 batch_addr, u32 seqno)
294 {
295 u32 dw[MAX_JOB_SIZE_DW], i = 0;
296 u32 ppgtt_flag = get_ppgtt_flag(job);
297 struct xe_gt *gt = job->q->gt;
298 struct xe_device *xe = gt_to_xe(gt);
299 bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE;
300
301 i = emit_copy_timestamp(lrc, dw, i);
302
303 dw[i++] = preparser_disable(true);
304
305 /* hsdes: 1809175790 */
306 if (has_aux_ccs(xe)) {
307 if (decode)
308 i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);
309 else
310 i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);
311 }
312
313 if (job->ring_ops_flush_tlb)
314 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
315 seqno, true, dw, i);
316
317 dw[i++] = preparser_disable(false);
318
319 if (!job->ring_ops_flush_tlb)
320 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
321 seqno, dw, i);
322
323 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
324
325 if (job->user_fence.used) {
326 i = emit_flush_dw(dw, i);
327 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
328 job->user_fence.value,
329 dw, i);
330 }
331
332 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, false, dw, i);
333
334 i = emit_user_interrupt(dw, i);
335
336 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
337
338 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
339 }
340
__emit_job_gen12_render_compute(struct xe_sched_job * job,struct xe_lrc * lrc,u64 batch_addr,u32 seqno)341 static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
342 struct xe_lrc *lrc,
343 u64 batch_addr, u32 seqno)
344 {
345 u32 dw[MAX_JOB_SIZE_DW], i = 0;
346 u32 ppgtt_flag = get_ppgtt_flag(job);
347 struct xe_gt *gt = job->q->gt;
348 struct xe_device *xe = gt_to_xe(gt);
349 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
350 u32 mask_flags = 0;
351
352 i = emit_copy_timestamp(lrc, dw, i);
353
354 dw[i++] = preparser_disable(true);
355 if (lacks_render)
356 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
357 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
358 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
359
360 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
361 i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);
362
363 /* hsdes: 1809175790 */
364 if (has_aux_ccs(xe))
365 i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);
366
367 dw[i++] = preparser_disable(false);
368
369 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
370 seqno, dw, i);
371
372 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
373
374 i = emit_render_cache_flush(job, dw, i);
375
376 if (job->user_fence.used)
377 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
378 job->user_fence.value,
379 dw, i);
380
381 i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
382
383 i = emit_user_interrupt(dw, i);
384
385 i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
386
387 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
388
389 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
390 }
391
emit_migration_job_gen12(struct xe_sched_job * job,struct xe_lrc * lrc,u32 seqno)392 static void emit_migration_job_gen12(struct xe_sched_job *job,
393 struct xe_lrc *lrc, u32 seqno)
394 {
395 u32 dw[MAX_JOB_SIZE_DW], i = 0;
396
397 i = emit_copy_timestamp(lrc, dw, i);
398
399 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
400 seqno, dw, i);
401
402 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
403
404 i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);
405
406 if (!IS_SRIOV_VF(gt_to_xe(job->q->gt))) {
407 /* XXX: Do we need this? Leaving for now. */
408 dw[i++] = preparser_disable(true);
409 i = emit_flush_invalidate(0, dw, i);
410 dw[i++] = preparser_disable(false);
411 }
412
413 i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);
414
415 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |
416 MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW;
417 dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;
418 dw[i++] = 0;
419 dw[i++] = seqno; /* value */
420
421 i = emit_user_interrupt(dw, i);
422
423 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
424
425 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
426 }
427
emit_job_gen12_gsc(struct xe_sched_job * job)428 static void emit_job_gen12_gsc(struct xe_sched_job *job)
429 {
430 struct xe_gt *gt = job->q->gt;
431
432 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */
433
434 __emit_job_gen12_simple(job, job->q->lrc[0],
435 job->ptrs[0].batch_addr,
436 xe_sched_job_lrc_seqno(job));
437 }
438
emit_job_gen12_copy(struct xe_sched_job * job)439 static void emit_job_gen12_copy(struct xe_sched_job *job)
440 {
441 int i;
442
443 if (xe_sched_job_is_migration(job->q)) {
444 emit_migration_job_gen12(job, job->q->lrc[0],
445 xe_sched_job_lrc_seqno(job));
446 return;
447 }
448
449 for (i = 0; i < job->q->width; ++i)
450 __emit_job_gen12_simple(job, job->q->lrc[i],
451 job->ptrs[i].batch_addr,
452 xe_sched_job_lrc_seqno(job));
453 }
454
emit_job_gen12_video(struct xe_sched_job * job)455 static void emit_job_gen12_video(struct xe_sched_job *job)
456 {
457 int i;
458
459 /* FIXME: Not doing parallel handshake for now */
460 for (i = 0; i < job->q->width; ++i)
461 __emit_job_gen12_video(job, job->q->lrc[i],
462 job->ptrs[i].batch_addr,
463 xe_sched_job_lrc_seqno(job));
464 }
465
emit_job_gen12_render_compute(struct xe_sched_job * job)466 static void emit_job_gen12_render_compute(struct xe_sched_job *job)
467 {
468 int i;
469
470 for (i = 0; i < job->q->width; ++i)
471 __emit_job_gen12_render_compute(job, job->q->lrc[i],
472 job->ptrs[i].batch_addr,
473 xe_sched_job_lrc_seqno(job));
474 }
475
476 static const struct xe_ring_ops ring_ops_gen12_gsc = {
477 .emit_job = emit_job_gen12_gsc,
478 };
479
480 static const struct xe_ring_ops ring_ops_gen12_copy = {
481 .emit_job = emit_job_gen12_copy,
482 };
483
484 static const struct xe_ring_ops ring_ops_gen12_video = {
485 .emit_job = emit_job_gen12_video,
486 };
487
488 static const struct xe_ring_ops ring_ops_gen12_render_compute = {
489 .emit_job = emit_job_gen12_render_compute,
490 };
491
492 const struct xe_ring_ops *
xe_ring_ops_get(struct xe_gt * gt,enum xe_engine_class class)493 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)
494 {
495 switch (class) {
496 case XE_ENGINE_CLASS_OTHER:
497 return &ring_ops_gen12_gsc;
498 case XE_ENGINE_CLASS_COPY:
499 return &ring_ops_gen12_copy;
500 case XE_ENGINE_CLASS_VIDEO_DECODE:
501 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
502 return &ring_ops_gen12_video;
503 case XE_ENGINE_CLASS_RENDER:
504 case XE_ENGINE_CLASS_COMPUTE:
505 return &ring_ops_gen12_render_compute;
506 default:
507 return NULL;
508 }
509 }
510