1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_ring_ops.h"
7
8 #include <generated/xe_wa_oob.h>
9
10 #include "instructions/xe_gpu_commands.h"
11 #include "instructions/xe_mi_commands.h"
12 #include "regs/xe_engine_regs.h"
13 #include "regs/xe_gt_regs.h"
14 #include "regs/xe_lrc_layout.h"
15 #include "xe_exec_queue_types.h"
16 #include "xe_gt.h"
17 #include "xe_lrc.h"
18 #include "xe_macros.h"
19 #include "xe_sched_job.h"
20 #include "xe_sriov.h"
21 #include "xe_vm_types.h"
22 #include "xe_vm.h"
23 #include "xe_wa.h"
24
25 /*
26 * 3D-related flags that can't be set on _engines_ that lack access to the 3D
27 * pipeline (i.e., CCS engines).
28 */
29 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\
30 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \
31 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
32 PIPE_CONTROL_TILE_CACHE_FLUSH | \
33 PIPE_CONTROL_DEPTH_STALL | \
34 PIPE_CONTROL_STALL_AT_SCOREBOARD | \
35 PIPE_CONTROL_PSD_SYNC | \
36 PIPE_CONTROL_AMFS_FLUSH | \
37 PIPE_CONTROL_VF_CACHE_INVALIDATE | \
38 PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET)
39
40 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */
41 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \
42 PIPE_CONTROL_3D_ENGINE_FLAGS | \
43 PIPE_CONTROL_INDIRECT_STATE_DISABLE | \
44 PIPE_CONTROL_FLUSH_ENABLE | \
45 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
46 PIPE_CONTROL_DC_FLUSH_ENABLE)
47
preparser_disable(bool state)48 static u32 preparser_disable(bool state)
49 {
50 return MI_ARB_CHECK | BIT(8) | state;
51 }
52
emit_aux_table_inv(struct xe_gt * gt,struct xe_reg reg,u32 * dw,int i)53 static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg,
54 u32 *dw, int i)
55 {
56 dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;
57 dw[i++] = reg.addr + gt->mmio.adj_offset;
58 dw[i++] = AUX_INV;
59 dw[i++] = MI_NOOP;
60
61 return i;
62 }
63
emit_user_interrupt(u32 * dw,int i)64 static int emit_user_interrupt(u32 *dw, int i)
65 {
66 dw[i++] = MI_USER_INTERRUPT;
67 dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
68 dw[i++] = MI_ARB_CHECK;
69
70 return i;
71 }
72
emit_store_imm_ggtt(u32 addr,u32 value,u32 * dw,int i)73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)
74 {
75 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
76 dw[i++] = addr;
77 dw[i++] = 0;
78 dw[i++] = value;
79
80 return i;
81 }
82
emit_flush_dw(u32 * dw,int i)83 static int emit_flush_dw(u32 *dw, int i)
84 {
85 dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;
86 dw[i++] = 0;
87 dw[i++] = 0;
88 dw[i++] = 0;
89
90 return i;
91 }
92
emit_flush_imm_ggtt(u32 addr,u32 value,u32 flags,u32 * dw,int i)93 static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i)
94 {
95 dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
96 flags;
97 dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
98 dw[i++] = 0;
99 dw[i++] = value;
100
101 return i;
102 }
103
emit_bb_start(u64 batch_addr,u32 ppgtt_flag,u32 * dw,int i)104 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
105 {
106 dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);
107 dw[i++] = lower_32_bits(batch_addr);
108 dw[i++] = upper_32_bits(batch_addr);
109
110 return i;
111 }
112
emit_flush_invalidate(u32 * dw,int i)113 static int emit_flush_invalidate(u32 *dw, int i)
114 {
115 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW |
116 MI_FLUSH_IMM_DW | MI_FLUSH_DW_STORE_INDEX;
117 dw[i++] = LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR;
118 dw[i++] = 0;
119 dw[i++] = 0;
120
121 return i;
122 }
123
124 static int
emit_pipe_control(u32 * dw,int i,u32 bit_group_0,u32 bit_group_1,u32 offset,u32 value)125 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)
126 {
127 dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;
128 dw[i++] = bit_group_1;
129 dw[i++] = offset;
130 dw[i++] = 0;
131 dw[i++] = value;
132 dw[i++] = 0;
133
134 return i;
135 }
136
emit_pipe_invalidate(u32 mask_flags,bool invalidate_tlb,u32 * dw,int i)137 static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
138 int i)
139 {
140 u32 flags0 = 0;
141 u32 flags1 = PIPE_CONTROL_CS_STALL |
142 PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
143 PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
144 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
145 PIPE_CONTROL_VF_CACHE_INVALIDATE |
146 PIPE_CONTROL_CONST_CACHE_INVALIDATE |
147 PIPE_CONTROL_STATE_CACHE_INVALIDATE |
148 PIPE_CONTROL_QW_WRITE |
149 PIPE_CONTROL_STORE_DATA_INDEX;
150
151 if (invalidate_tlb)
152 flags1 |= PIPE_CONTROL_TLB_INVALIDATE;
153
154 flags1 &= ~mask_flags;
155
156 if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE)
157 flags0 |= PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE;
158
159 return emit_pipe_control(dw, i, flags0, flags1,
160 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
161 }
162
emit_store_imm_ppgtt_posted(u64 addr,u64 value,u32 * dw,int i)163 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
164 u32 *dw, int i)
165 {
166 dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);
167 dw[i++] = lower_32_bits(addr);
168 dw[i++] = upper_32_bits(addr);
169 dw[i++] = lower_32_bits(value);
170 dw[i++] = upper_32_bits(value);
171
172 return i;
173 }
174
emit_render_cache_flush(struct xe_sched_job * job,u32 * dw,int i)175 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
176 {
177 struct xe_gt *gt = job->q->gt;
178 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
179 u32 flags;
180
181 if (XE_WA(gt, 14016712196))
182 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,
183 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
184
185 flags = (PIPE_CONTROL_CS_STALL |
186 PIPE_CONTROL_TILE_CACHE_FLUSH |
187 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
188 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
189 PIPE_CONTROL_DC_FLUSH_ENABLE |
190 PIPE_CONTROL_FLUSH_ENABLE);
191
192 if (XE_WA(gt, 1409600907))
193 flags |= PIPE_CONTROL_DEPTH_STALL;
194
195 if (lacks_render)
196 flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
197 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
198 flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
199
200 return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);
201 }
202
emit_pipe_control_to_ring_end(struct xe_hw_engine * hwe,u32 * dw,int i)203 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)
204 {
205 if (hwe->class != XE_ENGINE_CLASS_RENDER)
206 return i;
207
208 if (XE_WA(hwe->gt, 16020292621))
209 i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
210 RING_NOPID(hwe->mmio_base).addr, 0);
211
212 return i;
213 }
214
emit_pipe_imm_ggtt(u32 addr,u32 value,bool stall_only,u32 * dw,int i)215 static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,
216 int i)
217 {
218 u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB |
219 PIPE_CONTROL_QW_WRITE;
220
221 if (!stall_only)
222 flags |= PIPE_CONTROL_FLUSH_ENABLE;
223
224 return emit_pipe_control(dw, i, 0, flags, addr, value);
225 }
226
get_ppgtt_flag(struct xe_sched_job * job)227 static u32 get_ppgtt_flag(struct xe_sched_job *job)
228 {
229 if (job->q->vm && !job->ggtt)
230 return BIT(8);
231
232 return 0;
233 }
234
emit_copy_timestamp(struct xe_lrc * lrc,u32 * dw,int i)235 static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)
236 {
237 dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET;
238 dw[i++] = RING_CTX_TIMESTAMP(0).addr;
239 dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);
240 dw[i++] = 0;
241
242 return i;
243 }
244
245 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */
__emit_job_gen12_simple(struct xe_sched_job * job,struct xe_lrc * lrc,u64 batch_addr,u32 seqno)246 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc,
247 u64 batch_addr, u32 seqno)
248 {
249 u32 dw[MAX_JOB_SIZE_DW], i = 0;
250 u32 ppgtt_flag = get_ppgtt_flag(job);
251 struct xe_gt *gt = job->q->gt;
252
253 i = emit_copy_timestamp(lrc, dw, i);
254
255 if (job->ring_ops_flush_tlb) {
256 dw[i++] = preparser_disable(true);
257 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
258 seqno, MI_INVALIDATE_TLB, dw, i);
259 dw[i++] = preparser_disable(false);
260 } else {
261 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
262 seqno, dw, i);
263 }
264
265 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
266
267 if (job->user_fence.used) {
268 i = emit_flush_dw(dw, i);
269 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
270 job->user_fence.value,
271 dw, i);
272 }
273
274 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);
275
276 i = emit_user_interrupt(dw, i);
277
278 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
279
280 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
281 }
282
has_aux_ccs(struct xe_device * xe)283 static bool has_aux_ccs(struct xe_device *xe)
284 {
285 /*
286 * PVC is a special case that has no compression of either type
287 * (FlatCCS or AuxCCS). Also, AuxCCS is no longer used from Xe2
288 * onward, so any future platforms with no FlatCCS will not have
289 * AuxCCS either.
290 */
291 if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
292 return false;
293
294 return !xe->info.has_flat_ccs;
295 }
296
__emit_job_gen12_video(struct xe_sched_job * job,struct xe_lrc * lrc,u64 batch_addr,u32 seqno)297 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc,
298 u64 batch_addr, u32 seqno)
299 {
300 u32 dw[MAX_JOB_SIZE_DW], i = 0;
301 u32 ppgtt_flag = get_ppgtt_flag(job);
302 struct xe_gt *gt = job->q->gt;
303 struct xe_device *xe = gt_to_xe(gt);
304 bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE;
305
306 i = emit_copy_timestamp(lrc, dw, i);
307
308 dw[i++] = preparser_disable(true);
309
310 /* hsdes: 1809175790 */
311 if (has_aux_ccs(xe)) {
312 if (decode)
313 i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);
314 else
315 i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);
316 }
317
318 if (job->ring_ops_flush_tlb)
319 i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
320 seqno, MI_INVALIDATE_TLB, dw, i);
321
322 dw[i++] = preparser_disable(false);
323
324 if (!job->ring_ops_flush_tlb)
325 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
326 seqno, dw, i);
327
328 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
329
330 if (job->user_fence.used) {
331 i = emit_flush_dw(dw, i);
332 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
333 job->user_fence.value,
334 dw, i);
335 }
336
337 i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);
338
339 i = emit_user_interrupt(dw, i);
340
341 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
342
343 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
344 }
345
__emit_job_gen12_render_compute(struct xe_sched_job * job,struct xe_lrc * lrc,u64 batch_addr,u32 seqno)346 static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
347 struct xe_lrc *lrc,
348 u64 batch_addr, u32 seqno)
349 {
350 u32 dw[MAX_JOB_SIZE_DW], i = 0;
351 u32 ppgtt_flag = get_ppgtt_flag(job);
352 struct xe_gt *gt = job->q->gt;
353 struct xe_device *xe = gt_to_xe(gt);
354 bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
355 u32 mask_flags = 0;
356
357 i = emit_copy_timestamp(lrc, dw, i);
358
359 dw[i++] = preparser_disable(true);
360 if (lacks_render)
361 mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
362 else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
363 mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
364
365 /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
366 i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);
367
368 /* hsdes: 1809175790 */
369 if (has_aux_ccs(xe))
370 i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);
371
372 dw[i++] = preparser_disable(false);
373
374 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
375 seqno, dw, i);
376
377 i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
378
379 i = emit_render_cache_flush(job, dw, i);
380
381 if (job->user_fence.used)
382 i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
383 job->user_fence.value,
384 dw, i);
385
386 i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
387
388 i = emit_user_interrupt(dw, i);
389
390 i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
391
392 xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
393
394 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
395 }
396
emit_migration_job_gen12(struct xe_sched_job * job,struct xe_lrc * lrc,u32 seqno)397 static void emit_migration_job_gen12(struct xe_sched_job *job,
398 struct xe_lrc *lrc, u32 seqno)
399 {
400 u32 dw[MAX_JOB_SIZE_DW], i = 0;
401
402 i = emit_copy_timestamp(lrc, dw, i);
403
404 i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
405 seqno, dw, i);
406
407 dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
408
409 i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);
410
411 if (!IS_SRIOV_VF(gt_to_xe(job->q->gt))) {
412 /* XXX: Do we need this? Leaving for now. */
413 dw[i++] = preparser_disable(true);
414 i = emit_flush_invalidate(dw, i);
415 dw[i++] = preparser_disable(false);
416 }
417
418 i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);
419
420 dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |
421 MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW;
422 dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;
423 dw[i++] = 0;
424 dw[i++] = seqno; /* value */
425
426 i = emit_user_interrupt(dw, i);
427
428 xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
429
430 xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
431 }
432
emit_job_gen12_gsc(struct xe_sched_job * job)433 static void emit_job_gen12_gsc(struct xe_sched_job *job)
434 {
435 struct xe_gt *gt = job->q->gt;
436
437 xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */
438
439 __emit_job_gen12_simple(job, job->q->lrc[0],
440 job->ptrs[0].batch_addr,
441 xe_sched_job_lrc_seqno(job));
442 }
443
emit_job_gen12_copy(struct xe_sched_job * job)444 static void emit_job_gen12_copy(struct xe_sched_job *job)
445 {
446 int i;
447
448 if (xe_sched_job_is_migration(job->q)) {
449 emit_migration_job_gen12(job, job->q->lrc[0],
450 xe_sched_job_lrc_seqno(job));
451 return;
452 }
453
454 for (i = 0; i < job->q->width; ++i)
455 __emit_job_gen12_simple(job, job->q->lrc[i],
456 job->ptrs[i].batch_addr,
457 xe_sched_job_lrc_seqno(job));
458 }
459
emit_job_gen12_video(struct xe_sched_job * job)460 static void emit_job_gen12_video(struct xe_sched_job *job)
461 {
462 int i;
463
464 /* FIXME: Not doing parallel handshake for now */
465 for (i = 0; i < job->q->width; ++i)
466 __emit_job_gen12_video(job, job->q->lrc[i],
467 job->ptrs[i].batch_addr,
468 xe_sched_job_lrc_seqno(job));
469 }
470
emit_job_gen12_render_compute(struct xe_sched_job * job)471 static void emit_job_gen12_render_compute(struct xe_sched_job *job)
472 {
473 int i;
474
475 for (i = 0; i < job->q->width; ++i)
476 __emit_job_gen12_render_compute(job, job->q->lrc[i],
477 job->ptrs[i].batch_addr,
478 xe_sched_job_lrc_seqno(job));
479 }
480
481 static const struct xe_ring_ops ring_ops_gen12_gsc = {
482 .emit_job = emit_job_gen12_gsc,
483 };
484
485 static const struct xe_ring_ops ring_ops_gen12_copy = {
486 .emit_job = emit_job_gen12_copy,
487 };
488
489 static const struct xe_ring_ops ring_ops_gen12_video = {
490 .emit_job = emit_job_gen12_video,
491 };
492
493 static const struct xe_ring_ops ring_ops_gen12_render_compute = {
494 .emit_job = emit_job_gen12_render_compute,
495 };
496
497 const struct xe_ring_ops *
xe_ring_ops_get(struct xe_gt * gt,enum xe_engine_class class)498 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)
499 {
500 switch (class) {
501 case XE_ENGINE_CLASS_OTHER:
502 return &ring_ops_gen12_gsc;
503 case XE_ENGINE_CLASS_COPY:
504 return &ring_ops_gen12_copy;
505 case XE_ENGINE_CLASS_VIDEO_DECODE:
506 case XE_ENGINE_CLASS_VIDEO_ENHANCE:
507 return &ring_ops_gen12_video;
508 case XE_ENGINE_CLASS_RENDER:
509 case XE_ENGINE_CLASS_COMPUTE:
510 return &ring_ops_gen12_render_compute;
511 default:
512 return NULL;
513 }
514 }
515