xref: /linux/drivers/gpu/drm/xe/xe_ring_ops.c (revision 7a9b709e7cc5ce1ffb84ce07bf6d157e1de758df) 
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include "xe_ring_ops.h"
7 
8 #include <generated/xe_wa_oob.h>
9 
10 #include "instructions/xe_gpu_commands.h"
11 #include "instructions/xe_mi_commands.h"
12 #include "regs/xe_engine_regs.h"
13 #include "regs/xe_gt_regs.h"
14 #include "regs/xe_lrc_layout.h"
15 #include "xe_exec_queue_types.h"
16 #include "xe_gt.h"
17 #include "xe_lrc.h"
18 #include "xe_macros.h"
19 #include "xe_sched_job.h"
20 #include "xe_sriov.h"
21 #include "xe_vm_types.h"
22 #include "xe_vm.h"
23 #include "xe_wa.h"
24 
25 /*
26  * 3D-related flags that can't be set on _engines_ that lack access to the 3D
27  * pipeline (i.e., CCS engines).
28  */
29 #define PIPE_CONTROL_3D_ENGINE_FLAGS (\
30 		PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \
31 		PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
32 		PIPE_CONTROL_TILE_CACHE_FLUSH | \
33 		PIPE_CONTROL_DEPTH_STALL | \
34 		PIPE_CONTROL_STALL_AT_SCOREBOARD | \
35 		PIPE_CONTROL_PSD_SYNC | \
36 		PIPE_CONTROL_AMFS_FLUSH | \
37 		PIPE_CONTROL_VF_CACHE_INVALIDATE | \
38 		PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET)
39 
40 /* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */
41 #define PIPE_CONTROL_3D_ARCH_FLAGS ( \
42 		PIPE_CONTROL_3D_ENGINE_FLAGS | \
43 		PIPE_CONTROL_INDIRECT_STATE_DISABLE | \
44 		PIPE_CONTROL_FLUSH_ENABLE | \
45 		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
46 		PIPE_CONTROL_DC_FLUSH_ENABLE)
47 
48 static u32 preparser_disable(bool state)
49 {
50 	return MI_ARB_CHECK | BIT(8) | state;
51 }
52 
53 static int emit_aux_table_inv(struct xe_gt *gt, struct xe_reg reg,
54 			      u32 *dw, int i)
55 {
56 	dw[i++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(1) | MI_LRI_MMIO_REMAP_EN;
57 	dw[i++] = reg.addr + gt->mmio.adj_offset;
58 	dw[i++] = AUX_INV;
59 	dw[i++] = MI_NOOP;
60 
61 	return i;
62 }
63 
64 static int emit_user_interrupt(u32 *dw, int i)
65 {
66 	dw[i++] = MI_USER_INTERRUPT;
67 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE;
68 	dw[i++] = MI_ARB_CHECK;
69 
70 	return i;
71 }
72 
73 static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i)
74 {
75 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_GGTT | MI_SDI_NUM_DW(1);
76 	dw[i++] = addr;
77 	dw[i++] = 0;
78 	dw[i++] = value;
79 
80 	return i;
81 }
82 
83 static int emit_flush_dw(u32 *dw, int i)
84 {
85 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_IMM_DW;
86 	dw[i++] = 0;
87 	dw[i++] = 0;
88 	dw[i++] = 0;
89 
90 	return i;
91 }
92 
93 static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 flags, u32 *dw, int i)
94 {
95 	dw[i++] = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW |
96 		  flags;
97 	dw[i++] = addr | MI_FLUSH_DW_USE_GTT;
98 	dw[i++] = 0;
99 	dw[i++] = value;
100 
101 	return i;
102 }
103 
104 static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i)
105 {
106 	dw[i++] = MI_BATCH_BUFFER_START | ppgtt_flag | XE_INSTR_NUM_DW(3);
107 	dw[i++] = lower_32_bits(batch_addr);
108 	dw[i++] = upper_32_bits(batch_addr);
109 
110 	return i;
111 }
112 
113 static int emit_flush_invalidate(u32 *dw, int i)
114 {
115 	dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW |
116 		  MI_FLUSH_IMM_DW | MI_FLUSH_DW_STORE_INDEX;
117 	dw[i++] = LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR;
118 	dw[i++] = 0;
119 	dw[i++] = 0;
120 
121 	return i;
122 }
123 
124 static int
125 emit_pipe_control(u32 *dw, int i, u32 bit_group_0, u32 bit_group_1, u32 offset, u32 value)
126 {
127 	dw[i++] = GFX_OP_PIPE_CONTROL(6) | bit_group_0;
128 	dw[i++] = bit_group_1;
129 	dw[i++] = offset;
130 	dw[i++] = 0;
131 	dw[i++] = value;
132 	dw[i++] = 0;
133 
134 	return i;
135 }
136 
137 static int emit_pipe_invalidate(u32 mask_flags, bool invalidate_tlb, u32 *dw,
138 				int i)
139 {
140 	u32 flags0 = 0;
141 	u32 flags1 = PIPE_CONTROL_CS_STALL |
142 		PIPE_CONTROL_COMMAND_CACHE_INVALIDATE |
143 		PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE |
144 		PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
145 		PIPE_CONTROL_VF_CACHE_INVALIDATE |
146 		PIPE_CONTROL_CONST_CACHE_INVALIDATE |
147 		PIPE_CONTROL_STATE_CACHE_INVALIDATE |
148 		PIPE_CONTROL_QW_WRITE |
149 		PIPE_CONTROL_STORE_DATA_INDEX;
150 
151 	if (invalidate_tlb)
152 		flags1 |= PIPE_CONTROL_TLB_INVALIDATE;
153 
154 	flags1 &= ~mask_flags;
155 
156 	if (flags1 & PIPE_CONTROL_VF_CACHE_INVALIDATE)
157 		flags0 |= PIPE_CONTROL0_L3_READ_ONLY_CACHE_INVALIDATE;
158 
159 	return emit_pipe_control(dw, i, flags0, flags1,
160 				 LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
161 }
162 
163 static int emit_store_imm_ppgtt_posted(u64 addr, u64 value,
164 				       u32 *dw, int i)
165 {
166 	dw[i++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(1);
167 	dw[i++] = lower_32_bits(addr);
168 	dw[i++] = upper_32_bits(addr);
169 	dw[i++] = lower_32_bits(value);
170 	dw[i++] = upper_32_bits(value);
171 
172 	return i;
173 }
174 
175 static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i)
176 {
177 	struct xe_gt *gt = job->q->gt;
178 	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
179 	u32 flags;
180 
181 	if (XE_WA(gt, 14016712196))
182 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_DEPTH_CACHE_FLUSH,
183 				      LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR, 0);
184 
185 	flags = (PIPE_CONTROL_CS_STALL |
186 		 PIPE_CONTROL_TILE_CACHE_FLUSH |
187 		 PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH |
188 		 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
189 		 PIPE_CONTROL_DC_FLUSH_ENABLE |
190 		 PIPE_CONTROL_FLUSH_ENABLE);
191 
192 	if (XE_WA(gt, 1409600907))
193 		flags |= PIPE_CONTROL_DEPTH_STALL;
194 
195 	if (lacks_render)
196 		flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS;
197 	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
198 		flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS;
199 
200 	return emit_pipe_control(dw, i, PIPE_CONTROL0_HDC_PIPELINE_FLUSH, flags, 0, 0);
201 }
202 
203 static int emit_pipe_control_to_ring_end(struct xe_hw_engine *hwe, u32 *dw, int i)
204 {
205 	if (hwe->class != XE_ENGINE_CLASS_RENDER)
206 		return i;
207 
208 	if (XE_WA(hwe->gt, 16020292621))
209 		i = emit_pipe_control(dw, i, 0, PIPE_CONTROL_LRI_POST_SYNC,
210 				      RING_NOPID(hwe->mmio_base).addr, 0);
211 
212 	return i;
213 }
214 
215 static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw,
216 			      int i)
217 {
218 	u32 flags = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_GLOBAL_GTT_IVB |
219 		    PIPE_CONTROL_QW_WRITE;
220 
221 	if (!stall_only)
222 		flags |= PIPE_CONTROL_FLUSH_ENABLE;
223 
224 	return emit_pipe_control(dw, i, 0, flags, addr, value);
225 }
226 
227 static u32 get_ppgtt_flag(struct xe_sched_job *job)
228 {
229 	if (job->q->vm && !job->ggtt)
230 		return BIT(8);
231 
232 	return 0;
233 }
234 
235 static int emit_copy_timestamp(struct xe_lrc *lrc, u32 *dw, int i)
236 {
237 	dw[i++] = MI_COPY_MEM_MEM | MI_COPY_MEM_MEM_SRC_GGTT |
238 		MI_COPY_MEM_MEM_DST_GGTT;
239 	dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);
240 	dw[i++] = 0;
241 	dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc);
242 	dw[i++] = 0;
243 	dw[i++] = MI_NOOP;
244 
245 	return i;
246 }
247 
248 /* for engines that don't require any special HW handling (no EUs, no aux inval, etc) */
249 static void __emit_job_gen12_simple(struct xe_sched_job *job, struct xe_lrc *lrc,
250 				    u64 batch_addr, u32 seqno)
251 {
252 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
253 	u32 ppgtt_flag = get_ppgtt_flag(job);
254 	struct xe_gt *gt = job->q->gt;
255 
256 	i = emit_copy_timestamp(lrc, dw, i);
257 
258 	if (job->ring_ops_flush_tlb) {
259 		dw[i++] = preparser_disable(true);
260 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
261 					seqno, MI_INVALIDATE_TLB, dw, i);
262 		dw[i++] = preparser_disable(false);
263 	} else {
264 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
265 					seqno, dw, i);
266 	}
267 
268 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
269 
270 	if (job->user_fence.used) {
271 		i = emit_flush_dw(dw, i);
272 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
273 						job->user_fence.value,
274 						dw, i);
275 	}
276 
277 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);
278 
279 	i = emit_user_interrupt(dw, i);
280 
281 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
282 
283 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
284 }
285 
286 static bool has_aux_ccs(struct xe_device *xe)
287 {
288 	/*
289 	 * PVC is a special case that has no compression of either type
290 	 * (FlatCCS or AuxCCS).  Also, AuxCCS is no longer used from Xe2
291 	 * onward, so any future platforms with no FlatCCS will not have
292 	 * AuxCCS either.
293 	 */
294 	if (GRAPHICS_VER(xe) >= 20 || xe->info.platform == XE_PVC)
295 		return false;
296 
297 	return !xe->info.has_flat_ccs;
298 }
299 
300 static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc,
301 				   u64 batch_addr, u32 seqno)
302 {
303 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
304 	u32 ppgtt_flag = get_ppgtt_flag(job);
305 	struct xe_gt *gt = job->q->gt;
306 	struct xe_device *xe = gt_to_xe(gt);
307 	bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE;
308 
309 	i = emit_copy_timestamp(lrc, dw, i);
310 
311 	dw[i++] = preparser_disable(true);
312 
313 	/* hsdes: 1809175790 */
314 	if (has_aux_ccs(xe)) {
315 		if (decode)
316 			i = emit_aux_table_inv(gt, VD0_AUX_INV, dw, i);
317 		else
318 			i = emit_aux_table_inv(gt, VE0_AUX_INV, dw, i);
319 	}
320 
321 	if (job->ring_ops_flush_tlb)
322 		i = emit_flush_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
323 					seqno, MI_INVALIDATE_TLB, dw, i);
324 
325 	dw[i++] = preparser_disable(false);
326 
327 	if (!job->ring_ops_flush_tlb)
328 		i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
329 					seqno, dw, i);
330 
331 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
332 
333 	if (job->user_fence.used) {
334 		i = emit_flush_dw(dw, i);
335 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
336 						job->user_fence.value,
337 						dw, i);
338 	}
339 
340 	i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, 0, dw, i);
341 
342 	i = emit_user_interrupt(dw, i);
343 
344 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
345 
346 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
347 }
348 
349 static void __emit_job_gen12_render_compute(struct xe_sched_job *job,
350 					    struct xe_lrc *lrc,
351 					    u64 batch_addr, u32 seqno)
352 {
353 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
354 	u32 ppgtt_flag = get_ppgtt_flag(job);
355 	struct xe_gt *gt = job->q->gt;
356 	struct xe_device *xe = gt_to_xe(gt);
357 	bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK);
358 	u32 mask_flags = 0;
359 
360 	i = emit_copy_timestamp(lrc, dw, i);
361 
362 	dw[i++] = preparser_disable(true);
363 	if (lacks_render)
364 		mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS;
365 	else if (job->q->class == XE_ENGINE_CLASS_COMPUTE)
366 		mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS;
367 
368 	/* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */
369 	i = emit_pipe_invalidate(mask_flags, job->ring_ops_flush_tlb, dw, i);
370 
371 	/* hsdes: 1809175790 */
372 	if (has_aux_ccs(xe))
373 		i = emit_aux_table_inv(gt, CCS_AUX_INV, dw, i);
374 
375 	dw[i++] = preparser_disable(false);
376 
377 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
378 				seqno, dw, i);
379 
380 	i = emit_bb_start(batch_addr, ppgtt_flag, dw, i);
381 
382 	i = emit_render_cache_flush(job, dw, i);
383 
384 	if (job->user_fence.used)
385 		i = emit_store_imm_ppgtt_posted(job->user_fence.addr,
386 						job->user_fence.value,
387 						dw, i);
388 
389 	i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, lacks_render, dw, i);
390 
391 	i = emit_user_interrupt(dw, i);
392 
393 	i = emit_pipe_control_to_ring_end(job->q->hwe, dw, i);
394 
395 	xe_gt_assert(gt, i <= MAX_JOB_SIZE_DW);
396 
397 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
398 }
399 
400 static void emit_migration_job_gen12(struct xe_sched_job *job,
401 				     struct xe_lrc *lrc, u32 seqno)
402 {
403 	u32 dw[MAX_JOB_SIZE_DW], i = 0;
404 
405 	i = emit_copy_timestamp(lrc, dw, i);
406 
407 	i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc),
408 				seqno, dw, i);
409 
410 	dw[i++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; /* Enabled again below */
411 
412 	i = emit_bb_start(job->ptrs[0].batch_addr, BIT(8), dw, i);
413 
414 	if (!IS_SRIOV_VF(gt_to_xe(job->q->gt))) {
415 		/* XXX: Do we need this? Leaving for now. */
416 		dw[i++] = preparser_disable(true);
417 		i = emit_flush_invalidate(dw, i);
418 		dw[i++] = preparser_disable(false);
419 	}
420 
421 	i = emit_bb_start(job->ptrs[1].batch_addr, BIT(8), dw, i);
422 
423 	dw[i++] = MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags |
424 		MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_IMM_DW;
425 	dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT;
426 	dw[i++] = 0;
427 	dw[i++] = seqno; /* value */
428 
429 	i = emit_user_interrupt(dw, i);
430 
431 	xe_gt_assert(job->q->gt, i <= MAX_JOB_SIZE_DW);
432 
433 	xe_lrc_write_ring(lrc, dw, i * sizeof(*dw));
434 }
435 
436 static void emit_job_gen12_gsc(struct xe_sched_job *job)
437 {
438 	struct xe_gt *gt = job->q->gt;
439 
440 	xe_gt_assert(gt, job->q->width <= 1); /* no parallel submission for GSCCS */
441 
442 	__emit_job_gen12_simple(job, job->q->lrc[0],
443 				job->ptrs[0].batch_addr,
444 				xe_sched_job_lrc_seqno(job));
445 }
446 
447 static void emit_job_gen12_copy(struct xe_sched_job *job)
448 {
449 	int i;
450 
451 	if (xe_sched_job_is_migration(job->q)) {
452 		emit_migration_job_gen12(job, job->q->lrc[0],
453 					 xe_sched_job_lrc_seqno(job));
454 		return;
455 	}
456 
457 	for (i = 0; i < job->q->width; ++i)
458 		__emit_job_gen12_simple(job, job->q->lrc[i],
459 					job->ptrs[i].batch_addr,
460 					xe_sched_job_lrc_seqno(job));
461 }
462 
463 static void emit_job_gen12_video(struct xe_sched_job *job)
464 {
465 	int i;
466 
467 	/* FIXME: Not doing parallel handshake for now */
468 	for (i = 0; i < job->q->width; ++i)
469 		__emit_job_gen12_video(job, job->q->lrc[i],
470 				       job->ptrs[i].batch_addr,
471 				       xe_sched_job_lrc_seqno(job));
472 }
473 
474 static void emit_job_gen12_render_compute(struct xe_sched_job *job)
475 {
476 	int i;
477 
478 	for (i = 0; i < job->q->width; ++i)
479 		__emit_job_gen12_render_compute(job, job->q->lrc[i],
480 						job->ptrs[i].batch_addr,
481 						xe_sched_job_lrc_seqno(job));
482 }
483 
484 static const struct xe_ring_ops ring_ops_gen12_gsc = {
485 	.emit_job = emit_job_gen12_gsc,
486 };
487 
488 static const struct xe_ring_ops ring_ops_gen12_copy = {
489 	.emit_job = emit_job_gen12_copy,
490 };
491 
492 static const struct xe_ring_ops ring_ops_gen12_video = {
493 	.emit_job = emit_job_gen12_video,
494 };
495 
496 static const struct xe_ring_ops ring_ops_gen12_render_compute = {
497 	.emit_job = emit_job_gen12_render_compute,
498 };
499 
500 const struct xe_ring_ops *
501 xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class)
502 {
503 	switch (class) {
504 	case XE_ENGINE_CLASS_OTHER:
505 		return &ring_ops_gen12_gsc;
506 	case XE_ENGINE_CLASS_COPY:
507 		return &ring_ops_gen12_copy;
508 	case XE_ENGINE_CLASS_VIDEO_DECODE:
509 	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
510 		return &ring_ops_gen12_video;
511 	case XE_ENGINE_CLASS_RENDER:
512 	case XE_ENGINE_CLASS_COMPUTE:
513 		return &ring_ops_gen12_render_compute;
514 	default:
515 		return NULL;
516 	}
517 }
518